From 211375243e0e6fda0b677ad16d665edd49a385c6 Mon Sep 17 00:00:00 2001 From: Ali Date: Fri, 10 May 2024 15:55:06 +0200 Subject: [PATCH 01/39] - Some TODO's are done, wherever relavant, I implemented changes to `aiida-core`. - Refactor FirecrestScheduler to handle pagination for retrieving active jobs - dynamic_info() is added to retrieve machine information without user input. - put & get & copy passed manual test. - Added `dereference` option wherever relevant - Added new tests. The old tests, although more suffesticated is harder to debug. New test focuses only on aiida-firecrest by mocking pyfirecrest only, not the server. --- aiida_firecrest/remote_path.py | 617 ------------------------- aiida_firecrest/scheduler.py | 25 +- aiida_firecrest/transport.py | 819 +++++++++++++++++++++++++-------- aiida_firecrest/utils_test.py | 66 ++- tests/conftest.py | 4 + tests/test_computer.py | 4 +- tests_new/conftest.py | 191 ++++++++ tests_new/test_transport.py | 250 ++++++++++ 8 files changed, 1151 insertions(+), 825 deletions(-) delete mode 100644 aiida_firecrest/remote_path.py create mode 100644 tests_new/conftest.py create mode 100644 tests_new/test_transport.py diff --git a/aiida_firecrest/remote_path.py b/aiida_firecrest/remote_path.py deleted file mode 100644 index 107b0ce..0000000 --- a/aiida_firecrest/remote_path.py +++ /dev/null @@ -1,617 +0,0 @@ -"""A pathlib.Path-like object for accessing the file system via the Firecrest API. - -Note this is independent of AiiDA, -in fact it is awaiting possible inclusion in pyfirecrest: -https://github.com/eth-cscs/pyfirecrest/pull/43 -""" -from __future__ import annotations - -from collections.abc import Iterator -from dataclasses import dataclass -from functools import lru_cache -from io import BytesIO -import os -from pathlib import PurePosixPath -import stat -import tempfile -from typing import Callable, TypeVar - -from firecrest import ClientCredentialsAuth, Firecrest # type: ignore[attr-defined] - -from .utils import convert_header_exceptions - -# Note in python 3.11 could use typing.Self -SelfTv = TypeVar("SelfTv", bound="FcPath") - - -@dataclass -class ModeCache: - """A cache of the path's mode. - - This can be useful for limiting calls to the API, - particularly for paths generated via `/utilities/ls` - which already provides the st_mode for each file - """ - - st_mode: int | None = None - """The st_mode of the path, dereferencing symlinks.""" - lst_mode: int | None = None - """The st_mode of the path, without dereferencing symlinks.""" - - def reset(self) -> None: - """Reset the cache.""" - self.st_mode = None - self.lst_mode = None - - -class FcPath(os.PathLike[str]): - """A pathlib.Path-like object for accessing the file system via the Firecrest API.""" - - __slots__ = ("_client", "_machine", "_path", "_cache_enabled", "_cache") - - def __init__( - self, - client: Firecrest, - machine: str, - path: str | PurePosixPath, - *, - cache_enabled: bool = False, - _cache: None | ModeCache = None, - ) -> None: - """Construct a new FcPath instance. - - :param client: A Firecrest client object - :param machine: The machine name - :param path: The absolute path to the file or directory - :param cache_enabled: Enable caching of path statistics - This enables caching of path statistics, like mode, - which can be useful if you are using multiple methods on the same path, - as it avoids making multiple calls to the API. - You should only use this if you are sure that the file system is not being modified. - - """ - self._client = client - self._machine = machine - self._path = PurePosixPath(path) - if not self._path.is_absolute(): - raise ValueError(f"Path must be absolute: {str(self._path)!r}") - self._cache_enabled = cache_enabled - self._cache = _cache or ModeCache() - - @classmethod - def from_env_variables( - cls, machine: str, path: str | PurePosixPath, *, cache_enabled: bool = False - ) -> FcPath: - """Convenience method, to construct a new FcPath using environmental variables. - - The following environment variables are required: - - FIRECREST_URL - - FIRECREST_CLIENT_ID - - FIRECREST_CLIENT_SECRET - - AUTH_TOKEN_URL - """ - auth_obj = ClientCredentialsAuth( - os.environ["FIRECREST_CLIENT_ID"], - os.environ["FIRECREST_CLIENT_SECRET"], - os.environ["AUTH_TOKEN_URL"], - ) - client = Firecrest(os.environ["FIRECREST_URL"], authorization=auth_obj) - return cls(client, machine, path, cache_enabled=cache_enabled) - - @property - def client(self) -> Firecrest: - """The Firecrest client object.""" - return self._client - - @property - def machine(self) -> str: - """The machine name.""" - return self._machine - - @property - def path(self) -> str: - """Return the string representation of the path on the machine.""" - return str(self._path) - - @property - def pure_path(self) -> PurePosixPath: - """Return the pathlib representation of the path on the machine.""" - return self._path - - @property - def cache_enabled(self) -> bool: - """Enable caching of path statistics. - - This enables caching of path statistics, like mode, - which can be useful if you are using multiple methods on the same path, - as it avoids making multiple calls to the API. - - You should only use this if you are sure that the file system is not being modified. - """ - return self._cache_enabled - - @cache_enabled.setter - def cache_enabled(self, value: bool) -> None: - self._cache_enabled = value - - def enable_cache(self: SelfTv) -> SelfTv: - """Enable caching of path statistics.""" - self._cache_enabled = True - return self - - def clear_cache(self: SelfTv) -> SelfTv: - """Clear the cache of path statistics.""" - self._cache.reset() - return self - - def _new_path( - self: SelfTv, path: PurePosixPath, *, _cache: None | ModeCache = None - ) -> SelfTv: - """Construct a new FcPath object from a PurePosixPath object.""" - return self.__class__( - self._client, - self._machine, - path, - cache_enabled=self._cache_enabled, - _cache=_cache, - ) - - def __fspath__(self) -> str: - return str(self._path) - - def __str__(self) -> str: - return self.path - - def __repr__(self) -> str: - variables = [ - repr(self._client._firecrest_url), - repr(self._machine), - repr(self.path), - ] - if self._cache_enabled: - variables.append("CACHED") - return f"{self.__class__.__name__}({', '.join(variables)})" - - def as_posix(self) -> str: - """Return the string representation of the path.""" - return self._path.as_posix() - - @property - def name(self) -> str: - """The final path component, if any.""" - return self._path.name - - @property - def suffix(self) -> str: - """ - The final component's last suffix, if any. - - This includes the leading period. For example: '.txt' - """ - return self._path.suffix - - @property - def suffixes(self) -> list[str]: - """ - A list of the final component's suffixes, if any. - - These include the leading periods. For example: ['.tar', '.gz'] - """ - return self._path.suffixes - - @property - def stem(self) -> str: - """ - The final path component, minus its last suffix. - - If the final path component has no suffix, this is the same as name. - """ - return self._path.stem - - def with_name(self: SelfTv, name: str) -> SelfTv: - """Return a new path with the file name changed.""" - return self._new_path(self._path.with_name(name)) - - def with_suffix(self: SelfTv, suffix: str) -> SelfTv: - """Return a new path with the file suffix changed.""" - return self._new_path(self._path.with_suffix(suffix)) - - @property - def parts(self) -> tuple[str, ...]: - """The components of the path.""" - return self._path.parts - - @property - def parent(self: SelfTv) -> SelfTv: - """The path's parent directory.""" - return self._new_path(self._path.parent) - - def is_absolute(self) -> bool: - """Return True if the path is absolute.""" - return self._path.is_absolute() - - def __truediv__(self: SelfTv, other: str) -> SelfTv: - return self._new_path(self._path / other) - - def joinpath(self: SelfTv, *other: str) -> SelfTv: - """Combine this path with one or several arguments, and return a - new path representing either a subpath (if all arguments are relative - paths) or a totally different path (if one of the arguments is - anchored). - """ - return self._new_path(self._path.joinpath(*other)) - - def whoami(self) -> str: - """Return the username of the current user.""" - with convert_header_exceptions({"machine": self._machine}): - # TODO: use self._client.whoami(self._machine) - # requires https://github.com/eth-cscs/pyfirecrest/issues/58 - resp = self._client._get_request( - endpoint="/utilities/whoami", - additional_headers={"X-Machine-Name": self._machine}, - ) - return self._client._json_response([resp], 200)["output"] # type: ignore - - def checksum(self) -> str: - """Return the SHA256 (256-bit) checksum of the file.""" - # this is not part of the pathlib.Path API, but is useful - with convert_header_exceptions({"machine": self._machine, "path": self}): - return self._client.checksum(self._machine, self.path) - - # methods that utilise stat calls - - def _lstat_mode(self) -> int: - """Return the st_mode of the path, not following symlinks.""" - if self._cache_enabled and self._cache.lst_mode is not None: - return self._cache.lst_mode - self._cache.lst_mode = self.lstat().st_mode - if not stat.S_ISLNK(self._cache.lst_mode): - # if the path is not a symlink, - # then we also know the dereferenced mode - self._cache.st_mode = self._cache.lst_mode - return self._cache.lst_mode - - def _stat_mode(self) -> int: - """Return the st_mode of the path, following symlinks.""" - if self._cache_enabled and self._cache.st_mode is not None: - return self._cache.st_mode - self._cache.st_mode = self.stat().st_mode - return self._cache.st_mode - - def stat(self) -> os.stat_result: - """Return stat info for this path. - - If the path is a symbolic link, - stat will examine the file the link points to. - """ - with convert_header_exceptions( - {"machine": self._machine, "path": self}, - # TODO: remove this once issue fixed: https://github.com/eth-cscs/firecrest/issues/193 - {"X-A-Directory": FileNotFoundError}, - ): - stats = self._client.stat(self._machine, self.path, dereference=True) - return os.stat_result( - ( - stats["mode"], - stats["ino"], - stats["dev"], - stats["nlink"], - stats["uid"], - stats["gid"], - stats["size"], - stats["atime"], - stats["mtime"], - stats["ctime"], - ) - ) - - def lstat(self) -> os.stat_result: - """ - Like stat(), except if the path points to a symlink, the symlink's - status information is returned, rather than its target's. - """ - with convert_header_exceptions({"machine": self._machine, "path": self}): - stats = self._client.stat(self._machine, self.path, dereference=False) - return os.stat_result( - ( - stats["mode"], - stats["ino"], - stats["dev"], - stats["nlink"], - stats["uid"], - stats["gid"], - stats["size"], - stats["atime"], - stats["mtime"], - stats["ctime"], - ) - ) - - def exists(self) -> bool: - """Whether this path exists (follows symlinks).""" - try: - self.stat() - except FileNotFoundError: - return False - return True - - def is_dir(self) -> bool: - """Whether this path is a directory (follows symlinks).""" - try: - st_mode = self._stat_mode() - except FileNotFoundError: - return False - return stat.S_ISDIR(st_mode) - - def is_file(self) -> bool: - """Whether this path is a regular file (follows symlinks).""" - try: - st_mode = self._stat_mode() - except FileNotFoundError: - return False - return stat.S_ISREG(st_mode) - - def is_symlink(self) -> bool: - """Whether this path is a symbolic link.""" - try: - st_mode = self._lstat_mode() - except FileNotFoundError: - return False - return stat.S_ISLNK(st_mode) - - def is_block_device(self) -> bool: - """Whether this path is a block device (follows symlinks).""" - try: - st_mode = self._stat_mode() - except FileNotFoundError: - return False - return stat.S_ISBLK(st_mode) - - def is_char_device(self) -> bool: - """Whether this path is a character device (follows symlinks).""" - try: - st_mode = self._stat_mode() - except FileNotFoundError: - return False - return stat.S_ISCHR(st_mode) - - def is_fifo(self) -> bool: - """Whether this path is a FIFO (follows symlinks).""" - try: - st_mode = self._stat_mode() - except FileNotFoundError: - return False - return stat.S_ISFIFO(st_mode) - - def is_socket(self) -> bool: - """Whether this path is a socket (follows symlinks).""" - try: - st_mode = self._stat_mode() - except FileNotFoundError: - return False - return stat.S_ISSOCK(st_mode) - - def iterdir(self: SelfTv, hidden: bool = True) -> Iterator[SelfTv]: - """Iterate over the directory entries.""" - if not self.is_dir(): - return - with convert_header_exceptions({"machine": self._machine, "path": self}): - results = self._client.list_files( - self._machine, self.path, show_hidden=hidden - ) - for entry in results: - lst_mode = _ls_to_st_mode(entry["type"], entry["permissions"]) - # if the path is not a symlink, then we also know the dereferenced mode - st_mode = lst_mode if not stat.S_ISLNK(lst_mode) else None - yield self._new_path( - self._path / entry["name"], - _cache=ModeCache(lst_mode=lst_mode, st_mode=st_mode), - ) - - # operations that modify a file - - def chmod(self, mode: int | str) -> None: - """Change the mode of the path to the numeric mode. - - Note, if the path points to a symlink, - the symlink target's permissions are changed. - """ - # note: according to: - # https://www.gnu.org/software/coreutils/manual/html_node/chmod-invocation.html#chmod-invocation - # chmod never changes the permissions of symbolic links, - # i.e. this is chmod, not lchmod - if not isinstance(mode, (int, str)): - raise TypeError("mode must be an integer") - with convert_header_exceptions( - {"machine": self._machine, "path": self}, - {"X-Invalid-Mode": lambda p: ValueError(f"invalid mode: {mode}")}, - ): - self._client.chmod(self._machine, self.path, str(mode)) - self._cache.reset() - - def chown(self, uid: int | str, gid: int | str) -> None: - """Change the owner and group id of the path to the numeric uid and gid.""" - if not isinstance(uid, (str, int)): - raise TypeError("uid must be an integer") - if not isinstance(gid, (str, int)): - raise TypeError("gid must be an integer") - with convert_header_exceptions( - {"machine": self._machine, "path": self}, - { - "X-Invalid-Owner": lambda p: PermissionError(f"invalid uid: {uid}"), - "X-Invalid-Group": lambda p: PermissionError(f"invalid gid: {gid}"), - }, - ): - self._client.chown(self._machine, self.path, str(uid), str(gid)) - - def rename(self: SelfTv, target: str | os.PathLike[str]) -> SelfTv: - """Rename this path to the (absolute) target path. - - Returns the new Path instance pointing to the target path. - """ - target_path = self._new_path(PurePosixPath(target)) - with convert_header_exceptions({"machine": self._machine, "path": self}): - self._client.mv(self._machine, self.path, target_path.path) - return target_path - - def symlink_to(self, target: str | os.PathLike[str]) -> None: - """Make this path a symlink pointing to the target path.""" - target_path = PurePosixPath(target) - if not target_path.is_absolute(): - raise ValueError("target must be an absolute path") - with convert_header_exceptions( - {"machine": self._machine, "path": self}, - # TODO this is only here because of this bug: - # https://github.com/eth-cscs/firecrest/issues/190 - {"X-Error": FileExistsError}, - ): - self._client.symlink(self._machine, str(target_path), self.path) - - def copy_to(self: SelfTv, target: str | os.PathLike[str]) -> None: - """Copy this path to the target path - - Works for both files and directories (in which case the whole tree is copied). - """ - target_path = PurePosixPath(target) - if not target_path.is_absolute(): - raise ValueError("target must be an absolute path") - with convert_header_exceptions({"machine": self._machine, "path": self}): - # Note although this endpoint states that it is only for directories, - # it actually uses `cp -r`: - # https://github.com/eth-cscs/firecrest/blob/7f02d11b224e4faee7f4a3b35211acb9c1cc2c6a/src/utilities/utilities.py#L320 - self._client.copy(self._machine, self.path, str(target_path)) - - def mkdir( - self, mode: None = None, parents: bool = False, exist_ok: bool = False - ) -> None: - """Create a new directory at this given path.""" - if mode is not None: - raise NotImplementedError("mode is not supported yet") - try: - with convert_header_exceptions({"machine": self._machine, "path": self}): - self._client.mkdir(self._machine, self.path, p=parents) - except FileExistsError: - if not exist_ok: - raise - - def touch(self, mode: None = None, exist_ok: bool = True) -> None: - """Create a file at this given path. - - :param mode: ignored - :param exist_ok: if True, do not raise an exception if the path already exists - """ - if mode is not None: - raise NotImplementedError("mode is not supported yet") - if self.exists(): - if exist_ok: - return - raise FileExistsError(self) - try: - _, source_path = tempfile.mkstemp() - with convert_header_exceptions({"machine": self._machine, "path": self}): - self._client.simple_upload( - self._machine, source_path, self.parent.path, self.name - ) - finally: - os.remove(source_path) - - def read_bytes(self) -> bytes: - """Read the contents of the file as bytes. - - NOTE: This is only intended for small files. - """ - io = BytesIO() - with convert_header_exceptions({"machine": self._machine, "path": self}): - self._client.simple_download(self._machine, self.path, io) - return io.getvalue() - - def read_text(self, encoding: str = "utf-8", errors: str = "strict") -> str: - """Read the contents of the file as text. - - NOTE: This is only intended for small files. - """ - return self.read_bytes().decode(encoding, errors) - - def write_bytes(self, data: bytes) -> None: - """Write bytes to the file. - - NOTE: This is only intended for small files. - """ - buffer = BytesIO(data) - with convert_header_exceptions({"machine": self._machine, "path": self}): - self._client.simple_upload( - self._machine, buffer, self.parent.path, self.name - ) - - def write_text( - self, data: str, encoding: str = "utf-8", errors: str = "strict" - ) -> None: - """Write text to the file. - - NOTE: This is only intended for small files. - """ - self.write_bytes(data.encode(encoding, errors)) - - def unlink(self, missing_ok: bool = False) -> None: - """Remove this file.""" - # note /utilities/rm uses `rm -rf`, - # https://github.com/eth-cscs/firecrest/blob/7f02d11b224e4faee7f4a3b35211acb9c1cc2c6a/src/utilities/utilities.py#L347 - # so we have to be careful to check first what we are deleting - try: - st_mode = self._lstat_mode() - except FileNotFoundError: - if not missing_ok: - raise FileNotFoundError(self) from None - return - if stat.S_ISDIR(st_mode): - raise IsADirectoryError(self) - with convert_header_exceptions({"machine": self._machine, "path": self}): - self._client.simple_delete(self._machine, self.path) - self._cache.reset() - - def rmtree(self) -> None: - """Recursively delete a directory tree.""" - # note /utilities/rm uses `rm -rf`, - # https://github.com/eth-cscs/firecrest/blob/7f02d11b224e4faee7f4a3b35211acb9c1cc2c6a/src/utilities/utilities.py#L347 - # so we have to be careful to check first what we are deleting - try: - st_mode = self._lstat_mode() - except FileNotFoundError: - raise FileNotFoundError(self) from None - if not stat.S_ISDIR(st_mode): - raise NotADirectoryError(self) - with convert_header_exceptions({"machine": self._machine, "path": self}): - self._client.simple_delete(self._machine, self.path) - self._cache.reset() - - -@lru_cache(maxsize=256) -def _ls_to_st_mode(ftype: str, permissions: str) -> int: - """Use the return information from `utilities/ls` to create an st_mode value. - - Note, this does not dereference symlinks, and so is like lstat - - :param ftype: The file type, e.g. "-" for regular file, "d" for directory. - :param permissions: The file permissions, e.g. "rwxr-xr-x". - """ - ftypes = { - "b": "0060", # block device - "c": "0020", # character device - "d": "0040", # directory - "l": "0120", # Symbolic link - "s": "0140", # Socket. - "p": "0010", # FIFO - "-": "0100", # Regular file - } - if ftype not in ftypes: - raise ValueError(f"invalid file type: {ftype}") - p = permissions - r: Callable[[str], int] = lambda x: 4 if x == "r" else 0 # noqa: E731 - w: Callable[[str], int] = lambda x: 2 if x == "w" else 0 # noqa: E731 - x: Callable[[str], int] = lambda x: 1 if x == "x" else 0 # noqa: E731 - st_mode = ( - ((r(p[0]) + w(p[1]) + x(p[2])) * 100) - + ((r(p[3]) + w(p[4]) + x(p[5])) * 10) - + ((r(p[6]) + w(p[7]) + x(p[8])) * 1) - ) - return int(ftypes[ftype] + str(st_mode), 8) diff --git a/aiida_firecrest/scheduler.py b/aiida_firecrest/scheduler.py index 9a1f2bc..fe1870f 100644 --- a/aiida_firecrest/scheduler.py +++ b/aiida_firecrest/scheduler.py @@ -4,7 +4,7 @@ import re import string from typing import TYPE_CHECKING, Any, ClassVar - +import itertools from aiida.engine.processes.exit_code import ExitCode from aiida.schedulers import Scheduler, SchedulerError from aiida.schedulers.datastructures import JobInfo, JobState, JobTemplate @@ -26,9 +26,8 @@ class FirecrestScheduler(Scheduler): "can_query_by_user": False, } _logger = Scheduler._logger.getChild("firecrest") + _DEFAULT_PAGE_SIZE = 25 - # TODO if this is missing is causes plugin info to fail on verdi - is_process_function = False @classmethod def get_description(cls) -> str: @@ -211,21 +210,27 @@ def get_jobs( user: str | None = None, as_dict: bool = False, ) -> list[JobInfo] | dict[str, JobInfo]: + results = [] transport = self.transport with convert_header_exceptions({"machine": transport._machine}): # TODO handle pagination (pageSize, pageNumber) if many jobs + # This will do pagination, not manually tested becasue the server is damn slow. try: - results = transport._client.poll_active(transport._machine, jobs) + for page_iter in itertools.count(): + results += transport._client.poll_active(transport._machine, jobs, page_number=page_iter) + if len(results) < self._DEFAULT_PAGE_SIZE: + break except FirecrestException as exc: raise SchedulerError(str(exc)) from exc job_list = [] for raw_result in results: + # TODO: probably the if below is not needed, because recently, the server should return only the jobs of the current user if user is not None and raw_result["user"] != user: continue - this_job = JobInfo() # type: ignore this_job.job_id = raw_result["jobid"] + # TODO: firecrest does not return the annotation, so set to an empty string. To be investigated how important that is. this_job.annotation = "" job_state_raw = raw_result["state"] @@ -276,6 +281,7 @@ def get_jobs( ) ) + # TODO: The block below is commented, because the number of allocated cores is not returned by the FirecREST server # try: # this_job.num_mpiprocs = int(thisjob_dict['number_cpus']) # except ValueError: @@ -290,13 +296,15 @@ def get_jobs( # therefore it requires some parsing, that is unnecessary now. # I just store is as a raw string for the moment, and I leave # this_job.allocated_machines undefined - # if this_job.job_state == JobState.RUNNING: - # this_job.allocated_machines_raw = thisjob_dict['allocated_machines'] + if this_job.job_state == JobState.RUNNING: + this_job.allocated_machines_raw = raw_result["nodelist"] this_job.queue_name = raw_result["partition"] + # TODO: The block below is commented, because the time limit is not returned explicitly by the FirecREST server + # in any case, the time tags doesn't seem to be used by AiiDA anyway. # try: - # walltime = (self._convert_time(thisjob_dict['time_limit'])) + # walltime = (self._convert_time(raw_result['time_limit'])) # this_job.requested_wallclock_time_seconds = walltime # pylint: disable=invalid-name # except ValueError: # self.logger.warning(f'Error parsing the time limit for job id {this_job.job_id}') @@ -355,6 +363,7 @@ def kill_job(self, jobid: str) -> bool: return True + # see https://slurm.schedmd.com/squeue.html#lbAG # note firecrest returns full names, not abbreviations _MAP_STATUS_SLURM = { diff --git a/aiida_firecrest/transport.py b/aiida_firecrest/transport.py index ae1a35d..bb20e21 100644 --- a/aiida_firecrest/transport.py +++ b/aiida_firecrest/transport.py @@ -2,14 +2,17 @@ from __future__ import annotations import fnmatch +import hashlib import os from pathlib import Path import platform import posixpath import shutil +import tarfile import time from typing import Any, Callable, ClassVar, TypedDict from urllib import request +import uuid from aiida.cmdline.params.options.overridable import OverridableOption from aiida.transports import Transport @@ -18,8 +21,7 @@ from click.types import ParamType from firecrest import ClientCredentialsAuth, Firecrest # type: ignore[attr-defined] -from .remote_path import FcPath, convert_header_exceptions # type: ignore[attr-defined] - +from firecrest.path import FcPath class ValidAuthOption(TypedDict, total=False): option: OverridableOption | None # existing option @@ -31,6 +33,9 @@ class ValidAuthOption(TypedDict, total=False): help: str callback: Callable[..., Any] # for validation +class BuggyError(Exception): + # TODO: Remove this class when the code is stable + """Raised when something should absolutly not happen, but it does.""" class FirecrestTransport(Transport): """Transport interface for FirecREST.""" @@ -43,10 +48,132 @@ class FirecrestTransport(Transport): # but this would ideally require some "global" rate limiter, # across all transport instances # TODO upstream issue - # TODO also open an issue that the `verdi computer test won't work with a REST-API` _common_auth_options: ClassVar[list[Any]] = [] # type: ignore[misc] _DEFAULT_SAFE_OPEN_INTERVAL = 0.0 + def _create_secret_file(ctx, param, value) -> str: + """Create a secret file if the value is not a path to a secret file. + The path should be absolute, if it is not, the file will be created in ~/.firecrest. + """ + import uuid + from aiida.cmdline.utils import echo + from click import BadParameter + possible_path = Path(value) + if os.path.isabs(possible_path): + if not possible_path.exists(): + raise BadParameter(f'Secret file not found at {value}') + secret_path = possible_path + + else: + Path(f'~/.firecrest').expanduser().mkdir(parents=True, exist_ok=True) + _ = uuid.uuid4() + secret_path = Path(f'~/.firecrest/secret_{_}').expanduser() + while secret_path.exists(): + # instead of a random number one could use the label or pk of the computer being configured + secret_path = Path(f'~/.firecrest/secret_{_}').expanduser() + secret_path.write_text(value) + echo.echo_report(f"Secret file created at {secret_path}") + print(f"Client Secret stored at {secret_path}") + + return str(secret_path) + + + def _validate_temp_directory(ctx, param, value) -> str: + """Validate the temp directory on the server. + If it does not exist, create it. + If it is not empty, get a confimation from the user to empty it. + """ + + import click + firecrest_url= ctx.params['url'] + token_uri= ctx.params['token_uri'] + client_id= ctx.params['client_id'] + client_machine= ctx.params['client_machine'] + secret= ctx.params['client_secret']#)#.read_text() + small_file_size_mb= ctx.params['small_file_size_mb'] + + dummy = FirecrestTransport( + url=firecrest_url, + token_uri=token_uri, + client_id = client_id, + client_secret= secret, + client_machine= client_machine, + temp_directory= value, + small_file_size_mb = small_file_size_mb) + + # Temp directory rutine + if dummy._cwd.joinpath(dummy._temp_directory).is_file(): #self._temp_directory.is_file(): + raise click.BadParameter("Temp directory cannot be a file") + + if dummy.path_exists(dummy._temp_directory): + if dummy.listdir(dummy._temp_directory): + # if not configured: + confirm = click.confirm(f"Temp directory {dummy._temp_directory} is not empty. Do you want to flush it?") + if confirm: + for item in dummy.listdir(dummy._temp_directory): + # TODO: maybe do recursive delete + dummy.remove(dummy._temp_directory.joinpath(item)) + else: + click.echo(f"Please provide an empty temp directory on the server.") + raise click.BadParameter(f"Temp directory {dummy._temp_directory} is not empty") + # The block below could be moved to a maintanace delete function, if needed + # else: + # # There might still be some residual files in case of previous interupted connection + # for item in dummy.listdir(dummy._temp_directory): + # # this could be replace with a proper glob later + # if item[:4] == 'temp': + # dummy.remove(dummy._temp_directory.joinpath(item)) + + else: + try: + dummy.mkdir(dummy._temp_directory, ignore_existing=True) + except Exception as e: + raise OSError(f"Could not create temp directory {dummy._temp_directory} on server: {e}") + + return value + + + def _dynamic_info_direct_size(ctx, param, value) -> str: + """Get dynamic information from the server, if the user enters 0 for the small_file_size_mb. + This is done by connecting to the server and getting the value of UTILITIES_MAX_FILE_SIZE. + Below this size, file bytes will be sent in a single API call. Above this size, + the file will be downloaded(uploaded) from(to) the object store and downloaded in chunks. + + :param ctx: the `click.Context` + :param param: the parameter + :param value: the value passed for the parameter + + :return: the value of small_file_size_mb. + + """ + + if value > 0: + return value + + firecrest_url= ctx.params['url'] + token_uri= ctx.params['token_uri'] + client_id= ctx.params['client_id'] + client_machine= ctx.params['client_machine'] + secret= ctx.params['client_secret']#)#.read_text() + + dummy = FirecrestTransport( + url=firecrest_url, + token_uri=token_uri, + client_id = client_id, + client_secret= secret, + client_machine= client_machine, + temp_directory= '', + small_file_size_mb = 0.0) + + prameters= dummy._client.parameters() + utilities_max_file_size = next((item for item in prameters['utilities'] if item['name'] == 'UTILITIES_MAX_FILE_SIZE'), None) + if utilities_max_file_size is not None: + small_file_size_mb = float(utilities_max_file_size['value']) + else: + small_file_size_mb = 5.0 # default value + return small_file_size_mb + + _valid_auth_options: ClassVar[list[tuple[str, ValidAuthOption]]] = [ # type: ignore[misc] ( "url", @@ -81,19 +208,10 @@ class FirecrestTransport(Transport): "type": str, "non_interactive_default": False, "prompt": "Client Secret", - "help": "FirecREST client secret", + "help": "FirecREST client secret or Absolute path to an existing FirecREST Secret Key", + "callback": _create_secret_file, }, ), - # ( - # # TODO: format of secret file, and lookup secret by default in ~/.firecrest/secrets.json - # "secret_path", - # { - # "type": AbsolutePathOrEmptyParamType(dir_okay=False, exists=True), - # "non_interactive_default": False, - # "prompt": "Secret key file", - # "help": "Absolute path to file containing FirecREST client secret", - # }, - # ), ( "client_machine", { @@ -104,27 +222,24 @@ class FirecrestTransport(Transport): }, ), ( - # TODO you could potentially get this dynamically from server - # (via /status/parameters) "small_file_size_mb", { "type": float, - "default": 5.0, # limit set on the server is usually this + "default": 0, "non_interactive_default": True, - "prompt": "Maximum file size for direct transfer (MB)", + "prompt": "Maximum file size for direct transfer (MB) [Enter 0 to get this info from server]", "help": "Below this size, file bytes will be sent in a single API call.", - "callback": validate_positive_number, + "callback": _dynamic_info_direct_size, }, ), ( - "file_transfer_poll_interval", + "temp_directory", { - "type": float, - "default": 0.1, # TODO what default to choose? - "non_interactive_default": True, - "prompt": "File transfer poll interval (s)", - "help": "Poll interval when waiting for large file transfers.", - "callback": validate_positive_number, + "type": str, + "non_interactive_default": False, + "prompt": "Please enter a temp directory on server", + "help": "A temp directory on server for creating temporary files (compression, extraction, etc.)", + "callback": _validate_temp_directory, }, ), ] @@ -135,16 +250,32 @@ def __init__( url: str, token_uri: str, client_id: str, - client_secret: str | Path, + client_secret: str, #| Path, + # unfortunately we cannot store client_secret as a Path, because it is not JSON serializable client_machine: str, - small_file_size_mb: float = 5.0, - file_transfer_poll_interval: float = 0.1, + small_file_size_mb: float, + temp_directory: str, + # configured: bool = True, # note, machine is provided by default, # for the hostname, but we don't use that # TODO ideally hostname would not be necessary on a computer **kwargs: Any, ): - """Construct a FirecREST transport.""" + """Construct a FirecREST transport object. + + :param url: URL to the FirecREST server + :param token_uri: URI for retrieving FirecREST authentication tokens + :param client_id: FirecREST client ID + :param client_secret: FirecREST client secret or Absolute path to an existing FirecREST Secret Key + :param client_machine: FirecREST machine secret + :param small_file_size_mb: Maximum file size for direct transfer (MB) + :param temp_directory: A temp directory on server for creating temporary files (compression, extraction, etc.) + :param kwargs: Additional keyword arguments + """ + + + print("this is being done with firecrest transport") + # there is no overhead for "opening" a connection to a REST-API, # but still allow the user to set a safe interval if they really want to kwargs.setdefault("safe_interval", 0) @@ -153,40 +284,35 @@ def __init__( assert isinstance(url, str), "url must be a string" assert isinstance(token_uri, str), "token_uri must be a string" assert isinstance(client_id, str), "client_id must be a string" - assert isinstance( - client_secret, (str, Path) - ), "client_secret must be a string or Path" - + assert isinstance(client_secret, str), "client_secret must be a string" assert isinstance(client_machine, str), "client_machine must be a string" - assert isinstance( - small_file_size_mb, float - ), "small_file_size_mb must be a float" - assert isinstance( - file_transfer_poll_interval, float - ), "file_transfer_poll_interval must be a float" + assert isinstance(temp_directory, str), "temp_directory must be a string" + assert isinstance(small_file_size_mb, float), "small_file_size_mb must be a float" + self._machine = client_machine self._url = url self._token_uri = token_uri self._client_id = client_id + self._temp_directory = Path(temp_directory) self._small_file_size_bytes = int(small_file_size_mb * 1024 * 1024) - self._file_transfer_poll_interval = file_transfer_poll_interval - - secret = ( - client_secret.read_text() - if isinstance(client_secret, Path) - else client_secret - ) + secret = Path(client_secret).read_text() + + try: + self._client = Firecrest( + firecrest_url=self._url, + authorization=ClientCredentialsAuth(client_id, secret, token_uri), + ) + except Exception as e: + raise ValueError(f"Could not connect to FirecREST server: {e}") + + self._cwd: FcPath = FcPath(self._client, self._machine, "/", cache_enabled=True) + + + def __str__(self): + """Return the name of the plugin.""" + return self.__class__.__name__ - self._client = Firecrest( - firecrest_url=self._url, - authorization=ClientCredentialsAuth(client_id, secret, token_uri), - ) - - self._cwd: FcPath = FcPath(self._client, self._machine, "/") - - # TODO if this is missing is causes plugin info to fail on verdi - is_process_function = False @classmethod def get_description(cls) -> str: @@ -200,36 +326,46 @@ def get_description(cls) -> str: ) def open(self) -> None: # noqa: A003 + """Open the transport. + This is a no-op for the REST-API, as there is no connection to open. + """ pass def close(self) -> None: + """Close the transport. + This is a no-op for the REST-API, as there is no connection to close. + """ pass def getcwd(self) -> str: + """Return the current working directory.""" return str(self._cwd) def _get_path(self, *path: str) -> str: + """Return the path as a string.""" return posixpath.normpath(self._cwd.joinpath(*path)) def chdir(self, path: str) -> None: + """Change the current working directory.""" new_path = self._cwd.joinpath(path) if not new_path.is_dir(): raise OSError(f"'{new_path}' is not a valid directory") self._cwd = new_path - def normalize(self, path: str = ".") -> str: - return posixpath.normpath(path) - def chmod(self, path: str, mode: str) -> None: + """Change the mode of a file.""" self._cwd.joinpath(path).chmod(mode) def chown(self, path: str, uid: str, gid: str) -> None: + """Change the owner of a file.""" self._cwd.joinpath(path).chown(uid, gid) def path_exists(self, path: str) -> bool: + """Check if a path exists on the remote.""" return self._cwd.joinpath(path).exists() def get_attribute(self, path: str) -> FileAttribute: + """Get the attributes of a file.""" result = self._cwd.joinpath(path).stat() return FileAttribute( # type: ignore { @@ -243,13 +379,24 @@ def get_attribute(self, path: str) -> FileAttribute: ) def isdir(self, path: str) -> bool: + """Check if a path is a directory.""" return self._cwd.joinpath(path).is_dir() def isfile(self, path: str) -> bool: + """Check if a path is a file.""" return self._cwd.joinpath(path).is_file() - def listdir(self, path: str = ".", pattern: str | None = None) -> list[str]: - names = [p.name for p in self._cwd.joinpath(path).iterdir()] + def listdir(self, path: str = ".", pattern: str | None = None, recursive: bool = False) -> list[str]: + """List the contents of a directory. + + :param pattern: Unix shell-style wildcards to match the pattern: + - `*` matches everything + - `?` matches any single character + - `[seq]` matches any character in seq + - `[!seq]` matches any character not in seq + :param recursive: If True, list directories recursively + """ + names = [p.relpath(path).as_posix() for p in self._cwd.joinpath(path).iterdir(recursive=recursive)] if pattern is not None: names = fnmatch.filter(names, pattern) return names @@ -257,6 +404,18 @@ def listdir(self, path: str = ".", pattern: str | None = None) -> list[str]: # TODO the default implementations of glob / iglob could be overriden # to be more performant, using cached FcPaths and https://github.com/chrisjsewell/virtual-glob + def makedirs(self, path: str, ignore_existing: bool = False) -> None: + """Make directories on the remote.""" + self._cwd.joinpath(path).mkdir(parents=True, exist_ok=ignore_existing) + + def mkdir(self, path: str, ignore_existing: bool = False) -> None: + """Make a directory on the remote.""" + self._cwd.joinpath(path).mkdir(exist_ok=ignore_existing) + + def normalize(self, path: str = ".") -> str: + """Resolve the path.""" + return posixpath.normpath(path) + def write_binary(self, path: str, data: bytes) -> None: """Write bytes to a file on the remote.""" # Note this is not part of the Transport interface, but is useful for testing @@ -270,6 +429,7 @@ def read_binary(self, path: str) -> bytes: return self._cwd.joinpath(path).read_bytes() def symlink(self, remotesource: str, remotedestination: str) -> None: + """Create a symlink on the remote.""" source = self._cwd.joinpath(remotesource) destination = self._cwd.joinpath(remotedestination) destination.symlink_to(source) @@ -277,36 +437,52 @@ def symlink(self, remotesource: str, remotedestination: str) -> None: def copyfile( self, remotesource: str, remotedestination: str, dereference: bool = False ) -> None: - source = self._cwd.joinpath(remotesource).enable_cache() - destination = self._cwd.joinpath(remotedestination).enable_cache() - + """Copy a file on the remote. FirecREST does not support symlink copying. + + :param dereference: If True, copy the target of the symlink instead of the symlink itself. + Warning! even if deference is set to False, I'm not sure if the symlink will be functional after the copy. + """ + source = self._cwd.joinpath(remotesource)#.enable_cache() it's removed from from path.py to be investigated + destination = self._cwd.joinpath(remotedestination)#.enable_cache() it's removed from from path.py to be investigated + if dereference: + raise NotImplementedError("copyfile() does not support symlink dereference") if not source.exists(): raise FileNotFoundError(f"Source file does not exist: {source}") if not source.is_file(): raise FileNotFoundError(f"Source is not a file: {source}") - if not dereference and source.is_symlink(): - destination.symlink_to(source) - else: - source.copy_to(destination) + source.copy_to(destination) + # I removed symlink copy, becasue it's really not a file copy, it's a link copy + # and aiida-ssh have it in buggy manner, prrobably it's not used anyways + def copytree( self, remotesource: str, remotedestination: str, dereference: bool = False ) -> None: - source = self._cwd.joinpath(remotesource).enable_cache().enable_cache() + """Copy a directory on the remote. FirecREST does not support symlink copying. + + :param dereference: If True, copy the target of the symlink instead of the symlink itself. + Warning! even if deference is set to False, I'm not sure if the symlink will be functional after the copy. + """ + + source = self._cwd.joinpath(remotesource)#.enable_cache().enable_cache() it's removed from from path.py to be investigated destination = ( - self._cwd.joinpath(remotedestination).enable_cache().enable_cache() + self._cwd.joinpath(remotedestination)#.enable_cache().enable_cache() it's removed from from path.py to be investigated ) - + if dereference: + raise NotImplementedError("Dereferencing not implemented in FirecREST server") if not source.exists(): raise FileNotFoundError(f"Source file does not exist: {source}") if not source.is_dir(): raise FileNotFoundError(f"Source is not a directory: {source}") - if not dereference and source.is_symlink(): - destination.symlink_to(source) - else: - source.copy_to(destination) + source.copy_to(destination) + # TODO: the block belowe does not work for nested symlinks, if we really need that, + # we have to asked them to make this option for us in FirecREST. + # if not dereference and source.is_symlink(): + # destination.symlink_to(source) + # else: + # source.copy_to(destination) def copy( self, @@ -315,45 +491,67 @@ def copy( dereference: bool = False, recursive: bool = True, ) -> None: + """Copy a file or directory on the remote. FirecREST does not support symlink copying. + + :param recursive: If True, copy directories recursively. + note that the non-recursive option is not implemented in FirecREST server + + :param dereference: If True, copy the target of the symlink instead of the symlink itself. + Warning! even if deference is set to False, I'm not sure if the symlink will be functional after the copy. + """ + if not recursive: # TODO this appears to not actually be used upstream, so just remove there raise NotImplementedError("Non-recursive copy not implemented") - source = self._cwd.joinpath(remotesource).enable_cache() - destination = self._cwd.joinpath(remotedestination).enable_cache() + if dereference: + raise NotImplementedError("Dereferencing not implemented in FirecREST server") + source = self._cwd.joinpath(remotesource)#.enable_cache() it's removed from from path.py to be investigated + destination = self._cwd.joinpath(remotedestination)#.enable_cache() it's removed from from path.py to be investigated if not source.exists(): raise FileNotFoundError(f"Source file does not exist: {source}") - if not dereference and source.is_symlink(): - destination.symlink_to(source) - else: - source.copy_to(destination) + source.copy_to(destination) + # TODO: the block belowe does not work for nested symlinks, if we really need that, + # we have to asked them to make this option for us + # if not dereference and source.is_symlink(): + # destination.symlink_to(source) + # else: + # source.copy_to(destination) - def makedirs(self, path: str, ignore_existing: bool = False) -> None: - self._cwd.joinpath(path).mkdir(parents=True, exist_ok=ignore_existing) - - def mkdir(self, path: str, ignore_existing: bool = False) -> None: - self._cwd.joinpath(path).mkdir(exist_ok=ignore_existing) # TODO check symlink handling for get methods + # symlink handeling is done. # TODO do get/put methods need to handle glob patterns? + # Apparently not, but I'm not clear how glob() iglob() are going to behave here. We may need to implement them. def getfile( - self, remotepath: str | FcPath, localpath: str | Path, *args: Any, **kwargs: Any + self, remotepath: str | FcPath, localpath: str | Path, dereference:bool = True, *args: Any, **kwargs: Any ) -> None: + """Get a file from the remote. + + :param dereference: If True, follow symlinks. + note: we don't support downloading symlinks, so dereference is always should be True + + """ + + if not dereference: + raise NotImplementedError("Getting symlinks with `dereference=False` is not supported") + local = Path(localpath) if not local.is_absolute(): raise ValueError("Destination must be an absolute path") remote = ( remotepath if isinstance(remotepath, FcPath) - else self._cwd.joinpath(remotepath).enable_cache() + else self._cwd.joinpath(remotepath)#.enable_cache() it's removed from from path.py to be investigated ) if not remote.is_file(): raise FileNotFoundError(f"Source file does not exist: {remote}") remote_size = remote.lstat().st_size - - with convert_header_exceptions({"machine": self._machine, "path": remote}): + # if not local.exists(): + # local.mkdir(parents=True) + with self._cwd.convert_header_exceptions({"machine": self._machine, "path": remote}): if remote_size < self._small_file_size_bytes: self._client.simple_download(self._machine, str(remote), localpath) else: @@ -363,78 +561,209 @@ def getfile( # to concurrently initiate internal file transfers to the object store (a.k.a. "staging area") # and downloading from the object store to the local machine - # this initiates the internal transfer of the file to the "staging area" + # I investigated asyncio, but it's not performant for this use case. + # Becasue in the end, FirecREST server ends up serializing the requests. + # see here: https://github.com/eth-cscs/pyfirecrest/issues/94 down_obj = self._client.external_download(self._machine, str(remote)) + down_obj.finish_download(local) + + self._validate_checksum(local, remote) + + + def _validate_checksum(self,localpath: str | Path, remotepath: str | FcPath) -> None: + """Validate the checksum of a file. + Useful for checking if a file was transferred correctly. + it uses sha256 hash to compare the checksum of the local and remote files. + + Raises: ValueError: If the checksums do not match. + """ + + local = Path(localpath) + if not local.is_absolute(): + raise ValueError("Destination must be an absolute path") + remote = ( + remotepath + if isinstance(remotepath, FcPath) + else self._cwd.joinpath(remotepath)#.enable_cache() it's removed from from path.py to be investigated + ) + if not remote.is_file(): + raise FileNotFoundError(f"Cannot calculate checksum for a directory: {remote}") + + sha256_hash = hashlib.sha256() + with open(local,"rb") as f: + for byte_block in iter(lambda: f.read(4096),b""): + sha256_hash.update(byte_block) + local_hash = sha256_hash.hexdigest() - # this waits for the file to be moved to the staging area - # TODO handle the transfer stalling (timeout?) and optimise the polling interval - while down_obj.in_progress: - time.sleep(self._file_transfer_poll_interval) - - # this downloads the file from the "staging area" - url = down_obj.object_storage_data - if ( - os.environ.get("FIRECREST_LOCAL_TESTING") - and platform.system() == "Darwin" - ): - # TODO when using the demo server on a Mac, the wrong IP is provided - # and even then a 403 error is returned, due to a signature mismatch - # note you can directly directly download the file from: - # "/path/to/firecrest/deploy/demo/minio" + urlparse(url).path - url = url.replace("192.168.220.19", "localhost") - with request.urlopen(url) as response, local.open("wb") as handle: - shutil.copyfileobj(response, handle) - - # TODO use cwd.checksum to confirm download is not corrupted? + remote_hash = self._client.checksum(self._machine, remote) + + try: + assert local_hash == remote_hash + except AssertionError: + raise ValueError(f"Checksum mismatch between local and remote files: {local} and {remote}") + + + def _gettreetar( + self, remotepath: str | FcPath, localpath: str | Path, dereference: bool =False, *args: Any, **kwargs: Any + ) -> None: + """Get a directory from the remote as a tar file and extract it locally. + This is useful for downloading a directory with many files, as it is more efficient than downloading each file individually. + Note that this method is not part of the Transport interface, and is not meant to be used publicly. + + :param dereference: If True, follow symlinks. + note: FirecREST doesn't support `--dereference` for tar call, so dereference should always be False, for now. + """ + # TODO manual testing the submit behaviour + + if dereference: + raise NotImplementedError("Dereferencing compression not implemented in pyFirecREST.") + + _ = uuid.uuid4() # Attempt direct compress + remote_path_temp = self._temp_directory.joinpath(f"temp_{_}.tar") + try: + self._client.compress(self._machine, str(remotepath), remote_path_temp) + except Exception as e: + # TODO: pyfirecrest is providing a solution to this, but it's not yet merged. + # once done submit_compress_job should be done automaticaly by compress + # see: https://github.com/eth-cscs/pyfirecrest/pull/109 + raise NotImplementedError("Not implemeted for now") + comp_obj = self._client.submit_compress_job(self._machine, str(remotepath), remote_path_temp) + while comp_obj.in_progress: + time.sleep(self._file_transfer_poll_interval) + + # Download + localpath_temp = localpath.joinpath(f"temp_{_}.tar") + try: + self.getfile(remote_path_temp, localpath_temp) + finally: + self.remove(remote_path_temp) + + # Extract the downloaded file locally + # this is a bit hard coded, what I wanted to do: to extract the files in the same directory as the tar file + try: + with tarfile.open(localpath_temp, "r") as tar: + members = [m for m in tar.getmembers() if m.name.startswith(remotepath.name)] + for member in members: + member.name = os.path.relpath(member.name, remotepath.name) + tar.extract(member, path=localpath) + finally: + localpath_temp.unlink() + def gettree( - self, remotepath: str | FcPath, localpath: str | Path, *args: Any, **kwargs: Any + self, remotepath: str | FcPath, localpath: str | Path, dereference: bool =True, *args: Any, **kwargs: Any ) -> None: + """Get a directory from the remote. + + :param dereference: If True, follow symlinks. + note: dereference should be always True, otherwise the symlinks will not be functional. + """ local = Path(localpath) if not local.is_absolute(): raise ValueError("Destination must be an absolute path") if local.is_file(): raise OSError("Cannot copy a directory into a file") - local.mkdir(parents=True, exist_ok=True) remote = ( remotepath if isinstance(remotepath, FcPath) - else self._cwd.joinpath(remotepath).enable_cache() + else self._cwd.joinpath(remotepath)#.enable_cache() it's removed from from path.py to be investigated ) + local = Path(localpath) + if not remote.is_dir(): raise OSError(f"Source is not a directory: {remote}") - for remote_item in remote.iterdir(): - local_item = local.joinpath(remote_item.name) - if remote_item.is_dir(): - self.gettree(remote_item, local_item) - else: - self.getfile(remote_item, local_item) - - def get(self, remotepath: str, localpath: str, *args: Any, **kwargs: Any) -> None: - remote = self._cwd.joinpath(remotepath).enable_cache() + + # this block is added only to mimick the behavior that aiida expects + if local.exists(): + # Destination directory already exists, create remote directory name inside it + local = local.joinpath(remote.name) + local.mkdir(parents=True, exist_ok=True) + else: + # Destination directory does not exist, create and move content 69 inside it + local.mkdir(parents=True, exist_ok=False) + # SSH transport behaviour, 69 is a directory + # transport.get('somepath/69', 'someremotepath/') == transport.put('somepath/69', 'someremotepath') + # transport.get('somepath/69', 'someremotepath/') == transport.put('somepath/69/', 'someremotepath/') + # transport.get('someremotepath/69', 'somepath/69') --> if 69 exist, create 69 inside it ('somepath/69/69') + # transport.get('someremotepath/69', 'somepath/69') --> if 69 no texist, create 69 inside it ('somepath/69') + # transport.get('somepath/69', 'someremotepath/6889/69') --> create everything, make_parent = True + + if not dereference and self.payoff(remote): + # in this case send a request to the server to tar the files and then download the tar file + # unfortunately, the server does not provide a deferenced tar option, yet. + self._gettreetar(remote, local) + else: + # otherwise download the files one by one + for remote_item in remote.iterdir(recursive=True): + local_item = local.joinpath(remote_item.relpath(remote)) + + if dereference and remote_item.is_symlink(): + target_path = remote_item._cache.link_target + if not Path(target_path).is_absolute(): + target_path = remote_item.parent.joinpath(target_path).resolve() + + target_path = self._cwd.joinpath(target_path) + if target_path.is_dir(): + self.gettree(target_path, local_item, dereference=True) + else: + target_path = remote_item + + if not target_path.is_dir(): + self.getfile(target_path, local_item) + else: + local_item.mkdir(parents=True, exist_ok=True) + + + + def get(self, remotepath: str, localpath: str, ignore_nonexisting: bool = False, dereference: bool =True, *args: Any, **kwargs: Any) -> None: + """Get a file or directory from the remote. + + :param ignore_nonexisting: If True, do not raise an error if the source file does not exist. + :param dereference: If True, follow symlinks. + note: dereference should be always True, otherwise the symlinks will not be functional. + """ + remote = self._cwd.joinpath(remotepath)#.enable_cache() it's removed from from path.py to be investigated + if remote.is_dir(): self.gettree(remote, localpath) elif remote.is_file(): self.getfile(remote, localpath) - else: + elif not ignore_nonexisting: raise FileNotFoundError(f"Source file does not exist: {remote}") + def putfile( - self, localpath: str, remotepath: str, *args: Any, **kwargs: Any + self, localpath: str, remotepath: str, dereference:bool = True, *args: Any, **kwargs: Any , ) -> None: - if not Path(localpath).is_absolute(): + """Put a file from the remote. + + :param dereference: If True, follow symlinks. + note: we don't support uploading symlinks, so dereference is always should be True + + """ + + if not dereference: + raise NotImplementedError("Getting symlinks with `dereference=False` is not supported") + + + localpath = Path(localpath) + if not localpath.is_absolute(): raise ValueError("The localpath must be an absolute path") - if not Path(localpath).is_file(): + if not localpath.is_file(): raise ValueError(f"Input localpath is not a file: {localpath}") - local_size = Path(localpath).stat().st_size - remote = self._cwd.joinpath(remotepath).enable_cache() + remote = self._cwd.joinpath(remotepath)#.enable_cache() it's removed from from path.py to be investigated + + + if remote.is_dir(): + raise ValueError(f"Destination is a directory: {remote}") + + local_size = localpath.stat().st_size # note this allows overwriting of existing files - with convert_header_exceptions({"machine": self._machine, "path": remote}): + with self._cwd.convert_header_exceptions({"machine": self._machine, "path": remote}): if local_size < self._small_file_size_bytes: self._client.simple_upload( - self._machine, localpath, str(remote.parent), remote.name - ) + self._machine, str(localpath), str(remote.parent), remote.name) else: # TODO the following is a very basic implementation of uploading a large file # ideally though, if uploading multiple large files (i.e. in puttree), @@ -442,108 +771,212 @@ def putfile( # to concurrently upload to the object store (a.k.a. "staging area"), # then wait for all files to finish being transferred to the target location - # this simply retrieves a location to upload on the "staging area" - up_obj = self._client.external_upload( - self._machine, localpath, str(remote) - ) - if ( - os.environ.get("FIRECREST_LOCAL_TESTING") - and platform.system() == "Darwin" - ): - # TODO when using the demo server on a Mac, the wrong IP is provided - up_obj.object_storage_data["command"] = up_obj.object_storage_data[ - "command" - ].replace("192.168.220.19", "localhost") - - # this uploads the file to the "staging area" - # TODO this calls curl in a subcommand, but you could also use the python requests library - # see: https://github.com/chrisjsewell/fireflow/blob/d45d41a0aced6502b7946c5557712a3c3cb1bebb/src/fireflow/process.py#L177 + # I investigated asyncio, but it's not performant for this use case. + # Becasue in the end, FirecREST server ends up serializing the requests. + # see here: https://github.com/eth-cscs/pyfirecrest/issues/94 + up_obj = self._client.external_upload(self._machine, str(localpath), str(remote)) up_obj.finish_upload() - # this waits for the file in the staging area to be moved to the final location - # TODO handle the transfer stalling (timeout?) and optimise the polling interval - while up_obj.in_progress: - time.sleep(self._file_transfer_poll_interval) - # TODO use cwd.checksum to confirm upload is not corrupted? + self._validate_checksum(localpath, str(remote)) + + def payoff( + self, remotepath: str + ) -> bool: + """ + This function will be used to determine whether to tar the files before downloading + """ + # After discussing with the pyfirecrest team, it seems that server has some sort + # of serialization and "penalty" for sending multiple requests asycnhronusly or in a short time window. + # It responses in 1, 1.5, 3, 5, 7 seconds! + # So right now, I think if the number of files is more than 3, it pays off to tar everything + if len(self.listdir(remotepath,recursive=True)) > 3: + return True + else: + return False + + def _puttreetar( + self, localpath: str | Path, remotepath: str | FcPath, dereference: bool=True, *args: Any, **kwargs: Any + ) -> None: + """Put a directory to the remote by sending as tar file in backend. + This is useful for uploading a directory with many files, as it is more efficient than uploading each file individually. + Note that this method is not part of the Transport interface, and is not meant to be used publicly. + + :param dereference: If True, follow symlinks. If False, symlinks are ignored from sending over. + """ + # this function will be used to send a folder as a tar file to the server and extract it on the server + + + import tarfile + import uuid + _ = uuid.uuid4() + + localpath = Path(localpath) + tarpath = localpath.parent.joinpath(f"temp_{_}.tar") + remote_path_temp = self._temp_directory.joinpath(f"temp_{_}.tar") + with tarfile.open(tarpath, "w", dereference=dereference) as tar: + if dereference: + for root, dirs, files in os.walk(localpath): + for file in files: + full_path = os.path.join(root, file) + relative_path = os.path.relpath(full_path, localpath) + tar.add(full_path, arcname=relative_path) + else: + # iterdir() ignores symbolic links + for item in localpath.iterdir(): + tar.add(item, arcname=item.name) + + # Upload + try: + self.putfile(tarpath, remote_path_temp) + finally: + tarpath.unlink() + # Attempt direct extract + try: + self._client.extract(self._machine, remote_path_temp, str(remotepath)) + except Exception as e: + # TODO: pyfirecrest is providing a solution to this, but it's not yet merged + # once done submit_compress_job should be done automaticaly by compress + # see: https://github.com/eth-cscs/pyfirecrest/pull/109 + raise NotImplementedError("Not implemeted for now") + comp_obj = self._client.submit_extract_job(self._machine, remotepath.joinpath(f"_{_}.tar"), str(remotepath)) + while comp_obj.in_progress: + time.sleep(self._file_transfer_poll_interval) + finally: + self.remove(remote_path_temp) + def puttree( - self, localpath: str | Path, remotepath: str, *args: Any, **kwargs: Any + self, localpath: str | Path, remotepath: str, dereference: bool=True, *args: Any, **kwargs: Any ) -> None: + """Put a directory to the remote. + + :param dereference: If True, follow symlinks. + note: dereference should be always True, otherwise the symlinks will not be functional, therfore not supported. + """ + if not dereference: + raise NotImplementedError + localpath = Path(localpath) + remotepath = self._cwd.joinpath(remotepath) - # local checks if not localpath.is_absolute(): raise ValueError("The localpath must be an absolute path") if not localpath.exists(): raise OSError("The localpath does not exists") if not localpath.is_dir(): - raise ValueError(f"Input localpath is not a folder: {localpath}") + raise ValueError(f"Input localpath is not a directory: {localpath}") + + # this block is added only to mimick the behavior that aiida expects + if remotepath.exists(): + # Destination directory already exists, create local directory name inside it + remotepath = self._cwd.joinpath(remotepath, localpath.name) + self.mkdir(remotepath, ignore_existing=False) + else: + # Destination directory does not exist, create and move content 69 inside it + self.mkdir(remotepath, ignore_existing=False) + # SSH transport behaviour + # transport.put('somepath/69', 'someremotepath/') == transport.put('somepath/69', 'someremotepath') + # transport.put('somepath/69', 'someremotepath/') != transport.put('somepath/69/', 'someremotepath/') + # transport.put('somepath/69', 'someremotepath/67') --> if 67 not exist, create and move content 69 inside it (someremotepath/67) + # transport.put('somepath/69', 'someremotepath/67') --> if 67 exist, create 69 inside it (someremotepath/67/69) + # transport.put('somepath/69', 'someremotepath/6889/69') --> useless Error: OSError + # Weired + # SSH bug: + # transport.put('somepath/69', 'someremotepath/') --> assuming someremotepath exists, make 69 + # while + # transport.put('somepath/69/', 'someremotepath/') --> assuming someremotepath exists, OSError: cannot make someremotepath + + + if self.payoff(remotepath): + # in this case send send everything as a tar file + self._puttreetar(localpath, remotepath) + else: + # otherwise send the files one by one + for dirpath, _, filenames in os.walk(localpath): + rel_folder = os.path.relpath(path=dirpath, start=localpath) + + rm_parent_now = remotepath.joinpath(rel_folder) + self.mkdir(rm_parent_now, ignore_existing=True) - for dirpath, _, filenames in os.walk(localpath): - # Get the relative path - rel_folder = os.path.relpath(path=dirpath, start=localpath) + for filename in filenames: + localfile_path = os.path.join(localpath, rel_folder, filename) + remotefile_path = rm_parent_now.joinpath( filename) + self.putfile(localfile_path, remotefile_path) - if not self.path_exists(os.path.join(remotepath, rel_folder)): - self.mkdir(os.path.join(remotepath, rel_folder)) - for filename in filenames: - localfile_path = os.path.join(localpath, rel_folder, filename) - remotefile_path = os.path.join(remotepath, rel_folder, filename) - self.putfile(localfile_path, remotefile_path) + def put(self, localpath: str, remotepath: str, ignore_nonexisting: bool =False, dereference: bool=True, *args: Any, **kwargs: Any) -> None: + """Put a file or directory to the remote. - def put(self, localpath: str, remotepath: str, *args: Any, **kwargs: Any) -> None: + :param ignore_nonexisting: If True, do not raise an error if the source file does not exist. + :param dereference: If True, follow symlinks. + note: dereference should be always True, otherwise the symlinks will not be functional. + """ # TODO ssh does a lot more - if os.path.isdir(localpath): + # update to above TODO: I made a manual test with ssh. added some extra care in puttree and gettree and now it's working fine + + if not dereference: + raise NotImplementedError + + localpath = Path(localpath) + if not localpath.is_absolute(): + raise ValueError("The localpath must be an absolute path") + if not Path(localpath).exists() and not ignore_nonexisting: + raise FileNotFoundError(f"Source file does not exist: {localpath}") + + if localpath.is_dir(): self.puttree(localpath, remotepath) - elif os.path.isfile(localpath): - if self.isdir(remotepath): - remote = os.path.join(remotepath, os.path.split(localpath)[1]) - self.putfile(localpath, remote) - else: - self.putfile(localpath, remotepath) + elif localpath.is_file(): + self.putfile(localpath, remotepath) + def remove(self, path: str) -> None: + """Remove a file or directory on the remote.""" self._cwd.joinpath(path).unlink() def rename(self, oldpath: str, newpath: str) -> None: + """Rename a file or directory on the remote.""" self._cwd.joinpath(oldpath).rename(self._cwd.joinpath(newpath)) def rmdir(self, path: str) -> None: - # TODO check if empty - self._cwd.joinpath(path).rmtree() + """Remove a directory on the remote. + If the directory is not empty, an OSError is raised.""" + + if len(self.listdir(path)) == 0: + self._cwd.joinpath(path).rmtree() + else: + raise OSError(f"Directory not empty: {path}") def rmtree(self, path: str) -> None: - self._cwd.joinpath(path).rmtree() + """Remove a directory on the remote. + If the directory is not empty, it will be removed recursively, equivalent to `rm -rf`. + It does not raise an error if the directory does not exist. + """ + try: + self._cwd.joinpath(path).rmtree() + # TODO: this try&except is to mimick the behaviour of `aiida-ssh`` transport, TODO: raise an issue on aiida + except FileNotFoundError: + pass def whoami(self) -> str: - return self._cwd.whoami() + """Return the username of the current user.""" + return self._client.whoami(machine = self._machine) def gotocomputer_command(self, remotedir: str) -> str: + """Not possible for REST-API. + It's here only because it's an abstract method in the base class.""" # TODO remove from interface raise NotImplementedError("firecrest does not support gotocomputer_command") def _exec_command_internal(self, command: str, **kwargs: Any) -> Any: + """Not possible for REST-API. + It's here only because it's an abstract method in the base class.""" # TODO remove from interface raise NotImplementedError("firecrest does not support command execution") def exec_command_wait_bytes( self, command: str, stdin: Any = None, **kwargs: Any ) -> Any: + """Not possible for REST-API. + It's here only because it's an abstract method in the base class.""" # TODO remove from interface raise NotImplementedError("firecrest does not support command execution") - - -def validate_non_empty_string(ctx, param, value): # type: ignore - """Validate that the number passed to this parameter is a positive number. - - :param ctx: the `click.Context` - :param param: the parameter - :param value: the value passed for the parameter - :raises `click.BadParameter`: if the value is not a positive number - """ - if not isinstance(value, str) or not value.strip(): - from click import BadParameter - - raise BadParameter(f"{value} is not string or is empty") - - return value diff --git a/aiida_firecrest/utils_test.py b/aiida_firecrest/utils_test.py index f5c2801..a7a02e7 100644 --- a/aiida_firecrest/utils_test.py +++ b/aiida_firecrest/utils_test.py @@ -26,6 +26,7 @@ class FirecrestConfig: client_secret: str machine: str scratch_path: str + temp_directory: str small_file_size_mb: float = 1.0 @@ -47,10 +48,15 @@ def __init__( self._scratch = tmpdir / "scratch" self._scratch.mkdir() self._client_id = "test_client_id" - self._client_secret = "test_client_secret" + + Path(tmpdir / ".firecrest").mkdir() + self._client_secret = tmpdir / ".firecrest/secret" + self._client_secret.write_text("test_client_secret") + self._token_url = "https://test.auth.com/token" self._token_url_parsed = urlparse(self._token_url) self._username = "test_user" + self._temp_directory = tmpdir / "temp" self._slurm_job_id_counter = 0 self._slurm_jobs: dict[str, dict[str, Any]] = {} @@ -70,9 +76,10 @@ def config(self) -> FirecrestConfig: url=self._url, token_uri=self._token_url, client_id=self._client_id, - client_secret=self._client_secret, + client_secret=str(self._client_secret.absolute()), machine=self._machine, scratch_path=str(self._scratch.absolute()), + temp_directory=str(self._temp_directory.absolute()), ) def mock_request( @@ -117,6 +124,8 @@ def mock_request( self.utilities_file(params or {}, response) elif endpoint == "/utilities/ls": self.utilities_ls(params or {}, response) + elif endpoint == "/utilities/checksum": + self.utilities_checksum(params or {}, response) elif endpoint == "/utilities/symlink": self.utilities_symlink(data or {}, response) elif endpoint == "/utilities/mkdir": @@ -139,8 +148,9 @@ def mock_request( self.compute_jobs_path(data or {}, response) elif endpoint == "/compute/jobs": self.compute_jobs(params or {}, response) - elif endpoint.startswith("/tasks/"): - self.handle_task(endpoint[7:], response) + elif endpoint.startswith("/tasks"): + self.handle_task(params or {}, response) + # self.handle_task(endpoint[7:], response) elif endpoint == "/storage/xfer-external/upload": self.storage_xfer_external_upload(data or {}, response) elif endpoint == "/storage/xfer-external/download": @@ -383,11 +393,28 @@ def utilities_ls(self, params: dict[str, Any], response: Response) -> None: "permissions": stat.filemode(_stat.st_mode)[1:], "type": "l" if item.is_symlink() else "d" if item.is_dir() else "-", "size": _stat.st_size, + "link_target": item.readlink() if item.is_symlink() else None, } ) add_success_response(response, 200, data) + def utilities_checksum(self, params: dict[str, Any], response: Response) -> None: + path = Path(params["targetPath"]) + if not path.exists(): + response.status_code = 400 + response.headers["X-Invalid-Path"] = "" + return + import hashlib + # Firecrest uses sha256 + sha256_hash = hashlib.sha256() + with open(path,"rb") as f: + for byte_block in iter(lambda: f.read(4096),b""): + sha256_hash.update(byte_block) + + checksum = sha256_hash.hexdigest() + add_success_response(response, 200, checksum) + def utilities_chmod(self, data: dict[str, Any], response: Response) -> None: path = Path(data["targetPath"]) if not path.exists(): @@ -498,7 +525,9 @@ def utilities_download(self, params: dict[str, Any], response: Response) -> None response.status_code = 200 response.raw = io.BytesIO(path.read_bytes()) - def handle_task(self, task_id: str, response: Response) -> Response: + # def handle_task(self, task_id: str, response: Response) -> Response: + def handle_task(self, params: dict[str, Any], response: Response) -> Response: + task_id = params["tasks"].split(',')[0] if task_id not in self._tasks: return add_json_response( response, 404, {"error": f"Task {task_id} does not exist"} @@ -618,7 +647,32 @@ def task_storage_xfer_external_upload( # this skips statuses 111, 112 and 113, # see: https://github.com/eth-cscs/pyfirecrest/blob/5edbe85d11b977ed8f6943ca22e4fdc3d6f5e12d/firecrest/BasicClient.py#L143 # and so we are assuming that the file is uploaded to the server - + # I haven't updated the code to reflect this yet, but the new format of tasks is as follows: + # { + # 'b1fbee4afa1e52fb54a3a38aede7c246': { + # 'created_at': '2024-06-13T17:01:25', + # 'data': { + # 'hash_id': 'b1fbee4afa1e52fb54a3a38aede7c246', + # 'msg': 'Waiting for Presigned URL to upload file to staging area (Amazon S3 - Signature v4)', + # 'source': 'RM.mkv', + # 'status': '110', + # 'system_addr': 'domvm3.cscs.ch:22', + # 'system_name': 'dom', + # 'target': '/scratch/snx3000tds/akhosrav/delete_me/linkto_target', + # 'trace_id': '', + # 'user': 'akhosrav' + # }, + # 'description': 'Waiting for Form URL from Object Storage to be retrieved', + # 'hash_id': 'b1fbee4afa1e52fb54a3a38aede7c246', + # 'last_modify': '2024-06-13T17:01:25', + # 'service': 'storage', + # 'status': '110', + # 'system': 'dom', + # 'task_id': 'b1fbee4afa1e52fb54a3a38aede7c246', + # 'updated_at': '2024-06-13T17:01:25', + # 'user': 'akhosrav' + # } + # } if not task.form_retrieved: task.form_retrieved = True return add_json_response( diff --git a/tests/conftest.py b/tests/conftest.py index 2481be1..88aa401 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -109,6 +109,10 @@ def firecrest_server( monkeypatch.setattr(requests, "post", mock_request) monkeypatch.setattr(requests, "put", mock_request) monkeypatch.setattr(requests, "delete", mock_request) + monkeypatch.setattr(requests.Session, "get", mock_request) + monkeypatch.setattr(requests.Session, "post", mock_request) + monkeypatch.setattr(requests.Session, "put", mock_request) + monkeypatch.setattr(requests.Session, "delete", mock_request) yield server.config # save data on the server diff --git a/tests/test_computer.py b/tests/test_computer.py index 69faf82..a8a3179 100644 --- a/tests/test_computer.py +++ b/tests/test_computer.py @@ -32,6 +32,7 @@ def _firecrest_computer(firecrest_server: FirecrestConfig): client_secret=firecrest_server.client_secret, client_machine=firecrest_server.machine, small_file_size_mb=firecrest_server.small_file_size_mb, + temp_directory=firecrest_server.temp_directory, ) return computer @@ -40,7 +41,8 @@ def _firecrest_computer(firecrest_server: FirecrestConfig): def test_whoami(firecrest_computer: orm.Computer): """check if it is possible to determine the username.""" transport = firecrest_computer.get_transport() - assert isinstance(transport.whoami(), str) + assert transport.whoami() == 'test_user' + @pytest.mark.usefixtures("aiida_profile_clean") diff --git a/tests_new/conftest.py b/tests_new/conftest.py new file mode 100644 index 0000000..e51a640 --- /dev/null +++ b/tests_new/conftest.py @@ -0,0 +1,191 @@ +from __future__ import annotations + +from pathlib import Path +import os + +from _pytest.terminal import TerminalReporter +import firecrest.path +import pytest +import firecrest + + +class MockFirecrest: + def __init__(self, firecrest_url, *args, **kwargs): + self._firecrest_url = firecrest_url + self.args = args + self.kwargs = kwargs + self.whoami = mock_whomai + self.list_files = list_files + self.stat = stat_ + self.mkdir = mkdir + self.simple_delete = simple_delete + self.parameters = parameters + self.symlink = symlink + +class MockClientCredentialsAuth: + def __init__(self, *args, **kwargs): + self.args = args + self.kwargs = kwargs + +@pytest.fixture(scope="function") +def myfirecrest( + pytestconfig: pytest.Config, + monkeypatch, +): + + monkeypatch.setattr(firecrest, "Firecrest", MockFirecrest) + monkeypatch.setattr(firecrest, "ClientCredentialsAuth", MockClientCredentialsAuth) + # monkeypatch.setattr(firecrest.path, "_ls_to_st_mode", _ls_to_st_mode) + + + +def mock_whomai(machine: str): + assert machine == "MACHINE_NAME" + return "test_user" + + +import stat + +def list_files( + machine: str, target_path: str, recursive: bool = False, show_hidden: bool = False): + # this is mimiking the expected behaviour from the firecrest code. + + content_list = [] + for root, dirs, files in os.walk(target_path): + if not recursive and root != target_path: + continue + for name in dirs + files: + full_path = os.path.join(root, name) + relative_path = Path(os.path.relpath(root, target_path)).joinpath(name).as_posix() + # breakpoint() + print(relative_path) + if os.path.isdir(full_path): + content_type = 'd' + elif os.path.isfile(full_path): + content_type = '-' + elif os.path.islink(full_path): + content_type = 'l' + else: + content_type = 'NON' + link_target = os.readlink(full_path) if os.path.islink(full_path) else None + # permissions = stat.S_IMODE(os.lstat(full_path).st_mode) + permissions = stat.filemode(Path(full_path).lstat().st_mode)[1:] + # stat.S_ISREG(permissions) + if name.startswith('.') and not show_hidden: + continue + content_list.append({ + 'name': relative_path, + 'type': content_type, + 'link_target': link_target, + 'permissions': permissions, + }) + + return content_list + +# def _ls_to_st_mode(ftype: str, permissions: str) -> int: +# return int(permissions) + +def stat_(machine:str, targetpath: firecrest.path, dereference=True): + stats = os.stat(targetpath) + return { + "ino": stats.st_ino, + "dev": stats.st_dev, + "nlink": stats.st_nlink, + "uid": stats.st_uid, + "gid": stats.st_gid, + "size": stats.st_size, + "atime": stats.st_atime, + "mtime": stats.st_mtime, + "ctime": stats.st_ctime, + } + + +def mkdir(machine: str, target_path: str, p: bool = False): + if p: + os.makedirs(target_path) + else: + os.mkdir(target_path) + +def simple_delete(machine: str, target_path: str): + if not Path(target_path).exists(): + raise FileNotFoundError(f"File or folder {target_path} does not exist") + if os.path.isdir(target_path): + os.rmdir(target_path) + else: + os.remove(target_path) + +def symlink(machine: str, target_path: str, link_path: str): + os.symlink(target_path, link_path) + + +def parameters(): + # note: I took this from https://firecrest-tds.cscs.ch/ or https://firecrest.cscs.ch/ + # if code is not working but test passes, it means you need to update this dictionary + # with the latest FirecREST parameters + return { + "compute": [ + { + "description": "Type of resource and workload manager used in compute microservice", + "name": "WORKLOAD_MANAGER", + "unit": "", + "value": "Slurm" + } + ], + "storage": [ + { + "description": "Type of object storage, like `swift`, `s3v2` or `s3v4`.", + "name": "OBJECT_STORAGE", + "unit": "", + "value": "s3v4" + }, + { + "description": "Expiration time for temp URLs.", + "name": "STORAGE_TEMPURL_EXP_TIME", + "unit": "seconds", + "value": "86400" + }, + { + "description": "Maximum file size for temp URLs.", + "name": "STORAGE_MAX_FILE_SIZE", + "unit": "MB", + "value": "5120" + }, + { + "description": "Available filesystems through the API.", + "name": "FILESYSTEMS", + "unit": "", + "value": [ + { + "mounted": [ + "/project", + "/store", + "/scratch/snx3000tds" + ], + "system": "dom" + }, + { + "mounted": [ + "/project", + "/store", + "/capstor/scratch/cscs" + ], + "system": "pilatus" + } + ] + } + ], + "utilities": [ + { + "description": "The maximum allowable file size for various operations of the utilities microservice", + "name": "UTILITIES_MAX_FILE_SIZE", + "unit": "MB", + "value": "69" + }, + { + "description": "Maximum time duration for executing the commands in the cluster for the utilities microservice.", + "name": "UTILITIES_TIMEOUT", + "unit": "seconds", + "value": "5" + } + ] + } \ No newline at end of file diff --git a/tests_new/test_transport.py b/tests_new/test_transport.py new file mode 100644 index 0000000..288f6cb --- /dev/null +++ b/tests_new/test_transport.py @@ -0,0 +1,250 @@ +from pathlib import Path +import os + +import pytest +from unittest.mock import Mock +from click import BadParameter + +from aiida import orm + + +@pytest.fixture(name="firecrest_computer") +def _firecrest_computer(myfirecrest, tmpdir: Path): + """Create and return a computer configured for Firecrest. + + Note, the computer is not stored in the database. + """ + + # create a temp directory and set it as the workdir + _scratch = tmpdir / "scratch" + _scratch.mkdir() + _temp_directory = tmpdir / "temp" + + Path(tmpdir / ".firecrest").mkdir() + _secret_path = Path(tmpdir / ".firecrest/secret69") + _secret_path.write_text("SECRET_STRING") + + computer = orm.Computer( + label="test_computer", + description="test computer", + hostname="-", + workdir=str(_scratch), + transport_type="firecrest", + scheduler_type="firecrest", + ) + computer.set_minimum_job_poll_interval(5) + computer.set_default_mpiprocs_per_machine(1) + computer.configure( + url=' https://URI', + token_uri='https://TOKEN_URI', + client_id='CLIENT_ID', + client_secret=str(_secret_path), + client_machine='MACHINE_NAME', + small_file_size_mb=1.0, + temp_directory=str(_temp_directory), + ) + return computer + + +@pytest.mark.usefixtures("aiida_profile_clean") +def test_whoami(firecrest_computer: orm.Computer): + """check if it is possible to determine the username.""" + transport = firecrest_computer.get_transport() + assert transport.whoami() == 'test_user' + +def test_create_secret_file_with_existing_file(tmpdir: Path): + from aiida_firecrest.transport import FirecrestTransport + secret_file = Path(tmpdir / "secret") + secret_file.write_text("topsecret") + result = FirecrestTransport._create_secret_file(None, None, str(secret_file)) + assert isinstance(result, str) + assert result == str(secret_file) + assert Path(result).read_text() == "topsecret" + +def test_create_secret_file_with_nonexistent_file(tmp_path): + from aiida_firecrest.transport import FirecrestTransport + secret_file = tmp_path / "nonexistent" + with pytest.raises(BadParameter): + FirecrestTransport._create_secret_file(None, None, str(secret_file)) + +def test_create_secret_file_with_secret_value(tmp_path, monkeypatch): + from aiida_firecrest.transport import FirecrestTransport + secret = "topsecret!~/" + monkeypatch.setattr(Path, "expanduser", lambda x: tmp_path / str(x).lstrip("~/") if str(x).startswith("~/") else x) + result = FirecrestTransport._create_secret_file(None, None, secret) + assert Path(result).parent.parts[-1]== ".firecrest" + assert Path(result).read_text() == secret + +def test_validate_temp_directory(myfirecrest, monkeypatch, tmpdir: Path): + from aiida_firecrest.transport import FirecrestTransport + + monkeypatch.setattr('click.echo', lambda x: None) + # monkeypatch.setattr('click.BadParameter', lambda x: None) + secret_file = Path(tmpdir / "secret") + secret_file.write_text("topsecret") + ctx = Mock() + ctx.params = { + 'url': 'http://test.com', + 'token_uri': 'token_uri', + 'client_id': 'client_id', + 'client_machine': 'client_machine', + 'client_secret': secret_file.as_posix(), + 'small_file_size_mb': float(10) + } + + # should raise if is_file + Path(tmpdir / 'crap.txt').touch() + with pytest.raises(BadParameter): + result = FirecrestTransport._validate_temp_directory(ctx, None, Path(tmpdir /'crap.txt').as_posix()) + + # should create the directory if it doesn't exist + result = FirecrestTransport._validate_temp_directory(ctx, None, Path(tmpdir /'temp_on_server_directory').as_posix()) + assert result == Path(tmpdir /'temp_on_server_directory').as_posix() + assert Path(tmpdir /'temp_on_server_directory').exists() + + # should get a confirmation if the directory exists and is not empty + Path(tmpdir /'temp_on_server_directory' / 'crap.txt').touch() + monkeypatch.setattr('click.confirm', lambda x: False) + with pytest.raises(BadParameter): + result = FirecrestTransport._validate_temp_directory(ctx, None, Path(tmpdir /'temp_on_server_directory').as_posix()) + + # should delete the content if I confirm + monkeypatch.setattr('click.confirm', lambda x: True) + result = FirecrestTransport._validate_temp_directory(ctx, None, Path(tmpdir /'temp_on_server_directory').as_posix()) + assert result == Path(tmpdir /'temp_on_server_directory').as_posix() + assert not Path(tmpdir /'temp_on_server_directory' / 'crap.txt').exists() + +def test__dynamic_info(myfirecrest, monkeypatch, tmpdir: Path): + from aiida_firecrest.transport import FirecrestTransport + + monkeypatch.setattr('click.echo', lambda x: None) + # monkeypatch.setattr('click.BadParameter', lambda x: None) + secret_file = Path(tmpdir / "secret") + secret_file.write_text("topsecret") + ctx = Mock() + ctx.params = { + 'url': 'http://test.com', + 'token_uri': 'token_uri', + 'client_id': 'client_id', + 'client_machine': 'client_machine', + 'client_secret': secret_file.as_posix(), + 'small_file_size_mb': float(10) + } + + # should catch UTILITIES_MAX_FILE_SIZE if value is not provided + result = FirecrestTransport._dynamic_info_direct_size(ctx, None, 0) + assert result == 69 + + # should use the value if provided + # note: user cannot enter negative numbers anyways, click raise as this shoule be float not str + result = FirecrestTransport._dynamic_info_direct_size(ctx, None, 10) + assert result == 10 + + +@pytest.mark.usefixtures("aiida_profile_clean") +def test_mkdir(firecrest_computer: orm.Computer, tmpdir: Path): + transport = firecrest_computer.get_transport() + + _scratch = tmpdir / "sampledir" + transport.mkdir(_scratch) + assert _scratch.exists() + + _scratch = tmpdir / "sampledir2" / "subdir" + transport.makedirs(_scratch) + assert _scratch.exists() + +@pytest.mark.usefixtures("aiida_profile_clean") +def test_is_file(firecrest_computer: orm.Computer, tmpdir: Path): + transport = firecrest_computer.get_transport() + + _scratch = tmpdir / "samplefile" + Path(_scratch).touch() + + assert transport.isfile(_scratch) == True + assert transport.isfile("/does_not_exist") == False + +@pytest.mark.usefixtures("aiida_profile_clean") +def test_is_dir(firecrest_computer: orm.Computer, tmpdir: Path): + transport = firecrest_computer.get_transport() + + _scratch = tmpdir / "sampledir" + _scratch.mkdir() + + assert transport.isdir(_scratch) == True + assert transport.isdir("/does_not_exist") == False + +@pytest.mark.usefixtures("aiida_profile_clean") +def test_normalize(firecrest_computer: orm.Computer): + transport = firecrest_computer.get_transport() + assert transport.normalize("/path/to/dir") == os.path.normpath("/path/to/dir") + assert transport.normalize("path/to/dir") == os.path.normpath("path/to/dir") + assert transport.normalize("path/to/dir/") == os.path.normpath("path/to/dir/") + assert transport.normalize("path/to/../dir") == os.path.normpath("path/to/../dir") + assert transport.normalize("path/to/../../dir") == os.path.normpath("path/to/../../dir") + assert transport.normalize("path/to/../../dir/") == os.path.normpath("path/to/../../dir/") + assert transport.normalize("path/to/../../dir/../") == os.path.normpath("path/to/../../dir/../") + +@pytest.mark.usefixtures("aiida_profile_clean") +def test_remove(firecrest_computer: orm.Computer, tmpdir: Path): + transport = firecrest_computer.get_transport() + + _scratch = tmpdir / "samplefile" + Path(_scratch).touch() + transport.remove(_scratch) + assert not _scratch.exists() + + _scratch = tmpdir / "sampledir" + _scratch.mkdir() + transport.rmtree(_scratch) + assert not _scratch.exists() + + _scratch = tmpdir / "sampledir" + _scratch.mkdir() + Path(_scratch / "samplefile").touch() + with pytest.raises(OSError): + transport.rmdir(_scratch) + + os.remove(_scratch / "samplefile") + transport.rmdir(_scratch) + assert not _scratch.exists() + +@pytest.mark.usefixtures("aiida_profile_clean") +def test_symlink(firecrest_computer: orm.Computer, tmpdir: Path): + transport = firecrest_computer.get_transport() + + _scratch = Path(tmpdir / "samplefile-2sym") + Path(_scratch).touch() + _symlink = Path(tmpdir / "samplelink") + transport.symlink(_scratch, _symlink) + assert _symlink.is_symlink() + assert _symlink.resolve() == _scratch + + +@pytest.mark.usefixtures("aiida_profile_clean") +def test_listdir(firecrest_computer: orm.Computer, tmpdir: Path): + transport = firecrest_computer.get_transport() + + _scratch = tmpdir / "sampledir" + _scratch.mkdir() + # to test basics + Path(_scratch / "file1").touch() + Path(_scratch / "dir1").mkdir() + Path(_scratch / ".hidden").touch() + # to test recursive + Path(_scratch / "dir1" / "file2").touch() + + assert set(transport.listdir(_scratch)) == set(["file1", "dir1", ".hidden"]) + assert set(transport.listdir(_scratch, recursive=True)) == set(["file1", "dir1", ".hidden", + "dir1/file2"]) + # to test symlink + Path(_scratch / "dir1" / "dir2").mkdir() + Path(_scratch / "dir1" / "dir2" / "file3").touch() + os.symlink(_scratch / "dir1" / "dir2", _scratch / "dir2_link") + os.symlink(_scratch / "dir1" / "file2", _scratch / "file_link") + + assert set(transport.listdir(_scratch, recursive=True)) == set(["file1", "dir1", ".hidden", + "dir1/file2", "dir1/dir2", "dir1/dir2/file3", + "dir2_link", "file_link"]) + + assert set(transport.listdir(_scratch / "dir2_link", recursive=False)) == set(["file3"]) + From 9e3d30dfc32bc2fa5f823aed94a946646edbf1c4 Mon Sep 17 00:00:00 2001 From: Ali Date: Wed, 19 Jun 2024 09:32:25 +0200 Subject: [PATCH 02/39] added tests for transport:get --- aiida_firecrest/transport.py | 47 ++++-- tests_new/conftest.py | 78 +++++++--- tests_new/test_transport.py | 273 ++++++++++++++++++++++++++++++++++- 3 files changed, 360 insertions(+), 38 deletions(-) diff --git a/aiida_firecrest/transport.py b/aiida_firecrest/transport.py index bb20e21..e2f51f8 100644 --- a/aiida_firecrest/transport.py +++ b/aiida_firecrest/transport.py @@ -296,8 +296,10 @@ def __init__( self._client_id = client_id self._temp_directory = Path(temp_directory) self._small_file_size_bytes = int(small_file_size_mb * 1024 * 1024) + + self._payoff_override = None + secret = Path(client_secret).read_text() - try: self._client = Firecrest( firecrest_url=self._url, @@ -313,6 +315,15 @@ def __str__(self): """Return the name of the plugin.""" return self.__class__.__name__ + @property + def payoff_override(self): + return self._payoff_override + + @payoff_override.setter + def payoff_override(self, value): + if not isinstance(value, bool): + raise ValueError("payoff_override must be a boolean value") + self._payoff_override = value @classmethod def get_description(cls) -> str: @@ -531,7 +542,7 @@ def getfile( """Get a file from the remote. :param dereference: If True, follow symlinks. - note: we don't support downloading symlinks, so dereference is always should be True + note: we don't support downloading symlinks, so dereference should always be True """ @@ -615,8 +626,8 @@ def _gettreetar( """ # TODO manual testing the submit behaviour - if dereference: - raise NotImplementedError("Dereferencing compression not implemented in pyFirecREST.") + # if dereference: + # raise NotImplementedError("Dereferencing compression not implemented in pyFirecREST.") _ = uuid.uuid4() # Attempt direct compress remote_path_temp = self._temp_directory.joinpath(f"temp_{_}.tar") @@ -639,13 +650,13 @@ def _gettreetar( self.remove(remote_path_temp) # Extract the downloaded file locally - # this is a bit hard coded, what I wanted to do: to extract the files in the same directory as the tar file try: - with tarfile.open(localpath_temp, "r") as tar: - members = [m for m in tar.getmembers() if m.name.startswith(remotepath.name)] - for member in members: - member.name = os.path.relpath(member.name, remotepath.name) - tar.extract(member, path=localpath) + # with tarfile.open(localpath_temp, "r") as tar: + # members = [m for m in tar.getmembers() if m.name.startswith(remotepath.name)] + # for member in members: + # member.name = os.path.relpath(member.name, remotepath.name) + # tar.extract(member, path=localpath) + os.system(f"tar -xf '{localpath_temp}' --strip-components=1 -C {localpath}") finally: localpath_temp.unlink() @@ -683,13 +694,13 @@ def gettree( # Destination directory does not exist, create and move content 69 inside it local.mkdir(parents=True, exist_ok=False) # SSH transport behaviour, 69 is a directory - # transport.get('somepath/69', 'someremotepath/') == transport.put('somepath/69', 'someremotepath') - # transport.get('somepath/69', 'someremotepath/') == transport.put('somepath/69/', 'someremotepath/') + # transport.get('somepath/69', 'someremotepath/') == transport.get('somepath/69', 'someremotepath') + # transport.get('somepath/69', 'someremotepath/') == transport.get('somepath/69/', 'someremotepath/') # transport.get('someremotepath/69', 'somepath/69') --> if 69 exist, create 69 inside it ('somepath/69/69') # transport.get('someremotepath/69', 'somepath/69') --> if 69 no texist, create 69 inside it ('somepath/69') # transport.get('somepath/69', 'someremotepath/6889/69') --> create everything, make_parent = True - if not dereference and self.payoff(remote): + if self.payoff(remote): # in this case send a request to the server to tar the files and then download the tar file # unfortunately, the server does not provide a deferenced tar option, yet. self._gettreetar(remote, local) @@ -697,7 +708,6 @@ def gettree( # otherwise download the files one by one for remote_item in remote.iterdir(recursive=True): local_item = local.joinpath(remote_item.relpath(remote)) - if dereference and remote_item.is_symlink(): target_path = remote_item._cache.link_target if not Path(target_path).is_absolute(): @@ -724,6 +734,8 @@ def get(self, remotepath: str, localpath: str, ignore_nonexisting: bool = False, note: dereference should be always True, otherwise the symlinks will not be functional. """ remote = self._cwd.joinpath(remotepath)#.enable_cache() it's removed from from path.py to be investigated + local = Path(localpath) + if remote.is_dir(): self.gettree(remote, localpath) @@ -789,6 +801,11 @@ def payoff( # of serialization and "penalty" for sending multiple requests asycnhronusly or in a short time window. # It responses in 1, 1.5, 3, 5, 7 seconds! # So right now, I think if the number of files is more than 3, it pays off to tar everything + + # If payoff_override is set, return its value + if self.payoff_override is not None: + return self.payoff_override + if len(self.listdir(remotepath,recursive=True)) > 3: return True else: @@ -846,7 +863,7 @@ def _puttreetar( def puttree( - self, localpath: str | Path, remotepath: str, dereference: bool=True, *args: Any, **kwargs: Any + self, localpath: str | Path, remotepath: str, dereference: bool=True, *args: Any, **kwargs: Any ) -> None: """Put a directory to the remote. diff --git a/tests_new/conftest.py b/tests_new/conftest.py index e51a640..131a4a2 100644 --- a/tests_new/conftest.py +++ b/tests_new/conftest.py @@ -2,6 +2,8 @@ from pathlib import Path import os +import stat +import hashlib from _pytest.terminal import TerminalReporter import firecrest.path @@ -21,6 +23,11 @@ def __init__(self, firecrest_url, *args, **kwargs): self.simple_delete = simple_delete self.parameters = parameters self.symlink = symlink + self.checksum = checksum + self.simple_download = simple_download + self.compress = compress + self.extract = extract + class MockClientCredentialsAuth: def __init__(self, *args, **kwargs): @@ -32,20 +39,15 @@ def myfirecrest( pytestconfig: pytest.Config, monkeypatch, ): - + monkeypatch.setattr(firecrest, "Firecrest", MockFirecrest) monkeypatch.setattr(firecrest, "ClientCredentialsAuth", MockClientCredentialsAuth) - # monkeypatch.setattr(firecrest.path, "_ls_to_st_mode", _ls_to_st_mode) - - def mock_whomai(machine: str): assert machine == "MACHINE_NAME" return "test_user" -import stat - def list_files( machine: str, target_path: str, recursive: bool = False, show_hidden: bool = False): # this is mimiking the expected behaviour from the firecrest code. @@ -57,20 +59,19 @@ def list_files( for name in dirs + files: full_path = os.path.join(root, name) relative_path = Path(os.path.relpath(root, target_path)).joinpath(name).as_posix() - # breakpoint() - print(relative_path) - if os.path.isdir(full_path): - content_type = 'd' + if os.path.islink(full_path): + content_type = 'l' + link_target = os.readlink(full_path) if os.path.islink(full_path) else None elif os.path.isfile(full_path): content_type = '-' - elif os.path.islink(full_path): - content_type = 'l' + link_target = None + elif os.path.isdir(full_path): + content_type = 'd' + link_target = None else: content_type = 'NON' - link_target = os.readlink(full_path) if os.path.islink(full_path) else None - # permissions = stat.S_IMODE(os.lstat(full_path).st_mode) + link_target = None permissions = stat.filemode(Path(full_path).lstat().st_mode)[1:] - # stat.S_ISREG(permissions) if name.startswith('.') and not show_hidden: continue content_list.append({ @@ -86,7 +87,7 @@ def list_files( # return int(permissions) def stat_(machine:str, targetpath: firecrest.path, dereference=True): - stats = os.stat(targetpath) + stats = os.stat(targetpath, follow_symlinks= True if dereference else False) return { "ino": stats.st_ino, "dev": stats.st_dev, @@ -115,8 +116,49 @@ def simple_delete(machine: str, target_path: str): os.remove(target_path) def symlink(machine: str, target_path: str, link_path: str): - os.symlink(target_path, link_path) - + # this is how firecrest does it + os.system(f"ln -s {target_path} {link_path}") + +def simple_download(machine: str, remote_path: str, local_path: str): + # this procedure is complecated in firecrest, but I am simplifying it here + # we don't care about the details of the download, we just want to make sure + # that the aiida-firecrest code is calling the right functions at right time + if Path(remote_path).is_dir(): + raise IsADirectoryError(f"{remote_path} is a directory") + if not Path(remote_path).exists(): + raise FileNotFoundError(f"{remote_path} does not exist") + # print(f"{remote_path} {local_path}") + os.system(f"cp {remote_path} {local_path}") + +# def copy(machine: str, source_path: str, target_path: str): +# firecrest copy action = f"cp --force -dR --preserve=all -- '{sourcePath}' '{targetPath}'" +# https://github.com/eth-cscs/firecrest/blob/db6ba4ba273c11a79ecbe940872f19d5cb19ac5e/src/utilities/utilities.py#L451C1-L452C1 + +def compress(machine: str, source_path: str, target_path: str, dereference: bool = True): + # this is how firecrest does it + # https://github.com/eth-cscs/firecrest/blob/db6ba4ba273c11a79ecbe940872f19d5cb19ac5e/src/utilities/utilities.py#L460 + basedir = os.path.dirname(source_path) + file_path = os.path.basename(source_path) + deref = "--dereference" if dereference else "" + os.system(f"tar {deref} -czvf '{target_path}' -C '{basedir}' '{file_path}'") + +def extract(machine: str, source_path: str, target_path: str): + # this is how firecrest does it + # https://github.com/eth-cscs/firecrest/blob/db6ba4ba273c11a79ecbe940872f19d5cb19ac5e/src/common/cscs_api_common.py#L1110C18-L1110C65 + breakpoint() + os.system("tar -xf '{source_path}' -C '{target_path}'") + + +def checksum(machine: str, remote_path: str) -> int: + if not remote_path.exists(): + return False + # Firecrest uses sha256 + sha256_hash = hashlib.sha256() + with open(remote_path,"rb") as f: + for byte_block in iter(lambda: f.read(4096),b""): + sha256_hash.update(byte_block) + + return sha256_hash.hexdigest() def parameters(): # note: I took this from https://firecrest-tds.cscs.ch/ or https://firecrest.cscs.ch/ diff --git a/tests_new/test_transport.py b/tests_new/test_transport.py index 288f6cb..565b7d0 100644 --- a/tests_new/test_transport.py +++ b/tests_new/test_transport.py @@ -2,7 +2,7 @@ import os import pytest -from unittest.mock import Mock +from unittest.mock import Mock, patch from click import BadParameter from aiida import orm @@ -17,8 +17,9 @@ def _firecrest_computer(myfirecrest, tmpdir: Path): # create a temp directory and set it as the workdir _scratch = tmpdir / "scratch" - _scratch.mkdir() _temp_directory = tmpdir / "temp" + _scratch.mkdir() + _temp_directory.mkdir() Path(tmpdir / ".firecrest").mkdir() _secret_path = Path(tmpdir / ".firecrest/secret69") @@ -159,9 +160,8 @@ def test_is_file(firecrest_computer: orm.Computer, tmpdir: Path): _scratch = tmpdir / "samplefile" Path(_scratch).touch() - assert transport.isfile(_scratch) == True - assert transport.isfile("/does_not_exist") == False + assert transport.isfile(_scratch / "does_not_exist") == False @pytest.mark.usefixtures("aiida_profile_clean") def test_is_dir(firecrest_computer: orm.Computer, tmpdir: Path): @@ -171,7 +171,7 @@ def test_is_dir(firecrest_computer: orm.Computer, tmpdir: Path): _scratch.mkdir() assert transport.isdir(_scratch) == True - assert transport.isdir("/does_not_exist") == False + assert transport.isdir(_scratch / "does_not_exist") == False @pytest.mark.usefixtures("aiida_profile_clean") def test_normalize(firecrest_computer: orm.Computer): @@ -248,3 +248,266 @@ def test_listdir(firecrest_computer: orm.Computer, tmpdir: Path): assert set(transport.listdir(_scratch / "dir2_link", recursive=False)) == set(["file3"]) + +@pytest.mark.usefixtures("aiida_profile_clean") +def test_get(firecrest_computer: orm.Computer, tmpdir: Path): + """ + This is minimal test is to check if get() is raising errors as expected, + and directing to getfile() and gettree() as expected. + Mainly just checking error handeling and folder creation. + """ + transport = firecrest_computer.get_transport() + + _remote = tmpdir / "remotedir" + _local = tmpdir / "localdir" + _remote.mkdir() + _local.mkdir() + + + # check if the code is directing to getfile() or gettree() as expected + with patch.object(transport, 'gettree', autospec=True) as mock_gettree: + transport.get(_remote, _local) + mock_gettree.assert_called_once() + + with patch.object(transport, 'gettree', autospec=True) as mock_gettree: + os.symlink(_remote, tmpdir / "dir_link") + transport.get(tmpdir / "dir_link", _local) + mock_gettree.assert_called_once() + + with patch.object(transport, 'getfile', autospec=True) as mock_getfile: + Path(_remote / "file1").write_text("file1") + transport.get(_remote / "file1", _local / "file1") + mock_getfile.assert_called_once() + + with patch.object(transport, 'getfile', autospec=True) as mock_getfile: + os.symlink(_remote / "file1", _remote / "file1_link") + transport.get(_remote / "file1_link", _local / "file1_link") + mock_getfile.assert_called_once() + + # raise if remote file/folder does not exist + with pytest.raises(FileNotFoundError): + transport.get(_remote / "does_not_exist", _local) + transport.get(_remote / "does_not_exist", _local, ignore_nonexisting=True) + + # raise if localpath is relative + with pytest.raises(ValueError): + transport.get(_remote, Path(_local).relative_to(tmpdir)) + with pytest.raises(ValueError): + transport.get(_remote / "file1", Path(_local).relative_to(tmpdir)) + + +@pytest.mark.usefixtures("aiida_profile_clean") +def test_getfile(firecrest_computer: orm.Computer, tmpdir: Path): + transport = firecrest_computer.get_transport() + + _remote = tmpdir / "remotedir" + _local = tmpdir / "localdir" + _remote.mkdir() + _local.mkdir() + + Path(_remote / "file1").write_text("file1") + Path(_remote / ".hidden").write_text(".hidden") + os.symlink(_remote / "file1", _remote / "file1_link") + + + # raise if remote file does not exist + with pytest.raises(FileNotFoundError): + transport.getfile(_remote / "does_not_exist", _local) + + # raise if localfilename not provided + with pytest.raises(IsADirectoryError): + transport.getfile(_remote / "file1", _local) + + # raise if localpath is relative + with pytest.raises(ValueError): + transport.getfile(_remote / "file1", Path(_local / "file1").relative_to(tmpdir)) + + # don't mix up directory with file + with pytest.raises(FileNotFoundError): + transport.getfile(_remote, _local / "file1") + + # write where I tell you to + transport.getfile(_remote / "file1", _local / "file1") + transport.getfile(_remote / "file1", _local / "file1-prime") + assert Path(_local / "file1").read_text() == "file1" + assert Path(_local / "file1-prime").read_text() == "file1" + + # always overwrite + transport.getfile(_remote / "file1", _local / "file1") + assert Path(_local / "file1").read_text() == "file1" + + Path(_local / "file1").write_text("notfile1") + + transport.getfile(_remote / "file1", _local / "file1") + assert Path(_local / "file1").read_text() == "file1" + + # don't skip hidden files + transport.getfile(_remote / ".hidden", _local / ".hidden-prime") + assert Path(_local / ".hidden-prime").read_text() == ".hidden" + + # follow links + transport.getfile(_remote / "file1_link", _local / "file1_link") + assert Path(_local / "file1_link").read_text() == "file1" + assert not Path(_local / "file1_link").is_symlink() + + +@pytest.mark.usefixtures("aiida_profile_clean") +def test_gettree_notar(firecrest_computer: orm.Computer, tmpdir: Path, monkeypatch): + """ + This test is to check if the gettree function is working as expected. Through non tar mode. + payoff= False in this test, so just checking if getting files one by one is working as expected. + """ + transport = firecrest_computer.get_transport() + transport.payoff_override = False + + _remote = tmpdir / "remotedir" + _local = tmpdir / "localdir" + _remote.mkdir() + _local.mkdir() + # a typical tree + Path(_remote / "dir1").mkdir() + Path(_remote / "dir2").mkdir() + Path(_remote / "file1").write_text("file1") + Path(_remote / ".hidden").write_text(".hidden") + Path(_remote / "dir1" / "file2").write_text("file2") + Path(_remote / "dir2" / "file3").write_text("file3") + # with symlinks to a file even if pointing to a relative path + os.symlink(_remote / "file1", _remote / "dir1" / "file1_link") + os.symlink(Path("../file1"), _remote / "dir1" / "file10_link") + # with symlinks to a folder even if pointing to a relative path + os.symlink(_remote / "dir2", _remote / "dir1" / "dir2_link") + os.symlink(Path("../dir2" ), _remote / "dir1" / "dir20_link") + + + # raise if remote file does not exist + with pytest.raises(OSError): + transport.gettree(_remote / "does_not_exist", _local) + + # raise if local is a file + with pytest.raises(OSError): + Path(tmpdir / "isfile").touch() + transport.gettree(_remote, tmpdir / "isfile") + + # raise if localpath is relative + with pytest.raises(ValueError): + transport.gettree(_remote, Path(_local).relative_to(tmpdir)) + + # If destination directory does not exists, AiiDA expects transport make the new path as root not _remote.name + transport.gettree(_remote, _local / "newdir") + _root = _local / "newdir" + # tree should be copied recursively + assert Path(_root / "file1").read_text() == "file1" + assert Path(_root / ".hidden").read_text() == ".hidden" + assert Path(_root / "dir1" / "file2").read_text() == "file2" + assert Path(_root / "dir2" / "file3").read_text() == "file3" + # symlink should be followed + assert Path(_root / "dir1" / "file1_link").read_text() == "file1" + assert Path(_root / "dir1" / "dir2_link" / "file3").read_text() == "file3" + assert Path(_root / "dir1" / "file10_link").read_text() == "file1" + assert Path(_root / "dir1" / "dir20_link" / "file3").read_text() == "file3" + assert not Path(_root / "dir1" / "file1_link").is_symlink() + assert not Path(_root / "dir1" / "dir2_link" / "file3").is_symlink() + assert not Path(_root / "dir1" / "file10_link").is_symlink() + assert not Path(_root / "dir1" / "dir20_link" / "file3").is_symlink() + + + # If destination directory does exists, AiiDA expects transport make _remote.name and write into it + # however this might have changed in the newer versions of AiiDA ~ 2.6.0 (IDK) + transport.gettree(_remote, _local / "newdir") + _root = _local / "newdir" / Path(_remote).name + # tree should be copied recursively + assert Path(_root / "file1").read_text() == "file1" + assert Path(_root / ".hidden").read_text() == ".hidden" + assert Path(_root / "dir1" / "file2").read_text() == "file2" + assert Path(_root / "dir2" / "file3").read_text() == "file3" + # symlink should be followed + assert Path(_root / "dir1" / "file1_link").read_text() == "file1" + assert Path(_root / "dir1" / "dir2_link" / "file3").read_text() == "file3" + assert Path(_root / "dir1" / "file10_link").read_text() == "file1" + assert Path(_root / "dir1" / "dir20_link" / "file3").read_text() == "file3" + assert not Path(_root / "dir1" / "file1_link").is_symlink() + assert not Path(_root / "dir1" / "dir2_link" / "file3").is_symlink() + assert not Path(_root / "dir1" / "file10_link").is_symlink() + assert not Path(_root / "dir1" / "dir20_link" / "file3").is_symlink() + +@pytest.mark.usefixtures("aiida_profile_clean") +def test_gettree_bytar(firecrest_computer: orm.Computer, tmpdir: Path): + """ + This test is to check if the gettree function is working as expected. Through non tar mode. + bytar= True in this test. + """ + transport = firecrest_computer.get_transport() + transport.payoff_override = True + + _remote = tmpdir / "remotedir" + _local = tmpdir / "localdir" + _remote.mkdir() + _local.mkdir() + # a typical tree + Path(_remote / "file1").write_text("file1") + Path(_remote / ".hidden").write_text(".hidden") + Path(_remote / "dir1").mkdir() + Path(_remote / "dir1" / "file2").write_text("file2") + # with symlinks + Path(_remote / "dir2").mkdir() + Path(_remote / "dir2" / "file3").write_text("file3") + os.symlink(_remote / "file1", _remote / "dir1" / "file1_link") + os.symlink(_remote / "dir2", _remote / "dir1" / "dir2_link") + # if symlinks are pointing to a relative path + os.symlink(Path("../file1"), _remote / "dir1" / "file10_link") + os.symlink(Path("../dir2" ), _remote / "dir1" / "dir20_link") + + + + # raise if remote file does not exist + with pytest.raises(OSError): + transport.gettree(_remote / "does_not_exist", _local) + + # raise if local is a file + Path(tmpdir / "isfile").touch() + with pytest.raises(OSError): + transport.gettree(_remote, tmpdir / "isfile") + + # raise if localpath is relative + with pytest.raises(ValueError): + transport.gettree(_remote, Path(_local).relative_to(tmpdir)) + + # If destination directory does not exists, AiiDA expects transport make the new path as root not _remote.name + transport.gettree(_remote, _local / "newdir") + _root = _local / "newdir" + # tree should be copied recursively + assert Path(_root / "file1").read_text() == "file1" + assert Path(_root / ".hidden").read_text() == ".hidden" + assert Path(_root / "dir1" / "file2").read_text() == "file2" + assert Path(_root / "dir2" / "file3").read_text() == "file3" + # symlink should be followed + assert Path(_root / "dir1" / "file1_link").read_text() == "file1" + assert Path(_root / "dir1" / "dir2_link" / "file3").read_text() == "file3" + assert Path(_root / "dir1" / "file10_link").read_text() == "file1" + assert Path(_root / "dir1" / "dir20_link" / "file3").read_text() == "file3" + assert not Path(_root / "dir1" / "file1_link").is_symlink() + assert not Path(_root / "dir1" / "dir2_link" / "file3").is_symlink() + assert not Path(_root / "dir1" / "file10_link").is_symlink() + assert not Path(_root / "dir1" / "dir20_link" / "file3").is_symlink() + + + # If destination directory does exists, AiiDA expects transport make _remote.name and write into it + # however this might have changed in the newer versions of AiiDA ~ 2.6.0 (IDK) + transport.gettree(_remote, _local / "newdir") + _root = _local / "newdir" / Path(_remote).name + # tree should be copied recursively + assert Path(_root / "file1").read_text() == "file1" + assert Path(_root / ".hidden").read_text() == ".hidden" + assert Path(_root / "dir1" / "file2").read_text() == "file2" + assert Path(_root / "dir2" / "file3").read_text() == "file3" + # symlink should be followed + assert Path(_root / "dir1" / "file1_link").read_text() == "file1" + assert Path(_root / "dir1" / "dir2_link" / "file3").read_text() == "file3" + assert Path(_root / "dir1" / "file10_link").read_text() == "file1" + assert Path(_root / "dir1" / "dir20_link" / "file3").read_text() == "file3" + assert not Path(_root / "dir1" / "file1_link").is_symlink() + assert not Path(_root / "dir1" / "dir2_link" / "file3").is_symlink() + assert not Path(_root / "dir1" / "file10_link").is_symlink() + assert not Path(_root / "dir1" / "dir20_link" / "file3").is_symlink() + + From 56ff1ee9c9992b74769c5ea0769fe5f8ff9bce57 Mon Sep 17 00:00:00 2001 From: Ali Date: Wed, 19 Jun 2024 13:38:39 +0200 Subject: [PATCH 03/39] new tests moved as a subdirectory of the main tests --- .../tests_mocking_pyfirecrest}/conftest.py | 64 ++++++-- .../test_computer.py | 102 +++++++++++++ .../test_transport.py | 138 +----------------- 3 files changed, 156 insertions(+), 148 deletions(-) rename {tests_new => tests/tests_mocking_pyfirecrest}/conftest.py (80%) create mode 100644 tests/tests_mocking_pyfirecrest/test_computer.py rename {tests_new => tests/tests_mocking_pyfirecrest}/test_transport.py (74%) diff --git a/tests_new/conftest.py b/tests/tests_mocking_pyfirecrest/conftest.py similarity index 80% rename from tests_new/conftest.py rename to tests/tests_mocking_pyfirecrest/conftest.py index 131a4a2..a1fbb89 100644 --- a/tests_new/conftest.py +++ b/tests/tests_mocking_pyfirecrest/conftest.py @@ -9,6 +9,47 @@ import firecrest.path import pytest import firecrest +from aiida import orm + + +@pytest.fixture(name="firecrest_computer") +def _firecrest_computer(myfirecrest, tmpdir: Path): + """Create and return a computer configured for Firecrest. + + Note, the computer is not stored in the database. + """ + + # create a temp directory and set it as the workdir + _scratch = tmpdir / "scratch" + _temp_directory = tmpdir / "temp" + _scratch.mkdir() + _temp_directory.mkdir() + + Path(tmpdir / ".firecrest").mkdir() + _secret_path = Path(tmpdir / ".firecrest/secret69") + _secret_path.write_text("SECRET_STRING") + + computer = orm.Computer( + label="test_computer", + description="test computer", + hostname="-", + workdir=str(_scratch), + transport_type="firecrest", + scheduler_type="firecrest", + ) + computer.set_minimum_job_poll_interval(5) + computer.set_default_mpiprocs_per_machine(1) + computer.configure( + url=' https://URI', + token_uri='https://TOKEN_URI', + client_id='CLIENT_ID', + client_secret=str(_secret_path), + client_machine='MACHINE_NAME', + small_file_size_mb=1.0, + temp_directory=str(_temp_directory), + ) + return computer + class MockFirecrest: @@ -16,7 +57,7 @@ def __init__(self, firecrest_url, *args, **kwargs): self._firecrest_url = firecrest_url self.args = args self.kwargs = kwargs - self.whoami = mock_whomai + self.whoami = whomai self.list_files = list_files self.stat = stat_ self.mkdir = mkdir @@ -27,6 +68,8 @@ def __init__(self, firecrest_url, *args, **kwargs): self.simple_download = simple_download self.compress = compress self.extract = extract + self.submit = submit + # self.poll_active = poll_active class MockClientCredentialsAuth: @@ -43,11 +86,14 @@ def myfirecrest( monkeypatch.setattr(firecrest, "Firecrest", MockFirecrest) monkeypatch.setattr(firecrest, "ClientCredentialsAuth", MockClientCredentialsAuth) -def mock_whomai(machine: str): +# def poll_active(machine: str, job_id: str): +def submit(machine: str, script_str: str = None, script_remote_path: str = None, script_local_path: str = None): + pass + +def whomai(machine: str): assert machine == "MACHINE_NAME" return "test_user" - def list_files( machine: str, target_path: str, recursive: bool = False, show_hidden: bool = False): # this is mimiking the expected behaviour from the firecrest code. @@ -83,9 +129,6 @@ def list_files( return content_list -# def _ls_to_st_mode(ftype: str, permissions: str) -> int: -# return int(permissions) - def stat_(machine:str, targetpath: firecrest.path, dereference=True): stats = os.stat(targetpath, follow_symlinks= True if dereference else False) return { @@ -100,7 +143,6 @@ def stat_(machine:str, targetpath: firecrest.path, dereference=True): "ctime": stats.st_ctime, } - def mkdir(machine: str, target_path: str, p: bool = False): if p: os.makedirs(target_path) @@ -130,9 +172,10 @@ def simple_download(machine: str, remote_path: str, local_path: str): # print(f"{remote_path} {local_path}") os.system(f"cp {remote_path} {local_path}") -# def copy(machine: str, source_path: str, target_path: str): -# firecrest copy action = f"cp --force -dR --preserve=all -- '{sourcePath}' '{targetPath}'" -# https://github.com/eth-cscs/firecrest/blob/db6ba4ba273c11a79ecbe940872f19d5cb19ac5e/src/utilities/utilities.py#L451C1-L452C1 +def copy(machine: str, source_path: str, target_path: str): + # this is how firecrest does it + # https://github.com/eth-cscs/firecrest/blob/db6ba4ba273c11a79ecbe940872f19d5cb19ac5e/src/utilities/utilities.py#L451C1-L452C1 + os.system(f"cp --force -dR --preserve=all -- '{source_path}' '{target_path}'") def compress(machine: str, source_path: str, target_path: str, dereference: bool = True): # this is how firecrest does it @@ -148,7 +191,6 @@ def extract(machine: str, source_path: str, target_path: str): breakpoint() os.system("tar -xf '{source_path}' -C '{target_path}'") - def checksum(machine: str, remote_path: str) -> int: if not remote_path.exists(): return False diff --git a/tests/tests_mocking_pyfirecrest/test_computer.py b/tests/tests_mocking_pyfirecrest/test_computer.py new file mode 100644 index 0000000..edca7d1 --- /dev/null +++ b/tests/tests_mocking_pyfirecrest/test_computer.py @@ -0,0 +1,102 @@ +from pathlib import Path + +import pytest +from unittest.mock import Mock +from click import BadParameter + +from aiida import orm + + +@pytest.mark.usefixtures("aiida_profile_clean") +def test_whoami(firecrest_computer: orm.Computer): + """check if it is possible to determine the username.""" + transport = firecrest_computer.get_transport() + assert transport.whoami() == 'test_user' + +def test_create_secret_file_with_existing_file(tmpdir: Path): + from aiida_firecrest.transport import FirecrestTransport + secret_file = Path(tmpdir / "secret") + secret_file.write_text("topsecret") + result = FirecrestTransport._create_secret_file(None, None, str(secret_file)) + assert isinstance(result, str) + assert result == str(secret_file) + assert Path(result).read_text() == "topsecret" + +def test_create_secret_file_with_nonexistent_file(tmp_path): + from aiida_firecrest.transport import FirecrestTransport + secret_file = tmp_path / "nonexistent" + with pytest.raises(BadParameter): + FirecrestTransport._create_secret_file(None, None, str(secret_file)) + +def test_create_secret_file_with_secret_value(tmp_path, monkeypatch): + from aiida_firecrest.transport import FirecrestTransport + secret = "topsecret!~/" + monkeypatch.setattr(Path, "expanduser", lambda x: tmp_path / str(x).lstrip("~/") if str(x).startswith("~/") else x) + result = FirecrestTransport._create_secret_file(None, None, secret) + assert Path(result).parent.parts[-1]== ".firecrest" + assert Path(result).read_text() == secret + +def test_validate_temp_directory(myfirecrest, monkeypatch, tmpdir: Path): + from aiida_firecrest.transport import FirecrestTransport + + monkeypatch.setattr('click.echo', lambda x: None) + # monkeypatch.setattr('click.BadParameter', lambda x: None) + secret_file = Path(tmpdir / "secret") + secret_file.write_text("topsecret") + ctx = Mock() + ctx.params = { + 'url': 'http://test.com', + 'token_uri': 'token_uri', + 'client_id': 'client_id', + 'client_machine': 'client_machine', + 'client_secret': secret_file.as_posix(), + 'small_file_size_mb': float(10) + } + + # should raise if is_file + Path(tmpdir / 'crap.txt').touch() + with pytest.raises(BadParameter): + result = FirecrestTransport._validate_temp_directory(ctx, None, Path(tmpdir /'crap.txt').as_posix()) + + # should create the directory if it doesn't exist + result = FirecrestTransport._validate_temp_directory(ctx, None, Path(tmpdir /'temp_on_server_directory').as_posix()) + assert result == Path(tmpdir /'temp_on_server_directory').as_posix() + assert Path(tmpdir /'temp_on_server_directory').exists() + + # should get a confirmation if the directory exists and is not empty + Path(tmpdir /'temp_on_server_directory' / 'crap.txt').touch() + monkeypatch.setattr('click.confirm', lambda x: False) + with pytest.raises(BadParameter): + result = FirecrestTransport._validate_temp_directory(ctx, None, Path(tmpdir /'temp_on_server_directory').as_posix()) + + # should delete the content if I confirm + monkeypatch.setattr('click.confirm', lambda x: True) + result = FirecrestTransport._validate_temp_directory(ctx, None, Path(tmpdir /'temp_on_server_directory').as_posix()) + assert result == Path(tmpdir /'temp_on_server_directory').as_posix() + assert not Path(tmpdir /'temp_on_server_directory' / 'crap.txt').exists() + +def test__dynamic_info(myfirecrest, monkeypatch, tmpdir: Path): + from aiida_firecrest.transport import FirecrestTransport + + monkeypatch.setattr('click.echo', lambda x: None) + # monkeypatch.setattr('click.BadParameter', lambda x: None) + secret_file = Path(tmpdir / "secret") + secret_file.write_text("topsecret") + ctx = Mock() + ctx.params = { + 'url': 'http://test.com', + 'token_uri': 'token_uri', + 'client_id': 'client_id', + 'client_machine': 'client_machine', + 'client_secret': secret_file.as_posix(), + 'small_file_size_mb': float(10) + } + + # should catch UTILITIES_MAX_FILE_SIZE if value is not provided + result = FirecrestTransport._dynamic_info_direct_size(ctx, None, 0) + assert result == 69 + + # should use the value if provided + # note: user cannot enter negative numbers anyways, click raise as this shoule be float not str + result = FirecrestTransport._dynamic_info_direct_size(ctx, None, 10) + assert result == 10 diff --git a/tests_new/test_transport.py b/tests/tests_mocking_pyfirecrest/test_transport.py similarity index 74% rename from tests_new/test_transport.py rename to tests/tests_mocking_pyfirecrest/test_transport.py index 565b7d0..22c8184 100644 --- a/tests_new/test_transport.py +++ b/tests/tests_mocking_pyfirecrest/test_transport.py @@ -2,146 +2,10 @@ import os import pytest -from unittest.mock import Mock, patch -from click import BadParameter +from unittest.mock import patch from aiida import orm - -@pytest.fixture(name="firecrest_computer") -def _firecrest_computer(myfirecrest, tmpdir: Path): - """Create and return a computer configured for Firecrest. - - Note, the computer is not stored in the database. - """ - - # create a temp directory and set it as the workdir - _scratch = tmpdir / "scratch" - _temp_directory = tmpdir / "temp" - _scratch.mkdir() - _temp_directory.mkdir() - - Path(tmpdir / ".firecrest").mkdir() - _secret_path = Path(tmpdir / ".firecrest/secret69") - _secret_path.write_text("SECRET_STRING") - - computer = orm.Computer( - label="test_computer", - description="test computer", - hostname="-", - workdir=str(_scratch), - transport_type="firecrest", - scheduler_type="firecrest", - ) - computer.set_minimum_job_poll_interval(5) - computer.set_default_mpiprocs_per_machine(1) - computer.configure( - url=' https://URI', - token_uri='https://TOKEN_URI', - client_id='CLIENT_ID', - client_secret=str(_secret_path), - client_machine='MACHINE_NAME', - small_file_size_mb=1.0, - temp_directory=str(_temp_directory), - ) - return computer - - -@pytest.mark.usefixtures("aiida_profile_clean") -def test_whoami(firecrest_computer: orm.Computer): - """check if it is possible to determine the username.""" - transport = firecrest_computer.get_transport() - assert transport.whoami() == 'test_user' - -def test_create_secret_file_with_existing_file(tmpdir: Path): - from aiida_firecrest.transport import FirecrestTransport - secret_file = Path(tmpdir / "secret") - secret_file.write_text("topsecret") - result = FirecrestTransport._create_secret_file(None, None, str(secret_file)) - assert isinstance(result, str) - assert result == str(secret_file) - assert Path(result).read_text() == "topsecret" - -def test_create_secret_file_with_nonexistent_file(tmp_path): - from aiida_firecrest.transport import FirecrestTransport - secret_file = tmp_path / "nonexistent" - with pytest.raises(BadParameter): - FirecrestTransport._create_secret_file(None, None, str(secret_file)) - -def test_create_secret_file_with_secret_value(tmp_path, monkeypatch): - from aiida_firecrest.transport import FirecrestTransport - secret = "topsecret!~/" - monkeypatch.setattr(Path, "expanduser", lambda x: tmp_path / str(x).lstrip("~/") if str(x).startswith("~/") else x) - result = FirecrestTransport._create_secret_file(None, None, secret) - assert Path(result).parent.parts[-1]== ".firecrest" - assert Path(result).read_text() == secret - -def test_validate_temp_directory(myfirecrest, monkeypatch, tmpdir: Path): - from aiida_firecrest.transport import FirecrestTransport - - monkeypatch.setattr('click.echo', lambda x: None) - # monkeypatch.setattr('click.BadParameter', lambda x: None) - secret_file = Path(tmpdir / "secret") - secret_file.write_text("topsecret") - ctx = Mock() - ctx.params = { - 'url': 'http://test.com', - 'token_uri': 'token_uri', - 'client_id': 'client_id', - 'client_machine': 'client_machine', - 'client_secret': secret_file.as_posix(), - 'small_file_size_mb': float(10) - } - - # should raise if is_file - Path(tmpdir / 'crap.txt').touch() - with pytest.raises(BadParameter): - result = FirecrestTransport._validate_temp_directory(ctx, None, Path(tmpdir /'crap.txt').as_posix()) - - # should create the directory if it doesn't exist - result = FirecrestTransport._validate_temp_directory(ctx, None, Path(tmpdir /'temp_on_server_directory').as_posix()) - assert result == Path(tmpdir /'temp_on_server_directory').as_posix() - assert Path(tmpdir /'temp_on_server_directory').exists() - - # should get a confirmation if the directory exists and is not empty - Path(tmpdir /'temp_on_server_directory' / 'crap.txt').touch() - monkeypatch.setattr('click.confirm', lambda x: False) - with pytest.raises(BadParameter): - result = FirecrestTransport._validate_temp_directory(ctx, None, Path(tmpdir /'temp_on_server_directory').as_posix()) - - # should delete the content if I confirm - monkeypatch.setattr('click.confirm', lambda x: True) - result = FirecrestTransport._validate_temp_directory(ctx, None, Path(tmpdir /'temp_on_server_directory').as_posix()) - assert result == Path(tmpdir /'temp_on_server_directory').as_posix() - assert not Path(tmpdir /'temp_on_server_directory' / 'crap.txt').exists() - -def test__dynamic_info(myfirecrest, monkeypatch, tmpdir: Path): - from aiida_firecrest.transport import FirecrestTransport - - monkeypatch.setattr('click.echo', lambda x: None) - # monkeypatch.setattr('click.BadParameter', lambda x: None) - secret_file = Path(tmpdir / "secret") - secret_file.write_text("topsecret") - ctx = Mock() - ctx.params = { - 'url': 'http://test.com', - 'token_uri': 'token_uri', - 'client_id': 'client_id', - 'client_machine': 'client_machine', - 'client_secret': secret_file.as_posix(), - 'small_file_size_mb': float(10) - } - - # should catch UTILITIES_MAX_FILE_SIZE if value is not provided - result = FirecrestTransport._dynamic_info_direct_size(ctx, None, 0) - assert result == 69 - - # should use the value if provided - # note: user cannot enter negative numbers anyways, click raise as this shoule be float not str - result = FirecrestTransport._dynamic_info_direct_size(ctx, None, 10) - assert result == 10 - - @pytest.mark.usefixtures("aiida_profile_clean") def test_mkdir(firecrest_computer: orm.Computer, tmpdir: Path): transport = firecrest_computer.get_transport() From 83a4eb874d551e767a801c466bca966dc71ecff6 Mon Sep 17 00:00:00 2001 From: Ali Date: Wed, 19 Jun 2024 14:31:59 +0200 Subject: [PATCH 04/39] update readme.md --- README.md | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index ea22933..f6d8ee7 100644 --- a/README.md +++ b/README.md @@ -128,6 +128,13 @@ pre-commit run --all-files ### Testing +There are two types of tests: mocking the PyFirecREST or the FirecREST server. +While the former is a good practice to ensure that all three (`aiida-firecrest`, FirecREST, and PyFirecREST) work flawlessly, debugging may not always be easy because it may not always be obvious which of the three is causing a bug. +Because of this, we have another set of tests that only verify the functionality of `aiida-firecrest` by directly mocking PyFirecREST. Maintaining the second set in `tests/tests_mocking_pyfirecrest/` is simpler because we just need to monitor the return values of PyFirecREST​. While maintaining the former is more difficult as you have to keep up with both FirecREST and PyFirecREST. + + +#### Mocking FirecREST server + It is recommended to run the tests via [tox](https://tox.readthedocs.io/en/latest/). ```bash @@ -174,7 +181,7 @@ you can use the `--firecrest-requests` option: tox -- --firecrest-requests ``` -### Notes on using the demo server on MacOS +##### Notes on using the demo server on MacOS A few issues have been noted when using the demo server on MacOS (non-Mx): @@ -195,3 +202,13 @@ although it is of note that you can find these files directly where you your `fi [codecov-link]: https://codecov.io/gh/aiidateam/aiida-firecrest [black-badge]: https://img.shields.io/badge/code%20style-black-000000.svg [black-link]: https://github.com/ambv/black + + + +#### Mocking PyFirecREST + +These set of test do not gurantee that the firecrest protocol is working, but it's very usefull to quickly check if `aiida-firecrest` is behaving as it's expected to do. To run just simply use `pytest`. + + +If these tests, pass and still you have trouble in real deploymeny that means your installed version of pyfirecrest is behaving differently from what `aiida-firecrest` expects in `MockFirecrest` in `tests/tests_mocking_pyfirecrest/conftest.py`. +In order to solve that, first spot what is different and then solve or raise to `pyfirecrest` if relevant. From 4052ae7c3aa6e7a45c0fcbeb5cc257404af801e9 Mon Sep 17 00:00:00 2001 From: Ali Date: Wed, 19 Jun 2024 16:22:06 +0200 Subject: [PATCH 05/39] added test_transport::put* --- aiida_firecrest/transport.py | 23 +- tests/tests_mocking_pyfirecrest/conftest.py | 17 +- .../test_transport.py | 267 +++++++++++++++++- 3 files changed, 285 insertions(+), 22 deletions(-) diff --git a/aiida_firecrest/transport.py b/aiida_firecrest/transport.py index e2f51f8..1b6afa2 100644 --- a/aiida_firecrest/transport.py +++ b/aiida_firecrest/transport.py @@ -763,7 +763,9 @@ def putfile( if not localpath.is_absolute(): raise ValueError("The localpath must be an absolute path") if not localpath.is_file(): - raise ValueError(f"Input localpath is not a file: {localpath}") + if not localpath.exists(): + raise FileNotFoundError(f"Local file does not exist: {localpath}") + raise ValueError(f"Input localpath is not a file {localpath}") remote = self._cwd.joinpath(remotepath)#.enable_cache() it's removed from from path.py to be investigated @@ -831,16 +833,11 @@ def _puttreetar( tarpath = localpath.parent.joinpath(f"temp_{_}.tar") remote_path_temp = self._temp_directory.joinpath(f"temp_{_}.tar") with tarfile.open(tarpath, "w", dereference=dereference) as tar: - if dereference: - for root, dirs, files in os.walk(localpath): - for file in files: - full_path = os.path.join(root, file) - relative_path = os.path.relpath(full_path, localpath) - tar.add(full_path, arcname=relative_path) - else: - # iterdir() ignores symbolic links - for item in localpath.iterdir(): - tar.add(item, arcname=item.name) + for root, dirs, files in os.walk(localpath, followlinks=dereference): + for file in files: + full_path = os.path.join(root, file) + relative_path = os.path.relpath(full_path, localpath) + tar.add(full_path, arcname=relative_path) # Upload try: @@ -909,7 +906,7 @@ def puttree( self._puttreetar(localpath, remotepath) else: # otherwise send the files one by one - for dirpath, _, filenames in os.walk(localpath): + for dirpath, _, filenames in os.walk(localpath, followlinks=dereference): rel_folder = os.path.relpath(path=dirpath, start=localpath) rm_parent_now = remotepath.joinpath(rel_folder) @@ -929,7 +926,7 @@ def put(self, localpath: str, remotepath: str, ignore_nonexisting: bool =False, note: dereference should be always True, otherwise the symlinks will not be functional. """ # TODO ssh does a lot more - # update to above TODO: I made a manual test with ssh. added some extra care in puttree and gettree and now it's working fine + # update on the TODO: I made a manual test with ssh. added some extra care in puttree and gettree and now it's working fine if not dereference: raise NotImplementedError diff --git a/tests/tests_mocking_pyfirecrest/conftest.py b/tests/tests_mocking_pyfirecrest/conftest.py index a1fbb89..b59c223 100644 --- a/tests/tests_mocking_pyfirecrest/conftest.py +++ b/tests/tests_mocking_pyfirecrest/conftest.py @@ -66,6 +66,7 @@ def __init__(self, firecrest_url, *args, **kwargs): self.symlink = symlink self.checksum = checksum self.simple_download = simple_download + self.simple_upload = simple_upload self.compress = compress self.extract = extract self.submit = submit @@ -169,9 +170,20 @@ def simple_download(machine: str, remote_path: str, local_path: str): raise IsADirectoryError(f"{remote_path} is a directory") if not Path(remote_path).exists(): raise FileNotFoundError(f"{remote_path} does not exist") - # print(f"{remote_path} {local_path}") os.system(f"cp {remote_path} {local_path}") +def simple_upload(machine: str, local_path: str, remote_path: str, file_name: str = None): + # this procedure is complecated in firecrest, but I am simplifying it here + # we don't care about the details of the upload, we just want to make sure + # that the aiida-firecrest code is calling the right functions at right time + if Path(local_path).is_dir(): + raise IsADirectoryError(f"{local_path} is a directory") + if not Path(local_path).exists(): + raise FileNotFoundError(f"{local_path} does not exist") + if file_name: + remote_path = os.path.join(remote_path, file_name) + os.system(f"cp {local_path} {remote_path}") + def copy(machine: str, source_path: str, target_path: str): # this is how firecrest does it # https://github.com/eth-cscs/firecrest/blob/db6ba4ba273c11a79ecbe940872f19d5cb19ac5e/src/utilities/utilities.py#L451C1-L452C1 @@ -188,8 +200,7 @@ def compress(machine: str, source_path: str, target_path: str, dereference: bool def extract(machine: str, source_path: str, target_path: str): # this is how firecrest does it # https://github.com/eth-cscs/firecrest/blob/db6ba4ba273c11a79ecbe940872f19d5cb19ac5e/src/common/cscs_api_common.py#L1110C18-L1110C65 - breakpoint() - os.system("tar -xf '{source_path}' -C '{target_path}'") + os.system(f"tar -xf '{source_path}' -C '{target_path}'") def checksum(machine: str, remote_path: str) -> int: if not remote_path.exists(): diff --git a/tests/tests_mocking_pyfirecrest/test_transport.py b/tests/tests_mocking_pyfirecrest/test_transport.py index 22c8184..fafc8bf 100644 --- a/tests/tests_mocking_pyfirecrest/test_transport.py +++ b/tests/tests_mocking_pyfirecrest/test_transport.py @@ -159,7 +159,6 @@ def test_get(firecrest_computer: orm.Computer, tmpdir: Path): with pytest.raises(ValueError): transport.get(_remote / "file1", Path(_local).relative_to(tmpdir)) - @pytest.mark.usefixtures("aiida_profile_clean") def test_getfile(firecrest_computer: orm.Computer, tmpdir: Path): transport = firecrest_computer.get_transport() @@ -214,11 +213,10 @@ def test_getfile(firecrest_computer: orm.Computer, tmpdir: Path): assert Path(_local / "file1_link").read_text() == "file1" assert not Path(_local / "file1_link").is_symlink() - @pytest.mark.usefixtures("aiida_profile_clean") -def test_gettree_notar(firecrest_computer: orm.Computer, tmpdir: Path, monkeypatch): +def test_gettree_notar(firecrest_computer: orm.Computer, tmpdir: Path): """ - This test is to check if the gettree function is working as expected. Through non tar mode. + This test is to check `gettree` through non tar mode. payoff= False in this test, so just checking if getting files one by one is working as expected. """ transport = firecrest_computer.get_transport() @@ -256,7 +254,7 @@ def test_gettree_notar(firecrest_computer: orm.Computer, tmpdir: Path, monkeypat with pytest.raises(ValueError): transport.gettree(_remote, Path(_local).relative_to(tmpdir)) - # If destination directory does not exists, AiiDA expects transport make the new path as root not _remote.name + # If destination directory does not exists, AiiDA expects transport make the new path as root not using _remote.name transport.gettree(_remote, _local / "newdir") _root = _local / "newdir" # tree should be copied recursively @@ -297,7 +295,7 @@ def test_gettree_notar(firecrest_computer: orm.Computer, tmpdir: Path, monkeypat @pytest.mark.usefixtures("aiida_profile_clean") def test_gettree_bytar(firecrest_computer: orm.Computer, tmpdir: Path): """ - This test is to check if the gettree function is working as expected. Through non tar mode. + This test is to check `gettree` through non tar mode. bytar= True in this test. """ transport = firecrest_computer.get_transport() @@ -375,3 +373,260 @@ def test_gettree_bytar(firecrest_computer: orm.Computer, tmpdir: Path): assert not Path(_root / "dir1" / "dir20_link" / "file3").is_symlink() + +@pytest.mark.usefixtures("aiida_profile_clean") +def test_put(firecrest_computer: orm.Computer, tmpdir: Path): + """ + This is minimal test is to check if put() is raising errors as expected, + and directing to putfile() and puttree() as expected. + Mainly just checking error handeling and folder creation. + """ + transport = firecrest_computer.get_transport() + + _remote = tmpdir / "remotedir" + _local = tmpdir / "localdir" + _remote.mkdir() + _local.mkdir() + + + # check if the code is directing to putfile() or puttree() as expected + with patch.object(transport, 'puttree', autospec=True) as mock_puttree: + transport.put(_local, _remote) + mock_puttree.assert_called_once() + + with patch.object(transport, 'puttree', autospec=True) as mock_puttree: + os.symlink(_local, tmpdir / "dir_link") + transport.put(tmpdir / "dir_link", _remote) + mock_puttree.assert_called_once() + + with patch.object(transport, 'putfile', autospec=True) as mock_putfile: + Path(_local / "file1").write_text("file1") + transport.put(_local / "file1", _remote / "file1") + mock_putfile.assert_called_once() + + with patch.object(transport, 'putfile', autospec=True) as mock_putfile: + os.symlink(_local / "file1", _local / "file1_link") + transport.put(_local / "file1_link", _remote / "file1_link") + mock_putfile.assert_called_once() + + # raise if local file/folder does not exist + with pytest.raises(FileNotFoundError): + transport.put(_local / "does_not_exist", _remote) + transport.put(_local / "does_not_exist", _remote, ignore_nonexisting=True) + + # raise if localpath is relative + with pytest.raises(ValueError): + transport.put(Path(_local).relative_to(tmpdir), _remote) + with pytest.raises(ValueError): + transport.put(Path(_local / "file1").relative_to(tmpdir), _remote) + +@pytest.mark.usefixtures("aiida_profile_clean") +def test_putfile(firecrest_computer: orm.Computer, tmpdir: Path): + transport = firecrest_computer.get_transport() + + _remote = tmpdir / "remotedir" + _local = tmpdir / "localdir" + _remote.mkdir() + _local.mkdir() + + Path(_local / "file1").write_text("file1") + Path(_local / ".hidden").write_text(".hidden") + os.symlink(_local / "file1", _local / "file1_link") + + + # raise if local file does not exist + with pytest.raises(FileNotFoundError): + transport.putfile(_local/ "does_not_exist" ,_remote) + + # raise if remotefilename is not provided + with pytest.raises(ValueError): + transport.putfile(_local / "file1", _remote) + + # raise if localpath is relative + with pytest.raises(ValueError): + transport.putfile(Path(_local / "file1").relative_to(tmpdir), _remote / "file1") + + # don't mix up directory with file + with pytest.raises(ValueError): + transport.putfile(_local, _remote / "file1") + + # write where I tell you to + transport.putfile(_local / "file1", _remote / "file1") + transport.putfile(_local / "file1", _remote / "file1-prime") + assert Path(_remote / "file1").read_text() == "file1" + assert Path(_remote / "file1-prime").read_text() == "file1" + + # always overwrite + transport.putfile(_local / "file1", _remote / "file1") + assert Path(_remote / "file1").read_text() == "file1" + + Path(_remote / "file1").write_text("notfile1") + + transport.putfile(_local / "file1", _remote / "file1") + assert Path(_remote / "file1").read_text() == "file1" + + # don't skip hidden files + transport.putfile(_local / ".hidden", _remote / ".hidden-prime") + assert Path(_remote / ".hidden-prime").read_text() == ".hidden" + + # follow links + transport.putfile(_local / "file1_link", _remote / "file1_link") + assert Path(_remote / "file1_link").read_text() == "file1" + assert not Path(_remote / "file1_link").is_symlink() + +@pytest.mark.usefixtures("aiida_profile_clean") +def test_puttree_notar(firecrest_computer: orm.Computer, tmpdir: Path): + """ + This test is to check `puttree` through non tar mode. + payoff= False in this test, so just checking if putting files one by one is working as expected. + """ + transport = firecrest_computer.get_transport() + transport.payoff_override = False + + _remote = tmpdir / "remotedir" + _local = tmpdir / "localdir" + _remote.mkdir() + _local.mkdir() + # a typical tree + Path(_local / "dir1").mkdir() + Path(_local / "dir2").mkdir() + Path(_local / "file1").write_text("file1") + Path(_local / ".hidden").write_text(".hidden") + Path(_local / "dir1" / "file2").write_text("file2") + Path(_local / "dir2" / "file3").write_text("file3") + # with symlinks to a file even if pointing to a relative path + os.symlink(_local / "file1", _local / "dir1" / "file1_link") + os.symlink(Path("../file1"), _local / "dir1" / "file10_link") + # with symlinks to a folder even if pointing to a relative path + os.symlink(_local / "dir2", _local / "dir1" / "dir2_link") + os.symlink(Path("../dir2" ), _local / "dir1" / "dir20_link") + + # raise if local file does not exist + with pytest.raises(OSError): + transport.puttree(_local / "does_not_exist", _remote) + + # raise if local is a file + with pytest.raises(ValueError): + Path(tmpdir / "isfile").touch() + transport.puttree(tmpdir / "isfile", _remote) + + # raise if localpath is relative + with pytest.raises(ValueError): + transport.puttree(Path(_local).relative_to(tmpdir), _remote) + + # If destination directory does not exists, AiiDA expects transport make the new path as root not using _local.name + transport.puttree(_local, _remote / "newdir") + _root = _remote / "newdir" + # tree should be copied recursively + assert Path(_root / "file1").read_text() == "file1" + assert Path(_root / ".hidden").read_text() == ".hidden" + assert Path(_root / "dir1" / "file2").read_text() == "file2" + assert Path(_root / "dir2" / "file3").read_text() == "file3" + # symlink should be followed + assert Path(_root / "dir1" / "file1_link").read_text() == "file1" + assert Path(_root / "dir1" / "dir2_link" / "file3").read_text() == "file3" + assert Path(_root / "dir1" / "file10_link").read_text() == "file1" + assert Path(_root / "dir1" / "dir20_link" / "file3").read_text() == "file3" + assert not Path(_root / "dir1" / "file1_link").is_symlink() + assert not Path(_root / "dir1" / "dir2_link" / "file3").is_symlink() + assert not Path(_root / "dir1" / "file10_link").is_symlink() + assert not Path(_root / "dir1" / "dir20_link" / "file3").is_symlink() + + + # If destination directory does exists, AiiDA expects transport make _local.name and write into it + # however this might have changed in the newer versions of AiiDA ~ 2.6.0 (IDK) + transport.puttree(_local, _remote / "newdir") + _root = _remote / "newdir" / Path(_local).name + # tree should be copied recursively + assert Path(_root / "file1").read_text() == "file1" + assert Path(_root / ".hidden").read_text() == ".hidden" + assert Path(_root / "dir1" / "file2").read_text() == "file2" + assert Path(_root / "dir2" / "file3").read_text() == "file3" + # symlink should be followed + assert Path(_root / "dir1" / "file1_link").read_text() == "file1" + assert Path(_root / "dir1" / "dir2_link" / "file3").read_text() == "file3" + assert Path(_root / "dir1" / "file10_link").read_text() == "file1" + assert Path(_root / "dir1" / "dir20_link" / "file3").read_text() == "file3" + assert not Path(_root / "dir1" / "file1_link").is_symlink() + assert not Path(_root / "dir1" / "dir2_link" / "file3").is_symlink() + assert not Path(_root / "dir1" / "file10_link").is_symlink() + assert not Path(_root / "dir1" / "dir20_link" / "file3").is_symlink() + +@pytest.mark.usefixtures("aiida_profile_clean") +def test_puttree_bytar(firecrest_computer: orm.Computer, tmpdir: Path): + """ + This test is to check `puttree` through tar mode. + payoff= True in this test. + """ + transport = firecrest_computer.get_transport() + transport.payoff_override = True + + _remote = tmpdir / "remotedir" + _local = tmpdir / "localdir" + _remote.mkdir() + _local.mkdir() + # a typical tree + Path(_local / "dir1").mkdir() + Path(_local / "dir2").mkdir() + Path(_local / "file1").write_text("file1") + Path(_local / ".hidden").write_text(".hidden") + Path(_local / "dir1" / "file2").write_text("file2") + Path(_local / "dir2" / "file3").write_text("file3") + # with symlinks to a file even if pointing to a relative path + os.symlink(_local / "file1", _local / "dir1" / "file1_link") + os.symlink(Path("../file1"), _local / "dir1" / "file10_link") + # with symlinks to a folder even if pointing to a relative path + os.symlink(_local / "dir2", _local / "dir1" / "dir2_link") + os.symlink(Path("../dir2" ), _local / "dir1" / "dir20_link") + + # raise if local file does not exist + with pytest.raises(OSError): + transport.puttree(_local / "does_not_exist", _remote) + + # raise if local is a file + with pytest.raises(ValueError): + Path(tmpdir / "isfile").touch() + transport.puttree(tmpdir / "isfile", _remote) + + # raise if localpath is relative + with pytest.raises(ValueError): + transport.puttree(Path(_local).relative_to(tmpdir), _remote) + + # If destination directory does not exists, AiiDA expects transport make the new path as root not using _local.name + transport.puttree(_local, _remote / "newdir") + _root = _remote / "newdir" + # tree should be copied recursively + assert Path(_root / "file1").read_text() == "file1" + assert Path(_root / ".hidden").read_text() == ".hidden" + assert Path(_root / "dir1" / "file2").read_text() == "file2" + assert Path(_root / "dir2" / "file3").read_text() == "file3" + # symlink should be followed + assert Path(_root / "dir1" / "file1_link").read_text() == "file1" + assert Path(_root / "dir1" / "dir2_link" / "file3").read_text() == "file3" + assert Path(_root / "dir1" / "file10_link").read_text() == "file1" + assert Path(_root / "dir1" / "dir20_link" / "file3").read_text() == "file3" + assert not Path(_root / "dir1" / "file1_link").is_symlink() + assert not Path(_root / "dir1" / "dir2_link" / "file3").is_symlink() + assert not Path(_root / "dir1" / "file10_link").is_symlink() + assert not Path(_root / "dir1" / "dir20_link" / "file3").is_symlink() + + + # If destination directory does exists, AiiDA expects transport make _local.name and write into it + # however this might have changed in the newer versions of AiiDA ~ 2.6.0 (IDK) + transport.puttree(_local, _remote / "newdir") + _root = _remote / "newdir" / Path(_local).name + # tree should be copied recursively + assert Path(_root / "file1").read_text() == "file1" + assert Path(_root / ".hidden").read_text() == ".hidden" + assert Path(_root / "dir1" / "file2").read_text() == "file2" + assert Path(_root / "dir2" / "file3").read_text() == "file3" + # symlink should be followed + assert Path(_root / "dir1" / "file1_link").read_text() == "file1" + assert Path(_root / "dir1" / "dir2_link" / "file3").read_text() == "file3" + assert Path(_root / "dir1" / "file10_link").read_text() == "file1" + assert Path(_root / "dir1" / "dir20_link" / "file3").read_text() == "file3" + assert not Path(_root / "dir1" / "file1_link").is_symlink() + assert not Path(_root / "dir1" / "dir2_link" / "file3").is_symlink() + assert not Path(_root / "dir1" / "file10_link").is_symlink() + assert not Path(_root / "dir1" / "dir20_link" / "file3").is_symlink() + From a4588f816fc6e392ffc4803e27b1d33243f8921e Mon Sep 17 00:00:00 2001 From: Ali Date: Wed, 19 Jun 2024 18:28:21 +0200 Subject: [PATCH 06/39] added test_transport::copy* --- aiida_firecrest/transport.py | 35 +-- tests/tests_mocking_pyfirecrest/conftest.py | 3 +- .../test_transport.py | 265 ++++++++---------- 3 files changed, 129 insertions(+), 174 deletions(-) diff --git a/aiida_firecrest/transport.py b/aiida_firecrest/transport.py index 1b6afa2..f692e3f 100644 --- a/aiida_firecrest/transport.py +++ b/aiida_firecrest/transport.py @@ -451,7 +451,6 @@ def copyfile( """Copy a file on the remote. FirecREST does not support symlink copying. :param dereference: If True, copy the target of the symlink instead of the symlink itself. - Warning! even if deference is set to False, I'm not sure if the symlink will be functional after the copy. """ source = self._cwd.joinpath(remotesource)#.enable_cache() it's removed from from path.py to be investigated destination = self._cwd.joinpath(remotedestination)#.enable_cache() it's removed from from path.py to be investigated @@ -460,7 +459,9 @@ def copyfile( if not source.exists(): raise FileNotFoundError(f"Source file does not exist: {source}") if not source.is_file(): - raise FileNotFoundError(f"Source is not a file: {source}") + raise ValueError(f"Source is not a file: {source}") + if not destination.exists() and not source.is_file(): + raise FileNotFoundError(f"Destination file does not exist: {destination}") source.copy_to(destination) # I removed symlink copy, becasue it's really not a file copy, it's a link copy @@ -473,8 +474,8 @@ def copytree( """Copy a directory on the remote. FirecREST does not support symlink copying. :param dereference: If True, copy the target of the symlink instead of the symlink itself. - Warning! even if deference is set to False, I'm not sure if the symlink will be functional after the copy. """ + #TODO: check if deference is set to False, symlinks will be functional after the copy in Firecrest server. source = self._cwd.joinpath(remotesource)#.enable_cache().enable_cache() it's removed from from path.py to be investigated destination = ( @@ -485,15 +486,12 @@ def copytree( if not source.exists(): raise FileNotFoundError(f"Source file does not exist: {source}") if not source.is_dir(): - raise FileNotFoundError(f"Source is not a directory: {source}") + raise ValueError(f"Source is not a directory: {source}") + if not destination.exists(): + raise FileNotFoundError(f"Destination file does not exist: {destination}") source.copy_to(destination) - # TODO: the block belowe does not work for nested symlinks, if we really need that, - # we have to asked them to make this option for us in FirecREST. - # if not dereference and source.is_symlink(): - # destination.symlink_to(source) - # else: - # source.copy_to(destination) + def copy( self, @@ -505,11 +503,12 @@ def copy( """Copy a file or directory on the remote. FirecREST does not support symlink copying. :param recursive: If True, copy directories recursively. - note that the non-recursive option is not implemented in FirecREST server + note that the non-recursive option is not implemented in FirecREST server. + And it's not used in upstream, anyways... :param dereference: If True, copy the target of the symlink instead of the symlink itself. - Warning! even if deference is set to False, I'm not sure if the symlink will be functional after the copy. """ + # TODO: investigate overwrite (?) if not recursive: # TODO this appears to not actually be used upstream, so just remove there @@ -520,19 +519,13 @@ def copy( destination = self._cwd.joinpath(remotedestination)#.enable_cache() it's removed from from path.py to be investigated if not source.exists(): - raise FileNotFoundError(f"Source file does not exist: {source}") + raise FileNotFoundError(f"Source does not exist: {source}") + if not destination.exists() and not source.is_file(): + raise FileNotFoundError(f"Destination does not exist: {destination}") source.copy_to(destination) - # TODO: the block belowe does not work for nested symlinks, if we really need that, - # we have to asked them to make this option for us - # if not dereference and source.is_symlink(): - # destination.symlink_to(source) - # else: - # source.copy_to(destination) - # TODO check symlink handling for get methods - # symlink handeling is done. # TODO do get/put methods need to handle glob patterns? # Apparently not, but I'm not clear how glob() iglob() are going to behave here. We may need to implement them. diff --git a/tests/tests_mocking_pyfirecrest/conftest.py b/tests/tests_mocking_pyfirecrest/conftest.py index b59c223..95bc3c0 100644 --- a/tests/tests_mocking_pyfirecrest/conftest.py +++ b/tests/tests_mocking_pyfirecrest/conftest.py @@ -69,6 +69,7 @@ def __init__(self, firecrest_url, *args, **kwargs): self.simple_upload = simple_upload self.compress = compress self.extract = extract + self.copy = copy self.submit = submit # self.poll_active = poll_active @@ -195,7 +196,7 @@ def compress(machine: str, source_path: str, target_path: str, dereference: bool basedir = os.path.dirname(source_path) file_path = os.path.basename(source_path) deref = "--dereference" if dereference else "" - os.system(f"tar {deref} -czvf '{target_path}' -C '{basedir}' '{file_path}'") + os.system(f"tar {deref} -czf '{target_path}' -C '{basedir}' '{file_path}'") def extract(machine: str, source_path: str, target_path: str): # this is how firecrest does it diff --git a/tests/tests_mocking_pyfirecrest/test_transport.py b/tests/tests_mocking_pyfirecrest/test_transport.py index fafc8bf..c244c5a 100644 --- a/tests/tests_mocking_pyfirecrest/test_transport.py +++ b/tests/tests_mocking_pyfirecrest/test_transport.py @@ -83,7 +83,6 @@ def test_symlink(firecrest_computer: orm.Computer, tmpdir: Path): assert _symlink.is_symlink() assert _symlink.resolve() == _scratch - @pytest.mark.usefixtures("aiida_profile_clean") def test_listdir(firecrest_computer: orm.Computer, tmpdir: Path): transport = firecrest_computer.get_transport() @@ -213,93 +212,15 @@ def test_getfile(firecrest_computer: orm.Computer, tmpdir: Path): assert Path(_local / "file1_link").read_text() == "file1" assert not Path(_local / "file1_link").is_symlink() +@pytest.mark.parametrize("payoff", [True, False]) @pytest.mark.usefixtures("aiida_profile_clean") -def test_gettree_notar(firecrest_computer: orm.Computer, tmpdir: Path): - """ - This test is to check `gettree` through non tar mode. - payoff= False in this test, so just checking if getting files one by one is working as expected. - """ - transport = firecrest_computer.get_transport() - transport.payoff_override = False - - _remote = tmpdir / "remotedir" - _local = tmpdir / "localdir" - _remote.mkdir() - _local.mkdir() - # a typical tree - Path(_remote / "dir1").mkdir() - Path(_remote / "dir2").mkdir() - Path(_remote / "file1").write_text("file1") - Path(_remote / ".hidden").write_text(".hidden") - Path(_remote / "dir1" / "file2").write_text("file2") - Path(_remote / "dir2" / "file3").write_text("file3") - # with symlinks to a file even if pointing to a relative path - os.symlink(_remote / "file1", _remote / "dir1" / "file1_link") - os.symlink(Path("../file1"), _remote / "dir1" / "file10_link") - # with symlinks to a folder even if pointing to a relative path - os.symlink(_remote / "dir2", _remote / "dir1" / "dir2_link") - os.symlink(Path("../dir2" ), _remote / "dir1" / "dir20_link") - - - # raise if remote file does not exist - with pytest.raises(OSError): - transport.gettree(_remote / "does_not_exist", _local) - - # raise if local is a file - with pytest.raises(OSError): - Path(tmpdir / "isfile").touch() - transport.gettree(_remote, tmpdir / "isfile") - - # raise if localpath is relative - with pytest.raises(ValueError): - transport.gettree(_remote, Path(_local).relative_to(tmpdir)) - - # If destination directory does not exists, AiiDA expects transport make the new path as root not using _remote.name - transport.gettree(_remote, _local / "newdir") - _root = _local / "newdir" - # tree should be copied recursively - assert Path(_root / "file1").read_text() == "file1" - assert Path(_root / ".hidden").read_text() == ".hidden" - assert Path(_root / "dir1" / "file2").read_text() == "file2" - assert Path(_root / "dir2" / "file3").read_text() == "file3" - # symlink should be followed - assert Path(_root / "dir1" / "file1_link").read_text() == "file1" - assert Path(_root / "dir1" / "dir2_link" / "file3").read_text() == "file3" - assert Path(_root / "dir1" / "file10_link").read_text() == "file1" - assert Path(_root / "dir1" / "dir20_link" / "file3").read_text() == "file3" - assert not Path(_root / "dir1" / "file1_link").is_symlink() - assert not Path(_root / "dir1" / "dir2_link" / "file3").is_symlink() - assert not Path(_root / "dir1" / "file10_link").is_symlink() - assert not Path(_root / "dir1" / "dir20_link" / "file3").is_symlink() - - - # If destination directory does exists, AiiDA expects transport make _remote.name and write into it - # however this might have changed in the newer versions of AiiDA ~ 2.6.0 (IDK) - transport.gettree(_remote, _local / "newdir") - _root = _local / "newdir" / Path(_remote).name - # tree should be copied recursively - assert Path(_root / "file1").read_text() == "file1" - assert Path(_root / ".hidden").read_text() == ".hidden" - assert Path(_root / "dir1" / "file2").read_text() == "file2" - assert Path(_root / "dir2" / "file3").read_text() == "file3" - # symlink should be followed - assert Path(_root / "dir1" / "file1_link").read_text() == "file1" - assert Path(_root / "dir1" / "dir2_link" / "file3").read_text() == "file3" - assert Path(_root / "dir1" / "file10_link").read_text() == "file1" - assert Path(_root / "dir1" / "dir20_link" / "file3").read_text() == "file3" - assert not Path(_root / "dir1" / "file1_link").is_symlink() - assert not Path(_root / "dir1" / "dir2_link" / "file3").is_symlink() - assert not Path(_root / "dir1" / "file10_link").is_symlink() - assert not Path(_root / "dir1" / "dir20_link" / "file3").is_symlink() - -@pytest.mark.usefixtures("aiida_profile_clean") -def test_gettree_bytar(firecrest_computer: orm.Computer, tmpdir: Path): +def test_gettree(firecrest_computer: orm.Computer, tmpdir: Path, payoff: bool): """ This test is to check `gettree` through non tar mode. bytar= True in this test. """ transport = firecrest_computer.get_transport() - transport.payoff_override = True + transport.payoff_override = payoff _remote = tmpdir / "remotedir" _local = tmpdir / "localdir" @@ -372,8 +293,6 @@ def test_gettree_bytar(firecrest_computer: orm.Computer, tmpdir: Path): assert not Path(_root / "dir1" / "file10_link").is_symlink() assert not Path(_root / "dir1" / "dir20_link" / "file3").is_symlink() - - @pytest.mark.usefixtures("aiida_profile_clean") def test_put(firecrest_computer: orm.Computer, tmpdir: Path): """ @@ -474,14 +393,15 @@ def test_putfile(firecrest_computer: orm.Computer, tmpdir: Path): assert Path(_remote / "file1_link").read_text() == "file1" assert not Path(_remote / "file1_link").is_symlink() +@pytest.mark.parametrize("payoff", [True, False]) @pytest.mark.usefixtures("aiida_profile_clean") -def test_puttree_notar(firecrest_computer: orm.Computer, tmpdir: Path): +def test_puttree(firecrest_computer: orm.Computer, tmpdir: Path, payoff: bool): """ This test is to check `puttree` through non tar mode. payoff= False in this test, so just checking if putting files one by one is working as expected. """ transport = firecrest_computer.get_transport() - transport.payoff_override = False + transport.payoff_override = payoff _remote = tmpdir / "remotedir" _local = tmpdir / "localdir" @@ -552,81 +472,122 @@ def test_puttree_notar(firecrest_computer: orm.Computer, tmpdir: Path): assert not Path(_root / "dir1" / "file10_link").is_symlink() assert not Path(_root / "dir1" / "dir20_link" / "file3").is_symlink() + +@pytest.mark.parametrize("to_test", ['copy', 'copytree']) @pytest.mark.usefixtures("aiida_profile_clean") -def test_puttree_bytar(firecrest_computer: orm.Computer, tmpdir: Path): - """ - This test is to check `puttree` through tar mode. - payoff= True in this test. - """ +def test_copy(firecrest_computer: orm.Computer, tmpdir: Path, to_test: str): + transport = firecrest_computer.get_transport() - transport.payoff_override = True + if to_test == 'copy': + testing = transport.copy + elif to_test == 'copytree': + testing = transport.copytree + + + _remote_1 = tmpdir / "remotedir_1" + _remote_2 = tmpdir / "remotedir_2" + _remote_1.mkdir() + _remote_2.mkdir() + + # raise if source or destination does not exist + with pytest.raises(FileNotFoundError): + testing(_remote_1 / "does_not_exist", _remote_2) + with pytest.raises(FileNotFoundError): + testing(_remote_1, _remote_2 / "does_not_exist") + + + # raise if source is inappropriate + if to_test == 'copytree': + Path(tmpdir / "file1").touch() + with pytest.raises(ValueError): + testing(tmpdir / "file1", _remote_2) - _remote = tmpdir / "remotedir" - _local = tmpdir / "localdir" - _remote.mkdir() - _local.mkdir() # a typical tree - Path(_local / "dir1").mkdir() - Path(_local / "dir2").mkdir() - Path(_local / "file1").write_text("file1") - Path(_local / ".hidden").write_text(".hidden") - Path(_local / "dir1" / "file2").write_text("file2") - Path(_local / "dir2" / "file3").write_text("file3") + Path(_remote_1 / "dir1").mkdir() + Path(_remote_1 / "dir2").mkdir() + Path(_remote_1 / "file1").write_text("file1") + Path(_remote_1 / ".hidden").write_text(".hidden") + Path(_remote_1 / "dir1" / "file2").write_text("file2") + Path(_remote_1 / "dir2" / "file3").write_text("file3") # with symlinks to a file even if pointing to a relative path - os.symlink(_local / "file1", _local / "dir1" / "file1_link") - os.symlink(Path("../file1"), _local / "dir1" / "file10_link") + os.symlink(_remote_1 / "file1", _remote_1 / "dir1" / "file1_link") + os.symlink(Path("../file1"), _remote_1 / "dir1" / "file10_link") # with symlinks to a folder even if pointing to a relative path - os.symlink(_local / "dir2", _local / "dir1" / "dir2_link") - os.symlink(Path("../dir2" ), _local / "dir1" / "dir20_link") - - # raise if local file does not exist - with pytest.raises(OSError): - transport.puttree(_local / "does_not_exist", _remote) - - # raise if local is a file - with pytest.raises(ValueError): - Path(tmpdir / "isfile").touch() - transport.puttree(tmpdir / "isfile", _remote) - - # raise if localpath is relative - with pytest.raises(ValueError): - transport.puttree(Path(_local).relative_to(tmpdir), _remote) + os.symlink(_remote_1 / "dir2", _remote_1 / "dir1" / "dir2_link") + os.symlink(Path("../dir2" ), _remote_1 / "dir1" / "dir20_link") + + testing(_remote_1, _remote_2) + - # If destination directory does not exists, AiiDA expects transport make the new path as root not using _local.name - transport.puttree(_local, _remote / "newdir") - _root = _remote / "newdir" + _root_2 = _remote_2 / Path(_remote_1).name # tree should be copied recursively - assert Path(_root / "file1").read_text() == "file1" - assert Path(_root / ".hidden").read_text() == ".hidden" - assert Path(_root / "dir1" / "file2").read_text() == "file2" - assert Path(_root / "dir2" / "file3").read_text() == "file3" + assert Path(_root_2 / "dir1").exists() + assert Path(_root_2 / "dir2").exists() + assert Path(_root_2 / "file1").read_text() == "file1" + assert Path(_root_2 / ".hidden").read_text() == ".hidden" + assert Path(_root_2 / "dir1" / "file2").read_text () == "file2" + assert Path(_root_2 / "dir2" / "file3").read_text() == "file3" # symlink should be followed - assert Path(_root / "dir1" / "file1_link").read_text() == "file1" - assert Path(_root / "dir1" / "dir2_link" / "file3").read_text() == "file3" - assert Path(_root / "dir1" / "file10_link").read_text() == "file1" - assert Path(_root / "dir1" / "dir20_link" / "file3").read_text() == "file3" - assert not Path(_root / "dir1" / "file1_link").is_symlink() - assert not Path(_root / "dir1" / "dir2_link" / "file3").is_symlink() - assert not Path(_root / "dir1" / "file10_link").is_symlink() - assert not Path(_root / "dir1" / "dir20_link" / "file3").is_symlink() + assert Path(_root_2 / "dir1" / "dir2_link").exists() + assert Path(_root_2 / "dir1" / "dir20_link").exists() + assert Path(_root_2 / "dir1" / "file1_link").read_text() == "file1" + assert Path(_root_2 / "dir1" / "file10_link").read_text() == "file1" + assert Path(_root_2 / "dir1" / "file1_link").is_symlink() + assert Path(_root_2 / "dir1" / "dir2_link").is_symlink() + assert Path(_root_2 / "dir1" / "file10_link").is_symlink() + assert Path(_root_2 / "dir1" / "dir20_link").is_symlink() - # If destination directory does exists, AiiDA expects transport make _local.name and write into it - # however this might have changed in the newer versions of AiiDA ~ 2.6.0 (IDK) - transport.puttree(_local, _remote / "newdir") - _root = _remote / "newdir" / Path(_local).name - # tree should be copied recursively - assert Path(_root / "file1").read_text() == "file1" - assert Path(_root / ".hidden").read_text() == ".hidden" - assert Path(_root / "dir1" / "file2").read_text() == "file2" - assert Path(_root / "dir2" / "file3").read_text() == "file3" - # symlink should be followed - assert Path(_root / "dir1" / "file1_link").read_text() == "file1" - assert Path(_root / "dir1" / "dir2_link" / "file3").read_text() == "file3" - assert Path(_root / "dir1" / "file10_link").read_text() == "file1" - assert Path(_root / "dir1" / "dir20_link" / "file3").read_text() == "file3" - assert not Path(_root / "dir1" / "file1_link").is_symlink() - assert not Path(_root / "dir1" / "dir2_link" / "file3").is_symlink() - assert not Path(_root / "dir1" / "file10_link").is_symlink() - assert not Path(_root / "dir1" / "dir20_link" / "file3").is_symlink() + + +@pytest.mark.usefixtures("aiida_profile_clean") +def test_copyfile(firecrest_computer: orm.Computer, tmpdir: Path): + + transport = firecrest_computer.get_transport() + testing = transport.copyfile + + + _remote_1 = tmpdir / "remotedir_1" + _remote_2 = tmpdir / "remotedir_2" + _remote_1.mkdir() + _remote_2.mkdir() + + # raise if source or destination does not exist + with pytest.raises(FileNotFoundError): + testing(_remote_1 / "does_not_exist", _remote_2) + # in this case don't raise and just create the file + Path(tmpdir / "_").touch() + testing(tmpdir / "_", _remote_2 / "does_not_exist") + + # raise if source is unappropriate + with pytest.raises(ValueError): + testing(tmpdir, _remote_2) + + # a typical tree + Path(_remote_1 / "file1").write_text("file1") + Path(_remote_1 / ".hidden").write_text(".hidden") + # with symlinks to a file even if pointing to a relative path + os.symlink(_remote_1 / "file1", _remote_1 / "file1_link") + os.symlink(Path("file1"), _remote_1 / "file10_link") + + # write where I tell you to + testing(_remote_1 /"file1", _remote_2 / "file1") + assert Path(_remote_2 / "file1").read_text() == "file1" + + # always overwrite + Path(_remote_2 / "file1").write_text("notfile1") + testing(_remote_1 / "file1", _remote_2 / "file1") + assert Path(_remote_2 / "file1").read_text() == "file1" + + # don't skip hidden files + testing(_remote_1 / ".hidden", _remote_2 / ".hidden-prime") + assert Path(_remote_2 / ".hidden-prime").read_text() == ".hidden" + + # preserve links and don't follow them + testing(_remote_1 / "file1_link", _remote_2 / "file1_link") + assert Path(_remote_2 / "file1_link").read_text() == "file1" + assert Path(_remote_2 / "file1_link").is_symlink() + testing(_remote_1 / "file10_link", _remote_2 / "file10_link") + assert Path(_remote_2 / "file10_link").read_text() == "file1" + assert Path(_remote_2 / "file10_link").is_symlink() From c9ab8a6b48e3296e7e0ff202b7104fcb8589a256 Mon Sep 17 00:00:00 2001 From: Ali Date: Mon, 24 Jun 2024 13:53:32 +0200 Subject: [PATCH 07/39] added test_scheduler --- aiida_firecrest/scheduler.py | 87 ++++++++--- aiida_firecrest/transport.py | 8 -- tests/tests_mocking_pyfirecrest/conftest.py | 57 ++++++-- .../test_scheduler.py | 135 ++++++++++++++++++ 4 files changed, 247 insertions(+), 40 deletions(-) create mode 100644 tests/tests_mocking_pyfirecrest/test_scheduler.py diff --git a/aiida_firecrest/scheduler.py b/aiida_firecrest/scheduler.py index fe1870f..7184396 100644 --- a/aiida_firecrest/scheduler.py +++ b/aiida_firecrest/scheduler.py @@ -10,6 +10,8 @@ from aiida.schedulers.datastructures import JobInfo, JobState, JobTemplate from aiida.schedulers.plugins.slurm import SlurmJobResource from firecrest.FirecrestException import FirecrestException +from aiida.schedulers.plugins.slurm import _TIME_REGEXP +import datetime, time from .utils import convert_header_exceptions @@ -197,8 +199,7 @@ def submit_job(self, working_directory: str, filename: str) -> str | ExitCode: try: result = transport._client.submit( transport._machine, - transport._get_path(working_directory, filename), - local_file=False, + script_remote_path = transport._get_path(working_directory, filename), ) except FirecrestException as exc: raise SchedulerError(str(exc)) from exc @@ -214,11 +215,11 @@ def get_jobs( transport = self.transport with convert_header_exceptions({"machine": transport._machine}): # TODO handle pagination (pageSize, pageNumber) if many jobs - # This will do pagination, not manually tested becasue the server is damn slow. + # This will do pagination try: for page_iter in itertools.count(): results += transport._client.poll_active(transport._machine, jobs, page_number=page_iter) - if len(results) < self._DEFAULT_PAGE_SIZE: + if len(results) < self._DEFAULT_PAGE_SIZE*(page_iter+1): break except FirecrestException as exc: raise SchedulerError(str(exc)) from exc @@ -301,31 +302,31 @@ def get_jobs( this_job.queue_name = raw_result["partition"] - # TODO: The block below is commented, because the time limit is not returned explicitly by the FirecREST server - # in any case, the time tags doesn't seem to be used by AiiDA anyway. - # try: - # walltime = (self._convert_time(raw_result['time_limit'])) - # this_job.requested_wallclock_time_seconds = walltime # pylint: disable=invalid-name - # except ValueError: - # self.logger.warning(f'Error parsing the time limit for job id {this_job.job_id}') + try: + walltime = (self._convert_time(raw_result['time_left']) + self._convert_time(raw_result['start_time']) ) + this_job.requested_wallclock_time_seconds = walltime # pylint: disable=invalid-name + except ValueError: + self.logger.warning(f'Error parsing the time limit for job id {this_job.job_id}') # Only if it is RUNNING; otherwise it is not meaningful, # and may be not set (in my test, it is set to zero) - # if this_job.job_state == JobState.RUNNING: - # try: - # this_job.wallclock_time_seconds = (self._convert_time(thisjob_dict['time_used'])) - # except ValueError: - # self.logger.warning(f'Error parsing time_used for job id {this_job.job_id}') + if this_job.job_state == JobState.RUNNING: + try: + this_job.wallclock_time_seconds = self._convert_time(raw_result['start_time']) + except ValueError: + self.logger.warning(f'Error parsing time_used for job id {this_job.job_id}') + # TODO: The block below is commented, because dispatch_time is not returned explicitly by the FirecREST server + # in any case, the time tags doesn't seem to be used by AiiDA anyway. # try: # this_job.dispatch_time = self._parse_time_string(thisjob_dict['dispatch_time']) # except ValueError: # self.logger.warning(f'Error parsing dispatch_time for job id {this_job.job_id}') - # try: - # this_job.submission_time = self._parse_time_string(thisjob_dict['submission_time']) - # except ValueError: - # self.logger.warning(f'Error parsing submission_time for job id {this_job.job_id}') + try: + this_job.submission_time = self._parse_time_string(raw_result['time']) + except ValueError: + self.logger.warning(f'Error parsing submission_time for job id {this_job.job_id}') this_job.title = raw_result["name"] @@ -364,6 +365,52 @@ def kill_job(self, jobid: str) -> bool: + + def _convert_time(self, string): + """ + Note: this function was copied from the Slurm scheduler plugin + Convert a string in the format DD-HH:MM:SS to a number of seconds. + """ + if string == 'UNLIMITED': + return 2147483647 # == 2**31 - 1, largest 32-bit signed integer (68 years) + + if string == 'NOT_SET': + return None + + groups = _TIME_REGEXP.match(string) + if groups is None: + raise ValueError('Unrecognized format for time string.') + + groupdict = groups.groupdict() + # should not raise a ValueError, they all match digits only + days = int(groupdict['days'] if groupdict['days'] is not None else 0) + hours = int(groupdict['hours'] if groupdict['hours'] is not None else 0) + mins = int(groupdict['minutes'] if groupdict['minutes'] is not None else 0) + secs = int(groupdict['seconds'] if groupdict['seconds'] is not None else 0) + + return days * 86400 + hours * 3600 + mins * 60 + secs + + + def _parse_time_string(self, string, fmt='%Y-%m-%dT%H:%M:%S'): + """ + Note: this function was copied from the Slurm scheduler plugin + Parse a time string in the format returned from qstat -f and + returns a datetime object. + """ + + try: + time_struct = time.strptime(string, fmt) + except Exception as exc: + self.logger.debug(f'Unable to parse time string {string}, the message was {exc}') + raise ValueError('Problem parsing the time string.') + + # I convert from a time_struct to a datetime object going through + # the seconds since epoch, as suggested on stackoverflow: + # http://stackoverflow.com/questions/1697815 + return datetime.datetime.fromtimestamp(time.mktime(time_struct)) + + + # see https://slurm.schedmd.com/squeue.html#lbAG # note firecrest returns full names, not abbreviations _MAP_STATUS_SLURM = { diff --git a/aiida_firecrest/transport.py b/aiida_firecrest/transport.py index f692e3f..ed9d949 100644 --- a/aiida_firecrest/transport.py +++ b/aiida_firecrest/transport.py @@ -5,18 +5,14 @@ import hashlib import os from pathlib import Path -import platform import posixpath -import shutil import tarfile import time from typing import Any, Callable, ClassVar, TypedDict -from urllib import request import uuid from aiida.cmdline.params.options.overridable import OverridableOption from aiida.transports import Transport -from aiida.transports.transport import validate_positive_number from aiida.transports.util import FileAttribute from click.types import ParamType from firecrest import ClientCredentialsAuth, Firecrest # type: ignore[attr-defined] @@ -274,8 +270,6 @@ def __init__( """ - print("this is being done with firecrest transport") - # there is no overhead for "opening" a connection to a REST-API, # but still allow the user to set a safe interval if they really want to kwargs.setdefault("safe_interval", 0) @@ -818,8 +812,6 @@ def _puttreetar( # this function will be used to send a folder as a tar file to the server and extract it on the server - import tarfile - import uuid _ = uuid.uuid4() localpath = Path(localpath) diff --git a/tests/tests_mocking_pyfirecrest/conftest.py b/tests/tests_mocking_pyfirecrest/conftest.py index 95bc3c0..8655321 100644 --- a/tests/tests_mocking_pyfirecrest/conftest.py +++ b/tests/tests_mocking_pyfirecrest/conftest.py @@ -1,17 +1,18 @@ -from __future__ import annotations - from pathlib import Path -import os -import stat -import hashlib - -from _pytest.terminal import TerminalReporter +import os, stat +import random, hashlib import firecrest.path -import pytest import firecrest + +import pytest + from aiida import orm +class values: + _DEFAULT_PAGE_SIZE: int = 25 + + @pytest.fixture(name="firecrest_computer") def _firecrest_computer(myfirecrest, tmpdir: Path): """Create and return a computer configured for Firecrest. @@ -57,6 +58,7 @@ def __init__(self, firecrest_url, *args, **kwargs): self._firecrest_url = firecrest_url self.args = args self.kwargs = kwargs + self.whoami = whomai self.list_files = list_files self.stat = stat_ @@ -71,7 +73,7 @@ def __init__(self, firecrest_url, *args, **kwargs): self.extract = extract self.copy = copy self.submit = submit - # self.poll_active = poll_active + self.poll_active = poll_active class MockClientCredentialsAuth: @@ -88,9 +90,40 @@ def myfirecrest( monkeypatch.setattr(firecrest, "Firecrest", MockFirecrest) monkeypatch.setattr(firecrest, "ClientCredentialsAuth", MockClientCredentialsAuth) -# def poll_active(machine: str, job_id: str): -def submit(machine: str, script_str: str = None, script_remote_path: str = None, script_local_path: str = None): - pass +def submit(machine: str, script_str: str = None, script_remote_path: str = None, script_local_path: str = None, local_file=False): + if script_remote_path and not Path(script_remote_path).exists(): + raise FileNotFoundError(f"File {script_remote_path} does not exist") + job_id = random.randint(10000, 99999) + return {"jobid": job_id} + +def poll_active(machine: str, jobs: list[str], page_number: int = 0): + response = [] + # 12 satets are defined in firecrest + states = ["TIMEOUT", "SUSPENDED", "PREEMPTED", "CANCELLED", "NODE_FAIL", + "PENDING", "FAILED", "RUNNING", "CONFIGURING", "QUEUED", "COMPLETED", "COMPLETING"] + for i in range(len(jobs)): + response.append( + { + 'job_data_err': '', + 'job_data_out': '', + 'job_file': 'somefile.sh', + 'job_file_err': 'somefile-stderr.txt', + 'job_file_out': 'somefile-stdout.txt', + 'job_info_extra': 'Job info returned successfully', + 'jobid': f'{jobs[i]}', + 'name': 'aiida-45', + 'nodelist': 'nid00049', + 'nodes': '1', + 'partition': 'normal', + 'start_time': '0:03', + 'state': states[i%12], + 'time': '2024-06-21T10:44:42', + 'time_left': '29:57', + 'user': 'Prof. Wang' + } + ) + + return response[page_number*values._DEFAULT_PAGE_SIZE:(page_number+1)*values._DEFAULT_PAGE_SIZE] def whomai(machine: str): assert machine == "MACHINE_NAME" diff --git a/tests/tests_mocking_pyfirecrest/test_scheduler.py b/tests/tests_mocking_pyfirecrest/test_scheduler.py new file mode 100644 index 0000000..576875f --- /dev/null +++ b/tests/tests_mocking_pyfirecrest/test_scheduler.py @@ -0,0 +1,135 @@ +from pathlib import Path +import random +import pytest +from conftest import values + +from aiida import orm +from aiida_firecrest.scheduler import FirecrestScheduler +from aiida.schedulers.datastructures import CodeRunMode, JobTemplate + + +@pytest.mark.usefixtures("aiida_profile_clean") +def test_submit_job(firecrest_computer: orm.Computer, tmp_path: Path): + transport = firecrest_computer.get_transport() + scheduler = FirecrestScheduler() + scheduler.set_transport(transport) + + + with pytest.raises(Exception): + scheduler.submit_job(transport.getcwd(), "unknown.sh") + + _script = Path(tmp_path / "job.sh") + _script.write_text("#!/bin/bash\n\necho 'hello world'") + + job_id = scheduler.submit_job(transport.getcwd(), _script) + # this is how aiida expects the job_id to be returned + assert isinstance(job_id, str) + +@pytest.mark.usefixtures("aiida_profile_clean") +def test_get_jobs(firecrest_computer: orm.Computer): + transport = firecrest_computer.get_transport() + scheduler = FirecrestScheduler() + scheduler.set_transport(transport) + + # test pagaination + scheduler._DEFAULT_PAGE_SIZE = 2 + values._DEFAULT_PAGE_SIZE = 2 + + joblist = [random.randint(10000, 99999) for i in range(5)] + result = scheduler.get_jobs(joblist) + assert len(result) == 5 + for i in range(5): + assert result[i].job_id == str(joblist[i]) + # TODO: one could check states as well + + + +def test_write_script_full(): + # to avoid false positive (overwriting on existing file), + # we check the output of the script instead of using `file_regression`` + expectaion = """ + #!/bin/bash + #SBATCH -H + #SBATCH --requeue + #SBATCH --mail-user=True + #SBATCH --mail-type=BEGIN + #SBATCH --mail-type=FAIL + #SBATCH --mail-type=END + #SBATCH --job-name="test_job" + #SBATCH --get-user-env + #SBATCH --output=test.out + #SBATCH --error=test.err + #SBATCH --partition=test_queue + #SBATCH --account=test_account + #SBATCH --qos=test_qos + #SBATCH --nice=100 + #SBATCH --nodes=1 + #SBATCH --ntasks-per-node=1 + #SBATCH --time=01:00:00 + #SBATCH --mem=1 + test_command + """ + expectaion_flat = '\n'.join(line.strip() for line in expectaion.splitlines()).strip('\n') + scheduler = FirecrestScheduler() + template = JobTemplate( + { + "job_resource": scheduler.create_job_resource( + num_machines=1, num_mpiprocs_per_machine=1 + ), + "codes_info": [], + "codes_run_mode": CodeRunMode.SERIAL, + "submit_as_hold": True, + "rerunnable": True, + "email": True, + "email_on_started": True, + "email_on_terminated": True, + "job_name": "test_job", + "import_sys_environment": True, + "sched_output_path": "test.out", + "sched_error_path": "test.err", + "queue_name": "test_queue", + "account": "test_account", + "qos": "test_qos", + "priority": 100, + "max_wallclock_seconds": 3600, + "max_memory_kb": 1024, + "custom_scheduler_commands": "test_command", + } + ) + try: + assert scheduler.get_submit_script(template).rstrip() == expectaion_flat + except AssertionError: + print(scheduler.get_submit_script(template).rstrip()) + print(expectaion) + raise + + +def test_write_script_minimal(): + # to avoid false positive (overwriting on existing file), + # we check the output of the script instead of using `file_regression`` + expectaion = """ + #!/bin/bash + #SBATCH --no-requeue + #SBATCH --error=slurm-%j.err + #SBATCH --nodes=1 + #SBATCH --ntasks-per-node=1 + """ + + expectaion_flat = '\n'.join(line.strip() for line in expectaion.splitlines()).strip('\n') + scheduler = FirecrestScheduler() + template = JobTemplate( + { + "job_resource": scheduler.create_job_resource( + num_machines=1, num_mpiprocs_per_machine=1 + ), + "codes_info": [], + "codes_run_mode": CodeRunMode.SERIAL, + } + ) + + try: + assert scheduler.get_submit_script(template).rstrip() == expectaion_flat + except AssertionError: + print(scheduler.get_submit_script(template).rstrip()) + print(expectaion) + raise From f452999351f75b95bede977b23dc1d900a0b5f25 Mon Sep 17 00:00:00 2001 From: Ali Date: Mon, 24 Jun 2024 17:26:43 +0200 Subject: [PATCH 08/39] Applied ruff and mypy --- .pre-commit-config.yaml | 2 +- README.md | 2 +- aiida_firecrest/scheduler.py | 96 ++- aiida_firecrest/transport.py | 679 ++++++++++-------- aiida_firecrest/utils_test.py | 13 +- pyproject.toml | 2 +- tests/test_computer.py | 3 +- tests/tests_mocking_pyfirecrest/conftest.py | 283 +++++--- .../test_computer.py | 103 +-- .../test_scheduler.py | 34 +- .../test_transport.py | 186 +++-- 11 files changed, 811 insertions(+), 592 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index ce5828c..0601441 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -44,5 +44,5 @@ repos: additional_dependencies: - "types-PyYAML" - "types-requests" - - "pyfirecrest~=1.4.0" + - "pyfirecrest~=2.5.0" - "aiida-core~=2.4.0" diff --git a/README.md b/README.md index f6d8ee7..f823a19 100644 --- a/README.md +++ b/README.md @@ -211,4 +211,4 @@ These set of test do not gurantee that the firecrest protocol is working, but it If these tests, pass and still you have trouble in real deploymeny that means your installed version of pyfirecrest is behaving differently from what `aiida-firecrest` expects in `MockFirecrest` in `tests/tests_mocking_pyfirecrest/conftest.py`. -In order to solve that, first spot what is different and then solve or raise to `pyfirecrest` if relevant. +In order to solve that, first spot what is different and then solve or raise to `pyfirecrest` if relevant. diff --git a/aiida_firecrest/scheduler.py b/aiida_firecrest/scheduler.py index 7184396..59c9310 100644 --- a/aiida_firecrest/scheduler.py +++ b/aiida_firecrest/scheduler.py @@ -1,17 +1,18 @@ """Scheduler interface.""" from __future__ import annotations +import datetime +import itertools import re import string +import time from typing import TYPE_CHECKING, Any, ClassVar -import itertools + from aiida.engine.processes.exit_code import ExitCode from aiida.schedulers import Scheduler, SchedulerError from aiida.schedulers.datastructures import JobInfo, JobState, JobTemplate -from aiida.schedulers.plugins.slurm import SlurmJobResource +from aiida.schedulers.plugins.slurm import _TIME_REGEXP, SlurmJobResource from firecrest.FirecrestException import FirecrestException -from aiida.schedulers.plugins.slurm import _TIME_REGEXP -import datetime, time from .utils import convert_header_exceptions @@ -30,7 +31,6 @@ class FirecrestScheduler(Scheduler): _logger = Scheduler._logger.getChild("firecrest") _DEFAULT_PAGE_SIZE = 25 - @classmethod def get_description(cls) -> str: """Used by verdi to describe the plugin.""" @@ -199,7 +199,7 @@ def submit_job(self, working_directory: str, filename: str) -> str | ExitCode: try: result = transport._client.submit( transport._machine, - script_remote_path = transport._get_path(working_directory, filename), + script_remote_path=transport._get_path(working_directory, filename), ) except FirecrestException as exc: raise SchedulerError(str(exc)) from exc @@ -218,20 +218,24 @@ def get_jobs( # This will do pagination try: for page_iter in itertools.count(): - results += transport._client.poll_active(transport._machine, jobs, page_number=page_iter) - if len(results) < self._DEFAULT_PAGE_SIZE*(page_iter+1): + results += transport._client.poll_active( + transport._machine, jobs, page_number=page_iter + ) + if len(results) < self._DEFAULT_PAGE_SIZE * (page_iter + 1): break except FirecrestException as exc: raise SchedulerError(str(exc)) from exc job_list = [] for raw_result in results: - # TODO: probably the if below is not needed, because recently, the server should return only the jobs of the current user + # TODO: probably the if below is not needed, because recently + # the server should return only the jobs of the current user if user is not None and raw_result["user"] != user: continue this_job = JobInfo() # type: ignore this_job.job_id = raw_result["jobid"] - # TODO: firecrest does not return the annotation, so set to an empty string. To be investigated how important that is. + # TODO: firecrest does not return the annotation, so set to an empty string. + # To be investigated how important that is. this_job.annotation = "" job_state_raw = raw_result["state"] @@ -282,13 +286,15 @@ def get_jobs( ) ) - # TODO: The block below is commented, because the number of allocated cores is not returned by the FirecREST server + # TODO: The block below is commented, because the number of + # allocated cores is not returned by the FirecREST server # try: # this_job.num_mpiprocs = int(thisjob_dict['number_cpus']) # except ValueError: # self.logger.warning( # 'The number of allocated cores is not ' - # 'an integer ({}) for job id {}!'.format(thisjob_dict['number_cpus'], this_job.job_id) + # 'an integer ({}) for job id {}!'.format( + # thisjob_dict['number_cpus'], this_job.job_id) # ) # ALLOCATED NODES HERE @@ -303,20 +309,37 @@ def get_jobs( this_job.queue_name = raw_result["partition"] try: - walltime = (self._convert_time(raw_result['time_left']) + self._convert_time(raw_result['start_time']) ) - this_job.requested_wallclock_time_seconds = walltime # pylint: disable=invalid-name + time_left = self._convert_time(raw_result["time_left"]) + start_time = self._convert_time(raw_result["start_time"]) + + if time_left is None or start_time is None: + this_job.requested_wallclock_time_seconds = 0 + else: + this_job.requested_wallclock_time_seconds = time_left + start_time + except ValueError: - self.logger.warning(f'Error parsing the time limit for job id {this_job.job_id}') + self.logger.warning( + f"Error parsing the time limit for job id {this_job.job_id}" + ) # Only if it is RUNNING; otherwise it is not meaningful, # and may be not set (in my test, it is set to zero) if this_job.job_state == JobState.RUNNING: try: - this_job.wallclock_time_seconds = self._convert_time(raw_result['start_time']) + wallclock_time_seconds = self._convert_time( + raw_result["start_time"] + ) + if wallclock_time_seconds is not None: + this_job.wallclock_time_seconds = wallclock_time_seconds + else: + this_job.wallclock_time_seconds = 0 except ValueError: - self.logger.warning(f'Error parsing time_used for job id {this_job.job_id}') + self.logger.warning( + f"Error parsing time_used for job id {this_job.job_id}" + ) - # TODO: The block below is commented, because dispatch_time is not returned explicitly by the FirecREST server + # TODO: The block below is commented, because dispatch_time + # is not returned explicitly by the FirecREST server # in any case, the time tags doesn't seem to be used by AiiDA anyway. # try: # this_job.dispatch_time = self._parse_time_string(thisjob_dict['dispatch_time']) @@ -324,9 +347,11 @@ def get_jobs( # self.logger.warning(f'Error parsing dispatch_time for job id {this_job.job_id}') try: - this_job.submission_time = self._parse_time_string(raw_result['time']) + this_job.submission_time = self._parse_time_string(raw_result["time"]) except ValueError: - self.logger.warning(f'Error parsing submission_time for job id {this_job.job_id}') + self.logger.warning( + f"Error parsing submission_time for job id {this_job.job_id}" + ) this_job.title = raw_result["name"] @@ -363,35 +388,33 @@ def kill_job(self, jobid: str) -> bool: transport._client.cancel(transport._machine, jobid) return True - - - - def _convert_time(self, string): + def _convert_time(self, string: str) -> int | None: """ Note: this function was copied from the Slurm scheduler plugin Convert a string in the format DD-HH:MM:SS to a number of seconds. """ - if string == 'UNLIMITED': + if string == "UNLIMITED": return 2147483647 # == 2**31 - 1, largest 32-bit signed integer (68 years) - if string == 'NOT_SET': + if string == "NOT_SET" or string == "N/A": return None groups = _TIME_REGEXP.match(string) if groups is None: - raise ValueError('Unrecognized format for time string.') + raise ValueError("Unrecognized format for time string.") groupdict = groups.groupdict() # should not raise a ValueError, they all match digits only - days = int(groupdict['days'] if groupdict['days'] is not None else 0) - hours = int(groupdict['hours'] if groupdict['hours'] is not None else 0) - mins = int(groupdict['minutes'] if groupdict['minutes'] is not None else 0) - secs = int(groupdict['seconds'] if groupdict['seconds'] is not None else 0) + days = int(groupdict["days"] if groupdict["days"] is not None else 0) + hours = int(groupdict["hours"] if groupdict["hours"] is not None else 0) + mins = int(groupdict["minutes"] if groupdict["minutes"] is not None else 0) + secs = int(groupdict["seconds"] if groupdict["seconds"] is not None else 0) return days * 86400 + hours * 3600 + mins * 60 + secs - - def _parse_time_string(self, string, fmt='%Y-%m-%dT%H:%M:%S'): + def _parse_time_string( + self, string: str, fmt: str = "%Y-%m-%dT%H:%M:%S" + ) -> datetime.datetime: """ Note: this function was copied from the Slurm scheduler plugin Parse a time string in the format returned from qstat -f and @@ -401,8 +424,10 @@ def _parse_time_string(self, string, fmt='%Y-%m-%dT%H:%M:%S'): try: time_struct = time.strptime(string, fmt) except Exception as exc: - self.logger.debug(f'Unable to parse time string {string}, the message was {exc}') - raise ValueError('Problem parsing the time string.') + self.logger.debug( + f"Unable to parse time string {string}, the message was {exc}" + ) + raise ValueError("Problem parsing the time string.") from exc # I convert from a time_struct to a datetime object going through # the seconds since epoch, as suggested on stackoverflow: @@ -410,7 +435,6 @@ def _parse_time_string(self, string, fmt='%Y-%m-%dT%H:%M:%S'): return datetime.datetime.fromtimestamp(time.mktime(time_struct)) - # see https://slurm.schedmd.com/squeue.html#lbAG # note firecrest returns full names, not abbreviations _MAP_STATUS_SLURM = { diff --git a/aiida_firecrest/transport.py b/aiida_firecrest/transport.py index ed9d949..620d8d7 100644 --- a/aiida_firecrest/transport.py +++ b/aiida_firecrest/transport.py @@ -1,6 +1,7 @@ """Transport interface.""" from __future__ import annotations +from contextlib import suppress import fnmatch import hashlib import os @@ -11,14 +12,16 @@ from typing import Any, Callable, ClassVar, TypedDict import uuid +from aiida.cmdline.params.options.interactive import InteractiveOption from aiida.cmdline.params.options.overridable import OverridableOption from aiida.transports import Transport from aiida.transports.util import FileAttribute +from click.core import Context from click.types import ParamType from firecrest import ClientCredentialsAuth, Firecrest # type: ignore[attr-defined] - from firecrest.path import FcPath + class ValidAuthOption(TypedDict, total=False): option: OverridableOption | None # existing option switch: bool # whether the option is a boolean flag @@ -29,146 +32,174 @@ class ValidAuthOption(TypedDict, total=False): help: str callback: Callable[..., Any] # for validation + class BuggyError(Exception): # TODO: Remove this class when the code is stable """Raised when something should absolutly not happen, but it does.""" -class FirecrestTransport(Transport): - """Transport interface for FirecREST.""" - # override these options, because they don't really make sense for a REST-API, - # so we don't want the user having to provide them - # - `use_login_shell` you can't run bash on a REST-API - # - `safe_interval` there is no connection overhead for a REST-API - # although ideally you would rate-limit the number of requests, - # but this would ideally require some "global" rate limiter, - # across all transport instances - # TODO upstream issue - _common_auth_options: ClassVar[list[Any]] = [] # type: ignore[misc] - _DEFAULT_SAFE_OPEN_INTERVAL = 0.0 +def _create_secret_file(ctx: Context, param: InteractiveOption, value: str) -> str: + """Create a secret file if the value is not a path to a secret file. + The path should be absolute, if it is not, the file will be created in ~/.firecrest. + """ + import uuid - def _create_secret_file(ctx, param, value) -> str: - """Create a secret file if the value is not a path to a secret file. - The path should be absolute, if it is not, the file will be created in ~/.firecrest. - """ - import uuid - from aiida.cmdline.utils import echo - from click import BadParameter - possible_path = Path(value) - if os.path.isabs(possible_path): - if not possible_path.exists(): - raise BadParameter(f'Secret file not found at {value}') - secret_path = possible_path - - else: - Path(f'~/.firecrest').expanduser().mkdir(parents=True, exist_ok=True) - _ = uuid.uuid4() - secret_path = Path(f'~/.firecrest/secret_{_}').expanduser() - while secret_path.exists(): - # instead of a random number one could use the label or pk of the computer being configured - secret_path = Path(f'~/.firecrest/secret_{_}').expanduser() - secret_path.write_text(value) - echo.echo_report(f"Secret file created at {secret_path}") - print(f"Client Secret stored at {secret_path}") - - return str(secret_path) - - - def _validate_temp_directory(ctx, param, value) -> str: - """Validate the temp directory on the server. - If it does not exist, create it. - If it is not empty, get a confimation from the user to empty it. - """ + from aiida.cmdline.utils import echo + from click import BadParameter + + possible_path = Path(value) + if os.path.isabs(possible_path): + if not possible_path.exists(): + raise BadParameter(f"Secret file not found at {value}") + secret_path = possible_path + + else: + Path("~/.firecrest").expanduser().mkdir(parents=True, exist_ok=True) + _ = uuid.uuid4() + secret_path = Path(f"~/.firecrest/secret_{_}").expanduser() + while secret_path.exists(): + # instead of a random number one could use the label or pk of the computer being configured + secret_path = Path(f"~/.firecrest/secret_{_}").expanduser() + secret_path.write_text(value) + echo.echo_report(f"Secret file created at {secret_path}") + echo.echo_report(f"Client Secret stored at {secret_path}") + + return str(secret_path) + + +def _validate_temp_directory(ctx: Context, param: InteractiveOption, value: str) -> str: + """Validate the temp directory on the server. + If it does not exist, create it. + If it is not empty, get a confimation from the user to empty it. + """ + + import click - import click - firecrest_url= ctx.params['url'] - token_uri= ctx.params['token_uri'] - client_id= ctx.params['client_id'] - client_machine= ctx.params['client_machine'] - secret= ctx.params['client_secret']#)#.read_text() - small_file_size_mb= ctx.params['small_file_size_mb'] + firecrest_url = ctx.params["url"] + token_uri = ctx.params["token_uri"] + client_id = ctx.params["client_id"] + client_machine = ctx.params["client_machine"] + secret = ctx.params["client_secret"] # )#.read_text() + small_file_size_mb = ctx.params["small_file_size_mb"] - dummy = FirecrestTransport( + dummy = FirecrestTransport( url=firecrest_url, token_uri=token_uri, - client_id = client_id, - client_secret= secret, - client_machine= client_machine, - temp_directory= value, - small_file_size_mb = small_file_size_mb) - - # Temp directory rutine - if dummy._cwd.joinpath(dummy._temp_directory).is_file(): #self._temp_directory.is_file(): - raise click.BadParameter("Temp directory cannot be a file") - - if dummy.path_exists(dummy._temp_directory): - if dummy.listdir(dummy._temp_directory): - # if not configured: - confirm = click.confirm(f"Temp directory {dummy._temp_directory} is not empty. Do you want to flush it?") - if confirm: - for item in dummy.listdir(dummy._temp_directory): - # TODO: maybe do recursive delete - dummy.remove(dummy._temp_directory.joinpath(item)) - else: - click.echo(f"Please provide an empty temp directory on the server.") - raise click.BadParameter(f"Temp directory {dummy._temp_directory} is not empty") - # The block below could be moved to a maintanace delete function, if needed - # else: - # # There might still be some residual files in case of previous interupted connection - # for item in dummy.listdir(dummy._temp_directory): - # # this could be replace with a proper glob later - # if item[:4] == 'temp': - # dummy.remove(dummy._temp_directory.joinpath(item)) - - else: - try: - dummy.mkdir(dummy._temp_directory, ignore_existing=True) - except Exception as e: - raise OSError(f"Could not create temp directory {dummy._temp_directory} on server: {e}") - - return value + client_id=client_id, + client_secret=secret, + client_machine=client_machine, + temp_directory=value, + small_file_size_mb=small_file_size_mb, + ) + + # Temp directory rutine + if dummy._cwd.joinpath( + dummy._temp_directory + ).is_file(): # self._temp_directory.is_file(): + raise click.BadParameter("Temp directory cannot be a file") + + if dummy.path_exists(dummy._temp_directory): + if dummy.listdir(dummy._temp_directory): + # if not configured: + confirm = click.confirm( + f"Temp directory {dummy._temp_directory} is not empty. Do you want to flush it?" + ) + if confirm: + for item in dummy.listdir(dummy._temp_directory): + # TODO: maybe do recursive delete + dummy.remove(dummy._temp_directory.joinpath(item)) + else: + click.echo("Please provide an empty temp directory on the server.") + raise click.BadParameter( + f"Temp directory {dummy._temp_directory} is not empty" + ) + # The block below could be moved to a maintanace delete function, if needed + # else: + # # There might still be some residual files in case of previous interupted connection + # for item in dummy.listdir(dummy._temp_directory): + # # this could be replace with a proper glob later + # if item[:4] == 'temp': + # dummy.remove(dummy._temp_directory.joinpath(item)) + + else: + try: + dummy.mkdir(dummy._temp_directory, ignore_existing=True) + except Exception as e: + raise OSError( + f"Could not create temp directory {dummy._temp_directory} on server: {e}" + ) from e + return value - def _dynamic_info_direct_size(ctx, param, value) -> str: - """Get dynamic information from the server, if the user enters 0 for the small_file_size_mb. - This is done by connecting to the server and getting the value of UTILITIES_MAX_FILE_SIZE. - Below this size, file bytes will be sent in a single API call. Above this size, - the file will be downloaded(uploaded) from(to) the object store and downloaded in chunks. - :param ctx: the `click.Context` - :param param: the parameter - :param value: the value passed for the parameter +def _dynamic_info_direct_size( + ctx: Context, param: InteractiveOption, value: float +) -> float: + """Get dynamic information from the server, if the user enters 0 for the small_file_size_mb. + This is done by connecting to the server and getting the value of UTILITIES_MAX_FILE_SIZE. + Below this size, file bytes will be sent in a single API call. Above this size, + the file will be downloaded(uploaded) from(to) the object store and downloaded in chunks. - :return: the value of small_file_size_mb. + :param ctx: the `click.Context` + :param param: the parameter + :param value: the value passed for the parameter - """ + :return: the value of small_file_size_mb. - if value > 0: - return value + """ + from aiida.cmdline.utils import echo - firecrest_url= ctx.params['url'] - token_uri= ctx.params['token_uri'] - client_id= ctx.params['client_id'] - client_machine= ctx.params['client_machine'] - secret= ctx.params['client_secret']#)#.read_text() + if value > 0: + return value + + firecrest_url = ctx.params["url"] + token_uri = ctx.params["token_uri"] + client_id = ctx.params["client_id"] + client_machine = ctx.params["client_machine"] + secret = ctx.params["client_secret"] # )#.read_text() - dummy = FirecrestTransport( + dummy = FirecrestTransport( url=firecrest_url, token_uri=token_uri, - client_id = client_id, - client_secret= secret, - client_machine= client_machine, - temp_directory= '', - small_file_size_mb = 0.0) - - prameters= dummy._client.parameters() - utilities_max_file_size = next((item for item in prameters['utilities'] if item['name'] == 'UTILITIES_MAX_FILE_SIZE'), None) - if utilities_max_file_size is not None: - small_file_size_mb = float(utilities_max_file_size['value']) - else: - small_file_size_mb = 5.0 # default value - return small_file_size_mb + client_id=client_id, + client_secret=secret, + client_machine=client_machine, + temp_directory="", + small_file_size_mb=0.0, + ) + + prameters = dummy._client.parameters() + utilities_max_file_size = next( + ( + item + for item in prameters["utilities"] + if item["name"] == "UTILITIES_MAX_FILE_SIZE" + ), + None, + ) + small_file_size_mb = ( + float(utilities_max_file_size["value"]) + if utilities_max_file_size is not None + else 5.0 + ) + echo.echo_report(f"Maximum file size for direct transfer: {small_file_size_mb} MB") + return small_file_size_mb + + +class FirecrestTransport(Transport): + """Transport interface for FirecREST.""" + + # override these options, because they don't really make sense for a REST-API, + # so we don't want the user having to provide them + # - `use_login_shell` you can't run bash on a REST-API + # - `safe_interval` there is no connection overhead for a REST-API + # although ideally you would rate-limit the number of requests, + # but this would ideally require some "global" rate limiter, + # across all transport instances + # TODO upstream issue + _common_auth_options: ClassVar[list[Any]] = [] # type: ignore[misc] + _DEFAULT_SAFE_OPEN_INTERVAL = 0.0 _valid_auth_options: ClassVar[list[tuple[str, ValidAuthOption]]] = [ # type: ignore[misc] ( @@ -246,7 +277,7 @@ def __init__( url: str, token_uri: str, client_id: str, - client_secret: str, #| Path, + client_secret: str, # | Path, # unfortunately we cannot store client_secret as a Path, because it is not JSON serializable client_machine: str, small_file_size_mb: float, @@ -257,7 +288,7 @@ def __init__( # TODO ideally hostname would not be necessary on a computer **kwargs: Any, ): - """Construct a FirecREST transport object. + """Construct a FirecREST transport object. :param url: URL to the FirecREST server :param token_uri: URI for retrieving FirecREST authentication tokens @@ -268,7 +299,6 @@ def __init__( :param temp_directory: A temp directory on server for creating temporary files (compression, extraction, etc.) :param kwargs: Additional keyword arguments """ - # there is no overhead for "opening" a connection to a REST-API, # but still allow the user to set a safe interval if they really want to @@ -281,18 +311,18 @@ def __init__( assert isinstance(client_secret, str), "client_secret must be a string" assert isinstance(client_machine, str), "client_machine must be a string" assert isinstance(temp_directory, str), "temp_directory must be a string" - assert isinstance(small_file_size_mb, float), "small_file_size_mb must be a float" + assert isinstance( + small_file_size_mb, float + ), "small_file_size_mb must be a float" - self._machine = client_machine self._url = url self._token_uri = token_uri self._client_id = client_id - self._temp_directory = Path(temp_directory) self._small_file_size_bytes = int(small_file_size_mb * 1024 * 1024) - - self._payoff_override = None - + + self._payoff_override: bool | None = None + secret = Path(client_secret).read_text() try: self._client = Firecrest( @@ -300,21 +330,21 @@ def __init__( authorization=ClientCredentialsAuth(client_id, secret, token_uri), ) except Exception as e: - raise ValueError(f"Could not connect to FirecREST server: {e}") - + raise ValueError(f"Could not connect to FirecREST server: {e}") from e + self._cwd: FcPath = FcPath(self._client, self._machine, "/", cache_enabled=True) - - - def __str__(self): + self._temp_directory = self._cwd.joinpath(temp_directory) + + def __str__(self) -> str: """Return the name of the plugin.""" return self.__class__.__name__ @property - def payoff_override(self): + def payoff_override(self) -> bool | None: return self._payoff_override @payoff_override.setter - def payoff_override(self, value): + def payoff_override(self, value: bool) -> None: if not isinstance(value, bool): raise ValueError("payoff_override must be a boolean value") self._payoff_override = value @@ -331,7 +361,7 @@ def get_description(cls) -> str: ) def open(self) -> None: # noqa: A003 - """Open the transport. + """Open the transport. This is a no-op for the REST-API, as there is no connection to open. """ pass @@ -348,7 +378,7 @@ def getcwd(self) -> str: def _get_path(self, *path: str) -> str: """Return the path as a string.""" - return posixpath.normpath(self._cwd.joinpath(*path)) + return posixpath.normpath(self._cwd.joinpath(*path)) # type: ignore def chdir(self, path: str) -> None: """Change the current working directory.""" @@ -367,7 +397,7 @@ def chown(self, path: str, uid: str, gid: str) -> None: def path_exists(self, path: str) -> bool: """Check if a path exists on the remote.""" - return self._cwd.joinpath(path).exists() + return self._cwd.joinpath(path).exists() # type: ignore def get_attribute(self, path: str) -> FileAttribute: """Get the attributes of a file.""" @@ -385,15 +415,17 @@ def get_attribute(self, path: str) -> FileAttribute: def isdir(self, path: str) -> bool: """Check if a path is a directory.""" - return self._cwd.joinpath(path).is_dir() + return self._cwd.joinpath(path).is_dir() # type: ignore def isfile(self, path: str) -> bool: """Check if a path is a file.""" - return self._cwd.joinpath(path).is_file() + return self._cwd.joinpath(path).is_file() # type: ignore - def listdir(self, path: str = ".", pattern: str | None = None, recursive: bool = False) -> list[str]: + def listdir( + self, path: str = ".", pattern: str | None = None, recursive: bool = False + ) -> list[str]: """List the contents of a directory. - + :param pattern: Unix shell-style wildcards to match the pattern: - `*` matches everything - `?` matches any single character @@ -401,7 +433,10 @@ def listdir(self, path: str = ".", pattern: str | None = None, recursive: bool = - `[!seq]` matches any character not in seq :param recursive: If True, list directories recursively """ - names = [p.relpath(path).as_posix() for p in self._cwd.joinpath(path).iterdir(recursive=recursive)] + names = [ + p.relpath(path).as_posix() + for p in self._cwd.joinpath(path).iterdir(recursive=recursive) + ] if pattern is not None: names = fnmatch.filter(names, pattern) return names @@ -420,7 +455,7 @@ def mkdir(self, path: str, ignore_existing: bool = False) -> None: def normalize(self, path: str = ".") -> str: """Resolve the path.""" return posixpath.normpath(path) - + def write_binary(self, path: str, data: bytes) -> None: """Write bytes to a file on the remote.""" # Note this is not part of the Transport interface, but is useful for testing @@ -431,7 +466,7 @@ def read_binary(self, path: str) -> bytes: """Read bytes from a file on the remote.""" # Note this is not part of the Transport interface, but is useful for testing # TODO will fail for files exceeding small_file_size_mb - return self._cwd.joinpath(path).read_bytes() + return self._cwd.joinpath(path).read_bytes() # type: ignore def symlink(self, remotesource: str, remotedestination: str) -> None: """Create a symlink on the remote.""" @@ -446,8 +481,12 @@ def copyfile( :param dereference: If True, copy the target of the symlink instead of the symlink itself. """ - source = self._cwd.joinpath(remotesource)#.enable_cache() it's removed from from path.py to be investigated - destination = self._cwd.joinpath(remotedestination)#.enable_cache() it's removed from from path.py to be investigated + source = self._cwd.joinpath( + remotesource + ) # .enable_cache() it's removed from from path.py to be investigated + destination = self._cwd.joinpath( + remotedestination + ) # .enable_cache() it's removed from from path.py to be investigated if dereference: raise NotImplementedError("copyfile() does not support symlink dereference") if not source.exists(): @@ -461,22 +500,25 @@ def copyfile( # I removed symlink copy, becasue it's really not a file copy, it's a link copy # and aiida-ssh have it in buggy manner, prrobably it's not used anyways - def copytree( self, remotesource: str, remotedestination: str, dereference: bool = False ) -> None: """Copy a directory on the remote. FirecREST does not support symlink copying. - + :param dereference: If True, copy the target of the symlink instead of the symlink itself. """ - #TODO: check if deference is set to False, symlinks will be functional after the copy in Firecrest server. - - source = self._cwd.joinpath(remotesource)#.enable_cache().enable_cache() it's removed from from path.py to be investigated - destination = ( - self._cwd.joinpath(remotedestination)#.enable_cache().enable_cache() it's removed from from path.py to be investigated - ) + # TODO: check if deference is set to False, symlinks will be functional after the copy in Firecrest server. + + source = self._cwd.joinpath( + remotesource + ) # .enable_cache().enable_cache() it's removed from from path.py to be investigated + destination = self._cwd.joinpath( + remotedestination + ) # .enable_cache().enable_cache() it's removed from from path.py to be investigated if dereference: - raise NotImplementedError("Dereferencing not implemented in FirecREST server") + raise NotImplementedError( + "Dereferencing not implemented in FirecREST server" + ) if not source.exists(): raise FileNotFoundError(f"Source file does not exist: {source}") if not source.is_dir(): @@ -486,7 +528,6 @@ def copytree( source.copy_to(destination) - def copy( self, remotesource: str, @@ -497,9 +538,9 @@ def copy( """Copy a file or directory on the remote. FirecREST does not support symlink copying. :param recursive: If True, copy directories recursively. - note that the non-recursive option is not implemented in FirecREST server. + note that the non-recursive option is not implemented in FirecREST server. And it's not used in upstream, anyways... - + :param dereference: If True, copy the target of the symlink instead of the symlink itself. """ # TODO: investigate overwrite (?) @@ -508,9 +549,15 @@ def copy( # TODO this appears to not actually be used upstream, so just remove there raise NotImplementedError("Non-recursive copy not implemented") if dereference: - raise NotImplementedError("Dereferencing not implemented in FirecREST server") - source = self._cwd.joinpath(remotesource)#.enable_cache() it's removed from from path.py to be investigated - destination = self._cwd.joinpath(remotedestination)#.enable_cache() it's removed from from path.py to be investigated + raise NotImplementedError( + "Dereferencing not implemented in FirecREST server" + ) + source = self._cwd.joinpath( + remotesource + ) # .enable_cache() it's removed from from path.py to be investigated + destination = self._cwd.joinpath( + remotedestination + ) # .enable_cache() it's removed from from path.py to be investigated if not source.exists(): raise FileNotFoundError(f"Source does not exist: {source}") @@ -519,22 +566,28 @@ def copy( source.copy_to(destination) - # TODO do get/put methods need to handle glob patterns? # Apparently not, but I'm not clear how glob() iglob() are going to behave here. We may need to implement them. def getfile( - self, remotepath: str | FcPath, localpath: str | Path, dereference:bool = True, *args: Any, **kwargs: Any + self, + remotepath: str | FcPath, + localpath: str | Path, + dereference: bool = True, + *args: Any, + **kwargs: Any, ) -> None: """Get a file from the remote. :param dereference: If True, follow symlinks. note: we don't support downloading symlinks, so dereference should always be True - + """ if not dereference: - raise NotImplementedError("Getting symlinks with `dereference=False` is not supported") + raise NotImplementedError( + "Getting symlinks with `dereference=False` is not supported" + ) local = Path(localpath) if not local.is_absolute(): @@ -542,14 +595,16 @@ def getfile( remote = ( remotepath if isinstance(remotepath, FcPath) - else self._cwd.joinpath(remotepath)#.enable_cache() it's removed from from path.py to be investigated + else self._cwd.joinpath( + remotepath + ) # .enable_cache() it's removed from from path.py to be investigated ) if not remote.is_file(): raise FileNotFoundError(f"Source file does not exist: {remote}") remote_size = remote.lstat().st_size # if not local.exists(): # local.mkdir(parents=True) - with self._cwd.convert_header_exceptions({"machine": self._machine, "path": remote}): + with self._cwd.convert_header_exceptions(): if remote_size < self._small_file_size_bytes: self._client.simple_download(self._machine, str(remote), localpath) else: @@ -559,16 +614,17 @@ def getfile( # to concurrently initiate internal file transfers to the object store (a.k.a. "staging area") # and downloading from the object store to the local machine - # I investigated asyncio, but it's not performant for this use case. - # Becasue in the end, FirecREST server ends up serializing the requests. + # I investigated asyncio, but it's not performant for this use case. + # Becasue in the end, FirecREST server ends up serializing the requests. # see here: https://github.com/eth-cscs/pyfirecrest/issues/94 down_obj = self._client.external_download(self._machine, str(remote)) - down_obj.finish_download(local) + down_obj.finish_download(local) self._validate_checksum(local, remote) - - def _validate_checksum(self,localpath: str | Path, remotepath: str | FcPath) -> None: + def _validate_checksum( + self, localpath: str | Path, remotepath: str | FcPath + ) -> None: """Validate the checksum of a file. Useful for checking if a file was transferred correctly. it uses sha256 hash to compare the checksum of the local and remote files. @@ -582,14 +638,18 @@ def _validate_checksum(self,localpath: str | Path, remotepath: str | FcPath) -> remote = ( remotepath if isinstance(remotepath, FcPath) - else self._cwd.joinpath(remotepath)#.enable_cache() it's removed from from path.py to be investigated + else self._cwd.joinpath( + remotepath + ) # .enable_cache() it's removed from from path.py to be investigated ) if not remote.is_file(): - raise FileNotFoundError(f"Cannot calculate checksum for a directory: {remote}") - + raise FileNotFoundError( + f"Cannot calculate checksum for a directory: {remote}" + ) + sha256_hash = hashlib.sha256() - with open(local,"rb") as f: - for byte_block in iter(lambda: f.read(4096),b""): + with open(local, "rb") as f: + for byte_block in iter(lambda: f.read(4096), b""): sha256_hash.update(byte_block) local_hash = sha256_hash.hexdigest() @@ -597,40 +657,50 @@ def _validate_checksum(self,localpath: str | Path, remotepath: str | FcPath) -> try: assert local_hash == remote_hash - except AssertionError: - raise ValueError(f"Checksum mismatch between local and remote files: {local} and {remote}") - + except AssertionError as e: + raise ValueError( + f"Checksum mismatch between local and remote files: {local} and {remote}" + ) from e def _gettreetar( - self, remotepath: str | FcPath, localpath: str | Path, dereference: bool =False, *args: Any, **kwargs: Any + self, + remotepath: str | FcPath, + localpath: str | Path, + dereference: bool = False, + *args: Any, + **kwargs: Any, ) -> None: """Get a directory from the remote as a tar file and extract it locally. - This is useful for downloading a directory with many files, as it is more efficient than downloading each file individually. + This is useful for downloading a directory with many files, + as it might be more efficient than downloading each file individually. Note that this method is not part of the Transport interface, and is not meant to be used publicly. :param dereference: If True, follow symlinks. - note: FirecREST doesn't support `--dereference` for tar call, so dereference should always be False, for now. + note: FirecREST doesn't support `--dereference` for tar call, + so dereference should always be False, for now. """ # TODO manual testing the submit behaviour # if dereference: # raise NotImplementedError("Dereferencing compression not implemented in pyFirecREST.") - _ = uuid.uuid4() # Attempt direct compress + _ = uuid.uuid4() # Attempt direct compress remote_path_temp = self._temp_directory.joinpath(f"temp_{_}.tar") try: self._client.compress(self._machine, str(remotepath), remote_path_temp) except Exception as e: - # TODO: pyfirecrest is providing a solution to this, but it's not yet merged. + # TODO: pyfirecrest is providing a solution to this, but it's not yet merged. # once done submit_compress_job should be done automaticaly by compress # see: https://github.com/eth-cscs/pyfirecrest/pull/109 - raise NotImplementedError("Not implemeted for now") - comp_obj = self._client.submit_compress_job(self._machine, str(remotepath), remote_path_temp) + raise NotImplementedError("Not implemeted for now") from e + comp_obj = self._client.submit_compress_job( + self._machine, str(remotepath), remote_path_temp + ) while comp_obj.in_progress: time.sleep(self._file_transfer_poll_interval) - - # Download - localpath_temp = localpath.joinpath(f"temp_{_}.tar") + + # Download + localpath_temp = Path(localpath).joinpath(f"temp_{_}.tar") try: self.getfile(remote_path_temp, localpath_temp) finally: @@ -646,13 +716,17 @@ def _gettreetar( os.system(f"tar -xf '{localpath_temp}' --strip-components=1 -C {localpath}") finally: localpath_temp.unlink() - def gettree( - self, remotepath: str | FcPath, localpath: str | Path, dereference: bool =True, *args: Any, **kwargs: Any + self, + remotepath: str | FcPath, + localpath: str | Path, + dereference: bool = True, + *args: Any, + **kwargs: Any, ) -> None: """Get a directory from the remote. - + :param dereference: If True, follow symlinks. note: dereference should be always True, otherwise the symlinks will not be functional. """ @@ -665,14 +739,16 @@ def gettree( remote = ( remotepath if isinstance(remotepath, FcPath) - else self._cwd.joinpath(remotepath)#.enable_cache() it's removed from from path.py to be investigated + else self._cwd.joinpath( + remotepath + ) # .enable_cache() it's removed from from path.py to be investigated ) local = Path(localpath) if not remote.is_dir(): raise OSError(f"Source is not a directory: {remote}") - - # this block is added only to mimick the behavior that aiida expects + + # this block is added only to mimick the behavior that aiida expects if local.exists(): # Destination directory already exists, create remote directory name inside it local = local.joinpath(remote.name) @@ -680,16 +756,10 @@ def gettree( else: # Destination directory does not exist, create and move content 69 inside it local.mkdir(parents=True, exist_ok=False) - # SSH transport behaviour, 69 is a directory - # transport.get('somepath/69', 'someremotepath/') == transport.get('somepath/69', 'someremotepath') - # transport.get('somepath/69', 'someremotepath/') == transport.get('somepath/69/', 'someremotepath/') - # transport.get('someremotepath/69', 'somepath/69') --> if 69 exist, create 69 inside it ('somepath/69/69') - # transport.get('someremotepath/69', 'somepath/69') --> if 69 no texist, create 69 inside it ('somepath/69') - # transport.get('somepath/69', 'someremotepath/6889/69') --> create everything, make_parent = True - + if self.payoff(remote): # in this case send a request to the server to tar the files and then download the tar file - # unfortunately, the server does not provide a deferenced tar option, yet. + # unfortunately, the server does not provide a deferenced tar option, yet. self._gettreetar(remote, local) else: # otherwise download the files one by one @@ -699,7 +769,7 @@ def gettree( target_path = remote_item._cache.link_target if not Path(target_path).is_absolute(): target_path = remote_item.parent.joinpath(target_path).resolve() - + target_path = self._cwd.joinpath(target_path) if target_path.is_dir(): self.gettree(target_path, local_item, dereference=True) @@ -710,20 +780,26 @@ def gettree( self.getfile(target_path, local_item) else: local_item.mkdir(parents=True, exist_ok=True) - - - def get(self, remotepath: str, localpath: str, ignore_nonexisting: bool = False, dereference: bool =True, *args: Any, **kwargs: Any) -> None: + def get( + self, + remotepath: str, + localpath: str, + ignore_nonexisting: bool = False, + dereference: bool = True, + *args: Any, + **kwargs: Any, + ) -> None: """Get a file or directory from the remote. - + :param ignore_nonexisting: If True, do not raise an error if the source file does not exist. :param dereference: If True, follow symlinks. note: dereference should be always True, otherwise the symlinks will not be functional. """ - remote = self._cwd.joinpath(remotepath)#.enable_cache() it's removed from from path.py to be investigated - local = Path(localpath) - - + remote = self._cwd.joinpath( + remotepath + ) # .enable_cache() it's removed from from path.py to be investigated + if remote.is_dir(): self.gettree(remote, localpath) elif remote.is_file(): @@ -731,20 +807,25 @@ def get(self, remotepath: str, localpath: str, ignore_nonexisting: bool = False, elif not ignore_nonexisting: raise FileNotFoundError(f"Source file does not exist: {remote}") - def putfile( - self, localpath: str, remotepath: str, dereference:bool = True, *args: Any, **kwargs: Any , + self, + localpath: str | Path, + remotepath: str | FcPath, + dereference: bool = True, + *args: Any, + **kwargs: Any, ) -> None: """Put a file from the remote. :param dereference: If True, follow symlinks. note: we don't support uploading symlinks, so dereference is always should be True - + """ - - if not dereference: - raise NotImplementedError("Getting symlinks with `dereference=False` is not supported") + if not dereference: + raise NotImplementedError( + "Getting symlinks with `dereference=False` is not supported" + ) localpath = Path(localpath) if not localpath.is_absolute(): @@ -753,18 +834,20 @@ def putfile( if not localpath.exists(): raise FileNotFoundError(f"Local file does not exist: {localpath}") raise ValueError(f"Input localpath is not a file {localpath}") - remote = self._cwd.joinpath(remotepath)#.enable_cache() it's removed from from path.py to be investigated - + remote = self._cwd.joinpath( + str(remotepath) + ) # .enable_cache() it's removed from from path.py to be investigated if remote.is_dir(): raise ValueError(f"Destination is a directory: {remote}") local_size = localpath.stat().st_size # note this allows overwriting of existing files - with self._cwd.convert_header_exceptions({"machine": self._machine, "path": remote}): + with self._cwd.convert_header_exceptions(): if local_size < self._small_file_size_bytes: self._client.simple_upload( - self._machine, str(localpath), str(remote.parent), remote.name) + self._machine, str(localpath), str(remote.parent), remote.name + ) else: # TODO the following is a very basic implementation of uploading a large file # ideally though, if uploading multiple large files (i.e. in puttree), @@ -772,53 +855,57 @@ def putfile( # to concurrently upload to the object store (a.k.a. "staging area"), # then wait for all files to finish being transferred to the target location - # I investigated asyncio, but it's not performant for this use case. - # Becasue in the end, FirecREST server ends up serializing the requests. + # I investigated asyncio, but it's not performant for this use case. + # Becasue in the end, FirecREST server ends up serializing the requests. # see here: https://github.com/eth-cscs/pyfirecrest/issues/94 - up_obj = self._client.external_upload(self._machine, str(localpath), str(remote)) + up_obj = self._client.external_upload( + self._machine, str(localpath), str(remote) + ) up_obj.finish_upload() self._validate_checksum(localpath, str(remote)) - def payoff( - self, remotepath: str - ) -> bool: + def payoff(self, path: str | FcPath | Path) -> bool: """ This function will be used to determine whether to tar the files before downloading """ - # After discussing with the pyfirecrest team, it seems that server has some sort - # of serialization and "penalty" for sending multiple requests asycnhronusly or in a short time window. + # After discussing with the pyfirecrest team, it seems that server has some sort + # of serialization and "penalty" for sending multiple requests asycnhronusly or in a short time window. # It responses in 1, 1.5, 3, 5, 7 seconds! # So right now, I think if the number of files is more than 3, it pays off to tar everything - + # If payoff_override is set, return its value if self.payoff_override is not None: - return self.payoff_override + return bool(self.payoff_override) - if len(self.listdir(remotepath,recursive=True)) > 3: + if len(self.listdir(str(path), recursive=True)) > 3: return True - else: - return False + return False def _puttreetar( - self, localpath: str | Path, remotepath: str | FcPath, dereference: bool=True, *args: Any, **kwargs: Any + self, + localpath: str | Path, + remotepath: str | FcPath, + dereference: bool = True, + *args: Any, + **kwargs: Any, ) -> None: - """Put a directory to the remote by sending as tar file in backend. - This is useful for uploading a directory with many files, as it is more efficient than uploading each file individually. + """Put a directory to the remote by sending as tar file in backend. + This is useful for uploading a directory with many files, + as it might be more efficient than uploading each file individually. Note that this method is not part of the Transport interface, and is not meant to be used publicly. :param dereference: If True, follow symlinks. If False, symlinks are ignored from sending over. """ # this function will be used to send a folder as a tar file to the server and extract it on the server - _ = uuid.uuid4() localpath = Path(localpath) tarpath = localpath.parent.joinpath(f"temp_{_}.tar") remote_path_temp = self._temp_directory.joinpath(f"temp_{_}.tar") with tarfile.open(tarpath, "w", dereference=dereference) as tar: - for root, dirs, files in os.walk(localpath, followlinks=dereference): + for root, _, files in os.walk(localpath, followlinks=dereference): for file in files: full_path = os.path.join(root, file) relative_path = os.path.relpath(full_path, localpath) @@ -836,27 +923,34 @@ def _puttreetar( # TODO: pyfirecrest is providing a solution to this, but it's not yet merged # once done submit_compress_job should be done automaticaly by compress # see: https://github.com/eth-cscs/pyfirecrest/pull/109 - raise NotImplementedError("Not implemeted for now") - comp_obj = self._client.submit_extract_job(self._machine, remotepath.joinpath(f"_{_}.tar"), str(remotepath)) + raise NotImplementedError("Not implemeted for now") from e + comp_obj = self._client.submit_extract_job( + self._machine, remotepath.joinpath(f"_{_}.tar"), str(remotepath) + ) while comp_obj.in_progress: time.sleep(self._file_transfer_poll_interval) finally: self.remove(remote_path_temp) - def puttree( - self, localpath: str | Path, remotepath: str, dereference: bool=True, *args: Any, **kwargs: Any + self, + localpath: str | Path, + remotepath: str, + dereference: bool = True, + *args: Any, + **kwargs: Any, ) -> None: - """Put a directory to the remote. - - :param dereference: If True, follow symlinks. - note: dereference should be always True, otherwise the symlinks will not be functional, therfore not supported. + """Put a directory to the remote. + + :param dereference: If True, follow symlinks. + note: dereference should be always True, otherwise the symlinks + will not be functional, therfore not supported. """ if not dereference: raise NotImplementedError localpath = Path(localpath) - remotepath = self._cwd.joinpath(remotepath) + remote = self._cwd.joinpath(remotepath) if not localpath.is_absolute(): raise ValueError("The localpath must be an absolute path") @@ -866,71 +960,66 @@ def puttree( raise ValueError(f"Input localpath is not a directory: {localpath}") # this block is added only to mimick the behavior that aiida expects - if remotepath.exists(): + if remote.exists(): # Destination directory already exists, create local directory name inside it - remotepath = self._cwd.joinpath(remotepath, localpath.name) - self.mkdir(remotepath, ignore_existing=False) + remote = self._cwd.joinpath(remote, localpath.name) + self.mkdir(remote, ignore_existing=False) else: # Destination directory does not exist, create and move content 69 inside it - self.mkdir(remotepath, ignore_existing=False) - # SSH transport behaviour - # transport.put('somepath/69', 'someremotepath/') == transport.put('somepath/69', 'someremotepath') - # transport.put('somepath/69', 'someremotepath/') != transport.put('somepath/69/', 'someremotepath/') - # transport.put('somepath/69', 'someremotepath/67') --> if 67 not exist, create and move content 69 inside it (someremotepath/67) - # transport.put('somepath/69', 'someremotepath/67') --> if 67 exist, create 69 inside it (someremotepath/67/69) - # transport.put('somepath/69', 'someremotepath/6889/69') --> useless Error: OSError - # Weired - # SSH bug: - # transport.put('somepath/69', 'someremotepath/') --> assuming someremotepath exists, make 69 - # while - # transport.put('somepath/69/', 'someremotepath/') --> assuming someremotepath exists, OSError: cannot make someremotepath - - - if self.payoff(remotepath): + self.mkdir(remote, ignore_existing=False) + + if self.payoff(localpath): # in this case send send everything as a tar file - self._puttreetar(localpath, remotepath) + self._puttreetar(localpath, remote) else: # otherwise send the files one by one for dirpath, _, filenames in os.walk(localpath, followlinks=dereference): rel_folder = os.path.relpath(path=dirpath, start=localpath) - rm_parent_now = remotepath.joinpath(rel_folder) + rm_parent_now = remote.joinpath(rel_folder) self.mkdir(rm_parent_now, ignore_existing=True) for filename in filenames: localfile_path = os.path.join(localpath, rel_folder, filename) - remotefile_path = rm_parent_now.joinpath( filename) + remotefile_path = rm_parent_now.joinpath(filename) self.putfile(localfile_path, remotefile_path) - - def put(self, localpath: str, remotepath: str, ignore_nonexisting: bool =False, dereference: bool=True, *args: Any, **kwargs: Any) -> None: + def put( + self, + localpath: str, + remotepath: str, + ignore_nonexisting: bool = False, + dereference: bool = True, + *args: Any, + **kwargs: Any, + ) -> None: """Put a file or directory to the remote. :param ignore_nonexisting: If True, do not raise an error if the source file does not exist. :param dereference: If True, follow symlinks. - note: dereference should be always True, otherwise the symlinks will not be functional. + note: dereference should be always True, otherwise the symlinks will not be functional. """ # TODO ssh does a lot more - # update on the TODO: I made a manual test with ssh. added some extra care in puttree and gettree and now it's working fine - + # update on the TODO: I made a manual test with ssh. + # added some extra care in puttree and gettree and now it's working fine + if not dereference: raise NotImplementedError - - localpath = Path(localpath) - if not localpath.is_absolute(): + + local = Path(localpath) + if not local.is_absolute(): raise ValueError("The localpath must be an absolute path") - if not Path(localpath).exists() and not ignore_nonexisting: + if not Path(local).exists() and not ignore_nonexisting: raise FileNotFoundError(f"Source file does not exist: {localpath}") - - if localpath.is_dir(): + + if local.is_dir(): self.puttree(localpath, remotepath) - elif localpath.is_file(): + elif local.is_file(): self.putfile(localpath, remotepath) - - def remove(self, path: str) -> None: + def remove(self, path: str | FcPath) -> None: """Remove a file or directory on the remote.""" - self._cwd.joinpath(path).unlink() + self._cwd.joinpath(str(path)).unlink() def rename(self, oldpath: str, newpath: str) -> None: """Rename a file or directory on the remote.""" @@ -950,15 +1039,15 @@ def rmtree(self, path: str) -> None: If the directory is not empty, it will be removed recursively, equivalent to `rm -rf`. It does not raise an error if the directory does not exist. """ - try: + # TODO: suppress is to mimick the behaviour of `aiida-ssh`` transport, TODO: raise an issue on aiida + with suppress(FileNotFoundError): self._cwd.joinpath(path).rmtree() - # TODO: this try&except is to mimick the behaviour of `aiida-ssh`` transport, TODO: raise an issue on aiida - except FileNotFoundError: - pass - def whoami(self) -> str: - """Return the username of the current user.""" - return self._client.whoami(machine = self._machine) + def whoami(self) -> str | None: + """Return the username of the current user. + return None if the username cannot be determined. + """ + return self._client.whoami(machine=self._machine) def gotocomputer_command(self, remotedir: str) -> str: """Not possible for REST-API. diff --git a/aiida_firecrest/utils_test.py b/aiida_firecrest/utils_test.py index a7a02e7..27394e1 100644 --- a/aiida_firecrest/utils_test.py +++ b/aiida_firecrest/utils_test.py @@ -48,9 +48,9 @@ def __init__( self._scratch = tmpdir / "scratch" self._scratch.mkdir() self._client_id = "test_client_id" - + Path(tmpdir / ".firecrest").mkdir() - self._client_secret = tmpdir / ".firecrest/secret" + self._client_secret = tmpdir / ".firecrest/secret" self._client_secret.write_text("test_client_secret") self._token_url = "https://test.auth.com/token" @@ -406,12 +406,13 @@ def utilities_checksum(self, params: dict[str, Any], response: Response) -> None response.headers["X-Invalid-Path"] = "" return import hashlib + # Firecrest uses sha256 sha256_hash = hashlib.sha256() - with open(path,"rb") as f: - for byte_block in iter(lambda: f.read(4096),b""): + with open(path, "rb") as f: + for byte_block in iter(lambda: f.read(4096), b""): sha256_hash.update(byte_block) - + checksum = sha256_hash.hexdigest() add_success_response(response, 200, checksum) @@ -527,7 +528,7 @@ def utilities_download(self, params: dict[str, Any], response: Response) -> None # def handle_task(self, task_id: str, response: Response) -> Response: def handle_task(self, params: dict[str, Any], response: Response) -> Response: - task_id = params["tasks"].split(',')[0] + task_id = params["tasks"].split(",")[0] if task_id not in self._tasks: return add_json_response( response, 404, {"error": f"Task {task_id} does not exist"} diff --git a/pyproject.toml b/pyproject.toml index 97551be..ffcf89f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -26,7 +26,7 @@ requires-python = ">=3.9" dependencies = [ "aiida-core@git+https://github.com/chrisjsewell/aiida_core.git@aiida-firecrest#egg=aiida-core", "click", - "pyfirecrest~=1.4.0", + "pyfirecrest~=2.5.0", "pyyaml", "requests", ] diff --git a/tests/test_computer.py b/tests/test_computer.py index a8a3179..81d6c58 100644 --- a/tests/test_computer.py +++ b/tests/test_computer.py @@ -41,8 +41,7 @@ def _firecrest_computer(firecrest_server: FirecrestConfig): def test_whoami(firecrest_computer: orm.Computer): """check if it is possible to determine the username.""" transport = firecrest_computer.get_transport() - assert transport.whoami() == 'test_user' - + assert transport.whoami() == "test_user" @pytest.mark.usefixtures("aiida_profile_clean") diff --git a/tests/tests_mocking_pyfirecrest/conftest.py b/tests/tests_mocking_pyfirecrest/conftest.py index 8655321..2e4a116 100644 --- a/tests/tests_mocking_pyfirecrest/conftest.py +++ b/tests/tests_mocking_pyfirecrest/conftest.py @@ -1,15 +1,17 @@ +import hashlib +import os from pathlib import Path -import os, stat -import random, hashlib -import firecrest.path -import firecrest - -import pytest +import random +import stat +from typing import Optional from aiida import orm +import firecrest +import firecrest.path +import pytest -class values: +class Values: _DEFAULT_PAGE_SIZE: int = 25 @@ -41,18 +43,17 @@ def _firecrest_computer(myfirecrest, tmpdir: Path): computer.set_minimum_job_poll_interval(5) computer.set_default_mpiprocs_per_machine(1) computer.configure( - url=' https://URI', - token_uri='https://TOKEN_URI', - client_id='CLIENT_ID', + url=" https://URI", + token_uri="https://TOKEN_URI", + client_id="CLIENT_ID", client_secret=str(_secret_path), - client_machine='MACHINE_NAME', + client_machine="MACHINE_NAME", small_file_size_mb=1.0, temp_directory=str(_temp_directory), ) return computer - class MockFirecrest: def __init__(self, firecrest_url, *args, **kwargs): self._firecrest_url = firecrest_url @@ -81,56 +82,86 @@ def __init__(self, *args, **kwargs): self.args = args self.kwargs = kwargs + @pytest.fixture(scope="function") def myfirecrest( pytestconfig: pytest.Config, monkeypatch, ): - monkeypatch.setattr(firecrest, "Firecrest", MockFirecrest) monkeypatch.setattr(firecrest, "ClientCredentialsAuth", MockClientCredentialsAuth) -def submit(machine: str, script_str: str = None, script_remote_path: str = None, script_local_path: str = None, local_file=False): + +def submit( + machine: str, + script_str: Optional[str] = None, + script_remote_path: Optional[str] = None, + script_local_path: Optional[str] = None, + local_file=False, +): + if local_file: + raise DeprecationWarning("local_file is not supported") + if script_remote_path and not Path(script_remote_path).exists(): raise FileNotFoundError(f"File {script_remote_path} does not exist") - job_id = random.randint(10000, 99999) + job_id = random.randint(10000, 99999) return {"jobid": job_id} + def poll_active(machine: str, jobs: list[str], page_number: int = 0): response = [] # 12 satets are defined in firecrest - states = ["TIMEOUT", "SUSPENDED", "PREEMPTED", "CANCELLED", "NODE_FAIL", - "PENDING", "FAILED", "RUNNING", "CONFIGURING", "QUEUED", "COMPLETED", "COMPLETING"] + states = [ + "TIMEOUT", + "SUSPENDED", + "PREEMPTED", + "CANCELLED", + "NODE_FAIL", + "PENDING", + "FAILED", + "RUNNING", + "CONFIGURING", + "QUEUED", + "COMPLETED", + "COMPLETING", + ] for i in range(len(jobs)): response.append( { - 'job_data_err': '', - 'job_data_out': '', - 'job_file': 'somefile.sh', - 'job_file_err': 'somefile-stderr.txt', - 'job_file_out': 'somefile-stdout.txt', - 'job_info_extra': 'Job info returned successfully', - 'jobid': f'{jobs[i]}', - 'name': 'aiida-45', - 'nodelist': 'nid00049', - 'nodes': '1', - 'partition': 'normal', - 'start_time': '0:03', - 'state': states[i%12], - 'time': '2024-06-21T10:44:42', - 'time_left': '29:57', - 'user': 'Prof. Wang' - } + "job_data_err": "", + "job_data_out": "", + "job_file": "somefile.sh", + "job_file_err": "somefile-stderr.txt", + "job_file_out": "somefile-stdout.txt", + "job_info_extra": "Job info returned successfully", + "jobid": f"{jobs[i]}", + "name": "aiida-45", + "nodelist": "nid00049", + "nodes": "1", + "partition": "normal", + "start_time": "0:03", + "state": states[i % 12], + "time": "2024-06-21T10:44:42", + "time_left": "29:57", + "user": "Prof. Wang", + } ) - - return response[page_number*values._DEFAULT_PAGE_SIZE:(page_number+1)*values._DEFAULT_PAGE_SIZE] + + return response[ + page_number + * Values._DEFAULT_PAGE_SIZE : (page_number + 1) + * Values._DEFAULT_PAGE_SIZE + ] + def whomai(machine: str): assert machine == "MACHINE_NAME" return "test_user" + def list_files( - machine: str, target_path: str, recursive: bool = False, show_hidden: bool = False): + machine: str, target_path: str, recursive: bool = False, show_hidden: bool = False +): # this is mimiking the expected behaviour from the firecrest code. content_list = [] @@ -139,33 +170,42 @@ def list_files( continue for name in dirs + files: full_path = os.path.join(root, name) - relative_path = Path(os.path.relpath(root, target_path)).joinpath(name).as_posix() + relative_path = ( + Path(os.path.relpath(root, target_path)).joinpath(name).as_posix() + ) if os.path.islink(full_path): - content_type = 'l' - link_target = os.readlink(full_path) if os.path.islink(full_path) else None + content_type = "l" + link_target = ( + os.readlink(full_path) if os.path.islink(full_path) else None + ) elif os.path.isfile(full_path): - content_type = '-' + content_type = "-" link_target = None elif os.path.isdir(full_path): - content_type = 'd' + content_type = "d" link_target = None else: - content_type = 'NON' + content_type = "NON" link_target = None - permissions = stat.filemode(Path(full_path).lstat().st_mode)[1:] - if name.startswith('.') and not show_hidden: + permissions = stat.filemode(Path(full_path).lstat().st_mode)[1:] + if name.startswith(".") and not show_hidden: continue - content_list.append({ - 'name': relative_path, - 'type': content_type, - 'link_target': link_target, - 'permissions': permissions, - }) + content_list.append( + { + "name": relative_path, + "type": content_type, + "link_target": link_target, + "permissions": permissions, + } + ) return content_list -def stat_(machine:str, targetpath: firecrest.path, dereference=True): - stats = os.stat(targetpath, follow_symlinks= True if dereference else False) + +def stat_(machine: str, targetpath: firecrest.path, dereference=True): + stats = os.stat( + targetpath, follow_symlinks=bool(dereference) if dereference else False + ) return { "ino": stats.st_ino, "dev": stats.st_dev, @@ -178,12 +218,14 @@ def stat_(machine:str, targetpath: firecrest.path, dereference=True): "ctime": stats.st_ctime, } + def mkdir(machine: str, target_path: str, p: bool = False): if p: os.makedirs(target_path) else: os.mkdir(target_path) + def simple_delete(machine: str, target_path: str): if not Path(target_path).exists(): raise FileNotFoundError(f"File or folder {target_path} does not exist") @@ -192,10 +234,12 @@ def simple_delete(machine: str, target_path: str): else: os.remove(target_path) + def symlink(machine: str, target_path: str, link_path: str): # this is how firecrest does it os.system(f"ln -s {target_path} {link_path}") + def simple_download(machine: str, remote_path: str, local_path: str): # this procedure is complecated in firecrest, but I am simplifying it here # we don't care about the details of the download, we just want to make sure @@ -206,7 +250,10 @@ def simple_download(machine: str, remote_path: str, local_path: str): raise FileNotFoundError(f"{remote_path} does not exist") os.system(f"cp {remote_path} {local_path}") -def simple_upload(machine: str, local_path: str, remote_path: str, file_name: str = None): + +def simple_upload( + machine: str, local_path: str, remote_path: str, file_name: Optional[str] = None +): # this procedure is complecated in firecrest, but I am simplifying it here # we don't care about the details of the upload, we just want to make sure # that the aiida-firecrest code is calling the right functions at right time @@ -216,14 +263,18 @@ def simple_upload(machine: str, local_path: str, remote_path: str, file_name: st raise FileNotFoundError(f"{local_path} does not exist") if file_name: remote_path = os.path.join(remote_path, file_name) - os.system(f"cp {local_path} {remote_path}") + os.system(f"cp {local_path} {remote_path}") + def copy(machine: str, source_path: str, target_path: str): # this is how firecrest does it - # https://github.com/eth-cscs/firecrest/blob/db6ba4ba273c11a79ecbe940872f19d5cb19ac5e/src/utilities/utilities.py#L451C1-L452C1 + # https://github.com/eth-cscs/firecrest/blob/db6ba4ba273c11a79ecbe940872f19d5cb19ac5e/src/utilities/utilities.py#L451C1-L452C1 os.system(f"cp --force -dR --preserve=all -- '{source_path}' '{target_path}'") -def compress(machine: str, source_path: str, target_path: str, dereference: bool = True): + +def compress( + machine: str, source_path: str, target_path: str, dereference: bool = True +): # this is how firecrest does it # https://github.com/eth-cscs/firecrest/blob/db6ba4ba273c11a79ecbe940872f19d5cb19ac5e/src/utilities/utilities.py#L460 basedir = os.path.dirname(source_path) @@ -231,90 +282,88 @@ def compress(machine: str, source_path: str, target_path: str, dereference: bool deref = "--dereference" if dereference else "" os.system(f"tar {deref} -czf '{target_path}' -C '{basedir}' '{file_path}'") + def extract(machine: str, source_path: str, target_path: str): # this is how firecrest does it # https://github.com/eth-cscs/firecrest/blob/db6ba4ba273c11a79ecbe940872f19d5cb19ac5e/src/common/cscs_api_common.py#L1110C18-L1110C65 os.system(f"tar -xf '{source_path}' -C '{target_path}'") + def checksum(machine: str, remote_path: str) -> int: if not remote_path.exists(): return False # Firecrest uses sha256 sha256_hash = hashlib.sha256() - with open(remote_path,"rb") as f: - for byte_block in iter(lambda: f.read(4096),b""): + with open(remote_path, "rb") as f: + for byte_block in iter(lambda: f.read(4096), b""): sha256_hash.update(byte_block) return sha256_hash.hexdigest() + def parameters(): # note: I took this from https://firecrest-tds.cscs.ch/ or https://firecrest.cscs.ch/ - # if code is not working but test passes, it means you need to update this dictionary + # if code is not working but test passes, it means you need to update this dictionary # with the latest FirecREST parameters return { "compute": [ - { - "description": "Type of resource and workload manager used in compute microservice", - "name": "WORKLOAD_MANAGER", - "unit": "", - "value": "Slurm" - } + { + "description": "Type of resource and workload manager used in compute microservice", + "name": "WORKLOAD_MANAGER", + "unit": "", + "value": "Slurm", + } ], "storage": [ - { - "description": "Type of object storage, like `swift`, `s3v2` or `s3v4`.", - "name": "OBJECT_STORAGE", - "unit": "", - "value": "s3v4" - }, - { - "description": "Expiration time for temp URLs.", - "name": "STORAGE_TEMPURL_EXP_TIME", - "unit": "seconds", - "value": "86400" - }, - { - "description": "Maximum file size for temp URLs.", - "name": "STORAGE_MAX_FILE_SIZE", - "unit": "MB", - "value": "5120" - }, - { - "description": "Available filesystems through the API.", - "name": "FILESYSTEMS", - "unit": "", - "value": [ { - "mounted": [ - "/project", - "/store", - "/scratch/snx3000tds" - ], - "system": "dom" + "description": "Type of object storage, like `swift`, `s3v2` or `s3v4`.", + "name": "OBJECT_STORAGE", + "unit": "", + "value": "s3v4", }, { - "mounted": [ - "/project", - "/store", - "/capstor/scratch/cscs" + "description": "Expiration time for temp URLs.", + "name": "STORAGE_TEMPURL_EXP_TIME", + "unit": "seconds", + "value": "86400", + }, + { + "description": "Maximum file size for temp URLs.", + "name": "STORAGE_MAX_FILE_SIZE", + "unit": "MB", + "value": "5120", + }, + { + "description": "Available filesystems through the API.", + "name": "FILESYSTEMS", + "unit": "", + "value": [ + { + "mounted": ["/project", "/store", "/scratch/snx3000tds"], + "system": "dom", + }, + { + "mounted": ["/project", "/store", "/capstor/scratch/cscs"], + "system": "pilatus", + }, ], - "system": "pilatus" - } - ] - } + }, ], "utilities": [ - { - "description": "The maximum allowable file size for various operations of the utilities microservice", - "name": "UTILITIES_MAX_FILE_SIZE", - "unit": "MB", - "value": "69" - }, - { - "description": "Maximum time duration for executing the commands in the cluster for the utilities microservice.", - "name": "UTILITIES_TIMEOUT", - "unit": "seconds", - "value": "5" - } - ] - } \ No newline at end of file + { + "description": "The maximum allowable file size for various operations of the utilities microservice", + "name": "UTILITIES_MAX_FILE_SIZE", + "unit": "MB", + "value": "69", + }, + { + "description": ( + "Maximum time duration for executing the commands " + "in the cluster for the utilities microservice." + ), + "name": "UTILITIES_TIMEOUT", + "unit": "seconds", + "value": "5", + }, + ], + } diff --git a/tests/tests_mocking_pyfirecrest/test_computer.py b/tests/tests_mocking_pyfirecrest/test_computer.py index edca7d1..fe4e48e 100644 --- a/tests/tests_mocking_pyfirecrest/test_computer.py +++ b/tests/tests_mocking_pyfirecrest/test_computer.py @@ -1,102 +1,121 @@ from pathlib import Path - -import pytest from unittest.mock import Mock -from click import BadParameter from aiida import orm +from click import BadParameter +import pytest @pytest.mark.usefixtures("aiida_profile_clean") def test_whoami(firecrest_computer: orm.Computer): """check if it is possible to determine the username.""" transport = firecrest_computer.get_transport() - assert transport.whoami() == 'test_user' + assert transport.whoami() == "test_user" + def test_create_secret_file_with_existing_file(tmpdir: Path): - from aiida_firecrest.transport import FirecrestTransport + from aiida_firecrest.transport import _create_secret_file + secret_file = Path(tmpdir / "secret") secret_file.write_text("topsecret") - result = FirecrestTransport._create_secret_file(None, None, str(secret_file)) + result = _create_secret_file(None, None, str(secret_file)) assert isinstance(result, str) assert result == str(secret_file) assert Path(result).read_text() == "topsecret" + def test_create_secret_file_with_nonexistent_file(tmp_path): - from aiida_firecrest.transport import FirecrestTransport + from aiida_firecrest.transport import _create_secret_file + secret_file = tmp_path / "nonexistent" with pytest.raises(BadParameter): - FirecrestTransport._create_secret_file(None, None, str(secret_file)) + _create_secret_file(None, None, str(secret_file)) + def test_create_secret_file_with_secret_value(tmp_path, monkeypatch): - from aiida_firecrest.transport import FirecrestTransport + from aiida_firecrest.transport import _create_secret_file + secret = "topsecret!~/" - monkeypatch.setattr(Path, "expanduser", lambda x: tmp_path / str(x).lstrip("~/") if str(x).startswith("~/") else x) - result = FirecrestTransport._create_secret_file(None, None, secret) - assert Path(result).parent.parts[-1]== ".firecrest" + monkeypatch.setattr( + Path, + "expanduser", + lambda x: tmp_path / str(x).lstrip("~/") if str(x).startswith("~/") else x, + ) + result = _create_secret_file(None, None, secret) + assert Path(result).parent.parts[-1] == ".firecrest" assert Path(result).read_text() == secret + def test_validate_temp_directory(myfirecrest, monkeypatch, tmpdir: Path): - from aiida_firecrest.transport import FirecrestTransport + from aiida_firecrest.transport import _validate_temp_directory - monkeypatch.setattr('click.echo', lambda x: None) + monkeypatch.setattr("click.echo", lambda x: None) # monkeypatch.setattr('click.BadParameter', lambda x: None) secret_file = Path(tmpdir / "secret") secret_file.write_text("topsecret") ctx = Mock() ctx.params = { - 'url': 'http://test.com', - 'token_uri': 'token_uri', - 'client_id': 'client_id', - 'client_machine': 'client_machine', - 'client_secret': secret_file.as_posix(), - 'small_file_size_mb': float(10) + "url": "http://test.com", + "token_uri": "token_uri", + "client_id": "client_id", + "client_machine": "client_machine", + "client_secret": secret_file.as_posix(), + "small_file_size_mb": float(10), } # should raise if is_file - Path(tmpdir / 'crap.txt').touch() + Path(tmpdir / "crap.txt").touch() with pytest.raises(BadParameter): - result = FirecrestTransport._validate_temp_directory(ctx, None, Path(tmpdir /'crap.txt').as_posix()) + result = _validate_temp_directory( + ctx, None, Path(tmpdir / "crap.txt").as_posix() + ) # should create the directory if it doesn't exist - result = FirecrestTransport._validate_temp_directory(ctx, None, Path(tmpdir /'temp_on_server_directory').as_posix()) - assert result == Path(tmpdir /'temp_on_server_directory').as_posix() - assert Path(tmpdir /'temp_on_server_directory').exists() + result = _validate_temp_directory( + ctx, None, Path(tmpdir / "temp_on_server_directory").as_posix() + ) + assert result == Path(tmpdir / "temp_on_server_directory").as_posix() + assert Path(tmpdir / "temp_on_server_directory").exists() # should get a confirmation if the directory exists and is not empty - Path(tmpdir /'temp_on_server_directory' / 'crap.txt').touch() - monkeypatch.setattr('click.confirm', lambda x: False) + Path(tmpdir / "temp_on_server_directory" / "crap.txt").touch() + monkeypatch.setattr("click.confirm", lambda x: False) with pytest.raises(BadParameter): - result = FirecrestTransport._validate_temp_directory(ctx, None, Path(tmpdir /'temp_on_server_directory').as_posix()) + result = _validate_temp_directory( + ctx, None, Path(tmpdir / "temp_on_server_directory").as_posix() + ) # should delete the content if I confirm - monkeypatch.setattr('click.confirm', lambda x: True) - result = FirecrestTransport._validate_temp_directory(ctx, None, Path(tmpdir /'temp_on_server_directory').as_posix()) - assert result == Path(tmpdir /'temp_on_server_directory').as_posix() - assert not Path(tmpdir /'temp_on_server_directory' / 'crap.txt').exists() + monkeypatch.setattr("click.confirm", lambda x: True) + result = _validate_temp_directory( + ctx, None, Path(tmpdir / "temp_on_server_directory").as_posix() + ) + assert result == Path(tmpdir / "temp_on_server_directory").as_posix() + assert not Path(tmpdir / "temp_on_server_directory" / "crap.txt").exists() + def test__dynamic_info(myfirecrest, monkeypatch, tmpdir: Path): - from aiida_firecrest.transport import FirecrestTransport + from aiida_firecrest.transport import _dynamic_info_direct_size - monkeypatch.setattr('click.echo', lambda x: None) + monkeypatch.setattr("click.echo", lambda x: None) # monkeypatch.setattr('click.BadParameter', lambda x: None) secret_file = Path(tmpdir / "secret") secret_file.write_text("topsecret") ctx = Mock() ctx.params = { - 'url': 'http://test.com', - 'token_uri': 'token_uri', - 'client_id': 'client_id', - 'client_machine': 'client_machine', - 'client_secret': secret_file.as_posix(), - 'small_file_size_mb': float(10) + "url": "http://test.com", + "token_uri": "token_uri", + "client_id": "client_id", + "client_machine": "client_machine", + "client_secret": secret_file.as_posix(), + "small_file_size_mb": float(10), } # should catch UTILITIES_MAX_FILE_SIZE if value is not provided - result = FirecrestTransport._dynamic_info_direct_size(ctx, None, 0) + result = _dynamic_info_direct_size(ctx, None, 0) assert result == 69 # should use the value if provided # note: user cannot enter negative numbers anyways, click raise as this shoule be float not str - result = FirecrestTransport._dynamic_info_direct_size(ctx, None, 10) + result = _dynamic_info_direct_size(ctx, None, 10) assert result == 10 diff --git a/tests/tests_mocking_pyfirecrest/test_scheduler.py b/tests/tests_mocking_pyfirecrest/test_scheduler.py index 576875f..b4cb26f 100644 --- a/tests/tests_mocking_pyfirecrest/test_scheduler.py +++ b/tests/tests_mocking_pyfirecrest/test_scheduler.py @@ -1,11 +1,12 @@ from pathlib import Path import random -import pytest -from conftest import values from aiida import orm -from aiida_firecrest.scheduler import FirecrestScheduler from aiida.schedulers.datastructures import CodeRunMode, JobTemplate +import pytest + +from aiida_firecrest.scheduler import FirecrestScheduler +from conftest import Values @pytest.mark.usefixtures("aiida_profile_clean") @@ -14,26 +15,26 @@ def test_submit_job(firecrest_computer: orm.Computer, tmp_path: Path): scheduler = FirecrestScheduler() scheduler.set_transport(transport) - - with pytest.raises(Exception): + with pytest.raises(FileNotFoundError): scheduler.submit_job(transport.getcwd(), "unknown.sh") _script = Path(tmp_path / "job.sh") _script.write_text("#!/bin/bash\n\necho 'hello world'") - + job_id = scheduler.submit_job(transport.getcwd(), _script) # this is how aiida expects the job_id to be returned assert isinstance(job_id, str) + @pytest.mark.usefixtures("aiida_profile_clean") def test_get_jobs(firecrest_computer: orm.Computer): transport = firecrest_computer.get_transport() scheduler = FirecrestScheduler() scheduler.set_transport(transport) - # test pagaination - scheduler._DEFAULT_PAGE_SIZE = 2 - values._DEFAULT_PAGE_SIZE = 2 + # test pagaination + scheduler._DEFAULT_PAGE_SIZE = 2 + Values._DEFAULT_PAGE_SIZE = 2 joblist = [random.randint(10000, 99999) for i in range(5)] result = scheduler.get_jobs(joblist) @@ -41,11 +42,10 @@ def test_get_jobs(firecrest_computer: orm.Computer): for i in range(5): assert result[i].job_id == str(joblist[i]) # TODO: one could check states as well - def test_write_script_full(): - # to avoid false positive (overwriting on existing file), + # to avoid false positive (overwriting on existing file), # we check the output of the script instead of using `file_regression`` expectaion = """ #!/bin/bash @@ -69,7 +69,9 @@ def test_write_script_full(): #SBATCH --mem=1 test_command """ - expectaion_flat = '\n'.join(line.strip() for line in expectaion.splitlines()).strip('\n') + expectaion_flat = "\n".join(line.strip() for line in expectaion.splitlines()).strip( + "\n" + ) scheduler = FirecrestScheduler() template = JobTemplate( { @@ -105,7 +107,7 @@ def test_write_script_full(): def test_write_script_minimal(): - # to avoid false positive (overwriting on existing file), + # to avoid false positive (overwriting on existing file), # we check the output of the script instead of using `file_regression`` expectaion = """ #!/bin/bash @@ -114,8 +116,10 @@ def test_write_script_minimal(): #SBATCH --nodes=1 #SBATCH --ntasks-per-node=1 """ - - expectaion_flat = '\n'.join(line.strip() for line in expectaion.splitlines()).strip('\n') + + expectaion_flat = "\n".join(line.strip() for line in expectaion.splitlines()).strip( + "\n" + ) scheduler = FirecrestScheduler() template = JobTemplate( { diff --git a/tests/tests_mocking_pyfirecrest/test_transport.py b/tests/tests_mocking_pyfirecrest/test_transport.py index c244c5a..edd11ee 100644 --- a/tests/tests_mocking_pyfirecrest/test_transport.py +++ b/tests/tests_mocking_pyfirecrest/test_transport.py @@ -1,10 +1,10 @@ -from pathlib import Path import os - -import pytest +from pathlib import Path from unittest.mock import patch from aiida import orm +import pytest + @pytest.mark.usefixtures("aiida_profile_clean") def test_mkdir(firecrest_computer: orm.Computer, tmpdir: Path): @@ -18,14 +18,16 @@ def test_mkdir(firecrest_computer: orm.Computer, tmpdir: Path): transport.makedirs(_scratch) assert _scratch.exists() + @pytest.mark.usefixtures("aiida_profile_clean") def test_is_file(firecrest_computer: orm.Computer, tmpdir: Path): transport = firecrest_computer.get_transport() _scratch = tmpdir / "samplefile" Path(_scratch).touch() - assert transport.isfile(_scratch) == True - assert transport.isfile(_scratch / "does_not_exist") == False + assert transport.isfile(_scratch) + assert not transport.isfile(_scratch / "does_not_exist") + @pytest.mark.usefixtures("aiida_profile_clean") def test_is_dir(firecrest_computer: orm.Computer, tmpdir: Path): @@ -34,8 +36,9 @@ def test_is_dir(firecrest_computer: orm.Computer, tmpdir: Path): _scratch = tmpdir / "sampledir" _scratch.mkdir() - assert transport.isdir(_scratch) == True - assert transport.isdir(_scratch / "does_not_exist") == False + assert transport.isdir(_scratch) + assert not transport.isdir(_scratch / "does_not_exist") + @pytest.mark.usefixtures("aiida_profile_clean") def test_normalize(firecrest_computer: orm.Computer): @@ -44,9 +47,16 @@ def test_normalize(firecrest_computer: orm.Computer): assert transport.normalize("path/to/dir") == os.path.normpath("path/to/dir") assert transport.normalize("path/to/dir/") == os.path.normpath("path/to/dir/") assert transport.normalize("path/to/../dir") == os.path.normpath("path/to/../dir") - assert transport.normalize("path/to/../../dir") == os.path.normpath("path/to/../../dir") - assert transport.normalize("path/to/../../dir/") == os.path.normpath("path/to/../../dir/") - assert transport.normalize("path/to/../../dir/../") == os.path.normpath("path/to/../../dir/../") + assert transport.normalize("path/to/../../dir") == os.path.normpath( + "path/to/../../dir" + ) + assert transport.normalize("path/to/../../dir/") == os.path.normpath( + "path/to/../../dir/" + ) + assert transport.normalize("path/to/../../dir/../") == os.path.normpath( + "path/to/../../dir/../" + ) + @pytest.mark.usefixtures("aiida_profile_clean") def test_remove(firecrest_computer: orm.Computer, tmpdir: Path): @@ -72,6 +82,7 @@ def test_remove(firecrest_computer: orm.Computer, tmpdir: Path): transport.rmdir(_scratch) assert not _scratch.exists() + @pytest.mark.usefixtures("aiida_profile_clean") def test_symlink(firecrest_computer: orm.Computer, tmpdir: Path): transport = firecrest_computer.get_transport() @@ -83,6 +94,7 @@ def test_symlink(firecrest_computer: orm.Computer, tmpdir: Path): assert _symlink.is_symlink() assert _symlink.resolve() == _scratch + @pytest.mark.usefixtures("aiida_profile_clean") def test_listdir(firecrest_computer: orm.Computer, tmpdir: Path): transport = firecrest_computer.get_transport() @@ -96,25 +108,36 @@ def test_listdir(firecrest_computer: orm.Computer, tmpdir: Path): # to test recursive Path(_scratch / "dir1" / "file2").touch() - assert set(transport.listdir(_scratch)) == set(["file1", "dir1", ".hidden"]) - assert set(transport.listdir(_scratch, recursive=True)) == set(["file1", "dir1", ".hidden", - "dir1/file2"]) + assert set(transport.listdir(_scratch)) == {"file1", "dir1", ".hidden"} + assert set(transport.listdir(_scratch, recursive=True)) == { + "file1", + "dir1", + ".hidden", + "dir1/file2", + } # to test symlink Path(_scratch / "dir1" / "dir2").mkdir() Path(_scratch / "dir1" / "dir2" / "file3").touch() os.symlink(_scratch / "dir1" / "dir2", _scratch / "dir2_link") os.symlink(_scratch / "dir1" / "file2", _scratch / "file_link") - assert set(transport.listdir(_scratch, recursive=True)) == set(["file1", "dir1", ".hidden", - "dir1/file2", "dir1/dir2", "dir1/dir2/file3", - "dir2_link", "file_link"]) + assert set(transport.listdir(_scratch, recursive=True)) == { + "file1", + "dir1", + ".hidden", + "dir1/file2", + "dir1/dir2", + "dir1/dir2/file3", + "dir2_link", + "file_link", + } - assert set(transport.listdir(_scratch / "dir2_link", recursive=False)) == set(["file3"]) + assert set(transport.listdir(_scratch / "dir2_link", recursive=False)) == {"file3"} @pytest.mark.usefixtures("aiida_profile_clean") def test_get(firecrest_computer: orm.Computer, tmpdir: Path): - """ + """ This is minimal test is to check if get() is raising errors as expected, and directing to getfile() and gettree() as expected. Mainly just checking error handeling and folder creation. @@ -125,24 +148,23 @@ def test_get(firecrest_computer: orm.Computer, tmpdir: Path): _local = tmpdir / "localdir" _remote.mkdir() _local.mkdir() - # check if the code is directing to getfile() or gettree() as expected - with patch.object(transport, 'gettree', autospec=True) as mock_gettree: + with patch.object(transport, "gettree", autospec=True) as mock_gettree: transport.get(_remote, _local) mock_gettree.assert_called_once() - with patch.object(transport, 'gettree', autospec=True) as mock_gettree: + with patch.object(transport, "gettree", autospec=True) as mock_gettree: os.symlink(_remote, tmpdir / "dir_link") transport.get(tmpdir / "dir_link", _local) mock_gettree.assert_called_once() - with patch.object(transport, 'getfile', autospec=True) as mock_getfile: + with patch.object(transport, "getfile", autospec=True) as mock_getfile: Path(_remote / "file1").write_text("file1") transport.get(_remote / "file1", _local / "file1") mock_getfile.assert_called_once() - with patch.object(transport, 'getfile', autospec=True) as mock_getfile: + with patch.object(transport, "getfile", autospec=True) as mock_getfile: os.symlink(_remote / "file1", _remote / "file1_link") transport.get(_remote / "file1_link", _local / "file1_link") mock_getfile.assert_called_once() @@ -151,13 +173,14 @@ def test_get(firecrest_computer: orm.Computer, tmpdir: Path): with pytest.raises(FileNotFoundError): transport.get(_remote / "does_not_exist", _local) transport.get(_remote / "does_not_exist", _local, ignore_nonexisting=True) - + # raise if localpath is relative with pytest.raises(ValueError): transport.get(_remote, Path(_local).relative_to(tmpdir)) with pytest.raises(ValueError): transport.get(_remote / "file1", Path(_local).relative_to(tmpdir)) + @pytest.mark.usefixtures("aiida_profile_clean") def test_getfile(firecrest_computer: orm.Computer, tmpdir: Path): transport = firecrest_computer.get_transport() @@ -170,7 +193,6 @@ def test_getfile(firecrest_computer: orm.Computer, tmpdir: Path): Path(_remote / "file1").write_text("file1") Path(_remote / ".hidden").write_text(".hidden") os.symlink(_remote / "file1", _remote / "file1_link") - # raise if remote file does not exist with pytest.raises(FileNotFoundError): @@ -187,7 +209,7 @@ def test_getfile(firecrest_computer: orm.Computer, tmpdir: Path): # don't mix up directory with file with pytest.raises(FileNotFoundError): transport.getfile(_remote, _local / "file1") - + # write where I tell you to transport.getfile(_remote / "file1", _local / "file1") transport.getfile(_remote / "file1", _local / "file1-prime") @@ -212,16 +234,25 @@ def test_getfile(firecrest_computer: orm.Computer, tmpdir: Path): assert Path(_local / "file1_link").read_text() == "file1" assert not Path(_local / "file1_link").is_symlink() -@pytest.mark.parametrize("payoff", [True, False]) + +@pytest.mark.parametrize("payoff", [True, False]) @pytest.mark.usefixtures("aiida_profile_clean") def test_gettree(firecrest_computer: orm.Computer, tmpdir: Path, payoff: bool): - """ + """ This test is to check `gettree` through non tar mode. bytar= True in this test. """ transport = firecrest_computer.get_transport() transport.payoff_override = payoff + # Note: + # SSH transport behaviour, 69 is a directory + # transport.get('somepath/69', 'someremotepath/') == transport.get('somepath/69', 'someremotepath') + # transport.get('somepath/69', 'someremotepath/') == transport.get('somepath/69/', 'someremotepath/') + # transport.get('someremotepath/69', 'somepath/69')--> if 69 exist, create 69 inside it ('somepath/69/69') + # transport.get('someremotepath/69', 'somepath/69')--> if 69 no texist,create 69 inside it ('somepath/69') + # transport.get('somepath/69', 'someremotepath/6889/69') --> create everything, make_parent = True + _remote = tmpdir / "remotedir" _local = tmpdir / "localdir" _remote.mkdir() @@ -235,22 +266,20 @@ def test_gettree(firecrest_computer: orm.Computer, tmpdir: Path, payoff: bool): Path(_remote / "dir2").mkdir() Path(_remote / "dir2" / "file3").write_text("file3") os.symlink(_remote / "file1", _remote / "dir1" / "file1_link") - os.symlink(_remote / "dir2", _remote / "dir1" / "dir2_link") + os.symlink(_remote / "dir2", _remote / "dir1" / "dir2_link") # if symlinks are pointing to a relative path os.symlink(Path("../file1"), _remote / "dir1" / "file10_link") - os.symlink(Path("../dir2" ), _remote / "dir1" / "dir20_link") - - + os.symlink(Path("../dir2"), _remote / "dir1" / "dir20_link") # raise if remote file does not exist with pytest.raises(OSError): transport.gettree(_remote / "does_not_exist", _local) - + # raise if local is a file Path(tmpdir / "isfile").touch() with pytest.raises(OSError): transport.gettree(_remote, tmpdir / "isfile") - + # raise if localpath is relative with pytest.raises(ValueError): transport.gettree(_remote, Path(_local).relative_to(tmpdir)) @@ -268,12 +297,11 @@ def test_gettree(firecrest_computer: orm.Computer, tmpdir: Path, payoff: bool): assert Path(_root / "dir1" / "dir2_link" / "file3").read_text() == "file3" assert Path(_root / "dir1" / "file10_link").read_text() == "file1" assert Path(_root / "dir1" / "dir20_link" / "file3").read_text() == "file3" - assert not Path(_root / "dir1" / "file1_link").is_symlink() + assert not Path(_root / "dir1" / "file1_link").is_symlink() assert not Path(_root / "dir1" / "dir2_link" / "file3").is_symlink() assert not Path(_root / "dir1" / "file10_link").is_symlink() assert not Path(_root / "dir1" / "dir20_link" / "file3").is_symlink() - # If destination directory does exists, AiiDA expects transport make _remote.name and write into it # however this might have changed in the newer versions of AiiDA ~ 2.6.0 (IDK) transport.gettree(_remote, _local / "newdir") @@ -288,14 +316,15 @@ def test_gettree(firecrest_computer: orm.Computer, tmpdir: Path, payoff: bool): assert Path(_root / "dir1" / "dir2_link" / "file3").read_text() == "file3" assert Path(_root / "dir1" / "file10_link").read_text() == "file1" assert Path(_root / "dir1" / "dir20_link" / "file3").read_text() == "file3" - assert not Path(_root / "dir1" / "file1_link").is_symlink() + assert not Path(_root / "dir1" / "file1_link").is_symlink() assert not Path(_root / "dir1" / "dir2_link" / "file3").is_symlink() assert not Path(_root / "dir1" / "file10_link").is_symlink() assert not Path(_root / "dir1" / "dir20_link" / "file3").is_symlink() + @pytest.mark.usefixtures("aiida_profile_clean") def test_put(firecrest_computer: orm.Computer, tmpdir: Path): - """ + """ This is minimal test is to check if put() is raising errors as expected, and directing to putfile() and puttree() as expected. Mainly just checking error handeling and folder creation. @@ -306,24 +335,23 @@ def test_put(firecrest_computer: orm.Computer, tmpdir: Path): _local = tmpdir / "localdir" _remote.mkdir() _local.mkdir() - # check if the code is directing to putfile() or puttree() as expected - with patch.object(transport, 'puttree', autospec=True) as mock_puttree: + with patch.object(transport, "puttree", autospec=True) as mock_puttree: transport.put(_local, _remote) mock_puttree.assert_called_once() - with patch.object(transport, 'puttree', autospec=True) as mock_puttree: + with patch.object(transport, "puttree", autospec=True) as mock_puttree: os.symlink(_local, tmpdir / "dir_link") transport.put(tmpdir / "dir_link", _remote) mock_puttree.assert_called_once() - with patch.object(transport, 'putfile', autospec=True) as mock_putfile: + with patch.object(transport, "putfile", autospec=True) as mock_putfile: Path(_local / "file1").write_text("file1") transport.put(_local / "file1", _remote / "file1") mock_putfile.assert_called_once() - with patch.object(transport, 'putfile', autospec=True) as mock_putfile: + with patch.object(transport, "putfile", autospec=True) as mock_putfile: os.symlink(_local / "file1", _local / "file1_link") transport.put(_local / "file1_link", _remote / "file1_link") mock_putfile.assert_called_once() @@ -332,13 +360,14 @@ def test_put(firecrest_computer: orm.Computer, tmpdir: Path): with pytest.raises(FileNotFoundError): transport.put(_local / "does_not_exist", _remote) transport.put(_local / "does_not_exist", _remote, ignore_nonexisting=True) - + # raise if localpath is relative with pytest.raises(ValueError): transport.put(Path(_local).relative_to(tmpdir), _remote) with pytest.raises(ValueError): transport.put(Path(_local / "file1").relative_to(tmpdir), _remote) + @pytest.mark.usefixtures("aiida_profile_clean") def test_putfile(firecrest_computer: orm.Computer, tmpdir: Path): transport = firecrest_computer.get_transport() @@ -351,11 +380,10 @@ def test_putfile(firecrest_computer: orm.Computer, tmpdir: Path): Path(_local / "file1").write_text("file1") Path(_local / ".hidden").write_text(".hidden") os.symlink(_local / "file1", _local / "file1_link") - # raise if local file does not exist with pytest.raises(FileNotFoundError): - transport.putfile(_local/ "does_not_exist" ,_remote) + transport.putfile(_local / "does_not_exist", _remote) # raise if remotefilename is not provided with pytest.raises(ValueError): @@ -368,13 +396,13 @@ def test_putfile(firecrest_computer: orm.Computer, tmpdir: Path): # don't mix up directory with file with pytest.raises(ValueError): transport.putfile(_local, _remote / "file1") - + # write where I tell you to transport.putfile(_local / "file1", _remote / "file1") transport.putfile(_local / "file1", _remote / "file1-prime") assert Path(_remote / "file1").read_text() == "file1" assert Path(_remote / "file1-prime").read_text() == "file1" - + # always overwrite transport.putfile(_local / "file1", _remote / "file1") assert Path(_remote / "file1").read_text() == "file1" @@ -393,7 +421,8 @@ def test_putfile(firecrest_computer: orm.Computer, tmpdir: Path): assert Path(_remote / "file1_link").read_text() == "file1" assert not Path(_remote / "file1_link").is_symlink() -@pytest.mark.parametrize("payoff", [True, False]) + +@pytest.mark.parametrize("payoff", [True, False]) @pytest.mark.usefixtures("aiida_profile_clean") def test_puttree(firecrest_computer: orm.Computer, tmpdir: Path, payoff: bool): """ @@ -403,6 +432,21 @@ def test_puttree(firecrest_computer: orm.Computer, tmpdir: Path, payoff: bool): transport = firecrest_computer.get_transport() transport.payoff_override = payoff + # Note: + # SSH transport behaviour + # transport.put('somepath/69', 'someremotepath/') == transport.put('somepath/69', 'someremotepath') + # transport.put('somepath/69', 'someremotepath/') != transport.put('somepath/69/', 'someremotepath/') + # transport.put('somepath/69', 'someremotepath/67') --> if 67 not exist, create and move content 69 + # inside it (someremotepath/67) + # transport.put('somepath/69', 'someremotepath/67') --> if 67 exist, create 69 inside it (someremotepath/67/69) + # transport.put('somepath/69', 'someremotepath/6889/69') --> useless Error: OSError + # Weired + # SSH bug: + # transport.put('somepath/69', 'someremotepath/') --> assuming someremotepath exists, make 69 + # while + # transport.put('somepath/69/', 'someremotepath/') --> assuming someremotepath exists, OSError: + # cannot make someremotepath + _remote = tmpdir / "remotedir" _local = tmpdir / "localdir" _remote.mkdir() @@ -417,23 +461,23 @@ def test_puttree(firecrest_computer: orm.Computer, tmpdir: Path, payoff: bool): # with symlinks to a file even if pointing to a relative path os.symlink(_local / "file1", _local / "dir1" / "file1_link") os.symlink(Path("../file1"), _local / "dir1" / "file10_link") - # with symlinks to a folder even if pointing to a relative path - os.symlink(_local / "dir2", _local / "dir1" / "dir2_link") - os.symlink(Path("../dir2" ), _local / "dir1" / "dir20_link") - + # with symlinks to a folder even if pointing to a relative path + os.symlink(_local / "dir2", _local / "dir1" / "dir2_link") + os.symlink(Path("../dir2"), _local / "dir1" / "dir20_link") + # raise if local file does not exist with pytest.raises(OSError): transport.puttree(_local / "does_not_exist", _remote) - + # raise if local is a file with pytest.raises(ValueError): Path(tmpdir / "isfile").touch() transport.puttree(tmpdir / "isfile", _remote) - + # raise if localpath is relative with pytest.raises(ValueError): transport.puttree(Path(_local).relative_to(tmpdir), _remote) - + # If destination directory does not exists, AiiDA expects transport make the new path as root not using _local.name transport.puttree(_local, _remote / "newdir") _root = _remote / "newdir" @@ -452,7 +496,6 @@ def test_puttree(firecrest_computer: orm.Computer, tmpdir: Path, payoff: bool): assert not Path(_root / "dir1" / "file10_link").is_symlink() assert not Path(_root / "dir1" / "dir20_link" / "file3").is_symlink() - # If destination directory does exists, AiiDA expects transport make _local.name and write into it # however this might have changed in the newer versions of AiiDA ~ 2.6.0 (IDK) transport.puttree(_local, _remote / "newdir") @@ -473,17 +516,15 @@ def test_puttree(firecrest_computer: orm.Computer, tmpdir: Path, payoff: bool): assert not Path(_root / "dir1" / "dir20_link" / "file3").is_symlink() -@pytest.mark.parametrize("to_test", ['copy', 'copytree']) +@pytest.mark.parametrize("to_test", ["copy", "copytree"]) @pytest.mark.usefixtures("aiida_profile_clean") def test_copy(firecrest_computer: orm.Computer, tmpdir: Path, to_test: str): - transport = firecrest_computer.get_transport() - if to_test == 'copy': + if to_test == "copy": testing = transport.copy - elif to_test == 'copytree': + elif to_test == "copytree": testing = transport.copytree - _remote_1 = tmpdir / "remotedir_1" _remote_2 = tmpdir / "remotedir_2" _remote_1.mkdir() @@ -495,9 +536,8 @@ def test_copy(firecrest_computer: orm.Computer, tmpdir: Path, to_test: str): with pytest.raises(FileNotFoundError): testing(_remote_1, _remote_2 / "does_not_exist") - # raise if source is inappropriate - if to_test == 'copytree': + if to_test == "copytree": Path(tmpdir / "file1").touch() with pytest.raises(ValueError): testing(tmpdir / "file1", _remote_2) @@ -512,20 +552,19 @@ def test_copy(firecrest_computer: orm.Computer, tmpdir: Path, to_test: str): # with symlinks to a file even if pointing to a relative path os.symlink(_remote_1 / "file1", _remote_1 / "dir1" / "file1_link") os.symlink(Path("../file1"), _remote_1 / "dir1" / "file10_link") - # with symlinks to a folder even if pointing to a relative path - os.symlink(_remote_1 / "dir2", _remote_1 / "dir1" / "dir2_link") - os.symlink(Path("../dir2" ), _remote_1 / "dir1" / "dir20_link") + # with symlinks to a folder even if pointing to a relative path + os.symlink(_remote_1 / "dir2", _remote_1 / "dir1" / "dir2_link") + os.symlink(Path("../dir2"), _remote_1 / "dir1" / "dir20_link") testing(_remote_1, _remote_2) - _root_2 = _remote_2 / Path(_remote_1).name # tree should be copied recursively assert Path(_root_2 / "dir1").exists() assert Path(_root_2 / "dir2").exists() assert Path(_root_2 / "file1").read_text() == "file1" assert Path(_root_2 / ".hidden").read_text() == ".hidden" - assert Path(_root_2 / "dir1" / "file2").read_text () == "file2" + assert Path(_root_2 / "dir1" / "file2").read_text() == "file2" assert Path(_root_2 / "dir2" / "file3").read_text() == "file3" # symlink should be followed assert Path(_root_2 / "dir1" / "dir2_link").exists() @@ -538,15 +577,11 @@ def test_copy(firecrest_computer: orm.Computer, tmpdir: Path, to_test: str): assert Path(_root_2 / "dir1" / "dir20_link").is_symlink() - - @pytest.mark.usefixtures("aiida_profile_clean") def test_copyfile(firecrest_computer: orm.Computer, tmpdir: Path): - transport = firecrest_computer.get_transport() testing = transport.copyfile - _remote_1 = tmpdir / "remotedir_1" _remote_2 = tmpdir / "remotedir_2" _remote_1.mkdir() @@ -568,10 +603,10 @@ def test_copyfile(firecrest_computer: orm.Computer, tmpdir: Path): Path(_remote_1 / ".hidden").write_text(".hidden") # with symlinks to a file even if pointing to a relative path os.symlink(_remote_1 / "file1", _remote_1 / "file1_link") - os.symlink(Path("file1"), _remote_1 / "file10_link") + os.symlink(Path("file1"), _remote_1 / "file10_link") # write where I tell you to - testing(_remote_1 /"file1", _remote_2 / "file1") + testing(_remote_1 / "file1", _remote_2 / "file1") assert Path(_remote_2 / "file1").read_text() == "file1" # always overwrite @@ -590,4 +625,3 @@ def test_copyfile(firecrest_computer: orm.Computer, tmpdir: Path): testing(_remote_1 / "file10_link", _remote_2 / "file10_link") assert Path(_remote_2 / "file10_link").read_text() == "file1" assert Path(_remote_2 / "file10_link").is_symlink() - From 6304a75cd89ecf85138d17f1ba69ee72a42267f7 Mon Sep 17 00:00:00 2001 From: Ali Date: Mon, 24 Jun 2024 18:36:55 +0200 Subject: [PATCH 09/39] Readme updated --- README.md | 16 +++++++--------- aiida_firecrest/transport.py | 28 +++++++++++++++++----------- 2 files changed, 24 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index f823a19..078dc0a 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ AiiDA Transport/Scheduler plugins for interfacing with [FirecREST](https://products.cscs.ch/firecrest/), via [pyfirecrest](https://github.com/eth-cscs/pyfirecrest). -It is currently tested against [FirecREST v1.13.0](https://github.com/eth-cscs/firecrest/releases/tag/v1.13.0). +It is currently tested against [FirecREST v2.4.0](https://github.com/eth-cscs/firecrest/releases/tag/v2.4.0). **NOTE:** This plugin is currently dependent on a fork of `aiida-core` from [PR #6043](https://github.com/aiidateam/aiida-core/pull/6043) @@ -70,6 +70,7 @@ Client ID: username-client Client Secret: xyz Client Machine: daint Maximum file size for direct transfer (MB) [5.0]: +Temp directory on server: /scratch/something/ Report: Configuring computer firecrest-client for user chrisj_sewell@hotmail.com. Success: firecrest-client successfully configured for chrisj_sewell@hotmail.com ``` @@ -100,15 +101,9 @@ See [tests/test_calculation.py](tests/test_calculation.py) for a working example ### Current Issues -Simple calculations are now running successfully [in the tests](tests/test_calculation.py), however, there are still some critical issues, before this could be production ready: +Calculations are now running successfully, however, there are still issues regarding efficency, Could be improved: -1. Currently uploading via firecrest changes `_aiidasubmit.sh` to `aiidasubmit.sh` 😱 ([see #191](https://github.com/eth-cscs/firecrest/issues/191)), so `metadata.options.submit_script_filename` should be set to this. - -2. Handling of large (>5Mb) file uploads/downloads needs to be improved - -3. Handling of the client secret, which should likely not be stored in the database - -4. Monitoring / management of API request rates could to be improved +1. Monitoring / management of API request rates could to be improved. Currently it leaves it to hand of PyFirecREST. ## Development @@ -135,6 +130,9 @@ Because of this, we have another set of tests that only verify the functionality #### Mocking FirecREST server +These tests were successful against [FirecREST v1.13.0](https://github.com/eth-cscs/firecrest/releases/tag/v1.13.0). +For newer version please refer to tests Mocking PyFirecREST + It is recommended to run the tests via [tox](https://tox.readthedocs.io/en/latest/). ```bash diff --git a/aiida_firecrest/transport.py b/aiida_firecrest/transport.py index 620d8d7..e8f2e6a 100644 --- a/aiida_firecrest/transport.py +++ b/aiida_firecrest/transport.py @@ -42,15 +42,12 @@ def _create_secret_file(ctx: Context, param: InteractiveOption, value: str) -> s """Create a secret file if the value is not a path to a secret file. The path should be absolute, if it is not, the file will be created in ~/.firecrest. """ - import uuid - - from aiida.cmdline.utils import echo - from click import BadParameter + import click possible_path = Path(value) if os.path.isabs(possible_path): if not possible_path.exists(): - raise BadParameter(f"Secret file not found at {value}") + raise click.BadParameter(f"Secret file not found at {value}") secret_path = possible_path else: @@ -61,8 +58,10 @@ def _create_secret_file(ctx: Context, param: InteractiveOption, value: str) -> s # instead of a random number one could use the label or pk of the computer being configured secret_path = Path(f"~/.firecrest/secret_{_}").expanduser() secret_path.write_text(value) - echo.echo_report(f"Secret file created at {secret_path}") - echo.echo_report(f"Client Secret stored at {secret_path}") + click.echo( + click.style("Fireport: ", bold=True, fg="magenta") + + f"Client Secret stored at {secret_path}" + ) return str(secret_path) @@ -125,9 +124,13 @@ def _validate_temp_directory(ctx: Context, param: InteractiveOption, value: str) try: dummy.mkdir(dummy._temp_directory, ignore_existing=True) except Exception as e: - raise OSError( + raise click.BadParameter( f"Could not create temp directory {dummy._temp_directory} on server: {e}" ) from e + click.echo( + click.style("Fireport: ", bold=True, fg="magenta") + + f"Temp directory is set to {value}" + ) return value @@ -147,7 +150,7 @@ def _dynamic_info_direct_size( :return: the value of small_file_size_mb. """ - from aiida.cmdline.utils import echo + import click if value > 0: return value @@ -182,7 +185,10 @@ def _dynamic_info_direct_size( if utilities_max_file_size is not None else 5.0 ) - echo.echo_report(f"Maximum file size for direct transfer: {small_file_size_mb} MB") + click.echo( + click.style("Fireport: ", bold=True, fg="magenta") + + f"Maximum file size for direct transfer: {small_file_size_mb} MB" + ) return small_file_size_mb @@ -264,7 +270,7 @@ class FirecrestTransport(Transport): { "type": str, "non_interactive_default": False, - "prompt": "Please enter a temp directory on server", + "prompt": "Temp directory on server", "help": "A temp directory on server for creating temporary files (compression, extraction, etc.)", "callback": _validate_temp_directory, }, From 202e14fdc9133ee9771502542563597436aa6fe6 Mon Sep 17 00:00:00 2001 From: Ali Date: Tue, 25 Jun 2024 09:36:41 +0200 Subject: [PATCH 10/39] `_copy_to` moved from path.py directly to interface + updated aiida-core dependency --- .pre-commit-config.yaml | 2 +- aiida_firecrest/transport.py | 17 ++++++++++++++--- pyproject.toml | 2 +- 3 files changed, 16 insertions(+), 5 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 0601441..ea409d7 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -45,4 +45,4 @@ repos: - "types-PyYAML" - "types-requests" - "pyfirecrest~=2.5.0" - - "aiida-core~=2.4.0" + - "aiida-core~=2.5.1.post0" diff --git a/aiida_firecrest/transport.py b/aiida_firecrest/transport.py index e8f2e6a..9ba145f 100644 --- a/aiida_firecrest/transport.py +++ b/aiida_firecrest/transport.py @@ -502,10 +502,21 @@ def copyfile( if not destination.exists() and not source.is_file(): raise FileNotFoundError(f"Destination file does not exist: {destination}") - source.copy_to(destination) + self._copy_to(source, destination) # I removed symlink copy, becasue it's really not a file copy, it's a link copy # and aiida-ssh have it in buggy manner, prrobably it's not used anyways + def _copy_to(self, source: FcPath, target: FcPath) -> None: + """Copy source path to the target path. Both paths must be on remote. + + Works for both files and directories (in which case the whole tree is copied). + """ + with self._cwd.convert_header_exceptions(): + # Note although this endpoint states that it is only for directories, + # it actually uses `cp -r`: + # https://github.com/eth-cscs/firecrest/blob/7f02d11b224e4faee7f4a3b35211acb9c1cc2c6a/src/utilities/utilities.py#L320 + self._client.copy(self._machine, str(source), str(target)) + def copytree( self, remotesource: str, remotedestination: str, dereference: bool = False ) -> None: @@ -532,7 +543,7 @@ def copytree( if not destination.exists(): raise FileNotFoundError(f"Destination file does not exist: {destination}") - source.copy_to(destination) + self._copy_to(source, destination) def copy( self, @@ -570,7 +581,7 @@ def copy( if not destination.exists() and not source.is_file(): raise FileNotFoundError(f"Destination does not exist: {destination}") - source.copy_to(destination) + self._copy_to(source, destination) # TODO do get/put methods need to handle glob patterns? # Apparently not, but I'm not clear how glob() iglob() are going to behave here. We may need to implement them. diff --git a/pyproject.toml b/pyproject.toml index ffcf89f..67db52b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -24,7 +24,7 @@ classifiers = [ keywords = ["aiida", "firecrest"] requires-python = ">=3.9" dependencies = [ - "aiida-core@git+https://github.com/chrisjsewell/aiida_core.git@aiida-firecrest#egg=aiida-core", + "aiida-core@git+https://github.com/khsrali/aiida-core.git@aiida-firecrest#egg=aiida-core", "click", "pyfirecrest~=2.5.0", "pyyaml", From 3c3773f8a19c0d6e4281933aa4a8193c0244392e Mon Sep 17 00:00:00 2001 From: Ali Date: Wed, 26 Jun 2024 17:56:03 +0200 Subject: [PATCH 11/39] property _is_open added to skip AttributeError raised by aiida-core --- aiida_firecrest/scheduler.py | 6 +++--- aiida_firecrest/transport.py | 17 ++++++++++++++++- 2 files changed, 19 insertions(+), 4 deletions(-) diff --git a/aiida_firecrest/scheduler.py b/aiida_firecrest/scheduler.py index 59c9310..d5376b3 100644 --- a/aiida_firecrest/scheduler.py +++ b/aiida_firecrest/scheduler.py @@ -319,7 +319,7 @@ def get_jobs( except ValueError: self.logger.warning( - f"Error parsing the time limit for job id {this_job.job_id}" + f"Couldn't parse the time limit for job id {this_job.job_id}" ) # Only if it is RUNNING; otherwise it is not meaningful, @@ -335,7 +335,7 @@ def get_jobs( this_job.wallclock_time_seconds = 0 except ValueError: self.logger.warning( - f"Error parsing time_used for job id {this_job.job_id}" + f"Couldn't parse time_used for job id {this_job.job_id}" ) # TODO: The block below is commented, because dispatch_time @@ -350,7 +350,7 @@ def get_jobs( this_job.submission_time = self._parse_time_string(raw_result["time"]) except ValueError: self.logger.warning( - f"Error parsing submission_time for job id {this_job.job_id}" + f"Couldn't parse submission_time for job id {this_job.job_id}" ) this_job.title = raw_result["name"] diff --git a/aiida_firecrest/transport.py b/aiida_firecrest/transport.py index 9ba145f..8154372 100644 --- a/aiida_firecrest/transport.py +++ b/aiida_firecrest/transport.py @@ -341,10 +341,19 @@ def __init__( self._cwd: FcPath = FcPath(self._client, self._machine, "/", cache_enabled=True) self._temp_directory = self._cwd.joinpath(temp_directory) + # this makes no sense for firecrest, but we need to set this to True + # otherwise the aiida-core will complain that the transport is not open: + # aiida-core/src/aiida/orm/utils/remote:clean_remote() + self._is_open = True + def __str__(self) -> str: """Return the name of the plugin.""" return self.__class__.__name__ + @property + def is_open(self) -> bool: + return self._is_open + @property def payoff_override(self) -> bool | None: return self._payoff_override @@ -895,7 +904,12 @@ def payoff(self, path: str | FcPath | Path) -> bool: if self.payoff_override is not None: return bool(self.payoff_override) - if len(self.listdir(str(path), recursive=True)) > 3: + if ( + isinstance(path, FcPath) + and len(self.listdir(str(path), recursive=True)) > 3 + ): + return True + if isinstance(path, Path) and len(os.listdir(path)) > 3: return True return False @@ -1070,6 +1084,7 @@ def gotocomputer_command(self, remotedir: str) -> str: """Not possible for REST-API. It's here only because it's an abstract method in the base class.""" # TODO remove from interface + print(f"Trying to go for {remotedir}") raise NotImplementedError("firecrest does not support gotocomputer_command") def _exec_command_internal(self, command: str, **kwargs: Any) -> Any: From 96fe28687f716e95714cf0502efb2a58dcfca61e Mon Sep 17 00:00:00 2001 From: Ali Date: Thu, 27 Jun 2024 13:29:53 +0200 Subject: [PATCH 12/39] bug fix: when get_job() is retiereving a completed job bug fix: iglob() is sending relative path to listdir() compatibility with new version of firecrest: extract() and compress() will take care of submission if needed --- aiida_firecrest/scheduler.py | 6 ++++- aiida_firecrest/transport.py | 45 +++++++++++------------------------- 2 files changed, 18 insertions(+), 33 deletions(-) diff --git a/aiida_firecrest/scheduler.py b/aiida_firecrest/scheduler.py index d5376b3..3ed2062 100644 --- a/aiida_firecrest/scheduler.py +++ b/aiida_firecrest/scheduler.py @@ -213,6 +213,7 @@ def get_jobs( ) -> list[JobInfo] | dict[str, JobInfo]: results = [] transport = self.transport + with convert_header_exceptions({"machine": transport._machine}): # TODO handle pagination (pageSize, pageNumber) if many jobs # This will do pagination @@ -224,7 +225,10 @@ def get_jobs( if len(results) < self._DEFAULT_PAGE_SIZE * (page_iter + 1): break except FirecrestException as exc: - raise SchedulerError(str(exc)) from exc + # firecrest returns error if the job is completed + # TODO: check what type of error is returned and handle it properly + if "Invalid job id specified" not in str(exc): + raise SchedulerError(str(exc)) from exc job_list = [] for raw_result in results: # TODO: probably the if below is not needed, because recently diff --git a/aiida_firecrest/transport.py b/aiida_firecrest/transport.py index 8154372..c3b1237 100644 --- a/aiida_firecrest/transport.py +++ b/aiida_firecrest/transport.py @@ -8,7 +8,6 @@ from pathlib import Path import posixpath import tarfile -import time from typing import Any, Callable, ClassVar, TypedDict import uuid @@ -441,6 +440,9 @@ def listdir( ) -> list[str]: """List the contents of a directory. + :param path: this could be relative or absolute path. + Note igolb() will usually call this with relative path. + :param pattern: Unix shell-style wildcards to match the pattern: - `*` matches everything - `?` matches any single character @@ -448,9 +450,10 @@ def listdir( - `[!seq]` matches any character not in seq :param recursive: If True, list directories recursively """ + path_abs = self._cwd.joinpath(path) names = [ - p.relpath(path).as_posix() - for p in self._cwd.joinpath(path).iterdir(recursive=recursive) + p.relpath(path_abs).as_posix() + for p in path_abs.iterdir(recursive=recursive) ] if pattern is not None: names = fnmatch.filter(names, pattern) @@ -609,7 +612,6 @@ def getfile( note: we don't support downloading symlinks, so dereference should always be True """ - if not dereference: raise NotImplementedError( "Getting symlinks with `dereference=False` is not supported" @@ -705,25 +707,12 @@ def _gettreetar( note: FirecREST doesn't support `--dereference` for tar call, so dereference should always be False, for now. """ - # TODO manual testing the submit behaviour - - # if dereference: - # raise NotImplementedError("Dereferencing compression not implemented in pyFirecREST.") - _ = uuid.uuid4() # Attempt direct compress + _ = uuid.uuid4() remote_path_temp = self._temp_directory.joinpath(f"temp_{_}.tar") - try: - self._client.compress(self._machine, str(remotepath), remote_path_temp) - except Exception as e: - # TODO: pyfirecrest is providing a solution to this, but it's not yet merged. - # once done submit_compress_job should be done automaticaly by compress - # see: https://github.com/eth-cscs/pyfirecrest/pull/109 - raise NotImplementedError("Not implemeted for now") from e - comp_obj = self._client.submit_compress_job( - self._machine, str(remotepath), remote_path_temp - ) - while comp_obj.in_progress: - time.sleep(self._file_transfer_poll_interval) + + # Compress + self._client.compress(self._machine, str(remotepath), remote_path_temp) # Download localpath_temp = Path(localpath).joinpath(f"temp_{_}.tar") @@ -756,6 +745,7 @@ def gettree( :param dereference: If True, follow symlinks. note: dereference should be always True, otherwise the symlinks will not be functional. """ + local = Path(localpath) if not local.is_absolute(): raise ValueError("Destination must be an absolute path") @@ -947,19 +937,10 @@ def _puttreetar( self.putfile(tarpath, remote_path_temp) finally: tarpath.unlink() - # Attempt direct extract + + # Attempt extract try: self._client.extract(self._machine, remote_path_temp, str(remotepath)) - except Exception as e: - # TODO: pyfirecrest is providing a solution to this, but it's not yet merged - # once done submit_compress_job should be done automaticaly by compress - # see: https://github.com/eth-cscs/pyfirecrest/pull/109 - raise NotImplementedError("Not implemeted for now") from e - comp_obj = self._client.submit_extract_job( - self._machine, remotepath.joinpath(f"_{_}.tar"), str(remotepath) - ) - while comp_obj.in_progress: - time.sleep(self._file_transfer_poll_interval) finally: self.remove(remote_path_temp) From 0d45d6bd68c4c737aabb7bfc5a57b32f23c3e106 Mon Sep 17 00:00:00 2001 From: Ali Khosravi Date: Tue, 2 Jul 2024 12:17:03 +0200 Subject: [PATCH 13/39] Update FirecREST token url --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 078dc0a..c495665 100644 --- a/README.md +++ b/README.md @@ -65,7 +65,7 @@ $ verdi -p quicksetup computer configure firecrest firecrest-client Report: enter ? for help. Report: enter ! to ignore the default and set no value. Server URL: https://firecrest.cscs.ch -Token URI: https://auth.cscs.ch/auth/realms/cscs/protocol/openid-connect/token +Token URI: https://auth.cscs.ch/auth/realms/firecrest-clients/protocol/openid-connect/token Client ID: username-client Client Secret: xyz Client Machine: daint From 00c4be92922adcc5c2437fd535cfa06cd510ad09 Mon Sep 17 00:00:00 2001 From: Miki Bonacci <46074008+mikibonacci@users.noreply.github.com> Date: Tue, 2 Jul 2024 12:20:21 +0200 Subject: [PATCH 14/39] Update README.md with correct Token URI (#1) From f7519a4478a70f786b63d164afc177202d7ff265 Mon Sep 17 00:00:00 2001 From: Ali Date: Fri, 5 Jul 2024 11:52:14 +0200 Subject: [PATCH 15/39] added support for glob patterns in --- aiida_firecrest/transport.py | 37 ++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/aiida_firecrest/transport.py b/aiida_firecrest/transport.py index c3b1237..2963db0 100644 --- a/aiida_firecrest/transport.py +++ b/aiida_firecrest/transport.py @@ -581,6 +581,19 @@ def copy( raise NotImplementedError( "Dereferencing not implemented in FirecREST server" ) + + if self.has_magic(remotesource): # type: ignore + for item in self.iglob(remotesource): # type: ignore + # item is of str type, so we need to split it to get the file name + filename = item.split("/")[-1] if self.isfile(item) else "" + self.copy( + item, + remotedestination + filename, + dereference=dereference, + recursive=recursive, + ) + return + source = self._cwd.joinpath( remotesource ) # .enable_cache() it's removed from from path.py to be investigated @@ -820,6 +833,17 @@ def get( self.gettree(remote, localpath) elif remote.is_file(): self.getfile(remote, localpath) + elif self.has_magic(remotepath): # type: ignore + for item in self.iglob(remotepath): # type: ignore + # item is of str type, so we need to split it to get the file name + filename = item.split("/")[-1] if self.isfile(item) else "" + self.get( + item, + localpath + filename, + dereference=dereference, + ignore_nonexisting=ignore_nonexisting, + ) + return elif not ignore_nonexisting: raise FileNotFoundError(f"Source file does not exist: {remote}") @@ -1021,6 +1045,19 @@ def put( local = Path(localpath) if not local.is_absolute(): raise ValueError("The localpath must be an absolute path") + + if self.has_magic(localpath): # type: ignore + for item in self.iglob(localpath): # type: ignore + # item is of str type, so we need to split it to get the file name + filename = item.split("/")[-1] if self.isfile(item) else "" + self.put( + item, + remotepath + filename, + dereference=dereference, + ignore_nonexisting=ignore_nonexisting, + ) + return + if not Path(local).exists() and not ignore_nonexisting: raise FileNotFoundError(f"Source file does not exist: {localpath}") From 8ebe53120c9962151aef635f8ae929f7f5233ecf Mon Sep 17 00:00:00 2001 From: Ali Date: Sun, 14 Jul 2024 22:53:25 +0200 Subject: [PATCH 16/39] Enforce str for super method `has_magic()` --- aiida_firecrest/transport.py | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/aiida_firecrest/transport.py b/aiida_firecrest/transport.py index 2963db0..9384104 100644 --- a/aiida_firecrest/transport.py +++ b/aiida_firecrest/transport.py @@ -451,10 +451,7 @@ def listdir( :param recursive: If True, list directories recursively """ path_abs = self._cwd.joinpath(path) - names = [ - p.relpath(path_abs).as_posix() - for p in path_abs.iterdir(recursive=recursive) - ] + names = [p.relpath(path_abs) for p in path_abs.iterdir(recursive=recursive)] if pattern is not None: names = fnmatch.filter(names, pattern) return names @@ -582,7 +579,7 @@ def copy( "Dereferencing not implemented in FirecREST server" ) - if self.has_magic(remotesource): # type: ignore + if self.has_magic(str(remotesource)): # type: ignore for item in self.iglob(remotesource): # type: ignore # item is of str type, so we need to split it to get the file name filename = item.split("/")[-1] if self.isfile(item) else "" @@ -833,7 +830,7 @@ def get( self.gettree(remote, localpath) elif remote.is_file(): self.getfile(remote, localpath) - elif self.has_magic(remotepath): # type: ignore + elif self.has_magic(str(remotepath)): # type: ignore for item in self.iglob(remotepath): # type: ignore # item is of str type, so we need to split it to get the file name filename = item.split("/")[-1] if self.isfile(item) else "" @@ -1046,7 +1043,7 @@ def put( if not local.is_absolute(): raise ValueError("The localpath must be an absolute path") - if self.has_magic(localpath): # type: ignore + if self.has_magic(str(localpath)): # type: ignore for item in self.iglob(localpath): # type: ignore # item is of str type, so we need to split it to get the file name filename = item.split("/")[-1] if self.isfile(item) else "" From e493fc5d8f9731e2f17bda04d29713f395e63c4a Mon Sep 17 00:00:00 2001 From: Ali Khosravi Date: Mon, 15 Jul 2024 12:08:30 +0200 Subject: [PATCH 17/39] Apply suggestions from code review Co-authored-by: Rico Haeuselmann --- README.md | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index c495665..ec6b928 100644 --- a/README.md +++ b/README.md @@ -103,7 +103,7 @@ See [tests/test_calculation.py](tests/test_calculation.py) for a working example Calculations are now running successfully, however, there are still issues regarding efficency, Could be improved: -1. Monitoring / management of API request rates could to be improved. Currently it leaves it to hand of PyFirecREST. +1. Monitoring / management of API request rates could to be improved. Currently this is left up to PyFirecREST. ## Development @@ -124,7 +124,7 @@ pre-commit run --all-files ### Testing There are two types of tests: mocking the PyFirecREST or the FirecREST server. -While the former is a good practice to ensure that all three (`aiida-firecrest`, FirecREST, and PyFirecREST) work flawlessly, debugging may not always be easy because it may not always be obvious which of the three is causing a bug. +While the latter is a good practice to ensure that all three (`aiida-firecrest`, FirecREST, and PyFirecREST) work flawlessly, debugging may not always be easy because it may not always be obvious which of the three is causing a bug. Because of this, we have another set of tests that only verify the functionality of `aiida-firecrest` by directly mocking PyFirecREST. Maintaining the second set in `tests/tests_mocking_pyfirecrest/` is simpler because we just need to monitor the return values of PyFirecREST​. While maintaining the former is more difficult as you have to keep up with both FirecREST and PyFirecREST. @@ -205,8 +205,13 @@ although it is of note that you can find these files directly where you your `fi #### Mocking PyFirecREST -These set of test do not gurantee that the firecrest protocol is working, but it's very usefull to quickly check if `aiida-firecrest` is behaving as it's expected to do. To run just simply use `pytest`. +These set of test do not gurantee that the firecrest protocol is working, but it's very useful to quickly check if `aiida-firecrest` is behaving as it's expected to do. To run just simply use `pytest`. If these tests, pass and still you have trouble in real deploymeny that means your installed version of pyfirecrest is behaving differently from what `aiida-firecrest` expects in `MockFirecrest` in `tests/tests_mocking_pyfirecrest/conftest.py`. -In order to solve that, first spot what is different and then solve or raise to `pyfirecrest` if relevant. +If there is no version of `aiida-firecrest` available that supports your `pyfirecrest` version and if down/upgrading your `pyfirecrest` to a supported version is not an option, you might try the following: +- open an issue on the `aiida-firecrest` repository on GitHub to request supporting your version of pyfirecrest +- if you feel up to finding the discrepancy and fixing it within `aiida-firecrest`, open a PR instead +- if you think the problem is a bug in `pyfirecrest`, open an issue there + +Either way, make sure to report which version of `aiida-firecrest` and `pyfirecrest` you are using. From c83a89386d89aa14840d4a581faacf57296e1b3b Mon Sep 17 00:00:00 2001 From: Ali Date: Mon, 15 Jul 2024 14:30:02 +0200 Subject: [PATCH 18/39] Updated tests --- .firecrest-demo-config.json | 8 - .github/workflows/server-tests.yml | 96 --- .github/workflows/tests.yml | 2 +- aiida_firecrest/utils_test.py | 815 ------------------ firecrest_demo.py | 86 -- pyproject.toml | 2 +- tests/conftest.py | 469 +++++++--- tests/test_calculation.py | 183 ---- tests/test_computer.py | 158 ++-- tests/test_scheduler.py | 145 ++-- .../test_scheduler/test_write_script_full.txt | 20 - .../test_write_script_minimal.txt | 5 - tests/test_transport.py | 738 +++++++++++++--- tests/tests_mocking_pyfirecrest/conftest.py | 369 -------- .../test_computer.py | 121 --- .../test_scheduler.py | 139 --- .../test_transport.py | 627 -------------- 17 files changed, 1153 insertions(+), 2830 deletions(-) delete mode 100644 .firecrest-demo-config.json delete mode 100644 .github/workflows/server-tests.yml delete mode 100644 aiida_firecrest/utils_test.py delete mode 100644 firecrest_demo.py delete mode 100644 tests/test_calculation.py delete mode 100644 tests/test_scheduler/test_write_script_full.txt delete mode 100644 tests/test_scheduler/test_write_script_minimal.txt delete mode 100644 tests/tests_mocking_pyfirecrest/conftest.py delete mode 100644 tests/tests_mocking_pyfirecrest/test_computer.py delete mode 100644 tests/tests_mocking_pyfirecrest/test_scheduler.py delete mode 100644 tests/tests_mocking_pyfirecrest/test_transport.py diff --git a/.firecrest-demo-config.json b/.firecrest-demo-config.json deleted file mode 100644 index 9479e1d..0000000 --- a/.firecrest-demo-config.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "url": "http://localhost:8000/", - "token_uri": "http://localhost:8080/auth/realms/kcrealm/protocol/openid-connect/token", - "client_id": "firecrest-sample", - "client_secret": "b391e177-fa50-4987-beaf-e6d33ca93571", - "machine": "cluster", - "scratch_path": "/home/service-account-firecrest-sample" -} diff --git a/.github/workflows/server-tests.yml b/.github/workflows/server-tests.yml deleted file mode 100644 index 8b41a3f..0000000 --- a/.github/workflows/server-tests.yml +++ /dev/null @@ -1,96 +0,0 @@ -# Run pytest against an actual FirecREST server, -# rather than just a mock server. - -name: Server - -on: - push: - branches: [main] - tags: - - 'v*' - pull_request: - paths-ignore: - - README.md - - CHANGELOG.md - - "docs/**" - -jobs: - - tests: - - runs-on: ${{ matrix.os }} - strategy: - fail-fast: false - matrix: - os: ["ubuntu-latest"] - python-version: ["3.9"] - firecrest-version: ["v1.13.0"] - - services: - rabbitmq: - image: rabbitmq:3.8.14-management - ports: - - 5672:5672 - - 15672:15672 - - steps: - - uses: actions/checkout@v3 - - - name: checkout the firecrest repository - uses: actions/checkout@v3 - with: - repository: eth-cscs/firecrest - ref: ${{ matrix.firecrest-version }} - path: .demo-server - - - name: Cache Docker images - uses: jpribyl/action-docker-layer-caching@v0.1.1 - continue-on-error: true - with: - key: ${{ runner.os }}-docker-${{ matrix.firecrest-version }} - - # note, for some reason, the certificator image fails to build - # if you build them in order, so here we build everything except that first - # and then it seems to work - - name: Build the FirecREST images - run: | - docker-compose build f7t-base - docker-compose build compute - docker-compose build status - docker-compose build storage - docker-compose build tasks - docker-compose build utilities - docker-compose build reservations - docker-compose build client - docker-compose build cluster - docker-compose build keycloak - docker-compose build kong - docker-compose build minio - docker-compose build taskpersistence - docker-compose build opa - docker-compose build openapi - docker-compose build jaeger - - # docker-compose build certificator - working-directory: .demo-server/deploy/demo - - - name: Ensure permissions of SSH Keys - run: | - chmod 400 .demo-server/deploy/test-build/environment/keys/ca-key - chmod 400 .demo-server/deploy/test-build/environment/keys/user-key - - - name: Start the FirecREST server - run: docker-compose up --detach - working-directory: .demo-server/deploy/demo - - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.python-version }} - cache: 'pip' - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install -e .[dev] - - name: Test with pytest - run: pytest -vv --cov=aiida_firecrest --firecrest-config .firecrest-demo-config.json diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index e0ed17a..4f260ea 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -49,7 +49,7 @@ jobs: python -m pip install --upgrade pip pip install -e .[dev] - name: Test with pytest - run: pytest -vv --firecrest-requests --cov=aiida_firecrest --cov-report=xml --cov-report=term + run: pytest -vv --cov=aiida_firecrest --cov-report=xml --cov-report=term - name: Upload coverage reports to Codecov uses: codecov/codecov-action@v3 diff --git a/aiida_firecrest/utils_test.py b/aiida_firecrest/utils_test.py deleted file mode 100644 index 27394e1..0000000 --- a/aiida_firecrest/utils_test.py +++ /dev/null @@ -1,815 +0,0 @@ -"""Utilities mainly for testing.""" -from __future__ import annotations - -from dataclasses import dataclass, field -from datetime import datetime -import io -from json import dumps as json_dumps -from pathlib import Path -import shutil -import stat -import subprocess -from typing import Any, BinaryIO -from urllib.parse import urlparse - -import requests -from requests.models import Response - - -@dataclass -class FirecrestConfig: - """Configuration returned from tests fixtures.""" - - url: str - token_uri: str - client_id: str - client_secret: str - machine: str - scratch_path: str - temp_directory: str - small_file_size_mb: float = 1.0 - - -class FirecrestMockServer: - """A mock server to imitate Firecrest (v1.13.0). - - This minimally mimics accessing the filesystem and submitting jobs, - enough to make tests pass, without having to run a real Firecrest server. - - See also: https://github.com/eth-cscs/pyfirecrest/blob/4dadce90d7fb01949d203a5b1d2c247048a5a3a9/tests/test_storage.py - """ - - def __init__( - self, tmpdir: Path, url: str = "https://test.com", machine: str = "test" - ) -> None: - self._url = url - self._url_parsed = urlparse(url) - self._machine = machine - self._scratch = tmpdir / "scratch" - self._scratch.mkdir() - self._client_id = "test_client_id" - - Path(tmpdir / ".firecrest").mkdir() - self._client_secret = tmpdir / ".firecrest/secret" - self._client_secret.write_text("test_client_secret") - - self._token_url = "https://test.auth.com/token" - self._token_url_parsed = urlparse(self._token_url) - self._username = "test_user" - self._temp_directory = tmpdir / "temp" - - self._slurm_job_id_counter = 0 - self._slurm_jobs: dict[str, dict[str, Any]] = {} - - self._task_id_counter = 0 - self._tasks: dict[str, Task] = {} - - self.max_size_bytes = 5 * 1024 * 1024 - - @property - def scratch(self) -> Path: - return self._scratch - - @property - def config(self) -> FirecrestConfig: - return FirecrestConfig( - url=self._url, - token_uri=self._token_url, - client_id=self._client_id, - client_secret=str(self._client_secret.absolute()), - machine=self._machine, - scratch_path=str(self._scratch.absolute()), - temp_directory=str(self._temp_directory.absolute()), - ) - - def mock_request( - self, - url: str | bytes, - params: dict[str, Any] | None = None, - data: dict[str, Any] | None = None, - files: dict[str, tuple[str, BinaryIO]] | None = None, - **kwargs: Any, - ) -> Response: - """Mock a request to the Firecrest server.""" - response = Response() - response.encoding = "utf-8" - response.url = url if isinstance(url, str) else url.decode("utf-8") - parsed = urlparse(response.url) - endpoint = parsed.path - - if parsed.netloc == self._token_url_parsed.netloc: - if endpoint != "/token": - raise requests.exceptions.InvalidURL(f"Unknown endpoint: {endpoint}") - response.status_code = 200 - response.raw = io.BytesIO( - json_dumps( - { - "access_token": "test_access_token", - "expires_in": 3600, - } - ).encode(response.encoding) - ) - return response - - if parsed.netloc != self._url_parsed.netloc: - raise requests.exceptions.InvalidURL( - f"{parsed.netloc} != {self._url_parsed.netloc}" - ) - - if endpoint == "/utilities/whoami": - add_success_response(response, 200, self._username) - elif endpoint == "/utilities/stat": - self.utilities_stat(params or {}, response) - elif endpoint == "/utilities/file": - self.utilities_file(params or {}, response) - elif endpoint == "/utilities/ls": - self.utilities_ls(params or {}, response) - elif endpoint == "/utilities/checksum": - self.utilities_checksum(params or {}, response) - elif endpoint == "/utilities/symlink": - self.utilities_symlink(data or {}, response) - elif endpoint == "/utilities/mkdir": - self.utilities_mkdir(data or {}, response) - elif endpoint == "/utilities/rm": - self.utilities_rm(data or {}, response) - elif endpoint == "/utilities/copy": - self.utilities_copy(data or {}, response) - elif endpoint == "/utilities/chmod": - self.utilities_chmod(data or {}, response) - # elif endpoint == "/utilities/chown": - # utilities_chown(data or {}, response) - elif endpoint == "/utilities/rename": - self.utilities_rename(data or {}, response) - elif endpoint == "/utilities/upload": - self.utilities_upload(data or {}, files or {}, response) - elif endpoint == "/utilities/download": - self.utilities_download(params or {}, response) - elif endpoint == "/compute/jobs/path": - self.compute_jobs_path(data or {}, response) - elif endpoint == "/compute/jobs": - self.compute_jobs(params or {}, response) - elif endpoint.startswith("/tasks"): - self.handle_task(params or {}, response) - # self.handle_task(endpoint[7:], response) - elif endpoint == "/storage/xfer-external/upload": - self.storage_xfer_external_upload(data or {}, response) - elif endpoint == "/storage/xfer-external/download": - self.storage_xfer_external_download(data or {}, response) - else: - raise requests.exceptions.InvalidURL(f"Unknown endpoint: {endpoint}") - - return response - - def new_task_id(self) -> str: - self._task_id_counter += 1 - return f"{self._task_id_counter}" - - def task_url(self, task_id: str) -> str: - return f"TASK_IP/tasks/{task_id}" - - def new_slurm_job_id(self) -> str: - """Generate a new SLURM job ID.""" - self._slurm_job_id_counter += 1 - return f"{self._slurm_job_id_counter}" - - def compute_jobs(self, params: dict[str, Any], response: Response) -> None: - # TODO pageSize pageNumber - jobs: None | list[str] = params["jobs"].split(",") if "jobs" in params else None - task_id = self.new_task_id() - self._tasks[task_id] = ActiveSchedulerJobsTask(task_id=task_id, jobs=jobs) - add_json_response( - response, - 200, - { - "success": "Task created", - "task_id": task_id, - "task_url": self.task_url(task_id), - }, - ) - - def compute_jobs_path(self, data: dict[str, Any], response: Response) -> Response: - script_path = Path(data["targetPath"]) - if not script_path.is_file(): - return add_json_response( - response, - 400, - {"description": "Failed to submit job", "data": "File does not exist"}, - {"X-Invalid-Path": f"{script_path} is an invalid path."}, - ) - - job_id = self.new_slurm_job_id() - - # read the file - script_content = script_path.read_text("utf8") - - # TODO this could be more rigorous - - # check that the first line is a shebang - if not script_content.startswith("#!/bin/bash"): - return add_json_response( - response, - 400, - { - "description": "Finished with errors", - "data": "First line must be a shebang `#!/bin/bash`", - }, - ) - - # get all sbatch options (see https://slurm.schedmd.com/sbatch.html) - sbatch_options: dict[str, Any] = {} - - for line in script_content.splitlines(): - if not line.startswith("#SBATCH"): - continue - arg = line[7:].strip().split("=", 1) - if len(arg) == 1: - assert arg[0].startswith("--"), f"Invalid sbatch option: {arg[0]}" - sbatch_options[arg[0][2:]] = True - elif len(arg) == 2: - assert arg[0].startswith("--"), f"Invalid sbatch option: {arg[0]}" - sbatch_options[arg[0][2:]] = arg[1].strip() - - # set stdout and stderror file - out_file = error_file = "slurm-%j.out" - if "output" in sbatch_options: - out_file = sbatch_options["output"] - if "error" in sbatch_options: - error_file = sbatch_options["error"] - out_file = out_file.replace("%j", job_id) - error_file = error_file.replace("%j", job_id) - - # we now just run the job straight away and blocking, no scheduling - # run the script in a subprocess, in the script's directory - # pipe stdout and stderr to the slurm output file - script_path.chmod(0o755) # make sure the script is executable - if out_file == error_file: - with open(script_path.parent / out_file, "w") as out: - subprocess.run( - [str(script_path)], - cwd=script_path.parent, - stdout=out, - stderr=subprocess.STDOUT, - ) - else: - with open(script_path.parent / out_file, "w") as out, open( - script_path.parent / error_file, "w" - ) as err: - subprocess.run( - [str(script_path)], cwd=script_path.parent, stdout=out, stderr=err - ) - - task_id = self.new_task_id() - self._tasks[task_id] = ScheduledJobTask( - task_id=task_id, - job_id=job_id, - script_path=script_path, - stdout_path=script_path.parent / out_file, - stderr_path=script_path.parent / error_file, - ) - self._slurm_jobs[job_id] = {} - return add_json_response( - response, - 201, - { - "success": "Task created", - "task_url": self.task_url(task_id), - "task_id": task_id, - }, - ) - - def storage_xfer_external_download( - self, data: dict[str, Any], response: Response - ) -> None: - source = Path(data["sourcePath"]) - if not source.exists(): - response.status_code = 400 - response.headers["X-Not-Found"] = "" - return - if source.is_dir(): - response.status_code = 400 - response.headers["X-A-Directory"] = "" - return - if not source.is_file(): - response.status_code = 400 - response.headers["X-Invalid-Path"] = "" - return - task_id = self.new_task_id() - self._tasks[task_id] = StorageXferExternalDownloadTask( - task_id=task_id, source_path=source - ) - add_json_response( - response, - 201, - { - "success": "Task created", - "task_id": task_id, - "task_url": self.task_url(task_id), - }, - ) - - def storage_xfer_external_upload( - self, data: dict[str, Any], response: Response - ) -> None: - source = Path(data["sourcePath"]) - target = Path(data["targetPath"]) - if not target.parent.exists(): - response.status_code = 400 - response.headers["X-Not-Found"] = "" - return - task_id = self.new_task_id() - self._tasks[task_id] = StorageXferExternalUploadTask( - task_id=task_id, source=source, target=target - ) - add_json_response( - response, - 201, - { - "success": "Task created", - "task_id": task_id, - "task_url": self.task_url(task_id), - }, - ) - - def utilities_file(self, params: dict[str, Any], response: Response) -> None: - path = Path(params["targetPath"]) - if not path.exists(): - response.status_code = 400 - response.headers["X-Invalid-Path"] = "" - return - add_success_response(response, 200, "directory" if path.is_dir() else "text") - - def utilities_symlink(self, data: dict[str, Any], response: Response) -> None: - target = Path(data["targetPath"]) - link = Path(data["linkPath"]) - if not target.exists(): - response.status_code = 400 - response.headers["X-Invalid-Path"] = "" - return - if link.exists(): - response.status_code = 400 - response.headers["X-Exists"] = "" - return - link.symlink_to(target) - add_success_response(response, 201) - - def utilities_stat(self, params: dict[str, Any], response: Response) -> None: - path = Path(params["targetPath"]) - dereference = params.get("dereference", False) - if not path.exists(): - response.status_code = 400 - response.headers["X-Invalid-Path"] = "" - return - result = path.stat() if dereference else path.lstat() - add_success_response( - response, - 200, - { - "mode": result.st_mode, - "uid": result.st_uid, - "gid": result.st_gid, - "size": result.st_size, - "atime": result.st_atime, - "mtime": result.st_mtime, - "ctime": result.st_ctime, - "nlink": result.st_nlink, - "ino": result.st_ino, - "dev": result.st_dev, - }, - ) - - def utilities_ls(self, params: dict[str, Any], response: Response) -> None: - path = Path(params["targetPath"]) - if not path.is_dir(): - response.status_code = 400 - response.headers["X-Invalid-Path"] = "" - return - data = [] - # note ls returns the file if the path is a file - for item in path.iterdir() if path.is_dir() else [path]: - _stat = item.lstat() - data.append( - { - "name": item.name, - "permissions": stat.filemode(_stat.st_mode)[1:], - "type": "l" if item.is_symlink() else "d" if item.is_dir() else "-", - "size": _stat.st_size, - "link_target": item.readlink() if item.is_symlink() else None, - } - ) - - add_success_response(response, 200, data) - - def utilities_checksum(self, params: dict[str, Any], response: Response) -> None: - path = Path(params["targetPath"]) - if not path.exists(): - response.status_code = 400 - response.headers["X-Invalid-Path"] = "" - return - import hashlib - - # Firecrest uses sha256 - sha256_hash = hashlib.sha256() - with open(path, "rb") as f: - for byte_block in iter(lambda: f.read(4096), b""): - sha256_hash.update(byte_block) - - checksum = sha256_hash.hexdigest() - add_success_response(response, 200, checksum) - - def utilities_chmod(self, data: dict[str, Any], response: Response) -> None: - path = Path(data["targetPath"]) - if not path.exists(): - response.status_code = 400 - response.headers["X-Invalid-Path"] = "" - return - path.chmod(int(data["mode"], 8)) - add_success_response(response, 200) - - def utilities_rename(self, data: dict[str, Any], response: Response) -> None: - source = Path(data["sourcePath"]) - target = Path(data["targetPath"]) - if not source.exists(): - response.status_code = 400 - response.headers["X-Invalid-Path"] = "" - return - if target.exists(): - response.status_code = 400 - response.headers["X-Exists"] = "" - return - source.rename(target) - add_success_response(response, 200) - - def utilities_mkdir(self, data: dict[str, Any], response: Response) -> None: - path = Path(data["targetPath"]) - if path.exists(): - response.status_code = 400 - response.headers["X-Exists"] = "" - return - path.mkdir(parents=data.get("p", False)) - add_success_response(response, 201) - - def utilities_rm(self, data: dict[str, Any], response: Response) -> None: - path = Path(data["targetPath"]) - if not path.exists(): - response.status_code = 400 - response.headers["X-Invalid-Path"] = "" - return - if path.is_dir(): - shutil.rmtree(path) - else: - path.unlink() - add_success_response(response, 204) - - def utilities_copy(self, data: dict[str, Any], response: Response) -> None: - source = Path(data["sourcePath"]) - target = Path(data["targetPath"]) - if not source.exists(): - response.status_code = 400 - response.headers["X-Invalid-Path"] = "" - return - if target.exists(): - response.status_code = 400 - response.headers["X-Exists"] = "" - return - if source.is_dir(): - shutil.copytree(source, target) - else: - shutil.copy2(source, target) - add_success_response(response, 201) - - def utilities_upload( - self, - data: dict[str, Any], - files: dict[str, tuple[str, BinaryIO]], - response: Response, - ) -> None: - # TODO files["file"] can be a tuple (name, stream) or just a stream with a name - fname, fbuffer = files["file"] - path = Path(data["targetPath"]) / fname - if not path.parent.exists(): - response.status_code = 400 - response.headers["X-Invalid-Path"] = "" - return - - fbytes = fbuffer.read() - if len(fbytes) > self.max_size_bytes: - add_json_response( - response, - 413, - { - "description": f"Failed to upload file. The file is over {self.max_size_bytes} bytes" - }, - ) - return - - path.write_bytes(fbytes) - add_success_response(response, 201) - - def utilities_download(self, params: dict[str, Any], response: Response) -> None: - path = Path(params["sourcePath"]) - if not path.is_file(): - response.status_code = 400 - response.headers["X-Invalid-Path"] = "" - return - - if path.lstat().st_size >= self.max_size_bytes: - add_json_response( - response, - 400, - { - "description": f"Failed to download file. The file is over {self.max_size_bytes} bytes" - }, - {"X-Size-Limit": "File size exceeds limit"}, - ) - return - - response.status_code = 200 - response.raw = io.BytesIO(path.read_bytes()) - - # def handle_task(self, task_id: str, response: Response) -> Response: - def handle_task(self, params: dict[str, Any], response: Response) -> Response: - task_id = params["tasks"].split(",")[0] - if task_id not in self._tasks: - return add_json_response( - response, 404, {"error": f"Task {task_id} does not exist"} - ) - - task = self._tasks[task_id] - - if isinstance(task, ActiveSchedulerJobsTask): - return self.task_active_scheduler_jobs(task, response) - if isinstance(task, ScheduledJobTask): - return self.task_scheduled_job(task, response) - if isinstance(task, StorageXferExternalUploadTask): - return self.task_storage_xfer_external_upload(task, response) - if isinstance(task, StorageXferExternalDownloadTask): - return self.task_storage_xfer_external_download(task, response) - raise NotImplementedError(f"Unknown task type: {type(task)}") - - def task_active_scheduler_jobs( - self, task: ActiveSchedulerJobsTask, response: Response - ) -> Response: - if task.jobs is not None: - for job_id in task.jobs or []: - if job_id not in self._slurm_jobs: - return add_json_response( - response, - 400, - { - "description": "Failed to retrieve job information", - "error": f"{job_id} is not a valid job ID", - }, - ) - - # Note because we always run jobs straight away (see self.compute_jobs_path), - # then we can assume that there are never any active jobs. - # TODO add some basic way to simulate active jobs - job_data: dict[str, Any] = {} - - return add_json_response( - response, - 200, - { - "task": { - "task_id": task.task_id, - "service": "compute", - "status": "200", - "description": "Finished successfully", - "data": job_data, - "user": self._username, - "task_url": self.task_url(task.task_id), - "hash_id": task.task_id, - "created_at": task.created_str, - "last_modify": task.modified_str, - "updated_at": task.modified_str, - } - }, - ) - - def task_scheduled_job( - self, task: ScheduledJobTask, response: Response - ) -> Response: - return add_json_response( - response, - 200, - { - "task": { - "data": { - "job_data_err": "", - "job_data_out": "", - "job_file": str(task.script_path), - "job_file_err": str(str(task.stderr_path)), - "job_file_out": str(str(task.stdout_path)), - "job_info_extra": "Job info returned successfully", - "jobid": task.job_id, - "result": "Job submitted", - }, - "description": "Finished successfully", - "service": "compute", - "status": "200", - "task_id": task.task_id, - "user": self._username, - "task_url": self.task_url(task.task_id), - "hash_id": task.task_id, - "created_at": task.created_str, - "last_modify": task.modified_str, - "updated_at": task.modified_str, - } - }, - ) - - def task_storage_xfer_external_download( - self, task: StorageXferExternalDownloadTask, response: Response - ) -> Response: - # see: https://github.com/eth-cscs/pyfirecrest/blob/5edbe85d11b977ed8f6943ca22e4fdc3d6f5e12d/firecrest/BasicClient.py#L202 - return add_json_response( - response, - 200, - { - "task": { - "data": f"file://{task.source_path}", - "description": "Started upload from filesystem to Object Storage", - "hash_id": task.task_id, - "service": "storage", - "status": "117", - "task_id": task.task_id, - "task_url": self.task_url(task.task_id), - "user": "username", - "last_modify": task.modified_str, - } - }, - ) - - def task_storage_xfer_external_upload( - self, task: StorageXferExternalUploadTask, response: Response - ) -> Response: - # to mock this once the Form URL is retrieved (110), we move straight to the - # "Download from Object Storage to server has finished" (114) for the next request - # this skips statuses 111, 112 and 113, - # see: https://github.com/eth-cscs/pyfirecrest/blob/5edbe85d11b977ed8f6943ca22e4fdc3d6f5e12d/firecrest/BasicClient.py#L143 - # and so we are assuming that the file is uploaded to the server - # I haven't updated the code to reflect this yet, but the new format of tasks is as follows: - # { - # 'b1fbee4afa1e52fb54a3a38aede7c246': { - # 'created_at': '2024-06-13T17:01:25', - # 'data': { - # 'hash_id': 'b1fbee4afa1e52fb54a3a38aede7c246', - # 'msg': 'Waiting for Presigned URL to upload file to staging area (Amazon S3 - Signature v4)', - # 'source': 'RM.mkv', - # 'status': '110', - # 'system_addr': 'domvm3.cscs.ch:22', - # 'system_name': 'dom', - # 'target': '/scratch/snx3000tds/akhosrav/delete_me/linkto_target', - # 'trace_id': '', - # 'user': 'akhosrav' - # }, - # 'description': 'Waiting for Form URL from Object Storage to be retrieved', - # 'hash_id': 'b1fbee4afa1e52fb54a3a38aede7c246', - # 'last_modify': '2024-06-13T17:01:25', - # 'service': 'storage', - # 'status': '110', - # 'system': 'dom', - # 'task_id': 'b1fbee4afa1e52fb54a3a38aede7c246', - # 'updated_at': '2024-06-13T17:01:25', - # 'user': 'akhosrav' - # } - # } - if not task.form_retrieved: - task.form_retrieved = True - return add_json_response( - response, - 200, - { - "task": { - "data": { - "msg": { - "command": "echo 'mock'", - # "command": "curl --show-error -s -i -X POST http://192.168.220.19:9000/service-account-firecrest-sample -F 'key=fd690c43e6ee509359b9e2c3237f4cc5/file.txt' -F 'x-amz-algorithm=AWS4-HMAC-SHA256' -F 'x-amz-credential=storage_access_key/202306/us-east-1/s3/aws4_request' -F 'x-amz-date=20230630T155026Z' -F 'policy=xxx' -F 'x-amz-signature=yyy' -F file=@/private/var/folders/t2/xbl15_3n4tsb1vr_ccmmtmbr0000gn/T/pytest-of-chrisjsewell/pytest-340/test_putfile_large0/file.txt", # noqa: E501 - "parameters": { - # "data": { - # "key": "fd690c43e6ee509359b9e2c3237f4cc5/file.txt", - # "policy": "xxx", - # "x-amz-algorithm": "AWS4-HMAC-SHA256", - # "x-amz-credential": "storage_access_key/202306/us-east-1/s3/aws4_request", - # "x-amz-date": "20230630T155026Z", - # "x-amz-signature": "yyy", - # }, - "data": {}, - "files": str(task.target), - "headers": {}, - "json": {}, - "method": "POST", - "params": {}, - # "url": "http://192.168.220.19:9000/service-account-firecrest-sample", - }, - }, - "status": "111", - "source": str(task.source), - "target": str(task.target), - "hash_id": task.task_id, - "user": self._username, - "system_addr": "machine_addr", - "system_name": self._machine, - "trace_id": "trace", - }, - "description": "Form URL from Object Storage received", - "service": "storage", - "status": "111", - "user": self._username, - "task_id": task.task_id, - "hash_id": task.task_id, - "task_url": self.task_url(task.task_id), - "created_at": task.created_str, - "last_modify": task.modified_str, - "updated_at": task.modified_str, - }, - }, - ) - else: - if not task.target.exists(): - shutil.copy(task.source, task.target) - return add_json_response( - response, - 200, - { - "task": { - "data": "Download from Object Storage to server has finished", - "description": "Download from Object Storage to server has finished", - "service": "storage", - "status": "114", - "task_id": task.task_id, - "user": self._username, - "hash_id": task.task_id, - "updated_at": task.modified_str, - "last_modify": task.modified_str, - "created_at": task.created_str, - "task_url": self.task_url(task.task_id), - } - }, - ) - - -@dataclass # note in python 3.10 we can use `(kw_only=False)` -class Task: - task_id: str - created_at: datetime = field(default_factory=datetime.now, init=False) - last_modified: datetime = field(default_factory=datetime.now, init=False) - - @property - def created_str(self) -> str: - return self.created_at.strftime("%Y-%m-%dT%H:%M:%S") - - @property - def modified_str(self) -> str: - return self.last_modified.strftime("%Y-%m-%dT%H:%M:%S") - - -@dataclass -class ActiveSchedulerJobsTask(Task): - jobs: list[str] | None - - -@dataclass -class ScheduledJobTask(Task): - job_id: str - script_path: Path - stdout_path: Path - stderr_path: Path - - -@dataclass -class StorageXferExternalUploadTask(Task): - source: Path - target: Path - form_retrieved: bool = False - - -@dataclass -class StorageXferExternalDownloadTask(Task): - source_path: Path - - -def add_json_response( - response: Response, - status_code: int, - json_data: dict[str, Any], - headers: dict[str, str] | None = None, -) -> Response: - response.status_code = status_code - response.raw = io.BytesIO(json_dumps(json_data).encode(response.encoding or "utf8")) - if headers: - response.headers.update(headers) - return response - - -def add_success_response( - response: Response, status_code: int, output: Any = "" -) -> Response: - return add_json_response( - response, - status_code, - { - "description": "success", - "output": output, - }, - ) diff --git a/firecrest_demo.py b/firecrest_demo.py deleted file mode 100644 index 743d752..0000000 --- a/firecrest_demo.py +++ /dev/null @@ -1,86 +0,0 @@ -from __future__ import annotations - -from argparse import ArgumentParser -from pathlib import Path -from subprocess import check_call -from typing import Protocol - - -class CliArgs(Protocol): - folder: str - git_tag: str - git_url: str - build: bool - - -def parse_args(args: list[str] | None = None) -> CliArgs: - """Parse the command line arguments.""" - parser = ArgumentParser(description="Create a FirecREST demo server.") - parser.add_argument( - "--folder", - default=".demo-server", - type=str, - help="The folder to clone FirecREST into.", - ) - parser.add_argument( - "--git-url", - type=str, - default="https://github.com/eth-cscs/firecrest.git", - help="The URL to clone FirecREST from.", - ) - parser.add_argument( - "--git-tag", - type=str, - default="v1.13.0", - help="The tag to checkout FirecREST at.", - ) - parser.add_argument( - "--build", - action="store_true", - help="Don't build the docker environment.", - ) - return parser.parse_args(args) - - -def main(args: list[str] | None = None): - """A CLI to generate a FirecREST demo server.""" - # use argparse to get the folder to clone firecrest into - parsed = parse_args(args) - - folder = Path(parsed.folder).absolute() - - if not folder.exists(): - print(f"Cloning FirecREST into {parsed.folder}") - check_call( - ["git", "clone", "--branch", parsed.git_tag, parsed.git_url, str(folder)] - ) - else: - print(f"FirecREST already exists in {folder!r}") - - # build the docker environment - if not parsed.build: - print("Skipping building the docker environment") - else: - print("Building the docker environment") - check_call(["docker-compose", "build"], cwd=(folder / "deploy" / "demo")) - - # ensure permissions of SSH keys (chmod 400) - print("Ensuring permissions of SSH keys") - folder.joinpath("deploy", "test-build", "environment", "keys", "ca-key").chmod( - 0o400 - ) - folder.joinpath("deploy", "test-build", "environment", "keys", "user-key").chmod( - 0o400 - ) - - # run the docker environment - print("Running the docker environment") - # could fail if required port in use - # on MaOS, can use e.g. `lsof -i :8080` to check - # TODO on MacOS port 7000 is used by AirPlay (afs3-fileserver), - # https://github.com/cookiecutter/cookiecutter-django/issues/3499 - check_call(["docker-compose", "up", "--detach"], cwd=(folder / "deploy" / "demo")) - - -if __name__ == "__main__": - main() diff --git a/pyproject.toml b/pyproject.toml index 67db52b..943e8d3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -26,7 +26,7 @@ requires-python = ">=3.9" dependencies = [ "aiida-core@git+https://github.com/khsrali/aiida-core.git@aiida-firecrest#egg=aiida-core", "click", - "pyfirecrest~=2.5.0", + "pyfirecrest@git+https://github.com/khsrali/pyfirecrest.git@main#egg=pyfirecrest", "pyyaml", "requests", ] diff --git a/tests/conftest.py b/tests/conftest.py index 88aa401..2e4a116 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,142 +1,369 @@ -"""Pytest configuration for the aiida-firecrest tests. - -This sets up the Firecrest server to use, and telemetry for API requests. -""" -from __future__ import annotations - -from functools import partial -from json import load as json_load +import hashlib +import os from pathlib import Path -from typing import Any, Callable -from urllib.parse import urlparse +import random +import stat +from typing import Optional -from _pytest.terminal import TerminalReporter -import firecrest as f7t +from aiida import orm +import firecrest +import firecrest.path import pytest -import requests -import yaml -from aiida_firecrest.utils_test import FirecrestConfig, FirecrestMockServer +class Values: + _DEFAULT_PAGE_SIZE: int = 25 -def pytest_report_header(config): - if config.getoption("--firecrest-config"): - header = [ - "Running against FirecREST server: {}".format( - config.getoption("--firecrest-config") - ) - ] - if config.getoption("--firecrest-no-clean"): - header.append("Not cleaning up FirecREST server after tests!") - return header - return ["Running against Mock FirecREST server"] +@pytest.fixture(name="firecrest_computer") +def _firecrest_computer(myfirecrest, tmpdir: Path): + """Create and return a computer configured for Firecrest. -def pytest_terminal_summary( - terminalreporter: TerminalReporter, exitstatus: int, config: pytest.Config -): - """Called after all tests have run.""" - data = config.stash.get("firecrest_requests", None) - if data is None: - return - terminalreporter.write( - yaml.dump( - {"Firecrest requests telemetry": data}, - default_flow_style=False, - sort_keys=True, - ) + Note, the computer is not stored in the database. + """ + + # create a temp directory and set it as the workdir + _scratch = tmpdir / "scratch" + _temp_directory = tmpdir / "temp" + _scratch.mkdir() + _temp_directory.mkdir() + + Path(tmpdir / ".firecrest").mkdir() + _secret_path = Path(tmpdir / ".firecrest/secret69") + _secret_path.write_text("SECRET_STRING") + + computer = orm.Computer( + label="test_computer", + description="test computer", + hostname="-", + workdir=str(_scratch), + transport_type="firecrest", + scheduler_type="firecrest", + ) + computer.set_minimum_job_poll_interval(5) + computer.set_default_mpiprocs_per_machine(1) + computer.configure( + url=" https://URI", + token_uri="https://TOKEN_URI", + client_id="CLIENT_ID", + client_secret=str(_secret_path), + client_machine="MACHINE_NAME", + small_file_size_mb=1.0, + temp_directory=str(_temp_directory), ) + return computer + + +class MockFirecrest: + def __init__(self, firecrest_url, *args, **kwargs): + self._firecrest_url = firecrest_url + self.args = args + self.kwargs = kwargs + + self.whoami = whomai + self.list_files = list_files + self.stat = stat_ + self.mkdir = mkdir + self.simple_delete = simple_delete + self.parameters = parameters + self.symlink = symlink + self.checksum = checksum + self.simple_download = simple_download + self.simple_upload = simple_upload + self.compress = compress + self.extract = extract + self.copy = copy + self.submit = submit + self.poll_active = poll_active + + +class MockClientCredentialsAuth: + def __init__(self, *args, **kwargs): + self.args = args + self.kwargs = kwargs @pytest.fixture(scope="function") -def firecrest_server( +def myfirecrest( pytestconfig: pytest.Config, - request: pytest.FixtureRequest, monkeypatch, - tmp_path: Path, ): - """A fixture which provides a mock Firecrest server to test against.""" - config_path: str | None = request.config.getoption("--firecrest-config") - no_clean: bool = request.config.getoption("--firecrest-no-clean") - record_requests: bool = request.config.getoption("--firecrest-requests") - telemetry: RequestTelemetry | None = None - - if config_path is not None: - # if given, use this config - with open(config_path, encoding="utf8") as handle: - config = json_load(handle) - config = FirecrestConfig(**config) - # rather than use the scratch_path directly, we use a subfolder, - # which we can then clean - config.scratch_path = config.scratch_path + "/pytest_tmp" - - # we need to connect to the client here, - # to ensure that the scratch path exists and is empty - client = f7t.Firecrest( - firecrest_url=config.url, - authorization=f7t.ClientCredentialsAuth( - config.client_id, config.client_secret, config.token_uri - ), + monkeypatch.setattr(firecrest, "Firecrest", MockFirecrest) + monkeypatch.setattr(firecrest, "ClientCredentialsAuth", MockClientCredentialsAuth) + + +def submit( + machine: str, + script_str: Optional[str] = None, + script_remote_path: Optional[str] = None, + script_local_path: Optional[str] = None, + local_file=False, +): + if local_file: + raise DeprecationWarning("local_file is not supported") + + if script_remote_path and not Path(script_remote_path).exists(): + raise FileNotFoundError(f"File {script_remote_path} does not exist") + job_id = random.randint(10000, 99999) + return {"jobid": job_id} + + +def poll_active(machine: str, jobs: list[str], page_number: int = 0): + response = [] + # 12 satets are defined in firecrest + states = [ + "TIMEOUT", + "SUSPENDED", + "PREEMPTED", + "CANCELLED", + "NODE_FAIL", + "PENDING", + "FAILED", + "RUNNING", + "CONFIGURING", + "QUEUED", + "COMPLETED", + "COMPLETING", + ] + for i in range(len(jobs)): + response.append( + { + "job_data_err": "", + "job_data_out": "", + "job_file": "somefile.sh", + "job_file_err": "somefile-stderr.txt", + "job_file_out": "somefile-stdout.txt", + "job_info_extra": "Job info returned successfully", + "jobid": f"{jobs[i]}", + "name": "aiida-45", + "nodelist": "nid00049", + "nodes": "1", + "partition": "normal", + "start_time": "0:03", + "state": states[i % 12], + "time": "2024-06-21T10:44:42", + "time_left": "29:57", + "user": "Prof. Wang", + } ) - client.mkdir(config.machine, config.scratch_path, p=True) - if record_requests: - telemetry = RequestTelemetry() - monkeypatch.setattr(requests, "get", partial(telemetry.wrap, requests.get)) - monkeypatch.setattr( - requests, "post", partial(telemetry.wrap, requests.post) + return response[ + page_number + * Values._DEFAULT_PAGE_SIZE : (page_number + 1) + * Values._DEFAULT_PAGE_SIZE + ] + + +def whomai(machine: str): + assert machine == "MACHINE_NAME" + return "test_user" + + +def list_files( + machine: str, target_path: str, recursive: bool = False, show_hidden: bool = False +): + # this is mimiking the expected behaviour from the firecrest code. + + content_list = [] + for root, dirs, files in os.walk(target_path): + if not recursive and root != target_path: + continue + for name in dirs + files: + full_path = os.path.join(root, name) + relative_path = ( + Path(os.path.relpath(root, target_path)).joinpath(name).as_posix() ) - monkeypatch.setattr(requests, "put", partial(telemetry.wrap, requests.put)) - monkeypatch.setattr( - requests, "delete", partial(telemetry.wrap, requests.delete) + if os.path.islink(full_path): + content_type = "l" + link_target = ( + os.readlink(full_path) if os.path.islink(full_path) else None + ) + elif os.path.isfile(full_path): + content_type = "-" + link_target = None + elif os.path.isdir(full_path): + content_type = "d" + link_target = None + else: + content_type = "NON" + link_target = None + permissions = stat.filemode(Path(full_path).lstat().st_mode)[1:] + if name.startswith(".") and not show_hidden: + continue + content_list.append( + { + "name": relative_path, + "type": content_type, + "link_target": link_target, + "permissions": permissions, + } ) - yield config - # Note this shouldn't really work, for folders but it does :shrug: - # because they use `rm -r`: - # https://github.com/eth-cscs/firecrest/blob/7f02d11b224e4faee7f4a3b35211acb9c1cc2c6a/src/utilities/utilities.py#L347 - if not no_clean: - client.simple_delete(config.machine, config.scratch_path) + return content_list + + +def stat_(machine: str, targetpath: firecrest.path, dereference=True): + stats = os.stat( + targetpath, follow_symlinks=bool(dereference) if dereference else False + ) + return { + "ino": stats.st_ino, + "dev": stats.st_dev, + "nlink": stats.st_nlink, + "uid": stats.st_uid, + "gid": stats.st_gid, + "size": stats.st_size, + "atime": stats.st_atime, + "mtime": stats.st_mtime, + "ctime": stats.st_ctime, + } + + +def mkdir(machine: str, target_path: str, p: bool = False): + if p: + os.makedirs(target_path) + else: + os.mkdir(target_path) + + +def simple_delete(machine: str, target_path: str): + if not Path(target_path).exists(): + raise FileNotFoundError(f"File or folder {target_path} does not exist") + if os.path.isdir(target_path): + os.rmdir(target_path) else: - # otherwise use mock server - server = FirecrestMockServer(tmp_path) - if record_requests: - telemetry = RequestTelemetry() - mock_request = partial(telemetry.wrap, server.mock_request) - else: - mock_request = server.mock_request - monkeypatch.setattr(requests, "get", mock_request) - monkeypatch.setattr(requests, "post", mock_request) - monkeypatch.setattr(requests, "put", mock_request) - monkeypatch.setattr(requests, "delete", mock_request) - monkeypatch.setattr(requests.Session, "get", mock_request) - monkeypatch.setattr(requests.Session, "post", mock_request) - monkeypatch.setattr(requests.Session, "put", mock_request) - monkeypatch.setattr(requests.Session, "delete", mock_request) - yield server.config - - # save data on the server - if telemetry is not None: - test_name = request.node.name - pytestconfig.stash.setdefault("firecrest_requests", {})[ - test_name - ] = telemetry.counts - - -class RequestTelemetry: - """A to gather telemetry on requests.""" - - def __init__(self) -> None: - self.counts = {} - - def wrap( - self, - method: Callable[..., requests.Response], - url: str | bytes, - **kwargs: Any, - ) -> requests.Response: - """Wrap a requests method to gather telemetry.""" - endpoint = urlparse(url if isinstance(url, str) else url.decode("utf-8")).path - self.counts.setdefault(endpoint, 0) - self.counts[endpoint] += 1 - return method(url, **kwargs) + os.remove(target_path) + + +def symlink(machine: str, target_path: str, link_path: str): + # this is how firecrest does it + os.system(f"ln -s {target_path} {link_path}") + + +def simple_download(machine: str, remote_path: str, local_path: str): + # this procedure is complecated in firecrest, but I am simplifying it here + # we don't care about the details of the download, we just want to make sure + # that the aiida-firecrest code is calling the right functions at right time + if Path(remote_path).is_dir(): + raise IsADirectoryError(f"{remote_path} is a directory") + if not Path(remote_path).exists(): + raise FileNotFoundError(f"{remote_path} does not exist") + os.system(f"cp {remote_path} {local_path}") + + +def simple_upload( + machine: str, local_path: str, remote_path: str, file_name: Optional[str] = None +): + # this procedure is complecated in firecrest, but I am simplifying it here + # we don't care about the details of the upload, we just want to make sure + # that the aiida-firecrest code is calling the right functions at right time + if Path(local_path).is_dir(): + raise IsADirectoryError(f"{local_path} is a directory") + if not Path(local_path).exists(): + raise FileNotFoundError(f"{local_path} does not exist") + if file_name: + remote_path = os.path.join(remote_path, file_name) + os.system(f"cp {local_path} {remote_path}") + + +def copy(machine: str, source_path: str, target_path: str): + # this is how firecrest does it + # https://github.com/eth-cscs/firecrest/blob/db6ba4ba273c11a79ecbe940872f19d5cb19ac5e/src/utilities/utilities.py#L451C1-L452C1 + os.system(f"cp --force -dR --preserve=all -- '{source_path}' '{target_path}'") + + +def compress( + machine: str, source_path: str, target_path: str, dereference: bool = True +): + # this is how firecrest does it + # https://github.com/eth-cscs/firecrest/blob/db6ba4ba273c11a79ecbe940872f19d5cb19ac5e/src/utilities/utilities.py#L460 + basedir = os.path.dirname(source_path) + file_path = os.path.basename(source_path) + deref = "--dereference" if dereference else "" + os.system(f"tar {deref} -czf '{target_path}' -C '{basedir}' '{file_path}'") + + +def extract(machine: str, source_path: str, target_path: str): + # this is how firecrest does it + # https://github.com/eth-cscs/firecrest/blob/db6ba4ba273c11a79ecbe940872f19d5cb19ac5e/src/common/cscs_api_common.py#L1110C18-L1110C65 + os.system(f"tar -xf '{source_path}' -C '{target_path}'") + + +def checksum(machine: str, remote_path: str) -> int: + if not remote_path.exists(): + return False + # Firecrest uses sha256 + sha256_hash = hashlib.sha256() + with open(remote_path, "rb") as f: + for byte_block in iter(lambda: f.read(4096), b""): + sha256_hash.update(byte_block) + + return sha256_hash.hexdigest() + + +def parameters(): + # note: I took this from https://firecrest-tds.cscs.ch/ or https://firecrest.cscs.ch/ + # if code is not working but test passes, it means you need to update this dictionary + # with the latest FirecREST parameters + return { + "compute": [ + { + "description": "Type of resource and workload manager used in compute microservice", + "name": "WORKLOAD_MANAGER", + "unit": "", + "value": "Slurm", + } + ], + "storage": [ + { + "description": "Type of object storage, like `swift`, `s3v2` or `s3v4`.", + "name": "OBJECT_STORAGE", + "unit": "", + "value": "s3v4", + }, + { + "description": "Expiration time for temp URLs.", + "name": "STORAGE_TEMPURL_EXP_TIME", + "unit": "seconds", + "value": "86400", + }, + { + "description": "Maximum file size for temp URLs.", + "name": "STORAGE_MAX_FILE_SIZE", + "unit": "MB", + "value": "5120", + }, + { + "description": "Available filesystems through the API.", + "name": "FILESYSTEMS", + "unit": "", + "value": [ + { + "mounted": ["/project", "/store", "/scratch/snx3000tds"], + "system": "dom", + }, + { + "mounted": ["/project", "/store", "/capstor/scratch/cscs"], + "system": "pilatus", + }, + ], + }, + ], + "utilities": [ + { + "description": "The maximum allowable file size for various operations of the utilities microservice", + "name": "UTILITIES_MAX_FILE_SIZE", + "unit": "MB", + "value": "69", + }, + { + "description": ( + "Maximum time duration for executing the commands " + "in the cluster for the utilities microservice." + ), + "name": "UTILITIES_TIMEOUT", + "unit": "seconds", + "value": "5", + }, + ], + } diff --git a/tests/test_calculation.py b/tests/test_calculation.py deleted file mode 100644 index d308671..0000000 --- a/tests/test_calculation.py +++ /dev/null @@ -1,183 +0,0 @@ -"""Test for running calculations on a FireCREST computer.""" -from pathlib import Path - -from aiida import common, engine, manage, orm -from aiida.common.folders import Folder -from aiida.engine.processes.calcjobs.tasks import MAX_ATTEMPTS_OPTION -from aiida.manage.tests.pytest_fixtures import EntryPointManager -from aiida.parsers import Parser -import pytest - -from aiida_firecrest.utils_test import FirecrestConfig - - -@pytest.fixture(name="firecrest_computer") -def _firecrest_computer(firecrest_server: FirecrestConfig): - """Create and return a computer configured for Firecrest. - - Note, the computer is not stored in the database. - """ - computer = orm.Computer( - label="test_computer", - description="test computer", - hostname="-", - workdir=firecrest_server.scratch_path, - transport_type="firecrest", - scheduler_type="firecrest", - ) - computer.set_minimum_job_poll_interval(5) - computer.set_default_mpiprocs_per_machine(1) - computer.configure( - url=firecrest_server.url, - token_uri=firecrest_server.token_uri, - client_id=firecrest_server.client_id, - client_secret=firecrest_server.client_secret, - client_machine=firecrest_server.machine, - small_file_size_mb=firecrest_server.small_file_size_mb, - ) - computer.store() - return computer - - -@pytest.fixture(name="no_retries") -def _no_retries(): - """Remove calcjob retries, to make failing the test faster.""" - # TODO calculation seems to hang on errors still - max_attempts = manage.get_config().get_option(MAX_ATTEMPTS_OPTION) - manage.get_config().set_option(MAX_ATTEMPTS_OPTION, 1) - yield - manage.get_config().set_option(MAX_ATTEMPTS_OPTION, max_attempts) - - -@pytest.mark.usefixtures("aiida_profile_clean", "no_retries") -def test_calculation_basic(firecrest_computer: orm.Computer): - """Test running a simple `arithmetic.add` calculation.""" - code = orm.InstalledCode( - label="test_code", - description="test code", - default_calc_job_plugin="core.arithmetic.add", - computer=firecrest_computer, - filepath_executable="/bin/sh", - ) - code.store() - - builder = code.get_builder() - builder.x = orm.Int(1) - builder.y = orm.Int(2) - # TODO currently uploading via firecrest changes _aiidasubmit.sh to aiidasubmit.sh 😱 - # https://github.com/eth-cscs/firecrest/issues/191 - builder.metadata.options.submit_script_filename = "aiidasubmit.sh" - - _, node = engine.run_get_node(builder) - assert node.is_finished_ok - - -@pytest.mark.usefixtures("aiida_profile_clean", "no_retries") -def test_calculation_file_transfer( - firecrest_computer: orm.Computer, entry_points: EntryPointManager -): - """Test a calculation, with multiple files copied/uploaded/retrieved.""" - # add temporary entry points - entry_points.add(MultiFileCalcjob, "aiida.calculations:testing.multifile") - entry_points.add(NoopParser, "aiida.parsers:testing.noop") - - # add a remote file which is used remote_copy_list - firecrest_computer.get_transport()._cwd.joinpath( - firecrest_computer.get_workdir(), "remote_copy.txt" - ).touch() - - # setup the calculation - code = orm.InstalledCode( - label="test_code", - description="test code", - default_calc_job_plugin="testing.multifile", - computer=firecrest_computer, - filepath_executable="/bin/sh", - ) - code.store() - builder = code.get_builder() - - node: orm.CalcJobNode - _, node = engine.run_get_node(builder) - assert node.is_finished_ok - - if (retrieved := node.get_retrieved_node()) is None: - raise RuntimeError("No retrieved node found") - - paths = sorted([str(p) for p in retrieved.base.repository.glob()]) - assert paths == [ - "_scheduler-stderr.txt", - "_scheduler-stdout.txt", - "folder1", - "folder1/a", - "folder1/a/b.txt", - "folder1/a/c.txt", - "folder2", - "folder2/remote_copy.txt", - "folder2/x", - "folder2/y", - "folder2/y/z", - ] - - -class MultiFileCalcjob(engine.CalcJob): - """A complex CalcJob that creates/retrieves multiple files.""" - - @classmethod - def define(cls, spec): - """Define the process specification.""" - super().define(spec) - spec.input( - "metadata.options.submit_script_filename", - valid_type=str, - default="aiidasubmit.sh", - ) - spec.inputs["metadata"]["options"]["resources"].default = { - "num_machines": 1, - "num_mpiprocs_per_machine": 1, - } - spec.input( - "metadata.options.parser_name", valid_type=str, default="testing.noop" - ) - spec.exit_code(400, "ERROR", message="Calculation failed.") - - def prepare_for_submission(self, folder: Folder) -> common.CalcInfo: - """Prepare the calculation job for submission.""" - codeinfo = common.CodeInfo() - codeinfo.code_uuid = self.inputs.code.uuid - - path = Path(folder.get_abs_path("a")).parent - for subpath in [ - "i.txt", - "j.txt", - "folder1/a/b.txt", - "folder1/a/c.txt", - "folder1/a/c.in", - "folder1/c.txt", - "folder2/x", - "folder2/y/z", - ]: - path.joinpath(subpath).parent.mkdir(parents=True, exist_ok=True) - path.joinpath(subpath).touch() - - calcinfo = common.CalcInfo() - calcinfo.codes_info = [codeinfo] - calcinfo.retrieve_list = [("folder1/*/*.txt", ".", 99), ("folder2", ".", 99)] - comp: orm.Computer = self.inputs.code.computer - calcinfo.remote_copy_list = [ - ( - comp.uuid, - f"{comp.get_workdir()}/remote_copy.txt", - "folder2/remote_copy.txt", - ) - ] - # TODO also add remote_symlink_list - - return calcinfo - - -class NoopParser(Parser): - """Parser that does absolutely nothing!""" - - def parse(self, **kwargs): - pass diff --git a/tests/test_computer.py b/tests/test_computer.py index 81d6c58..fe4e48e 100644 --- a/tests/test_computer.py +++ b/tests/test_computer.py @@ -1,41 +1,10 @@ -"""Tests for setting up an AiiDA computer for Firecrest, -and basic functionality of the Firecrest transport and scheduler plugins. -""" from pathlib import Path +from unittest.mock import Mock from aiida import orm +from click import BadParameter import pytest -from aiida_firecrest.utils_test import FirecrestConfig - - -@pytest.fixture(name="firecrest_computer") -def _firecrest_computer(firecrest_server: FirecrestConfig): - """Create and return a computer configured for Firecrest. - - Note, the computer is not stored in the database. - """ - computer = orm.Computer( - label="test_computer", - description="test computer", - hostname="-", - workdir=firecrest_server.scratch_path, - transport_type="firecrest", - scheduler_type="firecrest", - ) - computer.set_minimum_job_poll_interval(5) - computer.set_default_mpiprocs_per_machine(1) - computer.configure( - url=firecrest_server.url, - token_uri=firecrest_server.token_uri, - client_id=firecrest_server.client_id, - client_secret=firecrest_server.client_secret, - client_machine=firecrest_server.machine, - small_file_size_mb=firecrest_server.small_file_size_mb, - temp_directory=firecrest_server.temp_directory, - ) - return computer - @pytest.mark.usefixtures("aiida_profile_clean") def test_whoami(firecrest_computer: orm.Computer): @@ -44,34 +13,109 @@ def test_whoami(firecrest_computer: orm.Computer): assert transport.whoami() == "test_user" -@pytest.mark.usefixtures("aiida_profile_clean") -def test_create_temp_file(firecrest_computer: orm.Computer, tmp_path: Path): - """Check if it is possible to create a temporary file - and then delete it in the work directory. - """ - transport = firecrest_computer.get_transport() - authinfo = firecrest_computer.get_authinfo(orm.User.collection.get_default()) - workdir = authinfo.get_workdir().format(username=transport.whoami()) - transport.chdir(workdir) +def test_create_secret_file_with_existing_file(tmpdir: Path): + from aiida_firecrest.transport import _create_secret_file - tmp_path.joinpath("test.txt").write_text("test") - transport.putfile(str(tmp_path.joinpath("test.txt")), "test.txt") + secret_file = Path(tmpdir / "secret") + secret_file.write_text("topsecret") + result = _create_secret_file(None, None, str(secret_file)) + assert isinstance(result, str) + assert result == str(secret_file) + assert Path(result).read_text() == "topsecret" - assert transport.path_exists("test.txt") - transport.getfile("test.txt", str(tmp_path.joinpath("test2.txt"))) +def test_create_secret_file_with_nonexistent_file(tmp_path): + from aiida_firecrest.transport import _create_secret_file - assert tmp_path.joinpath("test2.txt").read_text() == "test" + secret_file = tmp_path / "nonexistent" + with pytest.raises(BadParameter): + _create_secret_file(None, None, str(secret_file)) - transport.remove("test.txt") - assert not transport.path_exists("test.txt") +def test_create_secret_file_with_secret_value(tmp_path, monkeypatch): + from aiida_firecrest.transport import _create_secret_file - -@pytest.mark.usefixtures("aiida_profile_clean") -def test_get_jobs(firecrest_computer: orm.Computer): - """check if it is possible to determine the username.""" - transport = firecrest_computer.get_transport() - scheduler = firecrest_computer.get_scheduler() - scheduler.set_transport(transport) - assert isinstance(scheduler.get_jobs(), list) + secret = "topsecret!~/" + monkeypatch.setattr( + Path, + "expanduser", + lambda x: tmp_path / str(x).lstrip("~/") if str(x).startswith("~/") else x, + ) + result = _create_secret_file(None, None, secret) + assert Path(result).parent.parts[-1] == ".firecrest" + assert Path(result).read_text() == secret + + +def test_validate_temp_directory(myfirecrest, monkeypatch, tmpdir: Path): + from aiida_firecrest.transport import _validate_temp_directory + + monkeypatch.setattr("click.echo", lambda x: None) + # monkeypatch.setattr('click.BadParameter', lambda x: None) + secret_file = Path(tmpdir / "secret") + secret_file.write_text("topsecret") + ctx = Mock() + ctx.params = { + "url": "http://test.com", + "token_uri": "token_uri", + "client_id": "client_id", + "client_machine": "client_machine", + "client_secret": secret_file.as_posix(), + "small_file_size_mb": float(10), + } + + # should raise if is_file + Path(tmpdir / "crap.txt").touch() + with pytest.raises(BadParameter): + result = _validate_temp_directory( + ctx, None, Path(tmpdir / "crap.txt").as_posix() + ) + + # should create the directory if it doesn't exist + result = _validate_temp_directory( + ctx, None, Path(tmpdir / "temp_on_server_directory").as_posix() + ) + assert result == Path(tmpdir / "temp_on_server_directory").as_posix() + assert Path(tmpdir / "temp_on_server_directory").exists() + + # should get a confirmation if the directory exists and is not empty + Path(tmpdir / "temp_on_server_directory" / "crap.txt").touch() + monkeypatch.setattr("click.confirm", lambda x: False) + with pytest.raises(BadParameter): + result = _validate_temp_directory( + ctx, None, Path(tmpdir / "temp_on_server_directory").as_posix() + ) + + # should delete the content if I confirm + monkeypatch.setattr("click.confirm", lambda x: True) + result = _validate_temp_directory( + ctx, None, Path(tmpdir / "temp_on_server_directory").as_posix() + ) + assert result == Path(tmpdir / "temp_on_server_directory").as_posix() + assert not Path(tmpdir / "temp_on_server_directory" / "crap.txt").exists() + + +def test__dynamic_info(myfirecrest, monkeypatch, tmpdir: Path): + from aiida_firecrest.transport import _dynamic_info_direct_size + + monkeypatch.setattr("click.echo", lambda x: None) + # monkeypatch.setattr('click.BadParameter', lambda x: None) + secret_file = Path(tmpdir / "secret") + secret_file.write_text("topsecret") + ctx = Mock() + ctx.params = { + "url": "http://test.com", + "token_uri": "token_uri", + "client_id": "client_id", + "client_machine": "client_machine", + "client_secret": secret_file.as_posix(), + "small_file_size_mb": float(10), + } + + # should catch UTILITIES_MAX_FILE_SIZE if value is not provided + result = _dynamic_info_direct_size(ctx, None, 0) + assert result == 69 + + # should use the value if provided + # note: user cannot enter negative numbers anyways, click raise as this shoule be float not str + result = _dynamic_info_direct_size(ctx, None, 10) + assert result == 10 diff --git a/tests/test_scheduler.py b/tests/test_scheduler.py index 4126b61..b4cb26f 100644 --- a/tests/test_scheduler.py +++ b/tests/test_scheduler.py @@ -1,68 +1,77 @@ -"""Tests isolating only the Scheduler.""" -from aiida.schedulers import SchedulerError +from pathlib import Path +import random + +from aiida import orm from aiida.schedulers.datastructures import CodeRunMode, JobTemplate import pytest from aiida_firecrest.scheduler import FirecrestScheduler -from aiida_firecrest.transport import FirecrestTransport -from aiida_firecrest.utils_test import FirecrestConfig - - -@pytest.fixture(name="transport") -def _transport(firecrest_server: FirecrestConfig): - transport = FirecrestTransport( - url=firecrest_server.url, - token_uri=firecrest_server.token_uri, - client_id=firecrest_server.client_id, - client_secret=firecrest_server.client_secret, - client_machine=firecrest_server.machine, - small_file_size_mb=firecrest_server.small_file_size_mb, - ) - transport.chdir(firecrest_server.scratch_path) - yield transport - - -def test_get_jobs_empty(transport: FirecrestTransport): - scheduler = FirecrestScheduler() - scheduler.set_transport(transport) - - with pytest.raises(SchedulerError, match="Failed to retrieve job"): - scheduler.get_jobs(["unknown"]) - - assert isinstance(scheduler.get_jobs(), list) +from conftest import Values -def test_submit_job(transport: FirecrestTransport): +@pytest.mark.usefixtures("aiida_profile_clean") +def test_submit_job(firecrest_computer: orm.Computer, tmp_path: Path): + transport = firecrest_computer.get_transport() scheduler = FirecrestScheduler() scheduler.set_transport(transport) - with pytest.raises(SchedulerError, match="invalid path"): + with pytest.raises(FileNotFoundError): scheduler.submit_job(transport.getcwd(), "unknown.sh") - # create a job script in a folder - transport.mkdir("test_submission") - transport.chdir("test_submission") - transport.write_binary("job.sh", b"#!/bin/bash\n\necho 'hello world'") + _script = Path(tmp_path / "job.sh") + _script.write_text("#!/bin/bash\n\necho 'hello world'") - job_id = scheduler.submit_job(transport.getcwd(), "job.sh") + job_id = scheduler.submit_job(transport.getcwd(), _script) + # this is how aiida expects the job_id to be returned assert isinstance(job_id, str) -def test_write_script_minimal(file_regression): +@pytest.mark.usefixtures("aiida_profile_clean") +def test_get_jobs(firecrest_computer: orm.Computer): + transport = firecrest_computer.get_transport() scheduler = FirecrestScheduler() - template = JobTemplate( - { - "job_resource": scheduler.create_job_resource( - num_machines=1, num_mpiprocs_per_machine=1 - ), - "codes_info": [], - "codes_run_mode": CodeRunMode.SERIAL, - } - ) - file_regression.check(scheduler.get_submit_script(template).rstrip() + "\n") - + scheduler.set_transport(transport) -def test_write_script_full(file_regression): + # test pagaination + scheduler._DEFAULT_PAGE_SIZE = 2 + Values._DEFAULT_PAGE_SIZE = 2 + + joblist = [random.randint(10000, 99999) for i in range(5)] + result = scheduler.get_jobs(joblist) + assert len(result) == 5 + for i in range(5): + assert result[i].job_id == str(joblist[i]) + # TODO: one could check states as well + + +def test_write_script_full(): + # to avoid false positive (overwriting on existing file), + # we check the output of the script instead of using `file_regression`` + expectaion = """ + #!/bin/bash + #SBATCH -H + #SBATCH --requeue + #SBATCH --mail-user=True + #SBATCH --mail-type=BEGIN + #SBATCH --mail-type=FAIL + #SBATCH --mail-type=END + #SBATCH --job-name="test_job" + #SBATCH --get-user-env + #SBATCH --output=test.out + #SBATCH --error=test.err + #SBATCH --partition=test_queue + #SBATCH --account=test_account + #SBATCH --qos=test_qos + #SBATCH --nice=100 + #SBATCH --nodes=1 + #SBATCH --ntasks-per-node=1 + #SBATCH --time=01:00:00 + #SBATCH --mem=1 + test_command + """ + expectaion_flat = "\n".join(line.strip() for line in expectaion.splitlines()).strip( + "\n" + ) scheduler = FirecrestScheduler() template = JobTemplate( { @@ -89,4 +98,42 @@ def test_write_script_full(file_regression): "custom_scheduler_commands": "test_command", } ) - file_regression.check(scheduler.get_submit_script(template).rstrip() + "\n") + try: + assert scheduler.get_submit_script(template).rstrip() == expectaion_flat + except AssertionError: + print(scheduler.get_submit_script(template).rstrip()) + print(expectaion) + raise + + +def test_write_script_minimal(): + # to avoid false positive (overwriting on existing file), + # we check the output of the script instead of using `file_regression`` + expectaion = """ + #!/bin/bash + #SBATCH --no-requeue + #SBATCH --error=slurm-%j.err + #SBATCH --nodes=1 + #SBATCH --ntasks-per-node=1 + """ + + expectaion_flat = "\n".join(line.strip() for line in expectaion.splitlines()).strip( + "\n" + ) + scheduler = FirecrestScheduler() + template = JobTemplate( + { + "job_resource": scheduler.create_job_resource( + num_machines=1, num_mpiprocs_per_machine=1 + ), + "codes_info": [], + "codes_run_mode": CodeRunMode.SERIAL, + } + ) + + try: + assert scheduler.get_submit_script(template).rstrip() == expectaion_flat + except AssertionError: + print(scheduler.get_submit_script(template).rstrip()) + print(expectaion) + raise diff --git a/tests/test_scheduler/test_write_script_full.txt b/tests/test_scheduler/test_write_script_full.txt deleted file mode 100644 index 6b834d2..0000000 --- a/tests/test_scheduler/test_write_script_full.txt +++ /dev/null @@ -1,20 +0,0 @@ -#!/bin/bash -#SBATCH -H -#SBATCH --requeue -#SBATCH --mail-user=True -#SBATCH --mail-type=BEGIN -#SBATCH --mail-type=FAIL -#SBATCH --mail-type=END -#SBATCH --job-name="test_job" -#SBATCH --get-user-env -#SBATCH --output=test.out -#SBATCH --error=test.err -#SBATCH --partition=test_queue -#SBATCH --account=test_account -#SBATCH --qos=test_qos -#SBATCH --nice=100 -#SBATCH --nodes=1 -#SBATCH --ntasks-per-node=1 -#SBATCH --time=01:00:00 -#SBATCH --mem=1 -test_command diff --git a/tests/test_scheduler/test_write_script_minimal.txt b/tests/test_scheduler/test_write_script_minimal.txt deleted file mode 100644 index 5bc0101..0000000 --- a/tests/test_scheduler/test_write_script_minimal.txt +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/bash -#SBATCH --no-requeue -#SBATCH --error=slurm-%j.err -#SBATCH --nodes=1 -#SBATCH --ntasks-per-node=1 diff --git a/tests/test_transport.py b/tests/test_transport.py index e30db7e..edd11ee 100644 --- a/tests/test_transport.py +++ b/tests/test_transport.py @@ -1,153 +1,627 @@ -"""Tests isolating only the Transport.""" +import os from pathlib import Path -import platform +from unittest.mock import patch +from aiida import orm import pytest -from aiida_firecrest.transport import FirecrestTransport -from aiida_firecrest.utils_test import FirecrestConfig +@pytest.mark.usefixtures("aiida_profile_clean") +def test_mkdir(firecrest_computer: orm.Computer, tmpdir: Path): + transport = firecrest_computer.get_transport() -@pytest.fixture(name="transport") -def _transport(firecrest_server: FirecrestConfig): - transport = FirecrestTransport( - url=firecrest_server.url, - token_uri=firecrest_server.token_uri, - client_id=firecrest_server.client_id, - client_secret=firecrest_server.client_secret, - client_machine=firecrest_server.machine, - small_file_size_mb=firecrest_server.small_file_size_mb, - ) - transport.chdir(firecrest_server.scratch_path) - yield transport - - -def test_path_exists(firecrest_server: FirecrestConfig, transport: FirecrestTransport): - assert transport.path_exists(firecrest_server.scratch_path) - assert not transport.path_exists(firecrest_server.scratch_path + "/file.txt") - - -def test_get_attribute( - firecrest_server: FirecrestConfig, transport: FirecrestTransport -): - transport._cwd.joinpath("test.txt").touch() - attrs = transport.get_attribute(firecrest_server.scratch_path + "/test.txt") - assert set(attrs) == { - "st_size", - "st_atime", - "st_mode", - "st_gid", - "st_mtime", - "st_uid", - } - assert isinstance(attrs.st_mode, int) - - -def test_isdir(firecrest_server: FirecrestConfig, transport: FirecrestTransport): - assert transport.isdir(firecrest_server.scratch_path) - assert not transport.isdir(firecrest_server.scratch_path + "/other") - - -def test_mkdir(firecrest_server: FirecrestConfig, transport: FirecrestTransport): - transport.mkdir(firecrest_server.scratch_path + "/test") - assert transport.isdir(firecrest_server.scratch_path + "/test") - - -def test_large_file_transfers( - firecrest_server: FirecrestConfig, transport: FirecrestTransport, tmp_path: Path -): - """Large file transfers (> 5MB by default) have to be downloaded/uploaded via a different pathway.""" - content = "a" * (transport._small_file_size_bytes + 1) - - # upload - remote_path = firecrest_server.scratch_path + "/file.txt" - assert not transport.isfile(remote_path) - file_path = tmp_path.joinpath("file.txt") - file_path.write_text(content) - transport.putfile(str(file_path), remote_path) - assert transport.isfile(remote_path) - - # download - if transport._url.startswith("http://localhost") and platform.system() == "Darwin": - pytest.skip("Skipping large file download test on macOS with localhost server.") - # TODO this is a known issue whereby a 403 is returned when trying to download the supplied file url - # due to a signature mismatch - new_path = tmp_path.joinpath("file2.txt") - assert not new_path.is_file() - transport.getfile(remote_path, new_path) - assert new_path.is_file() - assert new_path.read_text() == content - + _scratch = tmpdir / "sampledir" + transport.mkdir(_scratch) + assert _scratch.exists() -def test_listdir(firecrest_server: FirecrestConfig, transport: FirecrestTransport): - assert transport.listdir(firecrest_server.scratch_path) == [] + _scratch = tmpdir / "sampledir2" / "subdir" + transport.makedirs(_scratch) + assert _scratch.exists() -def test_copyfile(firecrest_server: FirecrestConfig, transport: FirecrestTransport): - from_path = firecrest_server.scratch_path + "/copy_from.txt" - to_path = firecrest_server.scratch_path + "/copy_to.txt" +@pytest.mark.usefixtures("aiida_profile_clean") +def test_is_file(firecrest_computer: orm.Computer, tmpdir: Path): + transport = firecrest_computer.get_transport() - transport.write_binary(from_path, b"test") + _scratch = tmpdir / "samplefile" + Path(_scratch).touch() + assert transport.isfile(_scratch) + assert not transport.isfile(_scratch / "does_not_exist") - assert not transport.path_exists(to_path) - transport.copyfile(from_path, to_path) - assert transport.isfile(to_path) - assert transport.read_binary(to_path) == b"test" +@pytest.mark.usefixtures("aiida_profile_clean") +def test_is_dir(firecrest_computer: orm.Computer, tmpdir: Path): + transport = firecrest_computer.get_transport() -def test_copyfile_symlink_noderef( - firecrest_server: FirecrestConfig, transport: FirecrestTransport -): - from_path = firecrest_server.scratch_path + "/copy_from.txt" - from_path_symlink = firecrest_server.scratch_path + "/copy_from_symlink.txt" - to_path = firecrest_server.scratch_path + "/copy_to_symlink.txt" + _scratch = tmpdir / "sampledir" + _scratch.mkdir() - transport.write_binary(from_path, b"test") - transport.symlink(from_path, from_path_symlink) + assert transport.isdir(_scratch) + assert not transport.isdir(_scratch / "does_not_exist") - assert not transport.path_exists(to_path) - transport.copyfile(from_path_symlink, to_path, dereference=False) - assert transport.isfile(to_path) - assert transport.read_binary(to_path) == b"test" - - -def test_copyfile_symlink_deref( - firecrest_server: FirecrestConfig, transport: FirecrestTransport -): - from_path = firecrest_server.scratch_path + "/copy_from.txt" - from_path_symlink = firecrest_server.scratch_path + "/copy_from_symlink.txt" - to_path = firecrest_server.scratch_path + "/copy_to_symlink.txt" - - transport.write_binary(from_path, b"test") - transport.symlink(from_path, from_path_symlink) - - assert not transport.path_exists(to_path) - transport.copyfile(from_path_symlink, to_path, dereference=True) - assert transport.isfile(to_path) - assert transport.read_binary(to_path) == b"test" +@pytest.mark.usefixtures("aiida_profile_clean") +def test_normalize(firecrest_computer: orm.Computer): + transport = firecrest_computer.get_transport() + assert transport.normalize("/path/to/dir") == os.path.normpath("/path/to/dir") + assert transport.normalize("path/to/dir") == os.path.normpath("path/to/dir") + assert transport.normalize("path/to/dir/") == os.path.normpath("path/to/dir/") + assert transport.normalize("path/to/../dir") == os.path.normpath("path/to/../dir") + assert transport.normalize("path/to/../../dir") == os.path.normpath( + "path/to/../../dir" + ) + assert transport.normalize("path/to/../../dir/") == os.path.normpath( + "path/to/../../dir/" + ) + assert transport.normalize("path/to/../../dir/../") == os.path.normpath( + "path/to/../../dir/../" + ) -def test_remove(firecrest_server: FirecrestConfig, transport: FirecrestTransport): - transport.write_binary(firecrest_server.scratch_path + "/file.txt", b"test") - assert transport.path_exists(firecrest_server.scratch_path + "/file.txt") - transport.remove(firecrest_server.scratch_path + "/file.txt") - assert not transport.path_exists(firecrest_server.scratch_path + "/file.txt") +@pytest.mark.usefixtures("aiida_profile_clean") +def test_remove(firecrest_computer: orm.Computer, tmpdir: Path): + transport = firecrest_computer.get_transport() + + _scratch = tmpdir / "samplefile" + Path(_scratch).touch() + transport.remove(_scratch) + assert not _scratch.exists() + + _scratch = tmpdir / "sampledir" + _scratch.mkdir() + transport.rmtree(_scratch) + assert not _scratch.exists() + + _scratch = tmpdir / "sampledir" + _scratch.mkdir() + Path(_scratch / "samplefile").touch() + with pytest.raises(OSError): + transport.rmdir(_scratch) + + os.remove(_scratch / "samplefile") + transport.rmdir(_scratch) + assert not _scratch.exists() + + +@pytest.mark.usefixtures("aiida_profile_clean") +def test_symlink(firecrest_computer: orm.Computer, tmpdir: Path): + transport = firecrest_computer.get_transport() + + _scratch = Path(tmpdir / "samplefile-2sym") + Path(_scratch).touch() + _symlink = Path(tmpdir / "samplelink") + transport.symlink(_scratch, _symlink) + assert _symlink.is_symlink() + assert _symlink.resolve() == _scratch + + +@pytest.mark.usefixtures("aiida_profile_clean") +def test_listdir(firecrest_computer: orm.Computer, tmpdir: Path): + transport = firecrest_computer.get_transport() + + _scratch = tmpdir / "sampledir" + _scratch.mkdir() + # to test basics + Path(_scratch / "file1").touch() + Path(_scratch / "dir1").mkdir() + Path(_scratch / ".hidden").touch() + # to test recursive + Path(_scratch / "dir1" / "file2").touch() + + assert set(transport.listdir(_scratch)) == {"file1", "dir1", ".hidden"} + assert set(transport.listdir(_scratch, recursive=True)) == { + "file1", + "dir1", + ".hidden", + "dir1/file2", + } + # to test symlink + Path(_scratch / "dir1" / "dir2").mkdir() + Path(_scratch / "dir1" / "dir2" / "file3").touch() + os.symlink(_scratch / "dir1" / "dir2", _scratch / "dir2_link") + os.symlink(_scratch / "dir1" / "file2", _scratch / "file_link") + + assert set(transport.listdir(_scratch, recursive=True)) == { + "file1", + "dir1", + ".hidden", + "dir1/file2", + "dir1/dir2", + "dir1/dir2/file3", + "dir2_link", + "file_link", + } -def test_rmtree(firecrest_server: FirecrestConfig, transport: FirecrestTransport): - transport.mkdir(firecrest_server.scratch_path + "/test") - transport.write_binary(firecrest_server.scratch_path + "/test/file.txt", b"test") - assert transport.path_exists(firecrest_server.scratch_path + "/test/file.txt") - transport.rmtree(firecrest_server.scratch_path + "/test") - assert not transport.path_exists(firecrest_server.scratch_path + "/test") + assert set(transport.listdir(_scratch / "dir2_link", recursive=False)) == {"file3"} -def test_rename(firecrest_server: FirecrestConfig, transport: FirecrestTransport): - transport.write_binary(firecrest_server.scratch_path + "/file.txt", b"test") - assert transport.path_exists(firecrest_server.scratch_path + "/file.txt") - transport.rename( - firecrest_server.scratch_path + "/file.txt", - firecrest_server.scratch_path + "/file2.txt", - ) - assert not transport.path_exists(firecrest_server.scratch_path + "/file.txt") - assert transport.path_exists(firecrest_server.scratch_path + "/file2.txt") +@pytest.mark.usefixtures("aiida_profile_clean") +def test_get(firecrest_computer: orm.Computer, tmpdir: Path): + """ + This is minimal test is to check if get() is raising errors as expected, + and directing to getfile() and gettree() as expected. + Mainly just checking error handeling and folder creation. + """ + transport = firecrest_computer.get_transport() + + _remote = tmpdir / "remotedir" + _local = tmpdir / "localdir" + _remote.mkdir() + _local.mkdir() + + # check if the code is directing to getfile() or gettree() as expected + with patch.object(transport, "gettree", autospec=True) as mock_gettree: + transport.get(_remote, _local) + mock_gettree.assert_called_once() + + with patch.object(transport, "gettree", autospec=True) as mock_gettree: + os.symlink(_remote, tmpdir / "dir_link") + transport.get(tmpdir / "dir_link", _local) + mock_gettree.assert_called_once() + + with patch.object(transport, "getfile", autospec=True) as mock_getfile: + Path(_remote / "file1").write_text("file1") + transport.get(_remote / "file1", _local / "file1") + mock_getfile.assert_called_once() + + with patch.object(transport, "getfile", autospec=True) as mock_getfile: + os.symlink(_remote / "file1", _remote / "file1_link") + transport.get(_remote / "file1_link", _local / "file1_link") + mock_getfile.assert_called_once() + + # raise if remote file/folder does not exist + with pytest.raises(FileNotFoundError): + transport.get(_remote / "does_not_exist", _local) + transport.get(_remote / "does_not_exist", _local, ignore_nonexisting=True) + + # raise if localpath is relative + with pytest.raises(ValueError): + transport.get(_remote, Path(_local).relative_to(tmpdir)) + with pytest.raises(ValueError): + transport.get(_remote / "file1", Path(_local).relative_to(tmpdir)) + + +@pytest.mark.usefixtures("aiida_profile_clean") +def test_getfile(firecrest_computer: orm.Computer, tmpdir: Path): + transport = firecrest_computer.get_transport() + + _remote = tmpdir / "remotedir" + _local = tmpdir / "localdir" + _remote.mkdir() + _local.mkdir() + + Path(_remote / "file1").write_text("file1") + Path(_remote / ".hidden").write_text(".hidden") + os.symlink(_remote / "file1", _remote / "file1_link") + + # raise if remote file does not exist + with pytest.raises(FileNotFoundError): + transport.getfile(_remote / "does_not_exist", _local) + + # raise if localfilename not provided + with pytest.raises(IsADirectoryError): + transport.getfile(_remote / "file1", _local) + + # raise if localpath is relative + with pytest.raises(ValueError): + transport.getfile(_remote / "file1", Path(_local / "file1").relative_to(tmpdir)) + + # don't mix up directory with file + with pytest.raises(FileNotFoundError): + transport.getfile(_remote, _local / "file1") + + # write where I tell you to + transport.getfile(_remote / "file1", _local / "file1") + transport.getfile(_remote / "file1", _local / "file1-prime") + assert Path(_local / "file1").read_text() == "file1" + assert Path(_local / "file1-prime").read_text() == "file1" + + # always overwrite + transport.getfile(_remote / "file1", _local / "file1") + assert Path(_local / "file1").read_text() == "file1" + + Path(_local / "file1").write_text("notfile1") + + transport.getfile(_remote / "file1", _local / "file1") + assert Path(_local / "file1").read_text() == "file1" + + # don't skip hidden files + transport.getfile(_remote / ".hidden", _local / ".hidden-prime") + assert Path(_local / ".hidden-prime").read_text() == ".hidden" + + # follow links + transport.getfile(_remote / "file1_link", _local / "file1_link") + assert Path(_local / "file1_link").read_text() == "file1" + assert not Path(_local / "file1_link").is_symlink() + + +@pytest.mark.parametrize("payoff", [True, False]) +@pytest.mark.usefixtures("aiida_profile_clean") +def test_gettree(firecrest_computer: orm.Computer, tmpdir: Path, payoff: bool): + """ + This test is to check `gettree` through non tar mode. + bytar= True in this test. + """ + transport = firecrest_computer.get_transport() + transport.payoff_override = payoff + + # Note: + # SSH transport behaviour, 69 is a directory + # transport.get('somepath/69', 'someremotepath/') == transport.get('somepath/69', 'someremotepath') + # transport.get('somepath/69', 'someremotepath/') == transport.get('somepath/69/', 'someremotepath/') + # transport.get('someremotepath/69', 'somepath/69')--> if 69 exist, create 69 inside it ('somepath/69/69') + # transport.get('someremotepath/69', 'somepath/69')--> if 69 no texist,create 69 inside it ('somepath/69') + # transport.get('somepath/69', 'someremotepath/6889/69') --> create everything, make_parent = True + + _remote = tmpdir / "remotedir" + _local = tmpdir / "localdir" + _remote.mkdir() + _local.mkdir() + # a typical tree + Path(_remote / "file1").write_text("file1") + Path(_remote / ".hidden").write_text(".hidden") + Path(_remote / "dir1").mkdir() + Path(_remote / "dir1" / "file2").write_text("file2") + # with symlinks + Path(_remote / "dir2").mkdir() + Path(_remote / "dir2" / "file3").write_text("file3") + os.symlink(_remote / "file1", _remote / "dir1" / "file1_link") + os.symlink(_remote / "dir2", _remote / "dir1" / "dir2_link") + # if symlinks are pointing to a relative path + os.symlink(Path("../file1"), _remote / "dir1" / "file10_link") + os.symlink(Path("../dir2"), _remote / "dir1" / "dir20_link") + + # raise if remote file does not exist + with pytest.raises(OSError): + transport.gettree(_remote / "does_not_exist", _local) + + # raise if local is a file + Path(tmpdir / "isfile").touch() + with pytest.raises(OSError): + transport.gettree(_remote, tmpdir / "isfile") + + # raise if localpath is relative + with pytest.raises(ValueError): + transport.gettree(_remote, Path(_local).relative_to(tmpdir)) + + # If destination directory does not exists, AiiDA expects transport make the new path as root not _remote.name + transport.gettree(_remote, _local / "newdir") + _root = _local / "newdir" + # tree should be copied recursively + assert Path(_root / "file1").read_text() == "file1" + assert Path(_root / ".hidden").read_text() == ".hidden" + assert Path(_root / "dir1" / "file2").read_text() == "file2" + assert Path(_root / "dir2" / "file3").read_text() == "file3" + # symlink should be followed + assert Path(_root / "dir1" / "file1_link").read_text() == "file1" + assert Path(_root / "dir1" / "dir2_link" / "file3").read_text() == "file3" + assert Path(_root / "dir1" / "file10_link").read_text() == "file1" + assert Path(_root / "dir1" / "dir20_link" / "file3").read_text() == "file3" + assert not Path(_root / "dir1" / "file1_link").is_symlink() + assert not Path(_root / "dir1" / "dir2_link" / "file3").is_symlink() + assert not Path(_root / "dir1" / "file10_link").is_symlink() + assert not Path(_root / "dir1" / "dir20_link" / "file3").is_symlink() + + # If destination directory does exists, AiiDA expects transport make _remote.name and write into it + # however this might have changed in the newer versions of AiiDA ~ 2.6.0 (IDK) + transport.gettree(_remote, _local / "newdir") + _root = _local / "newdir" / Path(_remote).name + # tree should be copied recursively + assert Path(_root / "file1").read_text() == "file1" + assert Path(_root / ".hidden").read_text() == ".hidden" + assert Path(_root / "dir1" / "file2").read_text() == "file2" + assert Path(_root / "dir2" / "file3").read_text() == "file3" + # symlink should be followed + assert Path(_root / "dir1" / "file1_link").read_text() == "file1" + assert Path(_root / "dir1" / "dir2_link" / "file3").read_text() == "file3" + assert Path(_root / "dir1" / "file10_link").read_text() == "file1" + assert Path(_root / "dir1" / "dir20_link" / "file3").read_text() == "file3" + assert not Path(_root / "dir1" / "file1_link").is_symlink() + assert not Path(_root / "dir1" / "dir2_link" / "file3").is_symlink() + assert not Path(_root / "dir1" / "file10_link").is_symlink() + assert not Path(_root / "dir1" / "dir20_link" / "file3").is_symlink() + + +@pytest.mark.usefixtures("aiida_profile_clean") +def test_put(firecrest_computer: orm.Computer, tmpdir: Path): + """ + This is minimal test is to check if put() is raising errors as expected, + and directing to putfile() and puttree() as expected. + Mainly just checking error handeling and folder creation. + """ + transport = firecrest_computer.get_transport() + + _remote = tmpdir / "remotedir" + _local = tmpdir / "localdir" + _remote.mkdir() + _local.mkdir() + + # check if the code is directing to putfile() or puttree() as expected + with patch.object(transport, "puttree", autospec=True) as mock_puttree: + transport.put(_local, _remote) + mock_puttree.assert_called_once() + + with patch.object(transport, "puttree", autospec=True) as mock_puttree: + os.symlink(_local, tmpdir / "dir_link") + transport.put(tmpdir / "dir_link", _remote) + mock_puttree.assert_called_once() + + with patch.object(transport, "putfile", autospec=True) as mock_putfile: + Path(_local / "file1").write_text("file1") + transport.put(_local / "file1", _remote / "file1") + mock_putfile.assert_called_once() + + with patch.object(transport, "putfile", autospec=True) as mock_putfile: + os.symlink(_local / "file1", _local / "file1_link") + transport.put(_local / "file1_link", _remote / "file1_link") + mock_putfile.assert_called_once() + + # raise if local file/folder does not exist + with pytest.raises(FileNotFoundError): + transport.put(_local / "does_not_exist", _remote) + transport.put(_local / "does_not_exist", _remote, ignore_nonexisting=True) + + # raise if localpath is relative + with pytest.raises(ValueError): + transport.put(Path(_local).relative_to(tmpdir), _remote) + with pytest.raises(ValueError): + transport.put(Path(_local / "file1").relative_to(tmpdir), _remote) + + +@pytest.mark.usefixtures("aiida_profile_clean") +def test_putfile(firecrest_computer: orm.Computer, tmpdir: Path): + transport = firecrest_computer.get_transport() + + _remote = tmpdir / "remotedir" + _local = tmpdir / "localdir" + _remote.mkdir() + _local.mkdir() + + Path(_local / "file1").write_text("file1") + Path(_local / ".hidden").write_text(".hidden") + os.symlink(_local / "file1", _local / "file1_link") + + # raise if local file does not exist + with pytest.raises(FileNotFoundError): + transport.putfile(_local / "does_not_exist", _remote) + + # raise if remotefilename is not provided + with pytest.raises(ValueError): + transport.putfile(_local / "file1", _remote) + + # raise if localpath is relative + with pytest.raises(ValueError): + transport.putfile(Path(_local / "file1").relative_to(tmpdir), _remote / "file1") + + # don't mix up directory with file + with pytest.raises(ValueError): + transport.putfile(_local, _remote / "file1") + + # write where I tell you to + transport.putfile(_local / "file1", _remote / "file1") + transport.putfile(_local / "file1", _remote / "file1-prime") + assert Path(_remote / "file1").read_text() == "file1" + assert Path(_remote / "file1-prime").read_text() == "file1" + + # always overwrite + transport.putfile(_local / "file1", _remote / "file1") + assert Path(_remote / "file1").read_text() == "file1" + + Path(_remote / "file1").write_text("notfile1") + + transport.putfile(_local / "file1", _remote / "file1") + assert Path(_remote / "file1").read_text() == "file1" + + # don't skip hidden files + transport.putfile(_local / ".hidden", _remote / ".hidden-prime") + assert Path(_remote / ".hidden-prime").read_text() == ".hidden" + + # follow links + transport.putfile(_local / "file1_link", _remote / "file1_link") + assert Path(_remote / "file1_link").read_text() == "file1" + assert not Path(_remote / "file1_link").is_symlink() + + +@pytest.mark.parametrize("payoff", [True, False]) +@pytest.mark.usefixtures("aiida_profile_clean") +def test_puttree(firecrest_computer: orm.Computer, tmpdir: Path, payoff: bool): + """ + This test is to check `puttree` through non tar mode. + payoff= False in this test, so just checking if putting files one by one is working as expected. + """ + transport = firecrest_computer.get_transport() + transport.payoff_override = payoff + + # Note: + # SSH transport behaviour + # transport.put('somepath/69', 'someremotepath/') == transport.put('somepath/69', 'someremotepath') + # transport.put('somepath/69', 'someremotepath/') != transport.put('somepath/69/', 'someremotepath/') + # transport.put('somepath/69', 'someremotepath/67') --> if 67 not exist, create and move content 69 + # inside it (someremotepath/67) + # transport.put('somepath/69', 'someremotepath/67') --> if 67 exist, create 69 inside it (someremotepath/67/69) + # transport.put('somepath/69', 'someremotepath/6889/69') --> useless Error: OSError + # Weired + # SSH bug: + # transport.put('somepath/69', 'someremotepath/') --> assuming someremotepath exists, make 69 + # while + # transport.put('somepath/69/', 'someremotepath/') --> assuming someremotepath exists, OSError: + # cannot make someremotepath + + _remote = tmpdir / "remotedir" + _local = tmpdir / "localdir" + _remote.mkdir() + _local.mkdir() + # a typical tree + Path(_local / "dir1").mkdir() + Path(_local / "dir2").mkdir() + Path(_local / "file1").write_text("file1") + Path(_local / ".hidden").write_text(".hidden") + Path(_local / "dir1" / "file2").write_text("file2") + Path(_local / "dir2" / "file3").write_text("file3") + # with symlinks to a file even if pointing to a relative path + os.symlink(_local / "file1", _local / "dir1" / "file1_link") + os.symlink(Path("../file1"), _local / "dir1" / "file10_link") + # with symlinks to a folder even if pointing to a relative path + os.symlink(_local / "dir2", _local / "dir1" / "dir2_link") + os.symlink(Path("../dir2"), _local / "dir1" / "dir20_link") + + # raise if local file does not exist + with pytest.raises(OSError): + transport.puttree(_local / "does_not_exist", _remote) + + # raise if local is a file + with pytest.raises(ValueError): + Path(tmpdir / "isfile").touch() + transport.puttree(tmpdir / "isfile", _remote) + + # raise if localpath is relative + with pytest.raises(ValueError): + transport.puttree(Path(_local).relative_to(tmpdir), _remote) + + # If destination directory does not exists, AiiDA expects transport make the new path as root not using _local.name + transport.puttree(_local, _remote / "newdir") + _root = _remote / "newdir" + # tree should be copied recursively + assert Path(_root / "file1").read_text() == "file1" + assert Path(_root / ".hidden").read_text() == ".hidden" + assert Path(_root / "dir1" / "file2").read_text() == "file2" + assert Path(_root / "dir2" / "file3").read_text() == "file3" + # symlink should be followed + assert Path(_root / "dir1" / "file1_link").read_text() == "file1" + assert Path(_root / "dir1" / "dir2_link" / "file3").read_text() == "file3" + assert Path(_root / "dir1" / "file10_link").read_text() == "file1" + assert Path(_root / "dir1" / "dir20_link" / "file3").read_text() == "file3" + assert not Path(_root / "dir1" / "file1_link").is_symlink() + assert not Path(_root / "dir1" / "dir2_link" / "file3").is_symlink() + assert not Path(_root / "dir1" / "file10_link").is_symlink() + assert not Path(_root / "dir1" / "dir20_link" / "file3").is_symlink() + + # If destination directory does exists, AiiDA expects transport make _local.name and write into it + # however this might have changed in the newer versions of AiiDA ~ 2.6.0 (IDK) + transport.puttree(_local, _remote / "newdir") + _root = _remote / "newdir" / Path(_local).name + # tree should be copied recursively + assert Path(_root / "file1").read_text() == "file1" + assert Path(_root / ".hidden").read_text() == ".hidden" + assert Path(_root / "dir1" / "file2").read_text() == "file2" + assert Path(_root / "dir2" / "file3").read_text() == "file3" + # symlink should be followed + assert Path(_root / "dir1" / "file1_link").read_text() == "file1" + assert Path(_root / "dir1" / "dir2_link" / "file3").read_text() == "file3" + assert Path(_root / "dir1" / "file10_link").read_text() == "file1" + assert Path(_root / "dir1" / "dir20_link" / "file3").read_text() == "file3" + assert not Path(_root / "dir1" / "file1_link").is_symlink() + assert not Path(_root / "dir1" / "dir2_link" / "file3").is_symlink() + assert not Path(_root / "dir1" / "file10_link").is_symlink() + assert not Path(_root / "dir1" / "dir20_link" / "file3").is_symlink() + + +@pytest.mark.parametrize("to_test", ["copy", "copytree"]) +@pytest.mark.usefixtures("aiida_profile_clean") +def test_copy(firecrest_computer: orm.Computer, tmpdir: Path, to_test: str): + transport = firecrest_computer.get_transport() + if to_test == "copy": + testing = transport.copy + elif to_test == "copytree": + testing = transport.copytree + + _remote_1 = tmpdir / "remotedir_1" + _remote_2 = tmpdir / "remotedir_2" + _remote_1.mkdir() + _remote_2.mkdir() + + # raise if source or destination does not exist + with pytest.raises(FileNotFoundError): + testing(_remote_1 / "does_not_exist", _remote_2) + with pytest.raises(FileNotFoundError): + testing(_remote_1, _remote_2 / "does_not_exist") + + # raise if source is inappropriate + if to_test == "copytree": + Path(tmpdir / "file1").touch() + with pytest.raises(ValueError): + testing(tmpdir / "file1", _remote_2) + + # a typical tree + Path(_remote_1 / "dir1").mkdir() + Path(_remote_1 / "dir2").mkdir() + Path(_remote_1 / "file1").write_text("file1") + Path(_remote_1 / ".hidden").write_text(".hidden") + Path(_remote_1 / "dir1" / "file2").write_text("file2") + Path(_remote_1 / "dir2" / "file3").write_text("file3") + # with symlinks to a file even if pointing to a relative path + os.symlink(_remote_1 / "file1", _remote_1 / "dir1" / "file1_link") + os.symlink(Path("../file1"), _remote_1 / "dir1" / "file10_link") + # with symlinks to a folder even if pointing to a relative path + os.symlink(_remote_1 / "dir2", _remote_1 / "dir1" / "dir2_link") + os.symlink(Path("../dir2"), _remote_1 / "dir1" / "dir20_link") + + testing(_remote_1, _remote_2) + + _root_2 = _remote_2 / Path(_remote_1).name + # tree should be copied recursively + assert Path(_root_2 / "dir1").exists() + assert Path(_root_2 / "dir2").exists() + assert Path(_root_2 / "file1").read_text() == "file1" + assert Path(_root_2 / ".hidden").read_text() == ".hidden" + assert Path(_root_2 / "dir1" / "file2").read_text() == "file2" + assert Path(_root_2 / "dir2" / "file3").read_text() == "file3" + # symlink should be followed + assert Path(_root_2 / "dir1" / "dir2_link").exists() + assert Path(_root_2 / "dir1" / "dir20_link").exists() + assert Path(_root_2 / "dir1" / "file1_link").read_text() == "file1" + assert Path(_root_2 / "dir1" / "file10_link").read_text() == "file1" + assert Path(_root_2 / "dir1" / "file1_link").is_symlink() + assert Path(_root_2 / "dir1" / "dir2_link").is_symlink() + assert Path(_root_2 / "dir1" / "file10_link").is_symlink() + assert Path(_root_2 / "dir1" / "dir20_link").is_symlink() + + +@pytest.mark.usefixtures("aiida_profile_clean") +def test_copyfile(firecrest_computer: orm.Computer, tmpdir: Path): + transport = firecrest_computer.get_transport() + testing = transport.copyfile + + _remote_1 = tmpdir / "remotedir_1" + _remote_2 = tmpdir / "remotedir_2" + _remote_1.mkdir() + _remote_2.mkdir() + + # raise if source or destination does not exist + with pytest.raises(FileNotFoundError): + testing(_remote_1 / "does_not_exist", _remote_2) + # in this case don't raise and just create the file + Path(tmpdir / "_").touch() + testing(tmpdir / "_", _remote_2 / "does_not_exist") + + # raise if source is unappropriate + with pytest.raises(ValueError): + testing(tmpdir, _remote_2) + + # a typical tree + Path(_remote_1 / "file1").write_text("file1") + Path(_remote_1 / ".hidden").write_text(".hidden") + # with symlinks to a file even if pointing to a relative path + os.symlink(_remote_1 / "file1", _remote_1 / "file1_link") + os.symlink(Path("file1"), _remote_1 / "file10_link") + + # write where I tell you to + testing(_remote_1 / "file1", _remote_2 / "file1") + assert Path(_remote_2 / "file1").read_text() == "file1" + + # always overwrite + Path(_remote_2 / "file1").write_text("notfile1") + testing(_remote_1 / "file1", _remote_2 / "file1") + assert Path(_remote_2 / "file1").read_text() == "file1" + + # don't skip hidden files + testing(_remote_1 / ".hidden", _remote_2 / ".hidden-prime") + assert Path(_remote_2 / ".hidden-prime").read_text() == ".hidden" + + # preserve links and don't follow them + testing(_remote_1 / "file1_link", _remote_2 / "file1_link") + assert Path(_remote_2 / "file1_link").read_text() == "file1" + assert Path(_remote_2 / "file1_link").is_symlink() + testing(_remote_1 / "file10_link", _remote_2 / "file10_link") + assert Path(_remote_2 / "file10_link").read_text() == "file1" + assert Path(_remote_2 / "file10_link").is_symlink() diff --git a/tests/tests_mocking_pyfirecrest/conftest.py b/tests/tests_mocking_pyfirecrest/conftest.py deleted file mode 100644 index 2e4a116..0000000 --- a/tests/tests_mocking_pyfirecrest/conftest.py +++ /dev/null @@ -1,369 +0,0 @@ -import hashlib -import os -from pathlib import Path -import random -import stat -from typing import Optional - -from aiida import orm -import firecrest -import firecrest.path -import pytest - - -class Values: - _DEFAULT_PAGE_SIZE: int = 25 - - -@pytest.fixture(name="firecrest_computer") -def _firecrest_computer(myfirecrest, tmpdir: Path): - """Create and return a computer configured for Firecrest. - - Note, the computer is not stored in the database. - """ - - # create a temp directory and set it as the workdir - _scratch = tmpdir / "scratch" - _temp_directory = tmpdir / "temp" - _scratch.mkdir() - _temp_directory.mkdir() - - Path(tmpdir / ".firecrest").mkdir() - _secret_path = Path(tmpdir / ".firecrest/secret69") - _secret_path.write_text("SECRET_STRING") - - computer = orm.Computer( - label="test_computer", - description="test computer", - hostname="-", - workdir=str(_scratch), - transport_type="firecrest", - scheduler_type="firecrest", - ) - computer.set_minimum_job_poll_interval(5) - computer.set_default_mpiprocs_per_machine(1) - computer.configure( - url=" https://URI", - token_uri="https://TOKEN_URI", - client_id="CLIENT_ID", - client_secret=str(_secret_path), - client_machine="MACHINE_NAME", - small_file_size_mb=1.0, - temp_directory=str(_temp_directory), - ) - return computer - - -class MockFirecrest: - def __init__(self, firecrest_url, *args, **kwargs): - self._firecrest_url = firecrest_url - self.args = args - self.kwargs = kwargs - - self.whoami = whomai - self.list_files = list_files - self.stat = stat_ - self.mkdir = mkdir - self.simple_delete = simple_delete - self.parameters = parameters - self.symlink = symlink - self.checksum = checksum - self.simple_download = simple_download - self.simple_upload = simple_upload - self.compress = compress - self.extract = extract - self.copy = copy - self.submit = submit - self.poll_active = poll_active - - -class MockClientCredentialsAuth: - def __init__(self, *args, **kwargs): - self.args = args - self.kwargs = kwargs - - -@pytest.fixture(scope="function") -def myfirecrest( - pytestconfig: pytest.Config, - monkeypatch, -): - monkeypatch.setattr(firecrest, "Firecrest", MockFirecrest) - monkeypatch.setattr(firecrest, "ClientCredentialsAuth", MockClientCredentialsAuth) - - -def submit( - machine: str, - script_str: Optional[str] = None, - script_remote_path: Optional[str] = None, - script_local_path: Optional[str] = None, - local_file=False, -): - if local_file: - raise DeprecationWarning("local_file is not supported") - - if script_remote_path and not Path(script_remote_path).exists(): - raise FileNotFoundError(f"File {script_remote_path} does not exist") - job_id = random.randint(10000, 99999) - return {"jobid": job_id} - - -def poll_active(machine: str, jobs: list[str], page_number: int = 0): - response = [] - # 12 satets are defined in firecrest - states = [ - "TIMEOUT", - "SUSPENDED", - "PREEMPTED", - "CANCELLED", - "NODE_FAIL", - "PENDING", - "FAILED", - "RUNNING", - "CONFIGURING", - "QUEUED", - "COMPLETED", - "COMPLETING", - ] - for i in range(len(jobs)): - response.append( - { - "job_data_err": "", - "job_data_out": "", - "job_file": "somefile.sh", - "job_file_err": "somefile-stderr.txt", - "job_file_out": "somefile-stdout.txt", - "job_info_extra": "Job info returned successfully", - "jobid": f"{jobs[i]}", - "name": "aiida-45", - "nodelist": "nid00049", - "nodes": "1", - "partition": "normal", - "start_time": "0:03", - "state": states[i % 12], - "time": "2024-06-21T10:44:42", - "time_left": "29:57", - "user": "Prof. Wang", - } - ) - - return response[ - page_number - * Values._DEFAULT_PAGE_SIZE : (page_number + 1) - * Values._DEFAULT_PAGE_SIZE - ] - - -def whomai(machine: str): - assert machine == "MACHINE_NAME" - return "test_user" - - -def list_files( - machine: str, target_path: str, recursive: bool = False, show_hidden: bool = False -): - # this is mimiking the expected behaviour from the firecrest code. - - content_list = [] - for root, dirs, files in os.walk(target_path): - if not recursive and root != target_path: - continue - for name in dirs + files: - full_path = os.path.join(root, name) - relative_path = ( - Path(os.path.relpath(root, target_path)).joinpath(name).as_posix() - ) - if os.path.islink(full_path): - content_type = "l" - link_target = ( - os.readlink(full_path) if os.path.islink(full_path) else None - ) - elif os.path.isfile(full_path): - content_type = "-" - link_target = None - elif os.path.isdir(full_path): - content_type = "d" - link_target = None - else: - content_type = "NON" - link_target = None - permissions = stat.filemode(Path(full_path).lstat().st_mode)[1:] - if name.startswith(".") and not show_hidden: - continue - content_list.append( - { - "name": relative_path, - "type": content_type, - "link_target": link_target, - "permissions": permissions, - } - ) - - return content_list - - -def stat_(machine: str, targetpath: firecrest.path, dereference=True): - stats = os.stat( - targetpath, follow_symlinks=bool(dereference) if dereference else False - ) - return { - "ino": stats.st_ino, - "dev": stats.st_dev, - "nlink": stats.st_nlink, - "uid": stats.st_uid, - "gid": stats.st_gid, - "size": stats.st_size, - "atime": stats.st_atime, - "mtime": stats.st_mtime, - "ctime": stats.st_ctime, - } - - -def mkdir(machine: str, target_path: str, p: bool = False): - if p: - os.makedirs(target_path) - else: - os.mkdir(target_path) - - -def simple_delete(machine: str, target_path: str): - if not Path(target_path).exists(): - raise FileNotFoundError(f"File or folder {target_path} does not exist") - if os.path.isdir(target_path): - os.rmdir(target_path) - else: - os.remove(target_path) - - -def symlink(machine: str, target_path: str, link_path: str): - # this is how firecrest does it - os.system(f"ln -s {target_path} {link_path}") - - -def simple_download(machine: str, remote_path: str, local_path: str): - # this procedure is complecated in firecrest, but I am simplifying it here - # we don't care about the details of the download, we just want to make sure - # that the aiida-firecrest code is calling the right functions at right time - if Path(remote_path).is_dir(): - raise IsADirectoryError(f"{remote_path} is a directory") - if not Path(remote_path).exists(): - raise FileNotFoundError(f"{remote_path} does not exist") - os.system(f"cp {remote_path} {local_path}") - - -def simple_upload( - machine: str, local_path: str, remote_path: str, file_name: Optional[str] = None -): - # this procedure is complecated in firecrest, but I am simplifying it here - # we don't care about the details of the upload, we just want to make sure - # that the aiida-firecrest code is calling the right functions at right time - if Path(local_path).is_dir(): - raise IsADirectoryError(f"{local_path} is a directory") - if not Path(local_path).exists(): - raise FileNotFoundError(f"{local_path} does not exist") - if file_name: - remote_path = os.path.join(remote_path, file_name) - os.system(f"cp {local_path} {remote_path}") - - -def copy(machine: str, source_path: str, target_path: str): - # this is how firecrest does it - # https://github.com/eth-cscs/firecrest/blob/db6ba4ba273c11a79ecbe940872f19d5cb19ac5e/src/utilities/utilities.py#L451C1-L452C1 - os.system(f"cp --force -dR --preserve=all -- '{source_path}' '{target_path}'") - - -def compress( - machine: str, source_path: str, target_path: str, dereference: bool = True -): - # this is how firecrest does it - # https://github.com/eth-cscs/firecrest/blob/db6ba4ba273c11a79ecbe940872f19d5cb19ac5e/src/utilities/utilities.py#L460 - basedir = os.path.dirname(source_path) - file_path = os.path.basename(source_path) - deref = "--dereference" if dereference else "" - os.system(f"tar {deref} -czf '{target_path}' -C '{basedir}' '{file_path}'") - - -def extract(machine: str, source_path: str, target_path: str): - # this is how firecrest does it - # https://github.com/eth-cscs/firecrest/blob/db6ba4ba273c11a79ecbe940872f19d5cb19ac5e/src/common/cscs_api_common.py#L1110C18-L1110C65 - os.system(f"tar -xf '{source_path}' -C '{target_path}'") - - -def checksum(machine: str, remote_path: str) -> int: - if not remote_path.exists(): - return False - # Firecrest uses sha256 - sha256_hash = hashlib.sha256() - with open(remote_path, "rb") as f: - for byte_block in iter(lambda: f.read(4096), b""): - sha256_hash.update(byte_block) - - return sha256_hash.hexdigest() - - -def parameters(): - # note: I took this from https://firecrest-tds.cscs.ch/ or https://firecrest.cscs.ch/ - # if code is not working but test passes, it means you need to update this dictionary - # with the latest FirecREST parameters - return { - "compute": [ - { - "description": "Type of resource and workload manager used in compute microservice", - "name": "WORKLOAD_MANAGER", - "unit": "", - "value": "Slurm", - } - ], - "storage": [ - { - "description": "Type of object storage, like `swift`, `s3v2` or `s3v4`.", - "name": "OBJECT_STORAGE", - "unit": "", - "value": "s3v4", - }, - { - "description": "Expiration time for temp URLs.", - "name": "STORAGE_TEMPURL_EXP_TIME", - "unit": "seconds", - "value": "86400", - }, - { - "description": "Maximum file size for temp URLs.", - "name": "STORAGE_MAX_FILE_SIZE", - "unit": "MB", - "value": "5120", - }, - { - "description": "Available filesystems through the API.", - "name": "FILESYSTEMS", - "unit": "", - "value": [ - { - "mounted": ["/project", "/store", "/scratch/snx3000tds"], - "system": "dom", - }, - { - "mounted": ["/project", "/store", "/capstor/scratch/cscs"], - "system": "pilatus", - }, - ], - }, - ], - "utilities": [ - { - "description": "The maximum allowable file size for various operations of the utilities microservice", - "name": "UTILITIES_MAX_FILE_SIZE", - "unit": "MB", - "value": "69", - }, - { - "description": ( - "Maximum time duration for executing the commands " - "in the cluster for the utilities microservice." - ), - "name": "UTILITIES_TIMEOUT", - "unit": "seconds", - "value": "5", - }, - ], - } diff --git a/tests/tests_mocking_pyfirecrest/test_computer.py b/tests/tests_mocking_pyfirecrest/test_computer.py deleted file mode 100644 index fe4e48e..0000000 --- a/tests/tests_mocking_pyfirecrest/test_computer.py +++ /dev/null @@ -1,121 +0,0 @@ -from pathlib import Path -from unittest.mock import Mock - -from aiida import orm -from click import BadParameter -import pytest - - -@pytest.mark.usefixtures("aiida_profile_clean") -def test_whoami(firecrest_computer: orm.Computer): - """check if it is possible to determine the username.""" - transport = firecrest_computer.get_transport() - assert transport.whoami() == "test_user" - - -def test_create_secret_file_with_existing_file(tmpdir: Path): - from aiida_firecrest.transport import _create_secret_file - - secret_file = Path(tmpdir / "secret") - secret_file.write_text("topsecret") - result = _create_secret_file(None, None, str(secret_file)) - assert isinstance(result, str) - assert result == str(secret_file) - assert Path(result).read_text() == "topsecret" - - -def test_create_secret_file_with_nonexistent_file(tmp_path): - from aiida_firecrest.transport import _create_secret_file - - secret_file = tmp_path / "nonexistent" - with pytest.raises(BadParameter): - _create_secret_file(None, None, str(secret_file)) - - -def test_create_secret_file_with_secret_value(tmp_path, monkeypatch): - from aiida_firecrest.transport import _create_secret_file - - secret = "topsecret!~/" - monkeypatch.setattr( - Path, - "expanduser", - lambda x: tmp_path / str(x).lstrip("~/") if str(x).startswith("~/") else x, - ) - result = _create_secret_file(None, None, secret) - assert Path(result).parent.parts[-1] == ".firecrest" - assert Path(result).read_text() == secret - - -def test_validate_temp_directory(myfirecrest, monkeypatch, tmpdir: Path): - from aiida_firecrest.transport import _validate_temp_directory - - monkeypatch.setattr("click.echo", lambda x: None) - # monkeypatch.setattr('click.BadParameter', lambda x: None) - secret_file = Path(tmpdir / "secret") - secret_file.write_text("topsecret") - ctx = Mock() - ctx.params = { - "url": "http://test.com", - "token_uri": "token_uri", - "client_id": "client_id", - "client_machine": "client_machine", - "client_secret": secret_file.as_posix(), - "small_file_size_mb": float(10), - } - - # should raise if is_file - Path(tmpdir / "crap.txt").touch() - with pytest.raises(BadParameter): - result = _validate_temp_directory( - ctx, None, Path(tmpdir / "crap.txt").as_posix() - ) - - # should create the directory if it doesn't exist - result = _validate_temp_directory( - ctx, None, Path(tmpdir / "temp_on_server_directory").as_posix() - ) - assert result == Path(tmpdir / "temp_on_server_directory").as_posix() - assert Path(tmpdir / "temp_on_server_directory").exists() - - # should get a confirmation if the directory exists and is not empty - Path(tmpdir / "temp_on_server_directory" / "crap.txt").touch() - monkeypatch.setattr("click.confirm", lambda x: False) - with pytest.raises(BadParameter): - result = _validate_temp_directory( - ctx, None, Path(tmpdir / "temp_on_server_directory").as_posix() - ) - - # should delete the content if I confirm - monkeypatch.setattr("click.confirm", lambda x: True) - result = _validate_temp_directory( - ctx, None, Path(tmpdir / "temp_on_server_directory").as_posix() - ) - assert result == Path(tmpdir / "temp_on_server_directory").as_posix() - assert not Path(tmpdir / "temp_on_server_directory" / "crap.txt").exists() - - -def test__dynamic_info(myfirecrest, monkeypatch, tmpdir: Path): - from aiida_firecrest.transport import _dynamic_info_direct_size - - monkeypatch.setattr("click.echo", lambda x: None) - # monkeypatch.setattr('click.BadParameter', lambda x: None) - secret_file = Path(tmpdir / "secret") - secret_file.write_text("topsecret") - ctx = Mock() - ctx.params = { - "url": "http://test.com", - "token_uri": "token_uri", - "client_id": "client_id", - "client_machine": "client_machine", - "client_secret": secret_file.as_posix(), - "small_file_size_mb": float(10), - } - - # should catch UTILITIES_MAX_FILE_SIZE if value is not provided - result = _dynamic_info_direct_size(ctx, None, 0) - assert result == 69 - - # should use the value if provided - # note: user cannot enter negative numbers anyways, click raise as this shoule be float not str - result = _dynamic_info_direct_size(ctx, None, 10) - assert result == 10 diff --git a/tests/tests_mocking_pyfirecrest/test_scheduler.py b/tests/tests_mocking_pyfirecrest/test_scheduler.py deleted file mode 100644 index b4cb26f..0000000 --- a/tests/tests_mocking_pyfirecrest/test_scheduler.py +++ /dev/null @@ -1,139 +0,0 @@ -from pathlib import Path -import random - -from aiida import orm -from aiida.schedulers.datastructures import CodeRunMode, JobTemplate -import pytest - -from aiida_firecrest.scheduler import FirecrestScheduler -from conftest import Values - - -@pytest.mark.usefixtures("aiida_profile_clean") -def test_submit_job(firecrest_computer: orm.Computer, tmp_path: Path): - transport = firecrest_computer.get_transport() - scheduler = FirecrestScheduler() - scheduler.set_transport(transport) - - with pytest.raises(FileNotFoundError): - scheduler.submit_job(transport.getcwd(), "unknown.sh") - - _script = Path(tmp_path / "job.sh") - _script.write_text("#!/bin/bash\n\necho 'hello world'") - - job_id = scheduler.submit_job(transport.getcwd(), _script) - # this is how aiida expects the job_id to be returned - assert isinstance(job_id, str) - - -@pytest.mark.usefixtures("aiida_profile_clean") -def test_get_jobs(firecrest_computer: orm.Computer): - transport = firecrest_computer.get_transport() - scheduler = FirecrestScheduler() - scheduler.set_transport(transport) - - # test pagaination - scheduler._DEFAULT_PAGE_SIZE = 2 - Values._DEFAULT_PAGE_SIZE = 2 - - joblist = [random.randint(10000, 99999) for i in range(5)] - result = scheduler.get_jobs(joblist) - assert len(result) == 5 - for i in range(5): - assert result[i].job_id == str(joblist[i]) - # TODO: one could check states as well - - -def test_write_script_full(): - # to avoid false positive (overwriting on existing file), - # we check the output of the script instead of using `file_regression`` - expectaion = """ - #!/bin/bash - #SBATCH -H - #SBATCH --requeue - #SBATCH --mail-user=True - #SBATCH --mail-type=BEGIN - #SBATCH --mail-type=FAIL - #SBATCH --mail-type=END - #SBATCH --job-name="test_job" - #SBATCH --get-user-env - #SBATCH --output=test.out - #SBATCH --error=test.err - #SBATCH --partition=test_queue - #SBATCH --account=test_account - #SBATCH --qos=test_qos - #SBATCH --nice=100 - #SBATCH --nodes=1 - #SBATCH --ntasks-per-node=1 - #SBATCH --time=01:00:00 - #SBATCH --mem=1 - test_command - """ - expectaion_flat = "\n".join(line.strip() for line in expectaion.splitlines()).strip( - "\n" - ) - scheduler = FirecrestScheduler() - template = JobTemplate( - { - "job_resource": scheduler.create_job_resource( - num_machines=1, num_mpiprocs_per_machine=1 - ), - "codes_info": [], - "codes_run_mode": CodeRunMode.SERIAL, - "submit_as_hold": True, - "rerunnable": True, - "email": True, - "email_on_started": True, - "email_on_terminated": True, - "job_name": "test_job", - "import_sys_environment": True, - "sched_output_path": "test.out", - "sched_error_path": "test.err", - "queue_name": "test_queue", - "account": "test_account", - "qos": "test_qos", - "priority": 100, - "max_wallclock_seconds": 3600, - "max_memory_kb": 1024, - "custom_scheduler_commands": "test_command", - } - ) - try: - assert scheduler.get_submit_script(template).rstrip() == expectaion_flat - except AssertionError: - print(scheduler.get_submit_script(template).rstrip()) - print(expectaion) - raise - - -def test_write_script_minimal(): - # to avoid false positive (overwriting on existing file), - # we check the output of the script instead of using `file_regression`` - expectaion = """ - #!/bin/bash - #SBATCH --no-requeue - #SBATCH --error=slurm-%j.err - #SBATCH --nodes=1 - #SBATCH --ntasks-per-node=1 - """ - - expectaion_flat = "\n".join(line.strip() for line in expectaion.splitlines()).strip( - "\n" - ) - scheduler = FirecrestScheduler() - template = JobTemplate( - { - "job_resource": scheduler.create_job_resource( - num_machines=1, num_mpiprocs_per_machine=1 - ), - "codes_info": [], - "codes_run_mode": CodeRunMode.SERIAL, - } - ) - - try: - assert scheduler.get_submit_script(template).rstrip() == expectaion_flat - except AssertionError: - print(scheduler.get_submit_script(template).rstrip()) - print(expectaion) - raise diff --git a/tests/tests_mocking_pyfirecrest/test_transport.py b/tests/tests_mocking_pyfirecrest/test_transport.py deleted file mode 100644 index edd11ee..0000000 --- a/tests/tests_mocking_pyfirecrest/test_transport.py +++ /dev/null @@ -1,627 +0,0 @@ -import os -from pathlib import Path -from unittest.mock import patch - -from aiida import orm -import pytest - - -@pytest.mark.usefixtures("aiida_profile_clean") -def test_mkdir(firecrest_computer: orm.Computer, tmpdir: Path): - transport = firecrest_computer.get_transport() - - _scratch = tmpdir / "sampledir" - transport.mkdir(_scratch) - assert _scratch.exists() - - _scratch = tmpdir / "sampledir2" / "subdir" - transport.makedirs(_scratch) - assert _scratch.exists() - - -@pytest.mark.usefixtures("aiida_profile_clean") -def test_is_file(firecrest_computer: orm.Computer, tmpdir: Path): - transport = firecrest_computer.get_transport() - - _scratch = tmpdir / "samplefile" - Path(_scratch).touch() - assert transport.isfile(_scratch) - assert not transport.isfile(_scratch / "does_not_exist") - - -@pytest.mark.usefixtures("aiida_profile_clean") -def test_is_dir(firecrest_computer: orm.Computer, tmpdir: Path): - transport = firecrest_computer.get_transport() - - _scratch = tmpdir / "sampledir" - _scratch.mkdir() - - assert transport.isdir(_scratch) - assert not transport.isdir(_scratch / "does_not_exist") - - -@pytest.mark.usefixtures("aiida_profile_clean") -def test_normalize(firecrest_computer: orm.Computer): - transport = firecrest_computer.get_transport() - assert transport.normalize("/path/to/dir") == os.path.normpath("/path/to/dir") - assert transport.normalize("path/to/dir") == os.path.normpath("path/to/dir") - assert transport.normalize("path/to/dir/") == os.path.normpath("path/to/dir/") - assert transport.normalize("path/to/../dir") == os.path.normpath("path/to/../dir") - assert transport.normalize("path/to/../../dir") == os.path.normpath( - "path/to/../../dir" - ) - assert transport.normalize("path/to/../../dir/") == os.path.normpath( - "path/to/../../dir/" - ) - assert transport.normalize("path/to/../../dir/../") == os.path.normpath( - "path/to/../../dir/../" - ) - - -@pytest.mark.usefixtures("aiida_profile_clean") -def test_remove(firecrest_computer: orm.Computer, tmpdir: Path): - transport = firecrest_computer.get_transport() - - _scratch = tmpdir / "samplefile" - Path(_scratch).touch() - transport.remove(_scratch) - assert not _scratch.exists() - - _scratch = tmpdir / "sampledir" - _scratch.mkdir() - transport.rmtree(_scratch) - assert not _scratch.exists() - - _scratch = tmpdir / "sampledir" - _scratch.mkdir() - Path(_scratch / "samplefile").touch() - with pytest.raises(OSError): - transport.rmdir(_scratch) - - os.remove(_scratch / "samplefile") - transport.rmdir(_scratch) - assert not _scratch.exists() - - -@pytest.mark.usefixtures("aiida_profile_clean") -def test_symlink(firecrest_computer: orm.Computer, tmpdir: Path): - transport = firecrest_computer.get_transport() - - _scratch = Path(tmpdir / "samplefile-2sym") - Path(_scratch).touch() - _symlink = Path(tmpdir / "samplelink") - transport.symlink(_scratch, _symlink) - assert _symlink.is_symlink() - assert _symlink.resolve() == _scratch - - -@pytest.mark.usefixtures("aiida_profile_clean") -def test_listdir(firecrest_computer: orm.Computer, tmpdir: Path): - transport = firecrest_computer.get_transport() - - _scratch = tmpdir / "sampledir" - _scratch.mkdir() - # to test basics - Path(_scratch / "file1").touch() - Path(_scratch / "dir1").mkdir() - Path(_scratch / ".hidden").touch() - # to test recursive - Path(_scratch / "dir1" / "file2").touch() - - assert set(transport.listdir(_scratch)) == {"file1", "dir1", ".hidden"} - assert set(transport.listdir(_scratch, recursive=True)) == { - "file1", - "dir1", - ".hidden", - "dir1/file2", - } - # to test symlink - Path(_scratch / "dir1" / "dir2").mkdir() - Path(_scratch / "dir1" / "dir2" / "file3").touch() - os.symlink(_scratch / "dir1" / "dir2", _scratch / "dir2_link") - os.symlink(_scratch / "dir1" / "file2", _scratch / "file_link") - - assert set(transport.listdir(_scratch, recursive=True)) == { - "file1", - "dir1", - ".hidden", - "dir1/file2", - "dir1/dir2", - "dir1/dir2/file3", - "dir2_link", - "file_link", - } - - assert set(transport.listdir(_scratch / "dir2_link", recursive=False)) == {"file3"} - - -@pytest.mark.usefixtures("aiida_profile_clean") -def test_get(firecrest_computer: orm.Computer, tmpdir: Path): - """ - This is minimal test is to check if get() is raising errors as expected, - and directing to getfile() and gettree() as expected. - Mainly just checking error handeling and folder creation. - """ - transport = firecrest_computer.get_transport() - - _remote = tmpdir / "remotedir" - _local = tmpdir / "localdir" - _remote.mkdir() - _local.mkdir() - - # check if the code is directing to getfile() or gettree() as expected - with patch.object(transport, "gettree", autospec=True) as mock_gettree: - transport.get(_remote, _local) - mock_gettree.assert_called_once() - - with patch.object(transport, "gettree", autospec=True) as mock_gettree: - os.symlink(_remote, tmpdir / "dir_link") - transport.get(tmpdir / "dir_link", _local) - mock_gettree.assert_called_once() - - with patch.object(transport, "getfile", autospec=True) as mock_getfile: - Path(_remote / "file1").write_text("file1") - transport.get(_remote / "file1", _local / "file1") - mock_getfile.assert_called_once() - - with patch.object(transport, "getfile", autospec=True) as mock_getfile: - os.symlink(_remote / "file1", _remote / "file1_link") - transport.get(_remote / "file1_link", _local / "file1_link") - mock_getfile.assert_called_once() - - # raise if remote file/folder does not exist - with pytest.raises(FileNotFoundError): - transport.get(_remote / "does_not_exist", _local) - transport.get(_remote / "does_not_exist", _local, ignore_nonexisting=True) - - # raise if localpath is relative - with pytest.raises(ValueError): - transport.get(_remote, Path(_local).relative_to(tmpdir)) - with pytest.raises(ValueError): - transport.get(_remote / "file1", Path(_local).relative_to(tmpdir)) - - -@pytest.mark.usefixtures("aiida_profile_clean") -def test_getfile(firecrest_computer: orm.Computer, tmpdir: Path): - transport = firecrest_computer.get_transport() - - _remote = tmpdir / "remotedir" - _local = tmpdir / "localdir" - _remote.mkdir() - _local.mkdir() - - Path(_remote / "file1").write_text("file1") - Path(_remote / ".hidden").write_text(".hidden") - os.symlink(_remote / "file1", _remote / "file1_link") - - # raise if remote file does not exist - with pytest.raises(FileNotFoundError): - transport.getfile(_remote / "does_not_exist", _local) - - # raise if localfilename not provided - with pytest.raises(IsADirectoryError): - transport.getfile(_remote / "file1", _local) - - # raise if localpath is relative - with pytest.raises(ValueError): - transport.getfile(_remote / "file1", Path(_local / "file1").relative_to(tmpdir)) - - # don't mix up directory with file - with pytest.raises(FileNotFoundError): - transport.getfile(_remote, _local / "file1") - - # write where I tell you to - transport.getfile(_remote / "file1", _local / "file1") - transport.getfile(_remote / "file1", _local / "file1-prime") - assert Path(_local / "file1").read_text() == "file1" - assert Path(_local / "file1-prime").read_text() == "file1" - - # always overwrite - transport.getfile(_remote / "file1", _local / "file1") - assert Path(_local / "file1").read_text() == "file1" - - Path(_local / "file1").write_text("notfile1") - - transport.getfile(_remote / "file1", _local / "file1") - assert Path(_local / "file1").read_text() == "file1" - - # don't skip hidden files - transport.getfile(_remote / ".hidden", _local / ".hidden-prime") - assert Path(_local / ".hidden-prime").read_text() == ".hidden" - - # follow links - transport.getfile(_remote / "file1_link", _local / "file1_link") - assert Path(_local / "file1_link").read_text() == "file1" - assert not Path(_local / "file1_link").is_symlink() - - -@pytest.mark.parametrize("payoff", [True, False]) -@pytest.mark.usefixtures("aiida_profile_clean") -def test_gettree(firecrest_computer: orm.Computer, tmpdir: Path, payoff: bool): - """ - This test is to check `gettree` through non tar mode. - bytar= True in this test. - """ - transport = firecrest_computer.get_transport() - transport.payoff_override = payoff - - # Note: - # SSH transport behaviour, 69 is a directory - # transport.get('somepath/69', 'someremotepath/') == transport.get('somepath/69', 'someremotepath') - # transport.get('somepath/69', 'someremotepath/') == transport.get('somepath/69/', 'someremotepath/') - # transport.get('someremotepath/69', 'somepath/69')--> if 69 exist, create 69 inside it ('somepath/69/69') - # transport.get('someremotepath/69', 'somepath/69')--> if 69 no texist,create 69 inside it ('somepath/69') - # transport.get('somepath/69', 'someremotepath/6889/69') --> create everything, make_parent = True - - _remote = tmpdir / "remotedir" - _local = tmpdir / "localdir" - _remote.mkdir() - _local.mkdir() - # a typical tree - Path(_remote / "file1").write_text("file1") - Path(_remote / ".hidden").write_text(".hidden") - Path(_remote / "dir1").mkdir() - Path(_remote / "dir1" / "file2").write_text("file2") - # with symlinks - Path(_remote / "dir2").mkdir() - Path(_remote / "dir2" / "file3").write_text("file3") - os.symlink(_remote / "file1", _remote / "dir1" / "file1_link") - os.symlink(_remote / "dir2", _remote / "dir1" / "dir2_link") - # if symlinks are pointing to a relative path - os.symlink(Path("../file1"), _remote / "dir1" / "file10_link") - os.symlink(Path("../dir2"), _remote / "dir1" / "dir20_link") - - # raise if remote file does not exist - with pytest.raises(OSError): - transport.gettree(_remote / "does_not_exist", _local) - - # raise if local is a file - Path(tmpdir / "isfile").touch() - with pytest.raises(OSError): - transport.gettree(_remote, tmpdir / "isfile") - - # raise if localpath is relative - with pytest.raises(ValueError): - transport.gettree(_remote, Path(_local).relative_to(tmpdir)) - - # If destination directory does not exists, AiiDA expects transport make the new path as root not _remote.name - transport.gettree(_remote, _local / "newdir") - _root = _local / "newdir" - # tree should be copied recursively - assert Path(_root / "file1").read_text() == "file1" - assert Path(_root / ".hidden").read_text() == ".hidden" - assert Path(_root / "dir1" / "file2").read_text() == "file2" - assert Path(_root / "dir2" / "file3").read_text() == "file3" - # symlink should be followed - assert Path(_root / "dir1" / "file1_link").read_text() == "file1" - assert Path(_root / "dir1" / "dir2_link" / "file3").read_text() == "file3" - assert Path(_root / "dir1" / "file10_link").read_text() == "file1" - assert Path(_root / "dir1" / "dir20_link" / "file3").read_text() == "file3" - assert not Path(_root / "dir1" / "file1_link").is_symlink() - assert not Path(_root / "dir1" / "dir2_link" / "file3").is_symlink() - assert not Path(_root / "dir1" / "file10_link").is_symlink() - assert not Path(_root / "dir1" / "dir20_link" / "file3").is_symlink() - - # If destination directory does exists, AiiDA expects transport make _remote.name and write into it - # however this might have changed in the newer versions of AiiDA ~ 2.6.0 (IDK) - transport.gettree(_remote, _local / "newdir") - _root = _local / "newdir" / Path(_remote).name - # tree should be copied recursively - assert Path(_root / "file1").read_text() == "file1" - assert Path(_root / ".hidden").read_text() == ".hidden" - assert Path(_root / "dir1" / "file2").read_text() == "file2" - assert Path(_root / "dir2" / "file3").read_text() == "file3" - # symlink should be followed - assert Path(_root / "dir1" / "file1_link").read_text() == "file1" - assert Path(_root / "dir1" / "dir2_link" / "file3").read_text() == "file3" - assert Path(_root / "dir1" / "file10_link").read_text() == "file1" - assert Path(_root / "dir1" / "dir20_link" / "file3").read_text() == "file3" - assert not Path(_root / "dir1" / "file1_link").is_symlink() - assert not Path(_root / "dir1" / "dir2_link" / "file3").is_symlink() - assert not Path(_root / "dir1" / "file10_link").is_symlink() - assert not Path(_root / "dir1" / "dir20_link" / "file3").is_symlink() - - -@pytest.mark.usefixtures("aiida_profile_clean") -def test_put(firecrest_computer: orm.Computer, tmpdir: Path): - """ - This is minimal test is to check if put() is raising errors as expected, - and directing to putfile() and puttree() as expected. - Mainly just checking error handeling and folder creation. - """ - transport = firecrest_computer.get_transport() - - _remote = tmpdir / "remotedir" - _local = tmpdir / "localdir" - _remote.mkdir() - _local.mkdir() - - # check if the code is directing to putfile() or puttree() as expected - with patch.object(transport, "puttree", autospec=True) as mock_puttree: - transport.put(_local, _remote) - mock_puttree.assert_called_once() - - with patch.object(transport, "puttree", autospec=True) as mock_puttree: - os.symlink(_local, tmpdir / "dir_link") - transport.put(tmpdir / "dir_link", _remote) - mock_puttree.assert_called_once() - - with patch.object(transport, "putfile", autospec=True) as mock_putfile: - Path(_local / "file1").write_text("file1") - transport.put(_local / "file1", _remote / "file1") - mock_putfile.assert_called_once() - - with patch.object(transport, "putfile", autospec=True) as mock_putfile: - os.symlink(_local / "file1", _local / "file1_link") - transport.put(_local / "file1_link", _remote / "file1_link") - mock_putfile.assert_called_once() - - # raise if local file/folder does not exist - with pytest.raises(FileNotFoundError): - transport.put(_local / "does_not_exist", _remote) - transport.put(_local / "does_not_exist", _remote, ignore_nonexisting=True) - - # raise if localpath is relative - with pytest.raises(ValueError): - transport.put(Path(_local).relative_to(tmpdir), _remote) - with pytest.raises(ValueError): - transport.put(Path(_local / "file1").relative_to(tmpdir), _remote) - - -@pytest.mark.usefixtures("aiida_profile_clean") -def test_putfile(firecrest_computer: orm.Computer, tmpdir: Path): - transport = firecrest_computer.get_transport() - - _remote = tmpdir / "remotedir" - _local = tmpdir / "localdir" - _remote.mkdir() - _local.mkdir() - - Path(_local / "file1").write_text("file1") - Path(_local / ".hidden").write_text(".hidden") - os.symlink(_local / "file1", _local / "file1_link") - - # raise if local file does not exist - with pytest.raises(FileNotFoundError): - transport.putfile(_local / "does_not_exist", _remote) - - # raise if remotefilename is not provided - with pytest.raises(ValueError): - transport.putfile(_local / "file1", _remote) - - # raise if localpath is relative - with pytest.raises(ValueError): - transport.putfile(Path(_local / "file1").relative_to(tmpdir), _remote / "file1") - - # don't mix up directory with file - with pytest.raises(ValueError): - transport.putfile(_local, _remote / "file1") - - # write where I tell you to - transport.putfile(_local / "file1", _remote / "file1") - transport.putfile(_local / "file1", _remote / "file1-prime") - assert Path(_remote / "file1").read_text() == "file1" - assert Path(_remote / "file1-prime").read_text() == "file1" - - # always overwrite - transport.putfile(_local / "file1", _remote / "file1") - assert Path(_remote / "file1").read_text() == "file1" - - Path(_remote / "file1").write_text("notfile1") - - transport.putfile(_local / "file1", _remote / "file1") - assert Path(_remote / "file1").read_text() == "file1" - - # don't skip hidden files - transport.putfile(_local / ".hidden", _remote / ".hidden-prime") - assert Path(_remote / ".hidden-prime").read_text() == ".hidden" - - # follow links - transport.putfile(_local / "file1_link", _remote / "file1_link") - assert Path(_remote / "file1_link").read_text() == "file1" - assert not Path(_remote / "file1_link").is_symlink() - - -@pytest.mark.parametrize("payoff", [True, False]) -@pytest.mark.usefixtures("aiida_profile_clean") -def test_puttree(firecrest_computer: orm.Computer, tmpdir: Path, payoff: bool): - """ - This test is to check `puttree` through non tar mode. - payoff= False in this test, so just checking if putting files one by one is working as expected. - """ - transport = firecrest_computer.get_transport() - transport.payoff_override = payoff - - # Note: - # SSH transport behaviour - # transport.put('somepath/69', 'someremotepath/') == transport.put('somepath/69', 'someremotepath') - # transport.put('somepath/69', 'someremotepath/') != transport.put('somepath/69/', 'someremotepath/') - # transport.put('somepath/69', 'someremotepath/67') --> if 67 not exist, create and move content 69 - # inside it (someremotepath/67) - # transport.put('somepath/69', 'someremotepath/67') --> if 67 exist, create 69 inside it (someremotepath/67/69) - # transport.put('somepath/69', 'someremotepath/6889/69') --> useless Error: OSError - # Weired - # SSH bug: - # transport.put('somepath/69', 'someremotepath/') --> assuming someremotepath exists, make 69 - # while - # transport.put('somepath/69/', 'someremotepath/') --> assuming someremotepath exists, OSError: - # cannot make someremotepath - - _remote = tmpdir / "remotedir" - _local = tmpdir / "localdir" - _remote.mkdir() - _local.mkdir() - # a typical tree - Path(_local / "dir1").mkdir() - Path(_local / "dir2").mkdir() - Path(_local / "file1").write_text("file1") - Path(_local / ".hidden").write_text(".hidden") - Path(_local / "dir1" / "file2").write_text("file2") - Path(_local / "dir2" / "file3").write_text("file3") - # with symlinks to a file even if pointing to a relative path - os.symlink(_local / "file1", _local / "dir1" / "file1_link") - os.symlink(Path("../file1"), _local / "dir1" / "file10_link") - # with symlinks to a folder even if pointing to a relative path - os.symlink(_local / "dir2", _local / "dir1" / "dir2_link") - os.symlink(Path("../dir2"), _local / "dir1" / "dir20_link") - - # raise if local file does not exist - with pytest.raises(OSError): - transport.puttree(_local / "does_not_exist", _remote) - - # raise if local is a file - with pytest.raises(ValueError): - Path(tmpdir / "isfile").touch() - transport.puttree(tmpdir / "isfile", _remote) - - # raise if localpath is relative - with pytest.raises(ValueError): - transport.puttree(Path(_local).relative_to(tmpdir), _remote) - - # If destination directory does not exists, AiiDA expects transport make the new path as root not using _local.name - transport.puttree(_local, _remote / "newdir") - _root = _remote / "newdir" - # tree should be copied recursively - assert Path(_root / "file1").read_text() == "file1" - assert Path(_root / ".hidden").read_text() == ".hidden" - assert Path(_root / "dir1" / "file2").read_text() == "file2" - assert Path(_root / "dir2" / "file3").read_text() == "file3" - # symlink should be followed - assert Path(_root / "dir1" / "file1_link").read_text() == "file1" - assert Path(_root / "dir1" / "dir2_link" / "file3").read_text() == "file3" - assert Path(_root / "dir1" / "file10_link").read_text() == "file1" - assert Path(_root / "dir1" / "dir20_link" / "file3").read_text() == "file3" - assert not Path(_root / "dir1" / "file1_link").is_symlink() - assert not Path(_root / "dir1" / "dir2_link" / "file3").is_symlink() - assert not Path(_root / "dir1" / "file10_link").is_symlink() - assert not Path(_root / "dir1" / "dir20_link" / "file3").is_symlink() - - # If destination directory does exists, AiiDA expects transport make _local.name and write into it - # however this might have changed in the newer versions of AiiDA ~ 2.6.0 (IDK) - transport.puttree(_local, _remote / "newdir") - _root = _remote / "newdir" / Path(_local).name - # tree should be copied recursively - assert Path(_root / "file1").read_text() == "file1" - assert Path(_root / ".hidden").read_text() == ".hidden" - assert Path(_root / "dir1" / "file2").read_text() == "file2" - assert Path(_root / "dir2" / "file3").read_text() == "file3" - # symlink should be followed - assert Path(_root / "dir1" / "file1_link").read_text() == "file1" - assert Path(_root / "dir1" / "dir2_link" / "file3").read_text() == "file3" - assert Path(_root / "dir1" / "file10_link").read_text() == "file1" - assert Path(_root / "dir1" / "dir20_link" / "file3").read_text() == "file3" - assert not Path(_root / "dir1" / "file1_link").is_symlink() - assert not Path(_root / "dir1" / "dir2_link" / "file3").is_symlink() - assert not Path(_root / "dir1" / "file10_link").is_symlink() - assert not Path(_root / "dir1" / "dir20_link" / "file3").is_symlink() - - -@pytest.mark.parametrize("to_test", ["copy", "copytree"]) -@pytest.mark.usefixtures("aiida_profile_clean") -def test_copy(firecrest_computer: orm.Computer, tmpdir: Path, to_test: str): - transport = firecrest_computer.get_transport() - if to_test == "copy": - testing = transport.copy - elif to_test == "copytree": - testing = transport.copytree - - _remote_1 = tmpdir / "remotedir_1" - _remote_2 = tmpdir / "remotedir_2" - _remote_1.mkdir() - _remote_2.mkdir() - - # raise if source or destination does not exist - with pytest.raises(FileNotFoundError): - testing(_remote_1 / "does_not_exist", _remote_2) - with pytest.raises(FileNotFoundError): - testing(_remote_1, _remote_2 / "does_not_exist") - - # raise if source is inappropriate - if to_test == "copytree": - Path(tmpdir / "file1").touch() - with pytest.raises(ValueError): - testing(tmpdir / "file1", _remote_2) - - # a typical tree - Path(_remote_1 / "dir1").mkdir() - Path(_remote_1 / "dir2").mkdir() - Path(_remote_1 / "file1").write_text("file1") - Path(_remote_1 / ".hidden").write_text(".hidden") - Path(_remote_1 / "dir1" / "file2").write_text("file2") - Path(_remote_1 / "dir2" / "file3").write_text("file3") - # with symlinks to a file even if pointing to a relative path - os.symlink(_remote_1 / "file1", _remote_1 / "dir1" / "file1_link") - os.symlink(Path("../file1"), _remote_1 / "dir1" / "file10_link") - # with symlinks to a folder even if pointing to a relative path - os.symlink(_remote_1 / "dir2", _remote_1 / "dir1" / "dir2_link") - os.symlink(Path("../dir2"), _remote_1 / "dir1" / "dir20_link") - - testing(_remote_1, _remote_2) - - _root_2 = _remote_2 / Path(_remote_1).name - # tree should be copied recursively - assert Path(_root_2 / "dir1").exists() - assert Path(_root_2 / "dir2").exists() - assert Path(_root_2 / "file1").read_text() == "file1" - assert Path(_root_2 / ".hidden").read_text() == ".hidden" - assert Path(_root_2 / "dir1" / "file2").read_text() == "file2" - assert Path(_root_2 / "dir2" / "file3").read_text() == "file3" - # symlink should be followed - assert Path(_root_2 / "dir1" / "dir2_link").exists() - assert Path(_root_2 / "dir1" / "dir20_link").exists() - assert Path(_root_2 / "dir1" / "file1_link").read_text() == "file1" - assert Path(_root_2 / "dir1" / "file10_link").read_text() == "file1" - assert Path(_root_2 / "dir1" / "file1_link").is_symlink() - assert Path(_root_2 / "dir1" / "dir2_link").is_symlink() - assert Path(_root_2 / "dir1" / "file10_link").is_symlink() - assert Path(_root_2 / "dir1" / "dir20_link").is_symlink() - - -@pytest.mark.usefixtures("aiida_profile_clean") -def test_copyfile(firecrest_computer: orm.Computer, tmpdir: Path): - transport = firecrest_computer.get_transport() - testing = transport.copyfile - - _remote_1 = tmpdir / "remotedir_1" - _remote_2 = tmpdir / "remotedir_2" - _remote_1.mkdir() - _remote_2.mkdir() - - # raise if source or destination does not exist - with pytest.raises(FileNotFoundError): - testing(_remote_1 / "does_not_exist", _remote_2) - # in this case don't raise and just create the file - Path(tmpdir / "_").touch() - testing(tmpdir / "_", _remote_2 / "does_not_exist") - - # raise if source is unappropriate - with pytest.raises(ValueError): - testing(tmpdir, _remote_2) - - # a typical tree - Path(_remote_1 / "file1").write_text("file1") - Path(_remote_1 / ".hidden").write_text(".hidden") - # with symlinks to a file even if pointing to a relative path - os.symlink(_remote_1 / "file1", _remote_1 / "file1_link") - os.symlink(Path("file1"), _remote_1 / "file10_link") - - # write where I tell you to - testing(_remote_1 / "file1", _remote_2 / "file1") - assert Path(_remote_2 / "file1").read_text() == "file1" - - # always overwrite - Path(_remote_2 / "file1").write_text("notfile1") - testing(_remote_1 / "file1", _remote_2 / "file1") - assert Path(_remote_2 / "file1").read_text() == "file1" - - # don't skip hidden files - testing(_remote_1 / ".hidden", _remote_2 / ".hidden-prime") - assert Path(_remote_2 / ".hidden-prime").read_text() == ".hidden" - - # preserve links and don't follow them - testing(_remote_1 / "file1_link", _remote_2 / "file1_link") - assert Path(_remote_2 / "file1_link").read_text() == "file1" - assert Path(_remote_2 / "file1_link").is_symlink() - testing(_remote_1 / "file10_link", _remote_2 / "file10_link") - assert Path(_remote_2 / "file10_link").read_text() == "file1" - assert Path(_remote_2 / "file10_link").is_symlink() From b24563906c2eb8c62087a292add15577374d8aa9 Mon Sep 17 00:00:00 2001 From: Ali Date: Mon, 15 Jul 2024 14:54:23 +0200 Subject: [PATCH 19/39] temporarly turnned off Codecov --- .github/workflows/tests.yml | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 4f260ea..49fc7fe 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -51,11 +51,12 @@ jobs: - name: Test with pytest run: pytest -vv --cov=aiida_firecrest --cov-report=xml --cov-report=term - - name: Upload coverage reports to Codecov - uses: codecov/codecov-action@v3 - with: - name: aiida-firecrest-pytests - flags: pytests - file: ./coverage.xml - fail_ci_if_error: true - token: ${{ secrets.CODECOV_TOKEN }} + # Codecov failing, we need to fix token: https://github.com/aiidateam/aiida-firecrest/issues/38 + # - name: Upload coverage reports to Codecov + # uses: codecov/codecov-action@v3 + # with: + # name: aiida-firecrest-pytests + # flags: pytests + # file: ./coverage.xml + # fail_ci_if_error: true + # token: ${{ secrets.CODECOV_TOKEN }} From be9b52f72c1d4eb7a4b284b8622631ba324669ab Mon Sep 17 00:00:00 2001 From: Ali Khosravi Date: Mon, 15 Jul 2024 15:58:33 +0200 Subject: [PATCH 20/39] Apply suggestions from code review --- README.md | 2 +- aiida_firecrest/transport.py | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/README.md b/README.md index ec6b928..55ffbcf 100644 --- a/README.md +++ b/README.md @@ -208,7 +208,7 @@ although it is of note that you can find these files directly where you your `fi These set of test do not gurantee that the firecrest protocol is working, but it's very useful to quickly check if `aiida-firecrest` is behaving as it's expected to do. To run just simply use `pytest`. -If these tests, pass and still you have trouble in real deploymeny that means your installed version of pyfirecrest is behaving differently from what `aiida-firecrest` expects in `MockFirecrest` in `tests/tests_mocking_pyfirecrest/conftest.py`. +If these tests, pass and still you have trouble in real deployment, that means your installed version of pyfirecrest is behaving differently from what `aiida-firecrest` expects in `MockFirecrest` in `tests/tests_mocking_pyfirecrest/conftest.py`. If there is no version of `aiida-firecrest` available that supports your `pyfirecrest` version and if down/upgrading your `pyfirecrest` to a supported version is not an option, you might try the following: - open an issue on the `aiida-firecrest` repository on GitHub to request supporting your version of pyfirecrest - if you feel up to finding the discrepancy and fixing it within `aiida-firecrest`, open a PR instead diff --git a/aiida_firecrest/transport.py b/aiida_firecrest/transport.py index 9384104..8b2d031 100644 --- a/aiida_firecrest/transport.py +++ b/aiida_firecrest/transport.py @@ -287,7 +287,6 @@ def __init__( client_machine: str, small_file_size_mb: float, temp_directory: str, - # configured: bool = True, # note, machine is provided by default, # for the hostname, but we don't use that # TODO ideally hostname would not be necessary on a computer From f792b456e82e21822acb9ac0c31d5873a7e4d4cf Mon Sep 17 00:00:00 2001 From: Ali Date: Mon, 15 Jul 2024 16:18:48 +0200 Subject: [PATCH 21/39] review applied --- aiida_firecrest/scheduler.py | 8 +++----- aiida_firecrest/transport.py | 2 +- pyproject.toml | 1 - 3 files changed, 4 insertions(+), 7 deletions(-) diff --git a/aiida_firecrest/scheduler.py b/aiida_firecrest/scheduler.py index 3ed2062..e18158e 100644 --- a/aiida_firecrest/scheduler.py +++ b/aiida_firecrest/scheduler.py @@ -290,8 +290,7 @@ def get_jobs( ) ) - # TODO: The block below is commented, because the number of - # allocated cores is not returned by the FirecREST server + # See issue https://github.com/aiidateam/aiida-firecrest/issues/39 # try: # this_job.num_mpiprocs = int(thisjob_dict['number_cpus']) # except ValueError: @@ -342,9 +341,8 @@ def get_jobs( f"Couldn't parse time_used for job id {this_job.job_id}" ) - # TODO: The block below is commented, because dispatch_time - # is not returned explicitly by the FirecREST server - # in any case, the time tags doesn't seem to be used by AiiDA anyway. + # dispatch_time is not returned explicitly by the FirecREST server + # see: https://github.com/aiidateam/aiida-firecrest/issues/40 # try: # this_job.dispatch_time = self._parse_time_string(thisjob_dict['dispatch_time']) # except ValueError: diff --git a/aiida_firecrest/transport.py b/aiida_firecrest/transport.py index 8b2d031..a41f718 100644 --- a/aiida_firecrest/transport.py +++ b/aiida_firecrest/transport.py @@ -297,7 +297,7 @@ def __init__( :param url: URL to the FirecREST server :param token_uri: URI for retrieving FirecREST authentication tokens :param client_id: FirecREST client ID - :param client_secret: FirecREST client secret or Absolute path to an existing FirecREST Secret Key + :param client_secret: FirecREST client secret or str(Absolute path) to an existing FirecREST Secret Key :param client_machine: FirecREST machine secret :param small_file_size_mb: Maximum file size for direct transfer (MB) :param temp_directory: A temp directory on server for creating temporary files (compression, extraction, etc.) diff --git a/pyproject.toml b/pyproject.toml index 943e8d3..4fcc9d5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -28,7 +28,6 @@ dependencies = [ "click", "pyfirecrest@git+https://github.com/khsrali/pyfirecrest.git@main#egg=pyfirecrest", "pyyaml", - "requests", ] [project.urls] From a238dadfb36269057e7391eea3e05f62bcf6cf35 Mon Sep 17 00:00:00 2001 From: Ali Date: Mon, 15 Jul 2024 18:48:08 +0200 Subject: [PATCH 22/39] some info added in changelog --- CHANGELOG.md | 32 ++++++++++++++++++++++++++++++++ README.md | 2 +- aiida_firecrest/transport.py | 13 ++++++------- 3 files changed, 39 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e552098..dabfc42 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,37 @@ # Changelog +## v0.2.0 - 2024-07-15 (not released yet) + +### Transport plugin +- `dynamic_info()` is added to retrieve machine information without user input. +- Refactor `put` & `get` & `copy` now they mimic behavior `aiida-ssh` transport plugin. +- `put` & `get` & `copy` now support glob patterns. +- Added `dereference` option wherever relevant +- Added `recursive` functionality for `listdir` +- Added `_create_secret_file` to store user secret locally in `~/.firecrest/` +- Added `_validate_temp_directory` to allocate a temporary directory useful for `extract` and `compress` methods on FirecREST server. +- Added `_dynamic_info_direct_size` this is able to get info of direct transfer from the server rather than asking from users. Raise of user inputs fails to make a connection. +- Added `_validate_checksum` to check integrity of downloaded/uploaded files. +- Added `_gettreetar` & `_puttreetar` to transfer directories as tar files internally. +- Added `payoff` function to calculate when is gainful to transfer as zip, and when to transfer individually. + +### Scheduler plugin +- `get_job` now supports for pagination for retrieving active jobs +- `get_job` is parsing more data than before: `submission_time`, `wallclock_time_seconds`, `start_time`, `time_left`, `nodelist`. see open issues [39](https://github.com/aiidateam/aiida-firecrest/issues/39) & [40](https://github.com/aiidateam/aiida-firecrest/issues/40) +- bug fix: `get_job` won't raise if the job cannot be find (completed/error/etc.) +- `_convert_time` and `_parse_time_string` copied over from `slurm-plugin` see [open issue](https://github.com/aiidateam/aiida-firecrest/issues/42) + +### Tests + +- Tests has completely replaced with new ones. Previously tests were mocking FirecREST server. Those test were a good practice to ensure that all three (`aiida-firecrest`, FirecREST, and PyFirecREST) work flawlessly. +The downside was debugging wasn't easy at all. Not always obvious which of the three is causing a bug. +Because of this, a new set of tests only verify the functionality of `aiida-firecrest` by directly mocking PyFirecREST. Maintaining this set in `tests/` is simpler because we just need to monitor the return values of PyFirecREST​. While maintaining the former is more difficult as you have to keep up with both FirecREST and PyFirecREST. + + +### Miscellaneous + +- class `FcPath` is removed from interface here, as it has [merged](https://github.com/eth-cscs/pyfirecrest/pull/43) into `pyfirecrest` + ## v0.1.0 (December 2021) Initial release. diff --git a/README.md b/README.md index 55ffbcf..965f23b 100644 --- a/README.md +++ b/README.md @@ -101,7 +101,7 @@ See [tests/test_calculation.py](tests/test_calculation.py) for a working example ### Current Issues -Calculations are now running successfully, however, there are still issues regarding efficency, Could be improved: +Calculations are now running successfully, however, there are still issues regarding efficiency, Could be improved: 1. Monitoring / management of API request rates could to be improved. Currently this is left up to PyFirecREST. diff --git a/aiida_firecrest/transport.py b/aiida_firecrest/transport.py index a41f718..257f3b9 100644 --- a/aiida_firecrest/transport.py +++ b/aiida_firecrest/transport.py @@ -34,7 +34,7 @@ class ValidAuthOption(TypedDict, total=False): class BuggyError(Exception): # TODO: Remove this class when the code is stable - """Raised when something should absolutly not happen, but it does.""" + """Raised when something should absolutely not happen, but it does.""" def _create_secret_file(ctx: Context, param: InteractiveOption, value: str) -> str: @@ -68,7 +68,7 @@ def _create_secret_file(ctx: Context, param: InteractiveOption, value: str) -> s def _validate_temp_directory(ctx: Context, param: InteractiveOption, value: str) -> str: """Validate the temp directory on the server. If it does not exist, create it. - If it is not empty, get a confimation from the user to empty it. + If it is not empty, get a confirmation from the user to empty it. """ import click @@ -90,7 +90,7 @@ def _validate_temp_directory(ctx: Context, param: InteractiveOption, value: str) small_file_size_mb=small_file_size_mb, ) - # Temp directory rutine + # Temp directory routine if dummy._cwd.joinpath( dummy._temp_directory ).is_file(): # self._temp_directory.is_file(): @@ -111,9 +111,9 @@ def _validate_temp_directory(ctx: Context, param: InteractiveOption, value: str) raise click.BadParameter( f"Temp directory {dummy._temp_directory} is not empty" ) - # The block below could be moved to a maintanace delete function, if needed + # The block below could be moved to a maintenance delete function, if needed # else: - # # There might still be some residual files in case of previous interupted connection + # # There might still be some residual files in case of previous interrupted connection # for item in dummy.listdir(dummy._temp_directory): # # this could be replace with a proper glob later # if item[:4] == 'temp': @@ -455,7 +455,7 @@ def listdir( names = fnmatch.filter(names, pattern) return names - # TODO the default implementations of glob / iglob could be overriden + # TODO the default implementations of glob / iglob could be overridden # to be more performant, using cached FcPaths and https://github.com/chrisjsewell/virtual-glob def makedirs(self, path: str, ignore_existing: bool = False) -> None: @@ -1098,7 +1098,6 @@ def gotocomputer_command(self, remotedir: str) -> str: """Not possible for REST-API. It's here only because it's an abstract method in the base class.""" # TODO remove from interface - print(f"Trying to go for {remotedir}") raise NotImplementedError("firecrest does not support gotocomputer_command") def _exec_command_internal(self, command: str, **kwargs: Any) -> Any: From 689ef555bf73d66db14af800ec0465d2162db1b4 Mon Sep 17 00:00:00 2001 From: Ali Khosravi Date: Tue, 16 Jul 2024 09:26:12 +0200 Subject: [PATCH 23/39] Apply suggestions from code review Co-authored-by: Alexander Goscinski --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 4fcc9d5..df82a7f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -24,7 +24,7 @@ classifiers = [ keywords = ["aiida", "firecrest"] requires-python = ">=3.9" dependencies = [ - "aiida-core@git+https://github.com/khsrali/aiida-core.git@aiida-firecrest#egg=aiida-core", + "aiida-core@git+https://github.com/aiidateam/aiida-core.git@954cbdd3", "click", "pyfirecrest@git+https://github.com/khsrali/pyfirecrest.git@main#egg=pyfirecrest", "pyyaml", From 70450feba9b4f123fc03c7d2c246661b82869319 Mon Sep 17 00:00:00 2001 From: Ali Date: Tue, 16 Jul 2024 12:12:59 +0200 Subject: [PATCH 24/39] test_calculation.py retrieved --- .pre-commit-config.yaml | 4 +- CHANGELOG.md | 5 +- pyproject.toml | 4 +- tests/conftest.py | 22 ++++++ tests/test_calculation.py | 145 ++++++++++++++++++++++++++++++++++++++ 5 files changed, 173 insertions(+), 7 deletions(-) create mode 100644 tests/test_calculation.py diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index ea409d7..3c13f85 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -44,5 +44,5 @@ repos: additional_dependencies: - "types-PyYAML" - "types-requests" - - "pyfirecrest~=2.5.0" - - "aiida-core~=2.5.1.post0" + - "pyfirecrest>=2.5.0" # please change to 2.6.0 when released + - "aiida-core>=2.6.0" # please change to 2.6.2 when released diff --git a/CHANGELOG.md b/CHANGELOG.md index dabfc42..d378424 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,16 +1,15 @@ # Changelog -## v0.2.0 - 2024-07-15 (not released yet) +## v0.2.0 - (not released yet) ### Transport plugin -- `dynamic_info()` is added to retrieve machine information without user input. - Refactor `put` & `get` & `copy` now they mimic behavior `aiida-ssh` transport plugin. - `put` & `get` & `copy` now support glob patterns. - Added `dereference` option wherever relevant - Added `recursive` functionality for `listdir` - Added `_create_secret_file` to store user secret locally in `~/.firecrest/` - Added `_validate_temp_directory` to allocate a temporary directory useful for `extract` and `compress` methods on FirecREST server. -- Added `_dynamic_info_direct_size` this is able to get info of direct transfer from the server rather than asking from users. Raise of user inputs fails to make a connection. +- Added `_dynamic_info_direct_size` this is able to get info of direct transfer from the server rather than asking from users. Raise if fails to make a connection. - Added `_validate_checksum` to check integrity of downloaded/uploaded files. - Added `_gettreetar` & `_puttreetar` to transfer directories as tar files internally. - Added `payoff` function to calculate when is gainful to transfer as zip, and when to transfer individually. diff --git a/pyproject.toml b/pyproject.toml index df82a7f..c9eeba4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -24,9 +24,9 @@ classifiers = [ keywords = ["aiida", "firecrest"] requires-python = ">=3.9" dependencies = [ - "aiida-core@git+https://github.com/aiidateam/aiida-core.git@954cbdd3", + "aiida-core@git+https://github.com/aiidateam/aiida-core.git@954cbdd", "click", - "pyfirecrest@git+https://github.com/khsrali/pyfirecrest.git@main#egg=pyfirecrest", + "pyfirecrest@git+https://github.com/eth-cscs/pyfirecrest.git@6cae414", "pyyaml", ] diff --git a/tests/conftest.py b/tests/conftest.py index 2e4a116..d7bc3ef 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -51,6 +51,7 @@ def _firecrest_computer(myfirecrest, tmpdir: Path): small_file_size_mb=1.0, temp_directory=str(_temp_directory), ) + computer.store() return computer @@ -105,6 +106,27 @@ def submit( if script_remote_path and not Path(script_remote_path).exists(): raise FileNotFoundError(f"File {script_remote_path} does not exist") job_id = random.randint(10000, 99999) + + # Filter out lines starting with '#SBATCH' + with open(script_remote_path) as file: + lines = file.readlines() + command = "".join([line for line in lines if not line.strip().startswith("#")]) + + # Make the dummy files + for line in lines: + if "--error" in line: + error_file = line.split("=")[1].strip() + (Path(script_remote_path).parent / error_file).touch() + elif "--output" in line: + output_file = line.split("=")[1].strip() + (Path(script_remote_path).parent / output_file).touch() + + # Execute the job, this is useful for test_calculation.py + if "aiida.in" in command: + # skip blank command like: '/bin/bash' + os.chdir(Path(script_remote_path).parent) + os.system(command) + return {"jobid": job_id} diff --git a/tests/test_calculation.py b/tests/test_calculation.py new file mode 100644 index 0000000..fb34ae6 --- /dev/null +++ b/tests/test_calculation.py @@ -0,0 +1,145 @@ +"""Test for running calculations on a FireCREST computer.""" +from pathlib import Path + +from aiida import common, engine, manage, orm +from aiida.common.folders import Folder +from aiida.engine.processes.calcjobs.tasks import MAX_ATTEMPTS_OPTION +from aiida.manage.tests.pytest_fixtures import EntryPointManager +from aiida.parsers import Parser +import pytest + + +@pytest.fixture(name="no_retries") +def _no_retries(): + """Remove calcjob retries, to make failing the test faster.""" + # TODO calculation seems to hang on errors still + max_attempts = manage.get_config().get_option(MAX_ATTEMPTS_OPTION) + manage.get_config().set_option(MAX_ATTEMPTS_OPTION, 1) + yield + manage.get_config().set_option(MAX_ATTEMPTS_OPTION, max_attempts) + + +@pytest.mark.usefixtures("aiida_profile_clean", "no_retries") +def test_calculation_basic(firecrest_computer: orm.Computer): + """Test running a simple `arithmetic.add` calculation.""" + code = orm.InstalledCode( + label="test_code", + description="test code", + default_calc_job_plugin="core.arithmetic.add", + computer=firecrest_computer, + filepath_executable="/bin/sh", + ) + code.store() + + builder = code.get_builder() + builder.x = orm.Int(1) + builder.y = orm.Int(2) + + _, node = engine.run_get_node(builder) + assert node.is_finished_ok + + +@pytest.mark.usefixtures("aiida_profile_clean", "no_retries") +def test_calculation_file_transfer( + firecrest_computer: orm.Computer, entry_points: EntryPointManager +): + """Test a calculation, with multiple files copied/uploaded/retrieved.""" + # add temporary entry points + entry_points.add(MultiFileCalcjob, "aiida.calculations:testing.multifile") + entry_points.add(NoopParser, "aiida.parsers:testing.noop") + + # add a remote file which is used remote_copy_list + firecrest_computer.get_transport()._cwd.joinpath( + firecrest_computer.get_workdir(), "remote_copy.txt" + ).touch() + + # setup the calculation + code = orm.InstalledCode( + label="test_code", + description="test code", + default_calc_job_plugin="testing.multifile", + computer=firecrest_computer, + filepath_executable="/bin/sh", + ) + code.store() + builder = code.get_builder() + + node: orm.CalcJobNode + _, node = engine.run_get_node(builder) + assert node.is_finished_ok + + if (retrieved := node.get_retrieved_node()) is None: + raise RuntimeError("No retrieved node found") + + paths = sorted([str(p) for p in retrieved.base.repository.glob()]) + assert paths == [ + "_scheduler-stderr.txt", + "_scheduler-stdout.txt", + "folder1", + "folder1/a", + "folder1/a/b.txt", + "folder1/a/c.txt", + "folder2", + "folder2/remote_copy.txt", + "folder2/x", + "folder2/y", + "folder2/y/z", + ] + + +class MultiFileCalcjob(engine.CalcJob): + """A complex CalcJob that creates/retrieves multiple files.""" + + @classmethod + def define(cls, spec): + """Define the process specification.""" + super().define(spec) + spec.inputs["metadata"]["options"]["resources"].default = { + "num_machines": 1, + "num_mpiprocs_per_machine": 1, + } + spec.input( + "metadata.options.parser_name", valid_type=str, default="testing.noop" + ) + spec.exit_code(400, "ERROR", message="Calculation failed.") + + def prepare_for_submission(self, folder: Folder) -> common.CalcInfo: + """Prepare the calculation job for submission.""" + codeinfo = common.CodeInfo() + codeinfo.code_uuid = self.inputs.code.uuid + + path = Path(folder.get_abs_path("a")).parent + for subpath in [ + "i.txt", + "j.txt", + "folder1/a/b.txt", + "folder1/a/c.txt", + "folder1/a/c.in", + "folder1/c.txt", + "folder2/x", + "folder2/y/z", + ]: + path.joinpath(subpath).parent.mkdir(parents=True, exist_ok=True) + path.joinpath(subpath).touch() + + calcinfo = common.CalcInfo() + calcinfo.codes_info = [codeinfo] + calcinfo.retrieve_list = [("folder1/*/*.txt", ".", 99), ("folder2", ".", 99)] + comp: orm.Computer = self.inputs.code.computer + calcinfo.remote_copy_list = [ + ( + comp.uuid, + f"{comp.get_workdir()}/remote_copy.txt", + "folder2/remote_copy.txt", + ) + ] + # TODO also add remote_symlink_list + + return calcinfo + + +class NoopParser(Parser): + """Parser that does absolutely nothing!""" + + def parse(self, **kwargs): + pass From d2ccb9ecfa7a5f316294c6b9f3b7aa381e312ddd Mon Sep 17 00:00:00 2001 From: Ali Khosravi Date: Fri, 26 Jul 2024 12:14:31 +0200 Subject: [PATCH 25/39] Apply suggestions from code review Co-authored-by: Alexander Goscinski --- aiida_firecrest/transport.py | 6 +----- tests/test_computer.py | 2 +- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/aiida_firecrest/transport.py b/aiida_firecrest/transport.py index 257f3b9..35980e0 100644 --- a/aiida_firecrest/transport.py +++ b/aiida_firecrest/transport.py @@ -32,9 +32,6 @@ class ValidAuthOption(TypedDict, total=False): callback: Callable[..., Any] # for validation -class BuggyError(Exception): - # TODO: Remove this class when the code is stable - """Raised when something should absolutely not happen, but it does.""" def _create_secret_file(ctx: Context, param: InteractiveOption, value: str) -> str: @@ -282,8 +279,7 @@ def __init__( url: str, token_uri: str, client_id: str, - client_secret: str, # | Path, - # unfortunately we cannot store client_secret as a Path, because it is not JSON serializable + client_secret: str, client_machine: str, small_file_size_mb: float, temp_directory: str, diff --git a/tests/test_computer.py b/tests/test_computer.py index fe4e48e..4fde225 100644 --- a/tests/test_computer.py +++ b/tests/test_computer.py @@ -94,7 +94,7 @@ def test_validate_temp_directory(myfirecrest, monkeypatch, tmpdir: Path): assert not Path(tmpdir / "temp_on_server_directory" / "crap.txt").exists() -def test__dynamic_info(myfirecrest, monkeypatch, tmpdir: Path): +def test_dynamic_info(myfirecrest, monkeypatch, tmpdir: Path): from aiida_firecrest.transport import _dynamic_info_direct_size monkeypatch.setattr("click.echo", lambda x: None) From 64838a6d8e8ebc9df0fdc708e308eb97be95d0de Mon Sep 17 00:00:00 2001 From: Ali Date: Fri, 26 Jul 2024 12:21:08 +0200 Subject: [PATCH 26/39] appied review --- .pre-commit-config.yaml | 2 +- README.md | 5 ++--- aiida_firecrest/transport.py | 37 ++++++++++++++---------------------- tests/conftest.py | 8 ++++---- 4 files changed, 21 insertions(+), 31 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 3c13f85..f6012ab 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -44,5 +44,5 @@ repos: additional_dependencies: - "types-PyYAML" - "types-requests" - - "pyfirecrest>=2.5.0" # please change to 2.6.0 when released + - "pyfirecrest>=2.6.0" - "aiida-core>=2.6.0" # please change to 2.6.2 when released diff --git a/README.md b/README.md index 965f23b..88b5caa 100644 --- a/README.md +++ b/README.md @@ -5,9 +5,8 @@ AiiDA Transport/Scheduler plugins for interfacing with [FirecREST](https://products.cscs.ch/firecrest/), via [pyfirecrest](https://github.com/eth-cscs/pyfirecrest). -It is currently tested against [FirecREST v2.4.0](https://github.com/eth-cscs/firecrest/releases/tag/v2.4.0). +It is currently tested against [FirecREST v2.6.0](https://github.com/eth-cscs/pyfirecrest/tree/v2.6.0). -**NOTE:** This plugin is currently dependent on a fork of `aiida-core` from [PR #6043](https://github.com/aiidateam/aiida-core/pull/6043) ## Usage @@ -70,7 +69,7 @@ Client ID: username-client Client Secret: xyz Client Machine: daint Maximum file size for direct transfer (MB) [5.0]: -Temp directory on server: /scratch/something/ +Temp directory on server: /scratch/something/ # "A temp directory on user's space on the server for creating temporary files (compression, extraction, etc.)" Report: Configuring computer firecrest-client for user chrisj_sewell@hotmail.com. Success: firecrest-client successfully configured for chrisj_sewell@hotmail.com ``` diff --git a/aiida_firecrest/transport.py b/aiida_firecrest/transport.py index 35980e0..920d55a 100644 --- a/aiida_firecrest/transport.py +++ b/aiida_firecrest/transport.py @@ -32,8 +32,6 @@ class ValidAuthOption(TypedDict, total=False): callback: Callable[..., Any] # for validation - - def _create_secret_file(ctx: Context, param: InteractiveOption, value: str) -> str: """Create a secret file if the value is not a path to a secret file. The path should be absolute, if it is not, the file will be created in ~/.firecrest. @@ -77,7 +75,7 @@ def _validate_temp_directory(ctx: Context, param: InteractiveOption, value: str) secret = ctx.params["client_secret"] # )#.read_text() small_file_size_mb = ctx.params["small_file_size_mb"] - dummy = FirecrestTransport( + _client = FirecrestTransport( url=firecrest_url, token_uri=token_uri, client_id=client_id, @@ -88,40 +86,33 @@ def _validate_temp_directory(ctx: Context, param: InteractiveOption, value: str) ) # Temp directory routine - if dummy._cwd.joinpath( - dummy._temp_directory + if _client._cwd.joinpath( + _client._temp_directory ).is_file(): # self._temp_directory.is_file(): raise click.BadParameter("Temp directory cannot be a file") - if dummy.path_exists(dummy._temp_directory): - if dummy.listdir(dummy._temp_directory): + if _client.path_exists(_client._temp_directory): + if _client.listdir(_client._temp_directory): # if not configured: confirm = click.confirm( - f"Temp directory {dummy._temp_directory} is not empty. Do you want to flush it?" + f"Temp directory {_client._temp_directory} is not empty. Do you want to flush it?" ) if confirm: - for item in dummy.listdir(dummy._temp_directory): + for item in _client.listdir(_client._temp_directory): # TODO: maybe do recursive delete - dummy.remove(dummy._temp_directory.joinpath(item)) + _client.remove(_client._temp_directory.joinpath(item)) else: click.echo("Please provide an empty temp directory on the server.") raise click.BadParameter( - f"Temp directory {dummy._temp_directory} is not empty" + f"Temp directory {_client._temp_directory} is not empty" ) - # The block below could be moved to a maintenance delete function, if needed - # else: - # # There might still be some residual files in case of previous interrupted connection - # for item in dummy.listdir(dummy._temp_directory): - # # this could be replace with a proper glob later - # if item[:4] == 'temp': - # dummy.remove(dummy._temp_directory.joinpath(item)) else: try: - dummy.mkdir(dummy._temp_directory, ignore_existing=True) + _client.mkdir(_client._temp_directory, ignore_existing=True) except Exception as e: raise click.BadParameter( - f"Could not create temp directory {dummy._temp_directory} on server: {e}" + f"Could not create temp directory {_client._temp_directory} on server: {e}" ) from e click.echo( click.style("Fireport: ", bold=True, fg="magenta") @@ -157,7 +148,7 @@ def _dynamic_info_direct_size( client_machine = ctx.params["client_machine"] secret = ctx.params["client_secret"] # )#.read_text() - dummy = FirecrestTransport( + _client = FirecrestTransport( url=firecrest_url, token_uri=token_uri, client_id=client_id, @@ -167,11 +158,11 @@ def _dynamic_info_direct_size( small_file_size_mb=0.0, ) - prameters = dummy._client.parameters() + parameters = _client._client.parameters() utilities_max_file_size = next( ( item - for item in prameters["utilities"] + for item in parameters["utilities"] if item["name"] == "UTILITIES_MAX_FILE_SIZE" ), None, diff --git a/tests/conftest.py b/tests/conftest.py index d7bc3ef..f9bde2c 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -15,8 +15,8 @@ class Values: _DEFAULT_PAGE_SIZE: int = 25 -@pytest.fixture(name="firecrest_computer") -def _firecrest_computer(myfirecrest, tmpdir: Path): +@pytest.fixture() +def firecrest_computer(myfirecrest, tmpdir: Path): """Create and return a computer configured for Firecrest. Note, the computer is not stored in the database. @@ -61,7 +61,7 @@ def __init__(self, firecrest_url, *args, **kwargs): self.args = args self.kwargs = kwargs - self.whoami = whomai + self.whoami = whoami self.list_files = list_files self.stat = stat_ self.mkdir = mkdir @@ -176,7 +176,7 @@ def poll_active(machine: str, jobs: list[str], page_number: int = 0): ] -def whomai(machine: str): +def whoami(machine: str): assert machine == "MACHINE_NAME" return "test_user" From 08482c808be68cb83010b4d39acc090674f347fb Mon Sep 17 00:00:00 2001 From: Ali Date: Fri, 26 Jul 2024 15:04:13 +0200 Subject: [PATCH 27/39] added a functionality to check the api version --- aiida_firecrest/transport.py | 148 +++++++++++++++++++++++++---------- 1 file changed, 108 insertions(+), 40 deletions(-) diff --git a/aiida_firecrest/transport.py b/aiida_firecrest/transport.py index 920d55a..86953fe 100644 --- a/aiida_firecrest/transport.py +++ b/aiida_firecrest/transport.py @@ -19,6 +19,7 @@ from click.types import ParamType from firecrest import ClientCredentialsAuth, Firecrest # type: ignore[attr-defined] from firecrest.path import FcPath +from packaging.version import Version, parse class ValidAuthOption(TypedDict, total=False): @@ -71,48 +72,48 @@ def _validate_temp_directory(ctx: Context, param: InteractiveOption, value: str) firecrest_url = ctx.params["url"] token_uri = ctx.params["token_uri"] client_id = ctx.params["client_id"] - client_machine = ctx.params["client_machine"] - secret = ctx.params["client_secret"] # )#.read_text() - small_file_size_mb = ctx.params["small_file_size_mb"] + compute_resource = ctx.params["compute_resource"] + secret = ctx.params["client_secret"] - _client = FirecrestTransport( + transport = FirecrestTransport( url=firecrest_url, token_uri=token_uri, client_id=client_id, client_secret=secret, - client_machine=client_machine, + compute_resource=compute_resource, temp_directory=value, - small_file_size_mb=small_file_size_mb, + small_file_size_mb=1.0, # small_file_size_mb is irrelevant here + api_version="100.0.0", # version is irrelevant here ) # Temp directory routine - if _client._cwd.joinpath( - _client._temp_directory + if transport._cwd.joinpath( + transport._temp_directory ).is_file(): # self._temp_directory.is_file(): raise click.BadParameter("Temp directory cannot be a file") - if _client.path_exists(_client._temp_directory): - if _client.listdir(_client._temp_directory): + if transport.path_exists(transport._temp_directory): + if transport.listdir(transport._temp_directory): # if not configured: confirm = click.confirm( - f"Temp directory {_client._temp_directory} is not empty. Do you want to flush it?" + f"Temp directory {transport._temp_directory} is not empty. Do you want to flush it?" ) if confirm: - for item in _client.listdir(_client._temp_directory): + for item in transport.listdir(transport._temp_directory): # TODO: maybe do recursive delete - _client.remove(_client._temp_directory.joinpath(item)) + transport.remove(transport._temp_directory.joinpath(item)) else: click.echo("Please provide an empty temp directory on the server.") raise click.BadParameter( - f"Temp directory {_client._temp_directory} is not empty" + f"Temp directory {transport._temp_directory} is not empty" ) else: try: - _client.mkdir(_client._temp_directory, ignore_existing=True) + transport.mkdir(transport._temp_directory, ignore_existing=True) except Exception as e: raise click.BadParameter( - f"Could not create temp directory {_client._temp_directory} on server: {e}" + f"Could not create temp directory {transport._temp_directory} on server: {e}" ) from e click.echo( click.style("Fireport: ", bold=True, fg="magenta") @@ -122,6 +123,53 @@ def _validate_temp_directory(ctx: Context, param: InteractiveOption, value: str) return value +def _dynamic_info_firecrest_version( + ctx: Context, param: InteractiveOption, value: str +) -> str: + """Find the version of the FirecREST server.""" + # note: right now, unfortunately, the version is not exposed in the API. + # See issue https://github.com/eth-cscs/firecrest/issues/204 + # so here we just develope a workaround to get the version from the server + # basically we check if extract/compress endpoint is available + + import click + + if value != "0": + if parse(value) < parse("1.15.0"): + raise click.BadParameter(f"FirecREST api version {value} is not supported") + return value + + firecrest_url = ctx.params["url"] + token_uri = ctx.params["token_uri"] + client_id = ctx.params["client_id"] + compute_resource = ctx.params["compute_resource"] + secret = ctx.params["client_secret"] + temp_directory = ctx.params["temp_directory"] + + transport = FirecrestTransport( + url=firecrest_url, + token_uri=token_uri, + client_id=client_id, + client_secret=secret, + compute_resource=compute_resource, + temp_directory=temp_directory, + small_file_size_mb=0.0, + api_version="100.0.0", # version is irrelevant here + ) + try: + transport.listdir(transport._cwd.joinpath(temp_directory), recursive=True) + _version = "1.16.0" + except Exception: + # all sort of exceptions can be raised here, but we don't care. Since this is just a workaround + _version = "1.15.0" + + click.echo( + click.style("Fireport: ", bold=True, fg="magenta") + + f"Deployed version of FirecREST api: v{_version}" + ) + return _version + + def _dynamic_info_direct_size( ctx: Context, param: InteractiveOption, value: float ) -> float: @@ -145,20 +193,22 @@ def _dynamic_info_direct_size( firecrest_url = ctx.params["url"] token_uri = ctx.params["token_uri"] client_id = ctx.params["client_id"] - client_machine = ctx.params["client_machine"] - secret = ctx.params["client_secret"] # )#.read_text() + compute_resource = ctx.params["compute_resource"] + secret = ctx.params["client_secret"] + temp_directory = ctx.params["temp_directory"] - _client = FirecrestTransport( + transport = FirecrestTransport( url=firecrest_url, token_uri=token_uri, client_id=client_id, client_secret=secret, - client_machine=client_machine, - temp_directory="", + compute_resource=compute_resource, + temp_directory=temp_directory, small_file_size_mb=0.0, + api_version="100.0.0", # version is irrelevant here ) - parameters = _client._client.parameters() + parameters = transport._client.parameters() utilities_max_file_size = next( ( item @@ -233,12 +283,33 @@ class FirecrestTransport(Transport): }, ), ( - "client_machine", + "compute_resource", { "type": str, "non_interactive_default": False, - "prompt": "Client Machine", - "help": "FirecREST machine secret", + "prompt": "Compute resource (Machine)", + "help": "Compute resources, for example 'daint', 'eiger', etc.", + }, + ), + ( + "temp_directory", + { + "type": str, + "non_interactive_default": False, + "prompt": "Temp directory on server", + "help": "A temp directory on server for creating temporary files (compression, extraction, etc.)", + "callback": _validate_temp_directory, + }, + ), + ( + "api_version", + { + "type": str, + "default": "0", + "non_interactive_default": True, + "prompt": "FirecREST api version [Enter 0 to get this info from server]", + "help": "The version of the FirecREST api deployed on the server", + "callback": _dynamic_info_firecrest_version, }, ), ( @@ -252,16 +323,6 @@ class FirecrestTransport(Transport): "callback": _dynamic_info_direct_size, }, ), - ( - "temp_directory", - { - "type": str, - "non_interactive_default": False, - "prompt": "Temp directory on server", - "help": "A temp directory on server for creating temporary files (compression, extraction, etc.)", - "callback": _validate_temp_directory, - }, - ), ] def __init__( @@ -271,9 +332,10 @@ def __init__( token_uri: str, client_id: str, client_secret: str, - client_machine: str, - small_file_size_mb: float, + compute_resource: str, temp_directory: str, + small_file_size_mb: float, + api_version: str, # note, machine is provided by default, # for the hostname, but we don't use that # TODO ideally hostname would not be necessary on a computer @@ -285,7 +347,7 @@ def __init__( :param token_uri: URI for retrieving FirecREST authentication tokens :param client_id: FirecREST client ID :param client_secret: FirecREST client secret or str(Absolute path) to an existing FirecREST Secret Key - :param client_machine: FirecREST machine secret + :param compute_resource: Compute resources, for example 'daint', 'eiger', etc. :param small_file_size_mb: Maximum file size for direct transfer (MB) :param temp_directory: A temp directory on server for creating temporary files (compression, extraction, etc.) :param kwargs: Additional keyword arguments @@ -300,13 +362,14 @@ def __init__( assert isinstance(token_uri, str), "token_uri must be a string" assert isinstance(client_id, str), "client_id must be a string" assert isinstance(client_secret, str), "client_secret must be a string" - assert isinstance(client_machine, str), "client_machine must be a string" + assert isinstance(compute_resource, str), "compute_resource must be a string" assert isinstance(temp_directory, str), "temp_directory must be a string" + assert isinstance(api_version, str), "api_version must be a string" assert isinstance( small_file_size_mb, float ), "small_file_size_mb must be a float" - self._machine = client_machine + self._machine = compute_resource self._url = url self._token_uri = token_uri self._client_id = client_id @@ -326,6 +389,11 @@ def __init__( self._cwd: FcPath = FcPath(self._client, self._machine, "/", cache_enabled=True) self._temp_directory = self._cwd.joinpath(temp_directory) + self._api_version: Version = parse(api_version) + + if self._api_version < parse("1.16.0"): + self._payoff_override = False + # this makes no sense for firecrest, but we need to set this to True # otherwise the aiida-core will complain that the transport is not open: # aiida-core/src/aiida/orm/utils/remote:clean_remote() From f866a6fe6732d60c87ea05d2ec1d096642455e42 Mon Sep 17 00:00:00 2001 From: Ali Date: Fri, 26 Jul 2024 15:22:07 +0200 Subject: [PATCH 28/39] updated tests --- tests/conftest.py | 3 ++- tests/test_computer.py | 5 +++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index f9bde2c..92a8577 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -47,9 +47,10 @@ def firecrest_computer(myfirecrest, tmpdir: Path): token_uri="https://TOKEN_URI", client_id="CLIENT_ID", client_secret=str(_secret_path), - client_machine="MACHINE_NAME", + compute_resource="MACHINE_NAME", small_file_size_mb=1.0, temp_directory=str(_temp_directory), + api_version="2", ) computer.store() return computer diff --git a/tests/test_computer.py b/tests/test_computer.py index 4fde225..cee0181 100644 --- a/tests/test_computer.py +++ b/tests/test_computer.py @@ -58,7 +58,7 @@ def test_validate_temp_directory(myfirecrest, monkeypatch, tmpdir: Path): "url": "http://test.com", "token_uri": "token_uri", "client_id": "client_id", - "client_machine": "client_machine", + "compute_resource": "compute_resource", "client_secret": secret_file.as_posix(), "small_file_size_mb": float(10), } @@ -106,9 +106,10 @@ def test_dynamic_info(myfirecrest, monkeypatch, tmpdir: Path): "url": "http://test.com", "token_uri": "token_uri", "client_id": "client_id", - "client_machine": "client_machine", + "compute_resource": "compute_resource", "client_secret": secret_file.as_posix(), "small_file_size_mb": float(10), + "temp_directory": "temp_directory", } # should catch UTILITIES_MAX_FILE_SIZE if value is not provided From 6c7101c9fd3e34f3be4a588385b28a809b2e639f Mon Sep 17 00:00:00 2001 From: Alexander Goscinski Date: Fri, 26 Jul 2024 15:46:45 +0200 Subject: [PATCH 29/39] Bringing back the option to run firecrest with a config with a real server (#2) * Bringing back the option to run firecrest with a config with a real server --------- Co-authored-by: Ali Khosravi --- tests/conftest.py | 158 +++++++++++++++++++++++++++++++++++------ tests/test_computer.py | 4 +- 2 files changed, 137 insertions(+), 25 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 92a8577..b289222 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -3,54 +3,51 @@ from pathlib import Path import random import stat -from typing import Optional +from typing import Optional, Any, Callable +import json +from functools import partial +from urllib.parse import urlparse +from dataclasses import dataclass from aiida import orm import firecrest import firecrest.path import pytest +import requests class Values: _DEFAULT_PAGE_SIZE: int = 25 -@pytest.fixture() -def firecrest_computer(myfirecrest, tmpdir: Path): +@pytest.fixture +def firecrest_computer(firecrest_config): """Create and return a computer configured for Firecrest. Note, the computer is not stored in the database. """ # create a temp directory and set it as the workdir - _scratch = tmpdir / "scratch" - _temp_directory = tmpdir / "temp" - _scratch.mkdir() - _temp_directory.mkdir() - - Path(tmpdir / ".firecrest").mkdir() - _secret_path = Path(tmpdir / ".firecrest/secret69") - _secret_path.write_text("SECRET_STRING") computer = orm.Computer( label="test_computer", description="test computer", hostname="-", - workdir=str(_scratch), + workdir=firecrest_config.workdir, transport_type="firecrest", scheduler_type="firecrest", ) computer.set_minimum_job_poll_interval(5) computer.set_default_mpiprocs_per_machine(1) computer.configure( - url=" https://URI", - token_uri="https://TOKEN_URI", - client_id="CLIENT_ID", - client_secret=str(_secret_path), - compute_resource="MACHINE_NAME", - small_file_size_mb=1.0, - temp_directory=str(_temp_directory), - api_version="2", + url=firecrest_config.url, + token_uri=firecrest_config.token_uri, + client_id=firecrest_config.client_id, + client_secret=firecrest_config.client_secret, + compute_resource=firecrest_config.compute_resource, + small_file_size_mb=firecrest_config.small_file_size_mb, + temp_directory=firecrest_config.temp_directory, + api_version=firecrest_config.api_version, ) computer.store() return computer @@ -84,15 +81,130 @@ def __init__(self, *args, **kwargs): self.args = args self.kwargs = kwargs +@dataclass +class ComputerFirecrestConfig: + """Configuration of a computer using FirecREST as transport plugin.""" + + url: str + token_uri: str + client_id: str + client_secret: str + machine: str + temp_directory: str + workdir: str + small_file_size_mb: float = 1.0 + +class RequestTelemetry: + """A to gather telemetry on requests.""" + + def __init__(self) -> None: + self.counts = {} + + def wrap( + self, + method: Callable[..., requests.Response], + url: str | bytes, + **kwargs: Any, + ) -> requests.Response: + """Wrap a requests method to gather telemetry.""" + endpoint = urlparse(url if isinstance(url, str) else url.decode("utf-8")).path + self.counts.setdefault(endpoint, 0) + self.counts[endpoint] += 1 + return method(url, **kwargs) @pytest.fixture(scope="function") -def myfirecrest( +def firecrest_config( pytestconfig: pytest.Config, + request: pytest.FixtureRequest, monkeypatch, + tmp_path: Path ): - monkeypatch.setattr(firecrest, "Firecrest", MockFirecrest) - monkeypatch.setattr(firecrest, "ClientCredentialsAuth", MockClientCredentialsAuth) + """ + If a config file is given it sets up a client environment with the information + of the config file and uses pyfirecrest to communicate with the server. + ┌─────────────────┐───►┌─────────────┐───►┌──────────────────┐ + │ aiida_firecrest │ │ pyfirecrest │ │ FirecREST server │ + └─────────────────┘◄───└─────────────┘◄───└──────────────────┘ + + if `config_path` is not given, it monkeypatches pyfirecrest so we never + actually communicate with a server. + ┌─────────────────┐───►┌─────────────────────────────┐ + │ aiida_firecrest │ │ pyfirecrest (monkeypatched) │ + └─────────────────┘◄───└─────────────────────────────┘ + """ + config_path: str | None = request.config.getoption("--firecrest-config") + no_clean: bool = request.config.getoption("--firecrest-no-clean") + record_requests: bool = request.config.getoption("--firecrest-requests") + + if config_path is not None: + telemetry: RequestTelemetry | None = None + # if given, use this config + with open(config_path, encoding="utf8") as handle: + config = json.load(handle) + config = ComputerFirecrestConfig(**config) + # rather than use the scratch_path directly, we use a subfolder, + # which we can then clean + config.workdir = config.workdir + "/pytest_tmp" + + # we need to connect to the client here, + # to ensure that the scratch path exists and is empty + client = firecrest.Firecrest( + firecrest_url=config.url, + authorization=firecrest.ClientCredentialsAuth( + config.client_id, config.client_secret, config.token_uri + ), + ) + client.mkdir(config.machine, config.scratch_path, p=True) + if record_requests: + telemetry = RequestTelemetry() + monkeypatch.setattr(requests, "get", partial(telemetry.wrap, requests.get)) + monkeypatch.setattr( + requests, "post", partial(telemetry.wrap, requests.post) + ) + monkeypatch.setattr(requests, "put", partial(telemetry.wrap, requests.put)) + monkeypatch.setattr( + requests, "delete", partial(telemetry.wrap, requests.delete) + ) + yield config + # Note this shouldn't really work, for folders but it does :shrug: + # because they use `rm -r`: + # https://github.com/eth-cscs/firecrest/blob/7f02d11b224e4faee7f4a3b35211acb9c1cc2c6a/src/utilities/utilities.py#L347 + if not no_clean: + client.simple_delete(config.machine, config.scratch_path) + + if telemetry is not None: + test_name = request.node.name + pytestconfig.stash.setdefault("firecrest_requests", {})[ + test_name + ] = telemetry.counts + else: + if no_clean or record_requests: + raise ValueError("--firecrest-{no-clean,requests} options are only available when a config file is passed using --firecrest-config.") + + monkeypatch.setattr(firecrest, "Firecrest", MockFirecrest) + monkeypatch.setattr(firecrest, "ClientCredentialsAuth", MockClientCredentialsAuth) + + # dummy config + _temp_directory = tmp_path / "temp" + _temp_directory.mkdir() + + Path(tmp_path / ".firecrest").mkdir() + _secret_path = Path(tmp_path / ".firecrest/secret69") + _secret_path.write_text("secret_string") + + workdir = tmp_path / "scratch" + + yield ComputerFirecrestConfig( + url="https://URI", + token_uri="https://TOKEN_URI", + client_id="CLIENT_ID", + client_secret=str(_secret_path), + compute_resource="MACHINE_NAME", + small_file_size_mb=1.0, + temp_directory=str(_temp_directory), + api_version="2", + ) def submit( machine: str, diff --git a/tests/test_computer.py b/tests/test_computer.py index cee0181..6a183c0 100644 --- a/tests/test_computer.py +++ b/tests/test_computer.py @@ -46,7 +46,7 @@ def test_create_secret_file_with_secret_value(tmp_path, monkeypatch): assert Path(result).read_text() == secret -def test_validate_temp_directory(myfirecrest, monkeypatch, tmpdir: Path): +def test_validate_temp_directory(firecrest_config, monkeypatch, tmpdir: Path): from aiida_firecrest.transport import _validate_temp_directory monkeypatch.setattr("click.echo", lambda x: None) @@ -94,7 +94,7 @@ def test_validate_temp_directory(myfirecrest, monkeypatch, tmpdir: Path): assert not Path(tmpdir / "temp_on_server_directory" / "crap.txt").exists() -def test_dynamic_info(myfirecrest, monkeypatch, tmpdir: Path): +def test_dynamic_info(firecrest_config, monkeypatch, tmpdir: Path): from aiida_firecrest.transport import _dynamic_info_direct_size monkeypatch.setattr("click.echo", lambda x: None) From 364e3a4952bb692db9071969978b6836e29652f3 Mon Sep 17 00:00:00 2001 From: Ali Date: Fri, 26 Jul 2024 16:21:17 +0200 Subject: [PATCH 30/39] fix tests --- fixtests.patch | 48 +++++++++++++++++++++++++++++++++++++++++++++++ tests/conftest.py | 34 ++++++++++++++++++++++----------- 2 files changed, 71 insertions(+), 11 deletions(-) create mode 100644 fixtests.patch diff --git a/fixtests.patch b/fixtests.patch new file mode 100644 index 0000000..2dea180 --- /dev/null +++ b/fixtests.patch @@ -0,0 +1,48 @@ +commit 987dbb02f0bf483898b3a2d7b98411da111012c1 +Author: Alexander Goscinski +Date: Fri Jul 26 15:57:49 2024 +0200 + + fix tests + +diff --git a/tests/conftest.py b/tests/conftest.py +index b289222..81c5d0c 100644 +--- a/tests/conftest.py ++++ b/tests/conftest.py +@@ -89,9 +89,10 @@ class ComputerFirecrestConfig: + token_uri: str + client_id: str + client_secret: str +- machine: str ++ compute_resource: str + temp_directory: str + workdir: str ++ api_version: str + small_file_size_mb: float = 1.0 + + class RequestTelemetry: +@@ -154,7 +155,7 @@ def firecrest_config( + config.client_id, config.client_secret, config.token_uri + ), + ) +- client.mkdir(config.machine, config.scratch_path, p=True) ++ client.mkdir(config.compute_resource, config.scratch_path, p=True) + + if record_requests: + telemetry = RequestTelemetry() +@@ -171,7 +172,7 @@ def firecrest_config( + # because they use `rm -r`: + # https://github.com/eth-cscs/firecrest/blob/7f02d11b224e4faee7f4a3b35211acb9c1cc2c6a/src/utilities/utilities.py#L347 + if not no_clean: +- client.simple_delete(config.machine, config.scratch_path) ++ client.simple_delete(config.compute_resource, config.scratch_path) + + if telemetry is not None: + test_name = request.node.name +@@ -201,6 +202,7 @@ def firecrest_config( + client_id="CLIENT_ID", + client_secret=str(_secret_path), + compute_resource="MACHINE_NAME", ++ workdir=str(workdir), + small_file_size_mb=1.0, + temp_directory=str(_temp_directory), + api_version="2", diff --git a/tests/conftest.py b/tests/conftest.py index b289222..5a5a7e0 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,13 +1,13 @@ +from dataclasses import dataclass +from functools import partial import hashlib +import json import os from pathlib import Path import random import stat -from typing import Optional, Any, Callable -import json -from functools import partial +from typing import Any, Callable, Optional from urllib.parse import urlparse -from dataclasses import dataclass from aiida import orm import firecrest @@ -81,6 +81,7 @@ def __init__(self, *args, **kwargs): self.args = args self.kwargs = kwargs + @dataclass class ComputerFirecrestConfig: """Configuration of a computer using FirecREST as transport plugin.""" @@ -89,11 +90,13 @@ class ComputerFirecrestConfig: token_uri: str client_id: str client_secret: str - machine: str + compute_resource: str temp_directory: str workdir: str + api_version: str small_file_size_mb: float = 1.0 + class RequestTelemetry: """A to gather telemetry on requests.""" @@ -112,12 +115,13 @@ def wrap( self.counts[endpoint] += 1 return method(url, **kwargs) + @pytest.fixture(scope="function") def firecrest_config( pytestconfig: pytest.Config, request: pytest.FixtureRequest, monkeypatch, - tmp_path: Path + tmp_path: Path, ): """ If a config file is given it sets up a client environment with the information @@ -154,7 +158,7 @@ def firecrest_config( config.client_id, config.client_secret, config.token_uri ), ) - client.mkdir(config.machine, config.scratch_path, p=True) + client.mkdir(config.compute_resource, config.scratch_path, p=True) if record_requests: telemetry = RequestTelemetry() @@ -171,7 +175,7 @@ def firecrest_config( # because they use `rm -r`: # https://github.com/eth-cscs/firecrest/blob/7f02d11b224e4faee7f4a3b35211acb9c1cc2c6a/src/utilities/utilities.py#L347 if not no_clean: - client.simple_delete(config.machine, config.scratch_path) + client.simple_delete(config.compute_resource, config.scratch_path) if telemetry is not None: test_name = request.node.name @@ -180,10 +184,15 @@ def firecrest_config( ] = telemetry.counts else: if no_clean or record_requests: - raise ValueError("--firecrest-{no-clean,requests} options are only available when a config file is passed using --firecrest-config.") + raise ValueError( + "--firecrest-{no-clean,requests} options are only available" + " when a config file is passed using --firecrest-config." + ) monkeypatch.setattr(firecrest, "Firecrest", MockFirecrest) - monkeypatch.setattr(firecrest, "ClientCredentialsAuth", MockClientCredentialsAuth) + monkeypatch.setattr( + firecrest, "ClientCredentialsAuth", MockClientCredentialsAuth + ) # dummy config _temp_directory = tmp_path / "temp" @@ -192,8 +201,9 @@ def firecrest_config( Path(tmp_path / ".firecrest").mkdir() _secret_path = Path(tmp_path / ".firecrest/secret69") _secret_path.write_text("secret_string") - + workdir = tmp_path / "scratch" + workdir.mkdir() yield ComputerFirecrestConfig( url="https://URI", @@ -201,11 +211,13 @@ def firecrest_config( client_id="CLIENT_ID", client_secret=str(_secret_path), compute_resource="MACHINE_NAME", + workdir=str(workdir), small_file_size_mb=1.0, temp_directory=str(_temp_directory), api_version="2", ) + def submit( machine: str, script_str: Optional[str] = None, From 59942858bcbef3b90e8c9398ce20c7e1378ec671 Mon Sep 17 00:00:00 2001 From: Ali Date: Fri, 26 Jul 2024 16:24:19 +0200 Subject: [PATCH 31/39] fixtests.patch removed --- fixtests.patch | 48 ------------------------------------------------ 1 file changed, 48 deletions(-) delete mode 100644 fixtests.patch diff --git a/fixtests.patch b/fixtests.patch deleted file mode 100644 index 2dea180..0000000 --- a/fixtests.patch +++ /dev/null @@ -1,48 +0,0 @@ -commit 987dbb02f0bf483898b3a2d7b98411da111012c1 -Author: Alexander Goscinski -Date: Fri Jul 26 15:57:49 2024 +0200 - - fix tests - -diff --git a/tests/conftest.py b/tests/conftest.py -index b289222..81c5d0c 100644 ---- a/tests/conftest.py -+++ b/tests/conftest.py -@@ -89,9 +89,10 @@ class ComputerFirecrestConfig: - token_uri: str - client_id: str - client_secret: str -- machine: str -+ compute_resource: str - temp_directory: str - workdir: str -+ api_version: str - small_file_size_mb: float = 1.0 - - class RequestTelemetry: -@@ -154,7 +155,7 @@ def firecrest_config( - config.client_id, config.client_secret, config.token_uri - ), - ) -- client.mkdir(config.machine, config.scratch_path, p=True) -+ client.mkdir(config.compute_resource, config.scratch_path, p=True) - - if record_requests: - telemetry = RequestTelemetry() -@@ -171,7 +172,7 @@ def firecrest_config( - # because they use `rm -r`: - # https://github.com/eth-cscs/firecrest/blob/7f02d11b224e4faee7f4a3b35211acb9c1cc2c6a/src/utilities/utilities.py#L347 - if not no_clean: -- client.simple_delete(config.machine, config.scratch_path) -+ client.simple_delete(config.compute_resource, config.scratch_path) - - if telemetry is not None: - test_name = request.node.name -@@ -201,6 +202,7 @@ def firecrest_config( - client_id="CLIENT_ID", - client_secret=str(_secret_path), - compute_resource="MACHINE_NAME", -+ workdir=str(workdir), - small_file_size_mb=1.0, - temp_directory=str(_temp_directory), - api_version="2", From 63235473b64ee399b68fcc50aa3fb106e6511e88 Mon Sep 17 00:00:00 2001 From: Ali Date: Fri, 26 Jul 2024 16:32:57 +0200 Subject: [PATCH 32/39] __future__ added --- tests/conftest.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 5a5a7e0..b6df21f 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,3 +1,5 @@ +from __future__ import annotations + from dataclasses import dataclass from functools import partial import hashlib @@ -6,7 +8,7 @@ from pathlib import Path import random import stat -from typing import Any, Callable, Optional +from typing import Any, Callable from urllib.parse import urlparse from aiida import orm @@ -220,9 +222,9 @@ def firecrest_config( def submit( machine: str, - script_str: Optional[str] = None, - script_remote_path: Optional[str] = None, - script_local_path: Optional[str] = None, + script_str: str | None = None, + script_remote_path: str | None = None, + script_local_path: str | None = None, local_file=False, ): if local_file: @@ -399,7 +401,7 @@ def simple_download(machine: str, remote_path: str, local_path: str): def simple_upload( - machine: str, local_path: str, remote_path: str, file_name: Optional[str] = None + machine: str, local_path: str, remote_path: str, file_name: str | None = None ): # this procedure is complecated in firecrest, but I am simplifying it here # we don't care about the details of the upload, we just want to make sure From 7cc3e970f7ab39d121c6781f9075f56b1f18ffd5 Mon Sep 17 00:00:00 2001 From: Ali Date: Mon, 29 Jul 2024 19:27:58 +0200 Subject: [PATCH 33/39] Final updates on tests, now functional with server test. CI to be added in a separate PR. --- .firecrest-demo-config.json | 11 + .gitignore | 3 + aiida_firecrest/transport.py | 24 +- tests/conftest.py | 84 ++-- tests/test_computer.py | 78 ++-- tests/test_transport.py | 857 +++++++++++++++++++++-------------- 6 files changed, 628 insertions(+), 429 deletions(-) create mode 100644 .firecrest-demo-config.json diff --git a/.firecrest-demo-config.json b/.firecrest-demo-config.json new file mode 100644 index 0000000..271dd6b --- /dev/null +++ b/.firecrest-demo-config.json @@ -0,0 +1,11 @@ + { + "url": "", + "token_uri": "", + "client_id": "", + "client_secret": "", + "compute_resource": "", + "temp_directory": "", + "small_file_size_mb": 5.0, + "workdir": "", + "api_version": "1.16.0" +} diff --git a/.gitignore b/.gitignore index bbf1cd9..23708d4 100644 --- a/.gitignore +++ b/.gitignore @@ -132,3 +132,6 @@ dmypy.json .vscode/ .demo-server/ _archive/ + + +.firecrest-config.json diff --git a/aiida_firecrest/transport.py b/aiida_firecrest/transport.py index 86953fe..f30303a 100644 --- a/aiida_firecrest/transport.py +++ b/aiida_firecrest/transport.py @@ -57,7 +57,6 @@ def _create_secret_file(ctx: Context, param: InteractiveOption, value: str) -> s click.style("Fireport: ", bold=True, fg="magenta") + f"Client Secret stored at {secret_path}" ) - return str(secret_path) @@ -377,7 +376,7 @@ def __init__( self._payoff_override: bool | None = None - secret = Path(client_secret).read_text() + secret = Path(client_secret).read_text().strip() try: self._client = Firecrest( firecrest_url=self._url, @@ -515,11 +514,20 @@ def listdir( def makedirs(self, path: str, ignore_existing: bool = False) -> None: """Make directories on the remote.""" + new_path = self._cwd.joinpath(path) + if not ignore_existing and new_path.exists(): + # Note: FirecREST does not raise an error if the directory already exists, and parent is True. + # which makes sense, but following the Superclass, we should raise an OSError in that case. + # AiiDA expects an OSError, instead of a FileExistsError + raise OSError(f"'{path}' already exists") self._cwd.joinpath(path).mkdir(parents=True, exist_ok=ignore_existing) def mkdir(self, path: str, ignore_existing: bool = False) -> None: """Make a directory on the remote.""" - self._cwd.joinpath(path).mkdir(exist_ok=ignore_existing) + try: + self._cwd.joinpath(path).mkdir(exist_ok=ignore_existing) + except FileExistsError as err: + raise OSError(f"'{path}' already exists") from err def normalize(self, path: str = ".") -> str: """Resolve the path.""" @@ -758,7 +766,7 @@ def _gettreetar( self, remotepath: str | FcPath, localpath: str | Path, - dereference: bool = False, + dereference: bool = True, *args: Any, **kwargs: Any, ) -> None: @@ -768,15 +776,15 @@ def _gettreetar( Note that this method is not part of the Transport interface, and is not meant to be used publicly. :param dereference: If True, follow symlinks. - note: FirecREST doesn't support `--dereference` for tar call, - so dereference should always be False, for now. """ _ = uuid.uuid4() remote_path_temp = self._temp_directory.joinpath(f"temp_{_}.tar") # Compress - self._client.compress(self._machine, str(remotepath), remote_path_temp) + self._client.compress( + self._machine, str(remotepath), remote_path_temp, dereference=dereference + ) # Download localpath_temp = Path(localpath).joinpath(f"temp_{_}.tar") @@ -840,7 +848,7 @@ def gettree( if self.payoff(remote): # in this case send a request to the server to tar the files and then download the tar file # unfortunately, the server does not provide a deferenced tar option, yet. - self._gettreetar(remote, local) + self._gettreetar(remote, local, dereference=dereference) else: # otherwise download the files one by one for remote_item in remote.iterdir(recursive=True): diff --git a/tests/conftest.py b/tests/conftest.py index b6df21f..d95429e 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,12 +1,12 @@ from __future__ import annotations from dataclasses import dataclass -from functools import partial import hashlib import json import os from pathlib import Path import random +import shutil import stat from typing import Any, Callable from urllib.parse import urlparse @@ -51,7 +51,6 @@ def firecrest_computer(firecrest_config): temp_directory=firecrest_config.temp_directory, api_version=firecrest_config.api_version, ) - computer.store() return computer @@ -100,7 +99,10 @@ class ComputerFirecrestConfig: class RequestTelemetry: - """A to gather telemetry on requests.""" + """A to gather telemetry on requests. + This class is stale and not used in the current implementation. + We keep it here for future use, if needed. + """ def __init__(self) -> None: self.counts = {} @@ -120,7 +122,6 @@ def wrap( @pytest.fixture(scope="function") def firecrest_config( - pytestconfig: pytest.Config, request: pytest.FixtureRequest, monkeypatch, tmp_path: Path, @@ -143,47 +144,52 @@ def firecrest_config( record_requests: bool = request.config.getoption("--firecrest-requests") if config_path is not None: - telemetry: RequestTelemetry | None = None + # telemetry: RequestTelemetry | None = None # if given, use this config with open(config_path, encoding="utf8") as handle: config = json.load(handle) config = ComputerFirecrestConfig(**config) - # rather than use the scratch_path directly, we use a subfolder, - # which we can then clean + # # rather than use the scratch_path directly, we use a subfolder, + # # which we can then clean config.workdir = config.workdir + "/pytest_tmp" + config.temp_directory = config.temp_directory + "/pytest_tmp" - # we need to connect to the client here, - # to ensure that the scratch path exists and is empty + # # we need to connect to the client here, + # # to ensure that the scratch path exists and is empty client = firecrest.Firecrest( firecrest_url=config.url, authorization=firecrest.ClientCredentialsAuth( - config.client_id, config.client_secret, config.token_uri + config.client_id, + Path(config.client_secret).read_text().strip(), + config.token_uri, ), ) - client.mkdir(config.compute_resource, config.scratch_path, p=True) - - if record_requests: - telemetry = RequestTelemetry() - monkeypatch.setattr(requests, "get", partial(telemetry.wrap, requests.get)) - monkeypatch.setattr( - requests, "post", partial(telemetry.wrap, requests.post) - ) - monkeypatch.setattr(requests, "put", partial(telemetry.wrap, requests.put)) - monkeypatch.setattr( - requests, "delete", partial(telemetry.wrap, requests.delete) - ) + client.mkdir(config.compute_resource, config.workdir, p=True) + client.mkdir(config.compute_resource, config.temp_directory, p=True) + + # if record_requests: + # telemetry = RequestTelemetry() + # monkeypatch.setattr(requests, "get", partial(telemetry.wrap, requests.get)) + # monkeypatch.setattr( + # requests, "post", partial(telemetry.wrap, requests.post) + # ) + # monkeypatch.setattr(requests, "put", partial(telemetry.wrap, requests.put)) + # monkeypatch.setattr( + # requests, "delete", partial(telemetry.wrap, requests.delete) + # ) yield config - # Note this shouldn't really work, for folders but it does :shrug: - # because they use `rm -r`: - # https://github.com/eth-cscs/firecrest/blob/7f02d11b224e4faee7f4a3b35211acb9c1cc2c6a/src/utilities/utilities.py#L347 + # # Note this shouldn't really work, for folders but it does :shrug: + # # because they use `rm -r`: + # # https://github.com/eth-cscs/firecrest/blob/7f02d11b224e4faee7f4a3b35211acb9c1cc2c6a/src/utilities/utilities.py#L347 if not no_clean: - client.simple_delete(config.compute_resource, config.scratch_path) - - if telemetry is not None: - test_name = request.node.name - pytestconfig.stash.setdefault("firecrest_requests", {})[ - test_name - ] = telemetry.counts + client.simple_delete(config.compute_resource, config.workdir) + client.simple_delete(config.compute_resource, config.temp_directory) + + # if telemetry is not None: + # test_name = request.node.name + # pytestconfig.stash.setdefault("firecrest_requests", {})[ + # test_name + # ] = telemetry.counts else: if no_clean or record_requests: raise ValueError( @@ -368,18 +374,18 @@ def stat_(machine: str, targetpath: firecrest.path, dereference=True): } -def mkdir(machine: str, target_path: str, p: bool = False): - if p: - os.makedirs(target_path) - else: - os.mkdir(target_path) +def mkdir( + machine: str, target_path: str, p: bool = False, ignore_existing: bool = False +): + target = Path(target_path) + target.mkdir(exist_ok=ignore_existing, parents=p) def simple_delete(machine: str, target_path: str): if not Path(target_path).exists(): raise FileNotFoundError(f"File or folder {target_path} does not exist") if os.path.isdir(target_path): - os.rmdir(target_path) + shutil.rmtree(target_path) else: os.remove(target_path) @@ -503,7 +509,7 @@ def parameters(): "description": "The maximum allowable file size for various operations of the utilities microservice", "name": "UTILITIES_MAX_FILE_SIZE", "unit": "MB", - "value": "69", + "value": "5", }, { "description": ( diff --git a/tests/test_computer.py b/tests/test_computer.py index 6a183c0..6a2041a 100644 --- a/tests/test_computer.py +++ b/tests/test_computer.py @@ -10,7 +10,7 @@ def test_whoami(firecrest_computer: orm.Computer): """check if it is possible to determine the username.""" transport = firecrest_computer.get_transport() - assert transport.whoami() == "test_user" + assert isinstance(transport.whoami(), str) def test_create_secret_file_with_existing_file(tmpdir: Path): @@ -46,75 +46,85 @@ def test_create_secret_file_with_secret_value(tmp_path, monkeypatch): assert Path(result).read_text() == secret -def test_validate_temp_directory(firecrest_config, monkeypatch, tmpdir: Path): +@pytest.mark.usefixtures("aiida_profile_clean") +def test_validate_temp_directory( + firecrest_computer: orm.Computer, firecrest_config, monkeypatch, tmpdir: Path +): + """ + Test the validation of the temp directory. + Note: this test depends on a functional putfile() method, for consistency with the real server tests. + Before running this test, make sure putfile() is working, which is tested in `test_putfile_getfile`. + """ from aiida_firecrest.transport import _validate_temp_directory monkeypatch.setattr("click.echo", lambda x: None) - # monkeypatch.setattr('click.BadParameter', lambda x: None) - secret_file = Path(tmpdir / "secret") - secret_file.write_text("topsecret") ctx = Mock() ctx.params = { - "url": "http://test.com", - "token_uri": "token_uri", - "client_id": "client_id", - "compute_resource": "compute_resource", - "client_secret": secret_file.as_posix(), - "small_file_size_mb": float(10), + "url": f"{firecrest_config.url}", + "token_uri": f"{firecrest_config.token_uri}", + "client_id": f"{firecrest_config.client_id}", + "client_secret": f"{firecrest_config.client_secret}", + "compute_resource": f"{firecrest_config.compute_resource}", + "small_file_size_mb": float(5), + "api_version": f"{firecrest_config.api_version}", } + # prepare some files and directories for testing + transport = firecrest_computer.get_transport() + _remote = transport._temp_directory + _local = tmpdir + Path(tmpdir / "_.txt").touch() + transport.mkdir(_remote / "temp_on_server_directory") + transport.putfile(tmpdir / "_.txt", _remote / "_.txt") + transport.putfile(tmpdir / "_.txt", _remote / "temp_on_server_directory" / "_.txt") + # should raise if is_file - Path(tmpdir / "crap.txt").touch() with pytest.raises(BadParameter): - result = _validate_temp_directory( - ctx, None, Path(tmpdir / "crap.txt").as_posix() - ) + result = _validate_temp_directory(ctx, None, Path(_remote / "_.txt").as_posix()) # should create the directory if it doesn't exist result = _validate_temp_directory( - ctx, None, Path(tmpdir / "temp_on_server_directory").as_posix() + ctx, None, Path(_remote / "nonexisting").as_posix() ) - assert result == Path(tmpdir / "temp_on_server_directory").as_posix() - assert Path(tmpdir / "temp_on_server_directory").exists() + assert result == Path(_remote / "nonexisting").as_posix() + assert transport._cwd.joinpath(_remote / "nonexisting").exists() # should get a confirmation if the directory exists and is not empty - Path(tmpdir / "temp_on_server_directory" / "crap.txt").touch() monkeypatch.setattr("click.confirm", lambda x: False) with pytest.raises(BadParameter): result = _validate_temp_directory( - ctx, None, Path(tmpdir / "temp_on_server_directory").as_posix() + ctx, None, Path(_remote / "temp_on_server_directory").as_posix() ) # should delete the content if I confirm monkeypatch.setattr("click.confirm", lambda x: True) result = _validate_temp_directory( - ctx, None, Path(tmpdir / "temp_on_server_directory").as_posix() + ctx, None, Path(_remote / "temp_on_server_directory").as_posix() ) - assert result == Path(tmpdir / "temp_on_server_directory").as_posix() - assert not Path(tmpdir / "temp_on_server_directory" / "crap.txt").exists() + assert result == Path(_remote / "temp_on_server_directory").as_posix() + assert not transport._cwd.joinpath( + _remote / "temp_on_server_directory" / "_.txt" + ).exists() def test_dynamic_info(firecrest_config, monkeypatch, tmpdir: Path): from aiida_firecrest.transport import _dynamic_info_direct_size monkeypatch.setattr("click.echo", lambda x: None) - # monkeypatch.setattr('click.BadParameter', lambda x: None) - secret_file = Path(tmpdir / "secret") - secret_file.write_text("topsecret") ctx = Mock() ctx.params = { - "url": "http://test.com", - "token_uri": "token_uri", - "client_id": "client_id", - "compute_resource": "compute_resource", - "client_secret": secret_file.as_posix(), - "small_file_size_mb": float(10), - "temp_directory": "temp_directory", + "url": f"{firecrest_config.url}", + "token_uri": f"{firecrest_config.token_uri}", + "client_id": f"{firecrest_config.client_id}", + "client_secret": f"{firecrest_config.client_secret}", + "compute_resource": f"{firecrest_config.compute_resource}", + "temp_directory": f"{firecrest_config.temp_directory}", + "api_version": f"{firecrest_config.api_version}", } # should catch UTILITIES_MAX_FILE_SIZE if value is not provided result = _dynamic_info_direct_size(ctx, None, 0) - assert result == 69 + assert result == 5 # should use the value if provided # note: user cannot enter negative numbers anyways, click raise as this shoule be float not str diff --git a/tests/test_transport.py b/tests/test_transport.py index edd11ee..1726e7a 100644 --- a/tests/test_transport.py +++ b/tests/test_transport.py @@ -1,3 +1,7 @@ +""" +Note: order of tests is important, as some tests are dependent on the previous ones. +""" + import os from pathlib import Path from unittest.mock import patch @@ -7,8 +11,9 @@ @pytest.mark.usefixtures("aiida_profile_clean") -def test_mkdir(firecrest_computer: orm.Computer, tmpdir: Path): +def test_mkdir(firecrest_computer: orm.Computer): transport = firecrest_computer.get_transport() + tmpdir = transport._temp_directory _scratch = tmpdir / "sampledir" transport.mkdir(_scratch) @@ -18,23 +23,24 @@ def test_mkdir(firecrest_computer: orm.Computer, tmpdir: Path): transport.makedirs(_scratch) assert _scratch.exists() + # raise if directory already exists + with pytest.raises(OSError): + transport.mkdir(tmpdir / "sampledir") + with pytest.raises(OSError): + transport.makedirs(tmpdir / "sampledir2") -@pytest.mark.usefixtures("aiida_profile_clean") -def test_is_file(firecrest_computer: orm.Computer, tmpdir: Path): - transport = firecrest_computer.get_transport() - - _scratch = tmpdir / "samplefile" - Path(_scratch).touch() - assert transport.isfile(_scratch) - assert not transport.isfile(_scratch / "does_not_exist") + # don't raise if directory already exists and ignore_existing is True + transport.mkdir(tmpdir / "sampledir", ignore_existing=True) + transport.makedirs(tmpdir / "sampledir2", ignore_existing=True) @pytest.mark.usefixtures("aiida_profile_clean") -def test_is_dir(firecrest_computer: orm.Computer, tmpdir: Path): +def test_is_dir(firecrest_computer: orm.Computer): transport = firecrest_computer.get_transport() + tmpdir = transport._temp_directory _scratch = tmpdir / "sampledir" - _scratch.mkdir() + transport.mkdir(_scratch) assert transport.isdir(_scratch) assert not transport.isdir(_scratch / "does_not_exist") @@ -59,267 +65,206 @@ def test_normalize(firecrest_computer: orm.Computer): @pytest.mark.usefixtures("aiida_profile_clean") -def test_remove(firecrest_computer: orm.Computer, tmpdir: Path): +def test_putfile_getfile(firecrest_computer: orm.Computer, tmpdir: Path): + """ + Note: putfile() and getfile() should be tested together, as they are dependent on each other. + It's written this way to be compatible with the real server testings. + """ transport = firecrest_computer.get_transport() + tmpdir_remote = transport._temp_directory - _scratch = tmpdir / "samplefile" - Path(_scratch).touch() - transport.remove(_scratch) - assert not _scratch.exists() + _remote = tmpdir_remote / "remotedir" + _local = tmpdir / "localdir" + _local_download = tmpdir / "download" + _remote.mkdir() + _local.mkdir() + _local_download.mkdir() - _scratch = tmpdir / "sampledir" - _scratch.mkdir() - transport.rmtree(_scratch) - assert not _scratch.exists() + Path(_local / "file1").write_text("file1") + Path(_local / ".hidden").write_text(".hidden") + os.symlink(_local / "file1", _local / "file1_link") - _scratch = tmpdir / "sampledir" - _scratch.mkdir() - Path(_scratch / "samplefile").touch() - with pytest.raises(OSError): - transport.rmdir(_scratch) + # raise if file does not exist + with pytest.raises(FileNotFoundError): + transport.putfile(_local / "does_not_exist", _remote / "file1") + transport.getfile(_remote / "does_not_exist", _local) - os.remove(_scratch / "samplefile") - transport.rmdir(_scratch) - assert not _scratch.exists() + # raise if filename is not provided + with pytest.raises(ValueError): + transport.putfile(_local / "file1", _remote) + transport.getfile(_remote / "file1", _local) + # raise if localpath is relative + with pytest.raises(ValueError): + transport.putfile(Path(_local / "file1").relative_to(tmpdir), _remote / "file1") + transport.getfile(_remote / "file1", Path(_local / "file1").relative_to(tmpdir)) -@pytest.mark.usefixtures("aiida_profile_clean") -def test_symlink(firecrest_computer: orm.Computer, tmpdir: Path): - transport = firecrest_computer.get_transport() + # don't mix up directory with file + with pytest.raises(ValueError): + transport.putfile(_local, _remote / "file1") + transport.getfile(_remote, _local / "file1") - _scratch = Path(tmpdir / "samplefile-2sym") - Path(_scratch).touch() - _symlink = Path(tmpdir / "samplelink") - transport.symlink(_scratch, _symlink) - assert _symlink.is_symlink() - assert _symlink.resolve() == _scratch + # write where I tell you to + # note: if you change this block, you need to change the the following two blocks as well + transport.putfile(_local / "file1", _remote / "file1") + transport.putfile(_local / "file1", _remote / "file1-prime") + transport.getfile(_remote / "file1", _local_download / "file1") + transport.getfile(_remote / "file1-prime", _local_download / "differentName") -@pytest.mark.usefixtures("aiida_profile_clean") -def test_listdir(firecrest_computer: orm.Computer, tmpdir: Path): - transport = firecrest_computer.get_transport() + assert Path(_local_download / "file1").read_text() == "file1" + assert Path(_local_download / "differentName").read_text() == "file1" - _scratch = tmpdir / "sampledir" - _scratch.mkdir() - # to test basics - Path(_scratch / "file1").touch() - Path(_scratch / "dir1").mkdir() - Path(_scratch / ".hidden").touch() - # to test recursive - Path(_scratch / "dir1" / "file2").touch() - - assert set(transport.listdir(_scratch)) == {"file1", "dir1", ".hidden"} - assert set(transport.listdir(_scratch, recursive=True)) == { - "file1", - "dir1", - ".hidden", - "dir1/file2", - } - # to test symlink - Path(_scratch / "dir1" / "dir2").mkdir() - Path(_scratch / "dir1" / "dir2" / "file3").touch() - os.symlink(_scratch / "dir1" / "dir2", _scratch / "dir2_link") - os.symlink(_scratch / "dir1" / "file2", _scratch / "file_link") + # always overwrite for putfile + # this block assumes "file1" has already been uploaded with the content "file1". + # for efficiency reasons (when the test is run against a real server). I didn't that repeat here. + Path(_local / "file1").write_text("notfile1") + transport.putfile(_local / "file1", _remote / "file1") + transport.getfile(_remote / "file1", _local_download / "file1_uploaded") + assert Path(_local_download / "file1_uploaded").read_text() == "notfile1" - assert set(transport.listdir(_scratch, recursive=True)) == { - "file1", - "dir1", - ".hidden", - "dir1/file2", - "dir1/dir2", - "dir1/dir2/file3", - "dir2_link", - "file_link", - } + # always overwrite for getfile + # this block assumes "file1" has already been downloaded with the content "notfile1". + # for efficiency reasons (when the test is run against a real server). I didn't that repeat here. + transport.getfile(_remote / "file1", _local_download / "file1") + assert Path(_local_download / "file1").read_text() == "notfile1" - assert set(transport.listdir(_scratch / "dir2_link", recursive=False)) == {"file3"} + # don't skip hidden files + transport.putfile(_local / ".hidden", _remote / ".hidden") + transport.getfile(_remote / ".hidden", _local_download / ".hidden") + assert Path(_local_download / ".hidden").read_text() == ".hidden" + + # follow links + # for putfile() + Path(_local / "file1").write_text("file1") + transport.putfile(_local / "file1_link", _remote / "file1_link") + assert not transport._cwd.joinpath( + _remote / "file1_link" + ).is_symlink() # should be copied as a file + transport.getfile(_remote / "file1_link", _local_download / "file1_link") + assert Path(_local_download / "file1_link").read_text() == "file1" + # for getfile() + transport.putfile(_local / "file1", _remote / "file1") + transport.symlink(_remote / "file1", _remote / "remote_link") + transport.getfile(_remote / "remote_link", _local_download / "remote_link") + assert not Path(_local_download / "remote_link").is_symlink() + assert Path(_local_download / "remote_link").read_text() == "file1" @pytest.mark.usefixtures("aiida_profile_clean") -def test_get(firecrest_computer: orm.Computer, tmpdir: Path): - """ - This is minimal test is to check if get() is raising errors as expected, - and directing to getfile() and gettree() as expected. - Mainly just checking error handeling and folder creation. - """ +def test_remove(firecrest_computer: orm.Computer, tmpdir: Path): transport = firecrest_computer.get_transport() + tmpdir_remote = transport._temp_directory - _remote = tmpdir / "remotedir" - _local = tmpdir / "localdir" - _remote.mkdir() - _local.mkdir() + _remote = tmpdir_remote + _local = tmpdir - # check if the code is directing to getfile() or gettree() as expected - with patch.object(transport, "gettree", autospec=True) as mock_gettree: - transport.get(_remote, _local) - mock_gettree.assert_called_once() + Path(_local / "samplefile").touch() - with patch.object(transport, "gettree", autospec=True) as mock_gettree: - os.symlink(_remote, tmpdir / "dir_link") - transport.get(tmpdir / "dir_link", _local) - mock_gettree.assert_called_once() + # remove a non-empty directory with rmtree() + _scratch = transport._cwd.joinpath(_remote / "sampledir") + _scratch.mkdir() + transport.putfile(_local / "samplefile", _remote / "sampledir" / "samplefile") + transport.rmtree(_scratch) + assert not _scratch.exists() - with patch.object(transport, "getfile", autospec=True) as mock_getfile: - Path(_remote / "file1").write_text("file1") - transport.get(_remote / "file1", _local / "file1") - mock_getfile.assert_called_once() + # remove a non-empty directory should raise with rmdir() + transport.mkdir(_remote / "sampledir") + transport.putfile(_local / "samplefile", _remote / "sampledir" / "samplefile") + with pytest.raises(OSError): + transport.rmdir(_remote / "sampledir") - with patch.object(transport, "getfile", autospec=True) as mock_getfile: - os.symlink(_remote / "file1", _remote / "file1_link") - transport.get(_remote / "file1_link", _local / "file1_link") - mock_getfile.assert_called_once() + # remove a file with remove() + transport.remove(_remote / "sampledir" / "samplefile") + assert not transport._cwd.joinpath(_remote / "sampledir" / "samplefile").exists() - # raise if remote file/folder does not exist - with pytest.raises(FileNotFoundError): - transport.get(_remote / "does_not_exist", _local) - transport.get(_remote / "does_not_exist", _local, ignore_nonexisting=True) - - # raise if localpath is relative - with pytest.raises(ValueError): - transport.get(_remote, Path(_local).relative_to(tmpdir)) - with pytest.raises(ValueError): - transport.get(_remote / "file1", Path(_local).relative_to(tmpdir)) + # remove a empty directory with rmdir + transport.rmdir(_remote / "sampledir") + assert not _scratch.exists() @pytest.mark.usefixtures("aiida_profile_clean") -def test_getfile(firecrest_computer: orm.Computer, tmpdir: Path): +def test_is_file(firecrest_computer: orm.Computer, tmpdir: Path): transport = firecrest_computer.get_transport() + tmpdir_remote = transport._temp_directory - _remote = tmpdir / "remotedir" - _local = tmpdir / "localdir" - _remote.mkdir() - _local.mkdir() - - Path(_remote / "file1").write_text("file1") - Path(_remote / ".hidden").write_text(".hidden") - os.symlink(_remote / "file1", _remote / "file1_link") - - # raise if remote file does not exist - with pytest.raises(FileNotFoundError): - transport.getfile(_remote / "does_not_exist", _local) - - # raise if localfilename not provided - with pytest.raises(IsADirectoryError): - transport.getfile(_remote / "file1", _local) - - # raise if localpath is relative - with pytest.raises(ValueError): - transport.getfile(_remote / "file1", Path(_local / "file1").relative_to(tmpdir)) + _remote = tmpdir_remote + _local = tmpdir - # don't mix up directory with file - with pytest.raises(FileNotFoundError): - transport.getfile(_remote, _local / "file1") + Path(_local / "samplefile").touch() + transport.putfile(_local / "samplefile", _remote / "samplefile") + assert transport.isfile(_remote / "samplefile") + assert not transport.isfile(_remote / "does_not_exist") - # write where I tell you to - transport.getfile(_remote / "file1", _local / "file1") - transport.getfile(_remote / "file1", _local / "file1-prime") - assert Path(_local / "file1").read_text() == "file1" - assert Path(_local / "file1-prime").read_text() == "file1" - - # always overwrite - transport.getfile(_remote / "file1", _local / "file1") - assert Path(_local / "file1").read_text() == "file1" - Path(_local / "file1").write_text("notfile1") +@pytest.mark.usefixtures("aiida_profile_clean") +def test_symlink(firecrest_computer: orm.Computer, tmpdir: Path): + transport = firecrest_computer.get_transport() + tmpdir_remote = transport._temp_directory - transport.getfile(_remote / "file1", _local / "file1") - assert Path(_local / "file1").read_text() == "file1" + _remote = tmpdir_remote + _local = tmpdir - # don't skip hidden files - transport.getfile(_remote / ".hidden", _local / ".hidden-prime") - assert Path(_local / ".hidden-prime").read_text() == ".hidden" + Path(_local / "samplefile").touch() + transport.putfile(_local / "samplefile", _remote / "samplefile") + transport.symlink(_remote / "samplefile", _remote / "samplelink") - # follow links - transport.getfile(_remote / "file1_link", _local / "file1_link") - assert Path(_local / "file1_link").read_text() == "file1" - assert not Path(_local / "file1_link").is_symlink() + _symlink = transport._cwd.joinpath(_remote / "samplelink") + assert _symlink.is_symlink() + # TODO: check if the symlink is pointing to the correct file + # for this we need further development of FcPath.resolve() + # assert _symlink.resolve() == _remote / "samplefile" -@pytest.mark.parametrize("payoff", [True, False]) @pytest.mark.usefixtures("aiida_profile_clean") -def test_gettree(firecrest_computer: orm.Computer, tmpdir: Path, payoff: bool): - """ - This test is to check `gettree` through non tar mode. - bytar= True in this test. - """ +def test_listdir(firecrest_computer: orm.Computer, tmpdir: Path): transport = firecrest_computer.get_transport() - transport.payoff_override = payoff - - # Note: - # SSH transport behaviour, 69 is a directory - # transport.get('somepath/69', 'someremotepath/') == transport.get('somepath/69', 'someremotepath') - # transport.get('somepath/69', 'someremotepath/') == transport.get('somepath/69/', 'someremotepath/') - # transport.get('someremotepath/69', 'somepath/69')--> if 69 exist, create 69 inside it ('somepath/69/69') - # transport.get('someremotepath/69', 'somepath/69')--> if 69 no texist,create 69 inside it ('somepath/69') - # transport.get('somepath/69', 'someremotepath/6889/69') --> create everything, make_parent = True - - _remote = tmpdir / "remotedir" - _local = tmpdir / "localdir" - _remote.mkdir() - _local.mkdir() - # a typical tree - Path(_remote / "file1").write_text("file1") - Path(_remote / ".hidden").write_text(".hidden") - Path(_remote / "dir1").mkdir() - Path(_remote / "dir1" / "file2").write_text("file2") - # with symlinks - Path(_remote / "dir2").mkdir() - Path(_remote / "dir2" / "file3").write_text("file3") - os.symlink(_remote / "file1", _remote / "dir1" / "file1_link") - os.symlink(_remote / "dir2", _remote / "dir1" / "dir2_link") - # if symlinks are pointing to a relative path - os.symlink(Path("../file1"), _remote / "dir1" / "file10_link") - os.symlink(Path("../dir2"), _remote / "dir1" / "dir20_link") + tmpdir_remote = transport._temp_directory - # raise if remote file does not exist - with pytest.raises(OSError): - transport.gettree(_remote / "does_not_exist", _local) + _remote = tmpdir_remote + _local = tmpdir - # raise if local is a file - Path(tmpdir / "isfile").touch() - with pytest.raises(OSError): - transport.gettree(_remote, tmpdir / "isfile") + # test basic & recursive + Path(_local / "file1").touch() + Path(_local / "dir1").mkdir() + Path(_local / ".hidden").touch() + Path(_local / "dir1" / "file2").touch() + transport.putfile(_local / "file1", _remote / "file1") + transport.mkdir(_remote / "dir1") + transport.putfile(_local / "dir1" / "file2", _remote / "dir1" / "file2") + transport.putfile(_local / ".hidden", _remote / ".hidden") - # raise if localpath is relative - with pytest.raises(ValueError): - transport.gettree(_remote, Path(_local).relative_to(tmpdir)) + assert set(transport.listdir(_remote)) == {"file1", "dir1", ".hidden"} + assert set(transport.listdir(_remote, recursive=True)) == { + "file1", + "dir1", + ".hidden", + "dir1/file2", + } - # If destination directory does not exists, AiiDA expects transport make the new path as root not _remote.name - transport.gettree(_remote, _local / "newdir") - _root = _local / "newdir" - # tree should be copied recursively - assert Path(_root / "file1").read_text() == "file1" - assert Path(_root / ".hidden").read_text() == ".hidden" - assert Path(_root / "dir1" / "file2").read_text() == "file2" - assert Path(_root / "dir2" / "file3").read_text() == "file3" - # symlink should be followed - assert Path(_root / "dir1" / "file1_link").read_text() == "file1" - assert Path(_root / "dir1" / "dir2_link" / "file3").read_text() == "file3" - assert Path(_root / "dir1" / "file10_link").read_text() == "file1" - assert Path(_root / "dir1" / "dir20_link" / "file3").read_text() == "file3" - assert not Path(_root / "dir1" / "file1_link").is_symlink() - assert not Path(_root / "dir1" / "dir2_link" / "file3").is_symlink() - assert not Path(_root / "dir1" / "file10_link").is_symlink() - assert not Path(_root / "dir1" / "dir20_link" / "file3").is_symlink() + # to test symlink + Path(_local / "dir1" / "dir2").mkdir() + Path(_local / "dir1" / "dir2" / "file3").touch() + transport.mkdir(_remote / "dir1" / "dir2") + transport.putfile( + _local / "dir1" / "dir2" / "file3", _remote / "dir1" / "dir2" / "file3" + ) + transport.symlink(_remote / "dir1" / "dir2", _remote / "dir2_link") + transport.symlink(_remote / "dir1" / "file2", _remote / "file_link") - # If destination directory does exists, AiiDA expects transport make _remote.name and write into it - # however this might have changed in the newer versions of AiiDA ~ 2.6.0 (IDK) - transport.gettree(_remote, _local / "newdir") - _root = _local / "newdir" / Path(_remote).name - # tree should be copied recursively - assert Path(_root / "file1").read_text() == "file1" - assert Path(_root / ".hidden").read_text() == ".hidden" - assert Path(_root / "dir1" / "file2").read_text() == "file2" - assert Path(_root / "dir2" / "file3").read_text() == "file3" - # symlink should be followed - assert Path(_root / "dir1" / "file1_link").read_text() == "file1" - assert Path(_root / "dir1" / "dir2_link" / "file3").read_text() == "file3" - assert Path(_root / "dir1" / "file10_link").read_text() == "file1" - assert Path(_root / "dir1" / "dir20_link" / "file3").read_text() == "file3" - assert not Path(_root / "dir1" / "file1_link").is_symlink() - assert not Path(_root / "dir1" / "dir2_link" / "file3").is_symlink() - assert not Path(_root / "dir1" / "file10_link").is_symlink() - assert not Path(_root / "dir1" / "dir20_link" / "file3").is_symlink() + assert set(transport.listdir(_remote, recursive=True)) == { + "file1", + "dir1", + ".hidden", + "dir1/file2", + "dir1/dir2", + "dir1/dir2/file3", + "dir2_link", + "file_link", + } + # TODO: The following assert is not working as expected when testing against a real server, + # see the open issue on FirecREST: https://github.com/eth-cscs/firecrest/issues/205 + assert set(transport.listdir(_remote / "dir2_link", recursive=False)) == {"file3"} @pytest.mark.usefixtures("aiida_profile_clean") @@ -328,12 +273,14 @@ def test_put(firecrest_computer: orm.Computer, tmpdir: Path): This is minimal test is to check if put() is raising errors as expected, and directing to putfile() and puttree() as expected. Mainly just checking error handeling and folder creation. + For faster testing, we mock the subfucntions and don't actually do it. """ transport = firecrest_computer.get_transport() + tmpdir_remote = transport._temp_directory - _remote = tmpdir / "remotedir" + _remote = tmpdir_remote / "remotedir" _local = tmpdir / "localdir" - _remote.mkdir() + transport.mkdir(_remote) _local.mkdir() # check if the code is directing to putfile() or puttree() as expected @@ -369,65 +316,64 @@ def test_put(firecrest_computer: orm.Computer, tmpdir: Path): @pytest.mark.usefixtures("aiida_profile_clean") -def test_putfile(firecrest_computer: orm.Computer, tmpdir: Path): +def test_get(firecrest_computer: orm.Computer, tmpdir: Path): + """ + This is minimal test is to check if get() is raising errors as expected, + and directing to getfile() and gettree() as expected. + Mainly just checking error handeling and folder creation. + For faster testing, we mock the subfucntions and don't actually do it. + """ transport = firecrest_computer.get_transport() + tmpdir_remote = transport._temp_directory - _remote = tmpdir / "remotedir" + _remote = tmpdir_remote / "remotedir" _local = tmpdir / "localdir" _remote.mkdir() _local.mkdir() - Path(_local / "file1").write_text("file1") - Path(_local / ".hidden").write_text(".hidden") - os.symlink(_local / "file1", _local / "file1_link") + # check if the code is directing to getfile() or gettree() as expected + with patch.object(transport, "gettree", autospec=True) as mock_gettree: + transport.get(_remote, _local) + mock_gettree.assert_called_once() - # raise if local file does not exist - with pytest.raises(FileNotFoundError): - transport.putfile(_local / "does_not_exist", _remote) + with patch.object(transport, "gettree", autospec=True) as mock_gettree: + transport.symlink(_remote, tmpdir_remote / "dir_link") + transport.get(tmpdir_remote / "dir_link", _local) + mock_gettree.assert_called_once() - # raise if remotefilename is not provided - with pytest.raises(ValueError): - transport.putfile(_local / "file1", _remote) + with patch.object(transport, "getfile", autospec=True) as mock_getfile: + Path(_local / "file1").write_text("file1") + transport.putfile(_local / "file1", _remote / "file1") + transport.get(_remote / "file1", _local / "file1") + mock_getfile.assert_called_once() + + with patch.object(transport, "getfile", autospec=True) as mock_getfile: + transport.symlink(_remote / "file1", _remote / "file1_link") + transport.get(_remote / "file1_link", _local / "file1_link") + mock_getfile.assert_called_once() + + # raise if remote file/folder does not exist + with pytest.raises(FileNotFoundError): + transport.get(_remote / "does_not_exist", _local) + transport.get(_remote / "does_not_exist", _local, ignore_nonexisting=True) # raise if localpath is relative with pytest.raises(ValueError): - transport.putfile(Path(_local / "file1").relative_to(tmpdir), _remote / "file1") - - # don't mix up directory with file + transport.get(_remote, Path(_local).relative_to(tmpdir)) with pytest.raises(ValueError): - transport.putfile(_local, _remote / "file1") - - # write where I tell you to - transport.putfile(_local / "file1", _remote / "file1") - transport.putfile(_local / "file1", _remote / "file1-prime") - assert Path(_remote / "file1").read_text() == "file1" - assert Path(_remote / "file1-prime").read_text() == "file1" - - # always overwrite - transport.putfile(_local / "file1", _remote / "file1") - assert Path(_remote / "file1").read_text() == "file1" - - Path(_remote / "file1").write_text("notfile1") - - transport.putfile(_local / "file1", _remote / "file1") - assert Path(_remote / "file1").read_text() == "file1" - - # don't skip hidden files - transport.putfile(_local / ".hidden", _remote / ".hidden-prime") - assert Path(_remote / ".hidden-prime").read_text() == ".hidden" - - # follow links - transport.putfile(_local / "file1_link", _remote / "file1_link") - assert Path(_remote / "file1_link").read_text() == "file1" - assert not Path(_remote / "file1_link").is_symlink() + transport.get(_remote / "file1", Path(_local).relative_to(tmpdir)) +@pytest.mark.timeout(120) @pytest.mark.parametrize("payoff", [True, False]) @pytest.mark.usefixtures("aiida_profile_clean") def test_puttree(firecrest_computer: orm.Computer, tmpdir: Path, payoff: bool): """ - This test is to check `puttree` through non tar mode. - payoff= False in this test, so just checking if putting files one by one is working as expected. + This test is to check `puttree` through both recursive transfer and tar mode. + payoff= False would put files one by one. + payoff= True would use the tar mode. + Note: this test depends on a functional getfile() method, for consistency with the real server tests. + Before running this test, make sure getfile() is working, which is tested in `test_putfile_getfile`. """ transport = firecrest_computer.get_transport() transport.payoff_override = payoff @@ -441,16 +387,19 @@ def test_puttree(firecrest_computer: orm.Computer, tmpdir: Path, payoff: bool): # transport.put('somepath/69', 'someremotepath/67') --> if 67 exist, create 69 inside it (someremotepath/67/69) # transport.put('somepath/69', 'someremotepath/6889/69') --> useless Error: OSError # Weired - # SSH bug: + # SSH "bug": # transport.put('somepath/69', 'someremotepath/') --> assuming someremotepath exists, make 69 # while # transport.put('somepath/69/', 'someremotepath/') --> assuming someremotepath exists, OSError: # cannot make someremotepath - _remote = tmpdir / "remotedir" + tmpdir_remote = transport._temp_directory + _remote = tmpdir_remote / "remotedir" _local = tmpdir / "localdir" + _local_download = tmpdir / "download" _remote.mkdir() _local.mkdir() + _local_download.mkdir() # a typical tree Path(_local / "dir1").mkdir() Path(_local / "dir2").mkdir() @@ -481,54 +430,227 @@ def test_puttree(firecrest_computer: orm.Computer, tmpdir: Path, payoff: bool): # If destination directory does not exists, AiiDA expects transport make the new path as root not using _local.name transport.puttree(_local, _remote / "newdir") _root = _remote / "newdir" + + # GET block: to retrieve the files we depend on a functional getfile(), + # this limitation is a price to pay for real server testing. + transport.getfile(_root / "file1", _local_download / "file1") + transport.getfile(_root / ".hidden", _local_download / ".hidden") + Path(_local_download / "dir1").mkdir() + Path(_local_download / "dir2").mkdir() + transport.getfile(_root / "dir1" / "file2", _local_download / "dir1" / "file2") + transport.getfile(_root / "dir2" / "file3", _local_download / "dir2" / "file3") + # note links should have been dereferenced while uploading + transport.getfile( + _root / "dir1" / "file1_link", _local_download / "dir1" / "file1_link" + ) + Path(_local_download / "dir1" / "dir2_link").mkdir() + transport.getfile( + _root / "dir1" / "dir2_link" / "file3", + _local_download / "dir1" / "dir2_link" / "file3", + ) + transport.getfile( + _root / "dir1" / "file10_link", _local_download / "dir1" / "file10_link" + ) + Path(_local_download / "dir1" / "dir20_link").mkdir() + transport.getfile( + _root / "dir1" / "dir20_link" / "file3", + _local_download / "dir1" / "dir20_link" / "file3", + ) + # End of GET block + + # ASSERT block: tree should be copied recursively + assert Path(_local_download / "file1").read_text() == "file1" + assert Path(_local_download / ".hidden").read_text() == ".hidden" + assert Path(_local_download / "dir1" / "file2").read_text() == "file2" + assert Path(_local_download / "dir2" / "file3").read_text() == "file3" + # symlink should be followed + assert Path(_local_download / "dir1" / "file1_link").read_text() == "file1" + assert Path(_local_download / "dir1" / "dir2_link" / "file3").read_text() == "file3" + assert Path(_local_download / "dir1" / "file10_link").read_text() == "file1" + assert ( + Path(_local_download / "dir1" / "dir20_link" / "file3").read_text() == "file3" + ) + assert not transport._cwd.joinpath(_root / "dir1" / "file1_link").is_symlink() + assert not transport._cwd.joinpath( + _root / "dir1" / "dir2_link" / "file3" + ).is_symlink() + assert not transport._cwd.joinpath(_root / "dir1" / "file10_link").is_symlink() + assert not transport._cwd.joinpath( + _root / "dir1" / "dir20_link" / "file3" + ).is_symlink() + # End of ASSERT block + + # If destination directory does exists, AiiDA expects transport make _local.name and write into it + # however this might have changed in the newer versions of AiiDA ~ 2.6.0 (IDK) + transport.puttree(_local, _remote / "newdir") + _root = _remote / "newdir" / Path(_local).name + + # GET block: to retrieve the files we depend on a functional getfile(), + # this limitation is a price to pay for real server testing. + _local_download = tmpdir / "download2" + _local_download.mkdir() + transport.getfile(_root / "file1", _local_download / "file1") + transport.getfile(_root / ".hidden", _local_download / ".hidden") + Path(_local_download / "dir1").mkdir() + Path(_local_download / "dir2").mkdir() + transport.getfile(_root / "dir1" / "file2", _local_download / "dir1" / "file2") + transport.getfile(_root / "dir2" / "file3", _local_download / "dir2" / "file3") + # note links should have been dereferenced while uploading + transport.getfile( + _root / "dir1" / "file1_link", _local_download / "dir1" / "file1_link" + ) + Path(_local_download / "dir1" / "dir2_link").mkdir() + transport.getfile( + _root / "dir1" / "dir2_link" / "file3", + _local_download / "dir1" / "dir2_link" / "file3", + ) + transport.getfile( + _root / "dir1" / "file10_link", _local_download / "dir1" / "file10_link" + ) + Path(_local_download / "dir1" / "dir20_link").mkdir() + transport.getfile( + _root / "dir1" / "dir20_link" / "file3", + _local_download / "dir1" / "dir20_link" / "file3", + ) + # End of GET block + + # ASSERT block: tree should be copied recursively + assert Path(_local_download / "file1").read_text() == "file1" + assert Path(_local_download / ".hidden").read_text() == ".hidden" + assert Path(_local_download / "dir1" / "file2").read_text() == "file2" + assert Path(_local_download / "dir2" / "file3").read_text() == "file3" + # symlink should be followed + assert Path(_local_download / "dir1" / "file1_link").read_text() == "file1" + assert Path(_local_download / "dir1" / "dir2_link" / "file3").read_text() == "file3" + assert Path(_local_download / "dir1" / "file10_link").read_text() == "file1" + assert ( + Path(_local_download / "dir1" / "dir20_link" / "file3").read_text() == "file3" + ) + assert not transport._cwd.joinpath(_root / "dir1" / "file1_link").is_symlink() + assert not transport._cwd.joinpath( + _root / "dir1" / "dir2_link" / "file3" + ).is_symlink() + assert not transport._cwd.joinpath(_root / "dir1" / "file10_link").is_symlink() + assert not transport._cwd.joinpath( + _root / "dir1" / "dir20_link" / "file3" + ).is_symlink() + # End of ASSERT block + + +@pytest.mark.timeout(120) +@pytest.mark.parametrize("payoff", [True, False]) +@pytest.mark.usefixtures("aiida_profile_clean") +def test_gettree(firecrest_computer: orm.Computer, tmpdir: Path, payoff: bool): + """ + This test is to check `gettree` through both recursive transfer and tar mode. + payoff= False would get files one by one. + payoff= True would use the tar mode. + Note: this test depends on a functional putfile() method, for consistency with the real server tests. + Before running this test, make sure putfile() is working, which is tested in `test_putfile_getfile`. + """ + transport = firecrest_computer.get_transport() + transport.payoff_override = payoff + + # Note: + # SSH transport behaviour, 69 is a directory + # transport.get('somepath/69', 'someremotepath/') == transport.get('somepath/69', 'someremotepath') + # transport.get('somepath/69', 'someremotepath/') == transport.get('somepath/69/', 'someremotepath/') + # transport.get('someremotepath/69', 'somepath/69')--> if 69 exist, create 69 inside it ('somepath/69/69') + # transport.get('someremotepath/69', 'somepath/69')--> if 69 noexist,create 69 inside it ('somepath/69') + # transport.get('somepath/69', 'someremotepath/6889/69') --> create everything, make_parent = True + tmpdir_remote = transport._temp_directory + _remote = tmpdir_remote / "remotedir" + _remote.mkdir() + _local = tmpdir / "localdir" + _local_for_upload = tmpdir / "forupload" + _local.mkdir() + _local_for_upload.mkdir() + + # a typical tree with symlinks + Path(_local_for_upload / "file1").write_text("file1") + Path(_local_for_upload / ".hidden").write_text(".hidden") + Path(_local_for_upload / "dir1").mkdir() + Path(_local_for_upload / "dir1" / "file2").write_text("file2") + Path(_local_for_upload / "dir2").mkdir() + Path(_local_for_upload / "dir2" / "file3").write_text("file3") + transport.putfile(_local_for_upload / "file1", _remote / "file1") + transport.putfile(_local_for_upload / ".hidden", _remote / ".hidden") + transport.mkdir(_remote / "dir1") + transport.mkdir(_remote / "dir2") + transport.putfile(_local_for_upload / "dir1" / "file2", _remote / "dir1" / "file2") + transport.putfile(_local_for_upload / "dir2" / "file3", _remote / "dir2" / "file3") + + transport.symlink(_remote / "file1", _remote / "dir1" / "file1_link") + transport.symlink(_remote / "dir2", _remote / "dir1" / "dir2_link") + # I cannot create & check relative links, because we don't have access on the server side + # os.symlink(Path("../file1"), _local_for_upload / "dir1" / "file10_link") + # os.symlink(Path("../dir2"), _local_for_upload / "dir1" / "dir20_link") + + # raise if remote file does not exist + with pytest.raises(OSError): + transport.gettree(_remote / "does_not_exist", _local) + + # raise if local is a file + Path(tmpdir / "isfile").touch() + with pytest.raises(OSError): + transport.gettree(_remote, tmpdir / "isfile") + + # raise if localpath is relative + with pytest.raises(ValueError): + transport.gettree(_remote, Path(_local).relative_to(tmpdir)) + + # If destination directory does not exists, AiiDA expects from transport to make a new path as root not _remote.name + transport.gettree(_remote, _local / "newdir") + _root = _local / "newdir" # tree should be copied recursively assert Path(_root / "file1").read_text() == "file1" assert Path(_root / ".hidden").read_text() == ".hidden" assert Path(_root / "dir1" / "file2").read_text() == "file2" assert Path(_root / "dir2" / "file3").read_text() == "file3" # symlink should be followed - assert Path(_root / "dir1" / "file1_link").read_text() == "file1" - assert Path(_root / "dir1" / "dir2_link" / "file3").read_text() == "file3" - assert Path(_root / "dir1" / "file10_link").read_text() == "file1" - assert Path(_root / "dir1" / "dir20_link" / "file3").read_text() == "file3" assert not Path(_root / "dir1" / "file1_link").is_symlink() assert not Path(_root / "dir1" / "dir2_link" / "file3").is_symlink() - assert not Path(_root / "dir1" / "file10_link").is_symlink() - assert not Path(_root / "dir1" / "dir20_link" / "file3").is_symlink() + assert Path(_root / "dir1" / "file1_link").read_text() == "file1" + assert Path(_root / "dir1" / "dir2_link" / "file3").read_text() == "file3" - # If destination directory does exists, AiiDA expects transport make _local.name and write into it + # If destination directory does exists, AiiDA expects transport make _remote.name and write into it # however this might have changed in the newer versions of AiiDA ~ 2.6.0 (IDK) - transport.puttree(_local, _remote / "newdir") - _root = _remote / "newdir" / Path(_local).name + transport.gettree(_remote, _local / "newdir") + _root = _local / "newdir" / Path(_remote).name # tree should be copied recursively assert Path(_root / "file1").read_text() == "file1" assert Path(_root / ".hidden").read_text() == ".hidden" assert Path(_root / "dir1" / "file2").read_text() == "file2" assert Path(_root / "dir2" / "file3").read_text() == "file3" # symlink should be followed - assert Path(_root / "dir1" / "file1_link").read_text() == "file1" - assert Path(_root / "dir1" / "dir2_link" / "file3").read_text() == "file3" - assert Path(_root / "dir1" / "file10_link").read_text() == "file1" - assert Path(_root / "dir1" / "dir20_link" / "file3").read_text() == "file3" assert not Path(_root / "dir1" / "file1_link").is_symlink() assert not Path(_root / "dir1" / "dir2_link" / "file3").is_symlink() - assert not Path(_root / "dir1" / "file10_link").is_symlink() - assert not Path(_root / "dir1" / "dir20_link" / "file3").is_symlink() + assert Path(_root / "dir1" / "file1_link").read_text() == "file1" + assert Path(_root / "dir1" / "dir2_link" / "file3").read_text() == "file3" @pytest.mark.parametrize("to_test", ["copy", "copytree"]) @pytest.mark.usefixtures("aiida_profile_clean") def test_copy(firecrest_computer: orm.Computer, tmpdir: Path, to_test: str): + """ + This test is to check `copy` and `copytree`, + `copyfile` is that is only for files, and it is tested in `test_putfile_getfile`. + Also raising errors are somewhat method specific. + Note: this test depends on functional getfile() and putfile() methods, for consistency with the real server tests. + Before running this test, make sure `test_putfile_getfile` has passed. + """ transport = firecrest_computer.get_transport() if to_test == "copy": testing = transport.copy elif to_test == "copytree": testing = transport.copytree - _remote_1 = tmpdir / "remotedir_1" - _remote_2 = tmpdir / "remotedir_2" + tmpdir_remote = transport._temp_directory + _remote_1 = tmpdir_remote / "remotedir_1" + _remote_2 = tmpdir_remote / "remotedir_2" _remote_1.mkdir() _remote_2.mkdir() + _for_upload = tmpdir # raise if source or destination does not exist with pytest.raises(FileNotFoundError): @@ -536,54 +658,90 @@ def test_copy(firecrest_computer: orm.Computer, tmpdir: Path, to_test: str): with pytest.raises(FileNotFoundError): testing(_remote_1, _remote_2 / "does_not_exist") - # raise if source is inappropriate - if to_test == "copytree": - Path(tmpdir / "file1").touch() - with pytest.raises(ValueError): - testing(tmpdir / "file1", _remote_2) - - # a typical tree - Path(_remote_1 / "dir1").mkdir() - Path(_remote_1 / "dir2").mkdir() - Path(_remote_1 / "file1").write_text("file1") - Path(_remote_1 / ".hidden").write_text(".hidden") - Path(_remote_1 / "dir1" / "file2").write_text("file2") - Path(_remote_1 / "dir2" / "file3").write_text("file3") - # with symlinks to a file even if pointing to a relative path - os.symlink(_remote_1 / "file1", _remote_1 / "dir1" / "file1_link") - os.symlink(Path("../file1"), _remote_1 / "dir1" / "file10_link") - # with symlinks to a folder even if pointing to a relative path - os.symlink(_remote_1 / "dir2", _remote_1 / "dir1" / "dir2_link") - os.symlink(Path("../dir2"), _remote_1 / "dir1" / "dir20_link") + # a typical tree with symlinks to a file and a folder + Path(_for_upload / "file1").write_text("file1") + Path(_for_upload / ".hidden").write_text(".hidden") + Path(_for_upload / "dir1").mkdir() + Path(_for_upload / "dir2").mkdir() + Path(_for_upload / "dir1" / "file2").write_text("file2") + Path(_for_upload / "dir2" / "file3").write_text("file3") + + transport.putfile(_for_upload / "file1", _remote_1 / "file1") + transport.putfile(_for_upload / ".hidden", _remote_1 / ".hidden") + transport.mkdir(_remote_1 / "dir1") + transport.mkdir(_remote_1 / "dir2") + transport.putfile(_for_upload / "dir1" / "file2", _remote_1 / "dir1" / "file2") + transport.putfile(_for_upload / "dir2" / "file3", _remote_1 / "dir2" / "file3") + transport.symlink(_remote_1 / "file1", _remote_1 / "dir1" / "file1_link") + transport.symlink(_remote_1 / "dir2", _remote_1 / "dir1" / "dir2_link") + # I cannot create & check relative links, because we don't have access on the server side + # os.symlink(Path("../file1"), _remote_1 / "dir1" / "file10_link") + # os.symlink(Path("../dir2"), _remote_1 / "dir1" / "dir20_link") testing(_remote_1, _remote_2) _root_2 = _remote_2 / Path(_remote_1).name - # tree should be copied recursively - assert Path(_root_2 / "dir1").exists() - assert Path(_root_2 / "dir2").exists() - assert Path(_root_2 / "file1").read_text() == "file1" - assert Path(_root_2 / ".hidden").read_text() == ".hidden" - assert Path(_root_2 / "dir1" / "file2").read_text() == "file2" - assert Path(_root_2 / "dir2" / "file3").read_text() == "file3" - # symlink should be followed - assert Path(_root_2 / "dir1" / "dir2_link").exists() - assert Path(_root_2 / "dir1" / "dir20_link").exists() - assert Path(_root_2 / "dir1" / "file1_link").read_text() == "file1" - assert Path(_root_2 / "dir1" / "file10_link").read_text() == "file1" - assert Path(_root_2 / "dir1" / "file1_link").is_symlink() - assert Path(_root_2 / "dir1" / "dir2_link").is_symlink() - assert Path(_root_2 / "dir1" / "file10_link").is_symlink() - assert Path(_root_2 / "dir1" / "dir20_link").is_symlink() + + # GET block: to retrieve the files we depend on a functional getfile(), + # this limitation is a price to pay for real server testing. + _local_download = tmpdir / "download1" + _local_download.mkdir() + transport.getfile(_root_2 / "file1", _local_download / "file1") + transport.getfile(_root_2 / ".hidden", _local_download / ".hidden") + Path(_local_download / "dir1").mkdir() + Path(_local_download / "dir2").mkdir() + transport.getfile(_root_2 / "dir1" / "file2", _local_download / "dir1" / "file2") + transport.getfile(_root_2 / "dir2" / "file3", _local_download / "dir2" / "file3") + # note links should have been dereferenced while uploading + transport.getfile( + _root_2 / "dir1" / "file1_link", _local_download / "dir1" / "file1_link" + ) + Path(_local_download / "dir1" / "dir2_link").mkdir() + # TODO: The following is not working as expected when testing against a real server, see open issue on FirecREST: + # https://github.com/eth-cscs/firecrest/issues/205 + transport.getfile( + _root_2 / "dir1" / "dir2_link" / "file3", + _local_download / "dir1" / "dir2_link" / "file3", + ) + # End of GET block + + # ASSERT block: tree should be copied recursively symlink should be followed + assert Path(_local_download / "dir1").exists() + assert Path(_local_download / "dir2").exists() + assert Path(_local_download / "file1").read_text() == "file1" + assert Path(_local_download / ".hidden").read_text() == ".hidden" + assert Path(_local_download / "dir1" / "file2").read_text() == "file2" + assert Path(_local_download / "dir2" / "file3").read_text() == "file3" + assert Path(_local_download / "dir1" / "dir2_link").exists() + assert Path(_local_download / "dir1" / "file1_link").read_text() == "file1" + assert Path(_local_download / "dir1" / "dir2_link" / "file3").read_text() == "file3" + assert transport._cwd.joinpath(_root_2 / "dir1" / "file1_link").is_symlink() + assert transport._cwd.joinpath(_root_2 / "dir1" / "dir2_link").is_symlink() + + # End of ASSERT block + + # raise if source is inappropriate + if to_test == "copytree": + with pytest.raises(ValueError): + testing(_remote_1 / "file1", _remote_2) @pytest.mark.usefixtures("aiida_profile_clean") def test_copyfile(firecrest_computer: orm.Computer, tmpdir: Path): + """ + Note: this test depends on functional getfile() and putfile() methods, for consistency with the real server tests. + Before running this test, make sure `test_putfile_getfile` has passed. + """ transport = firecrest_computer.get_transport() testing = transport.copyfile - _remote_1 = tmpdir / "remotedir_1" - _remote_2 = tmpdir / "remotedir_2" + tmpdir_remote = transport._temp_directory + _remote_1 = tmpdir_remote / "remotedir_1" + _remote_2 = tmpdir_remote / "remotedir_2" + _for_upload = tmpdir / "forUpload" + _for_download = tmpdir / "forDownload" + _for_upload.mkdir() + _for_download.mkdir() _remote_1.mkdir() _remote_2.mkdir() @@ -591,37 +749,40 @@ def test_copyfile(firecrest_computer: orm.Computer, tmpdir: Path): with pytest.raises(FileNotFoundError): testing(_remote_1 / "does_not_exist", _remote_2) # in this case don't raise and just create the file - Path(tmpdir / "_").touch() - testing(tmpdir / "_", _remote_2 / "does_not_exist") + Path(_for_upload / "_").touch() + transport.putfile(_for_upload / "_", _remote_1 / "_") + testing(_remote_1 / "_", _remote_2 / "does_not_exist") - # raise if source is unappropriate + # raise if source is not a file with pytest.raises(ValueError): - testing(tmpdir, _remote_2) + testing(_remote_1, _remote_2) - # a typical tree - Path(_remote_1 / "file1").write_text("file1") - Path(_remote_1 / ".hidden").write_text(".hidden") - # with symlinks to a file even if pointing to a relative path - os.symlink(_remote_1 / "file1", _remote_1 / "file1_link") - os.symlink(Path("file1"), _remote_1 / "file10_link") + # main functionality, including symlinks + Path(_for_upload / "file1").write_text("file1") + Path(_for_upload / ".hidden").write_text(".hidden") + Path(_for_upload / "notfile1").write_text("notfile1") + transport.putfile(_for_upload / "file1", _remote_1 / "file1") + transport.putfile(_for_upload / ".hidden", _remote_1 / ".hidden") + transport.putfile(_for_upload / "notfile1", _remote_1 / "notfile1") + transport.symlink(_remote_1 / "file1", _remote_1 / "file1_link") # write where I tell you to testing(_remote_1 / "file1", _remote_2 / "file1") - assert Path(_remote_2 / "file1").read_text() == "file1" + transport.getfile(_remote_2 / "file1", _for_download / "file1") + assert Path(_for_download / "file1").read_text() == "file1" # always overwrite - Path(_remote_2 / "file1").write_text("notfile1") - testing(_remote_1 / "file1", _remote_2 / "file1") - assert Path(_remote_2 / "file1").read_text() == "file1" + testing(_remote_1 / "notfile1", _remote_2 / "file1") + transport.getfile(_remote_2 / "file1", _for_download / "file1-prime") + assert Path(_for_download / "file1-prime").read_text() == "notfile1" # don't skip hidden files testing(_remote_1 / ".hidden", _remote_2 / ".hidden-prime") - assert Path(_remote_2 / ".hidden-prime").read_text() == ".hidden" + transport.getfile(_remote_2 / ".hidden-prime", _for_download / ".hidden-prime") + assert Path(_for_download / ".hidden-prime").read_text() == ".hidden" # preserve links and don't follow them testing(_remote_1 / "file1_link", _remote_2 / "file1_link") - assert Path(_remote_2 / "file1_link").read_text() == "file1" - assert Path(_remote_2 / "file1_link").is_symlink() - testing(_remote_1 / "file10_link", _remote_2 / "file10_link") - assert Path(_remote_2 / "file10_link").read_text() == "file1" - assert Path(_remote_2 / "file10_link").is_symlink() + assert transport._cwd.joinpath(_remote_2 / "file1_link").is_symlink() + transport.getfile(_remote_2 / "file1_link", _for_download / "file1_link") + assert Path(_for_download / "file1_link").read_text() == "file1" From 1b38515fa6d71288ef6e63e9b14db3157b2e06df Mon Sep 17 00:00:00 2001 From: Ali Khosravi Date: Tue, 30 Jul 2024 10:41:51 +0200 Subject: [PATCH 34/39] Apply suggestions from code review Co-authored-by: Alexander Goscinski --- .pre-commit-config.yaml | 2 +- CHANGELOG.md | 2 +- tests/conftest.py | 6 ++++-- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index f6012ab..b20d7ff 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -45,4 +45,4 @@ repos: - "types-PyYAML" - "types-requests" - "pyfirecrest>=2.6.0" - - "aiida-core>=2.6.0" # please change to 2.6.2 when released + - "aiida-core>=2.6.0" # please change to 2.6.2 when released Issue #45 diff --git a/CHANGELOG.md b/CHANGELOG.md index d378424..cbfce08 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,7 +22,7 @@ ### Tests -- Tests has completely replaced with new ones. Previously tests were mocking FirecREST server. Those test were a good practice to ensure that all three (`aiida-firecrest`, FirecREST, and PyFirecREST) work flawlessly. +- The testing utils responsible for mocking the FirecREST server (specifically FirecrestMockServer) have been have been replaced with utils monkeypatching pyfirecrest. The FirecREST mocking utils introduced a maintenance overhead that is not in the responsibility of this repository. We still continue to support running with a real server and plan to continue running the tests with the [demo docker image](https://github.com/eth-cscs/firecrest/tree/master/deploy/demo) offered by CSCS. The downside was debugging wasn't easy at all. Not always obvious which of the three is causing a bug. Because of this, a new set of tests only verify the functionality of `aiida-firecrest` by directly mocking PyFirecREST. Maintaining this set in `tests/` is simpler because we just need to monitor the return values of PyFirecREST​. While maintaining the former is more difficult as you have to keep up with both FirecREST and PyFirecREST. diff --git a/tests/conftest.py b/tests/conftest.py index d95429e..7d7135e 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -55,6 +55,7 @@ def firecrest_computer(firecrest_config): class MockFirecrest: + """Mocks py:class:`pyfirecrest.Firecrest`.""" def __init__(self, firecrest_url, *args, **kwargs): self._firecrest_url = firecrest_url self.args = args @@ -78,6 +79,7 @@ def __init__(self, firecrest_url, *args, **kwargs): class MockClientCredentialsAuth: + """Mocks py:class:`pyfirecrest.ClientCredentialsAuth`.""" def __init__(self, *args, **kwargs): self.args = args self.kwargs = kwargs @@ -127,8 +129,8 @@ def firecrest_config( tmp_path: Path, ): """ - If a config file is given it sets up a client environment with the information - of the config file and uses pyfirecrest to communicate with the server. + If a config file is provided it sets up a client environment with the information + in the config file and uses pyfirecrest to communicate with the server. ┌─────────────────┐───►┌─────────────┐───►┌──────────────────┐ │ aiida_firecrest │ │ pyfirecrest │ │ FirecREST server │ └─────────────────┘◄───└─────────────┘◄───└──────────────────┘ From 28d481c80363e16a9aa3f476de7cf9421a5e60fd Mon Sep 17 00:00:00 2001 From: Ali Date: Tue, 30 Jul 2024 11:16:45 +0200 Subject: [PATCH 35/39] Server github action should skip for now --- .github/workflows/server-tests.yml | 96 ++++++++++++++++++++++++++++++ firecrest_demo.py | 93 +++++++++++++++++++++++++++++ tests/conftest.py | 2 + 3 files changed, 191 insertions(+) create mode 100644 .github/workflows/server-tests.yml create mode 100644 firecrest_demo.py diff --git a/.github/workflows/server-tests.yml b/.github/workflows/server-tests.yml new file mode 100644 index 0000000..175794e --- /dev/null +++ b/.github/workflows/server-tests.yml @@ -0,0 +1,96 @@ +# Run pytest against an actual FirecREST server, +# rather than just a mock server. + +name: Server + +# note: there are several bugs with docker image of FirecREST +# that failes this test. We skip this test for now, but should be addressed in a seperate PR than #36 +on: + push: + branches-ignore: + - '**' + pull_request: + branches-ignore: + - '**' + + +jobs: + + tests: + + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: ["ubuntu-latest"] + python-version: ["3.9"] + firecrest-version: ["v1.13.0"] + + services: + rabbitmq: + image: rabbitmq:3.8.14-management + ports: + - 5672:5672 + - 15672:15672 + + steps: + - uses: actions/checkout@v3 + + - name: checkout the firecrest repository + uses: actions/checkout@v3 + with: + repository: eth-cscs/firecrest + ref: ${{ matrix.firecrest-version }} + path: .demo-server + + - name: Cache Docker images + uses: jpribyl/action-docker-layer-caching@v0.1.1 + continue-on-error: true + with: + key: ${{ runner.os }}-docker-${{ matrix.firecrest-version }} + + # note, for some reason, the certificator image fails to build + # if you build them in order, so here we build everything except that first + # and then it seems to work + - name: Build the FirecREST images + run: | + docker-compose build f7t-base + docker-compose build compute + docker-compose build status + docker-compose build storage + docker-compose build tasks + docker-compose build utilities + docker-compose build reservations + docker-compose build client + docker-compose build cluster + docker-compose build keycloak + docker-compose build kong + docker-compose build minio + docker-compose build taskpersistence + docker-compose build opa + docker-compose build openapi + docker-compose build jaeger + + # docker-compose build certificator + working-directory: .demo-server/deploy/demo + + - name: Ensure permissions of SSH Keys + run: | + chmod 400 .demo-server/deploy/test-build/environment/keys/ca-key + chmod 400 .demo-server/deploy/test-build/environment/keys/user-key + + - name: Start the FirecREST server + run: docker-compose up --detach + working-directory: .demo-server/deploy/demo + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + cache: 'pip' + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -e .[dev] + - name: Test with pytest + run: pytest -vv --cov=aiida_firecrest --firecrest-config .firecrest-demo-config.json diff --git a/firecrest_demo.py b/firecrest_demo.py new file mode 100644 index 0000000..39065bb --- /dev/null +++ b/firecrest_demo.py @@ -0,0 +1,93 @@ +""" +This file is a CLI to generate a FirecREST demo server. +It clones FirecREST from a given URL and tag, builds the docker environment, and runs the docker environment. +It's useful for local testing and development. +Currently, it's not used in the CI/CD pipeline, and it's not part of the AiiDA plugin. +This code is not tested, with anything above version "v1.13.0" of FirecREST, and it's not maintained. +""" +from __future__ import annotations + +from argparse import ArgumentParser +from pathlib import Path +from subprocess import check_call +from typing import Protocol + + +class CliArgs(Protocol): + folder: str + git_tag: str + git_url: str + build: bool + + +def parse_args(args: list[str] | None = None) -> CliArgs: + """Parse the command line arguments.""" + parser = ArgumentParser(description="Create a FirecREST demo server.") + parser.add_argument( + "--folder", + default=".demo-server", + type=str, + help="The folder to clone FirecREST into.", + ) + parser.add_argument( + "--git-url", + type=str, + default="https://github.com/eth-cscs/firecrest.git", + help="The URL to clone FirecREST from.", + ) + parser.add_argument( + "--git-tag", + type=str, + default="v1.13.0", + help="The tag to checkout FirecREST at.", + ) + parser.add_argument( + "--build", + action="store_true", + help="Don't build the docker environment.", + ) + return parser.parse_args(args) + + +def main(args: list[str] | None = None): + """A CLI to generate a FirecREST demo server.""" + # use argparse to get the folder to clone firecrest into + parsed = parse_args(args) + + folder = Path(parsed.folder).absolute() + + if not folder.exists(): + print(f"Cloning FirecREST into {parsed.folder}") + check_call( + ["git", "clone", "--branch", parsed.git_tag, parsed.git_url, str(folder)] + ) + else: + print(f"FirecREST already exists in {folder!r}") + + # build the docker environment + if not parsed.build: + print("Skipping building the docker environment") + else: + print("Building the docker environment") + check_call(["docker-compose", "build"], cwd=(folder / "deploy" / "demo")) + + # ensure permissions of SSH keys (chmod 400) + print("Ensuring permissions of SSH keys") + folder.joinpath("deploy", "test-build", "environment", "keys", "ca-key").chmod( + 0o400 + ) + folder.joinpath("deploy", "test-build", "environment", "keys", "user-key").chmod( + 0o400 + ) + + # run the docker environment + print("Running the docker environment") + # could fail if required port in use + # on MaOS, can use e.g. `lsof -i :8080` to check + # TODO on MacOS port 7000 is used by AirPlay (afs3-fileserver), + # https://github.com/cookiecutter/cookiecutter-django/issues/3499 + check_call(["docker-compose", "up", "--detach"], cwd=(folder / "deploy" / "demo")) + + +if __name__ == "__main__": + main() diff --git a/tests/conftest.py b/tests/conftest.py index 7d7135e..7aa23e0 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -56,6 +56,7 @@ def firecrest_computer(firecrest_config): class MockFirecrest: """Mocks py:class:`pyfirecrest.Firecrest`.""" + def __init__(self, firecrest_url, *args, **kwargs): self._firecrest_url = firecrest_url self.args = args @@ -80,6 +81,7 @@ def __init__(self, firecrest_url, *args, **kwargs): class MockClientCredentialsAuth: """Mocks py:class:`pyfirecrest.ClientCredentialsAuth`.""" + def __init__(self, *args, **kwargs): self.args = args self.kwargs = kwargs From d143addea1d7d3ccdfcb6d80ba24553e61a5a7da Mon Sep 17 00:00:00 2001 From: Ali Date: Tue, 30 Jul 2024 11:34:42 +0200 Subject: [PATCH 36/39] check if codecov works --- .github/workflows/tests.yml | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 49fc7fe..b543581 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -51,12 +51,11 @@ jobs: - name: Test with pytest run: pytest -vv --cov=aiida_firecrest --cov-report=xml --cov-report=term - # Codecov failing, we need to fix token: https://github.com/aiidateam/aiida-firecrest/issues/38 - # - name: Upload coverage reports to Codecov - # uses: codecov/codecov-action@v3 - # with: - # name: aiida-firecrest-pytests - # flags: pytests - # file: ./coverage.xml - # fail_ci_if_error: true - # token: ${{ secrets.CODECOV_TOKEN }} + - name: Upload coverage reports to Codecov + uses: codecov/codecov-action@v4 + with: + name: aiida-firecrest-pytests + flags: pytests + file: ./coverage.xml + fail_ci_if_error: true + token: ${{ secrets.CODECOV_TOKEN }} From 6742b6c4fa01c39c413f9b4b73b5115cc087283e Mon Sep 17 00:00:00 2001 From: Ali Date: Tue, 30 Jul 2024 14:43:18 +0200 Subject: [PATCH 37/39] Readme updated. --- CHANGELOG.md | 3 +- README.md | 125 +++++++++++++++++------------------ aiida_firecrest/transport.py | 4 +- tests/conftest.py | 12 ++-- tests/test_transport.py | 26 ++++---- 5 files changed, 85 insertions(+), 85 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cbfce08..1e1e236 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ - Added `_create_secret_file` to store user secret locally in `~/.firecrest/` - Added `_validate_temp_directory` to allocate a temporary directory useful for `extract` and `compress` methods on FirecREST server. - Added `_dynamic_info_direct_size` this is able to get info of direct transfer from the server rather than asking from users. Raise if fails to make a connection. +- Added `_dynamic_info_firecrest_version` to fetch which version of FirecREST api is interacting with. - Added `_validate_checksum` to check integrity of downloaded/uploaded files. - Added `_gettreetar` & `_puttreetar` to transfer directories as tar files internally. - Added `payoff` function to calculate when is gainful to transfer as zip, and when to transfer individually. @@ -24,7 +25,7 @@ - The testing utils responsible for mocking the FirecREST server (specifically FirecrestMockServer) have been have been replaced with utils monkeypatching pyfirecrest. The FirecREST mocking utils introduced a maintenance overhead that is not in the responsibility of this repository. We still continue to support running with a real server and plan to continue running the tests with the [demo docker image](https://github.com/eth-cscs/firecrest/tree/master/deploy/demo) offered by CSCS. The downside was debugging wasn't easy at all. Not always obvious which of the three is causing a bug. -Because of this, a new set of tests only verify the functionality of `aiida-firecrest` by directly mocking PyFirecREST. Maintaining this set in `tests/` is simpler because we just need to monitor the return values of PyFirecREST​. While maintaining the former is more difficult as you have to keep up with both FirecREST and PyFirecREST. +Because of this, a new set of tests only verify the functionality of `aiida-firecrest` by directly mocking PyFirecREST. Maintaining this set in `tests/` is simpler because we just need to monitor the return values of PyFirecREST​. While maintaining the former was more difficult as you have to keep up with both FirecREST and PyFirecREST. ### Miscellaneous diff --git a/README.md b/README.md index 88b5caa..d8cf09b 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ AiiDA Transport/Scheduler plugins for interfacing with [FirecREST](https://produ It is currently tested against [FirecREST v2.6.0](https://github.com/eth-cscs/pyfirecrest/tree/v2.6.0). -## Usage +## Installation Install via GitHub or PyPI: @@ -43,68 +43,59 @@ You can then create a `Computer` in AiiDA: $ verdi computer setup Report: enter ? for help. Report: enter ! to ignore the default and set no value. -Computer label: firecrest-client -Hostname: unused -Description []: My FirecREST client plugin +Computer label: firecrest-client # your choice +Hostname: unused # your choice, irrelevant +Description []: My FirecREST client plugin # your choice Transport plugin: firecrest Scheduler plugin: firecrest Shebang line (first line of each script, starting with #!) [#!/bin/bash]: Work directory on the computer [/scratch/{username}/aiida/]: Mpirun command [mpirun -np {tot_num_mpiprocs}]: -Default number of CPUs per machine: 2 -Default amount of memory per machine (kB).: 100 +Default number of CPUs per machine: 2 # depending on your compute resource +Default amount of memory per machine (kB).: 100 # depending on your compute resource Escape CLI arguments in double quotes [y/N]: Success: Computer<3> firecrest-client created Report: Note: before the computer can be used, it has to be configured with the command: -Report: verdi -p quicksetup computer configure firecrest firecrest-client +Report: verdi -p MYPROFILE computer configure firecrest firecrest-client ``` ```console -$ verdi -p quicksetup computer configure firecrest firecrest-client +$ verdi -p MYPROFILE computer configure firecrest firecrest-client Report: enter ? for help. Report: enter ! to ignore the default and set no value. -Server URL: https://firecrest.cscs.ch +Server URL: https://firecrest.cscs.ch # this for CSCS Token URI: https://auth.cscs.ch/auth/realms/firecrest-clients/protocol/openid-connect/token Client ID: username-client Client Secret: xyz -Client Machine: daint -Maximum file size for direct transfer (MB) [5.0]: +Compute resource (Machine): daint Temp directory on server: /scratch/something/ # "A temp directory on user's space on the server for creating temporary files (compression, extraction, etc.)" +FirecREST api version [Enter 0 to get this info from server] [0]: 0 +Maximum file size for direct transfer (MB) [Enter 0 to get this info from server] [0]: 0 Report: Configuring computer firecrest-client for user chrisj_sewell@hotmail.com. Success: firecrest-client successfully configured for chrisj_sewell@hotmail.com ``` +You can always check your config with ```console $ verdi computer show firecrest-client ---------------------------- ------------------------------------ -Label firecrest-client -PK 3 -UUID 48813c55-1b2b-4afc-a1a1-e0d33a5b6868 -Description My FirecREST client plugin -Hostname unused -Transport type firecrest -Scheduler type firecrest -Work directory /scratch/{username}/aiida/ -Shebang #!/bin/bash -Mpirun command mpirun -np {tot_num_mpiprocs} -Default #procs/machine 2 -Default memory (kB)/machine 100 -Prepend text -Append text ---------------------------- ------------------------------------ ``` See also the [pyfirecrest CLI](https://github.com/eth-cscs/pyfirecrest), for directly interacting with a FirecREST server. + +After this, everything should function normally through AiiDA with no problems. See [tests/test_calculation.py](tests/test_calculation.py) for a working example of how to use the plugin, via the AiiDA API. +If you encounter any problems/bug, please don't hesitate to open an issue on this repository. + ### Current Issues Calculations are now running successfully, however, there are still issues regarding efficiency, Could be improved: 1. Monitoring / management of API request rates could to be improved. Currently this is left up to PyFirecREST. +2. Each transfer request includes 2 seconds of `sleep` time, imposed by `pyfirecrest`. One can takes use of their `async` client, but with current design of `aiida-core`, the gain will be minimum. (see the [closing comment of issue#94 on pyfirecrest](https://github.com/eth-cscs/pyfirecrest/issues/94) and [PR#6079 on aiida-core ](https://github.com/aiidateam/aiida-core/pull/6079)) -## Development +## For developers ```bash git clone @@ -122,43 +113,51 @@ pre-commit run --all-files ### Testing -There are two types of tests: mocking the PyFirecREST or the FirecREST server. -While the latter is a good practice to ensure that all three (`aiida-firecrest`, FirecREST, and PyFirecREST) work flawlessly, debugging may not always be easy because it may not always be obvious which of the three is causing a bug. -Because of this, we have another set of tests that only verify the functionality of `aiida-firecrest` by directly mocking PyFirecREST. Maintaining the second set in `tests/tests_mocking_pyfirecrest/` is simpler because we just need to monitor the return values of PyFirecREST​. While maintaining the former is more difficult as you have to keep up with both FirecREST and PyFirecREST. - - -#### Mocking FirecREST server - -These tests were successful against [FirecREST v1.13.0](https://github.com/eth-cscs/firecrest/releases/tag/v1.13.0). -For newer version please refer to tests Mocking PyFirecREST - It is recommended to run the tests via [tox](https://tox.readthedocs.io/en/latest/). ```bash tox ``` -By default, the tests are run using a mock FirecREST server, in a temporary folder -(see [aiida_fircrest.utils_test.FirecrestConfig](aiida_firecrest/utils_test.py)). -This allows for quick testing and debugging of the plugin, without needing to connect to a real server, -but is obviously not guaranteed to be fully representative of the real behaviour. +By default, the tests are run using a monkey patched PyFirecREST. +This allows for quick testing and debugging of the plugin, without needing to connect to a real server, but is obviously not guaranteed to be fully representative of the real behaviour. -You can also provide connections details to a real FirecREST server: +To have a guaranteed proof, you may also provide connections details to a real FirecREST server: ```bash tox -- --firecrest-config=".firecrest-demo-config.json" ``` -The format of the `.firecrest-demo-config.json` file is: + +If a config file is provided, tox sets up a client environment with the information +in the config file and uses pyfirecrest to communicate with the server. +```plaintext +┌─────────────────┐───►┌─────────────┐───►┌──────────────────┐ +│ aiida_firecrest │ │ pyfirecrest │ │ FirecREST server │ +└─────────────────┘◄───└─────────────┘◄───└──────────────────┘ +``` + +if a config file is not provided, it monkeypatches pyfirecrest so we never actually communicate with a server. +```plaintext +┌─────────────────┐───►┌─────────────────────────────┐ +│ aiida_firecrest │ │ pyfirecrest (monkeypatched) │ +└─────────────────┘◄───└─────────────────────────────┘ +``` + +The format of the `.firecrest-demo-config.json` file, for example is like: + ```json -{ - "url": "https://firecrest.cscs.ch", - "token_uri": "https://auth.cscs.ch/auth/realms/cscs/protocol/openid-connect/token", + { + "url": "https://firecrest-tds.cscs.ch", + "token_uri": "https://auth.cscs.ch/auth/realms/firecrest-clients/protocol/openid-connect/token", "client_id": "username-client", - "client_secret": "xyz", - "machine": "daint", - "scratch_path": "/scratch/snx3000/username" + "client_secret": "path-to-secret-file", + "compute_resource": "daint", + "temp_directory": "/scratch/snx3000/username/", + "small_file_size_mb": 5.0, + "workdir": "/scratch/snx3000/username/", + "api_version": "1.16.0" } ``` @@ -168,17 +167,18 @@ In this mode, if you want to inspect the generated files, after a failure, you c tox -- --firecrest-config=".firecrest-demo-config.json" --firecrest-no-clean ``` -See [firecrest_demo.py](firecrest_demo.py) for how to start up a demo server, -and also [server-tests.yml](.github/workflows/server-tests.yml) for how the tests are run against the demo server on GitHub Actions. +**These tests were successful against [FirecREST v1.16.0](https://github.com/eth-cscs/firecrest/releases/tag/v1.16.0), except those who require to list directories in a symlink directory, which fail due to a bug in FirecREST. [An issue](https://github.com/eth-cscs/firecrest/issues/205) is open on FirecREST repo about this.** -If you want to analyse statistics of the API requests made by each test, +Instead of a real server (which requires an account and credential), tests can also run against a docker image provided by FirecREST. See [firecrest_demo.py](firecrest_demo.py) for how to start up a demo server, and also [server-tests.yml](.github/workflows/server-tests.yml) for how the tests are run against the demo server on GitHub Actions. + + -##### Notes on using the demo server on MacOS +#### Notes on using the demo server on MacOS A few issues have been noted when using the demo server on MacOS (non-Mx): @@ -201,16 +201,11 @@ although it is of note that you can find these files directly where you your `fi [black-link]: https://github.com/ambv/black +### :bug: Fishing :bug: Bugs :bug: -#### Mocking PyFirecREST - -These set of test do not gurantee that the firecrest protocol is working, but it's very useful to quickly check if `aiida-firecrest` is behaving as it's expected to do. To run just simply use `pytest`. - +First, start with running tests locally with no `config` file given, that would monkeypatch `pyfirecrest`. These set of test do not guarantee that the whole firecrest protocol is working, but it's very useful to quickly check if `aiida-firecrest` is behaving as it's expected to do. To run just simply use `pytest` or `tox`. -If these tests, pass and still you have trouble in real deployment, that means your installed version of pyfirecrest is behaving differently from what `aiida-firecrest` expects in `MockFirecrest` in `tests/tests_mocking_pyfirecrest/conftest.py`. -If there is no version of `aiida-firecrest` available that supports your `pyfirecrest` version and if down/upgrading your `pyfirecrest` to a supported version is not an option, you might try the following: -- open an issue on the `aiida-firecrest` repository on GitHub to request supporting your version of pyfirecrest -- if you feel up to finding the discrepancy and fixing it within `aiida-firecrest`, open a PR instead -- if you think the problem is a bug in `pyfirecrest`, open an issue there +If these tests pass and the bug persists, consider providing a `config` file to run the tests on a docker image or directly on a real server. Be aware of versioning, `pyfirecrest` doesn't check which version of api it's interacting with. (TODO: open an issue on this) -Either way, make sure to report which version of `aiida-firecrest` and `pyfirecrest` you are using. +If the bug persists and test still passes, then most certainly it's a problem of `aiida-firecrest`. +If not, probably the issue is from FirecREST, you might open an issue to [`pyfirecrest`](https://github.com/eth-cscs/pyfirecrest) or [`FirecREST`](https://github.com/eth-cscs/firecrest). diff --git a/aiida_firecrest/transport.py b/aiida_firecrest/transport.py index f30303a..31a6743 100644 --- a/aiida_firecrest/transport.py +++ b/aiida_firecrest/transport.py @@ -842,7 +842,7 @@ def gettree( local = local.joinpath(remote.name) local.mkdir(parents=True, exist_ok=True) else: - # Destination directory does not exist, create and move content 69 inside it + # Destination directory does not exist, create and move content abc inside it local.mkdir(parents=True, exist_ok=False) if self.payoff(remote): @@ -1060,7 +1060,7 @@ def puttree( remote = self._cwd.joinpath(remote, localpath.name) self.mkdir(remote, ignore_existing=False) else: - # Destination directory does not exist, create and move content 69 inside it + # Destination directory does not exist, create and move content abc inside it self.mkdir(remote, ignore_existing=False) if self.payoff(localpath): diff --git a/tests/conftest.py b/tests/conftest.py index 7aa23e0..3a8918d 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -137,7 +137,7 @@ def firecrest_config( │ aiida_firecrest │ │ pyfirecrest │ │ FirecREST server │ └─────────────────┘◄───└─────────────┘◄───└──────────────────┘ - if `config_path` is not given, it monkeypatches pyfirecrest so we never + if a config file is not provided, it monkeypatches pyfirecrest so we never actually communicate with a server. ┌─────────────────┐───►┌─────────────────────────────┐ │ aiida_firecrest │ │ pyfirecrest (monkeypatched) │ @@ -145,7 +145,10 @@ def firecrest_config( """ config_path: str | None = request.config.getoption("--firecrest-config") no_clean: bool = request.config.getoption("--firecrest-no-clean") - record_requests: bool = request.config.getoption("--firecrest-requests") + # record_requests: bool = request.config.getoption("--firecrest-requests") + # record_requests: bool = request.config.getoption("--firecrest-requests") + # TODO: record_requests is un-maintained after PR#36, and practically not used. + # But let's keep it commented for future use, if needed. if config_path is not None: # telemetry: RequestTelemetry | None = None @@ -195,7 +198,8 @@ def firecrest_config( # test_name # ] = telemetry.counts else: - if no_clean or record_requests: + # if no_clean or record_requests: + if no_clean: raise ValueError( "--firecrest-{no-clean,requests} options are only available" " when a config file is passed using --firecrest-config." @@ -211,7 +215,7 @@ def firecrest_config( _temp_directory.mkdir() Path(tmp_path / ".firecrest").mkdir() - _secret_path = Path(tmp_path / ".firecrest/secret69") + _secret_path = Path(tmp_path / ".firecrest/secretabc") _secret_path.write_text("secret_string") workdir = tmp_path / "scratch" diff --git a/tests/test_transport.py b/tests/test_transport.py index 1726e7a..f9d970d 100644 --- a/tests/test_transport.py +++ b/tests/test_transport.py @@ -380,17 +380,17 @@ def test_puttree(firecrest_computer: orm.Computer, tmpdir: Path, payoff: bool): # Note: # SSH transport behaviour - # transport.put('somepath/69', 'someremotepath/') == transport.put('somepath/69', 'someremotepath') - # transport.put('somepath/69', 'someremotepath/') != transport.put('somepath/69/', 'someremotepath/') - # transport.put('somepath/69', 'someremotepath/67') --> if 67 not exist, create and move content 69 + # transport.put('somepath/abc', 'someremotepath/') == transport.put('somepath/abc', 'someremotepath') + # transport.put('somepath/abc', 'someremotepath/') != transport.put('somepath/abc/', 'someremotepath/') + # transport.put('somepath/abc', 'someremotepath/67') --> if 67 not exist, create and move content abc # inside it (someremotepath/67) - # transport.put('somepath/69', 'someremotepath/67') --> if 67 exist, create 69 inside it (someremotepath/67/69) - # transport.put('somepath/69', 'someremotepath/6889/69') --> useless Error: OSError + # transport.put('somepath/abc', 'someremotepath/67') --> if 67 exist, create abc inside it (someremotepath/67/abc) + # transport.put('somepath/abc', 'someremotepath/6889/abc') --> useless Error: OSError # Weired # SSH "bug": - # transport.put('somepath/69', 'someremotepath/') --> assuming someremotepath exists, make 69 + # transport.put('somepath/abc', 'someremotepath/') --> assuming someremotepath exists, make abc # while - # transport.put('somepath/69/', 'someremotepath/') --> assuming someremotepath exists, OSError: + # transport.put('somepath/abc/', 'someremotepath/') --> assuming someremotepath exists, OSError: # cannot make someremotepath tmpdir_remote = transport._temp_directory @@ -552,12 +552,12 @@ def test_gettree(firecrest_computer: orm.Computer, tmpdir: Path, payoff: bool): transport.payoff_override = payoff # Note: - # SSH transport behaviour, 69 is a directory - # transport.get('somepath/69', 'someremotepath/') == transport.get('somepath/69', 'someremotepath') - # transport.get('somepath/69', 'someremotepath/') == transport.get('somepath/69/', 'someremotepath/') - # transport.get('someremotepath/69', 'somepath/69')--> if 69 exist, create 69 inside it ('somepath/69/69') - # transport.get('someremotepath/69', 'somepath/69')--> if 69 noexist,create 69 inside it ('somepath/69') - # transport.get('somepath/69', 'someremotepath/6889/69') --> create everything, make_parent = True + # SSH transport behaviour, abc is a directory + # transport.get('somepath/abc', 'someremotepath/') == transport.get('somepath/abc', 'someremotepath') + # transport.get('somepath/abc', 'someremotepath/') == transport.get('somepath/abc/', 'someremotepath/') + # transport.get('someremotepath/abc', 'somepath/abc')--> if abc exist, create abc inside it ('somepath/abc/abc') + # transport.get('someremotepath/abc', 'somepath/abc')--> if abc noexist,create abc inside it ('somepath/abc') + # transport.get('somepath/abc', 'someremotepath/6889/abc') --> create everything, make_parent = True tmpdir_remote = transport._temp_directory _remote = tmpdir_remote / "remotedir" _remote.mkdir() From e2295618ce19cc88736c9b25f3923fc299a20d86 Mon Sep 17 00:00:00 2001 From: Ali Date: Tue, 30 Jul 2024 17:05:20 +0200 Subject: [PATCH 38/39] Applied changes from final review --- CHANGELOG.md | 6 +- README.md | 14 +- firecrest_demo.py | 1 + tests/conftest.py | 612 ++++++++++++++++++++++------------------------ 4 files changed, 308 insertions(+), 325 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1e1e236..1bc7fc1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,14 +23,12 @@ ### Tests -- The testing utils responsible for mocking the FirecREST server (specifically FirecrestMockServer) have been have been replaced with utils monkeypatching pyfirecrest. The FirecREST mocking utils introduced a maintenance overhead that is not in the responsibility of this repository. We still continue to support running with a real server and plan to continue running the tests with the [demo docker image](https://github.com/eth-cscs/firecrest/tree/master/deploy/demo) offered by CSCS. -The downside was debugging wasn't easy at all. Not always obvious which of the three is causing a bug. -Because of this, a new set of tests only verify the functionality of `aiida-firecrest` by directly mocking PyFirecREST. Maintaining this set in `tests/` is simpler because we just need to monitor the return values of PyFirecREST​. While maintaining the former was more difficult as you have to keep up with both FirecREST and PyFirecREST. +- The testing utils responsible for mocking the FirecREST server (specifically FirecrestMockServer) have been replaced with utils monkeypatching pyfirecrest. The FirecREST mocking utils introduced a maintenance overhead that is not in the responsibility of this repository. We still continue to support running with a real FirecREST server and plan to continue running the tests with the [demo docker image](https://github.com/eth-cscs/firecrest/tree/master/deploy/demo) offered by CSCS. The docker image has been disabled for the moment due to some problems (see issue #47). ### Miscellaneous -- class `FcPath` is removed from interface here, as it has [merged](https://github.com/eth-cscs/pyfirecrest/pull/43) into `pyfirecrest` +- class `FcPath` is removed from interface here, as it has [merged](https://github.com/eth-cscs/pyfirecrest/pull/43) into pyfirecrest ## v0.1.0 (December 2021) diff --git a/README.md b/README.md index d8cf09b..9eb6ea0 100644 --- a/README.md +++ b/README.md @@ -92,8 +92,8 @@ If you encounter any problems/bug, please don't hesitate to open an issue on thi Calculations are now running successfully, however, there are still issues regarding efficiency, Could be improved: -1. Monitoring / management of API request rates could to be improved. Currently this is left up to PyFirecREST. -2. Each transfer request includes 2 seconds of `sleep` time, imposed by `pyfirecrest`. One can takes use of their `async` client, but with current design of `aiida-core`, the gain will be minimum. (see the [closing comment of issue#94 on pyfirecrest](https://github.com/eth-cscs/pyfirecrest/issues/94) and [PR#6079 on aiida-core ](https://github.com/aiidateam/aiida-core/pull/6079)) +1. Monitoring / management of API request rates could to be improved. Currently this is left up to pyfirecrest. +2. Each transfer request includes 2 seconds of `sleep` time, imposed by pyfirecrest. One can takes use of their `async` client, but with current design of `aiida-core`, the gain will be minimum. (see the [closing comment of issue#94 on pyfirecrest](https://github.com/eth-cscs/pyfirecrest/issues/94) and [PR#6079 on aiida-core ](https://github.com/aiidateam/aiida-core/pull/6079)) ## For developers @@ -119,7 +119,7 @@ It is recommended to run the tests via [tox](https://tox.readthedocs.io/en/lates tox ``` -By default, the tests are run using a monkey patched PyFirecREST. +By default, the tests are run using a monkey patched pyfirecrest. This allows for quick testing and debugging of the plugin, without needing to connect to a real server, but is obviously not guaranteed to be fully representative of the real behaviour. To have a guaranteed proof, you may also provide connections details to a real FirecREST server: @@ -201,11 +201,11 @@ although it is of note that you can find these files directly where you your `fi [black-link]: https://github.com/ambv/black -### :bug: Fishing :bug: Bugs :bug: +### :bug: Fishing Bugs :bug: -First, start with running tests locally with no `config` file given, that would monkeypatch `pyfirecrest`. These set of test do not guarantee that the whole firecrest protocol is working, but it's very useful to quickly check if `aiida-firecrest` is behaving as it's expected to do. To run just simply use `pytest` or `tox`. +First, start with running tests locally with no `config` file given, that would monkeypatch pyfirecrest. These set of test do not guarantee that the whole firecrest protocol is working, but it's very useful to quickly check if `aiida-firecrest` is behaving as it's expected to do. To run just simply use `pytest` or `tox`. -If these tests pass and the bug persists, consider providing a `config` file to run the tests on a docker image or directly on a real server. Be aware of versioning, `pyfirecrest` doesn't check which version of api it's interacting with. (TODO: open an issue on this) +If these tests pass and the bug persists, consider providing a `config` file to run the tests on a docker image or directly on a real server. Be aware of versioning, pyfirecrest doesn't check which version of api it's interacting with. (see https://github.com/eth-cscs/pyfirecrest/issues/116) If the bug persists and test still passes, then most certainly it's a problem of `aiida-firecrest`. -If not, probably the issue is from FirecREST, you might open an issue to [`pyfirecrest`](https://github.com/eth-cscs/pyfirecrest) or [`FirecREST`](https://github.com/eth-cscs/firecrest). +If not, probably the issue is from FirecREST, you might open an issue to [pyfirecrest](https://github.com/eth-cscs/pyfirecrest) or [`FirecREST`](https://github.com/eth-cscs/firecrest). diff --git a/firecrest_demo.py b/firecrest_demo.py index 39065bb..21c5e30 100644 --- a/firecrest_demo.py +++ b/firecrest_demo.py @@ -4,6 +4,7 @@ It's useful for local testing and development. Currently, it's not used in the CI/CD pipeline, and it's not part of the AiiDA plugin. This code is not tested, with anything above version "v1.13.0" of FirecREST, and it's not maintained. +See also https://github.com/aiidateam/aiida-firecrest/issues/47 """ from __future__ import annotations diff --git a/tests/conftest.py b/tests/conftest.py index 3a8918d..7e333b0 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -62,21 +62,304 @@ def __init__(self, firecrest_url, *args, **kwargs): self.args = args self.kwargs = kwargs - self.whoami = whoami - self.list_files = list_files - self.stat = stat_ - self.mkdir = mkdir - self.simple_delete = simple_delete - self.parameters = parameters - self.symlink = symlink - self.checksum = checksum - self.simple_download = simple_download - self.simple_upload = simple_upload - self.compress = compress - self.extract = extract - self.copy = copy - self.submit = submit - self.poll_active = poll_active + def submit( + self, + machine: str, + script_str: str | None = None, + script_remote_path: str | None = None, + script_local_path: str | None = None, + local_file=False, + ): + if local_file: + raise DeprecationWarning("local_file is not supported") + + if script_remote_path and not Path(script_remote_path).exists(): + raise FileNotFoundError(f"File {script_remote_path} does not exist") + job_id = random.randint(10000, 99999) + + # Filter out lines starting with '#SBATCH' + with open(script_remote_path) as file: + lines = file.readlines() + command = "".join([line for line in lines if not line.strip().startswith("#")]) + + # Make the dummy files + for line in lines: + if "--error" in line: + error_file = line.split("=")[1].strip() + (Path(script_remote_path).parent / error_file).touch() + elif "--output" in line: + output_file = line.split("=")[1].strip() + (Path(script_remote_path).parent / output_file).touch() + + # Execute the job, this is useful for test_calculation.py + if "aiida.in" in command: + # skip blank command like: '/bin/bash' + os.chdir(Path(script_remote_path).parent) + os.system(command) + + return {"jobid": job_id} + + def poll_active(self, machine: str, jobs: list[str], page_number: int = 0): + response = [] + # 12 satets are defined in firecrest + states = [ + "TIMEOUT", + "SUSPENDED", + "PREEMPTED", + "CANCELLED", + "NODE_FAIL", + "PENDING", + "FAILED", + "RUNNING", + "CONFIGURING", + "QUEUED", + "COMPLETED", + "COMPLETING", + ] + for i in range(len(jobs)): + response.append( + { + "job_data_err": "", + "job_data_out": "", + "job_file": "somefile.sh", + "job_file_err": "somefile-stderr.txt", + "job_file_out": "somefile-stdout.txt", + "job_info_extra": "Job info returned successfully", + "jobid": f"{jobs[i]}", + "name": "aiida-45", + "nodelist": "nid00049", + "nodes": "1", + "partition": "normal", + "start_time": "0:03", + "state": states[i % 12], + "time": "2024-06-21T10:44:42", + "time_left": "29:57", + "user": "Prof. Wang", + } + ) + + return response[ + page_number + * Values._DEFAULT_PAGE_SIZE : (page_number + 1) + * Values._DEFAULT_PAGE_SIZE + ] + + def whoami(self, machine: str): + assert machine == "MACHINE_NAME" + return "test_user" + + def list_files( + self, + machine: str, + target_path: str, + recursive: bool = False, + show_hidden: bool = False, + ): + # this is mimiking the expected behaviour from the firecrest code. + + content_list = [] + for root, dirs, files in os.walk(target_path): + if not recursive and root != target_path: + continue + for name in dirs + files: + full_path = os.path.join(root, name) + relative_path = ( + Path(os.path.relpath(root, target_path)).joinpath(name).as_posix() + ) + if os.path.islink(full_path): + content_type = "l" + link_target = ( + os.readlink(full_path) if os.path.islink(full_path) else None + ) + elif os.path.isfile(full_path): + content_type = "-" + link_target = None + elif os.path.isdir(full_path): + content_type = "d" + link_target = None + else: + content_type = "NON" + link_target = None + permissions = stat.filemode(Path(full_path).lstat().st_mode)[1:] + if name.startswith(".") and not show_hidden: + continue + content_list.append( + { + "name": relative_path, + "type": content_type, + "link_target": link_target, + "permissions": permissions, + } + ) + + return content_list + + def stat(self, machine: str, targetpath: firecrest.path, dereference=True): + stats = os.stat( + targetpath, follow_symlinks=bool(dereference) if dereference else False + ) + return { + "ino": stats.st_ino, + "dev": stats.st_dev, + "nlink": stats.st_nlink, + "uid": stats.st_uid, + "gid": stats.st_gid, + "size": stats.st_size, + "atime": stats.st_atime, + "mtime": stats.st_mtime, + "ctime": stats.st_ctime, + } + + def mkdir( + self, + machine: str, + target_path: str, + p: bool = False, + ignore_existing: bool = False, + ): + target = Path(target_path) + target.mkdir(exist_ok=ignore_existing, parents=p) + + def simple_delete(self, machine: str, target_path: str): + if not Path(target_path).exists(): + raise FileNotFoundError(f"File or folder {target_path} does not exist") + if os.path.isdir(target_path): + shutil.rmtree(target_path) + else: + os.remove(target_path) + + def symlink(self, machine: str, target_path: str, link_path: str): + # this is how firecrest does it + os.system(f"ln -s {target_path} {link_path}") + + def simple_download(self, machine: str, remote_path: str, local_path: str): + # this procedure is complecated in firecrest, but I am simplifying it here + # we don't care about the details of the download, we just want to make sure + # that the aiida-firecrest code is calling the right functions at right time + if Path(remote_path).is_dir(): + raise IsADirectoryError(f"{remote_path} is a directory") + if not Path(remote_path).exists(): + raise FileNotFoundError(f"{remote_path} does not exist") + os.system(f"cp {remote_path} {local_path}") + + def simple_upload( + self, + machine: str, + local_path: str, + remote_path: str, + file_name: str | None = None, + ): + # this procedure is complecated in firecrest, but I am simplifying it here + # we don't care about the details of the upload, we just want to make sure + # that the aiida-firecrest code is calling the right functions at right time + if Path(local_path).is_dir(): + raise IsADirectoryError(f"{local_path} is a directory") + if not Path(local_path).exists(): + raise FileNotFoundError(f"{local_path} does not exist") + if file_name: + remote_path = os.path.join(remote_path, file_name) + os.system(f"cp {local_path} {remote_path}") + + def copy(self, machine: str, source_path: str, target_path: str): + # this is how firecrest does it + # https://github.com/eth-cscs/firecrest/blob/db6ba4ba273c11a79ecbe940872f19d5cb19ac5e/src/utilities/utilities.py#L451C1-L452C1 + os.system(f"cp --force -dR --preserve=all -- '{source_path}' '{target_path}'") + + def compress( + self, machine: str, source_path: str, target_path: str, dereference: bool = True + ): + # this is how firecrest does it + # https://github.com/eth-cscs/firecrest/blob/db6ba4ba273c11a79ecbe940872f19d5cb19ac5e/src/utilities/utilities.py#L460 + basedir = os.path.dirname(source_path) + file_path = os.path.basename(source_path) + deref = "--dereference" if dereference else "" + os.system(f"tar {deref} -czf '{target_path}' -C '{basedir}' '{file_path}'") + + def extract(self, machine: str, source_path: str, target_path: str): + # this is how firecrest does it + # https://github.com/eth-cscs/firecrest/blob/db6ba4ba273c11a79ecbe940872f19d5cb19ac5e/src/common/cscs_api_common.py#L1110C18-L1110C65 + os.system(f"tar -xf '{source_path}' -C '{target_path}'") + + def checksum(self, machine: str, remote_path: str) -> int: + if not remote_path.exists(): + return False + # Firecrest uses sha256 + sha256_hash = hashlib.sha256() + with open(remote_path, "rb") as f: + for byte_block in iter(lambda: f.read(4096), b""): + sha256_hash.update(byte_block) + + return sha256_hash.hexdigest() + + def parameters( + self, + ): + # note: I took this from https://firecrest-tds.cscs.ch/ or https://firecrest.cscs.ch/ + # if code is not working but test passes, it means you need to update this dictionary + # with the latest FirecREST parameters + return { + "compute": [ + { + "description": "Type of resource and workload manager used in compute microservice", + "name": "WORKLOAD_MANAGER", + "unit": "", + "value": "Slurm", + } + ], + "storage": [ + { + "description": "Type of object storage, like `swift`, `s3v2` or `s3v4`.", + "name": "OBJECT_STORAGE", + "unit": "", + "value": "s3v4", + }, + { + "description": "Expiration time for temp URLs.", + "name": "STORAGE_TEMPURL_EXP_TIME", + "unit": "seconds", + "value": "86400", + }, + { + "description": "Maximum file size for temp URLs.", + "name": "STORAGE_MAX_FILE_SIZE", + "unit": "MB", + "value": "5120", + }, + { + "description": "Available filesystems through the API.", + "name": "FILESYSTEMS", + "unit": "", + "value": [ + { + "mounted": ["/project", "/store", "/scratch/snx3000tds"], + "system": "dom", + }, + { + "mounted": ["/project", "/store", "/capstor/scratch/cscs"], + "system": "pilatus", + }, + ], + }, + ], + "utilities": [ + { + "description": "The maximum allowable file size for various operations" + " of the utilities microservice", + "name": "UTILITIES_MAX_FILE_SIZE", + "unit": "MB", + "value": "5", + }, + { + "description": ( + "Maximum time duration for executing the commands " + "in the cluster for the utilities microservice." + ), + "name": "UTILITIES_TIMEOUT", + "unit": "seconds", + "value": "5", + }, + ], + } class MockClientCredentialsAuth: @@ -146,7 +429,6 @@ def firecrest_config( config_path: str | None = request.config.getoption("--firecrest-config") no_clean: bool = request.config.getoption("--firecrest-no-clean") # record_requests: bool = request.config.getoption("--firecrest-requests") - # record_requests: bool = request.config.getoption("--firecrest-requests") # TODO: record_requests is un-maintained after PR#36, and practically not used. # But let's keep it commented for future use, if needed. @@ -232,301 +514,3 @@ def firecrest_config( temp_directory=str(_temp_directory), api_version="2", ) - - -def submit( - machine: str, - script_str: str | None = None, - script_remote_path: str | None = None, - script_local_path: str | None = None, - local_file=False, -): - if local_file: - raise DeprecationWarning("local_file is not supported") - - if script_remote_path and not Path(script_remote_path).exists(): - raise FileNotFoundError(f"File {script_remote_path} does not exist") - job_id = random.randint(10000, 99999) - - # Filter out lines starting with '#SBATCH' - with open(script_remote_path) as file: - lines = file.readlines() - command = "".join([line for line in lines if not line.strip().startswith("#")]) - - # Make the dummy files - for line in lines: - if "--error" in line: - error_file = line.split("=")[1].strip() - (Path(script_remote_path).parent / error_file).touch() - elif "--output" in line: - output_file = line.split("=")[1].strip() - (Path(script_remote_path).parent / output_file).touch() - - # Execute the job, this is useful for test_calculation.py - if "aiida.in" in command: - # skip blank command like: '/bin/bash' - os.chdir(Path(script_remote_path).parent) - os.system(command) - - return {"jobid": job_id} - - -def poll_active(machine: str, jobs: list[str], page_number: int = 0): - response = [] - # 12 satets are defined in firecrest - states = [ - "TIMEOUT", - "SUSPENDED", - "PREEMPTED", - "CANCELLED", - "NODE_FAIL", - "PENDING", - "FAILED", - "RUNNING", - "CONFIGURING", - "QUEUED", - "COMPLETED", - "COMPLETING", - ] - for i in range(len(jobs)): - response.append( - { - "job_data_err": "", - "job_data_out": "", - "job_file": "somefile.sh", - "job_file_err": "somefile-stderr.txt", - "job_file_out": "somefile-stdout.txt", - "job_info_extra": "Job info returned successfully", - "jobid": f"{jobs[i]}", - "name": "aiida-45", - "nodelist": "nid00049", - "nodes": "1", - "partition": "normal", - "start_time": "0:03", - "state": states[i % 12], - "time": "2024-06-21T10:44:42", - "time_left": "29:57", - "user": "Prof. Wang", - } - ) - - return response[ - page_number - * Values._DEFAULT_PAGE_SIZE : (page_number + 1) - * Values._DEFAULT_PAGE_SIZE - ] - - -def whoami(machine: str): - assert machine == "MACHINE_NAME" - return "test_user" - - -def list_files( - machine: str, target_path: str, recursive: bool = False, show_hidden: bool = False -): - # this is mimiking the expected behaviour from the firecrest code. - - content_list = [] - for root, dirs, files in os.walk(target_path): - if not recursive and root != target_path: - continue - for name in dirs + files: - full_path = os.path.join(root, name) - relative_path = ( - Path(os.path.relpath(root, target_path)).joinpath(name).as_posix() - ) - if os.path.islink(full_path): - content_type = "l" - link_target = ( - os.readlink(full_path) if os.path.islink(full_path) else None - ) - elif os.path.isfile(full_path): - content_type = "-" - link_target = None - elif os.path.isdir(full_path): - content_type = "d" - link_target = None - else: - content_type = "NON" - link_target = None - permissions = stat.filemode(Path(full_path).lstat().st_mode)[1:] - if name.startswith(".") and not show_hidden: - continue - content_list.append( - { - "name": relative_path, - "type": content_type, - "link_target": link_target, - "permissions": permissions, - } - ) - - return content_list - - -def stat_(machine: str, targetpath: firecrest.path, dereference=True): - stats = os.stat( - targetpath, follow_symlinks=bool(dereference) if dereference else False - ) - return { - "ino": stats.st_ino, - "dev": stats.st_dev, - "nlink": stats.st_nlink, - "uid": stats.st_uid, - "gid": stats.st_gid, - "size": stats.st_size, - "atime": stats.st_atime, - "mtime": stats.st_mtime, - "ctime": stats.st_ctime, - } - - -def mkdir( - machine: str, target_path: str, p: bool = False, ignore_existing: bool = False -): - target = Path(target_path) - target.mkdir(exist_ok=ignore_existing, parents=p) - - -def simple_delete(machine: str, target_path: str): - if not Path(target_path).exists(): - raise FileNotFoundError(f"File or folder {target_path} does not exist") - if os.path.isdir(target_path): - shutil.rmtree(target_path) - else: - os.remove(target_path) - - -def symlink(machine: str, target_path: str, link_path: str): - # this is how firecrest does it - os.system(f"ln -s {target_path} {link_path}") - - -def simple_download(machine: str, remote_path: str, local_path: str): - # this procedure is complecated in firecrest, but I am simplifying it here - # we don't care about the details of the download, we just want to make sure - # that the aiida-firecrest code is calling the right functions at right time - if Path(remote_path).is_dir(): - raise IsADirectoryError(f"{remote_path} is a directory") - if not Path(remote_path).exists(): - raise FileNotFoundError(f"{remote_path} does not exist") - os.system(f"cp {remote_path} {local_path}") - - -def simple_upload( - machine: str, local_path: str, remote_path: str, file_name: str | None = None -): - # this procedure is complecated in firecrest, but I am simplifying it here - # we don't care about the details of the upload, we just want to make sure - # that the aiida-firecrest code is calling the right functions at right time - if Path(local_path).is_dir(): - raise IsADirectoryError(f"{local_path} is a directory") - if not Path(local_path).exists(): - raise FileNotFoundError(f"{local_path} does not exist") - if file_name: - remote_path = os.path.join(remote_path, file_name) - os.system(f"cp {local_path} {remote_path}") - - -def copy(machine: str, source_path: str, target_path: str): - # this is how firecrest does it - # https://github.com/eth-cscs/firecrest/blob/db6ba4ba273c11a79ecbe940872f19d5cb19ac5e/src/utilities/utilities.py#L451C1-L452C1 - os.system(f"cp --force -dR --preserve=all -- '{source_path}' '{target_path}'") - - -def compress( - machine: str, source_path: str, target_path: str, dereference: bool = True -): - # this is how firecrest does it - # https://github.com/eth-cscs/firecrest/blob/db6ba4ba273c11a79ecbe940872f19d5cb19ac5e/src/utilities/utilities.py#L460 - basedir = os.path.dirname(source_path) - file_path = os.path.basename(source_path) - deref = "--dereference" if dereference else "" - os.system(f"tar {deref} -czf '{target_path}' -C '{basedir}' '{file_path}'") - - -def extract(machine: str, source_path: str, target_path: str): - # this is how firecrest does it - # https://github.com/eth-cscs/firecrest/blob/db6ba4ba273c11a79ecbe940872f19d5cb19ac5e/src/common/cscs_api_common.py#L1110C18-L1110C65 - os.system(f"tar -xf '{source_path}' -C '{target_path}'") - - -def checksum(machine: str, remote_path: str) -> int: - if not remote_path.exists(): - return False - # Firecrest uses sha256 - sha256_hash = hashlib.sha256() - with open(remote_path, "rb") as f: - for byte_block in iter(lambda: f.read(4096), b""): - sha256_hash.update(byte_block) - - return sha256_hash.hexdigest() - - -def parameters(): - # note: I took this from https://firecrest-tds.cscs.ch/ or https://firecrest.cscs.ch/ - # if code is not working but test passes, it means you need to update this dictionary - # with the latest FirecREST parameters - return { - "compute": [ - { - "description": "Type of resource and workload manager used in compute microservice", - "name": "WORKLOAD_MANAGER", - "unit": "", - "value": "Slurm", - } - ], - "storage": [ - { - "description": "Type of object storage, like `swift`, `s3v2` or `s3v4`.", - "name": "OBJECT_STORAGE", - "unit": "", - "value": "s3v4", - }, - { - "description": "Expiration time for temp URLs.", - "name": "STORAGE_TEMPURL_EXP_TIME", - "unit": "seconds", - "value": "86400", - }, - { - "description": "Maximum file size for temp URLs.", - "name": "STORAGE_MAX_FILE_SIZE", - "unit": "MB", - "value": "5120", - }, - { - "description": "Available filesystems through the API.", - "name": "FILESYSTEMS", - "unit": "", - "value": [ - { - "mounted": ["/project", "/store", "/scratch/snx3000tds"], - "system": "dom", - }, - { - "mounted": ["/project", "/store", "/capstor/scratch/cscs"], - "system": "pilatus", - }, - ], - }, - ], - "utilities": [ - { - "description": "The maximum allowable file size for various operations of the utilities microservice", - "name": "UTILITIES_MAX_FILE_SIZE", - "unit": "MB", - "value": "5", - }, - { - "description": ( - "Maximum time duration for executing the commands " - "in the cluster for the utilities microservice." - ), - "name": "UTILITIES_TIMEOUT", - "unit": "seconds", - "value": "5", - }, - ], - } From 7254af527545eb38e09915aebd34364433b3c460 Mon Sep 17 00:00:00 2001 From: Ali Date: Tue, 30 Jul 2024 17:21:18 +0200 Subject: [PATCH 39/39] change from random to itertools --- tests/conftest.py | 5 +++-- tests/test_scheduler.py | 3 +-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 7e333b0..7d13b7f 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -2,10 +2,10 @@ from dataclasses import dataclass import hashlib +import itertools import json import os from pathlib import Path -import random import shutil import stat from typing import Any, Callable @@ -61,6 +61,7 @@ def __init__(self, firecrest_url, *args, **kwargs): self._firecrest_url = firecrest_url self.args = args self.kwargs = kwargs + self.job_id_generator = itertools.cycle(range(1000, 999999)) def submit( self, @@ -75,7 +76,7 @@ def submit( if script_remote_path and not Path(script_remote_path).exists(): raise FileNotFoundError(f"File {script_remote_path} does not exist") - job_id = random.randint(10000, 99999) + job_id = next(self.job_id_generator) # Filter out lines starting with '#SBATCH' with open(script_remote_path) as file: diff --git a/tests/test_scheduler.py b/tests/test_scheduler.py index b4cb26f..5939e60 100644 --- a/tests/test_scheduler.py +++ b/tests/test_scheduler.py @@ -1,5 +1,4 @@ from pathlib import Path -import random from aiida import orm from aiida.schedulers.datastructures import CodeRunMode, JobTemplate @@ -36,7 +35,7 @@ def test_get_jobs(firecrest_computer: orm.Computer): scheduler._DEFAULT_PAGE_SIZE = 2 Values._DEFAULT_PAGE_SIZE = 2 - joblist = [random.randint(10000, 99999) for i in range(5)] + joblist = ["111", "222", "333", "444", "555"] result = scheduler.get_jobs(joblist) assert len(result) == 5 for i in range(5):