diff --git a/setup.py b/setup.py index 05b6ef70b..5f90e791d 100644 --- a/setup.py +++ b/setup.py @@ -179,7 +179,7 @@ def has_ext_modules(_placeholder): "pydantic==1.10.14", "pyzmq>=25.1.2", "pygithub>=2.3.0", - "numpy<2" + "numpy<2", ] # Add SmartRedis at specific version @@ -203,7 +203,7 @@ def has_ext_modules(_placeholder): "types-tqdm", "types-tensorflow==2.12.0.9", "types-setuptools", - "typing_extensions>=4.1.0", + "typing_extensions>=4.1.0,<4.6", ], # see smartsim/_core/_install/buildenv.py for more details **versions.ml_extras_required(), diff --git a/smartsim/_core/entrypoints/file_operations.py b/smartsim/_core/entrypoints/file_operations.py new file mode 100644 index 000000000..c57192ea8 --- /dev/null +++ b/smartsim/_core/entrypoints/file_operations.py @@ -0,0 +1,262 @@ +# BSD 2-Clause License +# +# Copyright (c) 2021-2024 Hewlett Packard Enterprise +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +from __future__ import annotations + +import argparse +import base64 +import functools +import os +import pathlib +import pickle +import shutil +import typing as t +from typing import Callable + +from ...log import get_logger + +logger = get_logger(__name__) + +"""Run file operations move, remove, symlink, copy, and configure +using command line arguments. +""" + + +def _abspath(input_path: str) -> pathlib.Path: + """Helper function to check that paths are absolute""" + path = pathlib.Path(input_path) + if not path.is_absolute(): + raise ValueError(f"path `{path}` must be absolute") + return path + + +def _make_substitution( + tag_name: str, replacement: str | int | float, tag_delimiter: str +) -> Callable[[str], str]: + """Helper function to replace tags""" + return lambda s: s.replace( + f"{tag_delimiter}{tag_name}{tag_delimiter}", str(replacement) + ) + + +def _replace_tags_in( + item: str, + substitutions: t.Sequence[Callable[[str], str]], +) -> str: + """Helper function to derive the lines in which to make the substitutions""" + return functools.reduce(lambda a, fn: fn(a), substitutions, item) + + +def move(parsed_args: argparse.Namespace) -> None: + """Move a source file or directory to another location. If dest is an + existing directory or a symlink to a directory, then the srouce will + be moved inside that directory. The destination path in that directory + must not already exist. If dest is an existing file, it will be overwritten. + + Sample usage: + .. highlight:: bash + .. code-block:: bash + python -m smartsim._core.entrypoints.file_operations \ + move /absolute/file/source/path /absolute/file/dest/path + + /absolute/file/source/path: File or directory to be moved + /absolute/file/dest/path: Path to a file or directory location + """ + shutil.move(parsed_args.source, parsed_args.dest) + + +def remove(parsed_args: argparse.Namespace) -> None: + """Remove a file or directory. + + Sample usage: + .. highlight:: bash + .. code-block:: bash + python -m smartsim._core.entrypoints.file_operations \ + remove /absolute/file/path + + /absolute/file/path: Path to the file or directory to be deleted + """ + if os.path.isdir(parsed_args.to_remove): + os.rmdir(parsed_args.to_remove) + else: + os.remove(parsed_args.to_remove) + + +def copy(parsed_args: argparse.Namespace) -> None: + """Copy the contents from the source file into the dest file. + If source is a directory, copy the entire directory tree source to dest. + + Sample usage: + .. highlight:: bash + .. code-block:: bash + python -m smartsim._core.entrypoints.file_operations copy \ + /absolute/file/source/path /absolute/file/dest/path \ + --dirs_exist_ok + + /absolute/file/source/path: Path to directory, or path to file to + copy to a new location + /absolute/file/dest/path: Path to destination directory or path to + destination file + --dirs_exist_ok: if the flag is included, the copying operation will + continue if the destination directory and files alrady exist, + and will be overwritten by corresponding files. If the flag is + not includedm and the destination file already exists, a + FileExistsError will be raised + """ + if os.path.isdir(parsed_args.source): + shutil.copytree( + parsed_args.source, + parsed_args.dest, + dirs_exist_ok=parsed_args.dirs_exist_ok, + ) + else: + shutil.copyfile(parsed_args.source, parsed_args.dest) + + +def symlink(parsed_args: argparse.Namespace) -> None: + """ + Create a symbolic link pointing to the exisiting source file + named link. + + Sample usage: + .. highlight:: bash + .. code-block:: bash + python -m smartsim._core.entrypoints.file_operations \ + symlink /absolute/file/source/path /absolute/file/dest/path + + /absolute/file/source/path: the exisiting source path + /absolute/file/dest/path: target name where the symlink will be created. + """ + os.symlink(parsed_args.source, parsed_args.dest) + + +def configure(parsed_args: argparse.Namespace) -> None: + """Set, search and replace the tagged parameters for the + configure operation within tagged files attached to an entity. + + User-formatted files can be attached using the `configure` argument. + These files will be modified during ``Application`` generation to replace + tagged sections in the user-formatted files with values from the `params` + initializer argument used during ``Application`` creation: + + Sample usage: + .. highlight:: bash + .. code-block:: bash + python -m smartsim._core.entrypoints.file_operations \ + configure /absolute/file/source/pat /absolute/file/dest/path \ + tag_deliminator param_dict + + /absolute/file/source/path: The tagged files the search and replace operations + to be performed upon + /absolute/file/dest/path: The destination for configured files to be + written to. + tag_delimiter: tag for the configure operation to search for, defaults to + semi-colon e.g. ";" + param_dict: A dict of parameter names and values set for the file + + """ + tag_delimiter = parsed_args.tag_delimiter + + decoded_dict = base64.b64decode(parsed_args.param_dict) + param_dict = pickle.loads(decoded_dict) + + if not param_dict: + raise ValueError("param dictionary is empty") + if not isinstance(param_dict, dict): + raise TypeError("param dict is not a valid dictionary") + + substitutions = tuple( + _make_substitution(k, v, tag_delimiter) for k, v in param_dict.items() + ) + + # Set the lines to iterate over + with open(parsed_args.source, "r+", encoding="utf-8") as file_stream: + lines = [_replace_tags_in(line, substitutions) for line in file_stream] + + # write configured file to destination specified + with open(parsed_args.dest, "w+", encoding="utf-8") as file_stream: + file_stream.writelines(lines) + + +def get_parser() -> argparse.ArgumentParser: + """Instantiate a parser to process command line arguments + + :returns: An argument parser ready to accept required command generator parameters + """ + arg_parser = argparse.ArgumentParser(description="Command Generator") + + subparsers = arg_parser.add_subparsers(help="file_operations") + + # Subparser for move op + move_parser = subparsers.add_parser("move") + move_parser.set_defaults(func=move) + move_parser.add_argument("source", type=_abspath) + move_parser.add_argument("dest", type=_abspath) + + # Subparser for remove op + remove_parser = subparsers.add_parser("remove") + remove_parser.set_defaults(func=remove) + remove_parser.add_argument("to_remove", type=_abspath) + + # Subparser for copy op + copy_parser = subparsers.add_parser("copy") + copy_parser.set_defaults(func=copy) + copy_parser.add_argument("source", type=_abspath) + copy_parser.add_argument("dest", type=_abspath) + copy_parser.add_argument("--dirs_exist_ok", action="store_true") + + # Subparser for symlink op + symlink_parser = subparsers.add_parser("symlink") + symlink_parser.set_defaults(func=symlink) + symlink_parser.add_argument("source", type=_abspath) + symlink_parser.add_argument("dest", type=_abspath) + + # Subparser for configure op + configure_parser = subparsers.add_parser("configure") + configure_parser.set_defaults(func=configure) + configure_parser.add_argument("source", type=_abspath) + configure_parser.add_argument("dest", type=_abspath) + configure_parser.add_argument("tag_delimiter", type=str, default=";") + configure_parser.add_argument("param_dict", type=str) + + return arg_parser + + +def parse_arguments() -> argparse.Namespace: + """Parse the command line arguments + + :returns: the parsed command line arguments + """ + parser = get_parser() + parsed_args = parser.parse_args() + return parsed_args + + +if __name__ == "__main__": + os.environ["PYTHONUNBUFFERED"] = "1" + + args = parse_arguments() + args.func(args) diff --git a/tests/test_configs/generator_files/easy/correct/invalidtag.txt b/tests/test_configs/generator_files/easy/correct/invalidtag.txt new file mode 100644 index 000000000..2165ae8d1 --- /dev/null +++ b/tests/test_configs/generator_files/easy/correct/invalidtag.txt @@ -0,0 +1,3 @@ +some text before +some params are valid and others are ;INVALID; but we mostly encounter valid params +some text after diff --git a/tests/test_configs/generator_files/easy/marked/invalidtag.txt b/tests/test_configs/generator_files/easy/marked/invalidtag.txt new file mode 100644 index 000000000..90a625319 --- /dev/null +++ b/tests/test_configs/generator_files/easy/marked/invalidtag.txt @@ -0,0 +1,3 @@ +some text before +some params are ;VALID; and others are ;INVALID; but we mostly encounter ;VALID; params +some text after diff --git a/tests/test_file_operations.py b/tests/test_file_operations.py new file mode 100644 index 000000000..6f516a569 --- /dev/null +++ b/tests/test_file_operations.py @@ -0,0 +1,681 @@ +# BSD 2-Clause License +# +# Copyright (c) 2021-2024, Hewlett Packard Enterprise +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import argparse +import base64 +import filecmp +import os +import pathlib +import pickle +import shutil +from glob import glob +from os import path as osp + +import pytest + +from smartsim._core.entrypoints import file_operations +from smartsim._core.entrypoints.file_operations import get_parser + +pytestmark = pytest.mark.group_a + + +def test_symlink_files(test_dir): + """ + Test operation to symlink files + """ + # Set source directory and file + source = pathlib.Path(test_dir) / "sym_source" + os.mkdir(source) + source_file = source / "sym_source.txt" + with open(source_file, "w+", encoding="utf-8") as dummy_file: + dummy_file.write("dummy") + + # Set path to be the destination directory + entity_path = os.path.join(test_dir, "entity_name") + + parser = get_parser() + cmd = f"symlink {source_file} {entity_path}" + args = cmd.split() + ns = parser.parse_args(args) + + file_operations.symlink(ns) + + # Assert the two files are the same file + link = pathlib.Path(test_dir) / "entity_name" + assert link.is_symlink() + assert os.readlink(link) == str(source_file) + + # Clean up the test directory + os.unlink(link) + os.remove(pathlib.Path(source) / "sym_source.txt") + os.rmdir(pathlib.Path(test_dir) / "sym_source") + + +def test_symlink_dir(test_dir): + """ + Test operation to symlink directories + """ + + source = pathlib.Path(test_dir) / "sym_source" + os.mkdir(source) + + # entity_path to be the dest dir + entity_path = os.path.join(test_dir, "entity_name") + + parser = get_parser() + cmd = f"symlink {source} {entity_path}" + args = cmd.split() + ns = parser.parse_args(args) + + file_operations.symlink(ns) + + link = pathlib.Path(test_dir) / "entity_name" + # Assert the two files are the same file + assert link.is_symlink() + assert os.readlink(link) == str(source) + + # Clean up the test directory + os.unlink(link) + os.rmdir(pathlib.Path(test_dir) / "sym_source") + + +def test_symlink_not_absolute(test_dir): + """Test that ValueError is raised when a relative path + is given to the symlink operation + """ + # Set source directory and file + source = pathlib.Path(test_dir) / "sym_source" + os.mkdir(source) + source_file = source / "sym_source.txt" + with open(source_file, "w+", encoding="utf-8") as dummy_file: + dummy_file.write("dummy") + + # Set path to be the destination directory + entity_path = ".." + + parser = get_parser() + cmd = f"symlink {source_file} {entity_path}" + args = cmd.split() + + with pytest.raises(SystemExit) as e: + parser.parse_args(args) + + assert isinstance(e.value.__context__, argparse.ArgumentError) + assert "invalid _abspath value" in e.value.__context__.message + + +def test_copy_op_file(test_dir): + """Test the operation to copy the content of the source file to the destination path + with an empty file of the same name already in the directory""" + + to_copy = os.path.join(test_dir, "to_copy") + os.mkdir(to_copy) + + source_file = pathlib.Path(to_copy) / "copy_file.txt" + with open(source_file, "w+", encoding="utf-8") as dummy_file: + dummy_file.write("dummy") + + entity_path = os.path.join(test_dir, "entity_name") + os.mkdir(entity_path) + + dest_file = os.path.join(test_dir, "entity_name", "copy_file.txt") + with open(dest_file, "w+", encoding="utf-8") as dummy_file: + dummy_file.write("") + + parser = get_parser() + cmd = f"copy {source_file} {dest_file}" + args = cmd.split() + ns = parser.parse_args(args) + + # Execute copy + file_operations.copy(ns) + + # Assert files were copied over + with open(dest_file, "r", encoding="utf-8") as dummy_file: + assert dummy_file.read() == "dummy" + + # Clean up + os.remove(pathlib.Path(to_copy) / "copy_file.txt") + os.rmdir(pathlib.Path(test_dir) / "to_copy") + + os.remove(pathlib.Path(entity_path) / "copy_file.txt") + os.rmdir(pathlib.Path(test_dir) / "entity_name") + + +def test_copy_op_dirs(test_dir): + """Test the operation that copies an entire directory tree source to a new location destination + that already exists""" + + to_copy = os.path.join(test_dir, "to_copy") + os.mkdir(to_copy) + + # write some test files in the dir + source_file = pathlib.Path(to_copy) / "copy_file.txt" + with open(source_file, "w+", encoding="utf-8") as dummy_file: + dummy_file.write("dummy1") + + source_file_2 = pathlib.Path(to_copy) / "copy_file_2.txt" + with open(source_file_2, "w+", encoding="utf-8") as dummy_file: + dummy_file.write("dummy2") + + # entity_path to be the dest dir + entity_path = os.path.join(test_dir, "entity_name") + os.mkdir(entity_path) + + parser = get_parser() + cmd = f"copy {to_copy} {entity_path} --dirs_exist_ok" + args = cmd.split() + ns = parser.parse_args(args) + + # Execute copy + file_operations.copy(ns) + + # Assert dirs were copied over + entity_files_1 = pathlib.Path(entity_path) / "copy_file.txt" + with open(entity_files_1, "r", encoding="utf-8") as dummy_file: + assert dummy_file.read() == "dummy1" + + entity_files_2 = pathlib.Path(entity_path) / "copy_file_2.txt" + with open(entity_files_2, "r", encoding="utf-8") as dummy_file: + assert dummy_file.read() == "dummy2" + + # Clean up + os.remove(pathlib.Path(to_copy) / "copy_file.txt") + os.remove(pathlib.Path(to_copy) / "copy_file_2.txt") + os.rmdir(pathlib.Path(test_dir) / "to_copy") + os.remove(pathlib.Path(entity_path) / "copy_file.txt") + os.remove(pathlib.Path(entity_path) / "copy_file_2.txt") + os.rmdir(pathlib.Path(test_dir) / "entity_name") + + +def test_copy_op_dirs_file_exists_error(test_dir): + """Test that a FileExistsError is raised when copying a directory tree source to a new location destination + when the destination already exists, and the flag --dirs_exist_ok is not included + """ + + to_copy = os.path.join(test_dir, "to_copy") + os.mkdir(to_copy) + + # write some test files in the dir + source_file = pathlib.Path(to_copy) / "copy_file.txt" + with open(source_file, "w+", encoding="utf-8") as dummy_file: + dummy_file.write("dummy1") + + source_file_2 = pathlib.Path(to_copy) / "copy_file_2.txt" + with open(source_file_2, "w+", encoding="utf-8") as dummy_file: + dummy_file.write("dummy2") + + # entity_path to be the dest dir + entity_path = os.path.join(test_dir, "entity_name") + os.mkdir(entity_path) + + parser = get_parser() + # command does not include the --dirs_exist_ok flag + cmd = f"copy {to_copy} {entity_path}" + args = cmd.split() + ns = parser.parse_args(args) + + # Execute copy + with pytest.raises(FileExistsError) as ex: + file_operations.copy(ns) + assert f"File exists" in ex.value.args + + # Clean up + os.remove(pathlib.Path(to_copy) / "copy_file.txt") + os.remove(pathlib.Path(to_copy) / "copy_file_2.txt") + os.rmdir(pathlib.Path(test_dir) / "to_copy") + os.rmdir(pathlib.Path(test_dir) / "entity_name") + + +def test_copy_op_bad_source_file(test_dir): + """Test that a FileNotFoundError is raised when there is a bad source file""" + + to_copy = os.path.join(test_dir, "to_copy") + os.mkdir(to_copy) + entity_path = os.path.join(test_dir, "entity_name") + os.mkdir(entity_path) + + bad_path = "/not/a/real/path" + # Execute copy + + parser = get_parser() + cmd = f"copy {bad_path} {entity_path}" + args = cmd.split() + ns = parser.parse_args(args) + + with pytest.raises(FileNotFoundError) as ex: + file_operations.copy(ns) + assert "No such file or directory" in ex.value.args + + # Clean up + os.rmdir(pathlib.Path(test_dir) / "to_copy") + os.rmdir(pathlib.Path(test_dir) / "entity_name") + + +def test_copy_op_bad_dest_path(test_dir): + """Test that a FileNotFoundError is raised when there is a bad destination file.""" + + to_copy = os.path.join(test_dir, "to_copy") + os.mkdir(to_copy) + + source_file = pathlib.Path(to_copy) / "copy_file.txt" + with open(source_file, "w+", encoding="utf-8") as dummy_file: + dummy_file.write("dummy1") + entity_path = os.path.join(test_dir, "entity_name") + os.mkdir(entity_path) + + bad_path = "/not/a/real/path" + + parser = get_parser() + cmd = f"copy {source_file} {bad_path}" + args = cmd.split() + ns = parser.parse_args(args) + + with pytest.raises(FileNotFoundError) as ex: + file_operations.copy(ns) + assert "No such file or directory" in ex.value.args + + # clean up + os.remove(pathlib.Path(to_copy) / "copy_file.txt") + os.rmdir(pathlib.Path(test_dir) / "to_copy") + os.rmdir(pathlib.Path(test_dir) / "entity_name") + + +def test_copy_not_absolute(test_dir): + + to_copy = os.path.join(test_dir, "to_copy") + os.mkdir(to_copy) + + source_file = pathlib.Path(to_copy) / "copy_file.txt" + with open(source_file, "w+", encoding="utf-8") as dummy_file: + dummy_file.write("dummy1") + entity_path = os.path.join(test_dir, "entity_name") + os.mkdir(entity_path) + + bad_path = ".." + + parser = get_parser() + cmd = f"copy {source_file} {bad_path}" + args = cmd.split() + + with pytest.raises(SystemExit) as e: + parser.parse_args(args) + + assert isinstance(e.value.__context__, argparse.ArgumentError) + assert "invalid _abspath value" in e.value.__context__.message + + # clean up + os.remove(pathlib.Path(to_copy) / "copy_file.txt") + os.rmdir(pathlib.Path(test_dir) / "to_copy") + os.rmdir(pathlib.Path(test_dir) / "entity_name") + + +def test_move_op(test_dir): + """Test the operation to move a file""" + + source_dir = os.path.join(test_dir, "from_here") + os.mkdir(source_dir) + dest_dir = os.path.join(test_dir, "to_here") + os.mkdir(dest_dir) + + dest_file = pathlib.Path(dest_dir) / "to_here.txt" + with open(dest_file, "w+", encoding="utf-8") as dummy_file: + dummy_file.write(" ") + + source_file = pathlib.Path(source_dir) / "app_move.txt" + with open(source_file, "w+", encoding="utf-8") as dummy_file: + dummy_file.write("dummy") + + assert osp.exists(source_file) + with open(source_file, "r", encoding="utf-8") as dummy_file: + assert dummy_file.read() == "dummy" + + parser = get_parser() + cmd = f"move {source_file} {dest_file}" + args = cmd.split() + ns = parser.parse_args(args) + + file_operations.move(ns) + + # Assert that the move was successful + assert not osp.exists(source_file) + assert osp.exists(dest_file) + with open(dest_file, "r", encoding="utf-8") as dummy_file: + assert dummy_file.read() == "dummy" + + # Clean up the directories + os.rmdir(source_dir) + os.remove(dest_file) + os.rmdir(dest_dir) + + +def test_move_not_absolute(test_dir): + """Test that a ValueError is raised when a relative + path is given to the move operation""" + + source_dir = os.path.join(test_dir, "from_here") + os.mkdir(source_dir) + dest_dir = os.path.join(test_dir, "to_here") + os.mkdir(dest_dir) + + dest_file = ".." + + source_file = pathlib.Path(source_dir) / "app_move.txt" + with open(source_file, "w+", encoding="utf-8") as dummy_file: + dummy_file.write("dummy") + + parser = get_parser() + cmd = f"move {source_file} {dest_file}" + args = cmd.split() + + with pytest.raises(SystemExit) as e: + parser.parse_args(args) + + assert isinstance(e.value.__context__, argparse.ArgumentError) + assert "invalid _abspath value" in e.value.__context__.message + + +def test_remove_op_file(test_dir): + """Test the operation to delete a file""" + + # Make a test file with dummy text + to_del = pathlib.Path(test_dir) / "app_del.txt" + with open(to_del, "w+", encoding="utf-8") as dummy_file: + dummy_file.write("dummy") + + assert osp.exists(to_del) + with open(to_del, "r", encoding="utf-8") as dummy_file: + assert dummy_file.read() == "dummy" + + parser = get_parser() + cmd = f"remove {to_del}" + args = cmd.split() + ns = parser.parse_args(args) + + file_operations.remove(ns) + + # Assert file has been deleted + assert not osp.exists(to_del) + + +def test_remove_op_dir(test_dir): + """Test the operation to delete a directory""" + + # Make a test file with dummy text + to_del = pathlib.Path(test_dir) / "dir_del" + os.mkdir(to_del) + + parser = get_parser() + cmd = f"remove {to_del}" + args = cmd.split() + ns = parser.parse_args(args) + + file_operations.remove(ns) + + # Assert directory has been deleted + assert not osp.exists(to_del) + + +def test_remove_op_bad_path(test_dir): + """Test that FileNotFoundError is raised when a bad path is given to the + operation to delete a file""" + + to_del = pathlib.Path(test_dir) / "not_real.txt" + + parser = get_parser() + cmd = f"remove {to_del}" + args = cmd.split() + ns = parser.parse_args(args) + + with pytest.raises(FileNotFoundError) as ex: + file_operations.remove(ns) + assert "No such file or directory" in ex.value.args + + +def test_remove_op_not_absolute(): + """Test that ValueError is raised when a relative path + is given to the operation to delete a file""" + + to_del = ".." + + parser = get_parser() + cmd = f"remove {to_del}" + args = cmd.split() + + with pytest.raises(SystemExit) as e: + parser.parse_args(args) + + assert isinstance(e.value.__context__, argparse.ArgumentError) + assert "invalid _abspath value" in e.value.__context__.message + + +@pytest.mark.parametrize( + ["param_dict", "error_type"], + [ + pytest.param( + { + "5": 10, + "FIRST": "SECOND", + "17": 20, + "65": "70", + "placeholder": "group leftupper region", + "1200": "120", + "VALID": "valid", + }, + "None", + id="correct dict", + ), + pytest.param( + ["list", "of", "values"], + "TypeError", + id="incorrect dict", + ), + pytest.param({}, "ValueError", id="empty dict"), + ], +) +def test_configure_op(test_dir, fileutils, param_dict, error_type): + """Test configure operation with correct parameter dictionary, empty dicitonary, and an incorrect type""" + + tag = ";" + + conf_path = fileutils.get_test_conf_path( + osp.join("generator_files", "easy", "marked/") + ) + # retrieve files to compare after test + correct_path = fileutils.get_test_conf_path( + osp.join("generator_files", "easy", "correct/") + ) + + # copy files to test directory + shutil.copytree(conf_path, test_dir, dirs_exist_ok=True) + assert osp.isdir(test_dir) + + tagged_files = sorted(glob(test_dir + "/*")) + correct_files = sorted(glob(correct_path + "/*")) + + # Pickle the dictionary + pickled_dict = pickle.dumps(param_dict) + + # Encode the pickled dictionary with Base64 + encoded_dict = base64.b64encode(pickled_dict).decode("ascii") + + # Run configure op on test files + for tagged_file in tagged_files: + parser = get_parser() + cmd = f"configure {tagged_file} {tagged_file} {tag} {encoded_dict}" + args = cmd.split() + ns = parser.parse_args(args) + + if error_type == "ValueError": + with pytest.raises(ValueError) as ex: + file_operations.configure(ns) + assert "param dictionary is empty" in ex.value.args[0] + elif error_type == "TypeError": + with pytest.raises(TypeError) as ex: + file_operations.configure(ns) + assert "param dict is not a valid dictionary" in ex.value.args[0] + else: + file_operations.configure(ns) + + if error_type == "None": + for written, correct in zip(tagged_files, correct_files): + assert filecmp.cmp(written, correct) + + +def test_configure_invalid_tags(fileutils): + """Test configure operation with an invalid tag""" + tagged_file = fileutils.get_test_conf_path( + osp.join("generator_files", "easy", "marked", "invalidtag.txt") + ) + + tag = ";" + param_dict = {"VALID": "valid"} + + # Pickle the dictionary + pickled_dict = pickle.dumps(param_dict) + + # Encode the pickled dictionary with Base64 + encoded_dict = base64.b64encode(pickled_dict).decode("ascii") + parser = get_parser() + cmd = f"configure {tagged_file} {tagged_file} {tag} {encoded_dict}" + args = cmd.split() + ns = parser.parse_args(args) + + file_operations.configure(ns) + + +def test_configure_not_absolute(): + """Test that ValueError is raised when tagged files + given to configure op are not absolute paths + """ + + tagged_file = ".." + tag = ";" + param_dict = {"5": 10} + # Pickle the dictionary + pickled_dict = pickle.dumps(param_dict) + + # Encode the pickled dictionary with Base64 + encoded_dict = base64.b64encode(pickled_dict) + parser = get_parser() + cmd = f"configure {tagged_file} {tagged_file} {tag} {encoded_dict}" + args = cmd.split() + + with pytest.raises(SystemExit) as e: + parser.parse_args(args) + + assert isinstance(e.value.__context__, argparse.ArgumentError) + assert "invalid _abspath value" in e.value.__context__.message + + +def test_parser_move(): + """Test that the parser succeeds when receiving expected args for the move operation""" + parser = get_parser() + + src_path = pathlib.Path("/absolute/file/src/path") + dest_path = pathlib.Path("/absolute/file/dest/path") + + cmd = f"move {src_path} {dest_path}" + args = cmd.split() + ns = parser.parse_args(args) + + assert ns.source == src_path + assert ns.dest == dest_path + + +def test_parser_remove(): + """Test that the parser succeeds when receiving expected args for the remove operation""" + parser = get_parser() + + file_path = pathlib.Path("/absolute/file/path") + cmd = f"remove {file_path}" + + args = cmd.split() + ns = parser.parse_args(args) + + assert ns.to_remove == file_path + + +def test_parser_symlink(): + """Test that the parser succeeds when receiving expected args for the symlink operation""" + parser = get_parser() + + src_path = pathlib.Path("/absolute/file/src/path") + dest_path = pathlib.Path("/absolute/file/dest/path") + cmd = f"symlink {src_path} {dest_path}" + + args = cmd.split() + + ns = parser.parse_args(args) + + assert ns.source == src_path + assert ns.dest == dest_path + + +def test_parser_copy(): + """Test that the parser succeeds when receiving expected args for the copy operation""" + parser = get_parser() + + src_path = pathlib.Path("/absolute/file/src/path") + dest_path = pathlib.Path("/absolute/file/dest/path") + + cmd = f"copy {src_path} {dest_path}" + + args = cmd.split() + ns = parser.parse_args(args) + + assert ns.source == src_path + assert ns.dest == dest_path + + +def test_parser_configure_parse(): + """Test that the parser succeeds when receiving expected args for the configure operation""" + parser = get_parser() + + src_path = pathlib.Path("/absolute/file/src/path") + dest_path = pathlib.Path("/absolute/file/dest/path") + tag_delimiter = ";" + + param_dict = { + "5": 10, + "FIRST": "SECOND", + "17": 20, + "65": "70", + "placeholder": "group leftupper region", + "1200": "120", + } + + pickled_dict = pickle.dumps(param_dict) + encoded_dict = base64.b64encode(pickled_dict) + + cmd = f"configure {src_path} {dest_path} {tag_delimiter} {encoded_dict}" + args = cmd.split() + ns = parser.parse_args(args) + + assert ns.source == src_path + assert ns.dest == dest_path + assert ns.tag_delimiter == tag_delimiter + assert ns.param_dict == str(encoded_dict)