From 1cbde0e83683f66ed90e4a4966ee2295ce6c57cb Mon Sep 17 00:00:00 2001 From: Christopher Woods Date: Mon, 24 May 2021 18:14:55 +0100 Subject: [PATCH 1/4] Fixed the bug where a list on the first line of the user variable file was mistakenly read as a CSV. Can now have lists as variables on any line, plus have added code to detect and interpret lists at input --- src/metawards/_interpret.py | 13 +++++++++++++ src/metawards/_variableset.py | 35 +++++++++++++++++++++++++++++++++-- tests/data/vertical3.dat | 1 + tests/test_read_variables.py | 12 ++++++++++++ 4 files changed, 59 insertions(+), 2 deletions(-) create mode 100644 tests/data/vertical3.dat diff --git a/src/metawards/_interpret.py b/src/metawards/_interpret.py index 49e4b5133..27b4bad67 100644 --- a/src/metawards/_interpret.py +++ b/src/metawards/_interpret.py @@ -159,6 +159,19 @@ def random_number(s: str = None, rng=None, minval: float = None, from .utils._ran_binomial import ran_uniform return rmin + ((rmax - rmin) * ran_uniform(rng)) + @staticmethod + def list(s: any): + """Interpret and return a list from the passed string 's'""" + try: + import ast + my_list = [] + for val in ast.literal_eval(s): + my_list.append(val) + + return my_list + except Exception: + raise ValueError(f"Cannot interpret a list from {s}") + @staticmethod def integer(s: any, rng=None, minval: int = None, maxval: int = None) -> int: diff --git a/src/metawards/_variableset.py b/src/metawards/_variableset.py index b8428b8b8..3ab5f19a2 100644 --- a/src/metawards/_variableset.py +++ b/src/metawards/_variableset.py @@ -227,7 +227,6 @@ def _interpret(value): elif canonical.startswith("i'"): # this is an integer return Interpret.integer(value[2:-1]) - elif canonical.startswith("s'"): # this is a string return Interpret.string(value[2:-1]) @@ -236,6 +235,12 @@ def _interpret(value): # this is a boolean return Interpret.boolean(value[2:-1]) + elif canonical.startswith("[") and canonical.endswith("]"): + return Interpret.list(value) + + elif canonical.startswith("(") and canonical.endswith(")"): + return Interpret.list(value) + # now we have to guess... try: v = float(value) @@ -778,6 +783,9 @@ def create_fingerprint(vals: _List[float], index: int = None, v = "T" else: v = "F" + elif isinstance(val, list): + v = [str(x) for x in val] + v = "|".join(v) else: v = float(val) @@ -1386,7 +1394,30 @@ def read(filename: str, line_numbers: _List[int] = None): # there is nothing to read? return VariableSets() - if len(lines[0]) > 1 and len(lines[0]) > 1 and lines[0][1] == "==": + if lines[0][0].find("=") != -1 or lines[0][0].find(":") != -1: + # this is a vertical file that should be split on spaces + lines = [] + for line in csvlines: + cleaned = [_clean(x) for x in line.split()] + + line = [] + + if len(cleaned) > 0: + for clean in cleaned: + if isinstance(clean, list) or isinstance(clean, tuple): + for c in clean: + if len(c) > 0: + line.append(c) + else: + line.append(clean) + + lines.append(line) + + if len(lines) == 0: + # there is nothing to read? + return VariableSets() + + if len(lines[0]) > 1 and lines[0][1] == "==": # this is a vertical file if line_numbers is not None: raise ValueError( diff --git a/tests/data/vertical3.dat b/tests/data/vertical3.dat new file mode 100644 index 000000000..6d990c5c9 --- /dev/null +++ b/tests/data/vertical3.dat @@ -0,0 +1 @@ +.myvar = ["cat", 42, 3.141] diff --git a/tests/test_read_variables.py b/tests/test_read_variables.py index 354b2b617..7f1fce0fb 100644 --- a/tests/test_read_variables.py +++ b/tests/test_read_variables.py @@ -261,6 +261,17 @@ def test_read_edgecase(): assert v[".date2"] == "five days ago" +def test_read_edgecase2(): + vertical3 = os.path.join(script_dir, "data", "vertical3.dat") + v = VariableSet.read(vertical3) + + print(v[".myvar"]) + + assert v[".myvar"][0] == "cat" + assert v[".myvar"][1] == 42 + assert v[".myvar"][2] == 3.141 + + if __name__ == "__main__": test_variableset() test_parameterset() @@ -268,3 +279,4 @@ def test_read_edgecase(): test_set_variables() test_set_custom() test_read_edgecase() + test_read_edgecase2() From a4c5e65099d77e71604c28dcbd6fc7cb275140e3 Mon Sep 17 00:00:00 2001 From: Christopher Woods Date: Mon, 24 May 2021 18:25:18 +0100 Subject: [PATCH 2/4] Added more detail to the docs so that people know that they have to install MetaWardsData. --- doc/source/install.rst | 10 +++++++--- doc/source/quickstart/01_R.rst | 6 ++++++ doc/source/quickstart/01_console.rst | 6 ++++++ doc/source/quickstart/01_python.rst | 6 ++++++ doc/source/quickstart/index.rst | 5 ++++- 5 files changed, 29 insertions(+), 4 deletions(-) diff --git a/doc/source/install.rst b/doc/source/install.rst index dcc198324..9c9f26304 100644 --- a/doc/source/install.rst +++ b/doc/source/install.rst @@ -4,7 +4,9 @@ Installation instructions MetaWards is a Python package, but it does come with R wrappers. This means that you can choose to install MetaWards either via Python or -via R. +via R. Note that MetaWards depends on data files that are +held in MetaWardsData. To use MetaWards, you must install MetaWardsData +following :doc:`these instructions `. Installation via Python ======================= @@ -85,7 +87,8 @@ Once installed, you can run `metawards` by typing metawards --version -This should print out some version information about MetaWards. +This should print out some version information about MetaWards, +including whether or not it found MetaWardsData. If this doesn't work, then it is possible that the directory into which `metawards` has been placed is not in your PATH (this is quite @@ -114,7 +117,8 @@ by default in R is too old to run MetaWards (MetaWards needs Python 3.7 or newer, while the default in reticulate is to install and use Python 3.6). -Once you have MetaWards installed in Python, you first need to +Once you have MetaWards (and MetaWardsData) +installed in Python, you first need to get the reticulate command that you will need to tell R which Python interpreter to use. We have written a function to do this for you. Open Python and type; diff --git a/doc/source/quickstart/01_R.rst b/doc/source/quickstart/01_R.rst index e256ea841..26df58236 100644 --- a/doc/source/quickstart/01_R.rst +++ b/doc/source/quickstart/01_R.rst @@ -18,6 +18,12 @@ the case by typing; If you don't see ``TRUE`` returned, then double-check your installation. +.. note:: + + You also need to have installed MetaWardsData. If you have + not installed MetaWardsData then you need to install it by + following :doc:`these instructions <../model_data>`. + Now make sure that you have installed the `tidyverse `__, e.g. via; diff --git a/doc/source/quickstart/01_console.rst b/doc/source/quickstart/01_console.rst index 71835f475..74deeef64 100644 --- a/doc/source/quickstart/01_console.rst +++ b/doc/source/quickstart/01_console.rst @@ -15,6 +15,12 @@ type; You should see that MetaWards version information is printed to the screen. If not, then you need to :doc:`install MetaWards <../install>`. +.. note:: + + The output should show that MetaWardsData has been found. If you have + not installed MetaWardsData then you need to install it by + following :doc:`these instructions <../model_data>`. + You also need to be able to use a text editor, e.g. notepad, vim, emacs, nano or pico. This quick start will use nano. Please use the editor that you prefer. diff --git a/doc/source/quickstart/01_python.rst b/doc/source/quickstart/01_python.rst index 36e1263e8..74ebaff1d 100644 --- a/doc/source/quickstart/01_python.rst +++ b/doc/source/quickstart/01_python.rst @@ -28,6 +28,12 @@ analysing and plotting the results. >>> import pandas as pd +.. note:: + + You also need to have installed MetaWardsData. If you have + not installed MetaWardsData then you need to install it by + following :doc:`these instructions <../model_data>`. + Importing metawards ------------------- diff --git a/doc/source/quickstart/index.rst b/doc/source/quickstart/index.rst index f6de116c3..84dd19249 100644 --- a/doc/source/quickstart/index.rst +++ b/doc/source/quickstart/index.rst @@ -6,7 +6,10 @@ Quick Start Guide Please make sure you have installed ``metawards``, e.g. by following these :doc:`installation instructions <../install>`. You can test this by - typing ``metawards --version`` on the terminal/console. + typing ``metawards --version`` on the terminal/console. Note that this + should show that MetaWardsData has been found. If you have + not installed MetaWardsData then you need to install it by + following :doc:`these instructions <../model_data>`. You have three choices for how you use MetaWards, and thus three choices for how you will follow this quick start guide; From 2190a14c8f2003a6dfb83c75600e47478d8f9e56 Mon Sep 17 00:00:00 2001 From: Christopher Woods Date: Tue, 25 May 2021 11:06:08 +0100 Subject: [PATCH 3/4] Fixed the problem of windows hanging when something is written to stderr (i.e. an exception). Also fixed the issue on windows where the additional seeds caused problems because they were written to the command line using single quotes. Needed to have double quotes. --- src/metawards/_run.py | 1414 +++++++++-------- .../iterators/_advance_additional.py | 2 +- 2 files changed, 710 insertions(+), 706 deletions(-) diff --git a/src/metawards/_run.py b/src/metawards/_run.py index 864f6faaf..1a91d9cda 100644 --- a/src/metawards/_run.py +++ b/src/metawards/_run.py @@ -1,705 +1,709 @@ -from __future__ import annotations - -from typing import Union as _Union -from typing import List as _List -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from ._disease import Disease - from ._demographics import Demographics - from ._demographic import Demographic - from ._parameters import Parameters - from ._wards import Wards - from ._ward import Ward - from ._variableset import VariableSet, VariableSets - - from datetime import date - - -__all__ = ["run", "find_mw_exe", "find_mw_include", - "get_reticulate_command"] - - -def _write_to_file(obj: any, filename: str, dir: str = ".", bzip: bool = False, - dry_run: bool = False) -> str: - """Write the passed object to a file called 'filename' in - directory 'dir', returning the - relative path to that file - """ - import os - - if dry_run: - return filename - - filename = os.path.join(dir, filename) - - if hasattr(obj, "to_json"): - return obj.to_json(filename, auto_bzip=bzip) - else: - raise IOError(f"Cannot convert {obj} to a file!") - - return filename - - -def _rmdir(directory): - """Function modified from one copied from 'mitch' on stackoverflow - https://stackoverflow.com/questions/13118029/deleting-folders-in-python-recursively - """ - if directory is None: - return - - from pathlib import Path - directory = Path(directory) - - # first, check for removing important directories such as $HOME or root - if directory == Path.home(): - raise FileExistsError(f"We WILL NOT remove your " - f"home directory ${directory}") - - if directory == Path("/"): - raise FileExistsError(f"We WILL NOT remove the root directory " - f"{directory}") - - # get the directory containing '$HOME' - if directory == Path.home().parent: - raise FileExistsError(f"We WILL NOT remove the users/home " - f"directory {directory}") - - if not directory.is_dir(): - directory.unlink() - return - - for item in directory.iterdir(): - if item.is_dir(): - _rmdir(item) - else: - item.unlink() - - directory.rmdir() - - -def _is_executable(filename): - import os - if not os.path.exists(filename): - return None - - if os.path.isdir(filename): - return None - - # determining if this is executable - # on windows is really difficult, so just - # assume it is... - return filename - - -def _find_metawards(dirname): - import os - m = _is_executable(os.path.join(dirname, "metawards")) - - if m: - return m - - m = _is_executable(os.path.join(dirname, "metawards.exe")) - - if m: - return m - - m = _is_executable(os.path.join(dirname, "Scripts", "metawards")) - - if m: - return m - - m = _is_executable(os.path.join(dirname, "Scripts", "metawards.exe")) - - if m: - return m - - m = _is_executable(os.path.join(dirname, "bin", "metawards")) - - if m: - return m - - m = _is_executable(os.path.join(dirname, "bin", "metawards.exe")) - - if m: - return m - - return None - - -def _find_metawards_include(dirname): - import os - - # this is from a metawards installation - m = os.path.abspath(os.path.join(dirname, "include", "metawards")) - - if os.path.exists(m): - return m - - # this is from a metawards source run (used for testing) - m = os.path.abspath(os.path.join(dirname, "src", "metawards")) - - if os.path.exists(m): - return m - - return None - - -def find_mw_include(): - """Try to find the directory containing the MetaWards include files. - This raises an exception if the include files cannot be found. - It returns the full path to the include files - """ - import metawards as _metawards - import os as _os - - # Search through the path based on where the metawards module - # has been installed. - modpath = _metawards.__file__ - - metawards = None - - # Loop only 100 times - this should break before now, - # We are not using a while loop to avoid an infinite loop - for i in range(0, 100): - metawards = _find_metawards_include(modpath) - - if metawards: - break - - newpath = _os.path.dirname(modpath) - - if newpath == modpath: - break - - modpath = newpath - - if metawards is None: - from .utils._console import Console - Console.error( - "Cannot find the metawards include directory, when starting from " - f"{_metawards.__file__}. Please could you " - "find it and then post an issue on the " - "GitHub repository (https://github.com/metawards/MetaWards) " - "as this may indicate a bug in the code.") - raise RuntimeError("Cannot locate the metawards include directory") - - return metawards - - -def find_mw_exe(): - """Try to find the MetaWards executable. This should be findable - if MetaWards has been installed. This raises an exception - if it cannot be found. It returns the full path to the - executable - """ - import metawards as _metawards - import os as _os - import sys as _sys - - # Search through the path based on where the metawards module - # has been installed. - modpath = _metawards.__file__ - - metawards = None - - # Loop only 100 times - this should break before now, - # We are not using a while loop to avoid an infinite loop - for i in range(0, 100): - metawards = _find_metawards(modpath) - - if metawards: - break - - newpath = _os.path.dirname(modpath) - - if newpath == modpath: - break - - modpath = newpath - - if metawards is None: - # We couldn't find it that way - try another route... - dirpath = _os.path.join(_os.path.dirname(_sys.executable)) - - for option in [_os.path.join(dirpath, "metawards.exe"), - _os.path.join(dirpath, "metawards"), - _os.path.join(dirpath, "Scripts", "metawards.exe"), - _os.path.join(dirpath, "Scripts", "metawards")]: - if _os.path.exists(option): - metawards = option - break - - if metawards is None: - # last attempt - is 'metawards' in the PATH? - from shutil import which - metawards = which("metawards") - - if metawards is None: - from .utils._console import Console - Console.error( - "Cannot find the metawards executable. Please could you find " - "it and add it to the PATH. Or please post an issue on the " - "GitHub repository (https://github.com/metawards/MetaWards) " - "as this may indicate a bug in the code.") - raise RuntimeError("Cannot locate the metawards executable") - - return metawards - - -def get_reticulate_command(): - """Print the reticulate command that you need to type - to be able to use the Python in which MetaWards is - installed - """ - import os as _os - import sys as _sys - pyexe = _os.path.abspath(_sys.executable) - return f"reticulate::use_python(\"{pyexe}\", required=TRUE)" - - -def run(help: bool = None, - version: bool = None, - dry_run: bool = None, - silent: bool = False, - auto_load: bool = False, - config: str = None, - input: _Union[str, VariableSet, VariableSets] = None, - line: int = None, - repeats: int = None, - seed: int = None, - additional: _Union[str, _List[str]] = None, - output: str = None, - disease: _Union[str, Disease] = None, - model: _Union[str, Wards, Ward] = None, - demographics: _Union[str, Demographics, Demographic] = None, - start_date: _Union[str, date] = None, - start_day: int = None, - parameters: _Union[str, Parameters] = None, - repository: str = None, - population: int = None, - nsteps: int = None, - user_variables: _Union[str, VariableSet] = None, - iterator: str = None, - extractor: str = None, - mixer: str = None, - mover: str = None, - star_as_E: bool = None, - star_as_R: bool = None, - disable_star: bool = None, - UV: float = None, - debug: bool = None, - debug_level: int = None, - outdir_scheme: str = None, - nthreads: int = None, - nprocs: int = None, - hostfile: str = None, - cores_per_node: int = None, - auto_bzip: bool = None, - no_auto_bzip: bool = None, - force_overwrite_output: bool = None, - profile: bool = None, - no_profile: bool = None, - mpi: bool = None, - scoop: bool = None) -> _Union[str, 'pandas.DataFrame']: - """Run a MetaWards simulation - - Parameters - ---------- - silent: bool - Run without printing the output to the screen - dry_run: bool - Don't run anything - just print what will be run - help: bool - Whether or not to print the full help - version: bool - Whether or not to print the metawards version info - output: str - The name of the directory in which to write the output. If this - is not set, then a new, random-named directory will be used. - force_overwrite_output: bool - Force overwriting the output directory - this will remove any - existing directory before running - auto_load: bool - Whether or not to automatically load and return a pandas dataframe - of the output/results.csv.bz2 file. If pandas is available then - this defaults to True, otherwise False - disease: Disease or str - The disease to model (or the filename of the json file containing - the disease, or name of the disease) - model: Ward, Wards or str - The network wards to run (of the filename of the json file - containing the network, or name of the network)) - - There are many more parameters, based on the arguments to - metawards --help. - - Please set "help" to True to print out a full list of - help for all of the arguments - - Returns - ------- - results: str or pandas.DataFrame - The file containing the output results (output/results.csv.bz2), - or, if auto_load is True, the pandas.DataFrame containing - those results - """ - import sys - import os - import tempfile - from .utils._console import Console - - metawards = find_mw_exe() - - args = [] - - tmpdir = None - - theme = "simple" - no_progress = True - no_spinner = True - - if help: - args.append("--help") - output = None - elif version: - args.append("--version") - output = None - else: - if output is None and not dry_run: - output = tempfile.mkdtemp(prefix="output_", dir=".") - force_overwrite_output = True - - if force_overwrite_output: - args.append("--force-overwrite-output") - else: - if output is None: - output = "output" - - while os.path.exists(output): - import metawards as _metawards - print(f"Output directory {output} exists.") - output = _metawards.input("Please choose a new directory: ", - default="error") - - if output is None: - return 0 - - output = output.strip() - if len(output) == 0: - return 0 - - if output.lower() == "error": - Console.error("You need to delete the directory or set " - "'force_overwrite_output' to TRUE") - return -1 - - try: - if config is not None: - args.append(f"--config {config}") - - if input is not None: - if not isinstance(input, str): - if tmpdir is None: - tmpdir = tempfile.mkdtemp(prefix="input_", dir=".") - - input = _write_to_file(input, "input.dat", dir=tmpdir, - bzip=False, dry_run=dry_run) - - args.append(f"--input {input}") - - if line is not None: - args.append(f"--line {int(line)}") - - if repeats is not None: - args.append(f"--repeats {int(repeats)}") - - if seed is not None: - args.append(f"--seed {int(seed)}") - - if additional is not None: - if isinstance(additional, list): - additional = "\\n".join(additional) - elif not isinstance(additional, str): - additional = str(int(additional)) - - if "'" in additional: - args.append(f"--additional \"{additional}\"") - else: - args.append(f"--additional '{additional}'") - - if output is not None: - args.append(f"--output {output}") - - if disease is not None: - if not isinstance(disease, str): - if tmpdir is None: - tmpdir = tempfile.mkdtemp(prefix="input_", dir=".") - - disease = _write_to_file(disease, "disease.json", - dir=tmpdir, - bzip=False, dry_run=dry_run) - - args.append(f"--disease {disease}") - - if model is not None: - from ._ward import Ward - from ._wards import Wards - - if isinstance(model, Ward): - m = Wards() - m.add(model) - model = m - - if not isinstance(model, str): - if tmpdir is None: - tmpdir = tempfile.mkdtemp(prefix="input_", dir=".") - - model = _write_to_file(model, "model.json", dir=tmpdir, - bzip=True, dry_run=dry_run) - - args.append(f"--model {model}") - - if demographics is not None: - from ._demographic import Demographic - from ._demographics import Demographics - - if isinstance(demographics, Demographic): - d = Demographics() - d.add(demographics) - demographics = demographics - - if not isinstance(demographics, str): - if tmpdir is None: - tmpdir = tempfile.mkdtemp(prefix="input_", dir=".") - - demographics = _write_to_file(demographics, - "demographics.json", - dir=tmpdir, - bzip=False, - dry_run=dry_run) - - args.append(f"--demographics {demographics}") - - if start_date is not None: - from datetime import date - if isinstance(start_date, date): - start_date = date.isoformat() - - args.append(f"--start-date {start_date}") - - if start_day is not None: - args.append(f"--start-day {int(start_day)}") - - if parameters is not None: - if not isinstance(parameters, str): - if tmpdir is None: - tmpdir = tempfile.mkdtemp(prefix="input_", dir=".") - - parameters = _write_to_file(parameters, "parameters.dat", - dir=tmpdir, bzip=False, - dry_run=dry_run) - - args.append(f"--parameters {parameters}") - - if repository is not None: - args.append(f"--repository {repository}") - - if population is not None: - args.append(f"--population {int(population)}") - - if nsteps is not None: - args.append(f"--nsteps {int(nsteps)}") - - if user_variables is not None: - if not isinstance(user_variables, str): - if tmpdir is None: - tmpdir = tempfile.mkdtemp(prefix="input_", dir=".") - - user_variables = _write_to_file(user_variables, - "user_variables.dat", - dir=tmpdir, - bzip=False, - dry_run=dry_run) - - args.append(f"--user {user_variables}") - - if iterator is not None: - args.append(f"--iterator {iterator}") - - if extractor is not None: - args.append(f"--extractor {extractor}") - - if mixer is not None: - args.append(f"--mixer {mixer}") - - if mover is not None: - args.append(f"--mover {mover}") - - if star_as_E: - args.append("--star-as-E") - elif star_as_R: - args.append("--star-as-R") - elif disable_star: - args.append("--disable-star") - - if UV is not None: - args.append(f"--UV {UV}") - - if theme is not None: - args.append(f"--theme {theme}") - - if no_spinner: - args.append("--no-spinner") - - if no_progress: - args.append("--no-progress") - - if debug: - args.append("--debug") - - if debug_level is not None: - args.append(f"--debug-level {debug_level}") - - if outdir_scheme is not None: - args.append(f"--outdir-scheme {outdir_scheme}") - - if nthreads is not None: - args.append(f"--nthreads {int(nthreads)}") - - if nprocs is not None: - args.append(f"--nprocs {int(nprocs)}") - - if hostfile is not None: - args.append(f"--hostfile {hostfile}") - - if cores_per_node is not None: - args.append(f"--cores-per-node {int(cores_per_node)}") - - if auto_bzip: - args.append("--auto-bzip") - elif no_auto_bzip: - args.append("--no-auto-bzip") - - if profile: - args.append("--profile") - elif no_profile: - args.append("--no-profile") - - if mpi: - args.append("--mpi") - - if scoop: - args.append("--scoop") - - except Exception as e: - Console.error(f"[ERROR] Error interpreting the arguments" - f"[ERROR] {e.__class__}: {e}") - _rmdir(tmpdir) - raise - return -1 - - cmd = f"{metawards} {' '.join(args)}" - - if dry_run: - Console.info(f"[DRY-RUN] {cmd}") - return_val = 0 - else: - if output is not None: - Console.info( - f"Writing output to directory {os.path.abspath(output)}") - - Console.info(f"[RUNNING] {cmd}") - - try: - if sys.platform.startswith("win"): - # shlex.split doesn't work, but the command can - # be passed as a single string - args = cmd - else: - import shlex - args = shlex.split(cmd) - - import subprocess - - # We have to specify all of the pipes (stdin, stdout, stderr) - # as below as otherwise we will break metawards on Windows - # (especially needed to allow metawards to run under - # reticulate via metawards$run. Without these specified - # we end up with Windows File Errors) - with subprocess.Popen(args, - stdin=subprocess.PIPE, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - bufsize=1, encoding="utf8", - errors="ignore", - text=True) as PROC: - while True: - line = PROC.stdout.readline() - - if not line: - break - - if not silent: - try: - sys.stdout.write(line) - sys.stdout.flush() - except UnicodeEncodeError: - # We get frequent unicode errors when run - # within RStudio. It is best just to ignore them - pass - except Exception as e: - Console.error(f"WRITE ERROR: {e.__class__} : {e}") - - return_val = PROC.poll() - - if return_val is None: - # get None if everything OK on Windows - # (sometimes windows returns 0 as None, which - # breaks things!) - return_val = 0 - - except Exception as e: - Console.error(f"[ERROR] {e.__class__}: {e}") - return_val = -1 - - if tmpdir is not None: - _rmdir(tmpdir) - - if dry_run: - return - - if output is None: - return - - if return_val == 0: - results = os.path.join(output, "results.csv") - - if not os.path.exists(results): - results += ".bz2" - - if auto_load: - try: - import pandas - except ImportError: - Console.error("Cannot import pandas:\n{e}") - auto_load = False - - if auto_load is None: - try: - import pandas - auto_load = True - except ImportError: - auto_load = False - - if auto_load: - import pandas as pd - return pd.read_csv(results) - else: - return results - else: - output_file = os.path.join(output, "console.log.bz2") - - Console.error(f"Something went wrong with the run. Please look " - f"at {output_file} for more information") - return None +from __future__ import annotations + +from typing import Union as _Union +from typing import List as _List +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from ._disease import Disease + from ._demographics import Demographics + from ._demographic import Demographic + from ._parameters import Parameters + from ._wards import Wards + from ._ward import Ward + from ._variableset import VariableSet, VariableSets + + from datetime import date + + +__all__ = ["run", "find_mw_exe", "find_mw_include", + "get_reticulate_command"] + + +def _write_to_file(obj: any, filename: str, dir: str = ".", bzip: bool = False, + dry_run: bool = False) -> str: + """Write the passed object to a file called 'filename' in + directory 'dir', returning the + relative path to that file + """ + import os + + if dry_run: + return filename + + filename = os.path.join(dir, filename) + + if hasattr(obj, "to_json"): + return obj.to_json(filename, auto_bzip=bzip) + else: + raise IOError(f"Cannot convert {obj} to a file!") + + return filename + + +def _rmdir(directory): + """Function modified from one copied from 'mitch' on stackoverflow + https://stackoverflow.com/questions/13118029/deleting-folders-in-python-recursively + """ + if directory is None: + return + + from pathlib import Path + directory = Path(directory) + + # first, check for removing important directories such as $HOME or root + if directory == Path.home(): + raise FileExistsError(f"We WILL NOT remove your " + f"home directory ${directory}") + + if directory == Path("/"): + raise FileExistsError(f"We WILL NOT remove the root directory " + f"{directory}") + + # get the directory containing '$HOME' + if directory == Path.home().parent: + raise FileExistsError(f"We WILL NOT remove the users/home " + f"directory {directory}") + + if not directory.is_dir(): + directory.unlink() + return + + for item in directory.iterdir(): + if item.is_dir(): + _rmdir(item) + else: + item.unlink() + + directory.rmdir() + + +def _is_executable(filename): + import os + if not os.path.exists(filename): + return None + + if os.path.isdir(filename): + return None + + # determining if this is executable + # on windows is really difficult, so just + # assume it is... + return filename + + +def _find_metawards(dirname): + import os + m = _is_executable(os.path.join(dirname, "metawards")) + + if m: + return m + + m = _is_executable(os.path.join(dirname, "metawards.exe")) + + if m: + return m + + m = _is_executable(os.path.join(dirname, "Scripts", "metawards")) + + if m: + return m + + m = _is_executable(os.path.join(dirname, "Scripts", "metawards.exe")) + + if m: + return m + + m = _is_executable(os.path.join(dirname, "bin", "metawards")) + + if m: + return m + + m = _is_executable(os.path.join(dirname, "bin", "metawards.exe")) + + if m: + return m + + return None + + +def _find_metawards_include(dirname): + import os + + # this is from a metawards installation + m = os.path.abspath(os.path.join(dirname, "include", "metawards")) + + if os.path.exists(m): + return m + + # this is from a metawards source run (used for testing) + m = os.path.abspath(os.path.join(dirname, "src", "metawards")) + + if os.path.exists(m): + return m + + return None + + +def find_mw_include(): + """Try to find the directory containing the MetaWards include files. + This raises an exception if the include files cannot be found. + It returns the full path to the include files + """ + import metawards as _metawards + import os as _os + + # Search through the path based on where the metawards module + # has been installed. + modpath = _metawards.__file__ + + metawards = None + + # Loop only 100 times - this should break before now, + # We are not using a while loop to avoid an infinite loop + for i in range(0, 100): + metawards = _find_metawards_include(modpath) + + if metawards: + break + + newpath = _os.path.dirname(modpath) + + if newpath == modpath: + break + + modpath = newpath + + if metawards is None: + from .utils._console import Console + Console.error( + "Cannot find the metawards include directory, when starting from " + f"{_metawards.__file__}. Please could you " + "find it and then post an issue on the " + "GitHub repository (https://github.com/metawards/MetaWards) " + "as this may indicate a bug in the code.") + raise RuntimeError("Cannot locate the metawards include directory") + + return metawards + + +def find_mw_exe(): + """Try to find the MetaWards executable. This should be findable + if MetaWards has been installed. This raises an exception + if it cannot be found. It returns the full path to the + executable + """ + import metawards as _metawards + import os as _os + import sys as _sys + + # Search through the path based on where the metawards module + # has been installed. + modpath = _metawards.__file__ + + metawards = None + + # Loop only 100 times - this should break before now, + # We are not using a while loop to avoid an infinite loop + for i in range(0, 100): + metawards = _find_metawards(modpath) + + if metawards: + break + + newpath = _os.path.dirname(modpath) + + if newpath == modpath: + break + + modpath = newpath + + if metawards is None: + # We couldn't find it that way - try another route... + dirpath = _os.path.join(_os.path.dirname(_sys.executable)) + + for option in [_os.path.join(dirpath, "metawards.exe"), + _os.path.join(dirpath, "metawards"), + _os.path.join(dirpath, "Scripts", "metawards.exe"), + _os.path.join(dirpath, "Scripts", "metawards")]: + if _os.path.exists(option): + metawards = option + break + + if metawards is None: + # last attempt - is 'metawards' in the PATH? + from shutil import which + metawards = which("metawards") + + if metawards is None: + from .utils._console import Console + Console.error( + "Cannot find the metawards executable. Please could you find " + "it and add it to the PATH. Or please post an issue on the " + "GitHub repository (https://github.com/metawards/MetaWards) " + "as this may indicate a bug in the code.") + raise RuntimeError("Cannot locate the metawards executable") + + return metawards + + +def get_reticulate_command(): + """Print the reticulate command that you need to type + to be able to use the Python in which MetaWards is + installed + """ + import os as _os + import sys as _sys + pyexe = _os.path.abspath(_sys.executable) + return f"reticulate::use_python(\"{pyexe}\", required=TRUE)" + + +def run(help: bool = None, + version: bool = None, + dry_run: bool = None, + silent: bool = False, + auto_load: bool = False, + config: str = None, + input: _Union[str, VariableSet, VariableSets] = None, + line: int = None, + repeats: int = None, + seed: int = None, + additional: _Union[str, _List[str]] = None, + output: str = None, + disease: _Union[str, Disease] = None, + model: _Union[str, Wards, Ward] = None, + demographics: _Union[str, Demographics, Demographic] = None, + start_date: _Union[str, date] = None, + start_day: int = None, + parameters: _Union[str, Parameters] = None, + repository: str = None, + population: int = None, + nsteps: int = None, + user_variables: _Union[str, VariableSet] = None, + iterator: str = None, + extractor: str = None, + mixer: str = None, + mover: str = None, + star_as_E: bool = None, + star_as_R: bool = None, + disable_star: bool = None, + UV: float = None, + debug: bool = None, + debug_level: int = None, + outdir_scheme: str = None, + nthreads: int = None, + nprocs: int = None, + hostfile: str = None, + cores_per_node: int = None, + auto_bzip: bool = None, + no_auto_bzip: bool = None, + force_overwrite_output: bool = None, + profile: bool = None, + no_profile: bool = None, + mpi: bool = None, + scoop: bool = None) -> _Union[str, 'pandas.DataFrame']: + """Run a MetaWards simulation + + Parameters + ---------- + silent: bool + Run without printing the output to the screen + dry_run: bool + Don't run anything - just print what will be run + help: bool + Whether or not to print the full help + version: bool + Whether or not to print the metawards version info + output: str + The name of the directory in which to write the output. If this + is not set, then a new, random-named directory will be used. + force_overwrite_output: bool + Force overwriting the output directory - this will remove any + existing directory before running + auto_load: bool + Whether or not to automatically load and return a pandas dataframe + of the output/results.csv.bz2 file. If pandas is available then + this defaults to True, otherwise False + disease: Disease or str + The disease to model (or the filename of the json file containing + the disease, or name of the disease) + model: Ward, Wards or str + The network wards to run (of the filename of the json file + containing the network, or name of the network)) + + There are many more parameters, based on the arguments to + metawards --help. + + Please set "help" to True to print out a full list of + help for all of the arguments + + Returns + ------- + results: str or pandas.DataFrame + The file containing the output results (output/results.csv.bz2), + or, if auto_load is True, the pandas.DataFrame containing + those results + """ + import sys + import os + import tempfile + from .utils._console import Console + + metawards = find_mw_exe() + + args = [] + + tmpdir = None + + theme = "simple" + no_progress = True + no_spinner = True + + if help: + args.append("--help") + output = None + elif version: + args.append("--version") + output = None + else: + if output is None and not dry_run: + output = tempfile.mkdtemp(prefix="output_", dir=".") + force_overwrite_output = True + + if force_overwrite_output: + args.append("--force-overwrite-output") + else: + if output is None: + output = "output" + + while os.path.exists(output): + import metawards as _metawards + print(f"Output directory {output} exists.") + output = _metawards.input("Please choose a new directory: ", + default="error") + + if output is None: + return 0 + + output = output.strip() + if len(output) == 0: + return 0 + + if output.lower() == "error": + Console.error("You need to delete the directory or set " + "'force_overwrite_output' to TRUE") + return -1 + + try: + if config is not None: + args.append(f"--config {config}") + + if input is not None: + if not isinstance(input, str): + if tmpdir is None: + tmpdir = tempfile.mkdtemp(prefix="input_", dir=".") + + input = _write_to_file(input, "input.dat", dir=tmpdir, + bzip=False, dry_run=dry_run) + + args.append(f"--input {input}") + + if line is not None: + args.append(f"--line {int(line)}") + + if repeats is not None: + args.append(f"--repeats {int(repeats)}") + + if seed is not None: + args.append(f"--seed {int(seed)}") + + if additional is not None: + if isinstance(additional, list): + additional = "\\n".join(additional) + elif not isinstance(additional, str): + additional = str(int(additional)) + + if "\"" in additional: + if sys.platform.startswith("win"): + additional.replace("\"", "'") + args.append(f"--additional \"{additional}\"") + else: + args.append(f"--additional '{additional}'") + else: + args.append(f"--additional \"{additional}\"") + + if output is not None: + args.append(f"--output {output}") + + if disease is not None: + if not isinstance(disease, str): + if tmpdir is None: + tmpdir = tempfile.mkdtemp(prefix="input_", dir=".") + + disease = _write_to_file(disease, "disease.json", + dir=tmpdir, + bzip=False, dry_run=dry_run) + + args.append(f"--disease {disease}") + + if model is not None: + from ._ward import Ward + from ._wards import Wards + + if isinstance(model, Ward): + m = Wards() + m.add(model) + model = m + + if not isinstance(model, str): + if tmpdir is None: + tmpdir = tempfile.mkdtemp(prefix="input_", dir=".") + + model = _write_to_file(model, "model.json", dir=tmpdir, + bzip=True, dry_run=dry_run) + + args.append(f"--model {model}") + + if demographics is not None: + from ._demographic import Demographic + from ._demographics import Demographics + + if isinstance(demographics, Demographic): + d = Demographics() + d.add(demographics) + demographics = demographics + + if not isinstance(demographics, str): + if tmpdir is None: + tmpdir = tempfile.mkdtemp(prefix="input_", dir=".") + + demographics = _write_to_file(demographics, + "demographics.json", + dir=tmpdir, + bzip=False, + dry_run=dry_run) + + args.append(f"--demographics {demographics}") + + if start_date is not None: + from datetime import date + if isinstance(start_date, date): + start_date = date.isoformat() + + args.append(f"--start-date {start_date}") + + if start_day is not None: + args.append(f"--start-day {int(start_day)}") + + if parameters is not None: + if not isinstance(parameters, str): + if tmpdir is None: + tmpdir = tempfile.mkdtemp(prefix="input_", dir=".") + + parameters = _write_to_file(parameters, "parameters.dat", + dir=tmpdir, bzip=False, + dry_run=dry_run) + + args.append(f"--parameters {parameters}") + + if repository is not None: + args.append(f"--repository {repository}") + + if population is not None: + args.append(f"--population {int(population)}") + + if nsteps is not None: + args.append(f"--nsteps {int(nsteps)}") + + if user_variables is not None: + if not isinstance(user_variables, str): + if tmpdir is None: + tmpdir = tempfile.mkdtemp(prefix="input_", dir=".") + + user_variables = _write_to_file(user_variables, + "user_variables.dat", + dir=tmpdir, + bzip=False, + dry_run=dry_run) + + args.append(f"--user {user_variables}") + + if iterator is not None: + args.append(f"--iterator {iterator}") + + if extractor is not None: + args.append(f"--extractor {extractor}") + + if mixer is not None: + args.append(f"--mixer {mixer}") + + if mover is not None: + args.append(f"--mover {mover}") + + if star_as_E: + args.append("--star-as-E") + elif star_as_R: + args.append("--star-as-R") + elif disable_star: + args.append("--disable-star") + + if UV is not None: + args.append(f"--UV {UV}") + + if theme is not None: + args.append(f"--theme {theme}") + + if no_spinner: + args.append("--no-spinner") + + if no_progress: + args.append("--no-progress") + + if debug: + args.append("--debug") + + if debug_level is not None: + args.append(f"--debug-level {debug_level}") + + if outdir_scheme is not None: + args.append(f"--outdir-scheme {outdir_scheme}") + + if nthreads is not None: + args.append(f"--nthreads {int(nthreads)}") + + if nprocs is not None: + args.append(f"--nprocs {int(nprocs)}") + + if hostfile is not None: + args.append(f"--hostfile {hostfile}") + + if cores_per_node is not None: + args.append(f"--cores-per-node {int(cores_per_node)}") + + if auto_bzip: + args.append("--auto-bzip") + elif no_auto_bzip: + args.append("--no-auto-bzip") + + if profile: + args.append("--profile") + elif no_profile: + args.append("--no-profile") + + if mpi: + args.append("--mpi") + + if scoop: + args.append("--scoop") + + except Exception as e: + Console.error(f"[ERROR] Error interpreting the arguments" + f"[ERROR] {e.__class__}: {e}") + _rmdir(tmpdir) + raise + return -1 + + cmd = f"{metawards} {' '.join(args)}" + + if dry_run: + Console.info(f"[DRY-RUN] {cmd}") + return_val = 0 + else: + if output is not None: + Console.info( + f"Writing output to directory {os.path.abspath(output)}") + + Console.info(f"[RUNNING] {cmd}") + + try: + if sys.platform.startswith("win"): + # shlex.split doesn't work, but the command can + # be passed as a single string + args = cmd + else: + import shlex + args = shlex.split(cmd) + + import subprocess + + # We have to specify all of the pipes (stdin, stdout, stderr) + # as below as otherwise we will break metawards on Windows + # (especially needed to allow metawards to run under + # reticulate via metawards$run. Without these specified + # we end up with Windows File Errors) + with subprocess.Popen(args, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + bufsize=1, encoding="utf8", + errors="ignore", + text=True) as PROC: + while True: + line = PROC.stdout.readline() + + if not line: + break + + if not silent: + try: + sys.stdout.write(line) + sys.stdout.flush() + except UnicodeEncodeError: + # We get frequent unicode errors when run + # within RStudio. It is best just to ignore them + pass + except Exception as e: + Console.error(f"WRITE ERROR: {e.__class__} : {e}") + + return_val = PROC.poll() + + if return_val is None: + # get None if everything OK on Windows + # (sometimes windows returns 0 as None, which + # breaks things!) + return_val = 0 + + except Exception as e: + Console.error(f"[ERROR] {e.__class__}: {e}") + return_val = -1 + + if tmpdir is not None: + _rmdir(tmpdir) + + if dry_run: + return + + if output is None: + return + + if return_val == 0: + results = os.path.join(output, "results.csv") + + if not os.path.exists(results): + results += ".bz2" + + if auto_load: + try: + import pandas + except ImportError: + Console.error("Cannot import pandas:\n{e}") + auto_load = False + + if auto_load is None: + try: + import pandas + auto_load = True + except ImportError: + auto_load = False + + if auto_load: + import pandas as pd + return pd.read_csv(results) + else: + return results + else: + output_file = os.path.join(output, "console.log.bz2") + + Console.error(f"Something went wrong with the run. Please look " + f"at {output_file} for more information") + return None diff --git a/src/metawards/iterators/_advance_additional.py b/src/metawards/iterators/_advance_additional.py index 963862f2a..95241b3f2 100644 --- a/src/metawards/iterators/_advance_additional.py +++ b/src/metawards/iterators/_advance_additional.py @@ -103,7 +103,7 @@ def _load_additional_seeds(network: _Union[Network, Networks], words = [] - # yes, the original files really do mix tabe and spaces... need + # yes, the original files really do mix tabs and spaces... need # to extract these separately! for l in line: rest_commented = False From c5448a71782378a0108222b2a9f8057be414d21b Mon Sep 17 00:00:00 2001 From: Christopher Woods Date: Tue, 25 May 2021 13:32:17 +0100 Subject: [PATCH 4/4] I think(!) I've got the cython code working on Windows so that dynamically compiled plugins can link to the metawards_random library. It all works on a local machine - now to test on GitHub Actions... --- setup.py | 34 ++++++++++++++++++++++++++++------ tests/test_cython_iterator.py | 6 ------ 2 files changed, 28 insertions(+), 12 deletions(-) diff --git a/setup.py b/setup.py index b51e97c09..e551d1342 100644 --- a/setup.py +++ b/setup.py @@ -153,7 +153,8 @@ def get_openmp_flag(): extra_postargs=[flag]) except Exception: # We are likely missing the path to the libomp library - get this path - print("\nCould not link - trying again, specifying the path to libomp") + print( + "\nCould not link - trying again, specifying the path to libomp") compiler_dir = os.path.dirname(compiler.compiler_so[0]) libomp = glob(f"{compiler_dir}/../lib*/libomp*") @@ -389,14 +390,35 @@ def no_cythonize(extensions, **_ignore): dev_requires = fp.read().strip().split("\n") if IS_WINDOWS: - mw_random_lib = glob("build/*/metawards.lib") + import sys + v = sys.version_info + print(os.getcwd()) + print(f"build/*{v.major}.{v.minor}/metawards_random.lib") + mw_random_lib = glob( + f"build/*{v.major}.{v.minor}/metawards_random.lib") + + if len(mw_random_lib) == 0: + if not is_build: + print("WARNING: CANNOT FIND metawards_random.lib!") + elif len(mw_random_lib) > 1: + mw_random_lib = [mw_random_lib[0]] + libs_dir = os.path.abspath(os.path.join(sys.prefix, "libs")) - if not os.path.exists(libs_dir): - print(f"Cannot find libs directory? {libs_dir}") - raise RuntimeError(f"Cannot find libs directory {libs_dir}") + if not is_build: + print(f"Installing {mw_random_lib} to {libs_dir}") else: - mw_random_lib = glob("build/*/libmetawards_random.a") + import sys + v = sys.version_info + mw_random_lib = glob( + f"build/*{v.major}.{v.minor}/libmetawards_random.a") + + if len(mw_random_lib) == 0: + if not is_build: + print("WARNING: CANNOT FIND metawards_random.lib!") + elif len(mw_random_lib) > 1: + mw_random_lib = [mw_random_lib[0]] + libs_dir = "lib" setup( diff --git a/tests/test_cython_iterator.py b/tests/test_cython_iterator.py index 42c0b0c4a..1851cd89d 100644 --- a/tests/test_cython_iterator.py +++ b/tests/test_cython_iterator.py @@ -14,12 +14,6 @@ def test_cython_iterator(): """Validate that we can compile and link cython iterators dynamically """ - if sys.platform.startswith("win"): - print( - "Skipping test as this functionality is not yet " - "supported on Windows.") - return - # load all of the parameters try: params = Parameters.load(parameters="march29")