From 41045f7f9cb1aa7578ec098c5f8cbade907a95b7 Mon Sep 17 00:00:00 2001 From: Krishna Kamath Date: Wed, 24 Apr 2024 19:10:43 +0000 Subject: [PATCH 1/3] feat: parallel_repeat basic functionality added: --- sciunit2/command/parallel_repeat.py | 69 +++++++++++++++++++++++++++++ sciunit2/command/parallel_script.py | 14 ++++++ 2 files changed, 83 insertions(+) create mode 100644 sciunit2/command/parallel_repeat.py create mode 100644 sciunit2/command/parallel_script.py diff --git a/sciunit2/command/parallel_repeat.py b/sciunit2/command/parallel_repeat.py new file mode 100644 index 0000000..8da35ac --- /dev/null +++ b/sciunit2/command/parallel_repeat.py @@ -0,0 +1,69 @@ +from __future__ import absolute_import + +from sciunit2.command import AbstractCommand +from sciunit2.command.context import CheckoutContext, CheckoutContext_Parallel +from sciunit2.exceptions import CommandLineError, MalformedExecutionId +import sciunit2.core +import re +from getopt import getopt +import sys +import subprocess +import shlex +import os + +class ParallelRepeatCommand(AbstractCommand): + name = 'parallel_repeat' + @staticmethod + def __to_rev(id_): + return 'e%d' % id_ + + @staticmethod + def __to_id_range(revrange): + r = re.match(r'^e([1-9]\d*)-e([1-9]\d*)?$', revrange) + if not r: + raise MalformedExecutionId + return tuple(int(x) if x is not None else x for x in r.groups()) + + + @property + def usage(self): + return [('parallel_repeat - []', + "Repeat the execution of to ")] + + def is_gnu_parallel_installed(self): + try: + subprocess.run(['parallel', '--version'], check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + return True + except FileNotFoundError: + return False + except subprocess.CalledProcessError as e: + return False + + def run(self, args): + optlist, args = getopt(args, '') + if len(args) == 0: + raise CommandLineError + + if not self.is_gnu_parallel_installed(): + print("GNU Parallel is not installed on the machine") + sys.exit(1) + + + start,end = self.__to_id_range(args[0]) + pkgdirs = '' + origs = '' + for curr in range(start, end+1): + with CheckoutContext_Parallel(self.__to_rev(curr)) as (pkgdir, orig): + pkgdirs += '\'' + pkgdir + '\' ' + origs += '\'' + str(orig).replace("'", '"') + '\' ' + + + + script_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'parallel_script.py') + + # command = f'parallel --link python {script_path} ::: {pkgdirs} ::: \'["python3", "test.py", "1"]\' \'["python3", "test.py", "2"]\' \'["python3", "test.py", "3"]\'' + command = f'parallel --link python {script_path} ::: {pkgdirs} ::: {origs}' + + subprocess.run(command, shell=True, executable='/bin/bash') + + sys.exit(0) diff --git a/sciunit2/command/parallel_script.py b/sciunit2/command/parallel_script.py new file mode 100644 index 0000000..5840385 --- /dev/null +++ b/sciunit2/command/parallel_script.py @@ -0,0 +1,14 @@ +import sys +import json +#from sciunit2.command import AbstractCommand +#from sciunit2.command.context import CheckoutContext +#from sciunit2.exceptions import CommandLineError +import sciunit2.core +if len(sys.argv) != 3: + print("Usage: python my_script.py pkgdir arg_list repeat_args") + sys.exit(1) + +pkgdir = sys.argv[1] +arg_list = json.loads(sys.argv[2]) + +sciunit2.core.repeat(pkgdir, arg_list,[]) \ No newline at end of file From 2d9b5f42dcc676bb789ecdffc2456e2cd86b272a Mon Sep 17 00:00:00 2001 From: Krishna Kamath Date: Wed, 24 Apr 2024 19:24:30 +0000 Subject: [PATCH 2/3] feat: parallel_repeat file changes --- sciunit2/cli.py | 3 ++- sciunit2/command/context.py | 16 +++++++++++++++- 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/sciunit2/cli.py b/sciunit2/cli.py index 82e3427..44a97d8 100644 --- a/sciunit2/cli.py +++ b/sciunit2/cli.py @@ -18,6 +18,7 @@ from sciunit2.command.diff import DiffCommand from sciunit2.command.export import ExportCommand from sciunit2.command.remove import RemoveCommand +from sciunit2.command.parallel_repeat import ParallelRepeatCommand import sys from getopt import getopt, GetoptError @@ -30,7 +31,7 @@ __cmds__ = [CreateCommand, OpenCommand, ExecCommand, RepeatCommand, ListCommand, ShowCommand, GivenCommand, CommitCommand, RmCommand, SortCommand, PushCommand, CopyCommand, PostInstallCommand, - DiffCommand, RemoveCommand, ExportCommand, CheckoutCommand] + DiffCommand, RemoveCommand, ExportCommand, CheckoutCommand, ParallelRepeatCommand] def short_usage(out): diff --git a/sciunit2/command/context.py b/sciunit2/command/context.py index 5e16891..39f9573 100644 --- a/sciunit2/command/context.py +++ b/sciunit2/command/context.py @@ -4,7 +4,7 @@ from contextlib import contextmanager import os - +import shutil # returns the pkgdir and original command used # to execute execution 'rev' @@ -28,3 +28,17 @@ def CheckoutContext_Diff(rev): repo.cleanup(pkgdir) repo.checkout(rev) yield pkgdir, orig + +@contextmanager +def CheckoutContext_Parallel(rev): + emgr, repo = sciunit2.workspace.current() + with emgr.exclusive(): + orig = emgr.get(rev).cmd + pkgdir = os.path.join(repo.location, 'cde-package') + + # creates a different path for each execution id + pkgdir_rev = os.path.join(repo.location, rev, 'cde-package') + repo.cleanup(pkgdir_rev) + repo.checkout(rev) + shutil.copytree(pkgdir, pkgdir_rev) + yield pkgdir_rev, orig \ No newline at end of file From 616e8e2153c324c977d4941e1aec3d056f4228bf Mon Sep 17 00:00:00 2001 From: Krishna Kamath Date: Tue, 30 Apr 2024 14:34:10 +0000 Subject: [PATCH 3/3] feat: parallel repeat --- sciunit2/command/parallel_repeat.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/sciunit2/command/parallel_repeat.py b/sciunit2/command/parallel_repeat.py index 8da35ac..209a680 100644 --- a/sciunit2/command/parallel_repeat.py +++ b/sciunit2/command/parallel_repeat.py @@ -62,8 +62,11 @@ def run(self, args): script_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'parallel_script.py') # command = f'parallel --link python {script_path} ::: {pkgdirs} ::: \'["python3", "test.py", "1"]\' \'["python3", "test.py", "2"]\' \'["python3", "test.py", "3"]\'' + command = f'parallel --link python {script_path} ::: {pkgdirs} ::: {origs}' subprocess.run(command, shell=True, executable='/bin/bash') + + # TODO: Delete all the new directories created sys.exit(0)