diff --git a/jobrunner/api/_commands.py b/jobrunner/api/_commands.py index 1d59112..ef9ca9a 100644 --- a/jobrunner/api/_commands.py +++ b/jobrunner/api/_commands.py @@ -21,6 +21,9 @@ def setup(dirlist, verbose=False, exit_on_failure=False): # set variable to determine console separator separator = False + # create an empty list for output + output = [] + # loop over dirlist for workdir in dirlist: @@ -51,7 +54,12 @@ def setup(dirlist, verbose=False, exit_on_failure=False): print(f'{" "*4}- {value.replace(basedir,"")}') # run a bash process - lib.BashProcess(basedir, workdir, "job.setup", verbose, exit_on_failure) + process = lib.BashProcess( + basedir, workdir, "job.setup", verbose, exit_on_failure + ) + + # append process to output + output.append(process) # set separator value separator = True @@ -59,6 +67,8 @@ def setup(dirlist, verbose=False, exit_on_failure=False): # Return to base directory os.chdir(basedir) + return output + def submit(dirlist, verbose=False, exit_on_failure=False): """ @@ -70,6 +80,9 @@ def submit(dirlist, verbose=False, exit_on_failure=False): # set variable to determine console separator separator = False + # create an empty list for output + output = [] + # loop over dirlist for workdir in dirlist: @@ -116,19 +129,26 @@ def submit(dirlist, verbose=False, exit_on_failure=False): # Submit job if config.schedular.command == "bash": - lib.BashProcess(basedir, workdir, "job.submit", verbose, exit_on_failure) + process = lib.BashProcess( + basedir, workdir, "job.submit", verbose, exit_on_failure + ) else: - lib.SchedularProcess( + process = lib.SchedularProcess( basedir, workdir, config.schedular.command, "job.submit" ) + # append process to output + output.append(process) + # set separator value separator = True # Return to base directory os.chdir(basedir) + return output + def clean(dirlist): """ diff --git a/jobrunner/lib/_console.py b/jobrunner/lib/_console.py index 68d5cfb..2bcfb44 100644 --- a/jobrunner/lib/_console.py +++ b/jobrunner/lib/_console.py @@ -39,6 +39,8 @@ def SchedularProcess(basedir, workdir, command, script): check=True, ) + return process + def BashProcess(basedir, workdir, script, verbose=False, exit_on_failure=False): """ @@ -73,11 +75,11 @@ def BashProcess(basedir, workdir, script, verbose=False, exit_on_failure=False): # output.write(line) else: - with open("job.output", "w") as output: + with open("job.output", "w") as output, open("job.error", "w") as error: process = subprocess.Popen( f"bash {script}".split(), stdout=output, - stderr=subprocess.STDOUT, + stderr=error, text=True, ) @@ -87,8 +89,8 @@ def BashProcess(basedir, workdir, script, verbose=False, exit_on_failure=False): if process.returncode != 0: if not verbose: - with open("job.output", "r") as output: - print("".join(output.readlines()[-8:])) + with open("job.error", "r") as error: + print("".join(error.readlines()[-8:])) if exit_on_failure: raise ValueError(f"{lib.Color.red}FAILURE {lib.Color.end}") @@ -101,3 +103,5 @@ def BashProcess(basedir, workdir, script, verbose=False, exit_on_failure=False): print( f'\n{lib.Color.purple}OUTPUT:{lib.Color.end} {workdir.replace(basedir,"")}/job.output' ) + + return process diff --git a/jobrunner/lib/_filetools.py b/jobrunner/lib/_filetools.py index 901b323..9e6d992 100644 --- a/jobrunner/lib/_filetools.py +++ b/jobrunner/lib/_filetools.py @@ -232,6 +232,7 @@ def RemoveNodeFiles(config, nodedir): nodedir + os.sep + "job.submit", nodedir + os.sep + "job.target", nodedir + os.sep + "job.output", + nodedir + os.sep + "job.error", ] # loop over list of files in nodedir and append to