From 477998d5677b4089ce03405ce428084afd5c1031 Mon Sep 17 00:00:00 2001 From: David Hiatt Date: Thu, 26 Apr 2018 16:39:39 -0700 Subject: [PATCH 1/3] Version 1.0 complete. Supports python3 and all runtypes. --- objects.py | 225 ++++++++++++++++++++++++++++++++++++++++---------- pycurses.py | 0 speccurses.py | 30 ++++++- 3 files changed, 211 insertions(+), 44 deletions(-) delete mode 100644 pycurses.py diff --git a/objects.py b/objects.py index 5bf1c2d..829bfb2 100644 --- a/objects.py +++ b/objects.py @@ -1,11 +1,12 @@ import json import multiprocessing import os +import shlex import time import datetime -from subprocess import Popen +from subprocess import Popen, PIPE, STDOUT class spec_config(): @@ -70,11 +71,11 @@ def __init__(self, fromjson = None): self.skip_report = False self.ignore_kit_validation = False else: - self.jdk = self._byteify(fromjson['jdk']) - self.jvm_options = self._byteify(fromjson['jvm_options']) - self.data_collection = self._byteify(fromjson['data_collection']) + self.jdk = fromjson['jdk'] + self.jvm_options = fromjson['jvm_options'] + self.data_collection =fromjson['data_collection'] self.num_runs = fromjson['num_runs'] - self.tag = self._byteify(fromjson['tag']) + self.tag = fromjson['tag'] self.run_type = fromjson['run_type'] self.properties = props(fromjson['props']) self.verbose = fromjson['verbose'] @@ -106,13 +107,19 @@ def tojson(self): "props" : self.properties.tojson() } + def _defHandle(msg): + print(msg) - def run(self, path = ""): + #handle = Any function that takes a single string argument. + #It can be used to intercept any output + #If set to None, then output will only be logged + def run(self, path = "", handle = _defHandle): if(not os.path.exists(self.jdk)): return 3 if(path == ""): path = os.getcwd() if(not os.path.exists(os.path.join(path, "specjbb2015.jar"))): + handle('Failed to locate "specjbb2015.jar" in "{}"'.format(path)) return 2 # orig = os.getcwd() # os.chdir(path) @@ -122,70 +129,203 @@ def run(self, path = ""): 'distributed_sut' : self.distributed_sut, 'multi' : self.run_multi } - ret = switch[self.run_type](path) + ret = switch[self.run_type](path, handle) # os.chdir(orig) return ret - def _byteify(self, data): - # if this is a unicode string, return its string representation - # # return data.encode('utf-8') - # if this is a list of values, return list of byteified values - if isinstance(data, list) and len(data) == 1: - return self._byteify(data[0]) - # if it's anything else, return it in its original form - return data - - def run_composite(self, path): - opts = self._spec_opts() + def run_composite(self, path, handle): + cmd = '{} {} -jar {}/specjbb2015.jar -m COMPOSITE {}'.format( + self.jdk, self.jvm_options, path, self._spec_opts()) for x in range(self.num_runs): result_dir = self._prerun(path) - #p = Popen([self.jdk, self.jvm_options, "-jar", "{}/specjbb2015.jar".format(path), "-m", "COMPOSITE", opts, "2>", "{}/composite.log".format(result_dir), "{}/composite.out".format(result_dir)]) - #p.wait() - # spec = os.subprocess.Popen(['/usr/bin/java' "{}/specjbb2015.jar".format(path)]) - os.system('{} {} -jar {}/specjbb2015.jar -m COMPOSITE {} 2> {}/composite.log > {}/composite.out &'.format(self.jdk, self.jvm_options, path, opts, result_dir, result_dir)) + handle('Starting run number {}.'.format(x)) + errout = open(os.path.join(result_dir, 'composite.out'), 'wb') + stdout = open(os.path.join(result_dir, 'composite.log'), 'wb') + p = Popen(shlex.split(cmd), cwd=path, stdout=PIPE, stderr=PIPE) + while(p.poll() is None): + e = p.stderr.readline() + if(e != ''): + errout.write(e) + handle(e) + o = p.stdout.readline() + if(o != ''): + stdout.write(o) + handle(o) + errout.close() + stdout.close() + p.wait() + return 0 - def run_distributed_ctrl_txl(self, path): + def run_distributed_ctrl_txl(self, path, handle): ctrl_ip = self.properties.root['specjbb.controller.host'].value #Check if ip responds here + pingcmd = 'ping -c 1 {}'.format(ctrl_ip) + FNULL = open(os.devnull, 'w') + ping = Popen(shlex.split(pingcmd), stderr=FNULL, stdout=FNULL) + exitcode = ping.wait() + FNULL.close() + if(exitcode != 0): + handle('ERROR: Failed to ping Controller host (specjbb.controller.host): {}'.format(ctrl_ip)) + return 4 + opts = self._spec_opts() tx_opts = self._tx_opts() result_dir= self._prerun(path) - controller = Popen([self.jdk, self.jvm_options, "-m", "DISTCONTROLLER", opts, "2>", "{}/composite.log".format(result_dir), "{}/composite.log".format(result_dir)]) - os.system('{} {} -jar {}/specjbb2015.jar -m DISTCONTROLLER {} 2> {}/controller.log > {}/controller.out &'.format(self.jdk, self.jvm_options, path, opts, result_dir, result_dir)) + cmd = '{} {} -jar {}/specjbb2015.jar -m DISTCONTROLLER {}'.format(self.jdk, self.jvm_options, path, opts) + tx_procs = [] + cont_std = open(os.path.join(result_dir, 'controller.log'), 'wb') + cont_err = open(os.path.join(result_dir, 'controller.out'), 'wb') + controller = Popen(shlex.split(cmd), cwd=path, stdout=PIPE, stderr=PIPE) + #os.system('{} {} -jar {}/specjbb2015.jar -m DISTCONTROLLER {} 2> {}/controller.log > {}/controller.out &'.format(self.jdk, self.jvm_options, path, opts, result_dir, result_dir)) for g in range(self.properties.root['specjbb.group.count'].value): for j in range(self.properties.root['specjbb.txi.pergroup.count'].value): ti_name = "{}Group{}.TxInjector.txiJVM{}".format(result_dir, g, j) - os.system('{} {} -jar {}/specjbb2015.jar -m TXINJECTOR -G={} 2> {}.log > {}.out &'.format(self.jdk, self.jvm_options, path, tx_opts, g, ti_name, ti_name)) + cmd = '{} {} -jar {}/specjbb2015.jar -m TXINJECTOR -G={}'.format(self.jdk, self.jvm_options, path, tx_opts, g) + tx_procs.append([Popen(shlex.split(cmd), cwd=path, stdout=PIPE, stderr=PIPE), + open(os.path.join(result_dir, '{}.log'.format(ti_name)), 'wb'), + open(os.path.join(result_dir, '{}.out'.format(ti_name)), 'wb')]) + #os.system('{} {} -jar {}/specjbb2015.jar -m TXINJECTOR -G={} 2> {}.log > {}.out &'.format(self.jdk, self.jvm_options, path, tx_opts, g, ti_name, ti_name)) + while (controller.poll() is None): + e = controller.stderr.readline() + if (e != ''): + cont_err.write(e) + handle(e) + o = controller.stdout.readline() + if (o != ''): + cont_std.write(o) + handle(o) + for p in tx_procs: + if(p[0].poll() is None): + e = p[0].stderr.readline() + if (e != ''): + p[2].write(e) + handle(e) + o = p[0].stdout.readline() + if (o != ''): + p[1].write(o) + handle(o) + cont_err.close() + cont_std.close() + controller.wait() + for p in tx_procs: + p[0].kill() + p[1].close() + p[2].close() return 0 - def distributed_sut(self, path): + def distributed_sut(self, path, handle): ctrl_ip = self.properties.root['specjbb.controller.host'].value - #Check if ip responds here + # Check if ip responds here + pingcmd = 'ping -c 1 {}'.format(ctrl_ip) + FNULL = open(os.devnull, 'wb') + ping = Popen(shlex.split(pingcmd), stderr=FNULL, stdout=FNULL) + exitcode = ping.wait() + FNULL.close() + if (exitcode != 0): + handle('ERROR: Failed to ping Controller host (specjbb.controller.host): {}'.format(ctrl_ip)) + return 4 opts = self._tx_opts() + procs = [] for g in range(self.properties.root['specjbb.group.count'].value): - os.system('java {} -jar {}/specjbb2015.jar -m BACKEND {} -G={} -J=beJVM Group{}.Backend.beJVM.log 2>&1 &'.format(self.jdk, self.jvm_options, path, opts, g, g, g)) - + be_name = 'beJVM Group{}.Backend.beJVM.log'.format(g) + cmd = '{} {} -jar {}/specjbb2015.jar -m BACKEND {} -G={} -J=beJVM'.format(self.jdk, self.jvm_options, path, opts, g) + procs.append([Popen(shlex.split(cmd), cwd=path, stdout=PIPE, stderr=STDOUT), + open(os.path.join(path, be_name), 'wb')]) + #os.system('java {} -jar {}/specjbb2015.jar -m BACKEND {} -G={} -J=beJVM Group{}.Backend.beJVM.log 2>&1 &'.format(self.jdk, self.jvm_options, path, opts, g, g, g)) + + dead = False + while not dead: + dead = True + for p in procs: + if(p[0].poll() is None): + dead = False + o = p[0].stdout.readline() + if (o != ''): + handle(o) + p[1].write(o) + for p in procs: + p[0].wait() + p[1].close() + + #Each process will continue until manually terminated. return 0 - def run_multi(self, path): + def run_multi(self, path, handle): result_dir = self._prerun(path) opts = self._spec_opts() tx_opts = self._tx_opts() for x in range(self.num_runs): - os.system('{} {} -jar {}/specjbb2015.jar -m MULTICONTROLLER {} 2> {}/controller.log > {}/controller.out &'.format(self.jdk, self.jvm_options, path, opts, x, result_dir, result_dir)) + cont_std = open(os.path.join(result_dir, 'controller.log'), 'wb') + cont_err = open(os.path.join(result_dir, 'controller.out'), 'wb') + cmd = '{} {} -jar {}/specjbb2015.jar -m MULTICONTROLLER {}'.format(self.jdk, self.jvm_options, path, opts) + controller = Popen(shlex.split(cmd), cwd=path, stdout=PIPE, stderr=PIPE) + tx_procs = [] + be_procs = [] + #os.system('{} {} -jar {}/specjbb2015.jar -m MULTICONTROLLER {} 2> {}/controller.log > {}/controller.out &'.format(self.jdk, self.jvm_options, path, opts, x, result_dir, result_dir)) for g in range(self.properties.root['specjbb.group.count'].value): for j in range(self.properties.root['specjbb.txi.pergroup.count'].value): ti_name = "{}Group{}.TxInjector.txiJVM{}".format(result_dir, g, j) - os.system('{} {} -jar {}/specjbb2015.jar -m TXINJECTOR {} -G={} 2> {}.log > {}.out &'.format(self.jdk, self.jvm_options, path, tx_opts, g, ti_name, ti_name)) + cmd = '{} {} -jar {}/specjbb2015.jar -m TXINJECTOR {} -G={}'.format(self.jdk, self.jvm_options, path, tx_opts, g, ti_name, ti_name) + tx_procs.append([Popen(shlex.split(cmd), cwd=path, stdout=PIPE, stderr=PIPE), + open(os.path.join(path, '{}.log'.format(ti_name)), 'wb'), + open(os.path.join(path, '{}.out'.format(ti_name)), 'wb')]) + #os.system('{} {} -jar {}/specjbb2015.jar -m TXINJECTOR {} -G={} 2> {}.log > {}.out &'.format(self.jdk, self.jvm_options, path, tx_opts, g, ti_name, ti_name)) be_name = "{}{}.Backend.beJVM".format(result_dir, g) - os.system( - '{} {} -jar {}/specjbb2015.jar -m BACKEND {} -G={} -J=beJVM 2> {}.log > {}.out &'.format(self.jdk, - self.jvm_options, + cmd = '{} {} -jar {}/specjbb2015.jar -m BACKEND {} -G={} -J=beJVM'.format(self.jdk,self.jvm_options, path, tx_opts, - g, be_name, - be_name)) + g) + be_procs.append([Popen(shlex.split(cmd), cwd=path, stdout=PIPE, stderr=PIPE), + open(os.path.join(path, '{}.log'.format(be_name)), 'wb'), + open(os.path.join(path, '{}.out'.format(be_name)), 'wb')]) + + while (controller.poll() is None): + e = controller.stderr.readline() + if (e != ''): + cont_err.write(e) + handle(e) + o = controller.stdout.readline() + if (o != ''): + cont_std.write(o) + handle(o) + for p in tx_procs: + if (p[0].poll() is None): + e = p[0].stderr.readline() + if (e != ''): + p[2].write(e) + handle(e) + o = p[0].stdout.readline() + if (o != ''): + p[1].write(o) + handle(o) + for p in be_procs: + if (p[0].poll() is None): + e = p[0].stderr.readline() + if (e != ''): + p[2].write(e) + handle(e) + o = p[0].stdout.readline() + if (o != ''): + p[1].write(o) + handle(o) + cont_err.close() + cont_std.close() + controller.wait() + for p in tx_procs: + p[0].kill() + p[1].close() + p[2].close() + for p in be_procs: + p[0].kill() + p[1].close() + p[2].close() + # os.system( + # '{} {} -jar {}/specjbb2015.jar -m BACKEND {} -G={} -J=beJVM 2> {}.log > {}.out &'.format(self.jdk, + # self.jvm_options, + ## path, tx_opts, + # g, be_name, + # be_name)) return 0 @@ -460,23 +600,24 @@ def __init__(self, fromjson = None): #dict(zip(lambda x:{x.prop: x}, defaults)) if(not fromjson is None): for p in fromjson['modified']: - self.root.update({p['prop'], p['value']}) + self.root.update({p['prop'] : p['value']}) def set(self, key, value): if(key in self.root and self.root[key].value_validator(value)): self.root[key].value = value def get_all(self): - return self.root.values() + return list(self.root.values()) def get_modified(self): - return [x for x in self.root.values() if x.value != x.def_value] + return [x for x in list(self.root.values()) if isinstance(x, propitem) and x.value != x.def_value] def writeconfig(self, path): with open(path, 'w') as f: f.write("#SPECjbb config") for p in self.root.values(): - p.write(f) + if( isinstance(p, propitem)): + p.write(f) #f.writelines(map(lambda x:"{}={}".format(x.prop, x.value), filter(lambda x:x.value != x.def_value, self.dict.itervalues()))) def tojson(self): diff --git a/pycurses.py b/pycurses.py deleted file mode 100644 index e69de29..0000000 diff --git a/speccurses.py b/speccurses.py index 8dcf0dd..b28e3f6 100644 --- a/speccurses.py +++ b/speccurses.py @@ -1,3 +1,5 @@ +from __future__ import print_function + import os import curses import json @@ -371,13 +373,37 @@ def run_config(stdscr): config, path = draw_get_config_path(stdscr) if (config is None): return - path = "" + +# stdscr.clear() +# stdscr.refresh() +# height, width = stdscr.getmaxyx() +# cury = draw_title(stdscr) +# index = 0 +# def _handle(msg): + # c = index % maxindex +# pad.addstr(0, 0, msg) + # if cury < maxy: + # cury += 1 +# pad.refresh(0, 0, 20, xoffset, 20, width - xoffset) + #log[index % maxindex] = msg + + #index += 1 + + #starty = y + #maxy = height - 3 + #log = [height - y - 3] + #maxindex = len(log) + +# pad = curses.newpad(height, width - 1 - xoffset) + stdscr.keypad(0) curses.echo() curses.nocbreak() curses.endwin() for r in config.runs: - result =r.run(path) + #draw_status_bar(stdscr, "Starting run '{}'".format(r.tag)) + + result = r.run(path) if(result == 2): curses.noecho() curses.cbreak() From ed83b344ff1d7c9eb1494cf17f879e3ea542ab0e Mon Sep 17 00:00:00 2001 From: David Hiatt Date: Thu, 26 Apr 2018 18:49:30 -0700 Subject: [PATCH 2/3] curses now uses a handle, still very cluttered --- objects.py | 552 ++++++++++++++++++++++++-------------------------- speccurses.py | 444 ++++++++++++++++++++++++---------------- 2 files changed, 538 insertions(+), 458 deletions(-) diff --git a/objects.py b/objects.py index 829bfb2..79e578e 100644 --- a/objects.py +++ b/objects.py @@ -10,31 +10,34 @@ class spec_config(): - #runs contains a list of spec_run's - def __init__(self, fromjson = None): + # runs contains a list of spec_run's + def __init__(self, fromjson=None): self.runs = [] - if(not fromjson is None): + if not fromjson is None: for r in fromjson['runs']: self.runs.append(spec_run(r)) def tojson(self): + """Returns dictionary of json terms. Should only be called internally""" return { - "_type" : "spec_config", - "runs" : list(map(lambda x:x.tojson(), self.runs)) + "_type": "spec_config", + "runs": list(map(lambda x: x.tojson(), self.runs)) } def save(self, path): + """Call this to save this config to a json file.""" with open(path, 'w') as f: - json.dump(self, f, cls=spec_encoder) - + json.dump(self, f, indent=4, cls=spec_encoder) class spec_encoder(json.JSONEncoder): def default(self, obj): - if(isinstance(obj, spec_config) or isinstance(obj, spec_run) or isinstance(obj, props) or isinstance(obj, propitem)): + if (isinstance(obj, spec_config) or isinstance(obj, spec_run) or isinstance(obj, props) or isinstance(obj, + propitem)): return obj.tojson() return super(spec_encoder, self).default(obj) + class spec_decoder(json.JSONDecoder): def __init__(self, *args, **kwargs): json.JSONDecoder.__init__(self, object_hook=self.object_hook, *args, **kwargs) @@ -48,16 +51,9 @@ def object_hook(self, obj): return obj -run_types = [ - 'composite', - 'distributed_ctrl_txl', - 'distributed_sut', - 'multi' -] - class spec_run: - def __init__(self, fromjson = None): - if(fromjson is None): + def __init__(self, fromjson=None): + if (fromjson is None): self.properties = props() self.jdk = "/usr/bin/java" self.jvm_options = "-Xms29g -Xmx29g -Xmn27g -XX:ParallelGCThreads=48" @@ -65,15 +61,15 @@ def __init__(self, fromjson = None): self.num_runs = 1 self.numa_nodes = 1 self.tag = "tag-name" - self.run_type = 'composite' #must be 'multi, 'distributed_ctrl_txl', 'distributed_sut', 'multi' + self.run_type = 'composite' # must be 'multi, 'distributed_ctrl_txl', 'distributed_sut', 'multi' self.verbose = False - self.report_level = 0 #must be between 0-3 + self.report_level = 0 # must be between 0-3 self.skip_report = False self.ignore_kit_validation = False else: self.jdk = fromjson['jdk'] self.jvm_options = fromjson['jvm_options'] - self.data_collection =fromjson['data_collection'] + self.data_collection = fromjson['data_collection'] self.num_runs = fromjson['num_runs'] self.tag = fromjson['tag'] self.run_type = fromjson['run_type'] @@ -85,108 +81,123 @@ def __init__(self, fromjson = None): self.ignore_kit_validation = fromjson['ignore_kit_validation'] def set_runtype(self, arg): - if(arg in run_types): + """Ensure that arg is a valid runtype before setting the runtype""" + if (arg in run_types): self.run_type = arg - def runtype(self): - return self.run_type - def tojson(self): + """Returns dictionary of json terms. Should only be called internally""" return { - "jdk" : self.jdk, - "jvm_options" : self.jvm_options, - "data_collection" : self.data_collection, - "num_runs" : self.num_runs, - "run_type" : self.run_type, - "tag" : self.tag, - "numa_nodes" : self.numa_nodes, - "verbose" : self.verbose, - "report_level" : self.report_level, - "skip_report" : self.skip_report, - "ignore_kit_validation" : self.ignore_kit_validation, - "props" : self.properties.tojson() + "jdk": self.jdk, + "jvm_options": self.jvm_options, + "data_collection": self.data_collection, + "num_runs": self.num_runs, + "run_type": self.run_type, + "tag": self.tag, + "numa_nodes": self.numa_nodes, + "verbose": self.verbose, + "report_level": self.report_level, + "skip_report": self.skip_report, + "ignore_kit_validation": self.ignore_kit_validation, + "props": self.properties.tojson() } def _defHandle(msg): print(msg) - #handle = Any function that takes a single string argument. - #It can be used to intercept any output - #If set to None, then output will only be logged - def run(self, path = "", handle = _defHandle): - if(not os.path.exists(self.jdk)): - return 3 - if(path == ""): + # handle = Any function that takes a single string argument. + # It can be used to intercept any output + # If set to None, then output will only be logged + def run(self, path="", handle=_defHandle): + """ + Runs with the current settings. Auto detects runtype, writes the config file, and executes all required processes + :param path: The path to a directory containing 'specjbb2015.jar' + :param handle: An output handler. Will receive byte encoded strings? + If left blank, all output will be 'printed' + :return: 0 -> All runs completed successfully + 2 -> Failed to located 'specjbb2015.jar' in the path + 4 -> Failed to ping the host controller + -1 -> An error ocurred executing specjbb + """ + if (not os.path.exists(self.jdk)): + return 2 + if (path == ""): path = os.getcwd() - if(not os.path.exists(os.path.join(path, "specjbb2015.jar"))): + if (not os.path.exists(os.path.join(path, "specjbb2015.jar"))): handle('Failed to locate "specjbb2015.jar" in "{}"'.format(path)) return 2 - # orig = os.getcwd() -# os.chdir(path) + # orig = os.getcwd() + # os.chdir(path) switch = { 'composite': self.run_composite, - 'distributed_ctrl_txl' : self.run_distributed_ctrl_txl, - 'distributed_sut' : self.distributed_sut, - 'multi' : self.run_multi + 'distributed_ctrl_txl': self.run_distributed_ctrl_txl, + 'distributed_sut': self.distributed_sut, + 'multi': self.run_multi } ret = switch[self.run_type](path, handle) - # os.chdir(orig) + # os.chdir(orig) return ret def run_composite(self, path, handle): + """Called internally only by this.run()""" cmd = '{} {} -jar {}/specjbb2015.jar -m COMPOSITE {}'.format( self.jdk, self.jvm_options, path, self._spec_opts()) for x in range(self.num_runs): result_dir = self._prerun(path) handle('Starting run number {}.'.format(x)) + handle('Using command: "{}"'.format(cmd)) errout = open(os.path.join(result_dir, 'composite.out'), 'wb') stdout = open(os.path.join(result_dir, 'composite.log'), 'wb') p = Popen(shlex.split(cmd), cwd=path, stdout=PIPE, stderr=PIPE) - while(p.poll() is None): + while (p.poll() is None): e = p.stderr.readline() - if(e != ''): + if (e != ''): errout.write(e) handle(e) o = p.stdout.readline() - if(o != ''): + if (o != ''): stdout.write(o) handle(o) errout.close() stdout.close() - p.wait() + exitcode = p.wait() + if (exitcode != 0): + return -1 return 0 def run_distributed_ctrl_txl(self, path, handle): + """Called internally only by this.run()""" ctrl_ip = self.properties.root['specjbb.controller.host'].value - #Check if ip responds here + # Check if ip responds here pingcmd = 'ping -c 1 {}'.format(ctrl_ip) FNULL = open(os.devnull, 'w') ping = Popen(shlex.split(pingcmd), stderr=FNULL, stdout=FNULL) exitcode = ping.wait() FNULL.close() - if(exitcode != 0): + if (exitcode != 0): handle('ERROR: Failed to ping Controller host (specjbb.controller.host): {}'.format(ctrl_ip)) return 4 opts = self._spec_opts() tx_opts = self._tx_opts() - result_dir= self._prerun(path) + result_dir = self._prerun(path) cmd = '{} {} -jar {}/specjbb2015.jar -m DISTCONTROLLER {}'.format(self.jdk, self.jvm_options, path, opts) tx_procs = [] cont_std = open(os.path.join(result_dir, 'controller.log'), 'wb') cont_err = open(os.path.join(result_dir, 'controller.out'), 'wb') controller = Popen(shlex.split(cmd), cwd=path, stdout=PIPE, stderr=PIPE) - #os.system('{} {} -jar {}/specjbb2015.jar -m DISTCONTROLLER {} 2> {}/controller.log > {}/controller.out &'.format(self.jdk, self.jvm_options, path, opts, result_dir, result_dir)) + # os.system('{} {} -jar {}/specjbb2015.jar -m DISTCONTROLLER {} 2> {}/controller.log > {}/controller.out &'.format(self.jdk, self.jvm_options, path, opts, result_dir, result_dir)) for g in range(self.properties.root['specjbb.group.count'].value): for j in range(self.properties.root['specjbb.txi.pergroup.count'].value): ti_name = "{}Group{}.TxInjector.txiJVM{}".format(result_dir, g, j) - cmd = '{} {} -jar {}/specjbb2015.jar -m TXINJECTOR -G={}'.format(self.jdk, self.jvm_options, path, tx_opts, g) + cmd = '{} {} -jar {}/specjbb2015.jar -m TXINJECTOR -G={}'.format(self.jdk, self.jvm_options, path, + tx_opts, g) tx_procs.append([Popen(shlex.split(cmd), cwd=path, stdout=PIPE, stderr=PIPE), open(os.path.join(result_dir, '{}.log'.format(ti_name)), 'wb'), open(os.path.join(result_dir, '{}.out'.format(ti_name)), 'wb')]) - #os.system('{} {} -jar {}/specjbb2015.jar -m TXINJECTOR -G={} 2> {}.log > {}.out &'.format(self.jdk, self.jvm_options, path, tx_opts, g, ti_name, ti_name)) + # os.system('{} {} -jar {}/specjbb2015.jar -m TXINJECTOR -G={} 2> {}.log > {}.out &'.format(self.jdk, self.jvm_options, path, tx_opts, g, ti_name, ti_name)) while (controller.poll() is None): e = controller.stderr.readline() if (e != ''): @@ -197,7 +208,7 @@ def run_distributed_ctrl_txl(self, path, handle): cont_std.write(o) handle(o) for p in tx_procs: - if(p[0].poll() is None): + if (p[0].poll() is None): e = p[0].stderr.readline() if (e != ''): p[2].write(e) @@ -208,14 +219,17 @@ def run_distributed_ctrl_txl(self, path, handle): handle(o) cont_err.close() cont_std.close() - controller.wait() + exitcode = controller.wait() for p in tx_procs: p[0].kill() p[1].close() p[2].close() + if (exitcode != 0): + return -1 return 0 def distributed_sut(self, path, handle): + """Called internally only by this.run()""" ctrl_ip = self.properties.root['specjbb.controller.host'].value # Check if ip responds here pingcmd = 'ping -c 1 {}'.format(ctrl_ip) @@ -230,16 +244,17 @@ def distributed_sut(self, path, handle): procs = [] for g in range(self.properties.root['specjbb.group.count'].value): be_name = 'beJVM Group{}.Backend.beJVM.log'.format(g) - cmd = '{} {} -jar {}/specjbb2015.jar -m BACKEND {} -G={} -J=beJVM'.format(self.jdk, self.jvm_options, path, opts, g) + cmd = '{} {} -jar {}/specjbb2015.jar -m BACKEND {} -G={} -J=beJVM'.format(self.jdk, self.jvm_options, path, + opts, g) procs.append([Popen(shlex.split(cmd), cwd=path, stdout=PIPE, stderr=STDOUT), open(os.path.join(path, be_name), 'wb')]) - #os.system('java {} -jar {}/specjbb2015.jar -m BACKEND {} -G={} -J=beJVM Group{}.Backend.beJVM.log 2>&1 &'.format(self.jdk, self.jvm_options, path, opts, g, g, g)) + # os.system('java {} -jar {}/specjbb2015.jar -m BACKEND {} -G={} -J=beJVM Group{}.Backend.beJVM.log 2>&1 &'.format(self.jdk, self.jvm_options, path, opts, g, g, g)) dead = False while not dead: dead = True for p in procs: - if(p[0].poll() is None): + if (p[0].poll() is None): dead = False o = p[0].stdout.readline() if (o != ''): @@ -249,10 +264,11 @@ def distributed_sut(self, path, handle): p[0].wait() p[1].close() - #Each process will continue until manually terminated. + # Each process will continue until manually terminated. return 0 def run_multi(self, path, handle): + """Called internally only by this.run()""" result_dir = self._prerun(path) opts = self._spec_opts() tx_opts = self._tx_opts() @@ -263,22 +279,24 @@ def run_multi(self, path, handle): controller = Popen(shlex.split(cmd), cwd=path, stdout=PIPE, stderr=PIPE) tx_procs = [] be_procs = [] - #os.system('{} {} -jar {}/specjbb2015.jar -m MULTICONTROLLER {} 2> {}/controller.log > {}/controller.out &'.format(self.jdk, self.jvm_options, path, opts, x, result_dir, result_dir)) + # os.system('{} {} -jar {}/specjbb2015.jar -m MULTICONTROLLER {} 2> {}/controller.log > {}/controller.out &'.format(self.jdk, self.jvm_options, path, opts, x, result_dir, result_dir)) for g in range(self.properties.root['specjbb.group.count'].value): for j in range(self.properties.root['specjbb.txi.pergroup.count'].value): ti_name = "{}Group{}.TxInjector.txiJVM{}".format(result_dir, g, j) - cmd = '{} {} -jar {}/specjbb2015.jar -m TXINJECTOR {} -G={}'.format(self.jdk, self.jvm_options, path, tx_opts, g, ti_name, ti_name) + cmd = '{} {} -jar {}/specjbb2015.jar -m TXINJECTOR {} -G={}'.format(self.jdk, self.jvm_options, + path, tx_opts, g, ti_name, + ti_name) tx_procs.append([Popen(shlex.split(cmd), cwd=path, stdout=PIPE, stderr=PIPE), open(os.path.join(path, '{}.log'.format(ti_name)), 'wb'), open(os.path.join(path, '{}.out'.format(ti_name)), 'wb')]) - #os.system('{} {} -jar {}/specjbb2015.jar -m TXINJECTOR {} -G={} 2> {}.log > {}.out &'.format(self.jdk, self.jvm_options, path, tx_opts, g, ti_name, ti_name)) + # os.system('{} {} -jar {}/specjbb2015.jar -m TXINJECTOR {} -G={} 2> {}.log > {}.out &'.format(self.jdk, self.jvm_options, path, tx_opts, g, ti_name, ti_name)) be_name = "{}{}.Backend.beJVM".format(result_dir, g) - cmd = '{} {} -jar {}/specjbb2015.jar -m BACKEND {} -G={} -J=beJVM'.format(self.jdk,self.jvm_options, - path, tx_opts, - g) + cmd = '{} {} -jar {}/specjbb2015.jar -m BACKEND {} -G={} -J=beJVM'.format(self.jdk, self.jvm_options, + path, tx_opts, + g) be_procs.append([Popen(shlex.split(cmd), cwd=path, stdout=PIPE, stderr=PIPE), - open(os.path.join(path, '{}.log'.format(be_name)), 'wb'), - open(os.path.join(path, '{}.out'.format(be_name)), 'wb')]) + open(os.path.join(path, '{}.log'.format(be_name)), 'wb'), + open(os.path.join(path, '{}.out'.format(be_name)), 'wb')]) while (controller.poll() is None): e = controller.stderr.readline() @@ -311,7 +329,7 @@ def run_multi(self, path, handle): handle(o) cont_err.close() cont_std.close() - controller.wait() + exitcode = controller.wait() for p in tx_procs: p[0].kill() p[1].close() @@ -320,18 +338,21 @@ def run_multi(self, path, handle): p[0].kill() p[1].close() p[2].close() - # os.system( - # '{} {} -jar {}/specjbb2015.jar -m BACKEND {} -G={} -J=beJVM 2> {}.log > {}.out &'.format(self.jdk, - # self.jvm_options, - ## path, tx_opts, - # g, be_name, - # be_name)) - + if (exitcode != 0): + return -1 + # os.system( + # '{} {} -jar {}/specjbb2015.jar -m BACKEND {} -G={} -J=beJVM 2> {}.log > {}.out &'.format(self.jdk, + # self.jvm_options, + ## path, tx_opts, + # g, be_name, + # be_name)) return 0 def _prerun(self, path): - result_dir = "{}/{}-{}".format(path, datetime.datetime.fromtimestamp(time.time()).strftime('+%y-%m-%d_%H%M%S'), self.tag) + """Called internally only. Builds a result directory and writes the current config to it.""" + result_dir = "{}/{}-{}".format(path, datetime.datetime.fromtimestamp(time.time()).strftime('+%y-%m-%d_%H%M%S'), + self.tag) os.makedirs(result_dir) if (not os.path.exists("{}/config".format(path))): os.makedirs("{}/config".format(path)) @@ -339,60 +360,67 @@ def _prerun(self, path): return result_dir def _tx_opts(self): - opts= "" - if(self.verbose): + """Called internally only. Obtains specjbb options available to TXINJECTOR and BACKEND""" + opts = "" + if (self.verbose): opts += " -v" - if(self.ignore_kit_validation): + if (self.ignore_kit_validation): opts += " -ikv" return opts def _spec_opts(self): - opts= "-l {}".format(self.report_level) - if(self.skip_report): + """Called internally only. Obtains specjbb options available to all except TXINJECTOR and BACKEND""" + opts = "-l {}".format(self.report_level) + if (self.skip_report): opts += " -skipReport" - if(self.verbose): + if (self.verbose): opts += " -v" - if(self.ignore_kit_validation): + if (self.ignore_kit_validation): opts += " -ikv" return opts - - class propitem: - def __init__(self, prop, def_value, desc, input_validator, value_validator, valid_opts = None, help_text = ""): + def __init__(self, prop, def_value, desc, input_validator, value_validator, valid_opts=None, help_text=""): self.prop = prop self.def_value = def_value self.desc = desc self.input_validator = input_validator self.value_validator = value_validator self.help_text = help_text - if(valid_opts is None): + if (valid_opts is None): self.valid_opts = [] else: self.valid_opts = valid_opts self.value = def_value def write(self, f): - if(self.value != self.def_value): + """Called internally only. Writes to a config file if different than the default value""" + if (self.value != self.def_value): f.write("{} = {}".format(self.prop, self.value)) def set(self, arg): - if(self.value_validator(arg)): + """Use this to set this property value using a validator""" + if (self.value_validator(arg)): self.value = arg def reset(self): + """Resets this property to the default value""" self.value = self.def_value - def tojson(self): + """Called internally only. Returns dictionary of json values""" return { - # "_type" : "propitem", - "prop" : self.prop, - "value" : self.value + "prop": self.prop, + "value": self.value } - +run_types = [ + 'composite', + 'distributed_ctrl_txl', + 'distributed_sut', + 'multi' +] loglevels = [ 'SEVERE', @@ -404,13 +432,6 @@ def tojson(self): 'FINEST' ] -def strtobool(str): - if str.lower() in ("yes", "true", "t", "1"): - return True - if str.lower() in ("no", "false", "f", "0"): - return False - return None - con_types = [ 'HBIR_RT', 'HBIR', @@ -422,205 +443,168 @@ def strtobool(str): must_be_positive = "Value must be greater than 0" -number_validator = lambda x:int(x)<=ord('9') and int(x)>=ord('0') -float_validator = lambda x:(x<=ord('9') and x>=ord('0')) or x == ord('.') -default_validator = lambda x:True - - +number_validator = lambda x: int(x) <= ord('9') and int(x) >= ord('0') +float_validator = lambda x: (x <= ord('9') and x >= ord('0')) or x == ord('.') +default_validator = lambda x: True defaults = [ -propitem('specjbb.controller.type', 'HBIR_RT', 'Controls phases being controlled by Controller.', default_validator, lambda x:x in con_types, con_types), - propitem('specjbb.comm.connect.client.pool.size', 256, 'Network connection pool size, i.e. number of sockets for I/O communication for each Agent.', - number_validator, lambda x:int(x)>0,help_text=must_be_positive), -propitem('specjbb.comm.connect.worker.pool.min', 1,'Minimum number of worker threads in the network connection pool.', - number_validator, lambda x:int(x)>0,help_text=must_be_positive), -propitem('specjbb.comm.connect.worker.pool.max', 256, 'Maximum number of worker threads in the network connection pool.', - number_validator, lambda x:int(x)>0,help_text=must_be_positive), -propitem('specjbb.comm.connect.selector.runner.count', 0,'Number of acceptor threads for handling new connections and scheduling existing ones.', - number_validator, lambda x:int(x)>=0,help_text="Value must an integer. Special '0' value will force to using the default connectivity provider setting"), - propitem('specjbb.comm.connect.timeouts.connect', 60000,'Timeout (in milliseconds) for I/O connection operation.', - number_validator, lambda x:int(x)>0,help_text=must_be_positive), - propitem('specjbb.comm.connect.timeouts.read' ,60000, 'Timeout (in milliseconds) for I/O read operation.', - number_validator, lambda x:int(x)>0,help_text=must_be_positive), - propitem('specjbb.comm.connect.timeouts.write' , 60000, 'Timeout (in milliseconds) for I/O write operation.', - number_validator, lambda x:int(x)>0,help_text=must_be_positive), - -propitem('specjbb.controller.host', "localhost", 'IP address / host name of the machine where Controller program will be launched.', - default_validator, default_validator,help_text=must_be_positive), -propitem('specjbb.controller.port', 24000, 'The network port to which Controller listener will bind.', number_validator, lambda x:int(x)>0,help_text=must_be_positive), - - propitem('specjbb.controller.handshake.period',5000, 'Time period (in milliseconds) for logging status of the initial Controller <-> Agent handshaking.', - number_validator, lambda x:int(x)>0,help_text=must_be_positive), - propitem('specjbb.controller.handshake.timeout' ,600000, 'Timeout (in milliseconds) for initial Controller <-> Agent handshaking.', - number_validator, lambda x:int(x)>0,help_text=must_be_positive), - propitem('specjbb.controller.maxir.maxFailedPoints',3, ' Number of points to try after max-jOPS was found to ensure there are no more passes and max-jOPS value is correct.', - number_validator, lambda x:int(x)>=0,help_text="Value must be greater than or equal to 0"), - -propitem('specjbb.controller.preset.ir', 1000, 'Sets IR for preset for controller type ', - number_validator, lambda x:x>=0,help_text="Value must be greater than or equal to 0"), -propitem('specjbb.controller.preset.duration', 600000, 'Sets duration in milliseconds for preset for controller type ', - number_validator, lambda x:int(x)>0,help_text="Value must be greater than or equal to 0"), - -propitem('specjbb.controller.rtcurve.duration.min', 60000, 'Sets duration of steady period of RT step level in milliseconds', - number_validator, lambda x:int(x)>0,help_text=must_be_positive), -propitem('specjbb.controller.rtcurve.duration.max', 90000, 'Sets duration of steady period of RT step level in milliseconds', - number_validator, lambda x:int(x)>0,help_text=must_be_positive), -propitem('specjbb.controller.rtcurve.start', 0, 'Sets the RT start percent', - number_validator, lambda x:int(x)>=0 and int(x) <= 100,help_text="Value must be between 0 and 100"), -propitem('specjbb.controller.rtcurve.step', 0.01,'Sets the RT step level to a percent', - float_validator, lambda x:float(x)>0 and float(x) <= 1,help_text="Value must be a decimal greater than 0 and less than or equal to 1"), -propitem('specjbb.controller.rtcurve.warmup.step', 0.1, 'Injection rate for warming up before response-time curve building defined as the percent of the high-bound.', - float_validator, lambda x:float(x)>0 and float(x) <= 1,help_text="Value must be a decimal greater than 0 and less than or equal to 1"), - -propitem('specjbb.controller.settle.time.min', 3000,'Sets duration of settle period of RT step level in milliseconds', number_validator, lambda x:int(x)>0,help_text=must_be_positive), -propitem('specjbb.controller.settle.time.max', 30000, 'Sets duration of settle period of RT step level in milliseconds',number_validator, lambda x:int(x)>0,help_text=must_be_positive), - - - - -#propitem('specjbb.customerDriver.threads', 64, 'Maximum number of threads in ThreadPoolExecutor for all three probe/saturate/service requests on the TxInjector side.', lambda x: isinstance(x, int) and x >= 64,help_text=must_be_positive) -propitem('specjbb.customerDriver.threads.saturate', 64, 'Maximum number of threads in ThreadPoolExecutor for saturate requests on the TxInjector side.', - number_validator, lambda x: int(x) >= 64,help_text="Value must be greater than or equal to 64"), -propitem('specjbb.customerDriver.threads.probe', 64, 'Maximum number of threads in ThreadPoolExecutor for probe requests on the TxInjector side.', number_validator, - lambda x: int(x) >= 64,help_text="Value must be greater than or equal to 64"), -propitem('specjbb.customerDriver.threads.service', 64, 'Maximum number of threads in ThreadPoolExecutor for service requests on the TxInjector side.', number_validator, - lambda x: int(x)>= 64,help_text="Value must be greater than or equal to 64"), -#propitem('specjbb.forkjoin.workers', multiprocessing.cpu_count() * 2, 'Maximum number of worker threads in ForkJoinPool in each tier on the Backend side.', lambda x:isinstance(x, int),help_text=must_be_positive) - propitem('specjbb.forkjoin.workers.Tier1', multiprocessing.cpu_count() * 2, 'Maximum number of worker threads in ForkJoinPool in tier 1 on the Backend side.', - number_validator, lambda x: int(x) > 0,help_text=must_be_positive), - propitem('specjbb.forkjoin.workers.Tier2', multiprocessing.cpu_count() * 2, 'Maximum number of worker threads in ForkJoinPool in tier 2 on the Backend side.', number_validator, - lambda x:int(x)>0,help_text=must_be_positive), - propitem('specjbb.forkjoin.workers.Tier3', multiprocessing.cpu_count() * 2, 'Maximum number of worker threads in ForkJoinPool in tier 3 on the Backend side.', number_validator, - lambda x:int(x)>0,help_text=must_be_positive), -propitem('specjbb.group.count', 1, 'Number of Groups for the run, where Group is TxInjector(s) mapped to Backend.', number_validator, lambda x:int(x)>0,help_text=must_be_positive), - - propitem('specjbb.heartbeat.period', 10000, 'How often (in milliseconds) Controller sends heartbeat message to an Agent checking it is alive',number_validator, - lambda x:int(x)>0,help_text=must_be_positive), -propitem('specjbb.heartbeat.threshold', 100000, 'How much time (in milliseconds) await for heartbeat response from an Agent.',number_validator, lambda x:int(x)>0,help_text=must_be_positive), - -propitem('specjbb.time.server', False, 'Enables Controller communication with Time Server.', default_validator, lambda x:x is bool,[False, True]), - propitem('specjbb.txi.pergroup.count', 1, 'Number of TxInjectors per Backend in one Group.', number_validator, lambda x:int(x)>0,help_text=must_be_positive), - -propitem('specjbb.run.datafile.dir', '.', 'Directory for storing binary log file of the run.', default_validator, default_validator,help_text="Enter a directory"), -propitem('specjbb.mapreducer.pool.size', 2, 'Controller ForkJoinPool size supporting parallel work of TxInjector/Backend agents.', number_validator, lambda x:int(x)>0,help_text=must_be_positive), - -propitem('specjbb.input.number_customers', 100000, 'Total number of customers', number_validator, lambda x:int(x)>0,help_text=must_be_positive), -propitem('specjbb.input.number_products', 100000, ' Number of products in each Supermarket', number_validator, lambda x:int(x)>0,help_text=must_be_positive), - -propitem('specjbb.logLevel', 'INFO' , 'Log level output', default_validator, lambda x:x in loglevels, loglevels), - -propitem('specjbb.time.check.interval', 10000, 'Time interval (in milliseconds) for periodic time check from Time Server', number_validator, lambda x:int(x)>0,help_text=must_be_positive), -propitem('specjbb.time.offset.max', 600000, 'Maximum time offset (in milliseconds) between Controller and Time Server on the benchmark start',number_validator, lambda x:int(x)>0,help_text=must_be_positive) -] - -hbir = [ - 'specjbb.controller.rtcurve.start', - 'specjbb.group.count', - 'specjbb.forkjoin.workers.Tier1', - 'specjbb.forkjoin.workers.Tier2', - 'specjbb.forkjoin.workers.Tier3', -] -loadlevels = [ - 'specjbb.controller.loadlevel.duration.min', - 'specjbb.controller.loadlevel.duration.max', - 'specjbb.controller.rtcurve.start', - 'specjbb.group.count', - 'specjbb.forkjoin.workers.Tier1', - 'specjbb.forkjoin.workers.Tier2', - 'specjbb.forkjoin.workers.Tier3', -] -presets =[ - 'specjbb.controller.preset.ir', - 'specjbb.controller.preset.duration', - 'specjbb.group.count', - 'specjbb.forkjoin.workers.Tier1', - 'specjbb.forkjoin.workers.Tier2', - 'specjbb.forkjoin.workers.Tier3', -] -multi_req = [ -'specjbb.group.count', -'specjbb.txi.pergroup.count' -] - -defhbir = [ -('specjbb.time.server',False), -('specjbb.comm.connect.client.pool.size', 192), -('specjbb.comm.connect.selector.runner.count', 4), -('specjbb.comm.connect.timeouts.connect', 650000), -('specjbb.comm.connect.timeouts.read', 650000), -('specjbb.comm.connect.timeouts.write', 650000), -('specjbb.comm.connect.worker.pool.max', 320), -('specjbb.customerDriver.threads', 64), -('specjbb.customerDriver.threads.saturate', 144), -('specjbb.customerDriver.threads.probe', 96), -('specjbb.mapreducer.pool.size', 27), -] -defloadlevels = [ -('specjbb.controller.loadlevel.start ', .95), -('specjbb.controller.loadlevel.step ', 1), -('specjbb.time.server', False), - -('specjbb.comm.connect.client.pool.size', 192), -('specjbb.comm.connect.selector.runner.count', 4), -('specjbb.comm.connect.timeouts.connect', 650000), -('specjbb.comm.connect.timeouts.read', 650000), -('specjbb.comm.connect.timeouts.write', 650000), -('specjbb.comm.connect.worker.pool.max', 320), -('specjbb.customerDriver.threads', 64), -('specjbb.customerDriver.threads.saturate', 144), -('specjbb.customerDriver.threads.probe', 96), -('specjbb.mapreducer.pool.size', 27), -] -defpresets =[ -('specjbb.comm.connect.client.pool.size', 192), -('specjbb.comm.connect.selector.runner.count', 4), -('specjbb.comm.connect.timeouts.connect', 650000), -('specjbb.comm.connect.timeouts.read', 650000), -('specjbb.comm.connect.timeouts.write', 650000), -('specjbb.comm.connect.worker.pool.max', 320), -('specjbb.customerDriver.threads', 64), -('specjbb.customerDriver.threads.saturate', 144), -('specjbb.customerDriver.threads.probe', 96), -('specjbb.mapreducer.pool.size', 27) + propitem('specjbb.controller.type', 'HBIR_RT', 'Controls phases being controlled by Controller.', default_validator, + lambda x: x in con_types, con_types), + propitem('specjbb.comm.connect.client.pool.size', 256, + 'Network connection pool size, i.e. number of sockets for I/O communication for each Agent.', + number_validator, lambda x: int(x) > 0, help_text=must_be_positive), + propitem('specjbb.comm.connect.worker.pool.min', 1, + 'Minimum number of worker threads in the network connection pool.', + number_validator, lambda x: int(x) > 0, help_text=must_be_positive), + propitem('specjbb.comm.connect.worker.pool.max', 256, + 'Maximum number of worker threads in the network connection pool.', + number_validator, lambda x: int(x) > 0, help_text=must_be_positive), + propitem('specjbb.comm.connect.selector.runner.count', 0, + 'Number of acceptor threads for handling new connections and scheduling existing ones.', + number_validator, lambda x: int(x) >= 0, + help_text="Value must an integer. Special '0' value will force to using the default connectivity provider setting"), + propitem('specjbb.comm.connect.timeouts.connect', 60000, 'Timeout (in milliseconds) for I/O connection operation.', + number_validator, lambda x: int(x) > 0, help_text=must_be_positive), + propitem('specjbb.comm.connect.timeouts.read', 60000, 'Timeout (in milliseconds) for I/O read operation.', + number_validator, lambda x: int(x) > 0, help_text=must_be_positive), + propitem('specjbb.comm.connect.timeouts.write', 60000, 'Timeout (in milliseconds) for I/O write operation.', + number_validator, lambda x: int(x) > 0, help_text=must_be_positive), + + propitem('specjbb.controller.host', "localhost", + 'IP address / host name of the machine where Controller program will be launched.', + default_validator, default_validator, help_text=must_be_positive), + propitem('specjbb.controller.port', 24000, 'The network port to which Controller listener will bind.', + number_validator, lambda x: int(x) > 0, help_text=must_be_positive), + + propitem('specjbb.controller.handshake.period', 5000, + 'Time period (in milliseconds) for logging status of the initial Controller <-> Agent handshaking.', + number_validator, lambda x: int(x) > 0, help_text=must_be_positive), + propitem('specjbb.controller.handshake.timeout', 600000, + 'Timeout (in milliseconds) for initial Controller <-> Agent handshaking.', + number_validator, lambda x: int(x) > 0, help_text=must_be_positive), + propitem('specjbb.controller.maxir.maxFailedPoints', 3, + ' Number of points to try after max-jOPS was found to ensure there are no more passes and max-jOPS value is correct.', + number_validator, lambda x: int(x) >= 0, help_text="Value must be greater than or equal to 0"), + + propitem('specjbb.controller.preset.ir', 1000, 'Sets IR for preset for controller type ', + number_validator, lambda x: x >= 0, help_text="Value must be greater than or equal to 0"), + propitem('specjbb.controller.preset.duration', 600000, + 'Sets duration in milliseconds for preset for controller type ', + number_validator, lambda x: int(x) > 0, help_text="Value must be greater than or equal to 0"), + + propitem('specjbb.controller.rtcurve.duration.min', 60000, + 'Sets duration of steady period of RT step level in milliseconds', + number_validator, lambda x: int(x) > 0, help_text=must_be_positive), + propitem('specjbb.controller.rtcurve.duration.max', 90000, + 'Sets duration of steady period of RT step level in milliseconds', + number_validator, lambda x: int(x) > 0, help_text=must_be_positive), + propitem('specjbb.controller.rtcurve.start', 0, 'Sets the RT start percent', + number_validator, lambda x: int(x) >= 0 and int(x) <= 100, help_text="Value must be between 0 and 100"), + propitem('specjbb.controller.rtcurve.step', 0.01, 'Sets the RT step level to a percent', + float_validator, lambda x: float(x) > 0 and float(x) <= 1, + help_text="Value must be a decimal greater than 0 and less than or equal to 1"), + propitem('specjbb.controller.rtcurve.warmup.step', 0.1, + 'Injection rate for warming up before response-time curve building defined as the percent of the high-bound.', + float_validator, lambda x: float(x) > 0 and float(x) <= 1, + help_text="Value must be a decimal greater than 0 and less than or equal to 1"), + + propitem('specjbb.controller.settle.time.min', 3000, + 'Sets duration of settle period of RT step level in milliseconds', number_validator, lambda x: int(x) > 0, + help_text=must_be_positive), + propitem('specjbb.controller.settle.time.max', 30000, + 'Sets duration of settle period of RT step level in milliseconds', number_validator, lambda x: int(x) > 0, + help_text=must_be_positive), + + # propitem('specjbb.customerDriver.threads', 64, 'Maximum number of threads in ThreadPoolExecutor for all three probe/saturate/service requests on the TxInjector side.', lambda x: isinstance(x, int) and x >= 64,help_text=must_be_positive) + propitem('specjbb.customerDriver.threads.saturate', 64, + 'Maximum number of threads in ThreadPoolExecutor for saturate requests on the TxInjector side.', + number_validator, lambda x: int(x) >= 64, help_text="Value must be greater than or equal to 64"), + propitem('specjbb.customerDriver.threads.probe', 64, + 'Maximum number of threads in ThreadPoolExecutor for probe requests on the TxInjector side.', + number_validator, + lambda x: int(x) >= 64, help_text="Value must be greater than or equal to 64"), + propitem('specjbb.customerDriver.threads.service', 64, + 'Maximum number of threads in ThreadPoolExecutor for service requests on the TxInjector side.', + number_validator, + lambda x: int(x) >= 64, help_text="Value must be greater than or equal to 64"), + # propitem('specjbb.forkjoin.workers', multiprocessing.cpu_count() * 2, 'Maximum number of worker threads in ForkJoinPool in each tier on the Backend side.', lambda x:isinstance(x, int),help_text=must_be_positive) + propitem('specjbb.forkjoin.workers.Tier1', multiprocessing.cpu_count() * 2, + 'Maximum number of worker threads in ForkJoinPool in tier 1 on the Backend side.', + number_validator, lambda x: int(x) > 0, help_text=must_be_positive), + propitem('specjbb.forkjoin.workers.Tier2', multiprocessing.cpu_count() * 2, + 'Maximum number of worker threads in ForkJoinPool in tier 2 on the Backend side.', number_validator, + lambda x: int(x) > 0, help_text=must_be_positive), + propitem('specjbb.forkjoin.workers.Tier3', multiprocessing.cpu_count() * 2, + 'Maximum number of worker threads in ForkJoinPool in tier 3 on the Backend side.', number_validator, + lambda x: int(x) > 0, help_text=must_be_positive), + propitem('specjbb.group.count', 1, 'Number of Groups for the run, where Group is TxInjector(s) mapped to Backend.', + number_validator, lambda x: int(x) > 0, help_text=must_be_positive), + + propitem('specjbb.heartbeat.period', 10000, + 'How often (in milliseconds) Controller sends heartbeat message to an Agent checking it is alive', + number_validator, + lambda x: int(x) > 0, help_text=must_be_positive), + propitem('specjbb.heartbeat.threshold', 100000, + 'How much time (in milliseconds) await for heartbeat response from an Agent.', number_validator, + lambda x: int(x) > 0, help_text=must_be_positive), + + propitem('specjbb.time.server', False, 'Enables Controller communication with Time Server.', default_validator, + lambda x: x is bool, [False, True]), + propitem('specjbb.txi.pergroup.count', 1, 'Number of TxInjectors per Backend in one Group.', number_validator, + lambda x: int(x) > 0, help_text=must_be_positive), + + propitem('specjbb.run.datafile.dir', '.', 'Directory for storing binary log file of the run.', default_validator, + default_validator, help_text="Enter a directory"), + propitem('specjbb.mapreducer.pool.size', 2, + 'Controller ForkJoinPool size supporting parallel work of TxInjector/Backend agents.', number_validator, + lambda x: int(x) > 0, help_text=must_be_positive), + + propitem('specjbb.input.number_customers', 100000, 'Total number of customers', number_validator, + lambda x: int(x) > 0, help_text=must_be_positive), + propitem('specjbb.input.number_products', 100000, ' Number of products in each Supermarket', number_validator, + lambda x: int(x) > 0, help_text=must_be_positive), + + propitem('specjbb.logLevel', 'INFO', 'Log level output', default_validator, lambda x: x in loglevels, loglevels), + + propitem('specjbb.time.check.interval', 10000, + 'Time interval (in milliseconds) for periodic time check from Time Server', number_validator, + lambda x: int(x) > 0, help_text=must_be_positive), + propitem('specjbb.time.offset.max', 600000, + 'Maximum time offset (in milliseconds) between Controller and Time Server on the benchmark start', + number_validator, lambda x: int(x) > 0, help_text=must_be_positive) ] -def get_important(type): - return { - 'HBIR_RT':1, - 'HBIR':1, - 'PRESET':1, - 'FIXED_RT':1, - 'FIXED_RT_LOADLEVELS':1, - 'FIXED_LOADLEVELS':1 - }.get(type) class props: - def __init__(self, fromjson = None): - self.root ={a.prop : a for a in defaults} - #dict(zip(lambda x:{x.prop: x}, defaults)) - if(not fromjson is None): + def __init__(self, fromjson=None): + self.root = {a.prop: a for a in defaults} + if (not fromjson is None): for p in fromjson['modified']: - self.root.update({p['prop'] : p['value']}) + self.root.update({p['prop']: p['value']}) def set(self, key, value): - if(key in self.root and self.root[key].value_validator(value)): + """Can be used to set a specific property to a value using a validator""" + if (key in self.root and self.root[key].value_validator(value)): self.root[key].value = value def get_all(self): + """Returns a list of 'propitem's""" return list(self.root.values()) def get_modified(self): + """Returns a list of modified 'propitem's""" return [x for x in list(self.root.values()) if isinstance(x, propitem) and x.value != x.def_value] def writeconfig(self, path): + """Called internally only before running any 'spec_run'""" with open(path, 'w') as f: f.write("#SPECjbb config") for p in self.root.values(): - if( isinstance(p, propitem)): + if (isinstance(p, propitem)): p.write(f) - #f.writelines(map(lambda x:"{}={}".format(x.prop, x.value), filter(lambda x:x.value != x.def_value, self.dict.itervalues()))) def tojson(self): + """Called internally only. Returns dictionary of json values""" return { - "modified" : list(map(lambda x:x.tojson(), self.get_modified())) + "modified": list(map(lambda x: x.tojson(), self.get_modified())) } diff --git a/speccurses.py b/speccurses.py index b28e3f6..467a97a 100644 --- a/speccurses.py +++ b/speccurses.py @@ -4,7 +4,7 @@ import curses import json import os.path -from simplejson import JSONDecodeError + import objects from objects import spec_decoder, spec_encoder, spec_config, spec_run @@ -26,13 +26,25 @@ STARTY = 2 xoffset = 5 -run_types = ['composite','distributed_ctrl_txl', 'distributed_sut','multi'] -#Displays a single run within a config file -def display_run(stdscr, runinfo, y =STARTY + 2): +run_types = ['composite', 'distributed_ctrl_txl', 'distributed_sut', 'multi'] + +def _remove_control_chars(s): + if(isinstance(s, str)): + return ''.join([i if ord(i) < 128 else '' for i in s]) + return _remove_control_chars(s.decode()) + +def display_run(stdscr, runinfo, y=STARTY + 2): + """ + Displays a single run within a config file + :param stdscr: The current screen + :param runinfo: The 'spec_run' to display + :param y: The current y value + :return: The new y value, as well as the max length of any property in the current run + """ mem = [attr for attr in dir(runinfo) if not callable(getattr(runinfo, attr)) and not attr.startswith("__")] valueoffset = len(max(mem, key=len)) + 2 for name in mem: - if(name == 'properties'): + if (name == 'properties'): stdscr.addstr(y, xoffset, name) else: stdscr.addstr(y, xoffset, "{}:".format(name)) @@ -40,8 +52,17 @@ def display_run(stdscr, runinfo, y =STARTY + 2): y += 1 return y, valueoffset -#Allows user to select from a list of valid options + def select_from(stdscr, x, y, value, list): + """ + Allows user to select from a list of valid options + :param stdscr: The current screen + :param x: The start x position to begin printing + :param y: The start y position to begin pritning + :param value: The current value chosen + :param list: A list of values to choose from + :return: A value within :param list + """ k = 0 pad = curses.newpad(1, 100) height, width = stdscr.getmaxyx() @@ -49,67 +70,102 @@ def select_from(stdscr, x, y, value, list): while (k != ENTER and k != ord('q')): pad.clear() draw_status_bar(stdscr, "Press 'q' to exit and 'UP' or 'DOWN' to select a value") - if(k == curses.KEY_UP and idx > 0): + if (k == curses.KEY_UP and idx > 0): idx -= 1 - elif(k == curses.KEY_DOWN and idx < len(list) - 1): + elif (k == curses.KEY_DOWN and idx < len(list) - 1): idx += 1 value = list[idx] - pad.addstr(0,0, str(value)) + pad.addstr(0, 0, str(value)) stdscr.move(y, x + len(str(value))) pad.refresh(0, 0, y, x, y, width - x) k = stdscr.getch() return value -#Allows user to input text and returns the result + def input_text(stdscr, x, y, value, validator): + """ + Allows user to input text and returns the result + :param stdscr: The current screen + :param x: The start x position to begin printing + :param y: The start y position to begin pritning + :param value: The current value chosen + :param validator: A validator to ensure only proper keys are entered. + :return: A validated value + """ k = 0 value = list(str(value)) idx = len(value) height, width = stdscr.getmaxyx() - pad = curses.newpad(1, width - xoffset) + xmax = width * 2 + pad = curses.newpad(1, xmax) + cursorx = min(x + len(value), width - 1) while (k != ENTER): pad.clear() - if(k == BACK and idx > 0): - if(idx < len(value)): - value = value[:idx-1] + value[idx:] + if (k == BACK and idx > 0): + if (idx < len(value)): + value = value[:idx - 1] + value[idx:] else: value = value[:idx - 1] idx -= 1 - elif(k >= 20 and k <= 126 and validator(k)): - if(idx == len(value)): + if(cursorx > x): + cursorx -= 1 + elif (k >= 20 and k <= 126 and validator(k)): + if (idx == len(value)): value += chr(k) + if(idx >= xmax): + xmax += width + pad = curses.newpad(1, xmax) else: value.insert(idx, chr(k)) idx += 1 - elif(k == DEL and idx < len(value)): - if(idx > 1): + if (cursorx < (width - 1)): + cursorx += 1 + elif (k == DEL and idx < len(value)): + if (idx > 1): value = value[:idx] + value[idx + 1:] else: value = value[1:] - elif(k == curses.KEY_LEFT and idx > 0 ): + elif (k == curses.KEY_LEFT and idx > 0): idx -= 1 - elif (k == curses.KEY_RIGHT and idx < len(value) ): + if (cursorx > x): + cursorx -= 1 + elif (k == curses.KEY_RIGHT and idx < len(value)): idx += 1 - - - stdscr.move(y, idx) - pad.addstr(0,0, "".join(value)) - stdscr.move(y, x + idx) - pad.refresh(0,0, y,x, y, width - x) + if (cursorx < (width - 1)): + cursorx += 1 + elif(k == curses.KEY_HOME and idx > 0): + idx = 0 + cursorx = x + elif (k == curses.KEY_END and idx < len(value)): + idx = len(value) + cursorx = min(len(value) + x, width - 1) + + stdscr.move(y, cursorx) + pad.addstr(0, 0, "".join(value)) + if(idx != (cursorx - x)): + #[0 1 2 3 4 5 6 7] + # o W + # o X + # I + # I - ((W - o) - (W - X)) + pad.refresh(0, idx - ((width - 1 - x) - (width - 1 - cursorx)), y, x, y, width - 1) + else: + pad.refresh(0, 0, y, x, y, width - 1) k = stdscr.getch() return str("".join(value)) -#Saves a config file and displays whether or not it was successful + +# Saves a config file and displays whether or not it was successful def draw_save_config(stdscr, config, path): try: - with open(path, 'w') as f: - json.dump(config, f, indent=4, cls=spec_encoder) + config.save(path) except: draw_show_message(stdscr, 'Unable to save to {}'.format(path)) return draw_show_message(stdscr, 'Saved to {}'.format(path)) + def draw_edit_props(stdscr, p): def _draw(): stdscr.clear() @@ -120,6 +176,7 @@ def _draw(): pad.refresh(index - (cury - starty), 0, starty, xoffset, height - 5, width - xoffset - 1) stdscr.move(cury, xoffset) stdscr.refresh() + index = 0 starty = STARTY + 2 cury = starty @@ -130,23 +187,23 @@ def _draw(): if k == curses.KEY_DOWN and index < len(p) - 1: index += 1 - if(cury < height - 5): + if (cury < height - 5): cury += 1 else: pad.refresh(index - (cury - starty), 0, starty, xoffset, height - 5, width - xoffset - 1) elif k == curses.KEY_UP and index > 0: index -= 1 - if(cury > starty): + if (cury > starty): cury -= 1 else: pad.refresh(index - (cury - starty), 0, starty, xoffset, height - 5, width - xoffset - 1) elif k == ENTER: - if(p[index].valid_opts != []): + if (p[index].valid_opts != []): value = select_from(stdscr, xoffset + startx, cury, str(p[index].value), p[index].valid_opts) p[index].value = value else: value = input_text(stdscr, xoffset + startx, cury, p[index].value, p[index].input_validator) - if(not p[index].value_validator(value)): + if (not p[index].value_validator(value)): draw_show_message(stdscr, p[index].help_text) else: p[index].value = value @@ -155,7 +212,9 @@ def _draw(): _draw() k = stdscr.getch() return -#Calls display_run, and allows user to select and edit any of the run options + + +# Calls display_run, and allows user to select and edit any of the run options def draw_edit_run(stdscr, runinfo): def _draw(): stdscr.clear() @@ -167,12 +226,13 @@ def _draw(): stdscr.move(cury, xoffset) stdscr.refresh() return ey, ex + k = 0 y = STARTY + 2 cury = y array = [attr for attr in dir(runinfo) if not callable(getattr(runinfo, attr)) and not attr.startswith("__")] endy, startx = _draw() - while (k != ord('q')): #27 == 'ESC' + while (k != ord('q')): # 27 == 'ESC' if k == curses.KEY_DOWN and cury < (endy - 1): cury = cury + 1 elif k == curses.KEY_UP and cury > y: @@ -183,43 +243,46 @@ def _draw(): value = select_from(stdscr, xoffset + startx, cury, getattr(runinfo, name), [True, False]) setattr(runinfo, name, value) elif (name == 'report_level'): - value = select_from(stdscr, xoffset + startx, cury, getattr(runinfo, name), [0,1,2]) + value = select_from(stdscr, xoffset + startx, cury, getattr(runinfo, name), [0, 1, 2]) setattr(runinfo, name, value) - elif(name == 'run_type'): - value = select_from(stdscr, xoffset + startx, cury, getattr(runinfo, name), run_types) + elif (name == 'run_type'): + value = select_from(stdscr, xoffset + startx, cury, getattr(runinfo, name), run_types) setattr(runinfo, name, value) - elif(name == 'properties'): + elif (name == 'properties'): draw_edit_props(stdscr, runinfo.properties.get_all()) elif (name == 'jdk'): value = input_text(stdscr, xoffset + startx, cury, getattr(runinfo, name), lambda x: True) # start editing at end of value - if(not os.path.exists(value)): + if (not os.path.exists(value)): draw_show_message(stdscr, "Warning: jdk path not found") setattr(runinfo, name, value) - elif(name == 'num_runs' or name == 'numa_nodes'): + elif (name == 'num_runs' or name == 'numa_nodes'): value = input_text(stdscr, xoffset + startx, cury, getattr(runinfo, name), objects.number_validator) # start editing at end of value setattr(runinfo, name, value) else: - value = input_text(stdscr, xoffset + startx, cury, getattr(runinfo, name), lambda x:True) #start editing at end of value + value = input_text(stdscr, xoffset + startx, cury, getattr(runinfo, name), + lambda x: True) # start editing at end of value setattr(runinfo, name, value) _draw() k = stdscr.getch() return runinfo -def edit_config(stdscr, config = None, path = ""): - if(config is None): + +def edit_config(stdscr, config=None, path=""): + if (config is None): config, path = draw_get_config_path(stdscr) - if(config is None): + if (config is None): return + def _draw(): stdscr.clear() stdscr.refresh() starty = draw_title(stdscr) stdscr.addstr(starty + len(config.runs), xoffset, "{}. Add new run:".format(len(config.runs) + 1)) draw_status_bar(stdscr, "Press 'ENTER' to edit a run | Press 'RIGHT' to adjust run position | Press q to quit") - if(position): + if (position): stdscr.move(cury, xoffset + 2) else: stdscr.move(cury, xoffset) @@ -230,7 +293,7 @@ def _draw(): def _getpad(): p = pad_runs(stdscr, config.runs) p.addstr(len(config.runs), 0, "{}. {}".format(len(config.runs), "Add new run")) - return p, min(starty + len(config.runs) , height - 1) + return p, min(starty + len(config.runs), height - 1) k = 0 height, width = stdscr.getmaxyx() @@ -239,63 +302,62 @@ def _getpad(): index = 0 pad, endy = _getpad() _draw() - while (k != ord('q')): #27 == 'ESC' + while (k != ord('q')): # 27 == 'ESC' if k == curses.KEY_DOWN: - if((not position and cury < endy) or (position and cury < endy - 1)): - if(position): + if ((not position and cury < endy) or (position and cury < endy - 1)): + if (position): config.runs[index + 1], config.runs[index] = config.runs[index], config.runs[index + 1] pad, endy = _getpad() pad.refresh(index - (cury - starty), 0, starty, xoffset, height - 2, width - xoffset - 1) cury += 1 index += 1 - elif((not position and index < len(config.runs) - 1) or (position and index < len(config.runs) - 1)): - if(position): + elif ((not position and index < len(config.runs) - 1) or (position and index < len(config.runs) - 1)): + if (position): config.runs[index + 1], config.runs[index] = config.runs[index], config.runs[index + 1] pad, endy = _getpad() pad.refresh(index - (cury - starty), 0, starty, xoffset, height - 2, width - xoffset - 1) index += 1 elif k == curses.KEY_UP: - if(cury > starty): + if (cury > starty): if (position): config.runs[index - 1], config.runs[index] = config.runs[index], config.runs[index - 1] pad, endy = _getpad() pad.refresh(index - (cury - starty), 0, starty, xoffset, height - 2, width - xoffset - 1) cury -= 1 index -= 1 - elif(index > 0): + elif (index > 0): if (position): config.runs[index - 1], config.runs[index] = config.runs[index], config.runs[index - 1] pad, endy = _getpad() pad.refresh(index - (cury - starty), 0, starty, xoffset, height - 2, width - xoffset - 1) index -= 1 - elif(k == curses.KEY_LEFT): + elif (k == curses.KEY_LEFT): position = False - elif (k == curses.KEY_RIGHT and index < len(config.runs) ): + elif (k == curses.KEY_RIGHT and index < len(config.runs)): position = True - elif(k == DEL and index < len(config.runs) - 1): + elif (k == DEL and index < len(config.runs) - 1): position = False draw_status_bar(stdscr, "Remove run {} from this config? (y/N)".format(config.runs[index])) resp = stdscr.getch() - if(resp == ord('y') or resp == ord('Y')): + if (resp == ord('y') or resp == ord('Y')): del config.runs[index] pad, endy = _getpad() elif k == ENTER: position = False - if(index == len(config.runs)): + if (index == len(config.runs)): config.runs.append(spec_run()) - config.runs[index] = draw_edit_run(stdscr, config.runs[index]) + config.runs[index] = draw_edit_run(stdscr, config.runs[index]) pad, endy = _getpad() _draw() - k = stdscr.getch() draw_save_config(stdscr, config, path) return -def draw_title(stdscr): +def draw_title(stdscr): height, width = stdscr.getmaxyx() title = "SPECtate - curses"[:width - 1] start_x_title = int((width // 2) - (len(title) // 2) - len(title) % 2) @@ -310,18 +372,20 @@ def draw_title(stdscr): # Turning off attributes for title stdscr.attroff(curses.color_pair(2)) stdscr.attroff(curses.A_BOLD) - return STARTY + 2 #the current y + return STARTY + 2 # the current y + def draw_status_bar(stdscr, statusbarstr): height, width = stdscr.getmaxyx() stdscr.attron(curses.color_pair(3)) - if(len(statusbarstr) > width): - statusbarstr= statusbarstr[:width - 1] + if (len(statusbarstr) > width): + statusbarstr = statusbarstr[:width - 1] stdscr.addstr(height - 1, 0, statusbarstr) stdscr.addstr(height - 1, len(statusbarstr), " " * (width - len(statusbarstr) - 1)) stdscr.attroff(curses.color_pair(3)) -#Displays some message and waits for the user to press a key + +# Displays some message and waits for the user to press a key def draw_show_message(stdscr, msg): stdscr.clear() y = draw_title(stdscr) @@ -331,22 +395,22 @@ def draw_show_message(stdscr, msg): stdscr.getch() -#Asks user for file and loads the json +# Asks user for file and loads the json def draw_get_config_path(stdscr): stdscr.clear() y = draw_title(stdscr) draw_status_bar(stdscr, "Please enter the path of a config") stdscr.addstr(y, xoffset, "Config file path:") - path = input_text(stdscr, xoffset + len("Config file path:") + 1, y, "", lambda x:True) - if(os.path.isfile(path)): - if(True): #do_validate(({'' : path}))): - config =load_config(path) - if(config is None): + path = input_text(stdscr, xoffset + len("Config file path:") + 1, y, "", lambda x: True) + if (os.path.isfile(path)): + if (True): # do_validate(({'' : path}))): + config = load_config(path) + if (config is None): import pdb pdb.set_trace() draw_show_message(stdscr, "Error loading config file") - elif(not isinstance(config, spec_config)): + elif (not isinstance(config, spec_config)): import pdb pdb.set_trace() draw_show_message(stdscr, "Invalid config file format") @@ -358,6 +422,7 @@ def draw_get_config_path(stdscr): draw_show_message(stdscr, "File not found") return None, "" + def draw_get_path(stdscr): stdscr.clear() y = draw_title(stdscr) @@ -368,102 +433,130 @@ def draw_get_path(stdscr): return input_text(stdscr, xoffset + len("Config file path:") + 1, y, "", lambda x: True) -#Disable ncurses and calls maincli +# Disable ncurses and calls maincli def run_config(stdscr): config, path = draw_get_config_path(stdscr) if (config is None): return - -# stdscr.clear() -# stdscr.refresh() -# height, width = stdscr.getmaxyx() -# cury = draw_title(stdscr) -# index = 0 -# def _handle(msg): - # c = index % maxindex -# pad.addstr(0, 0, msg) - # if cury < maxy: - # cury += 1 -# pad.refresh(0, 0, 20, xoffset, 20, width - xoffset) - #log[index % maxindex] = msg - - #index += 1 - - #starty = y - #maxy = height - 3 - #log = [height - y - 3] - #maxindex = len(log) - -# pad = curses.newpad(height, width - 1 - xoffset) - - stdscr.keypad(0) - curses.echo() - curses.nocbreak() - curses.endwin() + index = 0 + def _handle(msg): + msg = _remove_control_chars(msg) + if(msg.isspace()): + return + nonlocal index + nonlocal cury + nonlocal pad + nonlocal maxy + c = index % logmax + pad.addstr(c, 0, msg) # c == new bottom + if(index < maxy - cury): + pad.refresh(0, 0, cury, xoffset, cury + index, width - xoffset - 1) + else: + # if c + cury < maxy: + # pad.refresh(0, 0, cury, xoffset, cury + c, width - xoffset - 1) + # else: + #print 0 -> c + #print logmax - (c + pad.refresh(0,0, maxy - c, xoffset, maxy, width - xoffset - 1) + pad.refresh(logmax - c, 0, cury, xoffset, maxy - c - 1, width - xoffset - 1) + index += 1 + draw_status_bar(stdscr, 'c={}, index={}, msg=""'.format(c, index, msg)) + stdscr.refresh() + stdscr.clear() + stdscr.refresh() + height, width = stdscr.getmaxyx() + cury = draw_title(stdscr) + maxy = height - cury + logmax = maxy - cury + # def _handle(msg): + # c = index % maxindex + # pad.addstr(0, 0, msg) + # if cury < maxy: + # cury += 1 + # pad.refresh(0, 0, 20, xoffset, 20, width - xoffset) + # log[index % maxindex] = msg + + # index += 1 + + # starty = y + # maxy = height - 3 + # log = [height - y - 3] + # maxindex = len(log) + + pad = curses.newpad(height, width - 1 - xoffset) + + #stdscr.keypad(0) + #curses.echo() + #curses.nocbreak() + #curses.endwin() for r in config.runs: - #draw_status_bar(stdscr, "Starting run '{}'".format(r.tag)) + # draw_status_bar(stdscr, "Starting run '{}'".format(r.tag)) - result = r.run(path) - if(result == 2): - curses.noecho() - curses.cbreak() - stdscr.keypad(1) + result = r.run(path, _handle) + if (result == 2): + #curses.noecho() + #curses.cbreak() + #stdscr.keypad(1) path = draw_get_path(stdscr) - stdscr.keypad(0) - curses.echo() - curses.nocbreak() - curses.endwin() - result = r.run(path) - if(result == 2): - curses.noecho() - curses.cbreak() - stdscr.keypad(1) - draw_show_message(stdscr, "Invalid SPECjbb path") + #stdscr.keypad(0) + #curses.echo() + #curses.nocbreak() + #curses.endwin() + result = r.run(path, _handle) + if (result == 2): + #curses.noecho() + #curses.cbreak() + #stdscr.keypad(1) + draw_status_bar(stdscr, "Invalid SPECjbb path. Press any key to continue") + stdscr.getch() return - if(result == 1): - curses.noecho() - curses.cbreak() - stdscr.keypad(1) - draw_show_message(stdscr, "An error occured running the benchmark") - curses.echo() - curses.nocbreak() - curses.endwin() - curses.noecho() - curses.cbreak() - stdscr.keypad(1) - draw_show_message(stdscr, "All runs have been completed") - -#Requests file name from user, then calls edit_config -#Also clears the screen + if (result == -1): + #curses.noecho() + #curses.cbreak() + #stdscr.keypad(1) + draw_status_bar(stdscr, "An error occured running the benchmark. Press any key to continue") + stdscr.getch() + return + #curses.echo() + #curses.nocbreak() + #curses.endwin() + #curses.noecho() + #curses.cbreak() + #stdscr.keypad(1) + draw_status_bar(stdscr, "All runs have been completed - Press any key to continue") + stdscr.getch() + + +# Requests file name from user, then calls edit_config +# Also clears the screen def create_config(stdscr): stdscr.clear() stdscr.refresh() y = draw_title(stdscr) stdscr.addstr(y, xoffset, "Enter config name:") - - path = input_text(stdscr, xoffset + len("Enter config name:") + 1, y, "", lambda x:True).strip() - if(path.isspace() or path == ""): + path = input_text(stdscr, xoffset + len("Enter config name:") + 1, y, "", lambda x: True).strip() + if (path.isspace() or path == ""): draw_show_message(stdscr, "Config name cannot be blank") return - if(os.path.isfile(path)): + if (os.path.isfile(path)): stdscr.clear() stdscr.refresh() y = draw_title(stdscr) - stdscr.addstr(y,xoffset, "Warning: file {} already exists, overwrite? (y/N)") + stdscr.addstr(y, xoffset, "Warning: file {} already exists, overwrite? (y/N)".format(path)) k = stdscr.getch() - if(k != ord('Y') and k != ord('y')): + if (k != ord('Y') and k != ord('y')): return - edit_config(stdscr, spec_config(), path) + def pad_props(stdscr, proplist): height, width = stdscr.getmaxyx() if (proplist == []): return curses.newpad(1, width - 1 - xoffset), 0 pad = curses.newpad(len(proplist) + 1, width - 1 - xoffset) - valueoffset = len(max(map(lambda x:x.prop, proplist), key=len)) + valueoffset = len(max(map(lambda x: x.prop, proplist), key=len)) # add plus one to pad height for edit config to put 'Add run' option idx = 0 for p in proplist: @@ -473,18 +566,19 @@ def pad_props(stdscr, proplist): # pad.refresh(0,0,y,xoffset,min(height, y + len(runlist)), min(width - 1, xoffset)) return pad, valueoffset -#Returns a 'pad' containing a list of runs (tag names) + +# Returns a 'pad' containing a list of runs (tag names) def pad_runs(stdscr, runlist): height, width = stdscr.getmaxyx() if (runlist == []): return curses.newpad(1, width - 1 - xoffset) pad = curses.newpad(len(runlist) + 1, width - 1 - xoffset) - #add plus one to pad height for edit config to put 'Add run' option + # add plus one to pad height for edit config to put 'Add run' option idx = 0 for run in runlist: pad.addstr(idx, 0, "{}. {}".format(idx + 1, run.tag)) idx += 1 -# pad.refresh(0,0,y,xoffset,min(height, y + len(runlist)), min(width - 1, xoffset)) + # pad.refresh(0,0,y,xoffset,min(height, y + len(runlist)), min(width - 1, xoffset)) return pad @@ -493,42 +587,43 @@ def _draw(): stdscr.clear() stdscr.refresh() draw_title(stdscr) - if(show_all): + if (show_all): draw_status_bar(stdscr, "Press 'q' to exit | Press 'a' to hide defaults") else: draw_status_bar(stdscr, "Press 'q' to exit | Press 'a' to show all properties") - if(len(visprops) > 0): + if (len(visprops) > 0): stdscr.addstr(height - 3, 0, visprops[index].desc) pad.refresh(index - (cury - starty), 0, starty, xoffset, height - 5, width - xoffset - 1) stdscr.move(cury, xoffset) stdscr.refresh() + show_all = False index = 0 starty = STARTY + 2 cury = starty k = 0 visprops = p.get_modified() - pad, startx = pad_props(stdscr,visprops) - if(len(visprops) == 0): - pad.addstr(0,0, "No properties to display") + pad, startx = pad_props(stdscr, visprops) + if (len(visprops) == 0): + pad.addstr(0, 0, "No properties to display") height, width = stdscr.getmaxyx() while (k != ord('q')): if k == curses.KEY_DOWN and index < len(visprops) - 1: index += 1 - if(cury < height - 5): + if (cury < height - 5): cury += 1 else: pad.refresh(index - (cury - starty), 0, starty, xoffset, height - 5, width - xoffset - 1) elif k == curses.KEY_UP and index > 0: index -= 1 - if(cury > starty): + if (cury > starty): cury -= 1 else: pad.refresh(index - (cury - starty), 0, starty, xoffset, height - 5, width - xoffset - 1) - elif(k == ord('a') or k == ord('A')): + elif (k == ord('a') or k == ord('A')): show_all = not show_all - if(show_all): + if (show_all): visprops = p.get_all() else: visprops = p.get_modified() @@ -544,11 +639,11 @@ def _draw(): if (cury >= starty + len(visprops)): cury = len(visprops) - 1 - _draw() k = stdscr.getch() return + def draw_view_run(stdscr, runinfo): def _draw(): stdscr.clear() @@ -559,19 +654,22 @@ def _draw(): stdscr.refresh() return ey, ex + k = 0 _draw() - while (k != ord('q')): #27 == 'ESC' + while (k != ord('q')): # 27 == 'ESC' if k == ENTER: draw_view_props(stdscr, runinfo.properties) _draw() k = stdscr.getch() + def view_runs(stdscr): config, path = draw_get_config_path(stdscr) - if(config is None): + if (config is None): return + def _draw(stdscr, py, pad, index): stdscr.clear() stdscr.refresh() @@ -589,20 +687,20 @@ def _draw(stdscr, py, pad, index): y = len(config.runs) + cury pad = pad_runs(stdscr, config.runs) _draw(stdscr, cury, pad, index) - while(k != ord('q')): + while (k != ord('q')): if k == curses.KEY_DOWN: - if(cury < y - 1): + if (cury < y - 1): cury += 1 index += 1 - elif(index < len(config.runs) - 1): - #pad scrolling down + elif (index < len(config.runs) - 1): + # pad scrolling down index += 1 pad.refresh(index - (cury - starty), 0, starty, xoffset, height - 2, width - xoffset - 1) - elif k == curses.KEY_UP : - if(cury > starty): + elif k == curses.KEY_UP: + if (cury > starty): cury -= 1 - index -=1 - elif(index > 0): + index -= 1 + elif (index > 0): index -= 1 elif k == ENTER: @@ -611,14 +709,13 @@ def _draw(stdscr, py, pad, index): k = stdscr.getch() -#Utilities + +# Utilities def load_config(path): try: with open(path, 'r') as f: - return json.load(f, cls=spec_decoder) - except JSONDecodeError: - import pdb - pdb.set_trace() + return json.load(f, cls=spec_decoder) + except: return None @@ -632,7 +729,7 @@ def draw_menu(stdscr): edit_config, view_runs ] - current_config= None + current_config = None # Clear and refresh the screen for a blank canvas stdscr.clear() stdscr.refresh() @@ -646,16 +743,14 @@ def draw_menu(stdscr): while (k != ord('q')): # Initialization - if(k == ENTER): + if (k == ENTER): opts[cursor_y - (STARTY + 2)](stdscr) - elif(k >= ord('1') and k <= ord('4')): + elif (k >= ord('1') and k <= ord('4')): opts[int(k)](stdscr) stdscr.clear() stdscr.refresh() - - starty = y = draw_title(stdscr) # Declaration of strings stdscr.addstr(y, xoffset, "1. Run a config") @@ -671,7 +766,6 @@ def draw_menu(stdscr): # Render status bar draw_status_bar(stdscr, "Press 'q' to exit | MAIN MENU") - stdscr.move(cursor_y, xoffset) # Refresh the screen @@ -680,8 +774,10 @@ def draw_menu(stdscr): # Wait for next input k = stdscr.getch() + def main(): curses.wrapper(draw_menu) + if __name__ == "__main__": main() From fba70a35dff32dd13cbabe91cfe86cd1bc9f7b25 Mon Sep 17 00:00:00 2001 From: David Hiatt Date: Thu, 26 Apr 2018 19:11:23 -0700 Subject: [PATCH 3/3] added numa support --- objects.py | 26 ++++++++++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/objects.py b/objects.py index 79e578e..e70da72 100644 --- a/objects.py +++ b/objects.py @@ -272,6 +272,8 @@ def run_multi(self, path, handle): result_dir = self._prerun(path) opts = self._spec_opts() tx_opts = self._tx_opts() + has_numa = self._check_numa() and self.numa_nodes > 1 + numa_cmd = 'numactl --cpunodebind={} --localalloc' for x in range(self.num_runs): cont_std = open(os.path.join(result_dir, 'controller.log'), 'wb') cont_err = open(os.path.join(result_dir, 'controller.out'), 'wb') @@ -281,9 +283,15 @@ def run_multi(self, path, handle): be_procs = [] # os.system('{} {} -jar {}/specjbb2015.jar -m MULTICONTROLLER {} 2> {}/controller.log > {}/controller.out &'.format(self.jdk, self.jvm_options, path, opts, x, result_dir, result_dir)) for g in range(self.properties.root['specjbb.group.count'].value): + numa = numa_cmd.format((g - 1) % 4) for j in range(self.properties.root['specjbb.txi.pergroup.count'].value): ti_name = "{}Group{}.TxInjector.txiJVM{}".format(result_dir, g, j) - cmd = '{} {} -jar {}/specjbb2015.jar -m TXINJECTOR {} -G={}'.format(self.jdk, self.jvm_options, + if(has_numa): + cmd = '{} {} {} -jar {}/specjbb2015.jar -m TXINJECTOR {} -G={}'.format(numa, self.jdk, self.jvm_options, + path, tx_opts, g, ti_name, + ti_name) + else: + cmd = '{} {} -jar {}/specjbb2015.jar -m TXINJECTOR {} -G={}'.format(self.jdk, self.jvm_options, path, tx_opts, g, ti_name, ti_name) tx_procs.append([Popen(shlex.split(cmd), cwd=path, stdout=PIPE, stderr=PIPE), @@ -291,7 +299,13 @@ def run_multi(self, path, handle): open(os.path.join(path, '{}.out'.format(ti_name)), 'wb')]) # os.system('{} {} -jar {}/specjbb2015.jar -m TXINJECTOR {} -G={} 2> {}.log > {}.out &'.format(self.jdk, self.jvm_options, path, tx_opts, g, ti_name, ti_name)) be_name = "{}{}.Backend.beJVM".format(result_dir, g) - cmd = '{} {} -jar {}/specjbb2015.jar -m BACKEND {} -G={} -J=beJVM'.format(self.jdk, self.jvm_options, + if(has_numa): + cmd = '{} {} {} -jar {}/specjbb2015.jar -m BACKEND {} -G={} -J=beJVM'.format(numa, self.jdk, + self.jvm_options, + path, tx_opts, + g) + else: + cmd = '{} {} -jar {}/specjbb2015.jar -m BACKEND {} -G={} -J=beJVM'.format(self.jdk, self.jvm_options, path, tx_opts, g) be_procs.append([Popen(shlex.split(cmd), cwd=path, stdout=PIPE, stderr=PIPE), @@ -379,6 +393,14 @@ def _spec_opts(self): opts += " -ikv" return opts + def _check_numa(self): + p = Popen(shlex.split('which numactl'), stdout=PIPE) + o, e = p.communicate() + result = len(o) > 0 + p.wait() + return result + + class propitem: def __init__(self, prop, def_value, desc, input_validator, value_validator, valid_opts=None, help_text=""): self.prop = prop