From 03c86d95b9e84ed70658316f5006fcf0cc280b7f Mon Sep 17 00:00:00 2001 From: Richard Barella Date: Thu, 12 Dec 2019 17:04:46 -0800 Subject: [PATCH] Add elk-friendly output options in argparse options to better document usage instead of hiding behind os.environ Signed-off-by: Richard Barella --- pyclient/pymtt.py | 30 ++++++++++++++++ pylib/Tools/Executor/sequential.py | 20 +++++------ pylib/Utilities/ExecuteCmd.py | 9 +++-- pylib/Utilities/Logger.py | 57 +++++++++++++----------------- 4 files changed, 70 insertions(+), 46 deletions(-) diff --git a/pyclient/pymtt.py b/pyclient/pymtt.py index 3c18f9f8f..e9e6d975a 100755 --- a/pyclient/pymtt.py +++ b/pyclient/pymtt.py @@ -148,6 +148,36 @@ debugGroup.add_argument("--trial", action="store_true", dest="trial", default=False, help="Use when testing your MTT client setup; results that are generated and submitted to the database are marked as \"trials\" and are not included in normal reporting.") + +elkGroup = parser.add_argument_group('elkGroup','ELK-friendly output options') +elkGroup.add_argument("--elk_testcase", dest="elk_testcase", + default=os.environ['MTT_ELK_TESTCASE'] if 'MTT_ELK_TESTCASE' in os.environ else None, + help="Specifies which testcase to log results as when using elk-friendly output. You can also set this through the enviroment variable MTT_ELK_TESTCASE") +elkGroup.add_argument("--elk_testcycle", dest="elk_testcycle", + default=os.environ['MTT_ELK_TESTCYCLE'] if 'MTT_ELK_TESTCYCLE' in os.environ else None, + help="Specifies which testcycle to log results as when using elk-friendly output. You can also set this through the enviroment variable MTT_ELK_TESTCYCLE") +elkGroup.add_argument("--elk_id", dest="elk_id", + default=os.environ['MTT_ELK_ID'] if 'MTT_ELK_ID' in os.environ else None, + help="Specifies which execution id to log results as when using elk-friendly output. You can also set this through the enviroment variable MTT_ELK_ID") +elkGroup.add_argument("--elk_head", dest="elk_head", + default=os.environ['MTT_ELK_HEAD'] if 'MTT_ELK_HEAD' in os.environ else None, + help="Specifies which location to log _.elog files for elk-friendly output. You can also set this through the environment variable MTT_ELK_HEAD") +elkGroup.add_argument("--elk_chown", dest="elk_chown", + default=os.environ['MTT_ELK_CHOWN'] if 'MTT_ELK_CHOWN' in os.environ else None, + help="Specifies what user and group to set for *.elog file permissions. You can also set this through the environment variable MTT_ELK_CHOWN") +elkGroup.add_argument("--elk_maxsize", dest="elk_maxsize", + default=os.environ['MTT_ELK_MAXSIZE'] if 'MTT_ELK_MAXSIZE' in os.environ else None, + help="Specifies a max number of lines to log for stdout and stderr in elk-friendly output, and otherwise truncate. You can also set this through the environment variable MTT_ELK_MAXSIZE") +elkGroup.add_argument("--elk_nostdout", dest="elk_nostdout", action="store_true", + default=True if 'MTT_ELK_NOSTDOUT' in os.environ and os.environ['MTT_ELK_NOSTDOUT'].lower() != 'false' else False, + help="Specifies whether to include stdout in elk-friendly output. You can also set this through the environment variable MTT_ELK_NOSTDOUT") +elkGroup.add_argument("--elk_nostderr", dest="elk_nostderr", action="store_true", + default=True if 'MTT_ELK_NOSTDERR' in os.environ and os.environ['MTT_ELK_NOSTDERR'].lower() != 'false' else False, + help="Specifies whether to include stdout in elk-friendly output. You can also set this through the environment variable MTT_ELK_NOSTDERR") +elkGroup.add_argument("--elk_debug", dest="elk_debug", action="store_true", + default=True if 'MTT_ELK_DEBUG' in os.environ and os.environ['MTT_ELK_DEBUG'].lower() != 'false' else False, + help="Specifies whether to output everything logged to *.elog files to the screen as extra verbose output. You can also set this through the environment variable MTT_ELK_DEBUG") + args = parser.parse_args() # get any arguments set in MTT_ARGS environment variable and combine them with diff --git a/pylib/Tools/Executor/sequential.py b/pylib/Tools/Executor/sequential.py index a48f3595e..e4c6cc2c6 100755 --- a/pylib/Tools/Executor/sequential.py +++ b/pylib/Tools/Executor/sequential.py @@ -164,7 +164,7 @@ def execute_sections(self, testDef): # couldn't find the parent's log - cannot continue stageLog['status'] = 1 stageLog['stderr'] = ["Prior dependent step did not record a log"] - testDef.logger.logResults(disp_title, stageLog) + testDef.logger.logResults(disp_title, stageLog, testDef) testDef.plugin_trans_sem.acquire() continue try: @@ -173,14 +173,14 @@ def execute_sections(self, testDef): # cannot proceed here either stageLog['status'] = bldlog['status'] stageLog['stderr'] = ["Prior dependent step failed - cannot proceed"] - testDef.logger.logResults(disp_title, stageLog) + testDef.logger.logResults(disp_title, stageLog, testDef) testDef.plugin_trans_sem.acquire() continue except KeyError: # if it didn't report a status, we shouldn't rely on it stageLog['status'] = 1 stageLog['stderr'] = ["Prior dependent step failed to provide a status"] - testDef.logger.logResults(disp_title, stageLog) + testDef.logger.logResults(disp_title, stageLog, testDef) testDef.plugin_trans_sem.acquire() continue except KeyError: @@ -221,7 +221,7 @@ def execute_sections(self, testDef): if plugin is None: stageLog['status'] = 1 stageLog['stderr'] = "Specified plugin",module,"does not exist in stage",stage,"or in the available tools and utilities" - testDef.logger.logResults(disp_title, stageLog) + testDef.logger.logResults(disp_title, stageLog, testDef) testDef.plugin_trans_sem.acquire() continue else: @@ -253,7 +253,7 @@ def execute_sections(self, testDef): if plugin is None: stageLog['status'] = 1 stageLog['stderr'] = "Specified plugin",module,"does not exist in stage",stage,"or in the available tools and utilities" - testDef.logger.logResults(disp_title, stageLog) + testDef.logger.logResults(disp_title, stageLog, testDef) testDef.plugin_trans_sem.acquire() continue else: @@ -271,7 +271,7 @@ def execute_sections(self, testDef): # we really have no way of executing this stageLog['status'] = 1 stageLog['stderr'] = "Plugin for stage",stage,"was not specified, and no default is available" - testDef.logger.logResults(disp_title, stageLog) + testDef.logger.logResults(disp_title, stageLog, testDef) testDef.plugin_trans_sem.acquire() continue @@ -291,7 +291,7 @@ def execute_sections(self, testDef): stageLog['stderr'] = stageLog['stderr'].split("\n") # Log results for section - testDef.logger.logResults(disp_title, stageLog) + testDef.logger.logResults(disp_title, stageLog, testDef) # Print end of section testDef.logger.stage_end_print(disp_title, stageLog) @@ -319,7 +319,7 @@ def execute_sections(self, testDef): if not p._getIsActivated(): continue p.plugin_object.deactivate() - testDef.logger.logResults(disp_title, stageLog) + testDef.logger.logResults(disp_title, stageLog, testDef) testDef.logger.verbose_print("=======================================") testDef.logger.verbose_print("KeyboardInterrupt exception was raised: %s %s" \ % (type(e), str(e))) @@ -348,7 +348,7 @@ def execute_sections(self, testDef): testDef.logger.verbose_print("=======================================") stageLog['status'] = 1 stageLog['stderr'] = ["Exception was raised: %s %s" % (type(e), str(e))] - testDef.logger.logResults(disp_title, stageLog) + testDef.logger.logResults(disp_title, stageLog, testDef) self.status = 1 self.only_reporter = True continue @@ -395,7 +395,7 @@ def execute(self, testDef): if stageLog['status'] != 0: self.status = 1 self.only_reporter = True - testDef.logger.logResults("DefaultHarasser", stageLog) + testDef.logger.logResults("DefaultHarasser", stageLog, testDef) # Keep on looping as long as it's needed while self.looping or self.loop_count == 0 or (self.one_last_loop and self.looping): diff --git a/pylib/Utilities/ExecuteCmd.py b/pylib/Utilities/ExecuteCmd.py index e115fb291..82a38cee2 100644 --- a/pylib/Utilities/ExecuteCmd.py +++ b/pylib/Utilities/ExecuteCmd.py @@ -234,7 +234,8 @@ def execute(self, options, cmdargs, testDef, quiet=False): None, datetime.datetime.now(), datetime.datetime.now(), - 0, None) + 0, None, + testDef) return (1, [], ["MTT ExecuteCmd error: no cmdargs"], 0) # define storage to catch the output @@ -321,7 +322,8 @@ def execute(self, options, cmdargs, testDef, quiet=False): starttime, endtime, (endtime - starttime).total_seconds, - results['slurm_job_ids'] if 'slurm_job_ids' in results else None) + results['slurm_job_ids'] if 'slurm_job_ids' in results else None, + testDef) return results if time_exec: @@ -357,6 +359,7 @@ def execute(self, options, cmdargs, testDef, quiet=False): starttime, endtime, (endtime - starttime).total_seconds(), - results['slurm_job_ids'] if 'slurm_job_ids' in results else None) + results['slurm_job_ids'] if 'slurm_job_ids' in results else None, + testDef) return results diff --git a/pylib/Utilities/Logger.py b/pylib/Utilities/Logger.py index 9463ac7e3..459f778c8 100644 --- a/pylib/Utilities/Logger.py +++ b/pylib/Utilities/Logger.py @@ -34,10 +34,6 @@ def __init__(self): self.cmdtimestamp = False self.timestampeverything = False self.stage_start = {} - self.elk_id = None - self.elk_caseid = None - self.elk_cycleid = None - self.elk_head = None self.elk_log = None self.current_section = None self.execmds_stash = [] @@ -102,13 +98,6 @@ def open(self, testDef): self.cmdtimestamp = True except KeyError: pass - if 'MTT_ELK_TESTCASE' in os.environ: - self.elk_caseid = os.environ['MTT_ELK_TESTCASE'] - if 'MTT_ELK_TESTCYCLE' in os.environ: - self.elk_cycleid = os.environ['MTT_ELK_TESTCYCLE'] - if 'MTT_ELK_ID' in os.environ and 'MTT_ELK_HEAD' in os.environ: - self.elk_id = os.environ['MTT_ELK_ID'] - self.elk_head = os.environ['MTT_ELK_HEAD'] return def get_dict_contents(self, dic): @@ -120,18 +109,18 @@ def get_tuplelist_contents(self, tuplelist): max_keylen = max([len(key) for key,_ in tuplelist]) return ["%s = %s" % (k.rjust(max_keylen), o) for k,o in tuplelist] - def log_to_elk(self, result, logtype): + def log_to_elk(self, result, logtype, testDef): result = result.copy() - if self.elk_head is None or self.elk_id is None: + if testDef.options['elk_head'] is None or testDef.options['elk_id'] is None: self.verbose_print('Error: entered log_to_elk() function without elk_id and elk_head specified') return result = {k:(str(v) if isinstance(v, datetime.datetime) or isinstance(v, datetime.date) else v) for k,v in list(result.items())} - result['execid'] = self.elk_id - result['cycleid'] = self.elk_cycleid - result['caseid'] = self.elk_caseid + result['execid'] = testDef.options['elk_id'] + result['cycleid'] = testDef.options['elk_testcycle'] + result['caseid'] = testDef.options['elk_testcase'] result['logtype'] = logtype # drop the stderr and stdout, will use the copies that are part of the commands issues @@ -155,22 +144,24 @@ def log_to_elk(self, result, logtype): # convert profile, list of lists into a dictionary result['profile'] = { k:' '.join(v) for k,v in list(result['profile'].items()) } - #self.verbose_print('Logging to elk_head={}/{}.elog: {}'.format(self.elk_head, self.elk_id, result)) + if testDef.options['elk_debug']: + self.verbose_print('Logging to elk_head={}/{}.elog: {}'.format(testDef.options['elk_head'], testDef.options['elk_id'], result)) + if self.elk_log is None: allpath = '/' - for d in os.path.normpath(self.elk_head).split(os.path.sep): + for d in os.path.normpath(testDef.options['elk_head']).split(os.path.sep): allpath = os.path.join(allpath, d) if not os.path.exists(allpath): os.mkdir(allpath) uid = None gid = None - if 'MTT_ELK_CHOWN' in os.environ and ':' in os.environ['MTT_ELK_CHOWN']: - user = os.environ['MTT_ELK_CHOWN'].split(':')[0] - group = os.environ['MTT_ELK_CHOWN'].split(':')[1] + if testDef.options['elk_chown'] is not None and ':' in testDef.options['elk_chown']: + user = testDef.options['elk_chown'].split(':')[0] + group = testDef.options['elk_chown'].split(':')[1] uid = pwd.getpwnam(user).pw_uid gid = grp.getgrnam(group).gr_gid os.chown(allpath, uid, gid) - self.elk_log = open(os.path.join(self.elk_head, '{}-{}.elog'.format(self.elk_caseid, self.elk_id)), 'a+') + self.elk_log = open(os.path.join(testDef.options['elk_head'], '{}-{}.elog'.format(testDef.options['elk_testcase'], testDef.options['elk_id'])), 'a+') self.elk_log.write(json.dumps(result) + '\n') return @@ -187,14 +178,14 @@ def print_cmdline_args(self, testDef): for s in strs_to_print: self.verbose_print(s) self.verbose_print("") - if self.elk_id is not None: - self.log_to_elk({'environment': dict(os.environ), 'options': testDef.options}, 'mtt-env') + if testDef.options['elk_id'] is not None: + self.log_to_elk({'environment': dict(os.environ), 'options': testDef.options}, 'mtt-env', testDef) - def log_execmd_elk(self, cmdargs, status, stdout, stderr, timedout, starttime, endtime, elapsed_secs, slurm_job_ids): - if self.elk_id is not None: - if 'MTT_ELK_MAXSIZE' in os.environ: + def log_execmd_elk(self, cmdargs, status, stdout, stderr, timedout, starttime, endtime, elapsed_secs, slurm_job_ids, testDef): + if testDef.options['elk_id'] is not None: + if testDef.options['elk_maxsize'] is not None: try: - maxsize = int(os.environ['MTT_ELK_MAXSIZE']) + maxsize = int(testDef.options['elk_maxsize']) except: maxsize = None if maxsize is not None and len(stdout) > maxsize: @@ -209,8 +200,8 @@ def log_execmd_elk(self, cmdargs, status, stdout, stderr, timedout, starttime, e stderr = [''] self.execmds_stash.append({'cmdargs': ' '.join(cmdargs), 'status': status, - 'stdout': stdout if 'MTT_ELK_NOSTDOUT' not in os.environ else [''], - 'stderr': stderr if 'MTT_ELK_NOSTDERR' not in os.environ else [''], + 'stdout': stdout if testDef.options['elk_nostdout'] is not None else [''], + 'stderr': stderr if testDef.options['elk_nostderr'] is not None else [''], 'timedout': timedout, 'starttime': str(starttime), 'endtime': str(endtime), @@ -261,11 +252,11 @@ def close(self): self.fh.close() return - def logResults(self, title, result): + def logResults(self, title, result, testDef): self.verbose_print("LOGGING results for " + title) self.results.append(result) - if self.elk_id is not None: - self.log_to_elk(result, 'mtt-sec') + if testDef.options['elk_id'] is not None: + self.log_to_elk(result, 'mtt-sec', testDef) return def outputLog(self):