diff --git a/build/ReleaseManager/ReleaseManagerLib.py b/build/ReleaseManager/ReleaseManagerLib.py index 4beca2b32..2ba0aa86f 100644 --- a/build/ReleaseManager/ReleaseManagerLib.py +++ b/build/ReleaseManager/ReleaseManagerLib.py @@ -301,7 +301,7 @@ def __init__(self, rel, python_version, use_mock=True): "_binary_filedigest_algorithm": "md5", #'global __python': '%%{__python2}', #'py_byte_compile': '', - #%py_byte_compile %{__python2} %{buildroot}%{_datadir}/mypackage/foo + # %py_byte_compile %{__python2} %{buildroot}%{_datadir}/mypackage/foo } # __init__ @@ -394,7 +394,6 @@ def execute(self): class PackageExcludes: def __init__(self): - self.commonPattern = [ "CVS", ".DS_Store", diff --git a/creation/lib/cWDictFile.py b/creation/lib/cWDictFile.py index 30b5e57de..e123a5994 100644 --- a/creation/lib/cWDictFile.py +++ b/creation/lib/cWDictFile.py @@ -1771,6 +1771,7 @@ def __init__(self, monitor_dir, work_dir, work_subdir="monitor", monitordir_name # ################################################ + # helper class, used below class fileCommonDicts: def __init__(self): @@ -2237,7 +2238,6 @@ def erase(self, destroy_old_subs=True): # if false, the sub names will be prese return def load(self, destroy_old_subs=True): # if false, overwrite the subs you load, but leave the others as they are - self.main_dicts.load() if destroy_old_subs: self.sub_list = [] diff --git a/creation/lib/cgWDictFile.py b/creation/lib/cgWDictFile.py index 82e45af3c..85c8da109 100644 --- a/creation/lib/cgWDictFile.py +++ b/creation/lib/cgWDictFile.py @@ -208,6 +208,7 @@ def is_equal( # ################################################ + # internal, do not use from outside the module def get_common_dicts(submit_dir, stage_dir): common_dicts = { @@ -266,6 +267,7 @@ def get_entry_dicts(entry_submit_dir, entry_stage_dir, entry_name): # ################################################ + # internal, do not use from outside the module def load_common_dicts(dicts, description_el): # update in place # first submit dir ones (mutable) diff --git a/creation/lib/cgWParamDict.py b/creation/lib/cgWParamDict.py index 5d69d7b9d..a8ff5fec8 100644 --- a/creation/lib/cgWParamDict.py +++ b/creation/lib/cgWParamDict.py @@ -927,7 +927,7 @@ def sortit(self, unsorted_dict): i = i + 1 temp_list = sorted((x[1][1], x[0]) for x in list(d.items())) sortedList = [] - for (tmp, key) in temp_list: + for tmp, key in temp_list: sortedList.append(d[key][0]) return sortedList @@ -946,6 +946,7 @@ def new_SubDicts(self, sub_name): # ############################################################ + ############################################# # Add a user file residing in the stage area # file as described by Params.file_defaults @@ -1118,7 +1119,6 @@ def iter_to_dict(dictObject): def populate_factory_descript( work_dir, glidein_dict, active_sub_list, disabled_sub_list, conf # will be modified # will be modified ): - down_fname = os.path.join(work_dir, "glideinWMS.downtimes") sec_el = conf.get_child("security") diff --git a/creation/lib/cgWParams.py b/creation/lib/cgWParams.py index 879fe01c4..cf23b4f90 100644 --- a/creation/lib/cgWParams.py +++ b/creation/lib/cgWParams.py @@ -568,6 +568,7 @@ def get_xml_format(self): # ############################################################ + ##################################### # try to find out the base condor dir def find_condor_base_dir(): diff --git a/creation/lib/cvWCreate.py b/creation/lib/cvWCreate.py index 055b9c33a..f88353830 100644 --- a/creation/lib/cvWCreate.py +++ b/creation/lib/cvWCreate.py @@ -62,7 +62,7 @@ def create_client_mapfile(mapfile_fname, my_DN, factory_DNs, schedd_DNs, collect with open(mapfile_fname, "w") as fd: if my_DN: fd.write('GSI "^{}$" {}\n'.format(re.escape(my_DN), "me")) - for (uid, dns) in ( + for uid, dns in ( ("factory", factory_DNs), ("schedd", schedd_DNs), ("collector", collector_DNs), diff --git a/creation/lib/cvWDictFile.py b/creation/lib/cvWDictFile.py index c1c2c950e..46d744e60 100644 --- a/creation/lib/cvWDictFile.py +++ b/creation/lib/cvWDictFile.py @@ -81,6 +81,7 @@ def parse_val(self, line): # ################################################ + # internal, do not use from outside the module def get_common_dicts(work_dir, stage_dir, simple_work_dir): # if True, do not create params common_dicts = { @@ -148,6 +149,7 @@ def get_group_dicts(group_work_dir, group_stage_dir, group_name, simple_work_dir # ################################################ + # internal, do not use from outside the module def load_common_dicts(dicts, description_el): # update in place # first work dir ones (mutable) diff --git a/creation/lib/cvWParamDict.py b/creation/lib/cvWParamDict.py index daf899ea4..b557a6359 100644 --- a/creation/lib/cvWParamDict.py +++ b/creation/lib/cvWParamDict.py @@ -763,6 +763,7 @@ def new_SubDicts(self, sub_name): # ############################################################ + ####################### # Register an attribute # attr_obj as described by Params.attr_defaults @@ -833,7 +834,6 @@ def add_attr_unparsed_real(attr_name, params, dicts): ################################### # Create the frontend descript file def populate_frontend_descript(work_dir, frontend_dict, active_sub_list, params): # will be modified - frontend_dict.add("DowntimesFile", params.downtimes_file) frontend_dict.add("FrontendName", params.frontend_name) frontend_dict.add("WebURL", params.web_url) @@ -891,7 +891,6 @@ def populate_frontend_descript(work_dir, frontend_dict, active_sub_list, params) ####################### # Populate group descript def populate_group_descript(work_dir, group_descript_dict, sub_name, sub_params): # will be modified - group_descript_dict.add("GroupName", sub_name) group_descript_dict.add("MapFile", os.path.join(work_dir, cvWConsts.GROUP_MAP_FILE)) @@ -923,7 +922,6 @@ def populate_group_descript(work_dir, group_descript_dict, sub_name, sub_params) def apply_group_singularity_policy(descript_dict, sub_params, params): - glidein_singularity_use = None query_expr = descript_dict["FactoryQueryExpr"] match_expr = descript_dict["MatchExpr"] diff --git a/creation/lib/cvWParams.py b/creation/lib/cvWParams.py index c3bf66951..7b5b40d7c 100644 --- a/creation/lib/cvWParams.py +++ b/creation/lib/cvWParams.py @@ -797,6 +797,7 @@ def update_match_attrs(self): # Use the class method instead #################################################################### + # return attribute value in the proper python format def extract_attr_val(attr_obj): if not attr_obj.type in ("string", "int", "expr"): diff --git a/factory/glideFactory.py b/factory/glideFactory.py index 4bc642f2b..11af27232 100755 --- a/factory/glideFactory.py +++ b/factory/glideFactory.py @@ -202,7 +202,6 @@ def generate_log_tokens(startup_dir, glideinDescript): # Issue a token for each entry-recipient pair for entry in entries: - # Get the list of recipients if "LOG_RECIPIENTS_FACTORY" in glideFactoryConfig.JobParams(entry).data: log_recipients = glideFactoryConfig.JobParams(entry).data["LOG_RECIPIENTS_FACTORY"].split() diff --git a/factory/glideFactoryConfig.py b/factory/glideFactoryConfig.py index eeffee3fd..c2ad22295 100644 --- a/factory/glideFactoryConfig.py +++ b/factory/glideFactoryConfig.py @@ -46,6 +46,7 @@ def __init__(self): # ############################################################ + # loads a file composed of # NAME VAL # and creates diff --git a/factory/glideFactoryEntry.py b/factory/glideFactoryEntry.py index a6b2fee0f..f016a768f 100644 --- a/factory/glideFactoryEntry.py +++ b/factory/glideFactoryEntry.py @@ -1184,7 +1184,6 @@ def unit_work_v3( ) if entry.isSecurityClassInDowntime(client_security_name, credential_security_class): - # Cannot use proxy for submission but entry is not in downtime # since other proxies may map to valid security classes entry.log.warning( @@ -1851,7 +1850,6 @@ def update_entries_stats(factory_in_downtime, entry_list): updated_entries = [] for entry in entry_list: - # Add a heuristic to improve efficiency. Skip if no changes in the entry # if nothing_to_do: # continue @@ -1904,6 +1902,7 @@ def update_entries_stats(factory_in_downtime, entry_list): ############################################################ + # added by C.W. Murphy for glideFactoryEntryDescript def write_descript(entry_name, entryDescript, entryAttributes, entryParams, monitor_dir): entry_data = {entry_name: {}} diff --git a/factory/glideFactoryEntryGroup.py b/factory/glideFactoryEntryGroup.py index 0b4533d25..502bedffb 100755 --- a/factory/glideFactoryEntryGroup.py +++ b/factory/glideFactoryEntryGroup.py @@ -497,7 +497,6 @@ def iterate(parent_pid, sleep_time, advertize_rate, glideinDescript, frontendDes factory_downtimes = glideFactoryDowntimeLib.DowntimeFile(glideinDescript.data["DowntimesFile"]) while True: - # Check if parent is still active. If not cleanup and die. check_parent(parent_pid, glideinDescript, my_entries) diff --git a/factory/glideFactoryInterface.py b/factory/glideFactoryInterface.py index b25008a23..7937b734d 100644 --- a/factory/glideFactoryInterface.py +++ b/factory/glideFactoryInterface.py @@ -263,7 +263,7 @@ def findGroupWork( kel = data[k] el = {"requests": {}, "web": {}, "params": {}, "params_decrypted": {}, "monitor": {}, "internals": {}} - for (key, prefix) in ( + for key, prefix in ( ("requests", factoryConfig.client_req_prefix), ("web", factoryConfig.client_web_prefix), ("params", factoryConfig.glidein_param_prefix), @@ -308,7 +308,7 @@ def findGroupWork( invalid_classad = False - for (key, prefix) in (("params_decrypted", factoryConfig.encrypted_param_prefix),): + for key, prefix in (("params_decrypted", factoryConfig.encrypted_param_prefix),): # TODO: useless for, only one element plen = len(prefix) for attr in kel: @@ -476,7 +476,7 @@ def findWork( for k in list(data.keys()): kel = data[k] el = {"requests": {}, "web": {}, "params": {}, "params_decrypted": {}, "monitor": {}, "internals": {}} - for (key, prefix) in ( + for key, prefix in ( ("requests", factoryConfig.client_req_prefix), ("web", factoryConfig.client_web_prefix), ("params", factoryConfig.glidein_param_prefix), @@ -516,7 +516,7 @@ def findWork( continue # uh oh... either the client is misconfigured, or someone is trying to cheat invalid_classad = False - for (key, prefix) in (("params_decrypted", factoryConfig.encrypted_param_prefix),): + for key, prefix in (("params_decrypted", factoryConfig.encrypted_param_prefix),): plen = len(prefix) for attr in list(kel.keys()): if attr in reserved_names: @@ -632,7 +632,7 @@ def __init__( self.adParams["GlideinRequireGlideinProxy"] = "%s" % True # write out both the attributes, params and monitors - for (prefix, data) in ( + for prefix, data in ( (factoryConfig.glidein_attr_prefix, glidein_attrs), (factoryConfig.glidein_param_prefix, glidein_params), (factoryConfig.glidein_monitor_prefix, glidein_monitors), @@ -705,7 +705,6 @@ def __init__(self, factory_name, glidein_name, supported_signtypes, pub_key_obj) def advertizeGlobal( factory_name, glidein_name, supported_signtypes, pub_key_obj, stats_dict={}, factory_collector=DEFAULT_VAL ): - """ Creates the glidefactoryglobal classad and advertises. @@ -793,6 +792,7 @@ def deadvertizeFactory(factory_name, glidein_name, factory_collector=DEFAULT_VAL ############################################################ + # glidein_attrs is a dictionary of values to publish # like {"Arch":"INTEL","MinDisk":200000} # similar for glidein_params and glidein_monitor_monitors @@ -952,6 +952,7 @@ def writeToMultiClassadFile(self, filename=None, append=True): ############################## # Start INTERNAL + # glidein_attrs is a dictionary of values to publish # like {"Arch":"INTEL","MinDisk":200000} # similar for glidein_params and glidein_monitor_monitors @@ -1011,7 +1012,7 @@ def createGlideinClientMonitoringFile( fd.write("UpdateSequenceNumber = %i\n" % advertizeGFCCounter[client_name]) # write out both the attributes, params and monitors - for (prefix, data) in ( + for prefix, data in ( (factoryConfig.glidein_attr_prefix, glidein_attrs), (factoryConfig.glidein_param_prefix, client_params), (factoryConfig.glidein_monitor_prefix, client_monitors), diff --git a/factory/glideFactoryLib.py b/factory/glideFactoryLib.py index 913a7de82..b0781dc62 100644 --- a/factory/glideFactoryLib.py +++ b/factory/glideFactoryLib.py @@ -173,7 +173,6 @@ def secClass2Name(client_security_name, proxy_security_class): def getCondorQData(entry_name, client_name, schedd_name, factoryConfig=None): - """ Get Condor data, given the glidein name To be passed to the main functions @@ -405,7 +404,6 @@ def getCondorStatusData( client_startd_attribute=None, factoryConfig=None, ): - if factoryConfig is None: factoryConfig = globals()["factoryConfig"] @@ -580,7 +578,6 @@ def __init__( client_group_sign, factoryConfig=None, ): - if factoryConfig is None: factoryConfig = globals()["factoryConfig"] @@ -832,7 +829,6 @@ def clean_glidein_queue( log=logSupport.log, factoryConfig=None, ): - """Cleans up the glideins queue (removes any excesses) per the frontend request. We are not adjusting the glidein totals with what has been removed from the queue. It may take a cycle (or more) @@ -1000,7 +996,6 @@ def clean_glidein_queue( def sanitizeGlideins(condorq, log=logSupport.log, factoryConfig=None): - if factoryConfig is None: factoryConfig = globals()["factoryConfig"] @@ -1739,7 +1734,6 @@ def removeGlideins(schedd_name, jid_list, force=False, log=logSupport.log, facto is_not_first = 0 for jid in jid_list: - # Respect the max_removes limit and exit right away if required if len(removed_jids) >= factoryConfig.max_removes: break # limit reached, stop @@ -1823,7 +1817,6 @@ def get_submit_environment( log=logSupport.log, factoryConfig=None, ): - if factoryConfig is None: factoryConfig = globals()["factoryConfig"] @@ -2241,7 +2234,6 @@ class GlideinTotals: """Keeps track of all glidein totals.""" def __init__(self, entry_name, frontendDescript, jobDescript, entry_condorQ, log=logSupport.log): - # Initialize entry limits self.entry_name = entry_name self.entry_max_glideins = int(jobDescript.data["PerEntryMaxGlideins"]) @@ -2288,7 +2280,7 @@ def __init__(self, entry_name, frontendDescript, jobDescript, entry_condorQ, log ("PerFrontendMaxHeld", "max_held"), ) - for (jd_key, max_glideinstatus_key) in limits_keynames: + for jd_key, max_glideinstatus_key in limits_keynames: fe_glideins_param = jobDescript.data[jd_key] if fe_glideins_param.find(";") != -1: diff --git a/factory/glideFactoryMonitorAggregator.py b/factory/glideFactoryMonitorAggregator.py index d195575ec..fc7680124 100644 --- a/factory/glideFactoryMonitorAggregator.py +++ b/factory/glideFactoryMonitorAggregator.py @@ -826,7 +826,6 @@ def aggregateRRDStats(log=logSupport.log): tab = xmlFormat.DEFAULT_TAB for rrd in rrdstats_relname: - # assigns the data from every site to 'stats' stats = {} for entry in monitorAggregatorConfig.entries: diff --git a/factory/glideFactoryMonitoring.py b/factory/glideFactoryMonitoring.py index 47fcee987..d09c1b2fa 100644 --- a/factory/glideFactoryMonitoring.py +++ b/factory/glideFactoryMonitoring.py @@ -269,6 +269,7 @@ def write_rrd_multi_hetero(self, relative_fname, ds_desc_dict, time, val_dict): # ####################################################################################################################### + # TODO: ['Downtime'] is added to the self.data[client_name] dictionary only if logRequest is called before logSchedd, logClientMonitor # This is inconsistent and should be changed, Redmine [#17244] class condorQStats: @@ -1234,7 +1235,6 @@ def get_xml_updated(self, indent_tab=xmlFormat.DEFAULT_TAB, leading_tab=""): return xmlFormat.time2xml(self.updated, "updated", indent_tab, leading_tab) def write_file(self, monitoringConfig=None): - if monitoringConfig is None: monitoringConfig = globals()["monitoringConfig"] @@ -1488,7 +1488,8 @@ def fetchData(self, rrd_file, pathway, res, start, end): rrdtool fetch returns 3 tuples: a[0], a[1], & a[2]. [0] lists the resolution, start and end time, which can be specified as arugments of fetchData. [1] returns the names of the datasets. These names are listed in the key. - [2] is a list of tuples. each tuple contains data from every dataset. There is a tuple for each time data was collected.""" + [2] is a list of tuples. each tuple contains data from every dataset. There is a tuple for each time data was collected. + """ # use rrdtool to fetch data baseRRDSupport = rrdSupport.rrdSupport() @@ -1806,6 +1807,7 @@ def writeFile(self, path, xml_str, singleEntry=False): ############### P R I V A T E ################ + ################################################## def getAllJobTypes(): return ("validation", "idle", "badput", "nosuccess") diff --git a/factory/manageFactoryDowntimes.py b/factory/manageFactoryDowntimes.py index 1714a7758..d65d9b335 100755 --- a/factory/manageFactoryDowntimes.py +++ b/factory/manageFactoryDowntimes.py @@ -281,7 +281,6 @@ def vacuum(entry_or_id, opt_dict): def get_production_ress_entries(server, ref_dict_list): - production_entries = [] condor_obj = condorMonitor.CondorStatus(pool_name=server) diff --git a/factory/tools/analyze_entries b/factory/tools/analyze_entries index cde112a38..8a2e105fc 100755 --- a/factory/tools/analyze_entries +++ b/factory/tools/analyze_entries @@ -21,7 +21,6 @@ sys.path.append(os.path.join(STARTUP_DIR, "../../..")) def frontend_print(frontend, fe_mode, d, div, to_be_printed, fe_sorting, fe_attr_list, fe_sort_attribute): - fdiv = 1 if fe_attr_list[fe_sort_attribute] in ["total_time", "val", "used", "idle", "waste", "badput"]: fdiv = float(div) @@ -222,7 +221,6 @@ def lineprint(entry_name, entry, g_entry, g, to_be_printed, attr_list, sort_attr def main(): - usage = """ USAGE: -x [#] : interval to do verbose stats (default 24) @@ -416,15 +414,12 @@ USAGE: sys.exit(1) for entry in c_data["entries"]: - for frontend in c_data["entries"][entry]["frontends"]: - if filter_frontend != 0: if frontend != filter_frontend: continue for period in c_data["entries"][entry]["frontends"][frontend]["periods"]: - if int(period) not in data: data[int(period)] = {} if frontend not in data[int(period)]: @@ -601,7 +596,6 @@ Glidein log analysis for frontend %s - %s ################################################################################ if filter_frontend == 0 and frontend_mode == 0: - print( """ --------------------------------------- @@ -617,7 +611,6 @@ Per Entry (all frontends) stats for the past %s hours.\n""" totals_to_be_printed = [] for entry_name, entry in entry_data_all_frontends[period].items(): - if entry["glideins"] == 0 or entry["total_time"] == 0: continue if entry["validation"] == 0: diff --git a/factory/tools/analyze_frontends b/factory/tools/analyze_frontends index f8236f19f..f354579aa 100755 --- a/factory/tools/analyze_frontends +++ b/factory/tools/analyze_frontends @@ -22,7 +22,6 @@ sys.path.append(os.path.join(STARTUP_DIR, "../../..")) def list_print(frontend, zero_supp, entry_data, sorting, attr_list, sort_attribute, div): - to_be_printed = [] sum2 = 0 @@ -105,7 +104,6 @@ def list_print(frontend, zero_supp, entry_data, sorting, attr_list, sort_attribu def main(): - usage = """ USAGE: -x [#] : interval to do verbose stats (default 24) @@ -276,14 +274,12 @@ USAGE: for entry in rrd_data["entries"]: for frontend in rrd_data["entries"][entry]["frontends"]: - # if filtering, only choose selected frontend if filter_frontend != 0: if frontend != filter_frontend: continue for period, elements in rrd_data["entries"][entry]["frontends"][frontend]["periods"].items(): - if int(period) not in data: data[int(period)] = {} if frontend not in data[int(period)]: @@ -379,7 +375,6 @@ Status Attributes (Clients) analysis for %s - %s period_list = [period] for period in period_list: - title = "Past %.1f hours" % (float(period) / 3600) print( diff --git a/factory/tools/analyze_queues b/factory/tools/analyze_queues index a36ba5558..997e5c41d 100755 --- a/factory/tools/analyze_queues +++ b/factory/tools/analyze_queues @@ -22,7 +22,6 @@ sys.path.append(os.path.join(STARTUP_DIR, "../../..")) def list_print(frontend, zero_supp, entry_data, sorting, attr_list, sort_attribute, div): - to_be_printed = [] sum2 = 0 @@ -108,7 +107,6 @@ def list_print(frontend, zero_supp, entry_data, sorting, attr_list, sort_attribu def main(): - usage = """ USAGE: -x [#] : interval to do verbose stats (default 24) @@ -288,14 +286,12 @@ USAGE: for entry in rrd_data["entries"]: for frontend in rrd_data["entries"][entry]["frontends"]: - # if filtering, only choose selected frontend if filter_frontend != 0: if frontend != filter_frontend: continue for period, elements in rrd_data["entries"][entry]["frontends"][frontend]["periods"].items(): - if int(period) not in data: data[int(period)] = {} if frontend not in data[int(period)]: @@ -394,7 +390,6 @@ Status Attributes analysis for %s - %s period_list = [period] for period in period_list: - title = "Past %.1f hours" % (float(period) / 3600) p = period_data[period] status_total = ( diff --git a/factory/tools/cat_logs.py b/factory/tools/cat_logs.py index 6a171c173..55088b8a3 100755 --- a/factory/tools/cat_logs.py +++ b/factory/tools/cat_logs.py @@ -28,6 +28,7 @@ USAGE = "Usage: cat_logs.py YY/MM/DD [hh:mm:ss]" + # return a GlideinDescript with # factory_dir, date_arr and time_arr def parse_args(): diff --git a/factory/tools/find_StartdLogs.py b/factory/tools/find_StartdLogs.py index 10a7885d7..5f64b5240 100755 --- a/factory/tools/find_StartdLogs.py +++ b/factory/tools/find_StartdLogs.py @@ -29,6 +29,7 @@ USAGE = "Usage: find_StartdLogs.py YY/MM/DD [hh:mm:ss]" + # return a GlideinDescript with # factory_dir, date_arr and time_arr def parse_args(): diff --git a/factory/tools/find_logs.py b/factory/tools/find_logs.py index 934a1d665..bc8bf20fa 100755 --- a/factory/tools/find_logs.py +++ b/factory/tools/find_logs.py @@ -30,6 +30,7 @@ USAGE = "Usage: find_logs.py YY/MM/DD [hh:mm:ss]" + # return a GlideinDescript with # factory_dir, date_arr and time_arr def parse_args(): diff --git a/factory/tools/lib/analyze.py b/factory/tools/lib/analyze.py index e06b9ab67..e273cbe87 100644 --- a/factory/tools/lib/analyze.py +++ b/factory/tools/lib/analyze.py @@ -21,7 +21,6 @@ # Example: km(123456) returns "123 K" # and km(1234567) returns "1.23 M" def km(z): - if z < 0: neg = "-" else: diff --git a/frontend/glideinFrontend.py b/frontend/glideinFrontend.py index b790a7cb7..a794c068f 100755 --- a/frontend/glideinFrontend.py +++ b/frontend/glideinFrontend.py @@ -105,6 +105,7 @@ def poll_group_process(group_name, child): ############################################################ + # return the list of (group,walltime) pairs def spawn_iteration(work_dir, frontendDescript, groups, max_active, failure_dict, max_failures, action): childs = {} @@ -318,14 +319,12 @@ def spawn( logSupport.log.info("Frontend started with mode = %s" % mode) try: - # Service will exit on signal only. # This infinite loop is for the slave to go back into hibernation # once the master becomes alive. # Master never loops infinitely here, but instead it does in # the inner loop while(mode=='master') ... while True: - while hibernate: # If I am slave enter hibernation cycle while Master is alive logSupport.log.info("Master Frontend %s is online. Hibernating." % master_frontend_name) diff --git a/frontend/glideinFrontendConfig.py b/frontend/glideinFrontendConfig.py index e5a1ba008..78b437f1d 100644 --- a/frontend/glideinFrontendConfig.py +++ b/frontend/glideinFrontendConfig.py @@ -61,6 +61,7 @@ def get_group_dir(base_dir, group_name): # ############################################################ + # loads a file or URL composed of # NAME VAL # and creates @@ -252,6 +253,7 @@ def split_func(self, line, convert_function): # ############################################################ + # not everything is merged # the old element can still be accessed class ElementMergedDescript: diff --git a/frontend/glideinFrontendDowntimeLib.py b/frontend/glideinFrontendDowntimeLib.py index e13000426..16e951fe6 100644 --- a/frontend/glideinFrontendDowntimeLib.py +++ b/frontend/glideinFrontendDowntimeLib.py @@ -50,6 +50,7 @@ def read(self, raise_on_error=False): # INTERNAL - Do not use ############################# + # return a list of downtime periods (utimes) # a value of None idicates "forever" # for example: [(1215339200,1215439170),(1215439271,None)] @@ -116,7 +117,6 @@ def checkDowntime(fname, check_time=None): time_list = read(fname) for time_tuple in time_list: - if check_time < time_tuple[0]: # check_time before start continue @@ -155,7 +155,6 @@ def addPeriod(fname, start_time, end_time, create_if_empty=True): # end a downtime (not a scheduled one) # if end_time==None, use current time def endDowntime(fname, end_time=None): - if end_time is None: end_time = int(time.time()) diff --git a/frontend/glideinFrontendElement.py b/frontend/glideinFrontendElement.py index 2434934ab..2cac48100 100755 --- a/frontend/glideinFrontendElement.py +++ b/frontend/glideinFrontendElement.py @@ -1397,7 +1397,6 @@ def compute_glidein_min_idle( or (global_total_glideins >= self.global_total_max_glideins) or (global_total_idle_glideins >= self.global_total_max_vms_idle) ): - # Do not request more glideins under following conditions: # 1. Have all the running jobs I wanted # 2. Have enough idle vms/slots @@ -1504,7 +1503,6 @@ def identify_limits_triggered( global_total_idle_glideins, limits_triggered, ): - # Identify the limits triggered for advertizing in glideresource if count_status["Total"] >= self.max_running: limits_triggered["TotalGlideinsPerEntry"] = "count=%i, limit=%i" % (count_status["Total"], self.max_running) @@ -2059,7 +2057,6 @@ def get_condor_q(self, schedd_name): return condorq_dict def get_condor_status(self): - # All slots for this group status_dict = {} fe_counts = {"Idle": 0, "Total": 0} diff --git a/frontend/glideinFrontendInterface.py b/frontend/glideinFrontendInterface.py index 15177253e..6c7be85cb 100644 --- a/frontend/glideinFrontendInterface.py +++ b/frontend/glideinFrontendInterface.py @@ -261,7 +261,7 @@ def format_condor_dict(data): del kel[attr] # then move the parameters and monitoring - for (prefix, eldata) in ( + for prefix, eldata in ( (frontendConfig.glidein_param_prefix, el["params"]), (frontendConfig.glidein_monitor_prefix, el["monitor"]), ): @@ -799,7 +799,6 @@ def add( auth_method="Any", ha_mode="master", ): - params_obj = AdvertizeParams( request_name, glidein_name, @@ -1341,7 +1340,7 @@ def createAdvertizeWorkFile(self, factory_pool, params_obj, key_obj=None, file_i (frontendConfig.encrypted_param_prefix, encrypted_params), (frontendConfig.glidein_config_prefix, self.glidein_config_limits), ) - for (prefix, data) in classad_info_tuples: + for prefix, data in classad_info_tuples: for attr in list(data.keys()): writeTypedClassadAttrToFile(fd, f"{prefix}{attr}", data[attr]) @@ -1389,7 +1388,7 @@ def writeTypedClassadAttrToFile(fd, attr_name, attr_value): """ Given the FD, type check the value and write the info the classad file """ - if isinstance(attr_value, (int, int, float)): + if isinstance(attr_value, (int, float)): # don't quote numeric values fd.write(f"{attr_name} = {attr_value}\n") else: diff --git a/frontend/glideinFrontendLib.py b/frontend/glideinFrontendLib.py index a3d218fc5..4f5c5438d 100644 --- a/frontend/glideinFrontendLib.py +++ b/frontend/glideinFrontendLib.py @@ -783,7 +783,6 @@ def getCondorStatusNonDynamic(status_dict): def getIdleCondorStatus(status_dict): out = {} for collector_name in list(status_dict.keys()): - # Exclude partitionable slots with no free memory/cpus # Minimum memory required by CMS is 2500 MB # If the node had GPUs, there should be at least one available (requested by CMS) @@ -1236,6 +1235,7 @@ def getCondorStatusSchedds(collector_names, constraint=None, format_list=None, w # ############################################################ + # # Return a dictionary of schedds containing jobs of a certain type # Each element is a condorQ diff --git a/frontend/tools/glidein_off b/frontend/tools/glidein_off index 7b0ccf4b2..b01c365a7 100755 --- a/frontend/tools/glidein_off +++ b/frontend/tools/glidein_off @@ -199,6 +199,7 @@ def lookup_addrs(constr): ############################################################################# TP_FAIL = False + # thrad callback def run_off(addr, off_type): global TP_FAIL diff --git a/lib/condorMonitor.py b/lib/condorMonitor.py index b05e2f927..b859b8a0d 100644 --- a/lib/condorMonitor.py +++ b/lib/condorMonitor.py @@ -67,6 +67,7 @@ def htcondor_full_reload(): # Configuration # + # Set path to condor binaries def set_path(new_condor_bin_path): global condor_bin_path @@ -849,6 +850,7 @@ def getHash(self, hash_func): # ############################################################ + # check that req_format_els are present in in_format_list, and if not add them # return a new format_list def complete_format_list(in_format_list, req_format_els): @@ -890,6 +892,7 @@ def complete_format_list(in_format_list, req_format_els): # # + # 3 xml2list XML handler functions def xml2list_start_element(name, attrs): global xml2list_data, xml2list_inclassad, xml2list_inattr, xml2list_intype diff --git a/lib/config_util.py b/lib/config_util.py index 92a762418..71b8f92fc 100644 --- a/lib/config_util.py +++ b/lib/config_util.py @@ -273,7 +273,7 @@ def update(data, update_data, overwrite=True): if value is None: if key in data: del data[key] - elif isinstance(value, collections.Mapping): + elif isinstance(value, collections.abc.Mapping): sub_data = data.get(key, {}) if sub_data is not None: data[key] = update(sub_data, value, overwrite) diff --git a/lib/fork.py b/lib/fork.py index 25f8a3841..39cca2478 100644 --- a/lib/fork.py +++ b/lib/fork.py @@ -352,7 +352,6 @@ def fork_and_collect(self): return results def bounded_fork_and_collect(self, max_forks, log_progress=True, sleep_time=0.01): - post_work_info = {} nr_errors = 0 diff --git a/lib/pidSupport.py b/lib/pidSupport.py index 0bb1d2b45..35afd1e75 100644 --- a/lib/pidSupport.py +++ b/lib/pidSupport.py @@ -12,6 +12,7 @@ ############################################################ + # # Verify if the system knows about a pid # @@ -21,6 +22,7 @@ def check_pid(pid): ############################################################ + # this exception is raised when trying to register a pid # but another process is already owning the PID file class AlreadyRunning(RuntimeError): diff --git a/lib/timeConversion.py b/lib/timeConversion.py index 930c71a85..10fd77715 100644 --- a/lib/timeConversion.py +++ b/lib/timeConversion.py @@ -104,6 +104,7 @@ def get_time_in_format(now=None, time_format=None): # Internal ######################### + # time.daylight tells only if the computer support daylight saving time, # tm_isdst must be checked to see if it is in effect at time t # Some corner cases (changes in standard) are still uncovered, see https://bugs.python.org/issue1647654 diff --git a/lib/xmlParse.py b/lib/xmlParse.py index 7f60c518d..633fcd76d 100644 --- a/lib/xmlParse.py +++ b/lib/xmlParse.py @@ -87,7 +87,6 @@ def values(self): def xmlfile2dict( fname, use_ord_dict=False, always_singular_list=[] # if true, return OrderedDict instead of a regular dictionary ): # anything id listed here will be considered as a list - try: doc = xml.dom.minidom.parse(fname) except xml.parsers.expat.ExpatError as e: diff --git a/tools/lib/glideinCmd.py b/tools/lib/glideinCmd.py index 702c3156f..2eb1bf213 100644 --- a/tools/lib/glideinCmd.py +++ b/tools/lib/glideinCmd.py @@ -92,6 +92,7 @@ def exe_cmd_script(argv_func, argv=sys.argv): # INTERNAL - Do not use directly ###################################### + # create a file usable by a callback function for glideinMonitor.monitor def monitorScriptFromList(monitor_file_name, monitor_control_relname, script_list): # create the command file @@ -118,7 +119,6 @@ def argv2cmd(argv): # callback function for glideinMonitor.monitor # changes to the work dir and executes the command def createCmdMonitorFile(monitor_file_name, monitor_control_relname, argv, condor_status, monitorVM): - script_lines = [] # find work dir and execute command there # TODO: consider possible different directory for singularity jobs diff --git a/unittests/test_creation_lib_cWDictFile_DictFileTwoKeys.py b/unittests/test_creation_lib_cWDictFile_DictFileTwoKeys.py index 160535cfc..ca0838cca 100755 --- a/unittests/test_creation_lib_cWDictFile_DictFileTwoKeys.py +++ b/unittests/test_creation_lib_cWDictFile_DictFileTwoKeys.py @@ -168,7 +168,6 @@ def test_is_equal(self): ) def test_parse_val(self): - line = """foo 'bar'""" self.dict_file.parse_val(line) cpy = copy.deepcopy(self.dict_file) diff --git a/unittests/test_creation_lib_cgWDictFile.py b/unittests/test_creation_lib_cgWDictFile.py index 404804b60..46a29861b 100755 --- a/unittests/test_creation_lib_cgWDictFile.py +++ b/unittests/test_creation_lib_cgWDictFile.py @@ -65,6 +65,7 @@ XML_ENTRY = "fixtures/factory/config.d/Dev_Sites.xml" XML_ENTRY2 = "fixtures/factory/config.d/Dev_Sites2.xml" + # pylint: disable=maybe-no-member # class TestMonitorGroupDictFile(unittest.TestCase): class TestcgWDictFile(unittest.TestCase): diff --git a/unittests/test_creation_lib_cvWCreate.py b/unittests/test_creation_lib_cvWCreate.py index 89fe2fdaa..983866d77 100755 --- a/unittests/test_creation_lib_cvWCreate.py +++ b/unittests/test_creation_lib_cvWCreate.py @@ -66,7 +66,6 @@ def test_create_client_mapfile(self): os.remove(mapfile_fname) def test_get_template(self): - # test that we can fetch an existing template glideinWMS_dir = ".." template_name = "gwms-factory.service" diff --git a/unittests/test_factory_glideFactoryLib.py b/unittests/test_factory_glideFactoryLib.py index 258d7c366..7bff2b396 100755 --- a/unittests/test_factory_glideFactoryLib.py +++ b/unittests/test_factory_glideFactoryLib.py @@ -288,7 +288,6 @@ def test_is_str_safe(self): self.assertTrue(is_str_safe(s2)) def test_env_list2dict(self): - env = ["a=b", "c=d"] expected = {"a": "b", "c": "d"} self.assertEqual(expected, env_list2dict(env)) diff --git a/unittests/test_frontend.py b/unittests/test_frontend.py index 466a83538..9d2aa73e9 100755 --- a/unittests/test_frontend.py +++ b/unittests/test_frontend.py @@ -129,7 +129,6 @@ def setUp(self): def test_countMatch_missingKey(self): with mock.patch.object(glideinwms.frontend.glideinFrontendLib.logSupport.log, "debug") as m_debug: - match_obj = compile('glidein["attrs"]["FOO"] == 3', "", "eval") match_counts = glideinFrontendLib.countMatch(match_obj, self.condorq_dict, self.glidein_dict, {}, False) m_debug.assert_called_with( diff --git a/unittests/test_lib_xmlParse.py b/unittests/test_lib_xmlParse.py index 5f96cdf38..8e93f8f62 100755 --- a/unittests/test_lib_xmlParse.py +++ b/unittests/test_lib_xmlParse.py @@ -119,7 +119,6 @@ def test_update(self): self.assertTrue("foo" in ordered_dict) def test_values(self): - dict1 = xmlstring2dict(xmlstr, use_ord_dict=False, always_singular_list=[]) ordered_dict = OrderedDict(dict1) self.assertEqual(ordered_dict_values_repr, list(ordered_dict.values()).__repr__()) @@ -158,7 +157,6 @@ def test_get_xml_attributes(self): class TestIsSingularOf(unittest.TestCase): def test_is_singular_of(self): - self.assertEqual(True, is_singular_of(mysin="dog", myplu="dogs", always_singular_list=[])) self.assertEqual(True, is_singular_of(mysin="goose", myplu="geese", always_singular_list=["goose", "dog"])) self.assertEqual(False, is_singular_of(mysin="moose", myplu="meese", always_singular_list=["goose", "dog"]))