diff --git a/src/python/Utils/Utilities.py b/src/python/Utils/Utilities.py index 89b62211d0..b30839b939 100644 --- a/src/python/Utils/Utilities.py +++ b/src/python/Utils/Utilities.py @@ -10,6 +10,24 @@ import sys from types import ModuleType, FunctionType from gc import get_referents +from functools import reduce + + +def reduceReport(reportList, expectedValue='OK'): + """ + Simple function to aggregate a list of values (possibly outcomes of multiple + calls to couchDB or similar) to a single value. + :param report: A list with accumulated report values + :param exepctedValue: The value with which the initial report list is expected + to be filled. (any type) Default: 'OK' + :return: Either the expected value (could be of any type) in the + case when all the entries in the list are identical or + False in the case when any of them deviates from the expected value. + """ + if reduce(lambda x, y: x == y == expectedValue and expectedValue, reportList, expectedValue): + return expectedValue + return False + def lowerCmsHeaders(headers): """ diff --git a/src/python/WMCore/ReqMgr/Service/Request.py b/src/python/WMCore/ReqMgr/Service/Request.py index caa9d4ed67..13b0fa2985 100644 --- a/src/python/WMCore/ReqMgr/Service/Request.py +++ b/src/python/WMCore/ReqMgr/Service/Request.py @@ -33,6 +33,7 @@ isUserAllowed) from WMCore.Services.RequestDB.RequestDBWriter import RequestDBWriter from WMCore.Services.WorkQueue.WorkQueue import WorkQueue +from Utils.Utilities import reduceReport class Request(RESTEntity): @@ -423,27 +424,39 @@ def _handleNoStatusUpdate(self, workload, request_args, dn): cherrypy.log('Updated workqueue statistics of "{}", with: {}'.format(workload.name(), reqArgs)) return report - reqArgsNothandled = [] - for reqArg in reqArgs: - if reqArg == 'RequestPriority': - validate_request_priority(reqArgs) - # must update three places: GQ elements, workload_cache and workload spec - self.gq_service.updatePriority(workload.name(), reqArgs['RequestPriority']) - workload.setPriority(reqArgs['RequestPriority']) - cherrypy.log('Updated priority of "{}" to: {}'.format(workload.name(), reqArgs['RequestPriority'])) - elif reqArg == "SiteWhitelist": - workload.setSiteWhitelist(reqArgs["SiteWhitelist"]) - cherrypy.log('Updated SiteWhitelist of "{}", with: {}'.format(workload.name(), reqArgs['SiteWhitelist'])) - elif reqArg == "SiteBlacklist": - workload.setSiteBlacklist(reqArgs["SiteBlacklist"]) - cherrypy.log('Updated SiteBlacklist of "{}", with: {}'.format(workload.name(), reqArgs['SiteBlacklist'])) - else: - reqArgsNothandled.append(reqArg) - cherrypy.log("Unhandled argument for no-status update: %s" % reqArg) + # reqArgsNothandled = [] + # for reqArg in reqArgs: + # if reqArg == 'RequestPriority': + # validate_request_priority(reqArgs) + # # must update three places: GQ elements, workload_cache and workload spec + # self.gq_service.updatePriority(workload.name(), reqArgs['RequestPriority']) + # workload.setPriority(reqArgs['RequestPriority']) + # cherrypy.log('Updated priority of "{}" to: {}'.format(workload.name(), reqArgs['RequestPriority'])) + # elif reqArg == "SiteWhitelist": + # workload.setSiteWhitelist(reqArgs["SiteWhitelist"]) + # cherrypy.log('Updated SiteWhitelist of "{}", with: {}'.format(workload.name(), reqArgs['SiteWhitelist'])) + # elif reqArg == "SiteBlacklist": + # workload.setSiteBlacklist(reqArgs["SiteBlacklist"]) + # cherrypy.log('Updated SiteBlacklist of "{}", with: {}'.format(workload.name(), reqArgs['SiteBlacklist'])) + # else: + # reqArgsNothandled.append(reqArg) + # cherrypy.log("Unhandled argument for no-status update: %s" % reqArg) + + # TODO: map setters to key names: + # Creating a setter method map - directly in the workload object. + + # Update all workload parameters based on the full reqArgsDiff dictionary + workload.updateWorkloadArgs(reqArgsDiff) - if reqArgsNothandled: - msg = "There were unhandled arguments left for no-status update: %s" % reqArgsNothandled - raise InvalidSpecParameterValue(msg) + # Commit the changes of the current workload object to the database: + workload.saveCouchUrl(workload.specUrl()) + + # Commit all Global WorkQueue changes per workflow in a single go: + # self.gq_service.updateElementsByWorkflow(workload.name(), reqArgsDiff) + + # if reqArgsNothandled: + # msg = "There were unhandled arguments left for no-status update: %s" % reqArgsNothandled + # raise InvalidSpecParameterValue(msg) # Commit the changes of the current workload object to the database: workload.saveCouchUrl(workload.specUrl()) diff --git a/src/python/WMCore/WMSpec/WMWorkload.py b/src/python/WMCore/WMSpec/WMWorkload.py index 7aee473d81..e91a203f0b 100644 --- a/src/python/WMCore/WMSpec/WMWorkload.py +++ b/src/python/WMCore/WMSpec/WMWorkload.py @@ -9,6 +9,9 @@ from builtins import next, range from future.utils import viewitems, viewvalues +from inspect import signature +from collections import namedtuple +import inspect from Utils.Utilities import strToBool from WMCore.Configuration import ConfigSection @@ -59,6 +62,8 @@ class WMWorkloadException(WMException): pass +setterTuple = namedtuple('SetterTuple', ['reqArg', 'setterFunc', 'setterSignature']) + class WMWorkloadHelper(PersistencyHelper): """ _WMWorkloadHelper_ @@ -68,6 +73,52 @@ class WMWorkloadHelper(PersistencyHelper): def __init__(self, wmWorkload=None): self.data = wmWorkload + self.settersMap = {} + + def updateWorkloadArgs(self, reqArgs): + """ + Method to take a dictionary of arguments of the type: + {reqArg1: value, + reqArg2: value, + ...} + and update the workload by a predefined map of reqArg to setter methods. + :param reqArgs: A Dictionary of request arguments to be updated + :return: Nothing, Raises an error of type WMWorkloadException if + fails to apply the proper setter method + """ + # NOTE: So far we support only a single argument setter methods, like + # setSiteWhitelist or setPriority. This may change in the future, + # but it will require a change in the logic of how we validate and + # call the proper setter methods bellow. + + # populate the current instance settersMap + self.settersMap['RequestPriority'] = setterTuple('RequestPriority', self.setPriority, inspect.signature(self.setPriority)) + self.settersMap['SiteBlacklist'] = setterTuple('SiteBlacklist', self.setSiteBlacklist, inspect.signature(self.setSiteBlacklist)) + self.settersMap['SiteWhitelist'] = setterTuple('SiteWhitelist', self.setSiteWhitelist, inspect.signature(self.setSiteWhitelist)) + + # First validate if we can properly call the setter function given the reqArgs passed. + for reqArg, argValue in reqArgs.items(): + if not self.settersMap.get(reqArg, None): + msg = f"Unsupported or missing setter method for updating reqArg: {reqArg}." + raise WMWorkloadException(msg) + try: + self.settersMap[reqArg].setterSignature.bind(argValue) + except TypeError as ex: + msg = f"Setter's method signature does not match the method calls we currently support: Error: req{str(ex)}" + raise WMWorkloadException(msg) + + # Now go through the reqArg again and call every setter method according to the map + for reqArg, argValue in reqArgs.items(): + try: + self.settersMap[reqArg].setterFunc(argValue) + except Exception as ex: + currFrame = inspect.currentframe() + argsInfo = inspect.getargvalues(currFrame) + argVals = {arg: argsInfo.locals.get(arg) for arg in argsInfo.args} + msg = f"Failure while calling setter method {self.settersMap[reqArg].setterFunc.__name__} " + msg += f"With arguments: {argVals}" + msg += f"Full exception string: {str(ex)}" + raise WMWorkloadException(msg) def setSpecUrl(self, url): self.data.persistency.specUrl = sanitizeURL(url)["url"] diff --git a/test/python/Utils_t/Utilities_t.py b/test/python/Utils_t/Utilities_t.py index 96ac6bba69..750fc6f0de 100644 --- a/test/python/Utils_t/Utilities_t.py +++ b/test/python/Utils_t/Utilities_t.py @@ -8,7 +8,7 @@ from Utils.Utilities import makeList, makeNonEmptyList, strToBool, \ safeStr, rootUrlJoin, zipEncodeStr, lowerCmsHeaders, getSize, \ - encodeUnicodeToBytes, diskUse, numberCouchProcess + encodeUnicodeToBytes, diskUse, numberCouchProcess, reduceReport class UtilitiesTests(unittest.TestCase): @@ -16,6 +16,52 @@ class UtilitiesTests(unittest.TestCase): unittest for Utilities functions """ + def testReduceReport(self): + """ + Test reduceReport function + """ + testList = ['OK', 'OK', 'OK'] + self.assertEqual(reduceReport(testList), 'OK') + self.assertEqual(reduceReport([]), 'OK') + testList.append(None) + self.assertEqual(reduceReport(testList), False) + + testList = ['nonDefStr', 'nonDefStr', 'nonDefStr'] + self.assertEqual(reduceReport(testList, expectedValue='nonDefStr'), 'nonDefStr') + self.assertEqual(reduceReport([], expectedValue='nonDefStr'), 'nonDefStr') + testList.append(False) + self.assertEqual(reduceReport(testList, expectedValue='nonDefStr'), False) + + testList = [True, True, True] + self.assertEqual(reduceReport(testList, expectedValue=True), True) + self.assertEqual(reduceReport([], expectedValue=True), True) + testList.append(False) + self.assertEqual(reduceReport(testList, expectedValue=True), False) + + testList = [None, None, None] + self.assertEqual(reduceReport(testList, expectedValue=None), False) + self.assertEqual(reduceReport([], expectedValue=None), False) + testList.append(False) + self.assertEqual(reduceReport(testList, expectedValue=None), False) + + testList = [False, False, False] + self.assertEqual(reduceReport(testList, expectedValue=False), False) + self.assertEqual(reduceReport([], expectedValue=False), False) + testList.append(True) + self.assertEqual(reduceReport(testList, expectedValue=False), False) + + testList = [{'res': 'OK'}, {'res': 'OK'}, {'res': 'OK'}] + self.assertDictEqual(reduceReport(testList, expectedValue={'res': 'OK'}), {'res': 'OK'}) + self.assertDictEqual(reduceReport([], expectedValue={'res': 'OK'}), {'res': 'OK'}) + testList.append(False) + self.assertEqual(reduceReport(testList, expectedValue={'res': 'OK'}), False) + + testList = [['OK'], ['OK'], ['OK']] + self.assertListEqual(reduceReport(testList, expectedValue=['OK']), ['OK']) + self.assertListEqual(reduceReport([], expectedValue=['OK']), ['OK']) + testList.append(False) + self.assertEqual(reduceReport(testList, expectedValue=['OK']), False) + def testMakeList(self): """ Test the makeList function