Merge pull request #11283 from amaltaro/fix-11279

Skip alert for RelVals with deletable pileup

src/python/WMCore/MicroService/MSOutput/MSOutputTemplate.py

@@ -8,9 +8,9 @@
 
 from future.utils import viewitems
 from builtins import str, bytes
-
 from time import time
 from copy import deepcopy
+from WMCore.MicroService.Tools.Common import isRelVal
 
 
 class MSOutputTemplate(dict):
@@ -248,7 +248,7 @@ def _setRelVal(self, myDoc):
         Evaluates whether it's a release validation request, if so, set the flag to True
         :param myDoc: the request dictionary
         """
-        if myDoc.get('SubRequestType') in ['RelVal', 'HIRelVal']:
+        if isRelVal(myDoc):
             self.setKey('IsRelVal', True)
 
     def setTransferStatus(self, newStatus):

src/python/WMCore/MicroService/MSRuleCleaner/MSRuleCleaner.py

@@ -34,7 +34,7 @@
 from WMCore.WMException import WMException
 from WMCore.Services.LogDB.LogDB import LogDB
 from WMCore.Services.WMStatsServer.WMStatsServer import WMStatsServer
-from WMCore.MicroService.Tools.Common import findParent
+from WMCore.MicroService.Tools.Common import findParent, isRelVal
 from Utils.Pipeline import Pipeline, Functor
 from Utils.CertTools import ckey, cert
 
@@ -684,7 +684,8 @@ def getRucioRules(self, wflow, gran, rucioAcct):
                         msg = "Pileup container %s has the following container-level rules to be removed: %s."
                         msg += " However, this component is no longer removing pileup rules."
                         self.logger.info(msg, dataCont, ruleIds)
-                        self.alertDeletablePU(wflow['RequestName'], dataCont, ruleIds)
+                        if not isRelVal(wflow):
+                            self.alertDeletablePU(wflow['RequestName'], dataCont, ruleIds)
                     elif ruleIds:
                         wflow['RulesToClean'][currPline].extend(ruleIds)
                         msg = "Container %s has the following container-level rules to be removed: %s"

src/python/WMCore/MicroService/MSRuleCleaner/MSRuleCleanerWflow.py

@@ -169,6 +169,7 @@ def docSchema(self):
             {
             "RequestName": "ReqName",
             "RequestType": "Type",
+            "SubRequestType": "Type",
             "RequestStatus": "Status",
             "OutputDatasets": [],
             'RulesToClean': {'plineMSTrCont': [],
@@ -200,6 +201,7 @@ def docSchema(self):
         docTemplate = [
             ('RequestName', None, (bytes, str)),
             ('RequestType', None, (bytes, str)),
+            ('SubRequestType', None, (bytes, str)),
             ('RequestStatus', None, (bytes, str)),
             ('OutputDatasets', [], list),
             ('RulesToClean', {}, dict),

src/python/WMCore/MicroService/MSTransferor/DataStructs/Workflow.py

@@ -9,7 +9,7 @@
 
 from copy import copy, deepcopy
 from WMCore.DataStructs.LumiList import LumiList
-from WMCore.MicroService.Tools.Common import getMSLogger
+from WMCore.MicroService.Tools.Common import getMSLogger, isRelVal
 from WMCore.Services.Rucio.RucioUtils import GROUPING_DSET, GROUPING_ALL, NUM_COPIES_DEFAULT
 
 
@@ -366,21 +366,14 @@ def _updateDataCampaignMap(self, parentName):
                 break
         self.dataCampaignMap.append(newItem)
 
-    def isRelVal(self):
-        """
-        Helper function to evaluate whether this workflow corresponds to
-        a RelVal workflow or not.
-        """
-        return self.data.get('SubRequestType') in ['RelVal', 'HIRelVal']
-
     def getWorkflowGroup(self):
         """
         Defines a workflow according to its group/activity, such as:
           *) release validation workflows
           *) standard central production workflows
         :return: a string with the workflow class: relval, processing
         """
-        if self.isRelVal():
+        if isRelVal(self.data):
             return "relval"
         return "production"
 

src/python/WMCore/MicroService/MSTransferor/MSTransferor.py

@@ -24,7 +24,7 @@
 from Utils.IteratorTools import grouper
 from WMCore.MicroService.DataStructs.DefaultStructs import TRANSFEROR_REPORT,\
     TRANSFER_RECORD, TRANSFER_COUCH_DOC
-from WMCore.MicroService.Tools.Common import gigaBytes, teraBytes
+from WMCore.MicroService.Tools.Common import gigaBytes, teraBytes, isRelVal
 from WMCore.MicroService.MSCore import MSCore
 from WMCore.MicroService.MSTransferor.RequestInfo import RequestInfo
 from WMCore.MicroService.MSTransferor.DataStructs.RSEQuotas import RSEQuotas
@@ -364,7 +364,7 @@ def checkPUDataLocation(self, wflow):
                 secLocation = pileupInput[dsetName]['locations']
                 # and a special case for RelVal workflows, which do not define
                 # secondary datasets and their location
-                if wflow.isRelVal():
+                if isRelVal(wflow.data):
                     campSecLocations = wflowPnns
                 else:
                     campSecLocations = campConfig['Secondaries'].get(dsetName, [])
@@ -442,7 +442,7 @@ def makeTransferRequest(self, wflow):
                 # secondary already in place
                 continue
 
-            if wflow.getPURSElist() and not wflow.isRelVal():
+            if wflow.getPURSElist() and not isRelVal(wflow.data):
                 rses = list(wflow.getPURSElist() & self.rseQuotas.getAvailableRSEs())
             else:
                 rses = self._getValidSites(wflow, dataIn)
@@ -642,7 +642,7 @@ def _getValidSites(self, wflow, dataIn):
         self.logger.info("  final list of PSNs to be use: %s", psns)
         pnns = self._getPNNsFromPSNs(psns)
 
-        if wflow.isRelVal():
+        if isRelVal(wflow.data):
             self.logger.info("RelVal workflow '%s' ignores sites out of quota", wflow.getName())
             return list(pnns)
 

src/python/WMCore/MicroService/Tools/Common.py

@@ -66,6 +66,15 @@ def getMSLogger(verbose, logger=None):
     return logger
 
 
+def isRelVal(reqDict):
+    """
+    Helper function to evaluate whether the workflow is RelVal or not.
+    :param reqDict: dictionary with the workflow description
+    :return: True if it's a RelVal workflow, otherwise False
+    """
+    return reqDict.get("SubRequestType", "") in ['RelVal', 'HIRelVal']
+
+
 def dbsInfo(datasets, dbsUrl):
     "Provides DBS info about dataset blocks"
     datasetBlocks = {}

test/python/WMCore_t/MicroService_t/MSRuleCleaner_t/MSRuleCleanerWflow_t.py

@@ -52,6 +52,7 @@ def testTaskChainDefaults(self):
         wflow = MSRuleCleanerWflow({})
         self.assertEqual(wflow["RequestName"], None)
         self.assertEqual(wflow["RequestType"], None)
+        self.assertEqual(wflow["SubRequestType"], None)
         self.assertEqual(wflow["RequestStatus"], None)
         self.assertEqual(wflow["OutputDatasets"], [])
         self.assertEqual(wflow["RulesToClean"], {})
@@ -109,6 +110,7 @@ def testTaskChain(self):
                                                {u'DN': u'', u'Status': u'closed-out', u'UpdateTime': 1607347706},
                                                {u'DN': u'', u'Status': u'announced', u'UpdateTime': 1607359514}],
                          'RequestType': u'TaskChain',
+                         'SubRequestType': u'',
                          'RulesToClean': {},
                          'TargetStatus': None,
                          'TransferDone': False,
@@ -152,6 +154,7 @@ def testReReco(self):
                                                {u'DN': u'', u'Status': u'running-closed', u'UpdateTime': 1588775354},
                                                {u'DN': u'', u'Status': u'completed', u'UpdateTime': 1589041121}],
                          'RequestType': u'ReReco',
+                         'SubRequestType': u'',
                          'RulesToClean': {},
                          'TargetStatus': None,
                          'TransferDone': False,
@@ -190,6 +193,7 @@ def testIncludeParents(self):
                                                {u'DN': u'', u'Status': u'running-closed', u'UpdateTime': 1586861535},
                                                {u'DN': u'', u'Status': u'completed', u'UpdateTime': 1586863942}],
                          'RequestType': u'StepChain',
+                         'SubRequestType': u'',
                          'RulesToClean': {},
                          'TargetStatus': None,
                          'TransferDone': False,
@@ -231,6 +235,7 @@ def testMultiPU(self):
                                                {u'DN': u'', u'Status': u'running-closed', u'UpdateTime': 1579579378},
                                                {u'DN': u'', u'Status': u'completed', u'UpdateTime': 1579587203}],
                          'RequestType': u'StepChain',
+                         'SubRequestType': u'',
                          'RulesToClean': {},
                          'TargetStatus': None,
                          'TransferDone': False,

test/python/WMCore_t/MicroService_t/MSRuleCleaner_t/MSRuleCleaner_t.py

@@ -115,6 +115,7 @@ def testPipelineAgentBlock(self):
                                                {u'DN': u'', u'Status': u'closed-out', u'UpdateTime': 1607347706},
                                                {u'DN': u'', u'Status': u'announced', u'UpdateTime': 1607359514}],
                          'RequestType': u'TaskChain',
+                         'SubRequestType': u'',
                          'RulesToClean': {'plineAgentBlock': []},
                          'TargetStatus': None,
                          'TransferDone': False,
@@ -158,6 +159,7 @@ def testPipelineAgentCont(self):
                                                {u'DN': u'', u'Status': u'closed-out', u'UpdateTime': 1607347706},
                                                {u'DN': u'', u'Status': u'announced', u'UpdateTime': 1607359514}],
                          'RequestType': u'TaskChain',
+                         'SubRequestType': u'',
                          'RulesToClean': {'plineAgentCont': []},
                          'TargetStatus': None,
                          'TransferDone': False,
@@ -203,6 +205,7 @@ def testPipelineMSTrBlock(self):
                                                {u'DN': u'', u'Status': u'closed-out', u'UpdateTime': 1607347706},
                                                {u'DN': u'', u'Status': u'announced', u'UpdateTime': 1607359514}],
                          'RequestType': u'TaskChain',
+                         'SubRequestType': u'',
                          'RulesToClean': {'plineMSTrBlock': []},
                          'TargetStatus': None,
                          'TransferDone': False,
@@ -248,6 +251,7 @@ def testPipelineMSTrCont(self):
                                                {u'DN': u'', u'Status': u'closed-out', u'UpdateTime': 1607347706},
                                                {u'DN': u'', u'Status': u'announced', u'UpdateTime': 1607359514}],
                          'RequestType': u'TaskChain',
+                         'SubRequestType': u'',
                          'RulesToClean': {'plineMSTrCont': []},
                          'TargetStatus': None,
                          'TransferDone': False,
@@ -305,6 +309,7 @@ def testPipelineArchive(self):
                                                {u'DN': u'', u'Status': u'closed-out', u'UpdateTime': 1607347706},
                                                {u'DN': u'', u'Status': u'announced', u'UpdateTime': 1607359514}],
                          'RequestType': u'TaskChain',
+                         'SubRequestType': u'',
                          'RulesToClean': {'plineAgentBlock': [], 'plineAgentCont': []},
                          'TargetStatus': 'normal-archived',
                          'TransferDone': False,

test/python/WMCore_t/MicroService_t/MSTransferor_t/DataStructs_t/Workflow_t.py

@@ -285,29 +285,6 @@ def testGetInputData(self):
         self.assertItemsEqual(blockChunks, {"block_A", "block_B", "parent_A", "parent_B", "parent_C"})
         self.assertEqual(sizeChunks, 39)
 
-
-    def testIsRelVal(self):
-        """
-        Test the `isRelVal` method functionality
-        """
-        requestTypes = ("StepChain", "TaskChain", "ReReco")
-        for wflowType in requestTypes:
-            wflowObj = Workflow("wflow_test", {"RequestType": wflowType, "DbsUrl": "a_dbs_url"})
-            self.assertFalse(wflowObj.isRelVal())
-
-            wflowObj = Workflow("wflow_test", {"RequestType": wflowType, "SubRequestType": "ReDigi",
-                                               "DbsUrl": "a_dbs_url"})
-            self.assertFalse(wflowObj.isRelVal())
-
-        for wflowType in requestTypes:
-            wflowObj = Workflow("wflow_test", {"RequestType": wflowType, "SubRequestType": "RelVal",
-                                               "DbsUrl": "a_dbs_url"})
-            self.assertTrue(wflowObj.isRelVal())
-
-            wflowObj = Workflow("wflow_test", {"RequestType": wflowType, "SubRequestType": "HIRelVal",
-                                               "DbsUrl": "a_dbs_url"})
-            self.assertTrue(wflowObj.isRelVal())
-
     def testGetWorkflowGroup(self):
         """
         Test the `getWorkflowGroup` method functionality

test/python/WMCore_t/MicroService_t/Tools_t/Common_t.py

@@ -10,12 +10,12 @@
 
 from nose.plugins.attrib import attr
 
-from WMCore.MicroService.Tools.Common import (dbsInfo, getEventsLumis,
-                                              findParent, hasHTTPFailed)
+from WMCore.MicroService.Tools.Common import (dbsInfo, getEventsLumis, findParent,
+                                              hasHTTPFailed, isRelVal)
 
 
 class CommonTest(unittest.TestCase):
-    "Unit test for Common module"
+    """Unit test for Common module"""
 
     def setUp(self):
         self.dbsUrl = "https://cmsweb-prod.cern.ch/dbs/prod/global/DBSReader"
@@ -75,6 +75,19 @@ def testHasHTTPFailed(self):
         # result below should never happen, but let's test the status code
         self.assertFalse(hasHTTPFailed({'data': 1, 'code': 200, 'error': 'blah'}))
 
+    def testIsRelVal(self):
+        """
+        Test the `isRelVal` method functionality
+        """
+        badSubReqType = ("ReDigi", "TaskChain", "")
+        goodSubReqType = ("RelVal", "HIRelVal")
+        for subType in badSubReqType:
+            testReqDict = {"RequestType": "StepChain", "DbsUrl": "a_dbs_url", "SubRequestType": subType}
+            self.assertFalse(isRelVal(testReqDict))
+
+        for subType in goodSubReqType:
+            testReqDict = {"RequestType": "StepChain", "DbsUrl": "a_dbs_url", "SubRequestType": subType}
+            self.assertTrue(isRelVal(testReqDict))
 
 if __name__ == '__main__':
     unittest.main()