Merge pull request #11043 from vkuznet/fix-procver-schema2

Adjust processing version to be integer

src/python/WMComponent/DBS3Buffer/DBSBufferBlock.py

@@ -231,12 +231,14 @@ def setProcessingVer(self, procVer):
 
         Set the block's processing version.
         """
-        # compatibility statement for old style proc ver (still needed ?)
-        if procVer.count("-") == 1:
-            self.data["processing_era"]["processing_version"] = procVer.split("-v")[1]
-        else:
-            self.data["processing_era"]["processing_version"] = procVer
-
+        pver = procVer or 0
+        try:
+            pver = int(pver)
+        except TypeError:
+            msg = "Provided procVer=%s of type %s cannot be converted to int" \
+                    % (procVer, type(procVer))
+            raise TypeError(msg) from None
+        self.data["processing_era"]["processing_version"] = pver
         self.data["processing_era"]["create_by"] = "WMAgent"
         self.data["processing_era"]["description"] = ""
         return

src/python/WMComponent/DBS3Buffer/DBSBufferFile.py

@@ -387,8 +387,14 @@ def setProcessingVer(self, ver):
 
         Set the era
         """
-
-        self['processingVer'] = ver
+        procVer = ver or 0
+        try:
+            procVer = int(procVer)
+        except TypeError:
+            msg = "Provided ver=%s of type %s cannot be converted to int" \
+                    % (ver, type(ver))
+            raise TypeError(msg) from None
+        self['processingVer'] = procVer
         return
 
     def setAcquisitionEra(self, era):

src/python/WMComponent/DBS3Buffer/MySQL/Create.py

@@ -28,7 +28,7 @@ def __init__(self, logger=None, dbi=None, params=None):
             """CREATE TABLE dbsbuffer_dataset (
                  id              INTEGER      AUTO_INCREMENT,
                  path            VARCHAR(500) COLLATE latin1_general_cs NOT NULL,
-                 processing_ver  VARCHAR(255),
+                 processing_ver  INTEGER      NOT NULL,
                  acquisition_era VARCHAR(255),
                  valid_status    VARCHAR(20),
                  global_tag      VARCHAR(255),

src/python/WMComponent/DBS3Buffer/MySQL/FindDASToUpload.py

@@ -44,11 +44,6 @@ def makeDAS(self, results):
         """
         datasetalgos = []
         for result in results:
-
-            # compatibility statement for old style proc ver (still needed ?)
-            if result['procver'].count("-") == 1:
-                result['procver'] = result['procver'].split("-v")[1]
-
             datasetalgos.append( { 'DatasetPath' : result['datasetpath'],
                                    'AcquisitionEra' : result['acquera'],
                                    'ProcessingVer' : result['procver'] } )

src/python/WMComponent/DBS3Buffer/MySQL/UpdateDataset.py

@@ -26,11 +26,17 @@ class UpdateDataset(DBFormatter):
               WHERE id = :id
            """
     def execute(self, datasetId,
-                processingVer = None, acquisitionEra = None,
+                processingVer = 0, acquisitionEra = None,
                 validStatus = None, globalTag = None,
                 parent = None, prep_id = None,
                 conn = None, transaction = False):
 
+        # to be backward compatible with various places in WMCore codebase which
+        # assign processingVer to None default value. In DBS and WMCore databases
+        # the processsing version should be integer data type
+        if processingVer is None:
+            processingVer = 0
+
         bindVars = {"procVer" : processingVer,
                     "acqEra" : acquisitionEra,
                     "valid" : validStatus,

src/python/WMComponent/DBS3Buffer/Oracle/Create.py

@@ -27,7 +27,7 @@ def __init__(self, logger = None, dbi = None, params = None):
             """CREATE TABLE dbsbuffer_dataset (
                  id                INTEGER        NOT NULL,
                  path              VARCHAR2(500)  NOT NULL,
-                 processing_ver    VARCHAR2(255),
+                 processing_ver    INTEGER        NOT NULL,
                  acquisition_era   VARCHAR2(255),
                  valid_status      VARCHAR2(20),
                  global_tag        VARCHAR2(255),

src/python/WMCore/WMSpec/WMWorkload.py

@@ -949,6 +949,9 @@ def setProcessingVersion(self, processingVersions):
 
         Change the processing version for all tasks in the spec and then update
         all of the output LFNs and datasets to use the new processing version.
+
+        :param processingVersions: can be any data-type but it is set from StdBase
+        which performs the input data sanitization/type already.
         """
         stepNameMapping = self.getStepMapping()
 

test/python/WMComponent_t/DBS3Buffer_t/DBSBufferFile_t.py

@@ -947,7 +947,7 @@ def testProperties(self):
                                configContent="MOREGIBBERISH")
         testFileA.setDatasetPath("/Cosmics/CRUZET09-PromptReco-v1/RECO")
         testFileA.setValidStatus(validStatus="VALID")
-        testFileA.setProcessingVer(ver="ProcVer")
+        testFileA.setProcessingVer(ver="123")
         testFileA.setAcquisitionEra(era="AcqEra")
         testFileA.setGlobalTag(globalTag="GlobalTag")
         testFileA.setDatasetParent(datasetParent="Parent")

test/python/WMComponent_t/JobAccountant_t/JobAccountant_t.py

@@ -1005,7 +1005,7 @@ def testMergeSuccess(self):
         datasetInfo = myThread.dbi.processData("SELECT * FROM dbsbuffer_dataset")[0].fetchall()[0]
         self.assertEqual(datasetInfo[1],
                          "/Mu/IansMagicMushroomSoup-T0Test-AnalyzeThisAndGetAFreePhD-PreScaleThingy10-v9_29_pre14replaythingy_v5/AOD")
-        self.assertEqual(datasetInfo[2], "9")
+        self.assertEqual(datasetInfo[2], 9) # processing version, should be integer
         self.assertEqual(datasetInfo[3], "IansMagicMushroomSoup")
         self.assertEqual(datasetInfo[4], "Production")
         self.assertEqual(datasetInfo[5], "GT:Super")