Convert bin/ scripts to python3-only

Remove future and __future__ imports

bin/HWMon/wmcore-SysStat

@@ -5,8 +5,6 @@ wmcore-SysStat
 
 Hardware monitoring utility for launched sensor daemons.
 """
-from __future__ import print_function
-
 import sys
 import os
 import getopt

bin/adhoc-scripts/ParseSpecCmsswdist.py

@@ -104,7 +104,7 @@
 
 """
 
-from __future__ import division  # Jenkins CI
+  # Jenkins CI
 
 import argparse
 import os

bin/adhoc-scripts/checkDsetFileCount.py

@@ -4,14 +4,12 @@
 using both DBS and Rucio services. It prints any inconsistency among
 those two.
 """
-from __future__ import print_function, division
+
 
 import logging
 import sys
 from collections import Counter
 
-from future.utils import viewkeys, viewvalues
-
 from WMCore.Services.DBS.DBS3Reader import DBS3Reader
 from WMCore.Services.Rucio.Rucio import Rucio
 
@@ -95,20 +93,20 @@ def main():
     dbsOutput, dbsFilesCounter = getFromDBS(datasetName, logger)
 
     logger.info("*** Dataset: %s", datasetName)
-    logger.info("Rucio file count : %s", sum(viewvalues(rucioOutput)))
+    logger.info("Rucio file count : %s", sum(rucioOutput.values()))
     logger.info("DBS file count   : %s", dbsFilesCounter['valid'] + dbsFilesCounter['invalid'])
     logger.info(" - valid files   : %s", dbsFilesCounter['valid'])
     logger.info(" - invalid files : %s", dbsFilesCounter['invalid'])
-    logger.info("Blocks in Rucio but not in DBS: %s", set(viewkeys(rucioOutput)) - set(viewkeys(dbsOutput)))
-    logger.info("Blocks in DBS but not in Rucio: %s", set(viewkeys(dbsOutput)) - set(viewkeys(rucioOutput)))
+    logger.info("Blocks in Rucio but not in DBS: %s", set(rucioOutput.keys()) - set(dbsOutput.keys()))
+    logger.info("Blocks in DBS but not in Rucio: %s", set(dbsOutput.keys()) - set(rucioOutput.keys()))
 
     for blockname in rucioOutput:
         if blockname not in dbsOutput:
             logger.error("This block does not exist in DBS: %s", blockname)
             continue
-        if rucioOutput[blockname] != sum(viewvalues(dbsOutput[blockname])):
+        if rucioOutput[blockname] != sum(dbsOutput[blockname].values()):
             logger.warning("Block with file mismatch: %s", blockname)
-            logger.warning("\tRucio: %s\t\tDBS: %s", rucioOutput[blockname], sum(viewvalues(dbsOutput[blockname])))
+            logger.warning("\tRucio: %s\t\tDBS: %s", rucioOutput[blockname], sum(dbsOutput[blockname].values()))
 
 
 if __name__ == "__main__":

bin/adhoc-scripts/checkStuckLQE.py

@@ -8,10 +8,9 @@
 local workqueue elements and print a summary of work/data location
 for the elements sitting in the Available status
 """
-from __future__ import print_function, division
+
 
 from builtins import next
-from future.utils import viewvalues, listvalues
 
 import sys
 import os
@@ -58,14 +57,14 @@ def commonDataLocation(element):
     """
     commonLoc = set()
     if element['PileupData']:
-        commonLoc = set(next(iter(viewvalues(element['PileupData']))))
+        commonLoc = set(next(iter(element['PileupData'].values())))
     if element['Inputs']:
         if commonLoc:
-            commonLoc = commonLoc & set(next(iter(viewvalues(element['Inputs']))))
+            commonLoc = commonLoc & set(next(iter(element['Inputs'].values())))
         else:
             commonLoc = set(list(element['Inputs'].values())[0])
     if element['ParentData']:
-        tempLoc = listvalues(element['ParentData'])
+        tempLoc = list(element['ParentData'].values())
         parentLoc = set(tempLoc[0])
         for temp in tempLoc:
             parentLoc = parentLoc & set(temp)

bin/adhoc-scripts/drainAgent.py

@@ -15,10 +15,9 @@
 NOTE: you need to source the agent environment:
 source apps/wmagent/etc/profile.d/init.sh
  """
-from __future__ import print_function, division
+
 
 from builtins import range, str, bytes
-from future.utils import viewitems
 
 import argparse
 import logging
@@ -165,7 +164,7 @@ def getDsetAndWf(lfns, wfsDict):
 
     match = []
     for dset in uniqDsets:
-        for wf, values in viewitems(wfsDict):
+        for wf, values in wfsDict.items():
             if dset in values['OutputDatasets']:
                 match.append((wf, values['RequestStatus'], dset))
     if match:

bin/adhoc-scripts/getWQStatusByWorkflow.py

@@ -12,7 +12,7 @@
 NOTE: you need to source the agent environment:
 source apps/wmagent/etc/profile.d/init.sh
 """
-from __future__ import print_function, division
+
 
 import os
 import sys

bin/adhoc-scripts/parseUnifiedCampaigns.py

@@ -45,10 +45,9 @@
 python parseUnifiedCampaigns.py --fin=wmcore_campaign.json --url=https://alancc7-cloud2.cern.ch/reqmgr2
          --verbose=10 --testcamp
 """
-from __future__ import print_function, division
+
 
 from builtins import object
-from future.utils import viewitems
 
 import argparse
 import json
@@ -128,7 +127,7 @@ def getSecondaryAAA(initialValue, uniRecord):
       * under the secondaries dictionary.
     If it appears multiple times, we make an OR of the values.
     """
-    for _, innerDict in viewitems(uniRecord.get("secondaries", {})):
+    for _, innerDict in uniRecord.get("secondaries", {}).items():
         if "secondary_AAA" in innerDict:
             print("Found internal secondary_AAA for campaign: %s" % uniRecord['name'])
             initialValue = initialValue or innerDict["secondary_AAA"]
@@ -161,7 +160,7 @@ def getSecondaryLocation(initialValue, uniRecord):
       * under the secondaries dictionary.
     If it appears multiple times, we make an intersection of the values.
     """
-    for _, innerDict in viewitems(uniRecord.get("secondaries", {})):
+    for _, innerDict in uniRecord.get("secondaries", {}).items():
         if "SecondaryLocation" in innerDict:
             print("Found internal SecondaryLocation for campaign: %s" % uniRecord['name'])
             initialValue = intersect(initialValue, innerDict["SecondaryLocation"])
@@ -178,7 +177,7 @@ def getSecondaries(initialValue, uniRecord):
       * taken from the SiteWhitelist key or
       * taken from the SecondaryLocation one
     """
-    for dset, innerDict in viewitems(uniRecord.get("secondaries", {})):
+    for dset, innerDict in uniRecord.get("secondaries", {}).items():
         print("Found secondaries for campaign: %s" % uniRecord['name'])
         initialValue[dset] = intersect(innerDict.get("SiteWhitelist", []),
                                        innerDict.get("SecondaryLocation", []))
@@ -224,7 +223,7 @@ def parse(istream, verbose=0):
         conf = dict(confRec)
         # Set default value from top level campaign configuration
         # or use the default values defined above
-        for uniKey, wmKey in viewitems(remap):
+        for uniKey, wmKey in remap.items():
             conf[wmKey] = rec.get(uniKey, conf[wmKey])
 
         conf['SiteWhiteList'] = getSiteList("SiteWhitelist", conf['SiteWhiteList'], rec)
@@ -330,7 +329,7 @@ def main():
             campData = json.load(istream)
             if isinstance(campData, dict):
                 # then it's a Unified-like campaign schema
-                for key, val in viewitems(campData):
+                for key, val in campData.items():
                     rec = {'name': key}
                     rec.update(val)
                     data.append(rec)

bin/adhoc-scripts/setrequeststatus.py

@@ -2,10 +2,7 @@
 """
 This script can be used to update the status of a request in ReqMgr2.
 """
-from __future__ import print_function, division
 
-from future import standard_library
-standard_library.install_aliases()
 import http.client
 import json
 import os

bin/adhoc-scripts/summaryWMStatsFailures.py

@@ -4,12 +4,7 @@
 type of failures, files skipped per agent; as well as an overview of failures
 per task
 """
-from __future__ import print_function, division
 
-from future.utils import viewitems
-
-from future import standard_library
-standard_library.install_aliases()
 import http.client
 import json
 import os
@@ -47,7 +42,7 @@ def main():
         print("  Skipped files: %s" % pformat(data[agent]['skipped']))
         print("  Overall agent status:\t\t %s" % data[agent]['status'])
 
-        for task, values in viewitems(data[agent]['tasks']):
+        for task, values in data[agent]['tasks'].items():
             if values['jobtype'] not in ['Production', 'Processing', 'Merge', 'Harvest']:
                 # print("Skipping task type %s, for %s" % (values['jobtype'], task))
                 continue

bin/adhoc-scripts/updateTotalStats.py

@@ -109,7 +109,7 @@ def main():
     msDbg = MSDbg(msConfig, logger=logger)
 
     badWfList = []
-    reqList = msDbg.getRequestRecords(reqStatus).values()
+    reqList = list(msDbg.getRequestRecords(reqStatus).values())
 
     allReqTypes = {req['RequestType'] for req in reqList}
     logger.info("All possible request types: \n%s", pformat(allReqTypes))
@@ -120,7 +120,7 @@ def main():
 
     for wflow in reqList:
         # First  check if the current workflwo suffers the issue:
-        if not keySetToCheck < wflow.keys():
+        if not keySetToCheck < list(wflow.keys()):
             # Create a `tmp` key to fill in all the information we'll need in the follwoing steps.
             wflow['tmp'] = {}
 

bin/check-ACDC-parentage

@@ -1,18 +1,9 @@
 #!/usr/bin/env python
-from __future__ import print_function, division
-from future import standard_library
-standard_library.install_aliases()
-from future.utils import viewitems
 
 import json
-import time
 import http.client
 import os
 
-from pprint import pprint
-from collections import defaultdict
-
-from dbs.apis.dbsClient import DbsApi
 
 def callRESTAPI(restURL, url="cmsweb.cern.ch"):
     conn = http.client.HTTPSConnection(url, cert_file=os.getenv('X509_USER_CERT'),
@@ -35,7 +26,7 @@ def investigateACDCCollection(workflow):
     if "rows" in result:
         if result["rows"]:
             for row in result["rows"]:
-                for inFile, value in viewitems(row["doc"]["files"]):
+                for inFile, value in row["doc"]["files"].items():
                     if not value["merged"] or value["merged"] == "0":
                         if value["parents"] and "/unmerged/" in value["parents"][0]:
                             missingFiles.add(inFile)

bin/check-phedex-dbs-status

@@ -1,5 +1,4 @@
 #!/usr/bin/env python
-from __future__ import (print_function, division)
 
 from WMComponent.DBS3Buffer.DBSBufferUtil  import DBSBufferUtil
 from WMCore.WMInit import connectToDB

bin/check-request-wq-status

@@ -1,7 +1,4 @@
 #!/usr/bin/env python
-from __future__ import print_function, division
-
-from future.utils import viewvalues
 
 import os
 from argparse import ArgumentParser
@@ -34,14 +31,14 @@ def getAcceptableSites(ele):
 
     if ele['Inputs']:
         if commonSites:
-            commonSites = commonSites & set([y for x in viewvalues(ele['Inputs']) for y in x])
+            commonSites = commonSites & set([y for x in ele['Inputs'].values() for y in x])
         else:
-            commonSites = set([y for x in viewvalues(ele['Inputs']) for y in x])
+            commonSites = set([y for x in ele['Inputs'].values() for y in x])
     if ele['PileupData']:
         if commonSites:
-            commonSites = commonSites & set([y for x in viewvalues(ele['PileupData']) for y in x])
+            commonSites = commonSites & set([y for x in ele['PileupData'].values() for y in x])
         else:
-            commonSites = set([y for x in viewvalues(ele['PileupData']) for y in x])
+            commonSites = set([y for x in ele['PileupData'].values() for y in x])
 
     return commonSites
 

bin/combineMinifyWMStats.py

@@ -1,4 +1,4 @@
-from __future__ import print_function
+
 from glob import iglob
 import shutil
 import os

bin/couch_archiver.py

@@ -4,7 +4,7 @@
 
 Replicate an agent's jobdump to another couch machine.
 """
-from __future__ import print_function
+
 
 import os
 import sys

bin/createStoreResults.py

@@ -19,11 +19,6 @@
 
 """
 
-from __future__ import print_function, division
-
-from future import standard_library
-standard_library.install_aliases()
-
 import http.client
 import json
 import os

bin/dbsbuffer-file-fix.py

@@ -1,15 +1,10 @@
 """
 """
-from __future__ import print_function
-
-from future.utils import listvalues
 
 import logging
 import os
-import sys
 import threading
 
-from WMCore.Database.CMSCouch import Database
 from WMCore.Database.DBFormatter import DBFormatter
 from WMCore.WMInit import connectToDB
 
@@ -38,7 +33,7 @@ def fixDBSmissingFileAssoc():
     print("trimed %s lenth" % len(result))
     insertSQL = """INSERT INTO dbsbuffer_file_location (filename, location)
                VALUES (:fileid, :seid)"""
-    done = formatter.dbi.processData(insertSQL, listvalues(result))
+    done = formatter.dbi.processData(insertSQL, list(result.values()))
     print("inserted %s" % done)
 
 if __name__ == '__main__':

bin/fix-dbs-parentage

@@ -41,8 +41,6 @@ $manage execute-agent fix-dbs-parentage -p --insert -d /SingleMuon/Run2016D-03Fe
 
 """
 
-from __future__ import print_function, division
-
 from builtins import input
 import time
 import datetime

bin/inject-to-config-cache

@@ -4,7 +4,6 @@ _inject-to-config-cache_
 
 Add a config and it's meta data to the config cache.
 """
-from __future__ import print_function
 
 import os
 import sys

bin/kill-workflow-in-agent

@@ -4,9 +4,9 @@ Script for killing a workflow: delete from LQ, delete in WMBS and set cancelled
 This has to be executed in each agent the workflow is running in (Agent Level)
 
 """
-from __future__ import print_function
 from builtins import str
-import os, sys
+import os
+import sys
 from WMCore.Configuration import loadConfigurationFile
 from WMCore.WorkQueue.WorkQueueBackend import WorkQueueBackend
 from WMCore.WMInit import connectToDB

bin/kill-workflow-in-global

@@ -10,7 +10,6 @@ This script will mark the global workqueue elements - for a given
 workflow - as CancelRequested, such that the agents can proceed
 and acknowledge it, moving elements to status Canceled.
 """
-from __future__ import print_function
 
 import os
 import sys

bin/outputmodules-from-config

@@ -4,10 +4,6 @@ _outputmodules-from-config_
 
 Pull output module metadata from a CMSSW config.
 """
-from __future__ import print_function
-
-from future import standard_library
-standard_library.install_aliases()
 import urllib.request
 
 import imp

bin/paused-jobs

@@ -1,5 +1,4 @@
 #!/usr/bin/env python
-from __future__ import print_function
 
 from builtins import input, object