From f10a6849e4ce5219b911531fc532d789186ff733 Mon Sep 17 00:00:00 2001 From: Valentin Kuznetsov Date: Mon, 16 Sep 2024 13:37:56 -0400 Subject: [PATCH 1/4] Update dbs3-client version --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 86b31e543b..d9ed334f1e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -18,7 +18,7 @@ CMSCouchapp~=1.3.4 # wmcore,wmagent CMSMonitoring~=0.6.10 # wmcore,wmagent,reqmgr2,reqmon,global-workqueue,reqmgr2ms-unmerged,reqmgr2ms-output,reqmgr2ms-pileup,reqmgr2ms-monitor,reqmgr2ms-transferor,reqmgr2ms-rulecleaner coverage~=5.4 # wmcore,wmagent,wmagent-devtools cx-Oracle~=8.3.0 # wmcore,wmagent -dbs3-client~=4.0.12 # wmcore,wmagent,reqmgr2,reqmon,global-workqueue +dbs3-client~=4.0.19 # wmcore,wmagent,reqmgr2,reqmon,global-workqueue future~=0.18.2 # wmcore,wmagent,wmagent-devtools,reqmgr2,reqmon,acdcserver,global-workqueue,reqmgr2ms-unmerged,reqmgr2ms-output,reqmgr2ms-pileup,reqmgr2ms-monitor,reqmgr2ms-transferor,reqmgr2ms-rulecleaner gfal2-python~=1.11.0.post3 # wmcore,reqmgr2ms-unmerged httplib2~=0.20.4 # wmcore,wmagent,reqmgr2,reqmon,acdcserver,global-workqueue,reqmgr2ms-unmerged,reqmgr2ms-output,reqmgr2ms-pileup,reqmgr2ms-monitor,reqmgr2ms-transferor,reqmgr2ms-rulecleaner From 892a694b63f4496ed73fdf52af8f73df90fcd2fd Mon Sep 17 00:00:00 2001 From: Dario Mapelli Date: Thu, 19 Sep 2024 17:40:55 +0200 Subject: [PATCH 2/4] dbs3-client - fix specific version in requirements.txt with == --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index d9ed334f1e..511c72d2a2 100644 --- a/requirements.txt +++ b/requirements.txt @@ -18,7 +18,7 @@ CMSCouchapp~=1.3.4 # wmcore,wmagent CMSMonitoring~=0.6.10 # wmcore,wmagent,reqmgr2,reqmon,global-workqueue,reqmgr2ms-unmerged,reqmgr2ms-output,reqmgr2ms-pileup,reqmgr2ms-monitor,reqmgr2ms-transferor,reqmgr2ms-rulecleaner coverage~=5.4 # wmcore,wmagent,wmagent-devtools cx-Oracle~=8.3.0 # wmcore,wmagent -dbs3-client~=4.0.19 # wmcore,wmagent,reqmgr2,reqmon,global-workqueue +dbs3-client==4.0.19 # wmcore,wmagent,wmagent-devtools,reqmgr2,reqmon,global-workqueue,reqmgr2ms-output,reqmgr2ms-pileup,reqmgr2ms-rulecleaner,reqmgr2ms-transferor future~=0.18.2 # wmcore,wmagent,wmagent-devtools,reqmgr2,reqmon,acdcserver,global-workqueue,reqmgr2ms-unmerged,reqmgr2ms-output,reqmgr2ms-pileup,reqmgr2ms-monitor,reqmgr2ms-transferor,reqmgr2ms-rulecleaner gfal2-python~=1.11.0.post3 # wmcore,reqmgr2ms-unmerged httplib2~=0.20.4 # wmcore,wmagent,reqmgr2,reqmon,acdcserver,global-workqueue,reqmgr2ms-unmerged,reqmgr2ms-output,reqmgr2ms-pileup,reqmgr2ms-monitor,reqmgr2ms-transferor,reqmgr2ms-rulecleaner From 0190788f36c756a3b27d17937de95a9dec43b7c8 Mon Sep 17 00:00:00 2001 From: Dario Mapelli Date: Fri, 20 Sep 2024 16:20:58 +0200 Subject: [PATCH 3/4] dbs3-client - msoutput and mstransferor are the only ms that use it --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 511c72d2a2..c0e4c0bcfa 100644 --- a/requirements.txt +++ b/requirements.txt @@ -18,7 +18,7 @@ CMSCouchapp~=1.3.4 # wmcore,wmagent CMSMonitoring~=0.6.10 # wmcore,wmagent,reqmgr2,reqmon,global-workqueue,reqmgr2ms-unmerged,reqmgr2ms-output,reqmgr2ms-pileup,reqmgr2ms-monitor,reqmgr2ms-transferor,reqmgr2ms-rulecleaner coverage~=5.4 # wmcore,wmagent,wmagent-devtools cx-Oracle~=8.3.0 # wmcore,wmagent -dbs3-client==4.0.19 # wmcore,wmagent,wmagent-devtools,reqmgr2,reqmon,global-workqueue,reqmgr2ms-output,reqmgr2ms-pileup,reqmgr2ms-rulecleaner,reqmgr2ms-transferor +dbs3-client==4.0.19 # wmcore,wmagent,wmagent-devtools,reqmgr2,reqmon,global-workqueue,reqmgr2ms-output,reqmgr2ms-transferor future~=0.18.2 # wmcore,wmagent,wmagent-devtools,reqmgr2,reqmon,acdcserver,global-workqueue,reqmgr2ms-unmerged,reqmgr2ms-output,reqmgr2ms-pileup,reqmgr2ms-monitor,reqmgr2ms-transferor,reqmgr2ms-rulecleaner gfal2-python~=1.11.0.post3 # wmcore,reqmgr2ms-unmerged httplib2~=0.20.4 # wmcore,wmagent,reqmgr2,reqmon,acdcserver,global-workqueue,reqmgr2ms-unmerged,reqmgr2ms-output,reqmgr2ms-pileup,reqmgr2ms-monitor,reqmgr2ms-transferor,reqmgr2ms-rulecleaner From 4af7e2c4555a104dc4d6996f7a47c43aa61c43bf Mon Sep 17 00:00:00 2001 From: Dario Mapelli Date: Fri, 20 Sep 2024 17:20:40 +0200 Subject: [PATCH 4/4] dbs3-client - import it only in the functions that need it --- src/python/WMCore/MicroService/Tools/Common.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/src/python/WMCore/MicroService/Tools/Common.py b/src/python/WMCore/MicroService/Tools/Common.py index 6068c02b6d..908d90730a 100644 --- a/src/python/WMCore/MicroService/Tools/Common.py +++ b/src/python/WMCore/MicroService/Tools/Common.py @@ -25,10 +25,6 @@ from WMCore.Services.pycurl_manager import RequestHandler from WMCore.Services.pycurl_manager import getdata as multi_getdata -# DBS agregators -from dbs.apis.dbsClient import aggRuns, aggFileLumis - - # static variables STEP_PAT = re.compile(r'Step[0-9]') TASK_PAT = re.compile(r'Task[0-9]') @@ -279,6 +275,11 @@ def getFileLumisInBlock(blocks, dbsUrl, validFileOnly=1): :param validFileOnly: integer flag for valid files only or not :return: a dict of blocks with list of file/run/lumi info """ + # importing dbs3-client only in the functions where it is used so that + # we do not need to add it to the docker images of microservices that do + # not use it. + from dbs.apis.dbsClient import aggFileLumis + runLumisByBlock = {} urls = ['%s/filelumis?validFileOnly=%d&block_name=%s' % (dbsUrl, validFileOnly, quote(b)) for b in blocks] # limit it to 10 concurrent calls not to overload DBS @@ -343,6 +344,11 @@ def getRunsInBlock(blocks, dbsUrl): :param dbsUrl: string with the DBS URL :return: a dictionary of block names and a list of run numbers """ + # importing dbs3-client only in the functions where it is used so that + # we do not need to add it to the docker images of microservices that do + # not use it. + from dbs.apis.dbsClient import aggRuns + runsByBlock = {} urls = ['%s/runs?block_name=%s' % (dbsUrl, quote(b)) for b in blocks] logging.info("Executing %d requests against DBS 'runs' API", len(urls))