diff --git a/bin/adhoc-scripts/checkDsetFileCount.py b/bin/adhoc-scripts/checkDsetFileCount.py index 6ec8249c07..8ab209489b 100644 --- a/bin/adhoc-scripts/checkDsetFileCount.py +++ b/bin/adhoc-scripts/checkDsetFileCount.py @@ -18,7 +18,7 @@ RUCIO_ACCT = "wma_prod" RUCIO_HOST = "http://cms-rucio.cern.ch" RUCIO_AUTH = "https://cms-rucio-auth.cern.ch" -DBS_URL = "https://cmsweb.cern.ch/dbs/prod/global/DBSReader/" +DBS_URL = "https://cmsweb-prod.cern.ch/dbs/prod/global/DBSReader/" def loggerSetup(logLevel=logging.INFO): diff --git a/bin/adhoc-scripts/workflowCompletion.py b/bin/adhoc-scripts/workflowCompletion.py index 1c6c714a0f..bb083f5638 100644 --- a/bin/adhoc-scripts/workflowCompletion.py +++ b/bin/adhoc-scripts/workflowCompletion.py @@ -150,7 +150,7 @@ def main(): parser.error("You must provide either a workflow name or an input file name.") sys.exit(3) - cmswebUrl = "https://" + args.cms if args.cms else "https://cmsweb.cern.ch" + cmswebUrl = "https://" + args.cms if args.cms else "https://cmsweb-prod.cern.ch" reqmgrUrl = "https://" + args.reqmgr if args.reqmgr else "https://cmsweb.cern.ch" for reqName in listRequests: diff --git a/bin/createStoreResults.py b/bin/createStoreResults.py index b5669b121f..78fa36e456 100644 --- a/bin/createStoreResults.py +++ b/bin/createStoreResults.py @@ -48,7 +48,7 @@ "AcquisitionEra": "UPDATEME", "ProcessingVersion": 1, # will be updated too "Campaign": "StoreResults", - "DbsUrl": "https://cmsweb.cern.ch/dbs/prod/phys03/DBSReader", + "DbsUrl": "https://cmsweb-prod.cern.ch/dbs/prod/phys03/DBSReader", "GlobalTag": "crab3_tag", "Memory": 2000, "RequestPriority": 999999, @@ -86,7 +86,7 @@ def migrateDataset(dset, dbsInst): Migrate dataset from the user instance to the DBS prod one. It returns the origin site name, which should be used for assignment """ - dbsInst = "https://cmsweb.cern.ch/dbs/prod/%s/DBSReader" % dbsInst + dbsInst = "https://cmsweb-prod.cern.ch/dbs/prod/%s/DBSReader" % dbsInst migrateArgs = {'migration_url': dbsInst, 'migration_input': dset} dbsApi.submitMigration(migrateArgs) print("Migrating dataset %s from %s to prod/global" % (dset, dbsInst)) @@ -104,7 +104,7 @@ def buildRequest(userDict): newSchema = copy(DEFAULT_DICT) newSchema.update(userDict) - newSchema['DbsUrl'] = "https://cmsweb.cern.ch/dbs/prod/%s/DBSReader" % newSchema['DbsUrl'] + newSchema['DbsUrl'] = "https://cmsweb-prod.cern.ch/dbs/prod/%s/DBSReader" % newSchema['DbsUrl'] # Remove spaces from the Physics Group value newSchema['PhysicsGroup'] = newSchema['PhysicsGroup'].replace(" ", "") # Set PrepID according to the date and time diff --git a/bin/fix-dbs-parentage b/bin/fix-dbs-parentage index b11a0548c7..1deb073416 100755 --- a/bin/fix-dbs-parentage +++ b/bin/fix-dbs-parentage @@ -106,7 +106,7 @@ if __name__ == '__main__': end = time.mktime(datetime.datetime.strptime(args.end, "%d/%m/%Y").timetuple()) if args.prod: - dbsURL = 'https://cmsweb.cern.ch/dbs/prod/global/DBSWriter' + dbsURL = 'https://cmsweb-prod.cern.ch/dbs/prod/global/DBSWriter' logger.info("Using production: %s", dbsURL) else: dbsURL = 'https://cmsweb-testbed.cern.ch/dbs/int/global/DBSWriter' diff --git a/deploy/WMAgent.production b/deploy/WMAgent.production index 6e114dadeb..4aa4873b7a 100644 --- a/deploy/WMAgent.production +++ b/deploy/WMAgent.production @@ -15,7 +15,7 @@ WMSTATS_URL=https://cmsweb.cern.ch/couchdb/wmstats REQMGR_URL=https://cmsweb.cern.ch/reqmgr/rest ACDC_URL=https://cmsweb.cern.ch/couchdb/acdcserver WORKLOAD_SUMMARY_URL=https://cmsweb.cern.ch/couchdb/workloadsummary -DBS3_URL=https://cmsweb.cern.ch/dbs/prod/global/DBSWriter +DBS3_URL=https://cmsweb-prod.cern.ch/dbs/prod/global/DBSWriter PHEDEX_URL=https://cmsweb.cern.ch/phedex/datasvc/json/prod/ DQM_URL=https://cmsweb.cern.ch/dqm/offline DASHBOARD_URL=http://dashb-ssb.cern.ch/dashboard diff --git a/deploy/deploy-wmagent.sh b/deploy/deploy-wmagent.sh index ac052f171e..35c149e32d 100644 --- a/deploy/deploy-wmagent.sh +++ b/deploy/deploy-wmagent.sh @@ -24,8 +24,8 @@ ### Usage: -n <agent_number> Agent number to be set when more than 1 agent connected to the same team (defaults to 0) ### Usage: ### Usage: deploy-wmagent.sh -w <wma_version> -d <deployment_tag> -t <team_name> [-s <scram_arch>] [-r <repository>] [-n <agent_number>] -### Usage: Example: sh deploy-wmagent.sh -w 1.4.3.patch2 -d HG2012f -t production -n 30 -### Usage: Example: sh deploy-wmagent.sh -w 1.4.3.patch2 -d HG2012f -t testbed-vocms001 -p "9963 9959" -r comp=comp.amaltaro +### Usage: Example: sh deploy-wmagent.sh -w 1.4.5.patch1 -d HG2102e -t production -n 30 +### Usage: Example: sh deploy-wmagent.sh -w 1.4.5.patch2 -d HG2102e -t testbed-vocms001 -p "9963 9959" -r comp=comp.amaltaro ### Usage: IAM=`whoami` @@ -342,8 +342,8 @@ if [[ "$TEAMNAME" == relval ]]; then sed -i "s+config.RucioInjector.metaDIDProject = 'Production'+config.RucioInjector.metaDIDProject = 'RelVal'+" $MANAGE_DIR/config.py elif [[ "$TEAMNAME" == *testbed* ]] || [[ "$TEAMNAME" == *dev* ]]; then GLOBAL_DBS_URL=https://cmsweb-testbed.cern.ch/dbs/int/global/DBSReader - sed -i "s+DBSInterface.globalDBSUrl = 'https://cmsweb.cern.ch/dbs/prod/global/DBSReader'+DBSInterface.globalDBSUrl = '$GLOBAL_DBS_URL'+" $MANAGE_DIR/config.py - sed -i "s+DBSInterface.DBSUrl = 'https://cmsweb.cern.ch/dbs/prod/global/DBSReader'+DBSInterface.DBSUrl = '$GLOBAL_DBS_URL'+" $MANAGE_DIR/config.py + sed -i "s+DBSInterface.globalDBSUrl = 'https://cmsweb-prod.cern.ch/dbs/prod/global/DBSReader'+DBSInterface.globalDBSUrl = '$GLOBAL_DBS_URL'+" $MANAGE_DIR/config.py + sed -i "s+DBSInterface.DBSUrl = 'https://cmsweb-prod.cern.ch/dbs/prod/global/DBSReader'+DBSInterface.DBSUrl = '$GLOBAL_DBS_URL'+" $MANAGE_DIR/config.py sed -i "s+config.RucioInjector.metaDIDProject = 'Production'+config.RucioInjector.metaDIDProject = 'Test'+" $MANAGE_DIR/config.py fi diff --git a/doc/createSpecs/DQMHarvest_createSpec.json b/doc/createSpecs/DQMHarvest_createSpec.json index c9e3c9d317..cdb014a6bb 100644 --- a/doc/createSpecs/DQMHarvest_createSpec.json +++ b/doc/createSpecs/DQMHarvest_createSpec.json @@ -75,7 +75,7 @@ "type": "<type 'int'>" }, "DbsUrl": { - "default": "https://cmsweb.cern.ch/dbs/prod/global/DBSReader", + "default": "https://cmsweb-prod.cern.ch/dbs/prod/global/DBSReader", "optional": true, "type": "<type 'str'>" }, diff --git a/doc/createSpecs/ReReco_createSpec.json b/doc/createSpecs/ReReco_createSpec.json index 4b88a9ebb4..f6fa3ee22e 100644 --- a/doc/createSpecs/ReReco_createSpec.json +++ b/doc/createSpecs/ReReco_createSpec.json @@ -75,7 +75,7 @@ "type": "<type 'int'>" }, "DbsUrl": { - "default": "https://cmsweb.cern.ch/dbs/prod/global/DBSReader", + "default": "https://cmsweb-prod.cern.ch/dbs/prod/global/DBSReader", "optional": true, "type": "<type 'str'>" }, diff --git a/doc/createSpecs/StepChain_createSpec.json b/doc/createSpecs/StepChain_createSpec.json index 5795e68c5a..c51375b11a 100644 --- a/doc/createSpecs/StepChain_createSpec.json +++ b/doc/createSpecs/StepChain_createSpec.json @@ -70,7 +70,7 @@ "type": "<type 'int'>" }, "DbsUrl": { - "default": "https://cmsweb.cern.ch/dbs/prod/global/DBSReader", + "default": "https://cmsweb-prod.cern.ch/dbs/prod/global/DBSReader", "optional": true, "type": "<type 'str'>" }, diff --git a/doc/createSpecs/StoreResults_createSpec.json b/doc/createSpecs/StoreResults_createSpec.json index e6d3be910a..67abab64f6 100644 --- a/doc/createSpecs/StoreResults_createSpec.json +++ b/doc/createSpecs/StoreResults_createSpec.json @@ -80,7 +80,7 @@ "type": "<type 'str'>" }, "DbsUrl": { - "default": "https://cmsweb.cern.ch/dbs/prod/global/DBSReader", + "default": "https://cmsweb-prod.cern.ch/dbs/prod/global/DBSReader", "optional": true, "type": "<type 'str'>" }, diff --git a/doc/createSpecs/TaskChain_createSpec.json b/doc/createSpecs/TaskChain_createSpec.json index 6e4d2eb9eb..3dcc0cbd7b 100644 --- a/doc/createSpecs/TaskChain_createSpec.json +++ b/doc/createSpecs/TaskChain_createSpec.json @@ -65,7 +65,7 @@ "type": "<type 'int'>" }, "DbsUrl": { - "default": "https://cmsweb.cern.ch/dbs/prod/global/DBSReader", + "default": "https://cmsweb-prod.cern.ch/dbs/prod/global/DBSReader", "optional": true, "type": "<type 'str'>" }, diff --git a/etc/WMAgentConfig.py b/etc/WMAgentConfig.py index fc7e22fdc6..d4fbe2d46d 100644 --- a/etc/WMAgentConfig.py +++ b/etc/WMAgentConfig.py @@ -47,7 +47,7 @@ # DBS Information. localDBSVersion = "DBS_2_0_8" -globalDBSUrl = "https://cmsweb.cern.ch/dbs/prod/global/DBSReader" +globalDBSUrl = "https://cmsweb-prod.cern.ch/dbs/prod/global/DBSReader" globalDBSVersion = "DBS_2_0_8" # List of SE for T1 _Disk endpoints (TODO clean this up at some point) @@ -145,7 +145,7 @@ config.DBS3Upload.logLevel = globalLogLevel config.DBS3Upload.workerThreads = 1 config.DBS3Upload.pollInterval = 100 -# "https://cmsweb.cern.ch/dbs/prod/global/DBSWriter" - production one +# "https://cmsweb-prod.cern.ch/dbs/prod/global/DBSWriter" - production one config.DBS3Upload.dbsUrl = "OVERWRITE_BY_SECRETS" config.DBS3Upload.primaryDatasetType = "mc" config.DBS3Upload.dumpBlock = False # to dump block meta-data into a json file diff --git a/etc/dbsVerify.py b/etc/dbsVerify.py index 2fd74fdd80..c83b443ec1 100644 --- a/etc/dbsVerify.py +++ b/etc/dbsVerify.py @@ -59,7 +59,7 @@ def connectToDB(): blocks[row[0]].append(row[1]) args = {} -args["url"] = "https://cmsweb.cern.ch/dbs/prod/global/DBSReader" +args["url"] = "https://cmsweb-prod.cern.ch/dbs/prod/global/DBSReader" args["version"] = "DBS_2_0_9" args["mode"] = "GET" dbsApi = DbsApi(args) diff --git a/etc/harvestingInjector.py b/etc/harvestingInjector.py index 539425041e..a17fe84a92 100644 --- a/etc/harvestingInjector.py +++ b/etc/harvestingInjector.py @@ -94,7 +94,7 @@ def injectFilesFromDBS(inputFileset, datasetPath, runsWhiteList=[]): """ print("injecting files from %s into %s, please wait..." % (datasetPath, inputFileset.name)) args = {} - args["url"] = "https://cmsweb.cern.ch/dbs/prod/global/DBSReader" + args["url"] = "https://cmsweb-prod.cern.ch/dbs/prod/global/DBSReader" args["version"] = "DBS_2_1_1" args["mode"] = "GET" dbsApi = DbsApi(args) diff --git a/etc/injectReRecoWorkflow.py b/etc/injectReRecoWorkflow.py index 43d62bfd92..44ed22f20e 100644 --- a/etc/injectReRecoWorkflow.py +++ b/etc/injectReRecoWorkflow.py @@ -65,7 +65,7 @@ def injectFilesFromDBS(inputFileset, datasetPath): """ print("injecting files from %s into %s, please wait..." % (datasetPath, inputFileset.name)) args={} - args["url"] = "https://cmsweb.cern.ch/dbs/prod/global/DBSReader" + args["url"] = "https://cmsweb-prod.cern.ch/dbs/prod/global/DBSReader" args["version"] = "DBS_2_0_9" args["mode"] = "GET" dbsApi = DbsApi(args) diff --git a/etc/injectStoreResults.py b/etc/injectStoreResults.py index 2c0f4e73fa..7e2d3e4dd0 100644 --- a/etc/injectStoreResults.py +++ b/etc/injectStoreResults.py @@ -77,7 +77,7 @@ def injectFilesFromDBS(inputFileset, datasetPath): """ print("injecting files from %s into %s, please wait..." % (datasetPath, inputFileset.name)) args={} - args["url"] = "https://cmsweb.cern.ch/dbs/prod/global/DBSReader" + args["url"] = "https://cmsweb-prod.cern.ch/dbs/prod/global/DBSReader" args["mode"] = "GET" dbsApi = DbsApi(args) dbsResults = dbsApi.listFileArray(path = datasetPath, retriveList = ["retrive_lumi", "retrive_run"]) diff --git a/etc/submit.sh b/etc/submit.sh index 1c52b9d2bd..500699dd6b 100644 --- a/etc/submit.sh +++ b/etc/submit.sh @@ -96,19 +96,23 @@ echo -e "======== WMAgent validate arguments finished at $(TZ=GMT date) ======== echo "======== WMAgent CMS environment load starting at $(TZ=GMT date) ========" if [ -f "$VO_CMS_SW_DIR"/cmsset_default.sh ] then # LCG style -- + echo "WN with a LCG style environment, thus using VO_CMS_SW_DIR=$VO_CMS_SW_DIR" . $VO_CMS_SW_DIR/cmsset_default.sh elif [ -f "$OSG_APP"/cmssoft/cms/cmsset_default.sh ] then # OSG style -- + echo "WN with an OSG style environment, thus using OSG_APP=$OSG_APP" . $OSG_APP/cmssoft/cms/cmsset_default.sh CMSSW_3_3_2 elif [ -f "$CVMFS"/cms.cern.ch/cmsset_default.sh ] then + echo "WN with CVMFS environment, thus using CVMFS=$CVMFS" . $CVMFS/cms.cern.ch/cmsset_default.sh elif [ -f /cvmfs/cms.cern.ch/cmsset_default.sh ] then # ok, lets call it CVMFS then export CVMFS=/cvmfs/cms.cern.ch + echo "WN missing VO_CMS_SW_DIR/OSG_APP/CVMFS environment variable, forcing it to CVMFS=$CVMFS" . $CVMFS/cmsset_default.sh else - echo "Error during job bootstrap: VO_CMS_SW_DIR, OSG_APP, CVMFS or /cvmfs were not found." >&2 + echo "Error during job bootstrap: VO_CMS_SW_DIR, OSG_APP, CVMFS or /cvmfs were not found." >&2 echo " Because of this, we can't load CMSSW. Not good." >&2 exit 11003 fi @@ -137,8 +141,8 @@ elif [ -d "$CVMFS"/COMP/"$WMA_SCRAM_ARCH"/external/python ] then prefix="$CVMFS"/COMP/"$WMA_SCRAM_ARCH"/external/python else - echo "Error during job bootstrap: job environment does not contain the init.sh script." >&2 - echo " Because of this, we can't load CMSSW. Not good." >&2 + echo "Failed to find a COMP python installation in the worker node setup." >&2 + echo " Without a known python, there is nothing else we can do with this job. Quiting!" >&2 exit 11004 fi diff --git a/src/python/Utils/Utilities.py b/src/python/Utils/Utilities.py index 14813d4f19..71d74afbf3 100644 --- a/src/python/Utils/Utilities.py +++ b/src/python/Utils/Utilities.py @@ -224,6 +224,34 @@ def decodeBytesToUnicode(value, errors="strict"): return value +def encodeUnicodeToBytes(value, errors="strict"): + """ + Accepts an input "value" of generic type. + + If "value" is a string of type sequence of unicode (i.e. in py2 `unicode` or + `future.types.newstr.newstr`, in py3 `str`), then it is converted to + a sequence of bytes. + + This function is useful for encoding output data when using the + "unicode sandwich" approach, which involves converting unicode (i.e. strings + of type sequence of unicode codepoints) to bytes (i.e. strings of type + sequence of bytes, in py2 `str` or `future.types.newbytes.newbytes`, + in py3 `bytes`) as late as possible when passing a string to a third-party + function that only accepts bytes as input (pycurl's curl.setop is an + example). + py2: + - "errors" can be: "strict", "ignore", "replace", "xmlcharrefreplace" + - ref: https://docs.python.org/2/howto/unicode.html#the-unicode-type + py3: + - "errors" can be: "strict", "ignore", "replace", "backslashreplace", + "xmlcharrefreplace", "namereplace" + - ref: https://docs.python.org/3/howto/unicode.html#the-string-type + """ + if isinstance(value, str): + return value.encode("utf-8", errors) + return value + + # TODO: remove this function once we have completely migrated to Rucio def usingRucio(): """ diff --git a/src/python/WMComponent/DBS3Buffer/DBSUploadPoller.py b/src/python/WMComponent/DBS3Buffer/DBSUploadPoller.py index c13392c908..f266b7e29d 100644 --- a/src/python/WMComponent/DBS3Buffer/DBSUploadPoller.py +++ b/src/python/WMComponent/DBS3Buffer/DBSUploadPoller.py @@ -48,38 +48,6 @@ from WMCore.WorkerThreads.BaseWorkerThread import BaseWorkerThread -def createConfigForJSON(config): - """ - Turn a config object into a dictionary of dictionaries - - """ - - final = {} - for sectionName in config.listSections_(): - section = getattr(config, sectionName) - if hasattr(section, 'dictionary_'): - # Create a dictionary key for it - final[sectionName] = createDictionaryFromConfig(section) - - return final - - -def createDictionaryFromConfig(configSection): - """ - Recursively create dictionaries from config - - """ - - final = configSection.dictionary_() - - for key in final.keys(): - if hasattr(final[key], 'dictionary_'): - # Then we can turn it into a dictionary - final[key] = createDictionaryFromConfig(final[key]) - - return final - - def uploadWorker(workInput, results, dbsUrl): """ _uploadWorker_ @@ -141,11 +109,35 @@ def uploadWorker(workInput, results, dbsUrl): return +def isPassiveError(exceptionObj): + """ + This function will parse the exception object and report whether + the error message corresponds to a soft or hard error (hard errors + are supposed to let the component crash). + :param exceptionObj: any exception object + :return: True if it's a soft error, False otherwise + """ + passException = True + passiveErrorMsg = ['Service Unavailable', 'Service Temporarily Unavailable', + 'Proxy Error', 'Error reading from remote server', + 'Connection refused', 'timed out', 'Could not resolve', + 'OpenSSL SSL_connect: SSL_ERROR_SYSCALL'] + + excReason = getattr(exceptionObj, 'reason', '') + for passiveMsg in passiveErrorMsg: + if passiveMsg in excReason: + break + elif passiveMsg in str(exceptionObj): + break + else: + passException = False + return passException + + class DBSUploadException(WMException): """ Holds the exception info for all the things that will go wrong - """ @@ -344,19 +336,16 @@ def updateDatasetParentageCache(self): try: self.datasetParentageCache = self.wmstatsServerSvc.getChildParentDatasetMap() except Exception as ex: - reason = getattr(ex, 'reason', '') - msg = 'Failed to fetch parentage map from WMStats, skipping this cycle. ' - msg += 'Exception: %s. Reason: %s. ' % (type(ex).__name__, reason) - if 'Service Unavailable' in reason or 'Proxy Error' in reason or \ - 'Error reading from remote server' in reason: - pass - elif 'Connection refused' in str(ex) or 'timed out' in str(ex) or 'Could not resolve' in str(ex): - msg += 'Error: %s' % str(ex) + excReason = getattr(ex, 'reason', '') + errorMsg = 'Failed to fetch parentage map from WMStats, skipping this cycle. ' + errorMsg += 'Exception: {}. Reason: {}. Error: {}. '.format(type(ex).__name__, + excReason, str(ex)) + if isPassiveError(ex): + logging.warning(errorMsg) else: - msg = "Unknown failure while fetching parentage map from WMStats. Error: %s" % str(ex) - raise DBSUploadException(msg) - logging.warning(msg) - myThread.logdbClient.post("DBS3Upload_parentMap", msg, "warning") + errorMsg += 'Hit a terminal exception in DBSUploadPoller.' + raise DBSUploadException(errorMsg) + myThread.logdbClient.post("DBS3Upload_parentMap", errorMsg, "warning") success = False else: myThread.logdbClient.delete("DBS3Upload_parentMap", "warning", this_thread=True) @@ -376,7 +365,7 @@ def loadBlocks(self): # Load them if we don't have them blocksToLoad = [] for block in openBlocks: - if block['blockname'] not in self.blockCache.keys(): + if block['blockname'] not in self.blockCache: blocksToLoad.append(block['blockname']) # Now load the blocks diff --git a/src/python/WMComponent/RucioInjector/RucioInjectorPoller.py b/src/python/WMComponent/RucioInjector/RucioInjectorPoller.py index 0aa7a8ce40..d8f625fb03 100644 --- a/src/python/WMComponent/RucioInjector/RucioInjectorPoller.py +++ b/src/python/WMComponent/RucioInjector/RucioInjectorPoller.py @@ -287,6 +287,10 @@ def insertBlockRules(self): if not self._isBlockTierAllowed(item['blockname']): logging.debug("Component configured to skip block rule for: %s", item['blockname']) continue + # first, check if the block has already been created in Rucio + if not self.rucio.didExist(item['blockname']): + logging.warning("Block: %s not yet in Rucio. Retrying later..", item['blockname']) + continue kwargs = dict(activity="Production Output", account=self.rucioAcct, grouping="DATASET", comment="WMAgent automatic container rule", ignore_availability=True, meta=self.metaData) @@ -519,6 +523,10 @@ def insertContainerRules(self): container, rseName) subscriptionsMade.append(subInfo['id']) continue + # then check if the container has already been created in Rucio + if not self.rucio.didExist(container): + logging.warning("Container: %s not yet in Rucio. Retrying later..", container) + continue ruleKwargs = dict(ask_approval=False, activity=self._activityMap(rseName), diff --git a/src/python/WMCore/MicroService/DataStructs/Workflow.py b/src/python/WMCore/MicroService/DataStructs/Workflow.py index c66a4de3ea..ccbd085a7c 100644 --- a/src/python/WMCore/MicroService/DataStructs/Workflow.py +++ b/src/python/WMCore/MicroService/DataStructs/Workflow.py @@ -59,7 +59,8 @@ def getDbsUrl(self): """ Get the DbsUrl defined in this request """ - return self.data['DbsUrl'] + # TODO: this replace can be removed in one year from now, thus March 2022 + return self.data['DbsUrl'].replace("cmsweb.cern.ch", "cmsweb-prod.cern.ch") def getReqType(self): """ diff --git a/src/python/WMCore/ReqMgr/DataStructs/Request.py b/src/python/WMCore/ReqMgr/DataStructs/Request.py index 6827322f5a..db66000f15 100644 --- a/src/python/WMCore/ReqMgr/DataStructs/Request.py +++ b/src/python/WMCore/ReqMgr/DataStructs/Request.py @@ -3,8 +3,6 @@ define just 1 class. Derived from Python dict and implementing necessary conversion and validation extra methods possibly needed. -TODO/NOTE: - 'inputMode' should be removed by now (2013-07) since arguments validation #4705, arguments which are later validated during spec instantiation and which are not @@ -58,6 +56,7 @@ def initialize_request_args(request, config): request["CouchDBName"] = config.couch_config_cache_db generateRequestName(request) + replaceDbsProdUrl(request) def _replace_cloned_args(clone_args, user_args): @@ -75,6 +74,18 @@ def _replace_cloned_args(clone_args, user_args): return +def replaceDbsProdUrl(requestArgs): + """ + Function to update the DBS URL from the standard cmsweb.cern.ch + to cmsweb-prod.cern.ch, running in the k8s infra + :param requestArgs: dictionary with the request arguments + :return: same dictionary object (with updated dbs url) + """ + # TODO: this url conversion below can be removed in one year from now, thus March 2022 + if requestArgs.get("DbsUrl"): + requestArgs["DbsUrl"] = requestArgs["DbsUrl"].replace("cmsweb.cern.ch", "cmsweb-prod.cern.ch") + + def initialize_clone(requestArgs, originalArgs, argsDefinition, chainDefinition=None): """ Initialize arguments for a clone request by inheriting and overwriting argument @@ -111,6 +122,7 @@ def initialize_clone(requestArgs, originalArgs, argsDefinition, chainDefinition= # apply user override arguments at the end, such that it's validated at spec level incrementProcVer(cloneArgs, requestArgs) _replace_cloned_args(cloneArgs, requestArgs) + replaceDbsProdUrl(cloneArgs) return cloneArgs diff --git a/src/python/WMCore/ReqMgr/Tools/cms.py b/src/python/WMCore/ReqMgr/Tools/cms.py index fab8caf9c9..a8f4e7ddbe 100644 --- a/src/python/WMCore/ReqMgr/Tools/cms.py +++ b/src/python/WMCore/ReqMgr/Tools/cms.py @@ -202,7 +202,7 @@ def dqm_urls(): def dbs_urls(): "Return list of DBS urls" urls = [] - base = "https://cmsweb.cern.ch/dbs/prod/global/DBSReader/" + base = "https://cmsweb-prod.cern.ch/dbs/prod/global/DBSReader/" for inst in ["prod", "phys01", "phys02", "phys03"]: urls.append(base.replace("prod", inst)) return urls diff --git a/src/python/WMCore/ReqMgr/Utils/Validation.py b/src/python/WMCore/ReqMgr/Utils/Validation.py index 641a216529..479295cdba 100644 --- a/src/python/WMCore/ReqMgr/Utils/Validation.py +++ b/src/python/WMCore/ReqMgr/Utils/Validation.py @@ -259,6 +259,9 @@ def validateOutputDatasets(outDsets, dbsUrl): msg = "Bad output dataset name, check the processed dataset name.\n %s" % str(ex) raise InvalidSpecParameterValue(msg) + # TODO: this url conversion below can be removed in one year from now, thus March 2022 + dbsUrl = dbsUrl.replace("cmsweb.cern.ch", "cmsweb-prod.cern.ch") + # Verify whether the output datatiers are available in DBS _validateDatatier(datatier, dbsUrl) diff --git a/src/python/WMCore/Services/PhEDEx/PhEDEx.py b/src/python/WMCore/Services/PhEDEx/PhEDEx.py index 7cdabc1769..ee5f263ffc 100644 --- a/src/python/WMCore/Services/PhEDEx/PhEDEx.py +++ b/src/python/WMCore/Services/PhEDEx/PhEDEx.py @@ -15,7 +15,7 @@ class PhEDEx(Service): """ def __init__(self, httpDict=None, responseType="json", logger=None, - dbsUrl='https://cmsweb.cern.ch/dbs/prod/global/DBSReader'): + dbsUrl='https://cmsweb-prod.cern.ch/dbs/prod/global/DBSReader'): """ responseType will be either xml or json """ diff --git a/src/python/WMCore/Services/Rucio/Rucio.py b/src/python/WMCore/Services/Rucio/Rucio.py index e0b71bfab8..fd999e8b63 100644 --- a/src/python/WMCore/Services/Rucio/Rucio.py +++ b/src/python/WMCore/Services/Rucio/Rucio.py @@ -842,6 +842,19 @@ def getDID(self, didName, dynamic=True, scope='cms'): self.logger.error("Data identifier not found in Rucio: %s. Error: %s", didName, str(exc)) return response + def didExist(self, didName, scope='cms'): + """ + Provided a given DID, check whether it's already in the Rucio server. + Any kind of exception will return False (thus, data not yet in Rucio). + :param didName: a string with the DID name (container, block, or file) + :return: True if DID has been found, False otherwise + """ + try: + response = self.cli.get_did(scope=scope, name=didName) + except Exception: + response = dict() + return response.get("name") == didName + # FIXME we can likely delete this method (replaced by another implementation) def getDataLockedAndAvailable_old(self, **kwargs): """ diff --git a/src/python/WMCore/Services/pycurl_manager.py b/src/python/WMCore/Services/pycurl_manager.py index 48b7bd969f..6da402e21e 100644 --- a/src/python/WMCore/Services/pycurl_manager.py +++ b/src/python/WMCore/Services/pycurl_manager.py @@ -26,8 +26,8 @@ tfile = tempfile.NamedTemporaryFile() ckey = os.path.join(os.environ['HOME'], '.globus/userkey.pem') cert = os.path.join(os.environ['HOME'], '.globus/usercert.pem') -url1 = "https://cmsweb.cern.ch/dbs/prod/global/DBSReader/help" -url2 = "https://cmsweb.cern.ch/dbs/prod/global/DBSReader/datatiers" +url1 = "https://cmsweb-prod.cern.ch/dbs/prod/global/DBSReader/help" +url2 = "https://cmsweb-prod.cern.ch/dbs/prod/global/DBSReader/datatiers" url3 = "https://cms-gwmsmon.cern.ch/prodview/json/site_summary" cern_sso_cookie(url3, tfile.name, cert, ckey) cookie = {url3: tfile.name} @@ -44,6 +44,7 @@ from past.builtins import basestring from future.utils import viewitems + # system modules import json import logging @@ -56,6 +57,8 @@ import http.client from urllib.parse import urlencode +from Utils.Utilities import encodeUnicodeToBytes + class ResponseHeader(object): """ResponseHeader parses HTTP response header""" @@ -214,10 +217,10 @@ def set_opts(self, curl, url, params, headers, # we must pass url as a string data-type, otherwise pycurl will fail with error # TypeError: invalid arguments to setopt # see https://curl.haxx.se/mail/curlpython-2007-07/0001.html - curl.setopt(pycurl.URL, str(url)) + curl.setopt(pycurl.URL, encodeUnicodeToBytes(url)) if headers: curl.setopt(pycurl.HTTPHEADER, \ - ["%s: %s" % (k, v) for k, v in viewitems(headers)]) + [encodeUnicodeToBytes("%s: %s" % (k, v)) for k, v in viewitems(headers)]) bbuf = BytesIO() hbuf = BytesIO() curl.setopt(pycurl.WRITEFUNCTION, bbuf.write) diff --git a/src/python/WMCore/WMSpec/StdSpecs/StdBase.py b/src/python/WMCore/WMSpec/StdSpecs/StdBase.py index 26319c9545..0f10661ffb 100644 --- a/src/python/WMCore/WMSpec/StdSpecs/StdBase.py +++ b/src/python/WMCore/WMSpec/StdSpecs/StdBase.py @@ -74,6 +74,10 @@ def __call__(self, workloadName, arguments): except Exception as ex: raise WMSpecFactoryException("parameter %s: %s" % (arg, str(ex))) + # TODO: this replace can be removed in one year from now, thus March 2022 + if hasattr(self, "dbsUrl"): + self.dbsUrl = self.dbsUrl.replace("cmsweb.cern.ch", "cmsweb-prod.cern.ch") + return # static copy of the skim mapping @@ -297,7 +301,8 @@ def createWorkload(self): workload.setPriority(self.priority) workload.setCampaign(self.campaign) workload.setRequestType(self.requestType) - workload.setDbsUrl(self.dbsUrl) + # TODO: this replace can be removed in one year from now, thus March 2022 + workload.setDbsUrl(self.dbsUrl.replace("cmsweb.cern.ch", "cmsweb-prod.cern.ch")) workload.setPrepID(self.prepID) return workload @@ -1038,7 +1043,7 @@ def getWorkloadCreateArgs(): "validate": lambda x: x >= 0}, "GlobalTagConnect": {"null": True}, "LumiList": {"default": {}, "type": makeLumiList}, - "DbsUrl": {"default": "https://cmsweb.cern.ch/dbs/prod/global/DBSReader", + "DbsUrl": {"default": "https://cmsweb-prod.cern.ch/dbs/prod/global/DBSReader", "null": True, "validate": checkDBSURL}, "DashboardHost": {"default": "cms-jobmon.cern.ch"}, "DashboardPort": {"default": 8884, "type": int, diff --git a/src/python/WMCore/WMSpec/WMWorkload.py b/src/python/WMCore/WMSpec/WMWorkload.py index 9dc74b6b49..b1a6ef6566 100644 --- a/src/python/WMCore/WMSpec/WMWorkload.py +++ b/src/python/WMCore/WMSpec/WMWorkload.py @@ -1075,7 +1075,8 @@ def setDbsUrl(self, dbsUrl): Set the workload level DbsUrl. """ - self.data.dbsUrl = dbsUrl + # TODO: this replace can be removed in one year from now, thus March 2022 + self.data.dbsUrl = dbsUrl.replace("cmsweb.cern.ch", "cmsweb-prod.cern.ch") def getDbsUrl(self): """ @@ -1089,7 +1090,7 @@ def getDbsUrl(self): if hasattr(self.data, "request"): if hasattr(self.data.request, "schema"): if not getattr(self.data.request.schema, "DbsUrl", None): - return "https://cmsweb.cern.ch/dbs/prod/global/DBSReader" + return "https://cmsweb-prod.cern.ch/dbs/prod/global/DBSReader" return getattr(self.data.request.schema, "DbsUrl") diff --git a/src/python/WMCore/WMSpec/WMWorkloadTools.py b/src/python/WMCore/WMSpec/WMWorkloadTools.py index 03cf6609fe..94fe934090 100644 --- a/src/python/WMCore/WMSpec/WMWorkloadTools.py +++ b/src/python/WMCore/WMSpec/WMWorkloadTools.py @@ -133,7 +133,10 @@ def validateInputDatasSetAndParentFlag(arguments): mcpileup = _getChainKey(arguments, "MCPileup") datapileup = _getChainKey(arguments, "DataPileup") includeParents = _getChainKey(arguments, "IncludeParents") + # TODO: this replace can be removed in one year from now, thus March 2022 dbsURL = arguments.get("DbsUrl") + if dbsURL: + dbsURL = dbsURL.replace("cmsweb.cern.ch", "cmsweb-prod.cern.ch") if includeParents and not inputdataset: msg = "IncludeParents flag is True but InputDataset value has not been provided" diff --git a/src/python/WMQuality/Emulators/DBSClient/MockDbsApi.py b/src/python/WMQuality/Emulators/DBSClient/MockDbsApi.py index 0b283bd02c..191945ee3c 100644 --- a/src/python/WMQuality/Emulators/DBSClient/MockDbsApi.py +++ b/src/python/WMQuality/Emulators/DBSClient/MockDbsApi.py @@ -32,8 +32,8 @@ except IOError: mockData03 = {} -mockData['https://cmsweb.cern.ch/dbs/prod/global/DBSReader'] = mockDataGlobal -mockData['https://cmsweb.cern.ch/dbs/prod/phys03/DBSReader'] = mockData03 +mockData['https://cmsweb-prod.cern.ch/dbs/prod/global/DBSReader'] = mockDataGlobal +mockData['https://cmsweb-prod.cern.ch/dbs/prod/phys03/DBSReader'] = mockData03 class MockDbsApi(object): diff --git a/src/python/WMQuality/Emulators/DBSClient/MockedDBSGlobalCalls.py b/src/python/WMQuality/Emulators/DBSClient/MockedDBSGlobalCalls.py index d66a3adc05..3d85f3722e 100644 --- a/src/python/WMQuality/Emulators/DBSClient/MockedDBSGlobalCalls.py +++ b/src/python/WMQuality/Emulators/DBSClient/MockedDBSGlobalCalls.py @@ -2,7 +2,7 @@ from __future__ import (division, print_function) -endpoint = 'https://cmsweb.cern.ch/dbs/prod/global/DBSReader' +endpoint = 'https://cmsweb-prod.cern.ch/dbs/prod/global/DBSReader' # Datasets to get all blocks, array, and lumis from datasets = ['/HighPileUp/Run2011A-v1/RAW', '/MinimumBias/ComissioningHI-v1/RAW', '/Cosmics/ComissioningHI-v1/RAW', diff --git a/src/python/WMQuality/Emulators/DBSClient/MockedDBSphys03Calls.py b/src/python/WMQuality/Emulators/DBSClient/MockedDBSphys03Calls.py index 9e5c2bb1e2..e108cf4086 100644 --- a/src/python/WMQuality/Emulators/DBSClient/MockedDBSphys03Calls.py +++ b/src/python/WMQuality/Emulators/DBSClient/MockedDBSphys03Calls.py @@ -2,7 +2,7 @@ from __future__ import (division, print_function) -endpoint = 'https://cmsweb.cern.ch/dbs/prod/phys03/DBSReader' +endpoint = 'https://cmsweb-prod.cern.ch/dbs/prod/phys03/DBSReader' datasets = [] diff --git a/src/python/WMQuality/Emulators/PhEDExClient/MockPhEDExApi.py b/src/python/WMQuality/Emulators/PhEDExClient/MockPhEDExApi.py index 6d36743dac..b95ee3556a 100644 --- a/src/python/WMQuality/Emulators/PhEDExClient/MockPhEDExApi.py +++ b/src/python/WMQuality/Emulators/PhEDExClient/MockPhEDExApi.py @@ -14,7 +14,7 @@ from WMCore.Services.DBS.DBS3Reader import (DBS3Reader, DBSReaderError) from WMQuality.Emulators.DataBlockGenerator.DataBlockGenerator import DataBlockGenerator -PROD_DBS = 'https://cmsweb.cern.ch/dbs/prod/global/DBSReader' +PROD_DBS = 'https://cmsweb-prod.cern.ch/dbs/prod/global/DBSReader' NOT_EXIST_DATASET = 'thisdoesntexist' PILEUP_DATASET = '/HighPileUp/Run2011A-v1/RAW' @@ -35,7 +35,7 @@ class MockPhEDExApi(object): """ def __init__(self, dict=None, responseType="json", logger=None, - dbsUrl='https://cmsweb.cern.ch/dbs/prod/global/DBSReader'): + dbsUrl='https://cmsweb-prod.cern.ch/dbs/prod/global/DBSReader'): print("Using MockPhEDExApi") self.dbsUrl = dbsUrl dict = dict or {} diff --git a/src/python/WMQuality/Emulators/RucioClient/MockRucioApi.py b/src/python/WMQuality/Emulators/RucioClient/MockRucioApi.py index ee4b866931..9f8a8e3b62 100644 --- a/src/python/WMQuality/Emulators/RucioClient/MockRucioApi.py +++ b/src/python/WMQuality/Emulators/RucioClient/MockRucioApi.py @@ -11,7 +11,7 @@ from WMCore.Services.Rucio.Rucio import WMRucioException, WMRucioDIDNotFoundException from WMQuality.Emulators.DataBlockGenerator.DataBlockGenerator import DataBlockGenerator -PROD_DBS = 'https://cmsweb.cern.ch/dbs/prod/global/DBSReader' +PROD_DBS = 'https://cmsweb-prod.cern.ch/dbs/prod/global/DBSReader' NOT_EXIST_DATASET = 'thisdoesntexist' PILEUP_DATASET = '/HighPileUp/Run2011A-v1/RAW' diff --git a/src/python/WMQuality/Emulators/WMSpecGenerator/Samples/BasicProcessingWorkload.py b/src/python/WMQuality/Emulators/WMSpecGenerator/Samples/BasicProcessingWorkload.py index 0ea8f97b30..33805abef5 100644 --- a/src/python/WMQuality/Emulators/WMSpecGenerator/Samples/BasicProcessingWorkload.py +++ b/src/python/WMQuality/Emulators/WMSpecGenerator/Samples/BasicProcessingWorkload.py @@ -32,7 +32,7 @@ def createWorkload(name="BasicProcessing"): primary = "Cosmics", processed = "CRAFT09-PromptReco-v1", tier = "RECO", - dbsurl = "https://cmsweb.cern.ch/dbs/prod/global/DBSReader") + dbsurl = "https://cmsweb-prod.cern.ch/dbs/prod/global/DBSReader") # // # // rereco cmssw step diff --git a/src/python/WMQuality/Emulators/WMSpecGenerator/Samples/MultiTaskProcessingWorkload.py b/src/python/WMQuality/Emulators/WMSpecGenerator/Samples/MultiTaskProcessingWorkload.py index 591422da66..02f2aa1623 100644 --- a/src/python/WMQuality/Emulators/WMSpecGenerator/Samples/MultiTaskProcessingWorkload.py +++ b/src/python/WMQuality/Emulators/WMSpecGenerator/Samples/MultiTaskProcessingWorkload.py @@ -8,7 +8,7 @@ from WMCore.WMSpec.WMWorkload import newWorkload -DBSURL = "https://cmsweb.cern.ch/dbs/prod/global/DBSReader" +DBSURL = "https://cmsweb-prod.cern.ch/dbs/prod/global/DBSReader" # // # // Set up the basic workload task and step structure