diff --git a/src/python/TaskWorker/Actions/DBSDataDiscovery.py b/src/python/TaskWorker/Actions/DBSDataDiscovery.py index b5b5a2e52d..f82bcae095 100644 --- a/src/python/TaskWorker/Actions/DBSDataDiscovery.py +++ b/src/python/TaskWorker/Actions/DBSDataDiscovery.py @@ -419,6 +419,21 @@ def executeInternal(self, *args, **kwargs): secondaryBlocksWithLocation = secondaryLocationsMap.copy().keys() # filter out TAPE locations + # temporary hack until we have a new config. parameter: allow user to accpet a partial dataset + # by inserting '0' as (first element of) runRange + usePartialDataset = False + runRange = kwargs['task']['tm_split_args']['runs'] + if runRange and runRange[0] == '0': + usePartialDataset = True + # remove initial '0' from run list + # note that it caused an extra entry to be created in lumilis as well + kwargs['task']['tm_split_args']['runs'] = kwargs['task']['tm_split_args']['runs'][1:] + kwargs['task']['tm_split_args']['lumis'] = kwargs['task']['tm_split_args']['lumis'][1:] + runRange = kwargs['task']['tm_split_args']['runs'] + # or use Data.partialDataset configuration option in client + elif kwargs['task']['tm_user_config']['partialdataset']: + usePartialDataset = True + self.keepOnlyDiskRSEs(locationsMap) if not locationsMap: msg = "Task could not be submitted because there is no DISK replica for dataset %s" % inputDataset @@ -430,7 +445,7 @@ def executeInternal(self, *args, **kwargs): self.requestTapeRecall(blockList=blocksWithLocation, system='Rucio', msgHead=msg) if set(locationsMap.keys()) != set(blocksWithLocation): dataTier = inputDataset.split('/')[3] - if kwargs['task']['tm_user_config']['partialdataset']: + if usePartialDataset: msg = "Some blocks are on TAPE only and can not be reaed." msg += "\nSince you specified to accept a partial dataset, only blocks on disk will be processed" self.logger.warning(msg) @@ -450,7 +465,6 @@ def executeInternal(self, *args, **kwargs): # will not need lumi info if user has asked for split by file with no run/lumi mask splitAlgo = kwargs['task']['tm_split_algo'] lumiMask = kwargs['task']['tm_split_args']['lumis'] - runRange = kwargs['task']['tm_split_args']['runs'] needLumiInfo = splitAlgo != 'FileBased' or lumiMask != [] or runRange != [] # secondary dataset access relies on run/lumi info