Skip to content

Commit

Permalink
partial reverts 04d1c2c, add ranRange=0 as usePartialDataset back
Browse files Browse the repository at this point in the history
  • Loading branch information
Thanayut Seethongchuen committed Jun 27, 2022
1 parent 3647b4e commit 2f472c6
Showing 1 changed file with 16 additions and 2 deletions.
18 changes: 16 additions & 2 deletions src/python/TaskWorker/Actions/DBSDataDiscovery.py
Original file line number Diff line number Diff line change
Expand Up @@ -419,6 +419,21 @@ def executeInternal(self, *args, **kwargs):
secondaryBlocksWithLocation = secondaryLocationsMap.copy().keys()

# filter out TAPE locations
# temporary hack until we have a new config. parameter: allow user to accpet a partial dataset
# by inserting '0' as (first element of) runRange
usePartialDataset = False
runRange = kwargs['task']['tm_split_args']['runs']
if runRange and runRange[0] == '0':
usePartialDataset = True
# remove initial '0' from run list
# note that it caused an extra entry to be created in lumilis as well
kwargs['task']['tm_split_args']['runs'] = kwargs['task']['tm_split_args']['runs'][1:]
kwargs['task']['tm_split_args']['lumis'] = kwargs['task']['tm_split_args']['lumis'][1:]
runRange = kwargs['task']['tm_split_args']['runs']
# or use Data.partialDataset configuration option in client
elif kwargs['task']['tm_user_config']['partialdataset']:
usePartialDataset = True

self.keepOnlyDiskRSEs(locationsMap)
if not locationsMap:
msg = "Task could not be submitted because there is no DISK replica for dataset %s" % inputDataset
Expand All @@ -430,7 +445,7 @@ def executeInternal(self, *args, **kwargs):
self.requestTapeRecall(blockList=blocksWithLocation, system='Rucio', msgHead=msg)
if set(locationsMap.keys()) != set(blocksWithLocation):
dataTier = inputDataset.split('/')[3]
if kwargs['task']['tm_user_config']['partialdataset']:
if usePartialDataset:
msg = "Some blocks are on TAPE only and can not be reaed."
msg += "\nSince you specified to accept a partial dataset, only blocks on disk will be processed"
self.logger.warning(msg)
Expand All @@ -450,7 +465,6 @@ def executeInternal(self, *args, **kwargs):
# will not need lumi info if user has asked for split by file with no run/lumi mask
splitAlgo = kwargs['task']['tm_split_algo']
lumiMask = kwargs['task']['tm_split_args']['lumis']
runRange = kwargs['task']['tm_split_args']['runs']

needLumiInfo = splitAlgo != 'FileBased' or lumiMask != [] or runRange != []
# secondary dataset access relies on run/lumi info
Expand Down

0 comments on commit 2f472c6

Please sign in to comment.