diff --git a/dirac.cfg b/dirac.cfg index f34ff07a8e1..7424fd6a4d5 100644 --- a/dirac.cfg +++ b/dirac.cfg @@ -799,6 +799,7 @@ Resources SpaceReservation = LHCb-EOS # Space reservation name if any. Concept like SpaceToken ArchiveTimeout = 84600 # Timeout for the FTS archiving BringOnlineTimeout = 84600 # Timeout for the bring online operation used by FTS + WLCGTokenBasePath = /eos/lhcb # EXPERIMENTAL Path from which the token should be relative to # Protocol section, see http://dirac.readthedocs.io/en/latest/AdministratorGuide/Resources/Storages/index.html#available-protocol-plugins GFAL2_SRM2 { diff --git a/docs/source/AdministratorGuide/Resources/storage.rst b/docs/source/AdministratorGuide/Resources/storage.rst index 263a35a739b..eadf9e90738 100644 --- a/docs/source/AdministratorGuide/Resources/storage.rst +++ b/docs/source/AdministratorGuide/Resources/storage.rst @@ -59,6 +59,7 @@ Configuration options are: * ``SpaceReservation``: just a name of a zone of the physical storage which can have some space reserved. Extends the SRM ``SpaceToken`` concept. * ``ArchiveTimeout``: for tape SE only. If set to a value in seconds, enables the `FTS Archive Monitoring feature `_ * ``BringOnlineTimeout``: for tape SE only. If set to a value in seconds, specify the BringOnline parameter for FTS transfers. Otherwise, the default is whatever is in the ``FTS3Job`` class. +* ``WLCGTokenBasePath``: EXPERIMENTAL Path from which the token should be relative to VO specific paths ----------------- diff --git a/src/DIRAC/Resources/Storage/StorageBase.py b/src/DIRAC/Resources/Storage/StorageBase.py index 0aef091039b..270b3c79903 100755 --- a/src/DIRAC/Resources/Storage/StorageBase.py +++ b/src/DIRAC/Resources/Storage/StorageBase.py @@ -39,6 +39,8 @@ import shutil import tempfile +from pathlib import Path + from DIRAC import S_OK, S_ERROR from DIRAC.Core.Utilities.Pfn import pfnparse, pfnunparse from DIRAC.Core.Utilities.ReturnValues import returnSingleResult @@ -462,3 +464,16 @@ def getOccupancy(self, **kwargs): finally: # Clean the temporary dir shutil.rmtree(tmpDirName) + + def getWLCGTokenPath(self, lfn: str, wlcgTokenBasePath: str) -> str: + """ + Returns the path expected to be in a WLCG token + It basically consists of ``basepath - tokenBasePath + LFN`` + The tokenBasePath is a configuration on the storage side. + + """ + + allDict = dict.fromkeys(["Protocol", "Host", "Port", "Path", "FileName", "Options"], "") + allDict.update({"Path": self.protocolParameters["Path"], "FileName": lfn.lstrip("/")}) + fullPath = pfnunparse(allDict)["Value"] + return Path(fullPath).relative_to(Path(wlcgTokenBasePath)) diff --git a/src/DIRAC/Resources/Storage/StorageElement.py b/src/DIRAC/Resources/Storage/StorageElement.py index dbf00b070cf..8e22fa3191c 100755 --- a/src/DIRAC/Resources/Storage/StorageElement.py +++ b/src/DIRAC/Resources/Storage/StorageElement.py @@ -1,5 +1,6 @@ """ This is the StorageElement module. It implements The StorageElementItem as well as the caching system """ + # # custom duty @@ -20,7 +21,7 @@ from DIRAC.Core.Utilities import DErrno from DIRAC.Core.Utilities.File import convertSizeUnits from DIRAC.Core.Utilities.List import getIndexInList -from DIRAC.Core.Utilities.ReturnValues import S_OK, S_ERROR, returnSingleResult +from DIRAC.Core.Utilities.ReturnValues import S_OK, S_ERROR, returnSingleResult, convertToReturnValue from DIRAC.Core.Utilities.TimeUtilities import toEpochMilliSeconds from DIRAC.Resources.Storage.StorageFactory import StorageFactory from DIRAC.Core.Utilities.Pfn import pfnparse @@ -257,11 +258,13 @@ def __init__(self, name, protocolSections=None, vo=None, hideExceptions=False): self.localStageProtocolList = ( stageProto if stageProto - else accessProto - if accessProto - else globalStageProto - if globalStageProto - else self.localAccessProtocolList + else ( + accessProto + if accessProto + else globalStageProto + if globalStageProto + else self.localAccessProtocolList + ) ) self.log.debug(f"localStageProtocolList {self.localStageProtocolList}") @@ -291,6 +294,7 @@ def __init__(self, name, protocolSections=None, vo=None, hideExceptions=False): "isDirectory", "isFile", "getOccupancy", + "getWLCGTokenPath", ] self.okMethods = [ @@ -952,6 +956,25 @@ def getLFNFromURL(self, urls): # This is the generic wrapper for file operations # + @convertToReturnValue + def getWLCGTokenPath(self, lfn: str): + """ + EXPERIMENTAL + return the path to put in the token, relative to the vo path as configured + in the storage. + + """ + wlcgTokenBasePath = self.options.get("WLCGTokenBasePath") + if not wlcgTokenBasePath: + raise ValueError("WLCGTokenBasePath not configured") + + for storage in self.storages.values(): + try: + return storage.getWLCGTokenPath(lfn, wlcgTokenBasePath) + except Exception: + continue + raise RuntimeError("Could not get WLCGTokenPath") + def getURL(self, lfn, protocol=False, replicaDict=None): """execute 'getTransportURL' operation.