From 8276ec8f16b196dcaaa893efb2b223fb68acca0c Mon Sep 17 00:00:00 2001 From: Christophe Haen Date: Wed, 30 Aug 2023 10:15:20 +0200 Subject: [PATCH] feat (TransformationCleaningAgent): chunk the file removals --- .../Agent/TransformationCleaningAgent.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/src/DIRAC/TransformationSystem/Agent/TransformationCleaningAgent.py b/src/DIRAC/TransformationSystem/Agent/TransformationCleaningAgent.py index 85d8569fd4d..f09a2b26fac 100644 --- a/src/DIRAC/TransformationSystem/Agent/TransformationCleaningAgent.py +++ b/src/DIRAC/TransformationSystem/Agent/TransformationCleaningAgent.py @@ -386,20 +386,24 @@ def cleanContent(self, directory): # Executing with shifter proxy gConfigurationData.setOptionInCFG("/DIRAC/Security/UseServerCertificate", "false") - res = DataManager().removeFile(filesFound, force=True) + failed = {} + for chunkId, filesChunk in enumerate(breakListIntoChunks(filesFound, 500)): + self.log.info("Removing chunk", chunkId) + res = DataManager().removeFile(filesChunk, force=True) + if not res["OK"]: + failed.update(dict.fromkeys(filesChunk, res["Message"])) + failed.update(res["Value"]["Failed"]) gConfigurationData.setOptionInCFG("/DIRAC/Security/UseServerCertificate", "true") - if not res["OK"]: - return res realFailure = False - for lfn, reason in res["Value"]["Failed"].items(): + for lfn, reason in failed.items(): if "File does not exist" in str(reason): self.log.warn(f"File {lfn} not found in some catalog: ") else: self.log.error("Failed to remove file found in the catalog", f"{lfn} {reason}") realFailure = True if realFailure: - return S_ERROR("Failed to remove all files found in the catalog") + return S_ERROR("Failed to remove some files found in the catalog") return S_OK() def __getCatalogDirectoryContents(self, directories):