feat (TS): add getTransformationFilesAsJsonString

src/DIRAC/TransformationSystem/Client/TransformationClient.py

@@ -3,6 +3,7 @@
 from DIRAC import S_OK, S_ERROR, gLogger
 from DIRAC.Core.Base.Client import Client, createClient
 from DIRAC.Core.Utilities.List import breakListIntoChunks
+from DIRAC.Core.Utilities.JEncode import decode as jdecode
 from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations
 from DIRAC.TransformationSystem.Client import TransformationStatus
 from DIRAC.TransformationSystem.Client import TransformationFilesStatus
@@ -176,9 +177,12 @@ def getTransformationFiles(
             timeStamp = "LastUpdate"
 
         if "LFN" not in condDict:
-            res = rpcClient.getTransformationFiles(
+            res = rpcClient.getTransformationFilesAsJsonString(
                 condDict, older, newer, timeStamp, orderAttribute, offset, maxfiles, columns
             )
+            if not res["OK"]:
+                return res
+            res, _ = jdecode(res["Value"])
             # TransformationDB.getTransformationFiles includes a "Records"/"ParameterNames"
             # that we don't want to return to the client so explicitly return S_OK with the value
             if not res["OK"]:
@@ -204,9 +208,12 @@ def getTransformationFiles(
             # Apply the offset to the list of LFNs
             condDict["LFN"] = lfnList[offsetToApply : offsetToApply + limit]
             # No limit and no offset as the list is limited already
-            res = rpcClient.getTransformationFiles(
+            res = rpcClient.getTransformationFilesAsJsonString(
                 condDict, older, newer, timeStamp, orderAttribute, None, None, columns
             )
+            if not res["OK"]:
+                return res
+            res, _ = jdecode(res["Value"])
             if not res["OK"]:
                 gLogger.error(
                     "Error getting files for transformation %s (offset %d), %s"

src/DIRAC/TransformationSystem/DB/TransformationDB.py

@@ -603,6 +603,7 @@ def getTransformationFiles(
         offset=None,
         connection=False,
         columns=None,
+        include_web_records=True,
     ):
         """Get files for the supplied transformations with support for the web standard structure"""
         connection = self.__getConnection(connection)
@@ -640,11 +641,12 @@ def getTransformationFiles(
             return res
 
         resultList = [dict(zip(columns, row)) for row in res["Value"]]
-        webList = [[str(item) if not isinstance(item, int) else item for item in row] for row in res["Value"]]
 
         result = S_OK(resultList)
-        result["Records"] = webList
-        result["ParameterNames"] = columns
+        if include_web_records:
+            webList = [[str(item) if not isinstance(item, int) else item for item in row] for row in res["Value"]]
+            result["Records"] = webList
+            result["ParameterNames"] = columns
         return result
 
     def getFileSummary(self, lfns, connection=False):

src/DIRAC/TransformationSystem/Service/TransformationManagerHandler.py

@@ -1,9 +1,11 @@
 """ Service for interacting with TransformationDB
 """
+
 from DIRAC import S_OK, S_ERROR
 from DIRAC.Core.DISET.RequestHandler import RequestHandler
 from DIRAC.Core.Security.Properties import SecurityProperty
 from DIRAC.Core.Utilities.DEncode import ignoreEncodeWarning
+from DIRAC.Core.Utilities.JEncode import encode as jencode
 from DIRAC.Core.Utilities.ObjectLoader import ObjectLoader
 from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations
 from DIRAC.TransformationSystem.Client import TransformationFilesStatus
@@ -289,6 +291,7 @@ def export_getTransformationFiles(
         limit=None,
         offset=None,
         columns=None,
+        include_web_records=True,
     ):
         if not condDict:
             condDict = {}
@@ -302,8 +305,22 @@ def export_getTransformationFiles(
             offset=offset,
             connection=False,
             columns=columns,
+            include_web_records=include_web_records,
         )
 
+    types_getTransformationFilesAsJsonString = types_getTransformationFiles
+
+    def export_getTransformationFilesAsJsonString(self, *args, **kwargs):
+        """
+        DEncode cannot cope with nested structures of multiple millions items.
+        Encode everything as a json string, that DEncode can then transmit faster.
+
+        This will be the default as of v9.0
+        """
+        kwargs["include_web_records"] = False
+        res = self.export_getTransformationFiles(*args, **kwargs)
+        return S_OK(jencode(res))
+
     ####################################################################
     #
     # These are the methods to manipulate the TransformationTasks table