[Core] Import data/config: add versions, trees (#366)

* Upgrade importData function to support versionable values * Add cacheParams in treatElement(), and optimize validateVersion() * Add test on importDomain::saveValue params * importDomain::importData: Fix cache system and order * Add cache element counter (progress object) * importDomain: fix updateProgress translate key for tree, links and versions * importDomain.ts: remote links and verisons in stats
leav-solutions · Jan 10, 2024 · 7e967bc · 7e967bc
1 parent 77f406a
commit 7e967bc
Show file tree

Hide file tree

Showing 14 changed files with 543 additions and 242 deletions.
diff --git a/apps/core/package.json b/apps/core/package.json
@@ -7,8 +7,8 @@
     "start:watch": "npm run start:dev",
     "start:files": "ts-node-dev --inspect=0.0.0.0:9230 --respawn src/ --filesManager",
     "start:indexation": "ts-node-dev --respawn src/ --indexationManager",
-    "start:tasksMaster": "ts-node-dev --respawn src/ --tasksManager=master",
-    "start:tasksWorker": "ts-node-dev --respawn src/ --tasksManager=worker",
+    "start:tasksMaster": "ts-node-dev --inspect=0.0.0.0:9232 --respawn src/ --tasksManager=master",
+    "start:tasksWorker": "ts-node-dev --inspect=0.0.0.0:9234 --respawn src/ --tasksManager=worker",
     "start:dev": "ts-node-dev --inspect=0.0.0.0:9229 --respawn src/ --server",
     "test:watch": "jest -c jest-unit-config.json --watch",
     "test": "jest -c jest-unit-config.json",

diff --git a/apps/core/src/_types/import.ts b/apps/core/src/_types/import.ts
@@ -32,7 +32,13 @@ export interface IValue {
     library?: string; // only for tree attributes
     value: string | IMatch[];
     metadata?: IValueMetadata;
-    version?: IValueVersion;
+    version?: IImportValueVersion[];
+}
+
+export interface IImportValueVersion {
+    treeId: string;
+    element: IMatch[];
+    library?: string;
 }
 
 export interface IData {
@@ -46,7 +52,6 @@ export interface IElement {
     matches: IMatch[];
     mode: ImportMode;
     data: IData[];
-    links: IData[];
 }
 
 export interface ITree {
@@ -64,3 +69,9 @@ export interface IFile {
     elements: IElement[];
     trees: ITree[];
 }
+
+export interface ICacheParams {
+    cacheDataPath: string,
+    cacheKey: number,
+    isCacheActive: boolean,
+}
diff --git a/apps/core/src/app/core/attributeApp/attributeApp.ts b/apps/core/src/app/core/attributeApp/attributeApp.ts
@@ -393,7 +393,7 @@ export default function (deps: IDeps = {}): ICoreAttributeApp {
                             // TODO: this could be optimized if find() would allow searching for multiple IDs at once
                             return {
                                 ...attributeData.values_list,
-                                values: (attributeData.values_list.values as string[])
+                                values: (attributeData.values_list.values as string[]) || []
                                     .map(async recId => {
                                         const record = await recordDomain.find({
                                             params: {

diff --git a/apps/core/src/domain/import/import-config-schema.json b/apps/core/src/domain/import/import-config-schema.json
@@ -330,46 +330,54 @@
                     }
                 },
                 "libraries": {
-                    "type": "array",
+                    "type": "object",
                     "items": {
                         "type": "object",
                         "properties": {
-                            "library": {
-                                "type": "string"
+                            "allowMultiplePositions": {
+                                "type": "boolean"
                             },
-                            "settings": {
-                                "type": "object",
-                                "properties": {
-                                    "allowMultiplePositions": {
-                                        "type": "boolean"
-                                    },
-                                    "allowedAtRoot": {
-                                        "type": "boolean"
-                                    },
-                                    "allowedChildren": {
-                                        "type": "array",
-                                        "items": {
-                                            "type": "string"
-                                        }
-                                    }
-                                },
-                                "required": [
-                                    "allowMultiplePositions",
-                                    "allowedAtRoot",
-                                    "allowedChildren"
-                                ]
+                            "allowedAtRoot": {
+                                "type": "boolean"
+                            },
+                            "allowedChildren": {
+                                "type": "array",
+                                "items": {
+                                    "type": "string"
+                                }
                             }
                         },
                         "required": [
-                            "library",
-                            "settings"
+                            "allowMultiplePositions",
+                            "allowedAtRoot",
+                            "allowedChildren"
                         ]
                     }
                 }
             },
             "required": [
                 "id"
             ]
+        },
+        "version_profile": {
+            "type": "object",
+            "properties": {
+                "id": {
+                    "$ref": "#/$defs/id"
+                },
+                "label": {
+                    "$ref": "#/$defs/systemTranslation"
+                },
+                "description": {
+                    "$ref": "#/$defs/systemTranslation"
+                },
+                "trees": {
+                    "type": "array",
+                    "items": {
+                        "type": "string"
+                    }
+                }
+            }
         }
     },
     "properties": {
@@ -390,6 +398,12 @@
             "items": {
                 "$ref": "#/definitions/tree"
             }
+        },
+        "version_profiles": {
+            "type": "array",
+            "items": {
+                "$ref": "#/definitions/version_profile"
+            }
         }
     }
-}
+}
diff --git a/apps/core/src/domain/import/import-data-schema.json b/apps/core/src/domain/import/import-data-schema.json
@@ -2,6 +2,29 @@
   "$id": "import-data",
   "title": "LEAV Import Data",
   "definitions": {
+    "valueVersion": {
+      "type": "object",
+      "properties": {
+        "treeId": {
+          "type": "string"
+        },
+        "library": {
+          "type": "string"
+        },
+        "element": {
+          "description": "Object representing the element to process",
+          "type": ["array", "null"],
+          "items": {
+            "$ref": "#/definitions/match"
+          }
+        }
+      },
+      "required": [
+        "treeId",
+        "library",
+        "element"
+      ]
+    },
     "match": {
       "description": "Attribute/value pair used to find an element",
       "type": "object",
@@ -11,7 +34,7 @@
           "description": "Attribute key"
         },
         "value": {
-          "type": "string",
+          "type": ["string", "null"],
           "description": "Attribute value to search"
         }
       },
@@ -47,6 +70,12 @@
           "additionalProperties": {
             "type": "string"
           }
+        },
+        "version":{
+          "type": "array",
+          "items": {
+            "$ref": "#/definitions/valueVersion"
+          }
         }
       },
       "required": [
@@ -98,7 +127,9 @@
         "mode": {
           "type": "string",
           "enum": [
-            "update", "insert", "upsert"
+            "update",
+            "insert",
+            "upsert"
           ]
         },
         "data": {
@@ -107,20 +138,12 @@
           "items": {
             "$ref": "#/definitions/data"
           }
-        },
-        "links": {
-          "type": "array",
-          "description": "List of simple/advanced links attributes data to process",
-          "items": {
-            "$ref": "#/definitions/data"
-          }
         }
       },
       "required": [
         "library",
         "matches",
-        "data",
-        "links"
+        "data"
       ]
     },
     "tree": {
@@ -201,4 +224,4 @@
     "elements",
     "trees"
   ]
-}
+}