feat: generate a real world run (#1165)

.docs/content/2.guide/populate-database.md

@@ -53,15 +53,16 @@ chmod +x ./tools/mongodb/<script-name>.sh
 
 ### Available scripts
 
-| Script name                           | Description                                       |
-|---------------------------------------|---------------------------------------------------|
-| `export-all`                          | Export all collections in the `.database` folder. |
-| `clean-all`                           | Clean all collections. **Be careful!**            |
-| `generate-partitions`                 | Generate 100 partitions                           |
-| `generate-sessions`                   | Generate 100 sessions                             |
-| `generate-session-with-related-tasks` | Generate 1 session and 100 related tasks          |
-| `generate-tasks`                      | Generate 100 tasks                                |
-| `generate-results`                    | Generate 100 results                              |
+| Script name                           | Description                                                                         |
+|---------------------------------------|-------------------------------------------------------------------------------------|
+| `export-all`                          | Export all collections in the `.database` folder.                                   |
+| `clean-all`                           | Clean all collections. **Be careful!**                                              |
+| `generate-partitions`                 | Generate 100 partitions                                                             |
+| `generate-sessions`                   | Generate 100 sessions                                                               |
+| `generate-session-with-related-tasks` | Generate 1 session and 100 related tasks                                            |
+| `generate-tasks`                      | Generate 100 tasks                                                                  |
+| `generate-results`                    | Generate 100 results                                                                |
+| `generate-real-world-run`             | Generate 1 partition, 1 session, 1 application and 1 000 tasks with related results |
 
 ::alert{type="info"}
 To generate applications, you must generate tasks.

tools/mongodb/generate-real-world-run.sh

@@ -0,0 +1,5 @@
+#! /usr/bin/env bash
+
+DIR="$(realpath "$(dirname "${BASH_SOURCE[0]}")")"
+# Description: Generate results for ArmoniK
+"$DIR/utils/execute-mongo-shell-script.sh" generate-real-world-run

tools/mongodb/scripts/generate-real-world-run.js

@@ -0,0 +1,174 @@
+const localRequire = require("module").createRequire(__filename)
+const faker = localRequire("@faker-js/faker").fakerEN
+
+// Move to the correct database in MongoDB
+db = db.getSiblingDB("database")
+
+const application = {
+  ApplicationName: faker.commerce.productName().split(" ").join("."),
+  ApplicationService: faker.hacker.verb(),
+  ApplicationVersion: faker.system.semver(),
+  ApplicationNamespace: faker.commerce.productName().split(" ").join(".")
+}
+
+const partitionId = faker.string.uuid()
+
+db.PartitionData.insertOne({
+  _id: partitionId,
+  ParentPartitionIds: [],
+  PodConfiguration: null,
+  PodMax: faker.number.int({
+    min: 20,
+    max: 100
+  }),
+  PodReserved: faker.number.int({
+    min: 0,
+    max: 20
+  }),
+  PreemptionPercentage: faker.number.int({
+    min: 0,
+    max: 100
+  }),
+  Priority: faker.number.int({
+    min: 0,
+    max: 4
+  }),
+})
+
+const sessionId = faker.string.uuid()
+const sessionCreationDate = faker.date.past()
+
+const options = {
+  MaxDuration: "00:00:00",
+  MaxRetries: faker.number.int({
+    min: 0,
+    max: 10
+  }),
+  Options: {},
+  Priority: faker.number.int({
+    min: 0,
+    max: 4
+  }),
+  PartitionId: partitionId,
+  EngineType: faker.commerce.productAdjective(),
+  ...application
+}
+
+db.SessionData.insertOne({
+  _id: sessionId,
+  Status: 1, // Running
+  PartitionIds: [partitionId],
+  CreationDate: sessionCreationDate,
+  CancellationDate: null,
+  Options: {
+    ...options,
+  },
+})
+
+const tasksNumber = 1_000
+
+const resultsIds = []
+for (let i = 0; i < tasksNumber; i++) {
+  const { taskId, expectedOutputIds } = createTask()
+
+  resultsIds.push({
+    taskId: taskId,
+    expectedOutputIds: expectedOutputIds
+  })
+}
+
+resultsIds.forEach(({ taskId, expectedOutputIds }) => {
+  const creationDate = faker.date.past();
+  const completionDate = faker.date.future({ refDate: creationDate });
+
+  expectedOutputIds.forEach((expectedOutputId) => {
+    db.Result.insertOne({
+      SessionId: sessionId,
+      Name: faker.word.sample(),
+      OwnerTaskId: taskId,
+      Status: 2, // Completed
+      DependentTasks: [],
+      // Binary data
+      Data: faker.number.octal(),
+      CreationDate: creationDate,
+      CompletionDate: completionDate,
+      _id: expectedOutputId,
+    })
+  })
+})
+
+
+/**
+ * Create a task
+ *
+ * If id and outputsIds are given, the task is created from a retry.
+ * Only return the last task id when a retry is created.
+ *
+ * @param {string} id
+ * @param {string[]} retriedIds
+ * @param {string[]} outputsIds
+ */
+function createTask(id, retriedIds, outputsIds) {
+  const creationDate = faker.date.between({ from: sessionCreationDate, to: new Date() })
+  const submittedDate = faker.date.between({ from: creationDate, to: new Date() })
+  const startDate = faker.date.future({ refDate: submittedDate })
+  const endDate = faker.date.future({ refDate: startDate })
+
+  const expectedOutputIds = outputsIds ?? Array.from({
+    length: faker.number.int({
+      min: 0,
+      max: 2
+    })
+  }, () => faker.string.uuid())
+
+  const taskId = faker.string.uuid()
+
+  const isRetried = faker.datatype.boolean({
+    probability: 0.2
+  })
+
+  db.TaskData.insertOne({
+    _id: taskId,
+    SessionId: sessionId,
+    OwnerPodId: '',
+    OwnerPodName: '',
+    PayloadId: faker.string.uuid(),
+    ParentTaskIds: [],
+    DataDependencies: [],
+    RemainingDataDependencies: {},
+    ExpectedOutputIds: expectedOutputIds,
+    InitialTaskId: id ?? taskId,
+    RetryOfIds: retriedIds ?? [],
+    Status: isRetried ? 11 /* Retry */ : 4 /* Completed */,
+    StatusMessage: "",
+    Options: {
+      ...options,
+    },
+    CreationDate: creationDate,
+    SubmittedDate: submittedDate,
+    StartDate: startDate,
+    EndDate: endDate,
+    ReceptionDate: null,
+    AcquisitionDate: null,
+    PodTtl: null,
+    ProcessingToEndDuration: null,
+    CreationToEndDuration: null,
+    Output: {
+      Success: false,
+      Error: ""
+    }
+  })
+
+  if (isRetried) {
+    const { taskId: id } = createTask(taskId, retriedIds ? [taskId, ...retriedIds] : [taskId], expectedOutputIds)
+    return {
+      taskId: id,
+      expectedOutputIds: expectedOutputIds
+    }
+  }
+
+  return {
+    taskId: taskId,
+    expectedOutputIds: expectedOutputIds
+  }
+}