From d3820e4af0dec25052fa4d9a11169aea6b78e815 Mon Sep 17 00:00:00 2001 From: MontaGhanmy Date: Wed, 23 Oct 2024 14:17:14 +0100 Subject: [PATCH 1/3] feat: cli/seed db --- .../node/src/cli/cmds/dev_cmds/seed_db.ts | 179 ++++++++++++++++++ .../orm/connectors/abstract-connector.ts | 3 + .../database/services/orm/connectors/index.ts | 5 + 3 files changed, 187 insertions(+) create mode 100644 tdrive/backend/node/src/cli/cmds/dev_cmds/seed_db.ts diff --git a/tdrive/backend/node/src/cli/cmds/dev_cmds/seed_db.ts b/tdrive/backend/node/src/cli/cmds/dev_cmds/seed_db.ts new file mode 100644 index 00000000..bb7f4b40 --- /dev/null +++ b/tdrive/backend/node/src/cli/cmds/dev_cmds/seed_db.ts @@ -0,0 +1,179 @@ +import * as mongo from "mongodb"; +import { v4 as uuidv4 } from "uuid"; +import yargs from "yargs"; +import tdrive from "../../../tdrive"; +import gr from "../../../services/global-resolver"; +import { getInstance } from "../../../services/user/entities/user"; +import { getInstance as getCompanyInstance } from "../../../services/user/entities/company"; +import PasswordEncoder from "../../../utils/password-encoder"; +import { getDefaultDriveItem } from "../../../services/documents/utils"; + +type CLIArgs = { + start: number; +}; + +const services = [ + "auth", + "storage", + "user", + "files", + "counter", + "cron", + "message-queue", + "push", + "search", + "tracker", + "email-pusher", + "workspaces", + "console", + "applications", + "database", + "webserver", +]; +const createTree = async ( + depth: number, + folderData: any, + parent: string, + context: any, + client: mongo.MongoClient, +) => { + const documentRepo = await gr.services.documents.documents.repository; + // Create an array to hold promises + const createFolderPromises = []; + + // Loop from 0 to depth + for (let i = 0; i < depth; i++) { + // Modify folder data for each iteration + folderData.name = `folder_${i}`; + folderData.parent_id = parent; // All siblings share the same parent + folderData.id = uuidv4(); + + // Push the folder creation promise to the array + // createFolderPromises.push( + // gr.services.documents.documents.create( + // null, + // folderData, + // {}, + // { + // ...context, + // user: { + // ...context.user, + // server_request: false, + // }, + // }, + // ), + // ); + // createFolderPromises.push(documentRepo.save(getDefaultDriveItem(folderData, context))); + createFolderPromises.push(getDefaultDriveItem(folderData, context)); + } + /// const createdFolders = await documentRepo.saveAll(createFolderPromises); + await client.db("tdrive").collection("drive_files").insertMany(createFolderPromises); + + // Wait for all folder creation promises to resolve in parallel + // const createdFolders = await Promise.all(createFolderPromises); + + // Optionally, return the created folders + // return createdFolders; +}; + +const command: yargs.CommandModule = { + command: "seed", + describe: "Seed the db", + builder: { + start: { + default: 0, + type: "number", + description: "Start start for the users", + }, + output: { + default: "", + type: "string", + description: "Folder containing the exported data", + }, + }, + handler: async argv => { + console.log("🌱 Seeding the database with start: ", argv.start); + const usersNumber = 1000; + const defaultPassword = "password"; + const userRole = "admin"; + const folderTreeDepth = 1000; + + const platform = await tdrive.run(services); + await gr.doInit(platform); + + let client: mongo.MongoClient; + client = (await gr.database.getConnector()).getClient(); + + // Manage the default company + const companies = await gr.services.companies.getCompanies(); + let company = companies.getEntities()?.[0]; + if (!company) { + const newCompany = getCompanyInstance({ + name: "Tdrive", + plan: { name: "Local", limits: undefined, features: undefined }, + }); + company = await gr.services.companies.createCompany(newCompany); + } + + console.log("✅ Company created: ", company.id); + + // encoding the user password + const passwordEncoder = new PasswordEncoder(); + const encodedPassword = await passwordEncoder.encodePassword(defaultPassword); + // Step 1: Generate 10k users + const usersData = Array.from({ length: usersNumber }, (_, i) => { + const userNumber = i + argv.start * 1000; + return { + first_name: `User ${userNumber}`, + last_name: `Lastname ${userNumber}`, + username_canonical: `user${userNumber}`, + email_canonical: `user${userNumber}@example.com`, + password: encodedPassword, + cache: { + companies: [], + }, + }; + }); + + // for each user, get the user and add it to allUsers + const allUsers = []; + for (const userData of usersData) { + const newUser = getInstance(userData); + const user = await gr.services.users.create(newUser); + allUsers.push(user.entity); + } + console.log("✅ Users created:: ", allUsers); + + // // for each user, assign a role + for (const user of allUsers) { + console.log("User is:: ", user); + // Update user company + await gr.services.companies.setUserRole(company.id, user.id, userRole); + await gr.services.workspaces.processPendingUser(user); + } + console.log("✅ Users created"); + + let updateBegin = Date.now(); + let createTreePromises = []; + for (const user of allUsers) { + // console.log progress percentage + const context = { company, user }; + const parentDrive = `user_${user.id}`; + const folderData = await getDefaultDriveItem( + { + parent_id: parentDrive, + company_id: company.id, + is_directory: true, + }, + context, + ); + // create folder tree + await createTree(folderTreeDepth, folderData, parentDrive, context, client); + console.log(`✅ User ${user.id} folder tree created`); + } + // await Promise.all(createTreePromises); + console.log("✅ Finished execution, took: ", (Date.now() - updateBegin) / 1000, "s"); + }, +}; + +export default command; diff --git a/tdrive/backend/node/src/core/platform/services/database/services/orm/connectors/abstract-connector.ts b/tdrive/backend/node/src/core/platform/services/database/services/orm/connectors/abstract-connector.ts index a8ba3555..daaf5810 100644 --- a/tdrive/backend/node/src/core/platform/services/database/services/orm/connectors/abstract-connector.ts +++ b/tdrive/backend/node/src/core/platform/services/database/services/orm/connectors/abstract-connector.ts @@ -47,4 +47,7 @@ export abstract class AbstractConnector implements getType(): DatabaseType { return this.type; } + getClient() { + return this.getClient(); + } } diff --git a/tdrive/backend/node/src/core/platform/services/database/services/orm/connectors/index.ts b/tdrive/backend/node/src/core/platform/services/database/services/orm/connectors/index.ts index f1c6792e..6e1f50d6 100644 --- a/tdrive/backend/node/src/core/platform/services/database/services/orm/connectors/index.ts +++ b/tdrive/backend/node/src/core/platform/services/database/services/orm/connectors/index.ts @@ -99,6 +99,11 @@ export interface Connector extends Initializable { * @returns Pagination */ getOffsetPagination(options: Paginable): Pagination; + + /** + * Get the db client + */ + getClient(): any; } export declare type ConnectionOptions = MongoConnectionOptions | PostgresConnectionOptions; From d3e29da1e48b22828c86e5544c4f8761ee54a4b8 Mon Sep 17 00:00:00 2001 From: MontaGhanmy Date: Thu, 24 Oct 2024 14:43:05 +0100 Subject: [PATCH 2/3] =?UTF-8?q?=F0=9F=94=96=20Release=20Version=201.0.4-hf?= =?UTF-8?q?2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../services/orm/connectors/mongodb/mongodb.ts | 6 +----- tdrive/backend/node/src/version.ts | 6 +++--- tdrive/frontend/src/app/environment/version.ts | 4 ++-- .../app/features/drive/hooks/use-drive-item.tsx | 16 ++++++++-------- 4 files changed, 14 insertions(+), 18 deletions(-) diff --git a/tdrive/backend/node/src/core/platform/services/database/services/orm/connectors/mongodb/mongodb.ts b/tdrive/backend/node/src/core/platform/services/database/services/orm/connectors/mongodb/mongodb.ts index 7958110a..55c7c2bf 100644 --- a/tdrive/backend/node/src/core/platform/services/database/services/orm/connectors/mongodb/mongodb.ts +++ b/tdrive/backend/node/src/core/platform/services/database/services/orm/connectors/mongodb/mongodb.ts @@ -287,11 +287,7 @@ export class MongoConnector extends AbstractConnector { .find(query) .sort(sort) .skip(Math.max(0, parseInt(options.pagination.page_token || "0"))) - .limit(Math.max(0, parseInt(options.pagination.limitStr || "100"))) - .collation({ - locale: "en_US", - numericOrdering: true, - }); + .limit(Math.max(0, parseInt(options.pagination.limitStr || "100"))); const entities: Table[] = []; while (await cursor.hasNext()) { diff --git a/tdrive/backend/node/src/version.ts b/tdrive/backend/node/src/version.ts index b047764c..aba10f0d 100644 --- a/tdrive/backend/node/src/version.ts +++ b/tdrive/backend/node/src/version.ts @@ -1,7 +1,7 @@ export default { - current: /* @VERSION_DETAIL */ "1.0.4-hf1", + current: /* @VERSION_DETAIL */ "1.0.4-hf2", minimal: { - web: /* @MIN_VERSION_WEB */ "1.0.4-hf1", - mobile: /* @MIN_VERSION_MOBILE */ "1.0.4-hf1", + web: /* @MIN_VERSION_WEB */ "1.0.4-hf2", + mobile: /* @MIN_VERSION_MOBILE */ "1.0.4-hf2", }, }; diff --git a/tdrive/frontend/src/app/environment/version.ts b/tdrive/frontend/src/app/environment/version.ts index f98e2a81..1ae3445e 100644 --- a/tdrive/frontend/src/app/environment/version.ts +++ b/tdrive/frontend/src/app/environment/version.ts @@ -1,5 +1,5 @@ export default { - version: /* @VERSION */ '1.0.4-hf1', - version_detail: /* @VERSION_DETAIL */ '1.0.4-hf1', + version: /* @VERSION */ '1.0.4-hf2', + version_detail: /* @VERSION_DETAIL */ '1.0.4-hf2', version_name: /* @VERSION_NAME */ 'Ghost-Dog', }; \ No newline at end of file diff --git a/tdrive/frontend/src/app/features/drive/hooks/use-drive-item.tsx b/tdrive/frontend/src/app/features/drive/hooks/use-drive-item.tsx index 7f2716ca..dde7d72e 100644 --- a/tdrive/frontend/src/app/features/drive/hooks/use-drive-item.tsx +++ b/tdrive/frontend/src/app/features/drive/hooks/use-drive-item.tsx @@ -122,13 +122,13 @@ export const useDriveItem = (id: string) => { })); } // set children and remove duplicates - setChildren(prev => [ - ...prev, - ...details.children.filter( - (item, index, self) => - index === self.findIndex(t => t.id === item.id), - ), - ]); + setChildren(prev => { + // Create a Map for existing IDs for fast lookups + const existingIds = new Map(prev.map(item => [item.id, true])); + + // Filter children while adding them to the state + return [...prev, ...details.children.filter(item => !existingIds.has(item.id))]; + }); } catch (e) { // set pagination end to true set(DriveItemPagination, prev => ({ @@ -140,7 +140,7 @@ export const useDriveItem = (id: string) => { } finally { set(DriveItemPagination, prev => ({ ...prev, - page: (prev.page + prev.limit), + page: prev.page + prev.limit, })); } setLoading(false); From c8a54a1b66754a3b75048005deee180904158130 Mon Sep 17 00:00:00 2001 From: Eric Doughty-Papassideris Date: Mon, 28 Oct 2024 16:23:44 +0100 Subject: [PATCH 3/3] =?UTF-8?q?=F0=9F=9A=A8=20back:=20fixing=20linter?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- tdrive/backend/node/src/cli/cmds/dev_cmds/seed_db.ts | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/tdrive/backend/node/src/cli/cmds/dev_cmds/seed_db.ts b/tdrive/backend/node/src/cli/cmds/dev_cmds/seed_db.ts index bb7f4b40..1fbe39a8 100644 --- a/tdrive/backend/node/src/cli/cmds/dev_cmds/seed_db.ts +++ b/tdrive/backend/node/src/cli/cmds/dev_cmds/seed_db.ts @@ -37,7 +37,6 @@ const createTree = async ( context: any, client: mongo.MongoClient, ) => { - const documentRepo = await gr.services.documents.documents.repository; // Create an array to hold promises const createFolderPromises = []; @@ -101,8 +100,7 @@ const command: yargs.CommandModule = { const platform = await tdrive.run(services); await gr.doInit(platform); - let client: mongo.MongoClient; - client = (await gr.database.getConnector()).getClient(); + const client = (await gr.database.getConnector()).getClient(); // Manage the default company const companies = await gr.services.companies.getCompanies(); @@ -153,10 +151,8 @@ const command: yargs.CommandModule = { } console.log("✅ Users created"); - let updateBegin = Date.now(); - let createTreePromises = []; + const updateBegin = Date.now(); for (const user of allUsers) { - // console.log progress percentage const context = { company, user }; const parentDrive = `user_${user.id}`; const folderData = await getDefaultDriveItem( @@ -171,7 +167,6 @@ const command: yargs.CommandModule = { await createTree(folderTreeDepth, folderData, parentDrive, context, client); console.log(`✅ User ${user.id} folder tree created`); } - // await Promise.all(createTreePromises); console.log("✅ Finished execution, took: ", (Date.now() - updateBegin) / 1000, "s"); }, };