diff --git a/common/config/rush/command-line.json b/common/config/rush/command-line.json index c6e75745b45..2cff77a8497 100644 --- a/common/config/rush/command-line.json +++ b/common/config/rush/command-line.json @@ -4,13 +4,13 @@ */ { "$schema": "https://developer.microsoft.com/json-schemas/rush/v5/command-line.schema.json", - - "phases": [ { "name": "_phase:build", "dependencies": { - "upstream": ["_phase:build"] + "upstream": [ + "_phase:build" + ] }, "ignoreMissingScript": true, "allowWarningsOnSuccess": false @@ -18,8 +18,13 @@ { "name": "_phase:validate", "dependencies": { - "self": ["_phase:build"], - "upstream": ["_phase:validate", "_phase:build"] + "self": [ + "_phase:build" + ], + "upstream": [ + "_phase:validate", + "_phase:build" + ] }, "ignoreMissingScript": true, "allowWarningsOnSuccess": false @@ -27,8 +32,12 @@ { "name": "_phase:test", "dependencies": { - "self": ["_phase:build"], - "upstream": ["_phase:validate"] + "self": [ + "_phase:build" + ], + "upstream": [ + "_phase:validate" + ] }, "ignoreMissingScript": true, "allowWarningsOnSuccess": true @@ -36,7 +45,9 @@ { "name": "_phase:lint", "dependencies": { - "self": ["_phase:build"] + "self": [ + "_phase:build" + ] }, "ignoreMissingScript": true, "allowWarningsOnSuccess": false @@ -44,7 +55,9 @@ { "name": "_phase:bundle", "dependencies": { - "self": ["_phase:build"] + "self": [ + "_phase:build" + ] }, "ignoreMissingScript": true, "allowWarningsOnSuccess": false @@ -52,7 +65,9 @@ { "name": "_phase:format", "dependencies": { - "self": ["_phase:build"] + "self": [ + "_phase:build" + ] }, "ignoreMissingScript": true, "allowWarningsOnSuccess": false @@ -60,7 +75,9 @@ { "name": "_phase:svelte-check", "dependencies": { - "self": ["_phase:build"] + "self": [ + "_phase:build" + ] }, "ignoreMissingScript": true, "allowWarningsOnSuccess": true @@ -68,8 +85,12 @@ { "name": "_phase:package", "dependencies": { - "self": ["_phase:build"], - "upstream": ["_phase:package"] + "self": [ + "_phase:build" + ], + "upstream": [ + "_phase:package" + ] }, "ignoreMissingScript": true, "allowWarningsOnSuccess": false @@ -77,8 +98,16 @@ { "name": "_phase:docker-build", "dependencies": { - "self": ["_phase:build", "_phase:package", "_phase:bundle"], - "upstream": ["_phase:build", "_phase:bundle", "_phase:package"] + "self": [ + "_phase:build", + "_phase:package", + "_phase:bundle" + ], + "upstream": [ + "_phase:build", + "_phase:bundle", + "_phase:package" + ] }, "ignoreMissingScript": true, "allowWarningsOnSuccess": true @@ -86,7 +115,11 @@ { "name": "_phase:docker-staging", "dependencies": { - "self": ["_phase:build", "_phase:package", "_phase:bundle"] + "self": [ + "_phase:build", + "_phase:package", + "_phase:bundle" + ] }, "ignoreMissingScript": true, "allowWarningsOnSuccess": true @@ -103,26 +136,43 @@ "ignoreMissingScript": true, "safeForSimultaneousRushProcesses": true, "disableBuildCache": true - }, + }, + { + "commandKind": "global", + "name": "doformat", + "summary": "Do a format and show errors after it", + "description": "Do a format and show errors after it", + "safeForSimultaneousRushProcesses": true, + "shellCommand": "rush format && ./common/scripts/format-show.sh" + }, { "commandKind": "phased", "name": "build:watch", "summary": "Build and watch", - "phases": ["_phase:build", "_phase:validate"], + "phases": [ + "_phase:build", + "_phase:validate" + ], "description": "Perform build with tsc and watch for changes with rush", "enableParallelism": true, "incremental": true, "safeForSimultaneousRushProcesses": true, "watchOptions": { "alwaysWatch": true, - "watchPhases": ["_phase:build", "_phase:validate"] + "watchPhases": [ + "_phase:build", + "_phase:validate" + ] } }, - { + { "summary": "svelte-check", "commandKind": "phased", "name": "svelte-check", - "phases": ["_phase:build", "_phase:svelte-check"], + "phases": [ + "_phase:build", + "_phase:svelte-check" + ], "enableParallelism": true, "incremental": true }, @@ -130,34 +180,44 @@ "commandKind": "phased", "name": "build", "summary": "build", - "phases": ["_phase:build"], + "phases": [ + "_phase:build" + ], "enableParallelism": true, "incremental": true, "watchOptions": { "alwaysWatch": false, - "watchPhases": ["_phase:build"] + "watchPhases": [ + "_phase:build" + ] } }, { "commandKind": "phased", "name": "validate", - "phases": ["_phase:validate"], + "phases": [ + "_phase:validate" + ], "summary": "validate", "enableParallelism": true, - "incremental": true + "incremental": true }, { "commandKind": "phased", "name": "rebuild", "summary": "ReBuild and test all projects.", - "phases": ["_phase:build"], + "phases": [ + "_phase:build" + ], "enableParallelism": true, "incremental": false }, { "commandKind": "phased", "name": "dorevalidate", - "phases": ["_phase:validate"], + "phases": [ + "_phase:validate" + ], "summary": "dorevalidate", "enableParallelism": true, "incremental": false @@ -166,7 +226,10 @@ "commandKind": "phased", "summary": "Do testing", "name": "test", - "phases": ["_phase:build", "_phase:test"], + "phases": [ + "_phase:build", + "_phase:test" + ], "enableParallelism": true, "incremental": true }, @@ -174,16 +237,21 @@ "commandKind": "phased", "name": "retest", "summary": "Build and test all projects.", - "phases": ["_phase:build", "_phase:test"], + "phases": [ + "_phase:build", + "_phase:test" + ], "enableParallelism": true, "incremental": false }, - { "commandKind": "phased", "summary": "Do bundle", "name": "bundle", - "phases": ["_phase:build", "_phase:bundle"], + "phases": [ + "_phase:build", + "_phase:bundle" + ], "enableParallelism": true, "incremental": true }, @@ -191,16 +259,23 @@ "commandKind": "phased", "summary": "Do packaging", "name": "package", - "phases": ["_phase:build", "_phase:package"], + "phases": [ + "_phase:build", + "_phase:package" + ], "enableParallelism": true, "incremental": true }, - { "summary": "docker:build", "commandKind": "phased", "name": "docker:build", - "phases": ["_phase:build", "_phase:bundle", "_phase:package", "_phase:docker-build"], + "phases": [ + "_phase:build", + "_phase:bundle", + "_phase:package", + "_phase:docker-build" + ], "enableParallelism": true, "incremental": true }, @@ -208,7 +283,12 @@ "summary": "docker:rebuild", "commandKind": "phased", "name": "docker:rebuild", - "phases": ["_phase:build", "_phase:bundle", "_phase:package", "_phase:docker-build"], + "phases": [ + "_phase:build", + "_phase:bundle", + "_phase:package", + "_phase:docker-build" + ], "enableParallelism": true, "incremental": false }, @@ -216,7 +296,12 @@ "summary": "docker:staging", "commandKind": "phased", "name": "docker:staging", - "phases": ["_phase:build", "_phase:bundle", "_phase:package", "_phase:docker-staging"], + "phases": [ + "_phase:build", + "_phase:bundle", + "_phase:package", + "_phase:docker-staging" + ], "enableParallelism": true, "incremental": true }, @@ -239,7 +324,7 @@ "description": "use to build all docker containers required for platform", "safeForSimultaneousRushProcesses": true, "shellCommand": "rush docker:build -p 20 --to @hcengineering/pod-server --to @hcengineering/pod-front --to @hcengineering/prod --to @hcengineering/pod-account --to @hcengineering/pod-workspace --to @hcengineering/pod-collaborator --to @hcengineering/tool --to @hcengineering/pod-print --to @hcengineering/pod-sign --to @hcengineering/pod-analytics-collector --to @hcengineering/rekoni-service --to @hcengineering/pod-ai-bot --to @hcengineering/import-tool --to @hcengineering/pod-stats" - }, + }, { "commandKind": "global", "name": "docker:up", @@ -279,7 +364,7 @@ "description": "Clean typescript incremental cache", "safeForSimultaneousRushProcesses": true, "shellCommand": "find .|grep tsconfig.tsbuildinfo | xargs rm | pwd" - }, + }, { "commandKind": "global", "name": "revalidate", @@ -287,7 +372,7 @@ "description": "Clean typescript incremental cache", "safeForSimultaneousRushProcesses": true, "shellCommand": "find .|grep tsBuildInfoFile.info | xargs rm | pwd && rush dorevalidate" - }, + }, { "commandKind": "global", "name": "model-version", @@ -327,33 +412,40 @@ "description": "use to build all docker containers required for platform", "safeForSimultaneousRushProcesses": true, "shellCommand": "rush package -p 20 --to desktop && cd ./desktop-package && rushx dist" - }, + }, ], - /** * Custom "parameters" introduce new parameters for specified Rush command-line commands. * For example, you might define a "--production" parameter for the "rush build" command. */ "parameters": [ - { - "parameterKind": "flag", + { + "parameterKind": "flag", "longName": "--lite", "shortName": "-l", "description": "Enable Heft lite building option, will skip some phases.", - "associatedCommands": ["build", "rebuild"] + "associatedCommands": [ + "build", + "rebuild" + ] }, { - "parameterKind": "flag", + "parameterKind": "flag", "longName": "--clean", "description": "Enable Heft clean building option", - "associatedCommands": ["build", "rebuild"] + "associatedCommands": [ + "build", + "rebuild" + ] }, { - "parameterKind": "flag", - "longName": "--force", + "parameterKind": "flag", + "longName": "--force", "shortName": "-f", "description": "Force formatting", - "associatedCommands": ["format"] + "associatedCommands": [ + "format" + ] } // { // /** @@ -476,4 +568,4 @@ // ] // } ] -} +} \ No newline at end of file diff --git a/common/scripts/format-show.sh b/common/scripts/format-show.sh new file mode 100755 index 00000000000..57d92054b38 --- /dev/null +++ b/common/scripts/format-show.sh @@ -0,0 +1,19 @@ +#!/bin/sh + +roots=$(rush list -p --json | grep "path" | cut -f 2 -d ':' | cut -f 2 -d '"') +files="eslint.log prettier.log" +for file in $roots; do + for check in $files; do + f="$file/.format/$check" + if [ -f $f ]; then + if grep -q "error" "$f"; then + if ! grep -q "0 errors" "$f"; then + if ! grep -q "error.ts" "$f"; then + echo "Errors in $f" + cat "$f" + fi + fi + fi + fi + done +done \ No newline at end of file diff --git a/dev/tool/src/clean.ts b/dev/tool/src/clean.ts index 14a8d357b41..40cbfcbe8e5 100644 --- a/dev/tool/src/clean.ts +++ b/dev/tool/src/clean.ts @@ -15,26 +15,21 @@ import { getAccountDB, listWorkspacesRaw } from '@hcengineering/account' import calendar from '@hcengineering/calendar' -import chunter, { type ChatMessage } from '@hcengineering/chunter' import { loadCollaborativeDoc, saveCollaborativeDoc, yDocToBuffer } from '@hcengineering/collaboration' import contact from '@hcengineering/contact' import core, { type ArrOf, type AttachedDoc, type BackupClient, - type Class, - ClassifierKind, type CollaborativeDoc, type Client as CoreClient, DOMAIN_BENCHMARK, DOMAIN_DOC_INDEX_STATE, DOMAIN_MIGRATION, DOMAIN_MODEL, - DOMAIN_STATUS, DOMAIN_TX, type Doc, type DocumentUpdate, - type Domain, type Hierarchy, type Markup, type MeasureContext, @@ -43,36 +38,25 @@ import core, { type RefTo, type RelatedDocument, SortingOrder, - type Status, - type StatusCategory, type Tx, type TxCUD, type TxCollectionCUD, - type TxCreateDoc, - type TxMixin, TxOperations, - TxProcessor, type TxRemoveDoc, - type TxUpdateDoc, type WorkspaceId, generateId, - getObjectValue, getWorkspaceId, systemAccountEmail, - toIdMap, updateAttribute } from '@hcengineering/core' -import activity, { DOMAIN_ACTIVITY } from '@hcengineering/model-activity' +import activity from '@hcengineering/model-activity' import { DOMAIN_SPACE } from '@hcengineering/model-core' -import recruitModel, { defaultApplicantStatuses } from '@hcengineering/model-recruit' import { getMongoClient, getWorkspaceMongoDB } from '@hcengineering/mongo' -import recruit, { type Applicant, type Vacancy } from '@hcengineering/recruit' +import recruit from '@hcengineering/recruit' import { getTransactorEndpoint } from '@hcengineering/server-client' import { type StorageAdapter } from '@hcengineering/server-core' import { generateToken } from '@hcengineering/server-token' import { connect } from '@hcengineering/server-tool' -import tags, { type TagCategory, type TagElement, type TagReference } from '@hcengineering/tags' -import task, { type ProjectType, type Task, type TaskType } from '@hcengineering/task' import { updateYDocContent } from '@hcengineering/text' import tracker from '@hcengineering/tracker' import { deepEqual } from 'fast-equals' @@ -363,699 +347,6 @@ export async function cleanArchivedSpaces (workspaceId: WorkspaceId, transactorU } } -export async function fixCommentDoubleIdCreate (workspaceId: WorkspaceId, transactorUrl: string): Promise { - const connection = (await connect(transactorUrl, workspaceId, undefined, { - mode: 'backup' - })) as unknown as CoreClient & BackupClient - try { - const commentTxes = await connection.findAll(core.class.TxCollectionCUD, { - 'tx._class': core.class.TxCreateDoc, - 'tx.objectClass': chunter.class.ChatMessage - }) - const commentTxesRemoved = await connection.findAll(core.class.TxCollectionCUD, { - 'tx._class': core.class.TxRemoveDoc, - 'tx.objectClass': chunter.class.ChatMessage - }) - const removed = new Map(commentTxesRemoved.map((it) => [it.tx.objectId, it])) - // Do not checked removed - const objSet = new Set>() - const oldValue = new Map, string>() - for (const c of commentTxes) { - const cid = c.tx.objectId - if (removed.has(cid)) { - continue - } - const has = objSet.has(cid) - objSet.add(cid) - if (has) { - // We have found duplicate one, let's rename it. - const doc = TxProcessor.createDoc2Doc(c.tx as unknown as TxCreateDoc) - if (doc.message !== '' && doc.message.trim() !== '

') { - await connection.clean(DOMAIN_TX, [c._id]) - if (oldValue.get(cid) === doc.message.trim()) { - console.log('delete tx', cid, doc.message) - } else { - oldValue.set(doc._id, doc.message) - console.log('renaming', cid, doc.message) - // Remove previous transaction. - c.tx.objectId = generateId() - doc._id = c.tx.objectId as Ref - await connection.upload(DOMAIN_TX, [c]) - // Also we need to create snapsot - await connection.upload(DOMAIN_ACTIVITY, [doc]) - } - } - } - } - } catch (err: any) { - console.trace(err) - } finally { - await connection.close() - } -} - -const DOMAIN_TAGS = 'tags' as Domain -export async function fixSkills ( - mongoUrl: string, - workspaceId: WorkspaceId, - transactorUrl: string, - step: string -): Promise { - const connection = (await connect(transactorUrl, workspaceId, undefined, { - mode: 'backup' - })) as unknown as CoreClient & BackupClient - const client = getMongoClient(mongoUrl) - try { - const _client = await client.getClient() - const db = getWorkspaceMongoDB(_client, workspaceId) - - async function fixCount (): Promise { - console.log('fixing ref-count...') - const allTags = (await connection.findAll(tags.class.TagElement, {})) as TagElement[] - for (const tag of allTags) { - console.log('progress: ', ((allTags.indexOf(tag) + 1) * 100) / allTags.length) - const references = await connection.findAll(tags.class.TagReference, { tag: tag._id }, { total: true }) - if (references.total >= 0) { - await db.collection(DOMAIN_TAGS).updateOne({ _id: tag._id }, { $set: { refCount: references.total } }) - } - } - console.log('DONE: fixing ref-count') - } - - // STEP 1: all to Upper Case - if (step === '1') { - console.log('converting case') - const tagsToClean = (await connection.findAll(tags.class.TagElement, { - category: { - $in: ['recruit:category:Other', 'document:category:Other', 'tracker:category:Other'] as Ref[] - } - })) as TagElement[] - for (const tag of tagsToClean) { - await db - .collection(DOMAIN_TAGS) - .updateOne({ _id: tag._id }, { $set: { title: tag.title.trim().toUpperCase() } }) - } - console.log('DONE: converting case') - } - // STEP 2: Replace with same titles - if (step === '2') { - console.log('fixing titles') - const tagsToClean = (await connection.findAll(tags.class.TagElement, { - category: { - $in: ['recruit:category:Other', 'document:category:Other', 'tracker:category:Other'] as Ref[] - } - })) as TagElement[] - const groupped = groupBy(tagsToClean, 'title') - console.log('STEP2: Done grouping') - for (const key in groupped) { - const values = groupped[key] - if (values.length === 1) continue - // console.log('duplicates: ', values) - const goodTag = values[0] - for (const t of values) { - if (t._id === goodTag._id) continue - const references = await connection.findAll(tags.class.TagReference, { - attachedToClass: recruit.mixin.Candidate, - tag: t._id - }) - goodTag.refCount = (goodTag.refCount ?? 0) + references.length - for (const reference of references) { - await db - .collection(DOMAIN_TAGS) - .updateOne( - { _id: reference._id }, - { $set: { tag: goodTag._id, color: goodTag.color, title: goodTag.title } } - ) - } - await db.collection(DOMAIN_TAGS).deleteOne({ _id: t._id }) - } - await db.collection(DOMAIN_TAGS).updateOne({ _id: goodTag._id }, { $set: { refCount: goodTag.refCount } }) - } - console.log('STEP2 DONE') - } - // fix skills with + and - - if (step === '3') { - console.log('STEP 3') - const ops = new TxOperations(connection, core.account.System) - const regex = /\S+(?:[-+]\S+)+/g - const tagsToClean = (await connection.findAll(tags.class.TagElement, { - category: { - $in: ['recruit:category:Other', 'document:category:Other', 'tracker:category:Other'] as Ref[] - } - })) as TagElement[] - const tagsMatchingRegex = tagsToClean.filter((tag) => regex.test(tag.title)) - let goodTags = (await connection.findAll(tags.class.TagElement, { - category: { - $nin: ['recruit:category:Other', 'document:category:Other', 'tracker:category:Other'] as Ref[] - } - })) as TagElement[] - goodTags = goodTags.sort((a, b) => b.title.length - a.title.length).filter((t) => t.title.length > 2) - for (const wrongTag of tagsMatchingRegex) { - const incorrectStrings = wrongTag.title.match(regex) - if (incorrectStrings == null) continue - for (const str of incorrectStrings) { - const goodTag = goodTags.find((t) => t.title.toUpperCase() === str.replaceAll(/[+-]/g, '')) - if (goodTag === undefined) continue - const references = (await connection.findAll(tags.class.TagReference, { - attachedToClass: recruit.mixin.Candidate, - tag: wrongTag._id - })) as TagReference[] - for (const ref of references) { - await ops.addCollection( - tags.class.TagReference, - ref.space, - ref.attachedTo, - ref.attachedToClass, - ref.collection, - { - title: goodTag.title, - tag: goodTag._id, - color: ref.color - } - ) - await db - .collection(DOMAIN_TAGS) - .updateOne({ _id: ref._id }, { $set: { title: ref.title.replace(str, '') } }) - } - await db - .collection(DOMAIN_TAGS) - .updateOne({ _id: wrongTag._id }, { $set: { title: wrongTag.title.replace(str, goodTag.title) } }) - } - } - console.log('DONE: STEP 3') - } - // change incorrect skills and add good one - if (step === '4') { - console.log('step 4') - let goodTags = (await connection.findAll(tags.class.TagElement, { - category: { - $nin: ['recruit:category:Other', 'document:category:Other', 'tracker:category:Other'] as Ref[] - } - })) as TagElement[] - goodTags = goodTags.sort((a, b) => b.title.length - a.title.length).filter((t) => t.title.length > 2) - const ops = new TxOperations(connection, core.account.System) - const tagsToClean = (await connection.findAll(tags.class.TagElement, { - category: { - $in: ['recruit:category:Other', 'document:category:Other', 'tracker:category:Other'] as Ref[] - } - })) as TagElement[] - for (const incorrectTag of tagsToClean) { - console.log('tag progress: ', ((tagsToClean.indexOf(incorrectTag) + 1) * 100) / tagsToClean.length) - const toReplace = goodTags.filter((t) => incorrectTag.title.includes(t.title.toUpperCase())) - if (toReplace.length === 0) continue - const references = (await connection.findAll(tags.class.TagReference, { - attachedToClass: recruit.mixin.Candidate, - tag: incorrectTag._id - })) as TagReference[] - let title = incorrectTag.title - for (const ref of references) { - const refsForCand = ( - (await connection.findAll(tags.class.TagReference, { - attachedToClass: recruit.mixin.Candidate, - attachedTo: ref.attachedTo - })) as TagReference[] - ).map((r) => r.tag) - for (const gTag of toReplace) { - title = title.replace(gTag.title.toUpperCase(), '') - if ((refsForCand ?? []).includes(gTag._id)) continue - await ops.addCollection( - tags.class.TagReference, - ref.space, - ref.attachedTo, - ref.attachedToClass, - ref.collection, - { - title: gTag.title, - tag: gTag._id, - color: ref.color - } - ) - } - await db.collection(DOMAIN_TAGS).updateOne({ _id: ref._id }, { $set: { title } }) - } - await db.collection(DOMAIN_TAGS).updateOne({ _id: incorrectTag._id }, { $set: { title } }) - } - console.log('STEP4 DONE') - } - - // remove skills with space or empty string - if (step === '5') { - console.log('STEP 5') - const tagsToClean = (await connection.findAll(tags.class.TagElement, { - category: { - $in: ['recruit:category:Other', 'document:category:Other', 'tracker:category:Other'] as Ref[] - }, - title: { $in: [' ', ''] } - })) as TagElement[] - if (tagsToClean.length > 0) { - for (const t of tagsToClean) { - const references = (await connection.findAll(tags.class.TagReference, { - attachedToClass: recruit.mixin.Candidate, - tag: t._id - })) as TagReference[] - const ids = references.map((r) => r._id) - await db.collection(DOMAIN_TAGS).deleteMany({ _id: { $in: ids } }) - await db.collection(DOMAIN_TAGS).deleteOne({ _id: t._id }) - } - } - await fixCount() - console.log('DONE 5 STEP') - } - // remove skills with ref count less or equal to 10 - if (step === '6') { - console.log('STEP 6') - const tagsToClean = (await connection.findAll(tags.class.TagElement, { - category: { - $in: ['recruit:category:Other', 'document:category:Other', 'tracker:category:Other'] as Ref[] - } - })) as TagElement[] - for (const t of tagsToClean) { - if ((t?.refCount ?? 0) >= 10) continue - const references = (await connection.findAll(tags.class.TagReference, { - attachedToClass: recruit.mixin.Candidate, - tag: t._id - })) as TagReference[] - const ids = references.map((r) => r._id) - await db.collection(DOMAIN_TAGS).deleteMany({ _id: { $in: ids } }) - await db.collection(DOMAIN_TAGS).deleteOne({ _id: t._id }) - } - console.log('DONE 6 STEP') - } - // remove all skills that don't have letters in it - if (step === '7') { - console.log('STEP 7') - const tagsToClean = (await connection.findAll(tags.class.TagElement, { - category: { - $in: ['recruit:category:Other', 'document:category:Other', 'tracker:category:Other'] as Ref[] - } - })) as TagElement[] - const regex = /^((?![a-zA-Zа-яА-Я]).)*$/g - if (tagsToClean.length > 0) { - for (const t of tagsToClean) { - if (!regex.test(t.title)) continue - const references = (await connection.findAll(tags.class.TagReference, { - attachedToClass: recruit.mixin.Candidate, - tag: t._id - })) as TagReference[] - const ids = references.map((r) => r._id) - await db.collection(DOMAIN_TAGS).deleteMany({ _id: { $in: ids } }) - await db.collection(DOMAIN_TAGS).deleteOne({ _id: t._id }) - } - } - await fixCount() - console.log('DONE 7 STEP') - } - } catch (err: any) { - console.trace(err) - } finally { - client.close() - await connection.close() - } -} - -function groupBy (docs: T[], key: string): Record { - return docs.reduce((storage: Record, item: T) => { - const group = getObjectValue(key, item) ?? undefined - - storage[group] = storage[group] ?? [] - storage[group].push(item) - - return storage - }, {}) -} - -export async function restoreRecruitingTaskTypes ( - mongoUrl: string, - workspaceId: WorkspaceId, - transactorUrl: string -): Promise { - const connection = (await connect(transactorUrl, workspaceId, undefined, { - mode: 'backup', - model: 'upgrade' - })) as unknown as CoreClient & BackupClient - const client = getMongoClient(mongoUrl) - try { - const _client = await client.getClient() - const db = getWorkspaceMongoDB(_client, workspaceId) - - // Query all vacancy project types creations (in Model) - // We only update new project types in model here and not old ones in spaces - const vacancyTypes = await connection.findAll>(core.class.TxCreateDoc, { - objectClass: task.class.ProjectType, - objectSpace: core.space.Model, - 'attributes.descriptor': recruit.descriptors.VacancyType - }) - - console.log('Found ', vacancyTypes.length, ' vacancy types to check') - - if (vacancyTypes.length === 0) { - return - } - - const descr = connection.getModel().getObject(recruit.descriptors.VacancyType) - const knownCategories = [ - task.statusCategory.UnStarted, - task.statusCategory.Active, - task.statusCategory.Won, - task.statusCategory.Lost - ] - - function compareCategories (a: Ref, b: Ref): number { - const indexOfA = knownCategories.indexOf(a) - const indexOfB = knownCategories.indexOf(b) - - return indexOfA - indexOfB - } - - for (const vacType of vacancyTypes) { - for (const taskTypeId of vacType.attributes.tasks) { - // Check if task type create TX exists - const createTx = ( - await connection.findAll>(core.class.TxCreateDoc, { - objectClass: task.class.TaskType, - objectSpace: core.space.Model, - objectId: taskTypeId - }) - )[0] - - console.log('####################################') - console.log('Checking vacancy type: ', vacType.attributes.name) - - if (createTx !== undefined) { - console.log('Task type already exists in model') - continue - } - - console.log('Restoring task type: ', taskTypeId, ' in vacancy type: ', vacType.attributes.name) - - // Restore create task type tx - - // Get target class mixin - - const typeMixin = ( - await connection.findAll>(core.class.TxMixin, { - mixin: task.mixin.TaskTypeClass, - 'attributes.projectType': vacType.objectId, - 'attributes.taskType': taskTypeId - }) - )[0] - - if (typeMixin === undefined) { - console.error(new Error('No type mixin found for the task type being restored')) - continue - } - - // Get statuses and categories - const statusesIds = vacType.attributes.statuses.filter((s) => s.taskType === taskTypeId).map((s) => s._id) - if (statusesIds.length === 0) { - console.error(new Error('No statuses defined for the task type being restored')) - continue - } - - const statuses = await connection.findAll(core.class.Status, { - _id: { $in: statusesIds } - }) - const categoriesIds = new Set>() - - statuses.forEach((st) => { - if (st.category !== undefined) { - categoriesIds.add(st.category) - } - }) - - if (categoriesIds.size === 0) { - console.error(new Error('No categories found for the task type being restored')) - continue - } - - const statusCategories = Array.from(categoriesIds) - - statusCategories.sort(compareCategories) - - const createTxNew: TxCreateDoc = { - _id: generateId(), - _class: core.class.TxCreateDoc, - space: core.space.Tx, - objectId: taskTypeId, - objectClass: task.class.TaskType, - objectSpace: core.space.Model, - modifiedBy: core.account.ConfigUser, // So it's not removed during the next migration - modifiedOn: vacType.modifiedOn, - createdOn: vacType.createdOn, - attributes: { - name: 'Applicant', - descriptor: recruitModel.descriptors.Application, - ofClass: recruit.class.Applicant, - targetClass: typeMixin.objectId, - statusClass: core.class.Status, - allowedAsChildOf: [taskTypeId], - statuses: statusesIds, - statusCategories, - parent: vacType.objectId, - kind: 'both', - icon: descr.icon - } - } - - await db.collection(DOMAIN_TX).insertOne(createTxNew) - console.log('Successfully created new task type: ') - console.log(createTxNew) - - // If there were updates to the task type - move them to the model - const updateTxes = ( - await connection.findAll(core.class.TxCUD, { - objectClass: task.class.TaskType, - objectSpace: vacType.objectId as any, - objectId: taskTypeId - }) - ).filter((tx) => [core.class.TxUpdateDoc, core.class.TxRemoveDoc].includes(tx._class)) - - for (const updTx of updateTxes) { - await db.collection>(DOMAIN_TX).insertOne({ - ...updTx, - _id: generateId(), - objectSpace: core.space.Model - }) - } - console.log('Successfully restored ', updateTxes.length, ' CUD transactions') - } - } - } catch (err: any) { - console.trace(err) - } finally { - client.close() - await connection.close() - } -} - -export async function restoreHrTaskTypesFromUpdates ( - mongoUrl: string, - workspaceId: WorkspaceId, - transactorUrl: string -): Promise { - const connection = (await connect(transactorUrl, workspaceId, undefined, { - mode: 'backup', - model: 'upgrade' - })) as unknown as CoreClient & BackupClient - const client = getMongoClient(mongoUrl) - try { - const _client = await client.getClient() - const db = getWorkspaceMongoDB(_client, workspaceId) - const hierarchy = connection.getHierarchy() - const descr = connection.getModel().getObject(recruit.descriptors.VacancyType) - const knownCategories = [ - task.statusCategory.UnStarted, - task.statusCategory.Active, - task.statusCategory.Won, - task.statusCategory.Lost - ] - - const supersededStatusesCursor = db.collection(DOMAIN_STATUS).find({ __superseded: true }) - const supersededStatusesById = toIdMap(await supersededStatusesCursor.toArray()) - - // Query all vacancies - const vacancies = await connection.findAll(recruit.class.Vacancy, {}) - - for (const vacancy of vacancies) { - console.log('Checking vacancy: ', vacancy.name) - // Find if task type exists - const projectType = await connection.findOne(task.class.ProjectType, { - _id: vacancy.type - }) - - if (projectType !== undefined) { - console.log('Found project type for vacancy: ', vacancy.name) - continue - } - - console.log('Restoring project type for: ', vacancy.name) - - const projectTypeId = generateId() - - // Find task type from any task - const applicant = await connection.findOne(recruit.class.Applicant, { - space: vacancy._id - }) - - if (applicant === undefined) { - // there are no tasks, just make it of the default system type - console.log('No tasks found for the vacancy: ', vacancy.name) - console.log('Changing vacancy to default type') - await db - .collection(DOMAIN_SPACE) - .updateOne({ _id: vacancy._id }, { $set: { type: recruitModel.template.DefaultVacancy } }) - continue - } - - const taskTypeId = applicant.kind - - // Check if there's a create transaction for the task type - let createTaskTypeTx = await connection.findOne>(core.class.TxCreateDoc, { - objectId: taskTypeId - }) - - if (createTaskTypeTx === undefined) { - // Restore it based on the update transaction - const updateTaskTypeTx = await connection.findOne>(core.class.TxUpdateDoc, { - objectId: taskTypeId, - objectClass: task.class.TaskType, - 'operations.statuses': { $exists: true } - }) - - if (updateTaskTypeTx === undefined) { - console.error(new Error('No task type found for the vacancy ' + vacancy.name)) - continue - } - - const statuses = - updateTaskTypeTx.operations.statuses?.map((s) => { - const ssedStatus = supersededStatusesById.get(s) - if (ssedStatus === undefined) { - return s - } else { - const defStatus = defaultApplicantStatuses.find((st) => st.name === ssedStatus.name) - - if (defStatus === undefined) { - console.error(new Error('No default status found for the superseded status ' + ssedStatus.name)) - return s - } - - return defStatus.id - } - }) ?? [] - - const taskTargetClassId = `${taskTypeId}:type:mixin` as Ref> - const ofClassClass = hierarchy.getClass(recruit.class.Applicant) - - await db.collection>(DOMAIN_TX).insertOne({ - _id: generateId(), - _class: core.class.TxCreateDoc, - space: core.space.Tx, - objectId: taskTargetClassId, - objectClass: core.class.Mixin, - objectSpace: core.space.Model, - modifiedBy: core.account.ConfigUser, - modifiedOn: Date.now(), - createdBy: core.account.ConfigUser, - createdOn: Date.now(), - attributes: { - extends: recruit.class.Applicant, - kind: ClassifierKind.MIXIN, - label: ofClassClass.label, - icon: ofClassClass.icon - } - }) - - createTaskTypeTx = { - _id: generateId(), - _class: core.class.TxCreateDoc, - space: core.space.Tx, - objectId: taskTypeId, - objectClass: task.class.TaskType, - objectSpace: core.space.Model, - modifiedBy: core.account.ConfigUser, - modifiedOn: Date.now(), - createdBy: core.account.ConfigUser, - createdOn: Date.now(), - attributes: { - name: 'Applicant', - descriptor: recruitModel.descriptors.Application, - ofClass: recruit.class.Applicant, - targetClass: taskTargetClassId, - statusClass: core.class.Status, - allowedAsChildOf: [taskTypeId], - statuses, - statusCategories: knownCategories, - parent: projectTypeId, - kind: 'task', - icon: descr.icon - } - } - - await db.collection>(DOMAIN_TX).insertOne(createTaskTypeTx) - console.log('Restored task type id: ', taskTypeId) - } - - // Restore the project type - const targetClassId = `${projectTypeId}:type:mixin` as Ref> - const ofClassClass = hierarchy.getClass(recruit.class.Vacancy) - - await db.collection>(DOMAIN_TX).insertOne({ - _id: generateId(), - _class: core.class.TxCreateDoc, - space: core.space.Tx, - objectId: targetClassId, - objectClass: core.class.Mixin, - objectSpace: core.space.Model, - modifiedBy: core.account.ConfigUser, - modifiedOn: Date.now(), - createdBy: core.account.ConfigUser, - createdOn: Date.now(), - attributes: { - extends: recruit.class.Vacancy, - kind: ClassifierKind.MIXIN, - label: ofClassClass.label, - icon: ofClassClass.icon - } - }) - - const createProjectTypeTx: TxCreateDoc = { - _id: generateId(), - _class: core.class.TxCreateDoc, - space: core.space.Tx, - objectId: projectTypeId, - objectClass: task.class.ProjectType, - objectSpace: core.space.Model, - modifiedBy: core.account.ConfigUser, - modifiedOn: Date.now(), - createdBy: core.account.ConfigUser, - createdOn: Date.now(), - attributes: { - descriptor: recruit.descriptors.VacancyType, - tasks: [taskTypeId], - classic: true, - statuses: createTaskTypeTx.attributes.statuses.map((s) => ({ - _id: s, - taskType: taskTypeId - })), - targetClass: targetClassId, - name: vacancy.name, - description: '', - roles: 0 - } - } - - await db.collection>(DOMAIN_TX).insertOne(createProjectTypeTx) - console.log('Restored project type id: ', projectTypeId) - } - } catch (err: any) { - console.trace(err) - } finally { - client.close() - await connection.close() - } -} - export async function removeDuplicateIds ( ctx: MeasureContext, mongodbUri: string, diff --git a/dev/tool/src/elastic.ts b/dev/tool/src/elastic.ts deleted file mode 100644 index acd32175797..00000000000 --- a/dev/tool/src/elastic.ts +++ /dev/null @@ -1,71 +0,0 @@ -// -// Copyright © 2022 Hardcore Engineering Inc. -// -// Licensed under the Eclipse Public License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. You may -// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// -// See the License for the specific language governing permissions and -// limitations under the License. -// - -import { Client as ElasticClient } from '@elastic/elasticsearch' -import core, { DOMAIN_DOC_INDEX_STATE, toWorkspaceString, type WorkspaceId } from '@hcengineering/core' -import { getMongoClient, getWorkspaceMongoDB } from '@hcengineering/mongo' -import { type StorageAdapter } from '@hcengineering/server-core' - -export async function rebuildElastic ( - mongoUrl: string, - workspaceId: WorkspaceId, - storageAdapter: StorageAdapter, - elasticUrl: string -): Promise { - const client = getMongoClient(mongoUrl) - try { - const _client = await client.getClient() - const db = getWorkspaceMongoDB(_client, workspaceId) - await db - .collection(DOMAIN_DOC_INDEX_STATE) - .updateMany({ _class: core.class.DocIndexState }, { $set: { elastic: false } }) - } finally { - client.close() - } - - await dropElastic(elasticUrl, workspaceId) -} - -async function dropElastic (elasticUrl: string, workspaceId: WorkspaceId): Promise { - console.log('drop existing elastic docment') - const client = new ElasticClient({ - node: elasticUrl - }) - const productWs = toWorkspaceString(workspaceId) - await new Promise((resolve, reject) => { - client.indices.exists( - { - index: productWs - }, - (err: any, result: any) => { - if (err != null) reject(err) - if (result.body === true) { - client.indices.delete( - { - index: productWs - }, - (err: any, result: any) => { - if (err != null) reject(err) - resolve(result) - } - ) - } else { - resolve(result) - } - } - ) - }) - await client.close() -} diff --git a/dev/tool/src/index.ts b/dev/tool/src/index.ts index 762eb4ee9ed..158496ff623 100644 --- a/dev/tool/src/index.ts +++ b/dev/tool/src/index.ts @@ -56,7 +56,6 @@ import serverClientPlugin, { listAccountWorkspaces, updateBackupInfo } from '@hcengineering/server-client' -import { getServerPipeline, registerServerPlugins, registerStringLoaders } from '@hcengineering/server-pipeline' import serverToken, { decodeToken, generateToken } from '@hcengineering/server-token' import toolPlugin, { FileModelLogger } from '@hcengineering/server-tool' import { createWorkspace, upgradeWorkspace } from '@hcengineering/workspace-service' @@ -67,7 +66,7 @@ import { program, type Command } from 'commander' import { clearTelegramHistory } from './telegram' import { diffWorkspace, recreateElastic, updateField } from './workspace' -import core, { +import { AccountRole, generateId, getWorkspaceId, @@ -81,11 +80,9 @@ import core, { type Ref, type Tx, type Version, - type WorkspaceId, - type WorkspaceIdWithUrl + type WorkspaceId } from '@hcengineering/core' import { consoleModelLogger, type MigrateOperation } from '@hcengineering/model' -import contact from '@hcengineering/model-contact' import { getMongoClient, getWorkspaceMongoDB, shutdown } from '@hcengineering/mongo' import { backupDownload } from '@hcengineering/server-backup/src/backup' @@ -105,20 +102,14 @@ import { cleanArchivedSpaces, cleanRemovedTransactions, cleanWorkspace, - fixCommentDoubleIdCreate, fixMinioBW, - fixSkills, optimizeModel, - removeDuplicateIds, - restoreHrTaskTypesFromUpdates, - restoreRecruitingTaskTypes + removeDuplicateIds } from './clean' import { changeConfiguration } from './configuration' import { moveAccountDbFromMongoToPG, moveFromMongoToPG, moveWorkspaceFromMongoToPG } from './db' -import { fixJsonMarkup, migrateMarkup, restoreLostMarkup } from './markup' -import { fixMixinForeignAttributes, showMixinForeignAttributes } from './mixin' import { fixAccountEmails, renameAccount } from './renameAccount' -import { moveFiles, showLostFiles } from './storage' +import { moveFiles } from './storage' const colorConstants = { colorRed: '\u001b[31m', @@ -267,12 +258,11 @@ export function devTool ( .description('compact all db collections') .option('-w, --workspace ', 'A selected "workspace" only', '') .action(async (cmd: { workspace: string }) => { - const { dbUrl } = prepareTools() const mongodbUri = getMongoDBUrl() - await withDatabase(dbUrl, async (db) => { + await withDatabase(mongodbUri, async (db) => { console.log('compacting db ...') let gtotal: number = 0 - const client = getMongoClient(mongodbUri ?? dbUrl) + const client = getMongoClient(mongodbUri) const _client = await client.getClient() try { const workspaces = await listWorkspacesPure(db) @@ -509,13 +499,10 @@ export function devTool ( }) program - .command('list-unused-workspaces-mongo') - .description( - 'remove unused workspaces, please pass --remove to really delete them. Without it will only mark them disabled' - ) - .option('-r|--remove [remove]', 'Force remove', false) + .command('list-unused-workspaces') + .description('list unused workspaces') .option('-t|--timeout [timeout]', 'Timeout in days', '7') - .action(async (cmd: { remove: boolean, disable: boolean, exclude: string, timeout: string }) => { + .action(async (cmd: { exclude: string, timeout: string }) => { const { dbUrl } = prepareTools() await withDatabase(dbUrl, async (db) => { const workspaces = new Map((await listWorkspacesPure(db)).map((p) => [p._id.toString(), p])) @@ -526,48 +513,38 @@ export function devTool ( await withStorage(async (adapter) => { // We need to update workspaces with missing workspaceUrl - const mongodbUri = getMongoDBUrl() - const client = getMongoClient(mongodbUri ?? dbUrl) - const _client = await client.getClient() - try { - for (const a of accounts) { - const authored = a.workspaces - .map((it) => workspaces.get(it.toString())) - .filter((it) => it !== undefined && it.createdBy?.trim() === a.email?.trim()) as Workspace[] - authored.sort((a, b) => b.lastVisit - a.lastVisit) - if (authored.length > 0) { - const lastLoginDays = Math.floor((Date.now() - a.lastVisit) / 1000 / 3600 / 24) - toolCtx.info(a.email, { - workspaces: a.workspaces.length, - firstName: a.first, - lastName: a.last, - lastLoginDays - }) - for (const ws of authored) { - const lastVisitDays = Math.floor((Date.now() - ws.lastVisit) / 1000 / 3600 / 24) - - if (lastVisitDays > _timeout) { - toolCtx.warn(' --- unused', { - url: ws.workspaceUrl, - id: ws.workspace, - lastVisitDays - }) - if (cmd.remove) { - await dropWorkspaceFull(toolCtx, db, _client, null, ws.workspace, adapter) - } - } else { - toolCtx.warn(' +++ used', { - url: ws.workspaceUrl, - id: ws.workspace, - createdBy: ws.createdBy, - lastVisitDays - }) - } + for (const a of accounts) { + const authored = a.workspaces + .map((it) => workspaces.get(it.toString())) + .filter((it) => it !== undefined && it.createdBy?.trim() === a.email?.trim()) as Workspace[] + authored.sort((a, b) => b.lastVisit - a.lastVisit) + if (authored.length > 0) { + const lastLoginDays = Math.floor((Date.now() - a.lastVisit) / 1000 / 3600 / 24) + toolCtx.info(a.email, { + workspaces: a.workspaces.length, + firstName: a.first, + lastName: a.last, + lastLoginDays + }) + for (const ws of authored) { + const lastVisitDays = Math.floor((Date.now() - ws.lastVisit) / 1000 / 3600 / 24) + + if (lastVisitDays > _timeout) { + toolCtx.warn(' --- unused', { + url: ws.workspaceUrl, + id: ws.workspace, + lastVisitDays + }) + } else { + toolCtx.warn(' +++ used', { + url: ws.workspaceUrl, + id: ws.workspace, + createdBy: ws.createdBy, + lastVisitDays + }) } } } - } finally { - client.close() } }) }) @@ -711,32 +688,6 @@ export function devTool ( }) }) - program.command('fix-person-accounts-mongo').action(async () => { - const { dbUrl, version } = prepareTools() - const mongodbUri = getMongoDBUrl() - await withDatabase(dbUrl, async (db) => { - const ws = await listWorkspacesPure(db) - const client = getMongoClient(mongodbUri) - const _client = await client.getClient() - try { - for (const w of ws) { - const wsDb = getWorkspaceMongoDB(_client, { name: w.workspace }) - await wsDb.collection('tx').updateMany( - { - objectClass: contact.class.PersonAccount, - objectSpace: null - }, - { $set: { objectSpace: core.space.Model } } - ) - } - } finally { - client.close() - } - - console.log('latest model version:', JSON.stringify(version)) - }) - }) - program .command('show-accounts') .description('Show accounts') @@ -1234,86 +1185,6 @@ export function devTool ( } ) - program - .command('show-lost-files-mongo') - .option('-w, --workspace ', 'Selected workspace only', '') - .option('--disabled', 'Include disabled workspaces', false) - .option('--all', 'Show all files', false) - .action(async (cmd: { workspace: string, disabled: boolean, all: boolean }) => { - const { dbUrl } = prepareTools() - await withDatabase(dbUrl, async (db) => { - await withStorage(async (adapter) => { - const mongodbUri = getMongoDBUrl() - const client = getMongoClient(mongodbUri) - const _client = await client.getClient() - try { - let index = 1 - const workspaces = await listWorkspacesPure(db) - workspaces.sort((a, b) => b.lastVisit - a.lastVisit) - - for (const workspace of workspaces) { - if (workspace.disabled === true && !cmd.disabled) { - console.log('ignore disabled workspace', workspace.workspace) - continue - } - - if (cmd.workspace !== '' && workspace.workspace !== cmd.workspace) { - continue - } - - try { - console.log('start', workspace.workspace, index, '/', workspaces.length) - const workspaceId = getWorkspaceId(workspace.workspace) - const wsDb = getWorkspaceMongoDB(_client, { name: workspace.workspace }) - await showLostFiles(toolCtx, workspaceId, wsDb, adapter, { showAll: cmd.all }) - console.log('done', workspace.workspace) - } catch (err) { - console.error(err) - } - - index += 1 - } - } catch (err: any) { - console.error(err) - } finally { - client.close() - } - }) - }) - }) - - program.command('show-lost-markup ').action(async (workspace: string, cmd: any) => { - const { dbUrl } = prepareTools() - await withDatabase(dbUrl, async (db) => { - await withStorage(async (adapter) => { - try { - const workspaceId = getWorkspaceId(workspace) - const token = generateToken(systemAccountEmail, workspaceId) - const endpoint = await getTransactorEndpoint(token) - await restoreLostMarkup(toolCtx, workspaceId, endpoint, adapter, { command: 'show' }) - } catch (err: any) { - console.error(err) - } - }) - }) - }) - - program.command('restore-lost-markup ').action(async (workspace: string, cmd: any) => { - const { dbUrl } = prepareTools() - await withDatabase(dbUrl, async (db) => { - await withStorage(async (adapter) => { - try { - const workspaceId = getWorkspaceId(workspace) - const token = generateToken(systemAccountEmail, workspaceId) - const endpoint = await getTransactorEndpoint(token) - await restoreLostMarkup(toolCtx, workspaceId, endpoint, adapter, { command: 'restore' }) - } catch (err: any) { - console.error(err) - } - }) - }) - }) - program.command('fix-bw-workspace ').action(async (workspace: string) => { await withStorage(async (adapter) => { await fixMinioBW(toolCtx, getWorkspaceId(workspace), adapter) @@ -1340,42 +1211,6 @@ export function devTool ( await cleanArchivedSpaces(wsid, endpoint) }) - program - .command('chunter-fix-comments ') - .description('chunter-fix-comments') - .action(async (workspace: string, cmd: any) => { - const wsid = getWorkspaceId(workspace) - const token = generateToken(systemAccountEmail, wsid) - const endpoint = await getTransactorEndpoint(token) - await fixCommentDoubleIdCreate(wsid, endpoint) - }) - - program - .command('mixin-show-foreign-attributes ') - .description('mixin-show-foreign-attributes') - .option('--mixin ', 'Mixin class', '') - .option('--property ', 'Property name', '') - .option('--detail ', 'Show details', false) - .action(async (workspace: string, cmd: { detail: boolean, mixin: string, property: string }) => { - const wsid = getWorkspaceId(workspace) - const token = generateToken(systemAccountEmail, wsid) - const endpoint = await getTransactorEndpoint(token) - await showMixinForeignAttributes(wsid, endpoint, cmd) - }) - - program - .command('mixin-fix-foreign-attributes-mongo ') - .description('mixin-fix-foreign-attributes') - .option('--mixin ', 'Mixin class', '') - .option('--property ', 'Property name', '') - .action(async (workspace: string, cmd: { mixin: string, property: string }) => { - const mongodbUri = getMongoDBUrl() - const wsid = getWorkspaceId(workspace) - const token = generateToken(systemAccountEmail, wsid) - const endpoint = await getTransactorEndpoint(token) - await fixMixinForeignAttributes(mongodbUri, wsid, endpoint, cmd) - }) - program .command('configure ') .description('clean archived spaces') @@ -1504,39 +1339,6 @@ export function devTool ( await stressBenchmark(transactor, cmd.mode) }) - program - .command('fix-skills-mongo ') - .description('fix skills for workspace') - .action(async (workspace: string, step: string) => { - const mongodbUri = getMongoDBUrl() - const wsid = getWorkspaceId(workspace) - const token = generateToken(systemAccountEmail, wsid) - const endpoint = await getTransactorEndpoint(token) - await fixSkills(mongodbUri, wsid, endpoint, step) - }) - - program - .command('restore-ats-types-mongo ') - .description('Restore recruiting task types for workspace') - .action(async (workspace: string) => { - const mongodbUri = getMongoDBUrl() - console.log('Restoring recruiting task types in workspace ', workspace, '...') - const wsid = getWorkspaceId(workspace) - const endpoint = await getTransactorEndpoint(generateToken(systemAccountEmail, wsid), 'external') - await restoreRecruitingTaskTypes(mongodbUri, wsid, endpoint) - }) - - program - .command('restore-ats-types-2-mongo ') - .description('Restore recruiting task types for workspace 2') - .action(async (workspace: string) => { - const mongodbUri = getMongoDBUrl() - console.log('Restoring recruiting task types in workspace ', workspace, '...') - const wsid = getWorkspaceId(workspace) - const endpoint = await getTransactorEndpoint(generateToken(systemAccountEmail, wsid), 'external') - await restoreHrTaskTypesFromUpdates(mongodbUri, wsid, endpoint) - }) - program .command('change-field ') .description('change field value for the object') @@ -1557,86 +1359,12 @@ export function devTool ( ) program - .command('recreate-elastic-indexes-mongo ') + .command('recreate-elastic-indexes ') .description('reindex workspace to elastic') .action(async (workspace: string) => { - const mongodbUri = getMongoDBUrl() - const wsid = getWorkspaceId(workspace) - await recreateElastic(mongodbUri, wsid) - }) - - program - .command('recreate-all-elastic-indexes-mongo') - .description('reindex elastic') - .action(async () => { - const { dbUrl } = prepareTools() - const mongodbUri = getMongoDBUrl() - - await withDatabase(dbUrl, async (db) => { - const workspaces = await listWorkspacesRaw(db) - workspaces.sort((a, b) => b.lastVisit - a.lastVisit) - for (const workspace of workspaces) { - const wsid = getWorkspaceId(workspace.workspace) - await recreateElastic(mongodbUri ?? dbUrl, wsid) - } - }) - }) - - program - .command('fix-json-markup-mongo ') - .description('fixes double converted json markup') - .action(async (workspace: string) => { - const mongodbUri = getMongoDBUrl() - await withStorage(async (adapter) => { - const wsid = getWorkspaceId(workspace) - const endpoint = await getTransactorEndpoint(generateToken(systemAccountEmail, wsid), 'external') - await fixJsonMarkup(toolCtx, mongodbUri, adapter, wsid, endpoint) - }) - }) - - program - .command('migrate-markup-mongo') - .description('migrates collaborative markup to storage') - .option('-w, --workspace ', 'Selected workspace only', '') - .option('-c, --concurrency ', 'Number of documents being processed concurrently', '10') - .action(async (cmd: { workspace: string, concurrency: string }) => { const { dbUrl, txes } = prepareTools() - const mongodbUri = getMongoDBUrl() - await withDatabase(dbUrl, async (db) => { - await withStorage(async (adapter) => { - const workspaces = await listWorkspacesPure(db) - const client = getMongoClient(mongodbUri) - const _client = await client.getClient() - let index = 0 - try { - for (const workspace of workspaces) { - if (cmd.workspace !== '' && workspace.workspace !== cmd.workspace) { - continue - } - - const wsId = getWorkspaceId(workspace.workspace) - console.log('processing workspace', workspace.workspace, index, workspaces.length) - const wsUrl: WorkspaceIdWithUrl = { - name: workspace.workspace, - workspaceName: workspace.workspaceName ?? '', - workspaceUrl: workspace.workspaceUrl ?? '' - } - - registerServerPlugins() - registerStringLoaders() - - const { pipeline } = await getServerPipeline(toolCtx, txes, dbUrl, wsUrl) - - await migrateMarkup(toolCtx, adapter, wsId, _client, pipeline, parseInt(cmd.concurrency)) - - console.log('...done', workspace.workspace) - index++ - } - } finally { - client.close() - } - }) - }) + const wsid = getWorkspaceId(workspace) + await recreateElastic(dbUrl, wsid, txes) }) program diff --git a/dev/tool/src/markup.ts b/dev/tool/src/markup.ts deleted file mode 100644 index 8c2f7d154ae..00000000000 --- a/dev/tool/src/markup.ts +++ /dev/null @@ -1,336 +0,0 @@ -import { saveCollaborativeDoc } from '@hcengineering/collaboration' -import core, { - type AnyAttribute, - type Class, - type Client as CoreClient, - type CollaborativeDoc, - type Doc, - type DocIndexState, - type Domain, - type Hierarchy, - type Markup, - type MeasureContext, - type Ref, - type TxMixin, - type TxCreateDoc, - type TxUpdateDoc, - type WorkspaceId, - RateLimiter, - collaborativeDocParse, - makeCollaborativeDoc, - SortingOrder, - TxProcessor -} from '@hcengineering/core' -import { getMongoClient, getWorkspaceMongoDB } from '@hcengineering/mongo' -import { type Pipeline, type StorageAdapter } from '@hcengineering/server-core' -import { connect } from '@hcengineering/server-tool' -import { isEmptyMarkup, jsonToText, markupToYDoc } from '@hcengineering/text' -import { type Db, type FindCursor, type MongoClient } from 'mongodb' - -export async function fixJsonMarkup ( - ctx: MeasureContext, - mongoUrl: string, - storageAdapter: StorageAdapter, - workspaceId: WorkspaceId, - transactorUrl: string -): Promise { - const connection = (await connect(transactorUrl, workspaceId, undefined, { - mode: 'backup' - })) as unknown as CoreClient - const hierarchy = connection.getHierarchy() - - const client = getMongoClient(mongoUrl) - const _client = await client.getClient() - const db = getWorkspaceMongoDB(_client, workspaceId) - - try { - const classes = hierarchy.getDescendants(core.class.Doc) - for (const _class of classes) { - const domain = hierarchy.findDomain(_class) - if (domain === undefined) continue - - const attributes = hierarchy.getAllAttributes(_class) - const filtered = Array.from(attributes.values()).filter((attribute) => { - return hierarchy.isDerived(attribute.type._class, core.class.TypeMarkup) - }) - if (filtered.length === 0) continue - - await processFixJsonMarkupFor(ctx, domain, _class, filtered, workspaceId, db, storageAdapter) - } - } finally { - client.close() - await connection.close() - } -} - -async function processFixJsonMarkupFor ( - ctx: MeasureContext, - domain: Domain, - _class: Ref>, - attributes: AnyAttribute[], - workspaceId: WorkspaceId, - db: Db, - storageAdapter: StorageAdapter -): Promise { - const collection = db.collection(domain) - const docs = await collection.find({ _class }).toArray() - for (const doc of docs) { - const update: Record = {} - const remove = [] - - for (const attribute of attributes) { - try { - const value = (doc as any)[attribute.name] - if (value != null) { - let res = value - while (true) { - try { - const json = JSON.parse(res) - const text = jsonToText(json) - JSON.parse(text) - res = text - } catch { - break - } - } - if (res !== value) { - update[attribute.name] = res - remove.push(makeCollaborativeDoc(doc._id, attribute.name)) - } - } - } catch {} - } - - if (Object.keys(update).length > 0) { - try { - await collection.updateOne({ _id: doc._id }, { $set: update }) - } catch (err) { - console.error('failed to update document', doc._class, doc._id, err) - } - } - - if (remove.length > 0) { - try { - await storageAdapter.remove(ctx, workspaceId, remove) - } catch (err) { - console.error('failed to remove objects from storage', doc._class, doc._id, remove, err) - } - } - } -} - -export async function migrateMarkup ( - ctx: MeasureContext, - storageAdapter: StorageAdapter, - workspaceId: WorkspaceId, - client: MongoClient, - pipeline: Pipeline, - concurrency: number -): Promise { - const hierarchy = pipeline.context.hierarchy - - const workspaceDb = client.db(workspaceId.name) - - const classes = hierarchy.getDescendants(core.class.Doc) - for (const _class of classes) { - const domain = hierarchy.findDomain(_class) - if (domain === undefined) continue - - const allAttributes = hierarchy.getAllAttributes(_class) - const attributes = Array.from(allAttributes.values()).filter((attribute) => { - return hierarchy.isDerived(attribute.type._class, 'core:class:TypeCollaborativeMarkup' as Ref>) - }) - - if (attributes.length === 0) continue - if (hierarchy.isMixin(_class) && attributes.every((p) => p.attributeOf !== _class)) continue - - const collection = workspaceDb.collection(domain) - - const filter = hierarchy.isMixin(_class) ? { [_class]: { $exists: true } } : { _class } - const iterator = collection.find(filter) - - try { - await processMigrateMarkupFor(ctx, hierarchy, storageAdapter, workspaceId, attributes, iterator, concurrency) - } finally { - await iterator.close() - } - } -} - -async function processMigrateMarkupFor ( - ctx: MeasureContext, - hierarchy: Hierarchy, - storageAdapter: StorageAdapter, - workspaceId: WorkspaceId, - attributes: AnyAttribute[], - iterator: FindCursor, - concurrency: number -): Promise { - const rateLimiter = new RateLimiter(concurrency) - - let processed = 0 - - while (true) { - const doc = await iterator.next() - if (doc === null) break - - const timestamp = Date.now() - const revisionId = `${timestamp}` - - await rateLimiter.exec(async () => { - for (const attribute of attributes) { - const collaborativeDoc = makeCollaborativeDoc(doc._id, attribute.name, revisionId) - const { documentId } = collaborativeDocParse(collaborativeDoc) - - const value = hierarchy.isMixin(attribute.attributeOf) - ? ((doc as any)[attribute.attributeOf]?.[attribute.name] as string) - : ((doc as any)[attribute.name] as string) - - if (value != null && value.startsWith('{')) { - const blob = await storageAdapter.stat(ctx, workspaceId, documentId) - // only for documents not in storage - if (blob === undefined) { - try { - const ydoc = markupToYDoc(value, attribute.name) - await saveCollaborativeDoc(ctx, storageAdapter, workspaceId, collaborativeDoc, ydoc) - } catch (err) { - console.error('failed to process document', doc._class, doc._id, err) - } - } - } - } - }) - - processed += 1 - - if (processed % 100 === 0) { - await rateLimiter.waitProcessing() - console.log('...processing', processed) - } - } - - await rateLimiter.waitProcessing() - - console.log('processed', processed) -} - -export async function restoreLostMarkup ( - ctx: MeasureContext, - workspaceId: WorkspaceId, - transactorUrl: string, - storageAdapter: StorageAdapter, - { command }: { command: 'show' | 'restore' } -): Promise { - const connection = (await connect(transactorUrl, workspaceId, undefined, { - mode: 'backup' - })) as unknown as CoreClient - - try { - const hierarchy = connection.getHierarchy() - const classes = hierarchy.getDescendants(core.class.Doc) - - for (const _class of classes) { - const isAttachedDoc = hierarchy.isDerived(_class, core.class.AttachedDoc) - - const attributes = hierarchy.getAllAttributes(_class) - const attrs = Array.from(attributes.values()).filter((p) => p.type._class === core.class.TypeCollaborativeDoc) - - // ignore classes with no collaborative attributes - if (attrs.length === 0) continue - - const docs = await connection.findAll(_class, { _class }) - for (const doc of docs) { - for (const attr of attrs) { - const value = hierarchy.isMixin(attr.attributeOf) - ? ((doc as any)[attr.attributeOf]?.[attr.name] as CollaborativeDoc) - : ((doc as any)[attr.name] as CollaborativeDoc) - - if (value == null || value === '') continue - - const { documentId } = collaborativeDocParse(value) - const stat = await storageAdapter.stat(ctx, workspaceId, documentId) - if (stat !== undefined) continue - - const query = isAttachedDoc - ? { - 'tx.objectId': doc._id, - 'tx._class': { $in: [core.class.TxCreateDoc, core.class.TxUpdateDoc] } - } - : { - objectId: doc._id - } - - let restored = false - - // try to restore by txes - // we need last tx that modified the attribute - - const txes = await connection.findAll(isAttachedDoc ? core.class.TxCollectionCUD : core.class.TxCUD, query, { - sort: { modifiedOn: SortingOrder.Descending } - }) - for (const tx of txes) { - const innerTx = TxProcessor.extractTx(tx) - - let markup: string | undefined - if (innerTx._class === core.class.TxMixin) { - const mixinTx = innerTx as TxMixin - markup = (mixinTx.attributes as any)[attr.name] - } else if (innerTx._class === core.class.TxCreateDoc) { - const createTx = innerTx as TxCreateDoc - markup = (createTx.attributes as any)[attr.name] - } else if (innerTx._class === core.class.TxUpdateDoc) { - const updateTex = innerTx as TxUpdateDoc - markup = (updateTex.operations as any)[attr.name] - } else { - continue - } - - if (markup === undefined || !markup.startsWith('{')) continue - if (isEmptyMarkup(markup)) continue - - console.log(doc._class, doc._id, attr.name, markup) - if (command === 'restore') { - const ydoc = markupToYDoc(markup, attr.name) - await saveCollaborativeDoc(ctx, storageAdapter, workspaceId, value, ydoc) - } - restored = true - break - } - - if (restored) continue - - // try to restore by doc index state - const docIndexState = await connection.findOne(core.class.DocIndexState, { - _id: doc._id as Ref - }) - if (docIndexState !== undefined) { - // document:class:Document%content#content#base64 - const attrName = `${doc._class}%${attr.name}#content#base64` - const base64: string | undefined = docIndexState.attributes[attrName] - if (base64 !== undefined) { - const text = Buffer.from(base64, 'base64').toString() - if (text !== '') { - const markup: Markup = JSON.stringify({ - type: 'doc', - content: [ - { - type: 'paragraph', - content: [{ type: 'text', text, marks: [] }] - } - ] - }) - console.log(doc._class, doc._id, attr.name, markup) - if (command === 'restore') { - const ydoc = markupToYDoc(markup, attr.name) - await saveCollaborativeDoc(ctx, storageAdapter, workspaceId, value, ydoc) - } - } - } - } - } - } - } - } finally { - await connection.close() - } -} diff --git a/dev/tool/src/mixin.ts b/dev/tool/src/mixin.ts deleted file mode 100644 index d967b9a974c..00000000000 --- a/dev/tool/src/mixin.ts +++ /dev/null @@ -1,245 +0,0 @@ -// -// Copyright © 2023 Hardcore Engineering Inc. -// -// Licensed under the Eclipse Public License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. You may -// obtain a copy of the License at https://www.eclipse.org/legal/epl-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// -// See the License for the specific language governing permissions and -// limitations under the License. -// - -import core, { - type AnyAttribute, - type BackupClient, - type Class, - ClassifierKind, - type Client as CoreClient, - type Doc, - type Domain, - Hierarchy, - type Obj, - type Ref, - SortingOrder, - type WorkspaceId -} from '@hcengineering/core' -import { getMongoClient, getWorkspaceMongoDB } from '@hcengineering/mongo' -import { connect } from '@hcengineering/server-tool' - -interface PropertyInfo { - name: string - cValue: any - mValue: any - cModifiedOn: number - mModifiedOn: number -} - -interface ObjectPropertyInfo { - doc: Ref - properties: PropertyInfo[] -} - -export async function showMixinForeignAttributes ( - workspaceId: WorkspaceId, - transactorUrl: string, - cmd: { detail: boolean, mixin: string, property: string } -): Promise { - console.log(`Running mixin attribute check with ${JSON.stringify(cmd)}`) - - const connection = (await connect(transactorUrl, workspaceId, undefined, { - mode: 'backup' - })) as unknown as CoreClient & BackupClient - try { - const result = await getMixinWithForeignProperties(connection, cmd.mixin, cmd.property) - - console.log(`Found ${result.size} mixin(s)\n`) - - // print summary - for (const [mixin, objects] of result) { - const properties = [...new Set(objects.map((o) => o.properties.map((p) => p.name)).flat())].sort() - - console.log('*', mixin) - console.log(' -', properties.join(', ')) - } - - // print details - if (cmd.detail) { - for (const [mixin, objects] of result) { - console.log(mixin, '\n', JSON.stringify(objects, undefined, 2), '\n') - } - } - } catch (err: any) { - console.trace(err) - } finally { - await connection.close() - } -} - -export async function fixMixinForeignAttributes ( - mongoUrl: string, - workspaceId: WorkspaceId, - transactorUrl: string, - cmd: { mixin: string, property: string } -): Promise { - console.log(`Running mixin attribute check with ${JSON.stringify(cmd)}`) - - const connection = (await connect(transactorUrl, workspaceId, undefined, { - mode: 'backup' - })) as unknown as CoreClient & BackupClient - try { - const result = await getMixinWithForeignProperties(connection, cmd.mixin, cmd.property) - - console.log(`Found ${result.size} mixin(s)\n`) - for (const [mixin, objects] of result) { - console.log(mixin, '\n', JSON.stringify(objects, undefined, 2), '\n') - } - - if (result.size > 0) { - const client = getMongoClient(mongoUrl) - try { - const _client = await client.getClient() - const db = getWorkspaceMongoDB(_client, workspaceId) - - for (const [mixin, objects] of result) { - console.log('fixing', mixin) - - let domain: Domain | undefined - try { - domain = connection.getHierarchy().getDomain(mixin) - } catch (err: any) { - console.error('failed to get domain for', mixin) - } - if (domain === undefined) continue - - for (const { doc, properties } of objects) { - for (const property of properties) { - console.log('fixing', mixin, doc, property.name) - - const mixinPropertyName = `${mixin}.${property.name}` - const propertyName = property.name - - if (property.mModifiedOn > property.cModifiedOn) { - console.log(`- renaming: ${mixinPropertyName} -> ${propertyName}`) - await db.collection(domain).updateOne({ _id: doc }, { $rename: { [mixinPropertyName]: propertyName } }) - } else { - console.log(`- removing: ${mixinPropertyName}`) - await db.collection(domain).updateOne({ _id: doc }, { $unset: { [mixinPropertyName]: '' } }) - } - } - } - } - } finally { - client.close() - } - } - } catch (err: any) { - console.trace(err) - } finally { - await connection.close() - } -} - -/** - * Returns properties that present in the mixin but not declared as the mixin attribute. - * If the property is not an attribute, it won't be returned as 'foreign' - * @param doc - * @param mixin - * @param attributes - * @returns array of property names - */ -function getForeignProperties ( - doc: Doc, - mixin: Ref>, - attributes: Map, - attribute: string -): string[] { - if (!(mixin in doc)) return [] - - const mixinAttr = (doc as any)[mixin] - return Object.entries(mixinAttr) - .filter(([key]) => attribute === '' || key === attribute) - .filter(([key]) => (attributes.has(key) ? attributes.get(key)?.attributeOf !== mixin : false)) - .map(([key]) => key) -} - -async function getMixinWithForeignProperties ( - connection: CoreClient, - mixin: string, - attribute: string -): Promise>, ObjectPropertyInfo[]>> { - const hierarchy = connection.getHierarchy() - - const mixins = await connection.findAll(core.class.Class, { - kind: ClassifierKind.MIXIN, - ...(mixin !== '' ? { _id: mixin as Ref> } : {}) - }) - - const result = new Map>, ObjectPropertyInfo[]>() - - for (const [index, mixin] of mixins.entries()) { - console.log(`(${index + 1}/${mixins.length}) Processing ${mixin._id} ...`) - - const attributes = hierarchy.getAllAttributes(mixin._id) - - try { - // Find objects that have mixin attributes - const objects = await connection.findAll(mixin._id, { - [mixin._id]: { $exists: true } - }) - - for (const doc of objects) { - const foreignProperties = getForeignProperties(doc, mixin._id, attributes, attribute) - - if (foreignProperties.length > 0) { - const _doc = Hierarchy.toDoc(doc) - - const properties: PropertyInfo[] = [] - for (const property of foreignProperties) { - const cValue = (_doc as any)[property] - const mValue = (_doc as any)[mixin._id]?.[property] - - // check class and mixin transactions for the property - const updateDocTx = await connection.findAll( - core.class.TxUpdateDoc, - { objectId: doc._id, [`operations.${property}`]: { $exists: true } }, - { limit: 1, sort: { modifiedOn: SortingOrder.Descending } } - ) - - const mixinTx = await connection.findAll( - core.class.TxMixin, - { objectId: doc._id, [`attributes.${property}`]: { $exists: true } }, - { limit: 1, sort: { modifiedOn: SortingOrder.Descending } } - ) - - const collectionTx = await connection.findAll( - core.class.TxCollectionCUD, - { - 'tx._class': core.class.TxUpdateDoc, - 'tx.objectId': doc._id, - [`tx.operations.${property}`]: { $exists: true } - }, - { limit: 1, sort: { modifiedOn: SortingOrder.Descending } } - ) - - const cModifiedOn = Math.max(updateDocTx[0]?.modifiedOn ?? 0, collectionTx[0]?.modifiedOn ?? 0) - const mModifiedOn = mixinTx[0]?.modifiedOn ?? 0 - - properties.push({ name: property, cValue, mValue, cModifiedOn, mModifiedOn }) - } - - result.set(mixin._id, [...(result.get(mixin._id) ?? []), { doc: doc._id, properties }]) - } - } - const objectsCount = result.get(mixin._id)?.length ?? 0 - console.log(`(${index + 1}/${mixins.length}) ... processed ${mixin._id} done, found ${objectsCount} objects`) - } catch (err: any) { - console.error(err) - } - } - - return result -} diff --git a/dev/tool/src/storage.ts b/dev/tool/src/storage.ts index d13f254c123..3ec4e2ded71 100644 --- a/dev/tool/src/storage.ts +++ b/dev/tool/src/storage.ts @@ -13,16 +13,13 @@ // limitations under the License. // -import { type Attachment } from '@hcengineering/attachment' import { type Blob, type MeasureContext, type Ref, type WorkspaceId, RateLimiter } from '@hcengineering/core' -import { DOMAIN_ATTACHMENT } from '@hcengineering/model-attachment' import { type ListBlobResult, type StorageAdapter, type StorageAdapterEx, type UploadedObjectInfo } from '@hcengineering/server-core' -import { type Db } from 'mongodb' import { PassThrough } from 'stream' export interface MoveFilesParams { @@ -55,31 +52,6 @@ export async function moveFiles ( } } -export async function showLostFiles ( - ctx: MeasureContext, - workspaceId: WorkspaceId, - db: Db, - storageAdapter: StorageAdapter, - { showAll }: { showAll: boolean } -): Promise { - const iterator = db.collection(DOMAIN_ATTACHMENT).find({}) - - while (true) { - const attachment = await iterator.next() - if (attachment === null) break - - const { _id, _class, file, name, modifiedOn } = attachment - const date = new Date(modifiedOn).toISOString() - - const stat = await storageAdapter.stat(ctx, workspaceId, file) - if (stat === undefined) { - console.warn('-', date, _class, _id, file, name) - } else if (showAll) { - console.log('+', date, _class, _id, file, name) - } - } -} - async function processAdapter ( ctx: MeasureContext, exAdapter: StorageAdapterEx, diff --git a/dev/tool/src/workspace.ts b/dev/tool/src/workspace.ts index fe0d9d98af1..eba686d5e97 100644 --- a/dev/tool/src/workspace.ts +++ b/dev/tool/src/workspace.ts @@ -20,13 +20,16 @@ import core, { type Class, type Client as CoreClient, type Doc, + type DocIndexState, DOMAIN_DOC_INDEX_STATE, DOMAIN_TX, + MeasureMetricsContext, type Ref, type Tx, type WorkspaceId } from '@hcengineering/core' import { getMongoClient, getWorkspaceMongoDB } from '@hcengineering/mongo' +import { getServerPipeline } from '@hcengineering/server-pipeline' import { connect } from '@hcengineering/server-tool' import { generateModelDiff, printDiff } from './mdiff' @@ -97,15 +100,21 @@ export async function updateField ( } } -export async function recreateElastic (mongoUrl: string, workspaceId: WorkspaceId): Promise { - const client = getMongoClient(mongoUrl) - const _client = await client.getClient() +export async function recreateElastic (dbUrl: string, workspaceId: WorkspaceId, txes: Tx[]): Promise { + const pipeline = await getServerPipeline(new MeasureMetricsContext('tool', {}), txes, dbUrl, { + name: workspaceId.name, + workspaceName: workspaceId.name, + workspaceUrl: workspaceId.name + }) + try { - const db = getWorkspaceMongoDB(_client, workspaceId) - await db - .collection(DOMAIN_DOC_INDEX_STATE) - .updateMany({ _class: core.class.DocIndexState }, { $set: { stages: {}, needIndex: true } }) + await pipeline.pipeline.context.lowLevelStorage?.rawUpdate( + DOMAIN_DOC_INDEX_STATE, + { _class: core.class.DocIndexState }, + { needIndex: true } + ) } finally { - client.close() + await pipeline.pipeline.close() + await pipeline.storageAdapter.close() } } diff --git a/server-plugins/notification-resources/src/index.ts b/server-plugins/notification-resources/src/index.ts index 42b48c936ea..dea7acf68e8 100644 --- a/server-plugins/notification-resources/src/index.ts +++ b/server-plugins/notification-resources/src/index.ts @@ -990,7 +990,7 @@ export async function createCollabDocInfo ( const usersInfo = await ctx.with( 'get-user-info', {}, - async (ctx) => await getUsersInfo(ctx, [...Array.from(targets), originTx.modifiedBy as Ref], control) + (ctx) => getUsersInfo(ctx, [...Array.from(targets), originTx.modifiedBy as Ref], control) ) const sender: SenderInfo = usersInfo.get(originTx.modifiedBy) ?? { _id: originTx.modifiedBy @@ -1337,8 +1337,8 @@ async function collectionCollabDoc ( await ctx.with( 'create-collab-doc-info', {}, - async (ctx) => - await createCollabDocInfo( + (ctx) => + createCollabDocInfo( ctx, collaborators as Ref[], control,