UBERF-8557 Save collaborative content as JSON

.vscode/launch.json

@@ -286,12 +286,14 @@
         "MINIO_ENDPOINT": "localhost:9000",
         "TRANSACTOR_URL": "ws://localhost:3333",
         "MONGO_URL": "mongodb://localhost:27017",
+        "DB_URL": "mongodb://localhost:27017",
         "ACCOUNTS_URL": "http://localhost:3000",
         "TELEGRAM_DATABASE": "telegram-service",
         "ELASTIC_URL": "http://localhost:9200",
         "REKONI_URL": "http://localhost:4004",
         "MODEL_VERSION": "0.6.287"
       },
+      "runtimeVersion": "20",
       "runtimeArgs": ["--nolazy", "-r", "ts-node/register"],
       "sourceMaps": true,
       "outputCapture": "std",

dev/doc-import-tool/src/import.ts

@@ -15,14 +15,13 @@ import documents, {
 import core, {
   AttachedData,
   BackupClient,
-  CollaborativeDoc,
   Client as CoreClient,
   Data,
   MeasureContext,
   Ref,
   TxOperations,
   generateId,
-  makeCollaborativeDoc,
+  makeDocCollabId,
   systemAccountEmail,
   type Blob
 } from '@hcengineering/core'
@@ -101,7 +100,7 @@ async function createDocument (
     abstract: '',
     effectiveDate: 0,
     reviewInterval: DEFAULT_PERIODIC_REVIEW_INTERVAL,
-    content: makeCollaborativeDoc(generateId()),
+    content: null,
     snapshots: 0,
     plannedEffectiveDate: 0
   }
@@ -115,11 +114,6 @@ async function createDocument (
 
   console.log('Creating controlled doc from template')
 
-  const copyContent = async (source: CollaborativeDoc, target: CollaborativeDoc): Promise<void> => {
-    // intentionally left empty
-    // even though the template has some content, it won't be used
-  }
-
   const { success } = await createControlledDocFromTemplate(
     txops,
     templateId,
@@ -128,8 +122,7 @@ async function createDocument (
     space,
     undefined,
     undefined,
-    documents.class.ControlledDocument,
-    copyContent
+    documents.class.ControlledDocument
   )
   if (!success) {
     throw new Error('Failed to create controlled document from template')
@@ -184,7 +177,7 @@ async function createTemplateIfNotExist (
     approvers: [],
     coAuthors: [],
     changeControl: ccRecordId,
-    content: makeCollaborativeDoc(generateId()),
+    content: null,
     snapshots: 0,
     plannedEffectiveDate: 0
   }
@@ -229,9 +222,7 @@ async function createSections (
 
   console.log('Creating document content')
 
-  const collabId = doc.content
-
-  console.log(`Collab doc ID: ${collabId}`)
+  const collabId = makeDocCollabId(doc, 'content')
 
   try {
     let content: string = ''
@@ -245,7 +236,7 @@ async function createSections (
       content += `<h1>${section.title}</h1>${section.content}`
     }
 
-    await collaborator.updateContent(collabId, { content })
+    await collaborator.updateMarkup(collabId, content)
   } finally {
     // do nothing
   }

dev/import-tool/src/importer/importer.ts

@@ -14,18 +14,18 @@
 //
 import attachment, { type Attachment } from '@hcengineering/attachment'
 import chunter, { type ChatMessage } from '@hcengineering/chunter'
-import { yDocToBuffer } from '@hcengineering/collaboration'
 import { type Person } from '@hcengineering/contact'
 import core, {
   type Account,
   type AttachedData,
   type Class,
+  type Blob as PlatformBlob,
   type CollaborativeDoc,
   type Data,
   type Doc,
   type DocumentQuery,
   generateId,
-  makeCollaborativeDoc,
+  makeCollabId,
   type Mixin,
   type Ref,
   SortingOrder,
@@ -42,7 +42,7 @@ import task, {
   type TaskType,
   type TaskTypeWithFactory
 } from '@hcengineering/task'
-import { jsonToMarkup, jsonToYDocNoSchema, parseMessageMarkdown } from '@hcengineering/text'
+import { jsonToMarkup, parseMessageMarkdown } from '@hcengineering/text'
 import tracker, {
   type Issue,
   type IssueParentInfo,
@@ -52,7 +52,7 @@ import tracker, {
   TimeReportDayType
 } from '@hcengineering/tracker'
 import { type MarkdownPreprocessor, NoopMarkdownPreprocessor } from './preprocessor'
-import { type FileUploader, type UploadResult } from './uploader'
+import { type FileUploader } from './uploader'
 
 export interface ImportWorkspace {
   projectTypes?: ImportProjectType[]
@@ -281,14 +281,15 @@ export class WorkspaceImporter {
   ): Promise<Ref<Document>> {
     const id = doc.id ?? generateId<Document>()
     const content = await doc.descrProvider()
-    const collabId = await this.createCollaborativeContent(id, 'content', content, teamspaceId)
+    const collabId = makeCollabId(document.class.Document, id, 'content')
+    const contentId = await this.createCollaborativeContent(id, collabId, content, teamspaceId)
 
     const lastRank = await getFirstRank(this.client, teamspaceId, parentId)
     const rank = makeRank(lastRank, undefined)
 
     const attachedData: Data<Document> = {
       title: doc.title,
-      content: collabId,
+      content: contentId,
       parent: parentId,
       attachments: 0,
       embeddings: 0,
@@ -391,7 +392,8 @@ export class WorkspaceImporter {
   ): Promise<{ id: Ref<Issue>, identifier: string }> {
     const issueId = issue.id ?? generateId<Issue>()
     const content = await issue.descrProvider()
-    const collabId = await this.createCollaborativeContent(issueId, 'description', content, project._id)
+    const collabId = makeCollabId(tracker.class.Issue, issueId, 'description')
+    const contentId = await this.createCollaborativeContent(issueId, collabId, content, project._id)
 
     const { number, identifier } =
       issue.number !== undefined
@@ -412,7 +414,7 @@ export class WorkspaceImporter {
 
     const issueData: AttachedData<Issue> = {
       title: issue.title,
-      description: collabId,
+      description: contentId,
       assignee: issue.assignee ?? null,
       component: null,
       number,
@@ -532,14 +534,10 @@ export class WorkspaceImporter {
     }
 
     const file = new File([blob], attachment.title)
-    const attachmentId = await this.createAttachment(
-      attachment.id ?? generateId<Attachment>(),
-      file,
-      spaceId,
-      parentId,
-      parentClass
-    )
-    if (attachmentId === null) {
+
+    try {
+      await this.createAttachment(attachment.id ?? generateId<Attachment>(), file, spaceId, parentId, parentClass)
+    } catch {
       console.warn('Failed to upload attachment file: ', attachment.title)
     }
   }
@@ -550,56 +548,41 @@ export class WorkspaceImporter {
     spaceId: Ref<Space>,
     parentId: Ref<Doc>,
     parentClass: Ref<Class<Doc<Space>>>
-  ): Promise<Ref<Attachment> | null> {
-    const response = await this.fileUploader.uploadFile(id, id, file)
-    if (response.status !== 200) {
-      return null
-    }
-
-    const responseText = await response.text()
-    if (responseText === undefined) {
-      return null
-    }
-
-    const uploadResult = JSON.parse(responseText) as UploadResult[]
-    if (!Array.isArray(uploadResult) || uploadResult.length === 0) {
-      return null
-    }
-
+  ): Promise<Ref<Attachment>> {
+    const attachmentId = generateId<Attachment>()
+    const blobId = await this.fileUploader.uploadFile(id, file)
     await this.client.addCollection(
       attachment.class.Attachment,
       spaceId,
       parentId,
       parentClass,
       'attachments',
       {
-        file: uploadResult[0].id,
+        file: blobId,
         lastModified: Date.now(),
         name: file.name,
         size: file.size,
         type: file.type
       },
       id
     )
-    return id
+    return attachmentId
   }
 
   // Collaborative content handling
   private async createCollaborativeContent (
     id: Ref<Doc>,
-    field: string,
+    collabId: CollaborativeDoc,
     content: string,
     spaceId: Ref<Space>
-  ): Promise<CollaborativeDoc> {
+  ): Promise<Ref<PlatformBlob>> {
     const json = parseMessageMarkdown(content ?? '', 'image://')
     const processedJson = this.preprocessor.process(json, id, spaceId)
-    const collabId = makeCollaborativeDoc(id, 'description')
 
-    const yDoc = jsonToYDocNoSchema(processedJson, field)
-    const buffer = yDocToBuffer(yDoc)
+    const markup = jsonToMarkup(processedJson)
+    const buffer = Buffer.from(markup)
 
-    await this.fileUploader.uploadCollaborativeDoc(id, collabId, buffer)
-    return collabId
+    return await this.fileUploader.uploadCollaborativeDoc(collabId, buffer)
   }
 
   async findIssueStatusByName (name: string): Promise<Ref<IssueStatus>> {

dev/import-tool/src/importer/uploader.ts

@@ -13,25 +13,19 @@
 // limitations under the License.
 //
 import {
-  concatLink,
   type Ref,
   type Blob as PlatformBlob,
-  type Doc,
   type CollaborativeDoc,
-  collaborativeDocParse
+  concatLink,
+  makeCollabJsonId
 } from '@hcengineering/core'
 
 export interface FileUploader {
-  uploadFile: (id: Ref<Doc>, name: string, file: File, contentType?: string) => Promise<Response>
-  uploadCollaborativeDoc: (id: Ref<Doc>, collabId: CollaborativeDoc, data: Buffer) => Promise<Response>
+  uploadFile: (name: string, file: Blob) => Promise<Ref<PlatformBlob>>
+  uploadCollaborativeDoc: (collabId: CollaborativeDoc, data: Buffer) => Promise<Ref<PlatformBlob>>
   getFileUrl: (id: string) => string
 }
 
-export interface UploadResult {
-  key: 'file'
-  id: Ref<PlatformBlob>
-}
-
 export class FrontFileUploader implements FileUploader {
   constructor (
     private readonly frontUrl: string,
@@ -41,31 +35,32 @@ export class FrontFileUploader implements FileUploader {
     this.getFileUrl = this.getFileUrl.bind(this)
   }
 
-  public async uploadFile (id: Ref<Doc>, name: string, file: File, contentType?: string): Promise<Response> {
+  public async uploadFile (name: string, file: Blob): Promise<Ref<PlatformBlob>> {
     const form = new FormData()
     form.append('file', file, name)
-    form.append('type', contentType ?? file.type)
-    form.append('size', file.size.toString())
-    form.append('name', file.name)
-    form.append('id', id)
-    form.append('data', new Blob([file]))
 
-    return await fetch(concatLink(this.frontUrl, '/files'), {
+    const res = await fetch(concatLink(this.frontUrl, '/files'), {
       method: 'POST',
       headers: {
         Authorization: 'Bearer ' + this.token
       },
       body: form
     })
+
+    if (res.ok && res.status === 200) {
+      return name as Ref<PlatformBlob>
+    }
+
+    throw new Error('Failed to upload file')
   }
 
   public getFileUrl (id: string): string {
     return concatLink(this.frontUrl, `/files/${this.workspaceId}/${id}?file=${id}&workspace=${this.workspaceId}`)
   }
 
-  public async uploadCollaborativeDoc (id: Ref<Doc>, collabId: CollaborativeDoc, data: Buffer): Promise<Response> {
-    const file = new File([data], collabId)
-    const { documentId } = collaborativeDocParse(collabId)
-    return await this.uploadFile(id, documentId, file, 'application/ydoc')
+  public async uploadCollaborativeDoc (collabId: CollaborativeDoc, data: Buffer): Promise<Ref<PlatformBlob>> {
+    const blobId = makeCollabJsonId(collabId)
+    const blob = new Blob([data], { type: 'application/json' })
+    return await this.uploadFile(blobId, blob)
   }
 }