refactor: handle larger files in importer (#2692)

packages/files/src/Domain/Service/SuperConverterServiceInterface.ts

@@ -1,15 +1,6 @@
-import { FileItem } from '@standardnotes/models'
-import { GenerateUuid } from '@standardnotes/services'
-
 export interface SuperConverterServiceInterface {
   isValidSuperString(superString: string): boolean
   convertSuperStringToOtherFormat: (superString: string, toFormat: 'txt' | 'md' | 'html' | 'json') => Promise<string>
   convertOtherFormatToSuperString: (otherFormatString: string, fromFormat: 'txt' | 'md' | 'html' | 'json') => string
   getEmbeddedFileIDsFromSuperString(superString: string): string[]
-  uploadAndReplaceInlineFilesInSuperString(
-    superString: string,
-    uploadFile: (file: File) => Promise<FileItem | undefined>,
-    linkFile: (file: FileItem) => Promise<void>,
-    generateUuid: GenerateUuid,
-  ): Promise<string>
 }

packages/ui-services/src/Import/AegisConverter/AegisToAuthenticatorConverter.ts

@@ -1,5 +1,6 @@
 import { NativeFeatureIdentifier, NoteType } from '@standardnotes/features'
 import { Converter } from '../Converter'
+import { ConversionResult } from '../ConversionResult'
 
 type AegisData = {
   db: {
@@ -45,7 +46,7 @@ export class AegisToAuthenticatorConverter implements Converter {
     return false
   }
 
-  convert: Converter['convert'] = async (file, { createNote, readFileAsText }) => {
+  convert: Converter['convert'] = async (file, { insertNote, readFileAsText }) => {
     const content = await readFileAsText(file)
 
     const entries = this.parseEntries(content)
@@ -59,17 +60,22 @@ export class AegisToAuthenticatorConverter implements Converter {
     const title = file.name.split('.')[0]
     const text = JSON.stringify(entries)
 
-    return [
-      createNote({
-        createdAt,
-        updatedAt,
-        title,
-        text,
-        noteType: NoteType.Authentication,
-        editorIdentifier: NativeFeatureIdentifier.TYPES.TokenVaultEditor,
-        useSuperIfPossible: false,
-      }),
-    ]
+    const note = await insertNote({
+      createdAt,
+      updatedAt,
+      title,
+      text,
+      noteType: NoteType.Authentication,
+      editorIdentifier: NativeFeatureIdentifier.TYPES.TokenVaultEditor,
+      useSuperIfPossible: false,
+    })
+
+    const successful: ConversionResult['successful'] = [note]
+
+    return {
+      successful,
+      errored: [],
+    }
   }
 
   parseEntries(data: string): AuthenticatorEntry[] | null {

packages/ui-services/src/Import/ConversionResult.ts

@@ -0,0 +1,9 @@
+import { DecryptedItemInterface } from '@standardnotes/models'
+
+export type ConversionResult = {
+  successful: DecryptedItemInterface[]
+  errored: {
+    name: string
+    error: Error
+  }[]
+}

packages/ui-services/src/Import/Converter.ts

@@ -1,5 +1,6 @@
 import { NoteType } from '@standardnotes/features'
-import { DecryptedTransferPayload, ItemContent, NoteContent, TagContent } from '@standardnotes/models'
+import { DecryptedItemInterface, FileItem, ItemContent, NoteContent, SNNote, SNTag } from '@standardnotes/models'
+import { ConversionResult } from './ConversionResult'
 
 export interface Converter {
   getImportType(): string
@@ -12,17 +13,24 @@ export interface Converter {
   convert(
     file: File,
     dependencies: {
-      createNote: CreateNoteFn
-      createTag: CreateTagFn
+      insertNote: InsertNoteFn
+      insertTag: InsertTagFn
+      canUploadFiles: boolean
+      uploadFile: UploadFileFn
       canUseSuper: boolean
       convertHTMLToSuper: (html: string) => string
       convertMarkdownToSuper: (markdown: string) => string
       readFileAsText: (file: File) => Promise<string>
+      linkItems(
+        item: DecryptedItemInterface<ItemContent>,
+        itemToLink: DecryptedItemInterface<ItemContent>,
+      ): Promise<void>
+      cleanupItems(items: DecryptedItemInterface<ItemContent>[]): Promise<void>
     },
-  ): Promise<DecryptedTransferPayload<ItemContent>[]>
+  ): Promise<ConversionResult>
 }
 
-export type CreateNoteFn = (options: {
+export type InsertNoteFn = (options: {
   createdAt: Date
   updatedAt: Date
   title: string
@@ -33,10 +41,20 @@ export type CreateNoteFn = (options: {
   trashed?: boolean
   editorIdentifier?: NoteContent['editorIdentifier']
   useSuperIfPossible: boolean
-}) => DecryptedTransferPayload<NoteContent>
+}) => Promise<SNNote>
 
-export type CreateTagFn = (options: {
+export type InsertTagFn = (options: {
   createdAt: Date
   updatedAt: Date
   title: string
-}) => DecryptedTransferPayload<TagContent>
+  references: SNTag['references']
+}) => Promise<SNTag>
+
+export type UploadFileFn = (file: File) => Promise<FileItem | undefined>
+
+export type LinkItemsFn = (
+  item: DecryptedItemInterface<ItemContent>,
+  itemToLink: DecryptedItemInterface<ItemContent>,
+) => Promise<void>
+
+export type CleanupItemsFn = (items: DecryptedItemInterface<ItemContent>[]) => Promise<void>

packages/ui-services/src/Import/EvernoteConverter/EvernoteConverter.spec.ts

@@ -3,9 +3,9 @@
  */
 
 import { ContentType } from '@standardnotes/domain-core'
-import { DecryptedTransferPayload, NoteContent, TagContent } from '@standardnotes/models'
+import { SNNote, SNTag } from '@standardnotes/models'
 import { EvernoteConverter, EvernoteResource } from './EvernoteConverter'
-import { createTestResourceElement, enex, enexWithNoNoteOrTag } from './testData'
+import { createTestResourceElement, enex } from './testData'
 import { PureCryptoInterface } from '@standardnotes/sncrypto-common'
 import { GenerateUuid } from '@standardnotes/services'
 import { Converter } from '../Converter'
@@ -33,77 +33,80 @@ describe('EvernoteConverter', () => {
   const readFileAsText = async (file: File) => file as unknown as string
 
   const dependencies: Parameters<Converter['convert']>[1] = {
-    createNote: ({ text }) =>
+    insertNote: async ({ text }) =>
       ({
         content_type: ContentType.TYPES.Note,
         content: {
           text,
           references: [],
         },
-      }) as unknown as DecryptedTransferPayload<NoteContent>,
-    createTag: ({ title }) =>
+        uuid: generateUuid.execute().getValue(),
+      }) as unknown as SNNote,
+    insertTag: async ({ title }) =>
       ({
         content_type: ContentType.TYPES.Tag,
         content: {
           title,
           references: [],
         },
-      }) as unknown as DecryptedTransferPayload<TagContent>,
+        uuid: generateUuid.execute().getValue(),
+      }) as unknown as SNTag,
     convertHTMLToSuper: (data) => data,
     convertMarkdownToSuper: jest.fn(),
     readFileAsText,
     canUseSuper: false,
+    canUploadFiles: false,
+    uploadFile: async () => void 0,
+    linkItems: async (item, itemToLink) => {
+      itemToLink.content.references.push({
+        content_type: item.content_type,
+        uuid: item.uuid,
+      })
+    },
+    cleanupItems: async () => void 0,
   }
 
-  it('should throw error if no note or tag in enex', () => {
-    const converter = new EvernoteConverter(generateUuid)
-
-    expect(converter.convert(enexWithNoNoteOrTag as unknown as File, dependencies)).rejects.toThrowError()
-  })
-
   it('should parse and strip html', async () => {
     const converter = new EvernoteConverter(generateUuid)
 
-    const result = await converter.convert(enex as unknown as File, dependencies)
-
-    expect(result).not.toBeNull()
-    expect(result?.length).toBe(3)
-    expect(result?.[0].content_type).toBe(ContentType.TYPES.Note)
-    expect((result?.[0] as DecryptedTransferPayload<NoteContent>).content.text).toBe('This is a test.\nh e ')
-    expect(result?.[1].content_type).toBe(ContentType.TYPES.Note)
-    expect((result?.[1] as DecryptedTransferPayload<NoteContent>).content.text).toBe(
-      'Lorem ipsum dolor sit amet, consectetur adipiscing elit.',
-    )
-    expect(result?.[2].content_type).toBe(ContentType.TYPES.Tag)
-    expect((result?.[2] as DecryptedTransferPayload<TagContent>).content.title).toBe('distant reading')
-    expect((result?.[2] as DecryptedTransferPayload<TagContent>).content.references.length).toBe(2)
-    expect((result?.[2] as DecryptedTransferPayload<TagContent>).content.references[0].uuid).toBe(result?.[0].uuid)
-    expect((result?.[2] as DecryptedTransferPayload<TagContent>).content.references[1].uuid).toBe(result?.[1].uuid)
+    const { successful } = await converter.convert(enex as unknown as File, dependencies)
+
+    expect(successful).not.toBeNull()
+    expect(successful?.length).toBe(3)
+    expect(successful?.[0].content_type).toBe(ContentType.TYPES.Note)
+    expect((successful?.[0] as SNNote).content.text).toBe('This is a test.\nh e ')
+    expect(successful?.[1].content_type).toBe(ContentType.TYPES.Tag)
+    expect((successful?.[1] as SNTag).content.title).toBe('distant reading')
+    expect((successful?.[1] as SNTag).content.references.length).toBe(2)
+    expect((successful?.[1] as SNTag).content.references[0].uuid).toBe(successful?.[0].uuid)
+    expect((successful?.[1] as SNTag).content.references[1].uuid).toBe(successful?.[2].uuid)
+    expect(successful?.[2].content_type).toBe(ContentType.TYPES.Note)
+    expect((successful?.[2] as SNNote).content.text).toBe('Lorem ipsum dolor sit amet, consectetur adipiscing elit.')
   })
 
   it('should parse and not strip html', async () => {
     const converter = new EvernoteConverter(generateUuid)
 
-    const result = await converter.convert(enex as unknown as File, {
+    const { successful } = await converter.convert(enex as unknown as File, {
       ...dependencies,
       canUseSuper: true,
     })
 
-    expect(result).not.toBeNull()
-    expect(result?.length).toBe(3)
-    expect(result?.[0].content_type).toBe(ContentType.TYPES.Note)
-    expect((result?.[0] as DecryptedTransferPayload<NoteContent>).content.text).toBe(
+    expect(successful).not.toBeNull()
+    expect(successful?.length).toBe(3)
+    expect(successful?.[0].content_type).toBe(ContentType.TYPES.Note)
+    expect((successful?.[0] as SNNote).content.text).toBe(
       '<p>This is a test.</p><ul></ul><ol></ol><font><span>h </span><span>e </span></font>',
     )
-    expect(result?.[1].content_type).toBe(ContentType.TYPES.Note)
-    expect((result?.[1] as DecryptedTransferPayload<NoteContent>).content.text).toBe(
+    expect(successful?.[1].content_type).toBe(ContentType.TYPES.Tag)
+    expect((successful?.[1] as SNTag).content.title).toBe('distant reading')
+    expect((successful?.[1] as SNTag).content.references.length).toBe(2)
+    expect((successful?.[1] as SNTag).content.references[0].uuid).toBe(successful?.[0].uuid)
+    expect((successful?.[1] as SNTag).content.references[1].uuid).toBe(successful?.[2].uuid)
+    expect(successful?.[2].content_type).toBe(ContentType.TYPES.Note)
+    expect((successful?.[2] as SNNote).content.text).toBe(
       '<p>Lorem ipsum dolor sit amet, consectetur adipiscing elit.</p>',
     )
-    expect(result?.[2].content_type).toBe(ContentType.TYPES.Tag)
-    expect((result?.[2] as DecryptedTransferPayload<TagContent>).content.title).toBe('distant reading')
-    expect((result?.[2] as DecryptedTransferPayload<TagContent>).content.references.length).toBe(2)
-    expect((result?.[2] as DecryptedTransferPayload<TagContent>).content.references[0].uuid).toBe(result?.[0].uuid)
-    expect((result?.[2] as DecryptedTransferPayload<TagContent>).content.references[1].uuid).toBe(result?.[1].uuid)
   })
 
   it('should convert lists to super format if applicable', () => {
@@ -129,7 +132,7 @@ describe('EvernoteConverter', () => {
     expect(unorderedList2.getAttribute('__lexicallisttype')).toBeFalsy()
   })
 
-  it('should replace media elements with resources', () => {
+  it('should replace media elements with resources', async () => {
     const resources: EvernoteResource[] = [
       {
         hash: 'hash1',
@@ -152,9 +155,14 @@ describe('EvernoteConverter', () => {
     const array = [mediaElement1, mediaElement2, mediaElement3]
 
     const converter = new EvernoteConverter(generateUuid)
-    const replacedCount = converter.replaceMediaElementsWithResources(array, resources)
+    const { replacedElements } = await converter.replaceMediaElementsWithResources(
+      array,
+      resources,
+      false,
+      dependencies.uploadFile,
+    )
 
-    expect(replacedCount).toBe(1)
+    expect(replacedElements.length).toBe(1)
   })
 
   describe('getResourceFromElement', () => {