Skip to content

Commit

Permalink
feat: add note size threshold
Browse files Browse the repository at this point in the history
  • Loading branch information
@amanharwara
amanharwara committed Dec 8, 2023
1 parent 45512af commit 29cf551
Show file tree
Hide file tree
Showing 3 changed files with 170 additions and 105 deletions.
6 changes: 5 additions & 1 deletion packages/ui-services/src/Import/Converter.ts
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
import { NoteType } from '@standardnotes/features'
import { DecryptedItemInterface, ItemContent, NoteContent, SNNote, SNTag } from '@standardnotes/models'
import { DecryptedItemInterface, FileItem, ItemContent, NoteContent, SNNote, SNTag } from '@standardnotes/models'

export interface Converter {
getImportType(): string
Expand All @@ -14,6 +14,8 @@ export interface Converter {
dependencies: {
insertNote: InsertNoteFn
insertTag: InsertTagFn
canUploadFiles: boolean
uploadFile: UploadFileFn
canUseSuper: boolean
convertHTMLToSuper: (html: string) => string
convertMarkdownToSuper: (markdown: string) => string
Expand Down Expand Up @@ -46,6 +48,8 @@ export type InsertTagFn = (options: {
references: SNTag['references']
}) => Promise<SNTag>

export type UploadFileFn = (file: File) => Promise<FileItem | undefined>

export type LinkItemsFn = (
item: DecryptedItemInterface<ItemContent>,
itemToLink: DecryptedItemInterface<ItemContent>,
Expand Down
219 changes: 124 additions & 95 deletions packages/ui-services/src/Import/EvernoteConverter/EvernoteConverter.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ import utc from 'dayjs/plugin/utc'
import { GenerateUuid } from '@standardnotes/services'
import MD5 from 'crypto-js/md5'
import Base64 from 'crypto-js/enc-base64'
import { Converter } from '../Converter'
import { Converter, UploadFileFn } from '../Converter'
dayjs.extend(customParseFormat)
dayjs.extend(utc)

Expand Down Expand Up @@ -35,7 +35,7 @@ export class EvernoteConverter implements Converter {

convert: Converter['convert'] = async (
file,
{ insertNote, insertTag, linkItems, canUseSuper, convertHTMLToSuper, readFileAsText },
{ insertNote, insertTag, linkItems, canUploadFiles, canUseSuper, convertHTMLToSuper, readFileAsText, uploadFile },
) => {
const content = await readFileAsText(file)

Expand All @@ -50,83 +50,88 @@ export class EvernoteConverter implements Converter {
}

for (const [index, xmlNote] of Array.from(xmlNotes).entries()) {
const title = xmlNote.getElementsByTagName('title')[0].textContent
const created = xmlNote.getElementsByTagName('created')[0]?.textContent
const updatedNodes = xmlNote.getElementsByTagName('updated')
const updated = updatedNodes.length ? updatedNodes[0].textContent : null
const resources = Array.from(xmlNote.getElementsByTagName('resource'))
.map(this.getResourceFromElement)
.filter(Boolean) as EvernoteResource[]

const contentNode = xmlNote.getElementsByTagName('content')[0]
const contentXmlString = this.getXmlStringFromContentElement(contentNode)
if (!contentXmlString) {
continue
}
const contentXml = this.loadXMLString(contentXmlString, 'html')

const noteElement = contentXml.getElementsByTagName('en-note')[0] as HTMLElement
try {
const title = xmlNote.getElementsByTagName('title')[0].textContent
const created = xmlNote.getElementsByTagName('created')[0]?.textContent
const updatedNodes = xmlNote.getElementsByTagName('updated')
const updated = updatedNodes.length ? updatedNodes[0].textContent : null
const resources = Array.from(xmlNote.getElementsByTagName('resource'))
.map(this.getResourceFromElement)
.filter(Boolean) as EvernoteResource[]

const contentNode = xmlNote.getElementsByTagName('content')[0]
const contentXmlString = this.getXmlStringFromContentElement(contentNode)
if (!contentXmlString) {
continue
}
const contentXml = this.loadXMLString(contentXmlString, 'html')

const unorderedLists = Array.from(noteElement.getElementsByTagName('ul'))
const noteElement = contentXml.getElementsByTagName('en-note')[0] as HTMLElement

if (canUseSuper) {
this.convertTopLevelDivsToParagraphs(noteElement)
this.convertListsToSuperFormatIfApplicable(unorderedLists)
this.convertLeftPaddingToSuperIndent(noteElement)
}
const unorderedLists = Array.from(noteElement.getElementsByTagName('ul'))

this.removeEmptyAndOrphanListElements(noteElement)
this.removeUnnecessaryTopLevelBreaks(noteElement)
if (canUseSuper) {
this.convertTopLevelDivsToParagraphs(noteElement)
this.convertListsToSuperFormatIfApplicable(unorderedLists)
this.convertLeftPaddingToSuperIndent(noteElement)
}

const mediaElements = Array.from(noteElement.getElementsByTagName('en-media'))
this.replaceMediaElementsWithResources(mediaElements, resources)
this.removeEmptyAndOrphanListElements(noteElement)
this.removeUnnecessaryTopLevelBreaks(noteElement)

// Some notes have <font> tags that contain separate <span> tags with text
// which causes broken paragraphs in the note.
const fontElements = Array.from(noteElement.getElementsByTagName('font'))
for (const fontElement of fontElements) {
fontElement.childNodes.forEach((childNode) => {
childNode.textContent += ' '
})
fontElement.innerText = fontElement.textContent || ''
}
const mediaElements = Array.from(noteElement.getElementsByTagName('en-media'))
await this.replaceMediaElementsWithResources(mediaElements, resources, canUploadFiles, uploadFile)

let contentHTML = noteElement.innerHTML
if (!canUseSuper) {
contentHTML = contentHTML.replace(/<\/div>/g, '</div>\n')
contentHTML = contentHTML.replace(/<li[^>]*>/g, '\n')
contentHTML = contentHTML.trim()
}
const text = !canUseSuper ? this.stripHTML(contentHTML) : convertHTMLToSuper(contentHTML)

const createdAtDate = created ? dayjs.utc(created, dateFormat).toDate() : new Date()
const updatedAtDate = updated ? dayjs.utc(updated, dateFormat).toDate() : createdAtDate

const note = await insertNote({
createdAt: createdAtDate,
updatedAt: updatedAtDate,
title: !title ? `Imported note ${index + 1} from Evernote` : title,
text,
useSuperIfPossible: canUseSuper,
})

const xmlTags = xmlNote.getElementsByTagName('tag')
for (const tagXml of Array.from(xmlTags)) {
const tagName = tagXml.childNodes[0].nodeValue
let tag = findTag(tagName)

if (!tag) {
const now = new Date()
tag = await insertTag({
createdAt: now,
updatedAt: now,
title: tagName || `Imported tag ${index + 1} from Evernote`,
references: [],
// Some notes have <font> tags that contain separate <span> tags with text
// which causes broken paragraphs in the note.
const fontElements = Array.from(noteElement.getElementsByTagName('font'))
for (const fontElement of fontElements) {
fontElement.childNodes.forEach((childNode) => {
childNode.textContent += ' '
})
tags.push(tag)
fontElement.innerText = fontElement.textContent || ''
}

await linkItems(note, tag)
let contentHTML = noteElement.innerHTML
if (!canUseSuper) {
contentHTML = contentHTML.replace(/<\/div>/g, '</div>\n')
contentHTML = contentHTML.replace(/<li[^>]*>/g, '\n')
contentHTML = contentHTML.trim()
}
const text = !canUseSuper ? this.stripHTML(contentHTML) : convertHTMLToSuper(contentHTML)

const createdAtDate = created ? dayjs.utc(created, dateFormat).toDate() : new Date()
const updatedAtDate = updated ? dayjs.utc(updated, dateFormat).toDate() : createdAtDate

const note = await insertNote({
createdAt: createdAtDate,
updatedAt: updatedAtDate,
title: !title ? `Imported note ${index + 1} from Evernote` : title,
text,
useSuperIfPossible: canUseSuper,
})

const xmlTags = xmlNote.getElementsByTagName('tag')
for (const tagXml of Array.from(xmlTags)) {
const tagName = tagXml.childNodes[0].nodeValue
let tag = findTag(tagName)

if (!tag) {
const now = new Date()
tag = await insertTag({
createdAt: now,
updatedAt: now,
title: tagName || `Imported tag ${index + 1} from Evernote`,
references: [],
})
tags.push(tag)
}

await linkItems(note, tag)
}
} catch (error) {
console.error(error)
continue
}
}
}
Expand Down Expand Up @@ -250,41 +255,65 @@ export class EvernoteConverter implements Converter {
})
}

replaceMediaElementsWithResources(mediaElements: Element[], resources: EvernoteResource[]): number {
getHTMLElementFromResource(resource: EvernoteResource) {
let resourceElement: HTMLElement = document.createElement('object')
resourceElement.setAttribute('type', resource.mimeType)
resourceElement.setAttribute('data', resource.data)
if (resource.mimeType.startsWith('image/')) {
resourceElement = document.createElement('img')
resourceElement.setAttribute('src', resource.data)
resourceElement.setAttribute('data-mime-type', resource.mimeType)
} else if (resource.mimeType.startsWith('audio/')) {
resourceElement = document.createElement('audio')
resourceElement.setAttribute('controls', 'controls')
const sourceElement = document.createElement('source')
sourceElement.setAttribute('src', resource.data)
sourceElement.setAttribute('type', resource.mimeType)
resourceElement.appendChild(sourceElement)
} else if (resource.mimeType.startsWith('video/')) {
resourceElement = document.createElement('video')
resourceElement.setAttribute('controls', 'controls')
const sourceElement = document.createElement('source')
sourceElement.setAttribute('src', resource.data)
sourceElement.setAttribute('type', resource.mimeType)
resourceElement.appendChild(sourceElement)
}
resourceElement.setAttribute('data-filename', resource.fileName)
return resourceElement
}

async getFileFromResource(resource: EvernoteResource): Promise<File> {
const response = await fetch(resource.data)
const blob = await response.blob()
return new File([blob], resource.fileName, { type: resource.mimeType })
}

async replaceMediaElementsWithResources(
mediaElements: Element[],
resources: EvernoteResource[],
canUploadFiles: boolean,
uploadFile: UploadFileFn,
): Promise<number> {
let replacedElements = 0
for (const mediaElement of mediaElements) {
const hash = mediaElement.getAttribute('hash')
const resource = resources.find((resource) => resource && resource.hash === hash)
if (!resource) {
continue
}
let resourceElement: HTMLElement = document.createElement('object')
resourceElement.setAttribute('type', resource.mimeType)
resourceElement.setAttribute('data', resource.data)
if (resource.mimeType.startsWith('image/')) {
resourceElement = document.createElement('img')
resourceElement.setAttribute('src', resource.data)
resourceElement.setAttribute('data-mime-type', resource.mimeType)
} else if (resource.mimeType.startsWith('audio/')) {
resourceElement = document.createElement('audio')
resourceElement.setAttribute('controls', 'controls')
const sourceElement = document.createElement('source')
sourceElement.setAttribute('src', resource.data)
sourceElement.setAttribute('type', resource.mimeType)
resourceElement.appendChild(sourceElement)
} else if (resource.mimeType.startsWith('video/')) {
resourceElement = document.createElement('video')
resourceElement.setAttribute('controls', 'controls')
const sourceElement = document.createElement('source')
sourceElement.setAttribute('src', resource.data)
sourceElement.setAttribute('type', resource.mimeType)
resourceElement.appendChild(sourceElement)
}
resourceElement.setAttribute('data-filename', resource.fileName)
if (!mediaElement.parentNode) {
continue
}
mediaElement.parentNode.replaceChild(resourceElement, mediaElement)
const fileToUpload = canUploadFiles ? await this.getFileFromResource(resource) : undefined
const fileItem = fileToUpload ? await uploadFile(fileToUpload) : undefined
if (fileItem) {
const fileElement = document.createElement('span')
fileElement.setAttribute('data-lexical-file-uuid', fileItem.uuid)
mediaElement.parentNode.replaceChild(fileElement, mediaElement)
} else {
const resourceElement = this.getHTMLElementFromResource(resource)
mediaElement.parentNode.replaceChild(resourceElement, mediaElement)
}
replacedElements++
}
return replacedElements
Expand Down
Loading

0 comments on commit 29cf551

Please sign in to comment.