[WIP] Attempt to improve performance using Queues

package.json

@@ -25,6 +25,7 @@
     "@create-figma-plugin/ui": "^3.2",
     "base64-js": "^1.5",
     "classnames": "^2.5",
+    "fastq": "^1.17.1",
     "lru-cache": "^10.2",
     "preact": "^10.22",
     "react-hook-form": "^7.51",

plugin-src/Queue.ts

@@ -0,0 +1,30 @@
+import * as fastq from 'fastq';
+import type { queueAsPromised } from 'fastq';
+
+import { transformSceneNode } from '@plugin/transformers';
+
+import { PenpotNode } from '@ui/types';
+
+class Queue<T, R> {
+  private queue: queueAsPromised<T, R>;
+
+  constructor(worker: fastq.asyncWorker<unknown, T, R>, concurrency: number) {
+    this.queue = fastq.promise(worker, concurrency);
+  }
+
+  public enqueue(task: T): Promise<R> {
+    return this.queue.push(task);
+  }
+
+  public async waitIdle() {
+    await this.queue.drain();
+  }
+}
+
+export const nodeQueue = new Queue(
+  async ([sceneNode, position]: [SceneNode, number]): Promise<[PenpotNode | undefined, number]> => [
+    await transformSceneNode(sceneNode),
+    position
+  ],
+  1
+);

plugin-src/RemoteComponentLibrary.ts

@@ -1,33 +1,27 @@
-class RemoteComponentsLibrary {
-  private components: Record<string, ComponentNode | ComponentSetNode> = {};
-  private queue: string[] = [];
+import { PenpotNode } from '@ui/types';
 
-  public register(id: string, component: ComponentNode | ComponentSetNode) {
-    if (!Object.prototype.hasOwnProperty.call(this.components, id)) {
-      this.queue.push(id);
-    }
+class RemoteComponentsLibrary {
+  private components: PenpotNode[] = [];
+  private registry: Record<string, null> = {};
 
-    this.components[id] = component;
+  public add(component: PenpotNode) {
+    this.components.push(component);
   }
 
-  public get(id: string): ComponentNode | ComponentSetNode | undefined {
-    return this.components[id];
+  public register(id: string) {
+    this.registry[id] = null;
   }
 
-  public next(): ComponentNode | ComponentSetNode {
-    const lastKey = this.queue.pop();
-
-    if (!lastKey) throw new Error('No components to pop');
-
-    return this.components[lastKey];
+  public has(id: string): boolean {
+    return this.registry[id] === null;
   }
 
-  public remaining(): number {
-    return this.queue.length;
+  public total(): number {
+    return this.components.length;
   }
 
-  public total(): number {
-    return Object.keys(this.components).length;
+  public all(): PenpotNode[] {
+    return this.components;
   }
 }
 

plugin-src/transformers/partials/transformChildren.ts

@@ -6,13 +6,13 @@ const nodeActsAsMask = (node: SceneNode): boolean => {
   return 'isMask' in node && node.isMask;
 };
 
-export const transformChildren = async (node: ChildrenMixin): Promise<Children> => {
+export const transformChildren = (node: ChildrenMixin): Children => {
   const maskIndex = node.children.findIndex(nodeActsAsMask);
   const containsMask = maskIndex !== -1;
 
   return {
     children: containsMask
-      ? await translateMaskChildren(node.children, maskIndex)
-      : await translateChildren(node.children)
+      ? translateMaskChildren(node.children, maskIndex)
+      : translateChildren(node.children)
   };
 };

plugin-src/transformers/transformBooleanNode.ts

@@ -22,7 +22,7 @@ export const transformBooleanNode = async (node: BooleanOperationNode): Promise<
     name: node.name,
     boolType: translateBoolType(node.booleanOperation),
     ...transformFigmaIds(node),
-    ...(await transformChildren(node)),
+    ...transformChildren(node),
     ...transformFills(node),
     ...transformEffects(node),
     ...transformStrokes(node),

plugin-src/transformers/transformComponentNode.ts

@@ -18,7 +18,7 @@ import {
 
 import { ComponentRoot } from '@ui/types';
 
-export const transformComponentNode = async (node: ComponentNode): Promise<ComponentRoot> => {
+export const transformComponentNode = (node: ComponentNode): ComponentRoot => {
   componentsLibrary.register(node.id, {
     type: 'component',
     name: node.name,
@@ -33,7 +33,7 @@ export const transformComponentNode = async (node: ComponentNode): Promise<Compo
     ...transformProportion(node),
     ...transformLayoutAttributes(node, true),
     ...transformCornerRadius(node),
-    ...(await transformChildren(node)),
+    ...transformChildren(node),
     ...transformDimension(node),
     ...transformRotationAndPosition(node),
     ...transformConstraints(node),

plugin-src/transformers/transformDocumentNode.ts

@@ -1,5 +1,6 @@
 import { componentsLibrary } from '@plugin/ComponentLibrary';
 import { imagesLibrary } from '@plugin/ImageLibrary';
+import { nodeQueue } from '@plugin/Queue';
 import { remoteComponentLibrary } from '@plugin/RemoteComponentLibrary';
 import { translateRemoteChildren } from '@plugin/translators';
 import { sleep } from '@plugin/utils';
@@ -75,10 +76,12 @@ const processPages = async (node: DocumentNode): Promise<PenpotPage[]> => {
 export const transformDocumentNode = async (node: DocumentNode): Promise<PenpotDocument> => {
   const children = await processPages(node);
 
-  if (remoteComponentLibrary.remaining() > 0) {
+  await nodeQueue.waitIdle();
+
+  if (remoteComponentLibrary.total() > 0) {
     children.push({
       name: 'External Components',
-      children: await translateRemoteChildren()
+      children: remoteComponentLibrary.all()
     });
   }
 

plugin-src/transformers/transformFrameNode.ts

@@ -23,9 +23,9 @@ const isSectionNode = (node: FrameNode | SectionNode | ComponentSetNode): node i
   return node.type === 'SECTION';
 };
 
-export const transformFrameNode = async (
+export const transformFrameNode = (
   node: FrameNode | SectionNode | ComponentSetNode
-): Promise<FrameShape> => {
+): FrameShape => {
   let frameSpecificAttributes: Partial<FrameShape> = {};
   let referencePoint: Point = { x: node.absoluteTransform[0][2], y: node.absoluteTransform[1][2] };
 
@@ -60,7 +60,7 @@ export const transformFrameNode = async (
     ...referencePoint,
     ...frameSpecificAttributes,
     ...transformDimension(node),
-    ...(await transformChildren(node)),
+    ...transformChildren(node),
     ...transformSceneNode(node),
     ...transformOverrides(node)
   };

plugin-src/transformers/transformGroupNode.ts

@@ -11,20 +11,20 @@ import { transformChildren } from '@plugin/transformers/partials';
 
 import { GroupShape } from '@ui/lib/types/shapes/groupShape';
 
-export const transformGroupNode = async (node: GroupNode): Promise<GroupShape> => {
+export const transformGroupNode = (node: GroupNode): GroupShape => {
   return {
     ...transformFigmaIds(node),
     ...transformGroupNodeLike(node),
     ...transformEffects(node),
     ...transformBlend(node),
-    ...(await transformChildren(node)),
+    ...transformChildren(node),
     ...transformOverrides(node)
   };
 };
 
 export const transformGroupNodeLike = (
   node: BaseNodeMixin & LayoutMixin & SceneNodeMixin
-): GroupShape => {
+): Omit<GroupShape, 'children'> => {
   return {
     type: 'group',
     name: node.name,

plugin-src/transformers/transformInstanceNode.ts

@@ -1,4 +1,5 @@
 import { overridesLibrary } from '@plugin/OverridesLibrary';
+import { nodeQueue } from '@plugin/Queue';
 import { remoteComponentLibrary } from '@plugin/RemoteComponentLibrary';
 import {
   transformAutoLayout,
@@ -64,7 +65,7 @@ export const transformInstanceNode = async (
     ...transformRotationAndPosition(node),
     ...transformConstraints(node),
     ...transformAutoLayout(node),
-    ...(await transformChildren(node)),
+    ...transformChildren(node),
     ...transformOverrides(node)
   };
 };
@@ -78,11 +79,15 @@ const getPrimaryComponent = (mainComponent: ComponentNode): ComponentNode | Comp
 };
 
 const registerExternalComponents = (primaryComponent: ComponentNode | ComponentSetNode): void => {
-  if (remoteComponentLibrary.get(primaryComponent.id) !== undefined) {
+  if (remoteComponentLibrary.has(primaryComponent.id)) {
     return;
   }
 
-  remoteComponentLibrary.register(primaryComponent.id, primaryComponent);
+  remoteComponentLibrary.register(primaryComponent.id);
+
+  nodeQueue.enqueue([primaryComponent, 0]).then(([penpotNode, _]) => {
+    if (penpotNode) remoteComponentLibrary.add(penpotNode);
+  });
 };
 
 const getComponentTextPropertyOverrides = (

plugin-src/transformers/transformPageNode.ts

@@ -9,6 +9,6 @@ export const transformPageNode = async (node: PageNode): Promise<PenpotPage> =>
     options: {
       background: node.backgrounds.length ? translatePageFill(node.backgrounds[0]) : undefined
     },
-    children: await translateChildren(node.children)
+    children: translateChildren(node.children)
   };
 };

plugin-src/transformers/transformSceneNode.ts

@@ -54,7 +54,7 @@ export const transformSceneNode = async (node: SceneNode): Promise<PenpotNode |
       penpotNode = await transformBooleanNode(node);
       break;
     case 'COMPONENT':
-      penpotNode = await transformComponentNode(node);
+      penpotNode = transformComponentNode(node);
       break;
     case 'INSTANCE':
       penpotNode = await transformInstanceNode(node);

plugin-src/translators/translateChildren.ts

@@ -1,7 +1,6 @@
-import { remoteComponentLibrary } from '@plugin/RemoteComponentLibrary';
-import { transformGroupNodeLike, transformSceneNode } from '@plugin/transformers';
+import { nodeQueue } from '@plugin/Queue';
+import { transformGroupNodeLike } from '@plugin/transformers';
 import { transformMaskFigmaIds } from '@plugin/transformers/partials';
-import { sleep } from '@plugin/utils';
 
 import { PenpotNode } from '@ui/types';
 
@@ -14,14 +13,14 @@ import { PenpotNode } from '@ui/types';
  *
  * @maskIndex The index of the mask node in the children array
  */
-export const translateMaskChildren = async (
+export const translateMaskChildren = (
   children: readonly SceneNode[],
   maskIndex: number
-): Promise<PenpotNode[]> => {
+): PenpotNode[] => {
   const maskChild = children[maskIndex];
 
-  const unmaskedChildren = await translateChildren(children.slice(0, maskIndex));
-  const maskedChildren = await translateChildren(children.slice(maskIndex));
+  const unmaskedChildren = translateChildren(children.slice(0, maskIndex));
+  const maskedChildren = translateChildren(children.slice(maskIndex));
 
   if (
     maskChild.type === 'STICKY' ||
@@ -48,47 +47,14 @@ export const translateMaskChildren = async (
   return [...unmaskedChildren, maskGroup];
 };
 
-export const translateChildren = async (children: readonly SceneNode[]): Promise<PenpotNode[]> => {
+export const translateChildren = (children: readonly SceneNode[]): PenpotNode[] => {
   const transformedChildren: PenpotNode[] = [];
+  let count = 0;
 
   for (const child of children) {
-    const penpotNode = await transformSceneNode(child);
-
-    if (penpotNode) transformedChildren.push(penpotNode);
-
-    await sleep(0);
-  }
-
-  return transformedChildren;
-};
-
-export const translateRemoteChildren = async (): Promise<PenpotNode[]> => {
-  const transformedChildren: PenpotNode[] = [];
-  let currentRemote = 1;
-
-  figma.ui.postMessage({
-    type: 'PROGRESS_STEP',
-    data: 'remote'
-  });
-
-  while (remoteComponentLibrary.remaining() > 0) {
-    figma.ui.postMessage({
-      type: 'PROGRESS_TOTAL_ITEMS',
-      data: remoteComponentLibrary.total()
-    });
-
-    const child = remoteComponentLibrary.next();
-
-    const penpotNode = await transformSceneNode(child);
-
-    if (penpotNode) transformedChildren.push(penpotNode);
-
-    figma.ui.postMessage({
-      type: 'PROGRESS_PROCESSED_ITEMS',
-      data: currentRemote++
+    nodeQueue.enqueue([child, count++]).then(([penpotNode, position]) => {
+      if (penpotNode) transformedChildren[position] = penpotNode;
     });
-
-    await sleep(0);
   }
 
   return transformedChildren;