feat(ui): canvas error handling

invokeai/frontend/web/src/features/controlLayers/components/Filters/Filter.tsx

@@ -106,7 +106,6 @@ const FilterBox = memo(({ adapter }: { adapter: CanvasEntityAdapterRasterLayer |
           variant="ghost"
           leftIcon={<PiXBold />}
           onClick={adapter.filterer.cancel}
-          isLoading={isProcessing}
           loadingText={t('controlLayers.filter.cancel')}
         >
           {t('controlLayers.filter.cancel')}

invokeai/frontend/web/src/features/controlLayers/konva/CanvasCompositorModule.ts

@@ -14,7 +14,7 @@ import { selectAutoAddBoardId } from 'features/gallery/store/gallerySelectors';
 import { atom, computed } from 'nanostores';
 import type { Logger } from 'roarr';
 import type { UploadOptions } from 'services/api/endpoints/images';
-import { getImageDTO, uploadImage } from 'services/api/endpoints/images';
+import { getImageDTOSafe, uploadImage } from 'services/api/endpoints/images';
 import type { ImageDTO } from 'services/api/types';
 import stableHash from 'stable-hash';
 import { assert } from 'tsafe';
@@ -210,7 +210,7 @@ export class CanvasCompositorModule extends CanvasModuleBase {
     const cachedImageName = this.manager.cache.imageNameCache.get(hash);
 
     if (cachedImageName) {
-      imageDTO = await getImageDTO(cachedImageName);
+      imageDTO = await getImageDTOSafe(cachedImageName);
       if (imageDTO) {
         this.log.trace({ rect, imageName: cachedImageName, imageDTO }, 'Using cached composite raster layer image');
         return imageDTO;
@@ -374,7 +374,7 @@ export class CanvasCompositorModule extends CanvasModuleBase {
     const cachedImageName = this.manager.cache.imageNameCache.get(hash);
 
     if (cachedImageName) {
-      imageDTO = await getImageDTO(cachedImageName);
+      imageDTO = await getImageDTOSafe(cachedImageName);
       if (imageDTO) {
         this.log.trace({ rect, cachedImageName, imageDTO }, 'Using cached composite inpaint mask image');
         return imageDTO;

invokeai/frontend/web/src/features/controlLayers/konva/CanvasEntity/CanvasEntityFilterer.ts

@@ -1,5 +1,4 @@
-import type { SerializableObject } from 'common/types';
-import { withResultAsync } from 'common/util/result';
+import { withResult, withResultAsync } from 'common/util/result';
 import type { CanvasEntityAdapterControlLayer } from 'features/controlLayers/konva/CanvasEntity/CanvasEntityAdapterControlLayer';
 import type { CanvasEntityAdapterRasterLayer } from 'features/controlLayers/konva/CanvasEntity/CanvasEntityAdapterRasterLayer';
 import type { CanvasManager } from 'features/controlLayers/konva/CanvasManager';
@@ -13,9 +12,9 @@ import { imageDTOToImageObject } from 'features/controlLayers/store/util';
 import { debounce } from 'lodash-es';
 import { atom } from 'nanostores';
 import type { Logger } from 'roarr';
-import { getImageDTO } from 'services/api/endpoints/images';
+import { serializeError } from 'serialize-error';
 import { buildSelectModelConfig } from 'services/api/hooks/modelsByType';
-import { type BatchConfig, type ImageDTO, isControlNetOrT2IAdapterModelConfig, type S } from 'services/api/types';
+import { isControlNetOrT2IAdapterModelConfig } from 'services/api/types';
 import { assert } from 'tsafe';
 
 type CanvasEntityFiltererConfig = {
@@ -38,6 +37,11 @@ export class CanvasEntityFilterer extends CanvasModuleBase {
   subscriptions = new Set<() => void>();
   config: CanvasEntityFiltererConfig = DEFAULT_CONFIG;
 
+  /**
+   * The AbortController used to cancel the filter processing.
+   */
+  abortController: AbortController | null = null;
+
   $isFiltering = atom<boolean>(false);
   $hasProcessed = atom<boolean>(false);
   $isProcessing = atom<boolean>(false);
@@ -100,63 +104,82 @@ export class CanvasEntityFilterer extends CanvasModuleBase {
 
   processImmediate = async () => {
     const config = this.$filterConfig.get();
-    const isValid = IMAGE_FILTERS[config.type].validateConfig?.(config as never) ?? true;
+    const filterData = IMAGE_FILTERS[config.type];
+
+    // Cannot get TS to be happy with `config`, thinks it should be `never`... eh...
+    const isValid = filterData.validateConfig?.(config as never) ?? true;
     if (!isValid) {
+      this.log.error({ config }, 'Invalid filter config');
       return;
     }
 
-    this.log.trace({ config }, 'Previewing filter');
+    this.log.trace({ config }, 'Processing filter');
     const rect = this.parent.transformer.getRelativeRect();
-    const imageDTO = await this.parent.renderer.rasterize({ rect, attrs: { filters: [], opacity: 1 } });
-    const nodeId = getPrefixedId('filter_node');
-    const batch = this.buildBatchConfig(imageDTO, config, nodeId);
-
-    // Listen for the filter processing completion event
-    const completedListener = async (event: S['InvocationCompleteEvent']) => {
-      if (event.origin !== this.id || event.invocation_source_id !== nodeId) {
-        return;
-      }
-      this.manager.socket.off('invocation_complete', completedListener);
-      this.manager.socket.off('invocation_error', errorListener);
-
-      this.log.trace({ event } as SerializableObject, 'Handling filter processing completion');
 
-      const { result } = event;
-      assert(result.type === 'image_output', `Processor did not return an image output, got: ${result}`);
-
-      const imageDTO = await getImageDTO(result.image.image_name);
-      assert(imageDTO, "Failed to fetch processor output's image DTO");
+    const rasterizeResult = await withResultAsync(() =>
+      this.parent.renderer.rasterize({ rect, attrs: { filters: [], opacity: 1 } })
+    );
+    if (rasterizeResult.isErr()) {
+      this.log.error({ error: serializeError(rasterizeResult.error) }, 'Error rasterizing entity');
+      this.$isProcessing.set(false);
+      return;
+    }
 
-      this.imageState = imageDTOToImageObject(imageDTO);
+    this.$isProcessing.set(true);
 
-      await this.parent.bufferRenderer.setBuffer(this.imageState, true);
+    const imageDTO = rasterizeResult.value;
 
+    // Cannot get TS to be happy with `config`, thinks it should be `never`... eh...
+    const buildGraphResult = withResult(() => filterData.buildGraph(imageDTO, config as never));
+    if (buildGraphResult.isErr()) {
+      this.log.error({ error: serializeError(buildGraphResult.error) }, 'Error building filter graph');
       this.$isProcessing.set(false);
-      this.$hasProcessed.set(true);
-    };
-    const errorListener = (event: S['InvocationErrorEvent']) => {
-      if (event.origin !== this.id || event.invocation_source_id !== nodeId) {
-        return;
-      }
-      this.manager.socket.off('invocation_complete', completedListener);
-      this.manager.socket.off('invocation_error', errorListener);
-
-      this.log.error({ event } as SerializableObject, 'Error processing filter');
+      return;
+    }
+
+    const controller = new AbortController();
+    this.abortController = controller;
+
+    const { graph, outputNodeId } = buildGraphResult.value;
+    const filterResult = await withResultAsync(() =>
+      this.manager.stateApi.runGraphAndReturnImageOutput({
+        graph,
+        outputNodeId,
+        // The filter graph should always be prepended to the queue so it's processed ASAP.
+        prepend: true,
+        /**
+         * The filter node may need to download a large model. Currently, the models required by the filter nodes are
+         * downloaded just-in-time, as required by the filter. If we use a timeout here, we might get into a catch-22
+         * where the filter node is waiting for the model to download, but the download gets canceled if the filter
+         * node times out.
+         *
+         * (I suspect the model download will actually _not_ be canceled if the graph is canceled, but let's not chance it!)
+         *
+         * TODO(psyche): Figure out a better way to handle this. Probably need to download the models ahead of time.
+         */
+        // timeout: 5000,
+        /**
+         * The filter node should be able to cancel the request if it's taking too long. This will cancel the graph's
+         * queue item and clear any event listeners on the request.
+         */
+        signal: controller.signal,
+      })
+    );
+    if (filterResult.isErr()) {
+      this.log.error({ error: serializeError(filterResult.error) }, 'Error processing filter');
       this.$isProcessing.set(false);
-    };
+      this.abortController = null;
+      return;
+    }
 
-    this.manager.socket.on('invocation_complete', completedListener);
-    this.manager.socket.on('invocation_error', errorListener);
+    this.log.trace({ imageDTO: filterResult.value }, 'Filter processed');
+    this.imageState = imageDTOToImageObject(filterResult.value);
 
-    this.log.trace({ batch } as SerializableObject, 'Enqueuing filter batch');
+    await this.parent.bufferRenderer.setBuffer(this.imageState, true);
 
-    this.$isProcessing.set(true);
-    const req = this.manager.stateApi.enqueueBatch(batch);
-    const result = await withResultAsync(req.unwrap);
-    if (result.isErr()) {
-      this.$isProcessing.set(false);
-    }
-    req.reset();
+    this.$isProcessing.set(false);
+    this.$hasProcessed.set(true);
+    this.abortController = null;
   };
 
   process = debounce(this.processImmediate, this.config.processDebounceMs);
@@ -188,6 +211,8 @@ export class CanvasEntityFilterer extends CanvasModuleBase {
   reset = () => {
     this.log.trace('Resetting filter');
 
+    this.abortController?.abort();
+    this.abortController = null;
     this.parent.bufferRenderer.clearBuffer();
     this.parent.transformer.updatePosition();
     this.parent.renderer.syncCache(true);
@@ -205,31 +230,6 @@ export class CanvasEntityFilterer extends CanvasModuleBase {
     this.manager.stateApi.$filteringAdapter.set(null);
   };
 
-  buildBatchConfig = (imageDTO: ImageDTO, config: FilterConfig, id: string): BatchConfig => {
-    // TODO(psyche): I can't get TS to be happy, it thinkgs `config` is `never` but it should be inferred from the generic... I'll just cast it for now
-    const node = IMAGE_FILTERS[config.type].buildNode(imageDTO, config as never);
-    node.id = id;
-    const batch: BatchConfig = {
-      prepend: true,
-      batch: {
-        graph: {
-          nodes: {
-            [node.id]: {
-              ...node,
-              // filtered images are always intermediate - do not save to gallery
-              is_intermediate: true,
-            },
-          },
-          edges: [],
-        },
-        origin: this.id,
-        runs: 1,
-      },
-    };
-
-    return batch;
-  };
-
   repr = () => {
     return {
       id: this.id,

invokeai/frontend/web/src/features/controlLayers/konva/CanvasEntity/CanvasEntityObjectRenderer.ts

@@ -1,5 +1,6 @@
 import { $authToken } from 'app/store/nanostores/authToken';
 import { rgbColorToString } from 'common/util/colorCodeTransformers';
+import { withResult } from 'common/util/result';
 import { SyncableMap } from 'common/util/SyncableMap/SyncableMap';
 import type { CanvasEntityAdapter } from 'features/controlLayers/konva/CanvasEntity/types';
 import type { CanvasManager } from 'features/controlLayers/konva/CanvasManager';
@@ -27,7 +28,7 @@ import { debounce } from 'lodash-es';
 import { atom } from 'nanostores';
 import type { Logger } from 'roarr';
 import { serializeError } from 'serialize-error';
-import { getImageDTO, uploadImage } from 'services/api/endpoints/images';
+import { getImageDTOSafe, uploadImage } from 'services/api/endpoints/images';
 import type { ImageDTO } from 'services/api/types';
 import { assert } from 'tsafe';
 
@@ -356,14 +357,25 @@ export class CanvasEntityObjectRenderer extends CanvasModuleBase {
   };
 
   /**
-   * Rasterizes the parent entity. If the entity has a rasterization cache for the given rect, the cached image is
-   * returned. Otherwise, the entity is rasterized and the image is uploaded to the server.
+   * Rasterizes the parent entity, returning a promise that resolves to the image DTO.
+   *
+   * If the entity has a rasterization cache for the given rect, the cached image is returned. Otherwise, the entity is
+   * rasterized and the image is uploaded to the server.
    *
    * The rasterization cache is reset when the entity's state changes. The buffer object is not considered part of the
    * entity state for this purpose as it is a temporary object.
    *
-   * @param rect The rect to rasterize. If omitted, the entity's full rect will be used.
-   * @returns A promise that resolves to the rasterized image DTO.
+   * If rasterization fails for any reason, the promise will reject.
+   *
+   * @param options The rasterization options.
+   * @param options.rect The region of the entity to rasterize.
+   * @param options.replaceObjects Whether to replace the entity's objects with the rasterized image. If you just want
+   * the entity's image, omit or set this to false.
+   * @param options.attrs The Konva node attributes to apply to the rasterized image group. For example, you might want
+   * to disable filters or set the opacity to the rasterized image.
+   * @param options.bg Draws the entity on a canvas with the given background color. If omitted, the entity is drawn on
+   * a transparent canvas.
+   * @returns A promise that resolves to the rasterized image DTO or rejects if rasterization fails.
    */
   rasterize = async (options: {
     rect: Rect;
@@ -383,7 +395,7 @@ export class CanvasEntityObjectRenderer extends CanvasModuleBase {
     const cachedImageName = this.manager.cache.imageNameCache.get(hash);
 
     if (cachedImageName) {
-      imageDTO = await getImageDTO(cachedImageName);
+      imageDTO = await getImageDTOSafe(cachedImageName);
       if (imageDTO) {
         this.log.trace({ rect, cachedImageName, imageDTO }, 'Using cached rasterized image');
         return imageDTO;
@@ -423,26 +435,38 @@ export class CanvasEntityObjectRenderer extends CanvasModuleBase {
     if (this.parent.transformer.$isPendingRectCalculation.get()) {
       return;
     }
+
     const pixelRect = this.parent.transformer.$pixelRect.get();
     if (pixelRect.width === 0 || pixelRect.height === 0) {
       return;
     }
-    try {
-      // TODO(psyche): This is an internal Konva method, so it may break in the future. Can we make this API public?
-      const canvas = this.konva.objectGroup._getCachedSceneCanvas()._canvas as HTMLCanvasElement | undefined | null;
-      if (canvas) {
-        const nodeRect = this.parent.transformer.$nodeRect.get();
-        const rect = {
-          x: pixelRect.x - nodeRect.x,
-          y: pixelRect.y - nodeRect.y,
-          width: pixelRect.width,
-          height: pixelRect.height,
-        };
-        this.$canvasCache.set({ rect, canvas });
-      }
-    } catch (error) {
+
+    /**
+     * TODO(psyche): This is an internal Konva method, so it may break in the future. Can we make this API public?
+     *
+     * This method's API is unknown. It has been experimentally determined that it may throw, so we need to handle
+     * errors.
+     */
+    const getCacheCanvasResult = withResult(
+      () => this.konva.objectGroup._getCachedSceneCanvas()._canvas as HTMLCanvasElement | undefined | null
+    );
+    if (getCacheCanvasResult.isErr()) {
       // We are using an internal Konva method, so we need to catch any errors that may occur.
-      this.log.warn({ error: serializeError(error) }, 'Failed to update preview canvas');
+      this.log.warn({ error: serializeError(getCacheCanvasResult.error) }, 'Failed to update preview canvas');
+      return;
+    }
+
+    const canvas = getCacheCanvasResult.value;
+
+    if (canvas) {
+      const nodeRect = this.parent.transformer.$nodeRect.get();
+      const rect = {
+        x: pixelRect.x - nodeRect.x,
+        y: pixelRect.y - nodeRect.y,
+        width: pixelRect.width,
+        height: pixelRect.height,
+      };
+      this.$canvasCache.set({ rect, canvas });
     }
   }, 300);
 

invokeai/frontend/web/src/features/controlLayers/konva/CanvasEntity/CanvasEntityTransformer.ts

@@ -1,3 +1,4 @@
+import { withResultAsync } from 'common/util/result';
 import { roundToMultiple } from 'common/util/roundDownToMultiple';
 import type { CanvasEntityAdapter } from 'features/controlLayers/konva/CanvasEntity/types';
 import type { CanvasManager } from 'features/controlLayers/konva/CanvasManager';
@@ -15,6 +16,7 @@ import type { GroupConfig } from 'konva/lib/Group';
 import { debounce, get } from 'lodash-es';
 import { atom } from 'nanostores';
 import type { Logger } from 'roarr';
+import { serializeError } from 'serialize-error';
 import { assert } from 'tsafe';
 
 type CanvasEntityTransformerConfig = {
@@ -575,7 +577,12 @@ export class CanvasEntityTransformer extends CanvasModuleBase {
     this.log.debug('Applying transform');
     this.$isProcessing.set(true);
     const rect = this.getRelativeRect();
-    await this.parent.renderer.rasterize({ rect, replaceObjects: true, attrs: { opacity: 1, filters: [] } });
+    const rasterizeResult = await withResultAsync(() =>
+      this.parent.renderer.rasterize({ rect, replaceObjects: true, attrs: { opacity: 1, filters: [] } })
+    );
+    if (rasterizeResult.isErr()) {
+      this.log.error({ error: serializeError(rasterizeResult.error) }, 'Failed to rasterize entity');
+    }
     this.requestRectCalculation();
     this.stopTransform();
   };

invokeai/frontend/web/src/features/controlLayers/konva/CanvasObject/CanvasObjectImage.ts

@@ -11,7 +11,7 @@ import type { CanvasImageState } from 'features/controlLayers/store/types';
 import { t } from 'i18next';
 import Konva from 'konva';
 import type { Logger } from 'roarr';
-import { getImageDTO } from 'services/api/endpoints/images';
+import { getImageDTOSafe } from 'services/api/endpoints/images';
 
 export class CanvasObjectImage extends CanvasModuleBase {
   readonly type = 'object_image';
@@ -100,7 +100,7 @@ export class CanvasObjectImage extends CanvasModuleBase {
         this.konva.placeholder.text.text(t('common.loadingImage', 'Loading Image'));
       }
 
-      const imageDTO = await getImageDTO(imageName);
+      const imageDTO = await getImageDTOSafe(imageName);
       if (imageDTO === null) {
         this.onFailedToLoadImage();
         return;