quick demo of complex serialization

.github/workflows/node.js.yml

@@ -28,6 +28,6 @@ jobs:
     - run: npm install yarn
     - run: |
         XTERMJS=5.1.0 IMAGEADDON=${{ github.head_ref || github.ref_name }} ./bootstrap.sh
-    - run: cd xterm-addon-image/xterm.js/addons/xterm-addon-image && node_modules/.bin/inwasm out/base64.wasm.js
+    - run: cd xterm-addon-image/xterm.js/addons/xterm-addon-image && node_modules/.bin/inwasm out/*.wasm.js
     - run: cd xterm-addon-image/xterm.js && yarn test
     - run: cd xterm-addon-image/xterm.js && yarn test-api-chromium --headless

package.json

@@ -7,7 +7,7 @@
   "repository": "https://github.com/xtermjs/xterm.js",
   "license": "MIT",
   "scripts": {
-    "build": "../../node_modules/.bin/tsc -p src && node_modules/.bin/inwasm out/base64.wasm.js",
+    "build": "../../node_modules/.bin/tsc -p src && node_modules/.bin/inwasm out/*.wasm.js",
     "prepackage": "npm run build",
     "package": "../../node_modules/.bin/webpack",
     "prepublishOnly": "npm run package"

src/IIPHandler.ts

@@ -9,6 +9,7 @@ import { ImageStorage, CELL_SIZE_DEFAULT } from './ImageStorage';
 import { Base64Decoder } from './base64.wasm';
 import { HeaderParser, IHeaderFields, HeaderState } from './IIPHeaderParser';
 import { imageType, UNSUPPORTED_TYPE } from './IIPMetrics';
+import { QoiDecoder } from './QoiDecoder.wasm';
 
 
 // eslint-disable-next-line
@@ -34,6 +35,7 @@ export class IIPHandler implements IOscHandler, IResetHandler {
   private _header: IHeaderFields = DEFAULT_HEADER;
   private _dec = new Base64Decoder(KEEP_DATA);
   private _metrics = UNSUPPORTED_TYPE;
+  private _qoiDec = new QoiDecoder(KEEP_DATA);
 
   constructor(
     private readonly _opts: IImageAddonOptions,
@@ -95,7 +97,8 @@ export class IIPHandler implements IOscHandler, IResetHandler {
           w = this._metrics.width;
           h = this._metrics.height;
           if (cond = w && h && w * h < this._opts.pixelLimit) {
-            [w, h] = this._resize(w, h).map(Math.floor);
+            // ceil here to allow a 1px overprint (avoid stitching artefacts)
+            [w, h] = this._resize(w, h).map(Math.ceil);
             cond = w && h && w * h < this._opts.pixelLimit;
           }
         }
@@ -106,26 +109,48 @@ export class IIPHandler implements IOscHandler, IResetHandler {
       return true;
     }
 
-    const blob = new Blob([this._dec.data8], { type: this._metrics.mime });
+    let blob: Blob | ImageData;
+    if (this._metrics.mime === 'image/qoi') {
+      const data = this._qoiDec.decode(this._dec.data8);
+      blob = new ImageData(
+        new Uint8ClampedArray(data.buffer, data.byteOffset, data.byteLength),
+        this._qoiDec.width,
+        this._qoiDec.height
+      );
+      this._qoiDec.release();
+    } else {
+      blob = new Blob([this._dec.data8], { type: this._metrics.mime });
+    }
+
+    // const blob = new Blob([this._dec.data8], { type: this._metrics.mime });
     this._dec.release();
 
     const win = this._coreTerminal._core._coreBrowserService.window;
     if (!win.createImageBitmap) {
-      const url = URL.createObjectURL(blob);
-      const img = new Image();
-      return new Promise<boolean>(r => {
-        img.addEventListener('load', () => {
-          URL.revokeObjectURL(url);
-          const canvas = ImageRenderer.createCanvas(win, w, h);
-          canvas.getContext('2d')?.drawImage(img, 0, 0, w, h);
-          this._storage.addImage(canvas);
-          r(true);
+      if (blob instanceof Blob) {
+        const url = URL.createObjectURL(blob);
+        const img = new Image();
+        return new Promise<boolean>(r => {
+          img.addEventListener('load', () => {
+            URL.revokeObjectURL(url);
+            const canvas = ImageRenderer.createCanvas(win, w, h);
+            canvas.getContext('2d')?.drawImage(img, 0, 0, w, h);
+            this._storage.addImage(canvas);
+            r(true);
+          });
+          img.src = url;
+          // sanity measure to avoid terminal blocking from dangling promise
+          // happens from corrupt data (onload never gets fired)
+          setTimeout(() => r(true), 1000);
         });
-        img.src = url;
-        // sanity measure to avoid terminal blocking from dangling promise
-        // happens from corrupt data (onload never gets fired)
-        setTimeout(() => r(true), 1000);
-      });
+      }
+      // qoi path
+      const c1 = ImageRenderer.createCanvas(win, this._metrics.width, this._metrics.height);
+      c1.getContext('2d')?.putImageData(blob, 0, 0);
+      const c2 = ImageRenderer.createCanvas(win, w, h);
+      c2.getContext('2d')?.drawImage(c1, 0, 0, this._metrics.width, this._metrics.height, 0, 0, w, h);
+      this._storage.addImage(c2);
+      return true;
     }
     return win.createImageBitmap(blob, { resizeWidth: w, resizeHeight: h })
       .then(bm => {

src/IIPMetrics.ts

@@ -4,7 +4,7 @@
  */
 
 
-export type ImageType = 'image/png' | 'image/jpeg' | 'image/gif' | 'unsupported' | '';
+export type ImageType = 'image/png' | 'image/jpeg' | 'image/gif' | 'image/qoi' | 'unsupported' | '';
 
 export interface IMetrics {
   mime: ImageType;
@@ -50,6 +50,14 @@ export function imageType(d: Uint8Array): IMetrics {
       height: d[9] << 8 | d[8]
     };
   }
+  // QOI: qoif
+  if (d32[0] === 0x66696F71) {
+    return {
+      mime: 'image/qoi',
+      width: d[4] << 24 | d[5] << 16 | d[6] << 8 | d[7],
+      height: d[8] << 24 | d[9] << 16 | d[10] << 8 | d[11]
+    };
+  }
   return UNSUPPORTED_TYPE;
 }
 

src/ImageAddon.ts

@@ -9,6 +9,8 @@ import { ImageRenderer } from './ImageRenderer';
 import { ImageStorage, CELL_SIZE_DEFAULT } from './ImageStorage';
 import { SixelHandler } from './SixelHandler';
 import { ITerminalExt, IImageAddonOptions, IResetHandler } from './Types';
+import { QoiEncoder } from './QoiEncoder.wasm';
+import { Base64Encoder } from './base64.wasm';
 
 
 // default values of addon ctor options
@@ -314,4 +316,85 @@ export class ImageAddon implements ITerminalAddon {
     this._report(`\x1b[?${params[0]};${GaStatus.ITEM_ERROR}S`);
     return true;
   }
+
+  /**
+   * demo hack for complex terminal buffer serialization
+   */
+
+  private _td = new TextDecoder('latin1');
+  private _b64enc = new Base64Encoder(4194304);
+  public serializeLineQoi(num: number): string {
+    const canvas = this._storage!.extractLineCanvas(num);
+    if (!canvas) return '';
+    const w = this._terminal!.cols;
+    const rawData = this._qoiEnc.encode(
+      canvas.getContext('2d')!.getImageData(0, 0, canvas.width, canvas.height).data,
+      canvas.width,
+      canvas.height
+    );
+    const data = this._td.decode(this._b64enc.encode(rawData));
+    const iipSeq = `\x1b]1337;File=inline=1;width=${w};height=1;preserveAspectRatio=0;size=${rawData.length}:${data}`;
+    return '\r' + iipSeq + `\x1b[${w}C`; // CR + IIP sequence + cursor advance by line width
+  }
+  public serializeLinePng(num: number): string {
+    const canvas = this._storage!.extractLineCanvas(num);
+    if (!canvas) return '';
+    const w = this._terminal!.cols;
+    const data = canvas.toDataURL('image/png').slice(22);
+    const iipSeq = `\x1b]1337;File=inline=1;width=${w};height=1;preserveAspectRatio=0;size=${atob(data).length}:${data}`;
+    return '\r' + iipSeq + `\x1b[${w}C`; // CR + IIP sequence + cursor advance by line width
+  }
+
+  public serialize(start: number, end: number): string[] {
+    const st = Date.now();
+    const lines: string[] = [];
+    const buffer = this._terminal!._core.buffer;
+    for (let row = start; row < end; ++row) {
+      const line = buffer.lines.get(row);
+      if (!line) break;
+      // FIXME: hook into serialize addon instead of translateToString
+      // lines.push(line.translateToString(true) + this.serializeLinePng(row));
+      lines.push(line.translateToString(true) + this.serializeLineQoi(row));
+    }
+    console.log('duration', Date.now()-st);
+    return lines;
+  }
+
+  private _hCtx = document.createElement('canvas').getContext('2d', { willReadFrequently: true })!;
+  private _qoiEnc = new QoiEncoder(4194304);
+  public sQoi(): void {
+    const st = Date.now();
+    let src: CanvasRenderingContext2D;
+    let c = 0;
+    for (const spec of (this._storage as any)._images.values()) {
+      this._hCtx.canvas.width = spec.orig.width;
+      this._hCtx.canvas.height = spec.orig.height;
+      this._hCtx.drawImage(spec.orig, 0, 0);
+      src = this._hCtx;
+
+      // use custom QOI + base64 encoders
+      const data = this._qoiEnc.encode(
+        src.getImageData(0, 0, spec.orig.width, spec.orig.height).data,
+        spec.orig.width,
+        spec.orig.height
+      );
+      c += this._td.decode(this._b64enc.encode(data)).length;
+    }
+    console.log({ runtime: Date.now() - st, size: c });
+  }
+  public sPng(): void {
+    const st = Date.now();
+    let src: HTMLCanvasElement;
+    let c = 0;
+    for (const spec of (this._storage as any)._images.values()) {
+      this._hCtx.canvas.width = spec.orig.width;
+      this._hCtx.canvas.height = spec.orig.height;
+      this._hCtx.drawImage(spec.orig, 0, 0);
+      src = this._hCtx.canvas;
+
+      // use browser builtin PNG serialization
+      c += src.toDataURL('image/png').slice(22).length;
+    }
+    console.log({ runtime: Date.now() - st, size: c });
+  }
 }

src/ImageRenderer.ts

@@ -187,6 +187,39 @@ export class ImageRenderer implements IDisposable {
     );
   }
 
+  // TODO: cleanup, maybe merge with draw?
+  public lineDraw(ctx: CanvasRenderingContext2D, imgSpec: IImageSpec, tileId: number, col: number, row: number, count: number = 1): void {
+    const { width, height } = this.cellSize;
+
+    // Don't try to draw anything, if we cannot get valid renderer metrics.
+    if (width === -1 || height === -1) {
+      return;
+    }
+
+    this._rescaleImage(imgSpec, width, height);
+    const img = imgSpec.actual!;
+    const cols = Math.ceil(img.width / width);
+
+    const sx = (tileId % cols) * width;
+    const sy = Math.floor(tileId / cols) * height;
+    const dx = col * width;
+    // const dy = row * height;
+
+    // safari bug: never access image source out of bounds
+    const finalWidth = count * width + sx > img.width ? img.width - sx : count * width;
+    const finalHeight = sy + height > img.height ? img.height - sy : height;
+
+    // Floor all pixel offsets to get stable tile mapping without any overflows.
+    // Note: For not pixel perfect aligned cells like in the DOM renderer
+    // this will move a tile slightly to the top/left (subpixel range, thus ignore it).
+    // FIX #34: avoid striping on displays with pixelDeviceRatio != 1 by ceiling height and width
+    ctx.drawImage(
+      img,
+      Math.floor(sx), Math.floor(sy), Math.ceil(finalWidth), Math.ceil(finalHeight),
+      Math.floor(dx), 0, Math.ceil(finalWidth), Math.ceil(finalHeight)
+    );
+  }
+
   /**
    * Extract a single tile from an image.
    */
@@ -201,16 +234,26 @@ export class ImageRenderer implements IDisposable {
     const cols = Math.ceil(img.width / width);
     const sx = (tileId % cols) * width;
     const sy = Math.floor(tileId / cols) * height;
-    const finalWidth = width + sx > img.width ? img.width - sx : width;
-    const finalHeight = sy + height > img.height ? img.height - sy : height;
+
+    // Note on ceiling/flooring: We always floor left and ceil right borders
+    // for overprinting to avoid stitching artefact later on.
+    // This has a rather awkward downside of unstable tile dimensions (up to +2px)
+    // depending on the tile source position!
+    // This furthermore means, that any later composition from single tiles needs
+    // to shrink dimensions to fit back into place introducing resizing artefacts.
+    // Those resizing artefacts can be quite big for single cell tiles,
+    // as they have rather small base dimensions.
+    // FIXME: An better handling possible?
+    const finalWidth = Math.ceil(width + sx > img.width ? img.width - sx : width);
+    const finalHeight = Math.ceil(sy + height > img.height ? img.height - sy : height);
 
     const canvas = ImageRenderer.createCanvas(this._terminal._core._coreBrowserService.window, finalWidth, finalHeight);
     const ctx = canvas.getContext('2d');
     if (ctx) {
       ctx.drawImage(
         img,
-        Math.floor(sx), Math.floor(sy), Math.floor(finalWidth), Math.floor(finalHeight),
-        0, 0, Math.floor(finalWidth), Math.floor(finalHeight)
+        Math.floor(sx), Math.floor(sy), finalWidth, finalHeight,
+        0, 0, finalWidth, finalHeight
       );
       return canvas;
     }

src/ImageStorage.ts

@@ -223,7 +223,7 @@ export class ImageStorage implements IDisposable {
   /**
    * Method to add an image to the storage.
    */
-  public addImage(img: HTMLCanvasElement): void {
+  public addImage(img: HTMLCanvasElement | ImageBitmap): void {
     // never allow storage to exceed memory limit
     this._evictOldest(img.width * img.height);
 
@@ -232,8 +232,10 @@ export class ImageStorage implements IDisposable {
     if (cellSize.width === -1 || cellSize.height === -1) {
       cellSize = CELL_SIZE_DEFAULT;
     }
+    // cell coverage - ceil all to cover overprint pixels as well
+    // y-direction: height-1 to allow 1px overprint w'o cursor line progression
     const cols = Math.ceil(img.width / cellSize.width);
-    const rows = Math.ceil(img.height / cellSize.height);
+    const rows = Math.ceil((img.height - 1) / cellSize.height);
 
     const imageId = ++this._lastId;
 
@@ -482,6 +484,57 @@ export class ImageStorage implements IDisposable {
     }
   }
 
+  // TODO: cleanup, merge with render? Extend it to an x*y dim extraction method?
+  private _lineCtx = ImageRenderer.createCanvas(window, 0, 0).getContext('2d', { willReadFrequently: true })!;
+  public extractLineCanvas(num: number): HTMLCanvasElement | undefined {
+    if (!this._images.size) return;
+
+    const buffer = this._terminal._core.buffer;
+    const cols = this._terminal._core.cols;
+    const line = buffer.lines.get(num) as IBufferLineExt;
+    if (!line) return;
+
+    const cw = this._renderer.dimensions?.css.cell.width || CELL_SIZE_DEFAULT.width;
+    const ch = this._renderer.dimensions?.css.cell.height || CELL_SIZE_DEFAULT.height;
+    const width = this._renderer.dimensions?.css.canvas.width || cw * this._terminal.cols;
+
+    this._lineCtx.canvas.width = width;
+    this._lineCtx.canvas.height = Math.ceil(ch);
+
+    let hasTiles = false;
+
+    for (let col = 0; col < cols; ++col) {
+      if (line.getBg(col) & BgFlags.HAS_EXTENDED) {
+        let e: IExtendedAttrsImage = line._extendedAttrs[col] || EMPTY_ATTRS;
+        const imageId = e.imageId;
+        if (imageId === undefined || imageId === -1) {
+          continue;
+        }
+        const imgSpec = this._images.get(imageId);
+        if (e.tileId !== -1) {
+          const startTile = e.tileId;
+          const startCol = col;
+          let count = 1;
+          while (
+            ++col < cols
+            && (line.getBg(col) & BgFlags.HAS_EXTENDED)
+            && (e = line._extendedAttrs[col] || EMPTY_ATTRS)
+            && (e.imageId === imageId)
+            && (e.tileId === startTile + count)
+          ) {
+            count++;
+          }
+          col--;
+          if (imgSpec && imgSpec.actual) {
+            this._renderer.lineDraw(this._lineCtx, imgSpec, startTile, startCol, num, count);
+            hasTiles = true;
+          }
+        }
+      }
+    }
+    if (hasTiles) return this._lineCtx.canvas;
+  }
+
   /**
    * Extract active single tile at buffer position.
    */