Merge pull request #6125 from xtermjs/webp_avif_support

webp and avif support for IIP
2026-08-24 13:33:17 +02:00
parent 08ad9a4de9 0659493637
commit d3e32b344d
23 changed files with 125 additions and 23 deletions
@@ -98,7 +98,7 @@ terminal.loadAddon(imageAddon);
  Set by default, change it with `{iipSupport: true}`.

  The IIP implementation has the following features / restrictions (sequence will silently fail for unmet conditions):
-  - Supported formats: PNG, JPEG, GIF and QOI
+  - Supported formats: PNG, JPEG, GIF, QOI, WEBP and AVIF
  - No animation support.
  - Image type hinting is not supported (always deducted from data header).
  - File download is not supported.
@@ -112,6 +112,13 @@ terminal.loadAddon(imageAddon);
  - ReportCellSize sequence is supported.
  - Hi-res display support is currently limited (image gets blurred).

+- **Note on IIP image format support**  
+  For all image formats except QOI the addon only does the dimension extraction and leaves
+  the image decoding to the underlying browser engine. Note that some formats are implemented
+  and enabled with different constraints by browser vendors, especially newer formats
+  (e.g. under Firefox you might need to change the setting _image.avif.compliance_strictness_ to 0
+  to get broader AVIF support).
+
 - **Kitty Graphics Support (TGP)**  
  Set by default, change it with `{kittySupport: true}`.

@@ -39,7 +39,6 @@ export class IIPHandler implements IOscHandler, IResetHandler {
  private _header: IHeaderFields = DEFAULT_HEADER;
  private _dec: Base64Decoder;
  private _qoiDec: QoiDecoder;
-  private _metrics = UNSUPPORTED_TYPE;
  private _isMultipart = false;
  private _abortMulti = false;

@@ -63,7 +62,6 @@ export class IIPHandler implements IOscHandler, IResetHandler {

  public start(): void {
    this._aborted = false;
-    this._metrics  = UNSUPPORTED_TYPE;
    this._hp.reset();
  }

@@ -120,14 +118,14 @@ export class IIPHandler implements IOscHandler, IResetHandler {

    if (seqType === SequenceType.REPORTCELLSIZE) {
      // OSC 1337 ; ReportCellSize=[height];[width];[scale] ST
-      let width = CELL_SIZE_DEFAULT.width;
-      let height = CELL_SIZE_DEFAULT.height;
+      let w = CELL_SIZE_DEFAULT.width;
+      let h = CELL_SIZE_DEFAULT.height;
      if (this._renderer.dimensions) {
-        width = this._renderer.dimensions.css.canvas.width / this._coreTerminal.cols;
-        height = this._renderer.dimensions.css.canvas.height / this._coreTerminal.rows;
+        w = this._renderer.dimensions.css.canvas.width / this._coreTerminal.cols;
+        h = this._renderer.dimensions.css.canvas.height / this._coreTerminal.rows;
      }
      const scale = this._coreTerminal._core._coreBrowserService?.dpr ?? 1;
-      const report = `\x1b]1337;ReportCellSize=${height.toFixed(3)};${width.toFixed(3)};${scale.toFixed(3)}\x1b\\`;
+      const report = `\x1b]1337;ReportCellSize=${h.toFixed(3)};${w.toFixed(3)};${scale.toFixed(3)}\x1b\\`;
      this._coreTerminal.input(report, false);
      return true;
    }
@@ -154,17 +152,24 @@ export class IIPHandler implements IOscHandler, IResetHandler {

    // early exit condition chain
    let cond: number | boolean;
+    let metrics = UNSUPPORTED_TYPE;
    if (cond = success) {
      if (cond = !this._dec.end()) {
-        this._metrics = imageType(this._dec.data8);
-        if (cond = this._metrics.mime !== 'unsupported') {
-          w = this._metrics.width;
-          h = this._metrics.height;
+        metrics = imageType(this._dec.data8);
+        if (cond = metrics.mime !== 'unsupported') {
+          w = metrics.width;
+          h = metrics.height;
          if (cond = w && h && w * h < this._opts.pixelLimit) {
            [w, h] = this._resize(w, h).map(Math.floor);
            cond = w && h && w * h < this._opts.pixelLimit;
+          } else {
+            console.warn(`IIP: image dimension issue ${metrics.width}x${metrics.height}`);
          }
+        } else {
+          console.warn('IIP: unsupported image type');
        }
+      } else {
+        console.warn('IIP: error during BASE64 decoding');
      }
    }
    if (!cond) {
@@ -173,7 +178,7 @@ export class IIPHandler implements IOscHandler, IResetHandler {
    }

    let blob: Blob | ImageData;
-    if (this._metrics.mime === 'image/qoi') {
+    if (metrics.mime === 'image/qoi') {
      const data = this._qoiDec.decode(this._dec.data8);
      blob = new ImageData(
        new Uint8ClampedArray(data.buffer, data.byteOffset, data.byteLength),
@@ -190,13 +195,17 @@ export class IIPHandler implements IOscHandler, IResetHandler {
        return true;
      }
    } else {
-      blob = new Blob([this._dec.data8], { type: this._metrics.mime });
+      blob = new Blob([this._dec.data8], { type: metrics.mime });
    }
    this._dec.release();
    return createImageBitmap(blob, { resizeWidth: w, resizeHeight: h })
      .then(bm => {
        this._storage.addImage(bm);
        return true;
+      })
+      .catch(e => {
+        console.warn(`IIP: decoding error ${metrics.mime} ${metrics.width}x${metrics.height}`, e);
+        return true;
      });
  }

@@ -32,6 +32,28 @@ const TEST_IMAGES: [string, IMetrics][] = [
  ['agfa-makernotes.jpg', { mime: 'image/jpeg', width: 8, height: 8 }],
  ['sony-alpha-6000.jpg', { mime: 'image/jpeg', width: 6000, height: 4000 }],
  ['dice.qoi', { mime: 'image/qoi', width: 800, height: 600 }],
+  // VP8
+  ['1.webp', { mime: 'image/webp', width: 550, height: 368 }],
+  ['2.webp', { mime: 'image/webp', width: 550, height: 404 }],
+  ['3.webp', { mime: 'image/webp', width: 1280, height: 720 }],
+  ['4.webp', { mime: 'image/webp', width: 1024, height: 772 }],
+  ['5.webp', { mime: 'image/webp', width: 1024, height: 752 }],
+  // VP8X
+  ['1_webp_a.webp', { mime: 'image/webp', width: 400, height: 301 }],
+  ['2_webp_a.webp', { mime: 'image/webp', width: 386, height: 395 }],
+  ['3_webp_a.webp', { mime: 'image/webp', width: 800, height: 600 }],
+  ['4_webp_a.webp', { mime: 'image/webp', width: 421, height: 163 }],
+  ['5_webp_a.webp', { mime: 'image/webp', width: 300, height: 300 }],
+  // VP8L
+  ['1_webp_ll.webp', { mime: 'image/webp', width: 400, height: 301 }],
+  ['2_webp_ll.webp', { mime: 'image/webp', width: 386, height: 395 }],
+  ['3_webp_ll.webp', { mime: 'image/webp', width: 800, height: 600 }],
+  ['4_webp_ll.webp', { mime: 'image/webp', width: 421, height: 163 }],
+  ['5_webp_ll.webp', { mime: 'image/webp', width: 300, height: 300 }],
+  // some AVIF test images
+  ['fox.profile2.12bpc.yuv444.odd-width.odd-height.avif', { mime: 'image/avif', width: 1203, height: 799 }],
+  ['hato.profile0.8bpc.yuv420.avif', { mime: 'image/avif', width: 3082, height: 2048 }],
+  ['kimono.crop.avif', { mime: 'image/avif', width: 722, height: 1024 }],
 ];


@@ -4,7 +4,7 @@
 */


-export type ImageType = 'image/png' | 'image/jpeg' | 'image/gif' | 'image/qoi' | 'unsupported' | '';
+export type ImageType = 'image/png' | 'image/jpeg' | 'image/gif' | 'image/qoi' | 'image/webp' | 'image/avif' | 'unsupported' | '';

 export interface IMetrics {
  mime: ImageType;
@@ -19,16 +19,16 @@ export const UNSUPPORTED_TYPE: IMetrics = {
 };

 export function imageType(d: Uint8Array): IMetrics {
-  if (d.length < 24) {
+  if (d.length < 32) {
    return UNSUPPORTED_TYPE;
  }
-  const d32 = new Uint32Array(d.buffer, d.byteOffset, 6);
+  const d32 = new Uint32Array(d.buffer, d.byteOffset, 8);
  // PNG: 89 50 4E 47 0D 0A 1A 0A (8 first bytes == magic number for PNG)
  // + first chunk must be IHDR
  if (d32[0] === 0x474E5089 && d32[1] === 0x0A1A0A0D && d32[3] === 0x52444849) {
    return {
      mime: 'image/png',
-      width: d[16] << 24 | d[17] << 16 | d[18] << 8 | d[19],
+      width:  d[16] << 24 | d[17] << 16 | d[18] << 8 | d[19],
      height: d[20] << 24 | d[21] << 16 | d[22] << 8 | d[23]
    };
  }
@@ -41,7 +41,7 @@ export function imageType(d: Uint8Array): IMetrics {
  if (d32[0] === 0x38464947 && (d[4] === 0x37 || d[4] === 0x39) && d[5] === 0x61) {
    return {
      mime: 'image/gif',
-      width: d[7] << 8 | d[6],
+      width:  d[7] << 8 | d[6],
      height: d[9] << 8 | d[8]
    };
  }
@@ -49,13 +49,71 @@ export function imageType(d: Uint8Array): IMetrics {
  if (d32[0] === 0x66696F71) {
    return {
      mime: 'image/qoi',
-      width: d[4] << 24 | d[5] << 16 | d[6] << 8 | d[7],
+      width:  d[4] << 24 | d[5] << 16 | d[6] << 8 | d[7],
      height: d[8] << 24 | d[9] << 16 | d[10] << 8 | d[11]
    };
  }
+  // WEBP: RIFF | xxxx | WEBP | VP8x
+  if (d32[0] === 0x46464952 && d32[2] === 0x50424557 && (d32[3] & 0xFFFFFF) === 0x385056) {
+    switch (d[15]) {
+      case 0x58:  // Extended WebP VP8X --> "X"
+        return {
+          mime: 'image/webp',
+          width:  (d[24] | d[25] << 8 | d[26] << 16) + 1,
+          height: (d[27] | d[28] << 8 | d[29] << 16) + 1
+        };
+      case 0x4C:  // Lossless WebP VP8L --> "L"
+        if (d[20] !== 0x2f) return UNSUPPORTED_TYPE;
+        const dim = d[21] | d[22] << 8 | d[23] << 16 | d[24] << 24;
+        return {
+          mime: 'image/webp',
+          width:  (dim        & 0x3FFF) + 1,
+          height: (dim >>> 14 & 0x3FFF) + 1
+        };
+      case 0x20:  // Lossy WebP VP8  --> " "
+        if (d[23] !== 0x9d || d[24] !== 0x01 || d[25] !== 0x2a) return UNSUPPORTED_TYPE;
+        return {
+          mime: 'image/webp',
+          width:  (d[26] | d[27] << 8) & 0x3FFF,
+          height: (d[28] | d[29] << 8) & 0x3FFF
+        };
+    }
+    return UNSUPPORTED_TYPE;
+  }
+  // AVIF: Box size | ftyp | avif/avis
+  if (d32[1] === 0x70797466 && (d32[2] === 0x66697661 || d32[2] === 0x73697661)) {
+    let pos = -1;
+    // search for ispe box within first 1024 bytes
+    const limit = Math.min(d.length - 16, 1024);
+    for (let i = 8; i < limit; i++) {
+      // scan for ispe
+      if (d[i] === 0x69 && d[i + 1] === 0x73 && d[i + 2] === 0x70 && d[i + 3] === 0x65) {
+        pos = i;
+        break;
+      }
+    }
+    if (pos !== -1) {
+      // dimensions are in BE at +8 (width) at +12 (height)
+      const width =
+        d[pos +  8] << 24 |
+        d[pos +  9] << 16 |
+        d[pos + 10] <<  8 |
+        d[pos + 11];
+      const height =
+        d[pos + 12] << 24 |
+        d[pos + 13] << 16 |
+        d[pos + 14] <<  8 |
+        d[pos + 15];
+      if (width > 0 && height > 0) {
+        return { mime: 'image/avif', width, height };
+      }
+    }
+    return UNSUPPORTED_TYPE;
+  }
  return UNSUPPORTED_TYPE;
 }

+
 function jpgSize(d: Uint8Array): [number, number] {
  const len = d.length;
  let i = 4;
@@ -67,6 +67,8 @@ export class AddonImageWindow extends BaseWindow implements IControlWindow {
    dlIip.appendChild(dtIip);
    this._addDdWithButton(dlIip, 'image-demo3', 'palette (png File)');
    this._addDdWithButton(dlIip, 'image-demo4', 'dice (qoi MultipartFile)');
+    this._addDdWithButton(dlIip, 'image-demo5', 'rose (webp File)');
+    this._addDdWithButton(dlIip, 'image-demo6', 'kimono (avif MultipartFile)');
    container.appendChild(dlIip);

    // Kitty demos
@@ -173,11 +175,15 @@ export class AddonImageWindow extends BaseWindow implements IControlWindow {
    document.getElementById('image-demo2')!.addEventListener('click',
      sixelDemo('https://raw.githubusercontent.com/jerch/node-sixel/master/testfiles/test2.sixel'));
    document.getElementById('image-demo3')!.addEventListener('click',
-      iipDemo('https://raw.githubusercontent.com/jerch/node-sixel/master/palette.png'));
+      iipDemo('https://raw.githubusercontent.com/xtermjs/xterm.js/master/addons/addon-image/fixture/palette.png'));
    document.getElementById('image-demo4')!.addEventListener('click',
-      iipDemoMulti('https://raw.githubusercontent.com/jerch/xterm-wasm-parts/master/fixtures/qoi/dice.qoi'));
+      iipDemoMulti('https://raw.githubusercontent.com/xtermjs/xterm.js/master/addons/addon-image/fixture/testimages/dice.qoi'));
+    document.getElementById('image-demo5')!.addEventListener('click',
+      iipDemo('https://raw.githubusercontent.com/xtermjs/xterm.js/master/addons/addon-image/fixture/testimages/1_webp_a.webp'));
+    document.getElementById('image-demo6')!.addEventListener('click',
+      iipDemoMulti('https://raw.githubusercontent.com/xtermjs/xterm.js/master/addons/addon-image/fixture/testimages/kimono.crop.avif'));
    document.getElementById('image-demo-kitty1')!.addEventListener('click',
-      kittyDemo('https://raw.githubusercontent.com/jerch/node-sixel/master/palette.png'));
+      kittyDemo('https://raw.githubusercontent.com/xtermjs/xterm.js/master/addons/addon-image/fixture/palette.png'));

    // demo for image retrieval API
    this._terminal.element!.addEventListener('click', (ev: MouseEvent) => {