diff --git a/addons/addon-image/README.md b/addons/addon-image/README.md index 3b636d922..8ea6a1795 100644 --- a/addons/addon-image/README.md +++ b/addons/addon-image/README.md @@ -98,7 +98,7 @@ terminal.loadAddon(imageAddon); Set by default, change it with `{iipSupport: true}`. The IIP implementation has the following features / restrictions (sequence will silently fail for unmet conditions): - - Supported formats: PNG, JPEG, GIF and QOI + - Supported formats: PNG, JPEG, GIF, QOI, WEBP and AVIF - No animation support. - Image type hinting is not supported (always deducted from data header). - File download is not supported. @@ -112,6 +112,13 @@ terminal.loadAddon(imageAddon); - ReportCellSize sequence is supported. - Hi-res display support is currently limited (image gets blurred). +- **Note on IIP image format support** + For all image formats except QOI the addon only does the dimension extraction and leaves + the image decoding to the underlying browser engine. Note that some formats are implemented + and enabled with different constraints by browser vendors, especially newer formats + (e.g. under Firefox you might need to change the setting _image.avif.compliance_strictness_ to 0 + to get broader AVIF support). + - **Kitty Graphics Support (TGP)** Set by default, change it with `{kittySupport: true}`. diff --git a/addons/addon-image/fixture/testimages/1.webp b/addons/addon-image/fixture/testimages/1.webp new file mode 100644 index 000000000..122741b60 Binary files /dev/null and b/addons/addon-image/fixture/testimages/1.webp differ diff --git a/addons/addon-image/fixture/testimages/1_webp_a.webp b/addons/addon-image/fixture/testimages/1_webp_a.webp new file mode 100644 index 000000000..fd78f9335 Binary files /dev/null and b/addons/addon-image/fixture/testimages/1_webp_a.webp differ diff --git a/addons/addon-image/fixture/testimages/1_webp_ll.webp b/addons/addon-image/fixture/testimages/1_webp_ll.webp new file mode 100644 index 000000000..1953c59b3 Binary files /dev/null and b/addons/addon-image/fixture/testimages/1_webp_ll.webp differ diff --git a/addons/addon-image/fixture/testimages/2.webp b/addons/addon-image/fixture/testimages/2.webp new file mode 100644 index 000000000..c833c5d4a Binary files /dev/null and b/addons/addon-image/fixture/testimages/2.webp differ diff --git a/addons/addon-image/fixture/testimages/2_webp_a.webp b/addons/addon-image/fixture/testimages/2_webp_a.webp new file mode 100644 index 000000000..0a8560a77 Binary files /dev/null and b/addons/addon-image/fixture/testimages/2_webp_a.webp differ diff --git a/addons/addon-image/fixture/testimages/2_webp_ll.webp b/addons/addon-image/fixture/testimages/2_webp_ll.webp new file mode 100644 index 000000000..7c5de8033 Binary files /dev/null and b/addons/addon-image/fixture/testimages/2_webp_ll.webp differ diff --git a/addons/addon-image/fixture/testimages/3.webp b/addons/addon-image/fixture/testimages/3.webp new file mode 100644 index 000000000..e1cf65837 Binary files /dev/null and b/addons/addon-image/fixture/testimages/3.webp differ diff --git a/addons/addon-image/fixture/testimages/3_webp_a.webp b/addons/addon-image/fixture/testimages/3_webp_a.webp new file mode 100644 index 000000000..b87117bf2 Binary files /dev/null and b/addons/addon-image/fixture/testimages/3_webp_a.webp differ diff --git a/addons/addon-image/fixture/testimages/3_webp_ll.webp b/addons/addon-image/fixture/testimages/3_webp_ll.webp new file mode 100644 index 000000000..32ba54bca Binary files /dev/null and b/addons/addon-image/fixture/testimages/3_webp_ll.webp differ diff --git a/addons/addon-image/fixture/testimages/4.webp b/addons/addon-image/fixture/testimages/4.webp new file mode 100644 index 000000000..a608fc85c Binary files /dev/null and b/addons/addon-image/fixture/testimages/4.webp differ diff --git a/addons/addon-image/fixture/testimages/4_webp_a.webp b/addons/addon-image/fixture/testimages/4_webp_a.webp new file mode 100644 index 000000000..d4fe93206 Binary files /dev/null and b/addons/addon-image/fixture/testimages/4_webp_a.webp differ diff --git a/addons/addon-image/fixture/testimages/4_webp_ll.webp b/addons/addon-image/fixture/testimages/4_webp_ll.webp new file mode 100644 index 000000000..311feac36 Binary files /dev/null and b/addons/addon-image/fixture/testimages/4_webp_ll.webp differ diff --git a/addons/addon-image/fixture/testimages/5.webp b/addons/addon-image/fixture/testimages/5.webp new file mode 100644 index 000000000..89e6075fd Binary files /dev/null and b/addons/addon-image/fixture/testimages/5.webp differ diff --git a/addons/addon-image/fixture/testimages/5_webp_a.webp b/addons/addon-image/fixture/testimages/5_webp_a.webp new file mode 100644 index 000000000..c29e6758d Binary files /dev/null and b/addons/addon-image/fixture/testimages/5_webp_a.webp differ diff --git a/addons/addon-image/fixture/testimages/5_webp_ll.webp b/addons/addon-image/fixture/testimages/5_webp_ll.webp new file mode 100644 index 000000000..172ebaee9 Binary files /dev/null and b/addons/addon-image/fixture/testimages/5_webp_ll.webp differ diff --git a/addons/addon-image/fixture/testimages/fox.profile2.12bpc.yuv444.odd-width.odd-height.avif b/addons/addon-image/fixture/testimages/fox.profile2.12bpc.yuv444.odd-width.odd-height.avif new file mode 100644 index 000000000..bfc513dcf Binary files /dev/null and b/addons/addon-image/fixture/testimages/fox.profile2.12bpc.yuv444.odd-width.odd-height.avif differ diff --git a/addons/addon-image/fixture/testimages/hato.profile0.8bpc.yuv420.avif b/addons/addon-image/fixture/testimages/hato.profile0.8bpc.yuv420.avif new file mode 100644 index 000000000..d37c2734b Binary files /dev/null and b/addons/addon-image/fixture/testimages/hato.profile0.8bpc.yuv420.avif differ diff --git a/addons/addon-image/fixture/testimages/kimono.crop.avif b/addons/addon-image/fixture/testimages/kimono.crop.avif new file mode 100644 index 000000000..777813e69 Binary files /dev/null and b/addons/addon-image/fixture/testimages/kimono.crop.avif differ diff --git a/addons/addon-image/src/IIPHandler.ts b/addons/addon-image/src/IIPHandler.ts index 56f5cdf35..559b90741 100644 --- a/addons/addon-image/src/IIPHandler.ts +++ b/addons/addon-image/src/IIPHandler.ts @@ -39,7 +39,6 @@ export class IIPHandler implements IOscHandler, IResetHandler { private _header: IHeaderFields = DEFAULT_HEADER; private _dec: Base64Decoder; private _qoiDec: QoiDecoder; - private _metrics = UNSUPPORTED_TYPE; private _isMultipart = false; private _abortMulti = false; @@ -63,7 +62,6 @@ export class IIPHandler implements IOscHandler, IResetHandler { public start(): void { this._aborted = false; - this._metrics = UNSUPPORTED_TYPE; this._hp.reset(); } @@ -120,14 +118,14 @@ export class IIPHandler implements IOscHandler, IResetHandler { if (seqType === SequenceType.REPORTCELLSIZE) { // OSC 1337 ; ReportCellSize=[height];[width];[scale] ST - let width = CELL_SIZE_DEFAULT.width; - let height = CELL_SIZE_DEFAULT.height; + let w = CELL_SIZE_DEFAULT.width; + let h = CELL_SIZE_DEFAULT.height; if (this._renderer.dimensions) { - width = this._renderer.dimensions.css.canvas.width / this._coreTerminal.cols; - height = this._renderer.dimensions.css.canvas.height / this._coreTerminal.rows; + w = this._renderer.dimensions.css.canvas.width / this._coreTerminal.cols; + h = this._renderer.dimensions.css.canvas.height / this._coreTerminal.rows; } const scale = this._coreTerminal._core._coreBrowserService?.dpr ?? 1; - const report = `\x1b]1337;ReportCellSize=${height.toFixed(3)};${width.toFixed(3)};${scale.toFixed(3)}\x1b\\`; + const report = `\x1b]1337;ReportCellSize=${h.toFixed(3)};${w.toFixed(3)};${scale.toFixed(3)}\x1b\\`; this._coreTerminal.input(report, false); return true; } @@ -154,17 +152,24 @@ export class IIPHandler implements IOscHandler, IResetHandler { // early exit condition chain let cond: number | boolean; + let metrics = UNSUPPORTED_TYPE; if (cond = success) { if (cond = !this._dec.end()) { - this._metrics = imageType(this._dec.data8); - if (cond = this._metrics.mime !== 'unsupported') { - w = this._metrics.width; - h = this._metrics.height; + metrics = imageType(this._dec.data8); + if (cond = metrics.mime !== 'unsupported') { + w = metrics.width; + h = metrics.height; if (cond = w && h && w * h < this._opts.pixelLimit) { [w, h] = this._resize(w, h).map(Math.floor); cond = w && h && w * h < this._opts.pixelLimit; + } else { + console.warn(`IIP: image dimension issue ${metrics.width}x${metrics.height}`); } + } else { + console.warn('IIP: unsupported image type'); } + } else { + console.warn('IIP: error during BASE64 decoding'); } } if (!cond) { @@ -173,7 +178,7 @@ export class IIPHandler implements IOscHandler, IResetHandler { } let blob: Blob | ImageData; - if (this._metrics.mime === 'image/qoi') { + if (metrics.mime === 'image/qoi') { const data = this._qoiDec.decode(this._dec.data8); blob = new ImageData( new Uint8ClampedArray(data.buffer, data.byteOffset, data.byteLength), @@ -190,13 +195,17 @@ export class IIPHandler implements IOscHandler, IResetHandler { return true; } } else { - blob = new Blob([this._dec.data8], { type: this._metrics.mime }); + blob = new Blob([this._dec.data8], { type: metrics.mime }); } this._dec.release(); return createImageBitmap(blob, { resizeWidth: w, resizeHeight: h }) .then(bm => { this._storage.addImage(bm); return true; + }) + .catch(e => { + console.warn(`IIP: decoding error ${metrics.mime} ${metrics.width}x${metrics.height}`, e); + return true; }); } diff --git a/addons/addon-image/src/IIPMetrics.test.ts b/addons/addon-image/src/IIPMetrics.test.ts index cc381e5cb..4df320a2c 100644 --- a/addons/addon-image/src/IIPMetrics.test.ts +++ b/addons/addon-image/src/IIPMetrics.test.ts @@ -32,6 +32,28 @@ const TEST_IMAGES: [string, IMetrics][] = [ ['agfa-makernotes.jpg', { mime: 'image/jpeg', width: 8, height: 8 }], ['sony-alpha-6000.jpg', { mime: 'image/jpeg', width: 6000, height: 4000 }], ['dice.qoi', { mime: 'image/qoi', width: 800, height: 600 }], + // VP8 + ['1.webp', { mime: 'image/webp', width: 550, height: 368 }], + ['2.webp', { mime: 'image/webp', width: 550, height: 404 }], + ['3.webp', { mime: 'image/webp', width: 1280, height: 720 }], + ['4.webp', { mime: 'image/webp', width: 1024, height: 772 }], + ['5.webp', { mime: 'image/webp', width: 1024, height: 752 }], + // VP8X + ['1_webp_a.webp', { mime: 'image/webp', width: 400, height: 301 }], + ['2_webp_a.webp', { mime: 'image/webp', width: 386, height: 395 }], + ['3_webp_a.webp', { mime: 'image/webp', width: 800, height: 600 }], + ['4_webp_a.webp', { mime: 'image/webp', width: 421, height: 163 }], + ['5_webp_a.webp', { mime: 'image/webp', width: 300, height: 300 }], + // VP8L + ['1_webp_ll.webp', { mime: 'image/webp', width: 400, height: 301 }], + ['2_webp_ll.webp', { mime: 'image/webp', width: 386, height: 395 }], + ['3_webp_ll.webp', { mime: 'image/webp', width: 800, height: 600 }], + ['4_webp_ll.webp', { mime: 'image/webp', width: 421, height: 163 }], + ['5_webp_ll.webp', { mime: 'image/webp', width: 300, height: 300 }], + // some AVIF test images + ['fox.profile2.12bpc.yuv444.odd-width.odd-height.avif', { mime: 'image/avif', width: 1203, height: 799 }], + ['hato.profile0.8bpc.yuv420.avif', { mime: 'image/avif', width: 3082, height: 2048 }], + ['kimono.crop.avif', { mime: 'image/avif', width: 722, height: 1024 }], ]; diff --git a/addons/addon-image/src/IIPMetrics.ts b/addons/addon-image/src/IIPMetrics.ts index b377cf061..217a7fc93 100644 --- a/addons/addon-image/src/IIPMetrics.ts +++ b/addons/addon-image/src/IIPMetrics.ts @@ -4,7 +4,7 @@ */ -export type ImageType = 'image/png' | 'image/jpeg' | 'image/gif' | 'image/qoi' | 'unsupported' | ''; +export type ImageType = 'image/png' | 'image/jpeg' | 'image/gif' | 'image/qoi' | 'image/webp' | 'image/avif' | 'unsupported' | ''; export interface IMetrics { mime: ImageType; @@ -19,16 +19,16 @@ export const UNSUPPORTED_TYPE: IMetrics = { }; export function imageType(d: Uint8Array): IMetrics { - if (d.length < 24) { + if (d.length < 32) { return UNSUPPORTED_TYPE; } - const d32 = new Uint32Array(d.buffer, d.byteOffset, 6); + const d32 = new Uint32Array(d.buffer, d.byteOffset, 8); // PNG: 89 50 4E 47 0D 0A 1A 0A (8 first bytes == magic number for PNG) // + first chunk must be IHDR if (d32[0] === 0x474E5089 && d32[1] === 0x0A1A0A0D && d32[3] === 0x52444849) { return { mime: 'image/png', - width: d[16] << 24 | d[17] << 16 | d[18] << 8 | d[19], + width: d[16] << 24 | d[17] << 16 | d[18] << 8 | d[19], height: d[20] << 24 | d[21] << 16 | d[22] << 8 | d[23] }; } @@ -41,7 +41,7 @@ export function imageType(d: Uint8Array): IMetrics { if (d32[0] === 0x38464947 && (d[4] === 0x37 || d[4] === 0x39) && d[5] === 0x61) { return { mime: 'image/gif', - width: d[7] << 8 | d[6], + width: d[7] << 8 | d[6], height: d[9] << 8 | d[8] }; } @@ -49,13 +49,71 @@ export function imageType(d: Uint8Array): IMetrics { if (d32[0] === 0x66696F71) { return { mime: 'image/qoi', - width: d[4] << 24 | d[5] << 16 | d[6] << 8 | d[7], + width: d[4] << 24 | d[5] << 16 | d[6] << 8 | d[7], height: d[8] << 24 | d[9] << 16 | d[10] << 8 | d[11] }; } + // WEBP: RIFF | xxxx | WEBP | VP8x + if (d32[0] === 0x46464952 && d32[2] === 0x50424557 && (d32[3] & 0xFFFFFF) === 0x385056) { + switch (d[15]) { + case 0x58: // Extended WebP VP8X --> "X" + return { + mime: 'image/webp', + width: (d[24] | d[25] << 8 | d[26] << 16) + 1, + height: (d[27] | d[28] << 8 | d[29] << 16) + 1 + }; + case 0x4C: // Lossless WebP VP8L --> "L" + if (d[20] !== 0x2f) return UNSUPPORTED_TYPE; + const dim = d[21] | d[22] << 8 | d[23] << 16 | d[24] << 24; + return { + mime: 'image/webp', + width: (dim & 0x3FFF) + 1, + height: (dim >>> 14 & 0x3FFF) + 1 + }; + case 0x20: // Lossy WebP VP8 --> " " + if (d[23] !== 0x9d || d[24] !== 0x01 || d[25] !== 0x2a) return UNSUPPORTED_TYPE; + return { + mime: 'image/webp', + width: (d[26] | d[27] << 8) & 0x3FFF, + height: (d[28] | d[29] << 8) & 0x3FFF + }; + } + return UNSUPPORTED_TYPE; + } + // AVIF: Box size | ftyp | avif/avis + if (d32[1] === 0x70797466 && (d32[2] === 0x66697661 || d32[2] === 0x73697661)) { + let pos = -1; + // search for ispe box within first 1024 bytes + const limit = Math.min(d.length - 16, 1024); + for (let i = 8; i < limit; i++) { + // scan for ispe + if (d[i] === 0x69 && d[i + 1] === 0x73 && d[i + 2] === 0x70 && d[i + 3] === 0x65) { + pos = i; + break; + } + } + if (pos !== -1) { + // dimensions are in BE at +8 (width) at +12 (height) + const width = + d[pos + 8] << 24 | + d[pos + 9] << 16 | + d[pos + 10] << 8 | + d[pos + 11]; + const height = + d[pos + 12] << 24 | + d[pos + 13] << 16 | + d[pos + 14] << 8 | + d[pos + 15]; + if (width > 0 && height > 0) { + return { mime: 'image/avif', width, height }; + } + } + return UNSUPPORTED_TYPE; + } return UNSUPPORTED_TYPE; } + function jpgSize(d: Uint8Array): [number, number] { const len = d.length; let i = 4; diff --git a/demo/client/components/window/addonImageWindow.ts b/demo/client/components/window/addonImageWindow.ts index e5402182a..2631cdd41 100644 --- a/demo/client/components/window/addonImageWindow.ts +++ b/demo/client/components/window/addonImageWindow.ts @@ -67,6 +67,8 @@ export class AddonImageWindow extends BaseWindow implements IControlWindow { dlIip.appendChild(dtIip); this._addDdWithButton(dlIip, 'image-demo3', 'palette (png File)'); this._addDdWithButton(dlIip, 'image-demo4', 'dice (qoi MultipartFile)'); + this._addDdWithButton(dlIip, 'image-demo5', 'rose (webp File)'); + this._addDdWithButton(dlIip, 'image-demo6', 'kimono (avif MultipartFile)'); container.appendChild(dlIip); // Kitty demos @@ -173,11 +175,15 @@ export class AddonImageWindow extends BaseWindow implements IControlWindow { document.getElementById('image-demo2')!.addEventListener('click', sixelDemo('https://raw.githubusercontent.com/jerch/node-sixel/master/testfiles/test2.sixel')); document.getElementById('image-demo3')!.addEventListener('click', - iipDemo('https://raw.githubusercontent.com/jerch/node-sixel/master/palette.png')); + iipDemo('https://raw.githubusercontent.com/xtermjs/xterm.js/master/addons/addon-image/fixture/palette.png')); document.getElementById('image-demo4')!.addEventListener('click', - iipDemoMulti('https://raw.githubusercontent.com/jerch/xterm-wasm-parts/master/fixtures/qoi/dice.qoi')); + iipDemoMulti('https://raw.githubusercontent.com/xtermjs/xterm.js/master/addons/addon-image/fixture/testimages/dice.qoi')); + document.getElementById('image-demo5')!.addEventListener('click', + iipDemo('https://raw.githubusercontent.com/xtermjs/xterm.js/master/addons/addon-image/fixture/testimages/1_webp_a.webp')); + document.getElementById('image-demo6')!.addEventListener('click', + iipDemoMulti('https://raw.githubusercontent.com/xtermjs/xterm.js/master/addons/addon-image/fixture/testimages/kimono.crop.avif')); document.getElementById('image-demo-kitty1')!.addEventListener('click', - kittyDemo('https://raw.githubusercontent.com/jerch/node-sixel/master/palette.png')); + kittyDemo('https://raw.githubusercontent.com/xtermjs/xterm.js/master/addons/addon-image/fixture/palette.png')); // demo for image retrieval API this._terminal.element!.addEventListener('click', (ev: MouseEvent) => {