diff --git a/addons/addon-image/src/AnimationManager.ts b/addons/addon-image/src/AnimationManager.ts new file mode 100644 index 000000000..7ade4e66e --- /dev/null +++ b/addons/addon-image/src/AnimationManager.ts @@ -0,0 +1,609 @@ +/** + * Copyright (c) 2026 The xterm.js authors. All rights reserved. + * @license MIT + */ + +import { IDisposable } from '@xterm/xterm'; +import { ImageRenderer } from 'ImageRenderer'; +import { EMPTY_ATTRS, ImageStorage } from 'ImageStorage'; +import { BgFlags, IBufferLineExt, IImageSpec, ITerminalExt, IExtendedAttrsImage, IResetHandler } from 'Types'; + + +/** + * Note on Firefox Bug (https://bugzilla.mozilla.org/show_bug.cgi?id=2066911) + * + * ImageDecoder.decode() returns a corrupted VideoFrame, + * if the frame was closed from a previous decode. + * + * Therefore the AnimationManager class implements certain + * firefox shims gated by the isFirefox boolean: + * - IAnimation.highest tracks highest frameID ever decoded + * - whenever frameID is lesser or equal zu highest create new decoder + * - raw image bytes need to be held for new decoder + * + * To be removed once the bug got fixed. + */ + + +interface IAnimation { + /** image id in storage */ + id: number; + /** image bytes */ + data: Blob; + /** mime type */ + mime: string; + /** width */ + w: number; + /** height */ + h: number; + /** number of frames */ + numFrames: number; + frames: number[]; + /** ImageDecoder instance */ + decoder: ImageDecoder | undefined; + decoderPromise: Promise | undefined; + cache: BitmapBuffer, + /** start timestamp */ + startTime: number | undefined; + /** whether a decode task ist scheduled */ + scheduled: boolean; + /** whether the image is in viewport */ + inViewport: boolean; + /** firefox bug helper */ + highest: number; + /** playtime of animation */ + playtime: number; + /** frame timestamps */ + timestamps: number[]; + /** frame ID targeted by animation */ + current: number; + /** frame ID currently visible (owned by spec.actual) */ + actual: number; + /** idx of current on frames */ + frameIdx: number; +} + + +interface ImageDraw { + imageId: number; + imgSpec: IImageSpec; + tileId: number; + col: number; + row: number; + count: number; +} + + +interface IBufferSlot { + id: number; + bm: ImageBitmap | null; +} + + +const enum Constants { + BUFFERSIZE = 4, + BUFFERMOD = BUFFERSIZE - 1, + MAX_FRAMES = 65535 +} + + +class BitmapBuffer implements IDisposable { + private readonly _d: IBufferSlot[]; + private _isDisposed = false; + + constructor() { + this._d = Array(Constants.BUFFERSIZE).fill(0).map(() => ({ id: -1, bm: null })); + } + + public dispose(): void { + this.clear(); + this._isDisposed = true; + } + + public peek(id: number): ImageBitmap | null { + const slot = this._d[id & Constants.BUFFERMOD]; + return id === slot.id ? slot.bm : null; + } + + public pop(id: number): ImageBitmap | null { + const slot = this._d[id & Constants.BUFFERMOD]; + const res = id === slot.id + ? slot.bm + : (slot.bm?.close(), null); + slot.bm = null; + slot.id = -1; + return res; + } + + public push(id: number, bm: ImageBitmap): void { + /** + * A late decode task might still push bitmaps. + * Since push gives the buffer ownership of + * the bitmap, we use the isDisposed flag to + * free it right away. + */ + if (this._isDisposed) { + bm.close(); + return; + } + const slot = this._d[id & Constants.BUFFERMOD]; + slot.bm?.close(); + slot.bm = bm; + slot.id = id; + } + + public clear(): void { + for (let i = 0; i < Constants.BUFFERSIZE; ++i) { + const slot = this._d[i]; + slot.bm?.close(); + slot.bm = null; + slot.id = -1; + } + } +} + + + +declare const InstallTrigger: any; + + +// FIXME: move to AnimationManager ctor to control it from xterm side +const isChromium = (window as any).chrome !== undefined; +const isFirefox: boolean = typeof InstallTrigger !== 'undefined' || 'MozAppearance' in document.documentElement.style; + + +/** + * Helper to create a read to use ImageDecoder instance from a blob. + */ +async function BlobDecoder(blob: Blob, type: string): Promise { + /** + * Dealing with browser bugs - in theory this should be possible: + * + * const decoder = new ImageDecoder({ data: blob.stream(), type, preferAnimation: true }); + * await decoder.tracks.ready; + * return decoder; + * + * But this straightforward handling with blob.stream() uncovers several bugs: + * 1. Chromium: Fails to stream chunked blobs reliably. + * 2. Firefox: Native lifecycle race condition. Closing an old decoder breaks the + * shared underlying native stream of the same blob for new decoders, throwing + * an uncatchable "Closed decoder" DOMException during `tracks.ready`. + * + * So we use a safer approach by pulling full raw bytes over first and transferring ownership. + */ + const data = await blob.arrayBuffer(); + const decoder = new ImageDecoder({ data, type, preferAnimation: true, transfer: [data] }); + await decoder.tracks.ready; + if (isChromium && type === 'image/avif') { + // chromium bug: random frame access in AVIF is + // only possible after reading frame 0 + const res = await decoder.decode({ frameIndex: 0 }); + res.image.close(); + } + return decoder; +} + + +/** + * AnimationManager - orchestrates rendering of animated images. + * + * Current Workflow + * When IIPMetrics.imageType reports an animated image the IIP handler + * stores the image in the storage and registers an animation of it. + * + * The animation manager works mostly self-contained: + * - RAF loop + * - 3 frame ahead decoding + * - own bitmap ownership handling + * + * There are a few intersections with the ImageStorage and the ImageRenderer, + * which still need to be consolidated (TODOs): + * - ImageStorage: API for getting an ImageSpec + * - ImageStorage: API for announcing memory state, better eviction strategy + * - ImageRenderer: API for better drawing primitives + * + * In particular the ownership model of ImageSpec bitmaps (actual and orig) + * needs a better story. We prolly need here an API to flip bitmaps. + */ +export class AnimationManager implements IDisposable, IResetHandler { + private _animations = new Map(); + private _raf: number | null = null; + private _draws: ImageDraw[] = []; + + + constructor( + private _storage: ImageStorage, + private _renderer: ImageRenderer, + private _terminal: ITerminalExt) + {} + + + public dispose(): void { + this.reset(); + this._storage = undefined!; + this._terminal = undefined!; + this._renderer = undefined!; + } + + + /** + * Reset handler. + * Stops all animations and frees all ressources. + */ + public reset(): void { + for (const anim of this._animations.values()) { + anim.inViewport = false; + this._closeDecoder(anim); + anim.cache.dispose(); + } + this._animations.clear(); + this._draws = []; + if (this._raf) cancelAnimationFrame(this._raf); + this._raf = null; + } + + + /** + * Main hook to register an animated image for animation. + * + * @param id Image ID of the image storage the image was registered under. + * @param data Raw data bytes of the image. + * @param mime Mimetype of the image. + * @param w Output width in pixel. + * @param h Output height in pixel. + */ + public async registerAnimation(id: number, data: Blob, mime: string, w: number, h: number): Promise { + let decoder = await BlobDecoder(data, mime); + const track = decoder.tracks.selectedTrack; + if (!track || track.frameCount < 2) { + decoder.close(); + return; + } + + const frameCount = Math.min(track.frameCount, Constants.MAX_FRAMES); + const frames: number[] = []; + const timestamps: number[] = []; + const durations: number[] = []; + let elapsed = 0; + + for (let i = 0; i < frameCount; i++) { + try { + const result = await decoder.decode({ frameIndex: i }); + const frame = result.image; + const width = frame.displayWidth; + const height = frame.displayHeight + const duration = (frame.duration ?? 0) / 1000; + frame.close(); + // skip wrong dimensions and zero duration + // FIXME: needs orig image dimensions in IAnimation + if (width !== w || height !== h || !duration) { + if (durations.length) { + durations[durations.length - 1] += duration; + elapsed += duration; + } + continue; + } + frames.push(i); + timestamps.push(elapsed); + durations.push(duration); + elapsed += duration; + } catch (e) { + break; + } + } + + /** + * FIXME: still needs a consolidation of short frame durations: + * We don't track v-sync, but just decode the next 3 follow-up frames. + * This creates nonsense decoder pressure decoding frames, + * that effectively never make it to the screen. + * Currently our shortest time resolution is 1ms, + * thus with 60 fps we would decode 15 1ms frames for no reason. + * We prolly should introduce a lower bound of 8ms: + * - in sync with 120 fps + * - only every second frame is nonsense with 60 fps + */ + + if (frames.length < 2 || elapsed <= 0) { + decoder.close(); + return; + } + + if (isFirefox) { + decoder.close(); + decoder = await BlobDecoder(data, mime); + } + + const anim: IAnimation = { + id, data, mime, w, h, decoder, + numFrames: frames.length, + decoderPromise: undefined, + cache: new BitmapBuffer(), + frames, + startTime: undefined, + current: 0, + actual: 0, + scheduled: false, // set by _schedule + inViewport: true, // start with active prefetching + highest: -1, + playtime: elapsed, + timestamps, + frameIdx: 0 + }; + this._animations.set(id, anim); + this.viewportUpdated(); + // early schedule to give decoder time to fill the cache before RAF + this._schedule(anim); + if (this._raf === null) { + this._raf = requestAnimationFrame(this._loop); + } + } + + + /** + * Remove the animation for the given image id. + */ + public unregisterAnimation(id: number): void { + const a = this._animations.get(id); + if (a) { + a.inViewport = false; + this._closeDecoder(a); + a.cache.dispose(); + this._animations.delete(id); + } + } + + + /** + * onRender handler. + * Scratches the viewport buffer to collect draws for animation RAF + * and to pause/resume invisible/visible animations. + */ + public viewportUpdated(range?: { start: number, end: number }): any { + const rows = this._terminal.rows; + const cols = this._terminal.cols; + const buffer = this._terminal._core.buffer; + const actives: Set = new Set; + this._draws = []; + for (let row = 0; row < rows; ++row) { + const line = buffer.lines.get(row + buffer.ydisp) as IBufferLineExt; + if (!line) break; + for (let col = 0; col < cols; ++col) { + if (line.getBg(col) & BgFlags.HAS_EXTENDED) { + let e: IExtendedAttrsImage = line._extendedAttrs[col] ?? EMPTY_ATTRS; + const imageId = e.imageId; + if (imageId === undefined || imageId === -1) { + continue; + } + const imgSpec = (this._storage as any)._images.get(imageId); + if (e.tileId !== -1) { + const startTile = e.tileId; + const startCol = col; + let count = 1; + while ( + ++col < cols + && (line.getBg(col) & BgFlags.HAS_EXTENDED) + && (e = line._extendedAttrs[col] ?? EMPTY_ATTRS) + && (e.imageId === imageId) + && (e.tileId === startTile + count) + ) { + count++; + } + col--; + if (imgSpec && this._animations.has(imageId)) { + this._draws.push({ imageId, imgSpec, tileId: startTile, col: startCol, row, count }); + actives.add(imageId); + } + } + } + } + } + this._updateAnimations(actives); + } + + + /** + * Get the frameIdx for elapsed time. + * Note: The returned value is not the frame ID, + * but the index for IAnimation.frames. + */ + private _timedFrameIdx(a: IAnimation, elapsed: number): number { + elapsed %= a.playtime; + let low = 0; + let high = a.timestamps.length - 1; + while (low <= high) { + const mid = (low + high) >> 1; + const startTime = a.timestamps[mid]; + if (elapsed < startTime) { + high = mid - 1; + } else { + if (mid === a.timestamps.length - 1 || elapsed < a.timestamps[mid + 1]) { + return mid; + } + low = mid + 1; + } + } + return 0; + } + + /** + * Pause/resume decoding from viewport changes. + */ + private _updateAnimations(visibles: Set): void { + for (const [id, anim] of this._animations) { + const isVisible = visibles.has(id); + if (isVisible && !anim.inViewport) { + /** + * There are two strategies possible here: + * + * 1. Resume decoding and pumping frames as soon as possible. + * 2. Delay resuming further by a debounce to prevent flooding + * new decoder instances on heavy scrolling. + * + * Currently the code below does 1. as the impact is rather + * small from tests and gives the better user experience. + */ + anim.inViewport = true; + const frameIdx = this._timedFrameIdx(anim, performance.now() - anim.startTime!); + anim.frameIdx = frameIdx; + anim.current = anim.frames[frameIdx]; + if (!anim.decoder && !anim.decoderPromise) { + anim.decoderPromise = BlobDecoder(anim.data, anim.mime); + } + this._schedule(anim); + } else if (!isVisible && anim.inViewport) { + anim.inViewport = false; + anim.cache.clear(); + this._closeDecoder(anim); + } + } + if (visibles.size && this._raf === null) { + this._raf = requestAnimationFrame(this._loop); + } + } + + + /** + * RAF for animations in viewport. + */ + private _loop = (ts: number): void => { + if (!this._animations.size) { + this._raf = null; + return; + } + + let mustRender = false; + let anyVisible = false; + + for (const anim of this._animations.values()) { + if (!anim.inViewport) continue; + + const spec = (this._storage as any)._images.get(anim.id); + if (!spec) { + this.unregisterAnimation(anim.id); + continue; + } + anyVisible = true; + + if (anim.startTime === undefined) anim.startTime = ts; + + const frameIdx = this._timedFrameIdx(anim, ts - anim.startTime); + anim.frameIdx = frameIdx; + const current = anim.frames[frameIdx]; + + if (current !== anim.current) { + anim.current = current; + const bm = anim.cache.pop(current); + if (bm) { + // FIXME: setting spec.actual delegates draws from onRender back to ImageStorage + // --> should be avoided to prevent double draws within same rAF + + // we took ownership of spec.actual, thus have to close it here + // but only if we have more than 4 frames (ringbuffer size is 4) + if (anim.numFrames > 4) spec.actual?.close(); + else anim.cache.push(anim.actual, spec.actual); + spec.actual = bm; + anim.actual = current; + mustRender = true; + } + this._schedule(anim); + } + } + + // FIXME: merge clears and draws into col x row rects + if (mustRender) { + for (let i = 0; i < this._draws.length; ++i) { + const d = this._draws[i]; + this._renderer.clearLines(d.row, d.row); + this._renderer.draw(d.imgSpec, d.tileId, d.col, d.row, d.count); + } + } + this._raf = anyVisible ? requestAnimationFrame(this._loop) : null; + }; + + + /** + * Try to schedule frame decoding. + */ + private _schedule(a: IAnimation): void { + if (!a.scheduled) { + let needsSchedule = false; + // FIXME: do we need a == 4 test too? + if (a.numFrames > 3) { + needsSchedule = + !a.cache.peek(a.frames[(a.frameIdx + 1) % a.numFrames]) || + !a.cache.peek(a.frames[(a.frameIdx + 2) % a.numFrames]) || + !a.cache.peek(a.frames[(a.frameIdx + 3) % a.numFrames]); + } else if (a.numFrames === 3) { + needsSchedule = + !a.cache.peek(a.frames[(a.frameIdx + 1) % a.numFrames]) || + !a.cache.peek(a.frames[(a.frameIdx + 2) % a.numFrames]); + } else { + needsSchedule = + !a.cache.peek(a.frames[(a.frameIdx + 1) % a.numFrames]); + } + if (needsSchedule) { + a.scheduled = true; + setTimeout(() => this._decode(a), 0); + } + } + } + + + /** + * Decode up to next 3 frames. + */ + private async _decode(a: IAnimation): Promise { + if (!a.inViewport) { + a.scheduled = false; + return; + } + if (!a.decoder) { + const decP = a.decoderPromise; + a.decoderPromise = undefined; + a.decoder = await (decP ?? BlobDecoder(a.data, a.mime)); + } + const n1 = a.frames[(a.frameIdx + 1) % a.numFrames]; + const n2 = a.frames[(a.frameIdx + 2) % a.numFrames]; + const n3 = a.frames[(a.frameIdx + 3) % a.numFrames]; + for (const n of [n1, n2, n3]) { + if (!a.cache.peek(n)) { + if (isFirefox) { + if (n <= a.highest) { + a.highest = -1; + a.decoder?.close(); + a.decoder = await BlobDecoder(a.data, a.mime); + } + a.highest = Math.max(n, a.highest); + } + let res: ImageDecodeResult; + try { + res = await a.decoder.decode({ frameIndex: n }); + } catch (e) { continue; } + let bm: ImageBitmap; + try { + bm = await createImageBitmap(res.image, { resizeWidth: a.w, resizeHeight: a.h }); + } catch (e) { continue; } finally { res?.image?.close(); } + if (!a.inViewport) { + bm.close(); + break; + } + a.cache.push(n, bm); + } + } + a.scheduled = false; + } + + + /** + * Close the decoder defensively. + */ + private _closeDecoder(a: IAnimation): void { + const decoder = a.decoder; + a.decoder = undefined; + try { decoder?.close(); } catch (e) {} + const decP = a.decoderPromise; + a.decoderPromise = undefined; + decP?.then(dec => dec && dec !== decoder ? dec.close() : null).catch(() => {}); + } +} diff --git a/addons/addon-image/src/IIPHandler.ts b/addons/addon-image/src/IIPHandler.ts index 559b90741..230b39cb1 100644 --- a/addons/addon-image/src/IIPHandler.ts +++ b/addons/addon-image/src/IIPHandler.ts @@ -2,14 +2,15 @@ * Copyright (c) 2023 The xterm.js authors. All rights reserved. * @license MIT */ -import { IImageAddonOptions, IOscHandler, IResetHandler, ITerminalExt } from './Types'; +import type { IImageAddonOptions, IOscHandler, IResetHandler, ITerminalExt } from './Types'; import { ImageRenderer } from './ImageRenderer'; -import { IIPImageStorage } from './IIPImageStorage'; +import type { IIPImageStorage } from './IIPImageStorage'; import { CELL_SIZE_DEFAULT } from './ImageStorage'; import Base64Decoder from 'xterm-wasm-parts/lib/base64/Base64Decoder.wasm'; import QoiDecoder from 'xterm-wasm-parts/lib/qoi/QoiDecoder.wasm'; import { HeaderParser, IHeaderFields, HeaderState, SequenceType } from './IIPHeaderParser'; import { imageType, UNSUPPORTED_TYPE } from './IIPMetrics'; +import type { AnimationManager } from 'AnimationManager'; // Local const enum mirror - esbuild can't inline const enums from external packages const enum DecoderConst { @@ -46,7 +47,8 @@ export class IIPHandler implements IOscHandler, IResetHandler { private readonly _opts: IImageAddonOptions, private readonly _renderer: ImageRenderer, private readonly _storage: IIPImageStorage, - private readonly _coreTerminal: ITerminalExt + private readonly _coreTerminal: ITerminalExt, + private readonly _aniManager: AnimationManager | undefined ) { const maxEncodedBytes = Math.ceil(this._opts.iipSizeLimit * 4 / 3); const initialBytes = Math.min(DecoderConst.INITIAL_DATA, maxEncodedBytes); @@ -197,10 +199,17 @@ export class IIPHandler implements IOscHandler, IResetHandler { } else { blob = new Blob([this._dec.data8], { type: metrics.mime }); } + //let imgBytes: Uint8Array; + //if (this._aniManager && metrics.animated) { + // imgBytes = new Uint8Array(this._dec.data8); + //} this._dec.release(); return createImageBitmap(blob, { resizeWidth: w, resizeHeight: h }) .then(bm => { - this._storage.addImage(bm); + const id = this._storage.addImage(bm); + if (this._aniManager && metrics.animated) { + this._aniManager.registerAnimation(id, blob as Blob, metrics.mime, w, h); + } return true; }) .catch(e => { diff --git a/addons/addon-image/src/IIPImageStorage.ts b/addons/addon-image/src/IIPImageStorage.ts index 28037a897..be120d9e8 100644 --- a/addons/addon-image/src/IIPImageStorage.ts +++ b/addons/addon-image/src/IIPImageStorage.ts @@ -22,7 +22,7 @@ export class IIPImageStorage { * Add an IIP image to storage. * Always uses scrolling mode — cursor advances past the image. */ - public addImage(img: HTMLCanvasElement | ImageBitmap): void { - this._storage.addImage(img, this._addImageOpts); + public addImage(img: HTMLCanvasElement | ImageBitmap): number { + return this._storage.addImage(img, this._addImageOpts); } } diff --git a/addons/addon-image/src/IIPMetrics.test.ts b/addons/addon-image/src/IIPMetrics.test.ts index 4df320a2c..887ec869d 100644 --- a/addons/addon-image/src/IIPMetrics.test.ts +++ b/addons/addon-image/src/IIPMetrics.test.ts @@ -12,48 +12,48 @@ const fs = require('fs'); const TEST_IMAGES: [string, IMetrics][] = [ - ['w3c_home_256.gif', { mime: 'image/gif', width: 72, height: 48 }], - ['w3c_home_256.jpg', { mime: 'image/jpeg', width: 72, height: 48 }], - ['w3c_home_256.png', { mime: 'image/png', width: 72, height: 48 }], - ['w3c_home_2.gif', { mime: 'image/gif', width: 72, height: 48 }], - ['w3c_home_2.jpg', { mime: 'image/jpeg', width: 72, height: 48 }], - ['w3c_home_2.png', { mime: 'image/png', width: 72, height: 48 }], - ['w3c_home_animation.gif', { mime: 'image/gif', width: 72, height: 48 }], - ['w3c_home.gif', { mime: 'image/gif', width: 72, height: 48 }], - ['w3c_home_gray.gif', { mime: 'image/gif', width: 72, height: 48 }], - ['w3c_home_gray.jpg', { mime: 'image/jpeg', width: 72, height: 48 }], - ['w3c_home_gray.png', { mime: 'image/png', width: 72, height: 48 }], - ['w3c_home.jpg', { mime: 'image/jpeg', width: 72, height: 48 }], - ['w3c_home.png', { mime: 'image/png', width: 72, height: 48 }], - ['w3c_home_noexif.jpg', { mime: 'image/jpeg', width: 72, height: 48 }], - ['spinfox.png', { mime: 'image/png', width: 148, height: 148 }], - ['iphone_hdr_YES.jpg', { mime: 'image/jpeg', width: 3264, height: 2448 }], - ['nikon-e950.jpg', { mime: 'image/jpeg', width: 800, height: 600 }], - ['agfa-makernotes.jpg', { mime: 'image/jpeg', width: 8, height: 8 }], - ['sony-alpha-6000.jpg', { mime: 'image/jpeg', width: 6000, height: 4000 }], - ['dice.qoi', { mime: 'image/qoi', width: 800, height: 600 }], + ['w3c_home_256.gif', { mime: 'image/gif', width: 72, height: 48, animated: false }], + ['w3c_home_256.jpg', { mime: 'image/jpeg', width: 72, height: 48, animated: false }], + ['w3c_home_256.png', { mime: 'image/png', width: 72, height: 48, animated: false }], + ['w3c_home_2.gif', { mime: 'image/gif', width: 72, height: 48, animated: false }], + ['w3c_home_2.jpg', { mime: 'image/jpeg', width: 72, height: 48, animated: false }], + ['w3c_home_2.png', { mime: 'image/png', width: 72, height: 48, animated: false }], + ['w3c_home_animation.gif', { mime: 'image/gif', width: 72, height: 48, animated: true }], + ['w3c_home.gif', { mime: 'image/gif', width: 72, height: 48, animated: false }], + ['w3c_home_gray.gif', { mime: 'image/gif', width: 72, height: 48, animated: false }], + ['w3c_home_gray.jpg', { mime: 'image/jpeg', width: 72, height: 48, animated: false }], + ['w3c_home_gray.png', { mime: 'image/png', width: 72, height: 48, animated: false }], + ['w3c_home.jpg', { mime: 'image/jpeg', width: 72, height: 48, animated: false }], + ['w3c_home.png', { mime: 'image/png', width: 72, height: 48, animated: false }], + ['w3c_home_noexif.jpg', { mime: 'image/jpeg', width: 72, height: 48, animated: false }], + ['spinfox.png', { mime: 'image/png', width: 148, height: 148, animated: true }], + ['iphone_hdr_YES.jpg', { mime: 'image/jpeg', width: 3264, height: 2448, animated: false }], + ['nikon-e950.jpg', { mime: 'image/jpeg', width: 800, height: 600, animated: false }], + ['agfa-makernotes.jpg', { mime: 'image/jpeg', width: 8, height: 8, animated: false }], + ['sony-alpha-6000.jpg', { mime: 'image/jpeg', width: 6000, height: 4000, animated: false }], + ['dice.qoi', { mime: 'image/qoi', width: 800, height: 600, animated: false }], // VP8 - ['1.webp', { mime: 'image/webp', width: 550, height: 368 }], - ['2.webp', { mime: 'image/webp', width: 550, height: 404 }], - ['3.webp', { mime: 'image/webp', width: 1280, height: 720 }], - ['4.webp', { mime: 'image/webp', width: 1024, height: 772 }], - ['5.webp', { mime: 'image/webp', width: 1024, height: 752 }], + ['1.webp', { mime: 'image/webp', width: 550, height: 368, animated: false }], + ['2.webp', { mime: 'image/webp', width: 550, height: 404, animated: false }], + ['3.webp', { mime: 'image/webp', width: 1280, height: 720, animated: false }], + ['4.webp', { mime: 'image/webp', width: 1024, height: 772, animated: false }], + ['5.webp', { mime: 'image/webp', width: 1024, height: 752, animated: false }], // VP8X - ['1_webp_a.webp', { mime: 'image/webp', width: 400, height: 301 }], - ['2_webp_a.webp', { mime: 'image/webp', width: 386, height: 395 }], - ['3_webp_a.webp', { mime: 'image/webp', width: 800, height: 600 }], - ['4_webp_a.webp', { mime: 'image/webp', width: 421, height: 163 }], - ['5_webp_a.webp', { mime: 'image/webp', width: 300, height: 300 }], + ['1_webp_a.webp', { mime: 'image/webp', width: 400, height: 301, animated: false }], + ['2_webp_a.webp', { mime: 'image/webp', width: 386, height: 395, animated: false }], + ['3_webp_a.webp', { mime: 'image/webp', width: 800, height: 600, animated: false }], + ['4_webp_a.webp', { mime: 'image/webp', width: 421, height: 163, animated: false }], + ['5_webp_a.webp', { mime: 'image/webp', width: 300, height: 300, animated: false }], // VP8L - ['1_webp_ll.webp', { mime: 'image/webp', width: 400, height: 301 }], - ['2_webp_ll.webp', { mime: 'image/webp', width: 386, height: 395 }], - ['3_webp_ll.webp', { mime: 'image/webp', width: 800, height: 600 }], - ['4_webp_ll.webp', { mime: 'image/webp', width: 421, height: 163 }], - ['5_webp_ll.webp', { mime: 'image/webp', width: 300, height: 300 }], + ['1_webp_ll.webp', { mime: 'image/webp', width: 400, height: 301, animated: false }], + ['2_webp_ll.webp', { mime: 'image/webp', width: 386, height: 395, animated: false }], + ['3_webp_ll.webp', { mime: 'image/webp', width: 800, height: 600, animated: false }], + ['4_webp_ll.webp', { mime: 'image/webp', width: 421, height: 163, animated: false }], + ['5_webp_ll.webp', { mime: 'image/webp', width: 300, height: 300, animated: false }], // some AVIF test images - ['fox.profile2.12bpc.yuv444.odd-width.odd-height.avif', { mime: 'image/avif', width: 1203, height: 799 }], - ['hato.profile0.8bpc.yuv420.avif', { mime: 'image/avif', width: 3082, height: 2048 }], - ['kimono.crop.avif', { mime: 'image/avif', width: 722, height: 1024 }], + ['fox.profile2.12bpc.yuv444.odd-width.odd-height.avif', { mime: 'image/avif', width: 1203, height: 799, animated: false }], + ['hato.profile0.8bpc.yuv420.avif', { mime: 'image/avif', width: 3082, height: 2048, animated: false }], + ['kimono.crop.avif', { mime: 'image/avif', width: 722, height: 1024, animated: false }], ]; diff --git a/addons/addon-image/src/IIPMetrics.ts b/addons/addon-image/src/IIPMetrics.ts index 217a7fc93..4b09b7d04 100644 --- a/addons/addon-image/src/IIPMetrics.ts +++ b/addons/addon-image/src/IIPMetrics.ts @@ -10,12 +10,14 @@ export interface IMetrics { mime: ImageType; width: number; height: number; + animated: boolean; } export const UNSUPPORTED_TYPE: IMetrics = { mime: 'unsupported', width: 0, - height: 0 + height: 0, + animated: false }; export function imageType(d: Uint8Array): IMetrics { @@ -29,28 +31,31 @@ export function imageType(d: Uint8Array): IMetrics { return { mime: 'image/png', width: d[16] << 24 | d[17] << 16 | d[18] << 8 | d[19], - height: d[20] << 24 | d[21] << 16 | d[22] << 8 | d[23] + height: d[20] << 24 | d[21] << 16 | d[22] << 8 | d[23], + animated: isAPNG(d) }; } // JPEG: FF D8 FF if (d[0] === 0xFF && d[1] === 0xD8 && d[2] === 0xFF) { const [width, height] = jpgSize(d); - return { mime: 'image/jpeg', width, height }; + return { mime: 'image/jpeg', width, height, animated: false }; } // GIF: GIF87a or GIF89a if (d32[0] === 0x38464947 && (d[4] === 0x37 || d[4] === 0x39) && d[5] === 0x61) { return { mime: 'image/gif', - width: d[7] << 8 | d[6], - height: d[9] << 8 | d[8] + width: d[6] | d[7] << 8, + height: d[8] | d[9] << 8, + animated: isAGIF(d) }; } // QOI: qoif if (d32[0] === 0x66696F71) { return { mime: 'image/qoi', - width: d[4] << 24 | d[5] << 16 | d[6] << 8 | d[7], - height: d[8] << 24 | d[9] << 16 | d[10] << 8 | d[11] + width: d[4] << 24 | d[5] << 16 | d[ 6] << 8 | d[ 7], + height: d[8] << 24 | d[9] << 16 | d[10] << 8 | d[11], + animated: false }; } // WEBP: RIFF | xxxx | WEBP | VP8x @@ -60,7 +65,8 @@ export function imageType(d: Uint8Array): IMetrics { return { mime: 'image/webp', width: (d[24] | d[25] << 8 | d[26] << 16) + 1, - height: (d[27] | d[28] << 8 | d[29] << 16) + 1 + height: (d[27] | d[28] << 8 | d[29] << 16) + 1, + animated: (d[20] & 0x02) !== 0 }; case 0x4C: // Lossless WebP VP8L --> "L" if (d[20] !== 0x2f) return UNSUPPORTED_TYPE; @@ -68,14 +74,16 @@ export function imageType(d: Uint8Array): IMetrics { return { mime: 'image/webp', width: (dim & 0x3FFF) + 1, - height: (dim >>> 14 & 0x3FFF) + 1 + height: (dim >>> 14 & 0x3FFF) + 1, + animated: false }; case 0x20: // Lossy WebP VP8 --> " " if (d[23] !== 0x9d || d[24] !== 0x01 || d[25] !== 0x2a) return UNSUPPORTED_TYPE; return { mime: 'image/webp', width: (d[26] | d[27] << 8) & 0x3FFF, - height: (d[28] | d[29] << 8) & 0x3FFF + height: (d[28] | d[29] << 8) & 0x3FFF, + animated: false }; } return UNSUPPORTED_TYPE; @@ -83,12 +91,23 @@ export function imageType(d: Uint8Array): IMetrics { // AVIF: Box size | ftyp | avif/avis if (d32[1] === 0x70797466 && (d32[2] === 0x66697661 || d32[2] === 0x73697661)) { let pos = -1; - // search for ispe box within first 1024 bytes + let anim = d32[2] === 0x73697661; // avis is meant to be animated + // search for boxes within first 1024 bytes const limit = Math.min(d.length - 16, 1024); for (let i = 8; i < limit; i++) { - // scan for ispe - if (d[i] === 0x69 && d[i + 1] === 0x73 && d[i + 2] === 0x70 && d[i + 3] === 0x65) { + // scan for ispe for dimensions + if (pos === -1 && d[i] === 0x69 && d[i + 1] === 0x73 && d[i + 2] === 0x70 && d[i + 3] === 0x65) { pos = i; + } + // scan for stsz or moov + if (!anim) { + const isStsz = d[i] === 0x73 && d[i + 1] === 0x74 && d[i + 2] === 0x73 && d[i + 3] === 0x7A; + const isMoov = d[i] === 0x6D && d[i + 1] === 0x6F && d[i + 2] === 0x6F && d[i + 3] === 0x76; + if (isStsz || isMoov) { + anim = true; + } + } + if (pos !== -1 && anim) { break; } } @@ -105,7 +124,7 @@ export function imageType(d: Uint8Array): IMetrics { d[pos + 14] << 8 | d[pos + 15]; if (width > 0 && height > 0) { - return { mime: 'image/avif', width, height }; + return { mime: 'image/avif', width, height, animated: anim }; } } return UNSUPPORTED_TYPE; @@ -118,6 +137,7 @@ function jpgSize(d: Uint8Array): [number, number] { const len = d.length; let i = 4; let blockLength = d[i] << 8 | d[i + 1]; + if (blockLength < 2) return [0, 0]; while (true) { i += blockLength; if (i >= len) { @@ -138,5 +158,78 @@ function jpgSize(d: Uint8Array): [number, number] { } i += 2; blockLength = d[i] << 8 | d[i + 1]; + if (blockLength < 2) return [0, 0]; } } + + +function isAGIF(data: Uint8Array): boolean { + let pos = 10; + const gPacked = data[pos]; + const hasGP = (gPacked & 0x80) !== 0; + pos += 3; + if (hasGP) { + const gpSize = 2 << (gPacked & 0x07); + pos += 3 * gpSize; + } + let gceCount = 0; + while (pos < data.length) { + const blockType = data[pos]; + if (blockType === 0x21) { // Extension Block + if (pos + 1 >= data.length) return false; + const extLabel = data[pos + 1]; + if (extLabel === 0xF9) { // Graphic Control Extension + gceCount++; + if (gceCount > 1) return true; // early exit on 2nd frame + } + pos += 2; + while (pos < data.length) { + const blockSize = data[pos]; + if (blockSize === 0x00) break; + pos += blockSize + 1; + } + if (pos >= data.length) return false; + pos++; + } else if (blockType === 0x2C) { // Image Descriptor + if (pos + 10 >= data.length) return false; + pos += 10; + const lPacked = data[pos - 1]; + if ((lPacked & 0x80) !== 0) { + const lpSize = 2 << (lPacked & 0x07); + pos += 3 * lpSize; + } + if (pos >= data.length) return false; + pos++; + while (pos < data.length) { + const blockSize = data[pos]; + if (blockSize === 0x00) break; + pos += blockSize + 1; + } + if (pos >= data.length) return false; + pos++; + } else if (blockType === 0x3B) { + break; + } else { + return false; + } + } + return false; +} + + +function isAPNG(d: Uint8Array): boolean { + let p = 8; + while (p + 8 < d.length) { + const length = (d[p] << 24 | d[p + 1] << 16 | d[p + 2] << 8 | d[p + 3]) >>> 0; + // 'acTL' before 'IDAT': APNG + if (d[p + 4] === 0x61 && d[p + 5] === 0x63 && d[p + 6] === 0x54 && d[p + 7] === 0x4c) { + return true; + } + // 'IDAT' first: no APNG + if (d[p + 4] === 0x49 && d[p + 5] === 0x44 && d[p + 6] === 0x41 && d[p + 7] === 0x54) { + return false; + } + p += 12 + length; + } + return false; +} diff --git a/addons/addon-image/src/ImageAddon.ts b/addons/addon-image/src/ImageAddon.ts index 8fd395431..ae5153dc4 100644 --- a/addons/addon-image/src/ImageAddon.ts +++ b/addons/addon-image/src/ImageAddon.ts @@ -15,6 +15,7 @@ import { SixelHandler } from './SixelHandler'; import { SixelImageStorage } from './SixelImageStorage'; import { IIPImageStorage } from './IIPImageStorage'; import { ITerminalExt, IImageAddonOptions, IResetHandler } from './Types'; +import { AnimationManager } from 'AnimationManager'; /** @@ -69,6 +70,7 @@ const DEFAULT_OPTIONS: IImageAddonOptions = { showPlaceholder: true, iipSupport: true, iipSizeLimit: 33554432, + iipAnimation: true, // alpha kittySupport: true, kittySizeLimit: 33554432 }; @@ -104,6 +106,7 @@ export class ImageAddon implements ITerminalAddon, IImageApi { private _disposables: IDisposable[] = []; private _terminal: ITerminalExt | undefined; private _handlers: Map = new Map(); + private _aniManager: AnimationManager | undefined; private readonly _onImageAdded = new Emitter(); public readonly onImageAdded: IEvent = this._onImageAdded.event; @@ -144,6 +147,11 @@ export class ImageAddon implements ITerminalAddon, IImageApi { terminal.options.windowOptions = windowOps; } + if (this._opts.iipAnimation && typeof ImageDecoder === 'function') { + this._aniManager = new AnimationManager(this._storage, this._renderer, terminal); + this._disposeLater(terminal.onRender(range => this._aniManager!.viewportUpdated(range))); + } + this._disposeLater( this._renderer, this._storage, @@ -187,7 +195,7 @@ export class ImageAddon implements ITerminalAddon, IImageApi { // iTerm IIP handler if (this._opts.iipSupport) { const iipStorage = new IIPImageStorage(this._storage!); - const iipHandler = new IIPHandler(this._opts, this._renderer!, iipStorage, terminal); + const iipHandler = new IIPHandler(this._opts, this._renderer!, iipStorage, terminal, this._aniManager); this._handlers.set('iip', iipHandler); this._disposeLater( terminal._core._inputHandler._parser.registerOscHandler(1337, iipHandler) @@ -218,6 +226,7 @@ export class ImageAddon implements ITerminalAddon, IImageApi { for (const handler of this._handlers.values()) { handler.reset(); } + this._aniManager?.reset(); return false; } diff --git a/addons/addon-image/src/ImageStorage.ts b/addons/addon-image/src/ImageStorage.ts index 322b9e02f..58550ef3a 100644 --- a/addons/addon-image/src/ImageStorage.ts +++ b/addons/addon-image/src/ImageStorage.ts @@ -103,7 +103,7 @@ class ExtendedAttrsImage implements IExtendedAttrsImage { return this.underlineStyle === UnderlineStyle.NONE && this._urlId === 0 && this.imageId === -1; } } -const EMPTY_ATTRS = new ExtendedAttrsImage(); +export const EMPTY_ATTRS = new ExtendedAttrsImage(); /** diff --git a/addons/addon-image/src/Types.ts b/addons/addon-image/src/Types.ts index 53d357a97..a8e110cca 100644 --- a/addons/addon-image/src/Types.ts +++ b/addons/addon-image/src/Types.ts @@ -39,6 +39,7 @@ export interface IImageAddonOptions { sixelSizeLimit: number; iipSupport: boolean; iipSizeLimit: number; + iipAnimation: boolean; kittySupport: boolean; kittySizeLimit: number; }