/** * Copyright (c) 2025 The xterm.js authors. All rights reserved. * @license MIT */ import { IApcHandler, IImageAddonOptions, IResetHandler, ITerminalExt } from '../Types'; import { ImageRenderer } from '../ImageRenderer'; import { ImageStorage, CELL_SIZE_DEFAULT } from '../ImageStorage'; import Base64Decoder, { type DecodeStatus } from 'xterm-wasm-parts/lib/base64/Base64Decoder.wasm'; import { KittyAction, KittyFormat, KittyCompression, IKittyCommand, IPendingTransmission, IKittyImageData, BYTES_PER_PIXEL_RGB, BYTES_PER_PIXEL_RGBA, ALPHA_OPAQUE, parseKittyCommand } from './KittyGraphicsTypes'; // Memory limit for base64 decoder (4MB, same as IIPHandler) const DECODER_KEEP_DATA = 4194304; const DECODER_INITIAL_DATA = 4194304; // 4MB // Local mirror of const enum (esbuild can't inline const enums from external packages) const DECODER_OK: DecodeStatus.OK = 0; // Maximum control data size const MAX_CONTROL_DATA_SIZE = 512; // Semicolon codepoint const SEMICOLON = 0x3B; /** * Kitty graphics protocol handler with streaming base64 decoding. */ export class KittyGraphicsHandler implements IApcHandler, IResetHandler { private _aborted = false; private _decodeError = false; private _activeDecoder: Base64Decoder | null = null; private readonly _maxEncodedBytes: number; private readonly _initialEncodedBytes: number; // Streaming related states /** True while receiving control data (before semicolon). */ private _inControlData = true; /** Buffer for control data. */ private _controlData = new Uint32Array(MAX_CONTROL_DATA_SIZE); private _controlLength = 0; /** Pre-calculated encoded size limit */ private _encodedSizeLimit = 0; private _totalEncodedSize = 0; /** Parsed command. These are the control data before semicolon. */ private _parsedCommand: IKittyCommand | null = null; // Storage related states private _pendingTransmissions: Map = new Map(); private _nextImageId = 1; /** Maps Kitty protocol image ID → ImageStorage internal ID for deletion/lookup. */ private _kittyIdToStorageId: Map = new Map(); // TODO: Eliminate double storage — raw image data lives here (as Blob) AND rendered // ImageBitmaps live in ImageStorage. Currently we only use ImageStorage.addImage(bitmap) // for tiling + cursor movement + marker-based eviction. // // See: https://github.com/xtermjs/xterm.js/pull/5619#issuecomment-3853678815 private _images: Map = new Map(); constructor( private readonly _opts: IImageAddonOptions, private readonly _renderer: ImageRenderer, private readonly _storage: ImageStorage, private readonly _coreTerminal: ITerminalExt ) { // Convert decoded size limit -> max encoded bytes. this._maxEncodedBytes = Math.ceil(this._opts.kittySizeLimit * 4 / 3); // ensure we preallocate more than configured limit while using 4mb initial size. this._initialEncodedBytes = Math.min(DECODER_INITIAL_DATA, this._maxEncodedBytes); } public reset(): void { for (const pending of this._pendingTransmissions.values()) { pending.decoder.release(); } this._pendingTransmissions.clear(); if (this._activeDecoder) { this._activeDecoder.release(); this._activeDecoder = null; } this._images.clear(); this._kittyIdToStorageId.clear(); } public start(): void { this._aborted = false; this._decodeError = false; this._inControlData = true; this._controlLength = 0; this._parsedCommand = null; // Pre-calculate encoded limit once: base64 is 4 bytes encoded → 3 bytes decoded this._encodedSizeLimit = this._maxEncodedBytes; this._totalEncodedSize = 0; this._activeDecoder = null; } public put(data: Uint32Array, start: number, end: number): void { if (this._aborted) return; if (!this._inControlData) { this._streamPayload(data, start, end); } else { // Scan for semicolon let controlEnd = end; for (let i = start; i < end; i++) { if (data[i] === SEMICOLON) { this._inControlData = false; controlEnd = i; break; } } // Copy control data const copyLength = controlEnd - start; if (this._controlLength + copyLength > MAX_CONTROL_DATA_SIZE) { this._aborted = true; return; } this._controlData.set(data.subarray(start, controlEnd), this._controlLength); this._controlLength += copyLength; if (!this._inControlData) { // Found semicolon - parse control data early for validation this._parsedCommand = parseKittyCommand(this._parseControlDataString()); // Early validation: i+I conflict if (this._parsedCommand.id !== undefined && this._parsedCommand.imageNumber !== undefined) { this._sendResponse(this._parsedCommand.id, 'EINVAL:cannot specify both i and I keys', this._parsedCommand.quiet ?? 0); this._aborted = true; return; } // Delete action doesn't need payload - skip streaming if (this._parsedCommand.action === KittyAction.DELETE) { return; } // Stream remaining as payload const payloadStart = controlEnd + 1; if (payloadStart < end) { this._streamPayload(data, payloadStart, end); } } } } /** * Stream payload bytes into the base64 decoder. */ private _streamPayload(data: Uint32Array, start: number, end: number): void { if (this._aborted) return; // Check size limit (compare encoded bytes against pre-calculated limit) // Include cumulative size from pending transmission for multi-chunk images const pendingKey = this._parsedCommand?.id ?? 0; const pending = this._pendingTransmissions.get(pendingKey); const previousEncodedSize = pending?.totalEncodedSize ?? 0; this._totalEncodedSize += end - start; const cumulativeEncodedSize = previousEncodedSize + this._totalEncodedSize; if (cumulativeEncodedSize > this._encodedSizeLimit) { const decoderToRelease = this._activeDecoder ?? pending?.decoder; if (decoderToRelease) { decoderToRelease.release(); } this._activeDecoder = null; if (pending) { this._pendingTransmissions.delete(pendingKey); } this._aborted = true; return; } if (this._decodeError) return; if (pending?.decoder && !this._activeDecoder) { this._activeDecoder = pending.decoder; } if (!this._activeDecoder) { this._activeDecoder = new Base64Decoder(DECODER_KEEP_DATA, this._maxEncodedBytes, this._initialEncodedBytes); this._activeDecoder.init(); } if (this._activeDecoder.put(data.subarray(start, end)) !== DECODER_OK) { this._activeDecoder.release(); this._activeDecoder = null; this._decodeError = true; if (pending) { this._pendingTransmissions.delete(pendingKey); } } } public end(success: boolean): boolean | Promise { if (this._aborted || !success) { if (this._activeDecoder) { this._activeDecoder.release(); this._activeDecoder = null; } return true; } // No semicolon = no payload (delete, capability query) if (this._inControlData) { return this._handleNoPayloadCommand(); } // Use command parsed early in put() - i+I already validated there const cmd = this._parsedCommand!; // Delete action was handled by skipping payload - just execute if (cmd.action === KittyAction.DELETE) { return this._handleDelete(cmd); } const pendingKey = cmd.id ?? 0; const isMoreComing = cmd.more === 1; const pending = this._pendingTransmissions.get(pendingKey); if (isMoreComing) { if (this._activeDecoder) { if (pending) { pending.totalEncodedSize += this._totalEncodedSize; pending.decodeError = pending.decodeError || this._decodeError; } else { this._pendingTransmissions.set(pendingKey, { cmd: { ...cmd }, decoder: this._activeDecoder, totalEncodedSize: this._totalEncodedSize, decodeError: this._decodeError }); } this._activeDecoder = null; } return true; } let decodeError = this._decodeError; let finalCmd = cmd; let decoder = this._activeDecoder; if (pending) { finalCmd = pending.cmd; decoder = pending.decoder; decodeError = decodeError || pending.decodeError; this._pendingTransmissions.delete(pendingKey); } let imageBytes = new Uint8Array(0); if (decoder) { if (decoder.end() !== DECODER_OK) { decodeError = true; } imageBytes = decoder.data8; } this._activeDecoder = null; // Handle command first — handlers create Blob/ImageData from imageBytes, // which copies the data. Only then is it safe to release the decoder's // wasm memory that imageBytes points into. const result = this._handleCommandWithBytesAndCmd(finalCmd, imageBytes, decodeError); if (decoder) { decoder.release(); } return result; } // Command handling private _parseControlDataString(): string { let str = ''; for (let i = 0; i < this._controlLength; i++) { str += String.fromCodePoint(this._controlData[i]); } return str; } private _handleNoPayloadCommand(): boolean | Promise { const cmd = parseKittyCommand(this._parseControlDataString()); // Per spec: specifying both i and I is an error if (cmd.id !== undefined && cmd.imageNumber !== undefined) { this._sendResponse(cmd.id, 'EINVAL:cannot specify both i and I keys', cmd.quiet ?? 0); return true; } const action = cmd.action ?? 't'; switch (action) { case KittyAction.DELETE: return this._handleDelete(cmd); case KittyAction.QUERY: this._sendResponse(cmd.id ?? 0, 'OK', cmd.quiet ?? 0); return true; default: return true; } } private _handleCommandWithBytesAndCmd(cmd: IKittyCommand, bytes: Uint8Array, decodeError: boolean): boolean | Promise { const action = cmd.action ?? 't'; switch (action) { case KittyAction.TRANSMIT: { const result = this._handleTransmit(cmd, bytes, decodeError); if (cmd.id !== undefined && !decodeError && bytes.length > 0) { this._sendResponse(cmd.id, 'OK', cmd.quiet ?? 0); } return result; } case KittyAction.TRANSMIT_DISPLAY: return this._handleTransmitDisplay(cmd, bytes, decodeError); case KittyAction.QUERY: return this._handleQuery(cmd, bytes, decodeError); default: return true; } } private _handleTransmit(cmd: IKittyCommand, bytes: Uint8Array, decodeError: boolean): boolean { // TODO: Support file-based transmission modes (t=f, t=t, t=s) // Currently only supports direct transmission (t=d, the default). // - t=f (file): Payload is base64-encoded file path. Terminal reads image from that path. // - t=t (temp file): Payload is base64-encoded path in temp directory. Terminal reads, deletes. // - t=s Payload is base64-encoded POSIX shm name. Terminal reads from shared memory. // These modes require filesystem/IPC access not available in browsers. For Node.js/Electron: // 1. Check cmd.transmission (t key) before treating bytes as image data // 2. For t=f/t/s: decode bytes as UTF-8 string (the path/name), then read file contents // 3. For t=d: treat bytes as image data (current behavior) if (decodeError || bytes.length === 0) return true; const id = cmd.id ?? this._nextImageId++; const image: IKittyImageData = { id, data: new Blob([bytes as BlobPart]), width: cmd.width ?? 0, height: cmd.height ?? 0, format: (cmd.format ?? KittyFormat.RGBA) as 24 | 32 | 100, compression: cmd.compression ?? '' }; this._images.set(id, image); return true; } private _handleTransmitDisplay(cmd: IKittyCommand, bytes: Uint8Array, decodeError: boolean): boolean | Promise { if (decodeError) return true; const pendingKey = cmd.id ?? 0; this._handleTransmit(cmd, bytes, decodeError); // If still accumulating chunks, don't display yet if (this._pendingTransmissions.has(pendingKey)) return true; // Display the completed image const id = cmd.id ?? this._nextImageId - 1; const image = this._images.get(id); if (image) { const result = this._displayImage(image, cmd); if (cmd.id !== undefined) { return (result as Promise).then(r => { this._sendResponse(id, 'OK', cmd.quiet ?? 0); return r; }); } return result; } return true; } private _handleQuery(cmd: IKittyCommand, bytes: Uint8Array, decodeError: boolean): boolean { const id = cmd.id ?? 0; const quiet = cmd.quiet ?? 0; // Check decode error first (invalid base64) if (decodeError) { this._sendResponse(id, 'EINVAL:invalid base64 data', quiet); return true; } // Capability query (no payload) - just respond OK if (bytes.length === 0) { this._sendResponse(id, 'OK', quiet); return true; } const format = cmd.format ?? KittyFormat.RGBA; if (format === KittyFormat.PNG) { this._sendResponse(id, 'OK', quiet); } else { const width = cmd.width ?? 0; const height = cmd.height ?? 0; if (!width || !height) { this._sendResponse(id, 'EINVAL:width and height required for raw pixel data', quiet); return true; } const bytesPerPixel = format === KittyFormat.RGBA ? BYTES_PER_PIXEL_RGBA : BYTES_PER_PIXEL_RGB; const expectedBytes = width * height * bytesPerPixel; if (bytes.length < expectedBytes) { this._sendResponse(id, `EINVAL:insufficient pixel data`, quiet); return true; } this._sendResponse(id, 'OK', quiet); } return true; } private _handleDelete(cmd: IKittyCommand): boolean { const id = cmd.id; if (id !== undefined) { // Abort in-flight chunked upload for this specific image const pending = this._pendingTransmissions.get(id); if (pending) { pending.decoder.release(); this._pendingTransmissions.delete(id); } this._images.delete(id); const storageId = this._kittyIdToStorageId.get(id); if (storageId !== undefined) { this._storage.deleteImage(storageId); this._kittyIdToStorageId.delete(id); } } else { // Abort all in-flight chunked uploads for (const pending of this._pendingTransmissions.values()) { pending.decoder.release(); } this._pendingTransmissions.clear(); this._images.clear(); for (const storageId of this._kittyIdToStorageId.values()) { this._storage.deleteImage(storageId); } this._kittyIdToStorageId.clear(); } return true; } private _sendResponse(id: number, message: string, quiet: number): void { const isOk = message === 'OK'; if (isOk && quiet === 1) return; if (!isOk && quiet === 2) return; const response = `\x1b_Gi=${id};${message}\x1b\\`; this._coreTerminal._core.coreService.triggerDataEvent(response); } // Image display private _displayImage(image: IKittyImageData, cmd: IKittyCommand): boolean | Promise { return this._decodeAndDisplay(image, cmd) .then(() => true) .catch(() => true); } private async _decodeAndDisplay(image: IKittyImageData, cmd: IKittyCommand): Promise { const bitmap = await this._createBitmap(image); const cw = this._renderer.dimensions?.css.cell.width || CELL_SIZE_DEFAULT.width; const ch = this._renderer.dimensions?.css.cell.height || CELL_SIZE_DEFAULT.height; // Per spec: c/r default to image's natural cell dimensions const imgCols = cmd.columns ?? Math.ceil(bitmap.width / cw); const imgRows = cmd.rows ?? Math.ceil(bitmap.height / ch); let w = bitmap.width; let h = bitmap.height; // Scale bitmap to fit placement rectangle when c/r are specified if (cmd.columns !== undefined || cmd.rows !== undefined) { w = Math.round(imgCols * cw); h = Math.round(imgRows * ch); } if (w * h > this._opts.pixelLimit) return; // Save cursor position before addImage modifies it const buffer = this._coreTerminal._core.buffer; const savedX = buffer.x; const savedY = buffer.y; const savedYbase = buffer.ybase; let storageId: number; if (w !== bitmap.width || h !== bitmap.height) { const resized = await createImageBitmap(bitmap, { resizeWidth: w, resizeHeight: h }); storageId = this._storage.addImage(resized); } else { storageId = this._storage.addImage(bitmap); } this._kittyIdToStorageId.set(image.id, storageId); // Kitty cursor movement // Per spec: cursor placed at first column after last image column, // on the last row of the image. C=1 means don't move cursor. if (cmd.cursorMovement === 1) { // C=1: restore cursor to position before image was placed const scrolled = buffer.ybase - savedYbase; buffer.x = savedX; buffer.y = Math.max(savedY - scrolled, 0); } else { // Default (C=0): advance cursor horizontally past the image // addImage already positioned cursor on the last row via lineFeeds buffer.x = Math.min(savedX + imgCols, this._coreTerminal.cols); } } /** * Create ImageBitmap from already-decoded image data. */ private async _createBitmap(image: IKittyImageData): Promise { let bytes: Uint8Array = new Uint8Array(await image.data.arrayBuffer()); if (image.compression === KittyCompression.ZLIB) { bytes = await this._decompressZlib(bytes); } if (image.format === KittyFormat.PNG) { const blob = new Blob([bytes as BlobPart], { type: 'image/png' }); if (!window.createImageBitmap) { const url = URL.createObjectURL(blob); const img = new Image(); return new Promise((resolve, reject) => { img.addEventListener('load', () => { URL.revokeObjectURL(url); const canvas = ImageRenderer.createCanvas(window.document, img.width, img.height); canvas.getContext('2d')?.drawImage(img, 0, 0); createImageBitmap(canvas).then(resolve).catch(reject); }); img.addEventListener('error', reject); img.src = url; }); } return createImageBitmap(blob); } // Raw pixel data const width = image.width; const height = image.height; if (!width || !height) { throw new Error('Width and height required for raw pixel data'); } const bytesPerPixel = image.format === KittyFormat.RGBA ? BYTES_PER_PIXEL_RGBA : BYTES_PER_PIXEL_RGB; const expectedBytes = width * height * bytesPerPixel; if (bytes.length < expectedBytes) { throw new Error('Insufficient pixel data'); } const pixelCount = width * height; const data = new Uint8ClampedArray(pixelCount * BYTES_PER_PIXEL_RGBA); const isRgba = image.format === KittyFormat.RGBA; let srcOffset = 0; let dstOffset = 0; for (let i = 0; i < pixelCount; i++) { data[dstOffset] = bytes[srcOffset]; data[dstOffset + 1] = bytes[srcOffset + 1]; data[dstOffset + 2] = bytes[srcOffset + 2]; data[dstOffset + 3] = isRgba ? bytes[srcOffset + 3] : ALPHA_OPAQUE; srcOffset += bytesPerPixel; dstOffset += BYTES_PER_PIXEL_RGBA; } return createImageBitmap(new ImageData(data, width, height)); } private async _decompressZlib(compressed: Uint8Array): Promise { try { return await this._decompress(compressed, 'deflate'); } catch { return await this._decompress(compressed, 'deflate-raw'); } } private async _decompress(compressed: Uint8Array, format: 'deflate' | 'deflate-raw'): Promise { const ds = new DecompressionStream(format); const writer = ds.writable.getWriter(); writer.write(compressed as BufferSource); writer.close(); const chunks: Uint8Array[] = []; const reader = ds.readable.getReader(); while (true) { const { done, value } = await reader.read(); if (done) break; chunks.push(value); } const totalLength = chunks.reduce((sum, chunk) => sum + chunk.length, 0); const result = new Uint8Array(totalLength); let offset = 0; for (const chunk of chunks) { result.set(chunk, offset); offset += chunk.length; } return result; } public get images(): ReadonlyMap { return this._images; } public get pendingTransmissions(): ReadonlyMap { return this._pendingTransmissions; } }