Files
xterm.js/addons/addon-image/src/kitty/KittyGraphicsHandler.ts
T

634 lines
21 KiB
TypeScript

/**
* Copyright (c) 2025 The xterm.js authors. All rights reserved.
* @license MIT
*/
import { IApcHandler, IImageAddonOptions, IResetHandler, ITerminalExt } from '../Types';
import { ImageRenderer } from '../ImageRenderer';
import { ImageStorage, CELL_SIZE_DEFAULT } from '../ImageStorage';
import Base64Decoder, { type DecodeStatus } from 'xterm-wasm-parts/lib/base64/Base64Decoder.wasm';
import {
KittyAction,
KittyFormat,
KittyCompression,
IKittyCommand,
IPendingTransmission,
IKittyImageData,
BYTES_PER_PIXEL_RGB,
BYTES_PER_PIXEL_RGBA,
ALPHA_OPAQUE,
parseKittyCommand
} from './KittyGraphicsTypes';
// Memory limit for base64 decoder (4MB, same as IIPHandler)
const DECODER_KEEP_DATA = 4194304;
const DECODER_INITIAL_DATA = 4194304; // 4MB
// Local mirror of const enum (esbuild can't inline const enums from external packages)
const DECODER_OK: DecodeStatus.OK = 0;
// Maximum control data size
const MAX_CONTROL_DATA_SIZE = 512;
// Semicolon codepoint
const SEMICOLON = 0x3B;
/**
* Kitty graphics protocol handler with streaming base64 decoding.
*/
export class KittyGraphicsHandler implements IApcHandler, IResetHandler {
private _aborted = false;
private _decodeError = false;
private _activeDecoder: Base64Decoder | null = null;
private readonly _maxEncodedBytes: number;
private readonly _initialEncodedBytes: number;
// Streaming related states
/** True while receiving control data (before semicolon). */
private _inControlData = true;
/** Buffer for control data. */
private _controlData = new Uint32Array(MAX_CONTROL_DATA_SIZE);
private _controlLength = 0;
/** Pre-calculated encoded size limit */
private _encodedSizeLimit = 0;
private _totalEncodedSize = 0;
/** Parsed command. These are the control data before semicolon. */
private _parsedCommand: IKittyCommand | null = null;
// Storage related states
private _pendingTransmissions: Map<number, IPendingTransmission> = new Map();
private _nextImageId = 1;
/** Maps Kitty protocol image ID → ImageStorage internal ID for deletion/lookup. */
private _kittyIdToStorageId: Map<number, number> = new Map();
// TODO: Eliminate double storage — raw image data lives here (as Blob) AND rendered
// ImageBitmaps live in ImageStorage. Currently we only use ImageStorage.addImage(bitmap)
// for tiling + cursor movement + marker-based eviction.
//
// See: https://github.com/xtermjs/xterm.js/pull/5619#issuecomment-3853678815
private _images: Map<number, IKittyImageData> = new Map();
constructor(
private readonly _opts: IImageAddonOptions,
private readonly _renderer: ImageRenderer,
private readonly _storage: ImageStorage,
private readonly _coreTerminal: ITerminalExt
) {
// Convert decoded size limit -> max encoded bytes.
this._maxEncodedBytes = Math.ceil(this._opts.kittySizeLimit * 4 / 3);
// ensure we preallocate more than configured limit while using 4mb initial size.
this._initialEncodedBytes = Math.min(DECODER_INITIAL_DATA, this._maxEncodedBytes);
}
public reset(): void {
for (const pending of this._pendingTransmissions.values()) {
pending.decoder.release();
}
this._pendingTransmissions.clear();
if (this._activeDecoder) {
this._activeDecoder.release();
this._activeDecoder = null;
}
this._images.clear();
this._kittyIdToStorageId.clear();
}
public start(): void {
this._aborted = false;
this._decodeError = false;
this._inControlData = true;
this._controlLength = 0;
this._parsedCommand = null;
// Pre-calculate encoded limit once: base64 is 4 bytes encoded → 3 bytes decoded
this._encodedSizeLimit = this._maxEncodedBytes;
this._totalEncodedSize = 0;
this._activeDecoder = null;
}
public put(data: Uint32Array, start: number, end: number): void {
if (this._aborted) return;
if (!this._inControlData) {
this._streamPayload(data, start, end);
} else {
// Scan for semicolon
let controlEnd = end;
for (let i = start; i < end; i++) {
if (data[i] === SEMICOLON) {
this._inControlData = false;
controlEnd = i;
break;
}
}
// Copy control data
const copyLength = controlEnd - start;
if (this._controlLength + copyLength > MAX_CONTROL_DATA_SIZE) {
this._aborted = true;
return;
}
this._controlData.set(data.subarray(start, controlEnd), this._controlLength);
this._controlLength += copyLength;
if (!this._inControlData) {
// Found semicolon - parse control data early for validation
this._parsedCommand = parseKittyCommand(this._parseControlDataString());
// Early validation: i+I conflict
if (this._parsedCommand.id !== undefined && this._parsedCommand.imageNumber !== undefined) {
this._sendResponse(this._parsedCommand.id, 'EINVAL:cannot specify both i and I keys', this._parsedCommand.quiet ?? 0);
this._aborted = true;
return;
}
// Delete action doesn't need payload - skip streaming
if (this._parsedCommand.action === KittyAction.DELETE) {
return;
}
// Stream remaining as payload
const payloadStart = controlEnd + 1;
if (payloadStart < end) {
this._streamPayload(data, payloadStart, end);
}
}
}
}
/**
* Stream payload bytes into the base64 decoder.
*/
private _streamPayload(data: Uint32Array, start: number, end: number): void {
if (this._aborted) return;
// Check size limit (compare encoded bytes against pre-calculated limit)
// Include cumulative size from pending transmission for multi-chunk images
const pendingKey = this._parsedCommand?.id ?? 0;
const pending = this._pendingTransmissions.get(pendingKey);
const previousEncodedSize = pending?.totalEncodedSize ?? 0;
this._totalEncodedSize += end - start;
const cumulativeEncodedSize = previousEncodedSize + this._totalEncodedSize;
if (cumulativeEncodedSize > this._encodedSizeLimit) {
const decoderToRelease = this._activeDecoder ?? pending?.decoder;
if (decoderToRelease) {
decoderToRelease.release();
}
this._activeDecoder = null;
if (pending) {
this._pendingTransmissions.delete(pendingKey);
}
this._aborted = true;
return;
}
if (this._decodeError) return;
if (pending?.decoder && !this._activeDecoder) {
this._activeDecoder = pending.decoder;
}
if (!this._activeDecoder) {
this._activeDecoder = new Base64Decoder(DECODER_KEEP_DATA, this._maxEncodedBytes, this._initialEncodedBytes);
this._activeDecoder.init();
}
if (this._activeDecoder.put(data.subarray(start, end)) !== DECODER_OK) {
this._activeDecoder.release();
this._activeDecoder = null;
this._decodeError = true;
if (pending) {
this._pendingTransmissions.delete(pendingKey);
}
}
}
public end(success: boolean): boolean | Promise<boolean> {
if (this._aborted || !success) {
if (this._activeDecoder) {
this._activeDecoder.release();
this._activeDecoder = null;
}
return true;
}
// No semicolon = no payload (delete, capability query)
if (this._inControlData) {
return this._handleNoPayloadCommand();
}
// Use command parsed early in put() - i+I already validated there
const cmd = this._parsedCommand!;
// Delete action was handled by skipping payload - just execute
if (cmd.action === KittyAction.DELETE) {
return this._handleDelete(cmd);
}
const pendingKey = cmd.id ?? 0;
const isMoreComing = cmd.more === 1;
const pending = this._pendingTransmissions.get(pendingKey);
if (isMoreComing) {
if (this._activeDecoder) {
if (pending) {
pending.totalEncodedSize += this._totalEncodedSize;
pending.decodeError = pending.decodeError || this._decodeError;
} else {
this._pendingTransmissions.set(pendingKey, {
cmd: { ...cmd },
decoder: this._activeDecoder,
totalEncodedSize: this._totalEncodedSize,
decodeError: this._decodeError
});
}
this._activeDecoder = null;
}
return true;
}
let decodeError = this._decodeError;
let finalCmd = cmd;
let decoder = this._activeDecoder;
if (pending) {
finalCmd = pending.cmd;
decoder = pending.decoder;
decodeError = decodeError || pending.decodeError;
this._pendingTransmissions.delete(pendingKey);
}
let imageBytes = new Uint8Array(0);
if (decoder) {
if (decoder.end() !== DECODER_OK) {
decodeError = true;
}
imageBytes = decoder.data8;
}
this._activeDecoder = null;
// Handle command first — handlers create Blob/ImageData from imageBytes,
// which copies the data. Only then is it safe to release the decoder's
// wasm memory that imageBytes points into.
const result = this._handleCommandWithBytesAndCmd(finalCmd, imageBytes, decodeError);
if (decoder) {
decoder.release();
}
return result;
}
// Command handling
private _parseControlDataString(): string {
let str = '';
for (let i = 0; i < this._controlLength; i++) {
str += String.fromCodePoint(this._controlData[i]);
}
return str;
}
private _handleNoPayloadCommand(): boolean | Promise<boolean> {
const cmd = parseKittyCommand(this._parseControlDataString());
// Per spec: specifying both i and I is an error
if (cmd.id !== undefined && cmd.imageNumber !== undefined) {
this._sendResponse(cmd.id, 'EINVAL:cannot specify both i and I keys', cmd.quiet ?? 0);
return true;
}
const action = cmd.action ?? 't';
switch (action) {
case KittyAction.DELETE:
return this._handleDelete(cmd);
case KittyAction.QUERY:
this._sendResponse(cmd.id ?? 0, 'OK', cmd.quiet ?? 0);
return true;
default:
return true;
}
}
private _handleCommandWithBytesAndCmd(cmd: IKittyCommand, bytes: Uint8Array, decodeError: boolean): boolean | Promise<boolean> {
const action = cmd.action ?? 't';
switch (action) {
case KittyAction.TRANSMIT: {
const result = this._handleTransmit(cmd, bytes, decodeError);
if (cmd.id !== undefined && !decodeError && bytes.length > 0) {
this._sendResponse(cmd.id, 'OK', cmd.quiet ?? 0);
}
return result;
}
case KittyAction.TRANSMIT_DISPLAY:
return this._handleTransmitDisplay(cmd, bytes, decodeError);
case KittyAction.QUERY:
return this._handleQuery(cmd, bytes, decodeError);
default:
return true;
}
}
private _handleTransmit(cmd: IKittyCommand, bytes: Uint8Array, decodeError: boolean): boolean {
// TODO: Support file-based transmission modes (t=f, t=t, t=s)
// Currently only supports direct transmission (t=d, the default).
// - t=f (file): Payload is base64-encoded file path. Terminal reads image from that path.
// - t=t (temp file): Payload is base64-encoded path in temp directory. Terminal reads, deletes.
// - t=s Payload is base64-encoded POSIX shm name. Terminal reads from shared memory.
// These modes require filesystem/IPC access not available in browsers. For Node.js/Electron:
// 1. Check cmd.transmission (t key) before treating bytes as image data
// 2. For t=f/t/s: decode bytes as UTF-8 string (the path/name), then read file contents
// 3. For t=d: treat bytes as image data (current behavior)
if (decodeError || bytes.length === 0) return true;
const id = cmd.id ?? this._nextImageId++;
const image: IKittyImageData = {
id,
data: new Blob([bytes as BlobPart]),
width: cmd.width ?? 0,
height: cmd.height ?? 0,
format: (cmd.format ?? KittyFormat.RGBA) as 24 | 32 | 100,
compression: cmd.compression ?? ''
};
this._images.set(id, image);
return true;
}
private _handleTransmitDisplay(cmd: IKittyCommand, bytes: Uint8Array, decodeError: boolean): boolean | Promise<boolean> {
if (decodeError) return true;
const pendingKey = cmd.id ?? 0;
this._handleTransmit(cmd, bytes, decodeError);
// If still accumulating chunks, don't display yet
if (this._pendingTransmissions.has(pendingKey)) return true;
// Display the completed image
const id = cmd.id ?? this._nextImageId - 1;
const image = this._images.get(id);
if (image) {
const result = this._displayImage(image, cmd);
if (cmd.id !== undefined) {
return (result as Promise<boolean>).then(r => {
this._sendResponse(id, 'OK', cmd.quiet ?? 0);
return r;
});
}
return result;
}
return true;
}
private _handleQuery(cmd: IKittyCommand, bytes: Uint8Array, decodeError: boolean): boolean {
const id = cmd.id ?? 0;
const quiet = cmd.quiet ?? 0;
// Check decode error first (invalid base64)
if (decodeError) {
this._sendResponse(id, 'EINVAL:invalid base64 data', quiet);
return true;
}
// Capability query (no payload) - just respond OK
if (bytes.length === 0) {
this._sendResponse(id, 'OK', quiet);
return true;
}
const format = cmd.format ?? KittyFormat.RGBA;
if (format === KittyFormat.PNG) {
this._sendResponse(id, 'OK', quiet);
} else {
const width = cmd.width ?? 0;
const height = cmd.height ?? 0;
if (!width || !height) {
this._sendResponse(id, 'EINVAL:width and height required for raw pixel data', quiet);
return true;
}
const bytesPerPixel = format === KittyFormat.RGBA ? BYTES_PER_PIXEL_RGBA : BYTES_PER_PIXEL_RGB;
const expectedBytes = width * height * bytesPerPixel;
if (bytes.length < expectedBytes) {
this._sendResponse(id, `EINVAL:insufficient pixel data`, quiet);
return true;
}
this._sendResponse(id, 'OK', quiet);
}
return true;
}
private _handleDelete(cmd: IKittyCommand): boolean {
const id = cmd.id;
if (id !== undefined) {
// Abort in-flight chunked upload for this specific image
const pending = this._pendingTransmissions.get(id);
if (pending) {
pending.decoder.release();
this._pendingTransmissions.delete(id);
}
this._images.delete(id);
const storageId = this._kittyIdToStorageId.get(id);
if (storageId !== undefined) {
this._storage.deleteImage(storageId);
this._kittyIdToStorageId.delete(id);
}
} else {
// Abort all in-flight chunked uploads
for (const pending of this._pendingTransmissions.values()) {
pending.decoder.release();
}
this._pendingTransmissions.clear();
this._images.clear();
for (const storageId of this._kittyIdToStorageId.values()) {
this._storage.deleteImage(storageId);
}
this._kittyIdToStorageId.clear();
}
return true;
}
private _sendResponse(id: number, message: string, quiet: number): void {
const isOk = message === 'OK';
if (isOk && quiet === 1) return;
if (!isOk && quiet === 2) return;
const response = `\x1b_Gi=${id};${message}\x1b\\`;
this._coreTerminal._core.coreService.triggerDataEvent(response);
}
// Image display
private _displayImage(image: IKittyImageData, cmd: IKittyCommand): boolean | Promise<boolean> {
return this._decodeAndDisplay(image, cmd)
.then(() => true)
.catch(() => true);
}
private async _decodeAndDisplay(image: IKittyImageData, cmd: IKittyCommand): Promise<void> {
const bitmap = await this._createBitmap(image);
const cw = this._renderer.dimensions?.css.cell.width || CELL_SIZE_DEFAULT.width;
const ch = this._renderer.dimensions?.css.cell.height || CELL_SIZE_DEFAULT.height;
// Per spec: c/r default to image's natural cell dimensions
const imgCols = cmd.columns ?? Math.ceil(bitmap.width / cw);
const imgRows = cmd.rows ?? Math.ceil(bitmap.height / ch);
let w = bitmap.width;
let h = bitmap.height;
// Scale bitmap to fit placement rectangle when c/r are specified
if (cmd.columns !== undefined || cmd.rows !== undefined) {
w = Math.round(imgCols * cw);
h = Math.round(imgRows * ch);
}
if (w * h > this._opts.pixelLimit) return;
// Save cursor position before addImage modifies it
const buffer = this._coreTerminal._core.buffer;
const savedX = buffer.x;
const savedY = buffer.y;
const savedYbase = buffer.ybase;
let storageId: number;
if (w !== bitmap.width || h !== bitmap.height) {
const resized = await createImageBitmap(bitmap, { resizeWidth: w, resizeHeight: h });
storageId = this._storage.addImage(resized);
} else {
storageId = this._storage.addImage(bitmap);
}
this._kittyIdToStorageId.set(image.id, storageId);
// Kitty cursor movement
// Per spec: cursor placed at first column after last image column,
// on the last row of the image. C=1 means don't move cursor.
if (cmd.cursorMovement === 1) {
// C=1: restore cursor to position before image was placed
const scrolled = buffer.ybase - savedYbase;
buffer.x = savedX;
buffer.y = Math.max(savedY - scrolled, 0);
} else {
// Default (C=0): advance cursor horizontally past the image
// addImage already positioned cursor on the last row via lineFeeds
buffer.x = Math.min(savedX + imgCols, this._coreTerminal.cols);
}
}
/**
* Create ImageBitmap from already-decoded image data.
*/
private async _createBitmap(image: IKittyImageData): Promise<ImageBitmap> {
let bytes: Uint8Array = new Uint8Array(await image.data.arrayBuffer());
if (image.compression === KittyCompression.ZLIB) {
bytes = await this._decompressZlib(bytes);
}
if (image.format === KittyFormat.PNG) {
const blob = new Blob([bytes as BlobPart], { type: 'image/png' });
if (!window.createImageBitmap) {
const url = URL.createObjectURL(blob);
const img = new Image();
return new Promise<ImageBitmap>((resolve, reject) => {
img.addEventListener('load', () => {
URL.revokeObjectURL(url);
const canvas = ImageRenderer.createCanvas(window.document, img.width, img.height);
canvas.getContext('2d')?.drawImage(img, 0, 0);
createImageBitmap(canvas).then(resolve).catch(reject);
});
img.addEventListener('error', reject);
img.src = url;
});
}
return createImageBitmap(blob);
}
// Raw pixel data
const width = image.width;
const height = image.height;
if (!width || !height) {
throw new Error('Width and height required for raw pixel data');
}
const bytesPerPixel = image.format === KittyFormat.RGBA ? BYTES_PER_PIXEL_RGBA : BYTES_PER_PIXEL_RGB;
const expectedBytes = width * height * bytesPerPixel;
if (bytes.length < expectedBytes) {
throw new Error('Insufficient pixel data');
}
const pixelCount = width * height;
const data = new Uint8ClampedArray(pixelCount * BYTES_PER_PIXEL_RGBA);
const isRgba = image.format === KittyFormat.RGBA;
let srcOffset = 0;
let dstOffset = 0;
for (let i = 0; i < pixelCount; i++) {
data[dstOffset] = bytes[srcOffset];
data[dstOffset + 1] = bytes[srcOffset + 1];
data[dstOffset + 2] = bytes[srcOffset + 2];
data[dstOffset + 3] = isRgba ? bytes[srcOffset + 3] : ALPHA_OPAQUE;
srcOffset += bytesPerPixel;
dstOffset += BYTES_PER_PIXEL_RGBA;
}
return createImageBitmap(new ImageData(data, width, height));
}
private async _decompressZlib(compressed: Uint8Array): Promise<Uint8Array> {
try {
return await this._decompress(compressed, 'deflate');
} catch {
return await this._decompress(compressed, 'deflate-raw');
}
}
private async _decompress(compressed: Uint8Array, format: 'deflate' | 'deflate-raw'): Promise<Uint8Array> {
const ds = new DecompressionStream(format);
const writer = ds.writable.getWriter();
writer.write(compressed as BufferSource);
writer.close();
const chunks: Uint8Array[] = [];
const reader = ds.readable.getReader();
while (true) {
const { done, value } = await reader.read();
if (done) break;
chunks.push(value);
}
const totalLength = chunks.reduce((sum, chunk) => sum + chunk.length, 0);
const result = new Uint8Array(totalLength);
let offset = 0;
for (const chunk of chunks) {
result.set(chunk, offset);
offset += chunk.length;
}
return result;
}
public get images(): ReadonlyMap<number, IKittyImageData> {
return this._images;
}
public get pendingTransmissions(): ReadonlyMap<number, IPendingTransmission> {
return this._pendingTransmissions;
}
}