Start integrating KittyGraphics stuff into imageAddon as suggested

This commit is contained in:
Anthony Kim
2026-01-29 00:53:10 -08:00
parent 492d950b64
commit cb8159c4f2
31 changed files with 1034 additions and 1765 deletions
+13 -1
View File
@@ -8,6 +8,7 @@ import type { ImageAddon as IImageApi } from '@xterm/addon-image';
import { IIPHandler } from './IIPHandler';
import { ImageRenderer } from './ImageRenderer';
import { ImageStorage, CELL_SIZE_DEFAULT } from './ImageStorage';
import { KittyGraphicsHandler } from './kitty/KittyGraphicsHandler';
import { SixelHandler } from './SixelHandler';
import { ITerminalExt, IImageAddonOptions, IResetHandler } from './Types';
@@ -22,7 +23,9 @@ const DEFAULT_OPTIONS: IImageAddonOptions = {
storageLimit: 128,
showPlaceholder: true,
iipSupport: true,
iipSizeLimit: 20000000
iipSizeLimit: 20000000,
kittySupport: true,
kittySizeLimit: 20000000
};
// max palette size supported by the sixel lib (compile time setting)
@@ -144,6 +147,15 @@ export class ImageAddon implements ITerminalAddon , IImageApi {
terminal._core._inputHandler._parser.registerOscHandler(1337, iipHandler)
);
}
// Kitty graphics handler
if (this._opts.kittySupport) {
const kittyHandler = new KittyGraphicsHandler(this._opts, this._renderer!, this._storage!, terminal);
this._handlers.set('kitty', kittyHandler);
this._disposeLater(
terminal._core._inputHandler._parser.registerApcHandler(0x47, kittyHandler)
);
}
}
// Note: storageLimit is skipped here to not intoduce a surprising side effect.
+4 -2
View File
@@ -8,7 +8,7 @@ import { IDisposable, IMarker, Terminal } from '@xterm/xterm';
// private imports from base repo we build against
import { Attributes, BgFlags, Content, ExtFlags, UnderlineStyle } from 'common/buffer/Constants';
import type { AttributeData } from 'common/buffer/AttributeData';
import type { IParams, IDcsHandler, IOscHandler, IEscapeSequenceParser } from 'common/parser/Types';
import type { IParams, IDcsHandler, IOscHandler, IApcHandler, IEscapeSequenceParser } from 'common/parser/Types';
import type { IBufferLine, IExtendedAttrs, IInputHandler } from 'common/Types';
import type { ITerminal, ReadonlyColorSet } from 'browser/Types';
import type { IRenderDimensions } from 'browser/renderer/shared/Types';
@@ -22,7 +22,7 @@ export const enum Cell {
}
// export some privates for local usage
export { AttributeData, IParams, IDcsHandler, IOscHandler, BgFlags, IRenderDimensions, IRenderService, Content, ExtFlags, Attributes, UnderlineStyle, ReadonlyColorSet };
export { AttributeData, IParams, IDcsHandler, IOscHandler, IApcHandler, BgFlags, IRenderDimensions, IRenderService, Content, ExtFlags, Attributes, UnderlineStyle, ReadonlyColorSet };
/**
* Plugin ctor options.
@@ -38,6 +38,8 @@ export interface IImageAddonOptions {
sixelSizeLimit: number;
iipSupport: boolean;
iipSizeLimit: number;
kittySupport: boolean;
kittySizeLimit: number;
}
export interface IResetHandler {
@@ -0,0 +1,494 @@
/**
* Copyright (c) 2025 The xterm.js authors. All rights reserved.
* @license MIT
*
* Kitty graphics protocol APC handler.
* Implements IApcHandler to integrate with xterm.js parser.
*/
import { IApcHandler, IImageAddonOptions, IResetHandler, ITerminalExt } from '../Types';
import { ImageRenderer } from '../ImageRenderer';
import { ImageStorage, CELL_SIZE_DEFAULT } from '../ImageStorage';
import {
KittyAction,
KittyFormat,
KittyCompression,
IKittyCommand,
IPendingTransmission,
IKittyImageData,
BYTES_PER_PIXEL_RGB,
BYTES_PER_PIXEL_RGBA,
ALPHA_OPAQUE,
parseKittyCommand
} from './KittyGraphicsTypes';
/**
* Kitty graphics protocol handler.
*
* Handles APC sequences for the Kitty graphics protocol, supporting:
* - Image transmission (a=t)
* - Image transmission + display (a=T)
* - Query (a=q)
* - Delete (a=d)
*
* TODO: File transmission (t=f) is not supported since browsers cannot access the filesystem.
* Explore using the File System Access API for opt-in filesystem access when available.
*/
export class KittyGraphicsHandler implements IApcHandler, IResetHandler {
private _aborted = false;
private _data: string = '';
/**
* Pending chunked transmissions keyed by image ID.
* ID 0 is used for transmissions without an explicit ID (the "anonymous" transmission).
*/
private _pendingTransmissions: Map<number, IPendingTransmission> = new Map();
private _nextImageId = 1;
/**
* Stored images - kept for protocol compliance (transmit without display, then display later).
*/
private _images: Map<number, IKittyImageData> = new Map();
/**
* Cached decoded bitmaps.
*/
private _decodedImages: Map<number, ImageBitmap> = new Map();
constructor(
private readonly _opts: IImageAddonOptions,
private readonly _renderer: ImageRenderer,
private readonly _storage: ImageStorage,
private readonly _coreTerminal: ITerminalExt
) {}
public reset(): void {
// Clear pending transmissions
this._pendingTransmissions.clear();
// Clear stored images
this._images.clear();
// Close decoded bitmaps
for (const bitmap of this._decodedImages.values()) {
bitmap.close();
}
this._decodedImages.clear();
}
/**
* Called at the start of a new APC sequence.
*/
public start(): void {
this._aborted = false;
this._data = '';
}
/**
* Called for each chunk of data in the APC sequence.
* Accumulates codepoints as a string.
*/
public put(data: Uint32Array, start: number, end: number): void {
if (this._aborted) return;
// Check size limit
if (this._data.length + (end - start) > this._opts.kittySizeLimit) {
console.warn('[KittyHandler] Data exceeds size limit, aborting');
this._aborted = true;
return;
}
// Convert Uint32Array codepoints to string
// TODO: This atob + charCodeAt pattern has bad runtime. Consider using the wasm-based
// base64 decoder from xterm-wasm-parts which supports chunked data ingestion.
// For now, we accumulate as string and decode at the end.
for (let i = start; i < end; i++) {
this._data += String.fromCodePoint(data[i]);
}
}
/**
* Called at the end of the APC sequence.
*/
public end(success: boolean): boolean | Promise<boolean> {
if (this._aborted || !success) {
return true;
}
return this._handleCommand(this._data);
}
/**
* Handle a complete Kitty graphics command.
*/
private _handleCommand(data: string): boolean | Promise<boolean> {
const semiIdx = data.indexOf(';');
const controlData = semiIdx === -1 ? data : data.substring(0, semiIdx);
const payload = semiIdx === -1 ? '' : data.substring(semiIdx + 1);
const cmd = parseKittyCommand(controlData);
cmd.payload = payload;
const action = cmd.action ?? 't';
switch (action) {
case KittyAction.TRANSMIT:
return this._handleTransmit(cmd);
case KittyAction.TRANSMIT_DISPLAY:
return this._handleTransmitDisplay(cmd);
case KittyAction.QUERY:
return this._handleQuery(cmd);
case KittyAction.DELETE:
return this._handleDelete(cmd);
default:
return true;
}
}
private _handleTransmit(cmd: IKittyCommand): boolean {
const payload = cmd.payload ?? '';
const pendingKey = cmd.id ?? 0;
const isMoreComing = cmd.more === 1;
const pending = this._pendingTransmissions.get(pendingKey);
if (pending) {
pending.data += payload;
if (isMoreComing) {
return true;
}
const originalCmd = pending.cmd;
const fullPayload = pending.data;
this._pendingTransmissions.delete(pendingKey);
const id = originalCmd.id ?? this._nextImageId++;
const image: IKittyImageData = {
id,
data: fullPayload,
width: originalCmd.width ?? 0,
height: originalCmd.height ?? 0,
format: (originalCmd.format ?? KittyFormat.PNG) as 24 | 32 | 100,
compression: originalCmd.compression ?? ''
};
this._images.set(image.id, image);
if (originalCmd.action === KittyAction.TRANSMIT_DISPLAY) {
// Fire-and-forget the display, transmit itself succeeded
void this._displayImage(image, originalCmd.columns, originalCmd.rows);
return true;
}
cmd.id = id;
return true;
}
if (isMoreComing) {
this._pendingTransmissions.set(pendingKey, {
cmd: { ...cmd },
data: payload
});
return true;
}
const id = cmd.id ?? this._nextImageId++;
const image: IKittyImageData = {
id,
data: payload,
width: cmd.width ?? 0,
height: cmd.height ?? 0,
format: (cmd.format ?? KittyFormat.PNG) as 24 | 32 | 100,
compression: cmd.compression ?? ''
};
this._images.set(image.id, image);
cmd.id = id;
return true;
}
private _handleTransmitDisplay(cmd: IKittyCommand): boolean | Promise<boolean> {
const pendingKey = cmd.id ?? 0;
const wasPendingBefore = this._pendingTransmissions.has(pendingKey);
this._handleTransmit(cmd);
if (cmd.more === 1) {
return true;
}
if (wasPendingBefore) {
return true;
}
const id = cmd.id!;
const image = this._images.get(id);
if (image) {
return this._displayImage(image, cmd.columns, cmd.rows);
}
return true;
}
/**
* Handle query action (a=q).
*/
private _handleQuery(cmd: IKittyCommand): boolean {
const id = cmd.id ?? 0;
const quiet = cmd.quiet ?? 0;
const payload = cmd.payload || '';
if (!payload) {
this._sendResponse(id, 'OK', quiet);
return true;
}
try {
const binaryString = atob(payload);
const bytes = new Uint8Array(binaryString.length);
for (let i = 0; i < binaryString.length; i++) {
bytes[i] = binaryString.charCodeAt(i);
}
const format = cmd.format || KittyFormat.RGBA;
if (format === KittyFormat.PNG) {
this._sendResponse(id, 'OK', quiet);
} else {
const width = cmd.width || 0;
const height = cmd.height || 0;
if (!width || !height) {
this._sendResponse(id, 'EINVAL:width and height required for raw pixel data', quiet);
return true;
}
const bytesPerPixel = format === KittyFormat.RGBA ? BYTES_PER_PIXEL_RGBA : BYTES_PER_PIXEL_RGB;
const expectedBytes = width * height * bytesPerPixel;
if (bytes.length < expectedBytes) {
this._sendResponse(id, `EINVAL:insufficient pixel data, got ${bytes.length}, expected ${expectedBytes}`, quiet);
return true;
}
this._sendResponse(id, 'OK', quiet);
}
} catch (e) {
const errorMsg = e instanceof Error ? e.message : 'unknown error';
this._sendResponse(id, `EINVAL:${errorMsg}`, quiet);
}
return true;
}
private _handleDelete(cmd: IKittyCommand): boolean {
const id = cmd.id;
if (id !== undefined) {
this._images.delete(id);
const bitmap = this._decodedImages.get(id);
if (bitmap) {
bitmap.close();
this._decodedImages.delete(id);
}
} else {
this._images.clear();
for (const bitmap of this._decodedImages.values()) {
bitmap.close();
}
this._decodedImages.clear();
}
return true;
}
/**
* Send a response back to the client.
*/
private _sendResponse(id: number, message: string, quiet: number): void {
const isOk = message === 'OK';
if (isOk && quiet === 1) return;
if (!isOk && quiet === 2) return;
const response = `\x1b_Gi=${id};${message}\x1b\\`;
this._coreTerminal._core.coreService.triggerDataEvent(response);
}
/**
* Decode and display an image using the shared ImageStorage.
*/
private _displayImage(image: IKittyImageData, columns?: number, rows?: number): boolean | Promise<boolean> {
return this._decodeAndDisplay(image, columns, rows)
.then(() => true)
.catch(err => {
console.warn('[KittyHandler] Failed to decode/display image:', err);
return true;
});
}
private async _decodeAndDisplay(image: IKittyImageData, columns?: number, rows?: number): Promise<void> {
let bitmap = this._decodedImages.get(image.id);
if (!bitmap) {
bitmap = await this._decodeImage(image);
this._decodedImages.set(image.id, bitmap);
}
// Calculate display size
let w = bitmap.width;
let h = bitmap.height;
if (columns || rows) {
const cw = this._renderer.dimensions?.css.cell.width || CELL_SIZE_DEFAULT.width;
const ch = this._renderer.dimensions?.css.cell.height || CELL_SIZE_DEFAULT.height;
if (columns) {
w = columns * cw;
}
if (rows) {
h = rows * ch;
}
// Maintain aspect ratio if only one dimension specified
if (columns && !rows) {
h = Math.round(w * (bitmap.height / bitmap.width));
} else if (rows && !columns) {
w = Math.round(h * (bitmap.width / bitmap.height));
}
}
// Check pixel limit
if (w * h > this._opts.pixelLimit) {
console.warn('[KittyHandler] Image exceeds pixel limit');
return;
}
// Use shared ImageStorage to add the image
// This handles cursor movement and integration with the terminal
if (w !== bitmap.width || h !== bitmap.height) {
// Resize if needed
const resized = await createImageBitmap(bitmap, { resizeWidth: w, resizeHeight: h });
this._storage.addImage(resized);
} else {
this._storage.addImage(bitmap);
}
}
/**
* Decode base64 image data into an ImageBitmap.
*/
private async _decodeImage(image: IKittyImageData): Promise<ImageBitmap> {
const format = image.format;
const base64Data = image.data;
// TODO: This atob + charCodeAt loop has bad runtime and creates memory pressure with large
// images. Consider using the wasm-based base64 decoder from xterm-wasm-parts.
const binaryString = atob(base64Data);
let bytes = new Uint8Array(binaryString.length);
for (let i = 0; i < binaryString.length; i++) {
bytes[i] = binaryString.charCodeAt(i);
}
if (image.compression === KittyCompression.ZLIB) {
bytes = await this._decompressZlib(bytes) as Uint8Array<ArrayBuffer>;
}
if (format === KittyFormat.PNG) {
const blob = new Blob([bytes], { type: 'image/png' });
// Safari fallback pattern (from IIPHandler)
if (!window.createImageBitmap) {
const url = URL.createObjectURL(blob);
const img = new Image();
return new Promise<ImageBitmap>((resolve, reject) => {
img.addEventListener('load', () => {
URL.revokeObjectURL(url);
const canvas = ImageRenderer.createCanvas(window.document, img.width, img.height);
canvas.getContext('2d')?.drawImage(img, 0, 0);
createImageBitmap(canvas).then(resolve).catch(reject);
});
img.addEventListener('error', reject);
img.src = url;
});
}
return createImageBitmap(blob);
}
// Raw pixel data (RGB or RGBA)
const width = image.width;
const height = image.height;
if (!width || !height) {
throw new Error('Width and height required for raw pixel data');
}
const bytesPerPixel = format === KittyFormat.RGBA ? BYTES_PER_PIXEL_RGBA : BYTES_PER_PIXEL_RGB;
const expectedBytes = width * height * bytesPerPixel;
if (bytes.length < expectedBytes) {
throw new Error(`Insufficient pixel data: got ${bytes.length}, expected ${expectedBytes}`);
}
// Convert to RGBA ImageData
// TODO: For RGBA, use bytes directly. For RGB, use Uint32Array bit manipulation
// for 5-6x speedup.
const pixelCount = width * height;
const data = new Uint8ClampedArray(pixelCount * BYTES_PER_PIXEL_RGBA);
const isRgba = format === KittyFormat.RGBA;
let srcOffset = 0;
let dstOffset = 0;
for (let i = 0; i < pixelCount; i++) {
data[dstOffset ] = bytes[srcOffset ]; // R
data[dstOffset + 1] = bytes[srcOffset + 1]; // G
data[dstOffset + 2] = bytes[srcOffset + 2]; // B
data[dstOffset + 3] = isRgba ? bytes[srcOffset + 3] : ALPHA_OPAQUE;
srcOffset += bytesPerPixel;
dstOffset += BYTES_PER_PIXEL_RGBA;
}
return createImageBitmap(new ImageData(data, width, height));
}
/**
* Decompress zlib/deflate compressed data using the browser's DecompressionStream API.
*/
private async _decompressZlib(compressed: Uint8Array): Promise<Uint8Array> {
try {
return await this._decompress(compressed, 'deflate');
} catch {
return await this._decompress(compressed, 'deflate-raw');
}
}
private async _decompress(compressed: Uint8Array, format: 'deflate' | 'deflate-raw'): Promise<Uint8Array> {
const ds = new DecompressionStream(format);
const writer = ds.writable.getWriter();
writer.write(new Uint8Array(compressed) as Uint8Array<ArrayBuffer>);
writer.close();
const chunks: Uint8Array[] = [];
const reader = ds.readable.getReader();
while (true) {
const { done, value } = await reader.read();
if (done) break;
chunks.push(value);
}
const totalLength = chunks.reduce((sum, chunk) => sum + chunk.length, 0);
const result = new Uint8Array(totalLength);
let offset = 0;
for (const chunk of chunks) {
result.set(chunk, offset);
offset += chunk.length;
}
return result;
}
/**
* Get stored images (for testing/debugging).
*/
public get images(): ReadonlyMap<number, IKittyImageData> {
return this._images;
}
}
@@ -0,0 +1,80 @@
/**
* Copyright (c) 2025 The xterm.js authors. All rights reserved.
* @license MIT
*/
import { assert } from 'chai';
import { parseKittyCommand, KittyAction, KittyFormat } from './KittyGraphicsTypes';
describe('KittyGraphicsTypes', () => {
describe('parseKittyCommand', () => {
it('should parse control data with action and format', () => {
const cmd = parseKittyCommand('a=T,f=100');
assert.strictEqual(cmd.action, 'T');
assert.strictEqual(cmd.format, 100);
});
it('should parse control data with all options', () => {
const cmd = parseKittyCommand('a=t,f=32,i=5,s=10,v=20,c=3,r=2,m=1,q=2');
assert.strictEqual(cmd.action, 't');
assert.strictEqual(cmd.format, 32);
assert.strictEqual(cmd.id, 5);
assert.strictEqual(cmd.width, 10);
assert.strictEqual(cmd.height, 20);
assert.strictEqual(cmd.columns, 3);
assert.strictEqual(cmd.rows, 2);
assert.strictEqual(cmd.more, 1);
assert.strictEqual(cmd.quiet, 2);
});
it('should handle empty control data', () => {
const cmd = parseKittyCommand('');
assert.strictEqual(cmd.action, undefined);
assert.strictEqual(cmd.format, undefined);
});
it('should parse transmit action', () => {
const cmd = parseKittyCommand('a=t,f=100');
assert.strictEqual(cmd.action, KittyAction.TRANSMIT);
assert.strictEqual(cmd.format, KittyFormat.PNG);
});
it('should parse delete action', () => {
const cmd = parseKittyCommand('a=d,i=5');
assert.strictEqual(cmd.action, KittyAction.DELETE);
assert.strictEqual(cmd.id, 5);
});
it('should parse empty action as empty string', () => {
const cmd = parseKittyCommand('a=,f=100');
assert.strictEqual(cmd.action, '');
assert.strictEqual(cmd.format, 100);
});
it('should leave action undefined when key is not present', () => {
const cmd = parseKittyCommand('f=100,i=5');
assert.strictEqual(cmd.action, undefined);
assert.strictEqual(cmd.format, 100);
assert.strictEqual(cmd.id, 5);
});
it('should parse compression key', () => {
const cmd = parseKittyCommand('a=t,f=32,o=z');
assert.strictEqual(cmd.action, 't');
assert.strictEqual(cmd.format, 32);
assert.strictEqual(cmd.compression, 'z');
});
it('should parse x and y offset', () => {
const cmd = parseKittyCommand('a=T,x=10,y=20');
assert.strictEqual(cmd.x, 10);
assert.strictEqual(cmd.y, 20);
});
it('should handle keys without values', () => {
const cmd = parseKittyCommand('a=t,f=,i=5');
assert.strictEqual(cmd.action, 't');
assert.ok(isNaN(cmd.format!));
assert.strictEqual(cmd.id, 5);
});
});
});
@@ -0,0 +1,156 @@
/**
* Copyright (c) 2025 The xterm.js authors. All rights reserved.
* @license MIT
*
* Kitty graphics protocol types, constants, and parsing utilities.
*/
/**
* Kitty graphics protocol action types.
* See: https://sw.kovidgoyal.net/kitty/graphics-protocol/#control-data-reference under key 'a'.
*/
export const enum KittyAction {
TRANSMIT = 't',
TRANSMIT_DISPLAY = 'T',
QUERY = 'q',
PLACEMENT = 'p',
DELETE = 'd'
}
/**
* Kitty graphics protocol format types.
* See: https://sw.kovidgoyal.net/kitty/graphics-protocol/#control-data-reference
*/
export const enum KittyFormat {
RGB = 24,
RGBA = 32,
PNG = 100
}
/**
* Kitty graphics protocol compression types.
* See: https://sw.kovidgoyal.net/kitty/graphics-protocol/#control-data-reference under key 'o'.
*/
export const enum KittyCompression {
NONE = '',
ZLIB = 'z'
}
/**
* Kitty graphics protocol control data keys.
* See: https://sw.kovidgoyal.net/kitty/graphics-protocol/#control-data-reference
*/
export const enum KittyKey {
// Action to perform (t=transmit, T=transmit+display, q=query, p=placement, d=delete)
ACTION = 'a',
// Image format (24=RGB, 32=RGBA, 100=PNG)
FORMAT = 'f',
// Image ID for referencing stored images
ID = 'i',
// Source image width in pixels
WIDTH = 's',
// Source image height in pixels
HEIGHT = 'v',
// The left edge (in pixels) of the image area to display
X_OFFSET = 'x',
// The top edge (in pixels) of the image area to display
Y_OFFSET = 'y',
// Number of terminal columns to display the image over
COLUMNS = 'c',
// Number of terminal rows to display the image over
ROWS = 'r',
// More data flag (1=more chunks coming, 0=final chunk)
MORE = 'm',
// Compression type (z=zlib). This is essential for chunking larger images.
COMPRESSION = 'o',
// Quiet mode (1=suppress OK responses, 2=suppress error responses)
QUIET = 'q'
}
// Pixel format constants
export const BYTES_PER_PIXEL_RGB = 3;
export const BYTES_PER_PIXEL_RGBA = 4;
export const ALPHA_OPAQUE = 255;
/**
* Parsed Kitty graphics command.
*/
export interface IKittyCommand {
action?: string;
format?: number;
id?: number;
width?: number;
height?: number;
x?: number;
y?: number;
columns?: number;
rows?: number;
more?: number;
quiet?: number;
compression?: string;
payload?: string;
}
/**
* Pending chunked transmission state.
* Stores metadata from the first chunk while accumulating payload data.
*/
export interface IPendingTransmission {
/** The parsed command from the first chunk (contains action, format, dimensions, etc.) */
cmd: IKittyCommand;
/** Accumulated base64 payload data */
data: string;
}
/**
* Stored Kitty image data.
*/
export interface IKittyImageData {
id: number;
data: string;
width: number;
height: number;
format: 24 | 32 | 100;
compression?: string;
}
/**
* Parses Kitty graphics control data into a command object.
*/
export function parseKittyCommand(data: string): IKittyCommand {
const cmd: IKittyCommand = {};
const parts = data.split(',');
for (const part of parts) {
const eqIdx = part.indexOf('=');
if (eqIdx === -1) continue;
const key = part.substring(0, eqIdx);
const value = part.substring(eqIdx + 1);
// Handle string keys first
if (key === KittyKey.ACTION) {
cmd.action = value;
continue;
}
if (key === KittyKey.COMPRESSION) {
cmd.compression = value;
continue;
}
const numValue = parseInt(value);
switch (key) {
case KittyKey.FORMAT: cmd.format = numValue; break;
case KittyKey.ID: cmd.id = numValue; break;
case KittyKey.WIDTH: cmd.width = numValue; break;
case KittyKey.HEIGHT: cmd.height = numValue; break;
case KittyKey.X_OFFSET: cmd.x = numValue; break;
case KittyKey.Y_OFFSET: cmd.y = numValue; break;
case KittyKey.COLUMNS: cmd.columns = numValue; break;
case KittyKey.ROWS: cmd.rows = numValue; break;
case KittyKey.MORE: cmd.more = numValue; break;
case KittyKey.QUIET: cmd.quiet = numValue; break;
}
}
return cmd;
}