Merge pull request #5619 from xtermjs/anthonykim1/scaffoldKittyAddon

Support Kitty graphics protocol mvp
This commit is contained in:
Daniel Imms
2026-02-16 10:53:49 -08:00
committed by GitHub
16 changed files with 3323 additions and 102 deletions
+3 -1
View File
@@ -30,7 +30,9 @@ const customSettings: IImageAddonOptions = {
storageLimit: 128, // FIFO storage limit in MB
showPlaceholder: true, // whether to show a placeholder for evicted images
iipSupport: true, // enable iTerm IIP support
iipSizeLimit: 20000000 // size limit of a single IIP sequence
iipSizeLimit: 20000000, // size limit of a single IIP sequence
kittySupport: true, // enable Kitty graphics support
kittySizeLimit: 20000000 // size limit of a single Kitty sequence
}
// initialization
Binary file not shown.

After

Width:  |  Height:  |  Size: 123 B

Binary file not shown.

After

Width:  |  Height:  |  Size: 449 B

Binary file not shown.

After

Width:  |  Height:  |  Size: 131 B

+17 -1
View File
@@ -8,6 +8,8 @@ import type { ImageAddon as IImageApi } from '@xterm/addon-image';
import { IIPHandler } from './IIPHandler';
import { ImageRenderer } from './ImageRenderer';
import { ImageStorage, CELL_SIZE_DEFAULT } from './ImageStorage';
import { KittyGraphicsHandler } from './kitty/KittyGraphicsHandler';
import { KittyImageStorage } from './kitty/KittyImageStorage';
import { SixelHandler } from './SixelHandler';
import { SixelImageStorage } from './SixelImageStorage';
import { IIPImageStorage } from './IIPImageStorage';
@@ -24,7 +26,9 @@ const DEFAULT_OPTIONS: IImageAddonOptions = {
storageLimit: 128,
showPlaceholder: true,
iipSupport: true,
iipSizeLimit: 20000000
iipSizeLimit: 20000000,
kittySupport: true,
kittySizeLimit: 20000000
};
// max palette size supported by the sixel lib (compile time setting)
@@ -148,6 +152,18 @@ export class ImageAddon implements ITerminalAddon, IImageApi {
terminal._core._inputHandler._parser.registerOscHandler(1337, iipHandler)
);
}
// Kitty graphics handler
if (this._opts.kittySupport) {
const kittyStorage = new KittyImageStorage(this._storage!);
const kittyHandler = new KittyGraphicsHandler(this._opts, this._renderer!, kittyStorage, terminal);
this._handlers.set('kitty', kittyHandler);
this._disposeLater(
kittyStorage,
kittyHandler,
terminal._core._inputHandler._parser.registerApcHandler(0x47, kittyHandler)
);
}
}
// Note: storageLimit is skipped here to not intoduce a surprising side effect.
+85 -44
View File
@@ -5,7 +5,7 @@
import { toRGBA8888 } from 'sixel/lib/Colors';
import { IDisposable } from '@xterm/xterm';
import { ICellSize, ITerminalExt, IImageSpec, IRenderDimensions, IRenderService } from './Types';
import { ICellSize, ImageLayer, ITerminalExt, IImageSpec, IRenderDimensions, IRenderService } from './Types';
import { Disposable, MutableDisposable, toDisposable } from 'common/Lifecycle';
const PLACEHOLDER_LENGTH = 4096;
@@ -18,8 +18,9 @@ const PLACEHOLDER_HEIGHT = 24;
* - draw image tiles onRender
*/
export class ImageRenderer extends Disposable implements IDisposable {
public canvas: HTMLCanvasElement | undefined;
private _ctx: CanvasRenderingContext2D | null | undefined;
/** @deprecated Kept for backward compat — points to top layer canvas. */
public get canvas(): HTMLCanvasElement | undefined { return this._layers.get('top')?.canvas; }
private _layers = new Map<ImageLayer, CanvasRenderingContext2D>();
private _placeholder: HTMLCanvasElement | undefined;
private _placeholderBitmap: ImageBitmap | undefined;
private _optionsRefresh = this._register(new MutableDisposable());
@@ -86,6 +87,7 @@ export class ImageRenderer extends Disposable implements IDisposable {
});
this._register(toDisposable(() => {
this.removeLayerFromDom();
this.removeLayerFromDom('bottom');
if (this._terminal._core && this._oldOpen) {
this._terminal._core.open = this._oldOpen;
this._oldOpen = undefined;
@@ -95,8 +97,7 @@ export class ImageRenderer extends Disposable implements IDisposable {
this._oldSetRenderer = undefined;
}
this._renderService = undefined;
this.canvas = undefined;
this._ctx = undefined;
this._layers.clear();
this._placeholderBitmap?.close();
this._placeholderBitmap = undefined;
this._placeholder = undefined;
@@ -140,27 +141,38 @@ export class ImageRenderer extends Disposable implements IDisposable {
/**
* Clear a region of the image layer canvas.
*/
public clearLines(start: number, end: number): void {
this._ctx?.clearRect(
0,
start * (this.dimensions?.css.cell.height || 0),
this.dimensions?.css.canvas.width || 0,
(++end - start) * (this.dimensions?.css.cell.height || 0)
);
public clearLines(start: number, end: number, layer?: ImageLayer): void {
const y = start * (this.dimensions?.css.cell.height || 0);
const w = this.dimensions?.css.canvas.width || 0;
const h = (++end - start) * (this.dimensions?.css.cell.height || 0);
if (!layer || layer === 'top') {
this._layers.get('top')?.clearRect(0, y, w, h);
}
if (!layer || layer === 'bottom') {
this._layers.get('bottom')?.clearRect(0, y, w, h);
}
}
/**
* Clear whole image canvas.
*/
public clearAll(): void {
this._ctx?.clearRect(0, 0, this.canvas?.width || 0, this.canvas?.height || 0);
public clearAll(layer?: ImageLayer): void {
if (!layer || layer === 'top') {
const ctx = this._layers.get('top');
ctx?.clearRect(0, 0, ctx.canvas.width, ctx.canvas.height);
}
if (!layer || layer === 'bottom') {
const ctx = this._layers.get('bottom');
ctx?.clearRect(0, 0, ctx.canvas.width, ctx.canvas.height);
}
}
/**
* Draw neighboring tiles on the image layer canvas.
*/
public draw(imgSpec: IImageSpec, tileId: number, col: number, row: number, count: number = 1): void {
if (!this._ctx) {
const ctx = this._layers.get(imgSpec.layer);
if (!ctx) {
return;
}
const { width, height } = this.cellSize;
@@ -187,7 +199,7 @@ export class ImageRenderer extends Disposable implements IDisposable {
// Note: For not pixel perfect aligned cells like in the DOM renderer
// this will move a tile slightly to the top/left (subpixel range, thus ignore it).
// FIX #34: avoid striping on displays with pixelDeviceRatio != 1 by ceiling height and width
this._ctx.drawImage(
ctx.drawImage(
img,
Math.floor(sx), Math.floor(sy), Math.ceil(finalWidth), Math.ceil(finalHeight),
Math.floor(dx), Math.floor(dy), Math.ceil(finalWidth), Math.ceil(finalHeight)
@@ -227,7 +239,8 @@ export class ImageRenderer extends Disposable implements IDisposable {
* Draw a line with placeholder on the image layer canvas.
*/
public drawPlaceholder(col: number, row: number, count: number = 1): void {
if (this._ctx) {
const ctx = this._layers.get('top');
if (ctx) {
const { width, height } = this.cellSize;
// Don't try to draw anything, if we cannot get valid renderer metrics.
@@ -241,7 +254,7 @@ export class ImageRenderer extends Disposable implements IDisposable {
this._createPlaceHolder(height + 1);
}
if (!this._placeholder) return;
this._ctx.drawImage(
ctx.drawImage(
this._placeholderBitmap ?? this._placeholder!,
col * width,
(row * height) % 2 ? 0 : 1, // needs %2 offset correction
@@ -260,12 +273,13 @@ export class ImageRenderer extends Disposable implements IDisposable {
* Checked once from `ImageStorage.render`.
*/
public rescaleCanvas(): void {
if (!this.canvas) {
return;
}
if (this.canvas.width !== this.dimensions!.css.canvas.width || this.canvas.height !== this.dimensions!.css.canvas.height) {
this.canvas.width = this.dimensions!.css.canvas.width || 0;
this.canvas.height = this.dimensions!.css.canvas.height || 0;
const w = this.dimensions?.css.canvas.width || 0;
const h = this.dimensions?.css.canvas.height || 0;
for (const ctx of this._layers.values()) {
if (ctx.canvas.width !== w || ctx.canvas.height !== h) {
ctx.canvas.width = w;
ctx.canvas.height = h;
}
}
}
@@ -304,35 +318,62 @@ export class ImageRenderer extends Disposable implements IDisposable {
this._renderService = this._terminal._core._renderService;
this._oldSetRenderer = this._renderService.setRenderer.bind(this._renderService);
this._renderService.setRenderer = (renderer: any) => {
this.removeLayerFromDom();
for (const key of [...this._layers.keys()]) {
this.removeLayerFromDom(key);
}
this._oldSetRenderer?.call(this._renderService, renderer);
};
}
public insertLayerToDom(): void {
public insertLayerToDom(layer: ImageLayer = 'top'): void {
// make sure that the terminal is attached to a document and to DOM
if (this.document && this._terminal._core.screenElement) {
if (!this.canvas) {
this.canvas = ImageRenderer.createCanvas(
this.document, this.dimensions?.css.canvas.width || 0,
this.dimensions?.css.canvas.height || 0
);
this.canvas.classList.add('xterm-image-layer');
this._terminal._core.screenElement.appendChild(this.canvas);
this._ctx = this.canvas.getContext('2d', { alpha: true, desynchronized: true });
this.clearAll();
}
} else {
if (!this.document || !this._terminal._core.screenElement) {
console.warn('image addon: cannot insert output canvas to DOM, missing document or screenElement');
return;
}
if (this._layers.has(layer)) {
return;
}
const canvas = ImageRenderer.createCanvas(
this.document, this.dimensions?.css.canvas.width || 0,
this.dimensions?.css.canvas.height || 0
);
canvas.classList.add(`xterm-image-layer-${layer}`);
const screenElement = this._terminal._core.screenElement;
if (layer === 'bottom') {
// Use z-index:-1 so it paints behind non-positioned text elements.
// The screen element needs to be a stacking context to contain the
// negative z-index, otherwise it would go behind the entire terminal.
canvas.style.zIndex = '-1';
screenElement.style.zIndex = '0';
screenElement.insertBefore(canvas, screenElement.firstChild);
} else {
// Explicit z-index ensures the image canvas reliably stacks above
// the text layer (DOM renderer rows). z-index: 0 is below the
// selection overlay (z-index: 1).
canvas.style.zIndex = '0';
screenElement.style.zIndex = '0';
screenElement.appendChild(canvas);
}
const ctx = canvas.getContext('2d', { alpha: true, desynchronized: true });
if (!ctx) {
canvas.remove();
return;
}
this._layers.set(layer, ctx);
this.clearAll(layer);
}
public removeLayerFromDom(layer: ImageLayer = 'top'): void {
const ctx = this._layers.get(layer);
if (ctx) {
ctx.canvas.remove();
this._layers.delete(layer);
}
}
public removeLayerFromDom(): void {
if (this.canvas) {
this._ctx = undefined;
this.canvas.remove();
this.canvas = undefined;
}
public hasLayer(layer: ImageLayer): boolean {
return this._layers.has(layer);
}
private _createPlaceHolder(height: number = PLACEHOLDER_HEIGHT): void {
+117 -44
View File
@@ -5,7 +5,7 @@
import { IDisposable } from '@xterm/xterm';
import { ImageRenderer } from './ImageRenderer';
import { ITerminalExt, IExtendedAttrsImage, IImageAddonOptions, IImageSpec, IBufferLineExt, BgFlags, Cell, Content, ICellSize, ExtFlags, Attributes, UnderlineStyle } from './Types';
import { ITerminalExt, IExtendedAttrsImage, IImageAddonOptions, IImageSpec, IBufferLineExt, BgFlags, Cell, Content, ICellSize, ExtFlags, Attributes, UnderlineStyle, ImageLayer } from './Types';
// fallback default cell size
@@ -124,6 +124,7 @@ export class ImageStorage implements IDisposable {
private _pixelLimit: number = 2500000;
private _viewportMetrics: { cols: number, rows: number };
public onImageDeleted: ((storageId: number) => void) | undefined;
constructor(
private _terminal: ITerminalExt,
@@ -189,11 +190,13 @@ export class ImageStorage implements IDisposable {
private _delImg(id: number): void {
const spec = this._images.get(id);
if (!spec) return;
this._images.delete(id);
// FIXME: really ugly workaround to get bitmaps deallocated :(
if (spec && window.ImageBitmap && spec.orig instanceof ImageBitmap) {
if (window.ImageBitmap && spec.orig instanceof ImageBitmap) {
spec.orig.close();
}
this.onImageDeleted?.(id);
}
/**
@@ -216,14 +219,28 @@ export class ImageStorage implements IDisposable {
this._fullyCleared = false;
}
/**
* Delete an image by its internal storage ID.
* Used by protocols that support explicit deletion (e.g. Kitty a=d).
*/
public deleteImage(id: number): void {
const spec = this._images.get(id);
if (spec) {
spec.marker?.dispose();
this._delImg(id);
}
}
/**
* Method to add an image to the storage.
* @param img - The image to add (canvas or bitmap).
* @param scrolling - When true, cursor advances with the image (lineFeed per row).
* When false, image is placed at (0,0) and cursor is restored (DECSET 80 / sixel origin mode).
* @param layer - Which canvas layer to render on ('top' or 'bottom').
* @param zIndex - Z-index for image layering within the same layer.
* @returns The internal image ID assigned to the stored image.
*/
public addImage(img: HTMLCanvasElement | ImageBitmap, scrolling: boolean): number {
public addImage(img: HTMLCanvasElement | ImageBitmap, scrolling: boolean, layer: ImageLayer = 'top', zIndex: number = 0): number {
// never allow storage to exceed memory limit
this._evictOldest(img.width * img.height);
@@ -312,7 +329,9 @@ export class ImageStorage implements IDisposable {
actualCellSize: { ...cellSize }, // clone needed, since later modified
marker: endMarker || undefined,
tileCount,
bufferType: this._terminal.buffer.active.type
bufferType: this._terminal.buffer.active.type,
layer,
zIndex
};
// finally add the image
@@ -327,16 +346,30 @@ export class ImageStorage implements IDisposable {
*/
// TODO: Should we move this to the ImageRenderer?
public render(range: { start: number, end: number }): void {
// setup image canvas in case we have none yet, but have images in store
if (!this._renderer.canvas && this._images.size) {
this._renderer.insertLayerToDom();
// safety measure - in case we cannot spawn a canvas at all, just exit
if (!this._renderer.canvas) {
return;
// Determine which layers have images
let hasTopImages = false;
let hasBottomImages = false;
for (const spec of this._images.values()) {
if (spec.layer === 'bottom') {
hasBottomImages = true;
} else {
hasTopImages = true;
}
if (hasTopImages && hasBottomImages) break;
}
// Lazily insert layers that are needed
if (hasTopImages && !this._renderer.hasLayer('top')) {
this._renderer.insertLayerToDom('top');
if (!this._renderer.hasLayer('top')) return;
}
if (hasBottomImages && !this._renderer.hasLayer('bottom')) {
this._renderer.insertLayerToDom('bottom');
}
// rescale if needed
this._renderer.rescaleCanvas();
// exit early if we dont have any images to test for
if (!this._images.size) {
if (!this._fullyCleared) {
@@ -344,12 +377,25 @@ export class ImageStorage implements IDisposable {
this._fullyCleared = true;
this._needsFullClear = false;
}
if (this._renderer.canvas) {
this._renderer.removeLayerFromDom();
if (this._renderer.hasLayer('top')) {
this._renderer.removeLayerFromDom('top');
}
if (this._renderer.hasLayer('bottom')) {
this._renderer.removeLayerFromDom('bottom');
}
return;
}
// Remove layers no longer needed
if (!hasTopImages && this._renderer.hasLayer('top')) {
this._renderer.clearAll('top');
this._renderer.removeLayerFromDom('top');
}
if (!hasBottomImages && this._renderer.hasLayer('bottom')) {
this._renderer.clearAll('bottom');
this._renderer.removeLayerFromDom('bottom');
}
// buffer switches force a full clear
if (this._needsFullClear) {
this._renderer.clearAll();
@@ -364,50 +410,77 @@ export class ImageStorage implements IDisposable {
// clear drawing area
this._renderer.clearLines(start, end);
// walk all cells in viewport and draw tiles found
// Collect draw calls so we can sort by z-index (lower z drawn first).
const drawCalls: { imgSpec: IImageSpec, tileId: number, col: number, row: number, count: number }[] = [];
const placeholderCalls: { col: number, row: number, count: number }[] = [];
// walk all cells in viewport and collect tiles found
// Note: We check _extendedAttrs directly (not just HAS_EXTENDED flag)
// because text writes clear the BG flag but leave image tile data intact.
// This lets top-layer images survive text overwrites (kitty C=1 behavior).
for (let row = start; row <= end; ++row) {
const line = buffer.lines.get(row + buffer.ydisp) as IBufferLineExt;
if (!line) return;
for (let col = 0; col < cols; ++col) {
let e: IExtendedAttrsImage;
if (line.getBg(col) & BgFlags.HAS_EXTENDED) {
let e: IExtendedAttrsImage = line._extendedAttrs[col] ?? EMPTY_ATTRS;
const imageId = e.imageId;
if (imageId === undefined || imageId === -1) {
e = line._extendedAttrs[col] ?? EMPTY_ATTRS;
} else {
const maybeImg = line._extendedAttrs[col] as IExtendedAttrsImage | undefined;
if (!maybeImg || maybeImg.imageId === undefined || maybeImg.imageId === -1) {
continue;
}
const imgSpec = this._images.get(imageId);
if (e.tileId !== -1) {
const startTile = e.tileId;
const startCol = col;
let count = 1;
/**
* merge tiles to the right into a single draw call, if:
* - not at end of line
* - cell has same image id
* - cell has consecutive tile id
*/
while (
++col < cols
&& (line.getBg(col) & BgFlags.HAS_EXTENDED)
&& (e = line._extendedAttrs[col] ?? EMPTY_ATTRS)
&& (e.imageId === imageId)
&& (e.tileId === startTile + count)
) {
count++;
e = maybeImg;
}
const imageId = e.imageId;
if (imageId === undefined || imageId === -1) {
continue;
}
const imgSpec = this._images.get(imageId);
if (e.tileId !== -1) {
const startTile = e.tileId;
const startCol = col;
let count = 1;
/**
* merge tiles to the right into a single draw call, if:
* - not at end of line
* - cell has same image id
* - cell has consecutive tile id
* Also check _extendedAttrs directly for cells where text cleared HAS_EXTENDED.
*/
while (++col < cols) {
const nextE = line._extendedAttrs[col] as IExtendedAttrsImage | undefined;
if (!nextE || nextE.imageId !== imageId || nextE.tileId !== startTile + count) {
break;
}
col--;
if (imgSpec) {
if (imgSpec.actual) {
this._renderer.draw(imgSpec, startTile, startCol, row, count);
}
} else if (this._opts.showPlaceholder) {
this._renderer.drawPlaceholder(startCol, row, count);
}
this._fullyCleared = false;
e = nextE;
count++;
}
col--;
if (imgSpec) {
if (imgSpec.actual) {
drawCalls.push({ imgSpec, tileId: startTile, col: startCol, row, count });
}
} else if (this._opts.showPlaceholder) {
placeholderCalls.push({ col: startCol, row, count });
}
this._fullyCleared = false;
}
}
}
// Sort by z-index so lower z draws first (higher z renders on top)
drawCalls.sort((a, b) => a.imgSpec.zIndex - b.imgSpec.zIndex);
// Draw placeholders first (lowest priority)
for (const call of placeholderCalls) {
this._renderer.drawPlaceholder(call.col, call.row, call.count);
}
// Draw images in z-index order
for (const call of drawCalls) {
this._renderer.draw(call.imgSpec, call.tileId, call.col, call.row, call.count);
}
}
public viewportResize(metrics: { cols: number, rows: number }): void {
+8 -2
View File
@@ -8,7 +8,7 @@ import { IDisposable, IMarker, Terminal } from '@xterm/xterm';
// private imports from base repo we build against
import { Attributes, BgFlags, Content, ExtFlags, UnderlineStyle } from 'common/buffer/Constants';
import type { AttributeData } from 'common/buffer/AttributeData';
import type { IParams, IDcsHandler, IOscHandler, IEscapeSequenceParser } from 'common/parser/Types';
import type { IParams, IDcsHandler, IOscHandler, IApcHandler, IEscapeSequenceParser } from 'common/parser/Types';
import type { IBufferLine, IExtendedAttrs, IInputHandler } from 'common/Types';
import type { ITerminal, ReadonlyColorSet } from 'browser/Types';
import type { IRenderDimensions } from 'browser/renderer/shared/Types';
@@ -22,7 +22,7 @@ export const enum Cell {
}
// export some privates for local usage
export { AttributeData, IParams, IDcsHandler, IOscHandler, BgFlags, IRenderDimensions, IRenderService, Content, ExtFlags, Attributes, UnderlineStyle, ReadonlyColorSet };
export { AttributeData, IParams, IDcsHandler, IOscHandler, IApcHandler, BgFlags, IRenderDimensions, IRenderService, Content, ExtFlags, Attributes, UnderlineStyle, ReadonlyColorSet };
/**
* Plugin ctor options.
@@ -38,6 +38,8 @@ export interface IImageAddonOptions {
sixelSizeLimit: number;
iipSupport: boolean;
iipSizeLimit: number;
kittySupport: boolean;
kittySizeLimit: number;
}
export interface IResetHandler {
@@ -97,6 +99,8 @@ export interface ICellSize {
height: number;
}
export type ImageLayer = 'top' | 'bottom';
export interface IImageSpec {
orig: HTMLCanvasElement | ImageBitmap | undefined;
origCellSize: ICellSize;
@@ -105,4 +109,6 @@ export interface IImageSpec {
marker: IMarker | undefined;
tileCount: number;
bufferType: 'alternate' | 'normal';
layer: ImageLayer;
zIndex: number;
}
@@ -0,0 +1,721 @@
/**
* Copyright (c) 2026 The xterm.js authors. All rights reserved.
* @license MIT
*/
import { IDisposable } from '@xterm/xterm';
import { IApcHandler, IImageAddonOptions, IResetHandler, ITerminalExt, ImageLayer } from '../Types';
import { ImageRenderer } from '../ImageRenderer';
import { CELL_SIZE_DEFAULT } from '../ImageStorage';
import { KittyImageStorage } from './KittyImageStorage';
import Base64Decoder, { type DecodeStatus } from 'xterm-wasm-parts/lib/base64/Base64Decoder.wasm';
import {
KittyAction,
KittyFormat,
KittyCompression,
IKittyCommand,
IPendingTransmission,
IKittyImageData,
BYTES_PER_PIXEL_RGB,
BYTES_PER_PIXEL_RGBA,
ALPHA_OPAQUE,
parseKittyCommand
} from './KittyGraphicsTypes';
// Memory limit for base64 decoder (4MB, same as IIPHandler)
const DECODER_KEEP_DATA = 4194304;
const DECODER_INITIAL_DATA = 4194304; // 4MB
// Local mirror of const enum (esbuild can't inline const enums from external packages)
const DECODER_OK: DecodeStatus.OK = 0;
// Maximum control data size
const MAX_CONTROL_DATA_SIZE = 512;
// Semicolon codepoint
const SEMICOLON = 0x3B;
// Kitty graphics protocol handler with streaming base64 decoding.
export class KittyGraphicsHandler implements IApcHandler, IResetHandler, IDisposable {
private _aborted = false;
private _decodeError = false;
private _activeDecoder: Base64Decoder | null = null;
private readonly _maxEncodedBytes: number;
private readonly _initialEncodedBytes: number;
// Streaming related states
// True while receiving control data (before semicolon).
private _inControlData = true;
// Buffer for control data.
private _controlData = new Uint32Array(MAX_CONTROL_DATA_SIZE);
private _controlLength = 0;
// Pre-calculated encoded size limit
private _encodedSizeLimit = 0;
private _totalEncodedSize = 0;
// Parsed command. These are the control data before semicolon.
private _parsedCommand: IKittyCommand | null = null;
// Storage related states
private _pendingTransmissions: Map<number, IPendingTransmission> = new Map();
// Tracks the pending key of the most recently started chunked upload.
// Per spec, subsequent chunks only need m= (and optionally q=), without i=.
// When a chunk arrives with no i=, this key is used to find the pending upload.
private _lastPendingKey: number | undefined;
constructor(
private readonly _opts: IImageAddonOptions,
private readonly _renderer: ImageRenderer,
private readonly _kittyStorage: KittyImageStorage,
private readonly _coreTerminal: ITerminalExt
) {
// Convert decoded size limit -> max encoded bytes.
this._maxEncodedBytes = Math.ceil(this._opts.kittySizeLimit * 4 / 3);
// ensure we preallocate more than configured limit while using 4mb initial size.
this._initialEncodedBytes = Math.min(DECODER_INITIAL_DATA, this._maxEncodedBytes);
}
public reset(): void {
this._cleanupAllPending();
if (this._activeDecoder) {
this._activeDecoder.release();
this._activeDecoder = null;
}
this._kittyStorage.reset();
}
public dispose(): void {
this.reset();
}
private _removePendingEntry(key: number): void {
this._pendingTransmissions.delete(key);
if (this._lastPendingKey === key) {
this._lastPendingKey = undefined;
}
}
private _cleanupAllPending(): void {
for (const pending of this._pendingTransmissions.values()) {
pending.decoder.release();
}
this._pendingTransmissions.clear();
this._lastPendingKey = undefined;
}
public start(): void {
this._aborted = false;
this._decodeError = false;
this._inControlData = true;
this._controlLength = 0;
this._parsedCommand = null;
// Pre-calculate encoded limit once: base64 is 4 bytes encoded → 3 bytes decoded
this._encodedSizeLimit = this._maxEncodedBytes;
this._totalEncodedSize = 0;
this._activeDecoder = null;
}
public put(data: Uint32Array, start: number, end: number): void {
if (this._aborted) return;
if (!this._inControlData) {
this._streamPayload(data, start, end);
} else {
// Scan for semicolon
let controlEnd = end;
for (let i = start; i < end; i++) {
if (data[i] === SEMICOLON) {
this._inControlData = false;
controlEnd = i;
break;
}
}
// Copy control data
const copyLength = controlEnd - start;
if (this._controlLength + copyLength > MAX_CONTROL_DATA_SIZE) {
this._aborted = true;
return;
}
this._controlData.set(data.subarray(start, controlEnd), this._controlLength);
this._controlLength += copyLength;
if (!this._inControlData) {
// Found semicolon - parse control data early for validation
this._parsedCommand = parseKittyCommand(this._parseControlDataString());
// Early validation: i+I conflict
if (this._parsedCommand.id !== undefined && this._parsedCommand.imageNumber !== undefined) {
this._sendResponse(this._parsedCommand.id, 'EINVAL:cannot specify both i and I keys', this._parsedCommand.quiet ?? 0);
this._aborted = true;
return;
}
// Delete action doesn't need payload - skip streaming
if (this._parsedCommand.action === KittyAction.DELETE) {
return;
}
// Stream remaining as payload
const payloadStart = controlEnd + 1;
if (payloadStart < end) {
this._streamPayload(data, payloadStart, end);
}
}
}
}
// Stream payload bytes into the base64 decoder.
private _streamPayload(data: Uint32Array, start: number, end: number): void {
if (this._aborted) return;
// Check size limit (compare encoded bytes against pre-calculated limit)
// Include cumulative size from pending transmission for multi-chunk images.
// Per spec, subsequent chunks may omit i=, so fall back to _lastPendingKey.
const pendingKey = this._parsedCommand?.id ?? this._lastPendingKey ?? 0;
const pending = this._pendingTransmissions.get(pendingKey);
const previousEncodedSize = pending?.totalEncodedSize ?? 0;
this._totalEncodedSize += end - start;
const cumulativeEncodedSize = previousEncodedSize + this._totalEncodedSize;
if (cumulativeEncodedSize > this._encodedSizeLimit) {
const decoderToRelease = this._activeDecoder ?? pending?.decoder;
if (decoderToRelease) {
decoderToRelease.release();
}
this._activeDecoder = null;
if (pending) {
this._removePendingEntry(pendingKey);
}
this._aborted = true;
return;
}
if (this._decodeError) return;
if (pending?.decoder && !this._activeDecoder) {
this._activeDecoder = pending.decoder;
}
if (!this._activeDecoder) {
this._activeDecoder = new Base64Decoder(DECODER_KEEP_DATA, this._maxEncodedBytes, this._initialEncodedBytes);
this._activeDecoder.init();
}
if (this._activeDecoder.put(data.subarray(start, end)) !== DECODER_OK) {
this._activeDecoder.release();
this._activeDecoder = null;
this._decodeError = true;
if (pending) {
this._removePendingEntry(pendingKey);
}
}
}
public end(success: boolean): boolean | Promise<boolean> {
if (this._aborted || !success) {
if (this._activeDecoder) {
this._activeDecoder.release();
this._activeDecoder = null;
}
return true;
}
// No semicolon = no payload (delete, capability query)
if (this._inControlData) {
return this._handleNoPayloadCommand();
}
// Use command parsed early in put() - i+I already validated there
const cmd = this._parsedCommand!;
// Delete action was handled by skipping payload - just execute
if (cmd.action === KittyAction.DELETE) {
return this._handleDelete(cmd);
}
// Per spec, subsequent chunks may omit i=, so fall back to _lastPendingKey.
const pendingKey = cmd.id ?? this._lastPendingKey ?? 0;
const isMoreComing = cmd.more === 1;
const pending = this._pendingTransmissions.get(pendingKey);
if (isMoreComing) {
if (this._activeDecoder) {
if (pending) {
pending.totalEncodedSize += this._totalEncodedSize;
pending.decodeError = pending.decodeError || this._decodeError;
} else {
this._pendingTransmissions.set(pendingKey, {
cmd: { ...cmd },
decoder: this._activeDecoder,
totalEncodedSize: this._totalEncodedSize,
decodeError: this._decodeError
});
}
this._lastPendingKey = pendingKey;
this._activeDecoder = null;
}
return true;
}
// Final chunk received — clear the last pending key
if (pending) {
this._lastPendingKey = undefined;
}
let decodeError = this._decodeError;
let finalCmd = cmd;
let decoder = this._activeDecoder;
if (pending) {
finalCmd = pending.cmd;
decoder = pending.decoder;
decodeError = decodeError || pending.decodeError;
this._pendingTransmissions.delete(pendingKey);
}
let imageBytes = new Uint8Array(0);
if (decoder) {
if (decoder.end() !== DECODER_OK) {
decodeError = true;
}
imageBytes = decoder.data8;
}
this._activeDecoder = null;
// Handle command first — handlers create Blob/ImageData from imageBytes,
// which copies the data. Only then is it safe to release the decoder's
// wasm memory that imageBytes points into.
const result = this._handleCommandWithBytesAndCmd(finalCmd, imageBytes, decodeError);
if (decoder) {
decoder.release();
}
return result;
}
// Command handling
private _parseControlDataString(): string {
let str = '';
for (let i = 0; i < this._controlLength; i++) {
str += String.fromCodePoint(this._controlData[i]);
}
return str;
}
private _handleNoPayloadCommand(): boolean | Promise<boolean> {
const cmd = parseKittyCommand(this._parseControlDataString());
// Per spec: specifying both i and I is an error
if (cmd.id !== undefined && cmd.imageNumber !== undefined) {
this._sendResponse(cmd.id, 'EINVAL:cannot specify both i and I keys', cmd.quiet ?? 0);
return true;
}
const action = cmd.action ?? 't';
switch (action) {
case KittyAction.DELETE:
return this._handleDelete(cmd);
case KittyAction.QUERY:
this._sendResponse(cmd.id ?? 0, 'OK', cmd.quiet ?? 0);
return true;
default:
// TODO: Implement remaining actions when needed:
// - a=p (placement): place a previously transmitted image
// - a=f (frame): animation frame operations
// - a=a (animation): animation control
// - a=c (compose): compose images
if (cmd.id !== undefined) {
this._sendResponse(cmd.id, 'EINVAL:unsupported action', cmd.quiet ?? 0);
}
return true;
}
}
private _handleCommandWithBytesAndCmd(cmd: IKittyCommand, bytes: Uint8Array, decodeError: boolean): boolean | Promise<boolean> {
const action = cmd.action ?? 't';
switch (action) {
case KittyAction.TRANSMIT: {
const result = this._handleTransmit(cmd, bytes, decodeError);
// Only send response when _handleTransmit didn't already respond
// (it handles unsupported transmission medium responses internally)
if ((cmd.transmission ?? 'd') === 'd' && cmd.id !== undefined) {
if (decodeError) {
this._sendResponse(cmd.id, 'EINVAL:invalid base64 data', cmd.quiet ?? 0);
} else if (bytes.length > 0) {
this._sendResponse(cmd.id, 'OK', cmd.quiet ?? 0);
}
}
return result;
}
case KittyAction.TRANSMIT_DISPLAY:
return this._handleTransmitDisplay(cmd, bytes, decodeError);
case KittyAction.QUERY:
return this._handleQuery(cmd, bytes, decodeError);
default:
// TODO: Implement remaining actions when needed:
// - a=p (placement): place a previously transmitted image
// - a=f (frame): animation frame operations
// - a=a (animation): animation control
// - a=c (compose): compose images
if (cmd.id !== undefined) {
this._sendResponse(cmd.id, 'EINVAL:unsupported action', cmd.quiet ?? 0);
}
return true;
}
}
private _handleTransmit(cmd: IKittyCommand, bytes: Uint8Array, decodeError: boolean): boolean {
// TODO: Support file-based transmission modes (t=f, t=t, t=s)
// Currently only supports direct transmission (t=d, the default).
// - t=f (file): Payload is base64-encoded file path. Terminal reads image from that path.
// - t=t (temp file): Payload is base64-encoded path in temp directory. Terminal reads, deletes.
// - t=s: Payload is base64-encoded POSIX shm name. Terminal reads from shared memory.
// These modes require filesystem/IPC access not available in browsers. For Node.js/Electron:
// 1. Check cmd.transmission (t key) before treating bytes as image data
// 2. For t=f/t/s: decode bytes as UTF-8 string (the path/name), then read file contents
// 3. For t=d: treat bytes as image data (current behavior)
// When implementing, also update _handleQuery to accept these transmission mediums.
const transmission = cmd.transmission ?? 'd';
if (transmission !== 'd') {
if (cmd.id !== undefined) {
this._sendResponse(cmd.id, 'EINVAL:unsupported transmission medium', cmd.quiet ?? 0);
}
return true;
}
if (decodeError || bytes.length === 0) return true;
this._kittyStorage.storeImage(cmd.id, {
data: new Blob([bytes as BlobPart]),
width: cmd.width ?? 0,
height: cmd.height ?? 0,
format: (cmd.format ?? KittyFormat.RGBA) as 24 | 32 | 100,
compression: cmd.compression ?? ''
});
return true;
}
private _handleTransmitDisplay(cmd: IKittyCommand, bytes: Uint8Array, decodeError: boolean): boolean | Promise<boolean> {
if (decodeError) {
if (cmd.id !== undefined) {
this._sendResponse(cmd.id, 'EINVAL:invalid base64 data', cmd.quiet ?? 0);
}
return true;
}
this._handleTransmit(cmd, bytes, decodeError);
const id = cmd.id ?? this._kittyStorage.lastImageId;
const image = this._kittyStorage.getImage(id);
if (image) {
const result = this._displayImage(image, cmd);
if (cmd.id !== undefined) {
return result.then(success => {
this._sendResponse(id, success ? 'OK' : 'EINVAL:image rendering failed', cmd.quiet ?? 0);
return true;
});
}
return result.then(() => true);
}
return true;
}
private _handleQuery(cmd: IKittyCommand, bytes: Uint8Array, decodeError: boolean): boolean {
const id = cmd.id ?? 0;
const quiet = cmd.quiet ?? 0;
// Per spec: reject unsupported transmission mediums (only t=d is supported atm)
// TODO: When filesystem support is added (Node.js/Electron), update this to accept
// t=f (file), t=t (temp file), and t=s (shared memory) and respond OK for queries.
const transmission = cmd.transmission ?? 'd';
if (transmission !== 'd') {
this._sendResponse(id, 'EINVAL:unsupported transmission medium', quiet);
return true;
}
// Check decode error first (invalid base64)
if (decodeError) {
this._sendResponse(id, 'EINVAL:invalid base64 data', quiet);
return true;
}
// Capability query (no payload) - just respond OK
if (bytes.length === 0) {
this._sendResponse(id, 'OK', quiet);
return true;
}
const format = cmd.format ?? KittyFormat.RGBA;
if (format === KittyFormat.PNG) {
this._sendResponse(id, 'OK', quiet);
} else {
const width = cmd.width ?? 0;
const height = cmd.height ?? 0;
if (!width || !height) {
this._sendResponse(id, 'EINVAL:width and height required for raw pixel data', quiet);
return true;
}
const bytesPerPixel = format === KittyFormat.RGBA ? BYTES_PER_PIXEL_RGBA : BYTES_PER_PIXEL_RGB;
const expectedBytes = width * height * bytesPerPixel;
if (bytes.length < expectedBytes) {
this._sendResponse(id, `EINVAL:insufficient pixel data`, quiet);
return true;
}
this._sendResponse(id, 'OK', quiet);
}
return true;
}
private _handleDelete(cmd: IKittyCommand): boolean {
// Per spec: default delete selector is 'a' (delete all visible placements)
const selector = cmd.deleteSelector ?? 'a';
// TODO: Distinguish lowercase (delete placements only) from uppercase
// (delete placements + free stored image data). Currently both variants
// free everything since we don't separate stored data from placements.
switch (selector) {
case 'a':
case 'A':
this._cleanupAllPending();
this._kittyStorage.deleteAll();
break;
case 'i':
case 'I':
if (cmd.id !== undefined) {
const pending = this._pendingTransmissions.get(cmd.id);
if (pending) {
pending.decoder.release();
}
this._removePendingEntry(cmd.id);
this._kittyStorage.deleteById(cmd.id);
}
break;
default:
// Unsupported selectors (c, n, p, q, r, x, y, z, f) — ignore for now
break;
}
return true;
}
private _sendResponse(id: number, message: string, quiet: number): void {
const isOk = message === 'OK';
if (isOk && quiet === 1) return;
if (!isOk && quiet === 2) return;
const response = `\x1b_Gi=${id};${message}\x1b\\`;
this._coreTerminal._core.coreService.triggerDataEvent(response);
}
// Image display
private _displayImage(image: IKittyImageData, cmd: IKittyCommand): Promise<boolean> {
return this._decodeAndDisplay(image, cmd)
.then(() => true)
.catch(() => false);
}
private async _decodeAndDisplay(image: IKittyImageData, cmd: IKittyCommand): Promise<void> {
const bitmap = await this._createBitmap(image);
const cw = this._renderer.dimensions?.css.cell.width || CELL_SIZE_DEFAULT.width;
const ch = this._renderer.dimensions?.css.cell.height || CELL_SIZE_DEFAULT.height;
// Per spec: c/r default to image's natural cell dimensions
const imgCols = cmd.columns ?? Math.ceil(bitmap.width / cw);
const imgRows = cmd.rows ?? Math.ceil(bitmap.height / ch);
let w = bitmap.width;
let h = bitmap.height;
// Scale bitmap to fit placement rectangle when c/r are specified
if (cmd.columns !== undefined || cmd.rows !== undefined) {
w = Math.round(imgCols * cw);
h = Math.round(imgRows * ch);
}
if (w * h > this._opts.pixelLimit) {
throw new Error('image exceeds pixel limit');
}
// Save cursor position before addImage modifies it
const buffer = this._coreTerminal._core.buffer;
const savedX = buffer.x;
const savedY = buffer.y;
const savedYbase = buffer.ybase;
// Determine layer based on z-index: negative = behind text, 0+ = on top.
// When z<0 we always use the bottom layer even without allowTransparency —
// the image will simply be hidden behind the opaque text background, which
// is the correct behavior (client asked for "behind text").
const wantsBottom = cmd.zIndex !== undefined && cmd.zIndex < 0;
const layer: ImageLayer = wantsBottom ? 'bottom' : 'top';
const zIndex = cmd.zIndex ?? 0;
if (w !== bitmap.width || h !== bitmap.height) {
const resized = await createImageBitmap(bitmap, { resizeWidth: w, resizeHeight: h });
bitmap.close();
this._kittyStorage.addImage(image.id, resized, true, layer, zIndex);
} else {
this._kittyStorage.addImage(image.id, bitmap, true, layer, zIndex);
}
// Kitty cursor movement
// Per spec: cursor placed at first column after last image column,
// on the last row of the image. C=1 means don't move cursor.
if (cmd.cursorMovement === 1) {
// C=1: restore cursor to position before image was placed
const scrolled = buffer.ybase - savedYbase;
buffer.x = savedX;
// Can't restore cursor to scrollback?
buffer.y = Math.max(savedY - scrolled, 0);
} else {
// Default (C=0): advance cursor horizontally past the image
// addImage already positioned cursor on the last row via lineFeeds
buffer.x = Math.min(savedX + imgCols, this._coreTerminal.cols);
}
}
// Create ImageBitmap from already-decoded image data.
private async _createBitmap(image: IKittyImageData): Promise<ImageBitmap> {
let bytes: Uint8Array = new Uint8Array(await image.data.arrayBuffer());
if (image.compression === KittyCompression.ZLIB) {
bytes = await this._decompressZlib(bytes);
}
if (image.format === KittyFormat.PNG) {
const blob = new Blob([bytes as BlobPart], { type: 'image/png' });
if (!window.createImageBitmap) {
const url = URL.createObjectURL(blob);
const img = new Image();
return new Promise<ImageBitmap>((resolve, reject) => {
img.addEventListener('load', () => {
URL.revokeObjectURL(url);
const canvas = ImageRenderer.createCanvas(window.document, img.width, img.height);
canvas.getContext('2d')?.drawImage(img, 0, 0);
createImageBitmap(canvas).then(resolve).catch(reject);
});
img.addEventListener('error', () => {
URL.revokeObjectURL(url);
reject(new Error('Failed to load image'));
});
img.src = url;
});
}
return createImageBitmap(blob);
}
// Raw pixel data
const width = image.width;
const height = image.height;
if (!width || !height) {
throw new Error('Width and height required for raw pixel data');
}
const bytesPerPixel = image.format === KittyFormat.RGBA ? BYTES_PER_PIXEL_RGBA : BYTES_PER_PIXEL_RGB;
const expectedBytes = width * height * bytesPerPixel;
if (bytes.length < expectedBytes) {
throw new Error('Insufficient pixel data');
}
const pixelCount = width * height;
if (image.format === KittyFormat.RGBA) {
// RGBA: use bytes directly — no copy needed
return createImageBitmap(new ImageData(new Uint8ClampedArray(bytes.buffer as ArrayBuffer, bytes.byteOffset, pixelCount * BYTES_PER_PIXEL_RGBA), width, height));
}
// RGB→RGBA: interleave alpha using uint32 block processing (4 pixels per iteration).
// 3 uint32 reads + 4 uint32 writes per 4 pixels vs 28 byte reads/writes — ~6x faster.
// Assumes little-endian (all modern browsers/Node.js).
const data = new Uint8ClampedArray(pixelCount * BYTES_PER_PIXEL_RGBA);
const src32 = new Uint32Array(bytes.buffer, bytes.byteOffset, Math.floor(bytes.byteLength / 4));
const dst32 = new Uint32Array(data.buffer);
const alignedPixels = pixelCount & ~3; // round down to multiple of 4
let srcOffset = 0;
let dstOffset = 0;
for (let i = 0; i < alignedPixels; i += 4) {
const b0 = src32[srcOffset++];
const b1 = src32[srcOffset++];
const b2 = src32[srcOffset++];
// Little-endian: pixel bytes are [R,G,B] → uint32 ABGR layout
dst32[dstOffset++] = (b0 & 0x00FFFFFF) | 0xFF000000;
dst32[dstOffset++] = ((b0 >>> 24) | (b1 << 8)) & 0x00FFFFFF | 0xFF000000;
dst32[dstOffset++] = ((b1 >>> 16) | (b2 << 16)) & 0x00FFFFFF | 0xFF000000;
dst32[dstOffset++] = (b2 >>> 8) | 0xFF000000;
}
// Handle remaining 1–3 pixels
let srcByte = alignedPixels * BYTES_PER_PIXEL_RGB;
let dstByte = alignedPixels * BYTES_PER_PIXEL_RGBA;
for (let i = alignedPixels; i < pixelCount; i++) {
data[dstByte] = bytes[srcByte];
data[dstByte + 1] = bytes[srcByte + 1];
data[dstByte + 2] = bytes[srcByte + 2];
data[dstByte + 3] = ALPHA_OPAQUE;
srcByte += BYTES_PER_PIXEL_RGB;
dstByte += BYTES_PER_PIXEL_RGBA;
}
return createImageBitmap(new ImageData(data, width, height));
}
private async _decompressZlib(compressed: Uint8Array): Promise<Uint8Array> {
try {
return await this._decompress(compressed, 'deflate');
} catch {
return await this._decompress(compressed, 'deflate-raw');
}
}
private async _decompress(compressed: Uint8Array, format: 'deflate' | 'deflate-raw'): Promise<Uint8Array> {
const ds = new DecompressionStream(format);
const writer = ds.writable.getWriter();
writer.write(compressed as BufferSource);
writer.close();
const chunks: Uint8Array[] = [];
const reader = ds.readable.getReader();
while (true) {
const { done, value } = await reader.read();
if (done) break;
chunks.push(value);
}
const totalLength = chunks.reduce((sum, chunk) => sum + chunk.length, 0);
const result = new Uint8Array(totalLength);
let offset = 0;
for (const chunk of chunks) {
result.set(chunk, offset);
offset += chunk.length;
}
return result;
}
public get images(): ReadonlyMap<number, IKittyImageData> {
return this._kittyStorage.images;
}
public get _kittyIdToStorageId(): ReadonlyMap<number, number> {
return this._kittyStorage.kittyIdToStorageId;
}
public get pendingTransmissions(): ReadonlyMap<number, IPendingTransmission> {
return this._pendingTransmissions;
}
}
@@ -0,0 +1,149 @@
/**
* Copyright (c) 2026 The xterm.js authors. All rights reserved.
* @license MIT
*/
import { assert } from 'chai';
import { parseKittyCommand, KittyAction, KittyFormat } from './KittyGraphicsTypes';
describe('KittyGraphicsTypes', () => {
describe('parseKittyCommand', () => {
it('should parse control data with action and format', () => {
const cmd = parseKittyCommand('a=T,f=100');
assert.strictEqual(cmd.action, 'T');
assert.strictEqual(cmd.format, 100);
});
it('should parse control data with all options', () => {
const cmd = parseKittyCommand('a=t,f=32,i=5,s=10,v=20,c=3,r=2,m=1,q=2');
assert.strictEqual(cmd.action, 't');
assert.strictEqual(cmd.format, 32);
assert.strictEqual(cmd.id, 5);
assert.strictEqual(cmd.width, 10);
assert.strictEqual(cmd.height, 20);
assert.strictEqual(cmd.columns, 3);
assert.strictEqual(cmd.rows, 2);
assert.strictEqual(cmd.more, 1);
assert.strictEqual(cmd.quiet, 2);
});
it('should handle empty control data', () => {
const cmd = parseKittyCommand('');
assert.strictEqual(cmd.action, undefined);
assert.strictEqual(cmd.format, undefined);
});
it('should parse transmit action', () => {
const cmd = parseKittyCommand('a=t,f=100');
assert.strictEqual(cmd.action, KittyAction.TRANSMIT);
assert.strictEqual(cmd.format, KittyFormat.PNG);
});
it('should parse delete action', () => {
const cmd = parseKittyCommand('a=d,i=5');
assert.strictEqual(cmd.action, KittyAction.DELETE);
assert.strictEqual(cmd.id, 5);
});
it('should parse empty action as empty string', () => {
const cmd = parseKittyCommand('a=,f=100');
assert.strictEqual(cmd.action, '');
assert.strictEqual(cmd.format, 100);
});
it('should leave action undefined when key is not present', () => {
const cmd = parseKittyCommand('f=100,i=5');
assert.strictEqual(cmd.action, undefined);
assert.strictEqual(cmd.format, 100);
assert.strictEqual(cmd.id, 5);
});
it('should parse compression key', () => {
const cmd = parseKittyCommand('a=t,f=32,o=z');
assert.strictEqual(cmd.action, 't');
assert.strictEqual(cmd.format, 32);
assert.strictEqual(cmd.compression, 'z');
});
it('should parse cursor movement key', () => {
const cmd = parseKittyCommand('a=T,f=100,C=1');
assert.strictEqual(cmd.cursorMovement, 1);
});
it('should parse cursor movement key C=0', () => {
const cmd = parseKittyCommand('a=T,f=100,C=0');
assert.strictEqual(cmd.cursorMovement, 0);
});
it('should parse x and y offset', () => {
const cmd = parseKittyCommand('a=T,x=10,y=20');
assert.strictEqual(cmd.x, 10);
assert.strictEqual(cmd.y, 20);
});
it('should handle keys without values', () => {
const cmd = parseKittyCommand('a=t,f=,i=5');
assert.strictEqual(cmd.action, 't');
assert.ok(isNaN(cmd.format!));
assert.strictEqual(cmd.id, 5);
});
it('should parse z-index key with positive value', () => {
const cmd = parseKittyCommand('a=T,f=100,z=10');
assert.strictEqual(cmd.zIndex, 10);
});
it('should parse z-index key with zero', () => {
const cmd = parseKittyCommand('a=T,f=100,z=0');
assert.strictEqual(cmd.zIndex, 0);
});
it('should parse z-index key with negative value', () => {
const cmd = parseKittyCommand('a=T,f=100,z=-1');
assert.strictEqual(cmd.zIndex, -1);
});
it('should leave zIndex undefined when not specified', () => {
const cmd = parseKittyCommand('a=T,f=100');
assert.strictEqual(cmd.zIndex, undefined);
});
it('should parse delete selector key', () => {
const cmd = parseKittyCommand('a=d,d=i,i=5');
assert.strictEqual(cmd.action, 'd');
assert.strictEqual(cmd.deleteSelector, 'i');
assert.strictEqual(cmd.id, 5);
});
it('should parse uppercase delete selector', () => {
const cmd = parseKittyCommand('a=d,d=A');
assert.strictEqual(cmd.deleteSelector, 'A');
});
it('should parse delete selector d=a (all)', () => {
const cmd = parseKittyCommand('a=d,d=a');
assert.strictEqual(cmd.deleteSelector, 'a');
});
it('should leave deleteSelector undefined when not specified', () => {
const cmd = parseKittyCommand('a=d,i=5');
assert.strictEqual(cmd.deleteSelector, undefined);
});
it('should parse placement id key', () => {
const cmd = parseKittyCommand('a=d,d=i,i=5,p=3');
assert.strictEqual(cmd.placementId, 3);
assert.strictEqual(cmd.deleteSelector, 'i');
assert.strictEqual(cmd.id, 5);
});
it('should leave placementId undefined when not specified', () => {
const cmd = parseKittyCommand('a=d,d=i,i=5');
assert.strictEqual(cmd.placementId, undefined);
});
it('should parse image number key', () => {
const cmd = parseKittyCommand('a=t,f=100,I=42');
assert.strictEqual(cmd.imageNumber, 42);
});
});
});
@@ -0,0 +1,177 @@
/**
* Copyright (c) 2026 The xterm.js authors. All rights reserved.
* @license MIT
*
* Kitty graphics protocol types, constants, and parsing utilities.
*/
import type Base64Decoder from 'xterm-wasm-parts/lib/base64/Base64Decoder.wasm';
// Kitty graphics protocol action types.
// See: https://sw.kovidgoyal.net/kitty/graphics-protocol/#control-data-reference under key 'a'.
export const enum KittyAction {
TRANSMIT = 't',
TRANSMIT_DISPLAY = 'T',
QUERY = 'q',
PLACEMENT = 'p',
DELETE = 'd'
}
// Kitty graphics protocol format types.
// See: https://sw.kovidgoyal.net/kitty/graphics-protocol/#control-data-reference
export const enum KittyFormat {
RGB = 24,
RGBA = 32,
PNG = 100
}
// Kitty graphics protocol compression types.
// See: https://sw.kovidgoyal.net/kitty/graphics-protocol/#control-data-reference under key 'o'.
export const enum KittyCompression {
NONE = '',
ZLIB = 'z'
}
// Kitty graphics protocol control data keys.
// See: https://sw.kovidgoyal.net/kitty/graphics-protocol/#control-data-reference
export const enum KittyKey {
// Action to perform (t=transmit, T=transmit+display, q=query, p=placement, d=delete)
ACTION = 'a',
// Image format (24=RGB, 32=RGBA, 100=PNG)
FORMAT = 'f',
// Image ID for referencing stored images
ID = 'i',
// Image number (alternative to ID, terminal assigns ID)
IMAGE_NUMBER = 'I',
// Source image width in pixels
WIDTH = 's',
// Source image height in pixels
HEIGHT = 'v',
// The left edge (in pixels) of the image area to display
X_OFFSET = 'x',
// The top edge (in pixels) of the image area to display
Y_OFFSET = 'y',
// Number of terminal columns to display the image over
COLUMNS = 'c',
// Number of terminal rows to display the image over
ROWS = 'r',
// More data flag (1=more chunks coming, 0=final chunk)
MORE = 'm',
// Compression type (z=zlib). This is essential for chunking larger images.
COMPRESSION = 'o',
// Quiet mode (1=suppress OK responses, 2=suppress error responses)
QUIET = 'q',
// Cursor movement policy (0=move cursor after image, 1=don't move cursor)
CURSOR_MOVEMENT = 'C',
// Z-index for image layering (negative = behind text, 0+ = on top)
Z_INDEX = 'z',
// Transmission medium (d=direct, f=file, t=temp file, s=shared memory)
TRANSMISSION = 't',
// Delete selector (a/A=all, i/I=by id, c/C=at cursor, etc.) — only used when a=d
DELETE_SELECTOR = 'd',
// Placement ID for targeting specific placements
PLACEMENT_ID = 'p'
}
// Pixel format constants
export const BYTES_PER_PIXEL_RGB = 3;
export const BYTES_PER_PIXEL_RGBA = 4;
export const ALPHA_OPAQUE = 255;
// Parsed Kitty graphics command.
export interface IKittyCommand {
action?: string;
format?: number;
id?: number;
imageNumber?: number;
width?: number;
height?: number;
x?: number;
y?: number;
columns?: number;
rows?: number;
more?: number;
quiet?: number;
cursorMovement?: number;
zIndex?: number;
transmission?: string;
deleteSelector?: string;
placementId?: number;
compression?: string;
payload?: string;
}
// Pending chunked transmission state.
// Stores metadata from the first chunk while accumulating decoded payload data.
export interface IPendingTransmission {
// The parsed command from the first chunk (contains action, format, dimensions, etc.)
cmd: IKittyCommand;
// Decoder used across chunked payloads
decoder: Base64Decoder;
// Total encoded (base64) bytes received across all chunks - for size limit enforcement
totalEncodedSize: number;
// Whether any chunk has failed to decode
decodeError: boolean;
}
// Stored Kitty image data.
export interface IKittyImageData {
id: number;
// Decoded image data stored as Blob (off JS heap) to avoid 2GB heap limit
data: Blob;
width: number;
height: number;
format: 24 | 32 | 100;
compression?: string;
}
// Parses Kitty graphics control data into a command object.
export function parseKittyCommand(data: string): IKittyCommand {
const cmd: IKittyCommand = {};
const parts = data.split(',');
for (const part of parts) {
const eqIdx = part.indexOf('=');
if (eqIdx === -1) continue;
const key = part.substring(0, eqIdx);
const value = part.substring(eqIdx + 1);
// Handle string keys first
if (key === KittyKey.ACTION) {
cmd.action = value;
continue;
}
if (key === KittyKey.COMPRESSION) {
cmd.compression = value;
continue;
}
if (key === KittyKey.TRANSMISSION) {
cmd.transmission = value;
continue;
}
if (key === KittyKey.DELETE_SELECTOR) {
cmd.deleteSelector = value;
continue;
}
const numValue = parseInt(value);
switch (key) {
case KittyKey.FORMAT: cmd.format = numValue; break;
case KittyKey.ID: cmd.id = numValue; break;
case KittyKey.IMAGE_NUMBER: cmd.imageNumber = numValue; break;
case KittyKey.WIDTH: cmd.width = numValue; break;
case KittyKey.HEIGHT: cmd.height = numValue; break;
case KittyKey.X_OFFSET: cmd.x = numValue; break;
case KittyKey.Y_OFFSET: cmd.y = numValue; break;
case KittyKey.COLUMNS: cmd.columns = numValue; break;
case KittyKey.ROWS: cmd.rows = numValue; break;
case KittyKey.MORE: cmd.more = numValue; break;
case KittyKey.QUIET: cmd.quiet = numValue; break;
case KittyKey.CURSOR_MOVEMENT: cmd.cursorMovement = numValue; break;
case KittyKey.Z_INDEX: cmd.zIndex = numValue; break;
case KittyKey.PLACEMENT_ID: cmd.placementId = numValue; break;
}
}
return cmd;
}
@@ -0,0 +1,134 @@
/**
* Copyright (c) 2026 The xterm.js authors. All rights reserved.
* @license MIT
*/
import { IDisposable } from '@xterm/xterm';
import { ImageStorage } from '../ImageStorage';
import { ImageLayer } from '../Types';
import { IKittyImageData } from './KittyGraphicsTypes';
// Kitty-specific image storage controller.
//
// Wraps shared ImageStorage with kitty protocol semantics:
// - tracks transmitted image payloads by kitty image id
// - tracks kitty image id -> shared ImageStorage id mapping for displayed images
// - mirrors shared-storage evictions into kitty maps
// - applies protocol-level undisplayed-image eviction policy
export class KittyImageStorage implements IDisposable {
private static readonly _maxStoredImages = 256;
private _nextImageId = 1;
private readonly _images: Map<number, IKittyImageData> = new Map();
private readonly _kittyIdToStorageId: Map<number, number> = new Map();
private readonly _storageIdToKittyId: Map<number, number> = new Map();
private readonly _previousOnImageDeleted: ((storageId: number) => void) | undefined;
private readonly _wrappedOnImageDeleted: (storageId: number) => void;
private readonly _handleStorageImageDeleted = (storageId: number): void => {
const kittyId = this._storageIdToKittyId.get(storageId);
if (kittyId !== undefined) {
this._kittyIdToStorageId.delete(kittyId);
this._storageIdToKittyId.delete(storageId);
this._images.delete(kittyId);
}
};
constructor(
private readonly _storage: ImageStorage
) {
this._previousOnImageDeleted = this._storage.onImageDeleted;
this._wrappedOnImageDeleted = (storageId: number) => {
this._previousOnImageDeleted?.(storageId);
this._handleStorageImageDeleted(storageId);
};
this._storage.onImageDeleted = this._wrappedOnImageDeleted;
}
public reset(): void {
this._nextImageId = 1;
this._images.clear();
this._kittyIdToStorageId.clear();
this._storageIdToKittyId.clear();
}
public dispose(): void {
this.reset();
if (this._storage.onImageDeleted === this._wrappedOnImageDeleted) {
this._storage.onImageDeleted = this._previousOnImageDeleted;
}
}
public storeImage(id: number | undefined, imageData: Omit<IKittyImageData, 'id'>): number {
const imageId = id ?? this._nextImageId++;
const oldStorageId = this._kittyIdToStorageId.get(imageId);
if (oldStorageId !== undefined) {
this._storage.deleteImage(oldStorageId);
this._kittyIdToStorageId.delete(imageId);
this._storageIdToKittyId.delete(oldStorageId);
}
if (!this._images.has(imageId) && this._images.size >= KittyImageStorage._maxStoredImages) {
this._evictUndisplayedImages();
}
this._images.set(imageId, {
...imageData,
id: imageId
});
return imageId;
}
public addImage(kittyId: number, image: HTMLCanvasElement | ImageBitmap, scrolling: boolean, layer: ImageLayer, zIndex: number): void {
const storageId = this._storage.addImage(image, scrolling, layer, zIndex);
this._kittyIdToStorageId.set(kittyId, storageId);
this._storageIdToKittyId.set(storageId, kittyId);
}
public getImage(kittyId: number): IKittyImageData | undefined {
return this._images.get(kittyId);
}
public deleteById(kittyId: number): void {
this._images.delete(kittyId);
const storageId = this._kittyIdToStorageId.get(kittyId);
if (storageId !== undefined) {
this._storage.deleteImage(storageId);
this._kittyIdToStorageId.delete(kittyId);
this._storageIdToKittyId.delete(storageId);
}
}
public deleteAll(): void {
this._images.clear();
for (const storageId of this._kittyIdToStorageId.values()) {
this._storage.deleteImage(storageId);
}
this._kittyIdToStorageId.clear();
this._storageIdToKittyId.clear();
}
public get images(): ReadonlyMap<number, IKittyImageData> {
return this._images;
}
public get kittyIdToStorageId(): ReadonlyMap<number, number> {
return this._kittyIdToStorageId;
}
public get lastImageId(): number {
return this._nextImageId - 1;
}
private _evictUndisplayedImages(): void {
for (const [kittyId] of this._images) {
if (this._images.size <= KittyImageStorage._maxStoredImages / 2) {
break;
}
if (!this._kittyIdToStorageId.has(kittyId)) {
this._images.delete(kittyId);
}
}
}
}
+8 -2
View File
@@ -23,6 +23,8 @@ export interface IImageAddonOptions {
sixelSizeLimit: number;
iipSupport: boolean;
iipSizeLimit: number;
kittySupport: boolean;
kittySizeLimit: number;
}
// eslint-disable-next-line
@@ -134,7 +136,9 @@ test.describe('ImageAddon', () => {
storageLimit: 128,
showPlaceholder: true,
iipSupport: true,
iipSizeLimit: 20000000
iipSizeLimit: 20000000,
kittySupport: true,
kittySizeLimit: 20000000
};
deepStrictEqual(await ctx.page.evaluate(`window.imageAddon._opts`), DEFAULT_OPTIONS);
});
@@ -149,7 +153,9 @@ test.describe('ImageAddon', () => {
storageLimit: 10,
showPlaceholder: false,
iipSupport: false,
iipSizeLimit: 1000
iipSizeLimit: 1000,
kittySupport: false,
kittySizeLimit: 1000
};
await ctx.page.evaluate(opts => {
(window as any).imageAddonCustom = new ImageAddon(opts.opts);
File diff suppressed because it is too large Load Diff
+9
View File
@@ -76,6 +76,15 @@ declare module '@xterm/addon-image' {
iipSupport?: boolean;
/** IIP sequence size limit (default 20000000 bytes). */
iipSizeLimit?: number;
/**
* Kitty graphics protocol settings
*/
/** Whether Kitty graphics protocol is enabled (default is true). */
kittySupport?: boolean;
/** Kitty image size limit in bytes (default 20000000 bytes). */
kittySizeLimit?: number;
}
export class ImageAddon implements ITerminalAddon {
@@ -51,14 +51,30 @@ export class AddonImageWindow extends BaseWindow implements IControlWindow {
container.appendChild(document.createElement('br'));
container.appendChild(document.createElement('br'));
const dl = document.createElement('dl');
const dt = document.createElement('dt');
dt.textContent = 'Image Test';
dl.appendChild(dt);
this._addDdWithButton(dl, 'image-demo1', 'snake (sixel)');
this._addDdWithButton(dl, 'image-demo2', 'oranges (sixel)');
this._addDdWithButton(dl, 'image-demo3', 'palette (iip)');
container.appendChild(dl);
// Sixel demos
const dlSixel = document.createElement('dl');
const dtSixel = document.createElement('dt');
dtSixel.textContent = 'Sixel';
dlSixel.appendChild(dtSixel);
this._addDdWithButton(dlSixel, 'image-demo1', 'snake');
this._addDdWithButton(dlSixel, 'image-demo2', 'oranges');
container.appendChild(dlSixel);
// IIP demos
const dlIip = document.createElement('dl');
const dtIip = document.createElement('dt');
dtIip.textContent = 'IIP (iTerm)';
dlIip.appendChild(dtIip);
this._addDdWithButton(dlIip, 'image-demo3', 'palette');
container.appendChild(dlIip);
// Kitty demos
const dlKitty = document.createElement('dl');
const dtKitty = document.createElement('dt');
dtKitty.textContent = 'Kitty';
dlKitty.appendChild(dtKitty);
this._addDdWithButton(dlKitty, 'image-demo-kitty1', 'palette');
container.appendChild(dlKitty);
this._initImageAddonExposed();
}
@@ -125,12 +141,25 @@ export class AddonImageWindow extends BaseWindow implements IControlWindow {
this._terminal.write(`\x1b]1337;File=inline=1;size=${data.length}:${btoa(sdata)}\x1b\\`);
});
const kittyDemo = (url: string) => () => fetch(url)
.then(resp => resp.arrayBuffer())
.then(buffer => {
const data = new Uint8Array(buffer);
let sdata = '';
for (let i = 0; i < data.length; ++i) sdata += String.fromCharCode(data[i]);
const payload = btoa(sdata);
this._terminal.write('\r\n');
this._terminal.write(`\x1b_Ga=T,f=100;${payload}\x1b\\`);
});
document.getElementById('image-demo1')!.addEventListener('click',
sixelDemo('https://raw.githubusercontent.com/saitoha/libsixel/master/images/snake.six'));
document.getElementById('image-demo2')!.addEventListener('click',
sixelDemo('https://raw.githubusercontent.com/jerch/node-sixel/master/testfiles/test2.sixel'));
document.getElementById('image-demo3')!.addEventListener('click',
iipDemo('https://raw.githubusercontent.com/jerch/node-sixel/master/palette.png'));
document.getElementById('image-demo-kitty1')!.addEventListener('click',
kittyDemo('https://raw.githubusercontent.com/jerch/node-sixel/master/palette.png'));
// demo for image retrieval API
this._terminal.element!.addEventListener('click', (ev: MouseEvent) => {