Outsource a bunch to KittyApcHandler. TODO: go over this
This commit is contained in:
Anthony Kim
@@ -2,9 +2,11 @@
|
||||
* Copyright (c) 2025 The xterm.js authors. All rights reserved.
|
||||
* @license MIT
|
||||
*
|
||||
* Kitty graphics protocol types, constants, and parsing utilities.
|
||||
* Kitty graphics protocol types, constants, parsing, and APC handler.
|
||||
*/
|
||||
|
||||
import type { IKittyImage } from '@xterm/addon-kitty-graphics';
|
||||
|
||||
/**
|
||||
* Kitty graphics protocol action types.
|
||||
* See: https://sw.kovidgoyal.net/kitty/graphics-protocol/#control-data-reference under key 'a'.
|
||||
@@ -19,7 +21,7 @@ export const enum KittyAction {
|
||||
|
||||
/**
|
||||
* Kitty graphics protocol format types.
|
||||
* See: https://sw.kovidgoyal.net/kitty/graphics-protocol/#control-data-reference
|
||||
* See: https://sw.kovidgoyal.net/kitty/graphics-protocol/#control-data-reference where the format for *-bit came from.
|
||||
*/
|
||||
export const enum KittyFormat {
|
||||
RGB = 24,
|
||||
@@ -91,6 +93,35 @@ export interface IKittyCommand {
|
||||
payload?: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Pending chunked transmission state.
|
||||
* Stores metadata from the first chunk while accumulating payload data.
|
||||
*/
|
||||
interface IPendingTransmission {
|
||||
/** The parsed command from the first chunk (contains action, format, dimensions, etc.) */
|
||||
cmd: IKittyCommand;
|
||||
/** Accumulated base64 payload data */
|
||||
data: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Terminal interface for sending responses.
|
||||
* Question from Anthony: Do we really need this...
|
||||
*/
|
||||
interface ITerminal {
|
||||
input(data: string, wasUserInput: boolean): void;
|
||||
}
|
||||
|
||||
/**
|
||||
* Renderer interface for image display.
|
||||
*/
|
||||
interface IKittyRenderer {
|
||||
getCellSize(): { width: number, height: number };
|
||||
placeImage(bitmap: ImageBitmap, id: number, col?: number, row?: number, width?: number, height?: number): number;
|
||||
removeByImageId(id: number): void;
|
||||
clearAll(): void;
|
||||
}
|
||||
|
||||
/**
|
||||
* Parses Kitty graphics control data into a command object.
|
||||
*/
|
||||
@@ -131,3 +162,446 @@ export function parseKittyCommand(data: string): IKittyCommand {
|
||||
|
||||
return cmd;
|
||||
}
|
||||
|
||||
/**
|
||||
* Handles Kitty graphics protocol APC sequences.
|
||||
* Manages parsing, chunked transmissions, and dispatching to action handlers.
|
||||
*
|
||||
* TODO: Go over this with Daniel: Like SixelHandler, this receives direct references to storage
|
||||
* and terminal rather than using callbacks.
|
||||
*/
|
||||
export class KittyApcHandler {
|
||||
/**
|
||||
* Pending chunked transmissions keyed by image ID.
|
||||
* ID 0 is used for transmissions without an explicit ID (the "anonymous" transmission).
|
||||
*/
|
||||
private _pendingTransmissions: Map<number, IPendingTransmission> = new Map();
|
||||
private _nextImageId = 1;
|
||||
|
||||
constructor(
|
||||
private readonly _images: Map<number, IKittyImage>,
|
||||
private readonly _decodedImages: Map<number, ImageBitmap>,
|
||||
private readonly _renderer: IKittyRenderer | undefined,
|
||||
private readonly _terminal: ITerminal | undefined,
|
||||
private readonly _debug: boolean = false
|
||||
) {}
|
||||
|
||||
/**
|
||||
* Handle a Kitty graphics APC sequence.
|
||||
* @param data - The data portion of the APC sequence (after 'G')
|
||||
* @returns true if handled successfully
|
||||
*/
|
||||
public handle(data: string): boolean {
|
||||
const semiIdx = data.indexOf(';');
|
||||
const controlData = semiIdx === -1 ? data : data.substring(0, semiIdx);
|
||||
const payload = semiIdx === -1 ? '' : data.substring(semiIdx + 1);
|
||||
|
||||
const cmd = parseKittyCommand(controlData);
|
||||
cmd.payload = payload;
|
||||
|
||||
if (this._debug) {
|
||||
console.log('[KittyApcHandler] Received command:', cmd);
|
||||
}
|
||||
|
||||
const action = cmd.action ?? 't';
|
||||
|
||||
switch (action) {
|
||||
case KittyAction.TRANSMIT:
|
||||
return this._handleTransmit(cmd);
|
||||
case KittyAction.TRANSMIT_DISPLAY:
|
||||
return this._handleTransmitDisplay(cmd);
|
||||
case KittyAction.QUERY:
|
||||
return this._handleQuery(cmd);
|
||||
case KittyAction.DELETE:
|
||||
return this._handleDelete(cmd);
|
||||
default:
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Clear all pending transmissions. Called on dispose.
|
||||
*/
|
||||
public clearPendingTransmissions(): void {
|
||||
this._pendingTransmissions.clear();
|
||||
}
|
||||
|
||||
private _handleTransmit(cmd: IKittyCommand): boolean {
|
||||
const payload = cmd.payload ?? '';
|
||||
const pendingKey = cmd.id ?? 0;
|
||||
|
||||
const isMoreComing = cmd.more === 1;
|
||||
const pending = this._pendingTransmissions.get(pendingKey);
|
||||
|
||||
if (pending) {
|
||||
pending.data += payload;
|
||||
|
||||
if (this._debug) {
|
||||
console.log(`[KittyApcHandler] Chunk continuation for id=${pendingKey}, total size=${pending.data.length}, more=${isMoreComing}`);
|
||||
}
|
||||
|
||||
if (isMoreComing) {
|
||||
return true;
|
||||
}
|
||||
|
||||
const originalCmd = pending.cmd;
|
||||
const fullPayload = pending.data;
|
||||
this._pendingTransmissions.delete(pendingKey);
|
||||
|
||||
const id = originalCmd.id ?? this._nextImageId++;
|
||||
|
||||
const image: IKittyImage = {
|
||||
id,
|
||||
data: fullPayload,
|
||||
width: originalCmd.width ?? 0,
|
||||
height: originalCmd.height ?? 0,
|
||||
format: (originalCmd.format ?? KittyFormat.PNG) as 24 | 32 | 100,
|
||||
compression: originalCmd.compression ?? ''
|
||||
};
|
||||
|
||||
this._images.set(image.id, image);
|
||||
|
||||
if (this._debug) {
|
||||
console.log(`[KittyApcHandler] Stored chunked image ${id}, payload size=${fullPayload.length}`);
|
||||
}
|
||||
|
||||
if (originalCmd.action === KittyAction.TRANSMIT_DISPLAY) {
|
||||
this._displayImage(image, originalCmd.columns, originalCmd.rows);
|
||||
}
|
||||
|
||||
cmd.id = id;
|
||||
return true;
|
||||
}
|
||||
|
||||
if (isMoreComing) {
|
||||
this._pendingTransmissions.set(pendingKey, {
|
||||
cmd: { ...cmd },
|
||||
data: payload
|
||||
});
|
||||
|
||||
if (this._debug) {
|
||||
console.log(`[KittyApcHandler] Started chunked transmission, id=${pendingKey}, format=${cmd.format}, compression=${cmd.compression}, size=${cmd.width}x${cmd.height}, initial payload=${payload.length}`);
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
const id = cmd.id ?? this._nextImageId++;
|
||||
|
||||
const image: IKittyImage = {
|
||||
id,
|
||||
data: payload,
|
||||
width: cmd.width ?? 0,
|
||||
height: cmd.height ?? 0,
|
||||
format: (cmd.format ?? KittyFormat.PNG) as 24 | 32 | 100,
|
||||
compression: cmd.compression ?? ''
|
||||
};
|
||||
|
||||
this._images.set(image.id, image);
|
||||
|
||||
if (this._debug) {
|
||||
console.log(`[KittyApcHandler] Stored image ${id}`);
|
||||
}
|
||||
|
||||
cmd.id = id;
|
||||
return true;
|
||||
}
|
||||
|
||||
private _handleTransmitDisplay(cmd: IKittyCommand): boolean {
|
||||
const pendingKey = cmd.id ?? 0;
|
||||
const wasPendingBefore = this._pendingTransmissions.has(pendingKey);
|
||||
|
||||
this._handleTransmit(cmd);
|
||||
|
||||
if (cmd.more === 1) {
|
||||
return true;
|
||||
}
|
||||
|
||||
if (wasPendingBefore) {
|
||||
return true;
|
||||
}
|
||||
|
||||
const id = cmd.id!;
|
||||
const image = this._images.get(id);
|
||||
if (image) {
|
||||
this._displayImage(image, cmd.columns, cmd.rows);
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* Handle query action (a=q).
|
||||
*
|
||||
* Per the Kitty graphics protocol documentation:
|
||||
* "Sometimes, using an id is not appropriate... In that case, you can use the
|
||||
* query action, set a=q. Then the terminal emulator will try to load the image
|
||||
* and respond with either OK or an error, as above, but it will not replace an
|
||||
* existing image with the same id, nor will it store the image."
|
||||
*/
|
||||
private _handleQuery(cmd: IKittyCommand): boolean {
|
||||
const id = cmd.id ?? 0;
|
||||
const quiet = cmd.quiet ?? 0;
|
||||
|
||||
if (this._debug) {
|
||||
console.log(`[KittyApcHandler] Query received, id=${id}, quiet=${quiet}`);
|
||||
}
|
||||
|
||||
const payload = cmd.payload || '';
|
||||
|
||||
if (!payload) {
|
||||
this._sendResponse(id, 'OK', quiet);
|
||||
return true;
|
||||
}
|
||||
|
||||
try {
|
||||
const binaryString = atob(payload);
|
||||
const bytes = new Uint8Array(binaryString.length);
|
||||
for (let i = 0; i < binaryString.length; i++) {
|
||||
bytes[i] = binaryString.charCodeAt(i);
|
||||
}
|
||||
|
||||
const format = cmd.format || KittyFormat.RGBA;
|
||||
|
||||
if (format === KittyFormat.PNG) {
|
||||
this._sendResponse(id, 'OK', quiet);
|
||||
} else {
|
||||
const width = cmd.width || 0;
|
||||
const height = cmd.height || 0;
|
||||
|
||||
if (!width || !height) {
|
||||
this._sendResponse(id, 'EINVAL:width and height required for raw pixel data', quiet);
|
||||
return true;
|
||||
}
|
||||
|
||||
const bytesPerPixel = format === KittyFormat.RGBA ? BYTES_PER_PIXEL_RGBA : BYTES_PER_PIXEL_RGB;
|
||||
const expectedBytes = width * height * bytesPerPixel;
|
||||
|
||||
if (bytes.length < expectedBytes) {
|
||||
this._sendResponse(id, `EINVAL:insufficient pixel data, got ${bytes.length}, expected ${expectedBytes}`, quiet);
|
||||
return true;
|
||||
}
|
||||
|
||||
this._sendResponse(id, 'OK', quiet);
|
||||
}
|
||||
} catch (e) {
|
||||
const errorMsg = e instanceof Error ? e.message : 'unknown error';
|
||||
this._sendResponse(id, `EINVAL:${errorMsg}`, quiet);
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
private _handleDelete(cmd: IKittyCommand): boolean {
|
||||
const id = cmd.id;
|
||||
|
||||
if (id !== undefined) {
|
||||
this._images.delete(id);
|
||||
// Close and remove decoded bitmap
|
||||
const bitmap = this._decodedImages.get(id);
|
||||
if (bitmap) {
|
||||
bitmap.close();
|
||||
this._decodedImages.delete(id);
|
||||
}
|
||||
// Remove from renderer
|
||||
this._renderer?.removeByImageId(id);
|
||||
} else {
|
||||
this._images.clear();
|
||||
// Close all decoded bitmaps
|
||||
for (const bitmap of this._decodedImages.values()) {
|
||||
bitmap.close();
|
||||
}
|
||||
this._decodedImages.clear();
|
||||
// Clear all from renderer
|
||||
this._renderer?.clearAll();
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* Send a response back to the client via the terminal's input method.
|
||||
*/
|
||||
private _sendResponse(id: number, message: string, quiet: number): void {
|
||||
const isOk = message === 'OK';
|
||||
if (isOk && quiet === 1) {
|
||||
return;
|
||||
}
|
||||
if (!isOk && quiet === 2) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (!this._terminal) {
|
||||
return;
|
||||
}
|
||||
|
||||
const response = `\x1b_Gi=${id};${message}\x1b\\`;
|
||||
this._terminal.input(response, false);
|
||||
|
||||
if (this._debug) {
|
||||
console.log(`[KittyApcHandler] Sent response: i=${id};${message}`);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Decode and display an image at the cursor position.
|
||||
*/
|
||||
private _displayImage(image: IKittyImage, columns?: number, rows?: number): void {
|
||||
if (!this._renderer) {
|
||||
return;
|
||||
}
|
||||
|
||||
const storedImage = this._images.get(image.id);
|
||||
if (!storedImage) {
|
||||
if (this._debug) {
|
||||
console.log(`[KittyApcHandler] No image found for id=${image.id} after transmit`);
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
this._decodeAndDisplay(storedImage, columns, rows).catch(err => {
|
||||
if (this._debug) {
|
||||
console.error(`[KittyApcHandler] Failed to decode/display image ${image.id}:`, err);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Decode an image and display it at the cursor position.
|
||||
*/
|
||||
private async _decodeAndDisplay(image: IKittyImage, columns?: number, rows?: number): Promise<void> {
|
||||
if (!this._renderer) {
|
||||
return;
|
||||
}
|
||||
|
||||
let bitmap = this._decodedImages.get(image.id);
|
||||
|
||||
if (!bitmap) {
|
||||
bitmap = await this._decodeImage(image);
|
||||
this._decodedImages.set(image.id, bitmap);
|
||||
|
||||
if (this._debug) {
|
||||
console.log(`[KittyApcHandler] Decoded image ${image.id}: ${bitmap.width}x${bitmap.height}`);
|
||||
}
|
||||
}
|
||||
|
||||
let width = 0;
|
||||
let height = 0;
|
||||
if (columns || rows) {
|
||||
const cellSize = this._renderer.getCellSize();
|
||||
if (columns) {
|
||||
width = columns * cellSize.width;
|
||||
}
|
||||
if (rows) {
|
||||
height = rows * cellSize.height;
|
||||
}
|
||||
if (width && !height) {
|
||||
height = Math.round(width * (bitmap.height / bitmap.width));
|
||||
} else if (height && !width) {
|
||||
width = Math.round(height * (bitmap.width / bitmap.height));
|
||||
}
|
||||
}
|
||||
|
||||
this._renderer.placeImage(bitmap, image.id, undefined, undefined, width, height);
|
||||
|
||||
if (this._debug) {
|
||||
console.log(`[KittyApcHandler] Placed image ${image.id} at cursor, size: ${width || bitmap.width}x${height || bitmap.height}`);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Decode base64 image data into an ImageBitmap.
|
||||
*/
|
||||
private async _decodeImage(image: IKittyImage): Promise<ImageBitmap> {
|
||||
const format = image.format;
|
||||
const base64Data = image.data as string;
|
||||
|
||||
const binaryString = atob(base64Data);
|
||||
let bytes = new Uint8Array(binaryString.length);
|
||||
for (let i = 0; i < binaryString.length; i++) {
|
||||
bytes[i] = binaryString.charCodeAt(i);
|
||||
}
|
||||
|
||||
if (image.compression === KittyCompression.ZLIB) {
|
||||
bytes = await this._decompressZlib(bytes);
|
||||
if (this._debug) {
|
||||
console.log(`[KittyApcHandler] Decompressed ${binaryString.length} -> ${bytes.length} bytes`);
|
||||
}
|
||||
}
|
||||
|
||||
if (format === KittyFormat.PNG) {
|
||||
const blob = new Blob([bytes], { type: 'image/png' });
|
||||
return createImageBitmap(blob);
|
||||
}
|
||||
|
||||
const width = image.width;
|
||||
const height = image.height;
|
||||
|
||||
if (!width || !height) {
|
||||
throw new Error('Width and height required for raw pixel data');
|
||||
}
|
||||
|
||||
const bytesPerPixel = format === KittyFormat.RGBA ? BYTES_PER_PIXEL_RGBA : BYTES_PER_PIXEL_RGB;
|
||||
const expectedBytes = width * height * bytesPerPixel;
|
||||
|
||||
if (bytes.length < expectedBytes) {
|
||||
throw new Error(`Insufficient pixel data: got ${bytes.length}, expected ${expectedBytes}`);
|
||||
}
|
||||
|
||||
// Convert to RGBA ImageData
|
||||
// TODO: Get this checked by Daniel.
|
||||
const pixelCount = width * height;
|
||||
const data = new Uint8ClampedArray(pixelCount * BYTES_PER_PIXEL_RGBA);
|
||||
const isRgba = format === KittyFormat.RGBA;
|
||||
|
||||
let srcOffset = 0;
|
||||
let dstOffset = 0;
|
||||
for (let i = 0; i < pixelCount; i++) {
|
||||
data[dstOffset ] = bytes[srcOffset ]; // R
|
||||
data[dstOffset + 1] = bytes[srcOffset + 1]; // G
|
||||
data[dstOffset + 2] = bytes[srcOffset + 2]; // B
|
||||
data[dstOffset + 3] = isRgba ? bytes[srcOffset + 3] : ALPHA_OPAQUE;
|
||||
srcOffset += bytesPerPixel;
|
||||
dstOffset += BYTES_PER_PIXEL_RGBA;
|
||||
}
|
||||
|
||||
return createImageBitmap(new ImageData(data, width, height));
|
||||
}
|
||||
|
||||
/**
|
||||
* Decompress zlib/deflate compressed data using the browser's DecompressionStream API.
|
||||
*/
|
||||
private async _decompressZlib(compressed: Uint8Array): Promise<Uint8Array> {
|
||||
try {
|
||||
return await this._decompress(compressed, 'deflate');
|
||||
} catch {
|
||||
return await this._decompress(compressed, 'deflate-raw');
|
||||
}
|
||||
}
|
||||
|
||||
private async _decompress(compressed: Uint8Array, format: 'deflate' | 'deflate-raw'): Promise<Uint8Array> {
|
||||
const ds = new DecompressionStream(format);
|
||||
const writer = ds.writable.getWriter();
|
||||
writer.write(compressed);
|
||||
writer.close();
|
||||
|
||||
const chunks: Uint8Array[] = [];
|
||||
const reader = ds.readable.getReader();
|
||||
|
||||
while (true) {
|
||||
const { done, value } = await reader.read();
|
||||
if (done) break;
|
||||
chunks.push(value);
|
||||
}
|
||||
|
||||
const totalLength = chunks.reduce((sum, chunk) => sum + chunk.length, 0);
|
||||
const result = new Uint8Array(totalLength);
|
||||
let offset = 0;
|
||||
for (const chunk of chunks) {
|
||||
result.set(chunk, offset);
|
||||
offset += chunk.length;
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -7,42 +7,17 @@ import type { Terminal, ITerminalAddon, IDisposable } from '@xterm/xterm';
|
||||
import type { KittyGraphicsAddon as IKittyGraphicsApi, IKittyGraphicsOptions, IKittyImage } from '@xterm/addon-kitty-graphics';
|
||||
import { KittyImageRenderer } from './KittyImageRenderer';
|
||||
import type { ITerminalExt } from './Types';
|
||||
import {
|
||||
KittyAction,
|
||||
KittyFormat,
|
||||
KittyCompression,
|
||||
BYTES_PER_PIXEL_RGB,
|
||||
BYTES_PER_PIXEL_RGBA,
|
||||
ALPHA_OPAQUE,
|
||||
parseKittyCommand,
|
||||
type IKittyCommand
|
||||
} from './KittyApcHandler';
|
||||
|
||||
/**
|
||||
* Pending chunked transmission state.
|
||||
* Stores metadata from the first chunk while accumulating payload data.
|
||||
*/
|
||||
interface IPendingTransmission {
|
||||
/** The parsed command from the first chunk (contains action, format, dimensions, etc.) */
|
||||
cmd: IKittyCommand;
|
||||
/** Accumulated base64 payload data */
|
||||
data: string;
|
||||
}
|
||||
import { KittyApcHandler } from './KittyApcHandler';
|
||||
|
||||
export class KittyGraphicsAddon implements ITerminalAddon, IKittyGraphicsApi {
|
||||
private _terminal: ITerminalExt | undefined;
|
||||
private _apcHandler: IDisposable | undefined;
|
||||
private _kittyApcHandler: KittyApcHandler | undefined;
|
||||
private _renderer: KittyImageRenderer | undefined;
|
||||
// Question: ImageAddon has ImageStorage, lot going on there though comapred to IKittyImage atm.
|
||||
// Maybe add more, rename to IKittyImageStorage instead of IKittyImage?
|
||||
private _images: Map<number, IKittyImage> = new Map();
|
||||
private _decodedImages: Map<number, ImageBitmap> = new Map();
|
||||
/**
|
||||
* Pending chunked transmissions keyed by image ID.
|
||||
* ID 0 is used for transmissions without an explicit ID (the "anonymous" transmission).
|
||||
*/
|
||||
private _pendingTransmissions: Map<number, IPendingTransmission> = new Map();
|
||||
private _nextImageId = 1;
|
||||
private _debug: boolean;
|
||||
|
||||
constructor(options?: IKittyGraphicsOptions) {
|
||||
@@ -61,18 +36,26 @@ export class KittyGraphicsAddon implements ITerminalAddon, IKittyGraphicsApi {
|
||||
console.log('[KittyGraphicsAddon] Registering APC handler for G (0x47)');
|
||||
}
|
||||
|
||||
this._kittyApcHandler = new KittyApcHandler(
|
||||
this._images,
|
||||
this._decodedImages,
|
||||
this._renderer,
|
||||
this._terminal,
|
||||
this._debug
|
||||
);
|
||||
|
||||
// Register APC handler for 'G' (0x47) - Kitty graphics protocol
|
||||
// APC sequence format: ESC _ G <data> ESC \
|
||||
this._apcHandler = terminal.parser.registerApcHandler(0x47, (data: string) => {
|
||||
return this._handleKittyGraphics(data);
|
||||
return this._kittyApcHandler!.handle(data);
|
||||
});
|
||||
}
|
||||
|
||||
public dispose(): void {
|
||||
this._apcHandler?.dispose();
|
||||
this._kittyApcHandler?.clearPendingTransmissions();
|
||||
this._renderer?.dispose();
|
||||
this._images.clear();
|
||||
this._pendingTransmissions.clear();
|
||||
|
||||
// Close all decoded bitmaps
|
||||
for (const bitmap of this._decodedImages.values()) {
|
||||
@@ -81,468 +64,6 @@ export class KittyGraphicsAddon implements ITerminalAddon, IKittyGraphicsApi {
|
||||
this._decodedImages.clear();
|
||||
|
||||
this._terminal = undefined;
|
||||
}
|
||||
|
||||
private _handleKittyGraphics(data: string): boolean {
|
||||
const semiIdx = data.indexOf(';');
|
||||
const controlData = semiIdx === -1 ? data : data.substring(0, semiIdx);
|
||||
const payload = semiIdx === -1 ? '' : data.substring(semiIdx + 1);
|
||||
|
||||
const cmd = parseKittyCommand(controlData);
|
||||
cmd.payload = payload;
|
||||
|
||||
if (this._debug) {
|
||||
console.log('[KittyGraphicsAddon] Received command:', cmd);
|
||||
}
|
||||
|
||||
const action = cmd.action ?? 't';
|
||||
|
||||
// Actions from: https://sw.kovidgoyal.net/kitty/graphics-protocol/#control-data-reference
|
||||
switch (action) {
|
||||
case KittyAction.TRANSMIT:
|
||||
return this._handleTransmit(cmd);
|
||||
case KittyAction.TRANSMIT_DISPLAY:
|
||||
return this._handleTransmitDisplay(cmd);
|
||||
case KittyAction.QUERY:
|
||||
return this._handleQuery(cmd);
|
||||
case KittyAction.DELETE:
|
||||
return this._handleDelete(cmd);
|
||||
default:
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
private _handleTransmit(cmd: IKittyCommand): boolean {
|
||||
const payload = cmd.payload ?? '';
|
||||
// Use 0 as the key for anonymous transmissions (no explicit i=)
|
||||
const pendingKey = cmd.id ?? 0;
|
||||
|
||||
// Check if this is a continuation of a chunked transmission
|
||||
const isMoreComing = cmd.more === 1;
|
||||
const pending = this._pendingTransmissions.get(pendingKey);
|
||||
|
||||
if (pending) {
|
||||
// This is a continuation chunk - append data
|
||||
pending.data += payload;
|
||||
|
||||
if (this._debug) {
|
||||
console.log(`[KittyGraphicsAddon] Chunk continuation for id=${pendingKey}, total size=${pending.data.length}, more=${isMoreComing}`);
|
||||
}
|
||||
|
||||
if (isMoreComing) {
|
||||
// More chunks coming, keep waiting
|
||||
return true;
|
||||
}
|
||||
|
||||
// Final chunk - use the stored command's metadata
|
||||
const originalCmd = pending.cmd;
|
||||
const fullPayload = pending.data;
|
||||
this._pendingTransmissions.delete(pendingKey);
|
||||
|
||||
// Assign a real ID if this was anonymous (pendingKey === 0)
|
||||
const id = originalCmd.id ?? this._nextImageId++;
|
||||
|
||||
const image: IKittyImage = {
|
||||
id,
|
||||
data: fullPayload,
|
||||
width: originalCmd.width ?? 0,
|
||||
height: originalCmd.height ?? 0,
|
||||
format: (originalCmd.format ?? KittyFormat.PNG) as 24 | 32 | 100,
|
||||
compression: originalCmd.compression ?? ''
|
||||
};
|
||||
|
||||
this._images.set(id, image);
|
||||
|
||||
if (this._debug) {
|
||||
console.log(`[KittyGraphicsAddon] Stored chunked image ${id}, payload size=${fullPayload.length}`);
|
||||
}
|
||||
|
||||
// If the original action was 'T' (transmit+display), display the image now
|
||||
if (originalCmd.action === KittyAction.TRANSMIT_DISPLAY) {
|
||||
this._decodeAndDisplay(image, originalCmd.columns, originalCmd.rows).catch(err => {
|
||||
if (this._debug) {
|
||||
console.error(`[KittyGraphicsAddon] Failed to decode/display chunked image ${id}:`, err);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
cmd.id = id;
|
||||
return true;
|
||||
}
|
||||
|
||||
// This is the first chunk (or a non-chunked transmission)
|
||||
if (isMoreComing) {
|
||||
// First chunk of a multi-chunk transmission - store the command and start accumulating
|
||||
this._pendingTransmissions.set(pendingKey, {
|
||||
cmd: { ...cmd }, // Clone the command to preserve metadata
|
||||
data: payload
|
||||
});
|
||||
|
||||
if (this._debug) {
|
||||
console.log(`[KittyGraphicsAddon] Started chunked transmission, id=${pendingKey}, format=${cmd.format}, compression=${cmd.compression}, size=${cmd.width}x${cmd.height}, initial payload=${payload.length}`);
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
// Non-chunked transmission - store immediately
|
||||
const id = cmd.id ?? this._nextImageId++;
|
||||
|
||||
const image: IKittyImage = {
|
||||
id,
|
||||
data: payload,
|
||||
width: cmd.width ?? 0,
|
||||
height: cmd.height ?? 0,
|
||||
format: (cmd.format ?? KittyFormat.PNG) as 24 | 32 | 100,
|
||||
compression: cmd.compression ?? ''
|
||||
};
|
||||
|
||||
this._images.set(id, image);
|
||||
|
||||
if (this._debug) {
|
||||
console.log(`[KittyGraphicsAddon] Stored image ${id}`);
|
||||
}
|
||||
|
||||
// Update cmd.id for _handleTransmitDisplay
|
||||
cmd.id = id;
|
||||
return true;
|
||||
}
|
||||
|
||||
private _handleTransmitDisplay(cmd: IKittyCommand): boolean {
|
||||
const pendingKey = cmd.id ?? 0;
|
||||
const wasPendingBefore = this._pendingTransmissions.has(pendingKey);
|
||||
|
||||
// Delegate to _handleTransmit which will:
|
||||
// 1. Accumulate chunks if m=1
|
||||
// 2. Store the image when complete
|
||||
// 3. Trigger display when chunked transmission completes (checks original action)
|
||||
this._handleTransmit(cmd);
|
||||
|
||||
// If this was a chunked transmission (first chunk with m=1), _handleTransmit
|
||||
// stored it as pending and will display when complete. Don't display now.
|
||||
if (cmd.more === 1) {
|
||||
return true;
|
||||
}
|
||||
|
||||
// If there was a pending transmission that just completed, _handleTransmit
|
||||
// already triggered the display. Don't display again.
|
||||
if (wasPendingBefore) {
|
||||
return true;
|
||||
}
|
||||
|
||||
// For non-chunked transmission, display now
|
||||
const id = cmd.id!;
|
||||
const image = this._images.get(id);
|
||||
|
||||
if (!image) {
|
||||
if (this._debug) {
|
||||
console.log(`[KittyGraphicsAddon] No image found for id=${id} after transmit`);
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
// Decode and display with sizing
|
||||
this._decodeAndDisplay(image, cmd.columns, cmd.rows).catch(err => {
|
||||
if (this._debug) {
|
||||
console.error(`[KittyGraphicsAddon] Failed to decode/display image ${id}:`, err);
|
||||
}
|
||||
});
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* Decode base64 image data into an ImageBitmap.
|
||||
*/
|
||||
private async _decodeImage(image: IKittyImage): Promise<ImageBitmap> {
|
||||
const format = image.format;
|
||||
const base64Data = image.data as string;
|
||||
|
||||
// Decode base64 to binary
|
||||
const binaryString = atob(base64Data);
|
||||
let bytes = new Uint8Array(binaryString.length);
|
||||
for (let i = 0; i < binaryString.length; i++) {
|
||||
bytes[i] = binaryString.charCodeAt(i);
|
||||
}
|
||||
|
||||
// Decompress if needed (o=z means zlib/deflate compression)
|
||||
if (image.compression === KittyCompression.ZLIB) {
|
||||
bytes = await this._decompressZlib(bytes);
|
||||
if (this._debug) {
|
||||
console.log(`[KittyGraphicsAddon] Decompressed ${binaryString.length} -> ${bytes.length} bytes`);
|
||||
}
|
||||
}
|
||||
|
||||
if (format === KittyFormat.PNG) {
|
||||
// PNG: create blob and decode
|
||||
const blob = new Blob([bytes], { type: 'image/png' });
|
||||
return createImageBitmap(blob);
|
||||
}
|
||||
|
||||
// RGB (24) or RGBA (32): create ImageData
|
||||
const width = image.width;
|
||||
const height = image.height;
|
||||
|
||||
if (!width || !height) {
|
||||
throw new Error('Width and height required for raw pixel data');
|
||||
}
|
||||
|
||||
const bytesPerPixel = format === KittyFormat.RGBA ? BYTES_PER_PIXEL_RGBA : BYTES_PER_PIXEL_RGB;
|
||||
const expectedBytes = width * height * bytesPerPixel;
|
||||
|
||||
if (bytes.length < expectedBytes) {
|
||||
throw new Error(`Insufficient pixel data: got ${bytes.length}, expected ${expectedBytes}`);
|
||||
}
|
||||
|
||||
// Convert to RGBA ImageData
|
||||
// TODO: Get this checked by Daniel.
|
||||
const pixelCount = width * height;
|
||||
const data = new Uint8ClampedArray(pixelCount * BYTES_PER_PIXEL_RGBA);
|
||||
const isRgba = format === KittyFormat.RGBA;
|
||||
|
||||
let srcOffset = 0;
|
||||
let dstOffset = 0;
|
||||
for (let i = 0; i < pixelCount; i++) {
|
||||
data[dstOffset ] = bytes[srcOffset ]; // R
|
||||
data[dstOffset + 1] = bytes[srcOffset + 1]; // G
|
||||
data[dstOffset + 2] = bytes[srcOffset + 2]; // B
|
||||
data[dstOffset + 3] = isRgba ? bytes[srcOffset + 3] : ALPHA_OPAQUE;
|
||||
srcOffset += bytesPerPixel;
|
||||
dstOffset += BYTES_PER_PIXEL_RGBA;
|
||||
}
|
||||
|
||||
return createImageBitmap(new ImageData(data, width, height));
|
||||
}
|
||||
|
||||
/**
|
||||
* Decompress zlib/deflate compressed data using the browser's DecompressionStream API.
|
||||
*/
|
||||
private async _decompressZlib(compressed: Uint8Array): Promise<Uint8Array> {
|
||||
// Use DecompressionStream API (available in modern browsers)
|
||||
// 'deflate-raw' is the format used by zlib without headers
|
||||
// Try 'deflate' first (zlib with header), fall back to 'deflate-raw' if that fails
|
||||
try {
|
||||
return await this._decompress(compressed, 'deflate');
|
||||
} catch {
|
||||
// If 'deflate' fails, try 'deflate-raw'
|
||||
return await this._decompress(compressed, 'deflate-raw');
|
||||
}
|
||||
}
|
||||
|
||||
private async _decompress(compressed: Uint8Array, format: 'deflate' | 'deflate-raw'): Promise<Uint8Array> {
|
||||
const ds = new DecompressionStream(format);
|
||||
const writer = ds.writable.getWriter();
|
||||
writer.write(compressed);
|
||||
writer.close();
|
||||
|
||||
const chunks: Uint8Array[] = [];
|
||||
const reader = ds.readable.getReader();
|
||||
|
||||
while (true) {
|
||||
const { done, value } = await reader.read();
|
||||
if (done) break;
|
||||
chunks.push(value);
|
||||
}
|
||||
|
||||
// Combine chunks into single Uint8Array
|
||||
const totalLength = chunks.reduce((sum, chunk) => sum + chunk.length, 0);
|
||||
const result = new Uint8Array(totalLength);
|
||||
let offset = 0;
|
||||
for (const chunk of chunks) {
|
||||
result.set(chunk, offset);
|
||||
offset += chunk.length;
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Decode an image and display it at the cursor position.
|
||||
* @param image - The Kitty image to decode and display
|
||||
* @param columns - Optional: number of terminal columns to span
|
||||
* @param rows - Optional: number of terminal rows to span
|
||||
*/
|
||||
private async _decodeAndDisplay(image: IKittyImage, columns?: number, rows?: number): Promise<void> {
|
||||
if (!this._renderer) {
|
||||
return;
|
||||
}
|
||||
|
||||
// Check if already decoded
|
||||
let bitmap = this._decodedImages.get(image.id);
|
||||
|
||||
if (!bitmap) {
|
||||
bitmap = await this._decodeImage(image);
|
||||
this._decodedImages.set(image.id, bitmap);
|
||||
|
||||
if (this._debug) {
|
||||
console.log(`[KittyGraphicsAddon] Decoded image ${image.id}: ${bitmap.width}x${bitmap.height}`);
|
||||
}
|
||||
}
|
||||
|
||||
// Calculate pixel dimensions from columns/rows if specified
|
||||
let width = 0;
|
||||
let height = 0;
|
||||
if (columns || rows) {
|
||||
const cellSize = this._renderer.getCellSize();
|
||||
if (columns) {
|
||||
width = columns * cellSize.width;
|
||||
}
|
||||
if (rows) {
|
||||
height = rows * cellSize.height;
|
||||
}
|
||||
// If only one dimension specified, maintain aspect ratio
|
||||
if (width && !height) {
|
||||
height = Math.round(width * (bitmap.height / bitmap.width));
|
||||
} else if (height && !width) {
|
||||
width = Math.round(height * (bitmap.width / bitmap.height));
|
||||
}
|
||||
}
|
||||
|
||||
// Place at cursor position
|
||||
this._renderer.placeImage(bitmap, image.id, undefined, undefined, width, height);
|
||||
|
||||
if (this._debug) {
|
||||
console.log(`[KittyGraphicsAddon] Placed image ${image.id} at cursor, size: ${width || bitmap.width}x${height || bitmap.height}`);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Send a response back to the client via the terminal's data event.
|
||||
* Per the Kitty protocol, responses are sent when an image id (i=) is specified.
|
||||
* Format: ESC _ G i=<id>;message ESC \
|
||||
*
|
||||
* The quiet flag (q=) can suppress responses:
|
||||
* - q=1: suppress OK responses
|
||||
* - q=2: suppress error responses
|
||||
*
|
||||
* @param id - The image ID to include in the response
|
||||
* @param message - The message (e.g., 'OK' or 'EINVAL:error description')
|
||||
* @param quiet - The quiet flag value (0, 1, or 2)
|
||||
*/
|
||||
private _sendResponse(id: number, message: string, quiet: number): void {
|
||||
// Check quiet flag: q=1 suppresses OK, q=2 suppresses errors
|
||||
const isOk = message === 'OK';
|
||||
if (isOk && quiet === 1) {
|
||||
return;
|
||||
}
|
||||
if (!isOk && quiet === 2) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (!this._terminal) {
|
||||
return;
|
||||
}
|
||||
|
||||
const response = `\x1b_Gi=${id};${message}\x1b\\`;
|
||||
this._terminal.input(response, false);
|
||||
|
||||
if (this._debug) {
|
||||
console.log(`[KittyGraphicsAddon] Sent response: i=${id};${message}`);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Handle query action (a=q).
|
||||
*
|
||||
* Per the Kitty graphics protocol documentation:
|
||||
* "Sometimes, using an id is not appropriate... In that case, you can use the
|
||||
* query action, set a=q. Then the terminal emulator will try to load the image
|
||||
* and respond with either OK or an error, as above, but it will not replace an
|
||||
* existing image with the same id, nor will it store the image."
|
||||
*
|
||||
* The query is used by clients to check if the terminal supports the graphics
|
||||
* protocol. A typical query looks like:
|
||||
* ESC _ G i=31,s=1,v=1,a=q,t=d,f=24;AAAA ESC \ ESC [c
|
||||
*
|
||||
* If the terminal supports graphics, it responds with:
|
||||
* ESC _ G i=31;OK ESC \
|
||||
*
|
||||
* @param cmd - The parsed Kitty command
|
||||
*/
|
||||
private _handleQuery(cmd: IKittyCommand): boolean {
|
||||
const id = cmd.id ?? 0;
|
||||
const quiet = cmd.quiet ?? 0;
|
||||
|
||||
if (this._debug) {
|
||||
console.log(`[KittyGraphicsAddon] Query received, id=${id}, quiet=${quiet}`);
|
||||
}
|
||||
|
||||
// Try to validate the image data without storing it
|
||||
const payload = cmd.payload || '';
|
||||
|
||||
if (!payload) {
|
||||
// No payload - just checking if graphics protocol is supported
|
||||
// Respond with OK to indicate support
|
||||
this._sendResponse(id, 'OK', quiet);
|
||||
return true;
|
||||
}
|
||||
|
||||
// Validate the image data can be decoded
|
||||
try {
|
||||
// Decode base64 to verify it's valid
|
||||
const binaryString = atob(payload);
|
||||
const bytes = new Uint8Array(binaryString.length);
|
||||
for (let i = 0; i < binaryString.length; i++) {
|
||||
bytes[i] = binaryString.charCodeAt(i);
|
||||
}
|
||||
|
||||
const format = cmd.format || KittyFormat.RGBA;
|
||||
|
||||
if (format === KittyFormat.PNG) {
|
||||
// For PNG, we just verify base64 decoded successfully
|
||||
// Full PNG validation would require async createImageBitmap
|
||||
// which we can't do synchronously, so we trust the data is valid PNG
|
||||
this._sendResponse(id, 'OK', quiet);
|
||||
} else {
|
||||
// RGB (24) or RGBA (32): verify we have enough bytes
|
||||
const width = cmd.width || 0;
|
||||
const height = cmd.height || 0;
|
||||
|
||||
if (!width || !height) {
|
||||
this._sendResponse(id, 'EINVAL:width and height required for raw pixel data', quiet);
|
||||
return true;
|
||||
}
|
||||
|
||||
const bytesPerPixel = format === KittyFormat.RGBA ? BYTES_PER_PIXEL_RGBA : BYTES_PER_PIXEL_RGB;
|
||||
const expectedBytes = width * height * bytesPerPixel;
|
||||
|
||||
if (bytes.length < expectedBytes) {
|
||||
this._sendResponse(id, `EINVAL:insufficient pixel data, got ${bytes.length}, expected ${expectedBytes}`, quiet);
|
||||
return true;
|
||||
}
|
||||
|
||||
this._sendResponse(id, 'OK', quiet);
|
||||
}
|
||||
} catch (e) {
|
||||
// Base64 decode failed or other error
|
||||
const errorMsg = e instanceof Error ? e.message : 'unknown error';
|
||||
this._sendResponse(id, `EINVAL:${errorMsg}`, quiet);
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
private _handleDelete(cmd: IKittyCommand): boolean {
|
||||
if (cmd.id !== undefined) {
|
||||
this._images.delete(cmd.id);
|
||||
// Close and remove decoded bitmap
|
||||
const bitmap = this._decodedImages.get(cmd.id);
|
||||
if (bitmap) {
|
||||
bitmap.close();
|
||||
this._decodedImages.delete(cmd.id);
|
||||
}
|
||||
// Remove from renderer
|
||||
this._renderer?.removeByImageId(cmd.id);
|
||||
} else {
|
||||
this._images.clear();
|
||||
// Close all decoded bitmaps
|
||||
for (const bitmap of this._decodedImages.values()) {
|
||||
bitmap.close();
|
||||
}
|
||||
this._decodedImages.clear();
|
||||
// Clear all from renderer
|
||||
this._renderer?.clearAll();
|
||||
}
|
||||
return true;
|
||||
this._kittyApcHandler = undefined;
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user