eiac/xterm.js
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

Use sharding, decode with streaming

Browse Source
This commit is contained in:
Anthony Kim
2026-02-01 12:37:09 -08:00
parent bae314f67c
commit e4710d3286
3 changed files with 440 additions and 274 deletions
Download Patch File Download Diff File Expand all files Collapse all files
addons/addon-image/src/kitty/KittyGraphicsHandler.ts
+426 -268
View File
@@ -1,9 +1,6 @@
/**
* Copyright (c) 2025 The xterm.js authors. All rights reserved.
* @license MIT
*
* Kitty graphics protocol APC handler.
* Implements IApcHandler to integrate with xterm.js parser.
*/
import { IApcHandler, IImageAddonOptions, IResetHandler, ITerminalExt } from '../Types';
@@ -26,41 +23,53 @@ import {
// Memory limit for base64 decoder (4MB, same as IIPHandler)
const DECODER_KEEP_DATA = 4194304;
// Base64 shard size for chunked decoding (1MB encoded → ~768KB decoded)
// This prevents memory pressure with large images
// Base64 shard size (~1MB, must be divisible by 4)
const BASE64_SHARD_SIZE = 1048576;
// Decoded shard size: 1MB base64 → 768KB decoded
const DECODED_SHARD_SIZE = 786432;
// Maximum control data size
const MAX_CONTROL_DATA_SIZE = 4096;
// Semicolon codepoint
const SEMICOLON = 0x3B;
// Padding character '='
const EQUALS = 0x3D;
/**
* Kitty graphics protocol handler.
*
* Handles APC sequences for the Kitty graphics protocol, supporting:
* - Image transmission (a=t)
* - Image transmission + display (a=T)
* - Query (a=q)
* - Delete (a=d)
*
* TODO: File transmission (t=f) is not supported since browsers cannot access the filesystem.
* Explore using the File System Access API for opt-in filesystem access when available.
* Kitty graphics protocol handler with streaming base64 decoding.
*/
export class KittyGraphicsHandler implements IApcHandler, IResetHandler {
private _aborted = false;
private _data: string = '';
private _decodeError = false;
/** Reusable base64 decoder - avoids WASM cold-start on each chunk */
private _decoder = new Base64Decoder(DECODER_KEEP_DATA);
// Streaming related states
/** True while receiving control data (before semicolon). */
private _inControlData = true;
/** Buffer for control data. */
private _controlData = new Uint32Array(MAX_CONTROL_DATA_SIZE);
private _controlLength = 0;
/** Shard buffer for base64 bytes (filled until 4-byte aligned boundary). */
private _shardBuffer = new Uint32Array(BASE64_SHARD_SIZE);
private _shardBufferPos = 0;
/** Accumulated decoded chunks. */
private _decodedChunks: Uint8Array[] = [];
private _totalDecodedSize = 0;
// Storage related states
/**
* Pending chunked transmissions keyed by image ID.
* ID 0 is used for transmissions without an explicit ID (the "anonymous" transmission).
*/
private _pendingTransmissions: Map<number, IPendingTransmission> = new Map();
private _nextImageId = 1;
/**
* Stored images - kept for protocol compliance (transmit without display, then display later).
*/
private _images: Map<number, IKittyImageData> = new Map();
/**
* Cached decoded bitmaps.
*/
private _decodedImages: Map<number, ImageBitmap> = new Map();
constructor(
@@ -71,217 +80,473 @@ export class KittyGraphicsHandler implements IApcHandler, IResetHandler {
) {}
public reset(): void {
// Clear pending transmissions
this._pendingTransmissions.clear();
// Clear stored images
this._images.clear();
// Close decoded bitmaps
for (const bitmap of this._decodedImages.values()) {
bitmap.close();
}
this._decodedImages.clear();
}
/**
* Called at the start of a new APC sequence.
*/
public start(): void {
this._aborted = false;
this._data = '';
this._decodeError = false;
this._inControlData = true;
this._controlLength = 0;
this._shardBufferPos = 0;
this._decodedChunks = [];
this._totalDecodedSize = 0;
}
/**
* Called for each chunk of data in the APC sequence.
* Accumulates codepoints as a string for control data parsing.
* The base64 payload is decoded later using the wasm decoder.
*/
public put(data: Uint32Array, start: number, end: number): void {
if (this._aborted) return;
if (this._inControlData) {
// Scan for semicolon
let controlEnd = end;
for (let i = start; i < end; i++) {
if (data[i] === SEMICOLON) {
this._inControlData = false;
controlEnd = i;
break;
}
}
// Copy control data
const copyLength = controlEnd - start;
if (this._controlLength + copyLength > MAX_CONTROL_DATA_SIZE) {
this._aborted = true;
return;
}
this._controlData.set(data.subarray(start, controlEnd), this._controlLength);
this._controlLength += copyLength;
if (!this._inControlData) {
// Found semicolon - stream remaining as payload
const payloadStart = controlEnd + 1;
if (payloadStart < end) {
this._streamPayload(data, payloadStart, end);
}
}
} else {
this._streamPayload(data, start, end);
}
}
/**
* Stream payload bytes, decode at 4-byte aligned boundaries.
*/
private _streamPayload(data: Uint32Array, start: number, end: number): void {
if (this._aborted) return;
// Check size limit
if (this._data.length + (end - start) > this._opts.kittySizeLimit) {
console.warn('[KittyGraphicsHandler] Data exceeds size limit, aborting');
const estimatedTotal = this._totalDecodedSize + Math.ceil(this._shardBufferPos * 3 / 4) + Math.ceil((end - start) * 3 / 4);
if (estimatedTotal > this._opts.kittySizeLimit) {
this._aborted = true;
return;
}
// Convert Uint32Array codepoints to string for control data parsing
// Note: We accumulate as string because we need to parse control data (before semicolon).
// The base64 payload is decoded efficiently using the wasm-based decoder in _decodeImage().
for (let i = start; i < end; i++) {
this._data += String.fromCodePoint(data[i]);
const inputLength = end - start;
const freeSpace = BASE64_SHARD_SIZE - this._shardBufferPos;
if (inputLength < freeSpace) {
// Fits in current shard
this._shardBuffer.set(data.subarray(start, end), this._shardBufferPos);
this._shardBufferPos += inputLength;
return;
}
// Fill shard to capacity
this._shardBuffer.set(data.subarray(start, start + freeSpace), this._shardBufferPos);
this._shardBufferPos = BASE64_SHARD_SIZE;
// Decode full shard (exactly 1MB → 768KB)
if (!this._decodeFullShard()) {
this._aborted = true;
return;
}
// Reset and process remaining
this._shardBufferPos = 0;
const remaining = start + freeSpace;
if (remaining < end) {
this._streamPayload(data, remaining, end);
}
}
/**
* Called at the end of the APC sequence.
* Decode a full 1MB shard (768KB output).
*/
private _decodeFullShard(): boolean {
this._decoder.init(DECODED_SHARD_SIZE);
if (this._decoder.put(this._shardBuffer, 0, BASE64_SHARD_SIZE)) {
this._decoder.release();
this._decodeError = true;
return false;
}
if (this._decoder.end()) {
this._decoder.release();
this._decodeError = true;
return false;
}
const chunk = new Uint8Array(this._decoder.data8);
this._decodedChunks.push(chunk);
this._totalDecodedSize += chunk.length;
this._decoder.release();
return true;
}
public end(success: boolean): boolean | Promise<boolean> {
if (this._aborted || !success) {
return true;
}
return this._handleCommand(this._data);
// No semicolon = no payload (delete, capability query)
if (this._inControlData) {
return this._handleNoPayloadCommand();
}
// Parse command to check m=1 and get pending key
const cmd = parseKittyCommand(this._parseControlDataString());
const pendingKey = cmd.id ?? 0;
const isMoreComing = cmd.more === 1;
const pending = this._pendingTransmissions.get(pendingKey);
// If continuing a pending transmission, prepend leftover bytes
if (pending && pending.leftoverLength > 0) {
// Shift current buffer to make room for leftover
const newPos = pending.leftoverLength + this._shardBufferPos;
if (newPos <= BASE64_SHARD_SIZE) {
// Shift existing data right
for (let i = this._shardBufferPos - 1; i >= 0; i--) {
this._shardBuffer[i + pending.leftoverLength] = this._shardBuffer[i];
}
// Prepend leftover
this._shardBuffer.set(pending.leftover.subarray(0, pending.leftoverLength), 0);
this._shardBufferPos = newPos;
}
pending.leftoverLength = 0;
}
// Decode with 4-byte alignment, preserving leftover if m=1
if (this._shardBufferPos > 0) {
if (isMoreComing) {
this._decodePartialShardWithLeftover(pendingKey, cmd);
} else {
this._decodePartialShard();
}
}
// Combine all decoded chunks
const imageBytes = this._combineChunks();
return this._handleCommandWithBytesAndCmd(cmd, imageBytes, this._decodeError);
}
/**
* Handle a complete Kitty graphics command.
* Decode partial shard, storing leftover bytes for m=1 chunks.
*/
private _handleCommand(data: string): boolean | Promise<boolean> {
const semiIdx = data.indexOf(';');
const controlData = semiIdx === -1 ? data : data.substring(0, semiIdx);
const payload = semiIdx === -1 ? '' : data.substring(semiIdx + 1);
private _decodePartialShardWithLeftover(pendingKey: number, cmd: IKittyCommand): boolean {
let length = this._shardBufferPos;
if (length === 0) return true;
const cmd = parseKittyCommand(controlData);
cmd.payload = payload;
// Align to 4 bytes
const aligned = Math.floor(length / 4) * 4;
const leftoverCount = length - aligned;
// Store leftover bytes for next chunk
if (leftoverCount > 0) {
let pending = this._pendingTransmissions.get(pendingKey);
if (!pending) {
pending = {
cmd: { ...cmd },
chunks: [],
totalSize: 0,
leftover: new Uint32Array(4),
leftoverLength: 0
};
this._pendingTransmissions.set(pendingKey, pending);
}
pending.leftover.set(this._shardBuffer.subarray(aligned, length), 0);
pending.leftoverLength = leftoverCount;
}
if (aligned === 0) return true;
// Count padding (shouldn't have padding in m=1 chunks, but be safe)
let padding = 0;
if (aligned >= 1 && this._shardBuffer[aligned - 1] === EQUALS) padding++;
if (aligned >= 2 && this._shardBuffer[aligned - 2] === EQUALS) padding++;
const decodedSize = Math.floor(aligned * 3 / 4) - padding;
if (decodedSize <= 0) return true;
this._decoder.init(decodedSize);
if (this._decoder.put(this._shardBuffer, 0, aligned)) {
this._decoder.release();
this._decodeError = true;
return false;
}
if (this._decoder.end()) {
this._decoder.release();
this._decodeError = true;
return false;
}
const chunk = new Uint8Array(this._decoder.data8);
this._decodedChunks.push(chunk);
this._totalDecodedSize += chunk.length;
this._decoder.release();
return true;
}
/**
* Decode partial shard with proper 4-byte alignment.
* For final chunks, pad with = if not aligned instead of truncating.
*/
private _decodePartialShard(): boolean {
let length = this._shardBufferPos;
if (length === 0) return true;
// Count ORIGINAL padding BEFORE we add any
let padding = 0;
if (length >= 1 && this._shardBuffer[length - 1] === EQUALS) padding++;
if (length >= 2 && this._shardBuffer[length - 2] === EQUALS) padding++;
// Pad to 4-byte boundary with = if needed (instead of truncating)
const remainder = length % 4;
if (remainder > 0) {
const paddingNeeded = 4 - remainder;
for (let i = 0; i < paddingNeeded; i++) {
this._shardBuffer[length + i] = EQUALS;
}
length += paddingNeeded;
// Count the artificial padding too!
padding += paddingNeeded;
}
// Calculate decoded size
const decodedSize = Math.floor(length * 3 / 4) - padding;
if (decodedSize <= 0) return true;
this._decoder.init(decodedSize);
if (this._decoder.put(this._shardBuffer, 0, length)) {
this._decoder.release();
this._decodeError = true;
return false;
}
if (this._decoder.end()) {
this._decoder.release();
this._decodeError = true;
return false;
}
const chunk = new Uint8Array(this._decoder.data8);
this._decodedChunks.push(chunk);
this._totalDecodedSize += chunk.length;
this._decoder.release();
return true;
}
private _combineChunks(): Uint8Array {
if (this._decodedChunks.length === 0) return new Uint8Array(0);
if (this._decodedChunks.length === 1) return this._decodedChunks[0];
const result = new Uint8Array(this._totalDecodedSize);
let offset = 0;
for (const chunk of this._decodedChunks) {
result.set(chunk, offset);
offset += chunk.length;
}
return result;
}
// Command handling
private _parseControlDataString(): string {
let str = '';
for (let i = 0; i < this._controlLength; i++) {
str += String.fromCodePoint(this._controlData[i]);
}
return str;
}
private _handleNoPayloadCommand(): boolean | Promise<boolean> {
const cmd = parseKittyCommand(this._parseControlDataString());
const action = cmd.action ?? 't';
switch (action) {
case KittyAction.TRANSMIT:
return this._handleTransmit(cmd);
case KittyAction.TRANSMIT_DISPLAY:
return this._handleTransmitDisplay(cmd);
case KittyAction.QUERY:
return this._handleQuery(cmd);
case KittyAction.DELETE:
return this._handleDelete(cmd);
case KittyAction.QUERY:
this._sendResponse(cmd.id ?? 0, 'OK', cmd.quiet ?? 0);
return true;
default:
return true;
}
}
private _handleTransmit(cmd: IKittyCommand): boolean {
const payload = cmd.payload ?? '';
private _handleCommandWithBytesAndCmd(cmd: IKittyCommand, bytes: Uint8Array, decodeError: boolean): boolean | Promise<boolean> {
const action = cmd.action ?? 't';
switch (action) {
case KittyAction.TRANSMIT:
return this._handleTransmit(cmd, bytes, decodeError);
case KittyAction.TRANSMIT_DISPLAY:
return this._handleTransmitDisplay(cmd, bytes, decodeError);
case KittyAction.QUERY:
return this._handleQuery(cmd, bytes, decodeError);
default:
return true;
}
}
private _handleTransmit(cmd: IKittyCommand, bytes: Uint8Array, decodeError: boolean): boolean {
const pendingKey = cmd.id ?? 0;
const isMoreComing = cmd.more === 1;
const pending = this._pendingTransmissions.get(pendingKey);
if (pending) {
pending.data += payload;
if (isMoreComing) {
return true;
// For m=1 intermediate chunks, reject if decode error
if (decodeError && !pending && isMoreComing) return true;
if (pending) {
// Append to existing pending transmission (even if decode error, we keep accumulated data)
if (bytes.length > 0) {
pending.chunks.push(bytes);
pending.totalSize += bytes.length;
}
const originalCmd = pending.cmd;
const fullPayload = pending.data;
if (isMoreComing) return true;
// Final chunk - merge all (even if the final chunk had a decode error)
const fullData = this._mergeChunks(pending.chunks, pending.totalSize);
this._pendingTransmissions.delete(pendingKey);
const id = originalCmd.id ?? this._nextImageId++;
if (fullData.length === 0) return true; // Nothing to store
const id = pending.cmd.id ?? this._nextImageId++;
const image: IKittyImageData = {
id,
data: fullPayload,
width: originalCmd.width ?? 0,
height: originalCmd.height ?? 0,
format: (originalCmd.format ?? KittyFormat.PNG) as 24 | 32 | 100,
compression: originalCmd.compression ?? ''
data: fullData,
width: pending.cmd.width ?? 0,
height: pending.cmd.height ?? 0,
format: (pending.cmd.format ?? KittyFormat.PNG) as 24 | 32 | 100,
compression: pending.cmd.compression ?? ''
};
this._images.set(id, image);
this._images.set(image.id, image);
if (originalCmd.action === KittyAction.TRANSMIT_DISPLAY) {
// Fire-and-forget the display, transmit itself succeeded
void this._displayImage(image, originalCmd.columns, originalCmd.rows);
return true;
}
cmd.id = id;
// Note: Don't display here - _handleTransmitDisplay will do it
return true;
}
if (isMoreComing) {
// Start new pending transmission (reject if decode error for first chunk)
if (decodeError) return true;
this._pendingTransmissions.set(pendingKey, {
cmd: { ...cmd },
data: payload
chunks: bytes.length > 0 ? [bytes] : [],
totalSize: bytes.length,
leftover: new Uint32Array(4),
leftoverLength: 0
});
return true;
}
const id = cmd.id ?? this._nextImageId++;
// Single-chunk transmission - reject if decode error or no data
if (decodeError || bytes.length === 0) return true;
const id = cmd.id ?? this._nextImageId++;
const image: IKittyImageData = {
id,
data: payload,
data: bytes,
width: cmd.width ?? 0,
height: cmd.height ?? 0,
format: (cmd.format ?? KittyFormat.PNG) as 24 | 32 | 100,
compression: cmd.compression ?? ''
};
this._images.set(image.id, image);
cmd.id = id;
this._images.set(id, image);
return true;
}
private _handleTransmitDisplay(cmd: IKittyCommand): boolean | Promise<boolean> {
private _mergeChunks(chunks: Uint8Array[], totalSize: number): Uint8Array {
if (chunks.length === 1) return chunks[0];
const result = new Uint8Array(totalSize);
let offset = 0;
for (const chunk of chunks) {
result.set(chunk, offset);
offset += chunk.length;
}
return result;
}
private _handleTransmitDisplay(cmd: IKittyCommand, bytes: Uint8Array, decodeError: boolean): boolean | Promise<boolean> {
if (decodeError) return true;
const pendingKey = cmd.id ?? 0;
const wasPendingBefore = this._pendingTransmissions.has(pendingKey);
this._handleTransmit(cmd);
this._handleTransmit(cmd, bytes, decodeError);
if (cmd.more === 1) {
return true;
}
// If still accumulating chunks, don't display yet
if (this._pendingTransmissions.has(pendingKey)) return true;
if (wasPendingBefore) {
return true;
}
const id = cmd.id!;
// Display the completed image
const id = cmd.id ?? this._nextImageId - 1;
const image = this._images.get(id);
if (image) {
return this._displayImage(image, cmd.columns, cmd.rows);
}
return true;
}
/**
* Handle query action (a=q).
*/
private _handleQuery(cmd: IKittyCommand): boolean {
private _handleQuery(cmd: IKittyCommand, bytes: Uint8Array, decodeError: boolean): boolean {
const id = cmd.id ?? 0;
const quiet = cmd.quiet ?? 0;
const payload = cmd.payload || '';
if (!payload) {
// Check decode error first (invalid base64)
if (decodeError) {
this._sendResponse(id, 'EINVAL:invalid base64 data', quiet);
return true;
}
// Capability query (no payload) - just respond OK
if (bytes.length === 0) {
this._sendResponse(id, 'OK', quiet);
return true;
}
try {
const bytes = this._decodeBase64(payload);
const format = cmd.format ?? KittyFormat.RGBA;
const format = cmd.format || KittyFormat.RGBA;
if (format === KittyFormat.PNG) {
this._sendResponse(id, 'OK', quiet);
} else {
const width = cmd.width ?? 0;
const height = cmd.height ?? 0;
if (format === KittyFormat.PNG) {
this._sendResponse(id, 'OK', quiet);
} else {
const width = cmd.width || 0;
const height = cmd.height || 0;
if (!width || !height) {
this._sendResponse(id, 'EINVAL:width and height required for raw pixel data', quiet);
return true;
}
const bytesPerPixel = format === KittyFormat.RGBA ? BYTES_PER_PIXEL_RGBA : BYTES_PER_PIXEL_RGB;
const expectedBytes = width * height * bytesPerPixel;
if (bytes.length < expectedBytes) {
this._sendResponse(id, `EINVAL:insufficient pixel data, got ${bytes.length}, expected ${expectedBytes}`, quiet);
return true;
}
this._sendResponse(id, 'OK', quiet);
if (!width || !height) {
this._sendResponse(id, 'EINVAL:width and height required for raw pixel data', quiet);
return true;
}
} catch (e) {
const errorMsg = e instanceof Error ? e.message : 'unknown error';
this._sendResponse(id, `EINVAL:${errorMsg}`, quiet);
}
const bytesPerPixel = format === KittyFormat.RGBA ? BYTES_PER_PIXEL_RGBA : BYTES_PER_PIXEL_RGB;
const expectedBytes = width * height * bytesPerPixel;
if (bytes.length < expectedBytes) {
this._sendResponse(id, `EINVAL:insufficient pixel data`, quiet);
return true;
}
this._sendResponse(id, 'OK', quiet);
}
return true;
}
@@ -302,13 +567,9 @@ export class KittyGraphicsHandler implements IApcHandler, IResetHandler {
}
this._decodedImages.clear();
}
return true;
}
/**
* Send a response back to the client.
*/
private _sendResponse(id: number, message: string, quiet: number): void {
const isOk = message === 'OK';
if (isOk && quiet === 1) return;
@@ -318,27 +579,22 @@ export class KittyGraphicsHandler implements IApcHandler, IResetHandler {
this._coreTerminal._core.coreService.triggerDataEvent(response);
}
/**
* Decode and display an image using the shared ImageStorage.
*/
// Image display
private _displayImage(image: IKittyImageData, columns?: number, rows?: number): boolean | Promise<boolean> {
return this._decodeAndDisplay(image, columns, rows)
.then(() => true)
.catch(err => {
console.warn('[KittyHandler] Failed to decode/display image:', err);
return true;
});
.catch(() => true);
}
private async _decodeAndDisplay(image: IKittyImageData, columns?: number, rows?: number): Promise<void> {
let bitmap = this._decodedImages.get(image.id);
if (!bitmap) {
bitmap = await this._decodeImage(image);
bitmap = await this._createBitmap(image);
this._decodedImages.set(image.id, bitmap);
}
// Calculate display size
let w = bitmap.width;
let h = bitmap.height;
@@ -346,30 +602,16 @@ export class KittyGraphicsHandler implements IApcHandler, IResetHandler {
const cw = this._renderer.dimensions?.css.cell.width || CELL_SIZE_DEFAULT.width;
const ch = this._renderer.dimensions?.css.cell.height || CELL_SIZE_DEFAULT.height;
if (columns) {
w = columns * cw;
}
if (rows) {
h = rows * ch;
}
// Maintain aspect ratio if only one dimension specified
if (columns && !rows) {
h = Math.round(w * (bitmap.height / bitmap.width));
} else if (rows && !columns) {
w = Math.round(h * (bitmap.width / bitmap.height));
}
if (columns) w = columns * cw;
if (rows) h = rows * ch;
if (columns && !rows) h = Math.round(w * (bitmap.height / bitmap.width));
else if (rows && !columns) w = Math.round(h * (bitmap.width / bitmap.height));
}
// Check pixel limit
if (w * h > this._opts.pixelLimit) {
console.warn('[KittyHandler] Image exceeds pixel limit');
return;
}
if (w * h > this._opts.pixelLimit) return;
// Use shared ImageStorage to add the image
// This handles cursor movement and integration with the terminal
if (w !== bitmap.width || h !== bitmap.height) {
// Resize if needed
const resized = await createImageBitmap(bitmap, { resizeWidth: w, resizeHeight: h });
this._storage.addImage(resized);
} else {
@@ -378,23 +620,18 @@ export class KittyGraphicsHandler implements IApcHandler, IResetHandler {
}
/**
* Decode base64 image data into an ImageBitmap.
* Uses the wasm-based decoder from xterm-wasm-parts for better performance.
* Create ImageBitmap from already-decoded image data.
*/
private async _decodeImage(image: IKittyImageData): Promise<ImageBitmap> {
const format = image.format;
const base64Data = image.data;
private async _createBitmap(image: IKittyImageData): Promise<ImageBitmap> {
let bytes = image.data;
// TODO: Get this confirmed!
let bytes = this._decodeBase64(base64Data);
if (image.compression === KittyCompression.ZLIB) {
bytes = await this._decompressZlib(bytes) as Uint8Array<ArrayBuffer>;
bytes = await this._decompressZlib(bytes);
}
if (format === KittyFormat.PNG) {
const blob = new Blob([new Uint8Array(bytes)], { type: 'image/png' });
// Safari fallback pattern (from IIPHandler)
if (image.format === KittyFormat.PNG) {
const blob = new Blob([bytes as BlobPart], { type: 'image/png' });
if (!window.createImageBitmap) {
const url = URL.createObjectURL(blob);
const img = new Image();
@@ -412,7 +649,7 @@ export class KittyGraphicsHandler implements IApcHandler, IResetHandler {
return createImageBitmap(blob);
}
// Raw pixel data (RGB or RGBA)
// Raw pixel data
const width = image.width;
const height = image.height;
@@ -420,26 +657,23 @@ export class KittyGraphicsHandler implements IApcHandler, IResetHandler {
throw new Error('Width and height required for raw pixel data');
}
const bytesPerPixel = format === KittyFormat.RGBA ? BYTES_PER_PIXEL_RGBA : BYTES_PER_PIXEL_RGB;
const bytesPerPixel = image.format === KittyFormat.RGBA ? BYTES_PER_PIXEL_RGBA : BYTES_PER_PIXEL_RGB;
const expectedBytes = width * height * bytesPerPixel;
if (bytes.length < expectedBytes) {
throw new Error(`Insufficient pixel data: got ${bytes.length}, expected ${expectedBytes}`);
throw new Error('Insufficient pixel data');
}
// Convert to RGBA ImageData
// TODO: For RGBA, use bytes directly. For RGB, use Uint32Array bit manipulation
// for 5-6x speedup.
const pixelCount = width * height;
const data = new Uint8ClampedArray(pixelCount * BYTES_PER_PIXEL_RGBA);
const isRgba = format === KittyFormat.RGBA;
const isRgba = image.format === KittyFormat.RGBA;
let srcOffset = 0;
let dstOffset = 0;
for (let i = 0; i < pixelCount; i++) {
data[dstOffset ] = bytes[srcOffset ]; // R
data[dstOffset + 1] = bytes[srcOffset + 1]; // G
data[dstOffset + 2] = bytes[srcOffset + 2]; // B
data[dstOffset] = bytes[srcOffset];
data[dstOffset + 1] = bytes[srcOffset + 1];
data[dstOffset + 2] = bytes[srcOffset + 2];
data[dstOffset + 3] = isRgba ? bytes[srcOffset + 3] : ALPHA_OPAQUE;
srcOffset += bytesPerPixel;
dstOffset += BYTES_PER_PIXEL_RGBA;
@@ -448,13 +682,10 @@ export class KittyGraphicsHandler implements IApcHandler, IResetHandler {
return createImageBitmap(new ImageData(data, width, height));
}
/**
* Decompress zlib/deflate compressed data using the browser's DecompressionStream API.
*/
private async _decompressZlib(compressed: Uint8Array): Promise<Uint8Array> {
try {
return await this._decompress(compressed, 'deflate');
} catch {
} catch (e) {
return await this._decompress(compressed, 'deflate-raw');
}
}
@@ -462,7 +693,7 @@ export class KittyGraphicsHandler implements IApcHandler, IResetHandler {
private async _decompress(compressed: Uint8Array, format: 'deflate' | 'deflate-raw'): Promise<Uint8Array> {
const ds = new DecompressionStream(format);
const writer = ds.writable.getWriter();
writer.write(new Uint8Array(compressed) as Uint8Array<ArrayBuffer>);
writer.write(compressed as BufferSource);
writer.close();
const chunks: Uint8Array[] = [];
@@ -481,82 +712,9 @@ export class KittyGraphicsHandler implements IApcHandler, IResetHandler {
result.set(chunk, offset);
offset += chunk.length;
}
return result;
}
/**
* Decode base64 string using the wasm-based decoder for better performance.
* Uses shard-based decoding (1MB chunks) to prevent memory pressure with large images.
*/
private _decodeBase64(base64Data: string): Uint8Array {
const length = base64Data.length;
// For small data, decode directly without sharding
if (length <= BASE64_SHARD_SIZE) {
return this._decodeBase64Shard(base64Data);
}
// For large data, decode in 1MB shards and accumulate
// Base64 must be decoded in multiples of 4 chars, so align shard boundaries
const shardSize = BASE64_SHARD_SIZE - (BASE64_SHARD_SIZE % 4);
const chunks: Uint8Array[] = [];
let totalLength = 0;
for (let offset = 0; offset < length; offset += shardSize) {
const end = Math.min(offset + shardSize, length);
const shard = base64Data.substring(offset, end);
const decoded = this._decodeBase64Shard(shard);
chunks.push(decoded);
totalLength += decoded.length;
}
// Combine all chunks into final result
const result = new Uint8Array(totalLength);
let writeOffset = 0;
for (const chunk of chunks) {
result.set(chunk, writeOffset);
writeOffset += chunk.length;
}
return result;
}
/**
* Decode a single base64 shard using the wasm decoder.
*/
private _decodeBase64Shard(base64Data: string): Uint8Array {
// Calculate exact decoded size from base64 string
// Formula: (encoded_length * 3) / 4 - padding_count
let padding = 0;
if (base64Data.endsWith('==')) padding = 2;
else if (base64Data.endsWith('=')) padding = 1;
const decodedSize = Math.floor(base64Data.length * 3 / 4) - padding;
// Convert string to Uint32Array of codepoints for the wasm decoder
const codepoints = new Uint32Array(base64Data.length);
for (let i = 0; i < base64Data.length; i++) {
codepoints[i] = base64Data.charCodeAt(i);
}
const decoder = new Base64Decoder(DECODER_KEEP_DATA);
decoder.init(decodedSize);
decoder.put(codepoints, 0, codepoints.length);
if (decoder.end()) {
decoder.release();
throw new Error('Base64 decode failed');
}
// Copy the decoded data before releasing
const result = new Uint8Array(decoder.data8);
decoder.release();
return result;
}
/**
* Get stored images (for testing/debugging).
*/
public get images(): ReadonlyMap<number, IKittyImageData> {
return this._images;
}
addons/addon-image/src/kitty/KittyGraphicsTypes.ts
+11 -4
View File
@@ -93,13 +93,19 @@ export interface IKittyCommand {
/**
* Pending chunked transmission state.
* Stores metadata from the first chunk while accumulating payload data.
* Stores metadata from the first chunk while accumulating decoded payload data.
*/
export interface IPendingTransmission {
/** The parsed command from the first chunk (contains action, format, dimensions, etc.) */
cmd: IKittyCommand;
/** Accumulated base64 payload data */
data: string;
/** Accumulated decoded payload chunks */
chunks: Uint8Array[];
/** Total size of accumulated chunks */
totalSize: number;
/** Leftover base64 bytes (0-3) that weren't aligned in previous chunk */
leftover: Uint32Array;
/** Number of valid bytes in leftover */
leftoverLength: number;
}
/**
@@ -107,7 +113,8 @@ export interface IPendingTransmission {
*/
export interface IKittyImageData {
id: number;
data: string;
/** Decoded image bytes (already decoded from base64) */
data: Uint8Array;
width: number;
height: number;
format: 24 | 32 | 100;
addons/addon-image/test/ImageAddon.test.ts
+3 -2
View File
@@ -77,6 +77,7 @@ const TESTDATA_IIP: [string, [number, number]][] = [
// Kitty graphics test images
const KITTY_BLACK_1X1_BASE64 = readFileSync('./addons/addon-image/fixture/kitty/black-1x1.png').toString('base64');
const KITTY_BLACK_1X1_BYTES = Array.from(readFileSync('./addons/addon-image/fixture/kitty/black-1x1.png'));
const KITTY_RGB_3X1_BASE64 = readFileSync('./addons/addon-image/fixture/kitty/rgb-3x1.png').toString('base64');
let ctx: ITestContext;
@@ -386,8 +387,8 @@ test.describe('ImageAddon', () => {
await ctx.proxy.write(`\x1b_Ga=t,f=100,i=99;${part2}\x1b\\`);
await timeout(100);
const storedData = await ctx.page.evaluate(`window.imageAddon._handlers.get('kitty').images.get(99).data`);
strictEqual(storedData, KITTY_BLACK_1X1_BASE64);
const storedData = await ctx.page.evaluate(`Array.from(window.imageAddon._handlers.get('kitty').images.get(99).data)`);
deepStrictEqual(storedData, KITTY_BLACK_1X1_BYTES);
});
test('delete command (a=d) removes specific image by id', async () => {