Support Kitty graphics protocol

addons/addon-kitty-graphics/src/KittyGraphicsAddon.test.ts

@@ -0,0 +1,156 @@
+/**
+ * Copyright (c) 2025 The xterm.js authors. All rights reserved.
+ * @license MIT
+ */
+import { assert } from 'chai';
+import { Terminal } from 'browser/public/Terminal';
+import { KittyGraphicsAddon, parseKittyCommand } from './KittyGraphicsAddon';
+
+/**
+ * Write data to terminal and wait for completion.
+ */
+function writeP(terminal: Terminal, data: string | Uint8Array): Promise<void> {
+  return new Promise(r => terminal.write(data, r));
+}
+
+// Test image: 1x1 black PNG (captured from `send-png fixture/black-1x1.png`)
+// Get the below base64-encoded PNG file by: `python3 send-png addons/addon-kitty-graphics/fixture/black-1x1.png`
+const BLACK_1X1_BASE64 = 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAMAAAAoyzS7AAAAA1BMVEUAAACnej3aAAAACklEQVR4nGNgAAAAAgABSK+kcQAAAAt0RVh0Q29tbWVudAAA1LTqjgAAAApJREFUeJxjYAAAAGQA2AAAAAt0RVh0Q29tbWVudAAA1LTqjg5JREFUAAAAASUVORK5CYII=';
+
+// Test image: 3x1 RGB PNG (red, green, blue pixels)
+const RGB_3X1_BASE64 = 'iVBORw0KGgoAAAANSUhEUgAAAAMAAAABCAMAAAAsPuSGAAAACVBMVEX/AAAA/wAAAP8tSs2KAAAADElEQVR4nGNgYGQCAAAIAAQ24LCmAAAAHXRFWHRTb2Z0d2FyZQBAbHVuYXBhaW50L3BuZy1jb2RlY/VDGR4AAAAASUVORK5CYII=';
+
+// Currently tests the flow: write escape sequence -> addon stores image.
+// Pixel-level verification of rendered images is done in Playwright tests.
+describe('KittyGraphicsAddon', () => {
+  let terminal: Terminal;
+  let addon: KittyGraphicsAddon;
+
+  beforeEach(() => {
+    terminal = new Terminal({ cols: 80, rows: 24, allowProposedApi: true });
+    addon = new KittyGraphicsAddon({ debug: false });
+    terminal.loadAddon(addon);
+  });
+
+  describe('parseKittyCommand', () => {
+    it('should parse control data with action and format', () => {
+      const cmd = parseKittyCommand('a=T,f=100');
+      assert.equal(cmd.action, 'T');
+      assert.equal(cmd.format, 100);
+    });
+
+    it('should parse control data with all options', () => {
+      const cmd = parseKittyCommand('a=t,f=32,i=5,s=10,v=20,c=3,r=2,m=1,q=2');
+      assert.equal(cmd.action, 't');
+      assert.equal(cmd.format, 32);
+      assert.equal(cmd.id, 5);
+      assert.equal(cmd.width, 10);
+      assert.equal(cmd.height, 20);
+      assert.equal(cmd.columns, 3);
+      assert.equal(cmd.rows, 2);
+      assert.equal(cmd.more, 1);
+      assert.equal(cmd.quiet, 2);
+    });
+
+    it('should handle empty control data', () => {
+      const cmd = parseKittyCommand('');
+      assert.equal(cmd.action, undefined);
+      assert.equal(cmd.format, undefined);
+    });
+
+    it('should parse transmit action', () => {
+      const cmd = parseKittyCommand('a=t,f=100');
+      assert.equal(cmd.action, 't');
+      assert.equal(cmd.format, 100);
+    });
+
+    it('should parse delete action', () => {
+      const cmd = parseKittyCommand('a=d,i=5');
+      assert.equal(cmd.action, 'd');
+      assert.equal(cmd.id, 5);
+    });
+  });
+
+  describe('APC handler', () => {
+    it('should store image when transmit+display sequence is written', async () => {
+      // Write Kitty graphics sequence: ESC _ G <control>;<payload> ESC \
+      const sequence = `\x1b_Ga=T,f=100;${BLACK_1X1_BASE64}\x1b\\`;
+      await writeP(terminal, sequence);
+
+      // Addon should have stored the image
+      assert.equal(addon.images.size, 1);
+
+      const image = addon.images.get(1)!;
+      assert.exists(image);
+      assert.equal(image.format, 100); // PNG format
+      assert.equal(image.data, BLACK_1X1_BASE64);
+    });
+
+    it('should store RGB image with correct payload', async () => {
+      const sequence = `\x1b_Ga=T,f=100;${RGB_3X1_BASE64}\x1b\\`;
+      await writeP(terminal, sequence);
+
+      assert.equal(addon.images.size, 1);
+      const image = addon.images.get(1)!;
+      assert.equal(image.data, RGB_3X1_BASE64);
+    });
+
+    it('should use explicit image id when provided', async () => {
+      const sequence = `\x1b_Ga=T,f=100,i=42;${BLACK_1X1_BASE64}\x1b\\`;
+      await writeP(terminal, sequence);
+
+      assert.equal(addon.images.size, 1);
+      assert.isTrue(addon.images.has(42));
+      assert.isFalse(addon.images.has(1));
+    });
+
+    it('should handle transmit-only (a=t) without display', async () => {
+      const sequence = `\x1b_Ga=t,f=100;${BLACK_1X1_BASE64}\x1b\\`;
+      await writeP(terminal, sequence);
+
+      // Image should still be stored??
+      assert.equal(addon.images.size, 1);
+    });
+
+    it('should delete image by id', async () => {
+      // First store an image with id=5
+      await writeP(terminal, `\x1b_Ga=T,f=100,i=5;${BLACK_1X1_BASE64}\x1b\\`);
+      assert.equal(addon.images.size, 1);
+
+      // Delete it
+      await writeP(terminal, `\x1b_Ga=d,i=5\x1b\\`);
+      assert.equal(addon.images.size, 0);
+    });
+
+    it('should delete all images when no id specified', async () => {
+      // Store multiple images
+      await writeP(terminal, `\x1b_Ga=T,f=100,i=1;${BLACK_1X1_BASE64}\x1b\\`);
+      await writeP(terminal, `\x1b_Ga=T,f=100,i=2;${RGB_3X1_BASE64}\x1b\\`);
+      assert.equal(addon.images.size, 2);
+
+      // Delete all
+      await writeP(terminal, `\x1b_Ga=d\x1b\\`);
+      assert.equal(addon.images.size, 0);
+    });
+
+    it('should handle chunked transmission (m=1 flag)', async () => {
+      // Split payload into chunks using m=1 (more data coming)
+      const half = Math.floor(BLACK_1X1_BASE64.length / 2);
+      const chunk1 = BLACK_1X1_BASE64.substring(0, half);
+      const chunk2 = BLACK_1X1_BASE64.substring(half);
+
+      // First chunk with m=1
+      await writeP(terminal, `\x1b_Ga=t,f=100,i=10,m=1;${chunk1}\x1b\\`);
+      // Image not complete yet (pending)
+      assert.equal(addon.images.size, 0);
+
+      // Final chunk without m=1
+      await writeP(terminal, `\x1b_Ga=t,f=100,i=10;${chunk2}\x1b\\`);
+      // Now image should be stored
+      assert.equal(addon.images.size, 1);
+
+      const image = addon.images.get(10)!;
+      assert.equal(image.data, BLACK_1X1_BASE64);
+    });
+  });
+});

addons/addon-kitty-graphics/src/KittyGraphicsAddon.ts

@@ -0,0 +1,200 @@
+/**
+ * Copyright (c) 2025 The xterm.js authors. All rights reserved.
+ * @license MIT
+ */
+
+import type { Terminal, ITerminalAddon, IDisposable } from '@xterm/xterm';
+import type { KittyGraphicsAddon as IKittyGraphicsApi, IKittyGraphicsOptions, IKittyImage } from '@xterm/addon-kitty-graphics';
+
+/**
+ * Kitty graphics protocol action types.
+ */
+const enum KittyAction {
+  TRANSMIT = 't',
+  TRANSMIT_DISPLAY = 'T',
+  QUERY = 'q',
+  PLACEMENT = 'p',
+  DELETE = 'd'
+}
+
+/**
+ * Kitty graphics protocol format types.
+ */
+const enum KittyFormat {
+  RGB = 24,
+  RGBA = 32,
+  PNG = 100
+}
+
+/**
+ * Parsed Kitty graphics command.
+ */
+export interface IKittyCommand {
+  action?: string;
+  format?: number;
+  id?: number;
+  width?: number;
+  height?: number;
+  x?: number;
+  y?: number;
+  columns?: number;
+  rows?: number;
+  more?: number;
+  quiet?: number;
+  payload?: string;
+}
+
+/**
+ * Parses Kitty graphics control data into a command object.
+ * Exported for testing.
+ */
+export function parseKittyCommand(data: string): IKittyCommand {
+  const cmd: IKittyCommand = {};
+  const parts = data.split(',');
+
+  for (const part of parts) {
+    const eqIdx = part.indexOf('=');
+    if (eqIdx === -1) continue;
+
+    const key = part.substring(0, eqIdx);
+    const value = part.substring(eqIdx + 1);
+
+    switch (key) {
+      // Question: How do we know which radix to use?
+      case 'a': cmd.action = value; break;
+      case 'f': cmd.format = parseInt(value); break;
+      case 'i': cmd.id = parseInt(value); break;
+      case 's': cmd.width = parseInt(value); break;
+      case 'v': cmd.height = parseInt(value); break;
+      case 'x': cmd.x = parseInt(value); break;
+      case 'y': cmd.y = parseInt(value); break;
+      case 'c': cmd.columns = parseInt(value); break;
+      case 'r': cmd.rows = parseInt(value); break;
+      case 'm': cmd.more = parseInt(value); break;
+      case 'q': cmd.quiet = parseInt(value); break;
+    }
+  }
+
+  return cmd;
+}
+
+export class KittyGraphicsAddon implements ITerminalAddon, IKittyGraphicsApi {
+  private _terminal: Terminal | undefined;
+  private _apcHandler: IDisposable | undefined;
+  private _images: Map<number, IKittyImage> = new Map();
+  private _pendingData: Map<number, string> = new Map();
+  private _nextImageId = 1;
+  private _debug: boolean;
+
+  constructor(options?: IKittyGraphicsOptions) {
+    this._debug = options?.debug ?? false;
+  }
+
+  public get images(): ReadonlyMap<number, IKittyImage> {
+    return this._images;
+  }
+
+  public activate(terminal: Terminal): void {
+    this._terminal = terminal;
+    // TODO: Remove console log
+    console.log('[KittyGraphicsAddon] Activated');
+
+    // Register APC handler for 'G' (0x47) - Kitty graphics protocol
+    // APC sequence format: ESC _ G <data> ESC \
+    this._apcHandler = terminal.parser.registerApcHandler(0x47, (data: string) => {
+      return this._handleKittyGraphics(data);
+    });
+  }
+
+  public dispose(): void {
+    this._apcHandler?.dispose();
+    this._images.clear();
+    this._pendingData.clear();
+    this._terminal = undefined;
+  }
+
+  private _handleKittyGraphics(data: string): boolean {
+    const semiIdx = data.indexOf(';');
+    const controlData = semiIdx === -1 ? data : data.substring(0, semiIdx);
+    const payload = semiIdx === -1 ? '' : data.substring(semiIdx + 1);
+
+    const cmd = parseKittyCommand(controlData);
+    cmd.payload = payload;
+
+    if (this._debug) {
+      console.log('[KittyGraphicsAddon] Received command:', cmd);
+    }
+
+    const action = cmd.action || 't';
+
+    switch (action) {
+      case KittyAction.TRANSMIT:
+        return this._handleTransmit(cmd);
+      case KittyAction.TRANSMIT_DISPLAY:
+        return this._handleTransmitDisplay(cmd);
+      case KittyAction.QUERY:
+        return this._handleQuery(cmd);
+      case KittyAction.DELETE:
+        return this._handleDelete(cmd);
+      default:
+        return true;
+    }
+  }
+
+  private _handleTransmit(cmd: IKittyCommand): boolean {
+    const id = cmd.id || this._nextImageId++;
+    const payload = cmd.payload || '';
+
+    if (cmd.more === 1) {
+      const existing = this._pendingData.get(id) || '';
+      this._pendingData.set(id, existing + payload);
+      return true;
+    }
+
+    let fullPayload = payload;
+    if (this._pendingData.has(id)) {
+      fullPayload = this._pendingData.get(id)! + payload;
+      this._pendingData.delete(id);
+    }
+
+    const image: IKittyImage = {
+      id,
+      data: fullPayload,
+      width: cmd.width || 0,
+      height: cmd.height || 0,
+      format: (cmd.format || KittyFormat.PNG) as 24 | 32 | 100
+    };
+
+    this._images.set(id, image);
+
+    if (this._debug) {
+      console.log(`[KittyGraphicsAddon] Stored image ${id}`);
+    }
+
+    return true;
+  }
+
+  private _handleTransmitDisplay(cmd: IKittyCommand): boolean {
+    this._handleTransmit(cmd);
+    // TODO: Display image at cursor position (canvas layer)
+    return true;
+  }
+
+  private _handleQuery(cmd: IKittyCommand): boolean {
+    // TODO: Respond with APC sequence indicating graphics support
+    // Protocol: terminal should reply with ESC _ G i=<id>;OK ESC \
+    if (this._debug) {
+      console.log('[KittyGraphicsAddon] Query received');
+    }
+    return true;
+  }
+
+  private _handleDelete(cmd: IKittyCommand): boolean {
+    if (cmd.id !== undefined) {
+      this._images.delete(cmd.id);
+    } else {
+      this._images.clear();
+    }
+    return true;
+  }
+}

addons/addon-kitty-graphics/src/tsconfig.json

@@ -0,0 +1,27 @@
+{
+  "compilerOptions": {
+    "module": "commonjs",
+    "target": "es2021",
+    "lib": ["dom", "es2015"],
+    "rootDir": ".",
+    "outDir": "../out",
+    "sourceMap": true,
+    "removeComments": true,
+    "strict": true,
+    "types": ["../../../node_modules/@types/mocha"],
+    "paths": {
+      "browser/*": ["../../../src/browser/*"],
+      "vs/*": ["../../../src/vs/*"],
+      "@xterm/addon-kitty-graphics": ["../typings/addon-kitty-graphics.d.ts"]
+    }
+  },
+  "include": ["./**/*", "../../../typings/xterm.d.ts"],
+  "references": [
+    {
+      "path": "../../../src/browser"
+    },
+    {
+      "path": "../../../src/vs"
+    }
+  ]
+}