From 5c28e2706271904082c7111ede0ad7d331abf62b Mon Sep 17 00:00:00 2001 From: Anthony Kim Date: Thu, 19 Feb 2026 15:00:33 -0800 Subject: [PATCH 1/2] Implment kitty graphics placement action --- .../src/kitty/KittyGraphicsHandler.ts | 56 +++- .../src/kitty/KittyImageStorage.ts | 8 + addons/addon-image/test/KittyGraphics.test.ts | 290 +++++++++++++++++- 3 files changed, 336 insertions(+), 18 deletions(-) diff --git a/addons/addon-image/src/kitty/KittyGraphicsHandler.ts b/addons/addon-image/src/kitty/KittyGraphicsHandler.ts index a7bd0dd62..60f9b1727 100644 --- a/addons/addon-image/src/kitty/KittyGraphicsHandler.ts +++ b/addons/addon-image/src/kitty/KittyGraphicsHandler.ts @@ -323,9 +323,10 @@ export class KittyGraphicsHandler implements IApcHandler, IResetHandler, IDispos case KittyAction.QUERY: this._sendResponse(cmd.id ?? 0, 'OK', cmd.quiet ?? 0); return true; + case KittyAction.PLACEMENT: + return this._handlePlacement(cmd); default: // TODO: Implement remaining actions when needed: - // - a=p (placement): place a previously transmitted image // - a=f (frame): animation frame operations // - a=a (animation): animation control // - a=c (compose): compose images @@ -357,9 +358,11 @@ export class KittyGraphicsHandler implements IApcHandler, IResetHandler, IDispos return this._handleTransmitDisplay(cmd, bytes, decodeError); case KittyAction.QUERY: return this._handleQuery(cmd, bytes, decodeError); + case KittyAction.PLACEMENT: + // a=p ignores any payload — image data was already transmitted + return this._handlePlacement(cmd); default: // TODO: Implement remaining actions when needed: - // - a=p (placement): place a previously transmitted image // - a=f (frame): animation frame operations // - a=a (animation): animation control // - a=c (compose): compose images @@ -370,6 +373,22 @@ export class KittyGraphicsHandler implements IApcHandler, IResetHandler, IDispos } } + private _handlePlacement(cmd: IKittyCommand): boolean | Promise { + if (cmd.id === undefined) { + return true; + } + const image = this._kittyStorage.getImage(cmd.id); + if (!image) { + this._sendResponse(cmd.id, 'ENOENT:image not found', cmd.quiet ?? 0, cmd.placementId); + return true; + } + const result = this._displayImage(image, cmd); + return result.then(success => { + this._sendResponse(cmd.id!, success ? 'OK' : 'EINVAL:image rendering failed', cmd.quiet ?? 0, cmd.placementId); + return true; + }); + } + private _handleTransmit(cmd: IKittyCommand, bytes: Uint8Array, decodeError: boolean): boolean { // TODO: Support file-based transmission modes (t=f, t=t, t=s) // Currently only supports direct transmission (t=d, the default). @@ -508,12 +527,13 @@ export class KittyGraphicsHandler implements IApcHandler, IResetHandler, IDispos return true; } - private _sendResponse(id: number, message: string, quiet: number): void { + private _sendResponse(id: number, message: string, quiet: number, placementId?: number): void { const isOk = message === 'OK'; if (isOk && quiet === 1) return; if (!isOk && quiet === 2) return; - const response = `\x1b_Gi=${id};${message}\x1b\\`; + const pPart = placementId ? `,p=${placementId}` : ''; + const response = `\x1b_Gi=${id}${pPart};${message}\x1b\\`; this._coreTerminal._core.coreService.triggerDataEvent(response); } @@ -552,9 +572,23 @@ export class KittyGraphicsHandler implements IApcHandler, IResetHandler, IDispos const cw = this._renderer.dimensions?.css.cell.width || CELL_SIZE_DEFAULT.width; const ch = this._renderer.dimensions?.css.cell.height || CELL_SIZE_DEFAULT.height; - // Per spec: c/r default to image's natural cell dimensions - let imgCols = cmd.columns ?? Math.ceil(bitmap.width / cw); - let imgRows = cmd.rows ?? Math.ceil(bitmap.height / ch); + // Per spec: c/r default to image's natural cell dimensions. + // If only one of c/r is specified, compute the other from image aspect ratio. + let imgCols: number; + let imgRows: number; + if (cmd.columns !== undefined && cmd.rows !== undefined) { + imgCols = cmd.columns; + imgRows = cmd.rows; + } else if (cmd.columns !== undefined) { + imgCols = cmd.columns; + imgRows = Math.max(1, Math.ceil((bitmap.height / bitmap.width) * (imgCols * cw) / ch)); + } else if (cmd.rows !== undefined) { + imgRows = cmd.rows; + imgCols = Math.max(1, Math.ceil((bitmap.width / bitmap.height) * (imgRows * ch) / cw)); + } else { + imgCols = Math.ceil(bitmap.width / cw); + imgRows = Math.ceil(bitmap.height / ch); + } let w = bitmap.width; let h = bitmap.height; @@ -593,7 +627,13 @@ export class KittyGraphicsHandler implements IApcHandler, IResetHandler, IDispos const xOffset = Math.min(Math.max(0, cmd.xOffset ?? 0), cw - 1); const yOffset = Math.min(Math.max(0, cmd.yOffset ?? 0), ch - 1); if (xOffset !== 0 || yOffset !== 0) { - const offsetCanvas = ImageRenderer.createCanvas(window.document, finalBitmap.width + xOffset, finalBitmap.height + yOffset); + // Per spec: X/Y is not added to c/r area. When c/r are explicit, the + // total placement area remains c*cw × r*ch pixels and the offset image + // is clipped to fit. When c/r are unset, the padded canvas determines + // the natural cell dimensions. + const canvasW = (cmd.columns !== undefined) ? Math.round(imgCols * cw) : finalBitmap.width + xOffset; + const canvasH = (cmd.rows !== undefined) ? Math.round(imgRows * ch) : finalBitmap.height + yOffset; + const offsetCanvas = ImageRenderer.createCanvas(window.document, canvasW, canvasH); const offsetCtx = offsetCanvas.getContext('2d'); if (!offsetCtx) { finalBitmap.close(); diff --git a/addons/addon-image/src/kitty/KittyImageStorage.ts b/addons/addon-image/src/kitty/KittyImageStorage.ts index cd6e46dcb..146d854dd 100644 --- a/addons/addon-image/src/kitty/KittyImageStorage.ts +++ b/addons/addon-image/src/kitty/KittyImageStorage.ts @@ -81,6 +81,14 @@ export class KittyImageStorage implements IDisposable { } public addImage(kittyId: number, image: HTMLCanvasElement | ImageBitmap, scrolling: boolean, layer: ImageLayer, zIndex: number): void { + // Clean up stale reverse-mapping from a previous placement of the same + // kitty image. The old shared-storage entry is kept (it may still be + // visible on screen) but its reverse mapping is removed so that eviction + // of the old entry won't incorrectly delete the kitty image data. + const oldStorageId = this._kittyIdToStorageId.get(kittyId); + if (oldStorageId !== undefined) { + this._storageIdToKittyId.delete(oldStorageId); + } const storageId = this._storage.addImage(image, scrolling, layer, zIndex); this._kittyIdToStorageId.set(kittyId, storageId); this._storageIdToKittyId.set(storageId, kittyId); diff --git a/addons/addon-image/test/KittyGraphics.test.ts b/addons/addon-image/test/KittyGraphics.test.ts index 562527ca5..b28c3422a 100644 --- a/addons/addon-image/test/KittyGraphics.test.ts +++ b/addons/addon-image/test/KittyGraphics.test.ts @@ -975,21 +975,50 @@ test.describe('Kitty Graphics Protocol', () => { }); }); - test.describe('Unimplemented action responses', () => { - test('a=p with id responds EINVAL', async () => { + test.describe('Placement action (a=p)', () => { + test('displays a previously transmitted image at cursor', async () => { + // Transmit image without display + await ctx.proxy.write(`\x1b_Ga=t,f=100,i=210;${KITTY_BLACK_1X1_BASE64}\x1b\\`); + await timeout(100); + strictEqual(await getImageStorageLength(), 0); + + // Place the previously transmitted image + await ctx.proxy.write(`\x1b_Ga=p,i=210\x1b\\`); + await timeout(100); + strictEqual(await getImageStorageLength(), 1); + deepStrictEqual(await getPixel(0, 0, 0, 0), [0, 0, 0, 255]); + }); + + test('responds OK on successful placement', async () => { + await ctx.proxy.write(`\x1b_Ga=t,f=100,i=211;${KITTY_BLACK_1X1_BASE64}\x1b\\`); + await timeout(100); + await ctx.page.evaluate(() => { (window as any).kittyResponse = ''; (window as any).term.onData((data: string) => { (window as any).kittyResponse = data; }); }); - await ctx.proxy.write(`\x1b_Gi=210,a=p\x1b\\`); + await ctx.proxy.write(`\x1b_Ga=p,i=211\x1b\\`); + await timeout(100); + + const response = await ctx.page.evaluate('window.kittyResponse'); + strictEqual(response, '\x1b_Gi=211;OK\x1b\\'); + }); + + test('responds ENOENT for non-existent image id', async () => { + await ctx.page.evaluate(() => { + (window as any).kittyResponse = ''; + (window as any).term.onData((data: string) => { (window as any).kittyResponse = data; }); + }); + + await ctx.proxy.write(`\x1b_Ga=p,i=9999\x1b\\`); await timeout(100); const response: string = await ctx.page.evaluate('window.kittyResponse'); - strictEqual(response.startsWith('\x1b_Gi=210;EINVAL:'), true); + strictEqual(response, '\x1b_Gi=9999;ENOENT:image not found\x1b\\'); }); - test('a=p without id sends no response', async () => { + test('without id sends no response', async () => { await ctx.page.evaluate(() => { (window as any).kittyGotResponse = false; (window as any).term.onData(() => { (window as any).kittyGotResponse = true; }); @@ -1000,6 +1029,243 @@ test.describe('Kitty Graphics Protocol', () => { strictEqual(await ctx.page.evaluate('window.kittyGotResponse'), false); }); + + test('places at specified column/row size (c/r)', async () => { + await ctx.proxy.write(`\x1b_Ga=t,f=100,i=212;${KITTY_MULTICOLOR_200X100_BASE64}\x1b\\`); + await timeout(100); + + await ctx.proxy.write(`\x1b_Ga=p,i=212,c=5,r=3\x1b\\`); + await timeout(200); + + strictEqual(await getImageStorageLength(), 1); + deepStrictEqual(await getCursor(), [5, 2]); + }); + + test('cursor advances past placed image (default C=0)', async () => { + await ctx.proxy.write(`\x1b_Ga=t,f=100,i=213;${KITTY_BLACK_1X1_BASE64}\x1b\\`); + await timeout(100); + deepStrictEqual(await getCursor(), [0, 0]); + + await ctx.proxy.write(`\x1b_Ga=p,i=213\x1b\\`); + await timeout(100); + deepStrictEqual(await getCursor(), [1, 0]); + }); + + test('cursor does not move when C=1', async () => { + await ctx.proxy.write(`\x1b_Ga=t,f=100,i=214;${KITTY_MULTICOLOR_200X100_BASE64}\x1b\\`); + await timeout(100); + deepStrictEqual(await getCursor(), [0, 0]); + + await ctx.proxy.write(`\x1b_Ga=p,i=214,c=5,r=3,C=1\x1b\\`); + await timeout(200); + deepStrictEqual(await getCursor(), [0, 0]); + }); + + test('supports z-index (negative = bottom layer)', async () => { + await ctx.proxy.write(`\x1b_Ga=t,f=100,i=215;${KITTY_BLACK_1X1_BASE64}\x1b\\`); + await timeout(100); + + await ctx.proxy.write(`\x1b_Ga=p,i=215,z=-1\x1b\\`); + await timeout(100); + + strictEqual(await getImageStorageLength(), 1); + strictEqual(await ctx.page.evaluate(`window.imageAddon._storage._images.get(1).layer`), 'bottom'); + strictEqual(await ctx.page.evaluate(`window.imageAddon._storage._images.get(1).zIndex`), -1); + }); + + test('supports source crop via x/y', async () => { + await ctx.proxy.write(`\x1b_Ga=t,f=100,i=216;${KITTY_MULTICOLOR_200X100_BASE64}\x1b\\`); + await timeout(100); + + // Crop to the Orange rectangle (x=20, y=0, 20x50) + await ctx.proxy.write(`\x1b_Ga=p,i=216,x=20,y=0,w=20,h=50\x1b\\`); + await timeout(200); + + strictEqual(await getImageStorageLength(), 1); + deepStrictEqual(await getOrigSize(1), [20, 50]); + deepStrictEqual(await getPixel(0, 0, 0, 0), [255, 128, 0, 255]); + }); + + test('supports sub-cell offset via X/Y', async () => { + await ctx.proxy.write(`\x1b_Ga=t,f=100,i=217;${KITTY_BLACK_1X1_BASE64}\x1b\\`); + await timeout(100); + + await ctx.proxy.write(`\x1b_Ga=p,i=217,X=5,Y=3\x1b\\`); + await timeout(100); + + deepStrictEqual(await getPixel(0, 0, 0, 0), [0, 0, 0, 0]); + deepStrictEqual(await getPixel(0, 0, 4, 2), [0, 0, 0, 0]); + deepStrictEqual(await getPixel(0, 0, 5, 3), [0, 0, 0, 255]); + }); + + test('multiple placements of same image create separate displays', async () => { + await ctx.proxy.write(`\x1b_Ga=t,f=100,i=218;${KITTY_BLACK_1X1_BASE64}\x1b\\`); + await timeout(100); + strictEqual(await getImageStorageLength(), 0); + + // First placement + await ctx.proxy.write(`\x1b_Ga=p,i=218,p=1\x1b\\`); + await timeout(100); + strictEqual(await getImageStorageLength(), 1); + + // Move cursor down and place again with different placement ID + await ctx.proxy.write('\x1b[3;1H'); + await ctx.proxy.write(`\x1b_Ga=p,i=218,p=2\x1b\\`); + await timeout(100); + + // Both placements should be in shared storage + strictEqual(await getImageStorageLength(), 2); + }); + + test('image data remains available after placement for future placements', async () => { + await ctx.proxy.write(`\x1b_Ga=t,f=100,i=219;${KITTY_BLACK_1X1_BASE64}\x1b\\`); + await timeout(100); + + // Place three times + await ctx.proxy.write(`\x1b_Ga=p,i=219\x1b\\`); + await timeout(100); + await ctx.proxy.write('\x1b[2;1H'); + await ctx.proxy.write(`\x1b_Ga=p,i=219\x1b\\`); + await timeout(100); + await ctx.proxy.write('\x1b[3;1H'); + await ctx.proxy.write(`\x1b_Ga=p,i=219\x1b\\`); + await timeout(100); + + // Image data should still be available + strictEqual(await ctx.page.evaluate(`window.imageAddon._handlers.get('kitty').images.has(219)`), true); + }); + + test('OK response suppressed by q=1', async () => { + await ctx.proxy.write(`\x1b_Ga=t,f=100,i=220;${KITTY_BLACK_1X1_BASE64}\x1b\\`); + await timeout(100); + + await ctx.page.evaluate(() => { + (window as any).kittyGotResponse = false; + (window as any).term.onData(() => { (window as any).kittyGotResponse = true; }); + }); + + await ctx.proxy.write(`\x1b_Ga=p,i=220,q=1\x1b\\`); + await timeout(100); + + strictEqual(await ctx.page.evaluate('window.kittyGotResponse'), false); + strictEqual(await getImageStorageLength(), 1); + }); + + test('ENOENT error suppressed by q=2', async () => { + await ctx.page.evaluate(() => { + (window as any).kittyGotResponse = false; + (window as any).term.onData(() => { (window as any).kittyGotResponse = true; }); + }); + + await ctx.proxy.write(`\x1b_Ga=p,i=9998,q=2\x1b\\`); + await timeout(100); + + strictEqual(await ctx.page.evaluate('window.kittyGotResponse'), false); + }); + + test('ENOENT still reported when q=1 (only suppresses OK)', async () => { + await ctx.page.evaluate(() => { + (window as any).kittyResponse = ''; + (window as any).term.onData((data: string) => { (window as any).kittyResponse = data; }); + }); + + await ctx.proxy.write(`\x1b_Ga=p,i=9997,q=1\x1b\\`); + await timeout(100); + + const response: string = await ctx.page.evaluate('window.kittyResponse'); + strictEqual(response, '\x1b_Gi=9997;ENOENT:image not found\x1b\\'); + }); + + test('renders pixels correctly when placing a PNG image', async () => { + await ctx.proxy.write(`\x1b_Ga=t,f=100,i=221;${KITTY_RGB_3X1_BASE64}\x1b\\`); + await timeout(100); + + await ctx.proxy.write(`\x1b_Ga=p,i=221\x1b\\`); + await timeout(100); + + const pixels = await getPixels(0, 0, 0, 0, 3, 1); + deepStrictEqual(pixels?.slice(0, 4), [255, 0, 0, 255]); + deepStrictEqual(pixels?.slice(4, 8), [0, 255, 0, 255]); + deepStrictEqual(pixels?.slice(8, 12), [0, 0, 255, 255]); + }); + + test('renders pixels correctly when placing raw RGB image', async () => { + await ctx.proxy.write(`\x1b_Ga=t,f=24,s=3,v=1,i=222;${RAW_RGB_3X1}\x1b\\`); + await timeout(100); + + await ctx.proxy.write(`\x1b_Ga=p,i=222\x1b\\`); + await timeout(100); + + const pixels = await getPixels(0, 0, 0, 0, 3, 1); + deepStrictEqual(pixels?.slice(0, 4), [255, 0, 0, 255]); + deepStrictEqual(pixels?.slice(4, 8), [0, 255, 0, 255]); + deepStrictEqual(pixels?.slice(8, 12), [0, 0, 255, 255]); + }); + + test('renders pixels correctly when placing raw RGBA image', async () => { + await ctx.proxy.write(`\x1b_Ga=t,f=32,s=3,v=1,i=223;${RAW_RGBA_3X1}\x1b\\`); + await timeout(100); + + await ctx.proxy.write(`\x1b_Ga=p,i=223\x1b\\`); + await timeout(100); + + const pixels = await getPixels(0, 0, 0, 0, 3, 1); + deepStrictEqual(pixels?.slice(0, 4), [255, 0, 0, 255]); + deepStrictEqual(pixels?.slice(4, 8), [0, 255, 0, 255]); + deepStrictEqual(pixels?.slice(8, 12), [0, 0, 255, 255]); + }); + + test('response includes placement id when p is specified', async () => { + await ctx.proxy.write(`\x1b_Ga=t,f=100,i=224;${KITTY_BLACK_1X1_BASE64}\x1b\\`); + await timeout(100); + + await ctx.page.evaluate(() => { + (window as any).kittyResponse = ''; + (window as any).term.onData((data: string) => { (window as any).kittyResponse = data; }); + }); + + await ctx.proxy.write(`\x1b_Ga=p,i=224,p=42\x1b\\`); + await timeout(100); + + const response = await ctx.page.evaluate('window.kittyResponse'); + strictEqual(response, '\x1b_Gi=224,p=42;OK\x1b\\'); + }); + + test('only c specified computes r from aspect ratio', async () => { + // 200x100 image (2:1 aspect) with c=10. + // Per spec: r = ceil((h/w) * c * cw / ch) = ceil(0.5 * 10 * cw / ch) + await ctx.proxy.write(`\x1b_Ga=t,f=100,i=225;${KITTY_MULTICOLOR_200X100_BASE64}\x1b\\`); + await timeout(100); + + await ctx.proxy.write(`\x1b_Ga=p,i=225,c=10\x1b\\`); + await timeout(200); + + const cursor = await getCursor(); + const cellDims: number[] = await ctx.page.evaluate(() => { + const d = (window as any).term._core._renderService.dimensions.css.cell; + return [d.width, d.height]; + }); + const expectedR = Math.ceil((100 / 200) * 10 * cellDims[0] / cellDims[1]); + deepStrictEqual(cursor, [10, expectedR - 1]); + }); + + test('only r specified computes c from aspect ratio', async () => { + // 200x100 image (2:1 aspect) with r=5. + // Per spec: c = ceil((w/h) * r * ch / cw) = ceil(2 * 5 * ch / cw) + await ctx.proxy.write(`\x1b_Ga=t,f=100,i=226;${KITTY_MULTICOLOR_200X100_BASE64}\x1b\\`); + await timeout(100); + + await ctx.proxy.write(`\x1b_Ga=p,i=226,r=5\x1b\\`); + await timeout(200); + + const cursor = await getCursor(); + const cellDims: number[] = await ctx.page.evaluate(() => { + const d = (window as any).term._core._renderService.dimensions.css.cell; + return [d.width, d.height]; + }); + const expectedC = Math.ceil((200 / 100) * 5 * cellDims[1] / cellDims[0]); + deepStrictEqual(cursor, [expectedC, 4]); + }); }); test.describe('Cursor positioning', () => { @@ -1063,20 +1329,24 @@ test.describe('Kitty Graphics Protocol', () => { }); test('cursor should move right by cols when c specified', async () => { - // c=5: image displayed over 5 columns, r auto = ceil(1/cellHeight) = 1 + // c=5, r computed from 1x1 aspect ratio: r = ceil((1/1) * (5*cw) / ch) + // With 1:1 aspect, image height in pixels = 5*cw, so r = ceil(5*cw/ch) await ctx.proxy.write(`\x1b_Ga=T,f=100,c=5;${KITTY_BLACK_1X1_BASE64}\x1b\\`); await timeout(100); - deepStrictEqual(await getCursor(), [5, 0]); + const cursor = await getCursor(); + strictEqual(cursor[0], 5); + // r is computed from aspect ratio, so y > 0 for a square image at c=5 }); test('cursor should move down by rows when r specified', async () => { - // r=3: image displayed over 3 rows, c auto = ceil(1/cellWidth) = 1 + // r=3, c computed from 1x1 aspect ratio: c = ceil((1/1) * (3*ch) / cw) await ctx.proxy.write(`\x1b_Ga=T,f=100,r=3;${KITTY_BLACK_1X1_BASE64}\x1b\\`); await timeout(100); - // Cursor at first column after image (col 1), on last row (row 2) - deepStrictEqual(await getCursor(), [1, 2]); + const cursor = await getCursor(); + strictEqual(cursor[1], 2); + // c is computed from aspect ratio, so x > 1 for a square image at r=3 }); test('cursor should move by cols AND rows when both specified', async () => { From 2158827b25873b2743b59027e47acd4fc18ce959 Mon Sep 17 00:00:00 2001 From: Anthony Kim Date: Thu, 19 Feb 2026 15:51:22 -0800 Subject: [PATCH 2/2] remove non-null assertion, improve lifecycle for bitmap --- .../src/kitty/KittyGraphicsHandler.ts | 251 +++++++++--------- .../src/kitty/KittyImageStorage.ts | 9 + 2 files changed, 138 insertions(+), 122 deletions(-) diff --git a/addons/addon-image/src/kitty/KittyGraphicsHandler.ts b/addons/addon-image/src/kitty/KittyGraphicsHandler.ts index 60f9b1727..2e43accb4 100644 --- a/addons/addon-image/src/kitty/KittyGraphicsHandler.ts +++ b/addons/addon-image/src/kitty/KittyGraphicsHandler.ts @@ -377,14 +377,15 @@ export class KittyGraphicsHandler implements IApcHandler, IResetHandler, IDispos if (cmd.id === undefined) { return true; } - const image = this._kittyStorage.getImage(cmd.id); + const id = cmd.id; + const image = this._kittyStorage.getImage(id); if (!image) { - this._sendResponse(cmd.id, 'ENOENT:image not found', cmd.quiet ?? 0, cmd.placementId); + this._sendResponse(id, 'ENOENT:image not found', cmd.quiet ?? 0, cmd.placementId); return true; } const result = this._displayImage(image, cmd); return result.then(success => { - this._sendResponse(cmd.id!, success ? 'OK' : 'EINVAL:image rendering failed', cmd.quiet ?? 0, cmd.placementId); + this._sendResponse(id, success ? 'OK' : 'EINVAL:image rendering failed', cmd.quiet ?? 0, cmd.placementId); return true; }); } @@ -511,6 +512,10 @@ export class KittyGraphicsHandler implements IApcHandler, IResetHandler, IDispos break; case 'i': case 'I': + // TODO: When placement id tracking is implemented (see TODO in + // KittyImageStorage), d=i with p= should delete only that + // specific placement, while d=i without p should delete all + // placements for the image. if (cmd.id !== undefined) { const pending = this._pendingTransmissions.get(cmd.id); if (pending) { @@ -546,135 +551,137 @@ export class KittyGraphicsHandler implements IApcHandler, IResetHandler, IDispos } private async _decodeAndDisplay(image: IKittyImageData, cmd: IKittyCommand): Promise { - let bitmap = await this._createBitmap(image); + let bitmap: ImageBitmap | undefined = await this._createBitmap(image); - const cropX = Math.max(0, cmd.x ?? 0); - const cropY = Math.max(0, cmd.y ?? 0); - const cropW = cmd.sourceWidth || (bitmap.width - cropX); - const cropH = cmd.sourceHeight || (bitmap.height - cropY); + try { + const cropX = Math.max(0, cmd.x ?? 0); + const cropY = Math.max(0, cmd.y ?? 0); + const cropW = cmd.sourceWidth || (bitmap.width - cropX); + const cropH = cmd.sourceHeight || (bitmap.height - cropY); - const maxCropW = Math.max(0, bitmap.width - cropX); - const maxCropH = Math.max(0, bitmap.height - cropY); - const finalCropW = Math.max(0, Math.min(cropW, maxCropW)); - const finalCropH = Math.max(0, Math.min(cropH, maxCropH)); + const maxCropW = Math.max(0, bitmap.width - cropX); + const maxCropH = Math.max(0, bitmap.height - cropY); + const finalCropW = Math.max(0, Math.min(cropW, maxCropW)); + const finalCropH = Math.max(0, Math.min(cropH, maxCropH)); - if (finalCropW === 0 || finalCropH === 0) { - bitmap.close(); - throw new Error('invalid source rectangle'); - } - - if (cropX !== 0 || cropY !== 0 || finalCropW !== bitmap.width || finalCropH !== bitmap.height) { - const cropped = await createImageBitmap(bitmap, cropX, cropY, finalCropW, finalCropH); - bitmap.close(); - bitmap = cropped; - } - - const cw = this._renderer.dimensions?.css.cell.width || CELL_SIZE_DEFAULT.width; - const ch = this._renderer.dimensions?.css.cell.height || CELL_SIZE_DEFAULT.height; - - // Per spec: c/r default to image's natural cell dimensions. - // If only one of c/r is specified, compute the other from image aspect ratio. - let imgCols: number; - let imgRows: number; - if (cmd.columns !== undefined && cmd.rows !== undefined) { - imgCols = cmd.columns; - imgRows = cmd.rows; - } else if (cmd.columns !== undefined) { - imgCols = cmd.columns; - imgRows = Math.max(1, Math.ceil((bitmap.height / bitmap.width) * (imgCols * cw) / ch)); - } else if (cmd.rows !== undefined) { - imgRows = cmd.rows; - imgCols = Math.max(1, Math.ceil((bitmap.width / bitmap.height) * (imgRows * ch) / cw)); - } else { - imgCols = Math.ceil(bitmap.width / cw); - imgRows = Math.ceil(bitmap.height / ch); - } - - let w = bitmap.width; - let h = bitmap.height; - - // Scale bitmap to fit placement rectangle when c/r are specified - if (cmd.columns !== undefined || cmd.rows !== undefined) { - w = Math.round(imgCols * cw); - h = Math.round(imgRows * ch); - } - - if (w * h > this._opts.pixelLimit) { - bitmap.close(); - throw new Error('image exceeds pixel limit'); - } - - // Save cursor position before addImage modifies it - const buffer = this._coreTerminal._core.buffer; - const savedX = buffer.x; - const savedY = buffer.y; - const savedYbase = buffer.ybase; - - // Determine layer based on z-index: negative = behind text, 0+ = on top. - // When z<0 we always use the bottom layer even without allowTransparency — - // the image will simply be hidden behind the opaque text background, which - // is the correct behavior (client asked for "behind text"). - const wantsBottom = cmd.zIndex !== undefined && cmd.zIndex < 0; - const layer: ImageLayer = wantsBottom ? 'bottom' : 'top'; - - let finalBitmap = bitmap; - if (w !== bitmap.width || h !== bitmap.height) { - finalBitmap = await createImageBitmap(bitmap, { resizeWidth: w, resizeHeight: h }); - bitmap.close(); - } - - // Per spec: X/Y are pixel offsets within the first cell, so clamp to cell dimensions - const xOffset = Math.min(Math.max(0, cmd.xOffset ?? 0), cw - 1); - const yOffset = Math.min(Math.max(0, cmd.yOffset ?? 0), ch - 1); - if (xOffset !== 0 || yOffset !== 0) { - // Per spec: X/Y is not added to c/r area. When c/r are explicit, the - // total placement area remains c*cw × r*ch pixels and the offset image - // is clipped to fit. When c/r are unset, the padded canvas determines - // the natural cell dimensions. - const canvasW = (cmd.columns !== undefined) ? Math.round(imgCols * cw) : finalBitmap.width + xOffset; - const canvasH = (cmd.rows !== undefined) ? Math.round(imgRows * ch) : finalBitmap.height + yOffset; - const offsetCanvas = ImageRenderer.createCanvas(window.document, canvasW, canvasH); - const offsetCtx = offsetCanvas.getContext('2d'); - if (!offsetCtx) { - finalBitmap.close(); - throw new Error('Failed to create offset canvas context'); + if (finalCropW === 0 || finalCropH === 0) { + throw new Error('invalid source rectangle'); + } + + if (cropX !== 0 || cropY !== 0 || finalCropW !== bitmap.width || finalCropH !== bitmap.height) { + const cropped = await createImageBitmap(bitmap, cropX, cropY, finalCropW, finalCropH); + bitmap.close(); + bitmap = cropped; + } + + const cw = this._renderer.dimensions?.css.cell.width || CELL_SIZE_DEFAULT.width; + const ch = this._renderer.dimensions?.css.cell.height || CELL_SIZE_DEFAULT.height; + + // Per spec: c/r default to image's natural cell dimensions. + // If only one of c/r is specified, compute the other from image aspect ratio. + let imgCols: number; + let imgRows: number; + if (cmd.columns !== undefined && cmd.rows !== undefined) { + imgCols = cmd.columns; + imgRows = cmd.rows; + } else if (cmd.columns !== undefined) { + imgCols = cmd.columns; + imgRows = Math.max(1, Math.ceil((bitmap.height / bitmap.width) * (imgCols * cw) / ch)); + } else if (cmd.rows !== undefined) { + imgRows = cmd.rows; + imgCols = Math.max(1, Math.ceil((bitmap.width / bitmap.height) * (imgRows * ch) / cw)); + } else { + imgCols = Math.ceil(bitmap.width / cw); + imgRows = Math.ceil(bitmap.height / ch); + } + + let w = bitmap.width; + let h = bitmap.height; + + // Scale bitmap to fit placement rectangle when c/r are specified + if (cmd.columns !== undefined || cmd.rows !== undefined) { + w = Math.round(imgCols * cw); + h = Math.round(imgRows * ch); } - offsetCtx.drawImage(finalBitmap, xOffset, yOffset); - const offsetBitmap = await createImageBitmap(offsetCanvas); - offsetCanvas.width = offsetCanvas.height = 0; - finalBitmap.close(); - finalBitmap = offsetBitmap; - w = finalBitmap.width; - h = finalBitmap.height; if (w * h > this._opts.pixelLimit) { - finalBitmap.close(); throw new Error('image exceeds pixel limit'); } - if (cmd.columns === undefined) { - imgCols = Math.ceil(finalBitmap.width / cw); - } - if (cmd.rows === undefined) { - imgRows = Math.ceil(finalBitmap.height / ch); - } - } - const zIndex = cmd.zIndex ?? 0; - this._kittyStorage.addImage(image.id, finalBitmap, true, layer, zIndex); + // Save cursor position before addImage modifies it + const buffer = this._coreTerminal._core.buffer; + const savedX = buffer.x; + const savedY = buffer.y; + const savedYbase = buffer.ybase; - // Kitty cursor movement - // Per spec: cursor placed at first column after last image column, - // on the last row of the image. C=1 means don't move cursor. - if (cmd.cursorMovement === 1) { - // C=1: restore cursor to position before image was placed - const scrolled = buffer.ybase - savedYbase; - buffer.x = savedX; - // Can't restore cursor to scrollback? - buffer.y = Math.max(savedY - scrolled, 0); - } else { - // Default (C=0): advance cursor horizontally past the image - // addImage already positioned cursor on the last row via lineFeeds - buffer.x = Math.min(savedX + imgCols, this._coreTerminal.cols); + // Determine layer based on z-index: negative = behind text, 0+ = on top. + // When z<0 we always use the bottom layer even without allowTransparency — + // the image will simply be hidden behind the opaque text background, which + // is the correct behavior (client asked for "behind text"). + const wantsBottom = cmd.zIndex !== undefined && cmd.zIndex < 0; + const layer: ImageLayer = wantsBottom ? 'bottom' : 'top'; + + if (w !== bitmap.width || h !== bitmap.height) { + const scaled = await createImageBitmap(bitmap, { resizeWidth: w, resizeHeight: h }); + bitmap.close(); + bitmap = scaled; + } + + // Per spec: X/Y are pixel offsets within the first cell, so clamp to cell dimensions + const xOffset = Math.min(Math.max(0, cmd.xOffset ?? 0), cw - 1); + const yOffset = Math.min(Math.max(0, cmd.yOffset ?? 0), ch - 1); + if (xOffset !== 0 || yOffset !== 0) { + // Per spec: X/Y is not added to c/r area. When c/r are explicit, the + // total placement area remains c*cw × r*ch pixels and the offset image + // is clipped to fit. When c/r are unset, the padded canvas determines + // the natural cell dimensions. + const canvasW = (cmd.columns !== undefined) ? Math.round(imgCols * cw) : bitmap.width + xOffset; + const canvasH = (cmd.rows !== undefined) ? Math.round(imgRows * ch) : bitmap.height + yOffset; + const offsetCanvas = ImageRenderer.createCanvas(window.document, canvasW, canvasH); + const offsetCtx = offsetCanvas.getContext('2d'); + if (!offsetCtx) { + throw new Error('Failed to create offset canvas context'); + } + offsetCtx.drawImage(bitmap, xOffset, yOffset); + + const offsetBitmap = await createImageBitmap(offsetCanvas); + offsetCanvas.width = offsetCanvas.height = 0; + bitmap.close(); + bitmap = offsetBitmap; + w = bitmap.width; + h = bitmap.height; + if (w * h > this._opts.pixelLimit) { + throw new Error('image exceeds pixel limit'); + } + if (cmd.columns === undefined) { + imgCols = Math.ceil(bitmap.width / cw); + } + if (cmd.rows === undefined) { + imgRows = Math.ceil(bitmap.height / ch); + } + } + + const zIndex = cmd.zIndex ?? 0; + this._kittyStorage.addImage(image.id, bitmap, true, layer, zIndex); + bitmap = undefined; // ownership transferred to storage + + // Kitty cursor movement + // Per spec: cursor placed at first column after last image column, + // on the last row of the image. C=1 means don't move cursor. + if (cmd.cursorMovement === 1) { + // C=1: restore cursor to position before image was placed + const scrolled = buffer.ybase - savedYbase; + buffer.x = savedX; + // Can't restore cursor to scrollback? + buffer.y = Math.max(savedY - scrolled, 0); + } else { + // Default (C=0): advance cursor horizontally past the image + // addImage already positioned cursor on the last row via lineFeeds + buffer.x = Math.min(savedX + imgCols, this._coreTerminal.cols); + } + } catch (e) { + bitmap?.close(); + throw e; } } diff --git a/addons/addon-image/src/kitty/KittyImageStorage.ts b/addons/addon-image/src/kitty/KittyImageStorage.ts index 146d854dd..95ea594c7 100644 --- a/addons/addon-image/src/kitty/KittyImageStorage.ts +++ b/addons/addon-image/src/kitty/KittyImageStorage.ts @@ -20,6 +20,15 @@ export class KittyImageStorage implements IDisposable { private _nextImageId = 1; private readonly _images: Map = new Map(); + // TODO: Support multiple placements per image. The kitty spec identifies + // placements by an (image id, placement id) pair — same i + different p + // values should coexist, and same i + same p should replace the prior + // placement. Currently we track only one storage entry per kitty image id, + // so multiple placements of the same image overwrite each other. Fixing + // this requires changing these maps to Map> + // (kittyId → placementId → storageId) and updating addImage/deleteById + // accordingly. The underlying shared ImageStorage would also need to + // support multiple entries per logical image. private readonly _kittyIdToStorageId: Map = new Map(); private readonly _storageIdToKittyId: Map = new Map();