From dc1fab5d5d22dda1f723f2cf161880812c69b633 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Breitbart?= Date: Tue, 1 Sep 2026 11:16:43 +0200 Subject: [PATCH] multiple changes: - renderer: delete static primitives - renderer: use device pixels for drawing and clearing - renderer: simplify placeholder creation - renderer: localize dimensions in rescaleCanvas - renderer: rescale on drawing - storage: store raw bytes as blobs - storage: alter addImage interface adding blob and metrics data - storage: switch from pixel to byte counting - storage: getBlobUsage() - Drawable as storage/renderer handle - use metrics for all protocols - remove extractTileAtBufferCell - use device pixels in CSI t 14|16 reports --- addons/addon-image/src/IIPHandler.ts | 23 +- addons/addon-image/src/IIPImageStorage.ts | 6 +- addons/addon-image/src/ImageAddon.ts | 7 +- addons/addon-image/src/ImageRenderer.ts | 265 +++++++----------- addons/addon-image/src/ImageStorage.ts | 120 ++++---- .../{IIPMetrics.test.ts => Metrics.test.ts} | 3 +- .../src/{IIPMetrics.ts => Metrics.ts} | 7 +- addons/addon-image/src/Primitives.ts | 49 +++- addons/addon-image/src/SixelHandler.ts | 5 +- addons/addon-image/src/SixelImageStorage.ts | 11 +- addons/addon-image/src/Types.ts | 43 ++- .../src/kitty/KittyImageStorage.ts | 5 +- addons/addon-image/test/ImageAddon.test.ts | 44 +-- addons/addon-image/test/KittyGraphics.test.ts | 44 +-- addons/addon-image/typings/addon-image.d.ts | 5 - src/browser/CoreBrowserTerminal.ts | 8 +- 16 files changed, 313 insertions(+), 332 deletions(-) rename addons/addon-image/src/{IIPMetrics.test.ts => Metrics.test.ts} (97%) rename addons/addon-image/src/{IIPMetrics.ts => Metrics.ts} (94%) diff --git a/addons/addon-image/src/IIPHandler.ts b/addons/addon-image/src/IIPHandler.ts index d26a53faa..3f6673540 100644 --- a/addons/addon-image/src/IIPHandler.ts +++ b/addons/addon-image/src/IIPHandler.ts @@ -9,8 +9,8 @@ import { CELL_SIZE_DEFAULT } from './ImageStorage'; import Base64Decoder from 'xterm-wasm-parts/lib/base64/Base64Decoder.wasm'; import QoiDecoder from 'xterm-wasm-parts/lib/qoi/QoiDecoder.wasm'; import { HeaderParser, IHeaderFields, HeaderState, SequenceType } from './IIPHeaderParser'; -import { imageType, UNSUPPORTED_TYPE } from './IIPMetrics'; -import { createCanvas } from './Primitives'; +import { imageType, UNSUPPORTED_TYPE } from './Metrics'; +import { Drawable } from './Primitives'; // Local const enum mirror - esbuild can't inline const enums from external packages const enum DecoderConst { @@ -178,30 +178,23 @@ export class IIPHandler implements IOscHandler, IResetHandler { return true; } - let blob: Blob | ImageData; + let bmSrc: Blob | ImageData; + let imgBlob: Blob; if (metrics.mime === 'image/qoi') { const data = this._qoiDec.decode(this._dec.data8); - blob = new ImageData( + bmSrc = new ImageData( new Uint8ClampedArray(data.buffer, data.byteOffset, data.byteLength), this._qoiDec.width, this._qoiDec.height ); this._qoiDec.release(); - if (w === this._qoiDec.width && h === this._qoiDec.height) { - // use fast-path if we don't need to rescale - this._dec.release(); - const canvas = createCanvas(undefined, this._qoiDec.width, this._qoiDec.height); - canvas.getContext('2d')?.putImageData(blob, 0, 0); - this._storage.addImage(canvas); - return true; - } } else { - blob = new Blob([this._dec.data8], { type: metrics.mime }); + imgBlob = bmSrc = new Blob([this._dec.data8], { type: metrics.mime }); } this._dec.release(); - return createImageBitmap(blob, { resizeWidth: w, resizeHeight: h }) + return createImageBitmap(bmSrc, { resizeWidth: w, resizeHeight: h }) .then(bm => { - this._storage.addImage(bm); + this._storage.addImage(new Drawable(bm), imgBlob, metrics); return true; }) .catch(e => { diff --git a/addons/addon-image/src/IIPImageStorage.ts b/addons/addon-image/src/IIPImageStorage.ts index 28037a897..b2955970b 100644 --- a/addons/addon-image/src/IIPImageStorage.ts +++ b/addons/addon-image/src/IIPImageStorage.ts @@ -3,7 +3,7 @@ * @license MIT */ -import { IAddImageOpts } from './Types'; +import { IAddImageOpts, IDrawable, IMetrics } from './Types'; import { ImageStorage } from './ImageStorage'; /** @@ -22,7 +22,7 @@ export class IIPImageStorage { * Add an IIP image to storage. * Always uses scrolling mode — cursor advances past the image. */ - public addImage(img: HTMLCanvasElement | ImageBitmap): void { - this._storage.addImage(img, this._addImageOpts); + public addImage(src: IDrawable, data: Blob | undefined, metrics: IMetrics): void { + this._storage.addImage(src, data, metrics, this._addImageOpts); } } diff --git a/addons/addon-image/src/ImageAddon.ts b/addons/addon-image/src/ImageAddon.ts index 8fd395431..c7b8945f5 100644 --- a/addons/addon-image/src/ImageAddon.ts +++ b/addons/addon-image/src/ImageAddon.ts @@ -144,6 +144,9 @@ export class ImageAddon implements ITerminalAddon, IImageApi { terminal.options.windowOptions = windowOps; } + // apply placeholder setting + this._renderer.showPlaceholder(this._opts.showPlaceholder); + this._disposeLater( this._renderer, this._storage, @@ -250,10 +253,6 @@ export class ImageAddon implements ITerminalAddon, IImageApi { return this._storage?.getImageAtBufferCell(x, y); } - public extractTileAtBufferCell(x: number, y: number): HTMLCanvasElement | undefined { - return this._storage?.extractTileAtBufferCell(x, y); - } - private _report(s: string): void { this._terminal?._core.input(s, false); } diff --git a/addons/addon-image/src/ImageRenderer.ts b/addons/addon-image/src/ImageRenderer.ts index 84d83e5e5..d40ace665 100644 --- a/addons/addon-image/src/ImageRenderer.ts +++ b/addons/addon-image/src/ImageRenderer.ts @@ -3,20 +3,14 @@ * @license MIT */ -import { toRGBA8888 } from 'sixel/lib/Colors'; import { IDisposable } from '@xterm/xterm'; -import { ICellSize, ImageLayer, ITerminalExt, IImageSpec, IRenderDimensions, IRenderService } from './Types'; +import type { ICellSize, ImageLayer, ITerminalExt, IImageSpec, IRenderDimensions, IRenderService } from './Types'; import { Disposable, MutableDisposable, toDisposable } from 'common/Lifecycle'; import { createCanvas } from './Primitives'; -const enum Constants { - PLACEHOLDER_LENGTH = 4096, - PLACEHOLDER_HEIGHT = 24 -} /** * ImageRenderer - terminal frontend extension: - * - provide primitives for canvas, ImageData, Bitmap (static) * - add canvas layer to DOM (browser only for now) * - draw image tiles onRender */ @@ -25,12 +19,15 @@ export class ImageRenderer extends Disposable implements IDisposable { public get canvas(): HTMLCanvasElement | undefined { return this._layers.get('top')?.canvas; } private _layers = new Map(); private _placeholder: HTMLCanvasElement | undefined; - private _placeholderBitmap: ImageBitmap | undefined; private _optionsRefresh = this._register(new MutableDisposable()); private _oldOpen: ((parent: HTMLElement) => void) | undefined; private _renderService: IRenderService | undefined; private _oldSetRenderer: ((renderer: any) => void) | undefined; + // some local variables for faster access + private _dimensions: IRenderDimensions | undefined; + private _cellSize: ICellSize | undefined; + constructor(private _terminal: ITerminalExt) { super(); this._oldOpen = this._terminal._core.open; @@ -61,8 +58,6 @@ export class ImageRenderer extends Disposable implements IDisposable { } this._renderService = undefined; this._layers.clear(); - this._placeholderBitmap?.close(); - this._placeholderBitmap = undefined; this._placeholder = undefined; })); } @@ -71,15 +66,7 @@ export class ImageRenderer extends Disposable implements IDisposable { * Enable the placeholder. */ public showPlaceholder(value: boolean): void { - if (value) { - if (!this._placeholder && this.cellSize.height !== -1) { - this._createPlaceHolder(Math.max(this.cellSize.height + 1, Constants.PLACEHOLDER_HEIGHT)); - } - } else { - this._placeholderBitmap?.close(); - this._placeholderBitmap = undefined; - this._placeholder = undefined; - } + this._placeholder = value ? this._createPlaceHolder() : undefined; this._renderService?.refreshRows(0, this._terminal.rows); } @@ -92,22 +79,27 @@ export class ImageRenderer extends Disposable implements IDisposable { } /** - * Current cell size (float). + * Current cell size. */ - public get cellSize(): ICellSize { - return { - width: this.dimensions?.css.cell.width || -1, - height: this.dimensions?.css.cell.height || -1 - }; + public getCellSize(dimensions?: IRenderDimensions): ICellSize | undefined { + dimensions ??= this._terminal.dimensions; + if (dimensions) { + return { + width: dimensions.device.canvas.width / this._terminal.cols, + height: dimensions.device.canvas.height / this._terminal.rows, + }; + } } /** * Clear a region of the image layer canvas. */ public clearLines(start: number, end: number, layer?: ImageLayer): void { - const y = start * (this.dimensions?.css.cell.height || 0); - const w = this.dimensions?.css.canvas.width || 0; - const h = (end + 1 - start) * (this.dimensions?.css.cell.height || 0); + const deviceGrid = this._cellSize; + if (!deviceGrid || !this._dimensions) return; + const y = Math.floor(start * deviceGrid.height); + const w = this._dimensions.device.canvas.width; + const h = Math.ceil((end + 1 - start) * deviceGrid.height); if (!layer || layer === 'top') { this._layers.get('top')?.clearRect(0, y, w, h); } @@ -135,142 +127,95 @@ export class ImageRenderer extends Disposable implements IDisposable { */ public draw(imgSpec: IImageSpec, tileId: number, col: number, row: number, count: number = 1): void { const ctx = this._layers.get(imgSpec.layer); - if (!ctx) { - return; - } - const { width, height } = this.cellSize; + const initialGrid = imgSpec.cellSize; + const deviceGrid = this._cellSize; - // Don't try to draw anything, if we cannot get valid renderer metrics. - if (width === -1 || height === -1) { + if (!ctx || !imgSpec.src.native || !deviceGrid) { return; } - this._rescaleImage(imgSpec, width, height); - const img = imgSpec.actual!; - const cols = Math.ceil(img.width / width); + const img = imgSpec.src; + const cols = Math.ceil(img.width / initialGrid.width); - const sx = (tileId % cols) * width; - const sy = Math.floor(tileId / cols) * height; - const dx = col * width; - const dy = row * height; + const sx = (tileId % cols) * initialGrid.width; + const sy = Math.floor(tileId / cols) * initialGrid.height; + const dx = col * deviceGrid.width; + const dy = row * deviceGrid.height; - // safari bug: never access image source out of bounds - const finalWidth = count * width + sx > img.width ? img.width - sx : count * width; - const finalHeight = sy + height > img.height ? img.height - sy : height; + // safari bug: never access image source out of bounds, thus we clamp its dimensions + const sWidth = Math.min(count * initialGrid.width, img.width - sx); + const sHeight = Math.min(initialGrid.height, img.height - sy); + const dWidth = sWidth / initialGrid.width * deviceGrid.width; + const dHeight = sHeight / initialGrid.height * deviceGrid.height; // Floor all pixel offsets to get stable tile mapping without any overflows. // Note: For not pixel perfect aligned cells like in the DOM renderer // this will move a tile slightly to the top/left (subpixel range, thus ignore it). // FIX #34: avoid striping on displays with pixelDeviceRatio != 1 by ceiling height and width ctx.drawImage( - img, - Math.floor(sx), Math.floor(sy), Math.ceil(finalWidth), Math.ceil(finalHeight), - Math.floor(dx), Math.floor(dy), Math.ceil(finalWidth), Math.ceil(finalHeight) + img.native, + Math.floor(sx), Math.floor(sy), Math.ceil(sWidth), Math.ceil(sHeight), + Math.floor(dx), Math.floor(dy), Math.ceil(dWidth), Math.ceil(dHeight) ); } - /** - * Extract a single tile from an image. - */ - public extractTile(imgSpec: IImageSpec, tileId: number): HTMLCanvasElement | undefined { - const { width, height } = this.cellSize; - // Don't try to draw anything, if we cannot get valid renderer metrics. - if (width === -1 || height === -1) { - return; - } - this._rescaleImage(imgSpec, width, height); - const img = imgSpec.actual!; - const cols = Math.ceil(img.width / width); - const sx = (tileId % cols) * width; - const sy = Math.floor(tileId / cols) * height; - const finalWidth = width + sx > img.width ? img.width - sx : width; - const finalHeight = sy + height > img.height ? img.height - sy : height; - - const canvas = createCanvas(this.document, finalWidth, finalHeight); - const ctx = canvas.getContext('2d'); - if (ctx) { - ctx.drawImage( - img, - Math.floor(sx), Math.floor(sy), Math.floor(finalWidth), Math.floor(finalHeight), - 0, 0, Math.floor(finalWidth), Math.floor(finalHeight) - ); - return canvas; - } - } - /** * Draw a line with placeholder on the image layer canvas. */ public drawPlaceholder(col: number, row: number, count: number = 1): void { const ctx = this._layers.get('top'); - if (ctx) { - const { width, height } = this.cellSize; - - // Don't try to draw anything, if we cannot get valid renderer metrics. - if (width === -1 || height === -1) { - return; - } - - if (!this._placeholder) { - this._createPlaceHolder(Math.max(height + 1, Constants.PLACEHOLDER_HEIGHT)); - } else if (height >= this._placeholder!.height) { - this._createPlaceHolder(height + 1); - } - if (!this._placeholder) return; - ctx.drawImage( - this._placeholderBitmap ?? this._placeholder!, - col * width, - (row * height) % 2 ? 0 : 1, // needs %2 offset correction - width * count, - height, - col * width, - row * height, - width * count, - height - ); + const deviceGrid = this._cellSize; + if (!ctx || !deviceGrid) { + return; } + this._placeholder ??= this._createPlaceHolder(); + if (!this._placeholder) return; + ctx.drawImage( + this._placeholder, + 0, 0, 1, 1, + col * deviceGrid.width, + row * deviceGrid.height, + count * deviceGrid.width, + deviceGrid.height + ); } /** * Rescale image layer canvas if needed. * Checked once from `ImageStorage.render`. + * NOTE: This method updates dimensions on instance properties + * for faster access during draw and clear calls. + * So make sure to always call this before doing any draws. */ - public rescaleCanvas(): void { - const w = this.dimensions?.css.canvas.width || 0; - const h = this.dimensions?.css.canvas.height || 0; - for (const ctx of this._layers.values()) { - if (ctx.canvas.width !== w || ctx.canvas.height !== h) { - ctx.canvas.width = w; - ctx.canvas.height = h; + public rescaleCanvas(force: boolean = false): void { + const dimensions = this._terminal.dimensions; + if (dimensions) { + const cssW = dimensions.css.canvas.width; + const cssH = dimensions.css.canvas.height; + const devW = dimensions.device.canvas.width; + const devH = dimensions.device.canvas.height; + let recalc = force; + for (const ctx of this._layers.values()) { + if (ctx.canvas.width !== devW + || ctx.canvas.height !== devH + || ctx.canvas.style.width !== `${cssW}px` + || ctx.canvas.style.height !== `${cssH}px` + ) { + ctx.canvas.width = devW; + ctx.canvas.height = devH; + ctx.canvas.style.width = `${cssW}px`; + ctx.canvas.style.height = `${cssH}px`; + recalc = true; + } } - } - } - - /** - * Rescale image in storage if needed. - */ - private _rescaleImage(spec: IImageSpec, currentWidth: number, currentHeight: number): void { - if (currentWidth === spec.actualCellSize.width && currentHeight === spec.actualCellSize.height) { - return; - } - const { width: originalWidth, height: originalHeight } = spec.origCellSize; - if (currentWidth === originalWidth && currentHeight === originalHeight) { - spec.actual = spec.orig; - spec.actualCellSize.width = originalWidth; - spec.actualCellSize.height = originalHeight; - return; - } - const canvas = createCanvas( - this.document, - Math.ceil(spec.orig!.width * currentWidth / originalWidth), - Math.ceil(spec.orig!.height * currentHeight / originalHeight) - ); - const ctx = canvas.getContext('2d'); - if (ctx) { - ctx.drawImage(spec.orig!, 0, 0, canvas.width, canvas.height); - spec.actual = canvas; - spec.actualCellSize.width = currentWidth; - spec.actualCellSize.height = currentHeight; + if (recalc) { + this._dimensions = dimensions; + this._cellSize = this.getCellSize(dimensions); + } + } else { + // we were not able to get render dimensions: + // set cellsize to undefined so we don't try to draw anything + this._cellSize = this.getCellSize(); } } @@ -299,8 +244,10 @@ export class ImageRenderer extends Disposable implements IDisposable { } const canvas = createCanvas( this.document, + this.dimensions?.device.canvas.width || 0, + this.dimensions?.device.canvas.height || 0, this.dimensions?.css.canvas.width || 0, - this.dimensions?.css.canvas.height || 0 + this.dimensions?.css.canvas.height || 0, ); canvas.classList.add(`xterm-image-layer-${layer}`); const screenElement = this._terminal._core.screenElement; @@ -328,6 +275,8 @@ export class ImageRenderer extends Disposable implements IDisposable { return; } this._layers.set(layer, ctx); + // force rescaling to update stored cellSize (maybe not be populated yet) + this.rescaleCanvas(true); this.clearAll(layer); } @@ -343,41 +292,17 @@ export class ImageRenderer extends Disposable implements IDisposable { return this._layers.has(layer); } - private _createPlaceHolder(height: number = Constants.PLACEHOLDER_HEIGHT): void { - this._placeholderBitmap?.close(); - this._placeholderBitmap = undefined; - - // create blueprint to fill placeholder with - const bWidth = 32; // must be 2^n - const blueprint = createCanvas(this.document, bWidth, height); - const ctx = blueprint.getContext('2d', { alpha: false }); + /** + * Create a semi-transparent gray 1x1 placeholder + */ + private _createPlaceHolder(): HTMLCanvasElement | undefined { + const imgData = new ImageData(1, 1); + new Uint32Array(imgData.data.buffer).fill(128 << 24 | 128 << 16 | 128 << 8 | 128); + const canvas = createCanvas(this.document, 1, 1); + const ctx = canvas.getContext('2d', { alpha: true }); if (!ctx) return; - const imgData = new ImageData(bWidth, height); - const d32 = new Uint32Array(imgData.data.buffer); - const black = toRGBA8888(0, 0, 0); - const white = toRGBA8888(255, 255, 255); - d32.fill(black); - for (let y = 0; y < height; ++y) { - const shift = y % 2; - const offset = y * bWidth; - for (let x = 0; x < bWidth; x += 2) { - d32[offset + x + shift] = white; - } - } ctx.putImageData(imgData, 0, 0); - - // create placeholder line, width aligned to blueprint width - const width = (screen.width + bWidth - 1) & ~(bWidth - 1) || Constants.PLACEHOLDER_LENGTH; - this._placeholder = createCanvas(this.document, width, height); - const ctx2 = this._placeholder.getContext('2d', { alpha: false }); - if (!ctx2) { - this._placeholder = undefined; - return; - } - for (let i = 0; i < width; i += bWidth) { - ctx2.drawImage(blueprint, i, 0); - } - createImageBitmap(this._placeholder).then(bitmap => this._placeholderBitmap = bitmap); + return canvas; } public get document(): Document | undefined { diff --git a/addons/addon-image/src/ImageStorage.ts b/addons/addon-image/src/ImageStorage.ts index 17006f26d..cbbb71f57 100644 --- a/addons/addon-image/src/ImageStorage.ts +++ b/addons/addon-image/src/ImageStorage.ts @@ -6,7 +6,9 @@ import { IDisposable } from '@xterm/xterm'; import { ImageRenderer } from './ImageRenderer'; import type { - ITerminalExt, IImageAddonOptions, IImageSpec, ICellSize, IAddImageOpts + ITerminalExt, IImageAddonOptions, IImageSpec, ICellSize, IAddImageOpts, + IDrawable, + IMetrics } from './Types'; import type { IBufferLine } from 'common/buffer/Types'; import { CellData } from 'common/buffer/CellData'; @@ -44,7 +46,7 @@ export class ImageStorage implements IDisposable { // whether render should do a full clear private _needsFullClear = false; // hard limit of stored pixels (fallback limit of 10 MB) - private _pixelLimit: number = 2500000; + private _byteLimit: number = 10000000; private _workCell: CellData = new CellData(); private _viewportMetrics: { cols: number, rows: number }; @@ -84,43 +86,54 @@ export class ImageStorage implements IDisposable { this._renderer.clearAll(); } + /** + * FIXME: below we need to introduce separation of: + * - RAM usage + * - VRAM usage + * - Blob unsage + */ + + public getLimit(): number { - return this._pixelLimit * 4 / 1000000; + return this._byteLimit / 1000000; } public setLimit(value: number): void { if (value < 0.5 || value > 1000) { - throw RangeError('invalid storageLimit, should be at least 0.5 MB and not exceed 1G'); + throw RangeError('invalid storageLimit, should be at least 0.5 MB and not exceed 1GB'); } - this._pixelLimit = (value / 4 * 1000000) >>> 0; + this._byteLimit = (value * 1000000) >>> 0; this._evictOldest(0); } public getUsage(): number { - return this._getStoredPixels() * 4 / 1000000; + return this._getVRAMUsage() / 1000000; } - private _getStoredPixels(): number { - let storedPixels = 0; + private _getVRAMUsage(): number { + let bytes = 0; for (const spec of this._images.values()) { - if (spec.orig) { - storedPixels += spec.orig.width * spec.orig.height; - if (spec.actual && spec.actual !== spec.orig) { - storedPixels += spec.actual.width * spec.actual.height; - } + bytes += spec.src.bytes; + } + return bytes; + } + + public getBlobUsage(): number { + let size = 0; + for (const spec of this._images.values()) { + if (spec.data instanceof Blob) { + size += spec.data.size; } } - return storedPixels; + return size; } private _delImg(id: number): void { const spec = this._images.get(id); if (!spec) return; this._images.delete(id); - // FIXME: really ugly workaround to get bitmaps deallocated :( - if (window.ImageBitmap && spec.orig instanceof ImageBitmap) { - spec.orig.close(); - } + spec.src.close(); + spec.data = undefined; this.onImageDeleted?.(id); } @@ -157,8 +170,10 @@ export class ImageStorage implements IDisposable { } /** - * Method to add an image to the storage. - * @param img - The image to add (canvas or bitmap). + * Method to add an drawable to the storage. + * @param src - The drawable to add. + * @param data - Blob or Uint8Array of the original image data. + * @param metrics - IMetrics of the original image. * @param opts - Options for addImage: * - scrolling: When true, cursor advances with the image. * When false, image is placed at ORIGIN and cursor does not move. @@ -167,17 +182,14 @@ export class ImageStorage implements IDisposable { * - cursorPos: 'vt340' for bottom-left, 'iip' for bottom.right. * @returns The internal image ID assigned to the stored image. */ - public addImage(img: HTMLCanvasElement | ImageBitmap, opts: IAddImageOpts): number { + public addImage(src: IDrawable, data: Blob | undefined, metrics: IMetrics, opts: IAddImageOpts): number { // never allow storage to exceed memory limit - this._evictOldest(img.width * img.height); + this._evictOldest(src.bytes); // calc rows x cols needed to display the image - let cellSize = this._renderer.cellSize; - if (cellSize.width === -1 || cellSize.height === -1) { - cellSize = CELL_SIZE_DEFAULT; - } - const cols = Math.ceil(img.width / cellSize.width); - const rows = Math.ceil(img.height / cellSize.height); + const cellSize = this._renderer.getCellSize() ?? CELL_SIZE_DEFAULT; + const cols = Math.ceil(src.width / cellSize.width); + const rows = Math.ceil(src.height / cellSize.height); const imageId = ++this._lastId; @@ -254,10 +266,10 @@ export class ImageStorage implements IDisposable { // create storage entry const imgSpec: IImageSpec = { - orig: img, - origCellSize: cellSize, - actual: img, - actualCellSize: { ...cellSize }, // clone needed, since later modified + src, + data, + metrics, + cellSize, marker: endMarker || undefined, tileCount, bufferType: this._terminal.buffer.active.type, @@ -377,9 +389,7 @@ export class ImageStorage implements IDisposable { } col--; if (imgSpec) { - if (imgSpec.actual) { - drawCalls.push({ imgSpec, tileId: startTile, col: startCol, row, count }); - } + drawCalls.push({ imgSpec, tileId: startTile, col: startCol, row, count }); } else if (this._opts.showPlaceholder) { placeholderCalls.push({ col: startCol, row, count }); } @@ -434,7 +444,7 @@ export class ImageStorage implements IDisposable { continue; } // found an image tile at oldCol, check if it qualifies for right exapansion - const tilesPerRow = Math.ceil((imgSpec.actual?.width || 0) / imgSpec.actualCellSize.width); + const tilesPerRow = Math.ceil((imgSpec.src.width) / imgSpec.cellSize.width); if ((e.tileId % tilesPerRow) + 1 >= tilesPerRow) { continue; } @@ -471,30 +481,13 @@ export class ImageStorage implements IDisposable { if (line) { const e = line.getExtended(x)?.payload; if (e instanceof ImageTileInfo && e.imageId && e.imageId !== -1) { - const orig = this._images.get(e.imageId)?.orig; - if (window.ImageBitmap && orig instanceof ImageBitmap) { - const canvas = createCanvas(window.document, orig.width, orig.height); - canvas.getContext('2d')?.drawImage(orig, 0, 0, orig.width, orig.height); + const src = this._images.get(e.imageId)?.src; + if (src?.native instanceof ImageBitmap || src?.native instanceof VideoFrame) { + const canvas = createCanvas(window.document, src.width, src.height); + canvas.getContext('2d')?.drawImage(src.native, 0, 0, src.width, src.height); return canvas; } - return orig as HTMLCanvasElement; - } - } - } - - /** - * Extract active single tile at buffer position. - */ - public extractTileAtBufferCell(x: number, y: number): HTMLCanvasElement | undefined { - const buffer = this._terminal._core.buffer; - const line = buffer.lines.get(y); - if (line) { - const e = line.getExtended(x)?.payload; - if (e instanceof ImageTileInfo && e.imageId && e.imageId !== -1 && e.tileId !== -1) { - const spec = this._images.get(e.imageId); - if (spec) { - return this._renderer.extractTile(spec, e.tileId); - } + return src?.native; } } } @@ -502,15 +495,12 @@ export class ImageStorage implements IDisposable { // TODO: Do we need some blob offloading tricks here to avoid early eviction? // also see https://stackoverflow.com/questions/28307789/is-there-any-limitation-on-javascript-max-blob-size private _evictOldest(room: number): number { - const used = this._getStoredPixels(); + const used = this._getVRAMUsage(); let current = used; - while (this._pixelLimit < current + room && this._images.size) { + while (this._byteLimit < current + room && this._images.size) { const spec = this._images.get(++this._lowestId); - if (spec && spec.orig) { - current -= spec.orig.width * spec.orig.height; - if (spec.actual && spec.orig !== spec.actual) { - current -= spec.actual.width * spec.actual.height; - } + if (spec) { + current -= spec.src.bytes; spec.marker?.dispose(); this._delImg(this._lowestId); } diff --git a/addons/addon-image/src/IIPMetrics.test.ts b/addons/addon-image/src/Metrics.test.ts similarity index 97% rename from addons/addon-image/src/IIPMetrics.test.ts rename to addons/addon-image/src/Metrics.test.ts index 4df320a2c..ea8a6403c 100644 --- a/addons/addon-image/src/IIPMetrics.test.ts +++ b/addons/addon-image/src/Metrics.test.ts @@ -4,7 +4,8 @@ */ import { assert } from 'chai'; -import { imageType, IMetrics } from './IIPMetrics'; +import { imageType } from './Metrics'; +import type { IMetrics } from 'Types'; // fix missing nodejs decl declare const require: (s: string) => any; diff --git a/addons/addon-image/src/IIPMetrics.ts b/addons/addon-image/src/Metrics.ts similarity index 94% rename from addons/addon-image/src/IIPMetrics.ts rename to addons/addon-image/src/Metrics.ts index 217a7fc93..14b02e00f 100644 --- a/addons/addon-image/src/IIPMetrics.ts +++ b/addons/addon-image/src/Metrics.ts @@ -3,14 +3,9 @@ * @license MIT */ +import { IMetrics } from './Types'; -export type ImageType = 'image/png' | 'image/jpeg' | 'image/gif' | 'image/qoi' | 'image/webp' | 'image/avif' | 'unsupported' | ''; -export interface IMetrics { - mime: ImageType; - width: number; - height: number; -} export const UNSUPPORTED_TYPE: IMetrics = { mime: 'unsupported', diff --git a/addons/addon-image/src/Primitives.ts b/addons/addon-image/src/Primitives.ts index 86a3c5121..12797dba3 100644 --- a/addons/addon-image/src/Primitives.ts +++ b/addons/addon-image/src/Primitives.ts @@ -3,7 +3,15 @@ * @license MIT */ -export function createCanvas(localDocument: Document | undefined, width: number, height: number): HTMLCanvasElement { +import { IDrawable } from './Types'; + +export function createCanvas( + localDocument: Document | undefined, + width: number, + height: number, + cssWidth?: number, + cssHeight?: number +): HTMLCanvasElement { /** * NOTE: We normally dont care, from which document the canvas * gets created, so we can fall back to global document, @@ -16,6 +24,45 @@ export function createCanvas(localDocument: Document | undefined, width: number, const canvas = (localDocument ?? document).createElement('canvas'); canvas.width = width | 0; canvas.height = height | 0; + canvas.style.width = `${(cssWidth ?? width) | 0}px`; + canvas.style.height = `${(cssHeight ?? height) | 0}px`; return canvas; } + +/** + * Drawable is a resource the ImageRenderer knows to draw on the screen. + */ +export class Drawable implements IDrawable { + public readonly width: number; + public readonly height: number; + // @readonly: only for memory tracking + public bytes: number; + + constructor(public native: ImageBitmap | VideoFrame | HTMLCanvasElement) { + if (native instanceof VideoFrame) { + this.width = native.displayWidth; + this.height = native.displayHeight; + // NOTE: native.allocationSize() is too expensive + this.bytes = native.codedWidth * native.codedHeight * 4; + } else { + this.width = native.width; + this.height = native.height; + this.bytes = native.width * native.height * 4; + } + } + + public close(): void { + if (this.bytes === 0) return; + this.bytes = 0; + try { + if (this.native instanceof HTMLCanvasElement) { + this.native.width = 0; + this.native.height = 0; + } else { + this.native.close?.(); + } + } catch {} + this.native = null!; + } +} diff --git a/addons/addon-image/src/SixelHandler.ts b/addons/addon-image/src/SixelHandler.ts index 92b3f337c..1f79e77bb 100644 --- a/addons/addon-image/src/SixelHandler.ts +++ b/addons/addon-image/src/SixelHandler.ts @@ -9,7 +9,7 @@ import { toRGBA8888, BIG_ENDIAN, PALETTE_ANSI_256, PALETTE_VT340_COLOR } from 's import { RGBA8888 } from 'sixel/lib/Types'; import { DecoderAsync, Decoder } from 'sixel/lib/Decoder'; -import { createCanvas } from './Primitives'; +import { createCanvas, Drawable } from './Primitives'; // always free decoder ressources after decoding if it exceeds this limit const MEM_PERMA_LIMIT = 4194304; // 1024 pixels * 1024 pixels * 4 channels = 4MB @@ -103,7 +103,8 @@ export class SixelHandler implements IDcsHandler, IResetHandler { if (this._dec.memoryUsage > MEM_PERMA_LIMIT) { this._dec.release(); } - this._storage.addImage(canvas); + // FIXME: store sixel data in blob + this._storage.addImage(new Drawable(canvas), undefined, { width, height, mime: 'image/sixel' }); return true; } } diff --git a/addons/addon-image/src/SixelImageStorage.ts b/addons/addon-image/src/SixelImageStorage.ts index c2376a68a..b33d030fe 100644 --- a/addons/addon-image/src/SixelImageStorage.ts +++ b/addons/addon-image/src/SixelImageStorage.ts @@ -4,7 +4,7 @@ */ import { ImageStorage, CELL_SIZE_DEFAULT } from './ImageStorage'; -import { IImageAddonOptions, ITerminalExt, IAddImageOpts } from './Types'; +import { IImageAddonOptions, ITerminalExt, IAddImageOpts, IDrawable, IMetrics } from './Types'; import { ImageRenderer } from './ImageRenderer'; /** @@ -27,9 +27,9 @@ export class SixelImageStorage { * Add a sixel image to storage. * Cursor behavior depends on the sixelScrolling option (DECSET 80). */ - public addImage(img: HTMLCanvasElement | ImageBitmap): void { + public addImage(src: IDrawable, data: Blob | undefined, metrics: IMetrics): void { this._addImageOpts.scrolling = this._opts.sixelScrolling; - this._storage.addImage(img, this._addImageOpts); + this._storage.addImage(src, data, metrics, this._addImageOpts); } /** @@ -39,10 +39,7 @@ export class SixelImageStorage { */ public advanceCursor(height: number): void { if (this._opts.sixelScrolling) { - let cellSize = this._renderer.cellSize; - if (cellSize.width === -1 || cellSize.height === -1) { - cellSize = CELL_SIZE_DEFAULT; - } + const cellSize = this._renderer.getCellSize() ?? CELL_SIZE_DEFAULT; const rows = Math.ceil(height / cellSize.height); for (let i = 1; i < rows; ++i) { this._terminal._core._inputHandler.lineFeed(); diff --git a/addons/addon-image/src/Types.ts b/addons/addon-image/src/Types.ts index 04328420c..86d5679bb 100644 --- a/addons/addon-image/src/Types.ts +++ b/addons/addon-image/src/Types.ts @@ -96,14 +96,49 @@ export interface IAddImageOpts { cursorPos: CursorPos; } +export interface IDrawable { + native: ImageBitmap | VideoFrame | HTMLCanvasElement; + width: number; + height: number; + close(): void; + readonly bytes: number; +} + +export type ImageType = 'unsupported' + | 'image/png' + | 'image/jpeg' + | 'image/gif' + | 'image/qoi' + | 'image/webp' + | 'image/avif' + // sixel + | 'image/sixel' + // rgb|a blobs for kitty + | 'image/rgb' + | 'image/rgba'; + +export interface IMetrics { + mime: ImageType; + width: number; + height: number; +} + export interface IImageSpec { - orig: HTMLCanvasElement | ImageBitmap | undefined; - origCellSize: ICellSize; - actual: HTMLCanvasElement | ImageBitmap | undefined; - actualCellSize: ICellSize; + /** drawable for screen rendering */ + src: IDrawable; + /** image bytes as blob */ + data: Blob | undefined; + /** metrics about image like mime and dimensions */ + metrics: IMetrics; + /** cell size at time of insert */ + cellSize: ICellSize; + /** eviction marker */ marker: IMarker | undefined; + /** used tiles as eviction hint */ tileCount: number; + /** buffer it was placed on */ bufferType: 'alternate' | 'normal'; + /** layer it was placed on */ layer: ImageLayer; zIndex: number; } diff --git a/addons/addon-image/src/kitty/KittyImageStorage.ts b/addons/addon-image/src/kitty/KittyImageStorage.ts index 1f5c09ec9..5020ee5a9 100644 --- a/addons/addon-image/src/kitty/KittyImageStorage.ts +++ b/addons/addon-image/src/kitty/KittyImageStorage.ts @@ -7,6 +7,8 @@ import { IDisposable } from '@xterm/xterm'; import { ImageStorage } from '../ImageStorage'; import { ImageLayer, IAddImageOpts } from '../Types'; import { IKittyImageData } from './KittyGraphicsTypes'; +import { Drawable } from '../Primitives'; +import { UNSUPPORTED_TYPE } from '../Metrics'; // Kitty-specific image storage controller. // @@ -102,7 +104,8 @@ export class KittyImageStorage implements IDisposable { this._addImageOpts.scrolling = scrolling; this._addImageOpts.layer = layer; this._addImageOpts.zIndex = zIndex; - const storageId = this._storage.addImage(image, this._addImageOpts); + // FIXME: store image data in blob, fill metrics correctly + const storageId = this._storage.addImage(new Drawable(image), undefined, UNSUPPORTED_TYPE, this._addImageOpts); this._kittyIdToStorageId.set(kittyId, storageId); this._storageIdToKittyId.set(storageId, kittyId); } diff --git a/addons/addon-image/test/ImageAddon.test.ts b/addons/addon-image/test/ImageAddon.test.ts index f4e54d7f2..315112e1a 100644 --- a/addons/addon-image/test/ImageAddon.test.ts +++ b/addons/addon-image/test/ImageAddon.test.ts @@ -294,10 +294,10 @@ test.describe('ImageAddon', () => { test('size params is optional/informative', async () => { // too small size param await ctx.proxy.write(`\x1b]1337;File=inline=1;size=5:${PALETTE_PNG_BASE64}\x07`); - deepStrictEqual(await getOrigSize(1), [640, 80]); + deepStrictEqual(await getImageSize(1), [640, 80]); // no size aparm at all await ctx.proxy.write(`\x1b[10H\x1b]1337;File=inline=1:${PALETTE_QOI_BASE64}\x07`); - deepStrictEqual(await getOrigSize(2), [640, 80]); + deepStrictEqual(await getImageSize(2), [640, 80]); }); test.skip('FilePart support - bytewise', async () => { // opener @@ -308,7 +308,7 @@ test.describe('ImageAddon', () => { } // finalizer await ctx.proxy.write(`\x1b]1337;FileEnd\x07`); - deepStrictEqual(await getOrigSize(1), [640, 80]); + deepStrictEqual(await getImageSize(1), [640, 80]); }); test('FilePart support - chunks', async () => { // opener @@ -319,7 +319,7 @@ test.describe('ImageAddon', () => { } // finalizer await ctx.proxy.write(`\x1b]1337;FileEnd\x07`); - deepStrictEqual(await getOrigSize(1), [640, 80]); + deepStrictEqual(await getImageSize(1), [640, 80]); }); test('FilePart support - edgecases', async () => { // multiple opener do not harm @@ -347,39 +347,39 @@ test.describe('ImageAddon', () => { // finalizer skipped // write spinfox.png await ctx.proxy.write(TESTDATA_IIP[1][0]); - deepStrictEqual(await getOrigSize(1), TESTDATA_IIP[1][1]); + deepStrictEqual(await getImageSize(1), TESTDATA_IIP[1][1]); }); }); test.describe('IIP support - testimages', () => { test('palette.png', async () => { await ctx.proxy.write(TESTDATA_IIP[0][0]); - deepStrictEqual(await getOrigSize(1), TESTDATA_IIP[0][1]); + deepStrictEqual(await getImageSize(1), TESTDATA_IIP[0][1]); }); test('spinfox.png', async () => { await ctx.proxy.write(TESTDATA_IIP[1][0]); - deepStrictEqual(await getOrigSize(1), TESTDATA_IIP[1][1]); + deepStrictEqual(await getImageSize(1), TESTDATA_IIP[1][1]); }); test('w3c gif', async () => { await ctx.proxy.write(TESTDATA_IIP[2][0]); - deepStrictEqual(await getOrigSize(1), TESTDATA_IIP[2][1]); + deepStrictEqual(await getImageSize(1), TESTDATA_IIP[2][1]); }); test('w3c jpeg', async () => { await ctx.proxy.write(TESTDATA_IIP[3][0]); - deepStrictEqual(await getOrigSize(1), TESTDATA_IIP[3][1]); + deepStrictEqual(await getImageSize(1), TESTDATA_IIP[3][1]); }); test('w3c png', async () => { await ctx.proxy.write(TESTDATA_IIP[4][0]); - deepStrictEqual(await getOrigSize(1), TESTDATA_IIP[4][1]); + deepStrictEqual(await getImageSize(1), TESTDATA_IIP[4][1]); }); }); test.describe('IIP - QOI support', () => { test('palette should yield same bytes from PNG and QOI', async () => { await ctx.proxy.write(`\x1b]1337;File=inline=1;size=525:${PALETTE_PNG_BASE64}\x07`); - deepStrictEqual(await getOrigSize(1), [640, 80]); + deepStrictEqual(await getImageSize(1), [640, 80]); await ctx.proxy.write(`\x1b[10H\x1b]1337;File=inline=1;size=${qoiData.length}:${PALETTE_QOI_BASE64}\x07`); - deepStrictEqual(await getOrigSize(2), [640, 80]); + deepStrictEqual(await getImageSize(2), [640, 80]); const pngScrape = await getImageAtBufferCell(0, 0); const qoiScrape = await getImageAtBufferCell(0, 11); deepStrictEqual(qoiScrape, pngScrape); @@ -406,20 +406,20 @@ test.describe('ImageAddon', () => { const header = 'width=20;height=5;preserveAspectRatio=0'; await ctx.proxy.write(`\x1b]1337;File=inline=1;size=${size};${header}:${payload}\x07`); const dim = await getDimensions(); - deepStrictEqual(await getOrigSize(1), [dim.cellWidth * 20, dim.cellHeight * 5]); + deepStrictEqual(await getImageSize(1), [dim.cellWidth * 20, dim.cellHeight * 5]); }); test(name + ': Npx --> width=320px height=160px preserveAspectRatio=0', async () => { // pixel based resize const header = 'width=320px;height=160px;preserveAspectRatio=0'; await ctx.proxy.write(`\x1b]1337;File=inline=1;size=${size};${header}:${payload}\x07`); - deepStrictEqual(await getOrigSize(1), [320, 160]); + deepStrictEqual(await getImageSize(1), [320, 160]); }); test(name + ': N% --> width=50% height=30% preserveAspectRatio=0', async () => { // % of viewport resize const header = 'width=50%;height=30%;preserveAspectRatio=0'; await ctx.proxy.write(`\x1b]1337;File=inline=1;size=${size};${header}:${payload}\x07`); const dim = await getDimensions(); - deepStrictEqual(await getOrigSize(1), [Math.floor(dim.width * 0.5), Math.floor(dim.height * 0.3)]); + deepStrictEqual(await getImageSize(1), [Math.floor(dim.width * 0.5), Math.floor(dim.height * 0.3)]); }); test(name + ': ommitted dimension assumes preserveAspectRatio=1', async () => { // width provided in percent @@ -427,11 +427,11 @@ test.describe('ImageAddon', () => { await ctx.proxy.write(`\x1b]1337;File=inline=1;size=${size};${header}:${payload}\x07`); const dim = await getDimensions(); const width = Math.floor(dim.width * 0.5); - deepStrictEqual(await getOrigSize(1), [width, Math.floor(width * 80 / 640)]); + deepStrictEqual(await getImageSize(1), [width, Math.floor(width * 80 / 640)]); // height provided in pixel const header2 = 'height=200px'; await ctx.proxy.write(`\x1b]1337;File=inline=1;size=${size};${header2}:${payload}\x07`); - deepStrictEqual(await getOrigSize(2), [Math.floor(200 * 640 / 80), 200]); + deepStrictEqual(await getImageSize(2), [Math.floor(200 * 640 / 80), 200]); }); } }); @@ -471,10 +471,10 @@ async function getScrollbackPlusRows(): Promise { return ctx.page.evaluate('window.term.options.scrollback + window.term.rows'); } -async function getOrigSize(id: number): Promise<[number, number]> { +async function getImageSize(id: number): Promise<[number, number]> { return ctx.page.evaluate(`[ - window.imageAddon._storage._images.get(${id}).orig.width, - window.imageAddon._storage._images.get(${id}).orig.height + window.imageAddon._storage._images.get(${id}).src.width, + window.imageAddon._storage._images.get(${id}).src.height ]`); } @@ -483,12 +483,12 @@ async function getImageAtBufferCell(x: number, y: number): Promise { - return ctx.page.evaluate(`!!window.imageAddon.extractTileAtBufferCell(${x}, ${y})`); + return ctx.page.evaluate(`!!window.imageAddon.getImageAtBufferCell(${x}, ${y})`); } async function assertTextClearsTiles(imageSeq: string): Promise { await ctx.proxy.write('\x1b[H' + imageSeq); - await pollFor(ctx.page, '!!window.imageAddon.extractTileAtBufferCell(5, 1)', true); + await pollFor(ctx.page, '!!window.imageAddon.getImageAtBufferCell(5, 1)', true); ok(await hasTileAtBufferCell(0, 1)); await ctx.proxy.write('\x1b[2;6H#######'); for (let x = 5; x < 12; x++) { diff --git a/addons/addon-image/test/KittyGraphics.test.ts b/addons/addon-image/test/KittyGraphics.test.ts index 8c1833454..5f52ddeef 100644 --- a/addons/addon-image/test/KittyGraphics.test.ts +++ b/addons/addon-image/test/KittyGraphics.test.ts @@ -123,7 +123,7 @@ test.describe('Kitty Graphics Protocol', () => { await ctx.proxy.write(seq); await timeout(100); strictEqual(await getImageStorageLength(), 1); - deepStrictEqual(await getOrigSize(1), [1, 1]); + deepStrictEqual(await getImageSize(1), [1, 1]); }); test('stores 3x1 RGB PNG with a=T', async () => { @@ -131,7 +131,7 @@ test.describe('Kitty Graphics Protocol', () => { await ctx.proxy.write(seq); await timeout(100); strictEqual(await getImageStorageLength(), 1); - deepStrictEqual(await getOrigSize(1), [3, 1]); + deepStrictEqual(await getImageSize(1), [3, 1]); }); test('transmit only (a=t) does not display but stores in handler', async () => { @@ -1107,7 +1107,7 @@ test.describe('Kitty Graphics Protocol', () => { await timeout(200); strictEqual(await getImageStorageLength(), 1); - deepStrictEqual(await getOrigSize(1), [20, 50]); + deepStrictEqual(await getImageSize(1), [20, 50]); deepStrictEqual(await getPixel(0, 0, 0, 0), [255, 128, 0, 255]); }); @@ -1535,7 +1535,7 @@ test.describe('Kitty Graphics Protocol', () => { await ctx.proxy.write(`\x1b_Ga=T,f=100;${KITTY_MULTICOLOR_200X100_BASE64}\x1b\\`); await timeout(200); strictEqual(await getImageStorageLength(), 1); - deepStrictEqual(await getOrigSize(1), [200, 100]); + deepStrictEqual(await getImageSize(1), [200, 100]); }); test('transmit only (a=t) stores 200x100 image without display', async () => { @@ -1564,7 +1564,7 @@ test.describe('Kitty Graphics Protocol', () => { await ctx.proxy.write(`\x1b_Ga=T,f=100,i=500;${part2}\x1b\\`); await timeout(200); strictEqual(await getImageStorageLength(), 1); - deepStrictEqual(await getOrigSize(1), [200, 100]); + deepStrictEqual(await getImageSize(1), [200, 100]); }); test('handles 3-chunk transmission', async () => { @@ -1580,7 +1580,7 @@ test.describe('Kitty Graphics Protocol', () => { await ctx.proxy.write(`\x1b_Ga=T,f=100,i=501;${p3}\x1b\\`); await timeout(200); strictEqual(await getImageStorageLength(), 1); - deepStrictEqual(await getOrigSize(1), [200, 100]); + deepStrictEqual(await getImageSize(1), [200, 100]); }); test('verifies chunked data assembles correctly', async () => { @@ -1713,7 +1713,7 @@ test.describe('Kitty Graphics Protocol', () => { await ctx.proxy.write(`\x1b_Ga=T,f=100,x=20,y=0,w=20,h=50;${KITTY_MULTICOLOR_200X100_BASE64}\x1b\\`); await timeout(200); - deepStrictEqual(await getOrigSize(1), [20, 50]); + deepStrictEqual(await getImageSize(1), [20, 50]); deepStrictEqual(await getPixel(0, 0, 0, 0), [255, 128, 0, 255]); deepStrictEqual(await getPixel(0, 0, 19, 49), [255, 128, 0, 255]); }); @@ -1747,7 +1747,7 @@ test.describe('Kitty Graphics Protocol', () => { await ctx.proxy.write(`\x1b_Ga=T,f=100,w=0;${KITTY_MULTICOLOR_200X100_BASE64}\x1b\\`); await timeout(200); - deepStrictEqual(await getOrigSize(1), [200, 100]); + deepStrictEqual(await getImageSize(1), [200, 100]); deepStrictEqual(await getPixel(0, 0, 0, 0), [255, 0, 0, 255]); deepStrictEqual(await getPixel(0, 0, 199, 99), [255, 255, 255, 255]); }); @@ -1756,7 +1756,7 @@ test.describe('Kitty Graphics Protocol', () => { await ctx.proxy.write(`\x1b_Ga=T,f=100,h=0;${KITTY_MULTICOLOR_200X100_BASE64}\x1b\\`); await timeout(200); - deepStrictEqual(await getOrigSize(1), [200, 100]); + deepStrictEqual(await getImageSize(1), [200, 100]); deepStrictEqual(await getPixel(0, 0, 0, 0), [255, 0, 0, 255]); deepStrictEqual(await getPixel(0, 0, 0, 50), [255, 192, 203, 255]); }); @@ -1772,7 +1772,7 @@ test.describe('Kitty Graphics Protocol', () => { await ctx.proxy.write(`\x1b_Ga=T,f=100,x=-10,y=-10;${KITTY_MULTICOLOR_200X100_BASE64}\x1b\\`); await timeout(200); - deepStrictEqual(await getOrigSize(1), [200, 100]); + deepStrictEqual(await getImageSize(1), [200, 100]); deepStrictEqual(await getPixel(0, 0, 0, 0), [255, 0, 0, 255]); }); @@ -1893,28 +1893,28 @@ test.describe('Kitty Graphics Protocol', () => { await ctx.proxy.write(`\x1b_Ga=T,f=24,s=3,v=1;${RAW_RGB_3X1}\x1b\\`); await timeout(100); strictEqual(await getImageStorageLength(), 1); - deepStrictEqual(await getOrigSize(1), [3, 1]); + deepStrictEqual(await getImageSize(1), [3, 1]); }); test('stores image with correct original dimensions (2x2)', async () => { await ctx.proxy.write(`\x1b_Ga=T,f=24,s=2,v=2;${RAW_RGB_2X2}\x1b\\`); await timeout(100); strictEqual(await getImageStorageLength(), 1); - deepStrictEqual(await getOrigSize(1), [2, 2]); + deepStrictEqual(await getImageSize(1), [2, 2]); }); test('stores image with correct original dimensions (5x1)', async () => { await ctx.proxy.write(`\x1b_Ga=T,f=24,s=5,v=1;${RAW_RGB_5X1}\x1b\\`); await timeout(100); strictEqual(await getImageStorageLength(), 1); - deepStrictEqual(await getOrigSize(1), [5, 1]); + deepStrictEqual(await getImageSize(1), [5, 1]); }); test('stores image with correct original dimensions (4x2)', async () => { await ctx.proxy.write(`\x1b_Ga=T,f=24,s=4,v=2;${RAW_RGB_4X2}\x1b\\`); await timeout(100); strictEqual(await getImageStorageLength(), 1); - deepStrictEqual(await getOrigSize(1), [4, 2]); + deepStrictEqual(await getImageSize(1), [4, 2]); }); }); @@ -2034,21 +2034,21 @@ test.describe('Kitty Graphics Protocol', () => { await ctx.proxy.write(`\x1b_Ga=T,f=32,s=3,v=1;${RAW_RGBA_3X1}\x1b\\`); await timeout(100); strictEqual(await getImageStorageLength(), 1); - deepStrictEqual(await getOrigSize(1), [3, 1]); + deepStrictEqual(await getImageSize(1), [3, 1]); }); test('stores image with correct original dimensions (2x2)', async () => { await ctx.proxy.write(`\x1b_Ga=T,f=32,s=2,v=2;${RAW_RGBA_2X2}\x1b\\`); await timeout(100); strictEqual(await getImageStorageLength(), 1); - deepStrictEqual(await getOrigSize(1), [2, 2]); + deepStrictEqual(await getImageSize(1), [2, 2]); }); test('stores image with correct original dimensions (5x1)', async () => { await ctx.proxy.write(`\x1b_Ga=T,f=32,s=5,v=1;${RAW_RGBA_5X1}\x1b\\`); await timeout(100); strictEqual(await getImageStorageLength(), 1); - deepStrictEqual(await getOrigSize(1), [5, 1]); + deepStrictEqual(await getImageSize(1), [5, 1]); }); }); @@ -2217,7 +2217,7 @@ test.describe('Kitty Graphics Protocol', () => { test.describe('text overwrite removes tiles', () => { test('a=T placement', async () => { await ctx.proxy.write(`\x1b[H\x1b_Ga=T,f=100;${KITTY_MULTICOLOR_200X100_BASE64}\x1b\\`); - await pollFor(ctx.page, '!!window.imageAddon.extractTileAtBufferCell(5, 1)', true); + await pollFor(ctx.page, '!!window.imageAddon.getImageAtBufferCell(5, 1)', true); ok(await hasTileAtBufferCell(0, 1)); await ctx.proxy.write('\x1b[2;6H#######'); for (let x = 5; x < 12; x++) { @@ -2251,10 +2251,10 @@ async function getImageStorageLength(): Promise { return ctx.page.evaluate('window.imageAddon._storage._images.size'); } -async function getOrigSize(id: number): Promise<[number, number]> { +async function getImageSize(id: number): Promise<[number, number]> { return ctx.page.evaluate(`[ - window.imageAddon._storage._images.get(${id}).orig.width, - window.imageAddon._storage._images.get(${id}).orig.height + window.imageAddon._storage._images.get(${id}).src.width, + window.imageAddon._storage._images.get(${id}).src.height ]`); } @@ -2279,5 +2279,5 @@ async function getPixels(col: number, row: number, x: number, y: number, w: numb } async function hasTileAtBufferCell(x: number, y: number): Promise { - return ctx.page.evaluate(`!!window.imageAddon.extractTileAtBufferCell(${x}, ${y})`); + return ctx.page.evaluate(`!!window.imageAddon.getImageAtBufferCell(${x}, ${y})`); } diff --git a/addons/addon-image/typings/addon-image.d.ts b/addons/addon-image/typings/addon-image.d.ts index a3fc9e1c8..f7f4d9fac 100644 --- a/addons/addon-image/typings/addon-image.d.ts +++ b/addons/addon-image/typings/addon-image.d.ts @@ -125,10 +125,5 @@ declare module '@xterm/addon-image' { * Get original image canvas at buffer position. */ public getImageAtBufferCell(x: number, y: number): HTMLCanvasElement | undefined; - - /** - * Extract single tile canvas at buffer position. - */ - public extractTileAtBufferCell(x: number, y: number): HTMLCanvasElement | undefined; } } diff --git a/src/browser/CoreBrowserTerminal.ts b/src/browser/CoreBrowserTerminal.ts index 67893b796..97b1bb885 100644 --- a/src/browser/CoreBrowserTerminal.ts +++ b/src/browser/CoreBrowserTerminal.ts @@ -1137,13 +1137,13 @@ export class CoreBrowserTerminal extends CoreTerminal implements ITerminal { switch (type) { case WindowsOptionsReportType.GET_WIN_SIZE_PIXELS: - const canvasWidth = this._renderService.dimensions.css.canvas.width.toFixed(0); - const canvasHeight = this._renderService.dimensions.css.canvas.height.toFixed(0); + const canvasWidth = this._renderService.dimensions.device.canvas.width.toFixed(0); + const canvasHeight = this._renderService.dimensions.device.canvas.height.toFixed(0); this.coreService.triggerDataEvent(`${C0.ESC}[4;${canvasHeight};${canvasWidth}t`); break; case WindowsOptionsReportType.GET_CELL_SIZE_PIXELS: - const cellWidth = this._renderService.dimensions.css.cell.width.toFixed(0); - const cellHeight = this._renderService.dimensions.css.cell.height.toFixed(0); + const cellWidth = this._renderService.dimensions.device.cell.width.toFixed(0); + const cellHeight = this._renderService.dimensions.device.cell.height.toFixed(0); this.coreService.triggerDataEvent(`${C0.ESC}[6;${cellHeight};${cellWidth}t`); break; }