eiac/xterm.js
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

add painting primitives to IImageSpec, move IIP resize to late rendering

Browse Source
This commit is contained in:
Jörg Breitbart
2026-09-02 21:50:25 +02:00
parent c9aabf9cf5
commit 039e103ed8
10 changed files with 98 additions and 47 deletions
Download Patch File Download Diff File Expand all files Collapse all files
addons/addon-image/src/IIPHandler.ts
+26 -14
View File
@@ -3,7 +3,6 @@
* @license MIT
*/
import { IImageAddonOptions, IOscHandler, IResetHandler, ITerminalExt } from './Types';
import { ImageRenderer } from './ImageRenderer';
import { IIPImageStorage } from './IIPImageStorage';
import { CELL_SIZE_DEFAULT } from './ImageStorage';
import Base64Decoder from 'xterm-wasm-parts/lib/base64/Base64Decoder.wasm';
@@ -45,7 +44,6 @@ export class IIPHandler implements IOscHandler, IResetHandler {
constructor(
private readonly _opts: IImageAddonOptions,
private readonly _renderer: ImageRenderer,
private readonly _storage: IIPImageStorage,
private readonly _coreTerminal: ITerminalExt
) {
@@ -119,11 +117,13 @@ export class IIPHandler implements IOscHandler, IResetHandler {
if (seqType === SequenceType.REPORTCELLSIZE) {
// OSC 1337 ; ReportCellSize=[height];[width];[scale] ST
// IMPORTANT: ReportCellSize uses logical points (CSS pixels)
let w = CELL_SIZE_DEFAULT.width;
let h = CELL_SIZE_DEFAULT.height;
if (this._renderer.dimensions) {
w = this._renderer.dimensions.css.canvas.width / this._coreTerminal.cols;
h = this._renderer.dimensions.css.canvas.height / this._coreTerminal.rows;
const dimensions = this._coreTerminal.dimensions;
if (dimensions) {
w = dimensions.css.canvas.width / this._coreTerminal.cols;
h = dimensions.css.canvas.height / this._coreTerminal.rows;
}
const scale = this._coreTerminal._core._coreBrowserService?.dpr ?? 1;
const report = `\x1b]1337;ReportCellSize=${h.toFixed(3)};${w.toFixed(3)};${scale.toFixed(3)}\x1b\\`;
@@ -161,7 +161,7 @@ export class IIPHandler implements IOscHandler, IResetHandler {
w = metrics.width;
h = metrics.height;
if (cond = w && h && w * h < this._opts.pixelLimit) {
[w, h] = this._resize(w, h).map(Math.floor);
[w, h] = this._resize(w, h);
cond = w && h && w * h < this._opts.pixelLimit;
} else {
console.warn(`IIP: image dimension issue ${metrics.width}x${metrics.height}`);
@@ -192,9 +192,9 @@ export class IIPHandler implements IOscHandler, IResetHandler {
imgBlob = bmSrc = new Blob([this._dec.data8], { type: metrics.mime });
}
this._dec.release();
return createImageBitmap(bmSrc, { resizeWidth: w, resizeHeight: h })
return createImageBitmap(bmSrc)
.then(bm => {
this._storage.addImage(new Drawable(bm), imgBlob, metrics);
this._storage.addImage(new Drawable(bm), imgBlob, metrics, w / metrics.width, h / metrics.height);
return true;
})
.catch(e => {
@@ -204,13 +204,25 @@ export class IIPHandler implements IOscHandler, IResetHandler {
}
private _resize(w: number, h: number): [number, number] {
const cw = this._renderer.dimensions?.css.cell.width || CELL_SIZE_DEFAULT.width;
const ch = this._renderer.dimensions?.css.cell.height || CELL_SIZE_DEFAULT.height;
const width = this._renderer.dimensions?.css.canvas.width || cw * this._coreTerminal.cols;
const height = this._renderer.dimensions?.css.canvas.height || ch * this._coreTerminal.rows;
let cw;
let ch;
let width;
let height;
const dimensions = this._coreTerminal.dimensions;
if (dimensions) {
width = dimensions.device.canvas.width;
height = dimensions.device.canvas.height;
cw = width / this._coreTerminal.cols;
ch = height / this._coreTerminal.rows;
} else {
cw = CELL_SIZE_DEFAULT.width;
ch = CELL_SIZE_DEFAULT.height;
width = cw * this._coreTerminal.cols;
height = ch * this._coreTerminal.rows;
}
const rw = this._dim(this._header.width!, width, cw);
const rh = this._dim(this._header.height!, height, ch);
const rw = this._dim(this._header.width ?? '', width, cw);
const rh = this._dim(this._header.height ?? '', height, ch);
if (!rw && !rh) {
const wf = width / w; // TODO: should this respect initial cursor offset?
const hf = (height - ch) / h; // TODO: fix offset issues from float cell height
addons/addon-image/src/IIPImageStorage.ts
+3 -1
View File
@@ -22,7 +22,9 @@ export class IIPImageStorage {
* Add an IIP image to storage.
* Always uses scrolling mode — cursor advances past the image.
*/
public addImage(src: IDrawable, data: Blob | undefined, metrics: IMetrics): void {
public addImage(src: IDrawable, data: Blob | undefined, metrics: IMetrics, prescaleX: number, prescaleY: number): void {
this._addImageOpts.prescaleX = prescaleX;
this._addImageOpts.prescaleY = prescaleY;
this._storage.addImage(src, data, metrics, this._addImageOpts);
}
}
addons/addon-image/src/ImageAddon.ts
+3 -3
View File
@@ -190,7 +190,7 @@ export class ImageAddon implements ITerminalAddon, IImageApi {
// iTerm IIP handler
if (this._opts.iipSupport) {
const iipStorage = new IIPImageStorage(this._storage!);
const iipHandler = new IIPHandler(this._opts, this._renderer!, iipStorage, terminal);
const iipHandler = new IIPHandler(this._opts, iipStorage, terminal);
this._handlers.set('iip', iipHandler);
this._disposeLater(
terminal._core._inputHandler._parser.registerOscHandler(1337, iipHandler)
@@ -343,8 +343,8 @@ export class ImageAddon implements ITerminalAddon, IImageApi {
switch (params[1]) {
// we only implement read and read_max here
case GaAction.READ:
let width = this._renderer?.dimensions?.css.canvas.width;
let height = this._renderer?.dimensions?.css.canvas.height;
let width = this._renderer?.dimensions?.device.canvas.width;
let height = this._renderer?.dimensions?.device.canvas.height;
if (!width || !height) {
// for some reason we have no working image renderer
// --> fallback to default cell size
addons/addon-image/src/ImageRenderer.ts
+18 -10
View File
@@ -135,18 +135,26 @@ export class ImageRenderer extends Disposable implements IDisposable {
}
const img = imgSpec.src;
const cols = Math.ceil(img.width / initialGrid.width);
const cols = Math.ceil(img.width * imgSpec.prescaleX / initialGrid.width);
const sx = (tileId % cols) * initialGrid.width;
const sy = Math.floor(tileId / cols) * initialGrid.height;
const dx = col * deviceGrid.width;
const dy = row * deviceGrid.height;
const gridWidth = initialGrid.width / imgSpec.prescaleX;
const gridHeight = initialGrid.height / imgSpec.prescaleY;
// crops are initialGrid px
const sx = (tileId % cols) * gridWidth + imgSpec.precropX;
const sy = Math.floor(tileId / cols) * gridHeight + imgSpec.precropY;
const dx = Math.floor(col * deviceGrid.width + imgSpec.offsetX);
const dy = Math.floor(row * deviceGrid.height + imgSpec.offsetY);
// safari bug: never access image source out of bounds, thus we clamp its dimensions
const sWidth = Math.min(count * initialGrid.width, img.width - sx);
const sHeight = Math.min(initialGrid.height, img.height - sy);
const dWidth = sWidth / initialGrid.width * deviceGrid.width;
const dHeight = sHeight / initialGrid.height * deviceGrid.height;
const sWidth = Math.min(count * gridWidth, img.width - sx);
const sHeight = Math.min(gridHeight, img.height - sy);
const dWidth = Math.ceil(sWidth / gridWidth * deviceGrid.width * imgSpec.scaleX);
const dHeight = Math.ceil(sHeight / gridHeight * deviceGrid.height * imgSpec.scaleY);
if (imgSpec.blendMode === 'overwrite') {
ctx.clearRect(dx, dy, dWidth, dHeight);
}
// Floor all pixel offsets to get stable tile mapping without any overflows.
// Note: For not pixel perfect aligned cells like in the DOM renderer
@@ -155,7 +163,7 @@ export class ImageRenderer extends Disposable implements IDisposable {
ctx.drawImage(
img.native,
Math.floor(sx), Math.floor(sy), Math.ceil(sWidth), Math.ceil(sHeight),
Math.floor(dx), Math.floor(dy), Math.ceil(dWidth), Math.ceil(dHeight)
dx, dy, dWidth, dHeight
);
}
addons/addon-image/src/ImageStorage.ts
+14 -3
View File
@@ -187,9 +187,11 @@ export class ImageStorage implements IDisposable {
this._evictOldest(src.bytes);
// calc rows x cols needed to display the image
const prescaleX = opts.prescaleX ?? 1.0;
const prescaleY = opts.prescaleY ?? 1.0;
const cellSize = this._renderer.getCellSize() ?? CELL_SIZE_DEFAULT;
const cols = Math.ceil(src.width / cellSize.width);
const rows = Math.ceil(src.height / cellSize.height);
const cols = Math.ceil(src.width * prescaleX / cellSize.width);
const rows = Math.ceil(src.height * prescaleY / cellSize.height);
const imageId = ++this._lastId;
@@ -274,7 +276,16 @@ export class ImageStorage implements IDisposable {
tileCount,
bufferType: this._terminal.buffer.active.type,
layer: opts.layer,
zIndex: opts.zIndex
zIndex: opts.zIndex,
prescaleX,
prescaleY,
precropX: 0,
precropY: 0,
scaleX: 1.0,
scaleY: 1.0,
offsetX: 0,
offsetY: 0,
blendMode: 'alpha'
};
// finally add the image
addons/addon-image/src/SixelHandler.ts
-1
View File
@@ -103,7 +103,6 @@ export class SixelHandler implements IDcsHandler, IResetHandler {
if (this._dec.memoryUsage > MEM_PERMA_LIMIT) {
this._dec.release();
}
// FIXME: store sixel data in blob
this._storage.addImage(new Drawable(canvas), undefined, { width, height, mime: 'image/sixel' });
return true;
}
addons/addon-image/src/Types.ts
+19
View File
@@ -94,6 +94,8 @@ export interface IAddImageOpts {
layer: ImageLayer;
zIndex: number;
cursorPos: CursorPos;
prescaleX?: number;
prescaleY?: number;
}
export interface IDrawable {
@@ -124,12 +126,15 @@ export interface IMetrics {
}
export interface IImageSpec {
/** entries for a later IImageSource type */
/** drawable for screen rendering */
src: IDrawable;
/** image bytes as blob */
data: Blob | undefined;
/** metrics about image like mime and dimensions */
metrics: IMetrics;
/** entries for a later IPlacement type */
/** cell size at time of insert */
cellSize: ICellSize;
/** eviction marker */
@@ -141,4 +146,18 @@ export interface IImageSpec {
/** layer it was placed on */
layer: ImageLayer;
zIndex: number;
/** scaling factor on source dimensions, default = 1.0 */
prescaleX: number;
prescaleY: number;
/** crop offset into source, default = 0 (currently unused) */
precropX: number;
precropY: number;
/** output scaling, default = 1.0 (currently unused) */
scaleX: number;
scaleY: number;
/** output offset, default = 0 (currently unused) */
offsetX: number;
offsetY: number;
/** compositing mode on output, default = 'alpha' (currently unused) */
blendMode: 'overwrite' | 'alpha';
}
addons/addon-image/src/kitty/KittyGraphicsHandler.ts
+2 -2
View File
@@ -574,8 +574,8 @@ export class KittyGraphicsHandler implements IApcHandler, IResetHandler, IDispos
bitmap = cropped;
}
const cw = this._renderer.dimensions?.css.cell.width || CELL_SIZE_DEFAULT.width;
const ch = this._renderer.dimensions?.css.cell.height || CELL_SIZE_DEFAULT.height;
const cw = this._renderer.dimensions?.device.cell.width || CELL_SIZE_DEFAULT.width;
const ch = this._renderer.dimensions?.device.cell.height || CELL_SIZE_DEFAULT.height;
// Per spec: c/r default to image's natural cell dimensions.
// If only one of c/r is specified, compute the other from image aspect ratio.
addons/addon-image/test/ImageAddon.test.ts
+7 -7
View File
@@ -452,10 +452,10 @@ test.describe('ImageAddon', () => {
async function getDimensions(): Promise<IDimensions> {
const dimensions: any = await ctx.page.evaluate(`term.dimensions`);
return {
cellWidth: Math.round(dimensions.css.cell.width),
cellHeight: Math.round(dimensions.css.cell.height),
width: Math.round(dimensions.css.canvas.width),
height: Math.round(dimensions.css.canvas.height)
cellWidth: Math.round(dimensions.device.cell.width),
cellHeight: Math.round(dimensions.device.cell.height),
width: Math.round(dimensions.device.canvas.width),
height: Math.round(dimensions.device.canvas.height)
};
}
@@ -473,9 +473,9 @@ async function getScrollbackPlusRows(): Promise<number> {
async function getImageSize(id: number): Promise<[number, number]> {
return ctx.page.evaluate<any>(`[
window.imageAddon._storage._images.get(${id}).src.width,
window.imageAddon._storage._images.get(${id}).src.height
]`);
window.imageAddon._storage._images.get(${id}).src.width * window.imageAddon._storage._images.get(${id}).prescaleX,
window.imageAddon._storage._images.get(${id}).src.height * window.imageAddon._storage._images.get(${id}).prescaleY
]`).then(r => r.map(Math.floor));
}
async function getImageAtBufferCell(x: number, y: number): Promise<string | undefined> {
addons/addon-image/test/KittyGraphics.test.ts
+6 -6
View File
@@ -1283,7 +1283,7 @@ test.describe('Kitty Graphics Protocol', () => {
const cursor = await getCursor();
const cellDims: number[] = await ctx.page.evaluate(() => {
const d = (window as any).term._core._renderService.dimensions.css.cell;
const d = (window as any).term._core._renderService.dimensions.device.cell;
return [d.width, d.height];
});
const expectedR = Math.ceil((100 / 200) * 10 * cellDims[0] / cellDims[1]);
@@ -1301,7 +1301,7 @@ test.describe('Kitty Graphics Protocol', () => {
const cursor = await getCursor();
const cellDims: number[] = await ctx.page.evaluate(() => {
const d = (window as any).term._core._renderService.dimensions.css.cell;
const d = (window as any).term._core._renderService.dimensions.device.cell;
return [d.width, d.height];
});
const expectedC = Math.ceil((200 / 100) * 5 * cellDims[1] / cellDims[0]);
@@ -2236,10 +2236,10 @@ test.describe('Kitty Graphics Protocol', () => {
async function getDimensions(): Promise<IDimensions> {
const dimensions: any = await ctx.page.evaluate(`term.dimensions`);
return {
cellWidth: Math.round(dimensions.css.cell.width),
cellHeight: Math.round(dimensions.css.cell.height),
width: Math.round(dimensions.css.canvas.width),
height: Math.round(dimensions.css.canvas.height)
cellWidth: Math.round(dimensions.device.cell.width),
cellHeight: Math.round(dimensions.device.cell.height),
width: Math.round(dimensions.device.canvas.width),
height: Math.round(dimensions.device.canvas.height)
};
}