/** * Copyright (c) 2020 The xterm.js authors. All rights reserved. * @license MIT */ import test from '@playwright/test'; import { readFileSync } from 'fs'; import { FINALIZER, introducer, sixelEncode } from 'sixel'; import { ITestContext, createTestContext, openTerminal, pollFor, timeout } from '../../../test/playwright/TestUtils'; import { deepStrictEqual, ok, strictEqual } from 'assert'; import QoiEncoder from 'xterm-wasm-parts/lib/qoi/QoiEncoder.wasm'; /** * Plugin ctor options. */ export interface IImageAddonOptions { enableSizeReports: boolean; pixelLimit: number; storageLimit: number; showPlaceholder: boolean; sixelSupport: boolean; sixelScrolling: boolean; sixelPaletteLimit: number; sixelSizeLimit: number; iipSupport: boolean; iipSizeLimit: number; kittySupport: boolean; kittySizeLimit: number; } // eslint-disable-next-line declare const ImageAddon: { new(options?: Partial): any; }; interface ITestData { width: number; height: number; bytes: Uint8Array; palette: number[]; sixel: string; } interface IDimensions { cellWidth: number; cellHeight: number; width: number; height: number; } // image: 640 x 80, 512 color const TESTDATA: ITestData = (() => { const data8 = readFileSync('./addons/addon-image/fixture/palette.blob'); const data32 = new Uint32Array(data8.buffer); const palette = new Set(); for (let i = 0; i < data32.length; ++i) palette.add(data32[i]); const sixel = sixelEncode(data8, 640, 80, [...palette]); return { width: 640, height: 80, bytes: data8, palette: [...palette], sixel }; })(); const SIXEL_SEQ_0 = introducer(0) + TESTDATA.sixel + FINALIZER; // const SIXEL_SEQ_1 = introducer(1) + TESTDATA.sixel + FINALIZER; // const SIXEL_SEQ_2 = introducer(2) + TESTDATA.sixel + FINALIZER; // NOTE: the data is loaded as string for easier transport through playwright const TESTDATA_IIP: [string, [number, number]][] = [ [readFileSync('./addons/addon-image/fixture/iip/palette.iip', { encoding: 'utf-8' }), [640, 80]], [readFileSync('./addons/addon-image/fixture/iip/spinfox.iip', { encoding: 'utf-8' }), [148, 148]], [readFileSync('./addons/addon-image/fixture/iip/w3c_gif.iip', { encoding: 'utf-8' }), [72, 48]], [readFileSync('./addons/addon-image/fixture/iip/w3c_jpg.iip', { encoding: 'utf-8' }), [72, 48]], [readFileSync('./addons/addon-image/fixture/iip/w3c_png.iip', { encoding: 'utf-8' }), [72, 48]] ]; const PALETTE_PNG_BASE64 = readFileSync('./addons/addon-image/fixture/palette.png').toString('base64'); const qoiEnc = new QoiEncoder(1024*1024); const qoiData = qoiEnc.encode(TESTDATA.bytes, 640, 80); const PALETTE_QOI_BASE64 = Buffer.from(qoiData).toString('base64'); let ctx: ITestContext; test.beforeAll(async ({ browser }) => { ctx = await createTestContext(browser); await openTerminal(ctx, { cols: 80, rows: 24 }); }); test.afterAll(async () => await ctx.page.close()); test.describe('ImageAddon', () => { test.beforeEach(async ({}, testInfo) => { await ctx.page.evaluate(` window.term.reset() window.imageAddon?.dispose(); window.imageAddon = new ImageAddon({ sixelPaletteLimit: 512 }); window.term.loadAddon(window.imageAddon); `); }); test('test for private accessors', async () => { // terminal privates const accessors = [ '_core', '_core._renderService', '_core._inputHandler', '_core._inputHandler._parser', '_core._inputHandler._curAttrData', '_core._inputHandler._dirtyRowTracker', '_core._themeService.colors', '_core._coreBrowserService' ]; for (const prop of accessors) { strictEqual( await ctx.page.evaluate('(() => { const v = window.term.' + prop + '; return v !== undefined && v !== null; })()'), true, `problem at ${prop}` ); } // bufferline privates strictEqual(await ctx.page.evaluate('window.term._core.buffer.lines.get(0)._data instanceof Uint32Array'), true); strictEqual(await ctx.page.evaluate('window.term._core.buffer.lines.get(0)._extendedAttrs instanceof Object'), true); // inputhandler privates strictEqual(await ctx.page.evaluate('window.term._core._inputHandler._curAttrData.constructor.name'), '_AttributeData'); strictEqual(await ctx.page.evaluate('window.term._core._inputHandler._parser.constructor.name'), 'EscapeSequenceParser'); }); test.describe('ctor options', () => { test('empty settings should load defaults', async () => { const DEFAULT_OPTIONS: IImageAddonOptions = { enableSizeReports: true, pixelLimit: 16777216, sixelSupport: true, sixelScrolling: true, sixelPaletteLimit: 512, // set to 512 to get example image working sixelSizeLimit: 33554432, storageLimit: 128, showPlaceholder: true, iipSupport: true, iipSizeLimit: 33554432, kittySupport: true, kittySizeLimit: 33554432 }; deepStrictEqual(await ctx.page.evaluate(`window.imageAddon._opts`), DEFAULT_OPTIONS); }); test('custom settings should overload defaults', async () => { const customSettings: IImageAddonOptions = { enableSizeReports: false, pixelLimit: 5, sixelSupport: false, sixelScrolling: false, sixelPaletteLimit: 1024, sixelSizeLimit: 1000, storageLimit: 10, showPlaceholder: false, iipSupport: false, iipSizeLimit: 1000, kittySupport: false, kittySizeLimit: 1000 }; await ctx.page.evaluate(opts => { (window as any).imageAddonCustom = new ImageAddon(opts.opts); (window as any).term.loadAddon((window as any).imageAddonCustom); }, { opts: customSettings }); deepStrictEqual(await ctx.page.evaluate(`window.imageAddonCustom._opts`), customSettings); }); }); test.describe('scrolling & cursor modes', () => { test('testdata default (scrolling with VT240 cursor pos)', async () => { const dim = await getDimensions(); await ctx.proxy.write(SIXEL_SEQ_0); deepStrictEqual(await getCursor(), [0, Math.floor(TESTDATA.height/dim.cellHeight)]); // moved to right by 10 cells await ctx.proxy.write('#'.repeat(10) + SIXEL_SEQ_0); deepStrictEqual(await getCursor(), [10, Math.floor(TESTDATA.height/dim.cellHeight) * 2]); }); test('write testdata noScrolling', async () => { await ctx.proxy.write('\x1b[?80h' + SIXEL_SEQ_0); deepStrictEqual(await getCursor(), [0, 0]); // second draw does not change anything await ctx.proxy.write(SIXEL_SEQ_0); deepStrictEqual(await getCursor(), [0, 0]); }); test('testdata cursor always at VT240 pos', async () => { const dim = await getDimensions(); // offset 0 await ctx.proxy.write(SIXEL_SEQ_0); deepStrictEqual(await getCursor(), [0, Math.floor(TESTDATA.height/dim.cellHeight)]); // moved to right by 10 cells await ctx.proxy.write('#'.repeat(10) + SIXEL_SEQ_0); deepStrictEqual(await getCursor(), [10, Math.floor(TESTDATA.height/dim.cellHeight) * 2]); // moved by 30 cells (+10 prev) await ctx.proxy.write('#'.repeat(30) + SIXEL_SEQ_0); deepStrictEqual(await getCursor(), [10 + 30, Math.floor(TESTDATA.height/dim.cellHeight) * 3]); }); }); test.describe('image lifecycle & eviction', () => { test('onImageAdded fires for each image', async () => { await ctx.page.evaluate(` window._imageAddedCount = 0; window.imageAddon.onImageAdded(() => { window._imageAddedCount++; }); `); await ctx.proxy.write(SIXEL_SEQ_0); await pollFor(ctx.page, 'window._imageAddedCount', 1); await ctx.proxy.write(SIXEL_SEQ_0); await pollFor(ctx.page, 'window._imageAddedCount', 2); await ctx.proxy.write(SIXEL_SEQ_0); await pollFor(ctx.page, 'window._imageAddedCount', 3); }); test('delete image once scrolled off', async () => { await ctx.proxy.write(SIXEL_SEQ_0); pollFor(ctx.page, 'window.imageAddon._storage._images.size', 1); // scroll to scrollback + rows - 1 await ctx.page.evaluate( scrollback => new Promise(res => (window as any).term.write('\n'.repeat(scrollback), res)), (await getScrollbackPlusRows() - 1) ); // wait here, as we have to make sure, that eviction did not yet occur await timeout(100); pollFor(ctx.page, 'window.imageAddon._storage._images.size', 1); // scroll one further should delete the image await ctx.page.evaluate(() => new Promise(res => (window as any).term.write('\n', res))); pollFor(ctx.page, 'window.imageAddon._storage._images.size', 0); }); test('get storageUsage', async () => { strictEqual(await ctx.page.evaluate('window.imageAddon.storageUsage'), 0); await ctx.proxy.write(SIXEL_SEQ_0); ok(Math.abs((await ctx.page.evaluate('window.imageAddon.storageUsage')) - 640 * 80 * 4 / 1000000) < 0.05); }); test('get/set storageLimit', async () => { strictEqual(await ctx.page.evaluate('window.imageAddon.storageLimit'), 128); strictEqual(await ctx.page.evaluate('window.imageAddon.storageLimit = 1'), 1); strictEqual(await ctx.page.evaluate('window.imageAddon.storageLimit'), 1); }); test('remove images by storage limit pressure', async () => { strictEqual(await ctx.page.evaluate('window.imageAddon.storageLimit = 1'), 1); // never go beyond storage limit await ctx.proxy.write(SIXEL_SEQ_0); await ctx.proxy.write(SIXEL_SEQ_0); await ctx.proxy.write(SIXEL_SEQ_0); await ctx.proxy.write(SIXEL_SEQ_0); await timeout(100); const usage = await ctx.page.evaluate('window.imageAddon.storageUsage'); await ctx.proxy.write(SIXEL_SEQ_0); await ctx.proxy.write(SIXEL_SEQ_0); await ctx.proxy.write(SIXEL_SEQ_0); await ctx.proxy.write(SIXEL_SEQ_0); await timeout(100); strictEqual(await ctx.page.evaluate('window.imageAddon.storageUsage'), usage); strictEqual(usage as number < 1, true); }); test('set storageLimit removes images synchronously', async () => { await ctx.proxy.write(SIXEL_SEQ_0 + SIXEL_SEQ_0 + SIXEL_SEQ_0); const usage: number = await ctx.page.evaluate('window.imageAddon.storageUsage'); const newUsage: number = await ctx.page.evaluate('window.imageAddon.storageLimit = 0.5; window.imageAddon.storageUsage'); strictEqual(newUsage < usage, true); strictEqual(newUsage < 0.5, true); }); test('clear alternate images on buffer change', async () => { strictEqual(await ctx.page.evaluate('window.imageAddon.storageUsage'), 0); await ctx.proxy.write('\x1b[?1049h' + SIXEL_SEQ_0); ok(Math.abs((await ctx.page.evaluate('window.imageAddon.storageUsage')) - 640 * 80 * 4 / 1000000) < 0.05); await ctx.proxy.write('\x1b[?1049l'); strictEqual(await ctx.page.evaluate('window.imageAddon.storageUsage'), 0); }); test('evict tiles by in-place overwrites (only full overwrite tested)', async () => { await timeout(50); await ctx.proxy.write('\x1b[H' + SIXEL_SEQ_0 + '\x1b[100;100H'); await timeout(50); let usage = await ctx.page.evaluate('window.imageAddon.storageUsage'); while (usage === 0) { await timeout(50); usage = await ctx.page.evaluate('window.imageAddon.storageUsage'); } await ctx.proxy.write('\x1b[H' + SIXEL_SEQ_0 + '\x1b[100;100H'); await timeout(200); // wait some time and re-check strictEqual(await ctx.page.evaluate('window.imageAddon.storageUsage'), usage); }); test('manual eviction on alternate buffer must not miss images', async () => { await ctx.proxy.write('\x1b[?1049h'); await ctx.proxy.write(SIXEL_SEQ_0 + SIXEL_SEQ_0 + SIXEL_SEQ_0 + SIXEL_SEQ_0 + SIXEL_SEQ_0 + SIXEL_SEQ_0); await timeout(100); const usage: number = await ctx.page.evaluate('window.imageAddon.storageUsage'); await ctx.proxy.write(SIXEL_SEQ_0 + SIXEL_SEQ_0 + SIXEL_SEQ_0 + SIXEL_SEQ_0 + SIXEL_SEQ_0 + SIXEL_SEQ_0); await ctx.proxy.write(SIXEL_SEQ_0 + SIXEL_SEQ_0 + SIXEL_SEQ_0 + SIXEL_SEQ_0 + SIXEL_SEQ_0 + SIXEL_SEQ_0); await timeout(100); const newUsage: number = await ctx.page.evaluate('window.imageAddon.storageUsage'); strictEqual(newUsage, usage); }); }); test.describe('IIP support', () => { test('size params is optional/informative', async () => { // too small size param await ctx.proxy.write(`\x1b]1337;File=inline=1;size=5:${PALETTE_PNG_BASE64}\x07`); deepStrictEqual(await getOrigSize(1), [640, 80]); // no size aparm at all await ctx.proxy.write(`\x1b[10H\x1b]1337;File=inline=1:${PALETTE_QOI_BASE64}\x07`); deepStrictEqual(await getOrigSize(2), [640, 80]); }); test.skip('FilePart support - bytewise', async () => { // opener await ctx.proxy.write(`\x1b]1337;MultipartFile=inline=1\x07`); // byte-wise for (let i = 0; i < PALETTE_PNG_BASE64.length; ++i) { await ctx.proxy.write(`\x1b]1337;FilePart=${PALETTE_PNG_BASE64[i]}\x07`); } // finalizer await ctx.proxy.write(`\x1b]1337;FileEnd\x07`); deepStrictEqual(await getOrigSize(1), [640, 80]); }); test('FilePart support - chunks', async () => { // opener await ctx.proxy.write(`\x1b]1337;MultipartFile=inline=1\x07`); // chunks for (let i = 0; i < PALETTE_PNG_BASE64.length; i += 256) { await ctx.proxy.write(`\x1b]1337;FilePart=${PALETTE_PNG_BASE64.slice(i, i + 256)}\x07`); } // finalizer await ctx.proxy.write(`\x1b]1337;FileEnd\x07`); deepStrictEqual(await getOrigSize(1), [640, 80]); }); test('FilePart support - edgecases', async () => { // multiple opener do not harm for (let i = 0; i < 10; ++i) { await ctx.proxy.write(`\x1b]1337;MultipartFile=inline=1\x07`); } // multiple finalizer do not harm for (let i = 0; i < 5; ++i) { await ctx.proxy.write(`\x1b]1337;FileEnd\x07`); } // chunks w'o opener are ignored for (let i = 0; i < PALETTE_PNG_BASE64.length; i += 256) { await ctx.proxy.write(`\x1b]1337;FilePart=${PALETTE_PNG_BASE64.slice(i, i + 256)}\x07`); } await ctx.proxy.write(`\x1b]1337;FileEnd\x07`); pollFor(ctx.page, 'window.imageAddon._storage._images.size', 0); }); test('FilePart support - other IIP sequence cancels chunked image', async () => { // opener await ctx.proxy.write(`\x1b]1337;MultipartFile=inline=1\x07`); // chunks for (let i = 0; i < PALETTE_PNG_BASE64.length; i += 256) { await ctx.proxy.write(`\x1b]1337;FilePart=${PALETTE_PNG_BASE64.slice(i, i + 256)}\x07`); } // finalizer skipped // write spinfox.png await ctx.proxy.write(TESTDATA_IIP[1][0]); deepStrictEqual(await getOrigSize(1), TESTDATA_IIP[1][1]); }); }); test.describe('IIP support - testimages', () => { test('palette.png', async () => { await ctx.proxy.write(TESTDATA_IIP[0][0]); deepStrictEqual(await getOrigSize(1), TESTDATA_IIP[0][1]); }); test('spinfox.png', async () => { await ctx.proxy.write(TESTDATA_IIP[1][0]); deepStrictEqual(await getOrigSize(1), TESTDATA_IIP[1][1]); }); test('w3c gif', async () => { await ctx.proxy.write(TESTDATA_IIP[2][0]); deepStrictEqual(await getOrigSize(1), TESTDATA_IIP[2][1]); }); test('w3c jpeg', async () => { await ctx.proxy.write(TESTDATA_IIP[3][0]); deepStrictEqual(await getOrigSize(1), TESTDATA_IIP[3][1]); }); test('w3c png', async () => { await ctx.proxy.write(TESTDATA_IIP[4][0]); deepStrictEqual(await getOrigSize(1), TESTDATA_IIP[4][1]); }); }); test.describe('IIP - QOI support', () => { test('palette should yield same bytes from PNG and QOI', async () => { await ctx.proxy.write(`\x1b]1337;File=inline=1;size=525:${PALETTE_PNG_BASE64}\x07`); deepStrictEqual(await getOrigSize(1), [640, 80]); await ctx.proxy.write(`\x1b[10H\x1b]1337;File=inline=1;size=${qoiData.length}:${PALETTE_QOI_BASE64}\x07`); deepStrictEqual(await getOrigSize(2), [640, 80]); const pngScrape = await getImageAtBufferCell(0, 0); const qoiScrape = await getImageAtBufferCell(0, 11); deepStrictEqual(qoiScrape, pngScrape); }); }); test.describe('IIP - resizing', () => { /** * The correct resize behavior is wonky and needs to be tested against iTerm2, * especially for missing params to derive correct default behavior. * We document the current behavior here w'o claiming to be fully in line with iTerm2. * ref: https://iterm2.com/documentation-images.html * imgcat: https://iterm2.com/utilities/imgcat * * NOTE: QOI has a slightly different parse path, thus we test resizing explicitly */ const images = [ ['palette.png', 525, PALETTE_PNG_BASE64], ['palette.qoi', qoiData.length, PALETTE_QOI_BASE64] ]; for (const [name, size, payload] of images) { test(name + ': N --> width=20 height=5 preserveAspectRatio=0', async () => { // cell based resize const header = 'width=20;height=5;preserveAspectRatio=0'; await ctx.proxy.write(`\x1b]1337;File=inline=1;size=${size};${header}:${payload}\x07`); const dim = await getDimensions(); deepStrictEqual(await getOrigSize(1), [dim.cellWidth * 20, dim.cellHeight * 5]); }); test(name + ': Npx --> width=320px height=160px preserveAspectRatio=0', async () => { // pixel based resize const header = 'width=320px;height=160px;preserveAspectRatio=0'; await ctx.proxy.write(`\x1b]1337;File=inline=1;size=${size};${header}:${payload}\x07`); deepStrictEqual(await getOrigSize(1), [320, 160]); }); test(name + ': N% --> width=50% height=30% preserveAspectRatio=0', async () => { // % of viewport resize const header = 'width=50%;height=30%;preserveAspectRatio=0'; await ctx.proxy.write(`\x1b]1337;File=inline=1;size=${size};${header}:${payload}\x07`); const dim = await getDimensions(); deepStrictEqual(await getOrigSize(1), [Math.floor(dim.width * 0.5), Math.floor(dim.height * 0.3)]); }); test(name + ': ommitted dimension assumes preserveAspectRatio=1', async () => { // width provided in percent const header = 'width=50%'; await ctx.proxy.write(`\x1b]1337;File=inline=1;size=${size};${header}:${payload}\x07`); const dim = await getDimensions(); const width = Math.floor(dim.width * 0.5); deepStrictEqual(await getOrigSize(1), [width, Math.floor(width * 80 / 640)]); // height provided in pixel const header2 = 'height=200px'; await ctx.proxy.write(`\x1b]1337;File=inline=1;size=${size};${header2}:${payload}\x07`); deepStrictEqual(await getOrigSize(2), [Math.floor(200 * 640 / 80), 200]); }); } }); test.describe('text overwrite removes tiles', () => { test('sixel', async () => { await assertTextClearsTiles(SIXEL_SEQ_0); }); test('iip', async () => { await assertTextClearsTiles(TESTDATA_IIP[0][0]); }); }); }); /** * terminal access helpers. */ async function getDimensions(): Promise { const dimensions: any = await ctx.page.evaluate(`term.dimensions`); return { cellWidth: Math.round(dimensions.css.cell.width), cellHeight: Math.round(dimensions.css.cell.height), width: Math.round(dimensions.css.canvas.width), height: Math.round(dimensions.css.canvas.height) }; } async function getCursor(): Promise<[number, number]> { return ctx.page.evaluate('[window.term.buffer.active.cursorX, window.term.buffer.active.cursorY]'); } async function getImageStorageLength(): Promise { return ctx.page.evaluate('window.imageAddon._storage._images.size'); } async function getScrollbackPlusRows(): Promise { return ctx.page.evaluate('window.term.options.scrollback + window.term.rows'); } async function getOrigSize(id: number): Promise<[number, number]> { return ctx.page.evaluate(`[ window.imageAddon._storage._images.get(${id}).orig.width, window.imageAddon._storage._images.get(${id}).orig.height ]`); } async function getImageAtBufferCell(x: number, y: number): Promise { return ctx.page.evaluate(`window.imageAddon.getImageAtBufferCell(${x}, ${y})?.toDataURL('image/png')`); } async function hasTileAtBufferCell(x: number, y: number): Promise { return ctx.page.evaluate(`!!window.imageAddon.extractTileAtBufferCell(${x}, ${y})`); } async function assertTextClearsTiles(imageSeq: string): Promise { await ctx.proxy.write('\x1b[H' + imageSeq); await pollFor(ctx.page, '!!window.imageAddon.extractTileAtBufferCell(5, 1)', true); ok(await hasTileAtBufferCell(0, 1)); await ctx.proxy.write('\x1b[2;6H#######'); for (let x = 5; x < 12; x++) { strictEqual(await hasTileAtBufferCell(x, 1), false); } ok(await hasTileAtBufferCell(0, 1)); ok(await hasTileAtBufferCell(13, 1)); ok(((await ctx.page.evaluate('window.term.buffer.active.getLine(1).translateToString(true)')) as string).includes('#######')); }