New unicode-graphemes addon.
Handle grapheme cluster lookup as well as wcwidth. This is based on Unicode 15, and could replace the unicode11 addon.
This commit is contained in:
Per Bothner
@@ -0,0 +1,44 @@
|
||||
/**
|
||||
* Copyright (c) 2023 The xterm.js authors. All rights reserved.
|
||||
* @license MIT
|
||||
*/
|
||||
|
||||
import { IUnicodeVersionProvider } from 'xterm';
|
||||
import { UnicodeCharProperties, UnicodeCharWidth } from 'common/services/Services';
|
||||
import { UnicodeService } from 'common/services/UnicodeService';
|
||||
import * as UC from './UnicodeProperties';
|
||||
|
||||
export class UnicodeGraphemeProvider implements IUnicodeVersionProvider {
|
||||
public readonly version = '15-graphemes';
|
||||
|
||||
constructor() {
|
||||
}
|
||||
|
||||
charProperties(codepoint: number, preceding: UnicodeCharProperties): UnicodeCharProperties {
|
||||
let charInfo = UC.getInfo(codepoint);
|
||||
let w = UC.infoToWidthInfo(charInfo);
|
||||
let shouldJoin = false;
|
||||
if (w >= 2) {
|
||||
const preferWide = false; //this.ambiguousCharsAreWide(context);
|
||||
// Treat emoji_presentation_selector as WIDE.
|
||||
w = w == 3 || preferWide || codepoint === 0xfe0f ? 2 : 1;
|
||||
} else
|
||||
w = 1;
|
||||
if (preceding !== 0) {
|
||||
let oldWidth = UnicodeService.extractWidth(preceding);
|
||||
charInfo = UC.shouldJoin(UnicodeService.extractCharKind(preceding), charInfo);
|
||||
shouldJoin = charInfo > 0;
|
||||
if (shouldJoin) {
|
||||
if (oldWidth > w)
|
||||
w = oldWidth;
|
||||
else if (charInfo === 32) // FIXME UC.GRAPHEME_BREAK_SAW_Regional_Pair)
|
||||
w = 2;
|
||||
}
|
||||
}
|
||||
return UnicodeService.createPropertyValue(charInfo, w, shouldJoin);
|
||||
}
|
||||
|
||||
public wcwidth(num: number): UnicodeCharWidth {
|
||||
return UC.infoToWidth(UC.getInfo(num));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
/**
|
||||
* Copyright (c) 2023 The xterm.js authors. All rights reserved.
|
||||
* @license MIT
|
||||
*
|
||||
* UnicodeVersionProvider for V15 with grapeme cluster handleing.
|
||||
*/
|
||||
|
||||
import { Terminal, ITerminalAddon } from 'xterm';
|
||||
import { UnicodeGraphemeProvider } from './UnicodeGraphemeProvider';
|
||||
|
||||
|
||||
export class UnicodeGraphemesAddon implements ITerminalAddon {
|
||||
public activate(terminal: Terminal): void {
|
||||
terminal.unicode.register(new UnicodeGraphemeProvider());
|
||||
}
|
||||
public dispose(): void { }
|
||||
}
|
||||
@@ -0,0 +1,144 @@
|
||||
import UnicodeTrie from './unicode-trie';
|
||||
const trieRaw = "AAARAAAAAABwxwAAAb4LQfTtmw+sVmUdx58LL/ffe/kjzNBV80gW1F3yR+6CvbJiypoZa0paWmAWSluErSBbFtYkkuZykq6QamGJ4WRqo2kFGy6dYWtEq6G1MFAJbRbOVTQr+x7f5+x97q/n/3me87wXzm/3s+f/7/d7/p7znnvOlvGMbQM7wIPgEbAPHABPgcPgefAS+BfYwuv/F/Q2OulBxKcK6TMRPxu8FcwFbwcjYCFYDC4Cl4ArwNXgGvBJsA58UdBDwy+jbBO4La8DtoEd4H7wkNBuN+KPgn3gADgIngaHwFHwF/AyeAWMm4C+TGi3LdiJ/EnIex04A2RgFpgD5oKFYDG4CLwHXAo+IKSvAqt4/evA9bz9jWA6+Cq3dyvCP8HWNwX93wF38/ROcD94SCjP2+1B+BiPP4HwgOD/7xD/I08fRniMx48jPAFeBeuF+n29jE0G08FZvaPHYWZvh9mcEfAOjlhXx/qGfd2QvLO3zccmtMnzliC9lPt+GenD1nyMiK/LNf1cycs+gfAzPJ6vtxe4jhuQtx5sBLeA28G3eb3v8/Beif4HkPewxu5G6N/rMP4qfgEdvwZPgj+AZ8Cx3nYfxiE8Dk6AV0FfH/YEOB28AbwJDIPzQAtcAC4Gl/Z19F+J+NVCehWPr0b46b7RvixvdPg8yr7U10l/BfFN4La8DdgGdoAHwU/AI2AfOACeAofB8+AlcAKwfvyBKeCM/o7NrF9PXmdWv9/Ynot2I7ztIg8dF5I2a8i63CjZU+9Fm2Wcy4U4ZQVYyeOrwVoev57UuxHcJKRvFuJXgnU8/nUebtbYrKmpCUOx31P7UVNTU1NTU1NTU1OGLTz8Xr/77+W7+9vP0or0MxPMbXaizY8FW3sQ3wseB/t5/kGEh8DR/vbzwL8i/Af4Dy8fP8BYE0weaKenI/wV/DhrQG97JspngzlgLpgHzgPzwUhdVpfVZXVZXRa87HxwAVgQ4Pn5WEd85l5TUzOasvezFw/E3b/LoP9D4CpwrcTWWsGXNQOj748/G9k3G56d1KYxmbELwQbwKFiJvBM8nDWlHa5E+AOwCzwLzjkNeeB28NvTeB1OYyr0gQ1g99R23nGE50xj7MPgc+A+8K5Bxj4FHgB/G2z/T9XEzCZjd/S0WYX4Pc3/r/Nn5I0f6qQXIP5x8ENwBMyYyNhHJ3b0pOCuLrBvM941NTU1JyNHEp+BrC8dMyalt1/m3uWfhmeULzRGp9d3wf0WZSN8+prCr60Wz09tuNmx35sl9Y825HXvRN39KNveaL8flb9f913kbec67kHeTsR3gYcH2uV7ED4m2HhCYi/X9ZuBzvuXv0f8iKIfx5B/XCg7gTgbVPdvAsomCuWnD45eK28UyvL3Jt+s0fU2TVnOXJQvJHUWIb0ELAWXgCt4+UcMumSsEtpch/g6ouMGpG/ieZsc9N/q4YsLd3D9WyPbsWEbfNgO7hN82TWY/n8xKbmsC3xQsYKf+7sjrx2TH+u4H3vhx+OO6+X9hmtXN7C/4r15EPaeBs9J7L7YBeeED/k7wn8fbIf/Rji+yVizmd4vW6bB19cb/PU9w7MxMA60bzPHgM8+zG623+OnzOf55yNc3Gw/k303wveBy3nZcoTXgNVgLfiCRNcG5N3SbIebwZ08fhe4l8d/BH7K4yI/4+HPwS/BAfBks+PzIaHuc3x+ivSL4GUyZ68I6fwZYRNMG2qnz+Th2QjfMtTx/1zE5w61nyN+Q7C3aKgdin1dgrylYBn4INdhGn/Z2FfFiqH01/SUXMvnPD+jC+j85N/RqRhR/DYaS6T+P09K1mD+vzW+5zVqqeVUl0wTz2lK8odJHRGXfBufdGLSoSo3+ZFJ6sl0qvJVNmhI4z4i06mrZ6uT1le1z5h5HE3tMiHPtQ5javu+ItMXUr/MXpmwmyRL3D6U7UwIMyYfczGu0qdqb2pbhcw4xQkhWQBMerrZ/liXrGTbsQwTwrEu4zSczKLrd7fCSKiKn+zSo8BWXMe8myXWOivrUxWi60OPoQ7VIasbQ0S/Ukk3rZVullNhHEL1rYoxUF0PTfm6elWJzq54ZsU4z11ohOy0oxT2izFqCNj4TesXcWZo6+Jfqr1O+1O1beqDagypj2J9F1u2daucj3Eknmq/6PaHrK7Mb1o35DiW1a/a76LuhlDXZX25SOz11S33ErKxDb2/fc/bFKI6axskn+4/W90u9mOtbRf7smsoTdvOfwoRz0t6DaP9k81v6P7Re5aUQudTd303rX+bZzBl97/KR7E+Xbux9lLI+aNr1PfaYLpPDiW2/vrYTX1drMIeXbMye6HXlw8292Jl7ZXxLxRlxXbcaH9drjFlxfa3Qozx8NWRi834lPVZbD+SmN7EJPzc9TVCSVXXDps9L+513b2J7fMu176V2YOhx1A3JrJ8KrLxUumpcu5j/lYT+2tzLRVDZmhjO442a1Clu0ox9VPVXzE/lcS4V0k1D6LI1pJsz8fct9SGbO5l/rmKzTlvsxdj3IvRtC2uv0t1fotltvd2VaCy5Sp5m0EhnZG4CCNxXZrWp/VUIrOjapfnNw11ZNI0V/GWzKNuxtzGKKTEtJeR0NVmpojbtBuW5On0u0is9ZMxvU8ZM+8vEyadtu10oqtP9Q4rcJEm85+Two/QkpGwjI6YkgkhtUfzZOW6fFVexuRri+qj9TJJHZkdmW5abiu0rs6uj2TMfmx06bISUj9tZ9Lja8dVQtox6WpxTJKfW3M4MSTmvU4sWy1CU6BF4jIfdNeDjHWuO1lCWIm2Jr2ixNZvklD2fP0Q6+vsmO4hqN1hJvfDtV5G8mTlsvau4qPP1a64L1skT6QYEzEtq0PzGZOfCbSdSmcKTP7Qs86Ej/1hEpelaV6IMdT5ayu2+nT9tmnnO746XbLxE8t0qOrYtJWhmk9bvaLfsrotRVw1PnR+bcafSUKZ6Mps7smobybJLH2R6WqRkJa1DHV0UmbfUcksiSF0HExSpp+uY0zbTklMaCm7blzEtg8h1rNMXNaYi05ZXsbC75sQ/4+aUxFV2jL50Q3jE0rK2rVtN09By8OHoo1vH2LPSdE323mr2sdu0pUZiDkWLRKWnfeQY6taKzHF9n/GPv8jd/0/egiRvYMR24fU79iY3s9Qva9RlYR8n8HHtq9fMcT1HRWfdZXiHd9YInt/iI4PTaf+BimXKvdXYU+3hlRpHzs2dVK/cxhDn+xs0I2jzxjL5kpXz1VU72aLtkK/97sALKyQqu25SshvG6h08/cLrlKswRklKXvvXfa+pZt+y8nah5YUv2Oo/ap/X2URdRfico9K69hcp6r6XaCz5Wo/hs/iNTGF6N6tV92/9ZS0Wba9SlT3pKF/e6W674+x9ly+VRL73cPU8ygb31D3eSqfVd+iqET0y3YMYojoO11XqrTt2nPxmeq1HYeqxkmUMt8DiesjpoTSr+qDrD+qPZDiOZxMdH0pRPX8MFUfQtv0Xbs+a1a1NnRryNZ/2+tsaPG5ZoX0RXZei88yZGdo4UMPj/cwv/kMJboxLISuQbE+1VW12Mx7FWOrW3M9Hv7Y+uxyraPSo8B2TGPuLdOeZha+hBKf8Sjsm/oR+7pmsx/oeOraFWdXleeV6oyl41zm+mgSuq9C6ox1TsU8D+m4dwMmf8v2nz7Tm+fYfj7HV1K/x1HWjquvY+2dllxM64ue87Su772zzbXIVC+WxLZTRR9MdkMTypZNH1z6G0tUvoccwxA+hfLNdV+a7MaQqscztMi+7QnxDZXvd1dldWQOyMbApb1Jd2h91Ffx+y9Xfb7tClokboOvrRhrbVpFFO8z+65t2/u4su9MUx028znH01/TGVDmHAj13W1o+1USw+eUfYtpO+b82rRNsb6oPpV+1fdBqddB6n3WDXvdJDZrJ0QfQp6bsc/kqq4BIddHWXGdN1pmWveh58F1zYUW1zmOITHXWOg1XrZvZSWUf77tq1ofqear6muaT1lIQp3bofabSafJVlnfYo9B6LGr8uzz2Xchvzfw+T9PlgiV/A8=";
|
||||
let _data = null;
|
||||
{
|
||||
const bin = window.atob(trieRaw);
|
||||
_data = new Uint8Array(bin.length)
|
||||
for (let i = 0; i < bin.length; i++)
|
||||
_data[i] = bin.charCodeAt(i);
|
||||
}
|
||||
const trieData = new UnicodeTrie(_data);
|
||||
export const GRAPHEME_BREAK_MASK = 0xF;
|
||||
export const GRAPHEME_BREAK_SHIFT = 0;
|
||||
export const CHARWIDTH_MASK = 0x30;
|
||||
export const CHARWIDTH_SHIFT = 4;
|
||||
|
||||
// Values for the GRAPHEME_BREAK property
|
||||
export const GRAPHEME_BREAK_Other = 0; // includes CR, LF, Control
|
||||
export const GRAPHEME_BREAK_Prepend = 1;
|
||||
export const GRAPHEME_BREAK_Extend = 2;
|
||||
export const GRAPHEME_BREAK_Regional_Indicator = 3;
|
||||
export const GRAPHEME_BREAK_SpacingMark = 4;
|
||||
export const GRAPHEME_BREAK_Hangul_L = 5;
|
||||
export const GRAPHEME_BREAK_Hangul_V = 6;
|
||||
export const GRAPHEME_BREAK_Hangul_T = 7;
|
||||
export const GRAPHEME_BREAK_Hangul_LV = 8;
|
||||
export const GRAPHEME_BREAK_Hangul_LVT = 9;
|
||||
export const GRAPHEME_BREAK_ZWJ = 10;
|
||||
export const GRAPHEME_BREAK_ExtPic = 11;
|
||||
|
||||
// Only used as return value from shouldJoin/shouldJoinBackwards.
|
||||
// (Must be positive; distinct from other values;
|
||||
// and become GRAPHEME_BREAK_Other when masked with GRAPHEME_BREAK_MASK.)
|
||||
const GRAPHEME_BREAK_SAW_Regional_Pair = 32;
|
||||
|
||||
export const CHARWIDTH_NORMAL = 0;
|
||||
export const CHARWIDTH_FORCE_1COLUMN = 1;
|
||||
export const CHARWIDTH_EA_AMBIGUOUS = 2;
|
||||
export const CHARWIDTH_WIDE = 3;
|
||||
|
||||
// In the following 'info' is an encoded value from trie.get(codePoint)
|
||||
|
||||
// In the following 'info' is an encoded value from trie.get(codePoint)
|
||||
|
||||
export function infoToWidthInfo(info: number): number {
|
||||
return (info & CHARWIDTH_MASK) >> CHARWIDTH_SHIFT;
|
||||
}
|
||||
|
||||
export function infoToWidth(info: number, ambiguousIsWide = false): 0 | 1 |2 {
|
||||
const v = infoToWidthInfo(info);
|
||||
return v < CHARWIDTH_EA_AMBIGUOUS ? 1
|
||||
: v >= CHARWIDTH_WIDE || ambiguousIsWide ? 2 : 1;
|
||||
}
|
||||
|
||||
export function strWidth(str: string, preferWide: boolean): number {
|
||||
let width = 0;
|
||||
for (let i = 0; i < str.length;) {
|
||||
const codePoint = str.codePointAt(i) as number;
|
||||
width += infoToWidth(getInfo(codePoint), preferWide);
|
||||
i += (codePoint <= 0xffff) ? 1 : 2;
|
||||
}
|
||||
return width;
|
||||
}
|
||||
|
||||
export function columnToIndexInContext(str: string, startIndex: number, column: number, preferWide: boolean): number {
|
||||
let rv = 0;
|
||||
for (let i = startIndex; ;) {
|
||||
if (i >= str.length)
|
||||
return i;
|
||||
const codePoint = str.codePointAt(i) as number;
|
||||
const w = infoToWidth(getInfo(codePoint), preferWide);
|
||||
rv += w;
|
||||
if (rv > column)
|
||||
return i;
|
||||
i += (codePoint <= 0xffff) ? 1 : 2;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
// Test if should break between beforeState and afterCode.
|
||||
// Return <= 0 if should break; > 0 if should join.
|
||||
// 'beforeState' is the return value from the previous possible break;
|
||||
// the value 0 is start of string.
|
||||
// 'afterCode' is the GRAPHEME_BREAK_Xxx value for the following codepoint.
|
||||
export function shouldJoin(beforeState: number, afterInfo: number): number {
|
||||
let beforeCode = (beforeState & GRAPHEME_BREAK_MASK) >> GRAPHEME_BREAK_SHIFT;
|
||||
let afterCode = (afterInfo & GRAPHEME_BREAK_MASK) >> GRAPHEME_BREAK_SHIFT;
|
||||
if (_shouldJoin(beforeCode, afterCode)) {
|
||||
if (afterCode === GRAPHEME_BREAK_Regional_Indicator)
|
||||
return GRAPHEME_BREAK_SAW_Regional_Pair;
|
||||
else
|
||||
return afterCode + 16;
|
||||
} else
|
||||
return afterCode - 16;
|
||||
}
|
||||
|
||||
export function shouldJoinBackwards(beforeInfo: number, afterState: number): number {
|
||||
let afterCode = (afterState & GRAPHEME_BREAK_MASK) >> GRAPHEME_BREAK_SHIFT;
|
||||
let beforeCode = (beforeInfo & GRAPHEME_BREAK_MASK) >> GRAPHEME_BREAK_SHIFT;
|
||||
if (_shouldJoin(beforeCode, afterCode)) {
|
||||
if (beforeCode === GRAPHEME_BREAK_Regional_Indicator)
|
||||
return GRAPHEME_BREAK_SAW_Regional_Pair;
|
||||
else
|
||||
return beforeCode + 16;
|
||||
} else
|
||||
return beforeCode - 16;
|
||||
}
|
||||
|
||||
/** Doesn't handle an odd number of RI characters. */
|
||||
function _shouldJoin(beforeCode: number, afterCode: number): boolean {
|
||||
if (beforeCode >= GRAPHEME_BREAK_Hangul_L
|
||||
&& beforeCode <= GRAPHEME_BREAK_Hangul_LVT) {
|
||||
if (beforeCode == GRAPHEME_BREAK_Hangul_L // GB6
|
||||
&& (afterCode == GRAPHEME_BREAK_Hangul_L
|
||||
|| afterCode == GRAPHEME_BREAK_Hangul_V
|
||||
|| afterCode == GRAPHEME_BREAK_Hangul_LV
|
||||
|| afterCode == GRAPHEME_BREAK_Hangul_LVT))
|
||||
return true;
|
||||
if ((beforeCode == GRAPHEME_BREAK_Hangul_LV // GB7
|
||||
|| beforeCode == GRAPHEME_BREAK_Hangul_V)
|
||||
&& (afterCode == GRAPHEME_BREAK_Hangul_V
|
||||
|| afterCode == GRAPHEME_BREAK_Hangul_T))
|
||||
return true;
|
||||
if ((beforeCode == GRAPHEME_BREAK_Hangul_LVT // GB8
|
||||
|| beforeCode == GRAPHEME_BREAK_Hangul_T)
|
||||
&& afterCode == GRAPHEME_BREAK_Hangul_T)
|
||||
return true;
|
||||
}
|
||||
if (afterCode == GRAPHEME_BREAK_Extend // GB9
|
||||
|| afterCode == GRAPHEME_BREAK_ZWJ
|
||||
|| beforeCode == GRAPHEME_BREAK_Prepend // GB9a
|
||||
|| afterCode == GRAPHEME_BREAK_SpacingMark) // GB9b
|
||||
return true;
|
||||
if (beforeCode == GRAPHEME_BREAK_ZWJ // GB11
|
||||
&& afterCode == GRAPHEME_BREAK_ExtPic)
|
||||
return true;
|
||||
if (afterCode == GRAPHEME_BREAK_Regional_Indicator // GB12, GB13
|
||||
&& beforeCode == GRAPHEME_BREAK_Regional_Indicator)
|
||||
return true;
|
||||
return false;
|
||||
}
|
||||
|
||||
export function getInfo(codePoint: number): number {
|
||||
return trieData.get(codePoint);
|
||||
}
|
||||
@@ -0,0 +1,380 @@
|
||||
var TINF_OK = 0;
|
||||
var TINF_DATA_ERROR = -3;
|
||||
|
||||
class Tree {
|
||||
table = new Uint16Array(16); /* table of code length counts */
|
||||
trans = new Uint16Array(288); /* code -> symbol translation table */
|
||||
};
|
||||
|
||||
class Data {
|
||||
tag: number = 0;
|
||||
bitcount: number = 0;
|
||||
destLen: number = 0;
|
||||
ltree: Tree;
|
||||
dtree: Tree;
|
||||
source: Uint8Array;
|
||||
dest: Uint8Array;
|
||||
sourceIndex: number = 0;
|
||||
|
||||
constructor(source: Uint8Array, dest: Uint8Array) {
|
||||
this.source = source;
|
||||
this.dest = dest;
|
||||
this.ltree = new Tree(); /* dynamic length/symbol tree */
|
||||
this.dtree = new Tree(); /* dynamic distance tree */
|
||||
}
|
||||
}
|
||||
|
||||
/* --------------------------------------------------- *
|
||||
* -- uninitialized global data (static structures) -- *
|
||||
* --------------------------------------------------- */
|
||||
|
||||
var sltree = new Tree();
|
||||
var sdtree = new Tree();
|
||||
|
||||
/* extra bits and base tables for length codes */
|
||||
var length_bits = new Uint8Array(30);
|
||||
var length_base = new Uint16Array(30);
|
||||
|
||||
/* extra bits and base tables for distance codes */
|
||||
var dist_bits = new Uint8Array(30);
|
||||
var dist_base = new Uint16Array(30);
|
||||
|
||||
/* special ordering of code length codes */
|
||||
var clcidx = new Uint8Array([
|
||||
16, 17, 18, 0, 8, 7, 9, 6,
|
||||
10, 5, 11, 4, 12, 3, 13, 2,
|
||||
14, 1, 15
|
||||
]);
|
||||
|
||||
/* used by tinf_decode_trees, avoids allocations every call */
|
||||
const code_tree = new Tree();
|
||||
const lengths = new Uint8Array(288 + 32);
|
||||
|
||||
/* ----------------------- *
|
||||
* -- utility functions -- *
|
||||
* ----------------------- */
|
||||
|
||||
/* build extra bits and base tables */
|
||||
function tinf_build_bits_base(bits: Uint8Array, base: Uint16Array, delta: number, first: number): void {
|
||||
var i, sum;
|
||||
|
||||
/* build bits table */
|
||||
for (i = 0; i < delta; ++i) bits[i] = 0;
|
||||
for (i = 0; i < 30 - delta; ++i) bits[i + delta] = i / delta | 0;
|
||||
|
||||
/* build base table */
|
||||
for (sum = first, i = 0; i < 30; ++i) {
|
||||
base[i] = sum;
|
||||
sum += 1 << bits[i];
|
||||
}
|
||||
}
|
||||
|
||||
/* build the fixed huffman trees */
|
||||
function tinf_build_fixed_trees(lt: Tree, dt: Tree): void {
|
||||
var i;
|
||||
|
||||
/* build fixed length tree */
|
||||
for (i = 0; i < 7; ++i) lt.table[i] = 0;
|
||||
|
||||
lt.table[7] = 24;
|
||||
lt.table[8] = 152;
|
||||
lt.table[9] = 112;
|
||||
|
||||
for (i = 0; i < 24; ++i) lt.trans[i] = 256 + i;
|
||||
for (i = 0; i < 144; ++i) lt.trans[24 + i] = i;
|
||||
for (i = 0; i < 8; ++i) lt.trans[24 + 144 + i] = 280 + i;
|
||||
for (i = 0; i < 112; ++i) lt.trans[24 + 144 + 8 + i] = 144 + i;
|
||||
|
||||
/* build fixed distance tree */
|
||||
for (i = 0; i < 5; ++i) dt.table[i] = 0;
|
||||
|
||||
dt.table[5] = 32;
|
||||
|
||||
for (i = 0; i < 32; ++i) dt.trans[i] = i;
|
||||
}
|
||||
|
||||
/* given an array of code lengths, build a tree */
|
||||
var offs = new Uint16Array(16);
|
||||
|
||||
function tinf_build_tree(t: Tree, lengths: Uint8Array, off: number, num: number): void {
|
||||
var i, sum;
|
||||
|
||||
/* clear code length count table */
|
||||
for (i = 0; i < 16; ++i) t.table[i] = 0;
|
||||
|
||||
/* scan symbol lengths, and sum code length counts */
|
||||
for (i = 0; i < num; ++i) t.table[lengths[off + i]]++;
|
||||
|
||||
t.table[0] = 0;
|
||||
|
||||
/* compute offset table for distribution sort */
|
||||
for (sum = 0, i = 0; i < 16; ++i) {
|
||||
offs[i] = sum;
|
||||
sum += t.table[i];
|
||||
}
|
||||
|
||||
/* create code->symbol translation table (symbols sorted by code) */
|
||||
for (i = 0; i < num; ++i) {
|
||||
if (lengths[off + i]) t.trans[offs[lengths[off + i]]++] = i;
|
||||
}
|
||||
}
|
||||
|
||||
/* ---------------------- *
|
||||
* -- decode functions -- *
|
||||
* ---------------------- */
|
||||
|
||||
/* get one bit from source stream */
|
||||
function tinf_getbit(d: Data): number {
|
||||
/* check if tag is empty */
|
||||
if (!d.bitcount--) {
|
||||
/* load next tag */
|
||||
d.tag = d.source[d.sourceIndex++];
|
||||
d.bitcount = 7;
|
||||
}
|
||||
|
||||
/* shift bit out of tag */
|
||||
var bit = d.tag & 1;
|
||||
d.tag >>>= 1;
|
||||
|
||||
return bit;
|
||||
}
|
||||
|
||||
/* read a num bit value from a stream and add base */
|
||||
function tinf_read_bits(d: Data, num: number, base: number): number {
|
||||
if (!num)
|
||||
return base;
|
||||
|
||||
while (d.bitcount < 24) {
|
||||
d.tag |= d.source[d.sourceIndex++] << d.bitcount;
|
||||
d.bitcount += 8;
|
||||
}
|
||||
|
||||
var val = d.tag & (0xffff >>> (16 - num));
|
||||
d.tag >>>= num;
|
||||
d.bitcount -= num;
|
||||
return val + base;
|
||||
}
|
||||
|
||||
/* given a data stream and a tree, decode a symbol */
|
||||
function tinf_decode_symbol(d: Data, t: Tree): number {
|
||||
while (d.bitcount < 24) {
|
||||
d.tag |= d.source[d.sourceIndex++] << d.bitcount;
|
||||
d.bitcount += 8;
|
||||
}
|
||||
|
||||
var sum = 0, cur = 0, len = 0;
|
||||
var tag = d.tag;
|
||||
|
||||
/* get more bits while code value is above sum */
|
||||
do {
|
||||
cur = 2 * cur + (tag & 1);
|
||||
tag >>>= 1;
|
||||
++len;
|
||||
|
||||
sum += t.table[len];
|
||||
cur -= t.table[len];
|
||||
} while (cur >= 0);
|
||||
|
||||
d.tag = tag;
|
||||
d.bitcount -= len;
|
||||
|
||||
return t.trans[sum + cur];
|
||||
}
|
||||
|
||||
/* given a data stream, decode dynamic trees from it */
|
||||
function tinf_decode_trees(d: Data, lt: Tree, dt: Tree): void {
|
||||
var hlit, hdist, hclen;
|
||||
var i, num, length;
|
||||
|
||||
/* get 5 bits HLIT (257-286) */
|
||||
hlit = tinf_read_bits(d, 5, 257);
|
||||
|
||||
/* get 5 bits HDIST (1-32) */
|
||||
hdist = tinf_read_bits(d, 5, 1);
|
||||
|
||||
/* get 4 bits HCLEN (4-19) */
|
||||
hclen = tinf_read_bits(d, 4, 4);
|
||||
|
||||
for (i = 0; i < 19; ++i) lengths[i] = 0;
|
||||
|
||||
/* read code lengths for code length alphabet */
|
||||
for (i = 0; i < hclen; ++i) {
|
||||
/* get 3 bits code length (0-7) */
|
||||
var clen = tinf_read_bits(d, 3, 0);
|
||||
lengths[clcidx[i]] = clen;
|
||||
}
|
||||
|
||||
/* build code length tree */
|
||||
tinf_build_tree(code_tree, lengths, 0, 19);
|
||||
|
||||
/* decode code lengths for the dynamic trees */
|
||||
for (num = 0; num < hlit + hdist;) {
|
||||
var sym = tinf_decode_symbol(d, code_tree);
|
||||
|
||||
switch (sym) {
|
||||
case 16:
|
||||
/* copy previous code length 3-6 times (read 2 bits) */
|
||||
var prev = lengths[num - 1];
|
||||
for (length = tinf_read_bits(d, 2, 3); length; --length) {
|
||||
lengths[num++] = prev;
|
||||
}
|
||||
break;
|
||||
case 17:
|
||||
/* repeat code length 0 for 3-10 times (read 3 bits) */
|
||||
for (length = tinf_read_bits(d, 3, 3); length; --length) {
|
||||
lengths[num++] = 0;
|
||||
}
|
||||
break;
|
||||
case 18:
|
||||
/* repeat code length 0 for 11-138 times (read 7 bits) */
|
||||
for (length = tinf_read_bits(d, 7, 11); length; --length) {
|
||||
lengths[num++] = 0;
|
||||
}
|
||||
break;
|
||||
default:
|
||||
/* values 0-15 represent the actual code lengths */
|
||||
lengths[num++] = sym;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
/* build dynamic trees */
|
||||
tinf_build_tree(lt, lengths, 0, hlit);
|
||||
tinf_build_tree(dt, lengths, hlit, hdist);
|
||||
}
|
||||
|
||||
/* ----------------------------- *
|
||||
* -- block inflate functions -- *
|
||||
* ----------------------------- */
|
||||
|
||||
/* given a stream and two trees, inflate a block of data */
|
||||
function tinf_inflate_block_data(d: Data, lt: Tree, dt: Tree): number {
|
||||
for (;;) {
|
||||
var sym = tinf_decode_symbol(d, lt);
|
||||
|
||||
/* check for end of block */
|
||||
if (sym === 256) {
|
||||
return TINF_OK;
|
||||
}
|
||||
|
||||
if (sym < 256) {
|
||||
d.dest[d.destLen++] = sym;
|
||||
} else {
|
||||
var length, dist, offs;
|
||||
var i;
|
||||
|
||||
sym -= 257;
|
||||
|
||||
/* possibly get more bits from length code */
|
||||
length = tinf_read_bits(d, length_bits[sym], length_base[sym]);
|
||||
|
||||
dist = tinf_decode_symbol(d, dt);
|
||||
|
||||
/* possibly get more bits from distance code */
|
||||
offs = d.destLen - tinf_read_bits(d, dist_bits[dist], dist_base[dist]);
|
||||
|
||||
/* copy match */
|
||||
for (i = offs; i < offs + length; ++i) {
|
||||
d.dest[d.destLen++] = d.dest[i];
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/* inflate an uncompressed block of data */
|
||||
function tinf_inflate_uncompressed_block(d: Data) {
|
||||
var length, invlength;
|
||||
var i;
|
||||
|
||||
/* unread from bitbuffer */
|
||||
while (d.bitcount > 8) {
|
||||
d.sourceIndex--;
|
||||
d.bitcount -= 8;
|
||||
}
|
||||
|
||||
/* get length */
|
||||
length = d.source[d.sourceIndex + 1];
|
||||
length = 256 * length + d.source[d.sourceIndex];
|
||||
|
||||
/* get one's complement of length */
|
||||
invlength = d.source[d.sourceIndex + 3];
|
||||
invlength = 256 * invlength + d.source[d.sourceIndex + 2];
|
||||
|
||||
/* check length */
|
||||
if (length !== (~invlength & 0x0000ffff))
|
||||
return TINF_DATA_ERROR;
|
||||
|
||||
d.sourceIndex += 4;
|
||||
|
||||
/* copy block */
|
||||
for (i = length; i; --i)
|
||||
d.dest[d.destLen++] = d.source[d.sourceIndex++];
|
||||
|
||||
/* make sure we start next block on a byte boundary */
|
||||
d.bitcount = 0;
|
||||
|
||||
return TINF_OK;
|
||||
}
|
||||
|
||||
/* inflate stream from source to dest */
|
||||
function tinf_uncompress(source: Uint8Array, dest: Uint8Array) {
|
||||
var d = new Data(source, dest);
|
||||
var bfinal, btype, res;
|
||||
|
||||
do {
|
||||
/* read final block flag */
|
||||
bfinal = tinf_getbit(d);
|
||||
|
||||
/* read block type (2 bits) */
|
||||
btype = tinf_read_bits(d, 2, 0);
|
||||
|
||||
/* decompress block */
|
||||
switch (btype) {
|
||||
case 0:
|
||||
/* decompress uncompressed block */
|
||||
res = tinf_inflate_uncompressed_block(d);
|
||||
break;
|
||||
case 1:
|
||||
/* decompress block with fixed huffman trees */
|
||||
res = tinf_inflate_block_data(d, sltree, sdtree);
|
||||
break;
|
||||
case 2:
|
||||
/* decompress block with dynamic huffman trees */
|
||||
tinf_decode_trees(d, d.ltree, d.dtree);
|
||||
res = tinf_inflate_block_data(d, d.ltree, d.dtree);
|
||||
break;
|
||||
default:
|
||||
res = TINF_DATA_ERROR;
|
||||
}
|
||||
|
||||
if (res !== TINF_OK)
|
||||
throw new Error('Data error');
|
||||
|
||||
} while (!bfinal);
|
||||
|
||||
if (d.destLen < d.dest.length) {
|
||||
if (typeof d.dest.slice === 'function')
|
||||
return d.dest.slice(0, d.destLen);
|
||||
else
|
||||
return d.dest.subarray(0, d.destLen);
|
||||
}
|
||||
|
||||
return d.dest;
|
||||
}
|
||||
|
||||
/* -------------------- *
|
||||
* -- initialization -- *
|
||||
* -------------------- */
|
||||
|
||||
/* build fixed huffman trees */
|
||||
tinf_build_fixed_trees(sltree, sdtree);
|
||||
|
||||
/* build extra bits and base tables */
|
||||
tinf_build_bits_base(length_bits, length_base, 4, 3);
|
||||
tinf_build_bits_base(dist_bits, dist_base, 2, 1);
|
||||
|
||||
/* fix a special case */
|
||||
length_bits[28] = 0;
|
||||
length_base[28] = 258;
|
||||
|
||||
export default tinf_uncompress
|
||||
@@ -0,0 +1,33 @@
|
||||
{
|
||||
"compilerOptions": {
|
||||
"module": "commonjs",
|
||||
"target": "es2015",
|
||||
"lib": [
|
||||
"dom",
|
||||
"es2015"
|
||||
],
|
||||
"rootDir": ".",
|
||||
"outDir": "../out",
|
||||
"sourceMap": true,
|
||||
"removeComments": true,
|
||||
"strict": true,
|
||||
"baseUrl": ".",
|
||||
"paths": {
|
||||
"common/*": [
|
||||
"../../../src/common/*"
|
||||
]
|
||||
},
|
||||
"types": [
|
||||
"../../../node_modules/@types/mocha"
|
||||
]
|
||||
},
|
||||
"include": [
|
||||
"./**/*",
|
||||
"../../../typings/xterm.d.ts"
|
||||
],
|
||||
"references": [
|
||||
{
|
||||
"path": "../../../src/common"
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,134 @@
|
||||
import inflate from './tiny-inflate'
|
||||
|
||||
// Shift size for getting the index-1 table offset.
|
||||
const SHIFT_1 = 6 + 5;
|
||||
|
||||
// Shift size for getting the index-2 table offset.
|
||||
const SHIFT_2 = 5;
|
||||
|
||||
// Difference between the two shift sizes,
|
||||
// for getting an index-1 offset from an index-2 offset. 6=11-5
|
||||
const SHIFT_1_2 = SHIFT_1 - SHIFT_2;
|
||||
|
||||
// Number of index-1 entries for the BMP. 32=0x20
|
||||
// This part of the index-1 table is omitted from the serialized form.
|
||||
const OMITTED_BMP_INDEX_1_LENGTH = 0x10000 >> SHIFT_1;
|
||||
|
||||
// Number of entries in an index-2 block. 64=0x40
|
||||
const INDEX_2_BLOCK_LENGTH = 1 << SHIFT_1_2;
|
||||
|
||||
// Mask for getting the lower bits for the in-index-2-block offset. */
|
||||
const INDEX_2_MASK = INDEX_2_BLOCK_LENGTH - 1;
|
||||
|
||||
// Shift size for shifting left the index array values.
|
||||
// Increases possible data size with 16-bit index values at the cost
|
||||
// of compactability.
|
||||
// This requires data blocks to be aligned by DATA_GRANULARITY.
|
||||
const INDEX_SHIFT = 2;
|
||||
|
||||
// Number of entries in a data block. 32=0x20
|
||||
const DATA_BLOCK_LENGTH = 1 << SHIFT_2;
|
||||
|
||||
// Mask for getting the lower bits for the in-data-block offset.
|
||||
const DATA_MASK = DATA_BLOCK_LENGTH - 1;
|
||||
|
||||
// The part of the index-2 table for U+D800..U+DBFF stores values for
|
||||
// lead surrogate code _units_ not code _points_.
|
||||
// Values for lead surrogate code _points_ are indexed with this portion of the table.
|
||||
// Length=32=0x20=0x400>>SHIFT_2. (There are 1024=0x400 lead surrogates.)
|
||||
const LSCP_INDEX_2_OFFSET = 0x10000 >> SHIFT_2;
|
||||
const LSCP_INDEX_2_LENGTH = 0x400 >> SHIFT_2;
|
||||
|
||||
// Count the lengths of both BMP pieces. 2080=0x820
|
||||
const INDEX_2_BMP_LENGTH = LSCP_INDEX_2_OFFSET + LSCP_INDEX_2_LENGTH;
|
||||
|
||||
// The 2-byte UTF-8 version of the index-2 table follows at offset 2080=0x820.
|
||||
// Length 32=0x20 for lead bytes C0..DF, regardless of SHIFT_2.
|
||||
const UTF8_2B_INDEX_2_OFFSET = INDEX_2_BMP_LENGTH;
|
||||
const UTF8_2B_INDEX_2_LENGTH = 0x800 >> 6; // U+0800 is the first code point after 2-byte UTF-8
|
||||
|
||||
// The index-1 table, only used for supplementary code points, at offset 2112=0x840.
|
||||
// Variable length, for code points up to highStart, where the last single-value range starts.
|
||||
// Maximum length 512=0x200=0x100000>>SHIFT_1.
|
||||
// (For 0x100000 supplementary code points U+10000..U+10ffff.)
|
||||
//
|
||||
// The part of the index-2 table for supplementary code points starts
|
||||
// after this index-1 table.
|
||||
//
|
||||
// Both the index-1 table and the following part of the index-2 table
|
||||
// are omitted completely if there is only BMP data.
|
||||
const INDEX_1_OFFSET = UTF8_2B_INDEX_2_OFFSET + UTF8_2B_INDEX_2_LENGTH;
|
||||
|
||||
// The alignment size of a data block. Also the granularity for compaction.
|
||||
const DATA_GRANULARITY = 1 << INDEX_SHIFT;
|
||||
|
||||
const isBigEndian = (new Uint8Array(new Uint32Array([0x12345678]).buffer)[0] === 0x12);
|
||||
|
||||
class UnicodeTrie {
|
||||
private data: Uint32Array;
|
||||
private highStart: number;
|
||||
private errorValue: number;
|
||||
constructor(data: Uint8Array) {
|
||||
// read binary format
|
||||
|
||||
const view = new DataView(data.buffer);
|
||||
this.highStart = view.getUint32(0, true);
|
||||
this.errorValue = view.getUint32(4, true);
|
||||
let uncompressedLength = view.getUint32(8, true);
|
||||
data = data.subarray(12);
|
||||
|
||||
// double inflate the actual trie data
|
||||
data = inflate(data, new Uint8Array(uncompressedLength));
|
||||
data = inflate(data, new Uint8Array(uncompressedLength));
|
||||
|
||||
if (isBigEndian) {
|
||||
// swap bytes from little-endian
|
||||
const len = data.length;
|
||||
for (let i = 0; i < len; i += 4) {
|
||||
// Exchange data[i] and data[i + 3]:
|
||||
let x = data[i]; data[i] = data[i+3]; data[i+3] = x;
|
||||
// Exchange data[i + 1] and data[i + 2]:
|
||||
let y = data[i+1]; data[i+1] = data[i+2]; data[i+2] = y;
|
||||
}
|
||||
}
|
||||
|
||||
this.data = new Uint32Array(data.buffer);
|
||||
|
||||
}
|
||||
|
||||
get(codePoint: number): number {
|
||||
let index;
|
||||
if ((codePoint < 0) || (codePoint > 0x10ffff)) {
|
||||
return this.errorValue;
|
||||
}
|
||||
|
||||
if ((codePoint < 0xd800) || ((codePoint > 0xdbff) && (codePoint <= 0xffff))) {
|
||||
// Ordinary BMP code point, excluding leading surrogates.
|
||||
// BMP uses a single level lookup. BMP index starts at offset 0 in the index.
|
||||
// data is stored in the index array itself.
|
||||
index = (this.data[codePoint >> SHIFT_2] << INDEX_SHIFT) + (codePoint & DATA_MASK);
|
||||
return this.data[index];
|
||||
}
|
||||
|
||||
if (codePoint <= 0xffff) {
|
||||
// Lead Surrogate Code Point. A Separate index section is stored for
|
||||
// lead surrogate code units and code points.
|
||||
// The main index has the code unit data.
|
||||
// For this function, we need the code point data.
|
||||
index = (this.data[LSCP_INDEX_2_OFFSET + ((codePoint - 0xd800) >> SHIFT_2)] << INDEX_SHIFT) + (codePoint & DATA_MASK);
|
||||
return this.data[index];
|
||||
}
|
||||
|
||||
if (codePoint < this.highStart) {
|
||||
// Supplemental code point, use two-level lookup.
|
||||
index = this.data[(INDEX_1_OFFSET - OMITTED_BMP_INDEX_1_LENGTH) + (codePoint >> SHIFT_1)];
|
||||
index = this.data[index + ((codePoint >> SHIFT_2) & INDEX_2_MASK)];
|
||||
index = (index << INDEX_SHIFT) + (codePoint & DATA_MASK);
|
||||
return this.data[index];
|
||||
}
|
||||
|
||||
return this.data[this.data.length - DATA_GRANULARITY];
|
||||
}
|
||||
}
|
||||
|
||||
export default UnicodeTrie
|
||||
Reference in New Issue
Block a user