448 lines
13 KiB
JavaScript
448 lines
13 KiB
JavaScript
const fs = require('fs');
|
|
const path = require('path');
|
|
|
|
/**
|
|
* For decoder side:
|
|
* - reconstruct grays with f = lambda x: (x * 256 - x + 50) / 100 (needs value rounding check)
|
|
* - apply nameFilter
|
|
* - build lookup & extract functions (runtime decoding?)
|
|
* - transfer hashtable and color table
|
|
*/
|
|
|
|
// match color entry in rgb.txt
|
|
const rexFile = /^\s*(\d+)\s*(\d+)\s*(\d+)\s*?[\t]+(.*)$/;
|
|
// match greyXX|grayXX names
|
|
const rexGrey = /^gr[ae]y\d+/;
|
|
// name match
|
|
const rexName = /^\w?[A-Za-z0-9 ]+\w/;
|
|
|
|
function parseX11Colors(filename) {
|
|
const fileData = fs.readFileSync(filename, {encoding: 'utf8'});
|
|
const colors = [];
|
|
for (const line of fileData.split('\n')) {
|
|
const m = rexFile.exec(line);
|
|
if (m) {
|
|
colors.push({
|
|
color: [parseInt(m[1]), parseInt(m[2]), parseInt(m[3])],
|
|
name: m[4]
|
|
});
|
|
}
|
|
}
|
|
return colors;
|
|
}
|
|
|
|
// Use the FNV algorithm from http://isthe.com/chongo/tech/comp/fnv/
|
|
function hash(d, name) {
|
|
if (!d) d = 0x01000193;
|
|
for (const c of name) {
|
|
d = ( (d * 0x01000193) ^ c.charCodeAt(0) ) & 0xffffffff;
|
|
}
|
|
return d >>> 0;
|
|
}
|
|
|
|
// inspired from http://stevehanov.ca/blog/?id=119
|
|
function createMinimalPerfectHash(nameList) {
|
|
const nameMap = Object.create(null);
|
|
for (const [idx, name] of nameList.entries()) {
|
|
nameMap[name] = idx;
|
|
}
|
|
const size = nameList.length;
|
|
|
|
const buckets = [];
|
|
for (let i = 0; i < size; ++i) buckets.push([]);
|
|
const g = new Array(size).fill(0);
|
|
const v = new Array(size).fill(null);
|
|
|
|
for (const key of nameList) {
|
|
buckets[hash(0, key) % size].push(key);
|
|
}
|
|
|
|
buckets.sort((a, b) => b.length - a.length);
|
|
let breakIdx = 0;
|
|
for (let i = 0; i < size; ++i) {
|
|
const bucket = buckets[i];
|
|
if (bucket.length <= 1) {
|
|
breakIdx = i;
|
|
break;
|
|
}
|
|
let d = 1;
|
|
let item = 0;
|
|
let slots = [];
|
|
|
|
while (item < bucket.length) {
|
|
const slot = hash(d, bucket[item]) % size;
|
|
if (v[slot] !== null || slots.includes(slot)) {
|
|
d++;
|
|
item = 0;
|
|
slots = [];
|
|
} else {
|
|
slots.push(slot);
|
|
item++;
|
|
}
|
|
}
|
|
|
|
g[hash(0, bucket[0]) % size] = d;
|
|
for (let k = 0; k < bucket.length; ++k) {
|
|
v[slots[k]] = nameMap[bucket[k]];
|
|
}
|
|
}
|
|
|
|
const freeList = [];
|
|
for (let i = 0; i < size; ++i) {
|
|
if (v[i] === null) freeList.push(i);
|
|
}
|
|
|
|
for (let i = breakIdx; i < size; ++i) {
|
|
const bucket = buckets[i];
|
|
if (!bucket.length) break;
|
|
const slot = freeList.pop();
|
|
g[hash(0, bucket[0]) % size] = -slot - 1;
|
|
v[slot] = nameMap[bucket[0]];
|
|
}
|
|
|
|
return [g, v];
|
|
}
|
|
|
|
function lookupPerfectHash(g, v, key) {
|
|
const d = g[hash(0, key) % g.length];
|
|
if (d < 0) return v[-d - 1];
|
|
return v[hash(d, key) % v.length];
|
|
}
|
|
|
|
function checkPerfectHashTables(g, v, nameList) {
|
|
let allPassed = true;
|
|
for (const [idx, name] of nameList.entries()) {
|
|
const lookup = lookupPerfectHash(g, v, name);
|
|
if (lookup !== idx) {
|
|
console.log(`\x1b[33mmismatch: '${name}' returns ${lookup} (orig: ${idx})\x1b[m`);
|
|
allPassed = false;
|
|
}
|
|
}
|
|
return allPassed;
|
|
}
|
|
|
|
function crc10atm(name, crc) {
|
|
if (!crc) crc = 0;
|
|
for (const c of name) {
|
|
const v = c.charCodeAt(0);
|
|
crc ^= v << 2;
|
|
for (let k = 0; k < 8; k++) {
|
|
crc = crc & 0x200 ? (crc << 1) ^ 0x233 : crc << 1;
|
|
}
|
|
}
|
|
crc &= 0x3ff;
|
|
return crc >>> 0;
|
|
}
|
|
|
|
function checkColorList(g, v, crc, colorNames) {
|
|
let match = [];
|
|
for (const word of colorNames) {
|
|
if (!nameFilter(word)) continue;
|
|
const idx = lookupPerfectHash(g, v, word);
|
|
const crc10 = crc10atm(word);
|
|
if (crc[idx] === crc10) {
|
|
match.push({idx, word, colorName: colorNames[idx]});
|
|
}
|
|
}
|
|
return {tested: colorNames.length, match};
|
|
}
|
|
|
|
function checkWordlists(g, v, crc, colorNames, filename) {
|
|
const fileData = fs.readFileSync(filename, {encoding: 'utf8'});
|
|
const words = fileData.split('\n').filter(el => lookupPerfectHash.length !== 0);
|
|
let collisions = [];
|
|
for (const word of words) {
|
|
if (!nameFilter(word)) continue;
|
|
const idx = lookupPerfectHash(g, v, word);
|
|
const crc10 = crc10atm(word);
|
|
if (crc[idx] === crc10 && word !== colorNames[idx]) {
|
|
collisions.push({idx, word, colorName: colorNames[idx]});
|
|
}
|
|
}
|
|
return {tested: words.length, collisions};
|
|
}
|
|
|
|
function nameFilter(name) {
|
|
// length 3 - 22
|
|
if (name.length < 3 || name.length > 22) return;
|
|
// chars only in [A-Za-z0-9 ]
|
|
if (!rexName.exec(name)) return;
|
|
return name;
|
|
}
|
|
|
|
function compressTables(g, v, crc, al) {
|
|
// g | v | crc: all in 10 bit (<1024)
|
|
// --> 30 bit
|
|
// --> fits into 5 bytes of a 64-bit char alphabet
|
|
|
|
// g needs an offset for proper zero alignment
|
|
const gOffset = -Math.min(...g);
|
|
g = g.map(el => el + gOffset);
|
|
|
|
// assert we are in 0..2^10
|
|
if (Math.min(...g) < 0 || Math.min(...v) < 0 || Math.min(...crc) < 0
|
|
|| Math.max(...g) > 1023 || Math.max(...v) > 1023 || Math.max(...crc) > 1023
|
|
) {
|
|
console.log('\x1b[31mTables out of compressible range, manual fix needed.\x1b[m');
|
|
process.exit(1);
|
|
}
|
|
|
|
// assert we have same length
|
|
if (g.length !== v.length || g.length !== crc.length) {
|
|
console.log('\x1b[31mTables length mismatch, manual fix needed.\x1b[m');
|
|
process.exit(1);
|
|
}
|
|
|
|
// construct compressed data string
|
|
let result = '';
|
|
const length = g.length;
|
|
for (let i = 0; i < length; ++i) {
|
|
let value = (g[i] << 20) | (v[i] << 10) | crc[i];
|
|
let bucket = '';
|
|
for (let k = 0; k < 5; ++k) {
|
|
bucket += al[value % al.length];
|
|
value = Math.floor(value / al.length);
|
|
}
|
|
result += bucket.split('').reverse().join('');
|
|
}
|
|
|
|
return [result, length, gOffset];
|
|
}
|
|
|
|
function compressColors(colors, al) {
|
|
let result = '';
|
|
for (const color of colors) {
|
|
let value = (color[0] << 16) | (color[1] << 8) | color[2];
|
|
let bucket = '';
|
|
for (let k = 0; k < 4; ++k) {
|
|
bucket += al[value % al.length];
|
|
value = Math.floor(value / al.length);
|
|
}
|
|
result += bucket.split('').reverse().join('');
|
|
}
|
|
return result;
|
|
}
|
|
|
|
function loadData(bucket, al) {
|
|
let value = 0;
|
|
for (const c of bucket) {
|
|
value *= al.length;
|
|
value += al.indexOf(c);
|
|
}
|
|
return [value >>> 20, (value >> 10) & 0x3FF, value & 0x3FF];
|
|
}
|
|
|
|
function loadColor(al, data, idx) {
|
|
// color buckets are hardcoded to 4 chars
|
|
let value = 0;
|
|
for (let i = idx * 4; i < idx * 4 + 4; ++i) {
|
|
value *= al.length;
|
|
value += al.indexOf(data[i]);
|
|
}
|
|
return [value >>> 16, (value >> 8) & 0xFF, value & 0xFF];
|
|
}
|
|
|
|
function lookupIdx(al, data, length, gOffset, name) {
|
|
const bl = data.length / length;
|
|
let offset = (hash(0, name) % length) * bl;
|
|
let [g, v, crc] = loadData(data.slice(offset, offset + bl), al);
|
|
offset = g < gOffset
|
|
? (-(g - gOffset) - 1) * bl
|
|
: (hash(g - gOffset, name) % length) * bl;
|
|
[_, v, _] = loadData(data.slice(offset, offset + bl), al);
|
|
offset = v * bl;
|
|
[_, _, crc] = loadData(data.slice(offset, offset + bl), al);
|
|
return crc10atm(name) === crc ? v : -1;
|
|
}
|
|
|
|
function lookup(al, data, colorData, length, gOffset, name) {
|
|
const idx = lookupIdx(al, data, length, gOffset, name);
|
|
if (idx === -1) return;
|
|
return loadColor(al, colorData, idx);
|
|
}
|
|
|
|
function createModule(tableData, colorData, alphabet, gOffset, length) {
|
|
const TMPL = `/**
|
|
* Copyright (c) 2021 The xterm.js authors. All rights reserved.
|
|
* @license MIT
|
|
*/
|
|
|
|
/**
|
|
* This module enables X11 color name lookups with the help of a perfect hash function
|
|
* to not penalize the package size too much.
|
|
* saving: ~70% (from 17kB down to 5kB)
|
|
*/
|
|
|
|
// Note: module and table data created with fixtures/x11-colornames/create_module.js
|
|
const TABLE = '${tableData}';
|
|
const COLORS = '${colorData}';
|
|
const ALPHABET = '${alphabet}';
|
|
const OFFSET = ${gOffset};
|
|
const LENGTH = ${length};
|
|
const BUCKET_LENGTH = TABLE.length / LENGTH;
|
|
|
|
// match greyXX|grayXX names
|
|
const rexGrey = /^gr[ae]y(\\d+)/;
|
|
// name match
|
|
const rexName = /^\\w?[A-Za-z0-9 ]+\\w/;
|
|
|
|
function gray(n: number): [number, number, number] | undefined {
|
|
if (0 <= n && n <= 100) {
|
|
const v = Math.floor((n * 256 - n + 50) / 100);
|
|
return [v, v, v];
|
|
}
|
|
return;
|
|
}
|
|
|
|
function hash(d: number, name: string): number {
|
|
if (!d) d = 0x01000193;
|
|
for (const c of name) {
|
|
d = ((d * 0x01000193) ^ c.charCodeAt(0)) & 0xffffffff;
|
|
}
|
|
return d >>> 0;
|
|
}
|
|
|
|
function crc10(name: string, crc?: number): number {
|
|
if (!crc) crc = 0;
|
|
for (const c of name) {
|
|
crc ^= c.charCodeAt(0) << 2;
|
|
for (let k = 0; k < 8; k++) {
|
|
crc = crc & 0x200 ? (crc << 1) ^ 0x233 : crc << 1;
|
|
}
|
|
}
|
|
crc &= 0x3ff;
|
|
return crc >>> 0;
|
|
}
|
|
|
|
function loadData(idx: number): [number, number, number] {
|
|
let value = 0;
|
|
for (let i = idx * BUCKET_LENGTH; i < idx * BUCKET_LENGTH + BUCKET_LENGTH; ++i) {
|
|
value *= ALPHABET.length;
|
|
value += ALPHABET.indexOf(TABLE[i]);
|
|
}
|
|
return [value >>> 20, (value >> 10) & 0x3FF, value & 0x3FF];
|
|
}
|
|
|
|
function loadColor(idx: number): [number, number, number] {
|
|
// color buckets are hardcoded to 4 chars
|
|
let v = 0;
|
|
for (let i = idx * 4; i < idx * 4 + 4; ++i) {
|
|
v *= ALPHABET.length;
|
|
v += ALPHABET.indexOf(COLORS[i]);
|
|
}
|
|
return [v >>> 16, (v >> 8) & 0xFF, v & 0xFF];
|
|
}
|
|
|
|
function lookupIdx(name: string): number {
|
|
let b = loadData(hash(0, name) % LENGTH);
|
|
b = loadData(b[0] < OFFSET ? (-(b[0] - OFFSET) - 1) : hash(b[0] - OFFSET, name) % LENGTH);
|
|
const [ , , crc] = loadData(b[1]);
|
|
return crc10(name) === crc ? b[1] : -1;
|
|
}
|
|
|
|
export function getColorFromName(name: string): [number, number, number] | undefined {
|
|
// basic name filtering
|
|
if (name.length < 3 || name.length > 22 || !rexName.exec(name)) return;
|
|
|
|
// handle grays special
|
|
const m = rexGrey.exec(name);
|
|
if (m) return gray(parseInt(m[1]));
|
|
|
|
// grab crc checked idx from PHF
|
|
const idx = lookupIdx(name);
|
|
if (idx === -1) return;
|
|
return loadColor(idx);
|
|
}
|
|
`;
|
|
return TMPL;
|
|
}
|
|
|
|
|
|
function main() {
|
|
// parse definitions from X11 file
|
|
const colorList = parseX11Colors(path.join(__dirname, '/rgb.txt'))
|
|
.filter(el => !rexGrey.exec(el.name)); // remove greys, as we can reconstruct them later
|
|
const nameList = colorList.map(el => el.name);
|
|
|
|
// PHF creation and test
|
|
const [g, v] = createMinimalPerfectHash(nameList);
|
|
if (!checkPerfectHashTables(g, v, nameList)) {
|
|
console.error('\x1b[31mPHF creation failed.\x1b[m');
|
|
process.exit(1);
|
|
} else {
|
|
console.log('\x1b[32mPHF creation successful.\x1b[m');
|
|
}
|
|
|
|
// calc 10-bit CRCs to rule to avoid collisions at ~1:1024
|
|
const crc = nameList.map(el => crc10atm(el));
|
|
|
|
// collision check against colorNames - must all collide
|
|
console.log('Self collision test:')
|
|
const collSelf = checkColorList(g, v, crc, nameList);
|
|
console.log('self:', {
|
|
entries: collSelf.tested,
|
|
matches: collSelf.match.length,
|
|
rate: collSelf.tested / collSelf.match.length
|
|
});
|
|
if (collSelf.match.length !== collSelf.tested) {
|
|
console.error('\x1b[31mSelf collision tested failed.\x1b[m');
|
|
process.exit(1);
|
|
}
|
|
|
|
// basic collision checks against dictionaries (rates should be greater than 900:1)
|
|
console.log('Dictionary collision test:')
|
|
const collEnglish = checkWordlists(g, v, crc, nameList, '/usr/share/dict/words');
|
|
console.log('english:', {
|
|
entries: collEnglish.tested,
|
|
collisions: collEnglish.collisions.length,
|
|
rate: collEnglish.tested / collEnglish.collisions.length
|
|
});
|
|
//console.log(collEnglish.collisions);
|
|
const collnGerman = checkWordlists(g, v, crc, nameList, '/usr/share/dict/ngerman');
|
|
console.log('ngerman:', {
|
|
entries: collnGerman.tested,
|
|
collisions: collnGerman.collisions.length,
|
|
rate: collnGerman.tested / collnGerman.collisions.length
|
|
});
|
|
const colloGerman = checkWordlists(g, v, crc, nameList, '/usr/share/dict/ogerman');
|
|
console.log('ogerman:', {
|
|
entries: colloGerman.tested,
|
|
collisions: colloGerman.collisions.length,
|
|
rate: colloGerman.tested / colloGerman.collisions.length
|
|
});
|
|
|
|
// compression with custom alphabet
|
|
const ALPHABET = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789!?';
|
|
const [tableData, length, gOffset] = compressTables(g, v, crc, ALPHABET);
|
|
const colorData = compressColors(colorList.map(el => el.color), ALPHABET);
|
|
|
|
// test loading of compressed table data
|
|
let failed = false;
|
|
for (const name of nameList) {
|
|
const orig = lookupPerfectHash(g, v, name);
|
|
const idx = lookupIdx(ALPHABET, tableData, length, gOffset, name);
|
|
if (idx !== orig) {
|
|
console.log('\x1b[31mcompressed loading failed for:\x1b[m', {name, idx, orig});
|
|
failed = true;
|
|
}
|
|
}
|
|
if (failed) process.exit(1);
|
|
|
|
// test full color decoding
|
|
for (const color of colorList) {
|
|
const loaded = lookup(ALPHABET, tableData, colorData, length, gOffset, color.name);
|
|
if (loaded[0] !== color.color[0] || loaded[1] !== color.color[1] || loaded[2] !== color.color[2]) {
|
|
console.log('\x1b[31mcolor loading failed for:\x1b[m', {color, loaded});
|
|
failed = true;
|
|
}
|
|
}
|
|
if (failed) process.exit(1);
|
|
|
|
// if we made it up to here, create decoding boilerplate
|
|
const moduleData = createModule(tableData, colorData, ALPHABET, gOffset, length);
|
|
fs.writeFileSync(path.join(__dirname, '/ColorNames.ts'), moduleData);
|
|
console.log(`${moduleData.length} bytes written to ${path.join(__dirname, '/ColorNames.ts')}`);
|
|
}
|
|
|
|
main();
|