add TextDecoder for string to UTF32

src/core/input/TextDecoder.test.ts

@@ -0,0 +1,67 @@
+/**
+ * Copyright (c) 2019 The xterm.js authors. All rights reserved.
+ * @license MIT
+ */
+
+import { assert } from 'chai';
+import { StringToUtf32, stringFromCodePoint } from './TextDecoder';
+
+
+// convert UTF32 codepoints to string
+function toString(data: Uint32Array, length: number): string {
+  if ((String as any).fromCodePoint) {
+    return (String as any).fromCodePoint.apply(null, data.subarray(0, length));
+  }
+  let result = '';
+  for (let i = 0; i < length; ++i) {
+    result += stringFromCodePoint(data[i]);
+  }
+  return result;
+}
+
+describe('StringToUtf32 Decoder', () => {
+  describe('full codepoint test', () => {
+    it('0..65535', () => {
+      const decoder = new StringToUtf32();
+      const target = new Uint32Array(5);
+      for (let i = 0; i < 65536; ++i) {
+        // skip surrogate pairs
+        if (i >= 0xD800 && i <= 0xDFFF) {
+          continue;
+        }
+        const length = decoder.decode(String.fromCharCode(i), target);
+        assert.equal(length, 1);
+        assert.equal(target[0], i);
+        assert.equal(toString(target, length), String.fromCharCode(i));
+        decoder.clear();
+      }
+    });
+    it('65536..0x10FFFF (surrogates)', function(): void {
+      this.timeout(20000);
+      const decoder = new StringToUtf32();
+      const target = new Uint32Array(5);
+      for (let i = 65536; i < 0x10FFFF; ++i) {
+        const codePoint = i - 0x10000;
+        const s = String.fromCharCode((codePoint >> 10) + 0xD800) + String.fromCharCode((codePoint % 0x400) + 0xDC00);
+        const length = decoder.decode(s, target);
+        assert.equal(length, 1);
+        assert.equal(target[0], i);
+        assert.equal(toString(target, length), s);
+        decoder.clear();
+      }
+    });
+  });
+  describe('stream handling', () => {
+    it('surrogates mixed advance by 1', () => {
+      const decoder = new StringToUtf32();
+      const target = new Uint32Array(5);
+      const input = 'Ä€𝄞Ö𝄞€Ü𝄞€';
+      let decoded = '';
+      for (let i = 0; i < input.length; ++i) {
+        const written = decoder.decode(input[i], target);
+        decoded += toString(target, written);
+      }
+      assert(decoded, 'Ä€𝄞Ö𝄞€Ü𝄞€');
+    });
+  });
+});

src/core/input/TextDecoder.ts

@@ -0,0 +1,87 @@
+/**
+ * Copyright (c) 2019 The xterm.js authors. All rights reserved.
+ * @license MIT
+ */
+
+/**
+ * StringToUtf32 - decodes UTF16 sequences into UTF32 codepoints.
+ * To keep the decoder in line with JS strings it handles single surrogates as UCS2.
+ */
+export class StringToUtf32 {
+  private _interim: number = 0;
+
+  /**
+   * Clears interim and resets decoder to clean state.
+   */
+  public clear(): void {
+    this._interim = 0;
+  }
+
+  /**
+   * Decode JS string to UTF32 codepoints.
+   * The methods assumes stream input and will store partly transmitted
+   * surrogate pairs and decode them with the next data chunk.
+   * Note: The method does no bound checks for target, therefore make sure
+   * the provided input data does not exceed the size of `target`.
+   * Returns the number of written codepoints in `target`.
+   */
+  decode(input: string, target: Uint32Array): number {
+    const length = input.length;
+
+    if (!length) {
+      return 0;
+    }
+
+    let size = 0;
+    let startPos = 0;
+
+    // handle leftover surrogate high
+    if (this._interim) {
+      const second = input.charCodeAt(startPos++);
+      if (0xDC00 <= second && second <= 0xDFFF) {
+        target[size++] = (this._interim - 0xD800) * 0x400 + second - 0xDC00 + 0x10000;
+      } else {
+        // illegal codepoint (USC2 handling)
+        target[size++] = this._interim;
+        target[size++] = second;
+      }
+      this._interim = 0;
+    }
+
+    for (let i = startPos; i < length; ++i) {
+      const code = input.charCodeAt(i);
+      // surrogate pair first
+      if (0xD800 <= code && code <= 0xDBFF) {
+        if (++i >= length) {
+          this._interim = code;
+          return size;
+        }
+        const second = input.charCodeAt(i);
+        if (0xDC00 <= second && second <= 0xDFFF) {
+          target[size++] = (code - 0xD800) * 0x400 + second - 0xDC00 + 0x10000;
+        } else {
+          // illegal codepoint (USC2 handling)
+          target[size++] = code;
+          target[size++] = second;
+        }
+        continue;
+      }
+      target[size++] = code;
+    }
+    return size;
+  }
+}
+
+/**
+ * Polyfill - Convert UTF32 codepoint into JS string.
+ */
+export function stringFromCodePoint(codePoint: number): string {
+  if ((String as any).fromCodePoint) {
+    return (String as any).fromCodePoint(codePoint);
+  }
+  if (codePoint > 0xFFFF) {
+    codePoint -= 0x10000;
+    return String.fromCharCode((codePoint >> 10) + 0xD800) + String.fromCharCode((codePoint % 0x400) + 0xDC00);
+  }
+  return String.fromCharCode(codePoint);
+}