From 596634ac4189852a15d0178e49201fc7d52c968d Mon Sep 17 00:00:00 2001 From: jerch Date: Sat, 6 Jun 2026 18:02:43 +0200 Subject: [PATCH] fix #6003 (#6004) --- src/common/input/TextDecoder.test.ts | 64 +++++++++++++++++++--------- src/common/input/TextDecoder.ts | 4 +- 2 files changed, 47 insertions(+), 21 deletions(-) diff --git a/src/common/input/TextDecoder.test.ts b/src/common/input/TextDecoder.test.ts index b32d8a721..0276fc8c8 100644 --- a/src/common/input/TextDecoder.test.ts +++ b/src/common/input/TextDecoder.test.ts @@ -79,7 +79,7 @@ function assertDecodedRange( } const target = new Uint32Array(count); const length = decode(input, target); - assert.equal(length, count); + assert.strictEqual(length, count); let mismatchIndex = -1; let index = 0; for (let i = min; i < max; ++i) { @@ -92,8 +92,8 @@ function assertDecodedRange( } index++; } - assert.equal(mismatchIndex, -1); - assert.equal(outputToString(target, length), input); + assert.strictEqual(mismatchIndex, -1); + assert.strictEqual(outputToString(target, length), input); } const BATCH_SIZE = 8192; @@ -116,9 +116,9 @@ describe('text encodings', () => { const data = new Uint32Array(s.length); for (let i = 0; i < s.length; ++i) { data[i] = s.charCodeAt(i); - assert.equal(stringFromCodePoint(data[i]), s[i]); + assert.strictEqual(stringFromCodePoint(data[i]), s[i]); } - assert.equal(utf32ToString(data), s); + assert.strictEqual(utf32ToString(data), s); }); describe('StringToUtf32 decoder', () => { @@ -159,7 +159,7 @@ describe('text encodings', () => { const decoder = new StringToUtf32(); const target = new Uint32Array(5); const length = decoder.decode(String.fromCharCode(0xFEFF), target); - assert.equal(length, 0); + assert.strictEqual(length, 0); decoder.clear(); }); }); @@ -169,7 +169,7 @@ describe('text encodings', () => { const target = new Uint32Array(500); for (let i = 0; i < TEST_STRINGS.length; ++i) { const length = decoder.decode(TEST_STRINGS[i], target); - assert.equal(toString(target, length), TEST_STRINGS[i]); + assert.strictEqual(toString(target, length), TEST_STRINGS[i]); decoder.clear(); } }); @@ -184,7 +184,7 @@ describe('text encodings', () => { const written = decoder.decode(input[i], target); decoded += toString(target, written); } - assert(decoded, 'Γ„β‚¬π„žΓ–π„žβ‚¬Γœπ„žβ‚¬'); + assert.strictEqual(decoded, 'Γ„β‚¬π„žΓ–π„žβ‚¬Γœπ„žβ‚¬'); }); }); }); @@ -226,7 +226,7 @@ describe('text encodings', () => { const target = new Uint32Array(5); const utf8Data = stringToUtf8Bytes(String.fromCharCode(0xFEFF)); const length = decoder.decode(utf8Data, target); - assert.equal(length, 0); + assert.strictEqual(length, 0); decoder.clear(); }); }); @@ -237,7 +237,7 @@ describe('text encodings', () => { for (let i = 0; i < TEST_STRINGS.length; ++i) { const utf8Data = stringToUtf8Bytes(TEST_STRINGS[i]); const length = decoder.decode(utf8Data, target); - assert.equal(toString(target, length), TEST_STRINGS[i]); + assert.strictEqual(toString(target, length), TEST_STRINGS[i]); decoder.clear(); } }); @@ -252,7 +252,7 @@ describe('text encodings', () => { const written = decoder.decode(utf8Data.slice(i, i + 1), target); decoded += toString(target, written); } - assert(decoded, 'Γ„Γ–ΓœΓŸΓΆΓ€ΓΌ'); + assert.strictEqual(decoded, 'Γ„Γ–ΓœΓŸΓΆΓ€ΓΌ'); }); it('2/3 byte sequences - advance by 1', () => { @@ -264,7 +264,7 @@ describe('text encodings', () => { const written = decoder.decode(utf8Data.slice(i, i + 1), target); decoded += toString(target, written); } - assert(decoded, 'Γ„β‚¬Γ–β‚¬Γœβ‚¬ΓŸβ‚¬ΓΆβ‚¬Γ€β‚¬ΓΌ'); + assert.strictEqual(decoded, 'Γ„β‚¬Γ–β‚¬Γœβ‚¬ΓŸβ‚¬ΓΆβ‚¬Γ€β‚¬ΓΌ'); }); it('2/3/4 byte sequences - advance by 1', () => { @@ -276,7 +276,7 @@ describe('text encodings', () => { const written = decoder.decode(utf8Data.slice(i, i + 1), target); decoded += toString(target, written); } - assert(decoded, 'Γ„β‚¬π„žΓ–π„žβ‚¬Γœπ„žβ‚¬'); + assert.strictEqual(decoded, 'Γ„β‚¬π„žΓ–π„žβ‚¬Γœπ„žβ‚¬'); }); it('2/3/4 byte sequences - advance by 2', () => { @@ -288,7 +288,7 @@ describe('text encodings', () => { const written = decoder.decode(utf8Data.slice(i, i + 2), target); decoded += toString(target, written); } - assert(decoded, 'Γ„β‚¬π„žΓ–π„žβ‚¬Γœπ„žβ‚¬'); + assert.strictEqual(decoded, 'Γ„β‚¬π„žΓ–π„žβ‚¬Γœπ„žβ‚¬'); }); it('2/3/4 byte sequences - advance by 3', () => { @@ -300,7 +300,7 @@ describe('text encodings', () => { const written = decoder.decode(utf8Data.slice(i, i + 3), target); decoded += toString(target, written); } - assert(decoded, 'Γ„β‚¬π„žΓ–π„žβ‚¬Γœπ„žβ‚¬'); + assert.strictEqual(decoded, 'Γ„β‚¬π„žΓ–π„žβ‚¬Γœπ„žβ‚¬'); }); it('BOMs (3 byte sequences) - advance by 2', () => { @@ -312,7 +312,7 @@ describe('text encodings', () => { const written = decoder.decode(utf8Data.slice(i, i + 2), target); decoded += toString(target, written); } - assert.equal(decoded, ''); + assert.strictEqual(decoded, ''); }); it('test break after 3 bytes - issue #2495', () => { @@ -320,11 +320,37 @@ describe('text encodings', () => { const target = new Uint32Array(5); const utf8Data = fromByteString('\xf0\xa0\x9c\x8e'); let written = decoder.decode(utf8Data.slice(0, 3), target); - assert.equal(written, 0); + assert.strictEqual(written, 0); written = decoder.decode(utf8Data.slice(3), target); - assert.equal(written, 1); - assert(toString(target, written), '𠜎'); + assert.strictEqual(written, 1); + assert.strictEqual(toString(target, written), '𠜎'); }); + + describe('0x80 not swallowed in continuation', () => { + it('Aβ€”B', () => { + const decoder = new Utf8ToUtf32(); + const target = new Uint32Array(5); + const utf8Data = new TextEncoder().encode('Aβ€”BAβ€”BAβ€”BAβ€”BAβ€”B'); + let decoded = ''; + for (let i = 0; i < utf8Data.length; i += 2) { + const written = decoder.decode(utf8Data.slice(i, i + 2), target); + decoded += toString(target, written); + } + assert.strictEqual(decoded, 'Aβ€”BAβ€”BAβ€”BAβ€”BAβ€”B'); + }); + it('A𐀀B', () => { + const decoder = new Utf8ToUtf32(); + const target = new Uint32Array(5); + const utf8Data = new TextEncoder().encode('A𐀀BA𐀀BA𐀀BA𐀀BA𐀀B'); + let decoded = ''; + for (let i = 0; i < utf8Data.length; i += 2) { + const written = decoder.decode(utf8Data.slice(i, i + 2), target); + decoded += toString(target, written); + } + assert.strictEqual(decoded, 'A𐀀BA𐀀BA𐀀BA𐀀BA𐀀B'); + }); + }); + }); }); }); diff --git a/src/common/input/TextDecoder.ts b/src/common/input/TextDecoder.ts index ac7f8d155..7e0495630 100644 --- a/src/common/input/TextDecoder.ts +++ b/src/common/input/TextDecoder.ts @@ -158,9 +158,9 @@ export class Utf8ToUtf32 { cp &= ((((cp & 0xE0) === 0xC0)) ? 0x1F : (((cp & 0xF0) === 0xE0)) ? 0x0F : 0x07); let pos = 0; let tmp: number; - while ((tmp = this.interim[++pos] & 0x3F) && pos < 4) { + while ((tmp = this.interim[++pos]) && pos < 4) { cp <<= 6; - cp |= tmp; + cp |= tmp & 0x3F; } // missing bytes - read ahead from input const type = (((this.interim[0] & 0xE0) === 0xC0)) ? 2 : (((this.interim[0] & 0xF0) === 0xE0)) ? 3 : 4;