fix(parser): WASM scratch layout and scan cache for chunked parse
Place parser state above the linked transition table so scans emit ops correctly, and preserve or clear the WASM scan cache across nested and reused parse buffers. Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
@@ -71,6 +71,13 @@ function loadParams(host: IWasmParseHost, scan: ScanResult, opIndex: number): vo
|
||||
}
|
||||
}
|
||||
|
||||
/** True when a chunked WASM scan of `data` is still in progress (e.g. nested parse during print). */
|
||||
function isActiveWasmParse(scanCache: IWasmScanCache, data: Uint32Array, length: number): boolean {
|
||||
return scanCache.data === data
|
||||
&& (scanCache.inputOffset ?? 0) < length
|
||||
&& (scanCache.scan !== undefined || scanCache.chunkStart !== undefined);
|
||||
}
|
||||
|
||||
export function parseWithWasmScanner(
|
||||
host: IWasmParseHost,
|
||||
data: Uint32Array,
|
||||
@@ -85,10 +92,14 @@ export function parseWithWasmScanner(
|
||||
return resumeWasmParse(host, data, length, promiseResult, scanCache);
|
||||
}
|
||||
|
||||
scanCache.stateBeforeScan = host.currentState;
|
||||
scanCache.inputOffset = 0;
|
||||
scanCache.scan = undefined;
|
||||
scanCache.opIndex = 0;
|
||||
if (!isActiveWasmParse(scanCache, data, length)) {
|
||||
scanCache.stateBeforeScan = host.currentState;
|
||||
scanCache.inputOffset = 0;
|
||||
scanCache.scan = undefined;
|
||||
scanCache.opIndex = 0;
|
||||
scanCache.data = undefined;
|
||||
scanCache.chunkStart = undefined;
|
||||
}
|
||||
return runWasmScanChunks(host, data, length, promiseResult, scanCache);
|
||||
}
|
||||
|
||||
@@ -123,13 +134,18 @@ function runWasmScanChunks(
|
||||
host.collect = WasmEscapeScanner.collect;
|
||||
}
|
||||
|
||||
const result = dispatchScanOps(host, data, scanCache.scan, scanCache.opIndex, scanCache, promiseResult, stateBeforeScan);
|
||||
const scan = scanCache.scan;
|
||||
if (!scan) {
|
||||
throw new Error('Wasm parser scan cache missing during chunked parse');
|
||||
}
|
||||
|
||||
const result = dispatchScanOps(host, data, scan, scanCache.opIndex, scanCache, promiseResult, stateBeforeScan);
|
||||
if (result instanceof Promise) {
|
||||
scanCache.inputOffset = pos;
|
||||
return result;
|
||||
}
|
||||
|
||||
const nextOffset = scanCache.scan.nextOffset;
|
||||
const nextOffset = scan.nextOffset;
|
||||
if (nextOffset <= pos) {
|
||||
throw new Error(`Wasm parser scan did not advance (stuck at input offset ${pos})`);
|
||||
}
|
||||
@@ -141,6 +157,8 @@ function runWasmScanChunks(
|
||||
|
||||
scanCache.inputOffset = length;
|
||||
scanCache.scan = undefined;
|
||||
scanCache.data = undefined;
|
||||
scanCache.chunkStart = undefined;
|
||||
}
|
||||
|
||||
function resumeWasmParse(
|
||||
|
||||
@@ -5,9 +5,10 @@
|
||||
#include "transition_table.h"
|
||||
#include <stddef.h>
|
||||
|
||||
/* Place scratch memory above static data (transition table ~8.5KB in linear memory 0) */
|
||||
/* Static transition table is linked at 65536; scratch must sit above it. */
|
||||
#define MEM_BASE 65536
|
||||
#define MEM_STATE MEM_BASE
|
||||
#define PARSER_TABLE_BYTE_SIZE (PARSER_TABLE_SIZE * sizeof(uint16_t))
|
||||
#define MEM_STATE (((MEM_BASE + PARSER_TABLE_BYTE_SIZE) + 255u) & ~255u)
|
||||
/* ParserWasmState is ~2.3KB; keep header and op buffers past it */
|
||||
#define MEM_HEADER (MEM_STATE + 2048)
|
||||
#define MEM_KINDS (MEM_HEADER + 64)
|
||||
|
||||
Binary file not shown.
File diff suppressed because one or more lines are too long
Reference in New Issue
Block a user