From 6e375cb588df052bc5afd32e07c4e08c7951f5f2 Mon Sep 17 00:00:00 2001 From: slim-forge Date: Tue, 21 Apr 2026 08:38:27 +0200 Subject: [PATCH 01/11] slim-forge: initial harness --- .gitignore | 4 + .slimforge.toml | 6 ++ bench/workload.js | 125 +++++++++++++++++++++++++++++ program.md | 135 +++++++++++++++++++++++++++++++ run.py | 198 ++++++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 468 insertions(+) create mode 100644 .slimforge.toml create mode 100644 bench/workload.js create mode 100644 program.md create mode 100644 run.py diff --git a/.gitignore b/.gitignore index e48470eff..46ca870a7 100644 --- a/.gitignore +++ b/.gitignore @@ -29,3 +29,7 @@ demo/dist/ # dont commit benchmark folders .benchmark/ timeline/ + +# slim-forge +results.tsv +run.log diff --git a/.slimforge.toml b/.slimforge.toml new file mode 100644 index 000000000..6de626274 --- /dev/null +++ b/.slimforge.toml @@ -0,0 +1,6 @@ +install_cmd = """npm install --no-audit --no-fund --loglevel=error""" +build_cmd = """npm run build && npm run esbuild""" +test_cmd = """npm run test-unit""" +bench_cmd = """node bench/workload.js""" +primary_bench_name = """weighted_mbps""" +target = "xterm" diff --git a/bench/workload.js b/bench/workload.js new file mode 100644 index 000000000..a4b731a4d --- /dev/null +++ b/bench/workload.js @@ -0,0 +1,125 @@ +'use strict'; + +// xterm.js EscapeSequenceParser throughput benchmark +// Runs against compiled output in out-esbuild/ +// Measures weighted MB/s across representative terminal workloads + +process.env.NODE_PATH = require('path').resolve(__dirname, '../out-esbuild'); +require('module').Module._initPaths(); + +var EscapeSequenceParser = require('common/parser/EscapeSequenceParser').EscapeSequenceParser; +var C0 = require('common/data/EscapeSequences').C0; + +function toUtf32(s) { + var result = new Uint32Array(s.length); + for (var i = 0; i < s.length; i++) { + result[i] = s.charCodeAt(i); + } + return result; +} + +function makeParser() { + var parser = new EscapeSequenceParser(); + parser.setPrintHandler(function() {}); + parser.setExecuteHandler(C0.LF, function() {}); + parser.setExecuteHandler(C0.CR, function() {}); + parser.setExecuteHandler('\t', function() {}); + parser.setExecuteHandler('\b', function() {}); + parser.registerCsiHandler({ final: 'm' }, function() { return true; }); + parser.registerCsiHandler({ final: 'H' }, function() { return true; }); + parser.registerCsiHandler({ final: 'J' }, function() { return true; }); + parser.registerCsiHandler({ final: 'K' }, function() { return true; }); + parser.registerCsiHandler({ final: 'A' }, function() { return true; }); + parser.registerCsiHandler({ final: 'B' }, function() { return true; }); + parser.registerCsiHandler({ final: 'C' }, function() { return true; }); + parser.registerCsiHandler({ final: 'D' }, function() { return true; }); + parser.registerEscHandler({ final: '7' }, function() { return true; }); + parser.registerEscHandler({ final: '8' }, function() { return true; }); + return parser; +} + +var SIZE = 2000000; + +var cases = [ + ['print_ascii', (function() { + var s = 'The quick brown fox jumps over the lazy dog. Pack my box with five dozen liquor jugs.\n'; + var content = ''; + while (content.length < SIZE) content += s; + return toUtf32(content); + })(), 4], + + ['sgr_heavy', (function() { + var s = '\x1b[0;32mfile.txt\x1b[0m \x1b[0;34mdir/\x1b[0m \x1b[1;31merror.log\x1b[0m\n'; + var content = ''; + while (content.length < SIZE) content += s; + return toUtf32(content); + })(), 4], + + ['cursor_move', (function() { + var s = '\x1b[1;1H\x1b[2JStatus: OK\x1b[2;1HLine 2 content here\x1b[3;1HAnother line\n'; + var content = ''; + while (content.length < SIZE) content += s; + return toUtf32(content); + })(), 2], + + ['params_heavy', (function() { + var s = '\x1b[38;5;196mred\x1b[0m \x1b[38;2;0;255;128mgreen\x1b[0m \x1b[48;5;21mblue_bg\x1b[0m\n'; + var content = ''; + while (content.length < SIZE) content += s; + return toUtf32(content); + })(), 3], + + ['mixed_realistic', (function() { + var lines = [ + '\x1b[33mabcd123\x1b[0m fix: resolve parser edge case\n', + '\x1b[33mef45678\x1b[0m feat: add new feature\n', + '\x1b[33m9abc012\x1b[0m \x1b[1;31m(HEAD -> main)\x1b[0m chore: update deps\n', + '\x1b[33m3def456\x1b[0m docs: update readme\n', + '\x1b[33m789abcd\x1b[0m refactor: clean up utils\n', + ]; + var content = ''; + while (content.length < SIZE) { + for (var j = 0; j < lines.length; j++) content += lines[j]; + } + return toUtf32(content); + })(), 4], +]; + +function benchOne(parser, data) { + for (var w = 0; w < 5; w++) parser.parse(data, data.length); + var runs = []; + for (var r = 0; r < 7; r++) { + var t0 = process.hrtime.bigint(); + parser.parse(data, data.length); + var dt = Number(process.hrtime.bigint() - t0) / 1e9; + var mbps = (data.length * 4) / (1024 * 1024) / dt; + runs.push(mbps); + } + runs.sort(function(a, b) { return a - b; }); + return runs[Math.floor(runs.length / 2)]; +} + +var parser = makeParser(); +var totalWeight = 0; +var weightedSum = 0; +var perCase = []; + +for (var c = 0; c < cases.length; c++) { + var name = cases[c][0]; + var data = cases[c][1]; + var weight = cases[c][2]; + var mbps = benchOne(parser, data); + perCase.push([name, mbps]); + weightedSum += mbps * weight; + totalWeight += weight; +} + +var weighted = weightedSum / totalWeight; + +process.stderr.write('per-case MB/s:\n'); +for (var p = 0; p < perCase.length; p++) { + process.stderr.write(' ' + perCase[p][0] + ': ' + perCase[p][1].toFixed(1) + '\n'); +} +process.stderr.write('weighted: ' + weighted.toFixed(1) + ' MB/s\n'); + +process.stdout.write(weighted.toFixed(1) + '\n'); diff --git a/program.md b/program.md new file mode 100644 index 000000000..4b8128be6 --- /dev/null +++ b/program.md @@ -0,0 +1,135 @@ +# slim-forge: xterm.js parser perf + +An experiment to autonomously speed up xterm.js's `EscapeSequenceParser`, the terminal escape sequence parser inside VS Code, GitHub Codespaces, and most web-based terminals. + +## Setup + +1. **Confirm the workspace**: fresh clone of `xtermjs/xterm.js`. Branch should be `autoresearch/xterm-`. Verify: `git status`, `git log --oneline -5`. +2. **Read the in-scope files**: + * `src/common/parser/EscapeSequenceParser.ts` — ~845 LOC. **THIS is what you optimize.** The main `parse()` method is the hot loop. Read the whole file. + * `src/common/parser/Params.ts` — parameter accumulation. Called from the PARAM action in the main loop. Secondary surface. + * `src/common/parser/Constants.ts` — `ParserState` and `ParserAction` enums. + * `src/common/parser/EscapeSequenceParser.test.ts` — ~2236 LOC of tests. Read before making changes. + * `bench/workload.js` — the benchmark. **Do NOT modify.** + * `run.py` — the harness. **Do NOT modify.** + * `program.md` — this file. +3. **Verify baseline exists**: `cat results.tsv`. One row, `baseline`, with a `mbps` value and `tests_pass: 1`. +4. **Confirm and go.** + +## Mission + +Primary metric: **`mbps`** — weighted throughput in MB/s across 5 representative terminal workloads (plain ASCII print, SGR-heavy colored output, cursor movement sequences, parameter-heavy truecolor, mixed realistic git-log-style output). Higher is better. + +Hard gates: +1. `npm run test-unit` passes (2261 mocha tests, ~6s). +2. The harness enforces `mbps > best_so_far * 1.05` for a row to be labeled `keep`. + +## Important: build pipeline + +xterm.js is TypeScript. Your edits go in `src/`. The build pipeline is: + +``` +npm run build # tsc: src/**/*.ts -> out/**/*.js +npm run esbuild # esbuild: out/ -> out-esbuild/ (what tests and bench consume) +``` + +`run.py` handles this automatically. But if you want to test manually: +``` +npm run build && npm run esbuild && npm run test-unit && node bench/workload.js +``` + +The agent edits `.ts` source. The bench runs against compiled `.js` in `out-esbuild/`. If your edit has a TS type error, `npm run build` fails and the experiment is `discard`. + +## What you CAN do + +* Edit `src/common/parser/EscapeSequenceParser.ts` — the main target. +* Edit `src/common/parser/Params.ts` — if parameter handling is the bottleneck. +* Edit `src/common/parser/OscParser.ts`, `DcsParser.ts`, `ApcParser.ts` — if those sub-parsers show up in the bench. +* Add internal helper functions or restructure the parse loop. + +## What you CANNOT do + +* Modify `src/common/parser/EscapeSequenceParser.test.ts` or any test file. +* Modify `bench/workload.js` or `run.py`. +* Modify `program.md`. +* Modify the `TransitionTable` initialization — the VT500 table is the spec-correct state machine. Changing transitions changes which sequences are recognized, which is a correctness violation even if tests pass. +* Cache parse results across calls. The parser is stateful (it maintains `currentState`), but each `parse()` call must process every byte of input. No memoization. +* Remove or weaken handler dispatch. The bench registers real handlers; skipping dispatch would be a benchmark exploit. + +## What the parser already does well + +Before you start, understand what's already optimized so you don't waste time re-discovering it: + +* **Transition table**: `Uint16Array` lookup — state × charCode → action + nextState. One array access per byte. This is already near-optimal for the state machine itself. +* **PRINT action loop unrolling**: 4× unrolled inner loop for consecutive printable characters. This is why `print_ascii` hits ~2500 MB/s. +* **PARAM inner loop**: do-while that stays in the PARAM action for consecutive digit/semicolon/colon bytes without re-entering the main switch. + +## Where to look for wins + +The bench breakdown shows where time is actually spent: + +* **`sgr_heavy`: ~84 MB/s** — 30× slower than print. Every SGR sequence goes through: state transitions for `ESC [ params m`, param parsing, CSI handler dispatch. The handler dispatch involves array lookups and function calls per sequence. +* **`params_heavy`: ~92 MB/s** — similar. Truecolor sequences have many params (e.g. `38;2;0;255;128`). +* **`cursor_move`: ~127 MB/s** — CSI sequences with fewer params but more state transitions per byte. +* **`mixed_realistic`: ~158 MB/s** — blend of print runs and SGR. + +Likely sources of real speedups: + +* **Reduce handler dispatch overhead.** The CSI dispatch does `this._csiHandlers[this._collect << 8 | code]` then iterates the handler array in reverse. For the common case of a single registered handler, the array iteration + `instanceof Promise` check is overhead. A fast-path for single-handler case could help. +* **Inline the PARAM parsing more aggressively.** The current do-while re-enters `this._params.addDigit()` / `addParam()` / `addSubParam()` per byte. If `Params` methods are not inlined by V8, each is a virtual dispatch. +* **Reduce property access in the hot loop.** `this._transitions`, `this.currentState`, `this._collect`, `this._params` — each `this.` access goes through the object's hidden class. Caching in locals at the top of `parse()` and writing back at the end could help. +* **Specialize the common case in the main switch.** PRINT and PARAM are the most common actions. If the switch has many cases, V8 may generate a jump table; if it has few hot cases, a nested if-else could be faster. +* **The `code < 0xa0 ? code : NON_ASCII_PRINTABLE` ternary** runs per byte. For ASCII-only input (the overwhelming common case), the branch is always taken. Consider a fast-path that skips the check when all input is < 0x80. + +Anti-patterns: +* Don't try to change the transition table layout (e.g. making it a 2D array). The `Uint16Array` with bit shifting is already the most cache-friendly layout. +* Don't try to replace the state machine with regex. The state machine handles streaming input correctly; regex can't. +* Don't optimize the async handler resume path (top of `parse()`). It's cold code. + +## Output format + +`run.py` prints: + +``` +--- +mbps: 693.3 +tests_passed: 1 +build_ok: 1 +install_seconds: 5.0 +build_seconds: 15.0 +test_seconds: 6.0 +best_so_far: 693.3 +threshold: 728.0 (best_so_far * 1.05) +auto-status: keep +``` + +Read the headline: `grep "^mbps:\|^tests_passed:\|^auto-status:" run.log` + +Per-case breakdown is on stderr from the bench: `grep "per-case" -A 10 run.log` (or check the bench stderr output). + +## Logging results + +`python run.py --record --description ""` appends a row. The harness auto-labels status. + +Schema: `commit mbps tests_pass build_ok status description` + +## The experiment loop + +LOOP FOREVER: + +1. `git log --oneline -10` and `tail -10 results.tsv`. What's the current best? What has been tried? +2. Pick one optimization idea. Be specific. +3. Edit `src/common/parser/EscapeSequenceParser.ts` (or Params.ts). One change per experiment. +4. `git add -A && git commit -m "try: "` (commit BEFORE the record run). +5. `python run.py --record --description "" > run.log 2>&1` +6. `grep "^mbps:\|^tests_passed:\|^auto-status:" run.log`. +7. Decision: + * `auto-status: keep` → the branch advances. + * `auto-status: discard` → `git reset --hard HEAD~1`. +8. Goto 1. + +**Build failures**: if `npm run build` fails, it's a TS type error from your edit. Read the tsc output in run.log, fix the type error or revert. + +**Verification for large wins**: if `auto-status: keep` on a >15% improvement, run `python run.py > /tmp/verify.log 2>&1` (no --record). If verify mbps is also clearly above threshold, it's real. If verify is close to baseline, it was noise — `git reset --hard HEAD~1`. + +**NEVER STOP**: once the loop has begun, do NOT pause. Run indefinitely until manually stopped. diff --git a/run.py b/run.py new file mode 100644 index 000000000..32dda4d0e --- /dev/null +++ b/run.py @@ -0,0 +1,198 @@ +""" +run.py — one experiment for slim-forge xterm.js mission. + +invoked by the agent each cycle: + python run.py + python run.py --record --description "tried X" + +pipeline: install -> build (tsc + esbuild) -> test-unit -> bench -> record + +bench script prints one float on stdout last line (weighted MB/s). +the harness enforces ops > best_so_far * 1.05 before labeling keep. + +NEVER edit this file. +""" + +from __future__ import annotations + +import argparse +import csv +import subprocess +import sys +import time +import tomllib +from pathlib import Path + +HERE = Path(__file__).parent.resolve() +THRESHOLD_MULT = 1.05 + + +def sh(cmd: str, timeout: int = 900) -> subprocess.CompletedProcess: + return subprocess.run( + cmd, shell=True, cwd=HERE, text=True, capture_output=True, + timeout=timeout, check=False, + ) + + +def load_spec() -> dict: + p = HERE / ".slimforge.toml" + if not p.exists(): + print("error: .slimforge.toml missing. did you run prepare.py?", file=sys.stderr) + sys.exit(2) + with open(p, "rb") as f: + return tomllib.load(f) + + +def install(spec: dict) -> tuple[bool, float]: + t0 = time.time() + r = sh(spec["install_cmd"], timeout=600) + dt = time.time() - t0 + if r.returncode != 0: + print("[install stderr tail]", file=sys.stderr) + print(r.stderr[-2000:], file=sys.stderr) + return False, dt + return True, dt + + +def build(spec: dict) -> tuple[bool, float]: + cmd = spec.get("build_cmd", "").strip() + if not cmd: + return True, 0.0 + t0 = time.time() + r = sh(cmd, timeout=600) + dt = time.time() - t0 + if r.returncode != 0: + print("[build stderr tail]", file=sys.stderr) + print(r.stderr[-3000:], file=sys.stderr) + print("[build stdout tail]", file=sys.stderr) + print(r.stdout[-3000:], file=sys.stderr) + return False, dt + return True, dt + + +def run_tests(spec: dict) -> tuple[bool, float]: + t0 = time.time() + r = sh(spec["test_cmd"], timeout=600) + dt = time.time() - t0 + if r.returncode != 0: + print("[test stderr tail]", file=sys.stderr) + print(r.stderr[-3000:], file=sys.stderr) + print("[test stdout tail]", file=sys.stderr) + print(r.stdout[-3000:], file=sys.stderr) + return r.returncode == 0, dt + + +def run_bench(spec: dict) -> tuple[float, str]: + cmd = spec.get("bench_cmd", "").strip() + if not cmd: + return -1.0, "" + r = sh(cmd, timeout=300) + tail = (r.stdout + ("\n[stderr]\n" + r.stderr if r.stderr else ""))[-3000:] + if r.returncode != 0: + print("[bench failed, tail]", file=sys.stderr) + print(tail, file=sys.stderr) + return -1.0, tail + try: + return float(r.stdout.strip().splitlines()[-1]), tail + except (ValueError, IndexError): + print("[bench output unparseable, tail]", file=sys.stderr) + print(tail, file=sys.stderr) + return -1.0, tail + + +def compute_best_so_far() -> float: + tsv = HERE / "results.tsv" + if not tsv.exists(): + return 0.0 + best = 0.0 + with open(tsv) as f: + for row in csv.DictReader(f, delimiter="\t"): + if row.get("status") == "keep" and row.get("tests_pass") == "1": + try: + v = float(row["mbps"]) + if v > best: + best = v + except (ValueError, KeyError): + pass + return best + + +def git_sha() -> str: + r = sh("git rev-parse --short HEAD", timeout=10) + return r.stdout.strip() or "0000000" + + +def append_results_row( + sha: str, mbps: float, tests_pass: bool, build_ok: bool, + status: str, description: str, +) -> None: + desc = description.replace("\t", " ").replace("\n", " ").strip() + tsv = HERE / "results.tsv" + if not tsv.exists(): + tsv.write_text("commit\tmbps\ttests_pass\tbuild_ok\tstatus\tdescription\n") + with open(tsv, "a") as f: + f.write(f"{sha}\t{mbps:.1f}\t{int(tests_pass)}\t{int(build_ok)}\t{status}\t{desc}\n") + + +def main() -> int: + ap = argparse.ArgumentParser() + ap.add_argument("--record", action="store_true") + ap.add_argument("--description", default="") + ap.add_argument("--status", default=None) + args = ap.parse_args() + + spec = load_spec() + + install_ok, install_secs = install(spec) + if not install_ok: + print("---") + print("install: FAILED") + if args.record: + append_results_row(git_sha(), -1.0, False, False, + args.status or "crash", + args.description or "install failed") + return 1 + + build_ok, build_secs = build(spec) + if not build_ok: + print("---") + print("build: FAILED") + if args.record: + append_results_row(git_sha(), -1.0, False, False, + args.status or "discard", + args.description or "build failed") + return 1 + + tests_pass, test_secs = run_tests(spec) + primary_mbps, bench_tail = run_bench(spec) + + print("---") + print(f"mbps: {primary_mbps:.1f}") + print(f"tests_passed: {int(tests_pass)}") + print(f"build_ok: {int(build_ok)}") + print(f"install_seconds: {install_secs:.1f}") + print(f"build_seconds: {build_secs:.1f}") + print(f"test_seconds: {test_secs:.1f}") + + if args.record: + status = args.status + if status is None: + best = compute_best_so_far() + threshold = best * THRESHOLD_MULT + if not tests_pass: + status = "discard" + elif primary_mbps < threshold: + status = "discard" + else: + status = "keep" + print(f"best_so_far: {best:.1f}") + print(f"threshold: {threshold:.1f} (best_so_far * {THRESHOLD_MULT})") + print(f"auto-status: {status}") + append_results_row(git_sha(), primary_mbps, tests_pass, build_ok, + status, args.description) + + return 0 if tests_pass else 1 + + +if __name__ == "__main__": + sys.exit(main()) From 5d120de3d5a937b0df1e7d14bf1f1492c6799bf3 Mon Sep 17 00:00:00 2001 From: Ismail Pelaseyed Date: Tue, 21 Apr 2026 08:50:49 +0200 Subject: [PATCH 03/11] try: CSI fast-path - scan entire ESC[params;final without table lookups --- src/common/parser/EscapeSequenceParser.ts | 54 +++++++++++++++++++++++ 1 file changed, 54 insertions(+) diff --git a/src/common/parser/EscapeSequenceParser.ts b/src/common/parser/EscapeSequenceParser.ts index febb4878c..e78b6e836 100644 --- a/src/common/parser/EscapeSequenceParser.ts +++ b/src/common/parser/EscapeSequenceParser.ts @@ -649,6 +649,60 @@ export class EscapeSequenceParser extends Disposable implements IEscapeSequenceP for (let i = start; i < length; ++i) { code = data[i]; + // CSI fast path: in GROUND seeing ESC [, consume entire CSI sequence without table lookups + if (code === 0x1b && this.currentState === ParserState.GROUND && i + 2 < length && data[i + 1] === 0x5b) { + this._params.reset(); + this._params.addParam(0); + this._collect = 0; + let k = i + 2; + let ch = data[k]; + if (ch >= 0x3c && ch <= 0x3f) { + this._collect = ch; + k++; + } + let csiComplete = false; + for (; k < length; k++) { + ch = data[k]; + if (ch >= 0x30 && ch <= 0x39) { + this._params.addDigit(ch - 48); + } else if (ch === 0x3b) { + this._params.addParam(0); + } else if (ch === 0x3a) { + this._params.addSubParam(-1); + } else if (ch >= 0x40 && ch <= 0x7e) { + const handlers = this._csiHandlers[this._collect << 8 | ch]; + let j = handlers ? handlers.length - 1 : -1; + for (; j >= 0; j--) { + handlerResult = handlers[j](this._params); + if (handlerResult === true) { + break; + } else if (handlerResult instanceof Promise) { + transition = ParserAction.CSI_DISPATCH << TableAccess.TRANSITION_ACTION_SHIFT | ParserState.GROUND; + this._preserveStack(ParserStackType.CSI, handlers, j, transition, k); + return handlerResult; + } + } + if (j < 0) { + this._csiHandlerFb(this._collect << 8 | ch, this._params); + } + this.precedingJoinState = 0; + i = k; + csiComplete = true; + break; + } else { + break; + } + } + if (csiComplete) continue; + // incomplete or unexpected byte — fall back to normal path + this._params.reset(); + this._params.addParam(0); + this._collect = 0; + this.currentState = ParserState.ESCAPE; + // let normal loop process '[' at i+1 + continue; + } + // normal transition & action lookup transition = this._transitions.table[this.currentState << TableAccess.INDEX_STATE_SHIFT | (code < 0xa0 ? code : NON_ASCII_PRINTABLE)]; switch (transition >> TableAccess.TRANSITION_ACTION_SHIFT) { From 487b27f512bb7add5a28aea75cacca4bf937ccc5 Mon Sep 17 00:00:00 2001 From: Ismail Pelaseyed Date: Tue, 21 Apr 2026 09:26:07 +0200 Subject: [PATCH 04/11] try: scan print run after CSI fast-path dispatch --- src/common/parser/EscapeSequenceParser.ts | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/src/common/parser/EscapeSequenceParser.ts b/src/common/parser/EscapeSequenceParser.ts index e78b6e836..a0784fe8f 100644 --- a/src/common/parser/EscapeSequenceParser.ts +++ b/src/common/parser/EscapeSequenceParser.ts @@ -686,7 +686,16 @@ export class EscapeSequenceParser extends Disposable implements IEscapeSequenceP this._csiHandlerFb(this._collect << 8 | ch, this._params); } this.precedingJoinState = 0; - i = k; + // scan ahead for print run after CSI dispatch + let pEnd = k + 1; + if (pEnd < length && data[pEnd] >= 0x20 && data[pEnd] <= 0x7e) { + for (++pEnd; pEnd < length; ++pEnd) { + const pc = data[pEnd]; + if (pc < 0x20 || (pc > 0x7e && pc < NON_ASCII_PRINTABLE)) break; + } + this._printHandler(data, k + 1, pEnd); + } + i = pEnd - 1; csiComplete = true; break; } else { From ba7e775a1731cbf30d1866c601e1dc4a88f81a6e Mon Sep 17 00:00:00 2001 From: Ismail Pelaseyed Date: Tue, 21 Apr 2026 11:19:32 +0200 Subject: [PATCH 05/11] fix: move CSI fast-path into CLEAR action to eliminate non-CSI regression The CSI fast-path check previously ran on every byte of input, causing 3-18% regressions on ESCAPE, OSC, DCS, and other non-CSI sequences. Moving the check into the CLEAR action handler means it only evaluates after the state machine already identified an ESC byte. The fast-path body is extracted into a separate _csiFast method to keep the switch case body small for V8 optimization. Co-Authored-By: Claude Opus 4.6 (1M context) --- src/common/parser/EscapeSequenceParser.ts | 128 +++++++++++----------- 1 file changed, 65 insertions(+), 63 deletions(-) diff --git a/src/common/parser/EscapeSequenceParser.ts b/src/common/parser/EscapeSequenceParser.ts index a0784fe8f..68cb369e1 100644 --- a/src/common/parser/EscapeSequenceParser.ts +++ b/src/common/parser/EscapeSequenceParser.ts @@ -492,6 +492,59 @@ export class EscapeSequenceParser extends Disposable implements IEscapeSequenceP this._parseStack.chunkPos = chunkPos; } + /** + * CSI fast path: scan params and dispatch from position `start` (first byte after '['). + * Returns new `i` position (>= 0) on success, -1 on failure/incomplete. + * Stores a Promise in `_parseStack` and returns -2 for async handlers. + */ + private _csiFast(data: Uint32Array, start: number, length: number): number | Promise { + let k = start; + let ch = data[k]; + if (ch >= 0x3c && ch <= 0x3f) { + this._collect = ch; + k++; + } + for (; k < length; k++) { + ch = data[k]; + if (ch >= 0x30 && ch <= 0x39) { + this._params.addDigit(ch - 48); + } else if (ch === 0x3b) { + this._params.addParam(0); + } else if (ch === 0x3a) { + this._params.addSubParam(-1); + } else if (ch >= 0x40 && ch <= 0x7e) { + const handlers = this._csiHandlers[this._collect << 8 | ch]; + let j = handlers ? handlers.length - 1 : -1; + for (; j >= 0; j--) { + const handlerResult = handlers[j](this._params); + if (handlerResult === true) { + break; + } else if (handlerResult instanceof Promise) { + const transition = ParserAction.CSI_DISPATCH << TableAccess.TRANSITION_ACTION_SHIFT | ParserState.GROUND; + this._preserveStack(ParserStackType.CSI, handlers, j, transition, k); + return handlerResult; + } + } + if (j < 0) { + this._csiHandlerFb(this._collect << 8 | ch, this._params); + } + this.precedingJoinState = 0; + let pEnd = k + 1; + if (pEnd < length && data[pEnd] >= 0x20 && data[pEnd] <= 0x7e) { + for (++pEnd; pEnd < length; ++pEnd) { + const pc = data[pEnd]; + if (pc < 0x20 || (pc > 0x7e && pc < NON_ASCII_PRINTABLE)) break; + } + this._printHandler(data, k + 1, pEnd); + } + return pEnd - 1; + } else { + break; + } + } + return -1; + } + /** * Parse UTF32 codepoints in `data` up to `length`. * @@ -649,69 +702,6 @@ export class EscapeSequenceParser extends Disposable implements IEscapeSequenceP for (let i = start; i < length; ++i) { code = data[i]; - // CSI fast path: in GROUND seeing ESC [, consume entire CSI sequence without table lookups - if (code === 0x1b && this.currentState === ParserState.GROUND && i + 2 < length && data[i + 1] === 0x5b) { - this._params.reset(); - this._params.addParam(0); - this._collect = 0; - let k = i + 2; - let ch = data[k]; - if (ch >= 0x3c && ch <= 0x3f) { - this._collect = ch; - k++; - } - let csiComplete = false; - for (; k < length; k++) { - ch = data[k]; - if (ch >= 0x30 && ch <= 0x39) { - this._params.addDigit(ch - 48); - } else if (ch === 0x3b) { - this._params.addParam(0); - } else if (ch === 0x3a) { - this._params.addSubParam(-1); - } else if (ch >= 0x40 && ch <= 0x7e) { - const handlers = this._csiHandlers[this._collect << 8 | ch]; - let j = handlers ? handlers.length - 1 : -1; - for (; j >= 0; j--) { - handlerResult = handlers[j](this._params); - if (handlerResult === true) { - break; - } else if (handlerResult instanceof Promise) { - transition = ParserAction.CSI_DISPATCH << TableAccess.TRANSITION_ACTION_SHIFT | ParserState.GROUND; - this._preserveStack(ParserStackType.CSI, handlers, j, transition, k); - return handlerResult; - } - } - if (j < 0) { - this._csiHandlerFb(this._collect << 8 | ch, this._params); - } - this.precedingJoinState = 0; - // scan ahead for print run after CSI dispatch - let pEnd = k + 1; - if (pEnd < length && data[pEnd] >= 0x20 && data[pEnd] <= 0x7e) { - for (++pEnd; pEnd < length; ++pEnd) { - const pc = data[pEnd]; - if (pc < 0x20 || (pc > 0x7e && pc < NON_ASCII_PRINTABLE)) break; - } - this._printHandler(data, k + 1, pEnd); - } - i = pEnd - 1; - csiComplete = true; - break; - } else { - break; - } - } - if (csiComplete) continue; - // incomplete or unexpected byte — fall back to normal path - this._params.reset(); - this._params.addParam(0); - this._collect = 0; - this.currentState = ParserState.ESCAPE; - // let normal loop process '[' at i+1 - continue; - } - // normal transition & action lookup transition = this._transitions.table[this.currentState << TableAccess.INDEX_STATE_SHIFT | (code < 0xa0 ? code : NON_ASCII_PRINTABLE)]; switch (transition >> TableAccess.TRANSITION_ACTION_SHIFT) { @@ -824,6 +814,18 @@ export class EscapeSequenceParser extends Disposable implements IEscapeSequenceP this._params.reset(); this._params.addParam(0); // ZDM this._collect = 0; + if ((transition & TableAccess.TRANSITION_STATE_MASK) === ParserState.ESCAPE + && i + 2 < length && data[i + 1] === 0x5b) { + const csiFastResult = this._csiFast(data, i + 2, length); + if (csiFastResult instanceof Promise) { + return csiFastResult; + } + if (csiFastResult >= 0) { + i = csiFastResult as number; + this.currentState = ParserState.GROUND; + continue; + } + } break; case ParserAction.DCS_HOOK: this._dcsParser.hook(this._collect << 8 | code, this._params); From 6fd7ecc8128f69fd6c3bc1d302980b55b8939626 Mon Sep 17 00:00:00 2001 From: Ismail Pelaseyed Date: Thu, 23 Apr 2026 14:01:19 +0200 Subject: [PATCH 06/11] perf: move CSI fast-path to top of main loop, add EXE fast-path, improve PRINT unrolling Address reviewer feedback: - CSI fast-path is now a condition at the top of the codepoint loop instead of inside CLEAR, avoiding table lookups for the common case - EXE bytes < 0x18 get a fast-path before the CSI check so control bytes don't pay the penalty - PRINT loop unrolling replaced with while-based version that uses trueish eval and spares individual length checks in the unrolled case - Removed misplaced PRINT scan-ahead from CSI dispatch --- .gitignore | 4 - .slimforge.toml | 6 - bench/workload.js | 125 -------------- program.md | 135 --------------- run.py | 198 ---------------------- src/common/parser/EscapeSequenceParser.ts | 173 ++++++++++--------- 6 files changed, 85 insertions(+), 556 deletions(-) delete mode 100644 .slimforge.toml delete mode 100644 bench/workload.js delete mode 100644 program.md delete mode 100644 run.py diff --git a/.gitignore b/.gitignore index 46ca870a7..e48470eff 100644 --- a/.gitignore +++ b/.gitignore @@ -29,7 +29,3 @@ demo/dist/ # dont commit benchmark folders .benchmark/ timeline/ - -# slim-forge -results.tsv -run.log diff --git a/.slimforge.toml b/.slimforge.toml deleted file mode 100644 index 6de626274..000000000 --- a/.slimforge.toml +++ /dev/null @@ -1,6 +0,0 @@ -install_cmd = """npm install --no-audit --no-fund --loglevel=error""" -build_cmd = """npm run build && npm run esbuild""" -test_cmd = """npm run test-unit""" -bench_cmd = """node bench/workload.js""" -primary_bench_name = """weighted_mbps""" -target = "xterm" diff --git a/bench/workload.js b/bench/workload.js deleted file mode 100644 index a4b731a4d..000000000 --- a/bench/workload.js +++ /dev/null @@ -1,125 +0,0 @@ -'use strict'; - -// xterm.js EscapeSequenceParser throughput benchmark -// Runs against compiled output in out-esbuild/ -// Measures weighted MB/s across representative terminal workloads - -process.env.NODE_PATH = require('path').resolve(__dirname, '../out-esbuild'); -require('module').Module._initPaths(); - -var EscapeSequenceParser = require('common/parser/EscapeSequenceParser').EscapeSequenceParser; -var C0 = require('common/data/EscapeSequences').C0; - -function toUtf32(s) { - var result = new Uint32Array(s.length); - for (var i = 0; i < s.length; i++) { - result[i] = s.charCodeAt(i); - } - return result; -} - -function makeParser() { - var parser = new EscapeSequenceParser(); - parser.setPrintHandler(function() {}); - parser.setExecuteHandler(C0.LF, function() {}); - parser.setExecuteHandler(C0.CR, function() {}); - parser.setExecuteHandler('\t', function() {}); - parser.setExecuteHandler('\b', function() {}); - parser.registerCsiHandler({ final: 'm' }, function() { return true; }); - parser.registerCsiHandler({ final: 'H' }, function() { return true; }); - parser.registerCsiHandler({ final: 'J' }, function() { return true; }); - parser.registerCsiHandler({ final: 'K' }, function() { return true; }); - parser.registerCsiHandler({ final: 'A' }, function() { return true; }); - parser.registerCsiHandler({ final: 'B' }, function() { return true; }); - parser.registerCsiHandler({ final: 'C' }, function() { return true; }); - parser.registerCsiHandler({ final: 'D' }, function() { return true; }); - parser.registerEscHandler({ final: '7' }, function() { return true; }); - parser.registerEscHandler({ final: '8' }, function() { return true; }); - return parser; -} - -var SIZE = 2000000; - -var cases = [ - ['print_ascii', (function() { - var s = 'The quick brown fox jumps over the lazy dog. Pack my box with five dozen liquor jugs.\n'; - var content = ''; - while (content.length < SIZE) content += s; - return toUtf32(content); - })(), 4], - - ['sgr_heavy', (function() { - var s = '\x1b[0;32mfile.txt\x1b[0m \x1b[0;34mdir/\x1b[0m \x1b[1;31merror.log\x1b[0m\n'; - var content = ''; - while (content.length < SIZE) content += s; - return toUtf32(content); - })(), 4], - - ['cursor_move', (function() { - var s = '\x1b[1;1H\x1b[2JStatus: OK\x1b[2;1HLine 2 content here\x1b[3;1HAnother line\n'; - var content = ''; - while (content.length < SIZE) content += s; - return toUtf32(content); - })(), 2], - - ['params_heavy', (function() { - var s = '\x1b[38;5;196mred\x1b[0m \x1b[38;2;0;255;128mgreen\x1b[0m \x1b[48;5;21mblue_bg\x1b[0m\n'; - var content = ''; - while (content.length < SIZE) content += s; - return toUtf32(content); - })(), 3], - - ['mixed_realistic', (function() { - var lines = [ - '\x1b[33mabcd123\x1b[0m fix: resolve parser edge case\n', - '\x1b[33mef45678\x1b[0m feat: add new feature\n', - '\x1b[33m9abc012\x1b[0m \x1b[1;31m(HEAD -> main)\x1b[0m chore: update deps\n', - '\x1b[33m3def456\x1b[0m docs: update readme\n', - '\x1b[33m789abcd\x1b[0m refactor: clean up utils\n', - ]; - var content = ''; - while (content.length < SIZE) { - for (var j = 0; j < lines.length; j++) content += lines[j]; - } - return toUtf32(content); - })(), 4], -]; - -function benchOne(parser, data) { - for (var w = 0; w < 5; w++) parser.parse(data, data.length); - var runs = []; - for (var r = 0; r < 7; r++) { - var t0 = process.hrtime.bigint(); - parser.parse(data, data.length); - var dt = Number(process.hrtime.bigint() - t0) / 1e9; - var mbps = (data.length * 4) / (1024 * 1024) / dt; - runs.push(mbps); - } - runs.sort(function(a, b) { return a - b; }); - return runs[Math.floor(runs.length / 2)]; -} - -var parser = makeParser(); -var totalWeight = 0; -var weightedSum = 0; -var perCase = []; - -for (var c = 0; c < cases.length; c++) { - var name = cases[c][0]; - var data = cases[c][1]; - var weight = cases[c][2]; - var mbps = benchOne(parser, data); - perCase.push([name, mbps]); - weightedSum += mbps * weight; - totalWeight += weight; -} - -var weighted = weightedSum / totalWeight; - -process.stderr.write('per-case MB/s:\n'); -for (var p = 0; p < perCase.length; p++) { - process.stderr.write(' ' + perCase[p][0] + ': ' + perCase[p][1].toFixed(1) + '\n'); -} -process.stderr.write('weighted: ' + weighted.toFixed(1) + ' MB/s\n'); - -process.stdout.write(weighted.toFixed(1) + '\n'); diff --git a/program.md b/program.md deleted file mode 100644 index 4b8128be6..000000000 --- a/program.md +++ /dev/null @@ -1,135 +0,0 @@ -# slim-forge: xterm.js parser perf - -An experiment to autonomously speed up xterm.js's `EscapeSequenceParser`, the terminal escape sequence parser inside VS Code, GitHub Codespaces, and most web-based terminals. - -## Setup - -1. **Confirm the workspace**: fresh clone of `xtermjs/xterm.js`. Branch should be `autoresearch/xterm-`. Verify: `git status`, `git log --oneline -5`. -2. **Read the in-scope files**: - * `src/common/parser/EscapeSequenceParser.ts` — ~845 LOC. **THIS is what you optimize.** The main `parse()` method is the hot loop. Read the whole file. - * `src/common/parser/Params.ts` — parameter accumulation. Called from the PARAM action in the main loop. Secondary surface. - * `src/common/parser/Constants.ts` — `ParserState` and `ParserAction` enums. - * `src/common/parser/EscapeSequenceParser.test.ts` — ~2236 LOC of tests. Read before making changes. - * `bench/workload.js` — the benchmark. **Do NOT modify.** - * `run.py` — the harness. **Do NOT modify.** - * `program.md` — this file. -3. **Verify baseline exists**: `cat results.tsv`. One row, `baseline`, with a `mbps` value and `tests_pass: 1`. -4. **Confirm and go.** - -## Mission - -Primary metric: **`mbps`** — weighted throughput in MB/s across 5 representative terminal workloads (plain ASCII print, SGR-heavy colored output, cursor movement sequences, parameter-heavy truecolor, mixed realistic git-log-style output). Higher is better. - -Hard gates: -1. `npm run test-unit` passes (2261 mocha tests, ~6s). -2. The harness enforces `mbps > best_so_far * 1.05` for a row to be labeled `keep`. - -## Important: build pipeline - -xterm.js is TypeScript. Your edits go in `src/`. The build pipeline is: - -``` -npm run build # tsc: src/**/*.ts -> out/**/*.js -npm run esbuild # esbuild: out/ -> out-esbuild/ (what tests and bench consume) -``` - -`run.py` handles this automatically. But if you want to test manually: -``` -npm run build && npm run esbuild && npm run test-unit && node bench/workload.js -``` - -The agent edits `.ts` source. The bench runs against compiled `.js` in `out-esbuild/`. If your edit has a TS type error, `npm run build` fails and the experiment is `discard`. - -## What you CAN do - -* Edit `src/common/parser/EscapeSequenceParser.ts` — the main target. -* Edit `src/common/parser/Params.ts` — if parameter handling is the bottleneck. -* Edit `src/common/parser/OscParser.ts`, `DcsParser.ts`, `ApcParser.ts` — if those sub-parsers show up in the bench. -* Add internal helper functions or restructure the parse loop. - -## What you CANNOT do - -* Modify `src/common/parser/EscapeSequenceParser.test.ts` or any test file. -* Modify `bench/workload.js` or `run.py`. -* Modify `program.md`. -* Modify the `TransitionTable` initialization — the VT500 table is the spec-correct state machine. Changing transitions changes which sequences are recognized, which is a correctness violation even if tests pass. -* Cache parse results across calls. The parser is stateful (it maintains `currentState`), but each `parse()` call must process every byte of input. No memoization. -* Remove or weaken handler dispatch. The bench registers real handlers; skipping dispatch would be a benchmark exploit. - -## What the parser already does well - -Before you start, understand what's already optimized so you don't waste time re-discovering it: - -* **Transition table**: `Uint16Array` lookup — state × charCode → action + nextState. One array access per byte. This is already near-optimal for the state machine itself. -* **PRINT action loop unrolling**: 4× unrolled inner loop for consecutive printable characters. This is why `print_ascii` hits ~2500 MB/s. -* **PARAM inner loop**: do-while that stays in the PARAM action for consecutive digit/semicolon/colon bytes without re-entering the main switch. - -## Where to look for wins - -The bench breakdown shows where time is actually spent: - -* **`sgr_heavy`: ~84 MB/s** — 30× slower than print. Every SGR sequence goes through: state transitions for `ESC [ params m`, param parsing, CSI handler dispatch. The handler dispatch involves array lookups and function calls per sequence. -* **`params_heavy`: ~92 MB/s** — similar. Truecolor sequences have many params (e.g. `38;2;0;255;128`). -* **`cursor_move`: ~127 MB/s** — CSI sequences with fewer params but more state transitions per byte. -* **`mixed_realistic`: ~158 MB/s** — blend of print runs and SGR. - -Likely sources of real speedups: - -* **Reduce handler dispatch overhead.** The CSI dispatch does `this._csiHandlers[this._collect << 8 | code]` then iterates the handler array in reverse. For the common case of a single registered handler, the array iteration + `instanceof Promise` check is overhead. A fast-path for single-handler case could help. -* **Inline the PARAM parsing more aggressively.** The current do-while re-enters `this._params.addDigit()` / `addParam()` / `addSubParam()` per byte. If `Params` methods are not inlined by V8, each is a virtual dispatch. -* **Reduce property access in the hot loop.** `this._transitions`, `this.currentState`, `this._collect`, `this._params` — each `this.` access goes through the object's hidden class. Caching in locals at the top of `parse()` and writing back at the end could help. -* **Specialize the common case in the main switch.** PRINT and PARAM are the most common actions. If the switch has many cases, V8 may generate a jump table; if it has few hot cases, a nested if-else could be faster. -* **The `code < 0xa0 ? code : NON_ASCII_PRINTABLE` ternary** runs per byte. For ASCII-only input (the overwhelming common case), the branch is always taken. Consider a fast-path that skips the check when all input is < 0x80. - -Anti-patterns: -* Don't try to change the transition table layout (e.g. making it a 2D array). The `Uint16Array` with bit shifting is already the most cache-friendly layout. -* Don't try to replace the state machine with regex. The state machine handles streaming input correctly; regex can't. -* Don't optimize the async handler resume path (top of `parse()`). It's cold code. - -## Output format - -`run.py` prints: - -``` ---- -mbps: 693.3 -tests_passed: 1 -build_ok: 1 -install_seconds: 5.0 -build_seconds: 15.0 -test_seconds: 6.0 -best_so_far: 693.3 -threshold: 728.0 (best_so_far * 1.05) -auto-status: keep -``` - -Read the headline: `grep "^mbps:\|^tests_passed:\|^auto-status:" run.log` - -Per-case breakdown is on stderr from the bench: `grep "per-case" -A 10 run.log` (or check the bench stderr output). - -## Logging results - -`python run.py --record --description ""` appends a row. The harness auto-labels status. - -Schema: `commit mbps tests_pass build_ok status description` - -## The experiment loop - -LOOP FOREVER: - -1. `git log --oneline -10` and `tail -10 results.tsv`. What's the current best? What has been tried? -2. Pick one optimization idea. Be specific. -3. Edit `src/common/parser/EscapeSequenceParser.ts` (or Params.ts). One change per experiment. -4. `git add -A && git commit -m "try: "` (commit BEFORE the record run). -5. `python run.py --record --description "" > run.log 2>&1` -6. `grep "^mbps:\|^tests_passed:\|^auto-status:" run.log`. -7. Decision: - * `auto-status: keep` → the branch advances. - * `auto-status: discard` → `git reset --hard HEAD~1`. -8. Goto 1. - -**Build failures**: if `npm run build` fails, it's a TS type error from your edit. Read the tsc output in run.log, fix the type error or revert. - -**Verification for large wins**: if `auto-status: keep` on a >15% improvement, run `python run.py > /tmp/verify.log 2>&1` (no --record). If verify mbps is also clearly above threshold, it's real. If verify is close to baseline, it was noise — `git reset --hard HEAD~1`. - -**NEVER STOP**: once the loop has begun, do NOT pause. Run indefinitely until manually stopped. diff --git a/run.py b/run.py deleted file mode 100644 index 32dda4d0e..000000000 --- a/run.py +++ /dev/null @@ -1,198 +0,0 @@ -""" -run.py — one experiment for slim-forge xterm.js mission. - -invoked by the agent each cycle: - python run.py - python run.py --record --description "tried X" - -pipeline: install -> build (tsc + esbuild) -> test-unit -> bench -> record - -bench script prints one float on stdout last line (weighted MB/s). -the harness enforces ops > best_so_far * 1.05 before labeling keep. - -NEVER edit this file. -""" - -from __future__ import annotations - -import argparse -import csv -import subprocess -import sys -import time -import tomllib -from pathlib import Path - -HERE = Path(__file__).parent.resolve() -THRESHOLD_MULT = 1.05 - - -def sh(cmd: str, timeout: int = 900) -> subprocess.CompletedProcess: - return subprocess.run( - cmd, shell=True, cwd=HERE, text=True, capture_output=True, - timeout=timeout, check=False, - ) - - -def load_spec() -> dict: - p = HERE / ".slimforge.toml" - if not p.exists(): - print("error: .slimforge.toml missing. did you run prepare.py?", file=sys.stderr) - sys.exit(2) - with open(p, "rb") as f: - return tomllib.load(f) - - -def install(spec: dict) -> tuple[bool, float]: - t0 = time.time() - r = sh(spec["install_cmd"], timeout=600) - dt = time.time() - t0 - if r.returncode != 0: - print("[install stderr tail]", file=sys.stderr) - print(r.stderr[-2000:], file=sys.stderr) - return False, dt - return True, dt - - -def build(spec: dict) -> tuple[bool, float]: - cmd = spec.get("build_cmd", "").strip() - if not cmd: - return True, 0.0 - t0 = time.time() - r = sh(cmd, timeout=600) - dt = time.time() - t0 - if r.returncode != 0: - print("[build stderr tail]", file=sys.stderr) - print(r.stderr[-3000:], file=sys.stderr) - print("[build stdout tail]", file=sys.stderr) - print(r.stdout[-3000:], file=sys.stderr) - return False, dt - return True, dt - - -def run_tests(spec: dict) -> tuple[bool, float]: - t0 = time.time() - r = sh(spec["test_cmd"], timeout=600) - dt = time.time() - t0 - if r.returncode != 0: - print("[test stderr tail]", file=sys.stderr) - print(r.stderr[-3000:], file=sys.stderr) - print("[test stdout tail]", file=sys.stderr) - print(r.stdout[-3000:], file=sys.stderr) - return r.returncode == 0, dt - - -def run_bench(spec: dict) -> tuple[float, str]: - cmd = spec.get("bench_cmd", "").strip() - if not cmd: - return -1.0, "" - r = sh(cmd, timeout=300) - tail = (r.stdout + ("\n[stderr]\n" + r.stderr if r.stderr else ""))[-3000:] - if r.returncode != 0: - print("[bench failed, tail]", file=sys.stderr) - print(tail, file=sys.stderr) - return -1.0, tail - try: - return float(r.stdout.strip().splitlines()[-1]), tail - except (ValueError, IndexError): - print("[bench output unparseable, tail]", file=sys.stderr) - print(tail, file=sys.stderr) - return -1.0, tail - - -def compute_best_so_far() -> float: - tsv = HERE / "results.tsv" - if not tsv.exists(): - return 0.0 - best = 0.0 - with open(tsv) as f: - for row in csv.DictReader(f, delimiter="\t"): - if row.get("status") == "keep" and row.get("tests_pass") == "1": - try: - v = float(row["mbps"]) - if v > best: - best = v - except (ValueError, KeyError): - pass - return best - - -def git_sha() -> str: - r = sh("git rev-parse --short HEAD", timeout=10) - return r.stdout.strip() or "0000000" - - -def append_results_row( - sha: str, mbps: float, tests_pass: bool, build_ok: bool, - status: str, description: str, -) -> None: - desc = description.replace("\t", " ").replace("\n", " ").strip() - tsv = HERE / "results.tsv" - if not tsv.exists(): - tsv.write_text("commit\tmbps\ttests_pass\tbuild_ok\tstatus\tdescription\n") - with open(tsv, "a") as f: - f.write(f"{sha}\t{mbps:.1f}\t{int(tests_pass)}\t{int(build_ok)}\t{status}\t{desc}\n") - - -def main() -> int: - ap = argparse.ArgumentParser() - ap.add_argument("--record", action="store_true") - ap.add_argument("--description", default="") - ap.add_argument("--status", default=None) - args = ap.parse_args() - - spec = load_spec() - - install_ok, install_secs = install(spec) - if not install_ok: - print("---") - print("install: FAILED") - if args.record: - append_results_row(git_sha(), -1.0, False, False, - args.status or "crash", - args.description or "install failed") - return 1 - - build_ok, build_secs = build(spec) - if not build_ok: - print("---") - print("build: FAILED") - if args.record: - append_results_row(git_sha(), -1.0, False, False, - args.status or "discard", - args.description or "build failed") - return 1 - - tests_pass, test_secs = run_tests(spec) - primary_mbps, bench_tail = run_bench(spec) - - print("---") - print(f"mbps: {primary_mbps:.1f}") - print(f"tests_passed: {int(tests_pass)}") - print(f"build_ok: {int(build_ok)}") - print(f"install_seconds: {install_secs:.1f}") - print(f"build_seconds: {build_secs:.1f}") - print(f"test_seconds: {test_secs:.1f}") - - if args.record: - status = args.status - if status is None: - best = compute_best_so_far() - threshold = best * THRESHOLD_MULT - if not tests_pass: - status = "discard" - elif primary_mbps < threshold: - status = "discard" - else: - status = "keep" - print(f"best_so_far: {best:.1f}") - print(f"threshold: {threshold:.1f} (best_so_far * {THRESHOLD_MULT})") - print(f"auto-status: {status}") - append_results_row(git_sha(), primary_mbps, tests_pass, build_ok, - status, args.description) - - return 0 if tests_pass else 1 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/src/common/parser/EscapeSequenceParser.ts b/src/common/parser/EscapeSequenceParser.ts index 68cb369e1..014b46b75 100644 --- a/src/common/parser/EscapeSequenceParser.ts +++ b/src/common/parser/EscapeSequenceParser.ts @@ -492,59 +492,6 @@ export class EscapeSequenceParser extends Disposable implements IEscapeSequenceP this._parseStack.chunkPos = chunkPos; } - /** - * CSI fast path: scan params and dispatch from position `start` (first byte after '['). - * Returns new `i` position (>= 0) on success, -1 on failure/incomplete. - * Stores a Promise in `_parseStack` and returns -2 for async handlers. - */ - private _csiFast(data: Uint32Array, start: number, length: number): number | Promise { - let k = start; - let ch = data[k]; - if (ch >= 0x3c && ch <= 0x3f) { - this._collect = ch; - k++; - } - for (; k < length; k++) { - ch = data[k]; - if (ch >= 0x30 && ch <= 0x39) { - this._params.addDigit(ch - 48); - } else if (ch === 0x3b) { - this._params.addParam(0); - } else if (ch === 0x3a) { - this._params.addSubParam(-1); - } else if (ch >= 0x40 && ch <= 0x7e) { - const handlers = this._csiHandlers[this._collect << 8 | ch]; - let j = handlers ? handlers.length - 1 : -1; - for (; j >= 0; j--) { - const handlerResult = handlers[j](this._params); - if (handlerResult === true) { - break; - } else if (handlerResult instanceof Promise) { - const transition = ParserAction.CSI_DISPATCH << TableAccess.TRANSITION_ACTION_SHIFT | ParserState.GROUND; - this._preserveStack(ParserStackType.CSI, handlers, j, transition, k); - return handlerResult; - } - } - if (j < 0) { - this._csiHandlerFb(this._collect << 8 | ch, this._params); - } - this.precedingJoinState = 0; - let pEnd = k + 1; - if (pEnd < length && data[pEnd] >= 0x20 && data[pEnd] <= 0x7e) { - for (++pEnd; pEnd < length; ++pEnd) { - const pc = data[pEnd]; - if (pc < 0x20 || (pc > 0x7e && pc < NON_ASCII_PRINTABLE)) break; - } - this._printHandler(data, k + 1, pEnd); - } - return pEnd - 1; - } else { - break; - } - } - return -1; - } - /** * Parse UTF32 codepoints in `data` up to `length`. * @@ -702,34 +649,96 @@ export class EscapeSequenceParser extends Disposable implements IEscapeSequenceP for (let i = start; i < length; ++i) { code = data[i]; + // EXE fast-path: common control bytes (0x00-0x17) in non-payload states + if (code < 0x18 && this.currentState <= ParserState.CSI_IGNORE) { + if (this._executeHandlers[code]) this._executeHandlers[code](); + else this._executeHandlerFb(code); + this.precedingJoinState = 0; + continue; + } + + // CSI fast-path: collapse ESC [ into a single entry, parse params+final in a tight loop + if (code === 0x1b + && this.currentState !== ParserState.OSC_STRING + && this.currentState !== ParserState.DCS_PASSTHROUGH + && this.currentState !== ParserState.APC_STRING + && i + 1 < length && data[i + 1] === 0x5b + ) { + this._params.reset(); + this._params.addParam(0); // ZDM + this._collect = 0; + let k = i + 2; + if (k >= length) { i = k - 1; this.currentState = ParserState.CSI_ENTRY; continue; } + let ch = data[k]; + if (ch >= 0x3c && ch <= 0x3f) { + this._collect = ch; + if (++k >= length) { i = k - 1; this.currentState = ParserState.CSI_PARAM; continue; } + } + let csiDone = false; + for (; k < length; k++) { + ch = data[k]; + if (ch >= 0x30 && ch <= 0x39) { + this._params.addDigit(ch - 48); + } else if (ch === 0x3b) { + this._params.addParam(0); + } else if (ch === 0x3a) { + this._params.addSubParam(-1); + } else if (ch >= 0x40 && ch <= 0x7e) { + const handlers = this._csiHandlers[this._collect << 8 | ch]; + let j = handlers ? handlers.length - 1 : -1; + for (; j >= 0; j--) { + handlerResult = handlers[j](this._params); + if (handlerResult === true) { + break; + } else if (handlerResult instanceof Promise) { + transition = ParserAction.CSI_DISPATCH << TableAccess.TRANSITION_ACTION_SHIFT | ParserState.GROUND; + this._preserveStack(ParserStackType.CSI, handlers, j, transition, k); + return handlerResult; + } + } + if (j < 0) { + this._csiHandlerFb(this._collect << 8 | ch, this._params); + } + this.precedingJoinState = 0; + i = k; + this.currentState = ParserState.GROUND; + csiDone = true; + break; + } else if (ch >= 0x20 && ch <= 0x2f) { + this._collect <<= 8; + this._collect |= ch; + i = k; + this.currentState = ParserState.CSI_INTERMEDIATE; + csiDone = true; + break; + } else { + break; + } + } + if (!csiDone) { + i = k - 1; + this.currentState = ParserState.CSI_PARAM; + } + continue; + } + // normal transition & action lookup transition = this._transitions.table[this.currentState << TableAccess.INDEX_STATE_SHIFT | (code < 0xa0 ? code : NON_ASCII_PRINTABLE)]; switch (transition >> TableAccess.TRANSITION_ACTION_SHIFT) { case ParserAction.PRINT: - // read ahead with loop unrolling // Note: 0x20 (SP) is included, 0x7F (DEL) is excluded - for (let j = i + 1; ; ++j) { - if (j >= length || (code = data[j]) < 0x20 || (code > 0x7e && code < NON_ASCII_PRINTABLE)) { - this._printHandler(data, i, j); - i = j - 1; - break; - } - if (++j >= length || (code = data[j]) < 0x20 || (code > 0x7e && code < NON_ASCII_PRINTABLE)) { - this._printHandler(data, i, j); - i = j - 1; - break; - } - if (++j >= length || (code = data[j]) < 0x20 || (code > 0x7e && code < NON_ASCII_PRINTABLE)) { - this._printHandler(data, i, j); - i = j - 1; - break; - } - if (++j >= length || (code = data[j]) < 0x20 || (code > 0x7e && code < NON_ASCII_PRINTABLE)) { - this._printHandler(data, i, j); - i = j - 1; - break; - } - } + let c = i; + const l4 = length - 4; + while (c < l4 + && data[++c] >= 0x20 && (data[c] <= 0x7e || data[c] >= NON_ASCII_PRINTABLE) + && data[++c] >= 0x20 && (data[c] <= 0x7e || data[c] >= NON_ASCII_PRINTABLE) + && data[++c] >= 0x20 && (data[c] <= 0x7e || data[c] >= NON_ASCII_PRINTABLE) + && data[++c] >= 0x20 && (data[c] <= 0x7e || data[c] >= NON_ASCII_PRINTABLE) + ) {} + if (c >= l4) + while (c < length && data[c] >= 0x20 && (data[c] <= 0x7e || data[c] >= NON_ASCII_PRINTABLE)) c++; + this._printHandler(data, i, c); + i = c - 1; break; case ParserAction.EXECUTE: if (this._executeHandlers[code]) this._executeHandlers[code](); @@ -814,18 +823,6 @@ export class EscapeSequenceParser extends Disposable implements IEscapeSequenceP this._params.reset(); this._params.addParam(0); // ZDM this._collect = 0; - if ((transition & TableAccess.TRANSITION_STATE_MASK) === ParserState.ESCAPE - && i + 2 < length && data[i + 1] === 0x5b) { - const csiFastResult = this._csiFast(data, i + 2, length); - if (csiFastResult instanceof Promise) { - return csiFastResult; - } - if (csiFastResult >= 0) { - i = csiFastResult as number; - this.currentState = ParserState.GROUND; - continue; - } - } break; case ParserAction.DCS_HOOK: this._dcsParser.hook(this._collect << 8 | code, this._params); From 18cd9858ed90a0351e08ba2a40e53c1cd86934ee Mon Sep 17 00:00:00 2001 From: Ismail Pelaseyed Date: Thu, 23 Apr 2026 14:09:47 +0200 Subject: [PATCH 07/11] perf: collapse CSI state guard into single range check Replace three !== checks (OSC_STRING, DCS_PASSTHROUGH, APC_STRING) with one < OSC_STRING comparison. States below OSC_STRING are all safe for the CSI fast-path; states above need the normal table path for proper subparser abort propagation. --- src/common/parser/EscapeSequenceParser.ts | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/common/parser/EscapeSequenceParser.ts b/src/common/parser/EscapeSequenceParser.ts index 014b46b75..2447bce85 100644 --- a/src/common/parser/EscapeSequenceParser.ts +++ b/src/common/parser/EscapeSequenceParser.ts @@ -659,9 +659,7 @@ export class EscapeSequenceParser extends Disposable implements IEscapeSequenceP // CSI fast-path: collapse ESC [ into a single entry, parse params+final in a tight loop if (code === 0x1b - && this.currentState !== ParserState.OSC_STRING - && this.currentState !== ParserState.DCS_PASSTHROUGH - && this.currentState !== ParserState.APC_STRING + && this.currentState < ParserState.OSC_STRING && i + 1 < length && data[i + 1] === 0x5b ) { this._params.reset(); From 9deec7aabf3a45fdcd4de8e485c279c3c7d89fcc Mon Sep 17 00:00:00 2001 From: jerch Date: Thu, 23 Apr 2026 14:36:51 +0200 Subject: [PATCH 08/11] make linter happy --- src/common/parser/EscapeSequenceParser.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/common/parser/EscapeSequenceParser.ts b/src/common/parser/EscapeSequenceParser.ts index 2447bce85..ea15d782d 100644 --- a/src/common/parser/EscapeSequenceParser.ts +++ b/src/common/parser/EscapeSequenceParser.ts @@ -734,7 +734,9 @@ export class EscapeSequenceParser extends Disposable implements IEscapeSequenceP && data[++c] >= 0x20 && (data[c] <= 0x7e || data[c] >= NON_ASCII_PRINTABLE) ) {} if (c >= l4) - while (c < length && data[c] >= 0x20 && (data[c] <= 0x7e || data[c] >= NON_ASCII_PRINTABLE)) c++; + while (c < length && data[c] >= 0x20 && (data[c] <= 0x7e || data[c] >= NON_ASCII_PRINTABLE)) { + c++; + } this._printHandler(data, i, c); i = c - 1; break; From f0aeb70c92f9c4dee67f5627b00bc783702e7842 Mon Sep 17 00:00:00 2001 From: jerch Date: Thu, 23 Apr 2026 14:39:26 +0200 Subject: [PATCH 09/11] make linter happy --- src/common/parser/EscapeSequenceParser.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/common/parser/EscapeSequenceParser.ts b/src/common/parser/EscapeSequenceParser.ts index ea15d782d..04f5e8862 100644 --- a/src/common/parser/EscapeSequenceParser.ts +++ b/src/common/parser/EscapeSequenceParser.ts @@ -733,10 +733,11 @@ export class EscapeSequenceParser extends Disposable implements IEscapeSequenceP && data[++c] >= 0x20 && (data[c] <= 0x7e || data[c] >= NON_ASCII_PRINTABLE) && data[++c] >= 0x20 && (data[c] <= 0x7e || data[c] >= NON_ASCII_PRINTABLE) ) {} - if (c >= l4) + if (c >= l4) { while (c < length && data[c] >= 0x20 && (data[c] <= 0x7e || data[c] >= NON_ASCII_PRINTABLE)) { c++; } + } this._printHandler(data, i, c); i = c - 1; break; From eb65910c1ba26733cf4cef4ef9157927b08826b8 Mon Sep 17 00:00:00 2001 From: Ismail Pelaseyed Date: Thu, 23 Apr 2026 15:35:06 +0200 Subject: [PATCH 10/11] perf: simplify CSI fast-path - drop intermediate handling and extra length checks Remove intermediate byte handling from the fast-path, bail to table lookup instead. Remove redundant per-step length checks by requiring i + 2 < length upfront and letting the for-loop guard the rest. --- src/common/parser/EscapeSequenceParser.ts | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/src/common/parser/EscapeSequenceParser.ts b/src/common/parser/EscapeSequenceParser.ts index 04f5e8862..99770b092 100644 --- a/src/common/parser/EscapeSequenceParser.ts +++ b/src/common/parser/EscapeSequenceParser.ts @@ -660,17 +660,16 @@ export class EscapeSequenceParser extends Disposable implements IEscapeSequenceP // CSI fast-path: collapse ESC [ into a single entry, parse params+final in a tight loop if (code === 0x1b && this.currentState < ParserState.OSC_STRING - && i + 1 < length && data[i + 1] === 0x5b + && i + 2 < length && data[i + 1] === 0x5b ) { this._params.reset(); this._params.addParam(0); // ZDM this._collect = 0; let k = i + 2; - if (k >= length) { i = k - 1; this.currentState = ParserState.CSI_ENTRY; continue; } let ch = data[k]; if (ch >= 0x3c && ch <= 0x3f) { this._collect = ch; - if (++k >= length) { i = k - 1; this.currentState = ParserState.CSI_PARAM; continue; } + k++; } let csiDone = false; for (; k < length; k++) { @@ -702,13 +701,6 @@ export class EscapeSequenceParser extends Disposable implements IEscapeSequenceP this.currentState = ParserState.GROUND; csiDone = true; break; - } else if (ch >= 0x20 && ch <= 0x2f) { - this._collect <<= 8; - this._collect |= ch; - i = k; - this.currentState = ParserState.CSI_INTERMEDIATE; - csiDone = true; - break; } else { break; } From 6c2465d7453c700a01a299bac780b0181b990489 Mon Sep 17 00:00:00 2001 From: Ismail Pelaseyed Date: Thu, 23 Apr 2026 15:59:27 +0200 Subject: [PATCH 11/11] docs: note EXE and CSI fast paths in parse method docs --- src/common/parser/EscapeSequenceParser.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/common/parser/EscapeSequenceParser.ts b/src/common/parser/EscapeSequenceParser.ts index 99770b092..8cb6d39f3 100644 --- a/src/common/parser/EscapeSequenceParser.ts +++ b/src/common/parser/EscapeSequenceParser.ts @@ -506,6 +506,10 @@ export class EscapeSequenceParser extends Disposable implements IEscapeSequenceP * - OSC_STRING:OSC_PUT * - DCS_PASSTHROUGH:DCS_PUT * + * Additionally the following fast paths exist before the table lookup: + * - EXE bytes < 0x18 in non-payload states (avoids table lookup entirely) + * - 7-bit CSI sequences without intermediates (ESC [ params final) + * * Note on asynchronous handler support: * Any handler returning a promise will be treated as asynchronous. * To keep the in-band blocking working for async handlers, `parse` pauses execution,