From 6e375cb588df052bc5afd32e07c4e08c7951f5f2 Mon Sep 17 00:00:00 2001 From: slim-forge Date: Tue, 21 Apr 2026 08:38:27 +0200 Subject: [PATCH] slim-forge: initial harness --- .gitignore | 4 + .slimforge.toml | 6 ++ bench/workload.js | 125 +++++++++++++++++++++++++++++ program.md | 135 +++++++++++++++++++++++++++++++ run.py | 198 ++++++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 468 insertions(+) create mode 100644 .slimforge.toml create mode 100644 bench/workload.js create mode 100644 program.md create mode 100644 run.py diff --git a/.gitignore b/.gitignore index e48470eff..46ca870a7 100644 --- a/.gitignore +++ b/.gitignore @@ -29,3 +29,7 @@ demo/dist/ # dont commit benchmark folders .benchmark/ timeline/ + +# slim-forge +results.tsv +run.log diff --git a/.slimforge.toml b/.slimforge.toml new file mode 100644 index 000000000..6de626274 --- /dev/null +++ b/.slimforge.toml @@ -0,0 +1,6 @@ +install_cmd = """npm install --no-audit --no-fund --loglevel=error""" +build_cmd = """npm run build && npm run esbuild""" +test_cmd = """npm run test-unit""" +bench_cmd = """node bench/workload.js""" +primary_bench_name = """weighted_mbps""" +target = "xterm" diff --git a/bench/workload.js b/bench/workload.js new file mode 100644 index 000000000..a4b731a4d --- /dev/null +++ b/bench/workload.js @@ -0,0 +1,125 @@ +'use strict'; + +// xterm.js EscapeSequenceParser throughput benchmark +// Runs against compiled output in out-esbuild/ +// Measures weighted MB/s across representative terminal workloads + +process.env.NODE_PATH = require('path').resolve(__dirname, '../out-esbuild'); +require('module').Module._initPaths(); + +var EscapeSequenceParser = require('common/parser/EscapeSequenceParser').EscapeSequenceParser; +var C0 = require('common/data/EscapeSequences').C0; + +function toUtf32(s) { + var result = new Uint32Array(s.length); + for (var i = 0; i < s.length; i++) { + result[i] = s.charCodeAt(i); + } + return result; +} + +function makeParser() { + var parser = new EscapeSequenceParser(); + parser.setPrintHandler(function() {}); + parser.setExecuteHandler(C0.LF, function() {}); + parser.setExecuteHandler(C0.CR, function() {}); + parser.setExecuteHandler('\t', function() {}); + parser.setExecuteHandler('\b', function() {}); + parser.registerCsiHandler({ final: 'm' }, function() { return true; }); + parser.registerCsiHandler({ final: 'H' }, function() { return true; }); + parser.registerCsiHandler({ final: 'J' }, function() { return true; }); + parser.registerCsiHandler({ final: 'K' }, function() { return true; }); + parser.registerCsiHandler({ final: 'A' }, function() { return true; }); + parser.registerCsiHandler({ final: 'B' }, function() { return true; }); + parser.registerCsiHandler({ final: 'C' }, function() { return true; }); + parser.registerCsiHandler({ final: 'D' }, function() { return true; }); + parser.registerEscHandler({ final: '7' }, function() { return true; }); + parser.registerEscHandler({ final: '8' }, function() { return true; }); + return parser; +} + +var SIZE = 2000000; + +var cases = [ + ['print_ascii', (function() { + var s = 'The quick brown fox jumps over the lazy dog. Pack my box with five dozen liquor jugs.\n'; + var content = ''; + while (content.length < SIZE) content += s; + return toUtf32(content); + })(), 4], + + ['sgr_heavy', (function() { + var s = '\x1b[0;32mfile.txt\x1b[0m \x1b[0;34mdir/\x1b[0m \x1b[1;31merror.log\x1b[0m\n'; + var content = ''; + while (content.length < SIZE) content += s; + return toUtf32(content); + })(), 4], + + ['cursor_move', (function() { + var s = '\x1b[1;1H\x1b[2JStatus: OK\x1b[2;1HLine 2 content here\x1b[3;1HAnother line\n'; + var content = ''; + while (content.length < SIZE) content += s; + return toUtf32(content); + })(), 2], + + ['params_heavy', (function() { + var s = '\x1b[38;5;196mred\x1b[0m \x1b[38;2;0;255;128mgreen\x1b[0m \x1b[48;5;21mblue_bg\x1b[0m\n'; + var content = ''; + while (content.length < SIZE) content += s; + return toUtf32(content); + })(), 3], + + ['mixed_realistic', (function() { + var lines = [ + '\x1b[33mabcd123\x1b[0m fix: resolve parser edge case\n', + '\x1b[33mef45678\x1b[0m feat: add new feature\n', + '\x1b[33m9abc012\x1b[0m \x1b[1;31m(HEAD -> main)\x1b[0m chore: update deps\n', + '\x1b[33m3def456\x1b[0m docs: update readme\n', + '\x1b[33m789abcd\x1b[0m refactor: clean up utils\n', + ]; + var content = ''; + while (content.length < SIZE) { + for (var j = 0; j < lines.length; j++) content += lines[j]; + } + return toUtf32(content); + })(), 4], +]; + +function benchOne(parser, data) { + for (var w = 0; w < 5; w++) parser.parse(data, data.length); + var runs = []; + for (var r = 0; r < 7; r++) { + var t0 = process.hrtime.bigint(); + parser.parse(data, data.length); + var dt = Number(process.hrtime.bigint() - t0) / 1e9; + var mbps = (data.length * 4) / (1024 * 1024) / dt; + runs.push(mbps); + } + runs.sort(function(a, b) { return a - b; }); + return runs[Math.floor(runs.length / 2)]; +} + +var parser = makeParser(); +var totalWeight = 0; +var weightedSum = 0; +var perCase = []; + +for (var c = 0; c < cases.length; c++) { + var name = cases[c][0]; + var data = cases[c][1]; + var weight = cases[c][2]; + var mbps = benchOne(parser, data); + perCase.push([name, mbps]); + weightedSum += mbps * weight; + totalWeight += weight; +} + +var weighted = weightedSum / totalWeight; + +process.stderr.write('per-case MB/s:\n'); +for (var p = 0; p < perCase.length; p++) { + process.stderr.write(' ' + perCase[p][0] + ': ' + perCase[p][1].toFixed(1) + '\n'); +} +process.stderr.write('weighted: ' + weighted.toFixed(1) + ' MB/s\n'); + +process.stdout.write(weighted.toFixed(1) + '\n'); diff --git a/program.md b/program.md new file mode 100644 index 000000000..4b8128be6 --- /dev/null +++ b/program.md @@ -0,0 +1,135 @@ +# slim-forge: xterm.js parser perf + +An experiment to autonomously speed up xterm.js's `EscapeSequenceParser`, the terminal escape sequence parser inside VS Code, GitHub Codespaces, and most web-based terminals. + +## Setup + +1. **Confirm the workspace**: fresh clone of `xtermjs/xterm.js`. Branch should be `autoresearch/xterm-`. Verify: `git status`, `git log --oneline -5`. +2. **Read the in-scope files**: + * `src/common/parser/EscapeSequenceParser.ts` — ~845 LOC. **THIS is what you optimize.** The main `parse()` method is the hot loop. Read the whole file. + * `src/common/parser/Params.ts` — parameter accumulation. Called from the PARAM action in the main loop. Secondary surface. + * `src/common/parser/Constants.ts` — `ParserState` and `ParserAction` enums. + * `src/common/parser/EscapeSequenceParser.test.ts` — ~2236 LOC of tests. Read before making changes. + * `bench/workload.js` — the benchmark. **Do NOT modify.** + * `run.py` — the harness. **Do NOT modify.** + * `program.md` — this file. +3. **Verify baseline exists**: `cat results.tsv`. One row, `baseline`, with a `mbps` value and `tests_pass: 1`. +4. **Confirm and go.** + +## Mission + +Primary metric: **`mbps`** — weighted throughput in MB/s across 5 representative terminal workloads (plain ASCII print, SGR-heavy colored output, cursor movement sequences, parameter-heavy truecolor, mixed realistic git-log-style output). Higher is better. + +Hard gates: +1. `npm run test-unit` passes (2261 mocha tests, ~6s). +2. The harness enforces `mbps > best_so_far * 1.05` for a row to be labeled `keep`. + +## Important: build pipeline + +xterm.js is TypeScript. Your edits go in `src/`. The build pipeline is: + +``` +npm run build # tsc: src/**/*.ts -> out/**/*.js +npm run esbuild # esbuild: out/ -> out-esbuild/ (what tests and bench consume) +``` + +`run.py` handles this automatically. But if you want to test manually: +``` +npm run build && npm run esbuild && npm run test-unit && node bench/workload.js +``` + +The agent edits `.ts` source. The bench runs against compiled `.js` in `out-esbuild/`. If your edit has a TS type error, `npm run build` fails and the experiment is `discard`. + +## What you CAN do + +* Edit `src/common/parser/EscapeSequenceParser.ts` — the main target. +* Edit `src/common/parser/Params.ts` — if parameter handling is the bottleneck. +* Edit `src/common/parser/OscParser.ts`, `DcsParser.ts`, `ApcParser.ts` — if those sub-parsers show up in the bench. +* Add internal helper functions or restructure the parse loop. + +## What you CANNOT do + +* Modify `src/common/parser/EscapeSequenceParser.test.ts` or any test file. +* Modify `bench/workload.js` or `run.py`. +* Modify `program.md`. +* Modify the `TransitionTable` initialization — the VT500 table is the spec-correct state machine. Changing transitions changes which sequences are recognized, which is a correctness violation even if tests pass. +* Cache parse results across calls. The parser is stateful (it maintains `currentState`), but each `parse()` call must process every byte of input. No memoization. +* Remove or weaken handler dispatch. The bench registers real handlers; skipping dispatch would be a benchmark exploit. + +## What the parser already does well + +Before you start, understand what's already optimized so you don't waste time re-discovering it: + +* **Transition table**: `Uint16Array` lookup — state × charCode → action + nextState. One array access per byte. This is already near-optimal for the state machine itself. +* **PRINT action loop unrolling**: 4× unrolled inner loop for consecutive printable characters. This is why `print_ascii` hits ~2500 MB/s. +* **PARAM inner loop**: do-while that stays in the PARAM action for consecutive digit/semicolon/colon bytes without re-entering the main switch. + +## Where to look for wins + +The bench breakdown shows where time is actually spent: + +* **`sgr_heavy`: ~84 MB/s** — 30× slower than print. Every SGR sequence goes through: state transitions for `ESC [ params m`, param parsing, CSI handler dispatch. The handler dispatch involves array lookups and function calls per sequence. +* **`params_heavy`: ~92 MB/s** — similar. Truecolor sequences have many params (e.g. `38;2;0;255;128`). +* **`cursor_move`: ~127 MB/s** — CSI sequences with fewer params but more state transitions per byte. +* **`mixed_realistic`: ~158 MB/s** — blend of print runs and SGR. + +Likely sources of real speedups: + +* **Reduce handler dispatch overhead.** The CSI dispatch does `this._csiHandlers[this._collect << 8 | code]` then iterates the handler array in reverse. For the common case of a single registered handler, the array iteration + `instanceof Promise` check is overhead. A fast-path for single-handler case could help. +* **Inline the PARAM parsing more aggressively.** The current do-while re-enters `this._params.addDigit()` / `addParam()` / `addSubParam()` per byte. If `Params` methods are not inlined by V8, each is a virtual dispatch. +* **Reduce property access in the hot loop.** `this._transitions`, `this.currentState`, `this._collect`, `this._params` — each `this.` access goes through the object's hidden class. Caching in locals at the top of `parse()` and writing back at the end could help. +* **Specialize the common case in the main switch.** PRINT and PARAM are the most common actions. If the switch has many cases, V8 may generate a jump table; if it has few hot cases, a nested if-else could be faster. +* **The `code < 0xa0 ? code : NON_ASCII_PRINTABLE` ternary** runs per byte. For ASCII-only input (the overwhelming common case), the branch is always taken. Consider a fast-path that skips the check when all input is < 0x80. + +Anti-patterns: +* Don't try to change the transition table layout (e.g. making it a 2D array). The `Uint16Array` with bit shifting is already the most cache-friendly layout. +* Don't try to replace the state machine with regex. The state machine handles streaming input correctly; regex can't. +* Don't optimize the async handler resume path (top of `parse()`). It's cold code. + +## Output format + +`run.py` prints: + +``` +--- +mbps: 693.3 +tests_passed: 1 +build_ok: 1 +install_seconds: 5.0 +build_seconds: 15.0 +test_seconds: 6.0 +best_so_far: 693.3 +threshold: 728.0 (best_so_far * 1.05) +auto-status: keep +``` + +Read the headline: `grep "^mbps:\|^tests_passed:\|^auto-status:" run.log` + +Per-case breakdown is on stderr from the bench: `grep "per-case" -A 10 run.log` (or check the bench stderr output). + +## Logging results + +`python run.py --record --description ""` appends a row. The harness auto-labels status. + +Schema: `commit mbps tests_pass build_ok status description` + +## The experiment loop + +LOOP FOREVER: + +1. `git log --oneline -10` and `tail -10 results.tsv`. What's the current best? What has been tried? +2. Pick one optimization idea. Be specific. +3. Edit `src/common/parser/EscapeSequenceParser.ts` (or Params.ts). One change per experiment. +4. `git add -A && git commit -m "try: "` (commit BEFORE the record run). +5. `python run.py --record --description "" > run.log 2>&1` +6. `grep "^mbps:\|^tests_passed:\|^auto-status:" run.log`. +7. Decision: + * `auto-status: keep` → the branch advances. + * `auto-status: discard` → `git reset --hard HEAD~1`. +8. Goto 1. + +**Build failures**: if `npm run build` fails, it's a TS type error from your edit. Read the tsc output in run.log, fix the type error or revert. + +**Verification for large wins**: if `auto-status: keep` on a >15% improvement, run `python run.py > /tmp/verify.log 2>&1` (no --record). If verify mbps is also clearly above threshold, it's real. If verify is close to baseline, it was noise — `git reset --hard HEAD~1`. + +**NEVER STOP**: once the loop has begun, do NOT pause. Run indefinitely until manually stopped. diff --git a/run.py b/run.py new file mode 100644 index 000000000..32dda4d0e --- /dev/null +++ b/run.py @@ -0,0 +1,198 @@ +""" +run.py — one experiment for slim-forge xterm.js mission. + +invoked by the agent each cycle: + python run.py + python run.py --record --description "tried X" + +pipeline: install -> build (tsc + esbuild) -> test-unit -> bench -> record + +bench script prints one float on stdout last line (weighted MB/s). +the harness enforces ops > best_so_far * 1.05 before labeling keep. + +NEVER edit this file. +""" + +from __future__ import annotations + +import argparse +import csv +import subprocess +import sys +import time +import tomllib +from pathlib import Path + +HERE = Path(__file__).parent.resolve() +THRESHOLD_MULT = 1.05 + + +def sh(cmd: str, timeout: int = 900) -> subprocess.CompletedProcess: + return subprocess.run( + cmd, shell=True, cwd=HERE, text=True, capture_output=True, + timeout=timeout, check=False, + ) + + +def load_spec() -> dict: + p = HERE / ".slimforge.toml" + if not p.exists(): + print("error: .slimforge.toml missing. did you run prepare.py?", file=sys.stderr) + sys.exit(2) + with open(p, "rb") as f: + return tomllib.load(f) + + +def install(spec: dict) -> tuple[bool, float]: + t0 = time.time() + r = sh(spec["install_cmd"], timeout=600) + dt = time.time() - t0 + if r.returncode != 0: + print("[install stderr tail]", file=sys.stderr) + print(r.stderr[-2000:], file=sys.stderr) + return False, dt + return True, dt + + +def build(spec: dict) -> tuple[bool, float]: + cmd = spec.get("build_cmd", "").strip() + if not cmd: + return True, 0.0 + t0 = time.time() + r = sh(cmd, timeout=600) + dt = time.time() - t0 + if r.returncode != 0: + print("[build stderr tail]", file=sys.stderr) + print(r.stderr[-3000:], file=sys.stderr) + print("[build stdout tail]", file=sys.stderr) + print(r.stdout[-3000:], file=sys.stderr) + return False, dt + return True, dt + + +def run_tests(spec: dict) -> tuple[bool, float]: + t0 = time.time() + r = sh(spec["test_cmd"], timeout=600) + dt = time.time() - t0 + if r.returncode != 0: + print("[test stderr tail]", file=sys.stderr) + print(r.stderr[-3000:], file=sys.stderr) + print("[test stdout tail]", file=sys.stderr) + print(r.stdout[-3000:], file=sys.stderr) + return r.returncode == 0, dt + + +def run_bench(spec: dict) -> tuple[float, str]: + cmd = spec.get("bench_cmd", "").strip() + if not cmd: + return -1.0, "" + r = sh(cmd, timeout=300) + tail = (r.stdout + ("\n[stderr]\n" + r.stderr if r.stderr else ""))[-3000:] + if r.returncode != 0: + print("[bench failed, tail]", file=sys.stderr) + print(tail, file=sys.stderr) + return -1.0, tail + try: + return float(r.stdout.strip().splitlines()[-1]), tail + except (ValueError, IndexError): + print("[bench output unparseable, tail]", file=sys.stderr) + print(tail, file=sys.stderr) + return -1.0, tail + + +def compute_best_so_far() -> float: + tsv = HERE / "results.tsv" + if not tsv.exists(): + return 0.0 + best = 0.0 + with open(tsv) as f: + for row in csv.DictReader(f, delimiter="\t"): + if row.get("status") == "keep" and row.get("tests_pass") == "1": + try: + v = float(row["mbps"]) + if v > best: + best = v + except (ValueError, KeyError): + pass + return best + + +def git_sha() -> str: + r = sh("git rev-parse --short HEAD", timeout=10) + return r.stdout.strip() or "0000000" + + +def append_results_row( + sha: str, mbps: float, tests_pass: bool, build_ok: bool, + status: str, description: str, +) -> None: + desc = description.replace("\t", " ").replace("\n", " ").strip() + tsv = HERE / "results.tsv" + if not tsv.exists(): + tsv.write_text("commit\tmbps\ttests_pass\tbuild_ok\tstatus\tdescription\n") + with open(tsv, "a") as f: + f.write(f"{sha}\t{mbps:.1f}\t{int(tests_pass)}\t{int(build_ok)}\t{status}\t{desc}\n") + + +def main() -> int: + ap = argparse.ArgumentParser() + ap.add_argument("--record", action="store_true") + ap.add_argument("--description", default="") + ap.add_argument("--status", default=None) + args = ap.parse_args() + + spec = load_spec() + + install_ok, install_secs = install(spec) + if not install_ok: + print("---") + print("install: FAILED") + if args.record: + append_results_row(git_sha(), -1.0, False, False, + args.status or "crash", + args.description or "install failed") + return 1 + + build_ok, build_secs = build(spec) + if not build_ok: + print("---") + print("build: FAILED") + if args.record: + append_results_row(git_sha(), -1.0, False, False, + args.status or "discard", + args.description or "build failed") + return 1 + + tests_pass, test_secs = run_tests(spec) + primary_mbps, bench_tail = run_bench(spec) + + print("---") + print(f"mbps: {primary_mbps:.1f}") + print(f"tests_passed: {int(tests_pass)}") + print(f"build_ok: {int(build_ok)}") + print(f"install_seconds: {install_secs:.1f}") + print(f"build_seconds: {build_secs:.1f}") + print(f"test_seconds: {test_secs:.1f}") + + if args.record: + status = args.status + if status is None: + best = compute_best_so_far() + threshold = best * THRESHOLD_MULT + if not tests_pass: + status = "discard" + elif primary_mbps < threshold: + status = "discard" + else: + status = "keep" + print(f"best_so_far: {best:.1f}") + print(f"threshold: {threshold:.1f} (best_so_far * {THRESHOLD_MULT})") + print(f"auto-status: {status}") + append_results_row(git_sha(), primary_mbps, tests_pass, build_ok, + status, args.description) + + return 0 if tests_pass else 1 + + +if __name__ == "__main__": + sys.exit(main())