slim-forge: initial harness

.gitignore

@@ -29,3 +29,7 @@ demo/dist/
 # dont commit benchmark folders
 .benchmark/
 timeline/
+
+# slim-forge
+results.tsv
+run.log

.slimforge.toml

@@ -0,0 +1,6 @@
+install_cmd = """npm install --no-audit --no-fund --loglevel=error"""
+build_cmd = """npm run build && npm run esbuild"""
+test_cmd = """npm run test-unit"""
+bench_cmd = """node bench/workload.js"""
+primary_bench_name = """weighted_mbps"""
+target = "xterm"

bench/workload.js

@@ -0,0 +1,125 @@
+'use strict';
+
+// xterm.js EscapeSequenceParser throughput benchmark
+// Runs against compiled output in out-esbuild/
+// Measures weighted MB/s across representative terminal workloads
+
+process.env.NODE_PATH = require('path').resolve(__dirname, '../out-esbuild');
+require('module').Module._initPaths();
+
+var EscapeSequenceParser = require('common/parser/EscapeSequenceParser').EscapeSequenceParser;
+var C0 = require('common/data/EscapeSequences').C0;
+
+function toUtf32(s) {
+    var result = new Uint32Array(s.length);
+    for (var i = 0; i < s.length; i++) {
+        result[i] = s.charCodeAt(i);
+    }
+    return result;
+}
+
+function makeParser() {
+    var parser = new EscapeSequenceParser();
+    parser.setPrintHandler(function() {});
+    parser.setExecuteHandler(C0.LF, function() {});
+    parser.setExecuteHandler(C0.CR, function() {});
+    parser.setExecuteHandler('\t', function() {});
+    parser.setExecuteHandler('\b', function() {});
+    parser.registerCsiHandler({ final: 'm' }, function() { return true; });
+    parser.registerCsiHandler({ final: 'H' }, function() { return true; });
+    parser.registerCsiHandler({ final: 'J' }, function() { return true; });
+    parser.registerCsiHandler({ final: 'K' }, function() { return true; });
+    parser.registerCsiHandler({ final: 'A' }, function() { return true; });
+    parser.registerCsiHandler({ final: 'B' }, function() { return true; });
+    parser.registerCsiHandler({ final: 'C' }, function() { return true; });
+    parser.registerCsiHandler({ final: 'D' }, function() { return true; });
+    parser.registerEscHandler({ final: '7' }, function() { return true; });
+    parser.registerEscHandler({ final: '8' }, function() { return true; });
+    return parser;
+}
+
+var SIZE = 2000000;
+
+var cases = [
+    ['print_ascii', (function() {
+        var s = 'The quick brown fox jumps over the lazy dog. Pack my box with five dozen liquor jugs.\n';
+        var content = '';
+        while (content.length < SIZE) content += s;
+        return toUtf32(content);
+    })(), 4],
+
+    ['sgr_heavy', (function() {
+        var s = '\x1b[0;32mfile.txt\x1b[0m  \x1b[0;34mdir/\x1b[0m  \x1b[1;31merror.log\x1b[0m\n';
+        var content = '';
+        while (content.length < SIZE) content += s;
+        return toUtf32(content);
+    })(), 4],
+
+    ['cursor_move', (function() {
+        var s = '\x1b[1;1H\x1b[2JStatus: OK\x1b[2;1HLine 2 content here\x1b[3;1HAnother line\n';
+        var content = '';
+        while (content.length < SIZE) content += s;
+        return toUtf32(content);
+    })(), 2],
+
+    ['params_heavy', (function() {
+        var s = '\x1b[38;5;196mred\x1b[0m \x1b[38;2;0;255;128mgreen\x1b[0m \x1b[48;5;21mblue_bg\x1b[0m\n';
+        var content = '';
+        while (content.length < SIZE) content += s;
+        return toUtf32(content);
+    })(), 3],
+
+    ['mixed_realistic', (function() {
+        var lines = [
+            '\x1b[33mabcd123\x1b[0m fix: resolve parser edge case\n',
+            '\x1b[33mef45678\x1b[0m feat: add new feature\n',
+            '\x1b[33m9abc012\x1b[0m \x1b[1;31m(HEAD -> main)\x1b[0m chore: update deps\n',
+            '\x1b[33m3def456\x1b[0m docs: update readme\n',
+            '\x1b[33m789abcd\x1b[0m refactor: clean up utils\n',
+        ];
+        var content = '';
+        while (content.length < SIZE) {
+            for (var j = 0; j < lines.length; j++) content += lines[j];
+        }
+        return toUtf32(content);
+    })(), 4],
+];
+
+function benchOne(parser, data) {
+    for (var w = 0; w < 5; w++) parser.parse(data, data.length);
+    var runs = [];
+    for (var r = 0; r < 7; r++) {
+        var t0 = process.hrtime.bigint();
+        parser.parse(data, data.length);
+        var dt = Number(process.hrtime.bigint() - t0) / 1e9;
+        var mbps = (data.length * 4) / (1024 * 1024) / dt;
+        runs.push(mbps);
+    }
+    runs.sort(function(a, b) { return a - b; });
+    return runs[Math.floor(runs.length / 2)];
+}
+
+var parser = makeParser();
+var totalWeight = 0;
+var weightedSum = 0;
+var perCase = [];
+
+for (var c = 0; c < cases.length; c++) {
+    var name = cases[c][0];
+    var data = cases[c][1];
+    var weight = cases[c][2];
+    var mbps = benchOne(parser, data);
+    perCase.push([name, mbps]);
+    weightedSum += mbps * weight;
+    totalWeight += weight;
+}
+
+var weighted = weightedSum / totalWeight;
+
+process.stderr.write('per-case MB/s:\n');
+for (var p = 0; p < perCase.length; p++) {
+    process.stderr.write('  ' + perCase[p][0] + ': ' + perCase[p][1].toFixed(1) + '\n');
+}
+process.stderr.write('weighted: ' + weighted.toFixed(1) + ' MB/s\n');
+
+process.stdout.write(weighted.toFixed(1) + '\n');

program.md

@@ -0,0 +1,135 @@
+# slim-forge: xterm.js parser perf
+
+An experiment to autonomously speed up xterm.js's `EscapeSequenceParser`, the terminal escape sequence parser inside VS Code, GitHub Codespaces, and most web-based terminals.
+
+## Setup
+
+1. **Confirm the workspace**: fresh clone of `xtermjs/xterm.js`. Branch should be `autoresearch/xterm-<date>`. Verify: `git status`, `git log --oneline -5`.
+2. **Read the in-scope files**:
+   * `src/common/parser/EscapeSequenceParser.ts` — ~845 LOC. **THIS is what you optimize.** The main `parse()` method is the hot loop. Read the whole file.
+   * `src/common/parser/Params.ts` — parameter accumulation. Called from the PARAM action in the main loop. Secondary surface.
+   * `src/common/parser/Constants.ts` — `ParserState` and `ParserAction` enums.
+   * `src/common/parser/EscapeSequenceParser.test.ts` — ~2236 LOC of tests. Read before making changes.
+   * `bench/workload.js` — the benchmark. **Do NOT modify.**
+   * `run.py` — the harness. **Do NOT modify.**
+   * `program.md` — this file.
+3. **Verify baseline exists**: `cat results.tsv`. One row, `baseline`, with a `mbps` value and `tests_pass: 1`.
+4. **Confirm and go.**
+
+## Mission
+
+Primary metric: **`mbps`** — weighted throughput in MB/s across 5 representative terminal workloads (plain ASCII print, SGR-heavy colored output, cursor movement sequences, parameter-heavy truecolor, mixed realistic git-log-style output). Higher is better.
+
+Hard gates:
+1. `npm run test-unit` passes (2261 mocha tests, ~6s).
+2. The harness enforces `mbps > best_so_far * 1.05` for a row to be labeled `keep`.
+
+## Important: build pipeline
+
+xterm.js is TypeScript. Your edits go in `src/`. The build pipeline is:
+
+```
+npm run build     # tsc: src/**/*.ts -> out/**/*.js
+npm run esbuild   # esbuild: out/ -> out-esbuild/ (what tests and bench consume)
+```
+
+`run.py` handles this automatically. But if you want to test manually:
+```
+npm run build && npm run esbuild && npm run test-unit && node bench/workload.js
+```
+
+The agent edits `.ts` source. The bench runs against compiled `.js` in `out-esbuild/`. If your edit has a TS type error, `npm run build` fails and the experiment is `discard`.
+
+## What you CAN do
+
+* Edit `src/common/parser/EscapeSequenceParser.ts` — the main target.
+* Edit `src/common/parser/Params.ts` — if parameter handling is the bottleneck.
+* Edit `src/common/parser/OscParser.ts`, `DcsParser.ts`, `ApcParser.ts` — if those sub-parsers show up in the bench.
+* Add internal helper functions or restructure the parse loop.
+
+## What you CANNOT do
+
+* Modify `src/common/parser/EscapeSequenceParser.test.ts` or any test file.
+* Modify `bench/workload.js` or `run.py`.
+* Modify `program.md`.
+* Modify the `TransitionTable` initialization — the VT500 table is the spec-correct state machine. Changing transitions changes which sequences are recognized, which is a correctness violation even if tests pass.
+* Cache parse results across calls. The parser is stateful (it maintains `currentState`), but each `parse()` call must process every byte of input. No memoization.
+* Remove or weaken handler dispatch. The bench registers real handlers; skipping dispatch would be a benchmark exploit.
+
+## What the parser already does well
+
+Before you start, understand what's already optimized so you don't waste time re-discovering it:
+
+* **Transition table**: `Uint16Array` lookup — state × charCode → action + nextState. One array access per byte. This is already near-optimal for the state machine itself.
+* **PRINT action loop unrolling**: 4× unrolled inner loop for consecutive printable characters. This is why `print_ascii` hits ~2500 MB/s.
+* **PARAM inner loop**: do-while that stays in the PARAM action for consecutive digit/semicolon/colon bytes without re-entering the main switch.
+
+## Where to look for wins
+
+The bench breakdown shows where time is actually spent:
+
+* **`sgr_heavy`: ~84 MB/s** — 30× slower than print. Every SGR sequence goes through: state transitions for `ESC [ params m`, param parsing, CSI handler dispatch. The handler dispatch involves array lookups and function calls per sequence.
+* **`params_heavy`: ~92 MB/s** — similar. Truecolor sequences have many params (e.g. `38;2;0;255;128`).
+* **`cursor_move`: ~127 MB/s** — CSI sequences with fewer params but more state transitions per byte.
+* **`mixed_realistic`: ~158 MB/s** — blend of print runs and SGR.
+
+Likely sources of real speedups:
+
+* **Reduce handler dispatch overhead.** The CSI dispatch does `this._csiHandlers[this._collect << 8 | code]` then iterates the handler array in reverse. For the common case of a single registered handler, the array iteration + `instanceof Promise` check is overhead. A fast-path for single-handler case could help.
+* **Inline the PARAM parsing more aggressively.** The current do-while re-enters `this._params.addDigit()` / `addParam()` / `addSubParam()` per byte. If `Params` methods are not inlined by V8, each is a virtual dispatch.
+* **Reduce property access in the hot loop.** `this._transitions`, `this.currentState`, `this._collect`, `this._params` — each `this.` access goes through the object's hidden class. Caching in locals at the top of `parse()` and writing back at the end could help.
+* **Specialize the common case in the main switch.** PRINT and PARAM are the most common actions. If the switch has many cases, V8 may generate a jump table; if it has few hot cases, a nested if-else could be faster.
+* **The `code < 0xa0 ? code : NON_ASCII_PRINTABLE` ternary** runs per byte. For ASCII-only input (the overwhelming common case), the branch is always taken. Consider a fast-path that skips the check when all input is < 0x80.
+
+Anti-patterns:
+* Don't try to change the transition table layout (e.g. making it a 2D array). The `Uint16Array` with bit shifting is already the most cache-friendly layout.
+* Don't try to replace the state machine with regex. The state machine handles streaming input correctly; regex can't.
+* Don't optimize the async handler resume path (top of `parse()`). It's cold code.
+
+## Output format
+
+`run.py` prints:
+
+```
+---
+mbps:            693.3
+tests_passed:    1
+build_ok:        1
+install_seconds: 5.0
+build_seconds:   15.0
+test_seconds:    6.0
+best_so_far:     693.3
+threshold:       728.0  (best_so_far * 1.05)
+auto-status:     keep
+```
+
+Read the headline: `grep "^mbps:\|^tests_passed:\|^auto-status:" run.log`
+
+Per-case breakdown is on stderr from the bench: `grep "per-case" -A 10 run.log` (or check the bench stderr output).
+
+## Logging results
+
+`python run.py --record --description "<one line>"` appends a row. The harness auto-labels status.
+
+Schema: `commit  mbps  tests_pass  build_ok  status  description`
+
+## The experiment loop
+
+LOOP FOREVER:
+
+1. `git log --oneline -10` and `tail -10 results.tsv`. What's the current best? What has been tried?
+2. Pick one optimization idea. Be specific.
+3. Edit `src/common/parser/EscapeSequenceParser.ts` (or Params.ts). One change per experiment.
+4. `git add -A && git commit -m "try: <description>"` (commit BEFORE the record run).
+5. `python run.py --record --description "<description>" > run.log 2>&1`
+6. `grep "^mbps:\|^tests_passed:\|^auto-status:" run.log`.
+7. Decision:
+   * `auto-status: keep` → the branch advances.
+   * `auto-status: discard` → `git reset --hard HEAD~1`.
+8. Goto 1.
+
+**Build failures**: if `npm run build` fails, it's a TS type error from your edit. Read the tsc output in run.log, fix the type error or revert.
+
+**Verification for large wins**: if `auto-status: keep` on a >15% improvement, run `python run.py > /tmp/verify.log 2>&1` (no --record). If verify mbps is also clearly above threshold, it's real. If verify is close to baseline, it was noise — `git reset --hard HEAD~1`.
+
+**NEVER STOP**: once the loop has begun, do NOT pause. Run indefinitely until manually stopped.

run.py

@@ -0,0 +1,198 @@
+"""
+run.py — one experiment for slim-forge xterm.js mission.
+
+invoked by the agent each cycle:
+    python run.py
+    python run.py --record --description "tried X"
+
+pipeline: install -> build (tsc + esbuild) -> test-unit -> bench -> record
+
+bench script prints one float on stdout last line (weighted MB/s).
+the harness enforces ops > best_so_far * 1.05 before labeling keep.
+
+NEVER edit this file.
+"""
+
+from __future__ import annotations
+
+import argparse
+import csv
+import subprocess
+import sys
+import time
+import tomllib
+from pathlib import Path
+
+HERE = Path(__file__).parent.resolve()
+THRESHOLD_MULT = 1.05
+
+
+def sh(cmd: str, timeout: int = 900) -> subprocess.CompletedProcess:
+    return subprocess.run(
+        cmd, shell=True, cwd=HERE, text=True, capture_output=True,
+        timeout=timeout, check=False,
+    )
+
+
+def load_spec() -> dict:
+    p = HERE / ".slimforge.toml"
+    if not p.exists():
+        print("error: .slimforge.toml missing. did you run prepare.py?", file=sys.stderr)
+        sys.exit(2)
+    with open(p, "rb") as f:
+        return tomllib.load(f)
+
+
+def install(spec: dict) -> tuple[bool, float]:
+    t0 = time.time()
+    r = sh(spec["install_cmd"], timeout=600)
+    dt = time.time() - t0
+    if r.returncode != 0:
+        print("[install stderr tail]", file=sys.stderr)
+        print(r.stderr[-2000:], file=sys.stderr)
+        return False, dt
+    return True, dt
+
+
+def build(spec: dict) -> tuple[bool, float]:
+    cmd = spec.get("build_cmd", "").strip()
+    if not cmd:
+        return True, 0.0
+    t0 = time.time()
+    r = sh(cmd, timeout=600)
+    dt = time.time() - t0
+    if r.returncode != 0:
+        print("[build stderr tail]", file=sys.stderr)
+        print(r.stderr[-3000:], file=sys.stderr)
+        print("[build stdout tail]", file=sys.stderr)
+        print(r.stdout[-3000:], file=sys.stderr)
+        return False, dt
+    return True, dt
+
+
+def run_tests(spec: dict) -> tuple[bool, float]:
+    t0 = time.time()
+    r = sh(spec["test_cmd"], timeout=600)
+    dt = time.time() - t0
+    if r.returncode != 0:
+        print("[test stderr tail]", file=sys.stderr)
+        print(r.stderr[-3000:], file=sys.stderr)
+        print("[test stdout tail]", file=sys.stderr)
+        print(r.stdout[-3000:], file=sys.stderr)
+    return r.returncode == 0, dt
+
+
+def run_bench(spec: dict) -> tuple[float, str]:
+    cmd = spec.get("bench_cmd", "").strip()
+    if not cmd:
+        return -1.0, ""
+    r = sh(cmd, timeout=300)
+    tail = (r.stdout + ("\n[stderr]\n" + r.stderr if r.stderr else ""))[-3000:]
+    if r.returncode != 0:
+        print("[bench failed, tail]", file=sys.stderr)
+        print(tail, file=sys.stderr)
+        return -1.0, tail
+    try:
+        return float(r.stdout.strip().splitlines()[-1]), tail
+    except (ValueError, IndexError):
+        print("[bench output unparseable, tail]", file=sys.stderr)
+        print(tail, file=sys.stderr)
+        return -1.0, tail
+
+
+def compute_best_so_far() -> float:
+    tsv = HERE / "results.tsv"
+    if not tsv.exists():
+        return 0.0
+    best = 0.0
+    with open(tsv) as f:
+        for row in csv.DictReader(f, delimiter="\t"):
+            if row.get("status") == "keep" and row.get("tests_pass") == "1":
+                try:
+                    v = float(row["mbps"])
+                    if v > best:
+                        best = v
+                except (ValueError, KeyError):
+                    pass
+    return best
+
+
+def git_sha() -> str:
+    r = sh("git rev-parse --short HEAD", timeout=10)
+    return r.stdout.strip() or "0000000"
+
+
+def append_results_row(
+    sha: str, mbps: float, tests_pass: bool, build_ok: bool,
+    status: str, description: str,
+) -> None:
+    desc = description.replace("\t", " ").replace("\n", " ").strip()
+    tsv = HERE / "results.tsv"
+    if not tsv.exists():
+        tsv.write_text("commit\tmbps\ttests_pass\tbuild_ok\tstatus\tdescription\n")
+    with open(tsv, "a") as f:
+        f.write(f"{sha}\t{mbps:.1f}\t{int(tests_pass)}\t{int(build_ok)}\t{status}\t{desc}\n")
+
+
+def main() -> int:
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--record", action="store_true")
+    ap.add_argument("--description", default="")
+    ap.add_argument("--status", default=None)
+    args = ap.parse_args()
+
+    spec = load_spec()
+
+    install_ok, install_secs = install(spec)
+    if not install_ok:
+        print("---")
+        print("install:         FAILED")
+        if args.record:
+            append_results_row(git_sha(), -1.0, False, False,
+                               args.status or "crash",
+                               args.description or "install failed")
+        return 1
+
+    build_ok, build_secs = build(spec)
+    if not build_ok:
+        print("---")
+        print("build:           FAILED")
+        if args.record:
+            append_results_row(git_sha(), -1.0, False, False,
+                               args.status or "discard",
+                               args.description or "build failed")
+        return 1
+
+    tests_pass, test_secs = run_tests(spec)
+    primary_mbps, bench_tail = run_bench(spec)
+
+    print("---")
+    print(f"mbps:            {primary_mbps:.1f}")
+    print(f"tests_passed:    {int(tests_pass)}")
+    print(f"build_ok:        {int(build_ok)}")
+    print(f"install_seconds: {install_secs:.1f}")
+    print(f"build_seconds:   {build_secs:.1f}")
+    print(f"test_seconds:    {test_secs:.1f}")
+
+    if args.record:
+        status = args.status
+        if status is None:
+            best = compute_best_so_far()
+            threshold = best * THRESHOLD_MULT
+            if not tests_pass:
+                status = "discard"
+            elif primary_mbps < threshold:
+                status = "discard"
+            else:
+                status = "keep"
+            print(f"best_so_far:     {best:.1f}")
+            print(f"threshold:       {threshold:.1f}  (best_so_far * {THRESHOLD_MULT})")
+            print(f"auto-status:     {status}")
+        append_results_row(git_sha(), primary_mbps, tests_pass, build_ok,
+                           status, args.description)
+
+    return 0 if tests_pass else 1
+
+
+if __name__ == "__main__":
+    sys.exit(main())