slim-forge: initial harness
This commit is contained in:
slim-forge
@@ -29,3 +29,7 @@ demo/dist/
|
||||
# dont commit benchmark folders
|
||||
.benchmark/
|
||||
timeline/
|
||||
|
||||
# slim-forge
|
||||
results.tsv
|
||||
run.log
|
||||
|
||||
@@ -0,0 +1,6 @@
|
||||
install_cmd = """npm install --no-audit --no-fund --loglevel=error"""
|
||||
build_cmd = """npm run build && npm run esbuild"""
|
||||
test_cmd = """npm run test-unit"""
|
||||
bench_cmd = """node bench/workload.js"""
|
||||
primary_bench_name = """weighted_mbps"""
|
||||
target = "xterm"
|
||||
@@ -0,0 +1,125 @@
|
||||
'use strict';
|
||||
|
||||
// xterm.js EscapeSequenceParser throughput benchmark
|
||||
// Runs against compiled output in out-esbuild/
|
||||
// Measures weighted MB/s across representative terminal workloads
|
||||
|
||||
process.env.NODE_PATH = require('path').resolve(__dirname, '../out-esbuild');
|
||||
require('module').Module._initPaths();
|
||||
|
||||
var EscapeSequenceParser = require('common/parser/EscapeSequenceParser').EscapeSequenceParser;
|
||||
var C0 = require('common/data/EscapeSequences').C0;
|
||||
|
||||
function toUtf32(s) {
|
||||
var result = new Uint32Array(s.length);
|
||||
for (var i = 0; i < s.length; i++) {
|
||||
result[i] = s.charCodeAt(i);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
function makeParser() {
|
||||
var parser = new EscapeSequenceParser();
|
||||
parser.setPrintHandler(function() {});
|
||||
parser.setExecuteHandler(C0.LF, function() {});
|
||||
parser.setExecuteHandler(C0.CR, function() {});
|
||||
parser.setExecuteHandler('\t', function() {});
|
||||
parser.setExecuteHandler('\b', function() {});
|
||||
parser.registerCsiHandler({ final: 'm' }, function() { return true; });
|
||||
parser.registerCsiHandler({ final: 'H' }, function() { return true; });
|
||||
parser.registerCsiHandler({ final: 'J' }, function() { return true; });
|
||||
parser.registerCsiHandler({ final: 'K' }, function() { return true; });
|
||||
parser.registerCsiHandler({ final: 'A' }, function() { return true; });
|
||||
parser.registerCsiHandler({ final: 'B' }, function() { return true; });
|
||||
parser.registerCsiHandler({ final: 'C' }, function() { return true; });
|
||||
parser.registerCsiHandler({ final: 'D' }, function() { return true; });
|
||||
parser.registerEscHandler({ final: '7' }, function() { return true; });
|
||||
parser.registerEscHandler({ final: '8' }, function() { return true; });
|
||||
return parser;
|
||||
}
|
||||
|
||||
var SIZE = 2000000;
|
||||
|
||||
var cases = [
|
||||
['print_ascii', (function() {
|
||||
var s = 'The quick brown fox jumps over the lazy dog. Pack my box with five dozen liquor jugs.\n';
|
||||
var content = '';
|
||||
while (content.length < SIZE) content += s;
|
||||
return toUtf32(content);
|
||||
})(), 4],
|
||||
|
||||
['sgr_heavy', (function() {
|
||||
var s = '\x1b[0;32mfile.txt\x1b[0m \x1b[0;34mdir/\x1b[0m \x1b[1;31merror.log\x1b[0m\n';
|
||||
var content = '';
|
||||
while (content.length < SIZE) content += s;
|
||||
return toUtf32(content);
|
||||
})(), 4],
|
||||
|
||||
['cursor_move', (function() {
|
||||
var s = '\x1b[1;1H\x1b[2JStatus: OK\x1b[2;1HLine 2 content here\x1b[3;1HAnother line\n';
|
||||
var content = '';
|
||||
while (content.length < SIZE) content += s;
|
||||
return toUtf32(content);
|
||||
})(), 2],
|
||||
|
||||
['params_heavy', (function() {
|
||||
var s = '\x1b[38;5;196mred\x1b[0m \x1b[38;2;0;255;128mgreen\x1b[0m \x1b[48;5;21mblue_bg\x1b[0m\n';
|
||||
var content = '';
|
||||
while (content.length < SIZE) content += s;
|
||||
return toUtf32(content);
|
||||
})(), 3],
|
||||
|
||||
['mixed_realistic', (function() {
|
||||
var lines = [
|
||||
'\x1b[33mabcd123\x1b[0m fix: resolve parser edge case\n',
|
||||
'\x1b[33mef45678\x1b[0m feat: add new feature\n',
|
||||
'\x1b[33m9abc012\x1b[0m \x1b[1;31m(HEAD -> main)\x1b[0m chore: update deps\n',
|
||||
'\x1b[33m3def456\x1b[0m docs: update readme\n',
|
||||
'\x1b[33m789abcd\x1b[0m refactor: clean up utils\n',
|
||||
];
|
||||
var content = '';
|
||||
while (content.length < SIZE) {
|
||||
for (var j = 0; j < lines.length; j++) content += lines[j];
|
||||
}
|
||||
return toUtf32(content);
|
||||
})(), 4],
|
||||
];
|
||||
|
||||
function benchOne(parser, data) {
|
||||
for (var w = 0; w < 5; w++) parser.parse(data, data.length);
|
||||
var runs = [];
|
||||
for (var r = 0; r < 7; r++) {
|
||||
var t0 = process.hrtime.bigint();
|
||||
parser.parse(data, data.length);
|
||||
var dt = Number(process.hrtime.bigint() - t0) / 1e9;
|
||||
var mbps = (data.length * 4) / (1024 * 1024) / dt;
|
||||
runs.push(mbps);
|
||||
}
|
||||
runs.sort(function(a, b) { return a - b; });
|
||||
return runs[Math.floor(runs.length / 2)];
|
||||
}
|
||||
|
||||
var parser = makeParser();
|
||||
var totalWeight = 0;
|
||||
var weightedSum = 0;
|
||||
var perCase = [];
|
||||
|
||||
for (var c = 0; c < cases.length; c++) {
|
||||
var name = cases[c][0];
|
||||
var data = cases[c][1];
|
||||
var weight = cases[c][2];
|
||||
var mbps = benchOne(parser, data);
|
||||
perCase.push([name, mbps]);
|
||||
weightedSum += mbps * weight;
|
||||
totalWeight += weight;
|
||||
}
|
||||
|
||||
var weighted = weightedSum / totalWeight;
|
||||
|
||||
process.stderr.write('per-case MB/s:\n');
|
||||
for (var p = 0; p < perCase.length; p++) {
|
||||
process.stderr.write(' ' + perCase[p][0] + ': ' + perCase[p][1].toFixed(1) + '\n');
|
||||
}
|
||||
process.stderr.write('weighted: ' + weighted.toFixed(1) + ' MB/s\n');
|
||||
|
||||
process.stdout.write(weighted.toFixed(1) + '\n');
|
||||
+135
@@ -0,0 +1,135 @@
|
||||
# slim-forge: xterm.js parser perf
|
||||
|
||||
An experiment to autonomously speed up xterm.js's `EscapeSequenceParser`, the terminal escape sequence parser inside VS Code, GitHub Codespaces, and most web-based terminals.
|
||||
|
||||
## Setup
|
||||
|
||||
1. **Confirm the workspace**: fresh clone of `xtermjs/xterm.js`. Branch should be `autoresearch/xterm-<date>`. Verify: `git status`, `git log --oneline -5`.
|
||||
2. **Read the in-scope files**:
|
||||
* `src/common/parser/EscapeSequenceParser.ts` — ~845 LOC. **THIS is what you optimize.** The main `parse()` method is the hot loop. Read the whole file.
|
||||
* `src/common/parser/Params.ts` — parameter accumulation. Called from the PARAM action in the main loop. Secondary surface.
|
||||
* `src/common/parser/Constants.ts` — `ParserState` and `ParserAction` enums.
|
||||
* `src/common/parser/EscapeSequenceParser.test.ts` — ~2236 LOC of tests. Read before making changes.
|
||||
* `bench/workload.js` — the benchmark. **Do NOT modify.**
|
||||
* `run.py` — the harness. **Do NOT modify.**
|
||||
* `program.md` — this file.
|
||||
3. **Verify baseline exists**: `cat results.tsv`. One row, `baseline`, with a `mbps` value and `tests_pass: 1`.
|
||||
4. **Confirm and go.**
|
||||
|
||||
## Mission
|
||||
|
||||
Primary metric: **`mbps`** — weighted throughput in MB/s across 5 representative terminal workloads (plain ASCII print, SGR-heavy colored output, cursor movement sequences, parameter-heavy truecolor, mixed realistic git-log-style output). Higher is better.
|
||||
|
||||
Hard gates:
|
||||
1. `npm run test-unit` passes (2261 mocha tests, ~6s).
|
||||
2. The harness enforces `mbps > best_so_far * 1.05` for a row to be labeled `keep`.
|
||||
|
||||
## Important: build pipeline
|
||||
|
||||
xterm.js is TypeScript. Your edits go in `src/`. The build pipeline is:
|
||||
|
||||
```
|
||||
npm run build # tsc: src/**/*.ts -> out/**/*.js
|
||||
npm run esbuild # esbuild: out/ -> out-esbuild/ (what tests and bench consume)
|
||||
```
|
||||
|
||||
`run.py` handles this automatically. But if you want to test manually:
|
||||
```
|
||||
npm run build && npm run esbuild && npm run test-unit && node bench/workload.js
|
||||
```
|
||||
|
||||
The agent edits `.ts` source. The bench runs against compiled `.js` in `out-esbuild/`. If your edit has a TS type error, `npm run build` fails and the experiment is `discard`.
|
||||
|
||||
## What you CAN do
|
||||
|
||||
* Edit `src/common/parser/EscapeSequenceParser.ts` — the main target.
|
||||
* Edit `src/common/parser/Params.ts` — if parameter handling is the bottleneck.
|
||||
* Edit `src/common/parser/OscParser.ts`, `DcsParser.ts`, `ApcParser.ts` — if those sub-parsers show up in the bench.
|
||||
* Add internal helper functions or restructure the parse loop.
|
||||
|
||||
## What you CANNOT do
|
||||
|
||||
* Modify `src/common/parser/EscapeSequenceParser.test.ts` or any test file.
|
||||
* Modify `bench/workload.js` or `run.py`.
|
||||
* Modify `program.md`.
|
||||
* Modify the `TransitionTable` initialization — the VT500 table is the spec-correct state machine. Changing transitions changes which sequences are recognized, which is a correctness violation even if tests pass.
|
||||
* Cache parse results across calls. The parser is stateful (it maintains `currentState`), but each `parse()` call must process every byte of input. No memoization.
|
||||
* Remove or weaken handler dispatch. The bench registers real handlers; skipping dispatch would be a benchmark exploit.
|
||||
|
||||
## What the parser already does well
|
||||
|
||||
Before you start, understand what's already optimized so you don't waste time re-discovering it:
|
||||
|
||||
* **Transition table**: `Uint16Array` lookup — state × charCode → action + nextState. One array access per byte. This is already near-optimal for the state machine itself.
|
||||
* **PRINT action loop unrolling**: 4× unrolled inner loop for consecutive printable characters. This is why `print_ascii` hits ~2500 MB/s.
|
||||
* **PARAM inner loop**: do-while that stays in the PARAM action for consecutive digit/semicolon/colon bytes without re-entering the main switch.
|
||||
|
||||
## Where to look for wins
|
||||
|
||||
The bench breakdown shows where time is actually spent:
|
||||
|
||||
* **`sgr_heavy`: ~84 MB/s** — 30× slower than print. Every SGR sequence goes through: state transitions for `ESC [ params m`, param parsing, CSI handler dispatch. The handler dispatch involves array lookups and function calls per sequence.
|
||||
* **`params_heavy`: ~92 MB/s** — similar. Truecolor sequences have many params (e.g. `38;2;0;255;128`).
|
||||
* **`cursor_move`: ~127 MB/s** — CSI sequences with fewer params but more state transitions per byte.
|
||||
* **`mixed_realistic`: ~158 MB/s** — blend of print runs and SGR.
|
||||
|
||||
Likely sources of real speedups:
|
||||
|
||||
* **Reduce handler dispatch overhead.** The CSI dispatch does `this._csiHandlers[this._collect << 8 | code]` then iterates the handler array in reverse. For the common case of a single registered handler, the array iteration + `instanceof Promise` check is overhead. A fast-path for single-handler case could help.
|
||||
* **Inline the PARAM parsing more aggressively.** The current do-while re-enters `this._params.addDigit()` / `addParam()` / `addSubParam()` per byte. If `Params` methods are not inlined by V8, each is a virtual dispatch.
|
||||
* **Reduce property access in the hot loop.** `this._transitions`, `this.currentState`, `this._collect`, `this._params` — each `this.` access goes through the object's hidden class. Caching in locals at the top of `parse()` and writing back at the end could help.
|
||||
* **Specialize the common case in the main switch.** PRINT and PARAM are the most common actions. If the switch has many cases, V8 may generate a jump table; if it has few hot cases, a nested if-else could be faster.
|
||||
* **The `code < 0xa0 ? code : NON_ASCII_PRINTABLE` ternary** runs per byte. For ASCII-only input (the overwhelming common case), the branch is always taken. Consider a fast-path that skips the check when all input is < 0x80.
|
||||
|
||||
Anti-patterns:
|
||||
* Don't try to change the transition table layout (e.g. making it a 2D array). The `Uint16Array` with bit shifting is already the most cache-friendly layout.
|
||||
* Don't try to replace the state machine with regex. The state machine handles streaming input correctly; regex can't.
|
||||
* Don't optimize the async handler resume path (top of `parse()`). It's cold code.
|
||||
|
||||
## Output format
|
||||
|
||||
`run.py` prints:
|
||||
|
||||
```
|
||||
---
|
||||
mbps: 693.3
|
||||
tests_passed: 1
|
||||
build_ok: 1
|
||||
install_seconds: 5.0
|
||||
build_seconds: 15.0
|
||||
test_seconds: 6.0
|
||||
best_so_far: 693.3
|
||||
threshold: 728.0 (best_so_far * 1.05)
|
||||
auto-status: keep
|
||||
```
|
||||
|
||||
Read the headline: `grep "^mbps:\|^tests_passed:\|^auto-status:" run.log`
|
||||
|
||||
Per-case breakdown is on stderr from the bench: `grep "per-case" -A 10 run.log` (or check the bench stderr output).
|
||||
|
||||
## Logging results
|
||||
|
||||
`python run.py --record --description "<one line>"` appends a row. The harness auto-labels status.
|
||||
|
||||
Schema: `commit mbps tests_pass build_ok status description`
|
||||
|
||||
## The experiment loop
|
||||
|
||||
LOOP FOREVER:
|
||||
|
||||
1. `git log --oneline -10` and `tail -10 results.tsv`. What's the current best? What has been tried?
|
||||
2. Pick one optimization idea. Be specific.
|
||||
3. Edit `src/common/parser/EscapeSequenceParser.ts` (or Params.ts). One change per experiment.
|
||||
4. `git add -A && git commit -m "try: <description>"` (commit BEFORE the record run).
|
||||
5. `python run.py --record --description "<description>" > run.log 2>&1`
|
||||
6. `grep "^mbps:\|^tests_passed:\|^auto-status:" run.log`.
|
||||
7. Decision:
|
||||
* `auto-status: keep` → the branch advances.
|
||||
* `auto-status: discard` → `git reset --hard HEAD~1`.
|
||||
8. Goto 1.
|
||||
|
||||
**Build failures**: if `npm run build` fails, it's a TS type error from your edit. Read the tsc output in run.log, fix the type error or revert.
|
||||
|
||||
**Verification for large wins**: if `auto-status: keep` on a >15% improvement, run `python run.py > /tmp/verify.log 2>&1` (no --record). If verify mbps is also clearly above threshold, it's real. If verify is close to baseline, it was noise — `git reset --hard HEAD~1`.
|
||||
|
||||
**NEVER STOP**: once the loop has begun, do NOT pause. Run indefinitely until manually stopped.
|
||||
@@ -0,0 +1,198 @@
|
||||
"""
|
||||
run.py — one experiment for slim-forge xterm.js mission.
|
||||
|
||||
invoked by the agent each cycle:
|
||||
python run.py
|
||||
python run.py --record --description "tried X"
|
||||
|
||||
pipeline: install -> build (tsc + esbuild) -> test-unit -> bench -> record
|
||||
|
||||
bench script prints one float on stdout last line (weighted MB/s).
|
||||
the harness enforces ops > best_so_far * 1.05 before labeling keep.
|
||||
|
||||
NEVER edit this file.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import csv
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import tomllib
|
||||
from pathlib import Path
|
||||
|
||||
HERE = Path(__file__).parent.resolve()
|
||||
THRESHOLD_MULT = 1.05
|
||||
|
||||
|
||||
def sh(cmd: str, timeout: int = 900) -> subprocess.CompletedProcess:
|
||||
return subprocess.run(
|
||||
cmd, shell=True, cwd=HERE, text=True, capture_output=True,
|
||||
timeout=timeout, check=False,
|
||||
)
|
||||
|
||||
|
||||
def load_spec() -> dict:
|
||||
p = HERE / ".slimforge.toml"
|
||||
if not p.exists():
|
||||
print("error: .slimforge.toml missing. did you run prepare.py?", file=sys.stderr)
|
||||
sys.exit(2)
|
||||
with open(p, "rb") as f:
|
||||
return tomllib.load(f)
|
||||
|
||||
|
||||
def install(spec: dict) -> tuple[bool, float]:
|
||||
t0 = time.time()
|
||||
r = sh(spec["install_cmd"], timeout=600)
|
||||
dt = time.time() - t0
|
||||
if r.returncode != 0:
|
||||
print("[install stderr tail]", file=sys.stderr)
|
||||
print(r.stderr[-2000:], file=sys.stderr)
|
||||
return False, dt
|
||||
return True, dt
|
||||
|
||||
|
||||
def build(spec: dict) -> tuple[bool, float]:
|
||||
cmd = spec.get("build_cmd", "").strip()
|
||||
if not cmd:
|
||||
return True, 0.0
|
||||
t0 = time.time()
|
||||
r = sh(cmd, timeout=600)
|
||||
dt = time.time() - t0
|
||||
if r.returncode != 0:
|
||||
print("[build stderr tail]", file=sys.stderr)
|
||||
print(r.stderr[-3000:], file=sys.stderr)
|
||||
print("[build stdout tail]", file=sys.stderr)
|
||||
print(r.stdout[-3000:], file=sys.stderr)
|
||||
return False, dt
|
||||
return True, dt
|
||||
|
||||
|
||||
def run_tests(spec: dict) -> tuple[bool, float]:
|
||||
t0 = time.time()
|
||||
r = sh(spec["test_cmd"], timeout=600)
|
||||
dt = time.time() - t0
|
||||
if r.returncode != 0:
|
||||
print("[test stderr tail]", file=sys.stderr)
|
||||
print(r.stderr[-3000:], file=sys.stderr)
|
||||
print("[test stdout tail]", file=sys.stderr)
|
||||
print(r.stdout[-3000:], file=sys.stderr)
|
||||
return r.returncode == 0, dt
|
||||
|
||||
|
||||
def run_bench(spec: dict) -> tuple[float, str]:
|
||||
cmd = spec.get("bench_cmd", "").strip()
|
||||
if not cmd:
|
||||
return -1.0, ""
|
||||
r = sh(cmd, timeout=300)
|
||||
tail = (r.stdout + ("\n[stderr]\n" + r.stderr if r.stderr else ""))[-3000:]
|
||||
if r.returncode != 0:
|
||||
print("[bench failed, tail]", file=sys.stderr)
|
||||
print(tail, file=sys.stderr)
|
||||
return -1.0, tail
|
||||
try:
|
||||
return float(r.stdout.strip().splitlines()[-1]), tail
|
||||
except (ValueError, IndexError):
|
||||
print("[bench output unparseable, tail]", file=sys.stderr)
|
||||
print(tail, file=sys.stderr)
|
||||
return -1.0, tail
|
||||
|
||||
|
||||
def compute_best_so_far() -> float:
|
||||
tsv = HERE / "results.tsv"
|
||||
if not tsv.exists():
|
||||
return 0.0
|
||||
best = 0.0
|
||||
with open(tsv) as f:
|
||||
for row in csv.DictReader(f, delimiter="\t"):
|
||||
if row.get("status") == "keep" and row.get("tests_pass") == "1":
|
||||
try:
|
||||
v = float(row["mbps"])
|
||||
if v > best:
|
||||
best = v
|
||||
except (ValueError, KeyError):
|
||||
pass
|
||||
return best
|
||||
|
||||
|
||||
def git_sha() -> str:
|
||||
r = sh("git rev-parse --short HEAD", timeout=10)
|
||||
return r.stdout.strip() or "0000000"
|
||||
|
||||
|
||||
def append_results_row(
|
||||
sha: str, mbps: float, tests_pass: bool, build_ok: bool,
|
||||
status: str, description: str,
|
||||
) -> None:
|
||||
desc = description.replace("\t", " ").replace("\n", " ").strip()
|
||||
tsv = HERE / "results.tsv"
|
||||
if not tsv.exists():
|
||||
tsv.write_text("commit\tmbps\ttests_pass\tbuild_ok\tstatus\tdescription\n")
|
||||
with open(tsv, "a") as f:
|
||||
f.write(f"{sha}\t{mbps:.1f}\t{int(tests_pass)}\t{int(build_ok)}\t{status}\t{desc}\n")
|
||||
|
||||
|
||||
def main() -> int:
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--record", action="store_true")
|
||||
ap.add_argument("--description", default="")
|
||||
ap.add_argument("--status", default=None)
|
||||
args = ap.parse_args()
|
||||
|
||||
spec = load_spec()
|
||||
|
||||
install_ok, install_secs = install(spec)
|
||||
if not install_ok:
|
||||
print("---")
|
||||
print("install: FAILED")
|
||||
if args.record:
|
||||
append_results_row(git_sha(), -1.0, False, False,
|
||||
args.status or "crash",
|
||||
args.description or "install failed")
|
||||
return 1
|
||||
|
||||
build_ok, build_secs = build(spec)
|
||||
if not build_ok:
|
||||
print("---")
|
||||
print("build: FAILED")
|
||||
if args.record:
|
||||
append_results_row(git_sha(), -1.0, False, False,
|
||||
args.status or "discard",
|
||||
args.description or "build failed")
|
||||
return 1
|
||||
|
||||
tests_pass, test_secs = run_tests(spec)
|
||||
primary_mbps, bench_tail = run_bench(spec)
|
||||
|
||||
print("---")
|
||||
print(f"mbps: {primary_mbps:.1f}")
|
||||
print(f"tests_passed: {int(tests_pass)}")
|
||||
print(f"build_ok: {int(build_ok)}")
|
||||
print(f"install_seconds: {install_secs:.1f}")
|
||||
print(f"build_seconds: {build_secs:.1f}")
|
||||
print(f"test_seconds: {test_secs:.1f}")
|
||||
|
||||
if args.record:
|
||||
status = args.status
|
||||
if status is None:
|
||||
best = compute_best_so_far()
|
||||
threshold = best * THRESHOLD_MULT
|
||||
if not tests_pass:
|
||||
status = "discard"
|
||||
elif primary_mbps < threshold:
|
||||
status = "discard"
|
||||
else:
|
||||
status = "keep"
|
||||
print(f"best_so_far: {best:.1f}")
|
||||
print(f"threshold: {threshold:.1f} (best_so_far * {THRESHOLD_MULT})")
|
||||
print(f"auto-status: {status}")
|
||||
append_results_row(git_sha(), primary_mbps, tests_pass, build_ok,
|
||||
status, args.description)
|
||||
|
||||
return 0 if tests_pass else 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Reference in New Issue
Block a user