slim-forge: initial harness
This commit is contained in:
slim-forge
@@ -0,0 +1,198 @@
|
||||
"""
|
||||
run.py — one experiment for slim-forge xterm.js mission.
|
||||
|
||||
invoked by the agent each cycle:
|
||||
python run.py
|
||||
python run.py --record --description "tried X"
|
||||
|
||||
pipeline: install -> build (tsc + esbuild) -> test-unit -> bench -> record
|
||||
|
||||
bench script prints one float on stdout last line (weighted MB/s).
|
||||
the harness enforces ops > best_so_far * 1.05 before labeling keep.
|
||||
|
||||
NEVER edit this file.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import csv
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import tomllib
|
||||
from pathlib import Path
|
||||
|
||||
HERE = Path(__file__).parent.resolve()
|
||||
THRESHOLD_MULT = 1.05
|
||||
|
||||
|
||||
def sh(cmd: str, timeout: int = 900) -> subprocess.CompletedProcess:
|
||||
return subprocess.run(
|
||||
cmd, shell=True, cwd=HERE, text=True, capture_output=True,
|
||||
timeout=timeout, check=False,
|
||||
)
|
||||
|
||||
|
||||
def load_spec() -> dict:
|
||||
p = HERE / ".slimforge.toml"
|
||||
if not p.exists():
|
||||
print("error: .slimforge.toml missing. did you run prepare.py?", file=sys.stderr)
|
||||
sys.exit(2)
|
||||
with open(p, "rb") as f:
|
||||
return tomllib.load(f)
|
||||
|
||||
|
||||
def install(spec: dict) -> tuple[bool, float]:
|
||||
t0 = time.time()
|
||||
r = sh(spec["install_cmd"], timeout=600)
|
||||
dt = time.time() - t0
|
||||
if r.returncode != 0:
|
||||
print("[install stderr tail]", file=sys.stderr)
|
||||
print(r.stderr[-2000:], file=sys.stderr)
|
||||
return False, dt
|
||||
return True, dt
|
||||
|
||||
|
||||
def build(spec: dict) -> tuple[bool, float]:
|
||||
cmd = spec.get("build_cmd", "").strip()
|
||||
if not cmd:
|
||||
return True, 0.0
|
||||
t0 = time.time()
|
||||
r = sh(cmd, timeout=600)
|
||||
dt = time.time() - t0
|
||||
if r.returncode != 0:
|
||||
print("[build stderr tail]", file=sys.stderr)
|
||||
print(r.stderr[-3000:], file=sys.stderr)
|
||||
print("[build stdout tail]", file=sys.stderr)
|
||||
print(r.stdout[-3000:], file=sys.stderr)
|
||||
return False, dt
|
||||
return True, dt
|
||||
|
||||
|
||||
def run_tests(spec: dict) -> tuple[bool, float]:
|
||||
t0 = time.time()
|
||||
r = sh(spec["test_cmd"], timeout=600)
|
||||
dt = time.time() - t0
|
||||
if r.returncode != 0:
|
||||
print("[test stderr tail]", file=sys.stderr)
|
||||
print(r.stderr[-3000:], file=sys.stderr)
|
||||
print("[test stdout tail]", file=sys.stderr)
|
||||
print(r.stdout[-3000:], file=sys.stderr)
|
||||
return r.returncode == 0, dt
|
||||
|
||||
|
||||
def run_bench(spec: dict) -> tuple[float, str]:
|
||||
cmd = spec.get("bench_cmd", "").strip()
|
||||
if not cmd:
|
||||
return -1.0, ""
|
||||
r = sh(cmd, timeout=300)
|
||||
tail = (r.stdout + ("\n[stderr]\n" + r.stderr if r.stderr else ""))[-3000:]
|
||||
if r.returncode != 0:
|
||||
print("[bench failed, tail]", file=sys.stderr)
|
||||
print(tail, file=sys.stderr)
|
||||
return -1.0, tail
|
||||
try:
|
||||
return float(r.stdout.strip().splitlines()[-1]), tail
|
||||
except (ValueError, IndexError):
|
||||
print("[bench output unparseable, tail]", file=sys.stderr)
|
||||
print(tail, file=sys.stderr)
|
||||
return -1.0, tail
|
||||
|
||||
|
||||
def compute_best_so_far() -> float:
|
||||
tsv = HERE / "results.tsv"
|
||||
if not tsv.exists():
|
||||
return 0.0
|
||||
best = 0.0
|
||||
with open(tsv) as f:
|
||||
for row in csv.DictReader(f, delimiter="\t"):
|
||||
if row.get("status") == "keep" and row.get("tests_pass") == "1":
|
||||
try:
|
||||
v = float(row["mbps"])
|
||||
if v > best:
|
||||
best = v
|
||||
except (ValueError, KeyError):
|
||||
pass
|
||||
return best
|
||||
|
||||
|
||||
def git_sha() -> str:
|
||||
r = sh("git rev-parse --short HEAD", timeout=10)
|
||||
return r.stdout.strip() or "0000000"
|
||||
|
||||
|
||||
def append_results_row(
|
||||
sha: str, mbps: float, tests_pass: bool, build_ok: bool,
|
||||
status: str, description: str,
|
||||
) -> None:
|
||||
desc = description.replace("\t", " ").replace("\n", " ").strip()
|
||||
tsv = HERE / "results.tsv"
|
||||
if not tsv.exists():
|
||||
tsv.write_text("commit\tmbps\ttests_pass\tbuild_ok\tstatus\tdescription\n")
|
||||
with open(tsv, "a") as f:
|
||||
f.write(f"{sha}\t{mbps:.1f}\t{int(tests_pass)}\t{int(build_ok)}\t{status}\t{desc}\n")
|
||||
|
||||
|
||||
def main() -> int:
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--record", action="store_true")
|
||||
ap.add_argument("--description", default="")
|
||||
ap.add_argument("--status", default=None)
|
||||
args = ap.parse_args()
|
||||
|
||||
spec = load_spec()
|
||||
|
||||
install_ok, install_secs = install(spec)
|
||||
if not install_ok:
|
||||
print("---")
|
||||
print("install: FAILED")
|
||||
if args.record:
|
||||
append_results_row(git_sha(), -1.0, False, False,
|
||||
args.status or "crash",
|
||||
args.description or "install failed")
|
||||
return 1
|
||||
|
||||
build_ok, build_secs = build(spec)
|
||||
if not build_ok:
|
||||
print("---")
|
||||
print("build: FAILED")
|
||||
if args.record:
|
||||
append_results_row(git_sha(), -1.0, False, False,
|
||||
args.status or "discard",
|
||||
args.description or "build failed")
|
||||
return 1
|
||||
|
||||
tests_pass, test_secs = run_tests(spec)
|
||||
primary_mbps, bench_tail = run_bench(spec)
|
||||
|
||||
print("---")
|
||||
print(f"mbps: {primary_mbps:.1f}")
|
||||
print(f"tests_passed: {int(tests_pass)}")
|
||||
print(f"build_ok: {int(build_ok)}")
|
||||
print(f"install_seconds: {install_secs:.1f}")
|
||||
print(f"build_seconds: {build_secs:.1f}")
|
||||
print(f"test_seconds: {test_secs:.1f}")
|
||||
|
||||
if args.record:
|
||||
status = args.status
|
||||
if status is None:
|
||||
best = compute_best_so_far()
|
||||
threshold = best * THRESHOLD_MULT
|
||||
if not tests_pass:
|
||||
status = "discard"
|
||||
elif primary_mbps < threshold:
|
||||
status = "discard"
|
||||
else:
|
||||
status = "keep"
|
||||
print(f"best_so_far: {best:.1f}")
|
||||
print(f"threshold: {threshold:.1f} (best_so_far * {THRESHOLD_MULT})")
|
||||
print(f"auto-status: {status}")
|
||||
append_results_row(git_sha(), primary_mbps, tests_pass, build_ok,
|
||||
status, args.description)
|
||||
|
||||
return 0 if tests_pass else 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Reference in New Issue
Block a user