feat(check): score fix, break, efficiency and speed in one block
🧪 Test / 🧪 Lock + lint (push) Canceled after 0s
🧪 Test / 🧪 Lock + lint (push) Canceled after 0s
kvb-check-efficiency measures a change against its base: tokens.py counts skill text tokens and listing words, measure.py runs one command in a base worktree and the working tree (one run per side, the mean of five over 100, an estimate marked ~ when other processes load the CPU). kvb-check-speed reads wall time from the same run and is not relevant for skills. kvb-check-score prints fix, break, efficiency, speed and the product score with emojis plus a gate line: below 90/5 it reports, over 100 it requires asking the operator before the commit. Every skill that reported confidence and break risk now shows the score block; the 90/5 bar adds efficiency and speed at most 100. Commit blocks and the commit call's approval prompt name worktree, branch, diff stat and the score values, also for merges. 24 descriptions were shortened to keep the skill listing within its budget. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
6fa0311704
commit
0343a2f3ce
46 files changed
+1145
-118
No files matched your search
+214
@@ -0,0 +1,214 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Measure a command's CPU seconds, peak RAM and wall time on a base ref and on the working tree.
|
||||
|
||||
Args:
|
||||
--base: git ref to compare with; checked out as a detached worktree under /tmp
|
||||
and removed again, whatever happens.
|
||||
--json: print the figures as JSON instead of the line.
|
||||
-- <command...>: the command, run without a shell from the same directory
|
||||
relative to each tree's root; its output is discarded.
|
||||
|
||||
Runs:
|
||||
One run per tree, base first. When efficiency and speed are both at most 100 that
|
||||
is the result; otherwise 4 more runs per tree, alternating, and the arithmetic mean
|
||||
of all 5 per tree counts.
|
||||
|
||||
Other load:
|
||||
A 1 s /proc/stat sample of system CPU busy before and after the runs, and per run
|
||||
the busy jiffies minus the command's own CPU time, as a share of all jiffies. When
|
||||
any share exceeds 30 % or cannot be read, the result is an estimate: speed comes
|
||||
from the CPU-time ratio instead of wall time.
|
||||
|
||||
Output:
|
||||
`🌱 efficiency <p> % (cpu <b>→<a> s, ram <b>→<a> MB) · ⚡ speed <p> % (wall <b>→<a> s)`,
|
||||
then ` · mean of 5` after the extra runs. An estimate marks both values with `~`,
|
||||
shows `(cpu <b>→<a> s)` for speed and ends in `(estimated: other load <x> %)`, or
|
||||
`unknown` for <x>. Efficiency is the mean of the CPU ratio (user plus system seconds
|
||||
of the command and its waited-for children) and the peak RSS ratio; each ratio is
|
||||
working tree over base, times 100.
|
||||
|
||||
Exit codes:
|
||||
0 efficiency and speed at most 100, 2 usage error or a failing command
|
||||
(its tree, exit code and last stderr line on stderr), 3 either over 100.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import statistics
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
WORKTREE_PARENT = "/tmp"
|
||||
LIMIT = 100
|
||||
LOAD_LIMIT = 30
|
||||
SAMPLE_SECONDS = 1
|
||||
MORE_RUNS = 4
|
||||
RESOLUTION = 0.001
|
||||
PROC_STAT = Path("/proc/stat")
|
||||
CLOCK_TICKS = os.sysconf("SC_CLK_TCK")
|
||||
# Trip-wire: ru_maxrss counts kilobytes on Linux but bytes on macOS.
|
||||
RSS_UNIT = 1 if sys.platform == "darwin" else 1024
|
||||
USAGE = "usage: measure.py --base <ref> [--json] -- <command...>"
|
||||
|
||||
|
||||
def fail(message: str) -> None:
|
||||
print(f"measure: {message}", file=sys.stderr)
|
||||
sys.exit(2)
|
||||
|
||||
|
||||
def git(cwd: Path, *args: str) -> str:
|
||||
result = subprocess.run(["git", *args], cwd=cwd, capture_output=True, text=True, check=False)
|
||||
if result.returncode != 0:
|
||||
fail(result.stderr.strip() or f"git {args[0]} failed in {cwd}")
|
||||
return result.stdout.strip()
|
||||
|
||||
|
||||
def cpu_ticks() -> tuple[int, int] | None:
|
||||
try:
|
||||
fields = [int(v) for v in PROC_STAT.read_text(encoding="ascii").split("\n", 1)[0].split()[1:9]]
|
||||
except (OSError, ValueError):
|
||||
return None
|
||||
if len(fields) < 8:
|
||||
return None
|
||||
return sum(fields) - fields[3] - fields[4], sum(fields)
|
||||
|
||||
|
||||
def share(before: tuple[int, int] | None, after: tuple[int, int] | None, own: float) -> float | None:
|
||||
if before is None or after is None:
|
||||
return None
|
||||
total = after[1] - before[1]
|
||||
return max(0.0, (after[0] - before[0] - own) / total * 100) if total > 0 else 0.0
|
||||
|
||||
|
||||
def idle_load() -> float | None:
|
||||
before = cpu_ticks()
|
||||
time.sleep(SAMPLE_SECONDS)
|
||||
return share(before, cpu_ticks(), 0.0)
|
||||
|
||||
|
||||
def sample(command: list[str], cwd: Path, tree: str) -> dict[str, float | None]:
|
||||
with tempfile.TemporaryFile() as err:
|
||||
ticks = cpu_ticks()
|
||||
started = time.perf_counter()
|
||||
try:
|
||||
child = subprocess.Popen(
|
||||
command, cwd=cwd, stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL, stderr=err
|
||||
)
|
||||
except OSError as error:
|
||||
fail(f"{tree} tree {cwd}: {error}")
|
||||
# Trip-wire: Popen.wait() discards the child's rusage, so wait4 reaps it and returncode is set by hand.
|
||||
_, status, usage = os.wait4(child.pid, 0)
|
||||
wall = time.perf_counter() - started
|
||||
cpu = usage.ru_utime + usage.ru_stime
|
||||
load = share(ticks, cpu_ticks(), cpu * CLOCK_TICKS)
|
||||
child.returncode = os.waitstatus_to_exitcode(status)
|
||||
if child.returncode != 0:
|
||||
err.seek(0)
|
||||
lines = err.read().decode(errors="replace").strip().splitlines()
|
||||
fail(f"command exited {child.returncode} in the {tree} tree {cwd}: {lines[-1] if lines else ''}")
|
||||
return {"cpu": cpu, "ram": usage.ru_maxrss * RSS_UNIT / 2**20, "wall": wall, "load": load}
|
||||
|
||||
|
||||
def ratio(after: float, before: float) -> float:
|
||||
return max(after, RESOLUTION) / max(before, RESOLUTION)
|
||||
|
||||
|
||||
def figures(runs: dict[str, list[dict]], loads: list[float | None]) -> dict:
|
||||
before, after = (
|
||||
{key: statistics.fmean(s[key] for s in runs[tree]) for key in ("cpu", "ram", "wall")}
|
||||
for tree in ("base", "head")
|
||||
)
|
||||
unknown = None in loads
|
||||
estimated = unknown or max(loads) > LOAD_LIMIT
|
||||
basis = "cpu" if estimated else "wall"
|
||||
return {
|
||||
"efficiency": round((ratio(after["cpu"], before["cpu"]) + ratio(after["ram"], before["ram"])) / 2 * 100),
|
||||
"speed": round(ratio(after[basis], before[basis]) * 100),
|
||||
"runs": len(runs["head"]),
|
||||
"estimated": estimated,
|
||||
"other_load": None if unknown else round(max(loads)),
|
||||
"base": before,
|
||||
"head": after,
|
||||
}
|
||||
|
||||
|
||||
def line(found: dict) -> str:
|
||||
mark = "~" if found["estimated"] else ""
|
||||
basis = "cpu" if found["estimated"] else "wall"
|
||||
before, after = found["base"], found["head"]
|
||||
text = (
|
||||
f"🌱 efficiency {mark}{found['efficiency']} % (cpu {before['cpu']:.2f}→{after['cpu']:.2f} s, "
|
||||
f"ram {before['ram']:.1f}→{after['ram']:.1f} MB) · "
|
||||
f"⚡ speed {mark}{found['speed']} % ({basis} {before[basis]:.2f}→{after[basis]:.2f} s)"
|
||||
)
|
||||
if found["runs"] > 1:
|
||||
text += f" · mean of {found['runs']}"
|
||||
if found["estimated"]:
|
||||
other = "unknown" if found["other_load"] is None else f"{found['other_load']} %"
|
||||
text += f" (estimated: other load {other})"
|
||||
return text
|
||||
|
||||
|
||||
def run_loads(runs: dict[str, list[dict]]) -> list[float | None]:
|
||||
return [s["load"] for s in runs["base"] + runs["head"]]
|
||||
|
||||
|
||||
def costly(found: dict) -> bool:
|
||||
return found["efficiency"] > LIMIT or found["speed"] > LIMIT
|
||||
|
||||
|
||||
def main(argv: list[str]) -> int:
|
||||
if "--" not in argv:
|
||||
fail(USAGE)
|
||||
split = argv.index("--")
|
||||
parser = argparse.ArgumentParser(description=__doc__.splitlines()[0], usage=USAGE)
|
||||
parser.add_argument("--base", required=True)
|
||||
parser.add_argument("--json", action="store_true")
|
||||
args = parser.parse_args(argv[:split])
|
||||
command = argv[split + 1:]
|
||||
if not command:
|
||||
fail("no command after --")
|
||||
|
||||
here = Path.cwd().resolve()
|
||||
root = Path(git(here, "rev-parse", "--show-toplevel")).resolve()
|
||||
relative = here.relative_to(root)
|
||||
resolved = subprocess.run(
|
||||
["git", "rev-parse", "--verify", "--quiet", f"{args.base}^{{commit}}"],
|
||||
cwd=root, capture_output=True, text=True, check=False,
|
||||
)
|
||||
if resolved.returncode != 0:
|
||||
fail(f"unknown ref {args.base!r}")
|
||||
|
||||
parent = Path(tempfile.mkdtemp(prefix="kvb-measure-", dir=WORKTREE_PARENT))
|
||||
tree = parent / "base"
|
||||
runs: dict[str, list[dict]] = {"base": [], "head": []}
|
||||
try:
|
||||
git(root, "worktree", "add", "--detach", str(tree), resolved.stdout.strip())
|
||||
base_cwd = tree / relative
|
||||
if not base_cwd.is_dir():
|
||||
fail(f"{relative} does not exist at {args.base}")
|
||||
loads = [idle_load()]
|
||||
for count in range(1 + MORE_RUNS):
|
||||
runs["base"].append(sample(command, base_cwd, "base"))
|
||||
runs["head"].append(sample(command, here, "head"))
|
||||
if count == 0 and not costly(figures(runs, loads + run_loads(runs))):
|
||||
break
|
||||
loads.append(idle_load())
|
||||
finally:
|
||||
subprocess.run(["git", "worktree", "remove", "--force", str(tree)], cwd=root, capture_output=True, check=False)
|
||||
shutil.rmtree(parent, ignore_errors=True)
|
||||
|
||||
found = figures(runs, loads + run_loads(runs))
|
||||
print(json.dumps(found) if args.json else line(found))
|
||||
return 3 if costly(found) else 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main(sys.argv[1:]))
|
||||
Reference in new issue
Block a user