2026-11-04 17:07:06 +01:00
|
|
|
#!/usr/bin/env python3
|
|
|
|
|
"""Measure a command's CPU seconds, peak RAM and wall time on a base ref and on the working tree.
|
|
|
|
|
|
|
|
|
|
Args:
|
|
|
|
|
--base: git ref to compare with; checked out as a detached worktree under /tmp
|
|
|
|
|
and removed again, whatever happens.
|
|
|
|
|
--json: print the figures as JSON instead of the line.
|
|
|
|
|
-- <command...>: the command, run without a shell from the same directory
|
|
|
|
|
relative to each tree's root; its output is discarded.
|
|
|
|
|
|
|
|
|
|
Runs:
|
2026-11-13 12:32:54 +00:00
|
|
|
One run per tree, base first. When tokens and velocity are both at most 100 that
|
2026-11-04 17:07:06 +01:00
|
|
|
is the result; otherwise 4 more runs per tree, alternating, and the arithmetic mean
|
|
|
|
|
of all 5 per tree counts.
|
|
|
|
|
|
|
|
|
|
Other load:
|
|
|
|
|
A 1 s /proc/stat sample of system CPU busy before and after the runs, and per run
|
|
|
|
|
the busy jiffies minus the command's own CPU time, as a share of all jiffies. When
|
2026-11-13 12:32:54 +00:00
|
|
|
any share exceeds 30 % or cannot be read, the result is an estimate: velocity comes
|
2026-11-04 17:07:06 +01:00
|
|
|
from the CPU-time ratio instead of wall time.
|
|
|
|
|
|
|
|
|
|
Output:
|
2026-11-13 12:32:54 +00:00
|
|
|
`💸 tokens <p> % (cpu <b>→<a> s, ram <b>→<a> MB) · ⏱️ velocity <p> % (wall <b>→<a> s)`,
|
2026-11-04 17:07:06 +01:00
|
|
|
then ` · mean of 5` after the extra runs. An estimate marks both values with `~`,
|
2026-11-13 12:32:54 +00:00
|
|
|
shows `(cpu <b>→<a> s)` for velocity and ends in `(estimated: other load <x> %)`, or
|
|
|
|
|
`unknown` for <x>. Tokens is the mean of the CPU ratio (user plus system seconds
|
2026-11-04 17:07:06 +01:00
|
|
|
of the command and its waited-for children) and the peak RSS ratio; each ratio is
|
|
|
|
|
working tree over base, times 100.
|
|
|
|
|
|
|
|
|
|
Exit codes:
|
2026-11-13 12:32:54 +00:00
|
|
|
0 tokens and velocity at most 100, 2 usage error or a failing command
|
2026-11-04 17:07:06 +01:00
|
|
|
(its tree, exit code and last stderr line on stderr), 3 either over 100.
|
|
|
|
|
"""
|
|
|
|
|
|
|
|
|
|
from __future__ import annotations
|
|
|
|
|
|
|
|
|
|
import argparse
|
|
|
|
|
import json
|
|
|
|
|
import os
|
|
|
|
|
import shutil
|
|
|
|
|
import statistics
|
|
|
|
|
import subprocess
|
|
|
|
|
import sys
|
|
|
|
|
import tempfile
|
|
|
|
|
import time
|
|
|
|
|
from pathlib import Path
|
|
|
|
|
|
|
|
|
|
WORKTREE_PARENT = "/tmp"
|
|
|
|
|
LIMIT = 100
|
|
|
|
|
LOAD_LIMIT = 30
|
|
|
|
|
SAMPLE_SECONDS = 1
|
|
|
|
|
MORE_RUNS = 4
|
|
|
|
|
RESOLUTION = 0.001
|
|
|
|
|
PROC_STAT = Path("/proc/stat")
|
|
|
|
|
CLOCK_TICKS = os.sysconf("SC_CLK_TCK")
|
|
|
|
|
# Trip-wire: ru_maxrss counts kilobytes on Linux but bytes on macOS.
|
|
|
|
|
RSS_UNIT = 1 if sys.platform == "darwin" else 1024
|
|
|
|
|
USAGE = "usage: measure.py --base <ref> [--json] -- <command...>"
|
2026-11-13 12:32:54 +00:00
|
|
|
LABELS = Path(__file__).resolve().parents[2] / "kvb-check-score" / "labels.json"
|
|
|
|
|
LABEL_ERRORS = (OSError, ValueError, LookupError, TypeError)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def labelled(keys: tuple[str, ...]) -> dict[str, str]:
|
|
|
|
|
"""Args:
|
|
|
|
|
keys: metric keys of kvb-check-score's labels.json.
|
|
|
|
|
|
|
|
|
|
Returns:
|
|
|
|
|
Per key `<emoji> <label>`, the label in lower case; one of LABEL_ERRORS
|
|
|
|
|
for a missing or malformed file.
|
|
|
|
|
"""
|
|
|
|
|
metrics = json.loads(LABELS.read_text(encoding="utf-8"))["metrics"]
|
|
|
|
|
found = {key: (metrics[key]["emoji"], metrics[key]["label"]) for key in keys}
|
|
|
|
|
if not all(isinstance(part, str) for pair in found.values() for part in pair):
|
|
|
|
|
raise ValueError(f"needs a string emoji and label for {', '.join(keys)}")
|
|
|
|
|
return {key: f"{emoji} {label.lower()}" for key, (emoji, label) in found.items()}
|
2026-11-04 17:07:06 +01:00
|
|
|
|
|
|
|
|
|
|
|
|
|
def fail(message: str) -> None:
|
|
|
|
|
print(f"measure: {message}", file=sys.stderr)
|
|
|
|
|
sys.exit(2)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def git(cwd: Path, *args: str) -> str:
|
|
|
|
|
result = subprocess.run(["git", *args], cwd=cwd, capture_output=True, text=True, check=False)
|
|
|
|
|
if result.returncode != 0:
|
|
|
|
|
fail(result.stderr.strip() or f"git {args[0]} failed in {cwd}")
|
|
|
|
|
return result.stdout.strip()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def cpu_ticks() -> tuple[int, int] | None:
|
|
|
|
|
try:
|
|
|
|
|
fields = [int(v) for v in PROC_STAT.read_text(encoding="ascii").split("\n", 1)[0].split()[1:9]]
|
|
|
|
|
except (OSError, ValueError):
|
|
|
|
|
return None
|
|
|
|
|
if len(fields) < 8:
|
|
|
|
|
return None
|
|
|
|
|
return sum(fields) - fields[3] - fields[4], sum(fields)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def share(before: tuple[int, int] | None, after: tuple[int, int] | None, own: float) -> float | None:
|
|
|
|
|
if before is None or after is None:
|
|
|
|
|
return None
|
|
|
|
|
total = after[1] - before[1]
|
|
|
|
|
return max(0.0, (after[0] - before[0] - own) / total * 100) if total > 0 else 0.0
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def idle_load() -> float | None:
|
|
|
|
|
before = cpu_ticks()
|
|
|
|
|
time.sleep(SAMPLE_SECONDS)
|
|
|
|
|
return share(before, cpu_ticks(), 0.0)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def sample(command: list[str], cwd: Path, tree: str) -> dict[str, float | None]:
|
|
|
|
|
with tempfile.TemporaryFile() as err:
|
|
|
|
|
ticks = cpu_ticks()
|
|
|
|
|
started = time.perf_counter()
|
|
|
|
|
try:
|
|
|
|
|
child = subprocess.Popen(
|
|
|
|
|
command, cwd=cwd, stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL, stderr=err
|
|
|
|
|
)
|
|
|
|
|
except OSError as error:
|
|
|
|
|
fail(f"{tree} tree {cwd}: {error}")
|
|
|
|
|
# Trip-wire: Popen.wait() discards the child's rusage, so wait4 reaps it and returncode is set by hand.
|
|
|
|
|
_, status, usage = os.wait4(child.pid, 0)
|
|
|
|
|
wall = time.perf_counter() - started
|
|
|
|
|
cpu = usage.ru_utime + usage.ru_stime
|
|
|
|
|
load = share(ticks, cpu_ticks(), cpu * CLOCK_TICKS)
|
|
|
|
|
child.returncode = os.waitstatus_to_exitcode(status)
|
|
|
|
|
if child.returncode != 0:
|
|
|
|
|
err.seek(0)
|
|
|
|
|
lines = err.read().decode(errors="replace").strip().splitlines()
|
|
|
|
|
fail(f"command exited {child.returncode} in the {tree} tree {cwd}: {lines[-1] if lines else ''}")
|
|
|
|
|
return {"cpu": cpu, "ram": usage.ru_maxrss * RSS_UNIT / 2**20, "wall": wall, "load": load}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def ratio(after: float, before: float) -> float:
|
|
|
|
|
return max(after, RESOLUTION) / max(before, RESOLUTION)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def figures(runs: dict[str, list[dict]], loads: list[float | None]) -> dict:
|
|
|
|
|
before, after = (
|
|
|
|
|
{key: statistics.fmean(s[key] for s in runs[tree]) for key in ("cpu", "ram", "wall")}
|
|
|
|
|
for tree in ("base", "head")
|
|
|
|
|
)
|
|
|
|
|
unknown = None in loads
|
|
|
|
|
estimated = unknown or max(loads) > LOAD_LIMIT
|
|
|
|
|
basis = "cpu" if estimated else "wall"
|
|
|
|
|
return {
|
|
|
|
|
"efficiency": round((ratio(after["cpu"], before["cpu"]) + ratio(after["ram"], before["ram"])) / 2 * 100),
|
|
|
|
|
"speed": round(ratio(after[basis], before[basis]) * 100),
|
|
|
|
|
"runs": len(runs["head"]),
|
|
|
|
|
"estimated": estimated,
|
|
|
|
|
"other_load": None if unknown else round(max(loads)),
|
|
|
|
|
"base": before,
|
|
|
|
|
"head": after,
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
2026-11-13 12:32:54 +00:00
|
|
|
def line(found: dict, names: dict[str, str]) -> str:
|
2026-11-04 17:07:06 +01:00
|
|
|
mark = "~" if found["estimated"] else ""
|
|
|
|
|
basis = "cpu" if found["estimated"] else "wall"
|
|
|
|
|
before, after = found["base"], found["head"]
|
|
|
|
|
text = (
|
2026-11-13 12:32:54 +00:00
|
|
|
f"{names['tokens']} {mark}{found['efficiency']} % (cpu {before['cpu']:.2f}→{after['cpu']:.2f} s, "
|
2026-11-04 17:07:06 +01:00
|
|
|
f"ram {before['ram']:.1f}→{after['ram']:.1f} MB) · "
|
2026-11-13 12:32:54 +00:00
|
|
|
f"{names['velocity']} {mark}{found['speed']} % ({basis} {before[basis]:.2f}→{after[basis]:.2f} s)"
|
2026-11-04 17:07:06 +01:00
|
|
|
)
|
|
|
|
|
if found["runs"] > 1:
|
|
|
|
|
text += f" · mean of {found['runs']}"
|
|
|
|
|
if found["estimated"]:
|
|
|
|
|
other = "unknown" if found["other_load"] is None else f"{found['other_load']} %"
|
|
|
|
|
text += f" (estimated: other load {other})"
|
|
|
|
|
return text
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def run_loads(runs: dict[str, list[dict]]) -> list[float | None]:
|
|
|
|
|
return [s["load"] for s in runs["base"] + runs["head"]]
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def costly(found: dict) -> bool:
|
|
|
|
|
return found["efficiency"] > LIMIT or found["speed"] > LIMIT
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def main(argv: list[str]) -> int:
|
|
|
|
|
if "--" not in argv:
|
|
|
|
|
fail(USAGE)
|
|
|
|
|
split = argv.index("--")
|
|
|
|
|
parser = argparse.ArgumentParser(description=__doc__.splitlines()[0], usage=USAGE)
|
|
|
|
|
parser.add_argument("--base", required=True)
|
|
|
|
|
parser.add_argument("--json", action="store_true")
|
|
|
|
|
args = parser.parse_args(argv[:split])
|
|
|
|
|
command = argv[split + 1:]
|
|
|
|
|
if not command:
|
|
|
|
|
fail("no command after --")
|
2026-11-13 12:32:54 +00:00
|
|
|
try:
|
|
|
|
|
names = {} if args.json else labelled(("tokens", "velocity"))
|
|
|
|
|
except LABEL_ERRORS as error:
|
|
|
|
|
fail(f"labels {LABELS}: {error!r}")
|
2026-11-04 17:07:06 +01:00
|
|
|
|
|
|
|
|
here = Path.cwd().resolve()
|
|
|
|
|
root = Path(git(here, "rev-parse", "--show-toplevel")).resolve()
|
|
|
|
|
relative = here.relative_to(root)
|
|
|
|
|
resolved = subprocess.run(
|
|
|
|
|
["git", "rev-parse", "--verify", "--quiet", f"{args.base}^{{commit}}"],
|
|
|
|
|
cwd=root, capture_output=True, text=True, check=False,
|
|
|
|
|
)
|
|
|
|
|
if resolved.returncode != 0:
|
|
|
|
|
fail(f"unknown ref {args.base!r}")
|
|
|
|
|
|
|
|
|
|
parent = Path(tempfile.mkdtemp(prefix="kvb-measure-", dir=WORKTREE_PARENT))
|
|
|
|
|
tree = parent / "base"
|
|
|
|
|
runs: dict[str, list[dict]] = {"base": [], "head": []}
|
|
|
|
|
try:
|
|
|
|
|
git(root, "worktree", "add", "--detach", str(tree), resolved.stdout.strip())
|
|
|
|
|
base_cwd = tree / relative
|
|
|
|
|
if not base_cwd.is_dir():
|
|
|
|
|
fail(f"{relative} does not exist at {args.base}")
|
|
|
|
|
loads = [idle_load()]
|
|
|
|
|
for count in range(1 + MORE_RUNS):
|
|
|
|
|
runs["base"].append(sample(command, base_cwd, "base"))
|
|
|
|
|
runs["head"].append(sample(command, here, "head"))
|
|
|
|
|
if count == 0 and not costly(figures(runs, loads + run_loads(runs))):
|
|
|
|
|
break
|
|
|
|
|
loads.append(idle_load())
|
|
|
|
|
finally:
|
|
|
|
|
subprocess.run(["git", "worktree", "remove", "--force", str(tree)], cwd=root, capture_output=True, check=False)
|
|
|
|
|
shutil.rmtree(parent, ignore_errors=True)
|
|
|
|
|
|
|
|
|
|
found = figures(runs, loads + run_loads(runs))
|
2026-11-13 12:32:54 +00:00
|
|
|
print(json.dumps(found) if args.json else line(found, names))
|
2026-11-04 17:07:06 +01:00
|
|
|
return 3 if costly(found) else 0
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
if __name__ == "__main__":
|
|
|
|
|
sys.exit(main(sys.argv[1:]))
|