Files
skills/skills/kvb-check-efficiency/scripts/tokens.py
T
Kevin Veen-BirkenbachandClaude Opus 5.5 5b55c39b64 feat(kvb-check-score): score journal, backfill and per-metric burndown overlays (006)
score.py names the Cost and Time metrics Tokens and Velocity from one label file and appends every scored block to a score journal in the council root. backfill.py rebuilds that journal from commit messages with its own label mapping, deterministic and idempotent, so another host can import its history. The hub serves the journal, and the Gantt burndown draws each metric as its own overlay on a right 0-100 % axis, every overlay off by default and switched by legend chips whose state the view store keeps.

🎯 Fix: 90 %
💥 Break: 5 %
💸 Cost: 100 %
⏱️ Time: 100 %
🛡️ Security: 100 %
🏆 Score: 86
🚦 Gate: ✅ 90/5, cost and time <= 100, security 100 %

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-13 12:32:54 +00:00

405 lines
17 KiB
Python
Executable File

#!/usr/bin/env python3
"""Compare the estimated tokens of changed skill text files, and the listing words, with a base ref, and weigh their daily cost against their value.
Args:
--repo: skills repository root; its scripts/budget.py counts the listing.
--base: git ref the change starts from; the working tree is the after side.
--transcripts: Claude Code transcripts root, e.g. ~/.claude/projects; every */*.jsonl of the
last 7 days gives the requests (unique assistant message ids) and the invocations per skill
(unique Skill tool_use ids, a `<scope>:` prefix stripped). Read only; no text is printed.
--value: tokens per day the change saves; needs --evidence.
--evidence: one line that derives --value.
--json: print the figures as JSON instead of the lines.
value --repo <root> --since <commit> --transcripts <root> [--json]: measure the value of a
shipped change instead.
Output:
Line 1: `💸 tokens <p> % (skills <b>→<a> tokens; listing <b>→<a> words)`, p = after / before * 100,
or `💸 tokens new (+<a> tokens; listing <b>→<a> words)` when the changed files hold no
base tokens. Changed files are the *.md under skills/ that `git diff <ref>` names plus untracked
ones; a file missing on a side counts 0. Tokens are an estimate: characters / 4. The listing
is the sum of every skills/*/SKILL.md description word.
Line 2: `💰 value 1:<n> (cost <c>/day, value <v>/day; <evidence>)` when the value is at least
the cost, `<n>:1` below it, `0:1` without a cost, `1:0` without a value; without --value
`💰 value ?:? (cost <c>/day; pass --value with evidence)`. Cost per day = listing words delta
* 1.33 tokens * requests per day * 0.1 (cached) + per changed SKILL.md its tokens delta * the
skill's invocations per day. A skill without a base SKILL.md takes the median invocations of
its category's base skills (the name up to its second dash; all base skills when it has
none), and the line names it.
--json prints efficiency (null when new), the skills and listing totals, the tokens per skill
and the value figures.
value: `💰 measured value <v>/day (median daily context <b>→<a> tokens over <n> active days
before and <m> after <sha> of <day>)`: the context tokens per UTC day as context.py of
kvb-skill-silence sums them, days without a turn left out, over the 7 days before the
commit's day and the up to 7 full days after it; v = b - a.
Exit codes:
0 efficiency at most 100, 2 usage error, 3 over 100 or new; the value line never changes it.
value: 0 measured, 2 usage error or no active day on a side.
"""
from __future__ import annotations
import argparse
import datetime
import importlib.util
import json
import math
import statistics
import subprocess
import sys
from pathlib import Path, PurePosixPath
from types import ModuleType
CHARS_PER_TOKEN = 4
LIMIT = 100
SKILLS = "skills"
TOKENS_PER_WORD = 1.33
CACHE_FACTOR = 0.1
DAYS = 7
ROUND_ABOVE = 10
ALL_SKILLS = "all skills"
CONTEXT = Path(__file__).resolve().parents[2] / "kvb-skill-silence" / "scripts" / "context.py"
LABELS = Path(__file__).resolve().parents[2] / "kvb-check-score" / "labels.json"
LABEL_ERRORS = (OSError, ValueError, LookupError, TypeError)
def labelled(keys: tuple[str, ...]) -> dict[str, str]:
"""Args:
keys: metric keys of kvb-check-score's labels.json.
Returns:
Per key `<emoji> <label>`, the label in lower case; one of LABEL_ERRORS
for a missing or malformed file.
"""
metrics = json.loads(LABELS.read_text(encoding="utf-8"))["metrics"]
found = {key: (metrics[key]["emoji"], metrics[key]["label"]) for key in keys}
if not all(isinstance(part, str) for pair in found.values() for part in pair):
raise ValueError(f"needs a string emoji and label for {', '.join(keys)}")
return {key: f"{emoji} {label.lower()}" for key, (emoji, label) in found.items()}
def headline(name: str, efficiency: int | None, before: int, after: int, listing: str) -> str:
"""Args:
name: the tokens metric as labelled() gives it.
efficiency: after over before in percent, None without base tokens.
before: the base tokens of the changed files.
after: their tokens in the working tree.
listing: the listing words part.
Returns:
Line 1.
"""
if efficiency is None:
return f"{name} new (+{after} tokens; {listing})"
return f"{name} {efficiency} % (skills {before}→{after} tokens; {listing})"
def fail(message: str) -> None:
print(f"tokens: {message}", file=sys.stderr)
sys.exit(2)
def git(repo: Path, *args: str) -> subprocess.CompletedProcess:
return subprocess.run(["git", *args], cwd=repo, capture_output=True, check=False)
def paths(repo: Path, *args: str) -> list[str]:
result = git(repo, *args)
if result.returncode != 0:
fail(result.stderr.decode(errors="replace").strip())
return [p for p in result.stdout.decode().split("\0") if p]
def load(path: Path, name: str, role: str) -> ModuleType:
if not path.is_file():
fail(f"{path} is missing; it {role}")
spec = importlib.util.spec_from_file_location(name, path)
loaded = importlib.util.module_from_spec(spec)
spec.loader.exec_module(loaded)
return loaded
def at_base(repo: Path, ref: str, path: str) -> str:
result = git(repo, "show", f"{ref}:{path}")
return result.stdout.decode(errors="replace") if result.returncode == 0 else ""
def in_tree(repo: Path, path: str) -> str:
file = repo / path
return file.read_text(encoding="utf-8", errors="replace") if file.is_file() else ""
def tokens(chars: int) -> int:
return round(chars / CHARS_PER_TOKEN)
def is_skill_md(path: str) -> bool:
parts = PurePosixPath(path).parts
return len(parts) == 3 and parts[0] == SKILLS and parts[2] == "SKILL.md"
def category(skill: str) -> str:
return "-".join(skill.split("-")[:2])
def figure(amount: float) -> str:
if abs(amount) >= ROUND_ABOVE:
return str(round(amount))
return f"{round(amount, 1) + 0.0:g}"
def quotient(cost: float, value: float) -> str:
if cost <= 0:
return "0:1" if value > 0 else "0:0"
if value <= 0:
return "1:0"
if value >= cost:
return f"1:{figure(value / cost)}"
return f"{figure(cost / value)}:1"
def amount(text: str) -> float:
try:
value = float(text)
except ValueError:
raise argparse.ArgumentTypeError(f"{text!r} is not a number") from None
if not math.isfinite(value):
raise argparse.ArgumentTypeError(f"{text!r} is not finite")
return value
def directory(text: str) -> Path:
root = Path(text).expanduser()
if not root.is_dir():
fail(f"{root} is not a directory")
return root
def skill_uses(message: dict) -> list[tuple[str, str]]:
content = message.get("content")
found = []
for block in content if isinstance(content, list) else []:
if not isinstance(block, dict) or block.get("type") != "tool_use" or block.get("name") != "Skill":
continue
given = block.get("input")
skill = given.get("skill") if isinstance(given, dict) else None
if isinstance(skill, str) and block.get("id"):
found.append((skill.rpartition(":")[2], str(block["id"])))
return found
def usage(root: Path, since: float) -> tuple[int, dict[str, int]]:
requests: set[str] = set()
invocations: dict[str, set[str]] = {}
for path in root.glob("*/*.jsonl"):
if path.stat().st_mtime < since:
continue
with path.open(encoding="utf-8", errors="ignore") as handle:
for line in handle:
if '"assistant"' not in line:
continue
try:
entry = json.loads(line)
stamp = datetime.datetime.fromisoformat(entry["timestamp"]).timestamp()
except (ValueError, KeyError, TypeError):
continue
message = entry.get("message")
if entry.get("type") != "assistant" or stamp < since or not isinstance(message, dict):
continue
if message.get("id"):
requests.add(str(message["id"]))
for skill, use in skill_uses(message):
invocations.setdefault(skill, set()).add(use)
return len(requests), {skill: len(uses) for skill, uses in invocations.items()}
def weigh(root: Path, listing_delta: int, body: dict[str, int], new: set[str], base_skills: list[str]) -> dict:
requests, invocations = usage(root, datetime.datetime.now(datetime.UTC).timestamp() - DAYS * 86400)
per_day = {skill: count / DAYS for skill, count in invocations.items()}
rates, assumed = {}, {}
for skill in body:
if skill in new:
peers = [peer for peer in base_skills if category(peer) == category(skill)]
assumed[skill] = category(skill) if peers else ALL_SKILLS
pool = [per_day.get(peer, 0.0) for peer in peers or base_skills]
rates[skill] = statistics.median(pool) if pool else 0.0
else:
rates[skill] = per_day.get(skill, 0.0)
listing = listing_delta * TOKENS_PER_WORD * (requests / DAYS) * CACHE_FACTOR
return {
"cost": listing + sum(delta * rates[skill] for skill, delta in body.items()),
"requests_per_day": requests / DAYS,
"invocations_per_day": rates,
"median_for_new": assumed,
}
def value_line(weighed: dict, value: float | None, evidence: str | None) -> tuple[str, str]:
notes = [
f"{skill} new: {group} median {figure(weighed['invocations_per_day'][skill])} invocations/day"
for skill, group in sorted(weighed["median_for_new"].items())
]
cost = weighed["cost"]
if value is None:
return "?:?", f"💰 value ?:? ({'; '.join([f'cost {figure(cost)}/day', 'pass --value with evidence', *notes])})"
ratio = quotient(cost, value)
parts = [f"cost {figure(cost)}/day, value {figure(value)}/day", evidence, *notes]
return ratio, f"💰 value {ratio} ({'; '.join(parts)})"
def parse(argv: list[str]) -> argparse.Namespace:
parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
parser.add_argument("--repo", required=True)
parser.add_argument("--base", required=True)
parser.add_argument("--transcripts", required=True)
parser.add_argument("--value", type=amount)
parser.add_argument("--evidence")
parser.add_argument("--json", action="store_true")
args = parser.parse_args(argv)
if (args.value is None) != (args.evidence is None):
parser.error("--value and --evidence go together")
if args.evidence is not None and (not args.evidence.strip() or "\n" in args.evidence):
parser.error("--evidence must be one non-empty line")
return args
def measure(
repo: Path, base: str, base_files: set[str], changed: list[str]
) -> tuple[dict[str, list[int]], dict[str, int], set[str]]:
chars: dict[str, list[int]] = {}
body: dict[str, int] = {}
new: set[str] = set()
for path in sorted({p for p in changed if p.endswith(".md")}):
skill = PurePosixPath(path).parts[1]
old = at_base(repo, base, path) if path in base_files else ""
current = in_tree(repo, path)
side = chars.setdefault(skill, [0, 0])
side[0] += len(old)
side[1] += len(current)
if is_skill_md(path):
body[skill] = tokens(len(current)) - tokens(len(old))
if path not in base_files:
new.add(skill)
return chars, body, new
def check(argv: list[str]) -> int:
args = parse(argv)
try:
names = labelled(("tokens",))
except LABEL_ERRORS as error:
fail(f"labels {LABELS}: {error!r}")
repo = Path(args.repo).resolve()
if not (repo / SKILLS).is_dir():
fail(f"{repo} has no {SKILLS}/ directory")
if git(repo, "rev-parse", "--verify", "--quiet", f"{args.base}^{{commit}}").returncode != 0:
fail(f"unknown ref {args.base!r}")
root = directory(args.transcripts)
budget = load(repo / "scripts" / "budget.py", "budget", "counts the listing")
base_files = set(paths(repo, "ls-tree", "-r", "-z", "--name-only", args.base, "--", SKILLS))
# Trip-wire: with rename detection a moved file lists only its new path and drops the old one's tokens.
changed = paths(repo, "diff", "--no-renames", "-z", "--name-only", args.base, "--", SKILLS)
changed += paths(repo, "ls-files", "-z", "--others", "--exclude-standard", "--", SKILLS)
chars, body, new = measure(repo, args.base, base_files, changed)
before = tokens(sum(c[0] for c in chars.values()))
after = tokens(sum(c[1] for c in chars.values()))
base_skills = sorted(PurePosixPath(p).parts[1] for p in base_files if is_skill_md(p))
listing_before = sum(budget.words(at_base(repo, args.base, f"{SKILLS}/{s}/SKILL.md")) for s in base_skills)
listing_after = sum(budget.counts(repo, index=False).values())
weighed = weigh(root, listing_after - listing_before, body, new, base_skills)
ratio, money = value_line(weighed, args.value, args.evidence)
if before:
efficiency = round(after / before * 100)
else:
efficiency = None if after else LIMIT
if args.json:
print(json.dumps({
"efficiency": efficiency,
"skills": {"before": before, "after": after},
"listing": {"before": listing_before, "after": listing_after},
"per_skill": {name: {"before": tokens(b), "after": tokens(a)} for name, (b, a) in chars.items()},
"value": {
"quotient": ratio,
"cost": round(weighed["cost"], 2),
"value": args.value,
"evidence": args.evidence,
"requests_per_day": round(weighed["requests_per_day"], 2),
"invocations_per_day": {s: round(r, 2) for s, r in weighed["invocations_per_day"].items()},
"median_for_new": weighed["median_for_new"],
},
}))
else:
print(headline(names["tokens"], efficiency, before, after, f"listing {listing_before}→{listing_after} words"))
print(money)
return 3 if efficiency is None or efficiency > LIMIT else 0
def measured(argv: list[str]) -> int:
parser = argparse.ArgumentParser(
prog="tokens.py value", description="Measure the value of a shipped change from the daily context tokens."
)
parser.add_argument("--repo", required=True)
parser.add_argument("--since", required=True)
parser.add_argument("--transcripts", required=True)
parser.add_argument("--json", action="store_true")
args = parser.parse_args(argv)
repo = Path(args.repo).resolve()
if not repo.is_dir():
fail(f"{repo} is not a directory")
result = git(repo, "show", "-s", "--format=%ct %h", f"{args.since}^{{commit}}")
if result.returncode != 0:
fail(f"unknown commit {args.since!r}")
stamp, sha = result.stdout.decode().split()
root = directory(args.transcripts)
context = load(CONTEXT, "context", "sums the daily context tokens")
day = datetime.datetime.fromtimestamp(int(stamp), datetime.UTC).date()
today = datetime.datetime.now(datetime.UTC).date()
windows = (
[day - datetime.timedelta(days=n) for n in range(DAYS, 0, -1)],
[d for d in (day + datetime.timedelta(days=n) for n in range(1, DAYS + 1)) if d < today],
)
start = datetime.datetime.combine(windows[0][0], datetime.time.min, tzinfo=datetime.UTC).timestamp()
turns, _ = context.scan(root, start)
totals: dict[str, int] = {}
for turn_day, size in turns.values():
totals[turn_day] = totals.get(turn_day, 0) + size
sides = [[totals[d.isoformat()] for d in window if totals.get(d.isoformat())] for window in windows]
for name, side in zip(("before", "after"), sides, strict=True):
if not side:
fail(f"no active day in the {DAYS} days {name} {sha} of {day}")
median_before, median_after = (statistics.median(side) for side in sides)
value = median_before - median_after
if args.json:
print(json.dumps({
"since": sha,
"day": day.isoformat(),
"before": {"days": len(sides[0]), "median": median_before},
"after": {"days": len(sides[1]), "median": median_after},
"value": value,
}))
else:
print(
f"💰 measured value {figure(value)}/day (median daily context {figure(median_before)}→"
f"{figure(median_after)} tokens over {len(sides[0])} active days before and {len(sides[1])} after "
f"{sha} of {day})"
)
return 0
def main(argv: list[str]) -> int:
if argv[:1] == ["value"]:
return measured(argv[1:])
return check(argv)
if __name__ == "__main__":
sys.exit(main(sys.argv[1:]))