feat(kvb-check-score): score journal, backfill and per-metric burndown overlays (006)

score.py names the Cost and Time metrics Tokens and Velocity from one label file and appends every scored block to a score journal in the council root. backfill.py rebuilds that journal from commit messages with its own label mapping, deterministic and idempotent, so another host can import its history. The hub serves the journal, and the Gantt burndown draws each metric as its own overlay on a right 0-100 % axis, every overlay off by default and switched by legend chips whose state the view store keeps.

🎯 Fix: 90 %
💥 Break: 5 %
💸 Cost: 100 %
⏱️ Time: 100 %
🛡️ Security: 100 %
🏆 Score: 86
🚦 Gate: ✅ 90/5, cost and time <= 100, security 100 %

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Kevin Veen-BirkenbachandClaude Opus 5.5 committed 2026-11-13 12:32:54 +00:00
1 parent 3418738993
commit 5b55c39b64
30 files changed
+1806 -142

No files matched your search

+3
View File
@@ -11,6 +11,7 @@ from .diffs import DIFF_BODY_LIMIT
from .engines import STT_LIMIT
from .exchange import PLAIN_CSP, Exchange
from .resources import RESOURCES, sampled
from .scores import scored
BODY_LIMIT = 1 << 20
UPLOAD_LIMIT = 50 << 20
@@ -118,6 +119,8 @@ class Handler(Exchange):
self.answer(hub.repositories, HTTPStatus.OK)
elif path == "/api/resources":
self.answer(lambda: sampled(hub.kept / RESOURCES, hub.span(self.query(), time.time())), HTTPStatus.OK)
elif path == "/api/scores":
self.answer(lambda: scored(hub.council, hub.scripts.skills, hub.span(self.query(), time.time())), HTTPStatus.OK)
elif path == "/api/subagents":
self.answer(lambda: hub.subagents(self.query()), HTTPStatus.OK)
elif path == "/api/runner/jobs":
+44
View File
@@ -0,0 +1,44 @@
from __future__ import annotations
from pathlib import Path
from .records import epoch, parse_line, read_json, read_regular
SCORES = "scores.jsonl"
LABELS = "kvb-check-score/labels.json"
FILE_LIMIT = 64 << 20
def checked(labels: object) -> dict | None:
"""Args:
labels: the parsed label file, or None.
Returns:
labels when its "metrics" is a non-empty object of entries with a string
emoji and label, else None.
"""
metrics = labels.get("metrics") if isinstance(labels, dict) else None
valid = isinstance(metrics, dict) and metrics and all(isinstance(metric, dict) and isinstance(metric.get("emoji"), str) and isinstance(metric.get("label"), str) for metric in metrics.values())
return labels if valid else None
def scored(council: Path, skills: Path, span: tuple[float, float]) -> dict:
"""Args:
council: the council directory holding scores.jsonl, which score.py
and backfill.py of kvb-check-score append to.
skills: the skills directory holding kvb-check-score/labels.json.
span: (begin, end) of the burndown's time range in epoch seconds.
Returns:
{"since", "until", "labels", "rows"}: the label file, None when
unreadable or not as checked() wants it, and the journal rows whose time lies within the span, in
time order.
"""
begin, end = span
try:
lines = read_regular(council / SCORES, FILE_LIMIT).splitlines()
except (OSError, ValueError):
lines = []
rows = [row for row in map(parse_line, lines) if row is not None and isinstance(row.get("values"), dict)]
inside = [row for row in rows if (moment := epoch(row.get("time"))) is not None and begin <= moment <= end]
return {"since": begin, "until": end, "labels": checked(read_json(skills / LABELS)), "rows": sorted(inside, key=lambda row: row["time"])}
+72 -6
View File
@@ -1,5 +1,7 @@
const GANTT = {label: 240, right: 28, top: 34, lane: 30, width: 1180, bar: 16, ticks: 4};
const BURN = {width: 900, height: 340, left: 52, right: 24, top: 20, bottom: 40};
const BURN = {width: 900, height: 340, left: 52, right: 24, axis: 56, top: 20, bottom: 40};
const OVERLAY_PATH = "burn.overlays";
const scoreView = {key: "", rows: [], labels: null, busy: false, bound: false};
const EDGE_WORDS = {after: "Follows (rasci.py set --after)", base: "Branches from (base branch)", queue: "Waits behind (queue order)"};
const SERIES_WORDS = {scope: "Scope (burnup)", done: "Done (burnup)", remaining: "Remaining (burndown)", unestimated: "Unestimated tasks"};
const STEP_WORDS = [["started", "Started"], ["ended", "Ended"], ["joined", "Joined the scope"]];
@@ -80,11 +82,11 @@
return h("div", {class: "gantt-box"}, h("div", {class: "gantt-scroll"}, svg), legend);
}
function burnAxis(svg, from, to, top) {
const x = (t) => BURN.left + ((t - from) / Math.max(to - from, 1)) * (BURN.width - BURN.left - BURN.right);
function burnAxis(svg, from, to, top, right) {
const x = (t) => BURN.left + ((t - from) / Math.max(to - from, 1)) * (BURN.width - BURN.left - right);
const y = (value) => BURN.top + (1 - value / top) * (BURN.height - BURN.top - BURN.bottom);
for (const value of [0, top / 2, top]) {
svg.append(s("line", {class: "grid", x1: BURN.left, x2: BURN.width - BURN.right, y1: fixed(y(value)), y2: fixed(y(value))}));
svg.append(s("line", {class: "grid", x1: BURN.left, x2: BURN.width - right, y1: fixed(y(value)), y2: fixed(y(value))}));
svg.append(s("text", {class: "axis", x: BURN.left - 8, y: fixed(y(value)), dy: "0.35em", "text-anchor": "end"}, String(Math.round(value * 10) / 10)));
}
for (const [index, t] of [from, (from + to) / 2, to].entries()) {
@@ -165,6 +167,68 @@
return {reach, end: forecast === null ? picked.to : Math.min(forecast, reach)};
}
const overlaysReady = () => typeof ViewStore === "object" && typeof LegendChips === "object";
async function fetchScores(key, from, to) {
scoreView.busy = true;
scoreView.key = key;
try {
const response = await fetch(`/api/scores?from=${Math.floor(from)}&to=${Math.ceil(to)}`, {cache: "no-store"});
if (response.ok) {
const found = await response.json();
scoreView.rows = Array.isArray(found.rows) ? found.rows.filter((row) => row && row.values && typeof row.values === "object") : [];
scoreView.labels = Plan.overlayLabels(found.labels);
}
} catch {
ui.offline = true;
}
scoreView.busy = false;
taskView.sig = "";
drawTasks();
}
function overlayKeys() {
const kept = ViewStore.get(OVERLAY_PATH);
return Array.isArray(kept) && scoreView.labels ? Object.keys(scoreView.labels).filter((key) => kept.includes(key)) : [];
}
const burnRight = () => (overlaysReady() && overlayKeys().length ? BURN.axis : BURN.right);
function burnOverlays(svg, picked, x, right) {
const key = `${Math.floor(picked.from)}:${Math.floor(picked.to / 60)}`;
if (key !== scoreView.key && !scoreView.busy) {
fetchScores(key, picked.from, picked.to);
}
if (!scoreView.bound) {
scoreView.bound = true;
ViewStore.on(OVERLAY_PATH, () => {
taskView.sig = "";
drawTasks();
});
}
if (!scoreView.labels) {
return null;
}
const series = Plan.overlaySeries(picked.rows, scoreView.rows, overlayKeys(), picked);
const top = Plan.overlayTop(series);
const y = (value) => BURN.top + (1 - value / top) * (BURN.height - BURN.top - BURN.bottom);
if (series.length) {
for (const value of [0, top / 2, top]) {
svg.append(s("text", {class: "axis overlay-axis", x: BURN.width - right + 6, y: fixed(y(value)), dy: "0.35em", "text-anchor": "start"}, `${Math.round(value)} %`));
}
}
for (const {key: name, points} of series) {
const d = points.map((point, index) => `${index ? "L" : "M"}${fixed(x(point.t))},${fixed(y(point.value))}`).join(" ");
const group = s("g", {class: `burn-overlay k-${name}`, "data-overlay": name, "data-points": String(points.length)}, d ? s("path", {class: "burn-overlay-line", d}) : null);
for (const point of points) {
group.append(s("circle", {class: "burn-overlay-point", cx: fixed(x(point.t)), cy: fixed(y(point.value)), r: 3, "data-t": point.t, "data-value": point.value},
s("title", {}, `${absolute.format(stampOf(point.t))} · ${scoreView.labels[name]}: ${point.value} % · ${point.branch}`)));
}
svg.append(group);
}
return LegendChips.render(Object.entries(scoreView.labels).map(([key, label]) => ({id: key, label, tone: `k-${key}`})), OVERLAY_PATH, "shown");
}
function burnChart(rows, shown) {
const picked = Plan.scoped(rows, taskView.scope, {filtered: shown, since: state.since, until: state.until, now: state.now, holders: lineHolders()});
const series = Plan.burn(picked.rows, {estimation: estimation(), from: picked.from, to: picked.to}).points;
@@ -177,7 +241,8 @@
const top = Math.max(1, ...series.flatMap((point) => names.map((name) => point[name])), ...ends);
const unit = estimation() === "points" ? "story points; unestimated tasks as a line of their own" : "tasks";
const svg = s("svg", {class: "chart burn", viewBox: `0 0 ${BURN.width} ${BURN.height}`, role: "img", "aria-label": `Burndown and burnup in ${unit}; hover a point for the work packages of its step`});
const {x, y} = burnAxis(svg, picked.from, reach, top);
const right = burnRight();
const {x, y} = burnAxis(svg, picked.from, reach, top, right);
const last = series[series.length - 1];
if (reach > picked.to) {
svg.append(s("line", {class: "now", x1: fixed(x(picked.to)), x2: fixed(x(picked.to)), y1: BURN.top, y2: BURN.height - BURN.bottom}));
@@ -191,10 +256,11 @@
burnFinish(svg, segments, x, y);
}
svg.append(burnPoints(series, Plan.steps(picked.rows, series, picked.from), names, x, y));
const overlays = overlaysReady() ? burnOverlays(svg, picked, x, right) : null;
const stale = h("span", {class: "pill burn-stale", "data-tone": "warn", "data-stale": String(picked.stale.length), role: "img", ...named(`Stale: ${picked.stale.length}`, `Stale: ${picked.stale.length} tasks without a register change for ${Plan.STALE_DAYS} days, counted apart`)}, icon("bi-hourglass-bottom"), String(picked.stale.length));
const legend = h("div", {class: "legend"}, names.map((name) => h("span", {class: `burn-key k-${name}`, "data-series": name}, `${SERIES_WORDS[name]}: ${last[name]}`)),
settings.trend.mode === "off" ? null : finishNotes(trend, finish), stale);
return h("div", {class: "burn-box"}, svg, legend, h("p", {class: "muted"}, `${picked.rows.length} tasks in scope, counted in ${unit}`));
return h("div", {class: "burn-box", "data-panel": "burndown"}, svg, legend, overlays, h("p", {class: "muted"}, `${picked.rows.length} tasks in scope, counted in ${unit}`));
}
async function chooseEstimation(id) {
+39 -1
View File
@@ -262,6 +262,44 @@ const Plan = (() => {
return {fixed: reachable(fixed), growing: found.forecast};
}
const branchOf = (row) => (row.branch !== "-" ? row.branch : row.worktree_branch);
/**
* @param rows - the burndown scope's task rows with branch ("-" for none) and worktree_branch ("" for none).
* @param journal - /api/scores rows {time, branch, values: {metric key: number or null}}.
* @param keys - the metric keys switched on, in the label file's order.
* @param span - {from, to} in epoch seconds.
* @returns one {key, points: [{t, value, branch}]} per key, in keys order: the journal rows of a branch of the
* scope's rows within span whose value for the key is a number, in time order.
*/
function overlaySeries(rows, journal, keys, span) {
const branches = new Set(rows.map(branchOf).filter(Boolean));
const inside = journal.filter((entry) => branches.has(entry.branch) && entry.time >= span.from && entry.time <= span.to).sort((a, b) => a.time - b.time);
return keys.map((key) => ({
key, points: inside.filter((entry) => Number.isFinite(entry.values[key])).map((entry) => ({t: entry.time, value: entry.values[key], branch: entry.branch})),
}));
}
/**
* @param labels - the labels of /api/scores: {metrics: {key: {emoji, label}}}, or anything else.
* @returns {key: "<emoji> <label>"} in the file's order for the entries with a string emoji and label; null when
* none has both or labels is no such object.
*/
function overlayLabels(labels) {
const metrics = labels && typeof labels === "object" ? labels.metrics : null;
if (!metrics || typeof metrics !== "object" || Array.isArray(metrics)) {
return null;
}
const kept = Object.entries(metrics).filter(([, metric]) => metric && typeof metric.emoji === "string" && typeof metric.label === "string");
return kept.length ? Object.fromEntries(kept.map(([key, metric]) => [key, `${metric.emoji} ${metric.label}`])) : null;
}
/**
* @param series - overlaySeries().
* @returns the top of the right axis: 100, or the largest value when one exceeds it.
*/
const overlayTop = (series) => Math.max(100, ...series.flatMap((found) => found.points.map((point) => point.value)));
const KEYS = {
prio: () => null,
wsjf: (row) => (row.wsjf ? row.wsjf.score : null),
@@ -294,7 +332,7 @@ const Plan = (() => {
return keyed.map((entry) => entry.row);
}
return {FORECASTS, ESTIMATIONS, TRENDS, SORTS, SERIES, TRENDED, STATUS_ORDER, STALE_DAYS, median, medians, forecastLength, bars, etas, stale, scoped, burn, steps, trend, finishes, lineAt, sortRows};
return {FORECASTS, ESTIMATIONS, TRENDS, SORTS, SERIES, TRENDED, STATUS_ORDER, STALE_DAYS, median, medians, forecastLength, bars, etas, stale, scoped, burn, steps, trend, finishes, lineAt, overlaySeries, overlayLabels, overlayTop, sortRows};
})();
if (typeof module === "object" && module.exports) {
module.exports = Plan;
+9
View File
@@ -82,6 +82,15 @@ svg .burn-line.k-unestimated { stroke-dasharray: 6 4; }
svg .burn-trend { fill: none; stroke: var(--c); stroke-width: 1.6; stroke-dasharray: 2 5; stroke-linecap: round; }
svg .burn-point { fill: var(--c); stroke: transparent; stroke-width: 10; cursor: help; }
svg .burn-point:hover { r: 6px; }
svg .burn-overlay-line { fill: none; stroke: var(--c); stroke-width: 1.8; stroke-dasharray: 5 3; }
svg .burn-overlay-point { fill: var(--c); stroke: transparent; stroke-width: 10; cursor: help; }
svg.burn .overlay-axis { fill: var(--muted); }
svg.burn .burn-overlay.k-fix, .burn-box .legend-chip.k-fix { --c: var(--ok); }
svg.burn .burn-overlay.k-break, .burn-box .legend-chip.k-break { --c: var(--bad); }
svg.burn .burn-overlay.k-tokens, .burn-box .legend-chip.k-tokens { --c: var(--warn); }
svg.burn .burn-overlay.k-velocity, .burn-box .legend-chip.k-velocity { --c: var(--info); }
svg.burn .burn-overlay.k-security, .burn-box .legend-chip.k-security { --c: var(--violet); }
svg.burn .burn-overlay.k-score, .burn-box .legend-chip.k-score { --c: var(--accent); }
svg .forecast-line { stroke: var(--ok); stroke-width: 1.4; stroke-dasharray: 4 3; }
svg .forecast-label { fill: var(--ok); font-size: var(--fs-2xs); font-weight: 650; }
.burn-forecast { display: inline-flex; align-items: center; gap: 6px; }
+2 -2
View File
@@ -14,8 +14,8 @@ and never estimates one itself.
1. **Fix confidence** (percent): probability the change does what it claims.
**Break risk** (percent): probability it breaks anything else, from the
`kvb-check-break-risk` skill. Two numbers, never one blended figure.
2. **Score**: pass both, with cost from the `kvb-check-efficiency`
skill, time from the `kvb-check-speed` skill and security and the
2. **Score**: pass both, with tokens from the `kvb-check-efficiency`
skill, velocity from the `kvb-check-speed` skill and security and the
critical count from `security.py` of the `kvb-check-security` skill, to
`${CLAUDE_SKILL_DIR}/../kvb-check-score/scripts/score.py --fix <n> --break
<n> --efficiency <n|-> --speed <n|-> --security <n|~n> --critical <count>`
+2 -2
View File
@@ -4,7 +4,7 @@ description: >
Token cost of skill edits, CPU and RAM of code edits.
---
💸 Cost is the share of the base's resources a change uses: under 100 %
💸 Tokens is the share of the base's resources a change uses: under 100 %
it saves, over 100 % it costs more. Paste the script's lines, never a
paragraph about them. `<ref>` is the commit the change
starts from: `HEAD` for uncommitted work, the merge base with the default
@@ -20,7 +20,7 @@ It compares every changed `*.md` under `skills/`, untracked ones included,
at `<ref>` with the working tree, and the always-loaded listing: all
`description:` words, counted by the repository's `scripts/budget.py`.
Tokens are an estimate, characters / 4. Changed files without any base text
print `cost new`, which counts as over 100. Skills have no time.
print `tokens new`, which counts as over 100. Skills have no velocity.
A second line, `💰 value 1:<n>` (`<n>:1` when value < cost), weighs both
per day and never changes the exit code; pass its quotient or `?` to
+31 -10
View File
@@ -9,26 +9,26 @@ Args:
relative to each tree's root; its output is discarded.
Runs:
One run per tree, base first. When cost and time are both at most 100 that
One run per tree, base first. When tokens and velocity are both at most 100 that
is the result; otherwise 4 more runs per tree, alternating, and the arithmetic mean
of all 5 per tree counts.
Other load:
A 1 s /proc/stat sample of system CPU busy before and after the runs, and per run
the busy jiffies minus the command's own CPU time, as a share of all jiffies. When
any share exceeds 30 % or cannot be read, the result is an estimate: time comes
any share exceeds 30 % or cannot be read, the result is an estimate: velocity comes
from the CPU-time ratio instead of wall time.
Output:
`💸 cost <p> % (cpu <b>→<a> s, ram <b>→<a> MB) · ⏱️ time <p> % (wall <b>→<a> s)`,
`💸 tokens <p> % (cpu <b>→<a> s, ram <b>→<a> MB) · ⏱️ velocity <p> % (wall <b>→<a> s)`,
then ` · mean of 5` after the extra runs. An estimate marks both values with `~`,
shows `(cpu <b>→<a> s)` for time and ends in `(estimated: other load <x> %)`, or
`unknown` for <x>. Cost is the mean of the CPU ratio (user plus system seconds
shows `(cpu <b>→<a> s)` for velocity and ends in `(estimated: other load <x> %)`, or
`unknown` for <x>. Tokens is the mean of the CPU ratio (user plus system seconds
of the command and its waited-for children) and the peak RSS ratio; each ratio is
working tree over base, times 100.
Exit codes:
0 cost and time at most 100, 2 usage error or a failing command
0 tokens and velocity at most 100, 2 usage error or a failing command
(its tree, exit code and last stderr line on stderr), 3 either over 100.
"""
@@ -56,6 +56,23 @@ CLOCK_TICKS = os.sysconf("SC_CLK_TCK")
# Trip-wire: ru_maxrss counts kilobytes on Linux but bytes on macOS.
RSS_UNIT = 1 if sys.platform == "darwin" else 1024
USAGE = "usage: measure.py --base <ref> [--json] -- <command...>"
LABELS = Path(__file__).resolve().parents[2] / "kvb-check-score" / "labels.json"
LABEL_ERRORS = (OSError, ValueError, LookupError, TypeError)
def labelled(keys: tuple[str, ...]) -> dict[str, str]:
"""Args:
keys: metric keys of kvb-check-score's labels.json.
Returns:
Per key `<emoji> <label>`, the label in lower case; one of LABEL_ERRORS
for a missing or malformed file.
"""
metrics = json.loads(LABELS.read_text(encoding="utf-8"))["metrics"]
found = {key: (metrics[key]["emoji"], metrics[key]["label"]) for key in keys}
if not all(isinstance(part, str) for pair in found.values() for part in pair):
raise ValueError(f"needs a string emoji and label for {', '.join(keys)}")
return {key: f"{emoji} {label.lower()}" for key, (emoji, label) in found.items()}
def fail(message: str) -> None:
@@ -139,14 +156,14 @@ def figures(runs: dict[str, list[dict]], loads: list[float | None]) -> dict:
}
def line(found: dict) -> str:
def line(found: dict, names: dict[str, str]) -> str:
mark = "~" if found["estimated"] else ""
basis = "cpu" if found["estimated"] else "wall"
before, after = found["base"], found["head"]
text = (
f"💸 cost {mark}{found['efficiency']} % (cpu {before['cpu']:.2f}→{after['cpu']:.2f} s, "
f"{names['tokens']} {mark}{found['efficiency']} % (cpu {before['cpu']:.2f}→{after['cpu']:.2f} s, "
f"ram {before['ram']:.1f}→{after['ram']:.1f} MB) · "
f"⏱️ time {mark}{found['speed']} % ({basis} {before[basis]:.2f}→{after[basis]:.2f} s)"
f"{names['velocity']} {mark}{found['speed']} % ({basis} {before[basis]:.2f}→{after[basis]:.2f} s)"
)
if found["runs"] > 1:
text += f" · mean of {found['runs']}"
@@ -175,6 +192,10 @@ def main(argv: list[str]) -> int:
command = argv[split + 1:]
if not command:
fail("no command after --")
try:
names = {} if args.json else labelled(("tokens", "velocity"))
except LABEL_ERRORS as error:
fail(f"labels {LABELS}: {error!r}")
here = Path.cwd().resolve()
root = Path(git(here, "rev-parse", "--show-toplevel")).resolve()
@@ -206,7 +227,7 @@ def main(argv: list[str]) -> int:
shutil.rmtree(parent, ignore_errors=True)
found = figures(runs, loads + run_loads(runs))
print(json.dumps(found) if args.json else line(found))
print(json.dumps(found) if args.json else line(found, names))
return 3 if costly(found) else 0
+42 -7
View File
@@ -15,8 +15,8 @@ Args:
shipped change instead.
Output:
Line 1: `💸 cost <p> % (skills <b>→<a> tokens; listing <b>→<a> words)`, p = after / before * 100,
or `💸 cost new (+<a> tokens; listing <b>→<a> words)` when the changed files hold no
Line 1: `💸 tokens <p> % (skills <b>→<a> tokens; listing <b>→<a> words)`, p = after / before * 100,
or `💸 tokens new (+<a> tokens; listing <b>→<a> words)` when the changed files hold no
base tokens. Changed files are the *.md under skills/ that `git diff <ref>` names plus untracked
ones; a file missing on a side counts 0. Tokens are an estimate: characters / 4. The listing
is the sum of every skills/*/SKILL.md description word.
@@ -61,6 +61,41 @@ DAYS = 7
ROUND_ABOVE = 10
ALL_SKILLS = "all skills"
CONTEXT = Path(__file__).resolve().parents[2] / "kvb-skill-silence" / "scripts" / "context.py"
LABELS = Path(__file__).resolve().parents[2] / "kvb-check-score" / "labels.json"
LABEL_ERRORS = (OSError, ValueError, LookupError, TypeError)
def labelled(keys: tuple[str, ...]) -> dict[str, str]:
"""Args:
keys: metric keys of kvb-check-score's labels.json.
Returns:
Per key `<emoji> <label>`, the label in lower case; one of LABEL_ERRORS
for a missing or malformed file.
"""
metrics = json.loads(LABELS.read_text(encoding="utf-8"))["metrics"]
found = {key: (metrics[key]["emoji"], metrics[key]["label"]) for key in keys}
if not all(isinstance(part, str) for pair in found.values() for part in pair):
raise ValueError(f"needs a string emoji and label for {', '.join(keys)}")
return {key: f"{emoji} {label.lower()}" for key, (emoji, label) in found.items()}
def headline(name: str, efficiency: int | None, before: int, after: int, listing: str) -> str:
"""Args:
name: the tokens metric as labelled() gives it.
efficiency: after over before in percent, None without base tokens.
before: the base tokens of the changed files.
after: their tokens in the working tree.
listing: the listing words part.
Returns:
Line 1.
"""
if efficiency is None:
return f"{name} new (+{after} tokens; {listing})"
return f"{name} {efficiency} % (skills {before}→{after} tokens; {listing})"
def fail(message: str) -> None:
@@ -254,6 +289,10 @@ def measure(
def check(argv: list[str]) -> int:
args = parse(argv)
try:
names = labelled(("tokens",))
except LABEL_ERRORS as error:
fail(f"labels {LABELS}: {error!r}")
repo = Path(args.repo).resolve()
if not (repo / SKILLS).is_dir():
fail(f"{repo} has no {SKILLS}/ directory")
@@ -296,11 +335,7 @@ def check(argv: list[str]) -> int:
},
}))
else:
listing = f"listing {listing_before}→{listing_after} words"
if efficiency is None:
print(f"💸 cost new (+{after} tokens; {listing})")
else:
print(f"💸 cost {efficiency} % (skills {before}→{after} tokens; {listing})")
print(headline(names["tokens"], efficiency, before, after, f"listing {listing_before}→{listing_after} words"))
print(money)
return 3 if efficiency is None or efficiency > LIMIT else 0
+1 -1
View File
@@ -6,7 +6,7 @@ description: >
---
The gate rule, stated here and nowhere else: nothing is declared 90/5 (fix
confidence ≥ 90 %, break risk ≤ 5 %, cost and time ≤ 100 % in
confidence ≥ 90 %, break risk ≤ 5 %, tokens and velocity ≤ 100 % in
`kvb-check-score`; over 100 %, ask the operator before the commit, naming
the cause) before every `make test*` target of the project, each Makefile target named `test` or starting with
`test-`, ran green on the exact head with a clean tracked tree. Until then
+44 -12
View File
@@ -1,7 +1,7 @@
---
name: kvb-check-score
description: >
Score fix, break, cost, time and security; run wherever confidence is reported.
Score fix, break, tokens, velocity and security; run wherever confidence is reported.
---
Run it wherever fix confidence and break risk are reported: every
@@ -16,18 +16,22 @@ python3 ${CLAUDE_SKILL_DIR}/scripts/score.py --fix <0-100> --break <0-100> --eff
| --- | --- | --- |
| 🎯 | fix confidence | `kvb-check-confidence` |
| 💥 | break risk | `kvb-check-break-risk` |
| 💸 | cost, `--efficiency` | `tokens.py` (skills) or `measure.py` (code) of `kvb-check-efficiency` |
| ⏱️ | time, `--speed` | `measure.py`, see `kvb-check-speed` |
| 💸 | tokens, `--efficiency` | `tokens.py` (skills) or `measure.py` (code) of `kvb-check-efficiency` |
| ⏱️ | velocity, `--speed` | `measure.py`, see `kvb-check-speed` |
| 🛡️ | security, critical count | `security.py` of `kvb-check-security`; `~<n>` from a review by hand |
| 💰 | cost:value quotient, tables and message block only | 💰 line of `tokens.py` (skills) of `kvb-check-efficiency` |
| 🏆 | score | this script |
Cost and time: after over base in percent, over 100 worse. `-` means not
measured or not relevant (skills have no time) and counts as
Emojis and labels come from `labels.json`, one entry per metric key with
its former labels (Cost, Efficiency, Time, Speed); change a label there,
nowhere else.
Tokens and velocity: after over base in percent, over 100 worse. `-` means not
measured or not relevant (skills have no velocity) and counts as
100; it shows as `–`. `new` (changed skill text without a base) counts as 100
in the score, yet still asks. Never invent a value: measure it or pass `-`.
Score = `fix * (100 - break) / 100 * 100 / cost * 100 / time * security / 100`,
Score = `fix * (100 - break) / 100 * 100 / tokens * 100 / velocity * security / 100`,
capped to 0..100 and rounded.
## Gate
@@ -42,20 +46,22 @@ The second line is the verdict, first match wins:
in the block and commit only on the operator's word. Close the gap as the
last section of `kvb-check-break-risk` says.
- `⚠️ gate: <causes>: ask the operator before the commit` (exit 3), naming
each of `cost <c> % > 100`, `time <t> % > 100` and
each of `tokens <c> % > 100`, `velocity <t> % > 100` and
`security <n> % < 100`: ask with the block what the cost buys or why each
finding is acceptable; commit only on a yes, never carried over to
another commit.
- `✅ gate: 90/5, cost and time <= 100, security 100 %` (exit 0):
- `✅ gate: 90/5, tokens and velocity <= 100, security 100 %` (exit 0):
commit as usual. 90/5 still needs `gate: green` of `kvb-check-gate`.
- Exit 2: usage error.
- Exit 2: usage error, a malformed `labels.json` or a `--commit-table`
repository git cannot read, with one stderr line and nothing on stdout;
fix the cause and score again. A critical finding still exits 4 then.
## Tables and message block
Each flag replaces the two lines; values, gate and exit code stay:
- `--table`: one markdown table, `| 🎯 Fix | 💥 Break | 💸 Cost |
⏱️ Time | 🛡️ Security | 🏆 Score | Gate |`.
- `--table`: one markdown table, `| 🎯 Fix | 💥 Break | 💸 Tokens |
⏱️ Velocity | 🛡️ Security | 🏆 Score | Gate |`.
- `--commit-table <repo>`: that table led by `🌿 Branch | 📊 Diff stat`
from git, 📁 worktree path below, alone in a code block. Every commit
block is this table; after the commit, `--commit <sha>` shows that commit's diff stat
@@ -66,6 +72,32 @@ Each flag replaces the two lines; values, gate and exit code stay:
`-m` argument, so the approval prompt and git history show the values.
On a skill change add `--value <n:n|?>`, the quotient of the 💰 line (`?`
while it has no value): a `💰 Value` column after `💸 Cost`, or line
while it has no value): a `💰 Value` column after `💸 Tokens`, or line
after `🛡️ Security`. It changes neither score nor gate; alone it is a usage
error.
## Journal
Each `--message-block` render appends one row to
`${AGENTS_COUNCIL_ROOT:-/tmp/agents-council}/scores.jsonl`: time, origin
remote, top level path, branch, `sha` null, the six values and the block's
digest. The hub's burndown draws it as overlays; no gate reads it. A failed
append prints one stderr line and keeps the exit code; a linked council
root or a link, FIFO or other non-regular file at the journal fails it
without blocking, through `nofollow.py` of `kvb-org-organize`; when that
helper is missing or unusable, the line names it. A new council root gets mode
0700, a new journal 0600.
The journal lives in `/tmp`, so a reboot or another PC starts empty. Refill
it from the blocks in git history, without a model:
```bash
python3 ${CLAUDE_SKILL_DIR}/scripts/backfill.py --repo <dir>... [--since <rev>] [--labels <file>]
```
It reads every local and origin branch, takes the last block of each commit
message, old labels included, and adds one row per commit not yet keyed by
(remote, sha), so a second run or another clone adds nothing. The branch is
the one whose first-parent line holds the commit, the default branch first,
else the one a merge subject names. An unknown label is reported once and
skipped.
+14
View File
@@ -0,0 +1,14 @@
{
"metrics": {
"fix": {"emoji": "🎯", "label": "Fix", "former": []},
"break": {"emoji": "💥", "label": "Break", "former": []},
"tokens": {"emoji": "💸", "label": "Tokens", "former": ["Cost", "Efficiency"]},
"velocity": {"emoji": "⏱️", "label": "Velocity", "former": ["Time", "Speed"]},
"security": {"emoji": "🛡️", "label": "Security", "former": []},
"score": {"emoji": "🏆", "label": "Score", "former": []}
},
"extras": {
"value": {"emoji": "💰", "label": "Value", "former": []},
"gate": {"emoji": "🚦", "label": "Gate", "former": []}
}
}
+252
View File
@@ -0,0 +1,252 @@
#!/usr/bin/env python3
"""Backfill the score journal of journal.py from the message blocks in git history.
Args:
--repo <dir>...: repositories whose local and origin branches git log reads;
repeatable, every one given counts.
--since <rev>: only commits not reachable from <rev>, in every repository.
--labels <file>: the label file mapping current and former labels to metric
keys, ../labels.json when left out.
Output:
One line per repository, `<top level>: <n> blocks, <m> rows added`; one stderr
line per unknown block label, `backfill: unknown label <label>, skipped`.
Exit codes:
0 done, 2 usage error, a repository git cannot read or a journal that is
linked, not a regular file or not writable, with one stderr line; nothing
is written then.
"""
from __future__ import annotations
import argparse
import json
import re
import sys
from collections import deque
from pathlib import Path
from typing import NamedTuple
from journal import (
LABELS,
append,
entry,
git_read,
journal_path,
load_labels,
origin,
read,
)
FIELD = "\x00"
RECORD = "\x1e"
LOG_FORMAT = "%H%x00%P%x00%ct%x00%B%x1e"
REFS = ("refs/heads/", "refs/remotes/origin/")
LINE = re.compile(r"([^\x00-\x7f]\S*) ([A-Z][A-Za-z]*): (.+)")
AMOUNT = re.compile(r"~?(\d+(?:\.\d+)?)(?: ?%)?")
PARAGRAPH = re.compile(r"\n[ \t]*\n")
MERGES = (
re.compile(r"Merge pull request #\d+ from [^/\s]+/(\S+)"),
re.compile(r"Merge (?:remote-tracking )?branch '(?:origin/)?([^']+)'"),
re.compile(r"Merge (\S+) into \S+"),
)
class Commit(NamedTuple):
parents: list[str]
time: int
message: str
def aliases(labels: dict) -> tuple[dict[str, str], set[str]]:
"""Args:
labels: load_labels() of a label file.
Returns:
(metric of each current and former label, labels of the extra lines).
"""
named = {name: key for key, metric in labels["metrics"].items() for name in (metric["label"], *metric["former"])}
extras = {name for extra in labels["extras"].values() for name in (extra["label"], *extra["former"])}
return named, extras
def amount(text: str) -> float | int | None:
found = AMOUNT.fullmatch(text.strip())
if not found:
return None
value = float(found.group(1))
return int(value) if value.is_integer() else value
def parse(message: str, named: dict[str, str], extras: set[str], unknown: set[str]) -> tuple[dict, str] | None:
"""Args:
message: a commit message.
named: aliases() metrics.
extras: aliases() extra labels, read and left out.
unknown: collects the labels of block lines neither names.
Returns:
(values per metric key, block text) of the last block, None without one.
"""
found = None
for paragraph in PARAGRAPH.split(message):
lines = [match for match in (LINE.fullmatch(text.strip()) for text in paragraph.splitlines()) if match]
if not any(named.get(match.group(2)) == "score" for match in lines):
continue
values = dict.fromkeys(named.values())
for match in lines:
label = match.group(2)
if label in named:
values[named[label]] = amount(match.group(3))
elif label not in extras:
unknown.add(label)
if values["score"] is not None:
found = (values, "\n".join(match.group(0) for match in lines))
return found
def history(repo: Path, tips: list[str], since: str | None) -> dict[str, Commit]:
text = git_read(repo, "log", f"--format={LOG_FORMAT}", *tips, *([f"^{since}"] if since else []), "--")
if text is None:
raise LookupError(f"git log in {repo} failed")
commits = {}
for record in text.split(RECORD):
fields = record.strip("\n").split(FIELD, 3)
if len(fields) == 4:
sha, parents, stamp, message = fields
commits[sha] = Commit(parents.split(), int(stamp), message)
return commits
def heads(repo: Path) -> list[tuple[str, str]]:
"""Args:
repo: a git repository.
Returns:
(branch, tip sha) of every local and origin branch, the default branch
(origin/HEAD, else the checked out one) first, then by name, local first.
"""
listing = git_read(repo, "for-each-ref", "--format=%(refname)%00%(objectname)", *REFS)
if listing is None:
raise LookupError(f"git for-each-ref in {repo} failed")
pointed = git_read(repo, "symbolic-ref", "-q", "refs/remotes/origin/HEAD") or git_read(repo, "symbolic-ref", "-q", "HEAD") or ""
default = next((pointed.removeprefix(prefix) for prefix in REFS if pointed.startswith(prefix)), None)
found = []
for row in listing.splitlines():
ref, sha = row.split(FIELD)
rank, prefix = next((rank, prefix) for rank, prefix in enumerate(REFS) if ref.startswith(prefix))
name = ref.removeprefix(prefix)
if name != "HEAD":
found.append((name != default, name, rank, sha))
return [(name, sha) for _, name, _, sha in sorted(found)]
def merged(message: str) -> str | None:
subject = message.split("\n", 1)[0]
return next((match.group(1) for match in (pattern.match(subject) for pattern in MERGES) if match), None)
def owners(commits: dict[str, Commit], tips: list[tuple[str, str]]) -> dict[str, str]:
"""Args:
commits: history() of the repository.
tips: heads() of the repository.
Returns:
The branch of every commit: the first tip whose first-parent line holds
it, else the branch a merge subject names for its side, else the merge's.
"""
owner: dict[str, str] = {}
queue = deque((sha, name) for name, sha in tips)
while queue:
sha, name = queue.popleft()
while sha in commits and sha not in owner:
owner[sha] = name
commit = commits[sha]
side = merged(commit.message) or name
queue.extend((other, side) for other in commit.parents[1:])
sha = commit.parents[0] if commit.parents else None
return owner
def scan(repo: Path, since: str | None, named: dict[str, str], extras: set[str], unknown: set[str]) -> tuple[str, list[dict]]:
place = origin(repo)
if place[1] is None:
raise LookupError(f"{repo} is not a git work tree")
tips = heads(repo)
commits = history(repo, [sha for _, sha in tips], since) if tips else {}
owner = owners(commits, tips)
rows = []
for sha, commit in commits.items():
found = parse(commit.message, named, extras, unknown)
if found:
values, block = found
rows.append(entry(commit.time, place, owner.get(sha), sha, values, block))
return place[1], rows
def known(path: Path) -> set[tuple[str | None, str]]:
try:
text = read(path)
except FileNotFoundError:
return set()
keys = set()
for line in text.splitlines():
try:
row = json.loads(line)
except ValueError:
continue
if isinstance(row, dict) and isinstance(row.get("sha"), str):
keys.add((row.get("remote"), row["sha"]))
return keys
def main() -> int:
parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
parser.add_argument("--repo", type=Path, nargs="+", action="extend", required=True)
parser.add_argument("--since", metavar="REV")
parser.add_argument("--labels", type=Path, default=LABELS)
args = parser.parse_args()
try:
named, extras = aliases(load_labels(args.labels))
except (OSError, ValueError, LookupError, TypeError, AttributeError) as error:
parser.error(f"--labels {args.labels}: {error!r}")
unknown: set[str] = set()
try:
scanned = [scan(repo, args.since, named, extras, unknown) for repo in args.repo]
except LookupError as error:
print(f"backfill: {error}", file=sys.stderr)
return 2
path = journal_path()
try:
keys = known(path)
except (OSError, ValueError) as error:
return refused(path, error)
added, summary = [], []
for top, rows in scanned:
fresh = []
for row in sorted(rows, key=lambda row: (row["time"], row["sha"])):
key = (row["remote"], row["sha"])
if key not in keys:
keys.add(key)
fresh.append(row)
added += fresh
summary.append(f"{top}: {len(rows)} blocks, {len(fresh)} rows added")
if added:
try:
append(path, added)
except (OSError, ValueError) as error:
return refused(path, error)
print("\n".join(summary))
for label in sorted(unknown):
print(f"backfill: unknown label {label}, skipped", file=sys.stderr)
return 0
def refused(path: Path, error: Exception) -> int:
print(f"backfill: journal {path}: {getattr(error, 'strerror', None) or error}", file=sys.stderr)
return 2
if __name__ == "__main__":
sys.exit(main())
+184
View File
@@ -0,0 +1,184 @@
"""Labels and score journal shared by score.py and backfill.py of kvb-check-score.
Environment:
AGENTS_COUNCIL_ROOT: council directory shared by every agent on the host;
/tmp/agents-council when unset or empty. The journal is its scores.jsonl,
display data for the hub's burndown only, never a gate input.
"""
from __future__ import annotations
import hashlib
import importlib.util
import json
import os
import re
import subprocess
from collections.abc import Callable
from pathlib import Path
from types import ModuleType
from urllib.parse import urlsplit
LABELS = Path(__file__).resolve().parents[1] / "labels.json"
NOFOLLOW = Path(__file__).resolve().parents[2] / "kvb-org-organize" / "scripts" / "nofollow.py"
JOURNAL = "scores.jsonl"
COUNCIL = "/tmp/agents-council"
SCP = re.compile(r"(?:[^@/:]+@)?([^/:]+):(?!//)(.+)")
FILE_MODE = 0o600
def load_labels(path: Path) -> dict:
"""Args:
path: a labels.json with "metrics" and "extras", each key mapped to
{"emoji", "label", "former"}.
Returns:
The parsed file; metrics in block order.
"""
return json.loads(path.read_text(encoding="utf-8"))
def journal_path() -> Path:
return Path(os.environ.get("AGENTS_COUNCIL_ROOT") or COUNCIL) / JOURNAL
def git(repo: Path | str, *args: str) -> str:
"""Args:
repo: the directory git runs in.
args: the git arguments.
Returns:
The stripped stdout, bytes outside UTF-8 replaced; LookupError naming
the command and git's stderr on one line when git is missing, the
directory is gone or git fails.
"""
try:
done = subprocess.run(["git", *args], cwd=repo, capture_output=True, encoding="utf-8", errors="replace", check=False)
except OSError as error:
raise LookupError(f"git {args[0]} in {repo}: {error}") from None
if done.returncode != 0:
raise LookupError(f"git {args[0]} in {repo}: {' '.join(done.stderr.split())}")
return done.stdout.strip()
def git_read(repo: Path, *args: str) -> str | None:
"""Args:
repo: the directory git runs in.
args: the git arguments.
Returns:
The stdout of git(), or None when it raises.
"""
try:
return git(repo, *args)
except LookupError:
return None
def identity(url: str | None) -> str | None:
"""Args:
url: a git remote URL: https, ssh, scp-like or a local path.
Returns:
host/path without .git for a hosted remote, so ssh, scp-like and https
clones of one repository match; a local path unchanged; None for None.
"""
if url is None:
return None
scp = SCP.fullmatch(url)
if "://" in url:
parts = urlsplit(url)
host, path = parts.hostname, parts.path
elif scp:
host, path = scp.groups()
else:
return url
if not host:
return url
return f"{host.lower()}/{path.strip('/').removesuffix('.git')}"
def origin(repo: Path) -> tuple[str | None, str | None]:
"""Args:
repo: a directory inside a git work tree.
Returns:
(remote, path): the identity() of its origin remote and its top level
path; None for what git cannot tell.
"""
return identity(git_read(repo, "remote", "get-url", "origin")), git_read(repo, "rev-parse", "--show-toplevel")
def entry(time: int, place: tuple[str | None, str | None], branch: str | None, sha: str | None, values: dict, block: str) -> dict:
"""Args:
time: epoch seconds: the render for score.py, the commit for backfill.py.
place: origin() of the repository.
branch: the branch the block belongs to.
sha: the commit holding the block, None for a render.
values: per metric key its number, None where the block shows none.
block: the message block text the digest covers.
Returns:
One journal row.
"""
remote, path = place
digest = hashlib.sha256(block.encode("utf-8")).hexdigest()
return {"time": time, "remote": remote, "path": path, "branch": branch, "sha": sha, "values": values, "digest": digest}
def line(row: dict) -> str:
return json.dumps(row, ensure_ascii=False) + "\n"
def unusable(error: Exception) -> OSError:
return OSError(f"helper {NOFOLLOW} unusable ({type(error).__name__}): install or update kvb-org-organize")
def nofollow[T](use: Callable[[ModuleType], T]) -> T:
"""Args:
use: what to do with nofollow.py of kvb-org-organize.
Returns:
The result of use; its OSError unchanged, any other failure loading
or using the helper as one OSError naming it, never a
FileNotFoundError.
"""
try:
spec = importlib.util.spec_from_file_location("kvb_score_nofollow", NOFOLLOW)
loaded = importlib.util.module_from_spec(spec)
spec.loader.exec_module(loaded)
except Exception as error: # noqa: BLE001
raise unusable(error) from None
try:
return use(loaded)
except OSError:
raise
except Exception as error: # noqa: BLE001
raise unusable(error) from None
def append(path: Path, rows: list[dict]) -> None:
"""Args:
path: the journal in the council root, made with the helper's mode
0o700; a linked root, a link, FIFO or other non-regular file there
raises OSError without blocking.
rows: the rows to append, one JSON line each, in one write.
"""
text = "".join(line(row) for row in rows)
def write(helper: ModuleType) -> None:
path.parent.mkdir(mode=helper.DIRECTORY_MODE, parents=True, exist_ok=True)
helper.append(path.parent, path, text, FILE_MODE)
nofollow(write)
def read(path: Path) -> str:
"""Args:
path: the journal, guarded as for append(); a missing one raises
FileNotFoundError.
Returns:
Its text, undecodable bytes replaced.
"""
return nofollow(lambda helper: helper.read_text(path.parent, path))
+107 -55
View File
@@ -1,13 +1,13 @@
#!/usr/bin/env python3
"""Fold fix confidence, break risk, cost, time and security into one score and one gate verdict.
"""Fold fix confidence, break risk, tokens, velocity and security into one score and one gate verdict.
Args:
--fix: fix confidence in percent, 0 to 100.
--break: break risk in percent, 0 to 100.
--efficiency: cost in percent above 0, from kvb-check-efficiency; `-` when not
--efficiency: tokens in percent above 0, from kvb-check-efficiency; `-` when not
measured or not relevant, `new` when the change has no base. Both count as 100
in the score.
--speed: time in percent above 0, from kvb-check-speed; `-` or `new` as for
--speed: velocity in percent above 0, from kvb-check-speed; `-` or `new` as for
--efficiency.
--security: security in percent, 0 to 100, from security.py of kvb-check-security;
`~<n>` for a value from a review by hand.
@@ -19,24 +19,30 @@ Args:
--commit <sha>: with --commit-table, the diff stat of that commit instead of
the staged one, plus a commit column.
--message-block: print the commit message block instead of the two lines, one
value per line: `🎯 Fix: <f> %`, `💥 Break: <b> %`, `💸 Cost: <c>`,
`⏱️ Time: <t>`, `🛡️ Security: <n> %`, `💰 Value: <n:n>` with --value only,
`🏆 Score: <n>`, `🚦 Gate: <verdict>`; values and markers as in line 1.
value per line: `🎯 Fix: <f> %`, `💥 Break: <b> %`, `💸 Tokens: <c>`,
`⏱️ Velocity: <t>`, `🛡️ Security: <n> %`, `💰 Value: <n:n>` with --value only,
`🏆 Score: <n>`, `🚦 Gate: <verdict>`; values and markers as in line 1. Each
render appends one row to the score journal of journal.py; a failed append
prints one stderr line and keeps the exit code.
--value <n:n|?>: with --table, --commit-table or --message-block, a value after
the cost one (--message-block: after security): the cost:value quotient of
the tokens one (--message-block: after security): the cost:value quotient of
tokens.py of kvb-check-efficiency, `?` when unknown. It changes neither score nor gate.
Labels and emojis come from ../labels.json.
Output:
Line 1: `🎯 fix <f> % · 💥 break <b> % · 💸 cost <c> % · ⏱️ time <t> % ·
Line 1: `🎯 fix <f> % · 💥 break <b> % · 💸 tokens <c> % · ⏱️ velocity <t> % ·
🛡️ security <n> % · 🏆 score <n>`, `–` in place of `<x> %` for `-`, `new` for `new`.
Score is fix * (100 - break) / 100 * 100 / cost * 100 / time * security / 100,
Score is fix * (100 - break) / 100 * 100 / tokens * 100 / velocity * security / 100,
capped to 0..100, rounded half up.
Line 2: the gate verdict, `✅ gate: ...`, `⛔ gate: ...` or `⚠️ gate: ...`.
Exit codes:
0 gate passed, 1 fix under 90 or break over 5, 2 usage error,
3 cost or time over 100 or new, or security under 100: ask the operator
before the commit, 4 a critical security finding: do not commit.
0 gate passed, 1 fix under 90 or break over 5, 2 usage error, a malformed
labels.json or a --commit-table repository git cannot read, one stderr line
and nothing on stdout (4 still wins with a critical finding), 3 tokens or
velocity over 100 or new, or security under 100: ask the operator before the
commit, 4 a critical security finding: do not commit.
"""
from __future__ import annotations
@@ -44,8 +50,20 @@ from __future__ import annotations
import argparse
import math
import re
import subprocess
import sys
import time
from pathlib import Path
from journal import (
LABELS,
append,
entry,
git,
git_read,
journal_path,
load_labels,
origin,
)
FIX_MIN = 90
BREAK_MAX = 5
@@ -54,12 +72,10 @@ SAFE = 100
DASH = "-"
NEW = "new"
ESTIMATE = "~"
SCORE_HEADER = ("🎯 Fix", "💥 Break", "💸 Cost", "⏱️ Time", "🛡️ Security", "🏆 Score", "Gate")
COMMIT_HEADER = ("🌿 Branch", "📊 Diff stat")
WORKTREE = "📁 Worktree:"
VALUE_HEADER = "💰 Value"
VALUE_AT = SCORE_HEADER.index("💸 Cost") + 1
UNKNOWN = "?"
LABEL_ERRORS = (OSError, ValueError, LookupError, TypeError, AttributeError)
QUOTIENT = re.compile(r"\d+(?:\.\d+)?:\d+(?:\.\d+)?")
SHORTSTAT = re.compile(
r"(\d+) files? changed(?:, (\d+) insertions?\(\+\))?(?:, (\d+) deletions?\(-\))?"
@@ -137,35 +153,42 @@ def over(name: str, value: float | str) -> str | None:
return None
def gate(args: argparse.Namespace) -> tuple[str, str, int]:
def labels(keys: set[str]) -> tuple[dict, dict]:
"""Args:
keys: the metric keys score.py measures.
Returns:
(metrics, extras) of labels.json; ValueError when it lacks one of the
keys, holds another or an entry has no string emoji and label.
"""
found = load_labels(LABELS)
metrics, extras = found["metrics"], found["extras"]
named = [*metrics.values(), extras["value"], extras["gate"]]
if set(metrics) != keys or not all(isinstance(item["emoji"], str) and isinstance(item["label"], str) for item in named):
raise ValueError(f"needs a string emoji and label for {', '.join(sorted(keys))}, value and gate")
return metrics, extras
def gate(args: argparse.Namespace, metrics: dict) -> tuple[str, str, int]:
def word(key: str) -> str:
return metrics[key]["label"].lower()
if args.critical > 0:
return "⛔", "critical security finding: do not commit", 4
return "⛔", f"critical {word('security')} finding: do not commit", 4
failed = []
if args.fix < FIX_MIN:
failed.append(f"fix {number(args.fix)} % < {FIX_MIN}")
failed.append(f"{word('fix')} {number(args.fix)} % < {FIX_MIN}")
if args.risk > BREAK_MAX:
failed.append(f"break {number(args.risk)} % > {BREAK_MAX}")
failed.append(f"{word('break')} {number(args.risk)} % > {BREAK_MAX}")
value, mark = args.security
unsafe = f"security {mark}{number(value)} % < {SAFE}" if value < SAFE else None
asks = [text for text in (over("cost", args.efficiency), over("time", args.speed), unsafe) if text]
unsafe = f"{word('security')} {mark}{number(value)} % < {SAFE}" if value < SAFE else None
asks = [text for text in (over(word("tokens"), args.efficiency), over(word("velocity"), args.speed), unsafe) if text]
ask = f"{', '.join(asks)}: ask the operator before the commit" if asks else ""
if failed:
return "⛔", "; ".join(part for part in (", ".join(failed), ask) if part), 1
if asks:
return "⚠️", ask, 3
return "✅", f"{FIX_MIN}/{BREAK_MAX}, cost and time <= {COST_MAX}, security {SAFE} %", 0
def git(repo: str, *args: str) -> str:
try:
result = subprocess.run(["git", *args], cwd=repo, capture_output=True, text=True, check=False)
except OSError as error:
print(f"score: git {args[0]} in {repo}: {error}", file=sys.stderr)
sys.exit(2)
if result.returncode != 0:
print(f"score: git {args[0]} in {repo}: {result.stderr.strip()}", file=sys.stderr)
sys.exit(2)
return result.stdout.strip()
return "✅", f"{FIX_MIN}/{BREAK_MAX}, {word('tokens')} and {word('velocity')} <= {COST_MAX}, {word('security')} {SAFE} %", 0
def diffstat(shortstat: str) -> str:
@@ -217,38 +240,67 @@ def main() -> int:
parser.error("--value needs --table, --commit-table or --message-block")
total = score(args)
emoji, verdict, code = gate(args)
value, mark = args.security
values = [
f"{number(args.fix)} %", f"{number(args.risk)} %", shown(args.efficiency), shown(args.speed),
f"{mark}{number(value)} %", str(total), f"{emoji} {verdict}",
]
header, row = list(SCORE_HEADER), list(values)
measured = {
"fix": f"{number(args.fix)} %", "break": f"{number(args.risk)} %", "tokens": shown(args.efficiency),
"velocity": shown(args.speed), "security": f"{mark}{number(value)} %", "score": str(total),
}
try:
metrics, extras = labels(set(measured))
except LABEL_ERRORS as error:
print(f"score: labels {LABELS}: {error!r}", file=sys.stderr)
return 4 if args.critical > 0 else 2
emoji, verdict, code = gate(args, metrics)
verdict_cell = f"{emoji} {verdict}"
value_header = f"{extras['value']['emoji']} {extras['value']['label']}"
header = [*(f"{metric['emoji']} {metric['label']}" for metric in metrics.values()), extras["gate"]["label"]]
row = [*(measured[key] for key in metrics), verdict_cell]
if args.value:
header.insert(VALUE_AT, VALUE_HEADER)
row.insert(VALUE_AT, args.value)
header.insert(list(metrics).index("tokens") + 1, value_header)
row.insert(list(metrics).index("tokens") + 1, args.value)
if args.commit_table:
lead, cells = commit_cells(args.commit_table, args.commit)
top = git(args.commit_table, "rev-parse", "--show-toplevel")
try:
lead, cells = commit_cells(args.commit_table, args.commit)
top = git(args.commit_table, "rev-parse", "--show-toplevel")
except LookupError as error:
print(f"score: {error}", file=sys.stderr)
return 4 if code == 4 else 2
print(table([*lead, *header], [*cells, *row]))
print(f"\n{WORKTREE}\n\n```text\n{top}\n```")
elif args.table:
print(table(header, row))
elif args.message_block:
worth_line = [f"💰 Value: {args.value}"] if args.value else []
print("\n".join((
f"🎯 Fix: {values[0]}", f"💥 Break: {values[1]}", f"💸 Cost: {values[2]}",
f"⏱️ Time: {values[3]}", f"🛡️ Security: {values[4]}", *worth_line,
f"🏆 Score: {total}", f"🚦 Gate: {values[6]}",
)))
lines = [f"{metric['emoji']} {metric['label']}: {measured[key]}" for key, metric in metrics.items()]
if args.value:
lines.insert(list(metrics).index("score"), f"{value_header}: {args.value}")
gate_label = extras["gate"]
block = "\n".join((*lines, f"{gate_label['emoji']} {gate_label['label']}: {verdict_cell}"))
print(block, flush=True)
journal(args, total, block)
else:
print(
f"🎯 fix {values[0]} · 💥 break {values[1]} · 💸 cost {values[2]} · "
f"⏱️ time {values[3]} · 🛡️ security {values[4]} · 🏆 score {total}"
)
print(" · ".join(f"{metric['emoji']} {metric['label'].lower()} {measured[key]}" for key, metric in metrics.items()))
print(f"{emoji} gate: {verdict}")
return code
def plain(value: float | str) -> float | int | None:
if not isinstance(value, float):
return None
return int(value) if value.is_integer() else value
def journal(args: argparse.Namespace, total: int, block: str) -> None:
values = {
"fix": plain(args.fix), "break": plain(args.risk), "tokens": plain(args.efficiency),
"velocity": plain(args.speed), "security": plain(args.security[0]), "score": total,
}
path = journal_path()
try:
here = Path.cwd()
append(path, [entry(int(time.time()), origin(here), git_read(here, "branch", "--show-current") or None, None, values, block)])
except (OSError, ValueError) as error:
print(f"score: journal {path}: {getattr(error, 'strerror', None) or error}", file=sys.stderr)
if __name__ == "__main__":
sys.exit(main())
+3 -3
View File
@@ -4,7 +4,7 @@ description: >
Wall-clock speed of changed code versus its base commit.
---
Its ⏱️ time is the share of its base's wall-clock time a change needs:
Its ⏱️ velocity is the share of its base's wall-clock time a change needs:
under 100 % it is faster, over 100 % slower. It has no script of its own: one run
of the `kvb-check-efficiency` measurement yields both values.
@@ -12,7 +12,7 @@ of the `kvb-check-efficiency` measurement yields both values.
python3 ${CLAUDE_SKILL_DIR}/../kvb-check-efficiency/scripts/measure.py --base <ref> -- <command...>
```
Its `⏱️ time <p> % (wall <b>→<a> s)` compares wall seconds at `<ref>` and
Its `⏱️ velocity <p> % (wall <b>→<a> s)` compares wall seconds at `<ref>` and
in the working tree: one run each, or the mean of 5 each when a value
exceeds 100. The command, the runs, the `queue.py hold` wrapper for heavy
commands and the exit codes are those of the `kvb-check-efficiency` skill;
@@ -24,6 +24,6 @@ pasting its whole line reports both values at once.
renders) has no speed either: `--speed -`, and say why.
- **Other load**: wall time follows host load. Above 30 % other load the
script falls back to the CPU-time ratio and marks the estimate:
`⏱️ time ~<p> % (cpu <b>→<a> s)`, ending in
`⏱️ velocity ~<p> % (cpu <b>→<a> s)`, ending in
`(estimated: other load <x> %)`. Report the run you
made; never rerun to pick the best result.
+1 -1
View File
@@ -24,7 +24,7 @@ skill's Rules apply here too.
--task <register task> -- <command>`, which waits for the rule-based
grant and releases on exit. The fix enters the main line only
at the funnel's bar (the `kvb-org-swarm` skill's: fix confidence ≥ 90 %,
break risk ≤ 5 %, cost and time ≤ 100 % in the `kvb-check-score`
break risk ≤ 5 %, tokens and velocity ≤ 100 % in the `kvb-check-score`
block; over 100 %, ask the operator before the commit, naming the cause)
after its independent review; below the bar the cluster ends as a
proposal for the operator, never as a merge. Never mask a failure: no
+1 -1
View File
@@ -16,6 +16,6 @@ checklist (multi-select) with suggestions, free text allowed:
be changed (files, roles, services); whether new dependencies, toolchains
or languages may be introduced; whether infrastructure or CI configuration
may change; the merge bar (default: the `kvb-org-swarm` skill's bar of fix
confidence ≥ 90 %, break risk ≤ 5 %, cost and time ≤ 100 %, over
confidence ≥ 90 %, break risk ≤ 5 %, tokens and velocity ≤ 100 %, over
100 % only after asking the operator, naming the cause); a time cap per
delegated agent (default 60 minutes).
+2 -2
View File
@@ -11,14 +11,14 @@ description: >
2. Show in the message right before the `git commit` call, never only after
it, the commit table of the kvb-check-score skill (`score.py ...
--commit-table <worktree>`): 📁 worktree, 🌿 branch, 📊 staged diff stat,
then fix, break, cost, time, security, score and gate of the staged
then fix, break, tokens, velocity, security, score and gate of the staged
bundle, as the kvb-check-confidence skill produces them: break risk
derived through the kvb-check-break-risk skill, never estimated. Never
print a diff in the chat, only its stat.
3. Approval-prompt duty: the commit call's approval prompt (Claude Code: the
Bash `description`; other runtimes: their prompt text) names the worktree
path, the branch, the diff stat and the score values (🎯 fix, 💥 break,
💸 cost, ⏱️ time, 🛡️ security, 🏆 score). Missing any item: do not
💸 tokens, ⏱️ velocity, 🛡️ security, 🏆 score). Missing any item: do not
send the call. Gate exit 4 (critical security finding) means no commit:
the call is never sent, not even on the operator's word; fix and rescan
first. Exit 1 (below 90/5) is shown, the operator's word decides; exit 3
+2 -2
View File
@@ -24,9 +24,9 @@ confirms it.
branch fixes, each verified as the `kvb-git-issue-close` skill does.
5. **Confirm** with one `AskUserQuestion`: title (Conventional Commits),
base, state and labels. The state follows the bar of the `kvb-org-swarm`
skill (fix confidence ≥ 90 %, break risk ≤ 5 %, cost and time
skill (fix confidence ≥ 90 %, break risk ≤ 5 %, tokens and velocity
≤ 100 %): ready at or above it, a draft with the residuals in the body
below it. A cost or time over 100 % goes into this question with
below it. A tokens or velocity value over 100 % goes into this question with
its cause.
6. **Create**: `gh pr create --base <default> --head <branch> --title
"<title>" --body-file <file>`, plus `--draft` below the bar.
+2 -2
View File
@@ -12,7 +12,7 @@ holds:
failure on it, a red default-branch run or an issue that reproduces there;
name the run or issue.
- **`90/5`**: the pull request meets the bar of the `kvb-org-swarm` skill (fix
confidence ≥ 90 %, break risk ≤ 5 %, cost and time ≤ 100 %; over
confidence ≥ 90 %, break risk ≤ 5 %, tokens and velocity ≤ 100 %; over
100 %, ask the operator first, naming the cause), from the
`kvb-check-score` block in its body or freshly assessed, and only when
`gate.py status` of the `kvb-check-gate` skill prints `gate: green` with
@@ -27,7 +27,7 @@ holds:
3. A missing label is created once: `gh label create hotfix --color B60205
--description "Fixes a failure on the default branch"`, `gh label create
90/5 --color 0E8A16 --description
"Fix confidence >= 90 %, break risk <= 5 %, cost and time <= 100 %"`.
"Fix confidence >= 90 %, break risk <= 5 %, tokens and velocity <= 100 %"`.
4. Standalone, confirm the changes with one `AskUserQuestion`; from
`kvb-git-pr-create`, its confirmation covers them.
5. `gh pr edit <pr> --add-label <l>` or `--remove-label <l>`. A draft that
+2 -2
View File
@@ -38,8 +38,8 @@ One table, most-blocking first, one row per item:
- **State**: `open`, `running`, `blocked`, `done`. `done` means verified (a
passing run, a green check, a reproduction that now succeeds); work that
merely looks finished is `open`.
- **Score**: the `kvb-check-score` line (🎯 fix, 💥 break, 💸 cost,
⏱️ time, 🏆 score) per the `kvb-check-confidence` skill, only for rows
- **Score**: the `kvb-check-score` line (🎯 fix, 💥 break, 💸 tokens,
⏱️ velocity, 🏆 score) per the `kvb-check-confidence` skill, only for rows
that changed something; `-` for pure observation rows.
- **Evidence / next step**: for `done` the strongest proof line (what ran,
what it returned), otherwise the single next action.
+1 -1
View File
@@ -12,7 +12,7 @@ sidetree or leaves as a proposal.
## The bar
- **Fix confidence ≥ 90 %, break risk ≤ 5 %, cost and time ≤ 100 %**,
- **Fix confidence ≥ 90 %, break risk ≤ 5 %, tokens and velocity ≤ 100 %**,
all from the `kvb-check-score` block of `kvb-check-confidence`; over 100 %,
ask the operator before the commit, naming the cause.
- The operator may set a different bar. Never lower it to make a change fit.
+88
View File
@@ -0,0 +1,88 @@
from __future__ import annotations
import os
import shutil
import subprocess
import sys
from pathlib import Path
REPO_ROOT = Path(__file__).resolve().parents[1]
BACKFILL = REPO_ROOT / "skills" / "kvb-check-score" / "scripts" / "backfill.py"
EPOCH = 1_760_000_000
HANG = 60
OLD_BLOCK = "🎯 Fix: 95 %\n💥 Break: 2 %\n💸 Cost: 97 %\n⏱️ Time: –\n🛡️ Security: 100 %\n🏆 Score: 96\n🚦 Gate: ✅ 90/5, cost and time <= 100, security 100 %"
NEW_BLOCK = "🎯 Fix: 92 %\n💥 Break: 3 %\n💸 Tokens: 110 %\n⏱️ Velocity: 90 %\n🛡️ Security: 100 %\n💰 Value: 1:20\n🧪 Coverage: 80 %\n🏆 Score: 81\n🚦 Gate: ⚠️ tokens 110 % > 100: ask the operator before the commit"
ESTIMATED_BLOCK = "🎯 Fix: 90 %\n💥 Break: 5 %\n💸 Tokens: new\n⏱️ Velocity: 130 %\n🛡️ Security: ~80 %\n🌡️ Heat: 5\n🧪 Coverage: 70 %\n🏆 Score: 52\n🚦 Gate: ⚠️ tokens new: ask the operator before the commit"
MALFORMED_BLOCK = "🎯 Fix: 95 %\n💥 Break: 2 %\n🏆 Score: high"
TRAILER = "Co-Authored-By: Test <test@example.org>"
def git_env(home: Path) -> dict[str, str]:
"""Args:
home: a scratch HOME, so no user or system git configuration applies.
Returns:
The environment for fixture git calls.
"""
return {
"PATH": os.path.dirname(shutil.which("git")),
"HOME": str(home),
"GIT_CONFIG_NOSYSTEM": "1",
"GIT_AUTHOR_NAME": "Test",
"GIT_AUTHOR_EMAIL": "test@example.org",
"GIT_COMMITTER_NAME": "Test",
"GIT_COMMITTER_EMAIL": "test@example.org",
}
class Fixture:
"""Args:
base: a scratch directory for upstream.git, the work clone a and the clone b.
"""
def __init__(self, base: Path) -> None:
self.base = base
self.env = git_env(base)
self.upstream = base / "upstream.git"
self.a = base / "a"
self.b = base / "b"
self.step = 0
base.mkdir(parents=True)
self.git(base, "init", "-q", "--bare", "-b", "main", str(self.upstream))
self.a.mkdir()
self.git(self.a, "init", "-q", "-b", "main")
self.git(self.a, "remote", "add", "origin", str(self.upstream))
self.old = self.commit("feat: old labels", OLD_BLOCK)
self.git(self.a, "checkout", "-q", "-b", "feature/two")
self.new = self.commit("feat: new labels", NEW_BLOCK)
self.git(self.a, "checkout", "-q", "main")
self.tick()
self.git(self.a, "merge", "-q", "--no-ff", "-m", "Merge branch 'feature/two'", "feature/two")
self.estimated = self.commit("fix: estimated", ESTIMATED_BLOCK)
self.malformed = self.commit("chore: malformed", MALFORMED_BLOCK)
self.plain = self.commit("docs: no block", "")
self.git(self.a, "push", "-q", "origin", "main", "feature/two")
self.git(base, "clone", "-q", str(self.upstream), str(self.b))
def tick(self) -> None:
self.step += 1
stamp = f"{EPOCH + self.step * 3600} +0000"
self.env["GIT_AUTHOR_DATE"] = stamp
self.env["GIT_COMMITTER_DATE"] = stamp
def git(self, cwd: Path, *args: str) -> str:
return subprocess.run(["git", *args], cwd=cwd, env=self.env, capture_output=True, text=True, check=True).stdout.strip()
def commit(self, subject: str, block: str) -> str:
self.tick()
messages = [item for part in (subject, block, TRAILER) if part for item in ("-m", part)]
self.git(self.a, "commit", "-q", "--allow-empty", *messages)
return self.git(self.a, "rev-parse", "HEAD")
def time_of(self, sha: str) -> int:
return int(self.git(self.a, "show", "-s", "--format=%ct", sha))
def backfill(council: Path, *args: str) -> subprocess.CompletedProcess:
env = {"PATH": os.environ["PATH"], "AGENTS_COUNCIL_ROOT": str(council)}
return subprocess.run([sys.executable, "-B", str(BACKFILL), *args], env=env, capture_output=True, text=True, check=False, timeout=HANG)
+33 -8
View File
@@ -33,8 +33,8 @@ NEW_B = "💰 value ?:? (cost 0/day; pass --value with evidence; kvb-b new: all
CHEAP = "print('cheap')\n"
COSTLY = "data = list(range(2_000_000))\nprint(sum(data))\n"
LINE = re.compile(
r"^💸 cost (?P<mark>~?)(?P<efficiency>\d+) % \(cpu \d+\.\d\d→\d+\.\d\d s, ram \d+\.\d→\d+\.\d MB\) · "
r"⏱️ time (?P<speed_mark>~?)(?P<speed>\d+) % \((?P<basis>wall|cpu) \d+\.\d\d→\d+\.\d\d s\)(?P<tail>.*)$"
r"^💸 tokens (?P<mark>~?)(?P<efficiency>\d+) % \(cpu \d+\.\d\d→\d+\.\d\d s, ram \d+\.\d→\d+\.\d MB\) · "
r"⏱️ velocity (?P<speed_mark>~?)(?P<speed>\d+) % \((?P<basis>wall|cpu) \d+\.\d\d→\d+\.\d\d s\)(?P<tail>.*)$"
)
@@ -121,7 +121,7 @@ class TestTokens(GitRepo):
self.write("skills/kvb-a/scripts/tool.py", "print(1)\n" * 50)
result = self.tokens()
self.assertEqual(
result.stdout.splitlines(), ["💸 cost 50 % (skills 100→50 tokens; listing 3→3 words)", NO_VALUE]
result.stdout.splitlines(), ["💸 tokens 50 % (skills 100→50 tokens; listing 3→3 words)", NO_VALUE]
)
self.assertEqual(result.returncode, 0, result.stderr)
@@ -130,7 +130,7 @@ class TestTokens(GitRepo):
self.write("skills/kvb-b/SKILL.md", SKILL_B.ljust(200, "y"))
result = self.tokens()
self.assertEqual(
result.stdout.splitlines(), ["💸 cost 200 % (skills 100→200 tokens; listing 3→5 words)", NEW_B]
result.stdout.splitlines(), ["💸 tokens 200 % (skills 100→200 tokens; listing 3→5 words)", NEW_B]
)
self.assertEqual(result.returncode, 3)
result = self.tokens("--json")
@@ -157,7 +157,7 @@ class TestTokens(GitRepo):
def test_only_new_files_print_new(self):
self.write("skills/kvb-b/SKILL.md", SKILL_B.ljust(200, "y"))
result = self.tokens()
self.assertEqual(result.stdout.splitlines(), ["💸 cost new (+50 tokens; listing 3→5 words)", NEW_B])
self.assertEqual(result.stdout.splitlines(), ["💸 tokens new (+50 tokens; listing 3→5 words)", NEW_B])
self.assertEqual(result.returncode, 3)
self.assertIsNone(json.loads(self.tokens("--json").stdout)["efficiency"])
@@ -165,14 +165,14 @@ class TestTokens(GitRepo):
self.git("mv", "skills/kvb-a/notes.md", "skills/kvb-a/moved.md")
result = self.tokens()
self.assertEqual(
result.stdout.splitlines(), ["💸 cost 100 % (skills 100→100 tokens; listing 3→3 words)", NO_VALUE]
result.stdout.splitlines(), ["💸 tokens 100 % (skills 100→100 tokens; listing 3→3 words)", NO_VALUE]
)
self.assertEqual(result.returncode, 0)
def test_no_change_is_100(self):
result = self.tokens()
self.assertEqual(
result.stdout.splitlines(), ["💸 cost 100 % (skills 0→0 tokens; listing 3→3 words)", NO_VALUE]
result.stdout.splitlines(), ["💸 tokens 100 % (skills 0→0 tokens; listing 3→3 words)", NO_VALUE]
)
self.assertEqual(result.returncode, 0)
@@ -193,7 +193,7 @@ class TestTokens(GitRepo):
self.assertEqual(
result.stdout.splitlines(),
[
"💸 cost 869 % (skills 13→113 tokens; listing 3→5 words)",
"💸 tokens 869 % (skills 13→113 tokens; listing 3→5 words)",
"💰 value ?:? (cost 203/day; pass --value with evidence)",
],
)
@@ -235,6 +235,22 @@ class TestTokens(GitRepo):
)
self.assertEqual(result.returncode, 3)
def test_a_malformed_label_file_exits_2_with_one_stderr_line(self):
for text in ("{broken", json.dumps({"metrics": {"tokens": {"emoji": 1, "label": "Tokens"}}}), "[]"):
with self.subTest(labels=text):
labels = self.base / "labels.json"
labels.write_text(text, encoding="utf-8")
out, err = io.StringIO(), io.StringIO()
argv = ["--repo", str(self.repo), "--base", "HEAD", "--transcripts", str(self.base / "projects")]
with mock.patch.object(skill_tokens, "LABELS", labels), contextlib.redirect_stdout(out), contextlib.redirect_stderr(err):
try:
code = skill_tokens.main(argv)
except SystemExit as stop:
code = stop.code
self.assertEqual((code, out.getvalue()), (2, ""))
self.assertEqual(len(err.getvalue().splitlines()), 1, err.getvalue())
self.assertTrue(err.getvalue().startswith(f"tokens: labels {labels}: "), err.getvalue())
def test_usage_errors_exit_2(self):
self.assertEqual(self.tokens("--value", "5").returncode, 2)
self.assertEqual(self.tokens("--evidence", "why").returncode, 2)
@@ -381,6 +397,15 @@ class TestMeasure(GitRepo):
self.assertEqual(trees, [f"worktree {self.repo.resolve()}"])
self.assertLessEqual(set(Path("/tmp").glob("kvb-measure-*")), self.leftovers)
def test_a_malformed_label_file_exits_2_before_any_run(self):
labels = self.base / "labels.json"
labels.write_text(json.dumps({"metrics": {"tokens": {"emoji": "💸", "label": "Tokens"}}}), encoding="utf-8")
with mock.patch.object(measure, "LABELS", labels):
code, out, err, samples = self.measured("--base", "HEAD", "--", sys.executable, "work.py")
self.assertEqual((code, out, samples), (2, "", 0))
self.assertEqual(len(err.splitlines()), 1, err)
self.assertTrue(err.startswith(f"measure: labels {labels}: "), err)
def test_cheaper_head_stops_after_one_run_each(self):
self.write("pkg/work.py", CHEAP)
code, out, err, samples = self.measured("--base", "HEAD", "--", sys.executable, "work.py")
+232
View File
@@ -0,0 +1,232 @@
from __future__ import annotations
import json
import subprocess
import unittest
from hub_support import HUB, PLAN, REPO_ROOT
GANTT = HUB / "templates" / "page-tasks-gantt.js"
LABELS = REPO_ROOT / "skills" / "kvb-check-score" / "labels.json"
SERIES_PROBE = """
const P = require(process.argv[1]);
const rows = [
{task: "work/a", branch: "feature/a", worktree_branch: ""},
{task: "work/b", branch: "-", worktree_branch: "fix/b"},
{task: "work/c", branch: "-", worktree_branch: ""},
];
const values = (fix, score) => ({fix, break: 2, tokens: null, velocity: 130, security: 100, score});
const journal = [
{time: 300, branch: "feature/a", values: values(95, 90)},
{time: 100, branch: "fix/b", values: values(92, null)},
{time: 200, branch: "feature/other", values: values(10, 10)},
{time: 900, branch: "feature/a", values: values(1, 1)},
{time: 150, branch: "", values: values(5, 5)},
{time: 250, branch: "-", values: values(6, 6)},
];
const series = P.overlaySeries(rows, journal, ["fix", "tokens", "score"], {from: 100, to: 800});
console.log(JSON.stringify({
keys: Object.keys(P),
series,
none: P.overlaySeries(rows, journal, [], {from: 0, to: 1000}),
tops: [P.overlayTop(series), P.overlayTop(P.overlaySeries(rows, journal, ["velocity"], {from: 0, to: 1000})), P.overlayTop([])],
labels: [null, "x", {}, {metrics: []}, {metrics: {}}, {metrics: {fix: null}}, {metrics: {fix: {emoji: 1, label: "Fix"}}},
{metrics: {fix: {emoji: "🎯", label: "Fix"}, bad: {emoji: "x"}, score: {emoji: "🏆", label: "Score"}}}].map(P.overlayLabels),
}));
"""
BURN_PROBE = """
const fs = require("fs");
const [source, plan, labelFile] = process.argv.slice(1);
const P = require(plan);
const labels = JSON.parse(fs.readFileSync(labelFile, "utf8"));
const node = (tag, attrs, children) => ({tag, attrs, children: children.flat(Infinity).filter((child) => child !== null && child !== undefined && child !== false), append(...more) { this.children.push(...more); }});
const s = (tag, attrs, ...children) => node(tag, attrs, children);
const LEAF = /^[A-Za-z]{1,32}\\.[A-Za-z]{1,32}$/u;
const METRICS = Object.keys(labels.metrics);
const kept = {value: [], listeners: [], updates: 0};
const known = (path) => {
if (path !== "burn.overlays") {
throw new Error(`unknown view path ${path}`);
}
};
const ViewStore = {
get: (path) => {
known(path);
return [...kept.value];
},
set: (path) => {
known(path);
throw new Error(`${path} holds a list: change it through update`);
},
update: (path, change) => {
if (!LEAF.test(String(path)) || typeof change !== "function") {
throw new Error(`update needs a section.key path and a change function, not ${path}`);
}
known(path);
const next = change([...kept.value]);
if (!Array.isArray(next) || !next.every((item) => METRICS.includes(item)) || new Set(next).size !== next.length) {
throw new Error(`burn.overlays must list distinct metrics, not ${JSON.stringify(next)}`);
}
kept.value = next;
kept.updates += 1;
return Promise.resolve(true);
},
on: (path, callback) => {
known(path);
kept.listeners.push([path, callback]);
},
};
const rendered = [];
const LegendChips = {render: (series, path, mode) => {
if (mode !== "hidden" && mode !== "shown") {
throw new Error(`a legend chip mode must be hidden or shown, not ${mode}`);
}
if (!LEAF.test(String(path)) || !Array.isArray(series) || !series.every((entry) => entry && Object.keys(entry).sort().join() === "id,label,tone" && ["id", "label", "tone"].every((name) => typeof entry[name] === "string"))) {
throw new Error("legend chips need a section.key path and series [{id, label, tone}]");
}
const press = (id) => ViewStore.update(path, (list) => (list.includes(id) ? list.filter((item) => item !== id) : [...list, id]));
rendered.push({series, path, mode, press});
return node("div", {class: "legend-chips"}, []);
}};
const refused = (call) => {
try {
call();
return false;
} catch {
return true;
}
};
const contract = [
refused(() => LegendChips.render([{key: "fix", label: "Fix"}], "burn.overlays", "shown")),
refused(() => LegendChips.render([{id: "fix", label: "Fix", tone: "k-fix"}], "burn.overlays", "on")),
refused(() => LegendChips.render([{id: "fix", label: "Fix", tone: "k-fix"}], "burn.overlays")),
refused(() => ViewStore.set("burn.overlays", ["fix"])),
refused(() => ViewStore.update("burn.overlays", () => [null])),
];
const journal = [
{time: 600, branch: "feature/a", values: {fix: 95, break: 2, tokens: 97, velocity: null, security: 100, score: 90}},
{time: 300, branch: "feature/a", values: {fix: 92, break: 2, tokens: 97, velocity: null, security: 100, score: null}},
{time: 400, branch: "feature/other", values: {fix: 10, break: 2, tokens: 97, velocity: null, security: 100, score: 10}},
];
const fetched = [];
const fetch = async (url) => {
fetched.push(url);
return {ok: true, json: async () => ({since: 0, until: 1000, labels, rows: journal})};
};
const counts = {draws: 0};
const taskView = {sig: "drawn"};
const drawTasks = () => {
counts.draws += 1;
};
const build = new Function("Plan", "s", "h", "fixed", "stampOf", "absolute", "ViewStore", "LegendChips", "fetch", "drawTasks", "taskView", "ui", `${fs.readFileSync(source, "utf8")}\\nreturn {burnOverlays, overlaysReady, burnRight};`);
const helpers = [P, s, s, (value) => value.toFixed(1), (ts) => new Date(ts * 1000), {format: (date) => date.toISOString()}];
const api = build(...helpers, ViewStore, LegendChips, fetch, drawTasks, taskView, {});
const inert = build(...helpers, undefined, undefined, fetch, drawTasks, taskView, {});
const brokenStore = {get: () => ["fix"], update: () => Promise.resolve(true), on: () => {}};
const brokenFetch = async () => ({ok: true, json: async () => ({labels: {}, rows: "x"})});
const broken = build(...helpers, brokenStore, LegendChips, brokenFetch, () => {}, {sig: ""}, {});
const picked = {rows: [{task: "work/a", branch: "feature/a", worktree_branch: ""}], from: 0, to: 1000};
const shape = (svg) => ({
overlays: svg.children.filter((child) => child.tag === "g").map((group) => ({cls: group.attrs.class, key: group.attrs["data-overlay"], points: group.children.filter((child) => child.tag === "circle").map((circle) => [circle.attrs.cx, circle.attrs.cy])})),
axis: svg.children.filter((child) => child.tag === "text").map((text) => [text.attrs.x, text.children[0]]),
});
(async () => {
const pending = api.burnOverlays(s("svg", {}), picked, (t) => t, api.burnRight());
const brokenFirst = broken.burnOverlays(s("svg", {}), picked, (t) => t, broken.burnRight());
await new Promise((resolve) => setTimeout(resolve, 0));
const brokenSvg = s("svg", {});
const brokenAfter = broken.burnOverlays(brokenSvg, picked, (t) => t, broken.burnRight());
const brokenView = [brokenFirst, brokenAfter, brokenSvg.children.length, broken.burnRight()];
const off = s("svg", {});
const rights = {inert: [inert.overlaysReady(), inert.burnRight()], off: api.burnRight()};
const offChips = api.burnOverlays(off, picked, (t) => t, rights.off);
const stored = [await rendered[0].press("score"), await rendered[0].press("fix")];
const pressed = {stored, value: [...kept.value], updates: kept.updates};
rights.on = api.burnRight();
const on = s("svg", {});
api.burnOverlays(on, picked, (t) => t, rights.on);
const drawsBefore = counts.draws;
taskView.sig = "drawn";
kept.listeners[0][1]();
console.log(JSON.stringify({
ready: api.overlaysReady(), pending, fetched, firstDraws: drawsBefore, listeners: kept.listeners.map(([path]) => path), redrawn: [counts.draws - drawsBefore, taskView.sig],
off: shape(off), offChips: offChips.attrs.class, chips: rendered[0], on: shape(on), rights, brokenView, contract, pressed,
}));
})();
"""
class TestOverlaySeries(unittest.TestCase):
@classmethod
def setUpClass(cls):
done = subprocess.run(["node", "-e", SERIES_PROBE, str(PLAN)], capture_output=True, text=True, timeout=60, check=True)
cls.data = json.loads(done.stdout)
def test_only_rows_of_the_scope_branches_within_the_span_are_drawn_in_the_given_order(self):
self.assertIn("overlaySeries", self.data["keys"])
self.assertNotIn("OVERLAYS", self.data["keys"])
self.assertEqual(self.data["series"], [
{"key": "fix", "points": [{"t": 100, "value": 92, "branch": "fix/b"}, {"t": 300, "value": 95, "branch": "feature/a"}]},
{"key": "tokens", "points": []},
{"key": "score", "points": [{"t": 300, "value": 90, "branch": "feature/a"}]},
])
self.assertEqual(self.data["none"], [])
def test_the_right_axis_runs_to_100_or_the_largest_value_above_it(self):
self.assertEqual(self.data["tops"], [100, 130, 100])
def test_labels_of_another_shape_switch_the_overlays_off(self):
self.assertEqual(self.data["labels"], [None, None, None, None, None, None, None, {"fix": "🎯 Fix", "score": "🏆 Score"}])
class TestBurnOverlays(unittest.TestCase):
@classmethod
def setUpClass(cls):
done = subprocess.run(["node", "-e", BURN_PROBE, str(GANTT), str(PLAN), str(LABELS)], capture_output=True, text=True, timeout=60, check=True)
cls.data = json.loads(done.stdout)
def test_six_metric_chips_all_off_on_first_load_keyed_burn_overlays(self):
data = self.data
self.assertEqual((data["ready"], data["pending"], data["fetched"], data["firstDraws"]), (True, None, ["/api/scores?from=0&to=1000"], 1))
self.assertEqual((data["chips"]["path"], data["chips"]["mode"]), ("burn.overlays", "shown"))
self.assertEqual(data["chips"]["series"], [
{"id": key, "label": label, "tone": f"k-{key}"}
for key, label in (("fix", "🎯 Fix"), ("break", "💥 Break"), ("tokens", "💸 Tokens"), ("velocity", "⏱️ Velocity"), ("security", "🛡️ Security"), ("score", "🏆 Score"))
])
self.assertEqual((data["off"], data["offChips"]), ({"overlays": [], "axis": []}, "legend-chips"))
def test_the_contract_stub_refuses_another_shape_mode_or_write(self):
self.assertEqual(self.data["contract"], [True] * 5)
def test_a_chip_press_stores_its_metric_through_update(self):
self.assertEqual(self.data["pressed"], {"stored": [True, True], "value": ["score", "fix"], "updates": 2})
def test_labels_of_another_shape_draw_no_overlay_and_never_throw(self):
self.assertEqual(self.data["brokenView"], [None, None, 0, 24])
def test_the_right_margin_widens_only_while_an_overlay_is_on(self):
self.assertEqual(self.data["rights"], {"inert": [False, 24], "off": 24, "on": 56})
def test_a_switched_on_chip_draws_its_series_on_the_right_axis(self):
on = self.data["on"]
self.assertEqual(on["axis"], [[850, "0 %"], [850, "50 %"], [850, "100 %"]])
self.assertEqual(on["overlays"], [
{"cls": "burn-overlay k-fix", "key": "fix", "points": [["300.0", "42.4"], ["600.0", "34.0"]]},
{"cls": "burn-overlay k-score", "key": "score", "points": [["600.0", "48.0"]]},
])
def test_a_view_store_change_redraws_the_burndown(self):
self.assertEqual(self.data["listeners"], ["burn.overlays"])
self.assertEqual(self.data["redrawn"], [1, ""])
class TestOverlayTones(unittest.TestCase):
def test_every_metric_chip_has_a_tone_colour(self):
css = (HUB / "templates" / "style-tasks.css").read_text(encoding="utf-8")
for key in json.loads(LABELS.read_text(encoding="utf-8"))["metrics"]:
with self.subTest(key=key):
self.assertRegex(css, rf"\.burn-box \.legend-chip\.k-{key}\s*\{{\s*--c: var\(--[a-z]+\); \}}")
if __name__ == "__main__":
unittest.main()
+60
View File
@@ -0,0 +1,60 @@
from __future__ import annotations
import json
import unittest
from hub_case import HubCase
from score_fixture import REPO_ROOT, Fixture, backfill
LABELS = REPO_ROOT / "skills" / "kvb-check-score" / "labels.json"
T = 1_760_000_000
class TestHubScores(HubCase):
def fetch(self, query: str) -> tuple[int, bytes]:
status, _, body = self.get(f"/api/scores?{query}")
return status, body
def test_the_hub_serves_the_backfilled_journal_rows_of_a_window(self):
fixture = Fixture(self.council.parent / "fixture")
done = backfill(self.council, "--repo", str(fixture.a))
self.assertEqual(done.returncode, 0, done.stderr)
begin, end = fixture.time_of(fixture.new), fixture.time_of(fixture.estimated)
status, body = self.fetch(f"from={begin}&to={end}")
self.assertEqual(status, 200, body)
found = json.loads(body)
self.assertEqual((found["since"], found["until"]), (begin, end))
self.assertEqual(found["labels"], json.loads(LABELS.read_text(encoding="utf-8")))
self.assertEqual([(row["sha"], row["branch"]) for row in found["rows"]], [(fixture.new, "feature/two"), (fixture.estimated, "main")])
self.assertEqual([row["values"]["score"] for row in found["rows"]], [81, 52])
def test_broken_lines_and_rows_without_values_are_left_out_in_time_order(self):
self.council.mkdir(exist_ok=True)
late = {"time": T, "branch": "feature/a", "values": {"score": 90}}
early = {"time": T - 10, "branch": "feature/a", "values": {"score": 80}}
lines = [json.dumps(late), "{broken", json.dumps({"time": T}), json.dumps({"time": "x", "values": {}}), json.dumps(early), json.dumps({**late, "time": T + 500})]
(self.council / "scores.jsonl").write_text("\n".join(lines) + "\n", encoding="utf-8")
status, body = self.fetch(f"from={T - 100}&to={T + 100}")
self.assertEqual(status, 200, body)
self.assertEqual(json.loads(body)["rows"], [early, late])
self.assertEqual(self.fetch(f"from={T + 10}&to={T}")[0], 400)
def test_a_label_file_of_another_shape_is_served_as_null(self):
skills = self.council.parent / "skills"
labels = skills / "kvb-check-score" / "labels.json"
labels.parent.mkdir(parents=True)
self.scripts.skills = skills
good = {"metrics": {"fix": {"emoji": "🎯", "label": "Fix", "former": []}}}
for text, served in (("{}", None), ('{"metrics": []}', None), ('{"metrics": {}}', None), ('{"metrics": {"fix": {"emoji": 1, "label": "Fix"}}}', None), ("{broken", None), (json.dumps(good), good)):
with self.subTest(labels=text):
labels.write_text(text, encoding="utf-8")
status, body = self.fetch(f"from={T}&to={T + 1}")
self.assertEqual((status, json.loads(body)["labels"]), (200, served))
def test_a_missing_journal_answers_no_rows(self):
status, body = self.fetch(f"from={T}&to={T + 1}")
self.assertEqual((status, json.loads(body)["rows"]), (200, []))
if __name__ == "__main__":
unittest.main()
+25 -24
View File
@@ -12,11 +12,12 @@ from pathlib import Path
REPO_ROOT = Path(__file__).resolve().parents[1]
SCORE = REPO_ROOT / "skills" / "kvb-check-score" / "scripts" / "score.py"
PASS = "✅ gate: 90/5, cost and time <= 100, security 100 %"
PASS = "✅ gate: 90/5, tokens and velocity <= 100, security 100 %"
BASE = {"fix": "95", "break": "2", "efficiency": "100", "speed": "100", "security": "100", "critical": "0"}
HEADER = "| 🎯 Fix | 💥 Break | 💸 Cost | ⏱️ Time | 🛡️ Security | 🏆 Score | Gate |"
HEADER = "| 🎯 Fix | 💥 Break | 💸 Tokens | ⏱️ Velocity | 🛡️ Security | 🏆 Score | Gate |"
ASK = ": ask the operator before the commit"
ENV = {"PATH": os.environ["PATH"]}
COUNCIL = tempfile.TemporaryDirectory()
ENV = {"PATH": os.environ["PATH"], "AGENTS_COUNCIL_ROOT": COUNCIL.name}
def flags(values: dict[str, str]) -> list[str]:
@@ -37,12 +38,12 @@ class TestScore(unittest.TestCase):
def test_formula_and_line_format(self):
self.assertEqual(
self._score(fix="95", **{"break": "4"}),
(["🎯 fix 95 % · 💥 break 4 % · 💸 cost 100 % · ⏱️ time 100 % · 🛡️ security 100 % · 🏆 score 91",
(["🎯 fix 95 % · 💥 break 4 % · 💸 tokens 100 % · ⏱️ velocity 100 % · 🛡️ security 100 % · 🏆 score 91",
PASS], 0),
)
lines, code = self._score(efficiency="97", speed="-")
self.assertEqual(
lines[0], "🎯 fix 95 % · 💥 break 2 % · 💸 cost 97 % · ⏱️ time – · 🛡️ security 100 % · 🏆 score 96"
lines[0], "🎯 fix 95 % · 💥 break 2 % · 💸 tokens 97 % · ⏱️ velocity – · 🛡️ security 100 % · 🏆 score 96"
)
self.assertEqual(code, 0)
self.assertEqual(self._score(fix="92.5")[0][0].split(" · ")[0], "🎯 fix 92.5 %")
@@ -51,14 +52,14 @@ class TestScore(unittest.TestCase):
lines, code = self._score(fix="92", efficiency="-", speed="-", **{"break": "3"})
self.assertEqual(
(lines, code),
(["🎯 fix 92 % · 💥 break 3 % · 💸 cost – · ⏱️ time – · 🛡️ security 100 % · 🏆 score 89", PASS], 0),
(["🎯 fix 92 % · 💥 break 3 % · 💸 tokens – · ⏱️ velocity – · 🛡️ security 100 % · 🏆 score 89", PASS], 0),
)
def test_security_multiplies_and_asks_below_100(self):
lines, code = self._score(fix="100", efficiency="-", speed="-", security="75", **{"break": "0"})
self.assertEqual(
(lines, code),
(["🎯 fix 100 % · 💥 break 0 % · 💸 cost – · ⏱️ time – · 🛡️ security 75 % · 🏆 score 75",
(["🎯 fix 100 % · 💥 break 0 % · 💸 tokens – · ⏱️ velocity – · 🛡️ security 75 % · 🏆 score 75",
f"⚠️ gate: security 75 % < 100{ASK}"], 3),
)
lines, code = self._score(security="~80")
@@ -94,7 +95,7 @@ class TestScore(unittest.TestCase):
({"fix": "80", "break": "10"}, "⛔ gate: fix 80 % < 90, break 10 % > 5"),
(
{"fix": "85", "efficiency": "150", "speed": "new", "security": "50"},
f"⛔ gate: fix 85 % < 90; cost 150 % > 100, time new, security 50 % < 100{ASK}",
f"⛔ gate: fix 85 % < 90; tokens 150 % > 100, velocity new, security 50 % < 100{ASK}",
),
):
with self.subTest(values=values):
@@ -103,13 +104,13 @@ class TestScore(unittest.TestCase):
def test_costs_ask_the_operator_exit_3(self):
for values, verdict in (
({"efficiency": "120"}, f"⚠️ gate: cost 120 % > 100{ASK}"),
({"efficiency": "-", "speed": "130"}, f"⚠️ gate: time 130 % > 100{ASK}"),
({"efficiency": "100.5", "speed": "-"}, f"⚠️ gate: cost 100.5 % > 100{ASK}"),
({"efficiency": "new", "speed": "-"}, f"⚠️ gate: cost new{ASK}"),
({"efficiency": "120"}, f"⚠️ gate: tokens 120 % > 100{ASK}"),
({"efficiency": "-", "speed": "130"}, f"⚠️ gate: velocity 130 % > 100{ASK}"),
({"efficiency": "100.5", "speed": "-"}, f"⚠️ gate: tokens 100.5 % > 100{ASK}"),
({"efficiency": "new", "speed": "-"}, f"⚠️ gate: tokens new{ASK}"),
(
{"efficiency": "120", "speed": "130", "security": "90"},
f"⚠️ gate: cost 120 % > 100, time 130 % > 100, security 90 % < 100{ASK}",
f"⚠️ gate: tokens 120 % > 100, velocity 130 % > 100, security 90 % < 100{ASK}",
),
):
with self.subTest(values=values):
@@ -128,7 +129,7 @@ class TestScore(unittest.TestCase):
def test_new_counts_as_100_in_the_score(self):
lines, _ = self._score(efficiency="new", speed="-", **{"break": "0"})
self.assertEqual(
lines[0], "🎯 fix 95 % · 💥 break 0 % · 💸 cost new · ⏱️ time – · 🛡️ security 100 % · 🏆 score 95"
lines[0], "🎯 fix 95 % · 💥 break 0 % · 💸 tokens new · ⏱️ velocity – · 🛡️ security 100 % · 🏆 score 95"
)
def test_table(self):
@@ -138,7 +139,7 @@ class TestScore(unittest.TestCase):
[
HEADER,
"| --- | --- | --- | --- | --- | --- | --- |",
"| 95 % | 2 % | 97 % | – | 100 % | 96 | ✅ 90/5, cost and time <= 100, security 100 % |",
"| 95 % | 2 % | 97 % | – | 100 % | 96 | ✅ 90/5, tokens and velocity <= 100, security 100 % |",
],
)
self.assertEqual(result.returncode, 0)
@@ -152,8 +153,8 @@ class TestScore(unittest.TestCase):
(result.stdout.splitlines(), result.returncode),
(
[
"🎯 Fix: 95 %", "💥 Break: 2 %", "💸 Cost: 97 %", "⏱️ Time: –", "🛡️ Security: 100 %",
"🏆 Score: 96", "🚦 Gate: ✅ 90/5, cost and time <= 100, security 100 %",
"🎯 Fix: 95 %", "💥 Break: 2 %", "💸 Tokens: 97 %", "⏱️ Velocity: –", "🛡️ Security: 100 %",
"🏆 Score: 96", "🚦 Gate: ✅ 90/5, tokens and velocity <= 100, security 100 %",
],
0,
),
@@ -165,7 +166,7 @@ class TestScore(unittest.TestCase):
(result.stdout.splitlines(), result.returncode),
(
[
"🎯 Fix: 95 %", "💥 Break: 2 %", "💸 Cost: 100 %", "⏱️ Time: 100 %", "🛡️ Security: ~80 %",
"🎯 Fix: 95 %", "💥 Break: 2 %", "💸 Tokens: 100 %", "⏱️ Velocity: 100 %", "🛡️ Security: ~80 %",
"💰 Value: 1:20", "🏆 Score: 74", f"🚦 Gate: ⚠️ security ~80 % < 100{ASK}",
],
3,
@@ -188,9 +189,9 @@ class TestScore(unittest.TestCase):
def test_value_column_changes_neither_score_nor_gate(self):
for value, verdict, code in (
("1:20", "✅ 90/5, cost and time <= 100, security 100 %", 0),
("4.1:1", "✅ 90/5, cost and time <= 100, security 100 %", 0),
("?", "⚠️ cost 120 % > 100" + ASK, 3),
("1:20", "✅ 90/5, tokens and velocity <= 100, security 100 %", 0),
("4.1:1", "✅ 90/5, tokens and velocity <= 100, security 100 %", 0),
("?", "⚠️ tokens 120 % > 100" + ASK, 3),
):
with self.subTest(value=value):
efficiency = "97" if code == 0 else "120"
@@ -201,7 +202,7 @@ class TestScore(unittest.TestCase):
self.assertEqual(
result.stdout.splitlines(),
[
HEADER.replace("💸 Cost |", "💸 Cost | 💰 Value |"),
HEADER.replace("💸 Tokens |", "💸 Tokens | 💰 Value |"),
"|" + " --- |" * 8,
f"| 95 % | 2 % | {efficiency} % | {value} | – | 100 % | {score} | {verdict} |",
],
@@ -236,7 +237,7 @@ class TestScore(unittest.TestCase):
(repo / "b.txt").write_text("three\nfour\n", encoding="utf-8")
git("add", "a.txt", "b.txt")
head = ["🌿 Branch", "📊 Diff stat"]
score = "| 95 % | 2 % | 100 % | 100 % | 100 % | 93 | ✅ 90/5, cost and time <= 100, security 100 % |"
score = "| 95 % | 2 % | 100 % | 100 % | 100 % | 93 | ✅ 90/5, tokens and velocity <= 100, security 100 % |"
worktree = ["", "📁 Worktree:", "", "```text", str(repo.resolve()), "```"]
result = self._run(*flags(BASE), "--commit-table", str(repo), cwd=repo)
@@ -259,7 +260,7 @@ class TestScore(unittest.TestCase):
result = self._run(*flags(BASE), "--commit-table", str(repo), "--commit", "HEAD", "--value", "?", cwd=repo)
self.assertEqual(
result.stdout.splitlines()[0],
"| " + " | ".join([*head, "🔖 Commit"]) + " " + HEADER.replace("💸 Cost |", "💸 Cost | 💰 Value |"),
"| " + " | ".join([*head, "🔖 Commit"]) + " " + HEADER.replace("💸 Tokens |", "💸 Tokens | 💰 Value |"),
)
self.assertEqual(
result.stdout.splitlines()[2],
+165
View File
@@ -0,0 +1,165 @@
"""Exercise backfill.py of kvb-check-score over a fixture repository: rows, idempotence, determinism, labels, clones."""
from __future__ import annotations
import hashlib
import json
import os
import shutil
import subprocess
import sys
import tempfile
import unittest
from pathlib import Path
from score_fixture import (
BACKFILL,
ESTIMATED_BLOCK,
HANG,
NEW_BLOCK,
OLD_BLOCK,
REPO_ROOT,
Fixture,
backfill,
)
JOURNAL_NAME = "scores.jsonl"
LABELS = REPO_ROOT / "skills" / "kvb-check-score" / "labels.json"
UNKNOWN = ["backfill: unknown label Coverage, skipped", "backfill: unknown label Heat, skipped"]
class TestBackfill(unittest.TestCase):
@classmethod
def setUpClass(cls):
cls.tmp = tempfile.TemporaryDirectory()
cls.fixture = Fixture(Path(cls.tmp.name) / "fixture")
@classmethod
def tearDownClass(cls):
cls.tmp.cleanup()
def setUp(self):
scratch = tempfile.TemporaryDirectory()
self.addCleanup(scratch.cleanup)
self.scratch = Path(scratch.name)
def council(self, name: str) -> Path:
return self.scratch / name
def rows(self, council: Path) -> list[dict]:
return [json.loads(line) for line in (council / JOURNAL_NAME).read_text(encoding="utf-8").splitlines()]
def test_three_commits_with_blocks_give_three_rows_then_none(self):
council = self.council("one")
first = backfill(council, "--repo", str(self.fixture.a))
self.assertEqual(first.returncode, 0, first.stderr)
self.assertEqual(first.stdout, f"{self.fixture.a.resolve()}: 3 blocks, 3 rows added\n")
written = (council / JOURNAL_NAME).read_bytes()
second = backfill(council, "--repo", str(self.fixture.a))
self.assertEqual((second.returncode, second.stdout), (0, f"{self.fixture.a.resolve()}: 3 blocks, 0 rows added\n"))
self.assertEqual((council / JOURNAL_NAME).read_bytes(), written)
def test_two_runs_from_an_empty_journal_give_byte_identical_journals(self):
one, two = self.council("one"), self.council("two")
self.assertEqual(backfill(one, "--repo", str(self.fixture.a)).returncode, 0)
self.assertEqual(backfill(two, "--repo", str(self.fixture.a)).returncode, 0)
self.assertEqual((one / JOURNAL_NAME).read_bytes(), (two / JOURNAL_NAME).read_bytes())
def test_old_new_and_estimated_blocks_parse_and_a_malformed_one_is_skipped(self):
council = self.council("one")
backfill(council, "--repo", str(self.fixture.a))
rows = self.rows(council)
fixture = self.fixture
self.assertEqual([row["sha"] for row in rows], [fixture.old, fixture.new, fixture.estimated])
self.assertEqual([row["branch"] for row in rows], ["main", "feature/two", "main"])
self.assertEqual([row["time"] for row in rows], [fixture.time_of(sha) for sha in (fixture.old, fixture.new, fixture.estimated)])
self.assertEqual([row["values"] for row in rows], [
{"fix": 95, "break": 2, "tokens": 97, "velocity": None, "security": 100, "score": 96},
{"fix": 92, "break": 3, "tokens": 110, "velocity": 90, "security": 100, "score": 81},
{"fix": 90, "break": 5, "tokens": None, "velocity": 130, "security": 80, "score": 52},
])
self.assertEqual([row["digest"] for row in rows], [hashlib.sha256(block.encode("utf-8")).hexdigest() for block in (OLD_BLOCK, NEW_BLOCK, ESTIMATED_BLOCK)])
self.assertEqual({(row["remote"], row["path"]) for row in rows}, {(str(fixture.upstream), str(fixture.a.resolve()))})
def test_an_unknown_label_is_reported_once_per_label_and_skipped(self):
done = backfill(self.council("one"), "--repo", str(self.fixture.a))
self.assertEqual(done.stderr.splitlines(), UNKNOWN)
for row in self.rows(self.council("one")):
self.assertEqual(list(row["values"]), ["fix", "break", "tokens", "velocity", "security", "score"])
def test_old_labels_map_through_the_label_file(self):
labels = json.loads(LABELS.read_text(encoding="utf-8"))
labels["metrics"]["tokens"]["former"] = []
file = self.scratch / "labels.json"
file.write_text(json.dumps(labels, ensure_ascii=False), encoding="utf-8")
done = backfill(self.council("one"), "--repo", str(self.fixture.a), "--labels", str(file))
self.assertIn("backfill: unknown label Cost, skipped", done.stderr.splitlines())
self.assertIsNone(self.rows(self.council("one"))[0]["values"]["tokens"])
def test_a_clone_at_another_path_yields_the_same_keys(self):
one, two = self.council("one"), self.council("two")
backfill(one, "--repo", str(self.fixture.a))
backfill(two, "--repo", str(self.fixture.b))
keys = [[(row["remote"], row["sha"]) for row in self.rows(council)] for council in (one, two)]
self.assertEqual(keys[0], keys[1])
self.assertEqual([row["branch"] for row in self.rows(two)], ["main", "feature/two", "main"])
self.assertEqual({row["path"] for row in self.rows(two)}, {str(self.fixture.b.resolve())})
again = backfill(one, "--repo", str(self.fixture.b))
self.assertEqual(again.stdout, f"{self.fixture.b.resolve()}: 3 blocks, 0 rows added\n")
def test_since_reads_only_newer_commits_and_a_bad_repository_writes_nothing(self):
council = self.council("one")
done = backfill(council, "--repo", str(self.fixture.a), "--since", self.fixture.old)
self.assertEqual(done.stdout, f"{self.fixture.a.resolve()}: 2 blocks, 2 rows added\n")
empty = self.council("two")
failed = backfill(empty, "--repo", str(self.fixture.a), str(self.scratch))
self.assertEqual(failed.returncode, 2)
self.assertFalse((empty / JOURNAL_NAME).exists())
def test_a_fifo_or_a_linked_council_root_exits_2_with_one_stderr_line(self):
fifo = self.council("fifo")
fifo.mkdir()
os.mkfifo(fifo / JOURNAL_NAME)
real = self.scratch / "real"
real.mkdir()
linked = self.council("linked")
linked.symlink_to(real)
for council in (fifo, linked):
with self.subTest(council=council.name):
done = backfill(council, "--repo", str(self.fixture.a))
self.assertEqual(done.returncode, 2)
self.assertEqual(len(done.stderr.splitlines()), 1, done.stderr)
self.assertTrue(done.stderr.startswith(f"backfill: journal {council / JOURNAL_NAME}: "), done.stderr)
self.assertEqual(list(real.iterdir()), [])
def test_a_missing_nofollow_helper_is_named_in_the_one_stderr_line(self):
scripts = self.scratch.resolve() / "skills" / "kvb-check-score" / "scripts"
scripts.mkdir(parents=True)
for name in ("backfill.py", "journal.py"):
shutil.copy(BACKFILL.parent / name, scripts / name)
shutil.copy(LABELS, scripts.parent / "labels.json")
helper = scripts.parents[1] / "kvb-org-organize" / "scripts" / "nofollow.py"
council = self.council("missing")
env = {"PATH": os.environ["PATH"], "AGENTS_COUNCIL_ROOT": str(council)}
done = subprocess.run([sys.executable, "-B", str(scripts / "backfill.py"), "--repo", str(self.fixture.a)], env=env, capture_output=True, text=True, check=False, timeout=HANG)
self.assertEqual((done.returncode, done.stdout, done.stderr), (2, "", f"backfill: journal {council / JOURNAL_NAME}: helper {helper} unusable (FileNotFoundError): install or update kvb-org-organize\n"))
self.assertFalse(council.exists())
@unittest.skipIf(os.geteuid() == 0, "root writes into a mode 0500 directory")
def test_a_failed_append_prints_no_summary(self):
council = self.council("readonly")
council.mkdir(mode=0o500)
self.addCleanup(council.chmod, 0o700)
done = backfill(council, "--repo", str(self.fixture.a))
self.assertEqual((done.returncode, done.stdout), (2, ""))
self.assertEqual(len(done.stderr.splitlines()), 1, done.stderr)
self.assertTrue(done.stderr.startswith(f"backfill: journal {council / JOURNAL_NAME}: "), done.stderr)
self.assertEqual(list(council.iterdir()), [])
def test_a_repeated_repo_flag_reads_every_repository(self):
done = backfill(self.council("one"), "--repo", str(self.fixture.a), "--repo", str(self.fixture.b))
self.assertEqual(done.stdout.splitlines(), [f"{self.fixture.a.resolve()}: 3 blocks, 3 rows added", f"{self.fixture.b.resolve()}: 3 blocks, 0 rows added"])
if __name__ == "__main__":
unittest.main()
+343
View File
@@ -0,0 +1,343 @@
"""Exercise labels.json and the score journal of kvb-check-score: labels in every layout, one row per block, failed appends, no gate reader."""
from __future__ import annotations
import hashlib
import importlib.util
import json
import os
import re
import shutil
import stat
import subprocess
import sys
import tempfile
import time
import unittest
from pathlib import Path
from types import ModuleType
from unittest import mock
from score_fixture import git_env
REPO_ROOT = Path(__file__).resolve().parents[1]
SKILL = REPO_ROOT / "skills" / "kvb-check-score"
SCORE = SKILL / "scripts" / "score.py"
EFFICIENCY = REPO_ROOT / "skills" / "kvb-check-efficiency" / "scripts"
NOFOLLOW = REPO_ROOT / "skills" / "kvb-org-organize" / "scripts" / "nofollow.py"
HANG = 20
FORMER = re.compile(r"💸 Cost|⏱️ Time|(?i:💸 cost|⏱️ time|\bcost (?:and|or) time\b|\bcost, time\b|\b(?:cost|time) from the `kvb-check-(?:efficiency|speed)`)")
REMOTES = (
"git@github.com:org/repo.git",
"github.com:org/repo.git",
"GitHub.com:org/repo",
"https://github.com/org/repo",
"https://user@github.com:443/org/repo.git/",
"ssh://git@github.com/org/repo.git",
)
BASE = {"fix": "95", "break": "2", "efficiency": "97", "speed": "-", "security": "100", "critical": "0"}
VARIANTS = (
{},
{"efficiency": "120", "speed": "130"},
{"efficiency": "new", "speed": "new", "security": "~90"},
{"fix": "80", "break": "10", "efficiency": "150"},
{"critical": "1"},
)
LAYOUTS = ((), ("--table",), ("--message-block",))
OLD = re.compile(r"cost|time", re.IGNORECASE)
JOURNAL_NAME = "scores.jsonl"
READERS = {
"skills/kvb-check-score/SKILL.md",
"skills/kvb-check-score/scripts/journal.py",
"orchestrator/kvb_orchestrator/scores.py",
}
def flags(values: dict[str, str]) -> list[str]:
return [item for key, value in {**BASE, **values}.items() for item in (f"--{key}", value)]
def module(path: Path) -> ModuleType:
spec = importlib.util.spec_from_file_location(f"copied_{path.stem}", path)
found = importlib.util.module_from_spec(spec)
spec.loader.exec_module(found)
return found
class ScoreCase(unittest.TestCase):
def setUp(self):
tmp = tempfile.TemporaryDirectory()
self.addCleanup(tmp.cleanup)
self.tmp = Path(tmp.name)
self.council = self.tmp / "council"
self.env = {"PATH": os.environ["PATH"], "AGENTS_COUNCIL_ROOT": str(self.council)}
def score(self, script: Path, *args: str, cwd: Path) -> subprocess.CompletedProcess:
return subprocess.run([sys.executable, "-B", str(script), *args], cwd=cwd, env=self.env, capture_output=True, text=True, check=False)
def rows(self) -> list[dict]:
path = self.council / JOURNAL_NAME
return [json.loads(line) for line in path.read_text(encoding="utf-8").splitlines()] if path.exists() else []
def copy_skills(self, labels: str) -> Path:
"""Args:
labels: the text of the copied labels.json.
Returns:
The copied score.py, beside kvb-check-efficiency's copied tokens.py and measure.py.
"""
skill, efficiency = self.tmp / "skills" / "kvb-check-score", self.tmp / "skills" / "kvb-check-efficiency" / "scripts"
(skill / "scripts").mkdir(parents=True)
efficiency.mkdir(parents=True)
for name in ("score.py", "journal.py"):
shutil.copy(SKILL / "scripts" / name, skill / "scripts" / name)
for name in ("tokens.py", "measure.py"):
shutil.copy(EFFICIENCY / name, efficiency / name)
organize = self.tmp / "skills" / "kvb-org-organize" / "scripts"
organize.mkdir(parents=True)
shutil.copy(NOFOLLOW, organize / "nofollow.py")
(skill / "labels.json").write_text(labels, encoding="utf-8")
return skill / "scripts" / "score.py"
class TestScoreLabels(ScoreCase):
def test_line_table_and_block_print_tokens_and_velocity_never_cost_or_time(self):
expected = {(): ("💸 tokens", "⏱️ velocity"), ("--table",): ("| 💸 Tokens |", "| ⏱️ Velocity |"), ("--message-block",): ("💸 Tokens: ", "⏱️ Velocity: ")}
for values in VARIANTS:
for layout in LAYOUTS:
with self.subTest(values=values, layout=layout):
out = self.score(SCORE, *flags(values), *layout, cwd=self.tmp).stdout
for label in expected[layout]:
self.assertIn(label, out)
self.assertIsNone(OLD.search(out), out)
def test_no_skill_text_or_doc_names_the_former_labels(self):
found = [
f"{path.relative_to(REPO_ROOT)}:{number}"
for top in ("skills", "docs")
for path in sorted((REPO_ROOT / top).rglob("*"))
if path.is_file() and path.suffix != ".pyc"
for number, line in enumerate(path.read_text(encoding="utf-8", errors="replace").splitlines(), 1)
if FORMER.search(line)
]
self.assertEqual(found, [])
labels = json.loads((SKILL / "labels.json").read_text(encoding="utf-8"))["metrics"]
self.assertEqual({key: metric["former"] for key, metric in labels.items() if metric["former"]}, {"tokens": ["Cost", "Efficiency"], "velocity": ["Time", "Speed"]})
self.assertEqual((labels["tokens"]["emoji"], labels["velocity"]["emoji"]), ("💸", "⏱️"))
def test_a_label_changed_in_a_temporary_label_file_shows_in_every_layout(self):
labels = json.loads((SKILL / "labels.json").read_text(encoding="utf-8"))
labels["metrics"]["tokens"]["label"] = "Pennies"
labels["metrics"]["velocity"].update(emoji="🐢", label="Pace")
script = self.copy_skills(json.dumps(labels, ensure_ascii=False))
line = self.score(script, *flags({}), cwd=self.tmp).stdout.splitlines()
self.assertEqual(line[0], "🎯 fix 95 % · 💥 break 2 % · 💸 pennies 97 % · 🐢 pace – · 🛡️ security 100 % · 🏆 score 96")
self.assertEqual(line[1], "✅ gate: 90/5, pennies and pace <= 100, security 100 %")
table = self.score(script, *flags({}), "--table", "--value", "1:2", cwd=self.tmp).stdout.splitlines()
self.assertEqual(table[0], "| 🎯 Fix | 💥 Break | 💸 Pennies | 💰 Value | 🐢 Pace | 🛡️ Security | 🏆 Score | Gate |")
block = self.score(script, *flags({}), "--message-block", cwd=self.tmp).stdout.splitlines()
self.assertEqual(block[2:4], ["💸 Pennies: 97 %", "🐢 Pace: –"])
efficiency = self.tmp / "skills" / "kvb-check-efficiency" / "scripts"
tokens = module(efficiency / "tokens.py")
name = tokens.labelled(("tokens",))["tokens"]
self.assertEqual(tokens.headline(name, 97, 100, 97, "listing 3→3 words"), "💸 pennies 97 % (skills 100→97 tokens; listing 3→3 words)")
self.assertEqual(tokens.headline(name, None, 0, 50, "listing 3→5 words"), "💸 pennies new (+50 tokens; listing 3→5 words)")
side = {"cpu": 1.0, "ram": 10.0, "wall": 2.0}
found = {"estimated": False, "base": side, "head": side, "efficiency": 97, "speed": 90, "runs": 1}
measure = module(efficiency / "measure.py")
self.assertEqual(measure.line(found, measure.labelled(("tokens", "velocity"))), "💸 pennies 97 % (cpu 1.00→1.00 s, ram 10.0→10.0 MB) · 🐢 pace 90 % (wall 2.00→2.00 s)")
def test_a_malformed_label_file_exits_2_or_4_with_one_stderr_line_in_every_layout(self):
labels = json.loads((SKILL / "labels.json").read_text(encoding="utf-8"))
broken = [
"{broken",
json.dumps({"metrics": [], "extras": {}}),
json.dumps({**labels, "metrics": {key: value for key, value in labels["metrics"].items() if key != "velocity"}}),
json.dumps({**labels, "metrics": {**labels["metrics"], "fix": {"emoji": 1, "label": "Fix", "former": []}}}),
]
script = self.copy_skills("{}")
cases = [(index, text, layout, critical, code) for index, text in enumerate(broken) for layout in LAYOUTS for critical, code in (("0", 2), ("1", 4))]
for index, text, layout, critical, code in cases:
(script.parents[1] / "labels.json").write_text(text, encoding="utf-8")
with self.subTest(labels=index, layout=layout, critical=critical):
done = self.score(script, *flags({"critical": critical}), *layout, cwd=self.tmp)
self.assertEqual((done.returncode, done.stdout), (code, ""))
self.assertEqual(len(done.stderr.splitlines()), 1, done.stderr)
self.assertTrue(done.stderr.startswith("score: labels "), done.stderr)
class TestJournalPlace(unittest.TestCase):
def setUp(self):
self.journal = module(SKILL / "scripts" / "journal.py")
def test_an_empty_council_root_means_the_default(self):
for value in ("", None):
with self.subTest(value=value), mock.patch.dict(os.environ, {} if value is None else {"AGENTS_COUNCIL_ROOT": value}):
if value is None:
os.environ.pop("AGENTS_COUNCIL_ROOT", None)
self.assertEqual(self.journal.journal_path(), Path("/tmp/agents-council/scores.jsonl"))
def test_ssh_scp_and_https_remotes_share_one_key(self):
self.assertEqual({self.journal.identity(url) for url in REMOTES}, {"github.com/org/repo"})
for url in ("/srv/git/repo.git", "./repo", "file:///srv/git/repo.git", None):
with self.subTest(url=url):
self.assertEqual(self.journal.identity(url), url)
class TestScoreJournal(ScoreCase):
def repo(self) -> Path:
repo = self.tmp / "repo"
repo.mkdir()
env = git_env(self.tmp)
for args in (("init", "-q", "-b", "feature/journal"), ("remote", "add", "origin", "git@github.com:example/fixture.git"), ("commit", "-q", "--allow-empty", "-m", "base")):
subprocess.run(["git", *args], cwd=repo, env=env, capture_output=True, check=True)
return repo
def test_each_message_block_appends_exactly_one_row_with_the_journal_fields(self):
repo = self.repo()
before = int(time.time())
first = self.score(SCORE, *flags({}), "--message-block", cwd=repo)
second = self.score(SCORE, *flags({"security": "~80"}), "--message-block", "--value", "1:20", cwd=repo)
self.assertEqual((first.returncode, second.returncode, first.stderr, second.stderr), (0, 3, "", ""))
rows = self.rows()
self.assertEqual(len(rows), 2)
for row, done in zip(rows, (first, second)):
self.assertEqual(list(row), ["time", "remote", "path", "branch", "sha", "values", "digest"])
self.assertEqual((row["remote"], row["path"], row["branch"], row["sha"]), ("github.com/example/fixture", str(repo.resolve()), "feature/journal", None))
self.assertEqual(row["digest"], hashlib.sha256(done.stdout.rstrip("\n").encode("utf-8")).hexdigest())
self.assertLessEqual(before, row["time"])
self.assertLessEqual(row["time"], int(time.time()))
self.assertEqual(rows[0]["values"], {"fix": 95, "break": 2, "tokens": 97, "velocity": None, "security": 100, "score": 96})
self.assertEqual(rows[1]["values"], {"fix": 95, "break": 2, "tokens": 97, "velocity": None, "security": 80, "score": 77})
self.score(SCORE, *flags({}), cwd=repo)
self.score(SCORE, *flags({}), "--table", cwd=repo)
self.assertEqual(len(self.rows()), 2)
def test_a_failed_append_prints_one_stderr_line_and_keeps_the_exit_code(self):
self.council.write_text("not a directory", encoding="utf-8")
for values, code in (({}, 0), ({"efficiency": "120"}, 3), ({"break": "9"}, 1), ({"critical": "1"}, 4)):
with self.subTest(values=values):
done = self.score(SCORE, *flags(values), "--message-block", cwd=self.tmp)
self.assertEqual(done.returncode, code)
self.assertEqual(len(done.stderr.splitlines()), 1, done.stderr)
self.assertTrue(done.stderr.startswith(f"score: journal {self.council / JOURNAL_NAME}: "), done.stderr)
self.assertEqual(len(done.stdout.splitlines()), 7)
def test_a_deleted_working_directory_keeps_the_exit_code(self):
gone = self.tmp / "gone"
wrapper = "import os, subprocess, sys; os.chdir(sys.argv[1]); os.rmdir(sys.argv[1]); sys.exit(subprocess.run(sys.argv[2:]).returncode)"
for values, code in (({}, 0), ({"critical": "1"}, 4)):
with self.subTest(values=values):
gone.mkdir(exist_ok=True)
done = subprocess.run([sys.executable, "-B", "-c", wrapper, str(gone), sys.executable, "-B", str(SCORE), *flags(values), "--message-block"], env=self.env, capture_output=True, text=True, check=False)
self.assertEqual(done.returncode, code, done.stderr)
self.assertEqual(len(done.stderr.splitlines()), 1, done.stderr)
self.assertTrue(done.stderr.startswith(f"score: journal {self.council / JOURNAL_NAME}: "), done.stderr)
self.assertEqual(len(done.stdout.splitlines()), 7)
def test_git_output_outside_utf8_keeps_the_exit_code_and_the_row(self):
repo = self.repo()
subprocess.run([b"git", b"remote", b"set-url", b"origin", b"https://example.org/caf\xe9.git"], cwd=repo, env=git_env(self.tmp), capture_output=True, check=True)
done = self.score(SCORE, *flags({"critical": "1"}), "--message-block", cwd=repo)
self.assertEqual((done.returncode, done.stderr), (4, ""))
self.assertEqual(self.rows()[0]["remote"], "example.org/caf�")
def test_a_link_at_the_journal_is_not_followed(self):
self.council.mkdir()
target = self.tmp / "elsewhere"
target.write_text("kept\n", encoding="utf-8")
(self.council / JOURNAL_NAME).symlink_to(target)
done = self.score(SCORE, *flags({"critical": "1"}), "--message-block", cwd=self.tmp)
self.assertEqual(done.returncode, 4)
self.assertEqual(len(done.stderr.splitlines()), 1, done.stderr)
self.assertEqual(target.read_text(encoding="utf-8"), "kept\n")
def test_a_fifo_at_the_journal_does_not_block_the_exit_code(self):
self.council.mkdir()
os.mkfifo(self.council / JOURNAL_NAME)
done = subprocess.run([sys.executable, "-B", str(SCORE), *flags({"critical": "1"}), "--message-block"], cwd=self.tmp, env=self.env, capture_output=True, text=True, check=False, timeout=HANG)
self.assertEqual(done.returncode, 4)
self.assertEqual(len(done.stderr.splitlines()), 1, done.stderr)
self.assertTrue(done.stderr.startswith(f"score: journal {self.council / JOURNAL_NAME}: "), done.stderr)
def test_a_linked_council_root_is_refused(self):
real = self.tmp / "worktree"
real.mkdir()
(real / ".git").write_text("gitdir: /elsewhere\n", encoding="utf-8")
self.council.symlink_to(real)
done = self.score(SCORE, *flags({"critical": "1"}), "--message-block", cwd=self.tmp)
self.assertEqual(done.returncode, 4)
self.assertEqual(len(done.stderr.splitlines()), 1, done.stderr)
self.assertEqual(sorted(path.name for path in real.iterdir()), [".git"])
def test_a_missing_or_broken_nofollow_helper_is_named_in_the_one_stderr_line(self):
script = self.copy_skills((SKILL / "labels.json").read_text(encoding="utf-8"))
helper = self.tmp.resolve() / "skills" / "kvb-org-organize" / "scripts" / "nofollow.py"
cases = {"FileNotFoundError": None, "AttributeError": "VERSION = 0\n", "SyntaxError": "def (:\n"}
if os.geteuid() != 0:
cases["PermissionError"] = NOFOLLOW.read_text(encoding="utf-8")
for kind, text in cases.items():
helper.unlink(missing_ok=True)
if text is not None:
helper.write_text(text, encoding="utf-8")
if kind == "PermissionError":
helper.chmod(0)
for values, code in (({}, 0), ({"critical": "1"}, 4)):
with self.subTest(kind=kind, values=values):
done = self.score(script, *flags(values), "--message-block", cwd=self.tmp)
self.assertEqual(done.returncode, code)
self.assertEqual(done.stderr, f"score: journal {self.council / JOURNAL_NAME}: helper {helper} unusable ({kind}): install or update kvb-org-organize\n")
self.assertEqual(len(done.stdout.splitlines()), 7)
self.assertFalse(self.council.exists())
def test_a_branch_or_commit_outside_utf8_keeps_the_score_exit(self):
repo = self.tmp / "latin"
env = git_env(self.tmp)
subprocess.run(["git", "init", "-q", "-b", "caf\udce9", str(repo)], env=env, capture_output=True, check=True)
subprocess.run(["git", "commit", "-q", "--allow-empty", "-m", "base"], cwd=repo, env=env, capture_output=True, check=True)
for values, code in (({}, 0), ({"critical": "1"}, 4)):
with self.subTest(values=values):
done = self.score(SCORE, *flags(values), "--commit-table", str(repo), cwd=self.tmp)
self.assertEqual((done.returncode, done.stderr), (code, ""))
self.assertIn("| caf� |", done.stdout)
failed = self.score(SCORE, *flags(values), "--commit-table", str(repo), "--commit", "\udce9", cwd=self.tmp)
self.assertEqual((failed.returncode, failed.stdout), (code or 2, ""))
self.assertEqual(len(failed.stderr.splitlines()), 1, failed.stderr)
self.assertTrue(failed.stderr.startswith("score: git show in "), failed.stderr)
def test_a_critical_finding_wins_over_a_commit_table_git_cannot_read(self):
repo = self.repo()
plain = self.tmp / "plain"
plain.mkdir()
self.env["GIT_CEILING_DIRECTORIES"] = str(self.tmp)
tables = {"non-git": (str(plain),), "deleted": (str(self.tmp / "deleted"),), "unknown commit": (str(repo), "--commit", "0" * 40)}
for name, (target, *commit) in tables.items():
for values, code in (({}, 2), ({"fix": "80"}, 2), ({"critical": "1"}, 4), ({"fix": "80", "critical": "1"}, 4)):
with self.subTest(table=name, values=values):
done = self.score(SCORE, *flags(values), "--commit-table", target, *commit, cwd=self.tmp)
self.assertEqual((done.returncode, done.stdout), (code, ""))
self.assertEqual(len(done.stderr.splitlines()), 1, done.stderr)
self.assertTrue(done.stderr.startswith("score: git "), done.stderr)
def test_a_new_journal_and_council_root_are_private(self):
self.score(SCORE, *flags({}), "--message-block", cwd=self.tmp)
self.assertEqual(stat.S_IMODE((self.council / JOURNAL_NAME).stat().st_mode), 0o600)
self.assertEqual(stat.S_IMODE(self.council.stat().st_mode), 0o700)
def test_no_gate_reads_the_journal(self):
found = set()
for top in ("skills", "orchestrator", "scripts"):
for path in (REPO_ROOT / top).rglob("*"):
text = path.suffix in {".py", ".sh", ".md", ".js", ".j2", ".json", ".yml"} and path.is_file()
if text and JOURNAL_NAME in path.read_text(encoding="utf-8", errors="replace"):
found.add(str(path.relative_to(REPO_ROOT)))
self.assertEqual(found, READERS)
importers = {str(path.relative_to(REPO_ROOT)) for path in (REPO_ROOT / "skills").rglob("*.py") if re.search(r"^from journal import|^import journal", path.read_text(encoding="utf-8"), re.MULTILINE)}
self.assertEqual(importers, {"skills/kvb-check-score/scripts/score.py", "skills/kvb-check-score/scripts/backfill.py"})
self.council.mkdir()
(self.council / JOURNAL_NAME).write_text('{"time": 1, "values": {"score": 0}}\n{broken\n', encoding="utf-8")
self.assertEqual(self.score(SCORE, *flags({}), cwd=self.tmp).returncode, 0)
if __name__ == "__main__":
unittest.main()