Merge branch 'feature/score-metric-overlays' into unify/skills-20261112-night
This commit is contained in:
commit
7d9f25e289
30 files changed
+1806
-142
No files matched your search
@@ -11,6 +11,7 @@ from .diffs import DIFF_BODY_LIMIT
|
||||
from .engines import STT_LIMIT
|
||||
from .exchange import PLAIN_CSP, Exchange
|
||||
from .resources import RESOURCES, sampled
|
||||
from .scores import scored
|
||||
|
||||
BODY_LIMIT = 1 << 20
|
||||
UPLOAD_LIMIT = 50 << 20
|
||||
@@ -119,6 +120,8 @@ class Handler(Exchange):
|
||||
self.answer(hub.repositories, HTTPStatus.OK)
|
||||
elif path == "/api/resources":
|
||||
self.answer(lambda: sampled(hub.kept / RESOURCES, hub.span(self.query(), time.time())), HTTPStatus.OK)
|
||||
elif path == "/api/scores":
|
||||
self.answer(lambda: scored(hub.council, hub.scripts.skills, hub.span(self.query(), time.time())), HTTPStatus.OK)
|
||||
elif path == "/api/subagents":
|
||||
self.answer(lambda: hub.subagents(self.query()), HTTPStatus.OK)
|
||||
elif path == "/api/runner/jobs":
|
||||
|
||||
@@ -0,0 +1,44 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
from .records import epoch, parse_line, read_json, read_regular
|
||||
|
||||
SCORES = "scores.jsonl"
|
||||
LABELS = "kvb-check-score/labels.json"
|
||||
FILE_LIMIT = 64 << 20
|
||||
|
||||
|
||||
def checked(labels: object) -> dict | None:
|
||||
"""Args:
|
||||
labels: the parsed label file, or None.
|
||||
|
||||
Returns:
|
||||
labels when its "metrics" is a non-empty object of entries with a string
|
||||
emoji and label, else None.
|
||||
"""
|
||||
metrics = labels.get("metrics") if isinstance(labels, dict) else None
|
||||
valid = isinstance(metrics, dict) and metrics and all(isinstance(metric, dict) and isinstance(metric.get("emoji"), str) and isinstance(metric.get("label"), str) for metric in metrics.values())
|
||||
return labels if valid else None
|
||||
|
||||
|
||||
def scored(council: Path, skills: Path, span: tuple[float, float]) -> dict:
|
||||
"""Args:
|
||||
council: the council directory holding scores.jsonl, which score.py
|
||||
and backfill.py of kvb-check-score append to.
|
||||
skills: the skills directory holding kvb-check-score/labels.json.
|
||||
span: (begin, end) of the burndown's time range in epoch seconds.
|
||||
|
||||
Returns:
|
||||
{"since", "until", "labels", "rows"}: the label file, None when
|
||||
unreadable or not as checked() wants it, and the journal rows whose time lies within the span, in
|
||||
time order.
|
||||
"""
|
||||
begin, end = span
|
||||
try:
|
||||
lines = read_regular(council / SCORES, FILE_LIMIT).splitlines()
|
||||
except (OSError, ValueError):
|
||||
lines = []
|
||||
rows = [row for row in map(parse_line, lines) if row is not None and isinstance(row.get("values"), dict)]
|
||||
inside = [row for row in rows if (moment := epoch(row.get("time"))) is not None and begin <= moment <= end]
|
||||
return {"since": begin, "until": end, "labels": checked(read_json(skills / LABELS)), "rows": sorted(inside, key=lambda row: row["time"])}
|
||||
@@ -1,5 +1,7 @@
|
||||
const GANTT = {label: 240, right: 28, top: 34, lane: 30, width: 1180, bar: 16, ticks: 4};
|
||||
const BURN = {width: 900, height: 340, left: 52, right: 24, top: 20, bottom: 40};
|
||||
const BURN = {width: 900, height: 340, left: 52, right: 24, axis: 56, top: 20, bottom: 40};
|
||||
const OVERLAY_PATH = "burn.overlays";
|
||||
const scoreView = {key: "", rows: [], labels: null, busy: false, bound: false};
|
||||
const EDGE_WORDS = {after: "Follows (rasci.py set --after)", base: "Branches from (base branch)", queue: "Waits behind (queue order)"};
|
||||
const SERIES_WORDS = {scope: "Scope (burnup)", done: "Done (burnup)", remaining: "Remaining (burndown)", unestimated: "Unestimated tasks"};
|
||||
const STEP_WORDS = [["started", "Started"], ["ended", "Ended"], ["joined", "Joined the scope"]];
|
||||
@@ -80,11 +82,11 @@
|
||||
return h("div", {class: "gantt-box"}, h("div", {class: "gantt-scroll"}, svg), legend);
|
||||
}
|
||||
|
||||
function burnAxis(svg, from, to, top) {
|
||||
const x = (t) => BURN.left + ((t - from) / Math.max(to - from, 1)) * (BURN.width - BURN.left - BURN.right);
|
||||
function burnAxis(svg, from, to, top, right) {
|
||||
const x = (t) => BURN.left + ((t - from) / Math.max(to - from, 1)) * (BURN.width - BURN.left - right);
|
||||
const y = (value) => BURN.top + (1 - value / top) * (BURN.height - BURN.top - BURN.bottom);
|
||||
for (const value of [0, top / 2, top]) {
|
||||
svg.append(s("line", {class: "grid", x1: BURN.left, x2: BURN.width - BURN.right, y1: fixed(y(value)), y2: fixed(y(value))}));
|
||||
svg.append(s("line", {class: "grid", x1: BURN.left, x2: BURN.width - right, y1: fixed(y(value)), y2: fixed(y(value))}));
|
||||
svg.append(s("text", {class: "axis", x: BURN.left - 8, y: fixed(y(value)), dy: "0.35em", "text-anchor": "end"}, String(Math.round(value * 10) / 10)));
|
||||
}
|
||||
for (const [index, t] of [from, (from + to) / 2, to].entries()) {
|
||||
@@ -165,6 +167,68 @@
|
||||
return {reach, end: forecast === null ? picked.to : Math.min(forecast, reach)};
|
||||
}
|
||||
|
||||
const overlaysReady = () => typeof ViewStore === "object" && typeof LegendChips === "object";
|
||||
|
||||
async function fetchScores(key, from, to) {
|
||||
scoreView.busy = true;
|
||||
scoreView.key = key;
|
||||
try {
|
||||
const response = await fetch(`/api/scores?from=${Math.floor(from)}&to=${Math.ceil(to)}`, {cache: "no-store"});
|
||||
if (response.ok) {
|
||||
const found = await response.json();
|
||||
scoreView.rows = Array.isArray(found.rows) ? found.rows.filter((row) => row && row.values && typeof row.values === "object") : [];
|
||||
scoreView.labels = Plan.overlayLabels(found.labels);
|
||||
}
|
||||
} catch {
|
||||
ui.offline = true;
|
||||
}
|
||||
scoreView.busy = false;
|
||||
taskView.sig = "";
|
||||
drawTasks();
|
||||
}
|
||||
|
||||
function overlayKeys() {
|
||||
const kept = ViewStore.get(OVERLAY_PATH);
|
||||
return Array.isArray(kept) && scoreView.labels ? Object.keys(scoreView.labels).filter((key) => kept.includes(key)) : [];
|
||||
}
|
||||
|
||||
const burnRight = () => (overlaysReady() && overlayKeys().length ? BURN.axis : BURN.right);
|
||||
|
||||
function burnOverlays(svg, picked, x, right) {
|
||||
const key = `${Math.floor(picked.from)}:${Math.floor(picked.to / 60)}`;
|
||||
if (key !== scoreView.key && !scoreView.busy) {
|
||||
fetchScores(key, picked.from, picked.to);
|
||||
}
|
||||
if (!scoreView.bound) {
|
||||
scoreView.bound = true;
|
||||
ViewStore.on(OVERLAY_PATH, () => {
|
||||
taskView.sig = "";
|
||||
drawTasks();
|
||||
});
|
||||
}
|
||||
if (!scoreView.labels) {
|
||||
return null;
|
||||
}
|
||||
const series = Plan.overlaySeries(picked.rows, scoreView.rows, overlayKeys(), picked);
|
||||
const top = Plan.overlayTop(series);
|
||||
const y = (value) => BURN.top + (1 - value / top) * (BURN.height - BURN.top - BURN.bottom);
|
||||
if (series.length) {
|
||||
for (const value of [0, top / 2, top]) {
|
||||
svg.append(s("text", {class: "axis overlay-axis", x: BURN.width - right + 6, y: fixed(y(value)), dy: "0.35em", "text-anchor": "start"}, `${Math.round(value)} %`));
|
||||
}
|
||||
}
|
||||
for (const {key: name, points} of series) {
|
||||
const d = points.map((point, index) => `${index ? "L" : "M"}${fixed(x(point.t))},${fixed(y(point.value))}`).join(" ");
|
||||
const group = s("g", {class: `burn-overlay k-${name}`, "data-overlay": name, "data-points": String(points.length)}, d ? s("path", {class: "burn-overlay-line", d}) : null);
|
||||
for (const point of points) {
|
||||
group.append(s("circle", {class: "burn-overlay-point", cx: fixed(x(point.t)), cy: fixed(y(point.value)), r: 3, "data-t": point.t, "data-value": point.value},
|
||||
s("title", {}, `${absolute.format(stampOf(point.t))} · ${scoreView.labels[name]}: ${point.value} % · ${point.branch}`)));
|
||||
}
|
||||
svg.append(group);
|
||||
}
|
||||
return LegendChips.render(Object.entries(scoreView.labels).map(([key, label]) => ({id: key, label, tone: `k-${key}`})), OVERLAY_PATH, "shown");
|
||||
}
|
||||
|
||||
function burnChart(rows, shown) {
|
||||
const picked = Plan.scoped(rows, taskView.scope, {filtered: shown, since: state.since, until: state.until, now: state.now, holders: lineHolders()});
|
||||
const series = Plan.burn(picked.rows, {estimation: estimation(), from: picked.from, to: picked.to}).points;
|
||||
@@ -177,7 +241,8 @@
|
||||
const top = Math.max(1, ...series.flatMap((point) => names.map((name) => point[name])), ...ends);
|
||||
const unit = estimation() === "points" ? "story points; unestimated tasks as a line of their own" : "tasks";
|
||||
const svg = s("svg", {class: "chart burn", viewBox: `0 0 ${BURN.width} ${BURN.height}`, role: "img", "aria-label": `Burndown and burnup in ${unit}; hover a point for the work packages of its step`});
|
||||
const {x, y} = burnAxis(svg, picked.from, reach, top);
|
||||
const right = burnRight();
|
||||
const {x, y} = burnAxis(svg, picked.from, reach, top, right);
|
||||
const last = series[series.length - 1];
|
||||
if (reach > picked.to) {
|
||||
svg.append(s("line", {class: "now", x1: fixed(x(picked.to)), x2: fixed(x(picked.to)), y1: BURN.top, y2: BURN.height - BURN.bottom}));
|
||||
@@ -191,10 +256,11 @@
|
||||
burnFinish(svg, segments, x, y);
|
||||
}
|
||||
svg.append(burnPoints(series, Plan.steps(picked.rows, series, picked.from), names, x, y));
|
||||
const overlays = overlaysReady() ? burnOverlays(svg, picked, x, right) : null;
|
||||
const stale = h("span", {class: "pill burn-stale", "data-tone": "warn", "data-stale": String(picked.stale.length), role: "img", ...named(`Stale: ${picked.stale.length}`, `Stale: ${picked.stale.length} tasks without a register change for ${Plan.STALE_DAYS} days, counted apart`)}, icon("bi-hourglass-bottom"), String(picked.stale.length));
|
||||
const legend = h("div", {class: "legend"}, names.map((name) => h("span", {class: `burn-key k-${name}`, "data-series": name}, `${SERIES_WORDS[name]}: ${last[name]}`)),
|
||||
settings.trend.mode === "off" ? null : finishNotes(trend, finish), stale);
|
||||
return h("div", {class: "burn-box"}, svg, legend, h("p", {class: "muted"}, `${picked.rows.length} tasks in scope, counted in ${unit}`));
|
||||
return h("div", {class: "burn-box", "data-panel": "burndown"}, svg, legend, overlays, h("p", {class: "muted"}, `${picked.rows.length} tasks in scope, counted in ${unit}`));
|
||||
}
|
||||
|
||||
async function chooseEstimation(id) {
|
||||
|
||||
@@ -262,6 +262,44 @@ const Plan = (() => {
|
||||
return {fixed: reachable(fixed), growing: found.forecast};
|
||||
}
|
||||
|
||||
const branchOf = (row) => (row.branch !== "-" ? row.branch : row.worktree_branch);
|
||||
|
||||
/**
|
||||
* @param rows - the burndown scope's task rows with branch ("-" for none) and worktree_branch ("" for none).
|
||||
* @param journal - /api/scores rows {time, branch, values: {metric key: number or null}}.
|
||||
* @param keys - the metric keys switched on, in the label file's order.
|
||||
* @param span - {from, to} in epoch seconds.
|
||||
* @returns one {key, points: [{t, value, branch}]} per key, in keys order: the journal rows of a branch of the
|
||||
* scope's rows within span whose value for the key is a number, in time order.
|
||||
*/
|
||||
function overlaySeries(rows, journal, keys, span) {
|
||||
const branches = new Set(rows.map(branchOf).filter(Boolean));
|
||||
const inside = journal.filter((entry) => branches.has(entry.branch) && entry.time >= span.from && entry.time <= span.to).sort((a, b) => a.time - b.time);
|
||||
return keys.map((key) => ({
|
||||
key, points: inside.filter((entry) => Number.isFinite(entry.values[key])).map((entry) => ({t: entry.time, value: entry.values[key], branch: entry.branch})),
|
||||
}));
|
||||
}
|
||||
|
||||
/**
|
||||
* @param labels - the labels of /api/scores: {metrics: {key: {emoji, label}}}, or anything else.
|
||||
* @returns {key: "<emoji> <label>"} in the file's order for the entries with a string emoji and label; null when
|
||||
* none has both or labels is no such object.
|
||||
*/
|
||||
function overlayLabels(labels) {
|
||||
const metrics = labels && typeof labels === "object" ? labels.metrics : null;
|
||||
if (!metrics || typeof metrics !== "object" || Array.isArray(metrics)) {
|
||||
return null;
|
||||
}
|
||||
const kept = Object.entries(metrics).filter(([, metric]) => metric && typeof metric.emoji === "string" && typeof metric.label === "string");
|
||||
return kept.length ? Object.fromEntries(kept.map(([key, metric]) => [key, `${metric.emoji} ${metric.label}`])) : null;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param series - overlaySeries().
|
||||
* @returns the top of the right axis: 100, or the largest value when one exceeds it.
|
||||
*/
|
||||
const overlayTop = (series) => Math.max(100, ...series.flatMap((found) => found.points.map((point) => point.value)));
|
||||
|
||||
const KEYS = {
|
||||
prio: () => null,
|
||||
wsjf: (row) => (row.wsjf ? row.wsjf.score : null),
|
||||
@@ -294,7 +332,7 @@ const Plan = (() => {
|
||||
return keyed.map((entry) => entry.row);
|
||||
}
|
||||
|
||||
return {FORECASTS, ESTIMATIONS, TRENDS, SORTS, SERIES, TRENDED, STATUS_ORDER, STALE_DAYS, median, medians, forecastLength, bars, etas, stale, scoped, burn, steps, trend, finishes, lineAt, sortRows};
|
||||
return {FORECASTS, ESTIMATIONS, TRENDS, SORTS, SERIES, TRENDED, STATUS_ORDER, STALE_DAYS, median, medians, forecastLength, bars, etas, stale, scoped, burn, steps, trend, finishes, lineAt, overlaySeries, overlayLabels, overlayTop, sortRows};
|
||||
})();
|
||||
if (typeof module === "object" && module.exports) {
|
||||
module.exports = Plan;
|
||||
|
||||
@@ -82,6 +82,15 @@ svg .burn-line.k-unestimated { stroke-dasharray: 6 4; }
|
||||
svg .burn-trend { fill: none; stroke: var(--c); stroke-width: 1.6; stroke-dasharray: 2 5; stroke-linecap: round; }
|
||||
svg .burn-point { fill: var(--c); stroke: transparent; stroke-width: 10; cursor: help; }
|
||||
svg .burn-point:hover { r: 6px; }
|
||||
svg .burn-overlay-line { fill: none; stroke: var(--c); stroke-width: 1.8; stroke-dasharray: 5 3; }
|
||||
svg .burn-overlay-point { fill: var(--c); stroke: transparent; stroke-width: 10; cursor: help; }
|
||||
svg.burn .overlay-axis { fill: var(--muted); }
|
||||
svg.burn .burn-overlay.k-fix, .burn-box .legend-chip.k-fix { --c: var(--ok); }
|
||||
svg.burn .burn-overlay.k-break, .burn-box .legend-chip.k-break { --c: var(--bad); }
|
||||
svg.burn .burn-overlay.k-tokens, .burn-box .legend-chip.k-tokens { --c: var(--warn); }
|
||||
svg.burn .burn-overlay.k-velocity, .burn-box .legend-chip.k-velocity { --c: var(--info); }
|
||||
svg.burn .burn-overlay.k-security, .burn-box .legend-chip.k-security { --c: var(--violet); }
|
||||
svg.burn .burn-overlay.k-score, .burn-box .legend-chip.k-score { --c: var(--accent); }
|
||||
svg .forecast-line { stroke: var(--ok); stroke-width: 1.4; stroke-dasharray: 4 3; }
|
||||
svg .forecast-label { fill: var(--ok); font-size: var(--fs-2xs); font-weight: 650; }
|
||||
.burn-forecast { display: inline-flex; align-items: center; gap: 6px; }
|
||||
|
||||
@@ -14,8 +14,8 @@ and never estimates one itself.
|
||||
1. **Fix confidence** (percent): probability the change does what it claims.
|
||||
**Break risk** (percent): probability it breaks anything else, from the
|
||||
`kvb-check-break-risk` skill. Two numbers, never one blended figure.
|
||||
2. **Score**: pass both, with cost from the `kvb-check-efficiency`
|
||||
skill, time from the `kvb-check-speed` skill and security and the
|
||||
2. **Score**: pass both, with tokens from the `kvb-check-efficiency`
|
||||
skill, velocity from the `kvb-check-speed` skill and security and the
|
||||
critical count from `security.py` of the `kvb-check-security` skill, to
|
||||
`${CLAUDE_SKILL_DIR}/../kvb-check-score/scripts/score.py --fix <n> --break
|
||||
<n> --efficiency <n|-> --speed <n|-> --security <n|~n> --critical <count>`
|
||||
|
||||
@@ -4,7 +4,7 @@ description: >
|
||||
Token cost of skill edits, CPU and RAM of code edits.
|
||||
---
|
||||
|
||||
💸 Cost is the share of the base's resources a change uses: under 100 %
|
||||
💸 Tokens is the share of the base's resources a change uses: under 100 %
|
||||
it saves, over 100 % it costs more. Paste the script's lines, never a
|
||||
paragraph about them. `<ref>` is the commit the change
|
||||
starts from: `HEAD` for uncommitted work, the merge base with the default
|
||||
@@ -20,7 +20,7 @@ It compares every changed `*.md` under `skills/`, untracked ones included,
|
||||
at `<ref>` with the working tree, and the always-loaded listing: all
|
||||
`description:` words, counted by the repository's `scripts/budget.py`.
|
||||
Tokens are an estimate, characters / 4. Changed files without any base text
|
||||
print `cost new`, which counts as over 100. Skills have no time.
|
||||
print `tokens new`, which counts as over 100. Skills have no velocity.
|
||||
|
||||
A second line, `💰 value 1:<n>` (`<n>:1` when value < cost), weighs both
|
||||
per day and never changes the exit code; pass its quotient or `?` to
|
||||
|
||||
@@ -9,26 +9,26 @@ Args:
|
||||
relative to each tree's root; its output is discarded.
|
||||
|
||||
Runs:
|
||||
One run per tree, base first. When cost and time are both at most 100 that
|
||||
One run per tree, base first. When tokens and velocity are both at most 100 that
|
||||
is the result; otherwise 4 more runs per tree, alternating, and the arithmetic mean
|
||||
of all 5 per tree counts.
|
||||
|
||||
Other load:
|
||||
A 1 s /proc/stat sample of system CPU busy before and after the runs, and per run
|
||||
the busy jiffies minus the command's own CPU time, as a share of all jiffies. When
|
||||
any share exceeds 30 % or cannot be read, the result is an estimate: time comes
|
||||
any share exceeds 30 % or cannot be read, the result is an estimate: velocity comes
|
||||
from the CPU-time ratio instead of wall time.
|
||||
|
||||
Output:
|
||||
`💸 cost <p> % (cpu <b>→<a> s, ram <b>→<a> MB) · ⏱️ time <p> % (wall <b>→<a> s)`,
|
||||
`💸 tokens <p> % (cpu <b>→<a> s, ram <b>→<a> MB) · ⏱️ velocity <p> % (wall <b>→<a> s)`,
|
||||
then ` · mean of 5` after the extra runs. An estimate marks both values with `~`,
|
||||
shows `(cpu <b>→<a> s)` for time and ends in `(estimated: other load <x> %)`, or
|
||||
`unknown` for <x>. Cost is the mean of the CPU ratio (user plus system seconds
|
||||
shows `(cpu <b>→<a> s)` for velocity and ends in `(estimated: other load <x> %)`, or
|
||||
`unknown` for <x>. Tokens is the mean of the CPU ratio (user plus system seconds
|
||||
of the command and its waited-for children) and the peak RSS ratio; each ratio is
|
||||
working tree over base, times 100.
|
||||
|
||||
Exit codes:
|
||||
0 cost and time at most 100, 2 usage error or a failing command
|
||||
0 tokens and velocity at most 100, 2 usage error or a failing command
|
||||
(its tree, exit code and last stderr line on stderr), 3 either over 100.
|
||||
"""
|
||||
|
||||
@@ -56,6 +56,23 @@ CLOCK_TICKS = os.sysconf("SC_CLK_TCK")
|
||||
# Trip-wire: ru_maxrss counts kilobytes on Linux but bytes on macOS.
|
||||
RSS_UNIT = 1 if sys.platform == "darwin" else 1024
|
||||
USAGE = "usage: measure.py --base <ref> [--json] -- <command...>"
|
||||
LABELS = Path(__file__).resolve().parents[2] / "kvb-check-score" / "labels.json"
|
||||
LABEL_ERRORS = (OSError, ValueError, LookupError, TypeError)
|
||||
|
||||
|
||||
def labelled(keys: tuple[str, ...]) -> dict[str, str]:
|
||||
"""Args:
|
||||
keys: metric keys of kvb-check-score's labels.json.
|
||||
|
||||
Returns:
|
||||
Per key `<emoji> <label>`, the label in lower case; one of LABEL_ERRORS
|
||||
for a missing or malformed file.
|
||||
"""
|
||||
metrics = json.loads(LABELS.read_text(encoding="utf-8"))["metrics"]
|
||||
found = {key: (metrics[key]["emoji"], metrics[key]["label"]) for key in keys}
|
||||
if not all(isinstance(part, str) for pair in found.values() for part in pair):
|
||||
raise ValueError(f"needs a string emoji and label for {', '.join(keys)}")
|
||||
return {key: f"{emoji} {label.lower()}" for key, (emoji, label) in found.items()}
|
||||
|
||||
|
||||
def fail(message: str) -> None:
|
||||
@@ -139,14 +156,14 @@ def figures(runs: dict[str, list[dict]], loads: list[float | None]) -> dict:
|
||||
}
|
||||
|
||||
|
||||
def line(found: dict) -> str:
|
||||
def line(found: dict, names: dict[str, str]) -> str:
|
||||
mark = "~" if found["estimated"] else ""
|
||||
basis = "cpu" if found["estimated"] else "wall"
|
||||
before, after = found["base"], found["head"]
|
||||
text = (
|
||||
f"💸 cost {mark}{found['efficiency']} % (cpu {before['cpu']:.2f}→{after['cpu']:.2f} s, "
|
||||
f"{names['tokens']} {mark}{found['efficiency']} % (cpu {before['cpu']:.2f}→{after['cpu']:.2f} s, "
|
||||
f"ram {before['ram']:.1f}→{after['ram']:.1f} MB) · "
|
||||
f"⏱️ time {mark}{found['speed']} % ({basis} {before[basis]:.2f}→{after[basis]:.2f} s)"
|
||||
f"{names['velocity']} {mark}{found['speed']} % ({basis} {before[basis]:.2f}→{after[basis]:.2f} s)"
|
||||
)
|
||||
if found["runs"] > 1:
|
||||
text += f" · mean of {found['runs']}"
|
||||
@@ -175,6 +192,10 @@ def main(argv: list[str]) -> int:
|
||||
command = argv[split + 1:]
|
||||
if not command:
|
||||
fail("no command after --")
|
||||
try:
|
||||
names = {} if args.json else labelled(("tokens", "velocity"))
|
||||
except LABEL_ERRORS as error:
|
||||
fail(f"labels {LABELS}: {error!r}")
|
||||
|
||||
here = Path.cwd().resolve()
|
||||
root = Path(git(here, "rev-parse", "--show-toplevel")).resolve()
|
||||
@@ -206,7 +227,7 @@ def main(argv: list[str]) -> int:
|
||||
shutil.rmtree(parent, ignore_errors=True)
|
||||
|
||||
found = figures(runs, loads + run_loads(runs))
|
||||
print(json.dumps(found) if args.json else line(found))
|
||||
print(json.dumps(found) if args.json else line(found, names))
|
||||
return 3 if costly(found) else 0
|
||||
|
||||
|
||||
|
||||
@@ -15,8 +15,8 @@ Args:
|
||||
shipped change instead.
|
||||
|
||||
Output:
|
||||
Line 1: `💸 cost <p> % (skills <b>→<a> tokens; listing <b>→<a> words)`, p = after / before * 100,
|
||||
or `💸 cost new (+<a> tokens; listing <b>→<a> words)` when the changed files hold no
|
||||
Line 1: `💸 tokens <p> % (skills <b>→<a> tokens; listing <b>→<a> words)`, p = after / before * 100,
|
||||
or `💸 tokens new (+<a> tokens; listing <b>→<a> words)` when the changed files hold no
|
||||
base tokens. Changed files are the *.md under skills/ that `git diff <ref>` names plus untracked
|
||||
ones; a file missing on a side counts 0. Tokens are an estimate: characters / 4. The listing
|
||||
is the sum of every skills/*/SKILL.md description word.
|
||||
@@ -61,6 +61,41 @@ DAYS = 7
|
||||
ROUND_ABOVE = 10
|
||||
ALL_SKILLS = "all skills"
|
||||
CONTEXT = Path(__file__).resolve().parents[2] / "kvb-skill-silence" / "scripts" / "context.py"
|
||||
LABELS = Path(__file__).resolve().parents[2] / "kvb-check-score" / "labels.json"
|
||||
|
||||
|
||||
LABEL_ERRORS = (OSError, ValueError, LookupError, TypeError)
|
||||
|
||||
|
||||
def labelled(keys: tuple[str, ...]) -> dict[str, str]:
|
||||
"""Args:
|
||||
keys: metric keys of kvb-check-score's labels.json.
|
||||
|
||||
Returns:
|
||||
Per key `<emoji> <label>`, the label in lower case; one of LABEL_ERRORS
|
||||
for a missing or malformed file.
|
||||
"""
|
||||
metrics = json.loads(LABELS.read_text(encoding="utf-8"))["metrics"]
|
||||
found = {key: (metrics[key]["emoji"], metrics[key]["label"]) for key in keys}
|
||||
if not all(isinstance(part, str) for pair in found.values() for part in pair):
|
||||
raise ValueError(f"needs a string emoji and label for {', '.join(keys)}")
|
||||
return {key: f"{emoji} {label.lower()}" for key, (emoji, label) in found.items()}
|
||||
|
||||
|
||||
def headline(name: str, efficiency: int | None, before: int, after: int, listing: str) -> str:
|
||||
"""Args:
|
||||
name: the tokens metric as labelled() gives it.
|
||||
efficiency: after over before in percent, None without base tokens.
|
||||
before: the base tokens of the changed files.
|
||||
after: their tokens in the working tree.
|
||||
listing: the listing words part.
|
||||
|
||||
Returns:
|
||||
Line 1.
|
||||
"""
|
||||
if efficiency is None:
|
||||
return f"{name} new (+{after} tokens; {listing})"
|
||||
return f"{name} {efficiency} % (skills {before}→{after} tokens; {listing})"
|
||||
|
||||
|
||||
def fail(message: str) -> None:
|
||||
@@ -254,6 +289,10 @@ def measure(
|
||||
|
||||
def check(argv: list[str]) -> int:
|
||||
args = parse(argv)
|
||||
try:
|
||||
names = labelled(("tokens",))
|
||||
except LABEL_ERRORS as error:
|
||||
fail(f"labels {LABELS}: {error!r}")
|
||||
repo = Path(args.repo).resolve()
|
||||
if not (repo / SKILLS).is_dir():
|
||||
fail(f"{repo} has no {SKILLS}/ directory")
|
||||
@@ -296,11 +335,7 @@ def check(argv: list[str]) -> int:
|
||||
},
|
||||
}))
|
||||
else:
|
||||
listing = f"listing {listing_before}→{listing_after} words"
|
||||
if efficiency is None:
|
||||
print(f"💸 cost new (+{after} tokens; {listing})")
|
||||
else:
|
||||
print(f"💸 cost {efficiency} % (skills {before}→{after} tokens; {listing})")
|
||||
print(headline(names["tokens"], efficiency, before, after, f"listing {listing_before}→{listing_after} words"))
|
||||
print(money)
|
||||
return 3 if efficiency is None or efficiency > LIMIT else 0
|
||||
|
||||
|
||||
@@ -6,7 +6,7 @@ description: >
|
||||
---
|
||||
|
||||
The gate rule, stated here and nowhere else: nothing is declared 90/5 (fix
|
||||
confidence ≥ 90 %, break risk ≤ 5 %, cost and time ≤ 100 % in
|
||||
confidence ≥ 90 %, break risk ≤ 5 %, tokens and velocity ≤ 100 % in
|
||||
`kvb-check-score`; over 100 %, ask the operator before the commit, naming
|
||||
the cause) before every `make test*` target of the project, each Makefile target named `test` or starting with
|
||||
`test-`, ran green on the exact head with a clean tracked tree. Until then
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
---
|
||||
name: kvb-check-score
|
||||
description: >
|
||||
Score fix, break, cost, time and security; run wherever confidence is reported.
|
||||
Score fix, break, tokens, velocity and security; run wherever confidence is reported.
|
||||
---
|
||||
|
||||
Run it wherever fix confidence and break risk are reported: every
|
||||
@@ -16,18 +16,22 @@ python3 ${CLAUDE_SKILL_DIR}/scripts/score.py --fix <0-100> --break <0-100> --eff
|
||||
| --- | --- | --- |
|
||||
| 🎯 | fix confidence | `kvb-check-confidence` |
|
||||
| 💥 | break risk | `kvb-check-break-risk` |
|
||||
| 💸 | cost, `--efficiency` | `tokens.py` (skills) or `measure.py` (code) of `kvb-check-efficiency` |
|
||||
| ⏱️ | time, `--speed` | `measure.py`, see `kvb-check-speed` |
|
||||
| 💸 | tokens, `--efficiency` | `tokens.py` (skills) or `measure.py` (code) of `kvb-check-efficiency` |
|
||||
| ⏱️ | velocity, `--speed` | `measure.py`, see `kvb-check-speed` |
|
||||
| 🛡️ | security, critical count | `security.py` of `kvb-check-security`; `~<n>` from a review by hand |
|
||||
| 💰 | cost:value quotient, tables and message block only | 💰 line of `tokens.py` (skills) of `kvb-check-efficiency` |
|
||||
| 🏆 | score | this script |
|
||||
|
||||
Cost and time: after over base in percent, over 100 worse. `-` means not
|
||||
measured or not relevant (skills have no time) and counts as
|
||||
Emojis and labels come from `labels.json`, one entry per metric key with
|
||||
its former labels (Cost, Efficiency, Time, Speed); change a label there,
|
||||
nowhere else.
|
||||
|
||||
Tokens and velocity: after over base in percent, over 100 worse. `-` means not
|
||||
measured or not relevant (skills have no velocity) and counts as
|
||||
100; it shows as `–`. `new` (changed skill text without a base) counts as 100
|
||||
in the score, yet still asks. Never invent a value: measure it or pass `-`.
|
||||
|
||||
Score = `fix * (100 - break) / 100 * 100 / cost * 100 / time * security / 100`,
|
||||
Score = `fix * (100 - break) / 100 * 100 / tokens * 100 / velocity * security / 100`,
|
||||
capped to 0..100 and rounded.
|
||||
|
||||
## Gate
|
||||
@@ -42,20 +46,22 @@ The second line is the verdict, first match wins:
|
||||
in the block and commit only on the operator's word. Close the gap as the
|
||||
last section of `kvb-check-break-risk` says.
|
||||
- `⚠️ gate: <causes>: ask the operator before the commit` (exit 3), naming
|
||||
each of `cost <c> % > 100`, `time <t> % > 100` and
|
||||
each of `tokens <c> % > 100`, `velocity <t> % > 100` and
|
||||
`security <n> % < 100`: ask with the block what the cost buys or why each
|
||||
finding is acceptable; commit only on a yes, never carried over to
|
||||
another commit.
|
||||
- `✅ gate: 90/5, cost and time <= 100, security 100 %` (exit 0):
|
||||
- `✅ gate: 90/5, tokens and velocity <= 100, security 100 %` (exit 0):
|
||||
commit as usual. 90/5 still needs `gate: green` of `kvb-check-gate`.
|
||||
- Exit 2: usage error.
|
||||
- Exit 2: usage error, a malformed `labels.json` or a `--commit-table`
|
||||
repository git cannot read, with one stderr line and nothing on stdout;
|
||||
fix the cause and score again. A critical finding still exits 4 then.
|
||||
|
||||
## Tables and message block
|
||||
|
||||
Each flag replaces the two lines; values, gate and exit code stay:
|
||||
|
||||
- `--table`: one markdown table, `| 🎯 Fix | 💥 Break | 💸 Cost |
|
||||
⏱️ Time | 🛡️ Security | 🏆 Score | Gate |`.
|
||||
- `--table`: one markdown table, `| 🎯 Fix | 💥 Break | 💸 Tokens |
|
||||
⏱️ Velocity | 🛡️ Security | 🏆 Score | Gate |`.
|
||||
- `--commit-table <repo>`: that table led by `🌿 Branch | 📊 Diff stat`
|
||||
from git, 📁 worktree path below, alone in a code block. Every commit
|
||||
block is this table; after the commit, `--commit <sha>` shows that commit's diff stat
|
||||
@@ -66,6 +72,32 @@ Each flag replaces the two lines; values, gate and exit code stay:
|
||||
`-m` argument, so the approval prompt and git history show the values.
|
||||
|
||||
On a skill change add `--value <n:n|?>`, the quotient of the 💰 line (`?`
|
||||
while it has no value): a `💰 Value` column after `💸 Cost`, or line
|
||||
while it has no value): a `💰 Value` column after `💸 Tokens`, or line
|
||||
after `🛡️ Security`. It changes neither score nor gate; alone it is a usage
|
||||
error.
|
||||
|
||||
## Journal
|
||||
|
||||
Each `--message-block` render appends one row to
|
||||
`${AGENTS_COUNCIL_ROOT:-/tmp/agents-council}/scores.jsonl`: time, origin
|
||||
remote, top level path, branch, `sha` null, the six values and the block's
|
||||
digest. The hub's burndown draws it as overlays; no gate reads it. A failed
|
||||
append prints one stderr line and keeps the exit code; a linked council
|
||||
root or a link, FIFO or other non-regular file at the journal fails it
|
||||
without blocking, through `nofollow.py` of `kvb-org-organize`; when that
|
||||
helper is missing or unusable, the line names it. A new council root gets mode
|
||||
0700, a new journal 0600.
|
||||
|
||||
The journal lives in `/tmp`, so a reboot or another PC starts empty. Refill
|
||||
it from the blocks in git history, without a model:
|
||||
|
||||
```bash
|
||||
python3 ${CLAUDE_SKILL_DIR}/scripts/backfill.py --repo <dir>... [--since <rev>] [--labels <file>]
|
||||
```
|
||||
|
||||
It reads every local and origin branch, takes the last block of each commit
|
||||
message, old labels included, and adds one row per commit not yet keyed by
|
||||
(remote, sha), so a second run or another clone adds nothing. The branch is
|
||||
the one whose first-parent line holds the commit, the default branch first,
|
||||
else the one a merge subject names. An unknown label is reported once and
|
||||
skipped.
|
||||
@@ -0,0 +1,14 @@
|
||||
{
|
||||
"metrics": {
|
||||
"fix": {"emoji": "🎯", "label": "Fix", "former": []},
|
||||
"break": {"emoji": "💥", "label": "Break", "former": []},
|
||||
"tokens": {"emoji": "💸", "label": "Tokens", "former": ["Cost", "Efficiency"]},
|
||||
"velocity": {"emoji": "⏱️", "label": "Velocity", "former": ["Time", "Speed"]},
|
||||
"security": {"emoji": "🛡️", "label": "Security", "former": []},
|
||||
"score": {"emoji": "🏆", "label": "Score", "former": []}
|
||||
},
|
||||
"extras": {
|
||||
"value": {"emoji": "💰", "label": "Value", "former": []},
|
||||
"gate": {"emoji": "🚦", "label": "Gate", "former": []}
|
||||
}
|
||||
}
|
||||
Executable
+252
@@ -0,0 +1,252 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Backfill the score journal of journal.py from the message blocks in git history.
|
||||
|
||||
Args:
|
||||
--repo <dir>...: repositories whose local and origin branches git log reads;
|
||||
repeatable, every one given counts.
|
||||
--since <rev>: only commits not reachable from <rev>, in every repository.
|
||||
--labels <file>: the label file mapping current and former labels to metric
|
||||
keys, ../labels.json when left out.
|
||||
|
||||
Output:
|
||||
One line per repository, `<top level>: <n> blocks, <m> rows added`; one stderr
|
||||
line per unknown block label, `backfill: unknown label <label>, skipped`.
|
||||
|
||||
Exit codes:
|
||||
0 done, 2 usage error, a repository git cannot read or a journal that is
|
||||
linked, not a regular file or not writable, with one stderr line; nothing
|
||||
is written then.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
from collections import deque
|
||||
from pathlib import Path
|
||||
from typing import NamedTuple
|
||||
|
||||
from journal import (
|
||||
LABELS,
|
||||
append,
|
||||
entry,
|
||||
git_read,
|
||||
journal_path,
|
||||
load_labels,
|
||||
origin,
|
||||
read,
|
||||
)
|
||||
|
||||
FIELD = "\x00"
|
||||
RECORD = "\x1e"
|
||||
LOG_FORMAT = "%H%x00%P%x00%ct%x00%B%x1e"
|
||||
REFS = ("refs/heads/", "refs/remotes/origin/")
|
||||
LINE = re.compile(r"([^\x00-\x7f]\S*) ([A-Z][A-Za-z]*): (.+)")
|
||||
AMOUNT = re.compile(r"~?(\d+(?:\.\d+)?)(?: ?%)?")
|
||||
PARAGRAPH = re.compile(r"\n[ \t]*\n")
|
||||
MERGES = (
|
||||
re.compile(r"Merge pull request #\d+ from [^/\s]+/(\S+)"),
|
||||
re.compile(r"Merge (?:remote-tracking )?branch '(?:origin/)?([^']+)'"),
|
||||
re.compile(r"Merge (\S+) into \S+"),
|
||||
)
|
||||
|
||||
|
||||
class Commit(NamedTuple):
|
||||
parents: list[str]
|
||||
time: int
|
||||
message: str
|
||||
|
||||
|
||||
def aliases(labels: dict) -> tuple[dict[str, str], set[str]]:
|
||||
"""Args:
|
||||
labels: load_labels() of a label file.
|
||||
|
||||
Returns:
|
||||
(metric of each current and former label, labels of the extra lines).
|
||||
"""
|
||||
named = {name: key for key, metric in labels["metrics"].items() for name in (metric["label"], *metric["former"])}
|
||||
extras = {name for extra in labels["extras"].values() for name in (extra["label"], *extra["former"])}
|
||||
return named, extras
|
||||
|
||||
|
||||
def amount(text: str) -> float | int | None:
|
||||
found = AMOUNT.fullmatch(text.strip())
|
||||
if not found:
|
||||
return None
|
||||
value = float(found.group(1))
|
||||
return int(value) if value.is_integer() else value
|
||||
|
||||
|
||||
def parse(message: str, named: dict[str, str], extras: set[str], unknown: set[str]) -> tuple[dict, str] | None:
|
||||
"""Args:
|
||||
message: a commit message.
|
||||
named: aliases() metrics.
|
||||
extras: aliases() extra labels, read and left out.
|
||||
unknown: collects the labels of block lines neither names.
|
||||
|
||||
Returns:
|
||||
(values per metric key, block text) of the last block, None without one.
|
||||
"""
|
||||
found = None
|
||||
for paragraph in PARAGRAPH.split(message):
|
||||
lines = [match for match in (LINE.fullmatch(text.strip()) for text in paragraph.splitlines()) if match]
|
||||
if not any(named.get(match.group(2)) == "score" for match in lines):
|
||||
continue
|
||||
values = dict.fromkeys(named.values())
|
||||
for match in lines:
|
||||
label = match.group(2)
|
||||
if label in named:
|
||||
values[named[label]] = amount(match.group(3))
|
||||
elif label not in extras:
|
||||
unknown.add(label)
|
||||
if values["score"] is not None:
|
||||
found = (values, "\n".join(match.group(0) for match in lines))
|
||||
return found
|
||||
|
||||
|
||||
def history(repo: Path, tips: list[str], since: str | None) -> dict[str, Commit]:
|
||||
text = git_read(repo, "log", f"--format={LOG_FORMAT}", *tips, *([f"^{since}"] if since else []), "--")
|
||||
if text is None:
|
||||
raise LookupError(f"git log in {repo} failed")
|
||||
commits = {}
|
||||
for record in text.split(RECORD):
|
||||
fields = record.strip("\n").split(FIELD, 3)
|
||||
if len(fields) == 4:
|
||||
sha, parents, stamp, message = fields
|
||||
commits[sha] = Commit(parents.split(), int(stamp), message)
|
||||
return commits
|
||||
|
||||
|
||||
def heads(repo: Path) -> list[tuple[str, str]]:
|
||||
"""Args:
|
||||
repo: a git repository.
|
||||
|
||||
Returns:
|
||||
(branch, tip sha) of every local and origin branch, the default branch
|
||||
(origin/HEAD, else the checked out one) first, then by name, local first.
|
||||
"""
|
||||
listing = git_read(repo, "for-each-ref", "--format=%(refname)%00%(objectname)", *REFS)
|
||||
if listing is None:
|
||||
raise LookupError(f"git for-each-ref in {repo} failed")
|
||||
pointed = git_read(repo, "symbolic-ref", "-q", "refs/remotes/origin/HEAD") or git_read(repo, "symbolic-ref", "-q", "HEAD") or ""
|
||||
default = next((pointed.removeprefix(prefix) for prefix in REFS if pointed.startswith(prefix)), None)
|
||||
found = []
|
||||
for row in listing.splitlines():
|
||||
ref, sha = row.split(FIELD)
|
||||
rank, prefix = next((rank, prefix) for rank, prefix in enumerate(REFS) if ref.startswith(prefix))
|
||||
name = ref.removeprefix(prefix)
|
||||
if name != "HEAD":
|
||||
found.append((name != default, name, rank, sha))
|
||||
return [(name, sha) for _, name, _, sha in sorted(found)]
|
||||
|
||||
|
||||
def merged(message: str) -> str | None:
|
||||
subject = message.split("\n", 1)[0]
|
||||
return next((match.group(1) for match in (pattern.match(subject) for pattern in MERGES) if match), None)
|
||||
|
||||
|
||||
def owners(commits: dict[str, Commit], tips: list[tuple[str, str]]) -> dict[str, str]:
|
||||
"""Args:
|
||||
commits: history() of the repository.
|
||||
tips: heads() of the repository.
|
||||
|
||||
Returns:
|
||||
The branch of every commit: the first tip whose first-parent line holds
|
||||
it, else the branch a merge subject names for its side, else the merge's.
|
||||
"""
|
||||
owner: dict[str, str] = {}
|
||||
queue = deque((sha, name) for name, sha in tips)
|
||||
while queue:
|
||||
sha, name = queue.popleft()
|
||||
while sha in commits and sha not in owner:
|
||||
owner[sha] = name
|
||||
commit = commits[sha]
|
||||
side = merged(commit.message) or name
|
||||
queue.extend((other, side) for other in commit.parents[1:])
|
||||
sha = commit.parents[0] if commit.parents else None
|
||||
return owner
|
||||
|
||||
|
||||
def scan(repo: Path, since: str | None, named: dict[str, str], extras: set[str], unknown: set[str]) -> tuple[str, list[dict]]:
|
||||
place = origin(repo)
|
||||
if place[1] is None:
|
||||
raise LookupError(f"{repo} is not a git work tree")
|
||||
tips = heads(repo)
|
||||
commits = history(repo, [sha for _, sha in tips], since) if tips else {}
|
||||
owner = owners(commits, tips)
|
||||
rows = []
|
||||
for sha, commit in commits.items():
|
||||
found = parse(commit.message, named, extras, unknown)
|
||||
if found:
|
||||
values, block = found
|
||||
rows.append(entry(commit.time, place, owner.get(sha), sha, values, block))
|
||||
return place[1], rows
|
||||
|
||||
|
||||
def known(path: Path) -> set[tuple[str | None, str]]:
|
||||
try:
|
||||
text = read(path)
|
||||
except FileNotFoundError:
|
||||
return set()
|
||||
keys = set()
|
||||
for line in text.splitlines():
|
||||
try:
|
||||
row = json.loads(line)
|
||||
except ValueError:
|
||||
continue
|
||||
if isinstance(row, dict) and isinstance(row.get("sha"), str):
|
||||
keys.add((row.get("remote"), row["sha"]))
|
||||
return keys
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
|
||||
parser.add_argument("--repo", type=Path, nargs="+", action="extend", required=True)
|
||||
parser.add_argument("--since", metavar="REV")
|
||||
parser.add_argument("--labels", type=Path, default=LABELS)
|
||||
args = parser.parse_args()
|
||||
try:
|
||||
named, extras = aliases(load_labels(args.labels))
|
||||
except (OSError, ValueError, LookupError, TypeError, AttributeError) as error:
|
||||
parser.error(f"--labels {args.labels}: {error!r}")
|
||||
unknown: set[str] = set()
|
||||
try:
|
||||
scanned = [scan(repo, args.since, named, extras, unknown) for repo in args.repo]
|
||||
except LookupError as error:
|
||||
print(f"backfill: {error}", file=sys.stderr)
|
||||
return 2
|
||||
path = journal_path()
|
||||
try:
|
||||
keys = known(path)
|
||||
except (OSError, ValueError) as error:
|
||||
return refused(path, error)
|
||||
added, summary = [], []
|
||||
for top, rows in scanned:
|
||||
fresh = []
|
||||
for row in sorted(rows, key=lambda row: (row["time"], row["sha"])):
|
||||
key = (row["remote"], row["sha"])
|
||||
if key not in keys:
|
||||
keys.add(key)
|
||||
fresh.append(row)
|
||||
added += fresh
|
||||
summary.append(f"{top}: {len(rows)} blocks, {len(fresh)} rows added")
|
||||
if added:
|
||||
try:
|
||||
append(path, added)
|
||||
except (OSError, ValueError) as error:
|
||||
return refused(path, error)
|
||||
print("\n".join(summary))
|
||||
for label in sorted(unknown):
|
||||
print(f"backfill: unknown label {label}, skipped", file=sys.stderr)
|
||||
return 0
|
||||
|
||||
|
||||
def refused(path: Path, error: Exception) -> int:
|
||||
print(f"backfill: journal {path}: {getattr(error, 'strerror', None) or error}", file=sys.stderr)
|
||||
return 2
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,184 @@
|
||||
"""Labels and score journal shared by score.py and backfill.py of kvb-check-score.
|
||||
|
||||
Environment:
|
||||
AGENTS_COUNCIL_ROOT: council directory shared by every agent on the host;
|
||||
/tmp/agents-council when unset or empty. The journal is its scores.jsonl,
|
||||
display data for the hub's burndown only, never a gate input.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import importlib.util
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
from collections.abc import Callable
|
||||
from pathlib import Path
|
||||
from types import ModuleType
|
||||
from urllib.parse import urlsplit
|
||||
|
||||
LABELS = Path(__file__).resolve().parents[1] / "labels.json"
|
||||
NOFOLLOW = Path(__file__).resolve().parents[2] / "kvb-org-organize" / "scripts" / "nofollow.py"
|
||||
JOURNAL = "scores.jsonl"
|
||||
COUNCIL = "/tmp/agents-council"
|
||||
SCP = re.compile(r"(?:[^@/:]+@)?([^/:]+):(?!//)(.+)")
|
||||
FILE_MODE = 0o600
|
||||
|
||||
|
||||
def load_labels(path: Path) -> dict:
|
||||
"""Args:
|
||||
path: a labels.json with "metrics" and "extras", each key mapped to
|
||||
{"emoji", "label", "former"}.
|
||||
|
||||
Returns:
|
||||
The parsed file; metrics in block order.
|
||||
"""
|
||||
return json.loads(path.read_text(encoding="utf-8"))
|
||||
|
||||
|
||||
def journal_path() -> Path:
|
||||
return Path(os.environ.get("AGENTS_COUNCIL_ROOT") or COUNCIL) / JOURNAL
|
||||
|
||||
|
||||
def git(repo: Path | str, *args: str) -> str:
|
||||
"""Args:
|
||||
repo: the directory git runs in.
|
||||
args: the git arguments.
|
||||
|
||||
Returns:
|
||||
The stripped stdout, bytes outside UTF-8 replaced; LookupError naming
|
||||
the command and git's stderr on one line when git is missing, the
|
||||
directory is gone or git fails.
|
||||
"""
|
||||
try:
|
||||
done = subprocess.run(["git", *args], cwd=repo, capture_output=True, encoding="utf-8", errors="replace", check=False)
|
||||
except OSError as error:
|
||||
raise LookupError(f"git {args[0]} in {repo}: {error}") from None
|
||||
if done.returncode != 0:
|
||||
raise LookupError(f"git {args[0]} in {repo}: {' '.join(done.stderr.split())}")
|
||||
return done.stdout.strip()
|
||||
|
||||
|
||||
def git_read(repo: Path, *args: str) -> str | None:
|
||||
"""Args:
|
||||
repo: the directory git runs in.
|
||||
args: the git arguments.
|
||||
|
||||
Returns:
|
||||
The stdout of git(), or None when it raises.
|
||||
"""
|
||||
try:
|
||||
return git(repo, *args)
|
||||
except LookupError:
|
||||
return None
|
||||
|
||||
|
||||
def identity(url: str | None) -> str | None:
|
||||
"""Args:
|
||||
url: a git remote URL: https, ssh, scp-like or a local path.
|
||||
|
||||
Returns:
|
||||
host/path without .git for a hosted remote, so ssh, scp-like and https
|
||||
clones of one repository match; a local path unchanged; None for None.
|
||||
"""
|
||||
if url is None:
|
||||
return None
|
||||
scp = SCP.fullmatch(url)
|
||||
if "://" in url:
|
||||
parts = urlsplit(url)
|
||||
host, path = parts.hostname, parts.path
|
||||
elif scp:
|
||||
host, path = scp.groups()
|
||||
else:
|
||||
return url
|
||||
if not host:
|
||||
return url
|
||||
return f"{host.lower()}/{path.strip('/').removesuffix('.git')}"
|
||||
|
||||
|
||||
def origin(repo: Path) -> tuple[str | None, str | None]:
|
||||
"""Args:
|
||||
repo: a directory inside a git work tree.
|
||||
|
||||
Returns:
|
||||
(remote, path): the identity() of its origin remote and its top level
|
||||
path; None for what git cannot tell.
|
||||
"""
|
||||
return identity(git_read(repo, "remote", "get-url", "origin")), git_read(repo, "rev-parse", "--show-toplevel")
|
||||
|
||||
|
||||
def entry(time: int, place: tuple[str | None, str | None], branch: str | None, sha: str | None, values: dict, block: str) -> dict:
|
||||
"""Args:
|
||||
time: epoch seconds: the render for score.py, the commit for backfill.py.
|
||||
place: origin() of the repository.
|
||||
branch: the branch the block belongs to.
|
||||
sha: the commit holding the block, None for a render.
|
||||
values: per metric key its number, None where the block shows none.
|
||||
block: the message block text the digest covers.
|
||||
|
||||
Returns:
|
||||
One journal row.
|
||||
"""
|
||||
remote, path = place
|
||||
digest = hashlib.sha256(block.encode("utf-8")).hexdigest()
|
||||
return {"time": time, "remote": remote, "path": path, "branch": branch, "sha": sha, "values": values, "digest": digest}
|
||||
|
||||
|
||||
def line(row: dict) -> str:
|
||||
return json.dumps(row, ensure_ascii=False) + "\n"
|
||||
|
||||
|
||||
def unusable(error: Exception) -> OSError:
|
||||
return OSError(f"helper {NOFOLLOW} unusable ({type(error).__name__}): install or update kvb-org-organize")
|
||||
|
||||
|
||||
def nofollow[T](use: Callable[[ModuleType], T]) -> T:
|
||||
"""Args:
|
||||
use: what to do with nofollow.py of kvb-org-organize.
|
||||
|
||||
Returns:
|
||||
The result of use; its OSError unchanged, any other failure loading
|
||||
or using the helper as one OSError naming it, never a
|
||||
FileNotFoundError.
|
||||
"""
|
||||
try:
|
||||
spec = importlib.util.spec_from_file_location("kvb_score_nofollow", NOFOLLOW)
|
||||
loaded = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(loaded)
|
||||
except Exception as error: # noqa: BLE001
|
||||
raise unusable(error) from None
|
||||
try:
|
||||
return use(loaded)
|
||||
except OSError:
|
||||
raise
|
||||
except Exception as error: # noqa: BLE001
|
||||
raise unusable(error) from None
|
||||
|
||||
|
||||
def append(path: Path, rows: list[dict]) -> None:
|
||||
"""Args:
|
||||
path: the journal in the council root, made with the helper's mode
|
||||
0o700; a linked root, a link, FIFO or other non-regular file there
|
||||
raises OSError without blocking.
|
||||
rows: the rows to append, one JSON line each, in one write.
|
||||
"""
|
||||
text = "".join(line(row) for row in rows)
|
||||
|
||||
def write(helper: ModuleType) -> None:
|
||||
path.parent.mkdir(mode=helper.DIRECTORY_MODE, parents=True, exist_ok=True)
|
||||
helper.append(path.parent, path, text, FILE_MODE)
|
||||
|
||||
nofollow(write)
|
||||
|
||||
|
||||
def read(path: Path) -> str:
|
||||
"""Args:
|
||||
path: the journal, guarded as for append(); a missing one raises
|
||||
FileNotFoundError.
|
||||
|
||||
Returns:
|
||||
Its text, undecodable bytes replaced.
|
||||
"""
|
||||
return nofollow(lambda helper: helper.read_text(path.parent, path))
|
||||
@@ -1,13 +1,13 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Fold fix confidence, break risk, cost, time and security into one score and one gate verdict.
|
||||
"""Fold fix confidence, break risk, tokens, velocity and security into one score and one gate verdict.
|
||||
|
||||
Args:
|
||||
--fix: fix confidence in percent, 0 to 100.
|
||||
--break: break risk in percent, 0 to 100.
|
||||
--efficiency: cost in percent above 0, from kvb-check-efficiency; `-` when not
|
||||
--efficiency: tokens in percent above 0, from kvb-check-efficiency; `-` when not
|
||||
measured or not relevant, `new` when the change has no base. Both count as 100
|
||||
in the score.
|
||||
--speed: time in percent above 0, from kvb-check-speed; `-` or `new` as for
|
||||
--speed: velocity in percent above 0, from kvb-check-speed; `-` or `new` as for
|
||||
--efficiency.
|
||||
--security: security in percent, 0 to 100, from security.py of kvb-check-security;
|
||||
`~<n>` for a value from a review by hand.
|
||||
@@ -19,24 +19,30 @@ Args:
|
||||
--commit <sha>: with --commit-table, the diff stat of that commit instead of
|
||||
the staged one, plus a commit column.
|
||||
--message-block: print the commit message block instead of the two lines, one
|
||||
value per line: `🎯 Fix: <f> %`, `💥 Break: <b> %`, `💸 Cost: <c>`,
|
||||
`⏱️ Time: <t>`, `🛡️ Security: <n> %`, `💰 Value: <n:n>` with --value only,
|
||||
`🏆 Score: <n>`, `🚦 Gate: <verdict>`; values and markers as in line 1.
|
||||
value per line: `🎯 Fix: <f> %`, `💥 Break: <b> %`, `💸 Tokens: <c>`,
|
||||
`⏱️ Velocity: <t>`, `🛡️ Security: <n> %`, `💰 Value: <n:n>` with --value only,
|
||||
`🏆 Score: <n>`, `🚦 Gate: <verdict>`; values and markers as in line 1. Each
|
||||
render appends one row to the score journal of journal.py; a failed append
|
||||
prints one stderr line and keeps the exit code.
|
||||
--value <n:n|?>: with --table, --commit-table or --message-block, a value after
|
||||
the cost one (--message-block: after security): the cost:value quotient of
|
||||
the tokens one (--message-block: after security): the cost:value quotient of
|
||||
tokens.py of kvb-check-efficiency, `?` when unknown. It changes neither score nor gate.
|
||||
|
||||
Labels and emojis come from ../labels.json.
|
||||
|
||||
Output:
|
||||
Line 1: `🎯 fix <f> % · 💥 break <b> % · 💸 cost <c> % · ⏱️ time <t> % ·
|
||||
Line 1: `🎯 fix <f> % · 💥 break <b> % · 💸 tokens <c> % · ⏱️ velocity <t> % ·
|
||||
🛡️ security <n> % · 🏆 score <n>`, `–` in place of `<x> %` for `-`, `new` for `new`.
|
||||
Score is fix * (100 - break) / 100 * 100 / cost * 100 / time * security / 100,
|
||||
Score is fix * (100 - break) / 100 * 100 / tokens * 100 / velocity * security / 100,
|
||||
capped to 0..100, rounded half up.
|
||||
Line 2: the gate verdict, `✅ gate: ...`, `⛔ gate: ...` or `⚠️ gate: ...`.
|
||||
|
||||
Exit codes:
|
||||
0 gate passed, 1 fix under 90 or break over 5, 2 usage error,
|
||||
3 cost or time over 100 or new, or security under 100: ask the operator
|
||||
before the commit, 4 a critical security finding: do not commit.
|
||||
0 gate passed, 1 fix under 90 or break over 5, 2 usage error, a malformed
|
||||
labels.json or a --commit-table repository git cannot read, one stderr line
|
||||
and nothing on stdout (4 still wins with a critical finding), 3 tokens or
|
||||
velocity over 100 or new, or security under 100: ask the operator before the
|
||||
commit, 4 a critical security finding: do not commit.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -44,8 +50,20 @@ from __future__ import annotations
|
||||
import argparse
|
||||
import math
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
from journal import (
|
||||
LABELS,
|
||||
append,
|
||||
entry,
|
||||
git,
|
||||
git_read,
|
||||
journal_path,
|
||||
load_labels,
|
||||
origin,
|
||||
)
|
||||
|
||||
FIX_MIN = 90
|
||||
BREAK_MAX = 5
|
||||
@@ -54,12 +72,10 @@ SAFE = 100
|
||||
DASH = "-"
|
||||
NEW = "new"
|
||||
ESTIMATE = "~"
|
||||
SCORE_HEADER = ("🎯 Fix", "💥 Break", "💸 Cost", "⏱️ Time", "🛡️ Security", "🏆 Score", "Gate")
|
||||
COMMIT_HEADER = ("🌿 Branch", "📊 Diff stat")
|
||||
WORKTREE = "📁 Worktree:"
|
||||
VALUE_HEADER = "💰 Value"
|
||||
VALUE_AT = SCORE_HEADER.index("💸 Cost") + 1
|
||||
UNKNOWN = "?"
|
||||
LABEL_ERRORS = (OSError, ValueError, LookupError, TypeError, AttributeError)
|
||||
QUOTIENT = re.compile(r"\d+(?:\.\d+)?:\d+(?:\.\d+)?")
|
||||
SHORTSTAT = re.compile(
|
||||
r"(\d+) files? changed(?:, (\d+) insertions?\(\+\))?(?:, (\d+) deletions?\(-\))?"
|
||||
@@ -137,35 +153,42 @@ def over(name: str, value: float | str) -> str | None:
|
||||
return None
|
||||
|
||||
|
||||
def gate(args: argparse.Namespace) -> tuple[str, str, int]:
|
||||
def labels(keys: set[str]) -> tuple[dict, dict]:
|
||||
"""Args:
|
||||
keys: the metric keys score.py measures.
|
||||
|
||||
Returns:
|
||||
(metrics, extras) of labels.json; ValueError when it lacks one of the
|
||||
keys, holds another or an entry has no string emoji and label.
|
||||
"""
|
||||
found = load_labels(LABELS)
|
||||
metrics, extras = found["metrics"], found["extras"]
|
||||
named = [*metrics.values(), extras["value"], extras["gate"]]
|
||||
if set(metrics) != keys or not all(isinstance(item["emoji"], str) and isinstance(item["label"], str) for item in named):
|
||||
raise ValueError(f"needs a string emoji and label for {', '.join(sorted(keys))}, value and gate")
|
||||
return metrics, extras
|
||||
|
||||
|
||||
def gate(args: argparse.Namespace, metrics: dict) -> tuple[str, str, int]:
|
||||
def word(key: str) -> str:
|
||||
return metrics[key]["label"].lower()
|
||||
|
||||
if args.critical > 0:
|
||||
return "⛔", "critical security finding: do not commit", 4
|
||||
return "⛔", f"critical {word('security')} finding: do not commit", 4
|
||||
failed = []
|
||||
if args.fix < FIX_MIN:
|
||||
failed.append(f"fix {number(args.fix)} % < {FIX_MIN}")
|
||||
failed.append(f"{word('fix')} {number(args.fix)} % < {FIX_MIN}")
|
||||
if args.risk > BREAK_MAX:
|
||||
failed.append(f"break {number(args.risk)} % > {BREAK_MAX}")
|
||||
failed.append(f"{word('break')} {number(args.risk)} % > {BREAK_MAX}")
|
||||
value, mark = args.security
|
||||
unsafe = f"security {mark}{number(value)} % < {SAFE}" if value < SAFE else None
|
||||
asks = [text for text in (over("cost", args.efficiency), over("time", args.speed), unsafe) if text]
|
||||
unsafe = f"{word('security')} {mark}{number(value)} % < {SAFE}" if value < SAFE else None
|
||||
asks = [text for text in (over(word("tokens"), args.efficiency), over(word("velocity"), args.speed), unsafe) if text]
|
||||
ask = f"{', '.join(asks)}: ask the operator before the commit" if asks else ""
|
||||
if failed:
|
||||
return "⛔", "; ".join(part for part in (", ".join(failed), ask) if part), 1
|
||||
if asks:
|
||||
return "⚠️", ask, 3
|
||||
return "✅", f"{FIX_MIN}/{BREAK_MAX}, cost and time <= {COST_MAX}, security {SAFE} %", 0
|
||||
|
||||
|
||||
def git(repo: str, *args: str) -> str:
|
||||
try:
|
||||
result = subprocess.run(["git", *args], cwd=repo, capture_output=True, text=True, check=False)
|
||||
except OSError as error:
|
||||
print(f"score: git {args[0]} in {repo}: {error}", file=sys.stderr)
|
||||
sys.exit(2)
|
||||
if result.returncode != 0:
|
||||
print(f"score: git {args[0]} in {repo}: {result.stderr.strip()}", file=sys.stderr)
|
||||
sys.exit(2)
|
||||
return result.stdout.strip()
|
||||
return "✅", f"{FIX_MIN}/{BREAK_MAX}, {word('tokens')} and {word('velocity')} <= {COST_MAX}, {word('security')} {SAFE} %", 0
|
||||
|
||||
|
||||
def diffstat(shortstat: str) -> str:
|
||||
@@ -217,38 +240,67 @@ def main() -> int:
|
||||
parser.error("--value needs --table, --commit-table or --message-block")
|
||||
|
||||
total = score(args)
|
||||
emoji, verdict, code = gate(args)
|
||||
value, mark = args.security
|
||||
values = [
|
||||
f"{number(args.fix)} %", f"{number(args.risk)} %", shown(args.efficiency), shown(args.speed),
|
||||
f"{mark}{number(value)} %", str(total), f"{emoji} {verdict}",
|
||||
]
|
||||
header, row = list(SCORE_HEADER), list(values)
|
||||
measured = {
|
||||
"fix": f"{number(args.fix)} %", "break": f"{number(args.risk)} %", "tokens": shown(args.efficiency),
|
||||
"velocity": shown(args.speed), "security": f"{mark}{number(value)} %", "score": str(total),
|
||||
}
|
||||
try:
|
||||
metrics, extras = labels(set(measured))
|
||||
except LABEL_ERRORS as error:
|
||||
print(f"score: labels {LABELS}: {error!r}", file=sys.stderr)
|
||||
return 4 if args.critical > 0 else 2
|
||||
emoji, verdict, code = gate(args, metrics)
|
||||
verdict_cell = f"{emoji} {verdict}"
|
||||
value_header = f"{extras['value']['emoji']} {extras['value']['label']}"
|
||||
header = [*(f"{metric['emoji']} {metric['label']}" for metric in metrics.values()), extras["gate"]["label"]]
|
||||
row = [*(measured[key] for key in metrics), verdict_cell]
|
||||
if args.value:
|
||||
header.insert(VALUE_AT, VALUE_HEADER)
|
||||
row.insert(VALUE_AT, args.value)
|
||||
header.insert(list(metrics).index("tokens") + 1, value_header)
|
||||
row.insert(list(metrics).index("tokens") + 1, args.value)
|
||||
if args.commit_table:
|
||||
lead, cells = commit_cells(args.commit_table, args.commit)
|
||||
top = git(args.commit_table, "rev-parse", "--show-toplevel")
|
||||
try:
|
||||
lead, cells = commit_cells(args.commit_table, args.commit)
|
||||
top = git(args.commit_table, "rev-parse", "--show-toplevel")
|
||||
except LookupError as error:
|
||||
print(f"score: {error}", file=sys.stderr)
|
||||
return 4 if code == 4 else 2
|
||||
print(table([*lead, *header], [*cells, *row]))
|
||||
print(f"\n{WORKTREE}\n\n```text\n{top}\n```")
|
||||
elif args.table:
|
||||
print(table(header, row))
|
||||
elif args.message_block:
|
||||
worth_line = [f"💰 Value: {args.value}"] if args.value else []
|
||||
print("\n".join((
|
||||
f"🎯 Fix: {values[0]}", f"💥 Break: {values[1]}", f"💸 Cost: {values[2]}",
|
||||
f"⏱️ Time: {values[3]}", f"🛡️ Security: {values[4]}", *worth_line,
|
||||
f"🏆 Score: {total}", f"🚦 Gate: {values[6]}",
|
||||
)))
|
||||
lines = [f"{metric['emoji']} {metric['label']}: {measured[key]}" for key, metric in metrics.items()]
|
||||
if args.value:
|
||||
lines.insert(list(metrics).index("score"), f"{value_header}: {args.value}")
|
||||
gate_label = extras["gate"]
|
||||
block = "\n".join((*lines, f"{gate_label['emoji']} {gate_label['label']}: {verdict_cell}"))
|
||||
print(block, flush=True)
|
||||
journal(args, total, block)
|
||||
else:
|
||||
print(
|
||||
f"🎯 fix {values[0]} · 💥 break {values[1]} · 💸 cost {values[2]} · "
|
||||
f"⏱️ time {values[3]} · 🛡️ security {values[4]} · 🏆 score {total}"
|
||||
)
|
||||
print(" · ".join(f"{metric['emoji']} {metric['label'].lower()} {measured[key]}" for key, metric in metrics.items()))
|
||||
print(f"{emoji} gate: {verdict}")
|
||||
return code
|
||||
|
||||
|
||||
def plain(value: float | str) -> float | int | None:
|
||||
if not isinstance(value, float):
|
||||
return None
|
||||
return int(value) if value.is_integer() else value
|
||||
|
||||
|
||||
def journal(args: argparse.Namespace, total: int, block: str) -> None:
|
||||
values = {
|
||||
"fix": plain(args.fix), "break": plain(args.risk), "tokens": plain(args.efficiency),
|
||||
"velocity": plain(args.speed), "security": plain(args.security[0]), "score": total,
|
||||
}
|
||||
path = journal_path()
|
||||
try:
|
||||
here = Path.cwd()
|
||||
append(path, [entry(int(time.time()), origin(here), git_read(here, "branch", "--show-current") or None, None, values, block)])
|
||||
except (OSError, ValueError) as error:
|
||||
print(f"score: journal {path}: {getattr(error, 'strerror', None) or error}", file=sys.stderr)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -4,7 +4,7 @@ description: >
|
||||
Wall-clock speed of changed code versus its base commit.
|
||||
---
|
||||
|
||||
Its ⏱️ time is the share of its base's wall-clock time a change needs:
|
||||
Its ⏱️ velocity is the share of its base's wall-clock time a change needs:
|
||||
under 100 % it is faster, over 100 % slower. It has no script of its own: one run
|
||||
of the `kvb-check-efficiency` measurement yields both values.
|
||||
|
||||
@@ -12,7 +12,7 @@ of the `kvb-check-efficiency` measurement yields both values.
|
||||
python3 ${CLAUDE_SKILL_DIR}/../kvb-check-efficiency/scripts/measure.py --base <ref> -- <command...>
|
||||
```
|
||||
|
||||
Its `⏱️ time <p> % (wall <b>→<a> s)` compares wall seconds at `<ref>` and
|
||||
Its `⏱️ velocity <p> % (wall <b>→<a> s)` compares wall seconds at `<ref>` and
|
||||
in the working tree: one run each, or the mean of 5 each when a value
|
||||
exceeds 100. The command, the runs, the `queue.py hold` wrapper for heavy
|
||||
commands and the exit codes are those of the `kvb-check-efficiency` skill;
|
||||
@@ -24,6 +24,6 @@ pasting its whole line reports both values at once.
|
||||
renders) has no speed either: `--speed -`, and say why.
|
||||
- **Other load**: wall time follows host load. Above 30 % other load the
|
||||
script falls back to the CPU-time ratio and marks the estimate:
|
||||
`⏱️ time ~<p> % (cpu <b>→<a> s)`, ending in
|
||||
`⏱️ velocity ~<p> % (cpu <b>→<a> s)`, ending in
|
||||
`(estimated: other load <x> %)`. Report the run you
|
||||
made; never rerun to pick the best result.
|
||||
@@ -24,7 +24,7 @@ skill's Rules apply here too.
|
||||
--task <register task> -- <command>`, which waits for the rule-based
|
||||
grant and releases on exit. The fix enters the main line only
|
||||
at the funnel's bar (the `kvb-org-swarm` skill's: fix confidence ≥ 90 %,
|
||||
break risk ≤ 5 %, cost and time ≤ 100 % in the `kvb-check-score`
|
||||
break risk ≤ 5 %, tokens and velocity ≤ 100 % in the `kvb-check-score`
|
||||
block; over 100 %, ask the operator before the commit, naming the cause)
|
||||
after its independent review; below the bar the cluster ends as a
|
||||
proposal for the operator, never as a merge. Never mask a failure: no
|
||||
|
||||
@@ -16,6 +16,6 @@ checklist (multi-select) with suggestions, free text allowed:
|
||||
be changed (files, roles, services); whether new dependencies, toolchains
|
||||
or languages may be introduced; whether infrastructure or CI configuration
|
||||
may change; the merge bar (default: the `kvb-org-swarm` skill's bar of fix
|
||||
confidence ≥ 90 %, break risk ≤ 5 %, cost and time ≤ 100 %, over
|
||||
confidence ≥ 90 %, break risk ≤ 5 %, tokens and velocity ≤ 100 %, over
|
||||
100 % only after asking the operator, naming the cause); a time cap per
|
||||
delegated agent (default 60 minutes).
|
||||
@@ -11,14 +11,14 @@ description: >
|
||||
2. Show in the message right before the `git commit` call, never only after
|
||||
it, the commit table of the kvb-check-score skill (`score.py ...
|
||||
--commit-table <worktree>`): 📁 worktree, 🌿 branch, 📊 staged diff stat,
|
||||
then fix, break, cost, time, security, score and gate of the staged
|
||||
then fix, break, tokens, velocity, security, score and gate of the staged
|
||||
bundle, as the kvb-check-confidence skill produces them: break risk
|
||||
derived through the kvb-check-break-risk skill, never estimated. Never
|
||||
print a diff in the chat, only its stat.
|
||||
3. Approval-prompt duty: the commit call's approval prompt (Claude Code: the
|
||||
Bash `description`; other runtimes: their prompt text) names the worktree
|
||||
path, the branch, the diff stat and the score values (🎯 fix, 💥 break,
|
||||
💸 cost, ⏱️ time, 🛡️ security, 🏆 score). Missing any item: do not
|
||||
💸 tokens, ⏱️ velocity, 🛡️ security, 🏆 score). Missing any item: do not
|
||||
send the call. Gate exit 4 (critical security finding) means no commit:
|
||||
the call is never sent, not even on the operator's word; fix and rescan
|
||||
first. Exit 1 (below 90/5) is shown, the operator's word decides; exit 3
|
||||
|
||||
@@ -24,9 +24,9 @@ confirms it.
|
||||
branch fixes, each verified as the `kvb-git-issue-close` skill does.
|
||||
5. **Confirm** with one `AskUserQuestion`: title (Conventional Commits),
|
||||
base, state and labels. The state follows the bar of the `kvb-org-swarm`
|
||||
skill (fix confidence ≥ 90 %, break risk ≤ 5 %, cost and time
|
||||
skill (fix confidence ≥ 90 %, break risk ≤ 5 %, tokens and velocity
|
||||
≤ 100 %): ready at or above it, a draft with the residuals in the body
|
||||
below it. A cost or time over 100 % goes into this question with
|
||||
below it. A tokens or velocity value over 100 % goes into this question with
|
||||
its cause.
|
||||
6. **Create**: `gh pr create --base <default> --head <branch> --title
|
||||
"<title>" --body-file <file>`, plus `--draft` below the bar.
|
||||
|
||||
@@ -12,7 +12,7 @@ holds:
|
||||
failure on it, a red default-branch run or an issue that reproduces there;
|
||||
name the run or issue.
|
||||
- **`90/5`**: the pull request meets the bar of the `kvb-org-swarm` skill (fix
|
||||
confidence ≥ 90 %, break risk ≤ 5 %, cost and time ≤ 100 %; over
|
||||
confidence ≥ 90 %, break risk ≤ 5 %, tokens and velocity ≤ 100 %; over
|
||||
100 %, ask the operator first, naming the cause), from the
|
||||
`kvb-check-score` block in its body or freshly assessed, and only when
|
||||
`gate.py status` of the `kvb-check-gate` skill prints `gate: green` with
|
||||
@@ -27,7 +27,7 @@ holds:
|
||||
3. A missing label is created once: `gh label create hotfix --color B60205
|
||||
--description "Fixes a failure on the default branch"`, `gh label create
|
||||
90/5 --color 0E8A16 --description
|
||||
"Fix confidence >= 90 %, break risk <= 5 %, cost and time <= 100 %"`.
|
||||
"Fix confidence >= 90 %, break risk <= 5 %, tokens and velocity <= 100 %"`.
|
||||
4. Standalone, confirm the changes with one `AskUserQuestion`; from
|
||||
`kvb-git-pr-create`, its confirmation covers them.
|
||||
5. `gh pr edit <pr> --add-label <l>` or `--remove-label <l>`. A draft that
|
||||
|
||||
@@ -38,8 +38,8 @@ One table, most-blocking first, one row per item:
|
||||
- **State**: `open`, `running`, `blocked`, `done`. `done` means verified (a
|
||||
passing run, a green check, a reproduction that now succeeds); work that
|
||||
merely looks finished is `open`.
|
||||
- **Score**: the `kvb-check-score` line (🎯 fix, 💥 break, 💸 cost,
|
||||
⏱️ time, 🏆 score) per the `kvb-check-confidence` skill, only for rows
|
||||
- **Score**: the `kvb-check-score` line (🎯 fix, 💥 break, 💸 tokens,
|
||||
⏱️ velocity, 🏆 score) per the `kvb-check-confidence` skill, only for rows
|
||||
that changed something; `-` for pure observation rows.
|
||||
- **Evidence / next step**: for `done` the strongest proof line (what ran,
|
||||
what it returned), otherwise the single next action.
|
||||
|
||||
@@ -12,7 +12,7 @@ sidetree or leaves as a proposal.
|
||||
|
||||
## The bar
|
||||
|
||||
- **Fix confidence ≥ 90 %, break risk ≤ 5 %, cost and time ≤ 100 %**,
|
||||
- **Fix confidence ≥ 90 %, break risk ≤ 5 %, tokens and velocity ≤ 100 %**,
|
||||
all from the `kvb-check-score` block of `kvb-check-confidence`; over 100 %,
|
||||
ask the operator before the commit, naming the cause.
|
||||
- The operator may set a different bar. Never lower it to make a change fit.
|
||||
|
||||
@@ -0,0 +1,88 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parents[1]
|
||||
BACKFILL = REPO_ROOT / "skills" / "kvb-check-score" / "scripts" / "backfill.py"
|
||||
EPOCH = 1_760_000_000
|
||||
HANG = 60
|
||||
OLD_BLOCK = "🎯 Fix: 95 %\n💥 Break: 2 %\n💸 Cost: 97 %\n⏱️ Time: –\n🛡️ Security: 100 %\n🏆 Score: 96\n🚦 Gate: ✅ 90/5, cost and time <= 100, security 100 %"
|
||||
NEW_BLOCK = "🎯 Fix: 92 %\n💥 Break: 3 %\n💸 Tokens: 110 %\n⏱️ Velocity: 90 %\n🛡️ Security: 100 %\n💰 Value: 1:20\n🧪 Coverage: 80 %\n🏆 Score: 81\n🚦 Gate: ⚠️ tokens 110 % > 100: ask the operator before the commit"
|
||||
ESTIMATED_BLOCK = "🎯 Fix: 90 %\n💥 Break: 5 %\n💸 Tokens: new\n⏱️ Velocity: 130 %\n🛡️ Security: ~80 %\n🌡️ Heat: 5\n🧪 Coverage: 70 %\n🏆 Score: 52\n🚦 Gate: ⚠️ tokens new: ask the operator before the commit"
|
||||
MALFORMED_BLOCK = "🎯 Fix: 95 %\n💥 Break: 2 %\n🏆 Score: high"
|
||||
TRAILER = "Co-Authored-By: Test <test@example.org>"
|
||||
|
||||
|
||||
def git_env(home: Path) -> dict[str, str]:
|
||||
"""Args:
|
||||
home: a scratch HOME, so no user or system git configuration applies.
|
||||
|
||||
Returns:
|
||||
The environment for fixture git calls.
|
||||
"""
|
||||
return {
|
||||
"PATH": os.path.dirname(shutil.which("git")),
|
||||
"HOME": str(home),
|
||||
"GIT_CONFIG_NOSYSTEM": "1",
|
||||
"GIT_AUTHOR_NAME": "Test",
|
||||
"GIT_AUTHOR_EMAIL": "test@example.org",
|
||||
"GIT_COMMITTER_NAME": "Test",
|
||||
"GIT_COMMITTER_EMAIL": "test@example.org",
|
||||
}
|
||||
|
||||
|
||||
class Fixture:
|
||||
"""Args:
|
||||
base: a scratch directory for upstream.git, the work clone a and the clone b.
|
||||
"""
|
||||
|
||||
def __init__(self, base: Path) -> None:
|
||||
self.base = base
|
||||
self.env = git_env(base)
|
||||
self.upstream = base / "upstream.git"
|
||||
self.a = base / "a"
|
||||
self.b = base / "b"
|
||||
self.step = 0
|
||||
base.mkdir(parents=True)
|
||||
self.git(base, "init", "-q", "--bare", "-b", "main", str(self.upstream))
|
||||
self.a.mkdir()
|
||||
self.git(self.a, "init", "-q", "-b", "main")
|
||||
self.git(self.a, "remote", "add", "origin", str(self.upstream))
|
||||
self.old = self.commit("feat: old labels", OLD_BLOCK)
|
||||
self.git(self.a, "checkout", "-q", "-b", "feature/two")
|
||||
self.new = self.commit("feat: new labels", NEW_BLOCK)
|
||||
self.git(self.a, "checkout", "-q", "main")
|
||||
self.tick()
|
||||
self.git(self.a, "merge", "-q", "--no-ff", "-m", "Merge branch 'feature/two'", "feature/two")
|
||||
self.estimated = self.commit("fix: estimated", ESTIMATED_BLOCK)
|
||||
self.malformed = self.commit("chore: malformed", MALFORMED_BLOCK)
|
||||
self.plain = self.commit("docs: no block", "")
|
||||
self.git(self.a, "push", "-q", "origin", "main", "feature/two")
|
||||
self.git(base, "clone", "-q", str(self.upstream), str(self.b))
|
||||
|
||||
def tick(self) -> None:
|
||||
self.step += 1
|
||||
stamp = f"{EPOCH + self.step * 3600} +0000"
|
||||
self.env["GIT_AUTHOR_DATE"] = stamp
|
||||
self.env["GIT_COMMITTER_DATE"] = stamp
|
||||
|
||||
def git(self, cwd: Path, *args: str) -> str:
|
||||
return subprocess.run(["git", *args], cwd=cwd, env=self.env, capture_output=True, text=True, check=True).stdout.strip()
|
||||
|
||||
def commit(self, subject: str, block: str) -> str:
|
||||
self.tick()
|
||||
messages = [item for part in (subject, block, TRAILER) if part for item in ("-m", part)]
|
||||
self.git(self.a, "commit", "-q", "--allow-empty", *messages)
|
||||
return self.git(self.a, "rev-parse", "HEAD")
|
||||
|
||||
def time_of(self, sha: str) -> int:
|
||||
return int(self.git(self.a, "show", "-s", "--format=%ct", sha))
|
||||
|
||||
|
||||
def backfill(council: Path, *args: str) -> subprocess.CompletedProcess:
|
||||
env = {"PATH": os.environ["PATH"], "AGENTS_COUNCIL_ROOT": str(council)}
|
||||
return subprocess.run([sys.executable, "-B", str(BACKFILL), *args], env=env, capture_output=True, text=True, check=False, timeout=HANG)
|
||||
@@ -33,8 +33,8 @@ NEW_B = "💰 value ?:? (cost 0/day; pass --value with evidence; kvb-b new: all
|
||||
CHEAP = "print('cheap')\n"
|
||||
COSTLY = "data = list(range(2_000_000))\nprint(sum(data))\n"
|
||||
LINE = re.compile(
|
||||
r"^💸 cost (?P<mark>~?)(?P<efficiency>\d+) % \(cpu \d+\.\d\d→\d+\.\d\d s, ram \d+\.\d→\d+\.\d MB\) · "
|
||||
r"⏱️ time (?P<speed_mark>~?)(?P<speed>\d+) % \((?P<basis>wall|cpu) \d+\.\d\d→\d+\.\d\d s\)(?P<tail>.*)$"
|
||||
r"^💸 tokens (?P<mark>~?)(?P<efficiency>\d+) % \(cpu \d+\.\d\d→\d+\.\d\d s, ram \d+\.\d→\d+\.\d MB\) · "
|
||||
r"⏱️ velocity (?P<speed_mark>~?)(?P<speed>\d+) % \((?P<basis>wall|cpu) \d+\.\d\d→\d+\.\d\d s\)(?P<tail>.*)$"
|
||||
)
|
||||
|
||||
|
||||
@@ -121,7 +121,7 @@ class TestTokens(GitRepo):
|
||||
self.write("skills/kvb-a/scripts/tool.py", "print(1)\n" * 50)
|
||||
result = self.tokens()
|
||||
self.assertEqual(
|
||||
result.stdout.splitlines(), ["💸 cost 50 % (skills 100→50 tokens; listing 3→3 words)", NO_VALUE]
|
||||
result.stdout.splitlines(), ["💸 tokens 50 % (skills 100→50 tokens; listing 3→3 words)", NO_VALUE]
|
||||
)
|
||||
self.assertEqual(result.returncode, 0, result.stderr)
|
||||
|
||||
@@ -130,7 +130,7 @@ class TestTokens(GitRepo):
|
||||
self.write("skills/kvb-b/SKILL.md", SKILL_B.ljust(200, "y"))
|
||||
result = self.tokens()
|
||||
self.assertEqual(
|
||||
result.stdout.splitlines(), ["💸 cost 200 % (skills 100→200 tokens; listing 3→5 words)", NEW_B]
|
||||
result.stdout.splitlines(), ["💸 tokens 200 % (skills 100→200 tokens; listing 3→5 words)", NEW_B]
|
||||
)
|
||||
self.assertEqual(result.returncode, 3)
|
||||
result = self.tokens("--json")
|
||||
@@ -157,7 +157,7 @@ class TestTokens(GitRepo):
|
||||
def test_only_new_files_print_new(self):
|
||||
self.write("skills/kvb-b/SKILL.md", SKILL_B.ljust(200, "y"))
|
||||
result = self.tokens()
|
||||
self.assertEqual(result.stdout.splitlines(), ["💸 cost new (+50 tokens; listing 3→5 words)", NEW_B])
|
||||
self.assertEqual(result.stdout.splitlines(), ["💸 tokens new (+50 tokens; listing 3→5 words)", NEW_B])
|
||||
self.assertEqual(result.returncode, 3)
|
||||
self.assertIsNone(json.loads(self.tokens("--json").stdout)["efficiency"])
|
||||
|
||||
@@ -165,14 +165,14 @@ class TestTokens(GitRepo):
|
||||
self.git("mv", "skills/kvb-a/notes.md", "skills/kvb-a/moved.md")
|
||||
result = self.tokens()
|
||||
self.assertEqual(
|
||||
result.stdout.splitlines(), ["💸 cost 100 % (skills 100→100 tokens; listing 3→3 words)", NO_VALUE]
|
||||
result.stdout.splitlines(), ["💸 tokens 100 % (skills 100→100 tokens; listing 3→3 words)", NO_VALUE]
|
||||
)
|
||||
self.assertEqual(result.returncode, 0)
|
||||
|
||||
def test_no_change_is_100(self):
|
||||
result = self.tokens()
|
||||
self.assertEqual(
|
||||
result.stdout.splitlines(), ["💸 cost 100 % (skills 0→0 tokens; listing 3→3 words)", NO_VALUE]
|
||||
result.stdout.splitlines(), ["💸 tokens 100 % (skills 0→0 tokens; listing 3→3 words)", NO_VALUE]
|
||||
)
|
||||
self.assertEqual(result.returncode, 0)
|
||||
|
||||
@@ -193,7 +193,7 @@ class TestTokens(GitRepo):
|
||||
self.assertEqual(
|
||||
result.stdout.splitlines(),
|
||||
[
|
||||
"💸 cost 869 % (skills 13→113 tokens; listing 3→5 words)",
|
||||
"💸 tokens 869 % (skills 13→113 tokens; listing 3→5 words)",
|
||||
"💰 value ?:? (cost 203/day; pass --value with evidence)",
|
||||
],
|
||||
)
|
||||
@@ -235,6 +235,22 @@ class TestTokens(GitRepo):
|
||||
)
|
||||
self.assertEqual(result.returncode, 3)
|
||||
|
||||
def test_a_malformed_label_file_exits_2_with_one_stderr_line(self):
|
||||
for text in ("{broken", json.dumps({"metrics": {"tokens": {"emoji": 1, "label": "Tokens"}}}), "[]"):
|
||||
with self.subTest(labels=text):
|
||||
labels = self.base / "labels.json"
|
||||
labels.write_text(text, encoding="utf-8")
|
||||
out, err = io.StringIO(), io.StringIO()
|
||||
argv = ["--repo", str(self.repo), "--base", "HEAD", "--transcripts", str(self.base / "projects")]
|
||||
with mock.patch.object(skill_tokens, "LABELS", labels), contextlib.redirect_stdout(out), contextlib.redirect_stderr(err):
|
||||
try:
|
||||
code = skill_tokens.main(argv)
|
||||
except SystemExit as stop:
|
||||
code = stop.code
|
||||
self.assertEqual((code, out.getvalue()), (2, ""))
|
||||
self.assertEqual(len(err.getvalue().splitlines()), 1, err.getvalue())
|
||||
self.assertTrue(err.getvalue().startswith(f"tokens: labels {labels}: "), err.getvalue())
|
||||
|
||||
def test_usage_errors_exit_2(self):
|
||||
self.assertEqual(self.tokens("--value", "5").returncode, 2)
|
||||
self.assertEqual(self.tokens("--evidence", "why").returncode, 2)
|
||||
@@ -381,6 +397,15 @@ class TestMeasure(GitRepo):
|
||||
self.assertEqual(trees, [f"worktree {self.repo.resolve()}"])
|
||||
self.assertLessEqual(set(Path("/tmp").glob("kvb-measure-*")), self.leftovers)
|
||||
|
||||
def test_a_malformed_label_file_exits_2_before_any_run(self):
|
||||
labels = self.base / "labels.json"
|
||||
labels.write_text(json.dumps({"metrics": {"tokens": {"emoji": "💸", "label": "Tokens"}}}), encoding="utf-8")
|
||||
with mock.patch.object(measure, "LABELS", labels):
|
||||
code, out, err, samples = self.measured("--base", "HEAD", "--", sys.executable, "work.py")
|
||||
self.assertEqual((code, out, samples), (2, "", 0))
|
||||
self.assertEqual(len(err.splitlines()), 1, err)
|
||||
self.assertTrue(err.startswith(f"measure: labels {labels}: "), err)
|
||||
|
||||
def test_cheaper_head_stops_after_one_run_each(self):
|
||||
self.write("pkg/work.py", CHEAP)
|
||||
code, out, err, samples = self.measured("--base", "HEAD", "--", sys.executable, "work.py")
|
||||
|
||||
@@ -0,0 +1,232 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import subprocess
|
||||
import unittest
|
||||
|
||||
from hub_support import HUB, PLAN, REPO_ROOT
|
||||
|
||||
GANTT = HUB / "templates" / "page-tasks-gantt.js"
|
||||
LABELS = REPO_ROOT / "skills" / "kvb-check-score" / "labels.json"
|
||||
SERIES_PROBE = """
|
||||
const P = require(process.argv[1]);
|
||||
const rows = [
|
||||
{task: "work/a", branch: "feature/a", worktree_branch: ""},
|
||||
{task: "work/b", branch: "-", worktree_branch: "fix/b"},
|
||||
{task: "work/c", branch: "-", worktree_branch: ""},
|
||||
];
|
||||
const values = (fix, score) => ({fix, break: 2, tokens: null, velocity: 130, security: 100, score});
|
||||
const journal = [
|
||||
{time: 300, branch: "feature/a", values: values(95, 90)},
|
||||
{time: 100, branch: "fix/b", values: values(92, null)},
|
||||
{time: 200, branch: "feature/other", values: values(10, 10)},
|
||||
{time: 900, branch: "feature/a", values: values(1, 1)},
|
||||
{time: 150, branch: "", values: values(5, 5)},
|
||||
{time: 250, branch: "-", values: values(6, 6)},
|
||||
];
|
||||
const series = P.overlaySeries(rows, journal, ["fix", "tokens", "score"], {from: 100, to: 800});
|
||||
console.log(JSON.stringify({
|
||||
keys: Object.keys(P),
|
||||
series,
|
||||
none: P.overlaySeries(rows, journal, [], {from: 0, to: 1000}),
|
||||
tops: [P.overlayTop(series), P.overlayTop(P.overlaySeries(rows, journal, ["velocity"], {from: 0, to: 1000})), P.overlayTop([])],
|
||||
labels: [null, "x", {}, {metrics: []}, {metrics: {}}, {metrics: {fix: null}}, {metrics: {fix: {emoji: 1, label: "Fix"}}},
|
||||
{metrics: {fix: {emoji: "🎯", label: "Fix"}, bad: {emoji: "x"}, score: {emoji: "🏆", label: "Score"}}}].map(P.overlayLabels),
|
||||
}));
|
||||
"""
|
||||
BURN_PROBE = """
|
||||
const fs = require("fs");
|
||||
const [source, plan, labelFile] = process.argv.slice(1);
|
||||
const P = require(plan);
|
||||
const labels = JSON.parse(fs.readFileSync(labelFile, "utf8"));
|
||||
const node = (tag, attrs, children) => ({tag, attrs, children: children.flat(Infinity).filter((child) => child !== null && child !== undefined && child !== false), append(...more) { this.children.push(...more); }});
|
||||
const s = (tag, attrs, ...children) => node(tag, attrs, children);
|
||||
const LEAF = /^[A-Za-z]{1,32}\\.[A-Za-z]{1,32}$/u;
|
||||
const METRICS = Object.keys(labels.metrics);
|
||||
const kept = {value: [], listeners: [], updates: 0};
|
||||
const known = (path) => {
|
||||
if (path !== "burn.overlays") {
|
||||
throw new Error(`unknown view path ${path}`);
|
||||
}
|
||||
};
|
||||
const ViewStore = {
|
||||
get: (path) => {
|
||||
known(path);
|
||||
return [...kept.value];
|
||||
},
|
||||
set: (path) => {
|
||||
known(path);
|
||||
throw new Error(`${path} holds a list: change it through update`);
|
||||
},
|
||||
update: (path, change) => {
|
||||
if (!LEAF.test(String(path)) || typeof change !== "function") {
|
||||
throw new Error(`update needs a section.key path and a change function, not ${path}`);
|
||||
}
|
||||
known(path);
|
||||
const next = change([...kept.value]);
|
||||
if (!Array.isArray(next) || !next.every((item) => METRICS.includes(item)) || new Set(next).size !== next.length) {
|
||||
throw new Error(`burn.overlays must list distinct metrics, not ${JSON.stringify(next)}`);
|
||||
}
|
||||
kept.value = next;
|
||||
kept.updates += 1;
|
||||
return Promise.resolve(true);
|
||||
},
|
||||
on: (path, callback) => {
|
||||
known(path);
|
||||
kept.listeners.push([path, callback]);
|
||||
},
|
||||
};
|
||||
const rendered = [];
|
||||
const LegendChips = {render: (series, path, mode) => {
|
||||
if (mode !== "hidden" && mode !== "shown") {
|
||||
throw new Error(`a legend chip mode must be hidden or shown, not ${mode}`);
|
||||
}
|
||||
if (!LEAF.test(String(path)) || !Array.isArray(series) || !series.every((entry) => entry && Object.keys(entry).sort().join() === "id,label,tone" && ["id", "label", "tone"].every((name) => typeof entry[name] === "string"))) {
|
||||
throw new Error("legend chips need a section.key path and series [{id, label, tone}]");
|
||||
}
|
||||
const press = (id) => ViewStore.update(path, (list) => (list.includes(id) ? list.filter((item) => item !== id) : [...list, id]));
|
||||
rendered.push({series, path, mode, press});
|
||||
return node("div", {class: "legend-chips"}, []);
|
||||
}};
|
||||
const refused = (call) => {
|
||||
try {
|
||||
call();
|
||||
return false;
|
||||
} catch {
|
||||
return true;
|
||||
}
|
||||
};
|
||||
const contract = [
|
||||
refused(() => LegendChips.render([{key: "fix", label: "Fix"}], "burn.overlays", "shown")),
|
||||
refused(() => LegendChips.render([{id: "fix", label: "Fix", tone: "k-fix"}], "burn.overlays", "on")),
|
||||
refused(() => LegendChips.render([{id: "fix", label: "Fix", tone: "k-fix"}], "burn.overlays")),
|
||||
refused(() => ViewStore.set("burn.overlays", ["fix"])),
|
||||
refused(() => ViewStore.update("burn.overlays", () => [null])),
|
||||
];
|
||||
const journal = [
|
||||
{time: 600, branch: "feature/a", values: {fix: 95, break: 2, tokens: 97, velocity: null, security: 100, score: 90}},
|
||||
{time: 300, branch: "feature/a", values: {fix: 92, break: 2, tokens: 97, velocity: null, security: 100, score: null}},
|
||||
{time: 400, branch: "feature/other", values: {fix: 10, break: 2, tokens: 97, velocity: null, security: 100, score: 10}},
|
||||
];
|
||||
const fetched = [];
|
||||
const fetch = async (url) => {
|
||||
fetched.push(url);
|
||||
return {ok: true, json: async () => ({since: 0, until: 1000, labels, rows: journal})};
|
||||
};
|
||||
const counts = {draws: 0};
|
||||
const taskView = {sig: "drawn"};
|
||||
const drawTasks = () => {
|
||||
counts.draws += 1;
|
||||
};
|
||||
const build = new Function("Plan", "s", "h", "fixed", "stampOf", "absolute", "ViewStore", "LegendChips", "fetch", "drawTasks", "taskView", "ui", `${fs.readFileSync(source, "utf8")}\\nreturn {burnOverlays, overlaysReady, burnRight};`);
|
||||
const helpers = [P, s, s, (value) => value.toFixed(1), (ts) => new Date(ts * 1000), {format: (date) => date.toISOString()}];
|
||||
const api = build(...helpers, ViewStore, LegendChips, fetch, drawTasks, taskView, {});
|
||||
const inert = build(...helpers, undefined, undefined, fetch, drawTasks, taskView, {});
|
||||
const brokenStore = {get: () => ["fix"], update: () => Promise.resolve(true), on: () => {}};
|
||||
const brokenFetch = async () => ({ok: true, json: async () => ({labels: {}, rows: "x"})});
|
||||
const broken = build(...helpers, brokenStore, LegendChips, brokenFetch, () => {}, {sig: ""}, {});
|
||||
const picked = {rows: [{task: "work/a", branch: "feature/a", worktree_branch: ""}], from: 0, to: 1000};
|
||||
const shape = (svg) => ({
|
||||
overlays: svg.children.filter((child) => child.tag === "g").map((group) => ({cls: group.attrs.class, key: group.attrs["data-overlay"], points: group.children.filter((child) => child.tag === "circle").map((circle) => [circle.attrs.cx, circle.attrs.cy])})),
|
||||
axis: svg.children.filter((child) => child.tag === "text").map((text) => [text.attrs.x, text.children[0]]),
|
||||
});
|
||||
(async () => {
|
||||
const pending = api.burnOverlays(s("svg", {}), picked, (t) => t, api.burnRight());
|
||||
const brokenFirst = broken.burnOverlays(s("svg", {}), picked, (t) => t, broken.burnRight());
|
||||
await new Promise((resolve) => setTimeout(resolve, 0));
|
||||
const brokenSvg = s("svg", {});
|
||||
const brokenAfter = broken.burnOverlays(brokenSvg, picked, (t) => t, broken.burnRight());
|
||||
const brokenView = [brokenFirst, brokenAfter, brokenSvg.children.length, broken.burnRight()];
|
||||
const off = s("svg", {});
|
||||
const rights = {inert: [inert.overlaysReady(), inert.burnRight()], off: api.burnRight()};
|
||||
const offChips = api.burnOverlays(off, picked, (t) => t, rights.off);
|
||||
const stored = [await rendered[0].press("score"), await rendered[0].press("fix")];
|
||||
const pressed = {stored, value: [...kept.value], updates: kept.updates};
|
||||
rights.on = api.burnRight();
|
||||
const on = s("svg", {});
|
||||
api.burnOverlays(on, picked, (t) => t, rights.on);
|
||||
const drawsBefore = counts.draws;
|
||||
taskView.sig = "drawn";
|
||||
kept.listeners[0][1]();
|
||||
console.log(JSON.stringify({
|
||||
ready: api.overlaysReady(), pending, fetched, firstDraws: drawsBefore, listeners: kept.listeners.map(([path]) => path), redrawn: [counts.draws - drawsBefore, taskView.sig],
|
||||
off: shape(off), offChips: offChips.attrs.class, chips: rendered[0], on: shape(on), rights, brokenView, contract, pressed,
|
||||
}));
|
||||
})();
|
||||
"""
|
||||
|
||||
|
||||
class TestOverlaySeries(unittest.TestCase):
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
done = subprocess.run(["node", "-e", SERIES_PROBE, str(PLAN)], capture_output=True, text=True, timeout=60, check=True)
|
||||
cls.data = json.loads(done.stdout)
|
||||
|
||||
def test_only_rows_of_the_scope_branches_within_the_span_are_drawn_in_the_given_order(self):
|
||||
self.assertIn("overlaySeries", self.data["keys"])
|
||||
self.assertNotIn("OVERLAYS", self.data["keys"])
|
||||
self.assertEqual(self.data["series"], [
|
||||
{"key": "fix", "points": [{"t": 100, "value": 92, "branch": "fix/b"}, {"t": 300, "value": 95, "branch": "feature/a"}]},
|
||||
{"key": "tokens", "points": []},
|
||||
{"key": "score", "points": [{"t": 300, "value": 90, "branch": "feature/a"}]},
|
||||
])
|
||||
self.assertEqual(self.data["none"], [])
|
||||
|
||||
def test_the_right_axis_runs_to_100_or_the_largest_value_above_it(self):
|
||||
self.assertEqual(self.data["tops"], [100, 130, 100])
|
||||
|
||||
def test_labels_of_another_shape_switch_the_overlays_off(self):
|
||||
self.assertEqual(self.data["labels"], [None, None, None, None, None, None, None, {"fix": "🎯 Fix", "score": "🏆 Score"}])
|
||||
|
||||
|
||||
class TestBurnOverlays(unittest.TestCase):
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
done = subprocess.run(["node", "-e", BURN_PROBE, str(GANTT), str(PLAN), str(LABELS)], capture_output=True, text=True, timeout=60, check=True)
|
||||
cls.data = json.loads(done.stdout)
|
||||
|
||||
def test_six_metric_chips_all_off_on_first_load_keyed_burn_overlays(self):
|
||||
data = self.data
|
||||
self.assertEqual((data["ready"], data["pending"], data["fetched"], data["firstDraws"]), (True, None, ["/api/scores?from=0&to=1000"], 1))
|
||||
self.assertEqual((data["chips"]["path"], data["chips"]["mode"]), ("burn.overlays", "shown"))
|
||||
self.assertEqual(data["chips"]["series"], [
|
||||
{"id": key, "label": label, "tone": f"k-{key}"}
|
||||
for key, label in (("fix", "🎯 Fix"), ("break", "💥 Break"), ("tokens", "💸 Tokens"), ("velocity", "⏱️ Velocity"), ("security", "🛡️ Security"), ("score", "🏆 Score"))
|
||||
])
|
||||
self.assertEqual((data["off"], data["offChips"]), ({"overlays": [], "axis": []}, "legend-chips"))
|
||||
|
||||
def test_the_contract_stub_refuses_another_shape_mode_or_write(self):
|
||||
self.assertEqual(self.data["contract"], [True] * 5)
|
||||
|
||||
def test_a_chip_press_stores_its_metric_through_update(self):
|
||||
self.assertEqual(self.data["pressed"], {"stored": [True, True], "value": ["score", "fix"], "updates": 2})
|
||||
|
||||
def test_labels_of_another_shape_draw_no_overlay_and_never_throw(self):
|
||||
self.assertEqual(self.data["brokenView"], [None, None, 0, 24])
|
||||
|
||||
def test_the_right_margin_widens_only_while_an_overlay_is_on(self):
|
||||
self.assertEqual(self.data["rights"], {"inert": [False, 24], "off": 24, "on": 56})
|
||||
|
||||
def test_a_switched_on_chip_draws_its_series_on_the_right_axis(self):
|
||||
on = self.data["on"]
|
||||
self.assertEqual(on["axis"], [[850, "0 %"], [850, "50 %"], [850, "100 %"]])
|
||||
self.assertEqual(on["overlays"], [
|
||||
{"cls": "burn-overlay k-fix", "key": "fix", "points": [["300.0", "42.4"], ["600.0", "34.0"]]},
|
||||
{"cls": "burn-overlay k-score", "key": "score", "points": [["600.0", "48.0"]]},
|
||||
])
|
||||
|
||||
def test_a_view_store_change_redraws_the_burndown(self):
|
||||
self.assertEqual(self.data["listeners"], ["burn.overlays"])
|
||||
self.assertEqual(self.data["redrawn"], [1, ""])
|
||||
|
||||
|
||||
class TestOverlayTones(unittest.TestCase):
|
||||
def test_every_metric_chip_has_a_tone_colour(self):
|
||||
css = (HUB / "templates" / "style-tasks.css").read_text(encoding="utf-8")
|
||||
for key in json.loads(LABELS.read_text(encoding="utf-8"))["metrics"]:
|
||||
with self.subTest(key=key):
|
||||
self.assertRegex(css, rf"\.burn-box \.legend-chip\.k-{key}\s*\{{\s*--c: var\(--[a-z]+\); \}}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,60 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import unittest
|
||||
|
||||
from hub_case import HubCase
|
||||
from score_fixture import REPO_ROOT, Fixture, backfill
|
||||
|
||||
LABELS = REPO_ROOT / "skills" / "kvb-check-score" / "labels.json"
|
||||
T = 1_760_000_000
|
||||
|
||||
|
||||
class TestHubScores(HubCase):
|
||||
def fetch(self, query: str) -> tuple[int, bytes]:
|
||||
status, _, body = self.get(f"/api/scores?{query}")
|
||||
return status, body
|
||||
|
||||
def test_the_hub_serves_the_backfilled_journal_rows_of_a_window(self):
|
||||
fixture = Fixture(self.council.parent / "fixture")
|
||||
done = backfill(self.council, "--repo", str(fixture.a))
|
||||
self.assertEqual(done.returncode, 0, done.stderr)
|
||||
begin, end = fixture.time_of(fixture.new), fixture.time_of(fixture.estimated)
|
||||
status, body = self.fetch(f"from={begin}&to={end}")
|
||||
self.assertEqual(status, 200, body)
|
||||
found = json.loads(body)
|
||||
self.assertEqual((found["since"], found["until"]), (begin, end))
|
||||
self.assertEqual(found["labels"], json.loads(LABELS.read_text(encoding="utf-8")))
|
||||
self.assertEqual([(row["sha"], row["branch"]) for row in found["rows"]], [(fixture.new, "feature/two"), (fixture.estimated, "main")])
|
||||
self.assertEqual([row["values"]["score"] for row in found["rows"]], [81, 52])
|
||||
|
||||
def test_broken_lines_and_rows_without_values_are_left_out_in_time_order(self):
|
||||
self.council.mkdir(exist_ok=True)
|
||||
late = {"time": T, "branch": "feature/a", "values": {"score": 90}}
|
||||
early = {"time": T - 10, "branch": "feature/a", "values": {"score": 80}}
|
||||
lines = [json.dumps(late), "{broken", json.dumps({"time": T}), json.dumps({"time": "x", "values": {}}), json.dumps(early), json.dumps({**late, "time": T + 500})]
|
||||
(self.council / "scores.jsonl").write_text("\n".join(lines) + "\n", encoding="utf-8")
|
||||
status, body = self.fetch(f"from={T - 100}&to={T + 100}")
|
||||
self.assertEqual(status, 200, body)
|
||||
self.assertEqual(json.loads(body)["rows"], [early, late])
|
||||
self.assertEqual(self.fetch(f"from={T + 10}&to={T}")[0], 400)
|
||||
|
||||
def test_a_label_file_of_another_shape_is_served_as_null(self):
|
||||
skills = self.council.parent / "skills"
|
||||
labels = skills / "kvb-check-score" / "labels.json"
|
||||
labels.parent.mkdir(parents=True)
|
||||
self.scripts.skills = skills
|
||||
good = {"metrics": {"fix": {"emoji": "🎯", "label": "Fix", "former": []}}}
|
||||
for text, served in (("{}", None), ('{"metrics": []}', None), ('{"metrics": {}}', None), ('{"metrics": {"fix": {"emoji": 1, "label": "Fix"}}}', None), ("{broken", None), (json.dumps(good), good)):
|
||||
with self.subTest(labels=text):
|
||||
labels.write_text(text, encoding="utf-8")
|
||||
status, body = self.fetch(f"from={T}&to={T + 1}")
|
||||
self.assertEqual((status, json.loads(body)["labels"]), (200, served))
|
||||
|
||||
def test_a_missing_journal_answers_no_rows(self):
|
||||
status, body = self.fetch(f"from={T}&to={T + 1}")
|
||||
self.assertEqual((status, json.loads(body)["rows"]), (200, []))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
+25
-24
@@ -12,11 +12,12 @@ from pathlib import Path
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parents[1]
|
||||
SCORE = REPO_ROOT / "skills" / "kvb-check-score" / "scripts" / "score.py"
|
||||
PASS = "✅ gate: 90/5, cost and time <= 100, security 100 %"
|
||||
PASS = "✅ gate: 90/5, tokens and velocity <= 100, security 100 %"
|
||||
BASE = {"fix": "95", "break": "2", "efficiency": "100", "speed": "100", "security": "100", "critical": "0"}
|
||||
HEADER = "| 🎯 Fix | 💥 Break | 💸 Cost | ⏱️ Time | 🛡️ Security | 🏆 Score | Gate |"
|
||||
HEADER = "| 🎯 Fix | 💥 Break | 💸 Tokens | ⏱️ Velocity | 🛡️ Security | 🏆 Score | Gate |"
|
||||
ASK = ": ask the operator before the commit"
|
||||
ENV = {"PATH": os.environ["PATH"]}
|
||||
COUNCIL = tempfile.TemporaryDirectory()
|
||||
ENV = {"PATH": os.environ["PATH"], "AGENTS_COUNCIL_ROOT": COUNCIL.name}
|
||||
|
||||
|
||||
def flags(values: dict[str, str]) -> list[str]:
|
||||
@@ -37,12 +38,12 @@ class TestScore(unittest.TestCase):
|
||||
def test_formula_and_line_format(self):
|
||||
self.assertEqual(
|
||||
self._score(fix="95", **{"break": "4"}),
|
||||
(["🎯 fix 95 % · 💥 break 4 % · 💸 cost 100 % · ⏱️ time 100 % · 🛡️ security 100 % · 🏆 score 91",
|
||||
(["🎯 fix 95 % · 💥 break 4 % · 💸 tokens 100 % · ⏱️ velocity 100 % · 🛡️ security 100 % · 🏆 score 91",
|
||||
PASS], 0),
|
||||
)
|
||||
lines, code = self._score(efficiency="97", speed="-")
|
||||
self.assertEqual(
|
||||
lines[0], "🎯 fix 95 % · 💥 break 2 % · 💸 cost 97 % · ⏱️ time – · 🛡️ security 100 % · 🏆 score 96"
|
||||
lines[0], "🎯 fix 95 % · 💥 break 2 % · 💸 tokens 97 % · ⏱️ velocity – · 🛡️ security 100 % · 🏆 score 96"
|
||||
)
|
||||
self.assertEqual(code, 0)
|
||||
self.assertEqual(self._score(fix="92.5")[0][0].split(" · ")[0], "🎯 fix 92.5 %")
|
||||
@@ -51,14 +52,14 @@ class TestScore(unittest.TestCase):
|
||||
lines, code = self._score(fix="92", efficiency="-", speed="-", **{"break": "3"})
|
||||
self.assertEqual(
|
||||
(lines, code),
|
||||
(["🎯 fix 92 % · 💥 break 3 % · 💸 cost – · ⏱️ time – · 🛡️ security 100 % · 🏆 score 89", PASS], 0),
|
||||
(["🎯 fix 92 % · 💥 break 3 % · 💸 tokens – · ⏱️ velocity – · 🛡️ security 100 % · 🏆 score 89", PASS], 0),
|
||||
)
|
||||
|
||||
def test_security_multiplies_and_asks_below_100(self):
|
||||
lines, code = self._score(fix="100", efficiency="-", speed="-", security="75", **{"break": "0"})
|
||||
self.assertEqual(
|
||||
(lines, code),
|
||||
(["🎯 fix 100 % · 💥 break 0 % · 💸 cost – · ⏱️ time – · 🛡️ security 75 % · 🏆 score 75",
|
||||
(["🎯 fix 100 % · 💥 break 0 % · 💸 tokens – · ⏱️ velocity – · 🛡️ security 75 % · 🏆 score 75",
|
||||
f"⚠️ gate: security 75 % < 100{ASK}"], 3),
|
||||
)
|
||||
lines, code = self._score(security="~80")
|
||||
@@ -94,7 +95,7 @@ class TestScore(unittest.TestCase):
|
||||
({"fix": "80", "break": "10"}, "⛔ gate: fix 80 % < 90, break 10 % > 5"),
|
||||
(
|
||||
{"fix": "85", "efficiency": "150", "speed": "new", "security": "50"},
|
||||
f"⛔ gate: fix 85 % < 90; cost 150 % > 100, time new, security 50 % < 100{ASK}",
|
||||
f"⛔ gate: fix 85 % < 90; tokens 150 % > 100, velocity new, security 50 % < 100{ASK}",
|
||||
),
|
||||
):
|
||||
with self.subTest(values=values):
|
||||
@@ -103,13 +104,13 @@ class TestScore(unittest.TestCase):
|
||||
|
||||
def test_costs_ask_the_operator_exit_3(self):
|
||||
for values, verdict in (
|
||||
({"efficiency": "120"}, f"⚠️ gate: cost 120 % > 100{ASK}"),
|
||||
({"efficiency": "-", "speed": "130"}, f"⚠️ gate: time 130 % > 100{ASK}"),
|
||||
({"efficiency": "100.5", "speed": "-"}, f"⚠️ gate: cost 100.5 % > 100{ASK}"),
|
||||
({"efficiency": "new", "speed": "-"}, f"⚠️ gate: cost new{ASK}"),
|
||||
({"efficiency": "120"}, f"⚠️ gate: tokens 120 % > 100{ASK}"),
|
||||
({"efficiency": "-", "speed": "130"}, f"⚠️ gate: velocity 130 % > 100{ASK}"),
|
||||
({"efficiency": "100.5", "speed": "-"}, f"⚠️ gate: tokens 100.5 % > 100{ASK}"),
|
||||
({"efficiency": "new", "speed": "-"}, f"⚠️ gate: tokens new{ASK}"),
|
||||
(
|
||||
{"efficiency": "120", "speed": "130", "security": "90"},
|
||||
f"⚠️ gate: cost 120 % > 100, time 130 % > 100, security 90 % < 100{ASK}",
|
||||
f"⚠️ gate: tokens 120 % > 100, velocity 130 % > 100, security 90 % < 100{ASK}",
|
||||
),
|
||||
):
|
||||
with self.subTest(values=values):
|
||||
@@ -128,7 +129,7 @@ class TestScore(unittest.TestCase):
|
||||
def test_new_counts_as_100_in_the_score(self):
|
||||
lines, _ = self._score(efficiency="new", speed="-", **{"break": "0"})
|
||||
self.assertEqual(
|
||||
lines[0], "🎯 fix 95 % · 💥 break 0 % · 💸 cost new · ⏱️ time – · 🛡️ security 100 % · 🏆 score 95"
|
||||
lines[0], "🎯 fix 95 % · 💥 break 0 % · 💸 tokens new · ⏱️ velocity – · 🛡️ security 100 % · 🏆 score 95"
|
||||
)
|
||||
|
||||
def test_table(self):
|
||||
@@ -138,7 +139,7 @@ class TestScore(unittest.TestCase):
|
||||
[
|
||||
HEADER,
|
||||
"| --- | --- | --- | --- | --- | --- | --- |",
|
||||
"| 95 % | 2 % | 97 % | – | 100 % | 96 | ✅ 90/5, cost and time <= 100, security 100 % |",
|
||||
"| 95 % | 2 % | 97 % | – | 100 % | 96 | ✅ 90/5, tokens and velocity <= 100, security 100 % |",
|
||||
],
|
||||
)
|
||||
self.assertEqual(result.returncode, 0)
|
||||
@@ -152,8 +153,8 @@ class TestScore(unittest.TestCase):
|
||||
(result.stdout.splitlines(), result.returncode),
|
||||
(
|
||||
[
|
||||
"🎯 Fix: 95 %", "💥 Break: 2 %", "💸 Cost: 97 %", "⏱️ Time: –", "🛡️ Security: 100 %",
|
||||
"🏆 Score: 96", "🚦 Gate: ✅ 90/5, cost and time <= 100, security 100 %",
|
||||
"🎯 Fix: 95 %", "💥 Break: 2 %", "💸 Tokens: 97 %", "⏱️ Velocity: –", "🛡️ Security: 100 %",
|
||||
"🏆 Score: 96", "🚦 Gate: ✅ 90/5, tokens and velocity <= 100, security 100 %",
|
||||
],
|
||||
0,
|
||||
),
|
||||
@@ -165,7 +166,7 @@ class TestScore(unittest.TestCase):
|
||||
(result.stdout.splitlines(), result.returncode),
|
||||
(
|
||||
[
|
||||
"🎯 Fix: 95 %", "💥 Break: 2 %", "💸 Cost: 100 %", "⏱️ Time: 100 %", "🛡️ Security: ~80 %",
|
||||
"🎯 Fix: 95 %", "💥 Break: 2 %", "💸 Tokens: 100 %", "⏱️ Velocity: 100 %", "🛡️ Security: ~80 %",
|
||||
"💰 Value: 1:20", "🏆 Score: 74", f"🚦 Gate: ⚠️ security ~80 % < 100{ASK}",
|
||||
],
|
||||
3,
|
||||
@@ -188,9 +189,9 @@ class TestScore(unittest.TestCase):
|
||||
|
||||
def test_value_column_changes_neither_score_nor_gate(self):
|
||||
for value, verdict, code in (
|
||||
("1:20", "✅ 90/5, cost and time <= 100, security 100 %", 0),
|
||||
("4.1:1", "✅ 90/5, cost and time <= 100, security 100 %", 0),
|
||||
("?", "⚠️ cost 120 % > 100" + ASK, 3),
|
||||
("1:20", "✅ 90/5, tokens and velocity <= 100, security 100 %", 0),
|
||||
("4.1:1", "✅ 90/5, tokens and velocity <= 100, security 100 %", 0),
|
||||
("?", "⚠️ tokens 120 % > 100" + ASK, 3),
|
||||
):
|
||||
with self.subTest(value=value):
|
||||
efficiency = "97" if code == 0 else "120"
|
||||
@@ -201,7 +202,7 @@ class TestScore(unittest.TestCase):
|
||||
self.assertEqual(
|
||||
result.stdout.splitlines(),
|
||||
[
|
||||
HEADER.replace("💸 Cost |", "💸 Cost | 💰 Value |"),
|
||||
HEADER.replace("💸 Tokens |", "💸 Tokens | 💰 Value |"),
|
||||
"|" + " --- |" * 8,
|
||||
f"| 95 % | 2 % | {efficiency} % | {value} | – | 100 % | {score} | {verdict} |",
|
||||
],
|
||||
@@ -236,7 +237,7 @@ class TestScore(unittest.TestCase):
|
||||
(repo / "b.txt").write_text("three\nfour\n", encoding="utf-8")
|
||||
git("add", "a.txt", "b.txt")
|
||||
head = ["🌿 Branch", "📊 Diff stat"]
|
||||
score = "| 95 % | 2 % | 100 % | 100 % | 100 % | 93 | ✅ 90/5, cost and time <= 100, security 100 % |"
|
||||
score = "| 95 % | 2 % | 100 % | 100 % | 100 % | 93 | ✅ 90/5, tokens and velocity <= 100, security 100 % |"
|
||||
worktree = ["", "📁 Worktree:", "", "```text", str(repo.resolve()), "```"]
|
||||
|
||||
result = self._run(*flags(BASE), "--commit-table", str(repo), cwd=repo)
|
||||
@@ -259,7 +260,7 @@ class TestScore(unittest.TestCase):
|
||||
result = self._run(*flags(BASE), "--commit-table", str(repo), "--commit", "HEAD", "--value", "?", cwd=repo)
|
||||
self.assertEqual(
|
||||
result.stdout.splitlines()[0],
|
||||
"| " + " | ".join([*head, "🔖 Commit"]) + " " + HEADER.replace("💸 Cost |", "💸 Cost | 💰 Value |"),
|
||||
"| " + " | ".join([*head, "🔖 Commit"]) + " " + HEADER.replace("💸 Tokens |", "💸 Tokens | 💰 Value |"),
|
||||
)
|
||||
self.assertEqual(
|
||||
result.stdout.splitlines()[2],
|
||||
|
||||
@@ -0,0 +1,165 @@
|
||||
"""Exercise backfill.py of kvb-check-score over a fixture repository: rows, idempotence, determinism, labels, clones."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
from score_fixture import (
|
||||
BACKFILL,
|
||||
ESTIMATED_BLOCK,
|
||||
HANG,
|
||||
NEW_BLOCK,
|
||||
OLD_BLOCK,
|
||||
REPO_ROOT,
|
||||
Fixture,
|
||||
backfill,
|
||||
)
|
||||
|
||||
JOURNAL_NAME = "scores.jsonl"
|
||||
LABELS = REPO_ROOT / "skills" / "kvb-check-score" / "labels.json"
|
||||
UNKNOWN = ["backfill: unknown label Coverage, skipped", "backfill: unknown label Heat, skipped"]
|
||||
|
||||
|
||||
class TestBackfill(unittest.TestCase):
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
cls.tmp = tempfile.TemporaryDirectory()
|
||||
cls.fixture = Fixture(Path(cls.tmp.name) / "fixture")
|
||||
|
||||
@classmethod
|
||||
def tearDownClass(cls):
|
||||
cls.tmp.cleanup()
|
||||
|
||||
def setUp(self):
|
||||
scratch = tempfile.TemporaryDirectory()
|
||||
self.addCleanup(scratch.cleanup)
|
||||
self.scratch = Path(scratch.name)
|
||||
|
||||
def council(self, name: str) -> Path:
|
||||
return self.scratch / name
|
||||
|
||||
def rows(self, council: Path) -> list[dict]:
|
||||
return [json.loads(line) for line in (council / JOURNAL_NAME).read_text(encoding="utf-8").splitlines()]
|
||||
|
||||
def test_three_commits_with_blocks_give_three_rows_then_none(self):
|
||||
council = self.council("one")
|
||||
first = backfill(council, "--repo", str(self.fixture.a))
|
||||
self.assertEqual(first.returncode, 0, first.stderr)
|
||||
self.assertEqual(first.stdout, f"{self.fixture.a.resolve()}: 3 blocks, 3 rows added\n")
|
||||
written = (council / JOURNAL_NAME).read_bytes()
|
||||
second = backfill(council, "--repo", str(self.fixture.a))
|
||||
self.assertEqual((second.returncode, second.stdout), (0, f"{self.fixture.a.resolve()}: 3 blocks, 0 rows added\n"))
|
||||
self.assertEqual((council / JOURNAL_NAME).read_bytes(), written)
|
||||
|
||||
def test_two_runs_from_an_empty_journal_give_byte_identical_journals(self):
|
||||
one, two = self.council("one"), self.council("two")
|
||||
self.assertEqual(backfill(one, "--repo", str(self.fixture.a)).returncode, 0)
|
||||
self.assertEqual(backfill(two, "--repo", str(self.fixture.a)).returncode, 0)
|
||||
self.assertEqual((one / JOURNAL_NAME).read_bytes(), (two / JOURNAL_NAME).read_bytes())
|
||||
|
||||
def test_old_new_and_estimated_blocks_parse_and_a_malformed_one_is_skipped(self):
|
||||
council = self.council("one")
|
||||
backfill(council, "--repo", str(self.fixture.a))
|
||||
rows = self.rows(council)
|
||||
fixture = self.fixture
|
||||
self.assertEqual([row["sha"] for row in rows], [fixture.old, fixture.new, fixture.estimated])
|
||||
self.assertEqual([row["branch"] for row in rows], ["main", "feature/two", "main"])
|
||||
self.assertEqual([row["time"] for row in rows], [fixture.time_of(sha) for sha in (fixture.old, fixture.new, fixture.estimated)])
|
||||
self.assertEqual([row["values"] for row in rows], [
|
||||
{"fix": 95, "break": 2, "tokens": 97, "velocity": None, "security": 100, "score": 96},
|
||||
{"fix": 92, "break": 3, "tokens": 110, "velocity": 90, "security": 100, "score": 81},
|
||||
{"fix": 90, "break": 5, "tokens": None, "velocity": 130, "security": 80, "score": 52},
|
||||
])
|
||||
self.assertEqual([row["digest"] for row in rows], [hashlib.sha256(block.encode("utf-8")).hexdigest() for block in (OLD_BLOCK, NEW_BLOCK, ESTIMATED_BLOCK)])
|
||||
self.assertEqual({(row["remote"], row["path"]) for row in rows}, {(str(fixture.upstream), str(fixture.a.resolve()))})
|
||||
|
||||
def test_an_unknown_label_is_reported_once_per_label_and_skipped(self):
|
||||
done = backfill(self.council("one"), "--repo", str(self.fixture.a))
|
||||
self.assertEqual(done.stderr.splitlines(), UNKNOWN)
|
||||
for row in self.rows(self.council("one")):
|
||||
self.assertEqual(list(row["values"]), ["fix", "break", "tokens", "velocity", "security", "score"])
|
||||
|
||||
def test_old_labels_map_through_the_label_file(self):
|
||||
labels = json.loads(LABELS.read_text(encoding="utf-8"))
|
||||
labels["metrics"]["tokens"]["former"] = []
|
||||
file = self.scratch / "labels.json"
|
||||
file.write_text(json.dumps(labels, ensure_ascii=False), encoding="utf-8")
|
||||
done = backfill(self.council("one"), "--repo", str(self.fixture.a), "--labels", str(file))
|
||||
self.assertIn("backfill: unknown label Cost, skipped", done.stderr.splitlines())
|
||||
self.assertIsNone(self.rows(self.council("one"))[0]["values"]["tokens"])
|
||||
|
||||
def test_a_clone_at_another_path_yields_the_same_keys(self):
|
||||
one, two = self.council("one"), self.council("two")
|
||||
backfill(one, "--repo", str(self.fixture.a))
|
||||
backfill(two, "--repo", str(self.fixture.b))
|
||||
keys = [[(row["remote"], row["sha"]) for row in self.rows(council)] for council in (one, two)]
|
||||
self.assertEqual(keys[0], keys[1])
|
||||
self.assertEqual([row["branch"] for row in self.rows(two)], ["main", "feature/two", "main"])
|
||||
self.assertEqual({row["path"] for row in self.rows(two)}, {str(self.fixture.b.resolve())})
|
||||
again = backfill(one, "--repo", str(self.fixture.b))
|
||||
self.assertEqual(again.stdout, f"{self.fixture.b.resolve()}: 3 blocks, 0 rows added\n")
|
||||
|
||||
def test_since_reads_only_newer_commits_and_a_bad_repository_writes_nothing(self):
|
||||
council = self.council("one")
|
||||
done = backfill(council, "--repo", str(self.fixture.a), "--since", self.fixture.old)
|
||||
self.assertEqual(done.stdout, f"{self.fixture.a.resolve()}: 2 blocks, 2 rows added\n")
|
||||
empty = self.council("two")
|
||||
failed = backfill(empty, "--repo", str(self.fixture.a), str(self.scratch))
|
||||
self.assertEqual(failed.returncode, 2)
|
||||
self.assertFalse((empty / JOURNAL_NAME).exists())
|
||||
|
||||
def test_a_fifo_or_a_linked_council_root_exits_2_with_one_stderr_line(self):
|
||||
fifo = self.council("fifo")
|
||||
fifo.mkdir()
|
||||
os.mkfifo(fifo / JOURNAL_NAME)
|
||||
real = self.scratch / "real"
|
||||
real.mkdir()
|
||||
linked = self.council("linked")
|
||||
linked.symlink_to(real)
|
||||
for council in (fifo, linked):
|
||||
with self.subTest(council=council.name):
|
||||
done = backfill(council, "--repo", str(self.fixture.a))
|
||||
self.assertEqual(done.returncode, 2)
|
||||
self.assertEqual(len(done.stderr.splitlines()), 1, done.stderr)
|
||||
self.assertTrue(done.stderr.startswith(f"backfill: journal {council / JOURNAL_NAME}: "), done.stderr)
|
||||
self.assertEqual(list(real.iterdir()), [])
|
||||
|
||||
def test_a_missing_nofollow_helper_is_named_in_the_one_stderr_line(self):
|
||||
scripts = self.scratch.resolve() / "skills" / "kvb-check-score" / "scripts"
|
||||
scripts.mkdir(parents=True)
|
||||
for name in ("backfill.py", "journal.py"):
|
||||
shutil.copy(BACKFILL.parent / name, scripts / name)
|
||||
shutil.copy(LABELS, scripts.parent / "labels.json")
|
||||
helper = scripts.parents[1] / "kvb-org-organize" / "scripts" / "nofollow.py"
|
||||
council = self.council("missing")
|
||||
env = {"PATH": os.environ["PATH"], "AGENTS_COUNCIL_ROOT": str(council)}
|
||||
done = subprocess.run([sys.executable, "-B", str(scripts / "backfill.py"), "--repo", str(self.fixture.a)], env=env, capture_output=True, text=True, check=False, timeout=HANG)
|
||||
self.assertEqual((done.returncode, done.stdout, done.stderr), (2, "", f"backfill: journal {council / JOURNAL_NAME}: helper {helper} unusable (FileNotFoundError): install or update kvb-org-organize\n"))
|
||||
self.assertFalse(council.exists())
|
||||
|
||||
@unittest.skipIf(os.geteuid() == 0, "root writes into a mode 0500 directory")
|
||||
def test_a_failed_append_prints_no_summary(self):
|
||||
council = self.council("readonly")
|
||||
council.mkdir(mode=0o500)
|
||||
self.addCleanup(council.chmod, 0o700)
|
||||
done = backfill(council, "--repo", str(self.fixture.a))
|
||||
self.assertEqual((done.returncode, done.stdout), (2, ""))
|
||||
self.assertEqual(len(done.stderr.splitlines()), 1, done.stderr)
|
||||
self.assertTrue(done.stderr.startswith(f"backfill: journal {council / JOURNAL_NAME}: "), done.stderr)
|
||||
self.assertEqual(list(council.iterdir()), [])
|
||||
|
||||
def test_a_repeated_repo_flag_reads_every_repository(self):
|
||||
done = backfill(self.council("one"), "--repo", str(self.fixture.a), "--repo", str(self.fixture.b))
|
||||
self.assertEqual(done.stdout.splitlines(), [f"{self.fixture.a.resolve()}: 3 blocks, 3 rows added", f"{self.fixture.b.resolve()}: 3 blocks, 0 rows added"])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,343 @@
|
||||
"""Exercise labels.json and the score journal of kvb-check-score: labels in every layout, one row per block, failed appends, no gate reader."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import importlib.util
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import stat
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import time
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from types import ModuleType
|
||||
from unittest import mock
|
||||
|
||||
from score_fixture import git_env
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parents[1]
|
||||
SKILL = REPO_ROOT / "skills" / "kvb-check-score"
|
||||
SCORE = SKILL / "scripts" / "score.py"
|
||||
EFFICIENCY = REPO_ROOT / "skills" / "kvb-check-efficiency" / "scripts"
|
||||
NOFOLLOW = REPO_ROOT / "skills" / "kvb-org-organize" / "scripts" / "nofollow.py"
|
||||
HANG = 20
|
||||
FORMER = re.compile(r"💸 Cost|⏱️ Time|(?i:💸 cost|⏱️ time|\bcost (?:and|or) time\b|\bcost, time\b|\b(?:cost|time) from the `kvb-check-(?:efficiency|speed)`)")
|
||||
REMOTES = (
|
||||
"git@github.com:org/repo.git",
|
||||
"github.com:org/repo.git",
|
||||
"GitHub.com:org/repo",
|
||||
"https://github.com/org/repo",
|
||||
"https://user@github.com:443/org/repo.git/",
|
||||
"ssh://git@github.com/org/repo.git",
|
||||
)
|
||||
BASE = {"fix": "95", "break": "2", "efficiency": "97", "speed": "-", "security": "100", "critical": "0"}
|
||||
VARIANTS = (
|
||||
{},
|
||||
{"efficiency": "120", "speed": "130"},
|
||||
{"efficiency": "new", "speed": "new", "security": "~90"},
|
||||
{"fix": "80", "break": "10", "efficiency": "150"},
|
||||
{"critical": "1"},
|
||||
)
|
||||
LAYOUTS = ((), ("--table",), ("--message-block",))
|
||||
OLD = re.compile(r"cost|time", re.IGNORECASE)
|
||||
JOURNAL_NAME = "scores.jsonl"
|
||||
READERS = {
|
||||
"skills/kvb-check-score/SKILL.md",
|
||||
"skills/kvb-check-score/scripts/journal.py",
|
||||
"orchestrator/kvb_orchestrator/scores.py",
|
||||
}
|
||||
|
||||
|
||||
def flags(values: dict[str, str]) -> list[str]:
|
||||
return [item for key, value in {**BASE, **values}.items() for item in (f"--{key}", value)]
|
||||
|
||||
|
||||
def module(path: Path) -> ModuleType:
|
||||
spec = importlib.util.spec_from_file_location(f"copied_{path.stem}", path)
|
||||
found = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(found)
|
||||
return found
|
||||
|
||||
|
||||
class ScoreCase(unittest.TestCase):
|
||||
def setUp(self):
|
||||
tmp = tempfile.TemporaryDirectory()
|
||||
self.addCleanup(tmp.cleanup)
|
||||
self.tmp = Path(tmp.name)
|
||||
self.council = self.tmp / "council"
|
||||
self.env = {"PATH": os.environ["PATH"], "AGENTS_COUNCIL_ROOT": str(self.council)}
|
||||
|
||||
def score(self, script: Path, *args: str, cwd: Path) -> subprocess.CompletedProcess:
|
||||
return subprocess.run([sys.executable, "-B", str(script), *args], cwd=cwd, env=self.env, capture_output=True, text=True, check=False)
|
||||
|
||||
def rows(self) -> list[dict]:
|
||||
path = self.council / JOURNAL_NAME
|
||||
return [json.loads(line) for line in path.read_text(encoding="utf-8").splitlines()] if path.exists() else []
|
||||
|
||||
def copy_skills(self, labels: str) -> Path:
|
||||
"""Args:
|
||||
labels: the text of the copied labels.json.
|
||||
|
||||
Returns:
|
||||
The copied score.py, beside kvb-check-efficiency's copied tokens.py and measure.py.
|
||||
"""
|
||||
skill, efficiency = self.tmp / "skills" / "kvb-check-score", self.tmp / "skills" / "kvb-check-efficiency" / "scripts"
|
||||
(skill / "scripts").mkdir(parents=True)
|
||||
efficiency.mkdir(parents=True)
|
||||
for name in ("score.py", "journal.py"):
|
||||
shutil.copy(SKILL / "scripts" / name, skill / "scripts" / name)
|
||||
for name in ("tokens.py", "measure.py"):
|
||||
shutil.copy(EFFICIENCY / name, efficiency / name)
|
||||
organize = self.tmp / "skills" / "kvb-org-organize" / "scripts"
|
||||
organize.mkdir(parents=True)
|
||||
shutil.copy(NOFOLLOW, organize / "nofollow.py")
|
||||
(skill / "labels.json").write_text(labels, encoding="utf-8")
|
||||
return skill / "scripts" / "score.py"
|
||||
|
||||
|
||||
class TestScoreLabels(ScoreCase):
|
||||
def test_line_table_and_block_print_tokens_and_velocity_never_cost_or_time(self):
|
||||
expected = {(): ("💸 tokens", "⏱️ velocity"), ("--table",): ("| 💸 Tokens |", "| ⏱️ Velocity |"), ("--message-block",): ("💸 Tokens: ", "⏱️ Velocity: ")}
|
||||
for values in VARIANTS:
|
||||
for layout in LAYOUTS:
|
||||
with self.subTest(values=values, layout=layout):
|
||||
out = self.score(SCORE, *flags(values), *layout, cwd=self.tmp).stdout
|
||||
for label in expected[layout]:
|
||||
self.assertIn(label, out)
|
||||
self.assertIsNone(OLD.search(out), out)
|
||||
|
||||
def test_no_skill_text_or_doc_names_the_former_labels(self):
|
||||
found = [
|
||||
f"{path.relative_to(REPO_ROOT)}:{number}"
|
||||
for top in ("skills", "docs")
|
||||
for path in sorted((REPO_ROOT / top).rglob("*"))
|
||||
if path.is_file() and path.suffix != ".pyc"
|
||||
for number, line in enumerate(path.read_text(encoding="utf-8", errors="replace").splitlines(), 1)
|
||||
if FORMER.search(line)
|
||||
]
|
||||
self.assertEqual(found, [])
|
||||
labels = json.loads((SKILL / "labels.json").read_text(encoding="utf-8"))["metrics"]
|
||||
self.assertEqual({key: metric["former"] for key, metric in labels.items() if metric["former"]}, {"tokens": ["Cost", "Efficiency"], "velocity": ["Time", "Speed"]})
|
||||
self.assertEqual((labels["tokens"]["emoji"], labels["velocity"]["emoji"]), ("💸", "⏱️"))
|
||||
|
||||
def test_a_label_changed_in_a_temporary_label_file_shows_in_every_layout(self):
|
||||
labels = json.loads((SKILL / "labels.json").read_text(encoding="utf-8"))
|
||||
labels["metrics"]["tokens"]["label"] = "Pennies"
|
||||
labels["metrics"]["velocity"].update(emoji="🐢", label="Pace")
|
||||
script = self.copy_skills(json.dumps(labels, ensure_ascii=False))
|
||||
line = self.score(script, *flags({}), cwd=self.tmp).stdout.splitlines()
|
||||
self.assertEqual(line[0], "🎯 fix 95 % · 💥 break 2 % · 💸 pennies 97 % · 🐢 pace – · 🛡️ security 100 % · 🏆 score 96")
|
||||
self.assertEqual(line[1], "✅ gate: 90/5, pennies and pace <= 100, security 100 %")
|
||||
table = self.score(script, *flags({}), "--table", "--value", "1:2", cwd=self.tmp).stdout.splitlines()
|
||||
self.assertEqual(table[0], "| 🎯 Fix | 💥 Break | 💸 Pennies | 💰 Value | 🐢 Pace | 🛡️ Security | 🏆 Score | Gate |")
|
||||
block = self.score(script, *flags({}), "--message-block", cwd=self.tmp).stdout.splitlines()
|
||||
self.assertEqual(block[2:4], ["💸 Pennies: 97 %", "🐢 Pace: –"])
|
||||
efficiency = self.tmp / "skills" / "kvb-check-efficiency" / "scripts"
|
||||
tokens = module(efficiency / "tokens.py")
|
||||
name = tokens.labelled(("tokens",))["tokens"]
|
||||
self.assertEqual(tokens.headline(name, 97, 100, 97, "listing 3→3 words"), "💸 pennies 97 % (skills 100→97 tokens; listing 3→3 words)")
|
||||
self.assertEqual(tokens.headline(name, None, 0, 50, "listing 3→5 words"), "💸 pennies new (+50 tokens; listing 3→5 words)")
|
||||
side = {"cpu": 1.0, "ram": 10.0, "wall": 2.0}
|
||||
found = {"estimated": False, "base": side, "head": side, "efficiency": 97, "speed": 90, "runs": 1}
|
||||
measure = module(efficiency / "measure.py")
|
||||
self.assertEqual(measure.line(found, measure.labelled(("tokens", "velocity"))), "💸 pennies 97 % (cpu 1.00→1.00 s, ram 10.0→10.0 MB) · 🐢 pace 90 % (wall 2.00→2.00 s)")
|
||||
|
||||
def test_a_malformed_label_file_exits_2_or_4_with_one_stderr_line_in_every_layout(self):
|
||||
labels = json.loads((SKILL / "labels.json").read_text(encoding="utf-8"))
|
||||
broken = [
|
||||
"{broken",
|
||||
json.dumps({"metrics": [], "extras": {}}),
|
||||
json.dumps({**labels, "metrics": {key: value for key, value in labels["metrics"].items() if key != "velocity"}}),
|
||||
json.dumps({**labels, "metrics": {**labels["metrics"], "fix": {"emoji": 1, "label": "Fix", "former": []}}}),
|
||||
]
|
||||
script = self.copy_skills("{}")
|
||||
cases = [(index, text, layout, critical, code) for index, text in enumerate(broken) for layout in LAYOUTS for critical, code in (("0", 2), ("1", 4))]
|
||||
for index, text, layout, critical, code in cases:
|
||||
(script.parents[1] / "labels.json").write_text(text, encoding="utf-8")
|
||||
with self.subTest(labels=index, layout=layout, critical=critical):
|
||||
done = self.score(script, *flags({"critical": critical}), *layout, cwd=self.tmp)
|
||||
self.assertEqual((done.returncode, done.stdout), (code, ""))
|
||||
self.assertEqual(len(done.stderr.splitlines()), 1, done.stderr)
|
||||
self.assertTrue(done.stderr.startswith("score: labels "), done.stderr)
|
||||
|
||||
|
||||
class TestJournalPlace(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.journal = module(SKILL / "scripts" / "journal.py")
|
||||
|
||||
def test_an_empty_council_root_means_the_default(self):
|
||||
for value in ("", None):
|
||||
with self.subTest(value=value), mock.patch.dict(os.environ, {} if value is None else {"AGENTS_COUNCIL_ROOT": value}):
|
||||
if value is None:
|
||||
os.environ.pop("AGENTS_COUNCIL_ROOT", None)
|
||||
self.assertEqual(self.journal.journal_path(), Path("/tmp/agents-council/scores.jsonl"))
|
||||
|
||||
def test_ssh_scp_and_https_remotes_share_one_key(self):
|
||||
self.assertEqual({self.journal.identity(url) for url in REMOTES}, {"github.com/org/repo"})
|
||||
for url in ("/srv/git/repo.git", "./repo", "file:///srv/git/repo.git", None):
|
||||
with self.subTest(url=url):
|
||||
self.assertEqual(self.journal.identity(url), url)
|
||||
|
||||
|
||||
class TestScoreJournal(ScoreCase):
|
||||
def repo(self) -> Path:
|
||||
repo = self.tmp / "repo"
|
||||
repo.mkdir()
|
||||
env = git_env(self.tmp)
|
||||
for args in (("init", "-q", "-b", "feature/journal"), ("remote", "add", "origin", "git@github.com:example/fixture.git"), ("commit", "-q", "--allow-empty", "-m", "base")):
|
||||
subprocess.run(["git", *args], cwd=repo, env=env, capture_output=True, check=True)
|
||||
return repo
|
||||
|
||||
def test_each_message_block_appends_exactly_one_row_with_the_journal_fields(self):
|
||||
repo = self.repo()
|
||||
before = int(time.time())
|
||||
first = self.score(SCORE, *flags({}), "--message-block", cwd=repo)
|
||||
second = self.score(SCORE, *flags({"security": "~80"}), "--message-block", "--value", "1:20", cwd=repo)
|
||||
self.assertEqual((first.returncode, second.returncode, first.stderr, second.stderr), (0, 3, "", ""))
|
||||
rows = self.rows()
|
||||
self.assertEqual(len(rows), 2)
|
||||
for row, done in zip(rows, (first, second)):
|
||||
self.assertEqual(list(row), ["time", "remote", "path", "branch", "sha", "values", "digest"])
|
||||
self.assertEqual((row["remote"], row["path"], row["branch"], row["sha"]), ("github.com/example/fixture", str(repo.resolve()), "feature/journal", None))
|
||||
self.assertEqual(row["digest"], hashlib.sha256(done.stdout.rstrip("\n").encode("utf-8")).hexdigest())
|
||||
self.assertLessEqual(before, row["time"])
|
||||
self.assertLessEqual(row["time"], int(time.time()))
|
||||
self.assertEqual(rows[0]["values"], {"fix": 95, "break": 2, "tokens": 97, "velocity": None, "security": 100, "score": 96})
|
||||
self.assertEqual(rows[1]["values"], {"fix": 95, "break": 2, "tokens": 97, "velocity": None, "security": 80, "score": 77})
|
||||
self.score(SCORE, *flags({}), cwd=repo)
|
||||
self.score(SCORE, *flags({}), "--table", cwd=repo)
|
||||
self.assertEqual(len(self.rows()), 2)
|
||||
|
||||
def test_a_failed_append_prints_one_stderr_line_and_keeps_the_exit_code(self):
|
||||
self.council.write_text("not a directory", encoding="utf-8")
|
||||
for values, code in (({}, 0), ({"efficiency": "120"}, 3), ({"break": "9"}, 1), ({"critical": "1"}, 4)):
|
||||
with self.subTest(values=values):
|
||||
done = self.score(SCORE, *flags(values), "--message-block", cwd=self.tmp)
|
||||
self.assertEqual(done.returncode, code)
|
||||
self.assertEqual(len(done.stderr.splitlines()), 1, done.stderr)
|
||||
self.assertTrue(done.stderr.startswith(f"score: journal {self.council / JOURNAL_NAME}: "), done.stderr)
|
||||
self.assertEqual(len(done.stdout.splitlines()), 7)
|
||||
|
||||
def test_a_deleted_working_directory_keeps_the_exit_code(self):
|
||||
gone = self.tmp / "gone"
|
||||
wrapper = "import os, subprocess, sys; os.chdir(sys.argv[1]); os.rmdir(sys.argv[1]); sys.exit(subprocess.run(sys.argv[2:]).returncode)"
|
||||
for values, code in (({}, 0), ({"critical": "1"}, 4)):
|
||||
with self.subTest(values=values):
|
||||
gone.mkdir(exist_ok=True)
|
||||
done = subprocess.run([sys.executable, "-B", "-c", wrapper, str(gone), sys.executable, "-B", str(SCORE), *flags(values), "--message-block"], env=self.env, capture_output=True, text=True, check=False)
|
||||
self.assertEqual(done.returncode, code, done.stderr)
|
||||
self.assertEqual(len(done.stderr.splitlines()), 1, done.stderr)
|
||||
self.assertTrue(done.stderr.startswith(f"score: journal {self.council / JOURNAL_NAME}: "), done.stderr)
|
||||
self.assertEqual(len(done.stdout.splitlines()), 7)
|
||||
|
||||
def test_git_output_outside_utf8_keeps_the_exit_code_and_the_row(self):
|
||||
repo = self.repo()
|
||||
subprocess.run([b"git", b"remote", b"set-url", b"origin", b"https://example.org/caf\xe9.git"], cwd=repo, env=git_env(self.tmp), capture_output=True, check=True)
|
||||
done = self.score(SCORE, *flags({"critical": "1"}), "--message-block", cwd=repo)
|
||||
self.assertEqual((done.returncode, done.stderr), (4, ""))
|
||||
self.assertEqual(self.rows()[0]["remote"], "example.org/caf�")
|
||||
|
||||
def test_a_link_at_the_journal_is_not_followed(self):
|
||||
self.council.mkdir()
|
||||
target = self.tmp / "elsewhere"
|
||||
target.write_text("kept\n", encoding="utf-8")
|
||||
(self.council / JOURNAL_NAME).symlink_to(target)
|
||||
done = self.score(SCORE, *flags({"critical": "1"}), "--message-block", cwd=self.tmp)
|
||||
self.assertEqual(done.returncode, 4)
|
||||
self.assertEqual(len(done.stderr.splitlines()), 1, done.stderr)
|
||||
self.assertEqual(target.read_text(encoding="utf-8"), "kept\n")
|
||||
|
||||
def test_a_fifo_at_the_journal_does_not_block_the_exit_code(self):
|
||||
self.council.mkdir()
|
||||
os.mkfifo(self.council / JOURNAL_NAME)
|
||||
done = subprocess.run([sys.executable, "-B", str(SCORE), *flags({"critical": "1"}), "--message-block"], cwd=self.tmp, env=self.env, capture_output=True, text=True, check=False, timeout=HANG)
|
||||
self.assertEqual(done.returncode, 4)
|
||||
self.assertEqual(len(done.stderr.splitlines()), 1, done.stderr)
|
||||
self.assertTrue(done.stderr.startswith(f"score: journal {self.council / JOURNAL_NAME}: "), done.stderr)
|
||||
|
||||
def test_a_linked_council_root_is_refused(self):
|
||||
real = self.tmp / "worktree"
|
||||
real.mkdir()
|
||||
(real / ".git").write_text("gitdir: /elsewhere\n", encoding="utf-8")
|
||||
self.council.symlink_to(real)
|
||||
done = self.score(SCORE, *flags({"critical": "1"}), "--message-block", cwd=self.tmp)
|
||||
self.assertEqual(done.returncode, 4)
|
||||
self.assertEqual(len(done.stderr.splitlines()), 1, done.stderr)
|
||||
self.assertEqual(sorted(path.name for path in real.iterdir()), [".git"])
|
||||
|
||||
def test_a_missing_or_broken_nofollow_helper_is_named_in_the_one_stderr_line(self):
|
||||
script = self.copy_skills((SKILL / "labels.json").read_text(encoding="utf-8"))
|
||||
helper = self.tmp.resolve() / "skills" / "kvb-org-organize" / "scripts" / "nofollow.py"
|
||||
cases = {"FileNotFoundError": None, "AttributeError": "VERSION = 0\n", "SyntaxError": "def (:\n"}
|
||||
if os.geteuid() != 0:
|
||||
cases["PermissionError"] = NOFOLLOW.read_text(encoding="utf-8")
|
||||
for kind, text in cases.items():
|
||||
helper.unlink(missing_ok=True)
|
||||
if text is not None:
|
||||
helper.write_text(text, encoding="utf-8")
|
||||
if kind == "PermissionError":
|
||||
helper.chmod(0)
|
||||
for values, code in (({}, 0), ({"critical": "1"}, 4)):
|
||||
with self.subTest(kind=kind, values=values):
|
||||
done = self.score(script, *flags(values), "--message-block", cwd=self.tmp)
|
||||
self.assertEqual(done.returncode, code)
|
||||
self.assertEqual(done.stderr, f"score: journal {self.council / JOURNAL_NAME}: helper {helper} unusable ({kind}): install or update kvb-org-organize\n")
|
||||
self.assertEqual(len(done.stdout.splitlines()), 7)
|
||||
self.assertFalse(self.council.exists())
|
||||
|
||||
def test_a_branch_or_commit_outside_utf8_keeps_the_score_exit(self):
|
||||
repo = self.tmp / "latin"
|
||||
env = git_env(self.tmp)
|
||||
subprocess.run(["git", "init", "-q", "-b", "caf\udce9", str(repo)], env=env, capture_output=True, check=True)
|
||||
subprocess.run(["git", "commit", "-q", "--allow-empty", "-m", "base"], cwd=repo, env=env, capture_output=True, check=True)
|
||||
for values, code in (({}, 0), ({"critical": "1"}, 4)):
|
||||
with self.subTest(values=values):
|
||||
done = self.score(SCORE, *flags(values), "--commit-table", str(repo), cwd=self.tmp)
|
||||
self.assertEqual((done.returncode, done.stderr), (code, ""))
|
||||
self.assertIn("| caf� |", done.stdout)
|
||||
failed = self.score(SCORE, *flags(values), "--commit-table", str(repo), "--commit", "\udce9", cwd=self.tmp)
|
||||
self.assertEqual((failed.returncode, failed.stdout), (code or 2, ""))
|
||||
self.assertEqual(len(failed.stderr.splitlines()), 1, failed.stderr)
|
||||
self.assertTrue(failed.stderr.startswith("score: git show in "), failed.stderr)
|
||||
|
||||
def test_a_critical_finding_wins_over_a_commit_table_git_cannot_read(self):
|
||||
repo = self.repo()
|
||||
plain = self.tmp / "plain"
|
||||
plain.mkdir()
|
||||
self.env["GIT_CEILING_DIRECTORIES"] = str(self.tmp)
|
||||
tables = {"non-git": (str(plain),), "deleted": (str(self.tmp / "deleted"),), "unknown commit": (str(repo), "--commit", "0" * 40)}
|
||||
for name, (target, *commit) in tables.items():
|
||||
for values, code in (({}, 2), ({"fix": "80"}, 2), ({"critical": "1"}, 4), ({"fix": "80", "critical": "1"}, 4)):
|
||||
with self.subTest(table=name, values=values):
|
||||
done = self.score(SCORE, *flags(values), "--commit-table", target, *commit, cwd=self.tmp)
|
||||
self.assertEqual((done.returncode, done.stdout), (code, ""))
|
||||
self.assertEqual(len(done.stderr.splitlines()), 1, done.stderr)
|
||||
self.assertTrue(done.stderr.startswith("score: git "), done.stderr)
|
||||
|
||||
def test_a_new_journal_and_council_root_are_private(self):
|
||||
self.score(SCORE, *flags({}), "--message-block", cwd=self.tmp)
|
||||
self.assertEqual(stat.S_IMODE((self.council / JOURNAL_NAME).stat().st_mode), 0o600)
|
||||
self.assertEqual(stat.S_IMODE(self.council.stat().st_mode), 0o700)
|
||||
|
||||
def test_no_gate_reads_the_journal(self):
|
||||
found = set()
|
||||
for top in ("skills", "orchestrator", "scripts"):
|
||||
for path in (REPO_ROOT / top).rglob("*"):
|
||||
text = path.suffix in {".py", ".sh", ".md", ".js", ".j2", ".json", ".yml"} and path.is_file()
|
||||
if text and JOURNAL_NAME in path.read_text(encoding="utf-8", errors="replace"):
|
||||
found.add(str(path.relative_to(REPO_ROOT)))
|
||||
self.assertEqual(found, READERS)
|
||||
importers = {str(path.relative_to(REPO_ROOT)) for path in (REPO_ROOT / "skills").rglob("*.py") if re.search(r"^from journal import|^import journal", path.read_text(encoding="utf-8"), re.MULTILINE)}
|
||||
self.assertEqual(importers, {"skills/kvb-check-score/scripts/score.py", "skills/kvb-check-score/scripts/backfill.py"})
|
||||
self.council.mkdir()
|
||||
(self.council / JOURNAL_NAME).write_text('{"time": 1, "values": {"score": 0}}\n{broken\n', encoding="utf-8")
|
||||
self.assertEqual(self.score(SCORE, *flags({}), cwd=self.tmp).returncode, 0)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in new issue
Block a user