Merge branch 'feature/score-metric-overlays' into unify/skills-20261112-night
This commit is contained in:
commit
7d9f25e289
30 files changed
+1806
-142
No files matched your search
@@ -11,6 +11,7 @@ from .diffs import DIFF_BODY_LIMIT
|
|||||||
from .engines import STT_LIMIT
|
from .engines import STT_LIMIT
|
||||||
from .exchange import PLAIN_CSP, Exchange
|
from .exchange import PLAIN_CSP, Exchange
|
||||||
from .resources import RESOURCES, sampled
|
from .resources import RESOURCES, sampled
|
||||||
|
from .scores import scored
|
||||||
|
|
||||||
BODY_LIMIT = 1 << 20
|
BODY_LIMIT = 1 << 20
|
||||||
UPLOAD_LIMIT = 50 << 20
|
UPLOAD_LIMIT = 50 << 20
|
||||||
@@ -119,6 +120,8 @@ class Handler(Exchange):
|
|||||||
self.answer(hub.repositories, HTTPStatus.OK)
|
self.answer(hub.repositories, HTTPStatus.OK)
|
||||||
elif path == "/api/resources":
|
elif path == "/api/resources":
|
||||||
self.answer(lambda: sampled(hub.kept / RESOURCES, hub.span(self.query(), time.time())), HTTPStatus.OK)
|
self.answer(lambda: sampled(hub.kept / RESOURCES, hub.span(self.query(), time.time())), HTTPStatus.OK)
|
||||||
|
elif path == "/api/scores":
|
||||||
|
self.answer(lambda: scored(hub.council, hub.scripts.skills, hub.span(self.query(), time.time())), HTTPStatus.OK)
|
||||||
elif path == "/api/subagents":
|
elif path == "/api/subagents":
|
||||||
self.answer(lambda: hub.subagents(self.query()), HTTPStatus.OK)
|
self.answer(lambda: hub.subagents(self.query()), HTTPStatus.OK)
|
||||||
elif path == "/api/runner/jobs":
|
elif path == "/api/runner/jobs":
|
||||||
|
|||||||
@@ -0,0 +1,44 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
from .records import epoch, parse_line, read_json, read_regular
|
||||||
|
|
||||||
|
SCORES = "scores.jsonl"
|
||||||
|
LABELS = "kvb-check-score/labels.json"
|
||||||
|
FILE_LIMIT = 64 << 20
|
||||||
|
|
||||||
|
|
||||||
|
def checked(labels: object) -> dict | None:
|
||||||
|
"""Args:
|
||||||
|
labels: the parsed label file, or None.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
labels when its "metrics" is a non-empty object of entries with a string
|
||||||
|
emoji and label, else None.
|
||||||
|
"""
|
||||||
|
metrics = labels.get("metrics") if isinstance(labels, dict) else None
|
||||||
|
valid = isinstance(metrics, dict) and metrics and all(isinstance(metric, dict) and isinstance(metric.get("emoji"), str) and isinstance(metric.get("label"), str) for metric in metrics.values())
|
||||||
|
return labels if valid else None
|
||||||
|
|
||||||
|
|
||||||
|
def scored(council: Path, skills: Path, span: tuple[float, float]) -> dict:
|
||||||
|
"""Args:
|
||||||
|
council: the council directory holding scores.jsonl, which score.py
|
||||||
|
and backfill.py of kvb-check-score append to.
|
||||||
|
skills: the skills directory holding kvb-check-score/labels.json.
|
||||||
|
span: (begin, end) of the burndown's time range in epoch seconds.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
{"since", "until", "labels", "rows"}: the label file, None when
|
||||||
|
unreadable or not as checked() wants it, and the journal rows whose time lies within the span, in
|
||||||
|
time order.
|
||||||
|
"""
|
||||||
|
begin, end = span
|
||||||
|
try:
|
||||||
|
lines = read_regular(council / SCORES, FILE_LIMIT).splitlines()
|
||||||
|
except (OSError, ValueError):
|
||||||
|
lines = []
|
||||||
|
rows = [row for row in map(parse_line, lines) if row is not None and isinstance(row.get("values"), dict)]
|
||||||
|
inside = [row for row in rows if (moment := epoch(row.get("time"))) is not None and begin <= moment <= end]
|
||||||
|
return {"since": begin, "until": end, "labels": checked(read_json(skills / LABELS)), "rows": sorted(inside, key=lambda row: row["time"])}
|
||||||
@@ -1,5 +1,7 @@
|
|||||||
const GANTT = {label: 240, right: 28, top: 34, lane: 30, width: 1180, bar: 16, ticks: 4};
|
const GANTT = {label: 240, right: 28, top: 34, lane: 30, width: 1180, bar: 16, ticks: 4};
|
||||||
const BURN = {width: 900, height: 340, left: 52, right: 24, top: 20, bottom: 40};
|
const BURN = {width: 900, height: 340, left: 52, right: 24, axis: 56, top: 20, bottom: 40};
|
||||||
|
const OVERLAY_PATH = "burn.overlays";
|
||||||
|
const scoreView = {key: "", rows: [], labels: null, busy: false, bound: false};
|
||||||
const EDGE_WORDS = {after: "Follows (rasci.py set --after)", base: "Branches from (base branch)", queue: "Waits behind (queue order)"};
|
const EDGE_WORDS = {after: "Follows (rasci.py set --after)", base: "Branches from (base branch)", queue: "Waits behind (queue order)"};
|
||||||
const SERIES_WORDS = {scope: "Scope (burnup)", done: "Done (burnup)", remaining: "Remaining (burndown)", unestimated: "Unestimated tasks"};
|
const SERIES_WORDS = {scope: "Scope (burnup)", done: "Done (burnup)", remaining: "Remaining (burndown)", unestimated: "Unestimated tasks"};
|
||||||
const STEP_WORDS = [["started", "Started"], ["ended", "Ended"], ["joined", "Joined the scope"]];
|
const STEP_WORDS = [["started", "Started"], ["ended", "Ended"], ["joined", "Joined the scope"]];
|
||||||
@@ -80,11 +82,11 @@
|
|||||||
return h("div", {class: "gantt-box"}, h("div", {class: "gantt-scroll"}, svg), legend);
|
return h("div", {class: "gantt-box"}, h("div", {class: "gantt-scroll"}, svg), legend);
|
||||||
}
|
}
|
||||||
|
|
||||||
function burnAxis(svg, from, to, top) {
|
function burnAxis(svg, from, to, top, right) {
|
||||||
const x = (t) => BURN.left + ((t - from) / Math.max(to - from, 1)) * (BURN.width - BURN.left - BURN.right);
|
const x = (t) => BURN.left + ((t - from) / Math.max(to - from, 1)) * (BURN.width - BURN.left - right);
|
||||||
const y = (value) => BURN.top + (1 - value / top) * (BURN.height - BURN.top - BURN.bottom);
|
const y = (value) => BURN.top + (1 - value / top) * (BURN.height - BURN.top - BURN.bottom);
|
||||||
for (const value of [0, top / 2, top]) {
|
for (const value of [0, top / 2, top]) {
|
||||||
svg.append(s("line", {class: "grid", x1: BURN.left, x2: BURN.width - BURN.right, y1: fixed(y(value)), y2: fixed(y(value))}));
|
svg.append(s("line", {class: "grid", x1: BURN.left, x2: BURN.width - right, y1: fixed(y(value)), y2: fixed(y(value))}));
|
||||||
svg.append(s("text", {class: "axis", x: BURN.left - 8, y: fixed(y(value)), dy: "0.35em", "text-anchor": "end"}, String(Math.round(value * 10) / 10)));
|
svg.append(s("text", {class: "axis", x: BURN.left - 8, y: fixed(y(value)), dy: "0.35em", "text-anchor": "end"}, String(Math.round(value * 10) / 10)));
|
||||||
}
|
}
|
||||||
for (const [index, t] of [from, (from + to) / 2, to].entries()) {
|
for (const [index, t] of [from, (from + to) / 2, to].entries()) {
|
||||||
@@ -165,6 +167,68 @@
|
|||||||
return {reach, end: forecast === null ? picked.to : Math.min(forecast, reach)};
|
return {reach, end: forecast === null ? picked.to : Math.min(forecast, reach)};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const overlaysReady = () => typeof ViewStore === "object" && typeof LegendChips === "object";
|
||||||
|
|
||||||
|
async function fetchScores(key, from, to) {
|
||||||
|
scoreView.busy = true;
|
||||||
|
scoreView.key = key;
|
||||||
|
try {
|
||||||
|
const response = await fetch(`/api/scores?from=${Math.floor(from)}&to=${Math.ceil(to)}`, {cache: "no-store"});
|
||||||
|
if (response.ok) {
|
||||||
|
const found = await response.json();
|
||||||
|
scoreView.rows = Array.isArray(found.rows) ? found.rows.filter((row) => row && row.values && typeof row.values === "object") : [];
|
||||||
|
scoreView.labels = Plan.overlayLabels(found.labels);
|
||||||
|
}
|
||||||
|
} catch {
|
||||||
|
ui.offline = true;
|
||||||
|
}
|
||||||
|
scoreView.busy = false;
|
||||||
|
taskView.sig = "";
|
||||||
|
drawTasks();
|
||||||
|
}
|
||||||
|
|
||||||
|
function overlayKeys() {
|
||||||
|
const kept = ViewStore.get(OVERLAY_PATH);
|
||||||
|
return Array.isArray(kept) && scoreView.labels ? Object.keys(scoreView.labels).filter((key) => kept.includes(key)) : [];
|
||||||
|
}
|
||||||
|
|
||||||
|
const burnRight = () => (overlaysReady() && overlayKeys().length ? BURN.axis : BURN.right);
|
||||||
|
|
||||||
|
function burnOverlays(svg, picked, x, right) {
|
||||||
|
const key = `${Math.floor(picked.from)}:${Math.floor(picked.to / 60)}`;
|
||||||
|
if (key !== scoreView.key && !scoreView.busy) {
|
||||||
|
fetchScores(key, picked.from, picked.to);
|
||||||
|
}
|
||||||
|
if (!scoreView.bound) {
|
||||||
|
scoreView.bound = true;
|
||||||
|
ViewStore.on(OVERLAY_PATH, () => {
|
||||||
|
taskView.sig = "";
|
||||||
|
drawTasks();
|
||||||
|
});
|
||||||
|
}
|
||||||
|
if (!scoreView.labels) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
const series = Plan.overlaySeries(picked.rows, scoreView.rows, overlayKeys(), picked);
|
||||||
|
const top = Plan.overlayTop(series);
|
||||||
|
const y = (value) => BURN.top + (1 - value / top) * (BURN.height - BURN.top - BURN.bottom);
|
||||||
|
if (series.length) {
|
||||||
|
for (const value of [0, top / 2, top]) {
|
||||||
|
svg.append(s("text", {class: "axis overlay-axis", x: BURN.width - right + 6, y: fixed(y(value)), dy: "0.35em", "text-anchor": "start"}, `${Math.round(value)} %`));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for (const {key: name, points} of series) {
|
||||||
|
const d = points.map((point, index) => `${index ? "L" : "M"}${fixed(x(point.t))},${fixed(y(point.value))}`).join(" ");
|
||||||
|
const group = s("g", {class: `burn-overlay k-${name}`, "data-overlay": name, "data-points": String(points.length)}, d ? s("path", {class: "burn-overlay-line", d}) : null);
|
||||||
|
for (const point of points) {
|
||||||
|
group.append(s("circle", {class: "burn-overlay-point", cx: fixed(x(point.t)), cy: fixed(y(point.value)), r: 3, "data-t": point.t, "data-value": point.value},
|
||||||
|
s("title", {}, `${absolute.format(stampOf(point.t))} · ${scoreView.labels[name]}: ${point.value} % · ${point.branch}`)));
|
||||||
|
}
|
||||||
|
svg.append(group);
|
||||||
|
}
|
||||||
|
return LegendChips.render(Object.entries(scoreView.labels).map(([key, label]) => ({id: key, label, tone: `k-${key}`})), OVERLAY_PATH, "shown");
|
||||||
|
}
|
||||||
|
|
||||||
function burnChart(rows, shown) {
|
function burnChart(rows, shown) {
|
||||||
const picked = Plan.scoped(rows, taskView.scope, {filtered: shown, since: state.since, until: state.until, now: state.now, holders: lineHolders()});
|
const picked = Plan.scoped(rows, taskView.scope, {filtered: shown, since: state.since, until: state.until, now: state.now, holders: lineHolders()});
|
||||||
const series = Plan.burn(picked.rows, {estimation: estimation(), from: picked.from, to: picked.to}).points;
|
const series = Plan.burn(picked.rows, {estimation: estimation(), from: picked.from, to: picked.to}).points;
|
||||||
@@ -177,7 +241,8 @@
|
|||||||
const top = Math.max(1, ...series.flatMap((point) => names.map((name) => point[name])), ...ends);
|
const top = Math.max(1, ...series.flatMap((point) => names.map((name) => point[name])), ...ends);
|
||||||
const unit = estimation() === "points" ? "story points; unestimated tasks as a line of their own" : "tasks";
|
const unit = estimation() === "points" ? "story points; unestimated tasks as a line of their own" : "tasks";
|
||||||
const svg = s("svg", {class: "chart burn", viewBox: `0 0 ${BURN.width} ${BURN.height}`, role: "img", "aria-label": `Burndown and burnup in ${unit}; hover a point for the work packages of its step`});
|
const svg = s("svg", {class: "chart burn", viewBox: `0 0 ${BURN.width} ${BURN.height}`, role: "img", "aria-label": `Burndown and burnup in ${unit}; hover a point for the work packages of its step`});
|
||||||
const {x, y} = burnAxis(svg, picked.from, reach, top);
|
const right = burnRight();
|
||||||
|
const {x, y} = burnAxis(svg, picked.from, reach, top, right);
|
||||||
const last = series[series.length - 1];
|
const last = series[series.length - 1];
|
||||||
if (reach > picked.to) {
|
if (reach > picked.to) {
|
||||||
svg.append(s("line", {class: "now", x1: fixed(x(picked.to)), x2: fixed(x(picked.to)), y1: BURN.top, y2: BURN.height - BURN.bottom}));
|
svg.append(s("line", {class: "now", x1: fixed(x(picked.to)), x2: fixed(x(picked.to)), y1: BURN.top, y2: BURN.height - BURN.bottom}));
|
||||||
@@ -191,10 +256,11 @@
|
|||||||
burnFinish(svg, segments, x, y);
|
burnFinish(svg, segments, x, y);
|
||||||
}
|
}
|
||||||
svg.append(burnPoints(series, Plan.steps(picked.rows, series, picked.from), names, x, y));
|
svg.append(burnPoints(series, Plan.steps(picked.rows, series, picked.from), names, x, y));
|
||||||
|
const overlays = overlaysReady() ? burnOverlays(svg, picked, x, right) : null;
|
||||||
const stale = h("span", {class: "pill burn-stale", "data-tone": "warn", "data-stale": String(picked.stale.length), role: "img", ...named(`Stale: ${picked.stale.length}`, `Stale: ${picked.stale.length} tasks without a register change for ${Plan.STALE_DAYS} days, counted apart`)}, icon("bi-hourglass-bottom"), String(picked.stale.length));
|
const stale = h("span", {class: "pill burn-stale", "data-tone": "warn", "data-stale": String(picked.stale.length), role: "img", ...named(`Stale: ${picked.stale.length}`, `Stale: ${picked.stale.length} tasks without a register change for ${Plan.STALE_DAYS} days, counted apart`)}, icon("bi-hourglass-bottom"), String(picked.stale.length));
|
||||||
const legend = h("div", {class: "legend"}, names.map((name) => h("span", {class: `burn-key k-${name}`, "data-series": name}, `${SERIES_WORDS[name]}: ${last[name]}`)),
|
const legend = h("div", {class: "legend"}, names.map((name) => h("span", {class: `burn-key k-${name}`, "data-series": name}, `${SERIES_WORDS[name]}: ${last[name]}`)),
|
||||||
settings.trend.mode === "off" ? null : finishNotes(trend, finish), stale);
|
settings.trend.mode === "off" ? null : finishNotes(trend, finish), stale);
|
||||||
return h("div", {class: "burn-box"}, svg, legend, h("p", {class: "muted"}, `${picked.rows.length} tasks in scope, counted in ${unit}`));
|
return h("div", {class: "burn-box", "data-panel": "burndown"}, svg, legend, overlays, h("p", {class: "muted"}, `${picked.rows.length} tasks in scope, counted in ${unit}`));
|
||||||
}
|
}
|
||||||
|
|
||||||
async function chooseEstimation(id) {
|
async function chooseEstimation(id) {
|
||||||
|
|||||||
@@ -262,6 +262,44 @@ const Plan = (() => {
|
|||||||
return {fixed: reachable(fixed), growing: found.forecast};
|
return {fixed: reachable(fixed), growing: found.forecast};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const branchOf = (row) => (row.branch !== "-" ? row.branch : row.worktree_branch);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @param rows - the burndown scope's task rows with branch ("-" for none) and worktree_branch ("" for none).
|
||||||
|
* @param journal - /api/scores rows {time, branch, values: {metric key: number or null}}.
|
||||||
|
* @param keys - the metric keys switched on, in the label file's order.
|
||||||
|
* @param span - {from, to} in epoch seconds.
|
||||||
|
* @returns one {key, points: [{t, value, branch}]} per key, in keys order: the journal rows of a branch of the
|
||||||
|
* scope's rows within span whose value for the key is a number, in time order.
|
||||||
|
*/
|
||||||
|
function overlaySeries(rows, journal, keys, span) {
|
||||||
|
const branches = new Set(rows.map(branchOf).filter(Boolean));
|
||||||
|
const inside = journal.filter((entry) => branches.has(entry.branch) && entry.time >= span.from && entry.time <= span.to).sort((a, b) => a.time - b.time);
|
||||||
|
return keys.map((key) => ({
|
||||||
|
key, points: inside.filter((entry) => Number.isFinite(entry.values[key])).map((entry) => ({t: entry.time, value: entry.values[key], branch: entry.branch})),
|
||||||
|
}));
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @param labels - the labels of /api/scores: {metrics: {key: {emoji, label}}}, or anything else.
|
||||||
|
* @returns {key: "<emoji> <label>"} in the file's order for the entries with a string emoji and label; null when
|
||||||
|
* none has both or labels is no such object.
|
||||||
|
*/
|
||||||
|
function overlayLabels(labels) {
|
||||||
|
const metrics = labels && typeof labels === "object" ? labels.metrics : null;
|
||||||
|
if (!metrics || typeof metrics !== "object" || Array.isArray(metrics)) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
const kept = Object.entries(metrics).filter(([, metric]) => metric && typeof metric.emoji === "string" && typeof metric.label === "string");
|
||||||
|
return kept.length ? Object.fromEntries(kept.map(([key, metric]) => [key, `${metric.emoji} ${metric.label}`])) : null;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @param series - overlaySeries().
|
||||||
|
* @returns the top of the right axis: 100, or the largest value when one exceeds it.
|
||||||
|
*/
|
||||||
|
const overlayTop = (series) => Math.max(100, ...series.flatMap((found) => found.points.map((point) => point.value)));
|
||||||
|
|
||||||
const KEYS = {
|
const KEYS = {
|
||||||
prio: () => null,
|
prio: () => null,
|
||||||
wsjf: (row) => (row.wsjf ? row.wsjf.score : null),
|
wsjf: (row) => (row.wsjf ? row.wsjf.score : null),
|
||||||
@@ -294,7 +332,7 @@ const Plan = (() => {
|
|||||||
return keyed.map((entry) => entry.row);
|
return keyed.map((entry) => entry.row);
|
||||||
}
|
}
|
||||||
|
|
||||||
return {FORECASTS, ESTIMATIONS, TRENDS, SORTS, SERIES, TRENDED, STATUS_ORDER, STALE_DAYS, median, medians, forecastLength, bars, etas, stale, scoped, burn, steps, trend, finishes, lineAt, sortRows};
|
return {FORECASTS, ESTIMATIONS, TRENDS, SORTS, SERIES, TRENDED, STATUS_ORDER, STALE_DAYS, median, medians, forecastLength, bars, etas, stale, scoped, burn, steps, trend, finishes, lineAt, overlaySeries, overlayLabels, overlayTop, sortRows};
|
||||||
})();
|
})();
|
||||||
if (typeof module === "object" && module.exports) {
|
if (typeof module === "object" && module.exports) {
|
||||||
module.exports = Plan;
|
module.exports = Plan;
|
||||||
|
|||||||
@@ -82,6 +82,15 @@ svg .burn-line.k-unestimated { stroke-dasharray: 6 4; }
|
|||||||
svg .burn-trend { fill: none; stroke: var(--c); stroke-width: 1.6; stroke-dasharray: 2 5; stroke-linecap: round; }
|
svg .burn-trend { fill: none; stroke: var(--c); stroke-width: 1.6; stroke-dasharray: 2 5; stroke-linecap: round; }
|
||||||
svg .burn-point { fill: var(--c); stroke: transparent; stroke-width: 10; cursor: help; }
|
svg .burn-point { fill: var(--c); stroke: transparent; stroke-width: 10; cursor: help; }
|
||||||
svg .burn-point:hover { r: 6px; }
|
svg .burn-point:hover { r: 6px; }
|
||||||
|
svg .burn-overlay-line { fill: none; stroke: var(--c); stroke-width: 1.8; stroke-dasharray: 5 3; }
|
||||||
|
svg .burn-overlay-point { fill: var(--c); stroke: transparent; stroke-width: 10; cursor: help; }
|
||||||
|
svg.burn .overlay-axis { fill: var(--muted); }
|
||||||
|
svg.burn .burn-overlay.k-fix, .burn-box .legend-chip.k-fix { --c: var(--ok); }
|
||||||
|
svg.burn .burn-overlay.k-break, .burn-box .legend-chip.k-break { --c: var(--bad); }
|
||||||
|
svg.burn .burn-overlay.k-tokens, .burn-box .legend-chip.k-tokens { --c: var(--warn); }
|
||||||
|
svg.burn .burn-overlay.k-velocity, .burn-box .legend-chip.k-velocity { --c: var(--info); }
|
||||||
|
svg.burn .burn-overlay.k-security, .burn-box .legend-chip.k-security { --c: var(--violet); }
|
||||||
|
svg.burn .burn-overlay.k-score, .burn-box .legend-chip.k-score { --c: var(--accent); }
|
||||||
svg .forecast-line { stroke: var(--ok); stroke-width: 1.4; stroke-dasharray: 4 3; }
|
svg .forecast-line { stroke: var(--ok); stroke-width: 1.4; stroke-dasharray: 4 3; }
|
||||||
svg .forecast-label { fill: var(--ok); font-size: var(--fs-2xs); font-weight: 650; }
|
svg .forecast-label { fill: var(--ok); font-size: var(--fs-2xs); font-weight: 650; }
|
||||||
.burn-forecast { display: inline-flex; align-items: center; gap: 6px; }
|
.burn-forecast { display: inline-flex; align-items: center; gap: 6px; }
|
||||||
|
|||||||
@@ -14,8 +14,8 @@ and never estimates one itself.
|
|||||||
1. **Fix confidence** (percent): probability the change does what it claims.
|
1. **Fix confidence** (percent): probability the change does what it claims.
|
||||||
**Break risk** (percent): probability it breaks anything else, from the
|
**Break risk** (percent): probability it breaks anything else, from the
|
||||||
`kvb-check-break-risk` skill. Two numbers, never one blended figure.
|
`kvb-check-break-risk` skill. Two numbers, never one blended figure.
|
||||||
2. **Score**: pass both, with cost from the `kvb-check-efficiency`
|
2. **Score**: pass both, with tokens from the `kvb-check-efficiency`
|
||||||
skill, time from the `kvb-check-speed` skill and security and the
|
skill, velocity from the `kvb-check-speed` skill and security and the
|
||||||
critical count from `security.py` of the `kvb-check-security` skill, to
|
critical count from `security.py` of the `kvb-check-security` skill, to
|
||||||
`${CLAUDE_SKILL_DIR}/../kvb-check-score/scripts/score.py --fix <n> --break
|
`${CLAUDE_SKILL_DIR}/../kvb-check-score/scripts/score.py --fix <n> --break
|
||||||
<n> --efficiency <n|-> --speed <n|-> --security <n|~n> --critical <count>`
|
<n> --efficiency <n|-> --speed <n|-> --security <n|~n> --critical <count>`
|
||||||
|
|||||||
@@ -4,7 +4,7 @@ description: >
|
|||||||
Token cost of skill edits, CPU and RAM of code edits.
|
Token cost of skill edits, CPU and RAM of code edits.
|
||||||
---
|
---
|
||||||
|
|
||||||
💸 Cost is the share of the base's resources a change uses: under 100 %
|
💸 Tokens is the share of the base's resources a change uses: under 100 %
|
||||||
it saves, over 100 % it costs more. Paste the script's lines, never a
|
it saves, over 100 % it costs more. Paste the script's lines, never a
|
||||||
paragraph about them. `<ref>` is the commit the change
|
paragraph about them. `<ref>` is the commit the change
|
||||||
starts from: `HEAD` for uncommitted work, the merge base with the default
|
starts from: `HEAD` for uncommitted work, the merge base with the default
|
||||||
@@ -20,7 +20,7 @@ It compares every changed `*.md` under `skills/`, untracked ones included,
|
|||||||
at `<ref>` with the working tree, and the always-loaded listing: all
|
at `<ref>` with the working tree, and the always-loaded listing: all
|
||||||
`description:` words, counted by the repository's `scripts/budget.py`.
|
`description:` words, counted by the repository's `scripts/budget.py`.
|
||||||
Tokens are an estimate, characters / 4. Changed files without any base text
|
Tokens are an estimate, characters / 4. Changed files without any base text
|
||||||
print `cost new`, which counts as over 100. Skills have no time.
|
print `tokens new`, which counts as over 100. Skills have no velocity.
|
||||||
|
|
||||||
A second line, `💰 value 1:<n>` (`<n>:1` when value < cost), weighs both
|
A second line, `💰 value 1:<n>` (`<n>:1` when value < cost), weighs both
|
||||||
per day and never changes the exit code; pass its quotient or `?` to
|
per day and never changes the exit code; pass its quotient or `?` to
|
||||||
|
|||||||
@@ -9,26 +9,26 @@ Args:
|
|||||||
relative to each tree's root; its output is discarded.
|
relative to each tree's root; its output is discarded.
|
||||||
|
|
||||||
Runs:
|
Runs:
|
||||||
One run per tree, base first. When cost and time are both at most 100 that
|
One run per tree, base first. When tokens and velocity are both at most 100 that
|
||||||
is the result; otherwise 4 more runs per tree, alternating, and the arithmetic mean
|
is the result; otherwise 4 more runs per tree, alternating, and the arithmetic mean
|
||||||
of all 5 per tree counts.
|
of all 5 per tree counts.
|
||||||
|
|
||||||
Other load:
|
Other load:
|
||||||
A 1 s /proc/stat sample of system CPU busy before and after the runs, and per run
|
A 1 s /proc/stat sample of system CPU busy before and after the runs, and per run
|
||||||
the busy jiffies minus the command's own CPU time, as a share of all jiffies. When
|
the busy jiffies minus the command's own CPU time, as a share of all jiffies. When
|
||||||
any share exceeds 30 % or cannot be read, the result is an estimate: time comes
|
any share exceeds 30 % or cannot be read, the result is an estimate: velocity comes
|
||||||
from the CPU-time ratio instead of wall time.
|
from the CPU-time ratio instead of wall time.
|
||||||
|
|
||||||
Output:
|
Output:
|
||||||
`💸 cost <p> % (cpu <b>→<a> s, ram <b>→<a> MB) · ⏱️ time <p> % (wall <b>→<a> s)`,
|
`💸 tokens <p> % (cpu <b>→<a> s, ram <b>→<a> MB) · ⏱️ velocity <p> % (wall <b>→<a> s)`,
|
||||||
then ` · mean of 5` after the extra runs. An estimate marks both values with `~`,
|
then ` · mean of 5` after the extra runs. An estimate marks both values with `~`,
|
||||||
shows `(cpu <b>→<a> s)` for time and ends in `(estimated: other load <x> %)`, or
|
shows `(cpu <b>→<a> s)` for velocity and ends in `(estimated: other load <x> %)`, or
|
||||||
`unknown` for <x>. Cost is the mean of the CPU ratio (user plus system seconds
|
`unknown` for <x>. Tokens is the mean of the CPU ratio (user plus system seconds
|
||||||
of the command and its waited-for children) and the peak RSS ratio; each ratio is
|
of the command and its waited-for children) and the peak RSS ratio; each ratio is
|
||||||
working tree over base, times 100.
|
working tree over base, times 100.
|
||||||
|
|
||||||
Exit codes:
|
Exit codes:
|
||||||
0 cost and time at most 100, 2 usage error or a failing command
|
0 tokens and velocity at most 100, 2 usage error or a failing command
|
||||||
(its tree, exit code and last stderr line on stderr), 3 either over 100.
|
(its tree, exit code and last stderr line on stderr), 3 either over 100.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
@@ -56,6 +56,23 @@ CLOCK_TICKS = os.sysconf("SC_CLK_TCK")
|
|||||||
# Trip-wire: ru_maxrss counts kilobytes on Linux but bytes on macOS.
|
# Trip-wire: ru_maxrss counts kilobytes on Linux but bytes on macOS.
|
||||||
RSS_UNIT = 1 if sys.platform == "darwin" else 1024
|
RSS_UNIT = 1 if sys.platform == "darwin" else 1024
|
||||||
USAGE = "usage: measure.py --base <ref> [--json] -- <command...>"
|
USAGE = "usage: measure.py --base <ref> [--json] -- <command...>"
|
||||||
|
LABELS = Path(__file__).resolve().parents[2] / "kvb-check-score" / "labels.json"
|
||||||
|
LABEL_ERRORS = (OSError, ValueError, LookupError, TypeError)
|
||||||
|
|
||||||
|
|
||||||
|
def labelled(keys: tuple[str, ...]) -> dict[str, str]:
|
||||||
|
"""Args:
|
||||||
|
keys: metric keys of kvb-check-score's labels.json.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
Per key `<emoji> <label>`, the label in lower case; one of LABEL_ERRORS
|
||||||
|
for a missing or malformed file.
|
||||||
|
"""
|
||||||
|
metrics = json.loads(LABELS.read_text(encoding="utf-8"))["metrics"]
|
||||||
|
found = {key: (metrics[key]["emoji"], metrics[key]["label"]) for key in keys}
|
||||||
|
if not all(isinstance(part, str) for pair in found.values() for part in pair):
|
||||||
|
raise ValueError(f"needs a string emoji and label for {', '.join(keys)}")
|
||||||
|
return {key: f"{emoji} {label.lower()}" for key, (emoji, label) in found.items()}
|
||||||
|
|
||||||
|
|
||||||
def fail(message: str) -> None:
|
def fail(message: str) -> None:
|
||||||
@@ -139,14 +156,14 @@ def figures(runs: dict[str, list[dict]], loads: list[float | None]) -> dict:
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
def line(found: dict) -> str:
|
def line(found: dict, names: dict[str, str]) -> str:
|
||||||
mark = "~" if found["estimated"] else ""
|
mark = "~" if found["estimated"] else ""
|
||||||
basis = "cpu" if found["estimated"] else "wall"
|
basis = "cpu" if found["estimated"] else "wall"
|
||||||
before, after = found["base"], found["head"]
|
before, after = found["base"], found["head"]
|
||||||
text = (
|
text = (
|
||||||
f"💸 cost {mark}{found['efficiency']} % (cpu {before['cpu']:.2f}→{after['cpu']:.2f} s, "
|
f"{names['tokens']} {mark}{found['efficiency']} % (cpu {before['cpu']:.2f}→{after['cpu']:.2f} s, "
|
||||||
f"ram {before['ram']:.1f}→{after['ram']:.1f} MB) · "
|
f"ram {before['ram']:.1f}→{after['ram']:.1f} MB) · "
|
||||||
f"⏱️ time {mark}{found['speed']} % ({basis} {before[basis]:.2f}→{after[basis]:.2f} s)"
|
f"{names['velocity']} {mark}{found['speed']} % ({basis} {before[basis]:.2f}→{after[basis]:.2f} s)"
|
||||||
)
|
)
|
||||||
if found["runs"] > 1:
|
if found["runs"] > 1:
|
||||||
text += f" · mean of {found['runs']}"
|
text += f" · mean of {found['runs']}"
|
||||||
@@ -175,6 +192,10 @@ def main(argv: list[str]) -> int:
|
|||||||
command = argv[split + 1:]
|
command = argv[split + 1:]
|
||||||
if not command:
|
if not command:
|
||||||
fail("no command after --")
|
fail("no command after --")
|
||||||
|
try:
|
||||||
|
names = {} if args.json else labelled(("tokens", "velocity"))
|
||||||
|
except LABEL_ERRORS as error:
|
||||||
|
fail(f"labels {LABELS}: {error!r}")
|
||||||
|
|
||||||
here = Path.cwd().resolve()
|
here = Path.cwd().resolve()
|
||||||
root = Path(git(here, "rev-parse", "--show-toplevel")).resolve()
|
root = Path(git(here, "rev-parse", "--show-toplevel")).resolve()
|
||||||
@@ -206,7 +227,7 @@ def main(argv: list[str]) -> int:
|
|||||||
shutil.rmtree(parent, ignore_errors=True)
|
shutil.rmtree(parent, ignore_errors=True)
|
||||||
|
|
||||||
found = figures(runs, loads + run_loads(runs))
|
found = figures(runs, loads + run_loads(runs))
|
||||||
print(json.dumps(found) if args.json else line(found))
|
print(json.dumps(found) if args.json else line(found, names))
|
||||||
return 3 if costly(found) else 0
|
return 3 if costly(found) else 0
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -15,8 +15,8 @@ Args:
|
|||||||
shipped change instead.
|
shipped change instead.
|
||||||
|
|
||||||
Output:
|
Output:
|
||||||
Line 1: `💸 cost <p> % (skills <b>→<a> tokens; listing <b>→<a> words)`, p = after / before * 100,
|
Line 1: `💸 tokens <p> % (skills <b>→<a> tokens; listing <b>→<a> words)`, p = after / before * 100,
|
||||||
or `💸 cost new (+<a> tokens; listing <b>→<a> words)` when the changed files hold no
|
or `💸 tokens new (+<a> tokens; listing <b>→<a> words)` when the changed files hold no
|
||||||
base tokens. Changed files are the *.md under skills/ that `git diff <ref>` names plus untracked
|
base tokens. Changed files are the *.md under skills/ that `git diff <ref>` names plus untracked
|
||||||
ones; a file missing on a side counts 0. Tokens are an estimate: characters / 4. The listing
|
ones; a file missing on a side counts 0. Tokens are an estimate: characters / 4. The listing
|
||||||
is the sum of every skills/*/SKILL.md description word.
|
is the sum of every skills/*/SKILL.md description word.
|
||||||
@@ -61,6 +61,41 @@ DAYS = 7
|
|||||||
ROUND_ABOVE = 10
|
ROUND_ABOVE = 10
|
||||||
ALL_SKILLS = "all skills"
|
ALL_SKILLS = "all skills"
|
||||||
CONTEXT = Path(__file__).resolve().parents[2] / "kvb-skill-silence" / "scripts" / "context.py"
|
CONTEXT = Path(__file__).resolve().parents[2] / "kvb-skill-silence" / "scripts" / "context.py"
|
||||||
|
LABELS = Path(__file__).resolve().parents[2] / "kvb-check-score" / "labels.json"
|
||||||
|
|
||||||
|
|
||||||
|
LABEL_ERRORS = (OSError, ValueError, LookupError, TypeError)
|
||||||
|
|
||||||
|
|
||||||
|
def labelled(keys: tuple[str, ...]) -> dict[str, str]:
|
||||||
|
"""Args:
|
||||||
|
keys: metric keys of kvb-check-score's labels.json.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
Per key `<emoji> <label>`, the label in lower case; one of LABEL_ERRORS
|
||||||
|
for a missing or malformed file.
|
||||||
|
"""
|
||||||
|
metrics = json.loads(LABELS.read_text(encoding="utf-8"))["metrics"]
|
||||||
|
found = {key: (metrics[key]["emoji"], metrics[key]["label"]) for key in keys}
|
||||||
|
if not all(isinstance(part, str) for pair in found.values() for part in pair):
|
||||||
|
raise ValueError(f"needs a string emoji and label for {', '.join(keys)}")
|
||||||
|
return {key: f"{emoji} {label.lower()}" for key, (emoji, label) in found.items()}
|
||||||
|
|
||||||
|
|
||||||
|
def headline(name: str, efficiency: int | None, before: int, after: int, listing: str) -> str:
|
||||||
|
"""Args:
|
||||||
|
name: the tokens metric as labelled() gives it.
|
||||||
|
efficiency: after over before in percent, None without base tokens.
|
||||||
|
before: the base tokens of the changed files.
|
||||||
|
after: their tokens in the working tree.
|
||||||
|
listing: the listing words part.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
Line 1.
|
||||||
|
"""
|
||||||
|
if efficiency is None:
|
||||||
|
return f"{name} new (+{after} tokens; {listing})"
|
||||||
|
return f"{name} {efficiency} % (skills {before}→{after} tokens; {listing})"
|
||||||
|
|
||||||
|
|
||||||
def fail(message: str) -> None:
|
def fail(message: str) -> None:
|
||||||
@@ -254,6 +289,10 @@ def measure(
|
|||||||
|
|
||||||
def check(argv: list[str]) -> int:
|
def check(argv: list[str]) -> int:
|
||||||
args = parse(argv)
|
args = parse(argv)
|
||||||
|
try:
|
||||||
|
names = labelled(("tokens",))
|
||||||
|
except LABEL_ERRORS as error:
|
||||||
|
fail(f"labels {LABELS}: {error!r}")
|
||||||
repo = Path(args.repo).resolve()
|
repo = Path(args.repo).resolve()
|
||||||
if not (repo / SKILLS).is_dir():
|
if not (repo / SKILLS).is_dir():
|
||||||
fail(f"{repo} has no {SKILLS}/ directory")
|
fail(f"{repo} has no {SKILLS}/ directory")
|
||||||
@@ -296,11 +335,7 @@ def check(argv: list[str]) -> int:
|
|||||||
},
|
},
|
||||||
}))
|
}))
|
||||||
else:
|
else:
|
||||||
listing = f"listing {listing_before}→{listing_after} words"
|
print(headline(names["tokens"], efficiency, before, after, f"listing {listing_before}→{listing_after} words"))
|
||||||
if efficiency is None:
|
|
||||||
print(f"💸 cost new (+{after} tokens; {listing})")
|
|
||||||
else:
|
|
||||||
print(f"💸 cost {efficiency} % (skills {before}→{after} tokens; {listing})")
|
|
||||||
print(money)
|
print(money)
|
||||||
return 3 if efficiency is None or efficiency > LIMIT else 0
|
return 3 if efficiency is None or efficiency > LIMIT else 0
|
||||||
|
|
||||||
|
|||||||
@@ -6,7 +6,7 @@ description: >
|
|||||||
---
|
---
|
||||||
|
|
||||||
The gate rule, stated here and nowhere else: nothing is declared 90/5 (fix
|
The gate rule, stated here and nowhere else: nothing is declared 90/5 (fix
|
||||||
confidence ≥ 90 %, break risk ≤ 5 %, cost and time ≤ 100 % in
|
confidence ≥ 90 %, break risk ≤ 5 %, tokens and velocity ≤ 100 % in
|
||||||
`kvb-check-score`; over 100 %, ask the operator before the commit, naming
|
`kvb-check-score`; over 100 %, ask the operator before the commit, naming
|
||||||
the cause) before every `make test*` target of the project, each Makefile target named `test` or starting with
|
the cause) before every `make test*` target of the project, each Makefile target named `test` or starting with
|
||||||
`test-`, ran green on the exact head with a clean tracked tree. Until then
|
`test-`, ran green on the exact head with a clean tracked tree. Until then
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
---
|
---
|
||||||
name: kvb-check-score
|
name: kvb-check-score
|
||||||
description: >
|
description: >
|
||||||
Score fix, break, cost, time and security; run wherever confidence is reported.
|
Score fix, break, tokens, velocity and security; run wherever confidence is reported.
|
||||||
---
|
---
|
||||||
|
|
||||||
Run it wherever fix confidence and break risk are reported: every
|
Run it wherever fix confidence and break risk are reported: every
|
||||||
@@ -16,18 +16,22 @@ python3 ${CLAUDE_SKILL_DIR}/scripts/score.py --fix <0-100> --break <0-100> --eff
|
|||||||
| --- | --- | --- |
|
| --- | --- | --- |
|
||||||
| 🎯 | fix confidence | `kvb-check-confidence` |
|
| 🎯 | fix confidence | `kvb-check-confidence` |
|
||||||
| 💥 | break risk | `kvb-check-break-risk` |
|
| 💥 | break risk | `kvb-check-break-risk` |
|
||||||
| 💸 | cost, `--efficiency` | `tokens.py` (skills) or `measure.py` (code) of `kvb-check-efficiency` |
|
| 💸 | tokens, `--efficiency` | `tokens.py` (skills) or `measure.py` (code) of `kvb-check-efficiency` |
|
||||||
| ⏱️ | time, `--speed` | `measure.py`, see `kvb-check-speed` |
|
| ⏱️ | velocity, `--speed` | `measure.py`, see `kvb-check-speed` |
|
||||||
| 🛡️ | security, critical count | `security.py` of `kvb-check-security`; `~<n>` from a review by hand |
|
| 🛡️ | security, critical count | `security.py` of `kvb-check-security`; `~<n>` from a review by hand |
|
||||||
| 💰 | cost:value quotient, tables and message block only | 💰 line of `tokens.py` (skills) of `kvb-check-efficiency` |
|
| 💰 | cost:value quotient, tables and message block only | 💰 line of `tokens.py` (skills) of `kvb-check-efficiency` |
|
||||||
| 🏆 | score | this script |
|
| 🏆 | score | this script |
|
||||||
|
|
||||||
Cost and time: after over base in percent, over 100 worse. `-` means not
|
Emojis and labels come from `labels.json`, one entry per metric key with
|
||||||
measured or not relevant (skills have no time) and counts as
|
its former labels (Cost, Efficiency, Time, Speed); change a label there,
|
||||||
|
nowhere else.
|
||||||
|
|
||||||
|
Tokens and velocity: after over base in percent, over 100 worse. `-` means not
|
||||||
|
measured or not relevant (skills have no velocity) and counts as
|
||||||
100; it shows as `–`. `new` (changed skill text without a base) counts as 100
|
100; it shows as `–`. `new` (changed skill text without a base) counts as 100
|
||||||
in the score, yet still asks. Never invent a value: measure it or pass `-`.
|
in the score, yet still asks. Never invent a value: measure it or pass `-`.
|
||||||
|
|
||||||
Score = `fix * (100 - break) / 100 * 100 / cost * 100 / time * security / 100`,
|
Score = `fix * (100 - break) / 100 * 100 / tokens * 100 / velocity * security / 100`,
|
||||||
capped to 0..100 and rounded.
|
capped to 0..100 and rounded.
|
||||||
|
|
||||||
## Gate
|
## Gate
|
||||||
@@ -42,20 +46,22 @@ The second line is the verdict, first match wins:
|
|||||||
in the block and commit only on the operator's word. Close the gap as the
|
in the block and commit only on the operator's word. Close the gap as the
|
||||||
last section of `kvb-check-break-risk` says.
|
last section of `kvb-check-break-risk` says.
|
||||||
- `⚠️ gate: <causes>: ask the operator before the commit` (exit 3), naming
|
- `⚠️ gate: <causes>: ask the operator before the commit` (exit 3), naming
|
||||||
each of `cost <c> % > 100`, `time <t> % > 100` and
|
each of `tokens <c> % > 100`, `velocity <t> % > 100` and
|
||||||
`security <n> % < 100`: ask with the block what the cost buys or why each
|
`security <n> % < 100`: ask with the block what the cost buys or why each
|
||||||
finding is acceptable; commit only on a yes, never carried over to
|
finding is acceptable; commit only on a yes, never carried over to
|
||||||
another commit.
|
another commit.
|
||||||
- `✅ gate: 90/5, cost and time <= 100, security 100 %` (exit 0):
|
- `✅ gate: 90/5, tokens and velocity <= 100, security 100 %` (exit 0):
|
||||||
commit as usual. 90/5 still needs `gate: green` of `kvb-check-gate`.
|
commit as usual. 90/5 still needs `gate: green` of `kvb-check-gate`.
|
||||||
- Exit 2: usage error.
|
- Exit 2: usage error, a malformed `labels.json` or a `--commit-table`
|
||||||
|
repository git cannot read, with one stderr line and nothing on stdout;
|
||||||
|
fix the cause and score again. A critical finding still exits 4 then.
|
||||||
|
|
||||||
## Tables and message block
|
## Tables and message block
|
||||||
|
|
||||||
Each flag replaces the two lines; values, gate and exit code stay:
|
Each flag replaces the two lines; values, gate and exit code stay:
|
||||||
|
|
||||||
- `--table`: one markdown table, `| 🎯 Fix | 💥 Break | 💸 Cost |
|
- `--table`: one markdown table, `| 🎯 Fix | 💥 Break | 💸 Tokens |
|
||||||
⏱️ Time | 🛡️ Security | 🏆 Score | Gate |`.
|
⏱️ Velocity | 🛡️ Security | 🏆 Score | Gate |`.
|
||||||
- `--commit-table <repo>`: that table led by `🌿 Branch | 📊 Diff stat`
|
- `--commit-table <repo>`: that table led by `🌿 Branch | 📊 Diff stat`
|
||||||
from git, 📁 worktree path below, alone in a code block. Every commit
|
from git, 📁 worktree path below, alone in a code block. Every commit
|
||||||
block is this table; after the commit, `--commit <sha>` shows that commit's diff stat
|
block is this table; after the commit, `--commit <sha>` shows that commit's diff stat
|
||||||
@@ -66,6 +72,32 @@ Each flag replaces the two lines; values, gate and exit code stay:
|
|||||||
`-m` argument, so the approval prompt and git history show the values.
|
`-m` argument, so the approval prompt and git history show the values.
|
||||||
|
|
||||||
On a skill change add `--value <n:n|?>`, the quotient of the 💰 line (`?`
|
On a skill change add `--value <n:n|?>`, the quotient of the 💰 line (`?`
|
||||||
while it has no value): a `💰 Value` column after `💸 Cost`, or line
|
while it has no value): a `💰 Value` column after `💸 Tokens`, or line
|
||||||
after `🛡️ Security`. It changes neither score nor gate; alone it is a usage
|
after `🛡️ Security`. It changes neither score nor gate; alone it is a usage
|
||||||
error.
|
error.
|
||||||
|
|
||||||
|
## Journal
|
||||||
|
|
||||||
|
Each `--message-block` render appends one row to
|
||||||
|
`${AGENTS_COUNCIL_ROOT:-/tmp/agents-council}/scores.jsonl`: time, origin
|
||||||
|
remote, top level path, branch, `sha` null, the six values and the block's
|
||||||
|
digest. The hub's burndown draws it as overlays; no gate reads it. A failed
|
||||||
|
append prints one stderr line and keeps the exit code; a linked council
|
||||||
|
root or a link, FIFO or other non-regular file at the journal fails it
|
||||||
|
without blocking, through `nofollow.py` of `kvb-org-organize`; when that
|
||||||
|
helper is missing or unusable, the line names it. A new council root gets mode
|
||||||
|
0700, a new journal 0600.
|
||||||
|
|
||||||
|
The journal lives in `/tmp`, so a reboot or another PC starts empty. Refill
|
||||||
|
it from the blocks in git history, without a model:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
python3 ${CLAUDE_SKILL_DIR}/scripts/backfill.py --repo <dir>... [--since <rev>] [--labels <file>]
|
||||||
|
```
|
||||||
|
|
||||||
|
It reads every local and origin branch, takes the last block of each commit
|
||||||
|
message, old labels included, and adds one row per commit not yet keyed by
|
||||||
|
(remote, sha), so a second run or another clone adds nothing. The branch is
|
||||||
|
the one whose first-parent line holds the commit, the default branch first,
|
||||||
|
else the one a merge subject names. An unknown label is reported once and
|
||||||
|
skipped.
|
||||||
@@ -0,0 +1,14 @@
|
|||||||
|
{
|
||||||
|
"metrics": {
|
||||||
|
"fix": {"emoji": "🎯", "label": "Fix", "former": []},
|
||||||
|
"break": {"emoji": "💥", "label": "Break", "former": []},
|
||||||
|
"tokens": {"emoji": "💸", "label": "Tokens", "former": ["Cost", "Efficiency"]},
|
||||||
|
"velocity": {"emoji": "⏱️", "label": "Velocity", "former": ["Time", "Speed"]},
|
||||||
|
"security": {"emoji": "🛡️", "label": "Security", "former": []},
|
||||||
|
"score": {"emoji": "🏆", "label": "Score", "former": []}
|
||||||
|
},
|
||||||
|
"extras": {
|
||||||
|
"value": {"emoji": "💰", "label": "Value", "former": []},
|
||||||
|
"gate": {"emoji": "🚦", "label": "Gate", "former": []}
|
||||||
|
}
|
||||||
|
}
|
||||||
Executable
+252
@@ -0,0 +1,252 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""Backfill the score journal of journal.py from the message blocks in git history.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
--repo <dir>...: repositories whose local and origin branches git log reads;
|
||||||
|
repeatable, every one given counts.
|
||||||
|
--since <rev>: only commits not reachable from <rev>, in every repository.
|
||||||
|
--labels <file>: the label file mapping current and former labels to metric
|
||||||
|
keys, ../labels.json when left out.
|
||||||
|
|
||||||
|
Output:
|
||||||
|
One line per repository, `<top level>: <n> blocks, <m> rows added`; one stderr
|
||||||
|
line per unknown block label, `backfill: unknown label <label>, skipped`.
|
||||||
|
|
||||||
|
Exit codes:
|
||||||
|
0 done, 2 usage error, a repository git cannot read or a journal that is
|
||||||
|
linked, not a regular file or not writable, with one stderr line; nothing
|
||||||
|
is written then.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import json
|
||||||
|
import re
|
||||||
|
import sys
|
||||||
|
from collections import deque
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import NamedTuple
|
||||||
|
|
||||||
|
from journal import (
|
||||||
|
LABELS,
|
||||||
|
append,
|
||||||
|
entry,
|
||||||
|
git_read,
|
||||||
|
journal_path,
|
||||||
|
load_labels,
|
||||||
|
origin,
|
||||||
|
read,
|
||||||
|
)
|
||||||
|
|
||||||
|
FIELD = "\x00"
|
||||||
|
RECORD = "\x1e"
|
||||||
|
LOG_FORMAT = "%H%x00%P%x00%ct%x00%B%x1e"
|
||||||
|
REFS = ("refs/heads/", "refs/remotes/origin/")
|
||||||
|
LINE = re.compile(r"([^\x00-\x7f]\S*) ([A-Z][A-Za-z]*): (.+)")
|
||||||
|
AMOUNT = re.compile(r"~?(\d+(?:\.\d+)?)(?: ?%)?")
|
||||||
|
PARAGRAPH = re.compile(r"\n[ \t]*\n")
|
||||||
|
MERGES = (
|
||||||
|
re.compile(r"Merge pull request #\d+ from [^/\s]+/(\S+)"),
|
||||||
|
re.compile(r"Merge (?:remote-tracking )?branch '(?:origin/)?([^']+)'"),
|
||||||
|
re.compile(r"Merge (\S+) into \S+"),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class Commit(NamedTuple):
|
||||||
|
parents: list[str]
|
||||||
|
time: int
|
||||||
|
message: str
|
||||||
|
|
||||||
|
|
||||||
|
def aliases(labels: dict) -> tuple[dict[str, str], set[str]]:
|
||||||
|
"""Args:
|
||||||
|
labels: load_labels() of a label file.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
(metric of each current and former label, labels of the extra lines).
|
||||||
|
"""
|
||||||
|
named = {name: key for key, metric in labels["metrics"].items() for name in (metric["label"], *metric["former"])}
|
||||||
|
extras = {name for extra in labels["extras"].values() for name in (extra["label"], *extra["former"])}
|
||||||
|
return named, extras
|
||||||
|
|
||||||
|
|
||||||
|
def amount(text: str) -> float | int | None:
|
||||||
|
found = AMOUNT.fullmatch(text.strip())
|
||||||
|
if not found:
|
||||||
|
return None
|
||||||
|
value = float(found.group(1))
|
||||||
|
return int(value) if value.is_integer() else value
|
||||||
|
|
||||||
|
|
||||||
|
def parse(message: str, named: dict[str, str], extras: set[str], unknown: set[str]) -> tuple[dict, str] | None:
|
||||||
|
"""Args:
|
||||||
|
message: a commit message.
|
||||||
|
named: aliases() metrics.
|
||||||
|
extras: aliases() extra labels, read and left out.
|
||||||
|
unknown: collects the labels of block lines neither names.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
(values per metric key, block text) of the last block, None without one.
|
||||||
|
"""
|
||||||
|
found = None
|
||||||
|
for paragraph in PARAGRAPH.split(message):
|
||||||
|
lines = [match for match in (LINE.fullmatch(text.strip()) for text in paragraph.splitlines()) if match]
|
||||||
|
if not any(named.get(match.group(2)) == "score" for match in lines):
|
||||||
|
continue
|
||||||
|
values = dict.fromkeys(named.values())
|
||||||
|
for match in lines:
|
||||||
|
label = match.group(2)
|
||||||
|
if label in named:
|
||||||
|
values[named[label]] = amount(match.group(3))
|
||||||
|
elif label not in extras:
|
||||||
|
unknown.add(label)
|
||||||
|
if values["score"] is not None:
|
||||||
|
found = (values, "\n".join(match.group(0) for match in lines))
|
||||||
|
return found
|
||||||
|
|
||||||
|
|
||||||
|
def history(repo: Path, tips: list[str], since: str | None) -> dict[str, Commit]:
|
||||||
|
text = git_read(repo, "log", f"--format={LOG_FORMAT}", *tips, *([f"^{since}"] if since else []), "--")
|
||||||
|
if text is None:
|
||||||
|
raise LookupError(f"git log in {repo} failed")
|
||||||
|
commits = {}
|
||||||
|
for record in text.split(RECORD):
|
||||||
|
fields = record.strip("\n").split(FIELD, 3)
|
||||||
|
if len(fields) == 4:
|
||||||
|
sha, parents, stamp, message = fields
|
||||||
|
commits[sha] = Commit(parents.split(), int(stamp), message)
|
||||||
|
return commits
|
||||||
|
|
||||||
|
|
||||||
|
def heads(repo: Path) -> list[tuple[str, str]]:
|
||||||
|
"""Args:
|
||||||
|
repo: a git repository.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
(branch, tip sha) of every local and origin branch, the default branch
|
||||||
|
(origin/HEAD, else the checked out one) first, then by name, local first.
|
||||||
|
"""
|
||||||
|
listing = git_read(repo, "for-each-ref", "--format=%(refname)%00%(objectname)", *REFS)
|
||||||
|
if listing is None:
|
||||||
|
raise LookupError(f"git for-each-ref in {repo} failed")
|
||||||
|
pointed = git_read(repo, "symbolic-ref", "-q", "refs/remotes/origin/HEAD") or git_read(repo, "symbolic-ref", "-q", "HEAD") or ""
|
||||||
|
default = next((pointed.removeprefix(prefix) for prefix in REFS if pointed.startswith(prefix)), None)
|
||||||
|
found = []
|
||||||
|
for row in listing.splitlines():
|
||||||
|
ref, sha = row.split(FIELD)
|
||||||
|
rank, prefix = next((rank, prefix) for rank, prefix in enumerate(REFS) if ref.startswith(prefix))
|
||||||
|
name = ref.removeprefix(prefix)
|
||||||
|
if name != "HEAD":
|
||||||
|
found.append((name != default, name, rank, sha))
|
||||||
|
return [(name, sha) for _, name, _, sha in sorted(found)]
|
||||||
|
|
||||||
|
|
||||||
|
def merged(message: str) -> str | None:
|
||||||
|
subject = message.split("\n", 1)[0]
|
||||||
|
return next((match.group(1) for match in (pattern.match(subject) for pattern in MERGES) if match), None)
|
||||||
|
|
||||||
|
|
||||||
|
def owners(commits: dict[str, Commit], tips: list[tuple[str, str]]) -> dict[str, str]:
|
||||||
|
"""Args:
|
||||||
|
commits: history() of the repository.
|
||||||
|
tips: heads() of the repository.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
The branch of every commit: the first tip whose first-parent line holds
|
||||||
|
it, else the branch a merge subject names for its side, else the merge's.
|
||||||
|
"""
|
||||||
|
owner: dict[str, str] = {}
|
||||||
|
queue = deque((sha, name) for name, sha in tips)
|
||||||
|
while queue:
|
||||||
|
sha, name = queue.popleft()
|
||||||
|
while sha in commits and sha not in owner:
|
||||||
|
owner[sha] = name
|
||||||
|
commit = commits[sha]
|
||||||
|
side = merged(commit.message) or name
|
||||||
|
queue.extend((other, side) for other in commit.parents[1:])
|
||||||
|
sha = commit.parents[0] if commit.parents else None
|
||||||
|
return owner
|
||||||
|
|
||||||
|
|
||||||
|
def scan(repo: Path, since: str | None, named: dict[str, str], extras: set[str], unknown: set[str]) -> tuple[str, list[dict]]:
|
||||||
|
place = origin(repo)
|
||||||
|
if place[1] is None:
|
||||||
|
raise LookupError(f"{repo} is not a git work tree")
|
||||||
|
tips = heads(repo)
|
||||||
|
commits = history(repo, [sha for _, sha in tips], since) if tips else {}
|
||||||
|
owner = owners(commits, tips)
|
||||||
|
rows = []
|
||||||
|
for sha, commit in commits.items():
|
||||||
|
found = parse(commit.message, named, extras, unknown)
|
||||||
|
if found:
|
||||||
|
values, block = found
|
||||||
|
rows.append(entry(commit.time, place, owner.get(sha), sha, values, block))
|
||||||
|
return place[1], rows
|
||||||
|
|
||||||
|
|
||||||
|
def known(path: Path) -> set[tuple[str | None, str]]:
|
||||||
|
try:
|
||||||
|
text = read(path)
|
||||||
|
except FileNotFoundError:
|
||||||
|
return set()
|
||||||
|
keys = set()
|
||||||
|
for line in text.splitlines():
|
||||||
|
try:
|
||||||
|
row = json.loads(line)
|
||||||
|
except ValueError:
|
||||||
|
continue
|
||||||
|
if isinstance(row, dict) and isinstance(row.get("sha"), str):
|
||||||
|
keys.add((row.get("remote"), row["sha"]))
|
||||||
|
return keys
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> int:
|
||||||
|
parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
|
||||||
|
parser.add_argument("--repo", type=Path, nargs="+", action="extend", required=True)
|
||||||
|
parser.add_argument("--since", metavar="REV")
|
||||||
|
parser.add_argument("--labels", type=Path, default=LABELS)
|
||||||
|
args = parser.parse_args()
|
||||||
|
try:
|
||||||
|
named, extras = aliases(load_labels(args.labels))
|
||||||
|
except (OSError, ValueError, LookupError, TypeError, AttributeError) as error:
|
||||||
|
parser.error(f"--labels {args.labels}: {error!r}")
|
||||||
|
unknown: set[str] = set()
|
||||||
|
try:
|
||||||
|
scanned = [scan(repo, args.since, named, extras, unknown) for repo in args.repo]
|
||||||
|
except LookupError as error:
|
||||||
|
print(f"backfill: {error}", file=sys.stderr)
|
||||||
|
return 2
|
||||||
|
path = journal_path()
|
||||||
|
try:
|
||||||
|
keys = known(path)
|
||||||
|
except (OSError, ValueError) as error:
|
||||||
|
return refused(path, error)
|
||||||
|
added, summary = [], []
|
||||||
|
for top, rows in scanned:
|
||||||
|
fresh = []
|
||||||
|
for row in sorted(rows, key=lambda row: (row["time"], row["sha"])):
|
||||||
|
key = (row["remote"], row["sha"])
|
||||||
|
if key not in keys:
|
||||||
|
keys.add(key)
|
||||||
|
fresh.append(row)
|
||||||
|
added += fresh
|
||||||
|
summary.append(f"{top}: {len(rows)} blocks, {len(fresh)} rows added")
|
||||||
|
if added:
|
||||||
|
try:
|
||||||
|
append(path, added)
|
||||||
|
except (OSError, ValueError) as error:
|
||||||
|
return refused(path, error)
|
||||||
|
print("\n".join(summary))
|
||||||
|
for label in sorted(unknown):
|
||||||
|
print(f"backfill: unknown label {label}, skipped", file=sys.stderr)
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
def refused(path: Path, error: Exception) -> int:
|
||||||
|
print(f"backfill: journal {path}: {getattr(error, 'strerror', None) or error}", file=sys.stderr)
|
||||||
|
return 2
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
sys.exit(main())
|
||||||
@@ -0,0 +1,184 @@
|
|||||||
|
"""Labels and score journal shared by score.py and backfill.py of kvb-check-score.
|
||||||
|
|
||||||
|
Environment:
|
||||||
|
AGENTS_COUNCIL_ROOT: council directory shared by every agent on the host;
|
||||||
|
/tmp/agents-council when unset or empty. The journal is its scores.jsonl,
|
||||||
|
display data for the hub's burndown only, never a gate input.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import hashlib
|
||||||
|
import importlib.util
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import re
|
||||||
|
import subprocess
|
||||||
|
from collections.abc import Callable
|
||||||
|
from pathlib import Path
|
||||||
|
from types import ModuleType
|
||||||
|
from urllib.parse import urlsplit
|
||||||
|
|
||||||
|
LABELS = Path(__file__).resolve().parents[1] / "labels.json"
|
||||||
|
NOFOLLOW = Path(__file__).resolve().parents[2] / "kvb-org-organize" / "scripts" / "nofollow.py"
|
||||||
|
JOURNAL = "scores.jsonl"
|
||||||
|
COUNCIL = "/tmp/agents-council"
|
||||||
|
SCP = re.compile(r"(?:[^@/:]+@)?([^/:]+):(?!//)(.+)")
|
||||||
|
FILE_MODE = 0o600
|
||||||
|
|
||||||
|
|
||||||
|
def load_labels(path: Path) -> dict:
|
||||||
|
"""Args:
|
||||||
|
path: a labels.json with "metrics" and "extras", each key mapped to
|
||||||
|
{"emoji", "label", "former"}.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
The parsed file; metrics in block order.
|
||||||
|
"""
|
||||||
|
return json.loads(path.read_text(encoding="utf-8"))
|
||||||
|
|
||||||
|
|
||||||
|
def journal_path() -> Path:
|
||||||
|
return Path(os.environ.get("AGENTS_COUNCIL_ROOT") or COUNCIL) / JOURNAL
|
||||||
|
|
||||||
|
|
||||||
|
def git(repo: Path | str, *args: str) -> str:
|
||||||
|
"""Args:
|
||||||
|
repo: the directory git runs in.
|
||||||
|
args: the git arguments.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
The stripped stdout, bytes outside UTF-8 replaced; LookupError naming
|
||||||
|
the command and git's stderr on one line when git is missing, the
|
||||||
|
directory is gone or git fails.
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
done = subprocess.run(["git", *args], cwd=repo, capture_output=True, encoding="utf-8", errors="replace", check=False)
|
||||||
|
except OSError as error:
|
||||||
|
raise LookupError(f"git {args[0]} in {repo}: {error}") from None
|
||||||
|
if done.returncode != 0:
|
||||||
|
raise LookupError(f"git {args[0]} in {repo}: {' '.join(done.stderr.split())}")
|
||||||
|
return done.stdout.strip()
|
||||||
|
|
||||||
|
|
||||||
|
def git_read(repo: Path, *args: str) -> str | None:
|
||||||
|
"""Args:
|
||||||
|
repo: the directory git runs in.
|
||||||
|
args: the git arguments.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
The stdout of git(), or None when it raises.
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
return git(repo, *args)
|
||||||
|
except LookupError:
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def identity(url: str | None) -> str | None:
|
||||||
|
"""Args:
|
||||||
|
url: a git remote URL: https, ssh, scp-like or a local path.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
host/path without .git for a hosted remote, so ssh, scp-like and https
|
||||||
|
clones of one repository match; a local path unchanged; None for None.
|
||||||
|
"""
|
||||||
|
if url is None:
|
||||||
|
return None
|
||||||
|
scp = SCP.fullmatch(url)
|
||||||
|
if "://" in url:
|
||||||
|
parts = urlsplit(url)
|
||||||
|
host, path = parts.hostname, parts.path
|
||||||
|
elif scp:
|
||||||
|
host, path = scp.groups()
|
||||||
|
else:
|
||||||
|
return url
|
||||||
|
if not host:
|
||||||
|
return url
|
||||||
|
return f"{host.lower()}/{path.strip('/').removesuffix('.git')}"
|
||||||
|
|
||||||
|
|
||||||
|
def origin(repo: Path) -> tuple[str | None, str | None]:
|
||||||
|
"""Args:
|
||||||
|
repo: a directory inside a git work tree.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
(remote, path): the identity() of its origin remote and its top level
|
||||||
|
path; None for what git cannot tell.
|
||||||
|
"""
|
||||||
|
return identity(git_read(repo, "remote", "get-url", "origin")), git_read(repo, "rev-parse", "--show-toplevel")
|
||||||
|
|
||||||
|
|
||||||
|
def entry(time: int, place: tuple[str | None, str | None], branch: str | None, sha: str | None, values: dict, block: str) -> dict:
|
||||||
|
"""Args:
|
||||||
|
time: epoch seconds: the render for score.py, the commit for backfill.py.
|
||||||
|
place: origin() of the repository.
|
||||||
|
branch: the branch the block belongs to.
|
||||||
|
sha: the commit holding the block, None for a render.
|
||||||
|
values: per metric key its number, None where the block shows none.
|
||||||
|
block: the message block text the digest covers.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
One journal row.
|
||||||
|
"""
|
||||||
|
remote, path = place
|
||||||
|
digest = hashlib.sha256(block.encode("utf-8")).hexdigest()
|
||||||
|
return {"time": time, "remote": remote, "path": path, "branch": branch, "sha": sha, "values": values, "digest": digest}
|
||||||
|
|
||||||
|
|
||||||
|
def line(row: dict) -> str:
|
||||||
|
return json.dumps(row, ensure_ascii=False) + "\n"
|
||||||
|
|
||||||
|
|
||||||
|
def unusable(error: Exception) -> OSError:
|
||||||
|
return OSError(f"helper {NOFOLLOW} unusable ({type(error).__name__}): install or update kvb-org-organize")
|
||||||
|
|
||||||
|
|
||||||
|
def nofollow[T](use: Callable[[ModuleType], T]) -> T:
|
||||||
|
"""Args:
|
||||||
|
use: what to do with nofollow.py of kvb-org-organize.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
The result of use; its OSError unchanged, any other failure loading
|
||||||
|
or using the helper as one OSError naming it, never a
|
||||||
|
FileNotFoundError.
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
spec = importlib.util.spec_from_file_location("kvb_score_nofollow", NOFOLLOW)
|
||||||
|
loaded = importlib.util.module_from_spec(spec)
|
||||||
|
spec.loader.exec_module(loaded)
|
||||||
|
except Exception as error: # noqa: BLE001
|
||||||
|
raise unusable(error) from None
|
||||||
|
try:
|
||||||
|
return use(loaded)
|
||||||
|
except OSError:
|
||||||
|
raise
|
||||||
|
except Exception as error: # noqa: BLE001
|
||||||
|
raise unusable(error) from None
|
||||||
|
|
||||||
|
|
||||||
|
def append(path: Path, rows: list[dict]) -> None:
|
||||||
|
"""Args:
|
||||||
|
path: the journal in the council root, made with the helper's mode
|
||||||
|
0o700; a linked root, a link, FIFO or other non-regular file there
|
||||||
|
raises OSError without blocking.
|
||||||
|
rows: the rows to append, one JSON line each, in one write.
|
||||||
|
"""
|
||||||
|
text = "".join(line(row) for row in rows)
|
||||||
|
|
||||||
|
def write(helper: ModuleType) -> None:
|
||||||
|
path.parent.mkdir(mode=helper.DIRECTORY_MODE, parents=True, exist_ok=True)
|
||||||
|
helper.append(path.parent, path, text, FILE_MODE)
|
||||||
|
|
||||||
|
nofollow(write)
|
||||||
|
|
||||||
|
|
||||||
|
def read(path: Path) -> str:
|
||||||
|
"""Args:
|
||||||
|
path: the journal, guarded as for append(); a missing one raises
|
||||||
|
FileNotFoundError.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
Its text, undecodable bytes replaced.
|
||||||
|
"""
|
||||||
|
return nofollow(lambda helper: helper.read_text(path.parent, path))
|
||||||
@@ -1,13 +1,13 @@
|
|||||||
#!/usr/bin/env python3
|
#!/usr/bin/env python3
|
||||||
"""Fold fix confidence, break risk, cost, time and security into one score and one gate verdict.
|
"""Fold fix confidence, break risk, tokens, velocity and security into one score and one gate verdict.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
--fix: fix confidence in percent, 0 to 100.
|
--fix: fix confidence in percent, 0 to 100.
|
||||||
--break: break risk in percent, 0 to 100.
|
--break: break risk in percent, 0 to 100.
|
||||||
--efficiency: cost in percent above 0, from kvb-check-efficiency; `-` when not
|
--efficiency: tokens in percent above 0, from kvb-check-efficiency; `-` when not
|
||||||
measured or not relevant, `new` when the change has no base. Both count as 100
|
measured or not relevant, `new` when the change has no base. Both count as 100
|
||||||
in the score.
|
in the score.
|
||||||
--speed: time in percent above 0, from kvb-check-speed; `-` or `new` as for
|
--speed: velocity in percent above 0, from kvb-check-speed; `-` or `new` as for
|
||||||
--efficiency.
|
--efficiency.
|
||||||
--security: security in percent, 0 to 100, from security.py of kvb-check-security;
|
--security: security in percent, 0 to 100, from security.py of kvb-check-security;
|
||||||
`~<n>` for a value from a review by hand.
|
`~<n>` for a value from a review by hand.
|
||||||
@@ -19,24 +19,30 @@ Args:
|
|||||||
--commit <sha>: with --commit-table, the diff stat of that commit instead of
|
--commit <sha>: with --commit-table, the diff stat of that commit instead of
|
||||||
the staged one, plus a commit column.
|
the staged one, plus a commit column.
|
||||||
--message-block: print the commit message block instead of the two lines, one
|
--message-block: print the commit message block instead of the two lines, one
|
||||||
value per line: `🎯 Fix: <f> %`, `💥 Break: <b> %`, `💸 Cost: <c>`,
|
value per line: `🎯 Fix: <f> %`, `💥 Break: <b> %`, `💸 Tokens: <c>`,
|
||||||
`⏱️ Time: <t>`, `🛡️ Security: <n> %`, `💰 Value: <n:n>` with --value only,
|
`⏱️ Velocity: <t>`, `🛡️ Security: <n> %`, `💰 Value: <n:n>` with --value only,
|
||||||
`🏆 Score: <n>`, `🚦 Gate: <verdict>`; values and markers as in line 1.
|
`🏆 Score: <n>`, `🚦 Gate: <verdict>`; values and markers as in line 1. Each
|
||||||
|
render appends one row to the score journal of journal.py; a failed append
|
||||||
|
prints one stderr line and keeps the exit code.
|
||||||
--value <n:n|?>: with --table, --commit-table or --message-block, a value after
|
--value <n:n|?>: with --table, --commit-table or --message-block, a value after
|
||||||
the cost one (--message-block: after security): the cost:value quotient of
|
the tokens one (--message-block: after security): the cost:value quotient of
|
||||||
tokens.py of kvb-check-efficiency, `?` when unknown. It changes neither score nor gate.
|
tokens.py of kvb-check-efficiency, `?` when unknown. It changes neither score nor gate.
|
||||||
|
|
||||||
|
Labels and emojis come from ../labels.json.
|
||||||
|
|
||||||
Output:
|
Output:
|
||||||
Line 1: `🎯 fix <f> % · 💥 break <b> % · 💸 cost <c> % · ⏱️ time <t> % ·
|
Line 1: `🎯 fix <f> % · 💥 break <b> % · 💸 tokens <c> % · ⏱️ velocity <t> % ·
|
||||||
🛡️ security <n> % · 🏆 score <n>`, `–` in place of `<x> %` for `-`, `new` for `new`.
|
🛡️ security <n> % · 🏆 score <n>`, `–` in place of `<x> %` for `-`, `new` for `new`.
|
||||||
Score is fix * (100 - break) / 100 * 100 / cost * 100 / time * security / 100,
|
Score is fix * (100 - break) / 100 * 100 / tokens * 100 / velocity * security / 100,
|
||||||
capped to 0..100, rounded half up.
|
capped to 0..100, rounded half up.
|
||||||
Line 2: the gate verdict, `✅ gate: ...`, `⛔ gate: ...` or `⚠️ gate: ...`.
|
Line 2: the gate verdict, `✅ gate: ...`, `⛔ gate: ...` or `⚠️ gate: ...`.
|
||||||
|
|
||||||
Exit codes:
|
Exit codes:
|
||||||
0 gate passed, 1 fix under 90 or break over 5, 2 usage error,
|
0 gate passed, 1 fix under 90 or break over 5, 2 usage error, a malformed
|
||||||
3 cost or time over 100 or new, or security under 100: ask the operator
|
labels.json or a --commit-table repository git cannot read, one stderr line
|
||||||
before the commit, 4 a critical security finding: do not commit.
|
and nothing on stdout (4 still wins with a critical finding), 3 tokens or
|
||||||
|
velocity over 100 or new, or security under 100: ask the operator before the
|
||||||
|
commit, 4 a critical security finding: do not commit.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
@@ -44,8 +50,20 @@ from __future__ import annotations
|
|||||||
import argparse
|
import argparse
|
||||||
import math
|
import math
|
||||||
import re
|
import re
|
||||||
import subprocess
|
|
||||||
import sys
|
import sys
|
||||||
|
import time
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
from journal import (
|
||||||
|
LABELS,
|
||||||
|
append,
|
||||||
|
entry,
|
||||||
|
git,
|
||||||
|
git_read,
|
||||||
|
journal_path,
|
||||||
|
load_labels,
|
||||||
|
origin,
|
||||||
|
)
|
||||||
|
|
||||||
FIX_MIN = 90
|
FIX_MIN = 90
|
||||||
BREAK_MAX = 5
|
BREAK_MAX = 5
|
||||||
@@ -54,12 +72,10 @@ SAFE = 100
|
|||||||
DASH = "-"
|
DASH = "-"
|
||||||
NEW = "new"
|
NEW = "new"
|
||||||
ESTIMATE = "~"
|
ESTIMATE = "~"
|
||||||
SCORE_HEADER = ("🎯 Fix", "💥 Break", "💸 Cost", "⏱️ Time", "🛡️ Security", "🏆 Score", "Gate")
|
|
||||||
COMMIT_HEADER = ("🌿 Branch", "📊 Diff stat")
|
COMMIT_HEADER = ("🌿 Branch", "📊 Diff stat")
|
||||||
WORKTREE = "📁 Worktree:"
|
WORKTREE = "📁 Worktree:"
|
||||||
VALUE_HEADER = "💰 Value"
|
|
||||||
VALUE_AT = SCORE_HEADER.index("💸 Cost") + 1
|
|
||||||
UNKNOWN = "?"
|
UNKNOWN = "?"
|
||||||
|
LABEL_ERRORS = (OSError, ValueError, LookupError, TypeError, AttributeError)
|
||||||
QUOTIENT = re.compile(r"\d+(?:\.\d+)?:\d+(?:\.\d+)?")
|
QUOTIENT = re.compile(r"\d+(?:\.\d+)?:\d+(?:\.\d+)?")
|
||||||
SHORTSTAT = re.compile(
|
SHORTSTAT = re.compile(
|
||||||
r"(\d+) files? changed(?:, (\d+) insertions?\(\+\))?(?:, (\d+) deletions?\(-\))?"
|
r"(\d+) files? changed(?:, (\d+) insertions?\(\+\))?(?:, (\d+) deletions?\(-\))?"
|
||||||
@@ -137,35 +153,42 @@ def over(name: str, value: float | str) -> str | None:
|
|||||||
return None
|
return None
|
||||||
|
|
||||||
|
|
||||||
def gate(args: argparse.Namespace) -> tuple[str, str, int]:
|
def labels(keys: set[str]) -> tuple[dict, dict]:
|
||||||
|
"""Args:
|
||||||
|
keys: the metric keys score.py measures.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
(metrics, extras) of labels.json; ValueError when it lacks one of the
|
||||||
|
keys, holds another or an entry has no string emoji and label.
|
||||||
|
"""
|
||||||
|
found = load_labels(LABELS)
|
||||||
|
metrics, extras = found["metrics"], found["extras"]
|
||||||
|
named = [*metrics.values(), extras["value"], extras["gate"]]
|
||||||
|
if set(metrics) != keys or not all(isinstance(item["emoji"], str) and isinstance(item["label"], str) for item in named):
|
||||||
|
raise ValueError(f"needs a string emoji and label for {', '.join(sorted(keys))}, value and gate")
|
||||||
|
return metrics, extras
|
||||||
|
|
||||||
|
|
||||||
|
def gate(args: argparse.Namespace, metrics: dict) -> tuple[str, str, int]:
|
||||||
|
def word(key: str) -> str:
|
||||||
|
return metrics[key]["label"].lower()
|
||||||
|
|
||||||
if args.critical > 0:
|
if args.critical > 0:
|
||||||
return "⛔", "critical security finding: do not commit", 4
|
return "⛔", f"critical {word('security')} finding: do not commit", 4
|
||||||
failed = []
|
failed = []
|
||||||
if args.fix < FIX_MIN:
|
if args.fix < FIX_MIN:
|
||||||
failed.append(f"fix {number(args.fix)} % < {FIX_MIN}")
|
failed.append(f"{word('fix')} {number(args.fix)} % < {FIX_MIN}")
|
||||||
if args.risk > BREAK_MAX:
|
if args.risk > BREAK_MAX:
|
||||||
failed.append(f"break {number(args.risk)} % > {BREAK_MAX}")
|
failed.append(f"{word('break')} {number(args.risk)} % > {BREAK_MAX}")
|
||||||
value, mark = args.security
|
value, mark = args.security
|
||||||
unsafe = f"security {mark}{number(value)} % < {SAFE}" if value < SAFE else None
|
unsafe = f"{word('security')} {mark}{number(value)} % < {SAFE}" if value < SAFE else None
|
||||||
asks = [text for text in (over("cost", args.efficiency), over("time", args.speed), unsafe) if text]
|
asks = [text for text in (over(word("tokens"), args.efficiency), over(word("velocity"), args.speed), unsafe) if text]
|
||||||
ask = f"{', '.join(asks)}: ask the operator before the commit" if asks else ""
|
ask = f"{', '.join(asks)}: ask the operator before the commit" if asks else ""
|
||||||
if failed:
|
if failed:
|
||||||
return "⛔", "; ".join(part for part in (", ".join(failed), ask) if part), 1
|
return "⛔", "; ".join(part for part in (", ".join(failed), ask) if part), 1
|
||||||
if asks:
|
if asks:
|
||||||
return "⚠️", ask, 3
|
return "⚠️", ask, 3
|
||||||
return "✅", f"{FIX_MIN}/{BREAK_MAX}, cost and time <= {COST_MAX}, security {SAFE} %", 0
|
return "✅", f"{FIX_MIN}/{BREAK_MAX}, {word('tokens')} and {word('velocity')} <= {COST_MAX}, {word('security')} {SAFE} %", 0
|
||||||
|
|
||||||
|
|
||||||
def git(repo: str, *args: str) -> str:
|
|
||||||
try:
|
|
||||||
result = subprocess.run(["git", *args], cwd=repo, capture_output=True, text=True, check=False)
|
|
||||||
except OSError as error:
|
|
||||||
print(f"score: git {args[0]} in {repo}: {error}", file=sys.stderr)
|
|
||||||
sys.exit(2)
|
|
||||||
if result.returncode != 0:
|
|
||||||
print(f"score: git {args[0]} in {repo}: {result.stderr.strip()}", file=sys.stderr)
|
|
||||||
sys.exit(2)
|
|
||||||
return result.stdout.strip()
|
|
||||||
|
|
||||||
|
|
||||||
def diffstat(shortstat: str) -> str:
|
def diffstat(shortstat: str) -> str:
|
||||||
@@ -217,38 +240,67 @@ def main() -> int:
|
|||||||
parser.error("--value needs --table, --commit-table or --message-block")
|
parser.error("--value needs --table, --commit-table or --message-block")
|
||||||
|
|
||||||
total = score(args)
|
total = score(args)
|
||||||
emoji, verdict, code = gate(args)
|
|
||||||
value, mark = args.security
|
value, mark = args.security
|
||||||
values = [
|
measured = {
|
||||||
f"{number(args.fix)} %", f"{number(args.risk)} %", shown(args.efficiency), shown(args.speed),
|
"fix": f"{number(args.fix)} %", "break": f"{number(args.risk)} %", "tokens": shown(args.efficiency),
|
||||||
f"{mark}{number(value)} %", str(total), f"{emoji} {verdict}",
|
"velocity": shown(args.speed), "security": f"{mark}{number(value)} %", "score": str(total),
|
||||||
]
|
}
|
||||||
header, row = list(SCORE_HEADER), list(values)
|
try:
|
||||||
|
metrics, extras = labels(set(measured))
|
||||||
|
except LABEL_ERRORS as error:
|
||||||
|
print(f"score: labels {LABELS}: {error!r}", file=sys.stderr)
|
||||||
|
return 4 if args.critical > 0 else 2
|
||||||
|
emoji, verdict, code = gate(args, metrics)
|
||||||
|
verdict_cell = f"{emoji} {verdict}"
|
||||||
|
value_header = f"{extras['value']['emoji']} {extras['value']['label']}"
|
||||||
|
header = [*(f"{metric['emoji']} {metric['label']}" for metric in metrics.values()), extras["gate"]["label"]]
|
||||||
|
row = [*(measured[key] for key in metrics), verdict_cell]
|
||||||
if args.value:
|
if args.value:
|
||||||
header.insert(VALUE_AT, VALUE_HEADER)
|
header.insert(list(metrics).index("tokens") + 1, value_header)
|
||||||
row.insert(VALUE_AT, args.value)
|
row.insert(list(metrics).index("tokens") + 1, args.value)
|
||||||
if args.commit_table:
|
if args.commit_table:
|
||||||
lead, cells = commit_cells(args.commit_table, args.commit)
|
try:
|
||||||
top = git(args.commit_table, "rev-parse", "--show-toplevel")
|
lead, cells = commit_cells(args.commit_table, args.commit)
|
||||||
|
top = git(args.commit_table, "rev-parse", "--show-toplevel")
|
||||||
|
except LookupError as error:
|
||||||
|
print(f"score: {error}", file=sys.stderr)
|
||||||
|
return 4 if code == 4 else 2
|
||||||
print(table([*lead, *header], [*cells, *row]))
|
print(table([*lead, *header], [*cells, *row]))
|
||||||
print(f"\n{WORKTREE}\n\n```text\n{top}\n```")
|
print(f"\n{WORKTREE}\n\n```text\n{top}\n```")
|
||||||
elif args.table:
|
elif args.table:
|
||||||
print(table(header, row))
|
print(table(header, row))
|
||||||
elif args.message_block:
|
elif args.message_block:
|
||||||
worth_line = [f"💰 Value: {args.value}"] if args.value else []
|
lines = [f"{metric['emoji']} {metric['label']}: {measured[key]}" for key, metric in metrics.items()]
|
||||||
print("\n".join((
|
if args.value:
|
||||||
f"🎯 Fix: {values[0]}", f"💥 Break: {values[1]}", f"💸 Cost: {values[2]}",
|
lines.insert(list(metrics).index("score"), f"{value_header}: {args.value}")
|
||||||
f"⏱️ Time: {values[3]}", f"🛡️ Security: {values[4]}", *worth_line,
|
gate_label = extras["gate"]
|
||||||
f"🏆 Score: {total}", f"🚦 Gate: {values[6]}",
|
block = "\n".join((*lines, f"{gate_label['emoji']} {gate_label['label']}: {verdict_cell}"))
|
||||||
)))
|
print(block, flush=True)
|
||||||
|
journal(args, total, block)
|
||||||
else:
|
else:
|
||||||
print(
|
print(" · ".join(f"{metric['emoji']} {metric['label'].lower()} {measured[key]}" for key, metric in metrics.items()))
|
||||||
f"🎯 fix {values[0]} · 💥 break {values[1]} · 💸 cost {values[2]} · "
|
|
||||||
f"⏱️ time {values[3]} · 🛡️ security {values[4]} · 🏆 score {total}"
|
|
||||||
)
|
|
||||||
print(f"{emoji} gate: {verdict}")
|
print(f"{emoji} gate: {verdict}")
|
||||||
return code
|
return code
|
||||||
|
|
||||||
|
|
||||||
|
def plain(value: float | str) -> float | int | None:
|
||||||
|
if not isinstance(value, float):
|
||||||
|
return None
|
||||||
|
return int(value) if value.is_integer() else value
|
||||||
|
|
||||||
|
|
||||||
|
def journal(args: argparse.Namespace, total: int, block: str) -> None:
|
||||||
|
values = {
|
||||||
|
"fix": plain(args.fix), "break": plain(args.risk), "tokens": plain(args.efficiency),
|
||||||
|
"velocity": plain(args.speed), "security": plain(args.security[0]), "score": total,
|
||||||
|
}
|
||||||
|
path = journal_path()
|
||||||
|
try:
|
||||||
|
here = Path.cwd()
|
||||||
|
append(path, [entry(int(time.time()), origin(here), git_read(here, "branch", "--show-current") or None, None, values, block)])
|
||||||
|
except (OSError, ValueError) as error:
|
||||||
|
print(f"score: journal {path}: {getattr(error, 'strerror', None) or error}", file=sys.stderr)
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
sys.exit(main())
|
sys.exit(main())
|
||||||
@@ -4,7 +4,7 @@ description: >
|
|||||||
Wall-clock speed of changed code versus its base commit.
|
Wall-clock speed of changed code versus its base commit.
|
||||||
---
|
---
|
||||||
|
|
||||||
Its ⏱️ time is the share of its base's wall-clock time a change needs:
|
Its ⏱️ velocity is the share of its base's wall-clock time a change needs:
|
||||||
under 100 % it is faster, over 100 % slower. It has no script of its own: one run
|
under 100 % it is faster, over 100 % slower. It has no script of its own: one run
|
||||||
of the `kvb-check-efficiency` measurement yields both values.
|
of the `kvb-check-efficiency` measurement yields both values.
|
||||||
|
|
||||||
@@ -12,7 +12,7 @@ of the `kvb-check-efficiency` measurement yields both values.
|
|||||||
python3 ${CLAUDE_SKILL_DIR}/../kvb-check-efficiency/scripts/measure.py --base <ref> -- <command...>
|
python3 ${CLAUDE_SKILL_DIR}/../kvb-check-efficiency/scripts/measure.py --base <ref> -- <command...>
|
||||||
```
|
```
|
||||||
|
|
||||||
Its `⏱️ time <p> % (wall <b>→<a> s)` compares wall seconds at `<ref>` and
|
Its `⏱️ velocity <p> % (wall <b>→<a> s)` compares wall seconds at `<ref>` and
|
||||||
in the working tree: one run each, or the mean of 5 each when a value
|
in the working tree: one run each, or the mean of 5 each when a value
|
||||||
exceeds 100. The command, the runs, the `queue.py hold` wrapper for heavy
|
exceeds 100. The command, the runs, the `queue.py hold` wrapper for heavy
|
||||||
commands and the exit codes are those of the `kvb-check-efficiency` skill;
|
commands and the exit codes are those of the `kvb-check-efficiency` skill;
|
||||||
@@ -24,6 +24,6 @@ pasting its whole line reports both values at once.
|
|||||||
renders) has no speed either: `--speed -`, and say why.
|
renders) has no speed either: `--speed -`, and say why.
|
||||||
- **Other load**: wall time follows host load. Above 30 % other load the
|
- **Other load**: wall time follows host load. Above 30 % other load the
|
||||||
script falls back to the CPU-time ratio and marks the estimate:
|
script falls back to the CPU-time ratio and marks the estimate:
|
||||||
`⏱️ time ~<p> % (cpu <b>→<a> s)`, ending in
|
`⏱️ velocity ~<p> % (cpu <b>→<a> s)`, ending in
|
||||||
`(estimated: other load <x> %)`. Report the run you
|
`(estimated: other load <x> %)`. Report the run you
|
||||||
made; never rerun to pick the best result.
|
made; never rerun to pick the best result.
|
||||||
@@ -24,7 +24,7 @@ skill's Rules apply here too.
|
|||||||
--task <register task> -- <command>`, which waits for the rule-based
|
--task <register task> -- <command>`, which waits for the rule-based
|
||||||
grant and releases on exit. The fix enters the main line only
|
grant and releases on exit. The fix enters the main line only
|
||||||
at the funnel's bar (the `kvb-org-swarm` skill's: fix confidence ≥ 90 %,
|
at the funnel's bar (the `kvb-org-swarm` skill's: fix confidence ≥ 90 %,
|
||||||
break risk ≤ 5 %, cost and time ≤ 100 % in the `kvb-check-score`
|
break risk ≤ 5 %, tokens and velocity ≤ 100 % in the `kvb-check-score`
|
||||||
block; over 100 %, ask the operator before the commit, naming the cause)
|
block; over 100 %, ask the operator before the commit, naming the cause)
|
||||||
after its independent review; below the bar the cluster ends as a
|
after its independent review; below the bar the cluster ends as a
|
||||||
proposal for the operator, never as a merge. Never mask a failure: no
|
proposal for the operator, never as a merge. Never mask a failure: no
|
||||||
|
|||||||
@@ -16,6 +16,6 @@ checklist (multi-select) with suggestions, free text allowed:
|
|||||||
be changed (files, roles, services); whether new dependencies, toolchains
|
be changed (files, roles, services); whether new dependencies, toolchains
|
||||||
or languages may be introduced; whether infrastructure or CI configuration
|
or languages may be introduced; whether infrastructure or CI configuration
|
||||||
may change; the merge bar (default: the `kvb-org-swarm` skill's bar of fix
|
may change; the merge bar (default: the `kvb-org-swarm` skill's bar of fix
|
||||||
confidence ≥ 90 %, break risk ≤ 5 %, cost and time ≤ 100 %, over
|
confidence ≥ 90 %, break risk ≤ 5 %, tokens and velocity ≤ 100 %, over
|
||||||
100 % only after asking the operator, naming the cause); a time cap per
|
100 % only after asking the operator, naming the cause); a time cap per
|
||||||
delegated agent (default 60 minutes).
|
delegated agent (default 60 minutes).
|
||||||
@@ -11,14 +11,14 @@ description: >
|
|||||||
2. Show in the message right before the `git commit` call, never only after
|
2. Show in the message right before the `git commit` call, never only after
|
||||||
it, the commit table of the kvb-check-score skill (`score.py ...
|
it, the commit table of the kvb-check-score skill (`score.py ...
|
||||||
--commit-table <worktree>`): 📁 worktree, 🌿 branch, 📊 staged diff stat,
|
--commit-table <worktree>`): 📁 worktree, 🌿 branch, 📊 staged diff stat,
|
||||||
then fix, break, cost, time, security, score and gate of the staged
|
then fix, break, tokens, velocity, security, score and gate of the staged
|
||||||
bundle, as the kvb-check-confidence skill produces them: break risk
|
bundle, as the kvb-check-confidence skill produces them: break risk
|
||||||
derived through the kvb-check-break-risk skill, never estimated. Never
|
derived through the kvb-check-break-risk skill, never estimated. Never
|
||||||
print a diff in the chat, only its stat.
|
print a diff in the chat, only its stat.
|
||||||
3. Approval-prompt duty: the commit call's approval prompt (Claude Code: the
|
3. Approval-prompt duty: the commit call's approval prompt (Claude Code: the
|
||||||
Bash `description`; other runtimes: their prompt text) names the worktree
|
Bash `description`; other runtimes: their prompt text) names the worktree
|
||||||
path, the branch, the diff stat and the score values (🎯 fix, 💥 break,
|
path, the branch, the diff stat and the score values (🎯 fix, 💥 break,
|
||||||
💸 cost, ⏱️ time, 🛡️ security, 🏆 score). Missing any item: do not
|
💸 tokens, ⏱️ velocity, 🛡️ security, 🏆 score). Missing any item: do not
|
||||||
send the call. Gate exit 4 (critical security finding) means no commit:
|
send the call. Gate exit 4 (critical security finding) means no commit:
|
||||||
the call is never sent, not even on the operator's word; fix and rescan
|
the call is never sent, not even on the operator's word; fix and rescan
|
||||||
first. Exit 1 (below 90/5) is shown, the operator's word decides; exit 3
|
first. Exit 1 (below 90/5) is shown, the operator's word decides; exit 3
|
||||||
|
|||||||
@@ -24,9 +24,9 @@ confirms it.
|
|||||||
branch fixes, each verified as the `kvb-git-issue-close` skill does.
|
branch fixes, each verified as the `kvb-git-issue-close` skill does.
|
||||||
5. **Confirm** with one `AskUserQuestion`: title (Conventional Commits),
|
5. **Confirm** with one `AskUserQuestion`: title (Conventional Commits),
|
||||||
base, state and labels. The state follows the bar of the `kvb-org-swarm`
|
base, state and labels. The state follows the bar of the `kvb-org-swarm`
|
||||||
skill (fix confidence ≥ 90 %, break risk ≤ 5 %, cost and time
|
skill (fix confidence ≥ 90 %, break risk ≤ 5 %, tokens and velocity
|
||||||
≤ 100 %): ready at or above it, a draft with the residuals in the body
|
≤ 100 %): ready at or above it, a draft with the residuals in the body
|
||||||
below it. A cost or time over 100 % goes into this question with
|
below it. A tokens or velocity value over 100 % goes into this question with
|
||||||
its cause.
|
its cause.
|
||||||
6. **Create**: `gh pr create --base <default> --head <branch> --title
|
6. **Create**: `gh pr create --base <default> --head <branch> --title
|
||||||
"<title>" --body-file <file>`, plus `--draft` below the bar.
|
"<title>" --body-file <file>`, plus `--draft` below the bar.
|
||||||
|
|||||||
@@ -12,7 +12,7 @@ holds:
|
|||||||
failure on it, a red default-branch run or an issue that reproduces there;
|
failure on it, a red default-branch run or an issue that reproduces there;
|
||||||
name the run or issue.
|
name the run or issue.
|
||||||
- **`90/5`**: the pull request meets the bar of the `kvb-org-swarm` skill (fix
|
- **`90/5`**: the pull request meets the bar of the `kvb-org-swarm` skill (fix
|
||||||
confidence ≥ 90 %, break risk ≤ 5 %, cost and time ≤ 100 %; over
|
confidence ≥ 90 %, break risk ≤ 5 %, tokens and velocity ≤ 100 %; over
|
||||||
100 %, ask the operator first, naming the cause), from the
|
100 %, ask the operator first, naming the cause), from the
|
||||||
`kvb-check-score` block in its body or freshly assessed, and only when
|
`kvb-check-score` block in its body or freshly assessed, and only when
|
||||||
`gate.py status` of the `kvb-check-gate` skill prints `gate: green` with
|
`gate.py status` of the `kvb-check-gate` skill prints `gate: green` with
|
||||||
@@ -27,7 +27,7 @@ holds:
|
|||||||
3. A missing label is created once: `gh label create hotfix --color B60205
|
3. A missing label is created once: `gh label create hotfix --color B60205
|
||||||
--description "Fixes a failure on the default branch"`, `gh label create
|
--description "Fixes a failure on the default branch"`, `gh label create
|
||||||
90/5 --color 0E8A16 --description
|
90/5 --color 0E8A16 --description
|
||||||
"Fix confidence >= 90 %, break risk <= 5 %, cost and time <= 100 %"`.
|
"Fix confidence >= 90 %, break risk <= 5 %, tokens and velocity <= 100 %"`.
|
||||||
4. Standalone, confirm the changes with one `AskUserQuestion`; from
|
4. Standalone, confirm the changes with one `AskUserQuestion`; from
|
||||||
`kvb-git-pr-create`, its confirmation covers them.
|
`kvb-git-pr-create`, its confirmation covers them.
|
||||||
5. `gh pr edit <pr> --add-label <l>` or `--remove-label <l>`. A draft that
|
5. `gh pr edit <pr> --add-label <l>` or `--remove-label <l>`. A draft that
|
||||||
|
|||||||
@@ -38,8 +38,8 @@ One table, most-blocking first, one row per item:
|
|||||||
- **State**: `open`, `running`, `blocked`, `done`. `done` means verified (a
|
- **State**: `open`, `running`, `blocked`, `done`. `done` means verified (a
|
||||||
passing run, a green check, a reproduction that now succeeds); work that
|
passing run, a green check, a reproduction that now succeeds); work that
|
||||||
merely looks finished is `open`.
|
merely looks finished is `open`.
|
||||||
- **Score**: the `kvb-check-score` line (🎯 fix, 💥 break, 💸 cost,
|
- **Score**: the `kvb-check-score` line (🎯 fix, 💥 break, 💸 tokens,
|
||||||
⏱️ time, 🏆 score) per the `kvb-check-confidence` skill, only for rows
|
⏱️ velocity, 🏆 score) per the `kvb-check-confidence` skill, only for rows
|
||||||
that changed something; `-` for pure observation rows.
|
that changed something; `-` for pure observation rows.
|
||||||
- **Evidence / next step**: for `done` the strongest proof line (what ran,
|
- **Evidence / next step**: for `done` the strongest proof line (what ran,
|
||||||
what it returned), otherwise the single next action.
|
what it returned), otherwise the single next action.
|
||||||
|
|||||||
@@ -12,7 +12,7 @@ sidetree or leaves as a proposal.
|
|||||||
|
|
||||||
## The bar
|
## The bar
|
||||||
|
|
||||||
- **Fix confidence ≥ 90 %, break risk ≤ 5 %, cost and time ≤ 100 %**,
|
- **Fix confidence ≥ 90 %, break risk ≤ 5 %, tokens and velocity ≤ 100 %**,
|
||||||
all from the `kvb-check-score` block of `kvb-check-confidence`; over 100 %,
|
all from the `kvb-check-score` block of `kvb-check-confidence`; over 100 %,
|
||||||
ask the operator before the commit, naming the cause.
|
ask the operator before the commit, naming the cause.
|
||||||
- The operator may set a different bar. Never lower it to make a change fit.
|
- The operator may set a different bar. Never lower it to make a change fit.
|
||||||
|
|||||||
@@ -0,0 +1,88 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import os
|
||||||
|
import shutil
|
||||||
|
import subprocess
|
||||||
|
import sys
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
REPO_ROOT = Path(__file__).resolve().parents[1]
|
||||||
|
BACKFILL = REPO_ROOT / "skills" / "kvb-check-score" / "scripts" / "backfill.py"
|
||||||
|
EPOCH = 1_760_000_000
|
||||||
|
HANG = 60
|
||||||
|
OLD_BLOCK = "🎯 Fix: 95 %\n💥 Break: 2 %\n💸 Cost: 97 %\n⏱️ Time: –\n🛡️ Security: 100 %\n🏆 Score: 96\n🚦 Gate: ✅ 90/5, cost and time <= 100, security 100 %"
|
||||||
|
NEW_BLOCK = "🎯 Fix: 92 %\n💥 Break: 3 %\n💸 Tokens: 110 %\n⏱️ Velocity: 90 %\n🛡️ Security: 100 %\n💰 Value: 1:20\n🧪 Coverage: 80 %\n🏆 Score: 81\n🚦 Gate: ⚠️ tokens 110 % > 100: ask the operator before the commit"
|
||||||
|
ESTIMATED_BLOCK = "🎯 Fix: 90 %\n💥 Break: 5 %\n💸 Tokens: new\n⏱️ Velocity: 130 %\n🛡️ Security: ~80 %\n🌡️ Heat: 5\n🧪 Coverage: 70 %\n🏆 Score: 52\n🚦 Gate: ⚠️ tokens new: ask the operator before the commit"
|
||||||
|
MALFORMED_BLOCK = "🎯 Fix: 95 %\n💥 Break: 2 %\n🏆 Score: high"
|
||||||
|
TRAILER = "Co-Authored-By: Test <test@example.org>"
|
||||||
|
|
||||||
|
|
||||||
|
def git_env(home: Path) -> dict[str, str]:
|
||||||
|
"""Args:
|
||||||
|
home: a scratch HOME, so no user or system git configuration applies.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
The environment for fixture git calls.
|
||||||
|
"""
|
||||||
|
return {
|
||||||
|
"PATH": os.path.dirname(shutil.which("git")),
|
||||||
|
"HOME": str(home),
|
||||||
|
"GIT_CONFIG_NOSYSTEM": "1",
|
||||||
|
"GIT_AUTHOR_NAME": "Test",
|
||||||
|
"GIT_AUTHOR_EMAIL": "test@example.org",
|
||||||
|
"GIT_COMMITTER_NAME": "Test",
|
||||||
|
"GIT_COMMITTER_EMAIL": "test@example.org",
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
class Fixture:
|
||||||
|
"""Args:
|
||||||
|
base: a scratch directory for upstream.git, the work clone a and the clone b.
|
||||||
|
"""
|
||||||
|
|
||||||
|
def __init__(self, base: Path) -> None:
|
||||||
|
self.base = base
|
||||||
|
self.env = git_env(base)
|
||||||
|
self.upstream = base / "upstream.git"
|
||||||
|
self.a = base / "a"
|
||||||
|
self.b = base / "b"
|
||||||
|
self.step = 0
|
||||||
|
base.mkdir(parents=True)
|
||||||
|
self.git(base, "init", "-q", "--bare", "-b", "main", str(self.upstream))
|
||||||
|
self.a.mkdir()
|
||||||
|
self.git(self.a, "init", "-q", "-b", "main")
|
||||||
|
self.git(self.a, "remote", "add", "origin", str(self.upstream))
|
||||||
|
self.old = self.commit("feat: old labels", OLD_BLOCK)
|
||||||
|
self.git(self.a, "checkout", "-q", "-b", "feature/two")
|
||||||
|
self.new = self.commit("feat: new labels", NEW_BLOCK)
|
||||||
|
self.git(self.a, "checkout", "-q", "main")
|
||||||
|
self.tick()
|
||||||
|
self.git(self.a, "merge", "-q", "--no-ff", "-m", "Merge branch 'feature/two'", "feature/two")
|
||||||
|
self.estimated = self.commit("fix: estimated", ESTIMATED_BLOCK)
|
||||||
|
self.malformed = self.commit("chore: malformed", MALFORMED_BLOCK)
|
||||||
|
self.plain = self.commit("docs: no block", "")
|
||||||
|
self.git(self.a, "push", "-q", "origin", "main", "feature/two")
|
||||||
|
self.git(base, "clone", "-q", str(self.upstream), str(self.b))
|
||||||
|
|
||||||
|
def tick(self) -> None:
|
||||||
|
self.step += 1
|
||||||
|
stamp = f"{EPOCH + self.step * 3600} +0000"
|
||||||
|
self.env["GIT_AUTHOR_DATE"] = stamp
|
||||||
|
self.env["GIT_COMMITTER_DATE"] = stamp
|
||||||
|
|
||||||
|
def git(self, cwd: Path, *args: str) -> str:
|
||||||
|
return subprocess.run(["git", *args], cwd=cwd, env=self.env, capture_output=True, text=True, check=True).stdout.strip()
|
||||||
|
|
||||||
|
def commit(self, subject: str, block: str) -> str:
|
||||||
|
self.tick()
|
||||||
|
messages = [item for part in (subject, block, TRAILER) if part for item in ("-m", part)]
|
||||||
|
self.git(self.a, "commit", "-q", "--allow-empty", *messages)
|
||||||
|
return self.git(self.a, "rev-parse", "HEAD")
|
||||||
|
|
||||||
|
def time_of(self, sha: str) -> int:
|
||||||
|
return int(self.git(self.a, "show", "-s", "--format=%ct", sha))
|
||||||
|
|
||||||
|
|
||||||
|
def backfill(council: Path, *args: str) -> subprocess.CompletedProcess:
|
||||||
|
env = {"PATH": os.environ["PATH"], "AGENTS_COUNCIL_ROOT": str(council)}
|
||||||
|
return subprocess.run([sys.executable, "-B", str(BACKFILL), *args], env=env, capture_output=True, text=True, check=False, timeout=HANG)
|
||||||
@@ -33,8 +33,8 @@ NEW_B = "💰 value ?:? (cost 0/day; pass --value with evidence; kvb-b new: all
|
|||||||
CHEAP = "print('cheap')\n"
|
CHEAP = "print('cheap')\n"
|
||||||
COSTLY = "data = list(range(2_000_000))\nprint(sum(data))\n"
|
COSTLY = "data = list(range(2_000_000))\nprint(sum(data))\n"
|
||||||
LINE = re.compile(
|
LINE = re.compile(
|
||||||
r"^💸 cost (?P<mark>~?)(?P<efficiency>\d+) % \(cpu \d+\.\d\d→\d+\.\d\d s, ram \d+\.\d→\d+\.\d MB\) · "
|
r"^💸 tokens (?P<mark>~?)(?P<efficiency>\d+) % \(cpu \d+\.\d\d→\d+\.\d\d s, ram \d+\.\d→\d+\.\d MB\) · "
|
||||||
r"⏱️ time (?P<speed_mark>~?)(?P<speed>\d+) % \((?P<basis>wall|cpu) \d+\.\d\d→\d+\.\d\d s\)(?P<tail>.*)$"
|
r"⏱️ velocity (?P<speed_mark>~?)(?P<speed>\d+) % \((?P<basis>wall|cpu) \d+\.\d\d→\d+\.\d\d s\)(?P<tail>.*)$"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
@@ -121,7 +121,7 @@ class TestTokens(GitRepo):
|
|||||||
self.write("skills/kvb-a/scripts/tool.py", "print(1)\n" * 50)
|
self.write("skills/kvb-a/scripts/tool.py", "print(1)\n" * 50)
|
||||||
result = self.tokens()
|
result = self.tokens()
|
||||||
self.assertEqual(
|
self.assertEqual(
|
||||||
result.stdout.splitlines(), ["💸 cost 50 % (skills 100→50 tokens; listing 3→3 words)", NO_VALUE]
|
result.stdout.splitlines(), ["💸 tokens 50 % (skills 100→50 tokens; listing 3→3 words)", NO_VALUE]
|
||||||
)
|
)
|
||||||
self.assertEqual(result.returncode, 0, result.stderr)
|
self.assertEqual(result.returncode, 0, result.stderr)
|
||||||
|
|
||||||
@@ -130,7 +130,7 @@ class TestTokens(GitRepo):
|
|||||||
self.write("skills/kvb-b/SKILL.md", SKILL_B.ljust(200, "y"))
|
self.write("skills/kvb-b/SKILL.md", SKILL_B.ljust(200, "y"))
|
||||||
result = self.tokens()
|
result = self.tokens()
|
||||||
self.assertEqual(
|
self.assertEqual(
|
||||||
result.stdout.splitlines(), ["💸 cost 200 % (skills 100→200 tokens; listing 3→5 words)", NEW_B]
|
result.stdout.splitlines(), ["💸 tokens 200 % (skills 100→200 tokens; listing 3→5 words)", NEW_B]
|
||||||
)
|
)
|
||||||
self.assertEqual(result.returncode, 3)
|
self.assertEqual(result.returncode, 3)
|
||||||
result = self.tokens("--json")
|
result = self.tokens("--json")
|
||||||
@@ -157,7 +157,7 @@ class TestTokens(GitRepo):
|
|||||||
def test_only_new_files_print_new(self):
|
def test_only_new_files_print_new(self):
|
||||||
self.write("skills/kvb-b/SKILL.md", SKILL_B.ljust(200, "y"))
|
self.write("skills/kvb-b/SKILL.md", SKILL_B.ljust(200, "y"))
|
||||||
result = self.tokens()
|
result = self.tokens()
|
||||||
self.assertEqual(result.stdout.splitlines(), ["💸 cost new (+50 tokens; listing 3→5 words)", NEW_B])
|
self.assertEqual(result.stdout.splitlines(), ["💸 tokens new (+50 tokens; listing 3→5 words)", NEW_B])
|
||||||
self.assertEqual(result.returncode, 3)
|
self.assertEqual(result.returncode, 3)
|
||||||
self.assertIsNone(json.loads(self.tokens("--json").stdout)["efficiency"])
|
self.assertIsNone(json.loads(self.tokens("--json").stdout)["efficiency"])
|
||||||
|
|
||||||
@@ -165,14 +165,14 @@ class TestTokens(GitRepo):
|
|||||||
self.git("mv", "skills/kvb-a/notes.md", "skills/kvb-a/moved.md")
|
self.git("mv", "skills/kvb-a/notes.md", "skills/kvb-a/moved.md")
|
||||||
result = self.tokens()
|
result = self.tokens()
|
||||||
self.assertEqual(
|
self.assertEqual(
|
||||||
result.stdout.splitlines(), ["💸 cost 100 % (skills 100→100 tokens; listing 3→3 words)", NO_VALUE]
|
result.stdout.splitlines(), ["💸 tokens 100 % (skills 100→100 tokens; listing 3→3 words)", NO_VALUE]
|
||||||
)
|
)
|
||||||
self.assertEqual(result.returncode, 0)
|
self.assertEqual(result.returncode, 0)
|
||||||
|
|
||||||
def test_no_change_is_100(self):
|
def test_no_change_is_100(self):
|
||||||
result = self.tokens()
|
result = self.tokens()
|
||||||
self.assertEqual(
|
self.assertEqual(
|
||||||
result.stdout.splitlines(), ["💸 cost 100 % (skills 0→0 tokens; listing 3→3 words)", NO_VALUE]
|
result.stdout.splitlines(), ["💸 tokens 100 % (skills 0→0 tokens; listing 3→3 words)", NO_VALUE]
|
||||||
)
|
)
|
||||||
self.assertEqual(result.returncode, 0)
|
self.assertEqual(result.returncode, 0)
|
||||||
|
|
||||||
@@ -193,7 +193,7 @@ class TestTokens(GitRepo):
|
|||||||
self.assertEqual(
|
self.assertEqual(
|
||||||
result.stdout.splitlines(),
|
result.stdout.splitlines(),
|
||||||
[
|
[
|
||||||
"💸 cost 869 % (skills 13→113 tokens; listing 3→5 words)",
|
"💸 tokens 869 % (skills 13→113 tokens; listing 3→5 words)",
|
||||||
"💰 value ?:? (cost 203/day; pass --value with evidence)",
|
"💰 value ?:? (cost 203/day; pass --value with evidence)",
|
||||||
],
|
],
|
||||||
)
|
)
|
||||||
@@ -235,6 +235,22 @@ class TestTokens(GitRepo):
|
|||||||
)
|
)
|
||||||
self.assertEqual(result.returncode, 3)
|
self.assertEqual(result.returncode, 3)
|
||||||
|
|
||||||
|
def test_a_malformed_label_file_exits_2_with_one_stderr_line(self):
|
||||||
|
for text in ("{broken", json.dumps({"metrics": {"tokens": {"emoji": 1, "label": "Tokens"}}}), "[]"):
|
||||||
|
with self.subTest(labels=text):
|
||||||
|
labels = self.base / "labels.json"
|
||||||
|
labels.write_text(text, encoding="utf-8")
|
||||||
|
out, err = io.StringIO(), io.StringIO()
|
||||||
|
argv = ["--repo", str(self.repo), "--base", "HEAD", "--transcripts", str(self.base / "projects")]
|
||||||
|
with mock.patch.object(skill_tokens, "LABELS", labels), contextlib.redirect_stdout(out), contextlib.redirect_stderr(err):
|
||||||
|
try:
|
||||||
|
code = skill_tokens.main(argv)
|
||||||
|
except SystemExit as stop:
|
||||||
|
code = stop.code
|
||||||
|
self.assertEqual((code, out.getvalue()), (2, ""))
|
||||||
|
self.assertEqual(len(err.getvalue().splitlines()), 1, err.getvalue())
|
||||||
|
self.assertTrue(err.getvalue().startswith(f"tokens: labels {labels}: "), err.getvalue())
|
||||||
|
|
||||||
def test_usage_errors_exit_2(self):
|
def test_usage_errors_exit_2(self):
|
||||||
self.assertEqual(self.tokens("--value", "5").returncode, 2)
|
self.assertEqual(self.tokens("--value", "5").returncode, 2)
|
||||||
self.assertEqual(self.tokens("--evidence", "why").returncode, 2)
|
self.assertEqual(self.tokens("--evidence", "why").returncode, 2)
|
||||||
@@ -381,6 +397,15 @@ class TestMeasure(GitRepo):
|
|||||||
self.assertEqual(trees, [f"worktree {self.repo.resolve()}"])
|
self.assertEqual(trees, [f"worktree {self.repo.resolve()}"])
|
||||||
self.assertLessEqual(set(Path("/tmp").glob("kvb-measure-*")), self.leftovers)
|
self.assertLessEqual(set(Path("/tmp").glob("kvb-measure-*")), self.leftovers)
|
||||||
|
|
||||||
|
def test_a_malformed_label_file_exits_2_before_any_run(self):
|
||||||
|
labels = self.base / "labels.json"
|
||||||
|
labels.write_text(json.dumps({"metrics": {"tokens": {"emoji": "💸", "label": "Tokens"}}}), encoding="utf-8")
|
||||||
|
with mock.patch.object(measure, "LABELS", labels):
|
||||||
|
code, out, err, samples = self.measured("--base", "HEAD", "--", sys.executable, "work.py")
|
||||||
|
self.assertEqual((code, out, samples), (2, "", 0))
|
||||||
|
self.assertEqual(len(err.splitlines()), 1, err)
|
||||||
|
self.assertTrue(err.startswith(f"measure: labels {labels}: "), err)
|
||||||
|
|
||||||
def test_cheaper_head_stops_after_one_run_each(self):
|
def test_cheaper_head_stops_after_one_run_each(self):
|
||||||
self.write("pkg/work.py", CHEAP)
|
self.write("pkg/work.py", CHEAP)
|
||||||
code, out, err, samples = self.measured("--base", "HEAD", "--", sys.executable, "work.py")
|
code, out, err, samples = self.measured("--base", "HEAD", "--", sys.executable, "work.py")
|
||||||
|
|||||||
@@ -0,0 +1,232 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
|
import subprocess
|
||||||
|
import unittest
|
||||||
|
|
||||||
|
from hub_support import HUB, PLAN, REPO_ROOT
|
||||||
|
|
||||||
|
GANTT = HUB / "templates" / "page-tasks-gantt.js"
|
||||||
|
LABELS = REPO_ROOT / "skills" / "kvb-check-score" / "labels.json"
|
||||||
|
SERIES_PROBE = """
|
||||||
|
const P = require(process.argv[1]);
|
||||||
|
const rows = [
|
||||||
|
{task: "work/a", branch: "feature/a", worktree_branch: ""},
|
||||||
|
{task: "work/b", branch: "-", worktree_branch: "fix/b"},
|
||||||
|
{task: "work/c", branch: "-", worktree_branch: ""},
|
||||||
|
];
|
||||||
|
const values = (fix, score) => ({fix, break: 2, tokens: null, velocity: 130, security: 100, score});
|
||||||
|
const journal = [
|
||||||
|
{time: 300, branch: "feature/a", values: values(95, 90)},
|
||||||
|
{time: 100, branch: "fix/b", values: values(92, null)},
|
||||||
|
{time: 200, branch: "feature/other", values: values(10, 10)},
|
||||||
|
{time: 900, branch: "feature/a", values: values(1, 1)},
|
||||||
|
{time: 150, branch: "", values: values(5, 5)},
|
||||||
|
{time: 250, branch: "-", values: values(6, 6)},
|
||||||
|
];
|
||||||
|
const series = P.overlaySeries(rows, journal, ["fix", "tokens", "score"], {from: 100, to: 800});
|
||||||
|
console.log(JSON.stringify({
|
||||||
|
keys: Object.keys(P),
|
||||||
|
series,
|
||||||
|
none: P.overlaySeries(rows, journal, [], {from: 0, to: 1000}),
|
||||||
|
tops: [P.overlayTop(series), P.overlayTop(P.overlaySeries(rows, journal, ["velocity"], {from: 0, to: 1000})), P.overlayTop([])],
|
||||||
|
labels: [null, "x", {}, {metrics: []}, {metrics: {}}, {metrics: {fix: null}}, {metrics: {fix: {emoji: 1, label: "Fix"}}},
|
||||||
|
{metrics: {fix: {emoji: "🎯", label: "Fix"}, bad: {emoji: "x"}, score: {emoji: "🏆", label: "Score"}}}].map(P.overlayLabels),
|
||||||
|
}));
|
||||||
|
"""
|
||||||
|
BURN_PROBE = """
|
||||||
|
const fs = require("fs");
|
||||||
|
const [source, plan, labelFile] = process.argv.slice(1);
|
||||||
|
const P = require(plan);
|
||||||
|
const labels = JSON.parse(fs.readFileSync(labelFile, "utf8"));
|
||||||
|
const node = (tag, attrs, children) => ({tag, attrs, children: children.flat(Infinity).filter((child) => child !== null && child !== undefined && child !== false), append(...more) { this.children.push(...more); }});
|
||||||
|
const s = (tag, attrs, ...children) => node(tag, attrs, children);
|
||||||
|
const LEAF = /^[A-Za-z]{1,32}\\.[A-Za-z]{1,32}$/u;
|
||||||
|
const METRICS = Object.keys(labels.metrics);
|
||||||
|
const kept = {value: [], listeners: [], updates: 0};
|
||||||
|
const known = (path) => {
|
||||||
|
if (path !== "burn.overlays") {
|
||||||
|
throw new Error(`unknown view path ${path}`);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
const ViewStore = {
|
||||||
|
get: (path) => {
|
||||||
|
known(path);
|
||||||
|
return [...kept.value];
|
||||||
|
},
|
||||||
|
set: (path) => {
|
||||||
|
known(path);
|
||||||
|
throw new Error(`${path} holds a list: change it through update`);
|
||||||
|
},
|
||||||
|
update: (path, change) => {
|
||||||
|
if (!LEAF.test(String(path)) || typeof change !== "function") {
|
||||||
|
throw new Error(`update needs a section.key path and a change function, not ${path}`);
|
||||||
|
}
|
||||||
|
known(path);
|
||||||
|
const next = change([...kept.value]);
|
||||||
|
if (!Array.isArray(next) || !next.every((item) => METRICS.includes(item)) || new Set(next).size !== next.length) {
|
||||||
|
throw new Error(`burn.overlays must list distinct metrics, not ${JSON.stringify(next)}`);
|
||||||
|
}
|
||||||
|
kept.value = next;
|
||||||
|
kept.updates += 1;
|
||||||
|
return Promise.resolve(true);
|
||||||
|
},
|
||||||
|
on: (path, callback) => {
|
||||||
|
known(path);
|
||||||
|
kept.listeners.push([path, callback]);
|
||||||
|
},
|
||||||
|
};
|
||||||
|
const rendered = [];
|
||||||
|
const LegendChips = {render: (series, path, mode) => {
|
||||||
|
if (mode !== "hidden" && mode !== "shown") {
|
||||||
|
throw new Error(`a legend chip mode must be hidden or shown, not ${mode}`);
|
||||||
|
}
|
||||||
|
if (!LEAF.test(String(path)) || !Array.isArray(series) || !series.every((entry) => entry && Object.keys(entry).sort().join() === "id,label,tone" && ["id", "label", "tone"].every((name) => typeof entry[name] === "string"))) {
|
||||||
|
throw new Error("legend chips need a section.key path and series [{id, label, tone}]");
|
||||||
|
}
|
||||||
|
const press = (id) => ViewStore.update(path, (list) => (list.includes(id) ? list.filter((item) => item !== id) : [...list, id]));
|
||||||
|
rendered.push({series, path, mode, press});
|
||||||
|
return node("div", {class: "legend-chips"}, []);
|
||||||
|
}};
|
||||||
|
const refused = (call) => {
|
||||||
|
try {
|
||||||
|
call();
|
||||||
|
return false;
|
||||||
|
} catch {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
const contract = [
|
||||||
|
refused(() => LegendChips.render([{key: "fix", label: "Fix"}], "burn.overlays", "shown")),
|
||||||
|
refused(() => LegendChips.render([{id: "fix", label: "Fix", tone: "k-fix"}], "burn.overlays", "on")),
|
||||||
|
refused(() => LegendChips.render([{id: "fix", label: "Fix", tone: "k-fix"}], "burn.overlays")),
|
||||||
|
refused(() => ViewStore.set("burn.overlays", ["fix"])),
|
||||||
|
refused(() => ViewStore.update("burn.overlays", () => [null])),
|
||||||
|
];
|
||||||
|
const journal = [
|
||||||
|
{time: 600, branch: "feature/a", values: {fix: 95, break: 2, tokens: 97, velocity: null, security: 100, score: 90}},
|
||||||
|
{time: 300, branch: "feature/a", values: {fix: 92, break: 2, tokens: 97, velocity: null, security: 100, score: null}},
|
||||||
|
{time: 400, branch: "feature/other", values: {fix: 10, break: 2, tokens: 97, velocity: null, security: 100, score: 10}},
|
||||||
|
];
|
||||||
|
const fetched = [];
|
||||||
|
const fetch = async (url) => {
|
||||||
|
fetched.push(url);
|
||||||
|
return {ok: true, json: async () => ({since: 0, until: 1000, labels, rows: journal})};
|
||||||
|
};
|
||||||
|
const counts = {draws: 0};
|
||||||
|
const taskView = {sig: "drawn"};
|
||||||
|
const drawTasks = () => {
|
||||||
|
counts.draws += 1;
|
||||||
|
};
|
||||||
|
const build = new Function("Plan", "s", "h", "fixed", "stampOf", "absolute", "ViewStore", "LegendChips", "fetch", "drawTasks", "taskView", "ui", `${fs.readFileSync(source, "utf8")}\\nreturn {burnOverlays, overlaysReady, burnRight};`);
|
||||||
|
const helpers = [P, s, s, (value) => value.toFixed(1), (ts) => new Date(ts * 1000), {format: (date) => date.toISOString()}];
|
||||||
|
const api = build(...helpers, ViewStore, LegendChips, fetch, drawTasks, taskView, {});
|
||||||
|
const inert = build(...helpers, undefined, undefined, fetch, drawTasks, taskView, {});
|
||||||
|
const brokenStore = {get: () => ["fix"], update: () => Promise.resolve(true), on: () => {}};
|
||||||
|
const brokenFetch = async () => ({ok: true, json: async () => ({labels: {}, rows: "x"})});
|
||||||
|
const broken = build(...helpers, brokenStore, LegendChips, brokenFetch, () => {}, {sig: ""}, {});
|
||||||
|
const picked = {rows: [{task: "work/a", branch: "feature/a", worktree_branch: ""}], from: 0, to: 1000};
|
||||||
|
const shape = (svg) => ({
|
||||||
|
overlays: svg.children.filter((child) => child.tag === "g").map((group) => ({cls: group.attrs.class, key: group.attrs["data-overlay"], points: group.children.filter((child) => child.tag === "circle").map((circle) => [circle.attrs.cx, circle.attrs.cy])})),
|
||||||
|
axis: svg.children.filter((child) => child.tag === "text").map((text) => [text.attrs.x, text.children[0]]),
|
||||||
|
});
|
||||||
|
(async () => {
|
||||||
|
const pending = api.burnOverlays(s("svg", {}), picked, (t) => t, api.burnRight());
|
||||||
|
const brokenFirst = broken.burnOverlays(s("svg", {}), picked, (t) => t, broken.burnRight());
|
||||||
|
await new Promise((resolve) => setTimeout(resolve, 0));
|
||||||
|
const brokenSvg = s("svg", {});
|
||||||
|
const brokenAfter = broken.burnOverlays(brokenSvg, picked, (t) => t, broken.burnRight());
|
||||||
|
const brokenView = [brokenFirst, brokenAfter, brokenSvg.children.length, broken.burnRight()];
|
||||||
|
const off = s("svg", {});
|
||||||
|
const rights = {inert: [inert.overlaysReady(), inert.burnRight()], off: api.burnRight()};
|
||||||
|
const offChips = api.burnOverlays(off, picked, (t) => t, rights.off);
|
||||||
|
const stored = [await rendered[0].press("score"), await rendered[0].press("fix")];
|
||||||
|
const pressed = {stored, value: [...kept.value], updates: kept.updates};
|
||||||
|
rights.on = api.burnRight();
|
||||||
|
const on = s("svg", {});
|
||||||
|
api.burnOverlays(on, picked, (t) => t, rights.on);
|
||||||
|
const drawsBefore = counts.draws;
|
||||||
|
taskView.sig = "drawn";
|
||||||
|
kept.listeners[0][1]();
|
||||||
|
console.log(JSON.stringify({
|
||||||
|
ready: api.overlaysReady(), pending, fetched, firstDraws: drawsBefore, listeners: kept.listeners.map(([path]) => path), redrawn: [counts.draws - drawsBefore, taskView.sig],
|
||||||
|
off: shape(off), offChips: offChips.attrs.class, chips: rendered[0], on: shape(on), rights, brokenView, contract, pressed,
|
||||||
|
}));
|
||||||
|
})();
|
||||||
|
"""
|
||||||
|
|
||||||
|
|
||||||
|
class TestOverlaySeries(unittest.TestCase):
|
||||||
|
@classmethod
|
||||||
|
def setUpClass(cls):
|
||||||
|
done = subprocess.run(["node", "-e", SERIES_PROBE, str(PLAN)], capture_output=True, text=True, timeout=60, check=True)
|
||||||
|
cls.data = json.loads(done.stdout)
|
||||||
|
|
||||||
|
def test_only_rows_of_the_scope_branches_within_the_span_are_drawn_in_the_given_order(self):
|
||||||
|
self.assertIn("overlaySeries", self.data["keys"])
|
||||||
|
self.assertNotIn("OVERLAYS", self.data["keys"])
|
||||||
|
self.assertEqual(self.data["series"], [
|
||||||
|
{"key": "fix", "points": [{"t": 100, "value": 92, "branch": "fix/b"}, {"t": 300, "value": 95, "branch": "feature/a"}]},
|
||||||
|
{"key": "tokens", "points": []},
|
||||||
|
{"key": "score", "points": [{"t": 300, "value": 90, "branch": "feature/a"}]},
|
||||||
|
])
|
||||||
|
self.assertEqual(self.data["none"], [])
|
||||||
|
|
||||||
|
def test_the_right_axis_runs_to_100_or_the_largest_value_above_it(self):
|
||||||
|
self.assertEqual(self.data["tops"], [100, 130, 100])
|
||||||
|
|
||||||
|
def test_labels_of_another_shape_switch_the_overlays_off(self):
|
||||||
|
self.assertEqual(self.data["labels"], [None, None, None, None, None, None, None, {"fix": "🎯 Fix", "score": "🏆 Score"}])
|
||||||
|
|
||||||
|
|
||||||
|
class TestBurnOverlays(unittest.TestCase):
|
||||||
|
@classmethod
|
||||||
|
def setUpClass(cls):
|
||||||
|
done = subprocess.run(["node", "-e", BURN_PROBE, str(GANTT), str(PLAN), str(LABELS)], capture_output=True, text=True, timeout=60, check=True)
|
||||||
|
cls.data = json.loads(done.stdout)
|
||||||
|
|
||||||
|
def test_six_metric_chips_all_off_on_first_load_keyed_burn_overlays(self):
|
||||||
|
data = self.data
|
||||||
|
self.assertEqual((data["ready"], data["pending"], data["fetched"], data["firstDraws"]), (True, None, ["/api/scores?from=0&to=1000"], 1))
|
||||||
|
self.assertEqual((data["chips"]["path"], data["chips"]["mode"]), ("burn.overlays", "shown"))
|
||||||
|
self.assertEqual(data["chips"]["series"], [
|
||||||
|
{"id": key, "label": label, "tone": f"k-{key}"}
|
||||||
|
for key, label in (("fix", "🎯 Fix"), ("break", "💥 Break"), ("tokens", "💸 Tokens"), ("velocity", "⏱️ Velocity"), ("security", "🛡️ Security"), ("score", "🏆 Score"))
|
||||||
|
])
|
||||||
|
self.assertEqual((data["off"], data["offChips"]), ({"overlays": [], "axis": []}, "legend-chips"))
|
||||||
|
|
||||||
|
def test_the_contract_stub_refuses_another_shape_mode_or_write(self):
|
||||||
|
self.assertEqual(self.data["contract"], [True] * 5)
|
||||||
|
|
||||||
|
def test_a_chip_press_stores_its_metric_through_update(self):
|
||||||
|
self.assertEqual(self.data["pressed"], {"stored": [True, True], "value": ["score", "fix"], "updates": 2})
|
||||||
|
|
||||||
|
def test_labels_of_another_shape_draw_no_overlay_and_never_throw(self):
|
||||||
|
self.assertEqual(self.data["brokenView"], [None, None, 0, 24])
|
||||||
|
|
||||||
|
def test_the_right_margin_widens_only_while_an_overlay_is_on(self):
|
||||||
|
self.assertEqual(self.data["rights"], {"inert": [False, 24], "off": 24, "on": 56})
|
||||||
|
|
||||||
|
def test_a_switched_on_chip_draws_its_series_on_the_right_axis(self):
|
||||||
|
on = self.data["on"]
|
||||||
|
self.assertEqual(on["axis"], [[850, "0 %"], [850, "50 %"], [850, "100 %"]])
|
||||||
|
self.assertEqual(on["overlays"], [
|
||||||
|
{"cls": "burn-overlay k-fix", "key": "fix", "points": [["300.0", "42.4"], ["600.0", "34.0"]]},
|
||||||
|
{"cls": "burn-overlay k-score", "key": "score", "points": [["600.0", "48.0"]]},
|
||||||
|
])
|
||||||
|
|
||||||
|
def test_a_view_store_change_redraws_the_burndown(self):
|
||||||
|
self.assertEqual(self.data["listeners"], ["burn.overlays"])
|
||||||
|
self.assertEqual(self.data["redrawn"], [1, ""])
|
||||||
|
|
||||||
|
|
||||||
|
class TestOverlayTones(unittest.TestCase):
|
||||||
|
def test_every_metric_chip_has_a_tone_colour(self):
|
||||||
|
css = (HUB / "templates" / "style-tasks.css").read_text(encoding="utf-8")
|
||||||
|
for key in json.loads(LABELS.read_text(encoding="utf-8"))["metrics"]:
|
||||||
|
with self.subTest(key=key):
|
||||||
|
self.assertRegex(css, rf"\.burn-box \.legend-chip\.k-{key}\s*\{{\s*--c: var\(--[a-z]+\); \}}")
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
@@ -0,0 +1,60 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
|
import unittest
|
||||||
|
|
||||||
|
from hub_case import HubCase
|
||||||
|
from score_fixture import REPO_ROOT, Fixture, backfill
|
||||||
|
|
||||||
|
LABELS = REPO_ROOT / "skills" / "kvb-check-score" / "labels.json"
|
||||||
|
T = 1_760_000_000
|
||||||
|
|
||||||
|
|
||||||
|
class TestHubScores(HubCase):
|
||||||
|
def fetch(self, query: str) -> tuple[int, bytes]:
|
||||||
|
status, _, body = self.get(f"/api/scores?{query}")
|
||||||
|
return status, body
|
||||||
|
|
||||||
|
def test_the_hub_serves_the_backfilled_journal_rows_of_a_window(self):
|
||||||
|
fixture = Fixture(self.council.parent / "fixture")
|
||||||
|
done = backfill(self.council, "--repo", str(fixture.a))
|
||||||
|
self.assertEqual(done.returncode, 0, done.stderr)
|
||||||
|
begin, end = fixture.time_of(fixture.new), fixture.time_of(fixture.estimated)
|
||||||
|
status, body = self.fetch(f"from={begin}&to={end}")
|
||||||
|
self.assertEqual(status, 200, body)
|
||||||
|
found = json.loads(body)
|
||||||
|
self.assertEqual((found["since"], found["until"]), (begin, end))
|
||||||
|
self.assertEqual(found["labels"], json.loads(LABELS.read_text(encoding="utf-8")))
|
||||||
|
self.assertEqual([(row["sha"], row["branch"]) for row in found["rows"]], [(fixture.new, "feature/two"), (fixture.estimated, "main")])
|
||||||
|
self.assertEqual([row["values"]["score"] for row in found["rows"]], [81, 52])
|
||||||
|
|
||||||
|
def test_broken_lines_and_rows_without_values_are_left_out_in_time_order(self):
|
||||||
|
self.council.mkdir(exist_ok=True)
|
||||||
|
late = {"time": T, "branch": "feature/a", "values": {"score": 90}}
|
||||||
|
early = {"time": T - 10, "branch": "feature/a", "values": {"score": 80}}
|
||||||
|
lines = [json.dumps(late), "{broken", json.dumps({"time": T}), json.dumps({"time": "x", "values": {}}), json.dumps(early), json.dumps({**late, "time": T + 500})]
|
||||||
|
(self.council / "scores.jsonl").write_text("\n".join(lines) + "\n", encoding="utf-8")
|
||||||
|
status, body = self.fetch(f"from={T - 100}&to={T + 100}")
|
||||||
|
self.assertEqual(status, 200, body)
|
||||||
|
self.assertEqual(json.loads(body)["rows"], [early, late])
|
||||||
|
self.assertEqual(self.fetch(f"from={T + 10}&to={T}")[0], 400)
|
||||||
|
|
||||||
|
def test_a_label_file_of_another_shape_is_served_as_null(self):
|
||||||
|
skills = self.council.parent / "skills"
|
||||||
|
labels = skills / "kvb-check-score" / "labels.json"
|
||||||
|
labels.parent.mkdir(parents=True)
|
||||||
|
self.scripts.skills = skills
|
||||||
|
good = {"metrics": {"fix": {"emoji": "🎯", "label": "Fix", "former": []}}}
|
||||||
|
for text, served in (("{}", None), ('{"metrics": []}', None), ('{"metrics": {}}', None), ('{"metrics": {"fix": {"emoji": 1, "label": "Fix"}}}', None), ("{broken", None), (json.dumps(good), good)):
|
||||||
|
with self.subTest(labels=text):
|
||||||
|
labels.write_text(text, encoding="utf-8")
|
||||||
|
status, body = self.fetch(f"from={T}&to={T + 1}")
|
||||||
|
self.assertEqual((status, json.loads(body)["labels"]), (200, served))
|
||||||
|
|
||||||
|
def test_a_missing_journal_answers_no_rows(self):
|
||||||
|
status, body = self.fetch(f"from={T}&to={T + 1}")
|
||||||
|
self.assertEqual((status, json.loads(body)["rows"]), (200, []))
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
+25
-24
@@ -12,11 +12,12 @@ from pathlib import Path
|
|||||||
|
|
||||||
REPO_ROOT = Path(__file__).resolve().parents[1]
|
REPO_ROOT = Path(__file__).resolve().parents[1]
|
||||||
SCORE = REPO_ROOT / "skills" / "kvb-check-score" / "scripts" / "score.py"
|
SCORE = REPO_ROOT / "skills" / "kvb-check-score" / "scripts" / "score.py"
|
||||||
PASS = "✅ gate: 90/5, cost and time <= 100, security 100 %"
|
PASS = "✅ gate: 90/5, tokens and velocity <= 100, security 100 %"
|
||||||
BASE = {"fix": "95", "break": "2", "efficiency": "100", "speed": "100", "security": "100", "critical": "0"}
|
BASE = {"fix": "95", "break": "2", "efficiency": "100", "speed": "100", "security": "100", "critical": "0"}
|
||||||
HEADER = "| 🎯 Fix | 💥 Break | 💸 Cost | ⏱️ Time | 🛡️ Security | 🏆 Score | Gate |"
|
HEADER = "| 🎯 Fix | 💥 Break | 💸 Tokens | ⏱️ Velocity | 🛡️ Security | 🏆 Score | Gate |"
|
||||||
ASK = ": ask the operator before the commit"
|
ASK = ": ask the operator before the commit"
|
||||||
ENV = {"PATH": os.environ["PATH"]}
|
COUNCIL = tempfile.TemporaryDirectory()
|
||||||
|
ENV = {"PATH": os.environ["PATH"], "AGENTS_COUNCIL_ROOT": COUNCIL.name}
|
||||||
|
|
||||||
|
|
||||||
def flags(values: dict[str, str]) -> list[str]:
|
def flags(values: dict[str, str]) -> list[str]:
|
||||||
@@ -37,12 +38,12 @@ class TestScore(unittest.TestCase):
|
|||||||
def test_formula_and_line_format(self):
|
def test_formula_and_line_format(self):
|
||||||
self.assertEqual(
|
self.assertEqual(
|
||||||
self._score(fix="95", **{"break": "4"}),
|
self._score(fix="95", **{"break": "4"}),
|
||||||
(["🎯 fix 95 % · 💥 break 4 % · 💸 cost 100 % · ⏱️ time 100 % · 🛡️ security 100 % · 🏆 score 91",
|
(["🎯 fix 95 % · 💥 break 4 % · 💸 tokens 100 % · ⏱️ velocity 100 % · 🛡️ security 100 % · 🏆 score 91",
|
||||||
PASS], 0),
|
PASS], 0),
|
||||||
)
|
)
|
||||||
lines, code = self._score(efficiency="97", speed="-")
|
lines, code = self._score(efficiency="97", speed="-")
|
||||||
self.assertEqual(
|
self.assertEqual(
|
||||||
lines[0], "🎯 fix 95 % · 💥 break 2 % · 💸 cost 97 % · ⏱️ time – · 🛡️ security 100 % · 🏆 score 96"
|
lines[0], "🎯 fix 95 % · 💥 break 2 % · 💸 tokens 97 % · ⏱️ velocity – · 🛡️ security 100 % · 🏆 score 96"
|
||||||
)
|
)
|
||||||
self.assertEqual(code, 0)
|
self.assertEqual(code, 0)
|
||||||
self.assertEqual(self._score(fix="92.5")[0][0].split(" · ")[0], "🎯 fix 92.5 %")
|
self.assertEqual(self._score(fix="92.5")[0][0].split(" · ")[0], "🎯 fix 92.5 %")
|
||||||
@@ -51,14 +52,14 @@ class TestScore(unittest.TestCase):
|
|||||||
lines, code = self._score(fix="92", efficiency="-", speed="-", **{"break": "3"})
|
lines, code = self._score(fix="92", efficiency="-", speed="-", **{"break": "3"})
|
||||||
self.assertEqual(
|
self.assertEqual(
|
||||||
(lines, code),
|
(lines, code),
|
||||||
(["🎯 fix 92 % · 💥 break 3 % · 💸 cost – · ⏱️ time – · 🛡️ security 100 % · 🏆 score 89", PASS], 0),
|
(["🎯 fix 92 % · 💥 break 3 % · 💸 tokens – · ⏱️ velocity – · 🛡️ security 100 % · 🏆 score 89", PASS], 0),
|
||||||
)
|
)
|
||||||
|
|
||||||
def test_security_multiplies_and_asks_below_100(self):
|
def test_security_multiplies_and_asks_below_100(self):
|
||||||
lines, code = self._score(fix="100", efficiency="-", speed="-", security="75", **{"break": "0"})
|
lines, code = self._score(fix="100", efficiency="-", speed="-", security="75", **{"break": "0"})
|
||||||
self.assertEqual(
|
self.assertEqual(
|
||||||
(lines, code),
|
(lines, code),
|
||||||
(["🎯 fix 100 % · 💥 break 0 % · 💸 cost – · ⏱️ time – · 🛡️ security 75 % · 🏆 score 75",
|
(["🎯 fix 100 % · 💥 break 0 % · 💸 tokens – · ⏱️ velocity – · 🛡️ security 75 % · 🏆 score 75",
|
||||||
f"⚠️ gate: security 75 % < 100{ASK}"], 3),
|
f"⚠️ gate: security 75 % < 100{ASK}"], 3),
|
||||||
)
|
)
|
||||||
lines, code = self._score(security="~80")
|
lines, code = self._score(security="~80")
|
||||||
@@ -94,7 +95,7 @@ class TestScore(unittest.TestCase):
|
|||||||
({"fix": "80", "break": "10"}, "⛔ gate: fix 80 % < 90, break 10 % > 5"),
|
({"fix": "80", "break": "10"}, "⛔ gate: fix 80 % < 90, break 10 % > 5"),
|
||||||
(
|
(
|
||||||
{"fix": "85", "efficiency": "150", "speed": "new", "security": "50"},
|
{"fix": "85", "efficiency": "150", "speed": "new", "security": "50"},
|
||||||
f"⛔ gate: fix 85 % < 90; cost 150 % > 100, time new, security 50 % < 100{ASK}",
|
f"⛔ gate: fix 85 % < 90; tokens 150 % > 100, velocity new, security 50 % < 100{ASK}",
|
||||||
),
|
),
|
||||||
):
|
):
|
||||||
with self.subTest(values=values):
|
with self.subTest(values=values):
|
||||||
@@ -103,13 +104,13 @@ class TestScore(unittest.TestCase):
|
|||||||
|
|
||||||
def test_costs_ask_the_operator_exit_3(self):
|
def test_costs_ask_the_operator_exit_3(self):
|
||||||
for values, verdict in (
|
for values, verdict in (
|
||||||
({"efficiency": "120"}, f"⚠️ gate: cost 120 % > 100{ASK}"),
|
({"efficiency": "120"}, f"⚠️ gate: tokens 120 % > 100{ASK}"),
|
||||||
({"efficiency": "-", "speed": "130"}, f"⚠️ gate: time 130 % > 100{ASK}"),
|
({"efficiency": "-", "speed": "130"}, f"⚠️ gate: velocity 130 % > 100{ASK}"),
|
||||||
({"efficiency": "100.5", "speed": "-"}, f"⚠️ gate: cost 100.5 % > 100{ASK}"),
|
({"efficiency": "100.5", "speed": "-"}, f"⚠️ gate: tokens 100.5 % > 100{ASK}"),
|
||||||
({"efficiency": "new", "speed": "-"}, f"⚠️ gate: cost new{ASK}"),
|
({"efficiency": "new", "speed": "-"}, f"⚠️ gate: tokens new{ASK}"),
|
||||||
(
|
(
|
||||||
{"efficiency": "120", "speed": "130", "security": "90"},
|
{"efficiency": "120", "speed": "130", "security": "90"},
|
||||||
f"⚠️ gate: cost 120 % > 100, time 130 % > 100, security 90 % < 100{ASK}",
|
f"⚠️ gate: tokens 120 % > 100, velocity 130 % > 100, security 90 % < 100{ASK}",
|
||||||
),
|
),
|
||||||
):
|
):
|
||||||
with self.subTest(values=values):
|
with self.subTest(values=values):
|
||||||
@@ -128,7 +129,7 @@ class TestScore(unittest.TestCase):
|
|||||||
def test_new_counts_as_100_in_the_score(self):
|
def test_new_counts_as_100_in_the_score(self):
|
||||||
lines, _ = self._score(efficiency="new", speed="-", **{"break": "0"})
|
lines, _ = self._score(efficiency="new", speed="-", **{"break": "0"})
|
||||||
self.assertEqual(
|
self.assertEqual(
|
||||||
lines[0], "🎯 fix 95 % · 💥 break 0 % · 💸 cost new · ⏱️ time – · 🛡️ security 100 % · 🏆 score 95"
|
lines[0], "🎯 fix 95 % · 💥 break 0 % · 💸 tokens new · ⏱️ velocity – · 🛡️ security 100 % · 🏆 score 95"
|
||||||
)
|
)
|
||||||
|
|
||||||
def test_table(self):
|
def test_table(self):
|
||||||
@@ -138,7 +139,7 @@ class TestScore(unittest.TestCase):
|
|||||||
[
|
[
|
||||||
HEADER,
|
HEADER,
|
||||||
"| --- | --- | --- | --- | --- | --- | --- |",
|
"| --- | --- | --- | --- | --- | --- | --- |",
|
||||||
"| 95 % | 2 % | 97 % | – | 100 % | 96 | ✅ 90/5, cost and time <= 100, security 100 % |",
|
"| 95 % | 2 % | 97 % | – | 100 % | 96 | ✅ 90/5, tokens and velocity <= 100, security 100 % |",
|
||||||
],
|
],
|
||||||
)
|
)
|
||||||
self.assertEqual(result.returncode, 0)
|
self.assertEqual(result.returncode, 0)
|
||||||
@@ -152,8 +153,8 @@ class TestScore(unittest.TestCase):
|
|||||||
(result.stdout.splitlines(), result.returncode),
|
(result.stdout.splitlines(), result.returncode),
|
||||||
(
|
(
|
||||||
[
|
[
|
||||||
"🎯 Fix: 95 %", "💥 Break: 2 %", "💸 Cost: 97 %", "⏱️ Time: –", "🛡️ Security: 100 %",
|
"🎯 Fix: 95 %", "💥 Break: 2 %", "💸 Tokens: 97 %", "⏱️ Velocity: –", "🛡️ Security: 100 %",
|
||||||
"🏆 Score: 96", "🚦 Gate: ✅ 90/5, cost and time <= 100, security 100 %",
|
"🏆 Score: 96", "🚦 Gate: ✅ 90/5, tokens and velocity <= 100, security 100 %",
|
||||||
],
|
],
|
||||||
0,
|
0,
|
||||||
),
|
),
|
||||||
@@ -165,7 +166,7 @@ class TestScore(unittest.TestCase):
|
|||||||
(result.stdout.splitlines(), result.returncode),
|
(result.stdout.splitlines(), result.returncode),
|
||||||
(
|
(
|
||||||
[
|
[
|
||||||
"🎯 Fix: 95 %", "💥 Break: 2 %", "💸 Cost: 100 %", "⏱️ Time: 100 %", "🛡️ Security: ~80 %",
|
"🎯 Fix: 95 %", "💥 Break: 2 %", "💸 Tokens: 100 %", "⏱️ Velocity: 100 %", "🛡️ Security: ~80 %",
|
||||||
"💰 Value: 1:20", "🏆 Score: 74", f"🚦 Gate: ⚠️ security ~80 % < 100{ASK}",
|
"💰 Value: 1:20", "🏆 Score: 74", f"🚦 Gate: ⚠️ security ~80 % < 100{ASK}",
|
||||||
],
|
],
|
||||||
3,
|
3,
|
||||||
@@ -188,9 +189,9 @@ class TestScore(unittest.TestCase):
|
|||||||
|
|
||||||
def test_value_column_changes_neither_score_nor_gate(self):
|
def test_value_column_changes_neither_score_nor_gate(self):
|
||||||
for value, verdict, code in (
|
for value, verdict, code in (
|
||||||
("1:20", "✅ 90/5, cost and time <= 100, security 100 %", 0),
|
("1:20", "✅ 90/5, tokens and velocity <= 100, security 100 %", 0),
|
||||||
("4.1:1", "✅ 90/5, cost and time <= 100, security 100 %", 0),
|
("4.1:1", "✅ 90/5, tokens and velocity <= 100, security 100 %", 0),
|
||||||
("?", "⚠️ cost 120 % > 100" + ASK, 3),
|
("?", "⚠️ tokens 120 % > 100" + ASK, 3),
|
||||||
):
|
):
|
||||||
with self.subTest(value=value):
|
with self.subTest(value=value):
|
||||||
efficiency = "97" if code == 0 else "120"
|
efficiency = "97" if code == 0 else "120"
|
||||||
@@ -201,7 +202,7 @@ class TestScore(unittest.TestCase):
|
|||||||
self.assertEqual(
|
self.assertEqual(
|
||||||
result.stdout.splitlines(),
|
result.stdout.splitlines(),
|
||||||
[
|
[
|
||||||
HEADER.replace("💸 Cost |", "💸 Cost | 💰 Value |"),
|
HEADER.replace("💸 Tokens |", "💸 Tokens | 💰 Value |"),
|
||||||
"|" + " --- |" * 8,
|
"|" + " --- |" * 8,
|
||||||
f"| 95 % | 2 % | {efficiency} % | {value} | – | 100 % | {score} | {verdict} |",
|
f"| 95 % | 2 % | {efficiency} % | {value} | – | 100 % | {score} | {verdict} |",
|
||||||
],
|
],
|
||||||
@@ -236,7 +237,7 @@ class TestScore(unittest.TestCase):
|
|||||||
(repo / "b.txt").write_text("three\nfour\n", encoding="utf-8")
|
(repo / "b.txt").write_text("three\nfour\n", encoding="utf-8")
|
||||||
git("add", "a.txt", "b.txt")
|
git("add", "a.txt", "b.txt")
|
||||||
head = ["🌿 Branch", "📊 Diff stat"]
|
head = ["🌿 Branch", "📊 Diff stat"]
|
||||||
score = "| 95 % | 2 % | 100 % | 100 % | 100 % | 93 | ✅ 90/5, cost and time <= 100, security 100 % |"
|
score = "| 95 % | 2 % | 100 % | 100 % | 100 % | 93 | ✅ 90/5, tokens and velocity <= 100, security 100 % |"
|
||||||
worktree = ["", "📁 Worktree:", "", "```text", str(repo.resolve()), "```"]
|
worktree = ["", "📁 Worktree:", "", "```text", str(repo.resolve()), "```"]
|
||||||
|
|
||||||
result = self._run(*flags(BASE), "--commit-table", str(repo), cwd=repo)
|
result = self._run(*flags(BASE), "--commit-table", str(repo), cwd=repo)
|
||||||
@@ -259,7 +260,7 @@ class TestScore(unittest.TestCase):
|
|||||||
result = self._run(*flags(BASE), "--commit-table", str(repo), "--commit", "HEAD", "--value", "?", cwd=repo)
|
result = self._run(*flags(BASE), "--commit-table", str(repo), "--commit", "HEAD", "--value", "?", cwd=repo)
|
||||||
self.assertEqual(
|
self.assertEqual(
|
||||||
result.stdout.splitlines()[0],
|
result.stdout.splitlines()[0],
|
||||||
"| " + " | ".join([*head, "🔖 Commit"]) + " " + HEADER.replace("💸 Cost |", "💸 Cost | 💰 Value |"),
|
"| " + " | ".join([*head, "🔖 Commit"]) + " " + HEADER.replace("💸 Tokens |", "💸 Tokens | 💰 Value |"),
|
||||||
)
|
)
|
||||||
self.assertEqual(
|
self.assertEqual(
|
||||||
result.stdout.splitlines()[2],
|
result.stdout.splitlines()[2],
|
||||||
|
|||||||
@@ -0,0 +1,165 @@
|
|||||||
|
"""Exercise backfill.py of kvb-check-score over a fixture repository: rows, idempotence, determinism, labels, clones."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import shutil
|
||||||
|
import subprocess
|
||||||
|
import sys
|
||||||
|
import tempfile
|
||||||
|
import unittest
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
from score_fixture import (
|
||||||
|
BACKFILL,
|
||||||
|
ESTIMATED_BLOCK,
|
||||||
|
HANG,
|
||||||
|
NEW_BLOCK,
|
||||||
|
OLD_BLOCK,
|
||||||
|
REPO_ROOT,
|
||||||
|
Fixture,
|
||||||
|
backfill,
|
||||||
|
)
|
||||||
|
|
||||||
|
JOURNAL_NAME = "scores.jsonl"
|
||||||
|
LABELS = REPO_ROOT / "skills" / "kvb-check-score" / "labels.json"
|
||||||
|
UNKNOWN = ["backfill: unknown label Coverage, skipped", "backfill: unknown label Heat, skipped"]
|
||||||
|
|
||||||
|
|
||||||
|
class TestBackfill(unittest.TestCase):
|
||||||
|
@classmethod
|
||||||
|
def setUpClass(cls):
|
||||||
|
cls.tmp = tempfile.TemporaryDirectory()
|
||||||
|
cls.fixture = Fixture(Path(cls.tmp.name) / "fixture")
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def tearDownClass(cls):
|
||||||
|
cls.tmp.cleanup()
|
||||||
|
|
||||||
|
def setUp(self):
|
||||||
|
scratch = tempfile.TemporaryDirectory()
|
||||||
|
self.addCleanup(scratch.cleanup)
|
||||||
|
self.scratch = Path(scratch.name)
|
||||||
|
|
||||||
|
def council(self, name: str) -> Path:
|
||||||
|
return self.scratch / name
|
||||||
|
|
||||||
|
def rows(self, council: Path) -> list[dict]:
|
||||||
|
return [json.loads(line) for line in (council / JOURNAL_NAME).read_text(encoding="utf-8").splitlines()]
|
||||||
|
|
||||||
|
def test_three_commits_with_blocks_give_three_rows_then_none(self):
|
||||||
|
council = self.council("one")
|
||||||
|
first = backfill(council, "--repo", str(self.fixture.a))
|
||||||
|
self.assertEqual(first.returncode, 0, first.stderr)
|
||||||
|
self.assertEqual(first.stdout, f"{self.fixture.a.resolve()}: 3 blocks, 3 rows added\n")
|
||||||
|
written = (council / JOURNAL_NAME).read_bytes()
|
||||||
|
second = backfill(council, "--repo", str(self.fixture.a))
|
||||||
|
self.assertEqual((second.returncode, second.stdout), (0, f"{self.fixture.a.resolve()}: 3 blocks, 0 rows added\n"))
|
||||||
|
self.assertEqual((council / JOURNAL_NAME).read_bytes(), written)
|
||||||
|
|
||||||
|
def test_two_runs_from_an_empty_journal_give_byte_identical_journals(self):
|
||||||
|
one, two = self.council("one"), self.council("two")
|
||||||
|
self.assertEqual(backfill(one, "--repo", str(self.fixture.a)).returncode, 0)
|
||||||
|
self.assertEqual(backfill(two, "--repo", str(self.fixture.a)).returncode, 0)
|
||||||
|
self.assertEqual((one / JOURNAL_NAME).read_bytes(), (two / JOURNAL_NAME).read_bytes())
|
||||||
|
|
||||||
|
def test_old_new_and_estimated_blocks_parse_and_a_malformed_one_is_skipped(self):
|
||||||
|
council = self.council("one")
|
||||||
|
backfill(council, "--repo", str(self.fixture.a))
|
||||||
|
rows = self.rows(council)
|
||||||
|
fixture = self.fixture
|
||||||
|
self.assertEqual([row["sha"] for row in rows], [fixture.old, fixture.new, fixture.estimated])
|
||||||
|
self.assertEqual([row["branch"] for row in rows], ["main", "feature/two", "main"])
|
||||||
|
self.assertEqual([row["time"] for row in rows], [fixture.time_of(sha) for sha in (fixture.old, fixture.new, fixture.estimated)])
|
||||||
|
self.assertEqual([row["values"] for row in rows], [
|
||||||
|
{"fix": 95, "break": 2, "tokens": 97, "velocity": None, "security": 100, "score": 96},
|
||||||
|
{"fix": 92, "break": 3, "tokens": 110, "velocity": 90, "security": 100, "score": 81},
|
||||||
|
{"fix": 90, "break": 5, "tokens": None, "velocity": 130, "security": 80, "score": 52},
|
||||||
|
])
|
||||||
|
self.assertEqual([row["digest"] for row in rows], [hashlib.sha256(block.encode("utf-8")).hexdigest() for block in (OLD_BLOCK, NEW_BLOCK, ESTIMATED_BLOCK)])
|
||||||
|
self.assertEqual({(row["remote"], row["path"]) for row in rows}, {(str(fixture.upstream), str(fixture.a.resolve()))})
|
||||||
|
|
||||||
|
def test_an_unknown_label_is_reported_once_per_label_and_skipped(self):
|
||||||
|
done = backfill(self.council("one"), "--repo", str(self.fixture.a))
|
||||||
|
self.assertEqual(done.stderr.splitlines(), UNKNOWN)
|
||||||
|
for row in self.rows(self.council("one")):
|
||||||
|
self.assertEqual(list(row["values"]), ["fix", "break", "tokens", "velocity", "security", "score"])
|
||||||
|
|
||||||
|
def test_old_labels_map_through_the_label_file(self):
|
||||||
|
labels = json.loads(LABELS.read_text(encoding="utf-8"))
|
||||||
|
labels["metrics"]["tokens"]["former"] = []
|
||||||
|
file = self.scratch / "labels.json"
|
||||||
|
file.write_text(json.dumps(labels, ensure_ascii=False), encoding="utf-8")
|
||||||
|
done = backfill(self.council("one"), "--repo", str(self.fixture.a), "--labels", str(file))
|
||||||
|
self.assertIn("backfill: unknown label Cost, skipped", done.stderr.splitlines())
|
||||||
|
self.assertIsNone(self.rows(self.council("one"))[0]["values"]["tokens"])
|
||||||
|
|
||||||
|
def test_a_clone_at_another_path_yields_the_same_keys(self):
|
||||||
|
one, two = self.council("one"), self.council("two")
|
||||||
|
backfill(one, "--repo", str(self.fixture.a))
|
||||||
|
backfill(two, "--repo", str(self.fixture.b))
|
||||||
|
keys = [[(row["remote"], row["sha"]) for row in self.rows(council)] for council in (one, two)]
|
||||||
|
self.assertEqual(keys[0], keys[1])
|
||||||
|
self.assertEqual([row["branch"] for row in self.rows(two)], ["main", "feature/two", "main"])
|
||||||
|
self.assertEqual({row["path"] for row in self.rows(two)}, {str(self.fixture.b.resolve())})
|
||||||
|
again = backfill(one, "--repo", str(self.fixture.b))
|
||||||
|
self.assertEqual(again.stdout, f"{self.fixture.b.resolve()}: 3 blocks, 0 rows added\n")
|
||||||
|
|
||||||
|
def test_since_reads_only_newer_commits_and_a_bad_repository_writes_nothing(self):
|
||||||
|
council = self.council("one")
|
||||||
|
done = backfill(council, "--repo", str(self.fixture.a), "--since", self.fixture.old)
|
||||||
|
self.assertEqual(done.stdout, f"{self.fixture.a.resolve()}: 2 blocks, 2 rows added\n")
|
||||||
|
empty = self.council("two")
|
||||||
|
failed = backfill(empty, "--repo", str(self.fixture.a), str(self.scratch))
|
||||||
|
self.assertEqual(failed.returncode, 2)
|
||||||
|
self.assertFalse((empty / JOURNAL_NAME).exists())
|
||||||
|
|
||||||
|
def test_a_fifo_or_a_linked_council_root_exits_2_with_one_stderr_line(self):
|
||||||
|
fifo = self.council("fifo")
|
||||||
|
fifo.mkdir()
|
||||||
|
os.mkfifo(fifo / JOURNAL_NAME)
|
||||||
|
real = self.scratch / "real"
|
||||||
|
real.mkdir()
|
||||||
|
linked = self.council("linked")
|
||||||
|
linked.symlink_to(real)
|
||||||
|
for council in (fifo, linked):
|
||||||
|
with self.subTest(council=council.name):
|
||||||
|
done = backfill(council, "--repo", str(self.fixture.a))
|
||||||
|
self.assertEqual(done.returncode, 2)
|
||||||
|
self.assertEqual(len(done.stderr.splitlines()), 1, done.stderr)
|
||||||
|
self.assertTrue(done.stderr.startswith(f"backfill: journal {council / JOURNAL_NAME}: "), done.stderr)
|
||||||
|
self.assertEqual(list(real.iterdir()), [])
|
||||||
|
|
||||||
|
def test_a_missing_nofollow_helper_is_named_in_the_one_stderr_line(self):
|
||||||
|
scripts = self.scratch.resolve() / "skills" / "kvb-check-score" / "scripts"
|
||||||
|
scripts.mkdir(parents=True)
|
||||||
|
for name in ("backfill.py", "journal.py"):
|
||||||
|
shutil.copy(BACKFILL.parent / name, scripts / name)
|
||||||
|
shutil.copy(LABELS, scripts.parent / "labels.json")
|
||||||
|
helper = scripts.parents[1] / "kvb-org-organize" / "scripts" / "nofollow.py"
|
||||||
|
council = self.council("missing")
|
||||||
|
env = {"PATH": os.environ["PATH"], "AGENTS_COUNCIL_ROOT": str(council)}
|
||||||
|
done = subprocess.run([sys.executable, "-B", str(scripts / "backfill.py"), "--repo", str(self.fixture.a)], env=env, capture_output=True, text=True, check=False, timeout=HANG)
|
||||||
|
self.assertEqual((done.returncode, done.stdout, done.stderr), (2, "", f"backfill: journal {council / JOURNAL_NAME}: helper {helper} unusable (FileNotFoundError): install or update kvb-org-organize\n"))
|
||||||
|
self.assertFalse(council.exists())
|
||||||
|
|
||||||
|
@unittest.skipIf(os.geteuid() == 0, "root writes into a mode 0500 directory")
|
||||||
|
def test_a_failed_append_prints_no_summary(self):
|
||||||
|
council = self.council("readonly")
|
||||||
|
council.mkdir(mode=0o500)
|
||||||
|
self.addCleanup(council.chmod, 0o700)
|
||||||
|
done = backfill(council, "--repo", str(self.fixture.a))
|
||||||
|
self.assertEqual((done.returncode, done.stdout), (2, ""))
|
||||||
|
self.assertEqual(len(done.stderr.splitlines()), 1, done.stderr)
|
||||||
|
self.assertTrue(done.stderr.startswith(f"backfill: journal {council / JOURNAL_NAME}: "), done.stderr)
|
||||||
|
self.assertEqual(list(council.iterdir()), [])
|
||||||
|
|
||||||
|
def test_a_repeated_repo_flag_reads_every_repository(self):
|
||||||
|
done = backfill(self.council("one"), "--repo", str(self.fixture.a), "--repo", str(self.fixture.b))
|
||||||
|
self.assertEqual(done.stdout.splitlines(), [f"{self.fixture.a.resolve()}: 3 blocks, 3 rows added", f"{self.fixture.b.resolve()}: 3 blocks, 0 rows added"])
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
@@ -0,0 +1,343 @@
|
|||||||
|
"""Exercise labels.json and the score journal of kvb-check-score: labels in every layout, one row per block, failed appends, no gate reader."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import hashlib
|
||||||
|
import importlib.util
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import re
|
||||||
|
import shutil
|
||||||
|
import stat
|
||||||
|
import subprocess
|
||||||
|
import sys
|
||||||
|
import tempfile
|
||||||
|
import time
|
||||||
|
import unittest
|
||||||
|
from pathlib import Path
|
||||||
|
from types import ModuleType
|
||||||
|
from unittest import mock
|
||||||
|
|
||||||
|
from score_fixture import git_env
|
||||||
|
|
||||||
|
REPO_ROOT = Path(__file__).resolve().parents[1]
|
||||||
|
SKILL = REPO_ROOT / "skills" / "kvb-check-score"
|
||||||
|
SCORE = SKILL / "scripts" / "score.py"
|
||||||
|
EFFICIENCY = REPO_ROOT / "skills" / "kvb-check-efficiency" / "scripts"
|
||||||
|
NOFOLLOW = REPO_ROOT / "skills" / "kvb-org-organize" / "scripts" / "nofollow.py"
|
||||||
|
HANG = 20
|
||||||
|
FORMER = re.compile(r"💸 Cost|⏱️ Time|(?i:💸 cost|⏱️ time|\bcost (?:and|or) time\b|\bcost, time\b|\b(?:cost|time) from the `kvb-check-(?:efficiency|speed)`)")
|
||||||
|
REMOTES = (
|
||||||
|
"git@github.com:org/repo.git",
|
||||||
|
"github.com:org/repo.git",
|
||||||
|
"GitHub.com:org/repo",
|
||||||
|
"https://github.com/org/repo",
|
||||||
|
"https://user@github.com:443/org/repo.git/",
|
||||||
|
"ssh://git@github.com/org/repo.git",
|
||||||
|
)
|
||||||
|
BASE = {"fix": "95", "break": "2", "efficiency": "97", "speed": "-", "security": "100", "critical": "0"}
|
||||||
|
VARIANTS = (
|
||||||
|
{},
|
||||||
|
{"efficiency": "120", "speed": "130"},
|
||||||
|
{"efficiency": "new", "speed": "new", "security": "~90"},
|
||||||
|
{"fix": "80", "break": "10", "efficiency": "150"},
|
||||||
|
{"critical": "1"},
|
||||||
|
)
|
||||||
|
LAYOUTS = ((), ("--table",), ("--message-block",))
|
||||||
|
OLD = re.compile(r"cost|time", re.IGNORECASE)
|
||||||
|
JOURNAL_NAME = "scores.jsonl"
|
||||||
|
READERS = {
|
||||||
|
"skills/kvb-check-score/SKILL.md",
|
||||||
|
"skills/kvb-check-score/scripts/journal.py",
|
||||||
|
"orchestrator/kvb_orchestrator/scores.py",
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def flags(values: dict[str, str]) -> list[str]:
|
||||||
|
return [item for key, value in {**BASE, **values}.items() for item in (f"--{key}", value)]
|
||||||
|
|
||||||
|
|
||||||
|
def module(path: Path) -> ModuleType:
|
||||||
|
spec = importlib.util.spec_from_file_location(f"copied_{path.stem}", path)
|
||||||
|
found = importlib.util.module_from_spec(spec)
|
||||||
|
spec.loader.exec_module(found)
|
||||||
|
return found
|
||||||
|
|
||||||
|
|
||||||
|
class ScoreCase(unittest.TestCase):
|
||||||
|
def setUp(self):
|
||||||
|
tmp = tempfile.TemporaryDirectory()
|
||||||
|
self.addCleanup(tmp.cleanup)
|
||||||
|
self.tmp = Path(tmp.name)
|
||||||
|
self.council = self.tmp / "council"
|
||||||
|
self.env = {"PATH": os.environ["PATH"], "AGENTS_COUNCIL_ROOT": str(self.council)}
|
||||||
|
|
||||||
|
def score(self, script: Path, *args: str, cwd: Path) -> subprocess.CompletedProcess:
|
||||||
|
return subprocess.run([sys.executable, "-B", str(script), *args], cwd=cwd, env=self.env, capture_output=True, text=True, check=False)
|
||||||
|
|
||||||
|
def rows(self) -> list[dict]:
|
||||||
|
path = self.council / JOURNAL_NAME
|
||||||
|
return [json.loads(line) for line in path.read_text(encoding="utf-8").splitlines()] if path.exists() else []
|
||||||
|
|
||||||
|
def copy_skills(self, labels: str) -> Path:
|
||||||
|
"""Args:
|
||||||
|
labels: the text of the copied labels.json.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
The copied score.py, beside kvb-check-efficiency's copied tokens.py and measure.py.
|
||||||
|
"""
|
||||||
|
skill, efficiency = self.tmp / "skills" / "kvb-check-score", self.tmp / "skills" / "kvb-check-efficiency" / "scripts"
|
||||||
|
(skill / "scripts").mkdir(parents=True)
|
||||||
|
efficiency.mkdir(parents=True)
|
||||||
|
for name in ("score.py", "journal.py"):
|
||||||
|
shutil.copy(SKILL / "scripts" / name, skill / "scripts" / name)
|
||||||
|
for name in ("tokens.py", "measure.py"):
|
||||||
|
shutil.copy(EFFICIENCY / name, efficiency / name)
|
||||||
|
organize = self.tmp / "skills" / "kvb-org-organize" / "scripts"
|
||||||
|
organize.mkdir(parents=True)
|
||||||
|
shutil.copy(NOFOLLOW, organize / "nofollow.py")
|
||||||
|
(skill / "labels.json").write_text(labels, encoding="utf-8")
|
||||||
|
return skill / "scripts" / "score.py"
|
||||||
|
|
||||||
|
|
||||||
|
class TestScoreLabels(ScoreCase):
|
||||||
|
def test_line_table_and_block_print_tokens_and_velocity_never_cost_or_time(self):
|
||||||
|
expected = {(): ("💸 tokens", "⏱️ velocity"), ("--table",): ("| 💸 Tokens |", "| ⏱️ Velocity |"), ("--message-block",): ("💸 Tokens: ", "⏱️ Velocity: ")}
|
||||||
|
for values in VARIANTS:
|
||||||
|
for layout in LAYOUTS:
|
||||||
|
with self.subTest(values=values, layout=layout):
|
||||||
|
out = self.score(SCORE, *flags(values), *layout, cwd=self.tmp).stdout
|
||||||
|
for label in expected[layout]:
|
||||||
|
self.assertIn(label, out)
|
||||||
|
self.assertIsNone(OLD.search(out), out)
|
||||||
|
|
||||||
|
def test_no_skill_text_or_doc_names_the_former_labels(self):
|
||||||
|
found = [
|
||||||
|
f"{path.relative_to(REPO_ROOT)}:{number}"
|
||||||
|
for top in ("skills", "docs")
|
||||||
|
for path in sorted((REPO_ROOT / top).rglob("*"))
|
||||||
|
if path.is_file() and path.suffix != ".pyc"
|
||||||
|
for number, line in enumerate(path.read_text(encoding="utf-8", errors="replace").splitlines(), 1)
|
||||||
|
if FORMER.search(line)
|
||||||
|
]
|
||||||
|
self.assertEqual(found, [])
|
||||||
|
labels = json.loads((SKILL / "labels.json").read_text(encoding="utf-8"))["metrics"]
|
||||||
|
self.assertEqual({key: metric["former"] for key, metric in labels.items() if metric["former"]}, {"tokens": ["Cost", "Efficiency"], "velocity": ["Time", "Speed"]})
|
||||||
|
self.assertEqual((labels["tokens"]["emoji"], labels["velocity"]["emoji"]), ("💸", "⏱️"))
|
||||||
|
|
||||||
|
def test_a_label_changed_in_a_temporary_label_file_shows_in_every_layout(self):
|
||||||
|
labels = json.loads((SKILL / "labels.json").read_text(encoding="utf-8"))
|
||||||
|
labels["metrics"]["tokens"]["label"] = "Pennies"
|
||||||
|
labels["metrics"]["velocity"].update(emoji="🐢", label="Pace")
|
||||||
|
script = self.copy_skills(json.dumps(labels, ensure_ascii=False))
|
||||||
|
line = self.score(script, *flags({}), cwd=self.tmp).stdout.splitlines()
|
||||||
|
self.assertEqual(line[0], "🎯 fix 95 % · 💥 break 2 % · 💸 pennies 97 % · 🐢 pace – · 🛡️ security 100 % · 🏆 score 96")
|
||||||
|
self.assertEqual(line[1], "✅ gate: 90/5, pennies and pace <= 100, security 100 %")
|
||||||
|
table = self.score(script, *flags({}), "--table", "--value", "1:2", cwd=self.tmp).stdout.splitlines()
|
||||||
|
self.assertEqual(table[0], "| 🎯 Fix | 💥 Break | 💸 Pennies | 💰 Value | 🐢 Pace | 🛡️ Security | 🏆 Score | Gate |")
|
||||||
|
block = self.score(script, *flags({}), "--message-block", cwd=self.tmp).stdout.splitlines()
|
||||||
|
self.assertEqual(block[2:4], ["💸 Pennies: 97 %", "🐢 Pace: –"])
|
||||||
|
efficiency = self.tmp / "skills" / "kvb-check-efficiency" / "scripts"
|
||||||
|
tokens = module(efficiency / "tokens.py")
|
||||||
|
name = tokens.labelled(("tokens",))["tokens"]
|
||||||
|
self.assertEqual(tokens.headline(name, 97, 100, 97, "listing 3→3 words"), "💸 pennies 97 % (skills 100→97 tokens; listing 3→3 words)")
|
||||||
|
self.assertEqual(tokens.headline(name, None, 0, 50, "listing 3→5 words"), "💸 pennies new (+50 tokens; listing 3→5 words)")
|
||||||
|
side = {"cpu": 1.0, "ram": 10.0, "wall": 2.0}
|
||||||
|
found = {"estimated": False, "base": side, "head": side, "efficiency": 97, "speed": 90, "runs": 1}
|
||||||
|
measure = module(efficiency / "measure.py")
|
||||||
|
self.assertEqual(measure.line(found, measure.labelled(("tokens", "velocity"))), "💸 pennies 97 % (cpu 1.00→1.00 s, ram 10.0→10.0 MB) · 🐢 pace 90 % (wall 2.00→2.00 s)")
|
||||||
|
|
||||||
|
def test_a_malformed_label_file_exits_2_or_4_with_one_stderr_line_in_every_layout(self):
|
||||||
|
labels = json.loads((SKILL / "labels.json").read_text(encoding="utf-8"))
|
||||||
|
broken = [
|
||||||
|
"{broken",
|
||||||
|
json.dumps({"metrics": [], "extras": {}}),
|
||||||
|
json.dumps({**labels, "metrics": {key: value for key, value in labels["metrics"].items() if key != "velocity"}}),
|
||||||
|
json.dumps({**labels, "metrics": {**labels["metrics"], "fix": {"emoji": 1, "label": "Fix", "former": []}}}),
|
||||||
|
]
|
||||||
|
script = self.copy_skills("{}")
|
||||||
|
cases = [(index, text, layout, critical, code) for index, text in enumerate(broken) for layout in LAYOUTS for critical, code in (("0", 2), ("1", 4))]
|
||||||
|
for index, text, layout, critical, code in cases:
|
||||||
|
(script.parents[1] / "labels.json").write_text(text, encoding="utf-8")
|
||||||
|
with self.subTest(labels=index, layout=layout, critical=critical):
|
||||||
|
done = self.score(script, *flags({"critical": critical}), *layout, cwd=self.tmp)
|
||||||
|
self.assertEqual((done.returncode, done.stdout), (code, ""))
|
||||||
|
self.assertEqual(len(done.stderr.splitlines()), 1, done.stderr)
|
||||||
|
self.assertTrue(done.stderr.startswith("score: labels "), done.stderr)
|
||||||
|
|
||||||
|
|
||||||
|
class TestJournalPlace(unittest.TestCase):
|
||||||
|
def setUp(self):
|
||||||
|
self.journal = module(SKILL / "scripts" / "journal.py")
|
||||||
|
|
||||||
|
def test_an_empty_council_root_means_the_default(self):
|
||||||
|
for value in ("", None):
|
||||||
|
with self.subTest(value=value), mock.patch.dict(os.environ, {} if value is None else {"AGENTS_COUNCIL_ROOT": value}):
|
||||||
|
if value is None:
|
||||||
|
os.environ.pop("AGENTS_COUNCIL_ROOT", None)
|
||||||
|
self.assertEqual(self.journal.journal_path(), Path("/tmp/agents-council/scores.jsonl"))
|
||||||
|
|
||||||
|
def test_ssh_scp_and_https_remotes_share_one_key(self):
|
||||||
|
self.assertEqual({self.journal.identity(url) for url in REMOTES}, {"github.com/org/repo"})
|
||||||
|
for url in ("/srv/git/repo.git", "./repo", "file:///srv/git/repo.git", None):
|
||||||
|
with self.subTest(url=url):
|
||||||
|
self.assertEqual(self.journal.identity(url), url)
|
||||||
|
|
||||||
|
|
||||||
|
class TestScoreJournal(ScoreCase):
|
||||||
|
def repo(self) -> Path:
|
||||||
|
repo = self.tmp / "repo"
|
||||||
|
repo.mkdir()
|
||||||
|
env = git_env(self.tmp)
|
||||||
|
for args in (("init", "-q", "-b", "feature/journal"), ("remote", "add", "origin", "git@github.com:example/fixture.git"), ("commit", "-q", "--allow-empty", "-m", "base")):
|
||||||
|
subprocess.run(["git", *args], cwd=repo, env=env, capture_output=True, check=True)
|
||||||
|
return repo
|
||||||
|
|
||||||
|
def test_each_message_block_appends_exactly_one_row_with_the_journal_fields(self):
|
||||||
|
repo = self.repo()
|
||||||
|
before = int(time.time())
|
||||||
|
first = self.score(SCORE, *flags({}), "--message-block", cwd=repo)
|
||||||
|
second = self.score(SCORE, *flags({"security": "~80"}), "--message-block", "--value", "1:20", cwd=repo)
|
||||||
|
self.assertEqual((first.returncode, second.returncode, first.stderr, second.stderr), (0, 3, "", ""))
|
||||||
|
rows = self.rows()
|
||||||
|
self.assertEqual(len(rows), 2)
|
||||||
|
for row, done in zip(rows, (first, second)):
|
||||||
|
self.assertEqual(list(row), ["time", "remote", "path", "branch", "sha", "values", "digest"])
|
||||||
|
self.assertEqual((row["remote"], row["path"], row["branch"], row["sha"]), ("github.com/example/fixture", str(repo.resolve()), "feature/journal", None))
|
||||||
|
self.assertEqual(row["digest"], hashlib.sha256(done.stdout.rstrip("\n").encode("utf-8")).hexdigest())
|
||||||
|
self.assertLessEqual(before, row["time"])
|
||||||
|
self.assertLessEqual(row["time"], int(time.time()))
|
||||||
|
self.assertEqual(rows[0]["values"], {"fix": 95, "break": 2, "tokens": 97, "velocity": None, "security": 100, "score": 96})
|
||||||
|
self.assertEqual(rows[1]["values"], {"fix": 95, "break": 2, "tokens": 97, "velocity": None, "security": 80, "score": 77})
|
||||||
|
self.score(SCORE, *flags({}), cwd=repo)
|
||||||
|
self.score(SCORE, *flags({}), "--table", cwd=repo)
|
||||||
|
self.assertEqual(len(self.rows()), 2)
|
||||||
|
|
||||||
|
def test_a_failed_append_prints_one_stderr_line_and_keeps_the_exit_code(self):
|
||||||
|
self.council.write_text("not a directory", encoding="utf-8")
|
||||||
|
for values, code in (({}, 0), ({"efficiency": "120"}, 3), ({"break": "9"}, 1), ({"critical": "1"}, 4)):
|
||||||
|
with self.subTest(values=values):
|
||||||
|
done = self.score(SCORE, *flags(values), "--message-block", cwd=self.tmp)
|
||||||
|
self.assertEqual(done.returncode, code)
|
||||||
|
self.assertEqual(len(done.stderr.splitlines()), 1, done.stderr)
|
||||||
|
self.assertTrue(done.stderr.startswith(f"score: journal {self.council / JOURNAL_NAME}: "), done.stderr)
|
||||||
|
self.assertEqual(len(done.stdout.splitlines()), 7)
|
||||||
|
|
||||||
|
def test_a_deleted_working_directory_keeps_the_exit_code(self):
|
||||||
|
gone = self.tmp / "gone"
|
||||||
|
wrapper = "import os, subprocess, sys; os.chdir(sys.argv[1]); os.rmdir(sys.argv[1]); sys.exit(subprocess.run(sys.argv[2:]).returncode)"
|
||||||
|
for values, code in (({}, 0), ({"critical": "1"}, 4)):
|
||||||
|
with self.subTest(values=values):
|
||||||
|
gone.mkdir(exist_ok=True)
|
||||||
|
done = subprocess.run([sys.executable, "-B", "-c", wrapper, str(gone), sys.executable, "-B", str(SCORE), *flags(values), "--message-block"], env=self.env, capture_output=True, text=True, check=False)
|
||||||
|
self.assertEqual(done.returncode, code, done.stderr)
|
||||||
|
self.assertEqual(len(done.stderr.splitlines()), 1, done.stderr)
|
||||||
|
self.assertTrue(done.stderr.startswith(f"score: journal {self.council / JOURNAL_NAME}: "), done.stderr)
|
||||||
|
self.assertEqual(len(done.stdout.splitlines()), 7)
|
||||||
|
|
||||||
|
def test_git_output_outside_utf8_keeps_the_exit_code_and_the_row(self):
|
||||||
|
repo = self.repo()
|
||||||
|
subprocess.run([b"git", b"remote", b"set-url", b"origin", b"https://example.org/caf\xe9.git"], cwd=repo, env=git_env(self.tmp), capture_output=True, check=True)
|
||||||
|
done = self.score(SCORE, *flags({"critical": "1"}), "--message-block", cwd=repo)
|
||||||
|
self.assertEqual((done.returncode, done.stderr), (4, ""))
|
||||||
|
self.assertEqual(self.rows()[0]["remote"], "example.org/caf�")
|
||||||
|
|
||||||
|
def test_a_link_at_the_journal_is_not_followed(self):
|
||||||
|
self.council.mkdir()
|
||||||
|
target = self.tmp / "elsewhere"
|
||||||
|
target.write_text("kept\n", encoding="utf-8")
|
||||||
|
(self.council / JOURNAL_NAME).symlink_to(target)
|
||||||
|
done = self.score(SCORE, *flags({"critical": "1"}), "--message-block", cwd=self.tmp)
|
||||||
|
self.assertEqual(done.returncode, 4)
|
||||||
|
self.assertEqual(len(done.stderr.splitlines()), 1, done.stderr)
|
||||||
|
self.assertEqual(target.read_text(encoding="utf-8"), "kept\n")
|
||||||
|
|
||||||
|
def test_a_fifo_at_the_journal_does_not_block_the_exit_code(self):
|
||||||
|
self.council.mkdir()
|
||||||
|
os.mkfifo(self.council / JOURNAL_NAME)
|
||||||
|
done = subprocess.run([sys.executable, "-B", str(SCORE), *flags({"critical": "1"}), "--message-block"], cwd=self.tmp, env=self.env, capture_output=True, text=True, check=False, timeout=HANG)
|
||||||
|
self.assertEqual(done.returncode, 4)
|
||||||
|
self.assertEqual(len(done.stderr.splitlines()), 1, done.stderr)
|
||||||
|
self.assertTrue(done.stderr.startswith(f"score: journal {self.council / JOURNAL_NAME}: "), done.stderr)
|
||||||
|
|
||||||
|
def test_a_linked_council_root_is_refused(self):
|
||||||
|
real = self.tmp / "worktree"
|
||||||
|
real.mkdir()
|
||||||
|
(real / ".git").write_text("gitdir: /elsewhere\n", encoding="utf-8")
|
||||||
|
self.council.symlink_to(real)
|
||||||
|
done = self.score(SCORE, *flags({"critical": "1"}), "--message-block", cwd=self.tmp)
|
||||||
|
self.assertEqual(done.returncode, 4)
|
||||||
|
self.assertEqual(len(done.stderr.splitlines()), 1, done.stderr)
|
||||||
|
self.assertEqual(sorted(path.name for path in real.iterdir()), [".git"])
|
||||||
|
|
||||||
|
def test_a_missing_or_broken_nofollow_helper_is_named_in_the_one_stderr_line(self):
|
||||||
|
script = self.copy_skills((SKILL / "labels.json").read_text(encoding="utf-8"))
|
||||||
|
helper = self.tmp.resolve() / "skills" / "kvb-org-organize" / "scripts" / "nofollow.py"
|
||||||
|
cases = {"FileNotFoundError": None, "AttributeError": "VERSION = 0\n", "SyntaxError": "def (:\n"}
|
||||||
|
if os.geteuid() != 0:
|
||||||
|
cases["PermissionError"] = NOFOLLOW.read_text(encoding="utf-8")
|
||||||
|
for kind, text in cases.items():
|
||||||
|
helper.unlink(missing_ok=True)
|
||||||
|
if text is not None:
|
||||||
|
helper.write_text(text, encoding="utf-8")
|
||||||
|
if kind == "PermissionError":
|
||||||
|
helper.chmod(0)
|
||||||
|
for values, code in (({}, 0), ({"critical": "1"}, 4)):
|
||||||
|
with self.subTest(kind=kind, values=values):
|
||||||
|
done = self.score(script, *flags(values), "--message-block", cwd=self.tmp)
|
||||||
|
self.assertEqual(done.returncode, code)
|
||||||
|
self.assertEqual(done.stderr, f"score: journal {self.council / JOURNAL_NAME}: helper {helper} unusable ({kind}): install or update kvb-org-organize\n")
|
||||||
|
self.assertEqual(len(done.stdout.splitlines()), 7)
|
||||||
|
self.assertFalse(self.council.exists())
|
||||||
|
|
||||||
|
def test_a_branch_or_commit_outside_utf8_keeps_the_score_exit(self):
|
||||||
|
repo = self.tmp / "latin"
|
||||||
|
env = git_env(self.tmp)
|
||||||
|
subprocess.run(["git", "init", "-q", "-b", "caf\udce9", str(repo)], env=env, capture_output=True, check=True)
|
||||||
|
subprocess.run(["git", "commit", "-q", "--allow-empty", "-m", "base"], cwd=repo, env=env, capture_output=True, check=True)
|
||||||
|
for values, code in (({}, 0), ({"critical": "1"}, 4)):
|
||||||
|
with self.subTest(values=values):
|
||||||
|
done = self.score(SCORE, *flags(values), "--commit-table", str(repo), cwd=self.tmp)
|
||||||
|
self.assertEqual((done.returncode, done.stderr), (code, ""))
|
||||||
|
self.assertIn("| caf� |", done.stdout)
|
||||||
|
failed = self.score(SCORE, *flags(values), "--commit-table", str(repo), "--commit", "\udce9", cwd=self.tmp)
|
||||||
|
self.assertEqual((failed.returncode, failed.stdout), (code or 2, ""))
|
||||||
|
self.assertEqual(len(failed.stderr.splitlines()), 1, failed.stderr)
|
||||||
|
self.assertTrue(failed.stderr.startswith("score: git show in "), failed.stderr)
|
||||||
|
|
||||||
|
def test_a_critical_finding_wins_over_a_commit_table_git_cannot_read(self):
|
||||||
|
repo = self.repo()
|
||||||
|
plain = self.tmp / "plain"
|
||||||
|
plain.mkdir()
|
||||||
|
self.env["GIT_CEILING_DIRECTORIES"] = str(self.tmp)
|
||||||
|
tables = {"non-git": (str(plain),), "deleted": (str(self.tmp / "deleted"),), "unknown commit": (str(repo), "--commit", "0" * 40)}
|
||||||
|
for name, (target, *commit) in tables.items():
|
||||||
|
for values, code in (({}, 2), ({"fix": "80"}, 2), ({"critical": "1"}, 4), ({"fix": "80", "critical": "1"}, 4)):
|
||||||
|
with self.subTest(table=name, values=values):
|
||||||
|
done = self.score(SCORE, *flags(values), "--commit-table", target, *commit, cwd=self.tmp)
|
||||||
|
self.assertEqual((done.returncode, done.stdout), (code, ""))
|
||||||
|
self.assertEqual(len(done.stderr.splitlines()), 1, done.stderr)
|
||||||
|
self.assertTrue(done.stderr.startswith("score: git "), done.stderr)
|
||||||
|
|
||||||
|
def test_a_new_journal_and_council_root_are_private(self):
|
||||||
|
self.score(SCORE, *flags({}), "--message-block", cwd=self.tmp)
|
||||||
|
self.assertEqual(stat.S_IMODE((self.council / JOURNAL_NAME).stat().st_mode), 0o600)
|
||||||
|
self.assertEqual(stat.S_IMODE(self.council.stat().st_mode), 0o700)
|
||||||
|
|
||||||
|
def test_no_gate_reads_the_journal(self):
|
||||||
|
found = set()
|
||||||
|
for top in ("skills", "orchestrator", "scripts"):
|
||||||
|
for path in (REPO_ROOT / top).rglob("*"):
|
||||||
|
text = path.suffix in {".py", ".sh", ".md", ".js", ".j2", ".json", ".yml"} and path.is_file()
|
||||||
|
if text and JOURNAL_NAME in path.read_text(encoding="utf-8", errors="replace"):
|
||||||
|
found.add(str(path.relative_to(REPO_ROOT)))
|
||||||
|
self.assertEqual(found, READERS)
|
||||||
|
importers = {str(path.relative_to(REPO_ROOT)) for path in (REPO_ROOT / "skills").rglob("*.py") if re.search(r"^from journal import|^import journal", path.read_text(encoding="utf-8"), re.MULTILINE)}
|
||||||
|
self.assertEqual(importers, {"skills/kvb-check-score/scripts/score.py", "skills/kvb-check-score/scripts/backfill.py"})
|
||||||
|
self.council.mkdir()
|
||||||
|
(self.council / JOURNAL_NAME).write_text('{"time": 1, "values": {"score": 0}}\n{broken\n', encoding="utf-8")
|
||||||
|
self.assertEqual(self.score(SCORE, *flags({}), cwd=self.tmp).returncode, 0)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
Reference in new issue
Block a user