Merge branch 'feature/score-metric-overlays' into unify/skills-20261112-night

This commit is contained in:
Kevin Veen-Birkenbach committed 2026-11-13 12:33:10 +00:00
commit 7d9f25e289
30 files changed
+1806 -142

No files matched your search

+88
View File
@@ -0,0 +1,88 @@
from __future__ import annotations
import os
import shutil
import subprocess
import sys
from pathlib import Path
REPO_ROOT = Path(__file__).resolve().parents[1]
BACKFILL = REPO_ROOT / "skills" / "kvb-check-score" / "scripts" / "backfill.py"
EPOCH = 1_760_000_000
HANG = 60
OLD_BLOCK = "🎯 Fix: 95 %\n💥 Break: 2 %\n💸 Cost: 97 %\n⏱️ Time: –\n🛡️ Security: 100 %\n🏆 Score: 96\n🚦 Gate: ✅ 90/5, cost and time <= 100, security 100 %"
NEW_BLOCK = "🎯 Fix: 92 %\n💥 Break: 3 %\n💸 Tokens: 110 %\n⏱️ Velocity: 90 %\n🛡️ Security: 100 %\n💰 Value: 1:20\n🧪 Coverage: 80 %\n🏆 Score: 81\n🚦 Gate: ⚠️ tokens 110 % > 100: ask the operator before the commit"
ESTIMATED_BLOCK = "🎯 Fix: 90 %\n💥 Break: 5 %\n💸 Tokens: new\n⏱️ Velocity: 130 %\n🛡️ Security: ~80 %\n🌡️ Heat: 5\n🧪 Coverage: 70 %\n🏆 Score: 52\n🚦 Gate: ⚠️ tokens new: ask the operator before the commit"
MALFORMED_BLOCK = "🎯 Fix: 95 %\n💥 Break: 2 %\n🏆 Score: high"
TRAILER = "Co-Authored-By: Test <test@example.org>"
def git_env(home: Path) -> dict[str, str]:
"""Args:
home: a scratch HOME, so no user or system git configuration applies.
Returns:
The environment for fixture git calls.
"""
return {
"PATH": os.path.dirname(shutil.which("git")),
"HOME": str(home),
"GIT_CONFIG_NOSYSTEM": "1",
"GIT_AUTHOR_NAME": "Test",
"GIT_AUTHOR_EMAIL": "test@example.org",
"GIT_COMMITTER_NAME": "Test",
"GIT_COMMITTER_EMAIL": "test@example.org",
}
class Fixture:
"""Args:
base: a scratch directory for upstream.git, the work clone a and the clone b.
"""
def __init__(self, base: Path) -> None:
self.base = base
self.env = git_env(base)
self.upstream = base / "upstream.git"
self.a = base / "a"
self.b = base / "b"
self.step = 0
base.mkdir(parents=True)
self.git(base, "init", "-q", "--bare", "-b", "main", str(self.upstream))
self.a.mkdir()
self.git(self.a, "init", "-q", "-b", "main")
self.git(self.a, "remote", "add", "origin", str(self.upstream))
self.old = self.commit("feat: old labels", OLD_BLOCK)
self.git(self.a, "checkout", "-q", "-b", "feature/two")
self.new = self.commit("feat: new labels", NEW_BLOCK)
self.git(self.a, "checkout", "-q", "main")
self.tick()
self.git(self.a, "merge", "-q", "--no-ff", "-m", "Merge branch 'feature/two'", "feature/two")
self.estimated = self.commit("fix: estimated", ESTIMATED_BLOCK)
self.malformed = self.commit("chore: malformed", MALFORMED_BLOCK)
self.plain = self.commit("docs: no block", "")
self.git(self.a, "push", "-q", "origin", "main", "feature/two")
self.git(base, "clone", "-q", str(self.upstream), str(self.b))
def tick(self) -> None:
self.step += 1
stamp = f"{EPOCH + self.step * 3600} +0000"
self.env["GIT_AUTHOR_DATE"] = stamp
self.env["GIT_COMMITTER_DATE"] = stamp
def git(self, cwd: Path, *args: str) -> str:
return subprocess.run(["git", *args], cwd=cwd, env=self.env, capture_output=True, text=True, check=True).stdout.strip()
def commit(self, subject: str, block: str) -> str:
self.tick()
messages = [item for part in (subject, block, TRAILER) if part for item in ("-m", part)]
self.git(self.a, "commit", "-q", "--allow-empty", *messages)
return self.git(self.a, "rev-parse", "HEAD")
def time_of(self, sha: str) -> int:
return int(self.git(self.a, "show", "-s", "--format=%ct", sha))
def backfill(council: Path, *args: str) -> subprocess.CompletedProcess:
env = {"PATH": os.environ["PATH"], "AGENTS_COUNCIL_ROOT": str(council)}
return subprocess.run([sys.executable, "-B", str(BACKFILL), *args], env=env, capture_output=True, text=True, check=False, timeout=HANG)
+33 -8
View File
@@ -33,8 +33,8 @@ NEW_B = "💰 value ?:? (cost 0/day; pass --value with evidence; kvb-b new: all
CHEAP = "print('cheap')\n"
COSTLY = "data = list(range(2_000_000))\nprint(sum(data))\n"
LINE = re.compile(
r"^💸 cost (?P<mark>~?)(?P<efficiency>\d+) % \(cpu \d+\.\d\d→\d+\.\d\d s, ram \d+\.\d→\d+\.\d MB\) · "
r"⏱️ time (?P<speed_mark>~?)(?P<speed>\d+) % \((?P<basis>wall|cpu) \d+\.\d\d→\d+\.\d\d s\)(?P<tail>.*)$"
r"^💸 tokens (?P<mark>~?)(?P<efficiency>\d+) % \(cpu \d+\.\d\d→\d+\.\d\d s, ram \d+\.\d→\d+\.\d MB\) · "
r"⏱️ velocity (?P<speed_mark>~?)(?P<speed>\d+) % \((?P<basis>wall|cpu) \d+\.\d\d→\d+\.\d\d s\)(?P<tail>.*)$"
)
@@ -121,7 +121,7 @@ class TestTokens(GitRepo):
self.write("skills/kvb-a/scripts/tool.py", "print(1)\n" * 50)
result = self.tokens()
self.assertEqual(
result.stdout.splitlines(), ["💸 cost 50 % (skills 100→50 tokens; listing 3→3 words)", NO_VALUE]
result.stdout.splitlines(), ["💸 tokens 50 % (skills 100→50 tokens; listing 3→3 words)", NO_VALUE]
)
self.assertEqual(result.returncode, 0, result.stderr)
@@ -130,7 +130,7 @@ class TestTokens(GitRepo):
self.write("skills/kvb-b/SKILL.md", SKILL_B.ljust(200, "y"))
result = self.tokens()
self.assertEqual(
result.stdout.splitlines(), ["💸 cost 200 % (skills 100→200 tokens; listing 3→5 words)", NEW_B]
result.stdout.splitlines(), ["💸 tokens 200 % (skills 100→200 tokens; listing 3→5 words)", NEW_B]
)
self.assertEqual(result.returncode, 3)
result = self.tokens("--json")
@@ -157,7 +157,7 @@ class TestTokens(GitRepo):
def test_only_new_files_print_new(self):
self.write("skills/kvb-b/SKILL.md", SKILL_B.ljust(200, "y"))
result = self.tokens()
self.assertEqual(result.stdout.splitlines(), ["💸 cost new (+50 tokens; listing 3→5 words)", NEW_B])
self.assertEqual(result.stdout.splitlines(), ["💸 tokens new (+50 tokens; listing 3→5 words)", NEW_B])
self.assertEqual(result.returncode, 3)
self.assertIsNone(json.loads(self.tokens("--json").stdout)["efficiency"])
@@ -165,14 +165,14 @@ class TestTokens(GitRepo):
self.git("mv", "skills/kvb-a/notes.md", "skills/kvb-a/moved.md")
result = self.tokens()
self.assertEqual(
result.stdout.splitlines(), ["💸 cost 100 % (skills 100→100 tokens; listing 3→3 words)", NO_VALUE]
result.stdout.splitlines(), ["💸 tokens 100 % (skills 100→100 tokens; listing 3→3 words)", NO_VALUE]
)
self.assertEqual(result.returncode, 0)
def test_no_change_is_100(self):
result = self.tokens()
self.assertEqual(
result.stdout.splitlines(), ["💸 cost 100 % (skills 0→0 tokens; listing 3→3 words)", NO_VALUE]
result.stdout.splitlines(), ["💸 tokens 100 % (skills 0→0 tokens; listing 3→3 words)", NO_VALUE]
)
self.assertEqual(result.returncode, 0)
@@ -193,7 +193,7 @@ class TestTokens(GitRepo):
self.assertEqual(
result.stdout.splitlines(),
[
"💸 cost 869 % (skills 13→113 tokens; listing 3→5 words)",
"💸 tokens 869 % (skills 13→113 tokens; listing 3→5 words)",
"💰 value ?:? (cost 203/day; pass --value with evidence)",
],
)
@@ -235,6 +235,22 @@ class TestTokens(GitRepo):
)
self.assertEqual(result.returncode, 3)
def test_a_malformed_label_file_exits_2_with_one_stderr_line(self):
for text in ("{broken", json.dumps({"metrics": {"tokens": {"emoji": 1, "label": "Tokens"}}}), "[]"):
with self.subTest(labels=text):
labels = self.base / "labels.json"
labels.write_text(text, encoding="utf-8")
out, err = io.StringIO(), io.StringIO()
argv = ["--repo", str(self.repo), "--base", "HEAD", "--transcripts", str(self.base / "projects")]
with mock.patch.object(skill_tokens, "LABELS", labels), contextlib.redirect_stdout(out), contextlib.redirect_stderr(err):
try:
code = skill_tokens.main(argv)
except SystemExit as stop:
code = stop.code
self.assertEqual((code, out.getvalue()), (2, ""))
self.assertEqual(len(err.getvalue().splitlines()), 1, err.getvalue())
self.assertTrue(err.getvalue().startswith(f"tokens: labels {labels}: "), err.getvalue())
def test_usage_errors_exit_2(self):
self.assertEqual(self.tokens("--value", "5").returncode, 2)
self.assertEqual(self.tokens("--evidence", "why").returncode, 2)
@@ -381,6 +397,15 @@ class TestMeasure(GitRepo):
self.assertEqual(trees, [f"worktree {self.repo.resolve()}"])
self.assertLessEqual(set(Path("/tmp").glob("kvb-measure-*")), self.leftovers)
def test_a_malformed_label_file_exits_2_before_any_run(self):
labels = self.base / "labels.json"
labels.write_text(json.dumps({"metrics": {"tokens": {"emoji": "💸", "label": "Tokens"}}}), encoding="utf-8")
with mock.patch.object(measure, "LABELS", labels):
code, out, err, samples = self.measured("--base", "HEAD", "--", sys.executable, "work.py")
self.assertEqual((code, out, samples), (2, "", 0))
self.assertEqual(len(err.splitlines()), 1, err)
self.assertTrue(err.startswith(f"measure: labels {labels}: "), err)
def test_cheaper_head_stops_after_one_run_each(self):
self.write("pkg/work.py", CHEAP)
code, out, err, samples = self.measured("--base", "HEAD", "--", sys.executable, "work.py")
+232
View File
@@ -0,0 +1,232 @@
from __future__ import annotations
import json
import subprocess
import unittest
from hub_support import HUB, PLAN, REPO_ROOT
GANTT = HUB / "templates" / "page-tasks-gantt.js"
LABELS = REPO_ROOT / "skills" / "kvb-check-score" / "labels.json"
SERIES_PROBE = """
const P = require(process.argv[1]);
const rows = [
{task: "work/a", branch: "feature/a", worktree_branch: ""},
{task: "work/b", branch: "-", worktree_branch: "fix/b"},
{task: "work/c", branch: "-", worktree_branch: ""},
];
const values = (fix, score) => ({fix, break: 2, tokens: null, velocity: 130, security: 100, score});
const journal = [
{time: 300, branch: "feature/a", values: values(95, 90)},
{time: 100, branch: "fix/b", values: values(92, null)},
{time: 200, branch: "feature/other", values: values(10, 10)},
{time: 900, branch: "feature/a", values: values(1, 1)},
{time: 150, branch: "", values: values(5, 5)},
{time: 250, branch: "-", values: values(6, 6)},
];
const series = P.overlaySeries(rows, journal, ["fix", "tokens", "score"], {from: 100, to: 800});
console.log(JSON.stringify({
keys: Object.keys(P),
series,
none: P.overlaySeries(rows, journal, [], {from: 0, to: 1000}),
tops: [P.overlayTop(series), P.overlayTop(P.overlaySeries(rows, journal, ["velocity"], {from: 0, to: 1000})), P.overlayTop([])],
labels: [null, "x", {}, {metrics: []}, {metrics: {}}, {metrics: {fix: null}}, {metrics: {fix: {emoji: 1, label: "Fix"}}},
{metrics: {fix: {emoji: "🎯", label: "Fix"}, bad: {emoji: "x"}, score: {emoji: "🏆", label: "Score"}}}].map(P.overlayLabels),
}));
"""
BURN_PROBE = """
const fs = require("fs");
const [source, plan, labelFile] = process.argv.slice(1);
const P = require(plan);
const labels = JSON.parse(fs.readFileSync(labelFile, "utf8"));
const node = (tag, attrs, children) => ({tag, attrs, children: children.flat(Infinity).filter((child) => child !== null && child !== undefined && child !== false), append(...more) { this.children.push(...more); }});
const s = (tag, attrs, ...children) => node(tag, attrs, children);
const LEAF = /^[A-Za-z]{1,32}\\.[A-Za-z]{1,32}$/u;
const METRICS = Object.keys(labels.metrics);
const kept = {value: [], listeners: [], updates: 0};
const known = (path) => {
if (path !== "burn.overlays") {
throw new Error(`unknown view path ${path}`);
}
};
const ViewStore = {
get: (path) => {
known(path);
return [...kept.value];
},
set: (path) => {
known(path);
throw new Error(`${path} holds a list: change it through update`);
},
update: (path, change) => {
if (!LEAF.test(String(path)) || typeof change !== "function") {
throw new Error(`update needs a section.key path and a change function, not ${path}`);
}
known(path);
const next = change([...kept.value]);
if (!Array.isArray(next) || !next.every((item) => METRICS.includes(item)) || new Set(next).size !== next.length) {
throw new Error(`burn.overlays must list distinct metrics, not ${JSON.stringify(next)}`);
}
kept.value = next;
kept.updates += 1;
return Promise.resolve(true);
},
on: (path, callback) => {
known(path);
kept.listeners.push([path, callback]);
},
};
const rendered = [];
const LegendChips = {render: (series, path, mode) => {
if (mode !== "hidden" && mode !== "shown") {
throw new Error(`a legend chip mode must be hidden or shown, not ${mode}`);
}
if (!LEAF.test(String(path)) || !Array.isArray(series) || !series.every((entry) => entry && Object.keys(entry).sort().join() === "id,label,tone" && ["id", "label", "tone"].every((name) => typeof entry[name] === "string"))) {
throw new Error("legend chips need a section.key path and series [{id, label, tone}]");
}
const press = (id) => ViewStore.update(path, (list) => (list.includes(id) ? list.filter((item) => item !== id) : [...list, id]));
rendered.push({series, path, mode, press});
return node("div", {class: "legend-chips"}, []);
}};
const refused = (call) => {
try {
call();
return false;
} catch {
return true;
}
};
const contract = [
refused(() => LegendChips.render([{key: "fix", label: "Fix"}], "burn.overlays", "shown")),
refused(() => LegendChips.render([{id: "fix", label: "Fix", tone: "k-fix"}], "burn.overlays", "on")),
refused(() => LegendChips.render([{id: "fix", label: "Fix", tone: "k-fix"}], "burn.overlays")),
refused(() => ViewStore.set("burn.overlays", ["fix"])),
refused(() => ViewStore.update("burn.overlays", () => [null])),
];
const journal = [
{time: 600, branch: "feature/a", values: {fix: 95, break: 2, tokens: 97, velocity: null, security: 100, score: 90}},
{time: 300, branch: "feature/a", values: {fix: 92, break: 2, tokens: 97, velocity: null, security: 100, score: null}},
{time: 400, branch: "feature/other", values: {fix: 10, break: 2, tokens: 97, velocity: null, security: 100, score: 10}},
];
const fetched = [];
const fetch = async (url) => {
fetched.push(url);
return {ok: true, json: async () => ({since: 0, until: 1000, labels, rows: journal})};
};
const counts = {draws: 0};
const taskView = {sig: "drawn"};
const drawTasks = () => {
counts.draws += 1;
};
const build = new Function("Plan", "s", "h", "fixed", "stampOf", "absolute", "ViewStore", "LegendChips", "fetch", "drawTasks", "taskView", "ui", `${fs.readFileSync(source, "utf8")}\\nreturn {burnOverlays, overlaysReady, burnRight};`);
const helpers = [P, s, s, (value) => value.toFixed(1), (ts) => new Date(ts * 1000), {format: (date) => date.toISOString()}];
const api = build(...helpers, ViewStore, LegendChips, fetch, drawTasks, taskView, {});
const inert = build(...helpers, undefined, undefined, fetch, drawTasks, taskView, {});
const brokenStore = {get: () => ["fix"], update: () => Promise.resolve(true), on: () => {}};
const brokenFetch = async () => ({ok: true, json: async () => ({labels: {}, rows: "x"})});
const broken = build(...helpers, brokenStore, LegendChips, brokenFetch, () => {}, {sig: ""}, {});
const picked = {rows: [{task: "work/a", branch: "feature/a", worktree_branch: ""}], from: 0, to: 1000};
const shape = (svg) => ({
overlays: svg.children.filter((child) => child.tag === "g").map((group) => ({cls: group.attrs.class, key: group.attrs["data-overlay"], points: group.children.filter((child) => child.tag === "circle").map((circle) => [circle.attrs.cx, circle.attrs.cy])})),
axis: svg.children.filter((child) => child.tag === "text").map((text) => [text.attrs.x, text.children[0]]),
});
(async () => {
const pending = api.burnOverlays(s("svg", {}), picked, (t) => t, api.burnRight());
const brokenFirst = broken.burnOverlays(s("svg", {}), picked, (t) => t, broken.burnRight());
await new Promise((resolve) => setTimeout(resolve, 0));
const brokenSvg = s("svg", {});
const brokenAfter = broken.burnOverlays(brokenSvg, picked, (t) => t, broken.burnRight());
const brokenView = [brokenFirst, brokenAfter, brokenSvg.children.length, broken.burnRight()];
const off = s("svg", {});
const rights = {inert: [inert.overlaysReady(), inert.burnRight()], off: api.burnRight()};
const offChips = api.burnOverlays(off, picked, (t) => t, rights.off);
const stored = [await rendered[0].press("score"), await rendered[0].press("fix")];
const pressed = {stored, value: [...kept.value], updates: kept.updates};
rights.on = api.burnRight();
const on = s("svg", {});
api.burnOverlays(on, picked, (t) => t, rights.on);
const drawsBefore = counts.draws;
taskView.sig = "drawn";
kept.listeners[0][1]();
console.log(JSON.stringify({
ready: api.overlaysReady(), pending, fetched, firstDraws: drawsBefore, listeners: kept.listeners.map(([path]) => path), redrawn: [counts.draws - drawsBefore, taskView.sig],
off: shape(off), offChips: offChips.attrs.class, chips: rendered[0], on: shape(on), rights, brokenView, contract, pressed,
}));
})();
"""
class TestOverlaySeries(unittest.TestCase):
@classmethod
def setUpClass(cls):
done = subprocess.run(["node", "-e", SERIES_PROBE, str(PLAN)], capture_output=True, text=True, timeout=60, check=True)
cls.data = json.loads(done.stdout)
def test_only_rows_of_the_scope_branches_within_the_span_are_drawn_in_the_given_order(self):
self.assertIn("overlaySeries", self.data["keys"])
self.assertNotIn("OVERLAYS", self.data["keys"])
self.assertEqual(self.data["series"], [
{"key": "fix", "points": [{"t": 100, "value": 92, "branch": "fix/b"}, {"t": 300, "value": 95, "branch": "feature/a"}]},
{"key": "tokens", "points": []},
{"key": "score", "points": [{"t": 300, "value": 90, "branch": "feature/a"}]},
])
self.assertEqual(self.data["none"], [])
def test_the_right_axis_runs_to_100_or_the_largest_value_above_it(self):
self.assertEqual(self.data["tops"], [100, 130, 100])
def test_labels_of_another_shape_switch_the_overlays_off(self):
self.assertEqual(self.data["labels"], [None, None, None, None, None, None, None, {"fix": "🎯 Fix", "score": "🏆 Score"}])
class TestBurnOverlays(unittest.TestCase):
@classmethod
def setUpClass(cls):
done = subprocess.run(["node", "-e", BURN_PROBE, str(GANTT), str(PLAN), str(LABELS)], capture_output=True, text=True, timeout=60, check=True)
cls.data = json.loads(done.stdout)
def test_six_metric_chips_all_off_on_first_load_keyed_burn_overlays(self):
data = self.data
self.assertEqual((data["ready"], data["pending"], data["fetched"], data["firstDraws"]), (True, None, ["/api/scores?from=0&to=1000"], 1))
self.assertEqual((data["chips"]["path"], data["chips"]["mode"]), ("burn.overlays", "shown"))
self.assertEqual(data["chips"]["series"], [
{"id": key, "label": label, "tone": f"k-{key}"}
for key, label in (("fix", "🎯 Fix"), ("break", "💥 Break"), ("tokens", "💸 Tokens"), ("velocity", "⏱️ Velocity"), ("security", "🛡️ Security"), ("score", "🏆 Score"))
])
self.assertEqual((data["off"], data["offChips"]), ({"overlays": [], "axis": []}, "legend-chips"))
def test_the_contract_stub_refuses_another_shape_mode_or_write(self):
self.assertEqual(self.data["contract"], [True] * 5)
def test_a_chip_press_stores_its_metric_through_update(self):
self.assertEqual(self.data["pressed"], {"stored": [True, True], "value": ["score", "fix"], "updates": 2})
def test_labels_of_another_shape_draw_no_overlay_and_never_throw(self):
self.assertEqual(self.data["brokenView"], [None, None, 0, 24])
def test_the_right_margin_widens_only_while_an_overlay_is_on(self):
self.assertEqual(self.data["rights"], {"inert": [False, 24], "off": 24, "on": 56})
def test_a_switched_on_chip_draws_its_series_on_the_right_axis(self):
on = self.data["on"]
self.assertEqual(on["axis"], [[850, "0 %"], [850, "50 %"], [850, "100 %"]])
self.assertEqual(on["overlays"], [
{"cls": "burn-overlay k-fix", "key": "fix", "points": [["300.0", "42.4"], ["600.0", "34.0"]]},
{"cls": "burn-overlay k-score", "key": "score", "points": [["600.0", "48.0"]]},
])
def test_a_view_store_change_redraws_the_burndown(self):
self.assertEqual(self.data["listeners"], ["burn.overlays"])
self.assertEqual(self.data["redrawn"], [1, ""])
class TestOverlayTones(unittest.TestCase):
def test_every_metric_chip_has_a_tone_colour(self):
css = (HUB / "templates" / "style-tasks.css").read_text(encoding="utf-8")
for key in json.loads(LABELS.read_text(encoding="utf-8"))["metrics"]:
with self.subTest(key=key):
self.assertRegex(css, rf"\.burn-box \.legend-chip\.k-{key}\s*\{{\s*--c: var\(--[a-z]+\); \}}")
if __name__ == "__main__":
unittest.main()
+60
View File
@@ -0,0 +1,60 @@
from __future__ import annotations
import json
import unittest
from hub_case import HubCase
from score_fixture import REPO_ROOT, Fixture, backfill
LABELS = REPO_ROOT / "skills" / "kvb-check-score" / "labels.json"
T = 1_760_000_000
class TestHubScores(HubCase):
def fetch(self, query: str) -> tuple[int, bytes]:
status, _, body = self.get(f"/api/scores?{query}")
return status, body
def test_the_hub_serves_the_backfilled_journal_rows_of_a_window(self):
fixture = Fixture(self.council.parent / "fixture")
done = backfill(self.council, "--repo", str(fixture.a))
self.assertEqual(done.returncode, 0, done.stderr)
begin, end = fixture.time_of(fixture.new), fixture.time_of(fixture.estimated)
status, body = self.fetch(f"from={begin}&to={end}")
self.assertEqual(status, 200, body)
found = json.loads(body)
self.assertEqual((found["since"], found["until"]), (begin, end))
self.assertEqual(found["labels"], json.loads(LABELS.read_text(encoding="utf-8")))
self.assertEqual([(row["sha"], row["branch"]) for row in found["rows"]], [(fixture.new, "feature/two"), (fixture.estimated, "main")])
self.assertEqual([row["values"]["score"] for row in found["rows"]], [81, 52])
def test_broken_lines_and_rows_without_values_are_left_out_in_time_order(self):
self.council.mkdir(exist_ok=True)
late = {"time": T, "branch": "feature/a", "values": {"score": 90}}
early = {"time": T - 10, "branch": "feature/a", "values": {"score": 80}}
lines = [json.dumps(late), "{broken", json.dumps({"time": T}), json.dumps({"time": "x", "values": {}}), json.dumps(early), json.dumps({**late, "time": T + 500})]
(self.council / "scores.jsonl").write_text("\n".join(lines) + "\n", encoding="utf-8")
status, body = self.fetch(f"from={T - 100}&to={T + 100}")
self.assertEqual(status, 200, body)
self.assertEqual(json.loads(body)["rows"], [early, late])
self.assertEqual(self.fetch(f"from={T + 10}&to={T}")[0], 400)
def test_a_label_file_of_another_shape_is_served_as_null(self):
skills = self.council.parent / "skills"
labels = skills / "kvb-check-score" / "labels.json"
labels.parent.mkdir(parents=True)
self.scripts.skills = skills
good = {"metrics": {"fix": {"emoji": "🎯", "label": "Fix", "former": []}}}
for text, served in (("{}", None), ('{"metrics": []}', None), ('{"metrics": {}}', None), ('{"metrics": {"fix": {"emoji": 1, "label": "Fix"}}}', None), ("{broken", None), (json.dumps(good), good)):
with self.subTest(labels=text):
labels.write_text(text, encoding="utf-8")
status, body = self.fetch(f"from={T}&to={T + 1}")
self.assertEqual((status, json.loads(body)["labels"]), (200, served))
def test_a_missing_journal_answers_no_rows(self):
status, body = self.fetch(f"from={T}&to={T + 1}")
self.assertEqual((status, json.loads(body)["rows"]), (200, []))
if __name__ == "__main__":
unittest.main()
+25 -24
View File
@@ -12,11 +12,12 @@ from pathlib import Path
REPO_ROOT = Path(__file__).resolve().parents[1]
SCORE = REPO_ROOT / "skills" / "kvb-check-score" / "scripts" / "score.py"
PASS = "✅ gate: 90/5, cost and time <= 100, security 100 %"
PASS = "✅ gate: 90/5, tokens and velocity <= 100, security 100 %"
BASE = {"fix": "95", "break": "2", "efficiency": "100", "speed": "100", "security": "100", "critical": "0"}
HEADER = "| 🎯 Fix | 💥 Break | 💸 Cost | ⏱️ Time | 🛡️ Security | 🏆 Score | Gate |"
HEADER = "| 🎯 Fix | 💥 Break | 💸 Tokens | ⏱️ Velocity | 🛡️ Security | 🏆 Score | Gate |"
ASK = ": ask the operator before the commit"
ENV = {"PATH": os.environ["PATH"]}
COUNCIL = tempfile.TemporaryDirectory()
ENV = {"PATH": os.environ["PATH"], "AGENTS_COUNCIL_ROOT": COUNCIL.name}
def flags(values: dict[str, str]) -> list[str]:
@@ -37,12 +38,12 @@ class TestScore(unittest.TestCase):
def test_formula_and_line_format(self):
self.assertEqual(
self._score(fix="95", **{"break": "4"}),
(["🎯 fix 95 % · 💥 break 4 % · 💸 cost 100 % · ⏱️ time 100 % · 🛡️ security 100 % · 🏆 score 91",
(["🎯 fix 95 % · 💥 break 4 % · 💸 tokens 100 % · ⏱️ velocity 100 % · 🛡️ security 100 % · 🏆 score 91",
PASS], 0),
)
lines, code = self._score(efficiency="97", speed="-")
self.assertEqual(
lines[0], "🎯 fix 95 % · 💥 break 2 % · 💸 cost 97 % · ⏱️ time – · 🛡️ security 100 % · 🏆 score 96"
lines[0], "🎯 fix 95 % · 💥 break 2 % · 💸 tokens 97 % · ⏱️ velocity – · 🛡️ security 100 % · 🏆 score 96"
)
self.assertEqual(code, 0)
self.assertEqual(self._score(fix="92.5")[0][0].split(" · ")[0], "🎯 fix 92.5 %")
@@ -51,14 +52,14 @@ class TestScore(unittest.TestCase):
lines, code = self._score(fix="92", efficiency="-", speed="-", **{"break": "3"})
self.assertEqual(
(lines, code),
(["🎯 fix 92 % · 💥 break 3 % · 💸 cost – · ⏱️ time – · 🛡️ security 100 % · 🏆 score 89", PASS], 0),
(["🎯 fix 92 % · 💥 break 3 % · 💸 tokens – · ⏱️ velocity – · 🛡️ security 100 % · 🏆 score 89", PASS], 0),
)
def test_security_multiplies_and_asks_below_100(self):
lines, code = self._score(fix="100", efficiency="-", speed="-", security="75", **{"break": "0"})
self.assertEqual(
(lines, code),
(["🎯 fix 100 % · 💥 break 0 % · 💸 cost – · ⏱️ time – · 🛡️ security 75 % · 🏆 score 75",
(["🎯 fix 100 % · 💥 break 0 % · 💸 tokens – · ⏱️ velocity – · 🛡️ security 75 % · 🏆 score 75",
f"⚠️ gate: security 75 % < 100{ASK}"], 3),
)
lines, code = self._score(security="~80")
@@ -94,7 +95,7 @@ class TestScore(unittest.TestCase):
({"fix": "80", "break": "10"}, "⛔ gate: fix 80 % < 90, break 10 % > 5"),
(
{"fix": "85", "efficiency": "150", "speed": "new", "security": "50"},
f"⛔ gate: fix 85 % < 90; cost 150 % > 100, time new, security 50 % < 100{ASK}",
f"⛔ gate: fix 85 % < 90; tokens 150 % > 100, velocity new, security 50 % < 100{ASK}",
),
):
with self.subTest(values=values):
@@ -103,13 +104,13 @@ class TestScore(unittest.TestCase):
def test_costs_ask_the_operator_exit_3(self):
for values, verdict in (
({"efficiency": "120"}, f"⚠️ gate: cost 120 % > 100{ASK}"),
({"efficiency": "-", "speed": "130"}, f"⚠️ gate: time 130 % > 100{ASK}"),
({"efficiency": "100.5", "speed": "-"}, f"⚠️ gate: cost 100.5 % > 100{ASK}"),
({"efficiency": "new", "speed": "-"}, f"⚠️ gate: cost new{ASK}"),
({"efficiency": "120"}, f"⚠️ gate: tokens 120 % > 100{ASK}"),
({"efficiency": "-", "speed": "130"}, f"⚠️ gate: velocity 130 % > 100{ASK}"),
({"efficiency": "100.5", "speed": "-"}, f"⚠️ gate: tokens 100.5 % > 100{ASK}"),
({"efficiency": "new", "speed": "-"}, f"⚠️ gate: tokens new{ASK}"),
(
{"efficiency": "120", "speed": "130", "security": "90"},
f"⚠️ gate: cost 120 % > 100, time 130 % > 100, security 90 % < 100{ASK}",
f"⚠️ gate: tokens 120 % > 100, velocity 130 % > 100, security 90 % < 100{ASK}",
),
):
with self.subTest(values=values):
@@ -128,7 +129,7 @@ class TestScore(unittest.TestCase):
def test_new_counts_as_100_in_the_score(self):
lines, _ = self._score(efficiency="new", speed="-", **{"break": "0"})
self.assertEqual(
lines[0], "🎯 fix 95 % · 💥 break 0 % · 💸 cost new · ⏱️ time – · 🛡️ security 100 % · 🏆 score 95"
lines[0], "🎯 fix 95 % · 💥 break 0 % · 💸 tokens new · ⏱️ velocity – · 🛡️ security 100 % · 🏆 score 95"
)
def test_table(self):
@@ -138,7 +139,7 @@ class TestScore(unittest.TestCase):
[
HEADER,
"| --- | --- | --- | --- | --- | --- | --- |",
"| 95 % | 2 % | 97 % | – | 100 % | 96 | ✅ 90/5, cost and time <= 100, security 100 % |",
"| 95 % | 2 % | 97 % | – | 100 % | 96 | ✅ 90/5, tokens and velocity <= 100, security 100 % |",
],
)
self.assertEqual(result.returncode, 0)
@@ -152,8 +153,8 @@ class TestScore(unittest.TestCase):
(result.stdout.splitlines(), result.returncode),
(
[
"🎯 Fix: 95 %", "💥 Break: 2 %", "💸 Cost: 97 %", "⏱️ Time: –", "🛡️ Security: 100 %",
"🏆 Score: 96", "🚦 Gate: ✅ 90/5, cost and time <= 100, security 100 %",
"🎯 Fix: 95 %", "💥 Break: 2 %", "💸 Tokens: 97 %", "⏱️ Velocity: –", "🛡️ Security: 100 %",
"🏆 Score: 96", "🚦 Gate: ✅ 90/5, tokens and velocity <= 100, security 100 %",
],
0,
),
@@ -165,7 +166,7 @@ class TestScore(unittest.TestCase):
(result.stdout.splitlines(), result.returncode),
(
[
"🎯 Fix: 95 %", "💥 Break: 2 %", "💸 Cost: 100 %", "⏱️ Time: 100 %", "🛡️ Security: ~80 %",
"🎯 Fix: 95 %", "💥 Break: 2 %", "💸 Tokens: 100 %", "⏱️ Velocity: 100 %", "🛡️ Security: ~80 %",
"💰 Value: 1:20", "🏆 Score: 74", f"🚦 Gate: ⚠️ security ~80 % < 100{ASK}",
],
3,
@@ -188,9 +189,9 @@ class TestScore(unittest.TestCase):
def test_value_column_changes_neither_score_nor_gate(self):
for value, verdict, code in (
("1:20", "✅ 90/5, cost and time <= 100, security 100 %", 0),
("4.1:1", "✅ 90/5, cost and time <= 100, security 100 %", 0),
("?", "⚠️ cost 120 % > 100" + ASK, 3),
("1:20", "✅ 90/5, tokens and velocity <= 100, security 100 %", 0),
("4.1:1", "✅ 90/5, tokens and velocity <= 100, security 100 %", 0),
("?", "⚠️ tokens 120 % > 100" + ASK, 3),
):
with self.subTest(value=value):
efficiency = "97" if code == 0 else "120"
@@ -201,7 +202,7 @@ class TestScore(unittest.TestCase):
self.assertEqual(
result.stdout.splitlines(),
[
HEADER.replace("💸 Cost |", "💸 Cost | 💰 Value |"),
HEADER.replace("💸 Tokens |", "💸 Tokens | 💰 Value |"),
"|" + " --- |" * 8,
f"| 95 % | 2 % | {efficiency} % | {value} | – | 100 % | {score} | {verdict} |",
],
@@ -236,7 +237,7 @@ class TestScore(unittest.TestCase):
(repo / "b.txt").write_text("three\nfour\n", encoding="utf-8")
git("add", "a.txt", "b.txt")
head = ["🌿 Branch", "📊 Diff stat"]
score = "| 95 % | 2 % | 100 % | 100 % | 100 % | 93 | ✅ 90/5, cost and time <= 100, security 100 % |"
score = "| 95 % | 2 % | 100 % | 100 % | 100 % | 93 | ✅ 90/5, tokens and velocity <= 100, security 100 % |"
worktree = ["", "📁 Worktree:", "", "```text", str(repo.resolve()), "```"]
result = self._run(*flags(BASE), "--commit-table", str(repo), cwd=repo)
@@ -259,7 +260,7 @@ class TestScore(unittest.TestCase):
result = self._run(*flags(BASE), "--commit-table", str(repo), "--commit", "HEAD", "--value", "?", cwd=repo)
self.assertEqual(
result.stdout.splitlines()[0],
"| " + " | ".join([*head, "🔖 Commit"]) + " " + HEADER.replace("💸 Cost |", "💸 Cost | 💰 Value |"),
"| " + " | ".join([*head, "🔖 Commit"]) + " " + HEADER.replace("💸 Tokens |", "💸 Tokens | 💰 Value |"),
)
self.assertEqual(
result.stdout.splitlines()[2],
+165
View File
@@ -0,0 +1,165 @@
"""Exercise backfill.py of kvb-check-score over a fixture repository: rows, idempotence, determinism, labels, clones."""
from __future__ import annotations
import hashlib
import json
import os
import shutil
import subprocess
import sys
import tempfile
import unittest
from pathlib import Path
from score_fixture import (
BACKFILL,
ESTIMATED_BLOCK,
HANG,
NEW_BLOCK,
OLD_BLOCK,
REPO_ROOT,
Fixture,
backfill,
)
JOURNAL_NAME = "scores.jsonl"
LABELS = REPO_ROOT / "skills" / "kvb-check-score" / "labels.json"
UNKNOWN = ["backfill: unknown label Coverage, skipped", "backfill: unknown label Heat, skipped"]
class TestBackfill(unittest.TestCase):
@classmethod
def setUpClass(cls):
cls.tmp = tempfile.TemporaryDirectory()
cls.fixture = Fixture(Path(cls.tmp.name) / "fixture")
@classmethod
def tearDownClass(cls):
cls.tmp.cleanup()
def setUp(self):
scratch = tempfile.TemporaryDirectory()
self.addCleanup(scratch.cleanup)
self.scratch = Path(scratch.name)
def council(self, name: str) -> Path:
return self.scratch / name
def rows(self, council: Path) -> list[dict]:
return [json.loads(line) for line in (council / JOURNAL_NAME).read_text(encoding="utf-8").splitlines()]
def test_three_commits_with_blocks_give_three_rows_then_none(self):
council = self.council("one")
first = backfill(council, "--repo", str(self.fixture.a))
self.assertEqual(first.returncode, 0, first.stderr)
self.assertEqual(first.stdout, f"{self.fixture.a.resolve()}: 3 blocks, 3 rows added\n")
written = (council / JOURNAL_NAME).read_bytes()
second = backfill(council, "--repo", str(self.fixture.a))
self.assertEqual((second.returncode, second.stdout), (0, f"{self.fixture.a.resolve()}: 3 blocks, 0 rows added\n"))
self.assertEqual((council / JOURNAL_NAME).read_bytes(), written)
def test_two_runs_from_an_empty_journal_give_byte_identical_journals(self):
one, two = self.council("one"), self.council("two")
self.assertEqual(backfill(one, "--repo", str(self.fixture.a)).returncode, 0)
self.assertEqual(backfill(two, "--repo", str(self.fixture.a)).returncode, 0)
self.assertEqual((one / JOURNAL_NAME).read_bytes(), (two / JOURNAL_NAME).read_bytes())
def test_old_new_and_estimated_blocks_parse_and_a_malformed_one_is_skipped(self):
council = self.council("one")
backfill(council, "--repo", str(self.fixture.a))
rows = self.rows(council)
fixture = self.fixture
self.assertEqual([row["sha"] for row in rows], [fixture.old, fixture.new, fixture.estimated])
self.assertEqual([row["branch"] for row in rows], ["main", "feature/two", "main"])
self.assertEqual([row["time"] for row in rows], [fixture.time_of(sha) for sha in (fixture.old, fixture.new, fixture.estimated)])
self.assertEqual([row["values"] for row in rows], [
{"fix": 95, "break": 2, "tokens": 97, "velocity": None, "security": 100, "score": 96},
{"fix": 92, "break": 3, "tokens": 110, "velocity": 90, "security": 100, "score": 81},
{"fix": 90, "break": 5, "tokens": None, "velocity": 130, "security": 80, "score": 52},
])
self.assertEqual([row["digest"] for row in rows], [hashlib.sha256(block.encode("utf-8")).hexdigest() for block in (OLD_BLOCK, NEW_BLOCK, ESTIMATED_BLOCK)])
self.assertEqual({(row["remote"], row["path"]) for row in rows}, {(str(fixture.upstream), str(fixture.a.resolve()))})
def test_an_unknown_label_is_reported_once_per_label_and_skipped(self):
done = backfill(self.council("one"), "--repo", str(self.fixture.a))
self.assertEqual(done.stderr.splitlines(), UNKNOWN)
for row in self.rows(self.council("one")):
self.assertEqual(list(row["values"]), ["fix", "break", "tokens", "velocity", "security", "score"])
def test_old_labels_map_through_the_label_file(self):
labels = json.loads(LABELS.read_text(encoding="utf-8"))
labels["metrics"]["tokens"]["former"] = []
file = self.scratch / "labels.json"
file.write_text(json.dumps(labels, ensure_ascii=False), encoding="utf-8")
done = backfill(self.council("one"), "--repo", str(self.fixture.a), "--labels", str(file))
self.assertIn("backfill: unknown label Cost, skipped", done.stderr.splitlines())
self.assertIsNone(self.rows(self.council("one"))[0]["values"]["tokens"])
def test_a_clone_at_another_path_yields_the_same_keys(self):
one, two = self.council("one"), self.council("two")
backfill(one, "--repo", str(self.fixture.a))
backfill(two, "--repo", str(self.fixture.b))
keys = [[(row["remote"], row["sha"]) for row in self.rows(council)] for council in (one, two)]
self.assertEqual(keys[0], keys[1])
self.assertEqual([row["branch"] for row in self.rows(two)], ["main", "feature/two", "main"])
self.assertEqual({row["path"] for row in self.rows(two)}, {str(self.fixture.b.resolve())})
again = backfill(one, "--repo", str(self.fixture.b))
self.assertEqual(again.stdout, f"{self.fixture.b.resolve()}: 3 blocks, 0 rows added\n")
def test_since_reads_only_newer_commits_and_a_bad_repository_writes_nothing(self):
council = self.council("one")
done = backfill(council, "--repo", str(self.fixture.a), "--since", self.fixture.old)
self.assertEqual(done.stdout, f"{self.fixture.a.resolve()}: 2 blocks, 2 rows added\n")
empty = self.council("two")
failed = backfill(empty, "--repo", str(self.fixture.a), str(self.scratch))
self.assertEqual(failed.returncode, 2)
self.assertFalse((empty / JOURNAL_NAME).exists())
def test_a_fifo_or_a_linked_council_root_exits_2_with_one_stderr_line(self):
fifo = self.council("fifo")
fifo.mkdir()
os.mkfifo(fifo / JOURNAL_NAME)
real = self.scratch / "real"
real.mkdir()
linked = self.council("linked")
linked.symlink_to(real)
for council in (fifo, linked):
with self.subTest(council=council.name):
done = backfill(council, "--repo", str(self.fixture.a))
self.assertEqual(done.returncode, 2)
self.assertEqual(len(done.stderr.splitlines()), 1, done.stderr)
self.assertTrue(done.stderr.startswith(f"backfill: journal {council / JOURNAL_NAME}: "), done.stderr)
self.assertEqual(list(real.iterdir()), [])
def test_a_missing_nofollow_helper_is_named_in_the_one_stderr_line(self):
scripts = self.scratch.resolve() / "skills" / "kvb-check-score" / "scripts"
scripts.mkdir(parents=True)
for name in ("backfill.py", "journal.py"):
shutil.copy(BACKFILL.parent / name, scripts / name)
shutil.copy(LABELS, scripts.parent / "labels.json")
helper = scripts.parents[1] / "kvb-org-organize" / "scripts" / "nofollow.py"
council = self.council("missing")
env = {"PATH": os.environ["PATH"], "AGENTS_COUNCIL_ROOT": str(council)}
done = subprocess.run([sys.executable, "-B", str(scripts / "backfill.py"), "--repo", str(self.fixture.a)], env=env, capture_output=True, text=True, check=False, timeout=HANG)
self.assertEqual((done.returncode, done.stdout, done.stderr), (2, "", f"backfill: journal {council / JOURNAL_NAME}: helper {helper} unusable (FileNotFoundError): install or update kvb-org-organize\n"))
self.assertFalse(council.exists())
@unittest.skipIf(os.geteuid() == 0, "root writes into a mode 0500 directory")
def test_a_failed_append_prints_no_summary(self):
council = self.council("readonly")
council.mkdir(mode=0o500)
self.addCleanup(council.chmod, 0o700)
done = backfill(council, "--repo", str(self.fixture.a))
self.assertEqual((done.returncode, done.stdout), (2, ""))
self.assertEqual(len(done.stderr.splitlines()), 1, done.stderr)
self.assertTrue(done.stderr.startswith(f"backfill: journal {council / JOURNAL_NAME}: "), done.stderr)
self.assertEqual(list(council.iterdir()), [])
def test_a_repeated_repo_flag_reads_every_repository(self):
done = backfill(self.council("one"), "--repo", str(self.fixture.a), "--repo", str(self.fixture.b))
self.assertEqual(done.stdout.splitlines(), [f"{self.fixture.a.resolve()}: 3 blocks, 3 rows added", f"{self.fixture.b.resolve()}: 3 blocks, 0 rows added"])
if __name__ == "__main__":
unittest.main()
+343
View File
@@ -0,0 +1,343 @@
"""Exercise labels.json and the score journal of kvb-check-score: labels in every layout, one row per block, failed appends, no gate reader."""
from __future__ import annotations
import hashlib
import importlib.util
import json
import os
import re
import shutil
import stat
import subprocess
import sys
import tempfile
import time
import unittest
from pathlib import Path
from types import ModuleType
from unittest import mock
from score_fixture import git_env
REPO_ROOT = Path(__file__).resolve().parents[1]
SKILL = REPO_ROOT / "skills" / "kvb-check-score"
SCORE = SKILL / "scripts" / "score.py"
EFFICIENCY = REPO_ROOT / "skills" / "kvb-check-efficiency" / "scripts"
NOFOLLOW = REPO_ROOT / "skills" / "kvb-org-organize" / "scripts" / "nofollow.py"
HANG = 20
FORMER = re.compile(r"💸 Cost|⏱️ Time|(?i:💸 cost|⏱️ time|\bcost (?:and|or) time\b|\bcost, time\b|\b(?:cost|time) from the `kvb-check-(?:efficiency|speed)`)")
REMOTES = (
"git@github.com:org/repo.git",
"github.com:org/repo.git",
"GitHub.com:org/repo",
"https://github.com/org/repo",
"https://user@github.com:443/org/repo.git/",
"ssh://git@github.com/org/repo.git",
)
BASE = {"fix": "95", "break": "2", "efficiency": "97", "speed": "-", "security": "100", "critical": "0"}
VARIANTS = (
{},
{"efficiency": "120", "speed": "130"},
{"efficiency": "new", "speed": "new", "security": "~90"},
{"fix": "80", "break": "10", "efficiency": "150"},
{"critical": "1"},
)
LAYOUTS = ((), ("--table",), ("--message-block",))
OLD = re.compile(r"cost|time", re.IGNORECASE)
JOURNAL_NAME = "scores.jsonl"
READERS = {
"skills/kvb-check-score/SKILL.md",
"skills/kvb-check-score/scripts/journal.py",
"orchestrator/kvb_orchestrator/scores.py",
}
def flags(values: dict[str, str]) -> list[str]:
return [item for key, value in {**BASE, **values}.items() for item in (f"--{key}", value)]
def module(path: Path) -> ModuleType:
spec = importlib.util.spec_from_file_location(f"copied_{path.stem}", path)
found = importlib.util.module_from_spec(spec)
spec.loader.exec_module(found)
return found
class ScoreCase(unittest.TestCase):
def setUp(self):
tmp = tempfile.TemporaryDirectory()
self.addCleanup(tmp.cleanup)
self.tmp = Path(tmp.name)
self.council = self.tmp / "council"
self.env = {"PATH": os.environ["PATH"], "AGENTS_COUNCIL_ROOT": str(self.council)}
def score(self, script: Path, *args: str, cwd: Path) -> subprocess.CompletedProcess:
return subprocess.run([sys.executable, "-B", str(script), *args], cwd=cwd, env=self.env, capture_output=True, text=True, check=False)
def rows(self) -> list[dict]:
path = self.council / JOURNAL_NAME
return [json.loads(line) for line in path.read_text(encoding="utf-8").splitlines()] if path.exists() else []
def copy_skills(self, labels: str) -> Path:
"""Args:
labels: the text of the copied labels.json.
Returns:
The copied score.py, beside kvb-check-efficiency's copied tokens.py and measure.py.
"""
skill, efficiency = self.tmp / "skills" / "kvb-check-score", self.tmp / "skills" / "kvb-check-efficiency" / "scripts"
(skill / "scripts").mkdir(parents=True)
efficiency.mkdir(parents=True)
for name in ("score.py", "journal.py"):
shutil.copy(SKILL / "scripts" / name, skill / "scripts" / name)
for name in ("tokens.py", "measure.py"):
shutil.copy(EFFICIENCY / name, efficiency / name)
organize = self.tmp / "skills" / "kvb-org-organize" / "scripts"
organize.mkdir(parents=True)
shutil.copy(NOFOLLOW, organize / "nofollow.py")
(skill / "labels.json").write_text(labels, encoding="utf-8")
return skill / "scripts" / "score.py"
class TestScoreLabels(ScoreCase):
def test_line_table_and_block_print_tokens_and_velocity_never_cost_or_time(self):
expected = {(): ("💸 tokens", "⏱️ velocity"), ("--table",): ("| 💸 Tokens |", "| ⏱️ Velocity |"), ("--message-block",): ("💸 Tokens: ", "⏱️ Velocity: ")}
for values in VARIANTS:
for layout in LAYOUTS:
with self.subTest(values=values, layout=layout):
out = self.score(SCORE, *flags(values), *layout, cwd=self.tmp).stdout
for label in expected[layout]:
self.assertIn(label, out)
self.assertIsNone(OLD.search(out), out)
def test_no_skill_text_or_doc_names_the_former_labels(self):
found = [
f"{path.relative_to(REPO_ROOT)}:{number}"
for top in ("skills", "docs")
for path in sorted((REPO_ROOT / top).rglob("*"))
if path.is_file() and path.suffix != ".pyc"
for number, line in enumerate(path.read_text(encoding="utf-8", errors="replace").splitlines(), 1)
if FORMER.search(line)
]
self.assertEqual(found, [])
labels = json.loads((SKILL / "labels.json").read_text(encoding="utf-8"))["metrics"]
self.assertEqual({key: metric["former"] for key, metric in labels.items() if metric["former"]}, {"tokens": ["Cost", "Efficiency"], "velocity": ["Time", "Speed"]})
self.assertEqual((labels["tokens"]["emoji"], labels["velocity"]["emoji"]), ("💸", "⏱️"))
def test_a_label_changed_in_a_temporary_label_file_shows_in_every_layout(self):
labels = json.loads((SKILL / "labels.json").read_text(encoding="utf-8"))
labels["metrics"]["tokens"]["label"] = "Pennies"
labels["metrics"]["velocity"].update(emoji="🐢", label="Pace")
script = self.copy_skills(json.dumps(labels, ensure_ascii=False))
line = self.score(script, *flags({}), cwd=self.tmp).stdout.splitlines()
self.assertEqual(line[0], "🎯 fix 95 % · 💥 break 2 % · 💸 pennies 97 % · 🐢 pace – · 🛡️ security 100 % · 🏆 score 96")
self.assertEqual(line[1], "✅ gate: 90/5, pennies and pace <= 100, security 100 %")
table = self.score(script, *flags({}), "--table", "--value", "1:2", cwd=self.tmp).stdout.splitlines()
self.assertEqual(table[0], "| 🎯 Fix | 💥 Break | 💸 Pennies | 💰 Value | 🐢 Pace | 🛡️ Security | 🏆 Score | Gate |")
block = self.score(script, *flags({}), "--message-block", cwd=self.tmp).stdout.splitlines()
self.assertEqual(block[2:4], ["💸 Pennies: 97 %", "🐢 Pace: –"])
efficiency = self.tmp / "skills" / "kvb-check-efficiency" / "scripts"
tokens = module(efficiency / "tokens.py")
name = tokens.labelled(("tokens",))["tokens"]
self.assertEqual(tokens.headline(name, 97, 100, 97, "listing 3→3 words"), "💸 pennies 97 % (skills 100→97 tokens; listing 3→3 words)")
self.assertEqual(tokens.headline(name, None, 0, 50, "listing 3→5 words"), "💸 pennies new (+50 tokens; listing 3→5 words)")
side = {"cpu": 1.0, "ram": 10.0, "wall": 2.0}
found = {"estimated": False, "base": side, "head": side, "efficiency": 97, "speed": 90, "runs": 1}
measure = module(efficiency / "measure.py")
self.assertEqual(measure.line(found, measure.labelled(("tokens", "velocity"))), "💸 pennies 97 % (cpu 1.00→1.00 s, ram 10.0→10.0 MB) · 🐢 pace 90 % (wall 2.00→2.00 s)")
def test_a_malformed_label_file_exits_2_or_4_with_one_stderr_line_in_every_layout(self):
labels = json.loads((SKILL / "labels.json").read_text(encoding="utf-8"))
broken = [
"{broken",
json.dumps({"metrics": [], "extras": {}}),
json.dumps({**labels, "metrics": {key: value for key, value in labels["metrics"].items() if key != "velocity"}}),
json.dumps({**labels, "metrics": {**labels["metrics"], "fix": {"emoji": 1, "label": "Fix", "former": []}}}),
]
script = self.copy_skills("{}")
cases = [(index, text, layout, critical, code) for index, text in enumerate(broken) for layout in LAYOUTS for critical, code in (("0", 2), ("1", 4))]
for index, text, layout, critical, code in cases:
(script.parents[1] / "labels.json").write_text(text, encoding="utf-8")
with self.subTest(labels=index, layout=layout, critical=critical):
done = self.score(script, *flags({"critical": critical}), *layout, cwd=self.tmp)
self.assertEqual((done.returncode, done.stdout), (code, ""))
self.assertEqual(len(done.stderr.splitlines()), 1, done.stderr)
self.assertTrue(done.stderr.startswith("score: labels "), done.stderr)
class TestJournalPlace(unittest.TestCase):
def setUp(self):
self.journal = module(SKILL / "scripts" / "journal.py")
def test_an_empty_council_root_means_the_default(self):
for value in ("", None):
with self.subTest(value=value), mock.patch.dict(os.environ, {} if value is None else {"AGENTS_COUNCIL_ROOT": value}):
if value is None:
os.environ.pop("AGENTS_COUNCIL_ROOT", None)
self.assertEqual(self.journal.journal_path(), Path("/tmp/agents-council/scores.jsonl"))
def test_ssh_scp_and_https_remotes_share_one_key(self):
self.assertEqual({self.journal.identity(url) for url in REMOTES}, {"github.com/org/repo"})
for url in ("/srv/git/repo.git", "./repo", "file:///srv/git/repo.git", None):
with self.subTest(url=url):
self.assertEqual(self.journal.identity(url), url)
class TestScoreJournal(ScoreCase):
def repo(self) -> Path:
repo = self.tmp / "repo"
repo.mkdir()
env = git_env(self.tmp)
for args in (("init", "-q", "-b", "feature/journal"), ("remote", "add", "origin", "git@github.com:example/fixture.git"), ("commit", "-q", "--allow-empty", "-m", "base")):
subprocess.run(["git", *args], cwd=repo, env=env, capture_output=True, check=True)
return repo
def test_each_message_block_appends_exactly_one_row_with_the_journal_fields(self):
repo = self.repo()
before = int(time.time())
first = self.score(SCORE, *flags({}), "--message-block", cwd=repo)
second = self.score(SCORE, *flags({"security": "~80"}), "--message-block", "--value", "1:20", cwd=repo)
self.assertEqual((first.returncode, second.returncode, first.stderr, second.stderr), (0, 3, "", ""))
rows = self.rows()
self.assertEqual(len(rows), 2)
for row, done in zip(rows, (first, second)):
self.assertEqual(list(row), ["time", "remote", "path", "branch", "sha", "values", "digest"])
self.assertEqual((row["remote"], row["path"], row["branch"], row["sha"]), ("github.com/example/fixture", str(repo.resolve()), "feature/journal", None))
self.assertEqual(row["digest"], hashlib.sha256(done.stdout.rstrip("\n").encode("utf-8")).hexdigest())
self.assertLessEqual(before, row["time"])
self.assertLessEqual(row["time"], int(time.time()))
self.assertEqual(rows[0]["values"], {"fix": 95, "break": 2, "tokens": 97, "velocity": None, "security": 100, "score": 96})
self.assertEqual(rows[1]["values"], {"fix": 95, "break": 2, "tokens": 97, "velocity": None, "security": 80, "score": 77})
self.score(SCORE, *flags({}), cwd=repo)
self.score(SCORE, *flags({}), "--table", cwd=repo)
self.assertEqual(len(self.rows()), 2)
def test_a_failed_append_prints_one_stderr_line_and_keeps_the_exit_code(self):
self.council.write_text("not a directory", encoding="utf-8")
for values, code in (({}, 0), ({"efficiency": "120"}, 3), ({"break": "9"}, 1), ({"critical": "1"}, 4)):
with self.subTest(values=values):
done = self.score(SCORE, *flags(values), "--message-block", cwd=self.tmp)
self.assertEqual(done.returncode, code)
self.assertEqual(len(done.stderr.splitlines()), 1, done.stderr)
self.assertTrue(done.stderr.startswith(f"score: journal {self.council / JOURNAL_NAME}: "), done.stderr)
self.assertEqual(len(done.stdout.splitlines()), 7)
def test_a_deleted_working_directory_keeps_the_exit_code(self):
gone = self.tmp / "gone"
wrapper = "import os, subprocess, sys; os.chdir(sys.argv[1]); os.rmdir(sys.argv[1]); sys.exit(subprocess.run(sys.argv[2:]).returncode)"
for values, code in (({}, 0), ({"critical": "1"}, 4)):
with self.subTest(values=values):
gone.mkdir(exist_ok=True)
done = subprocess.run([sys.executable, "-B", "-c", wrapper, str(gone), sys.executable, "-B", str(SCORE), *flags(values), "--message-block"], env=self.env, capture_output=True, text=True, check=False)
self.assertEqual(done.returncode, code, done.stderr)
self.assertEqual(len(done.stderr.splitlines()), 1, done.stderr)
self.assertTrue(done.stderr.startswith(f"score: journal {self.council / JOURNAL_NAME}: "), done.stderr)
self.assertEqual(len(done.stdout.splitlines()), 7)
def test_git_output_outside_utf8_keeps_the_exit_code_and_the_row(self):
repo = self.repo()
subprocess.run([b"git", b"remote", b"set-url", b"origin", b"https://example.org/caf\xe9.git"], cwd=repo, env=git_env(self.tmp), capture_output=True, check=True)
done = self.score(SCORE, *flags({"critical": "1"}), "--message-block", cwd=repo)
self.assertEqual((done.returncode, done.stderr), (4, ""))
self.assertEqual(self.rows()[0]["remote"], "example.org/caf�")
def test_a_link_at_the_journal_is_not_followed(self):
self.council.mkdir()
target = self.tmp / "elsewhere"
target.write_text("kept\n", encoding="utf-8")
(self.council / JOURNAL_NAME).symlink_to(target)
done = self.score(SCORE, *flags({"critical": "1"}), "--message-block", cwd=self.tmp)
self.assertEqual(done.returncode, 4)
self.assertEqual(len(done.stderr.splitlines()), 1, done.stderr)
self.assertEqual(target.read_text(encoding="utf-8"), "kept\n")
def test_a_fifo_at_the_journal_does_not_block_the_exit_code(self):
self.council.mkdir()
os.mkfifo(self.council / JOURNAL_NAME)
done = subprocess.run([sys.executable, "-B", str(SCORE), *flags({"critical": "1"}), "--message-block"], cwd=self.tmp, env=self.env, capture_output=True, text=True, check=False, timeout=HANG)
self.assertEqual(done.returncode, 4)
self.assertEqual(len(done.stderr.splitlines()), 1, done.stderr)
self.assertTrue(done.stderr.startswith(f"score: journal {self.council / JOURNAL_NAME}: "), done.stderr)
def test_a_linked_council_root_is_refused(self):
real = self.tmp / "worktree"
real.mkdir()
(real / ".git").write_text("gitdir: /elsewhere\n", encoding="utf-8")
self.council.symlink_to(real)
done = self.score(SCORE, *flags({"critical": "1"}), "--message-block", cwd=self.tmp)
self.assertEqual(done.returncode, 4)
self.assertEqual(len(done.stderr.splitlines()), 1, done.stderr)
self.assertEqual(sorted(path.name for path in real.iterdir()), [".git"])
def test_a_missing_or_broken_nofollow_helper_is_named_in_the_one_stderr_line(self):
script = self.copy_skills((SKILL / "labels.json").read_text(encoding="utf-8"))
helper = self.tmp.resolve() / "skills" / "kvb-org-organize" / "scripts" / "nofollow.py"
cases = {"FileNotFoundError": None, "AttributeError": "VERSION = 0\n", "SyntaxError": "def (:\n"}
if os.geteuid() != 0:
cases["PermissionError"] = NOFOLLOW.read_text(encoding="utf-8")
for kind, text in cases.items():
helper.unlink(missing_ok=True)
if text is not None:
helper.write_text(text, encoding="utf-8")
if kind == "PermissionError":
helper.chmod(0)
for values, code in (({}, 0), ({"critical": "1"}, 4)):
with self.subTest(kind=kind, values=values):
done = self.score(script, *flags(values), "--message-block", cwd=self.tmp)
self.assertEqual(done.returncode, code)
self.assertEqual(done.stderr, f"score: journal {self.council / JOURNAL_NAME}: helper {helper} unusable ({kind}): install or update kvb-org-organize\n")
self.assertEqual(len(done.stdout.splitlines()), 7)
self.assertFalse(self.council.exists())
def test_a_branch_or_commit_outside_utf8_keeps_the_score_exit(self):
repo = self.tmp / "latin"
env = git_env(self.tmp)
subprocess.run(["git", "init", "-q", "-b", "caf\udce9", str(repo)], env=env, capture_output=True, check=True)
subprocess.run(["git", "commit", "-q", "--allow-empty", "-m", "base"], cwd=repo, env=env, capture_output=True, check=True)
for values, code in (({}, 0), ({"critical": "1"}, 4)):
with self.subTest(values=values):
done = self.score(SCORE, *flags(values), "--commit-table", str(repo), cwd=self.tmp)
self.assertEqual((done.returncode, done.stderr), (code, ""))
self.assertIn("| caf� |", done.stdout)
failed = self.score(SCORE, *flags(values), "--commit-table", str(repo), "--commit", "\udce9", cwd=self.tmp)
self.assertEqual((failed.returncode, failed.stdout), (code or 2, ""))
self.assertEqual(len(failed.stderr.splitlines()), 1, failed.stderr)
self.assertTrue(failed.stderr.startswith("score: git show in "), failed.stderr)
def test_a_critical_finding_wins_over_a_commit_table_git_cannot_read(self):
repo = self.repo()
plain = self.tmp / "plain"
plain.mkdir()
self.env["GIT_CEILING_DIRECTORIES"] = str(self.tmp)
tables = {"non-git": (str(plain),), "deleted": (str(self.tmp / "deleted"),), "unknown commit": (str(repo), "--commit", "0" * 40)}
for name, (target, *commit) in tables.items():
for values, code in (({}, 2), ({"fix": "80"}, 2), ({"critical": "1"}, 4), ({"fix": "80", "critical": "1"}, 4)):
with self.subTest(table=name, values=values):
done = self.score(SCORE, *flags(values), "--commit-table", target, *commit, cwd=self.tmp)
self.assertEqual((done.returncode, done.stdout), (code, ""))
self.assertEqual(len(done.stderr.splitlines()), 1, done.stderr)
self.assertTrue(done.stderr.startswith("score: git "), done.stderr)
def test_a_new_journal_and_council_root_are_private(self):
self.score(SCORE, *flags({}), "--message-block", cwd=self.tmp)
self.assertEqual(stat.S_IMODE((self.council / JOURNAL_NAME).stat().st_mode), 0o600)
self.assertEqual(stat.S_IMODE(self.council.stat().st_mode), 0o700)
def test_no_gate_reads_the_journal(self):
found = set()
for top in ("skills", "orchestrator", "scripts"):
for path in (REPO_ROOT / top).rglob("*"):
text = path.suffix in {".py", ".sh", ".md", ".js", ".j2", ".json", ".yml"} and path.is_file()
if text and JOURNAL_NAME in path.read_text(encoding="utf-8", errors="replace"):
found.add(str(path.relative_to(REPO_ROOT)))
self.assertEqual(found, READERS)
importers = {str(path.relative_to(REPO_ROOT)) for path in (REPO_ROOT / "skills").rglob("*.py") if re.search(r"^from journal import|^import journal", path.read_text(encoding="utf-8"), re.MULTILINE)}
self.assertEqual(importers, {"skills/kvb-check-score/scripts/score.py", "skills/kvb-check-score/scripts/backfill.py"})
self.council.mkdir()
(self.council / JOURNAL_NAME).write_text('{"time": 1, "values": {"score": 0}}\n{broken\n', encoding="utf-8")
self.assertEqual(self.score(SCORE, *flags({}), cwd=self.tmp).returncode, 0)
if __name__ == "__main__":
unittest.main()