Merge branch 'feature/score-metric-overlays' into unify/skills-20261112-night
This commit is contained in:
commit
7d9f25e289
30 files changed
+1806
-142
No files matched your search
@@ -0,0 +1,88 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parents[1]
|
||||
BACKFILL = REPO_ROOT / "skills" / "kvb-check-score" / "scripts" / "backfill.py"
|
||||
EPOCH = 1_760_000_000
|
||||
HANG = 60
|
||||
OLD_BLOCK = "🎯 Fix: 95 %\n💥 Break: 2 %\n💸 Cost: 97 %\n⏱️ Time: –\n🛡️ Security: 100 %\n🏆 Score: 96\n🚦 Gate: ✅ 90/5, cost and time <= 100, security 100 %"
|
||||
NEW_BLOCK = "🎯 Fix: 92 %\n💥 Break: 3 %\n💸 Tokens: 110 %\n⏱️ Velocity: 90 %\n🛡️ Security: 100 %\n💰 Value: 1:20\n🧪 Coverage: 80 %\n🏆 Score: 81\n🚦 Gate: ⚠️ tokens 110 % > 100: ask the operator before the commit"
|
||||
ESTIMATED_BLOCK = "🎯 Fix: 90 %\n💥 Break: 5 %\n💸 Tokens: new\n⏱️ Velocity: 130 %\n🛡️ Security: ~80 %\n🌡️ Heat: 5\n🧪 Coverage: 70 %\n🏆 Score: 52\n🚦 Gate: ⚠️ tokens new: ask the operator before the commit"
|
||||
MALFORMED_BLOCK = "🎯 Fix: 95 %\n💥 Break: 2 %\n🏆 Score: high"
|
||||
TRAILER = "Co-Authored-By: Test <test@example.org>"
|
||||
|
||||
|
||||
def git_env(home: Path) -> dict[str, str]:
|
||||
"""Args:
|
||||
home: a scratch HOME, so no user or system git configuration applies.
|
||||
|
||||
Returns:
|
||||
The environment for fixture git calls.
|
||||
"""
|
||||
return {
|
||||
"PATH": os.path.dirname(shutil.which("git")),
|
||||
"HOME": str(home),
|
||||
"GIT_CONFIG_NOSYSTEM": "1",
|
||||
"GIT_AUTHOR_NAME": "Test",
|
||||
"GIT_AUTHOR_EMAIL": "test@example.org",
|
||||
"GIT_COMMITTER_NAME": "Test",
|
||||
"GIT_COMMITTER_EMAIL": "test@example.org",
|
||||
}
|
||||
|
||||
|
||||
class Fixture:
|
||||
"""Args:
|
||||
base: a scratch directory for upstream.git, the work clone a and the clone b.
|
||||
"""
|
||||
|
||||
def __init__(self, base: Path) -> None:
|
||||
self.base = base
|
||||
self.env = git_env(base)
|
||||
self.upstream = base / "upstream.git"
|
||||
self.a = base / "a"
|
||||
self.b = base / "b"
|
||||
self.step = 0
|
||||
base.mkdir(parents=True)
|
||||
self.git(base, "init", "-q", "--bare", "-b", "main", str(self.upstream))
|
||||
self.a.mkdir()
|
||||
self.git(self.a, "init", "-q", "-b", "main")
|
||||
self.git(self.a, "remote", "add", "origin", str(self.upstream))
|
||||
self.old = self.commit("feat: old labels", OLD_BLOCK)
|
||||
self.git(self.a, "checkout", "-q", "-b", "feature/two")
|
||||
self.new = self.commit("feat: new labels", NEW_BLOCK)
|
||||
self.git(self.a, "checkout", "-q", "main")
|
||||
self.tick()
|
||||
self.git(self.a, "merge", "-q", "--no-ff", "-m", "Merge branch 'feature/two'", "feature/two")
|
||||
self.estimated = self.commit("fix: estimated", ESTIMATED_BLOCK)
|
||||
self.malformed = self.commit("chore: malformed", MALFORMED_BLOCK)
|
||||
self.plain = self.commit("docs: no block", "")
|
||||
self.git(self.a, "push", "-q", "origin", "main", "feature/two")
|
||||
self.git(base, "clone", "-q", str(self.upstream), str(self.b))
|
||||
|
||||
def tick(self) -> None:
|
||||
self.step += 1
|
||||
stamp = f"{EPOCH + self.step * 3600} +0000"
|
||||
self.env["GIT_AUTHOR_DATE"] = stamp
|
||||
self.env["GIT_COMMITTER_DATE"] = stamp
|
||||
|
||||
def git(self, cwd: Path, *args: str) -> str:
|
||||
return subprocess.run(["git", *args], cwd=cwd, env=self.env, capture_output=True, text=True, check=True).stdout.strip()
|
||||
|
||||
def commit(self, subject: str, block: str) -> str:
|
||||
self.tick()
|
||||
messages = [item for part in (subject, block, TRAILER) if part for item in ("-m", part)]
|
||||
self.git(self.a, "commit", "-q", "--allow-empty", *messages)
|
||||
return self.git(self.a, "rev-parse", "HEAD")
|
||||
|
||||
def time_of(self, sha: str) -> int:
|
||||
return int(self.git(self.a, "show", "-s", "--format=%ct", sha))
|
||||
|
||||
|
||||
def backfill(council: Path, *args: str) -> subprocess.CompletedProcess:
|
||||
env = {"PATH": os.environ["PATH"], "AGENTS_COUNCIL_ROOT": str(council)}
|
||||
return subprocess.run([sys.executable, "-B", str(BACKFILL), *args], env=env, capture_output=True, text=True, check=False, timeout=HANG)
|
||||
@@ -33,8 +33,8 @@ NEW_B = "💰 value ?:? (cost 0/day; pass --value with evidence; kvb-b new: all
|
||||
CHEAP = "print('cheap')\n"
|
||||
COSTLY = "data = list(range(2_000_000))\nprint(sum(data))\n"
|
||||
LINE = re.compile(
|
||||
r"^💸 cost (?P<mark>~?)(?P<efficiency>\d+) % \(cpu \d+\.\d\d→\d+\.\d\d s, ram \d+\.\d→\d+\.\d MB\) · "
|
||||
r"⏱️ time (?P<speed_mark>~?)(?P<speed>\d+) % \((?P<basis>wall|cpu) \d+\.\d\d→\d+\.\d\d s\)(?P<tail>.*)$"
|
||||
r"^💸 tokens (?P<mark>~?)(?P<efficiency>\d+) % \(cpu \d+\.\d\d→\d+\.\d\d s, ram \d+\.\d→\d+\.\d MB\) · "
|
||||
r"⏱️ velocity (?P<speed_mark>~?)(?P<speed>\d+) % \((?P<basis>wall|cpu) \d+\.\d\d→\d+\.\d\d s\)(?P<tail>.*)$"
|
||||
)
|
||||
|
||||
|
||||
@@ -121,7 +121,7 @@ class TestTokens(GitRepo):
|
||||
self.write("skills/kvb-a/scripts/tool.py", "print(1)\n" * 50)
|
||||
result = self.tokens()
|
||||
self.assertEqual(
|
||||
result.stdout.splitlines(), ["💸 cost 50 % (skills 100→50 tokens; listing 3→3 words)", NO_VALUE]
|
||||
result.stdout.splitlines(), ["💸 tokens 50 % (skills 100→50 tokens; listing 3→3 words)", NO_VALUE]
|
||||
)
|
||||
self.assertEqual(result.returncode, 0, result.stderr)
|
||||
|
||||
@@ -130,7 +130,7 @@ class TestTokens(GitRepo):
|
||||
self.write("skills/kvb-b/SKILL.md", SKILL_B.ljust(200, "y"))
|
||||
result = self.tokens()
|
||||
self.assertEqual(
|
||||
result.stdout.splitlines(), ["💸 cost 200 % (skills 100→200 tokens; listing 3→5 words)", NEW_B]
|
||||
result.stdout.splitlines(), ["💸 tokens 200 % (skills 100→200 tokens; listing 3→5 words)", NEW_B]
|
||||
)
|
||||
self.assertEqual(result.returncode, 3)
|
||||
result = self.tokens("--json")
|
||||
@@ -157,7 +157,7 @@ class TestTokens(GitRepo):
|
||||
def test_only_new_files_print_new(self):
|
||||
self.write("skills/kvb-b/SKILL.md", SKILL_B.ljust(200, "y"))
|
||||
result = self.tokens()
|
||||
self.assertEqual(result.stdout.splitlines(), ["💸 cost new (+50 tokens; listing 3→5 words)", NEW_B])
|
||||
self.assertEqual(result.stdout.splitlines(), ["💸 tokens new (+50 tokens; listing 3→5 words)", NEW_B])
|
||||
self.assertEqual(result.returncode, 3)
|
||||
self.assertIsNone(json.loads(self.tokens("--json").stdout)["efficiency"])
|
||||
|
||||
@@ -165,14 +165,14 @@ class TestTokens(GitRepo):
|
||||
self.git("mv", "skills/kvb-a/notes.md", "skills/kvb-a/moved.md")
|
||||
result = self.tokens()
|
||||
self.assertEqual(
|
||||
result.stdout.splitlines(), ["💸 cost 100 % (skills 100→100 tokens; listing 3→3 words)", NO_VALUE]
|
||||
result.stdout.splitlines(), ["💸 tokens 100 % (skills 100→100 tokens; listing 3→3 words)", NO_VALUE]
|
||||
)
|
||||
self.assertEqual(result.returncode, 0)
|
||||
|
||||
def test_no_change_is_100(self):
|
||||
result = self.tokens()
|
||||
self.assertEqual(
|
||||
result.stdout.splitlines(), ["💸 cost 100 % (skills 0→0 tokens; listing 3→3 words)", NO_VALUE]
|
||||
result.stdout.splitlines(), ["💸 tokens 100 % (skills 0→0 tokens; listing 3→3 words)", NO_VALUE]
|
||||
)
|
||||
self.assertEqual(result.returncode, 0)
|
||||
|
||||
@@ -193,7 +193,7 @@ class TestTokens(GitRepo):
|
||||
self.assertEqual(
|
||||
result.stdout.splitlines(),
|
||||
[
|
||||
"💸 cost 869 % (skills 13→113 tokens; listing 3→5 words)",
|
||||
"💸 tokens 869 % (skills 13→113 tokens; listing 3→5 words)",
|
||||
"💰 value ?:? (cost 203/day; pass --value with evidence)",
|
||||
],
|
||||
)
|
||||
@@ -235,6 +235,22 @@ class TestTokens(GitRepo):
|
||||
)
|
||||
self.assertEqual(result.returncode, 3)
|
||||
|
||||
def test_a_malformed_label_file_exits_2_with_one_stderr_line(self):
|
||||
for text in ("{broken", json.dumps({"metrics": {"tokens": {"emoji": 1, "label": "Tokens"}}}), "[]"):
|
||||
with self.subTest(labels=text):
|
||||
labels = self.base / "labels.json"
|
||||
labels.write_text(text, encoding="utf-8")
|
||||
out, err = io.StringIO(), io.StringIO()
|
||||
argv = ["--repo", str(self.repo), "--base", "HEAD", "--transcripts", str(self.base / "projects")]
|
||||
with mock.patch.object(skill_tokens, "LABELS", labels), contextlib.redirect_stdout(out), contextlib.redirect_stderr(err):
|
||||
try:
|
||||
code = skill_tokens.main(argv)
|
||||
except SystemExit as stop:
|
||||
code = stop.code
|
||||
self.assertEqual((code, out.getvalue()), (2, ""))
|
||||
self.assertEqual(len(err.getvalue().splitlines()), 1, err.getvalue())
|
||||
self.assertTrue(err.getvalue().startswith(f"tokens: labels {labels}: "), err.getvalue())
|
||||
|
||||
def test_usage_errors_exit_2(self):
|
||||
self.assertEqual(self.tokens("--value", "5").returncode, 2)
|
||||
self.assertEqual(self.tokens("--evidence", "why").returncode, 2)
|
||||
@@ -381,6 +397,15 @@ class TestMeasure(GitRepo):
|
||||
self.assertEqual(trees, [f"worktree {self.repo.resolve()}"])
|
||||
self.assertLessEqual(set(Path("/tmp").glob("kvb-measure-*")), self.leftovers)
|
||||
|
||||
def test_a_malformed_label_file_exits_2_before_any_run(self):
|
||||
labels = self.base / "labels.json"
|
||||
labels.write_text(json.dumps({"metrics": {"tokens": {"emoji": "💸", "label": "Tokens"}}}), encoding="utf-8")
|
||||
with mock.patch.object(measure, "LABELS", labels):
|
||||
code, out, err, samples = self.measured("--base", "HEAD", "--", sys.executable, "work.py")
|
||||
self.assertEqual((code, out, samples), (2, "", 0))
|
||||
self.assertEqual(len(err.splitlines()), 1, err)
|
||||
self.assertTrue(err.startswith(f"measure: labels {labels}: "), err)
|
||||
|
||||
def test_cheaper_head_stops_after_one_run_each(self):
|
||||
self.write("pkg/work.py", CHEAP)
|
||||
code, out, err, samples = self.measured("--base", "HEAD", "--", sys.executable, "work.py")
|
||||
|
||||
@@ -0,0 +1,232 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import subprocess
|
||||
import unittest
|
||||
|
||||
from hub_support import HUB, PLAN, REPO_ROOT
|
||||
|
||||
GANTT = HUB / "templates" / "page-tasks-gantt.js"
|
||||
LABELS = REPO_ROOT / "skills" / "kvb-check-score" / "labels.json"
|
||||
SERIES_PROBE = """
|
||||
const P = require(process.argv[1]);
|
||||
const rows = [
|
||||
{task: "work/a", branch: "feature/a", worktree_branch: ""},
|
||||
{task: "work/b", branch: "-", worktree_branch: "fix/b"},
|
||||
{task: "work/c", branch: "-", worktree_branch: ""},
|
||||
];
|
||||
const values = (fix, score) => ({fix, break: 2, tokens: null, velocity: 130, security: 100, score});
|
||||
const journal = [
|
||||
{time: 300, branch: "feature/a", values: values(95, 90)},
|
||||
{time: 100, branch: "fix/b", values: values(92, null)},
|
||||
{time: 200, branch: "feature/other", values: values(10, 10)},
|
||||
{time: 900, branch: "feature/a", values: values(1, 1)},
|
||||
{time: 150, branch: "", values: values(5, 5)},
|
||||
{time: 250, branch: "-", values: values(6, 6)},
|
||||
];
|
||||
const series = P.overlaySeries(rows, journal, ["fix", "tokens", "score"], {from: 100, to: 800});
|
||||
console.log(JSON.stringify({
|
||||
keys: Object.keys(P),
|
||||
series,
|
||||
none: P.overlaySeries(rows, journal, [], {from: 0, to: 1000}),
|
||||
tops: [P.overlayTop(series), P.overlayTop(P.overlaySeries(rows, journal, ["velocity"], {from: 0, to: 1000})), P.overlayTop([])],
|
||||
labels: [null, "x", {}, {metrics: []}, {metrics: {}}, {metrics: {fix: null}}, {metrics: {fix: {emoji: 1, label: "Fix"}}},
|
||||
{metrics: {fix: {emoji: "🎯", label: "Fix"}, bad: {emoji: "x"}, score: {emoji: "🏆", label: "Score"}}}].map(P.overlayLabels),
|
||||
}));
|
||||
"""
|
||||
BURN_PROBE = """
|
||||
const fs = require("fs");
|
||||
const [source, plan, labelFile] = process.argv.slice(1);
|
||||
const P = require(plan);
|
||||
const labels = JSON.parse(fs.readFileSync(labelFile, "utf8"));
|
||||
const node = (tag, attrs, children) => ({tag, attrs, children: children.flat(Infinity).filter((child) => child !== null && child !== undefined && child !== false), append(...more) { this.children.push(...more); }});
|
||||
const s = (tag, attrs, ...children) => node(tag, attrs, children);
|
||||
const LEAF = /^[A-Za-z]{1,32}\\.[A-Za-z]{1,32}$/u;
|
||||
const METRICS = Object.keys(labels.metrics);
|
||||
const kept = {value: [], listeners: [], updates: 0};
|
||||
const known = (path) => {
|
||||
if (path !== "burn.overlays") {
|
||||
throw new Error(`unknown view path ${path}`);
|
||||
}
|
||||
};
|
||||
const ViewStore = {
|
||||
get: (path) => {
|
||||
known(path);
|
||||
return [...kept.value];
|
||||
},
|
||||
set: (path) => {
|
||||
known(path);
|
||||
throw new Error(`${path} holds a list: change it through update`);
|
||||
},
|
||||
update: (path, change) => {
|
||||
if (!LEAF.test(String(path)) || typeof change !== "function") {
|
||||
throw new Error(`update needs a section.key path and a change function, not ${path}`);
|
||||
}
|
||||
known(path);
|
||||
const next = change([...kept.value]);
|
||||
if (!Array.isArray(next) || !next.every((item) => METRICS.includes(item)) || new Set(next).size !== next.length) {
|
||||
throw new Error(`burn.overlays must list distinct metrics, not ${JSON.stringify(next)}`);
|
||||
}
|
||||
kept.value = next;
|
||||
kept.updates += 1;
|
||||
return Promise.resolve(true);
|
||||
},
|
||||
on: (path, callback) => {
|
||||
known(path);
|
||||
kept.listeners.push([path, callback]);
|
||||
},
|
||||
};
|
||||
const rendered = [];
|
||||
const LegendChips = {render: (series, path, mode) => {
|
||||
if (mode !== "hidden" && mode !== "shown") {
|
||||
throw new Error(`a legend chip mode must be hidden or shown, not ${mode}`);
|
||||
}
|
||||
if (!LEAF.test(String(path)) || !Array.isArray(series) || !series.every((entry) => entry && Object.keys(entry).sort().join() === "id,label,tone" && ["id", "label", "tone"].every((name) => typeof entry[name] === "string"))) {
|
||||
throw new Error("legend chips need a section.key path and series [{id, label, tone}]");
|
||||
}
|
||||
const press = (id) => ViewStore.update(path, (list) => (list.includes(id) ? list.filter((item) => item !== id) : [...list, id]));
|
||||
rendered.push({series, path, mode, press});
|
||||
return node("div", {class: "legend-chips"}, []);
|
||||
}};
|
||||
const refused = (call) => {
|
||||
try {
|
||||
call();
|
||||
return false;
|
||||
} catch {
|
||||
return true;
|
||||
}
|
||||
};
|
||||
const contract = [
|
||||
refused(() => LegendChips.render([{key: "fix", label: "Fix"}], "burn.overlays", "shown")),
|
||||
refused(() => LegendChips.render([{id: "fix", label: "Fix", tone: "k-fix"}], "burn.overlays", "on")),
|
||||
refused(() => LegendChips.render([{id: "fix", label: "Fix", tone: "k-fix"}], "burn.overlays")),
|
||||
refused(() => ViewStore.set("burn.overlays", ["fix"])),
|
||||
refused(() => ViewStore.update("burn.overlays", () => [null])),
|
||||
];
|
||||
const journal = [
|
||||
{time: 600, branch: "feature/a", values: {fix: 95, break: 2, tokens: 97, velocity: null, security: 100, score: 90}},
|
||||
{time: 300, branch: "feature/a", values: {fix: 92, break: 2, tokens: 97, velocity: null, security: 100, score: null}},
|
||||
{time: 400, branch: "feature/other", values: {fix: 10, break: 2, tokens: 97, velocity: null, security: 100, score: 10}},
|
||||
];
|
||||
const fetched = [];
|
||||
const fetch = async (url) => {
|
||||
fetched.push(url);
|
||||
return {ok: true, json: async () => ({since: 0, until: 1000, labels, rows: journal})};
|
||||
};
|
||||
const counts = {draws: 0};
|
||||
const taskView = {sig: "drawn"};
|
||||
const drawTasks = () => {
|
||||
counts.draws += 1;
|
||||
};
|
||||
const build = new Function("Plan", "s", "h", "fixed", "stampOf", "absolute", "ViewStore", "LegendChips", "fetch", "drawTasks", "taskView", "ui", `${fs.readFileSync(source, "utf8")}\\nreturn {burnOverlays, overlaysReady, burnRight};`);
|
||||
const helpers = [P, s, s, (value) => value.toFixed(1), (ts) => new Date(ts * 1000), {format: (date) => date.toISOString()}];
|
||||
const api = build(...helpers, ViewStore, LegendChips, fetch, drawTasks, taskView, {});
|
||||
const inert = build(...helpers, undefined, undefined, fetch, drawTasks, taskView, {});
|
||||
const brokenStore = {get: () => ["fix"], update: () => Promise.resolve(true), on: () => {}};
|
||||
const brokenFetch = async () => ({ok: true, json: async () => ({labels: {}, rows: "x"})});
|
||||
const broken = build(...helpers, brokenStore, LegendChips, brokenFetch, () => {}, {sig: ""}, {});
|
||||
const picked = {rows: [{task: "work/a", branch: "feature/a", worktree_branch: ""}], from: 0, to: 1000};
|
||||
const shape = (svg) => ({
|
||||
overlays: svg.children.filter((child) => child.tag === "g").map((group) => ({cls: group.attrs.class, key: group.attrs["data-overlay"], points: group.children.filter((child) => child.tag === "circle").map((circle) => [circle.attrs.cx, circle.attrs.cy])})),
|
||||
axis: svg.children.filter((child) => child.tag === "text").map((text) => [text.attrs.x, text.children[0]]),
|
||||
});
|
||||
(async () => {
|
||||
const pending = api.burnOverlays(s("svg", {}), picked, (t) => t, api.burnRight());
|
||||
const brokenFirst = broken.burnOverlays(s("svg", {}), picked, (t) => t, broken.burnRight());
|
||||
await new Promise((resolve) => setTimeout(resolve, 0));
|
||||
const brokenSvg = s("svg", {});
|
||||
const brokenAfter = broken.burnOverlays(brokenSvg, picked, (t) => t, broken.burnRight());
|
||||
const brokenView = [brokenFirst, brokenAfter, brokenSvg.children.length, broken.burnRight()];
|
||||
const off = s("svg", {});
|
||||
const rights = {inert: [inert.overlaysReady(), inert.burnRight()], off: api.burnRight()};
|
||||
const offChips = api.burnOverlays(off, picked, (t) => t, rights.off);
|
||||
const stored = [await rendered[0].press("score"), await rendered[0].press("fix")];
|
||||
const pressed = {stored, value: [...kept.value], updates: kept.updates};
|
||||
rights.on = api.burnRight();
|
||||
const on = s("svg", {});
|
||||
api.burnOverlays(on, picked, (t) => t, rights.on);
|
||||
const drawsBefore = counts.draws;
|
||||
taskView.sig = "drawn";
|
||||
kept.listeners[0][1]();
|
||||
console.log(JSON.stringify({
|
||||
ready: api.overlaysReady(), pending, fetched, firstDraws: drawsBefore, listeners: kept.listeners.map(([path]) => path), redrawn: [counts.draws - drawsBefore, taskView.sig],
|
||||
off: shape(off), offChips: offChips.attrs.class, chips: rendered[0], on: shape(on), rights, brokenView, contract, pressed,
|
||||
}));
|
||||
})();
|
||||
"""
|
||||
|
||||
|
||||
class TestOverlaySeries(unittest.TestCase):
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
done = subprocess.run(["node", "-e", SERIES_PROBE, str(PLAN)], capture_output=True, text=True, timeout=60, check=True)
|
||||
cls.data = json.loads(done.stdout)
|
||||
|
||||
def test_only_rows_of_the_scope_branches_within_the_span_are_drawn_in_the_given_order(self):
|
||||
self.assertIn("overlaySeries", self.data["keys"])
|
||||
self.assertNotIn("OVERLAYS", self.data["keys"])
|
||||
self.assertEqual(self.data["series"], [
|
||||
{"key": "fix", "points": [{"t": 100, "value": 92, "branch": "fix/b"}, {"t": 300, "value": 95, "branch": "feature/a"}]},
|
||||
{"key": "tokens", "points": []},
|
||||
{"key": "score", "points": [{"t": 300, "value": 90, "branch": "feature/a"}]},
|
||||
])
|
||||
self.assertEqual(self.data["none"], [])
|
||||
|
||||
def test_the_right_axis_runs_to_100_or_the_largest_value_above_it(self):
|
||||
self.assertEqual(self.data["tops"], [100, 130, 100])
|
||||
|
||||
def test_labels_of_another_shape_switch_the_overlays_off(self):
|
||||
self.assertEqual(self.data["labels"], [None, None, None, None, None, None, None, {"fix": "🎯 Fix", "score": "🏆 Score"}])
|
||||
|
||||
|
||||
class TestBurnOverlays(unittest.TestCase):
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
done = subprocess.run(["node", "-e", BURN_PROBE, str(GANTT), str(PLAN), str(LABELS)], capture_output=True, text=True, timeout=60, check=True)
|
||||
cls.data = json.loads(done.stdout)
|
||||
|
||||
def test_six_metric_chips_all_off_on_first_load_keyed_burn_overlays(self):
|
||||
data = self.data
|
||||
self.assertEqual((data["ready"], data["pending"], data["fetched"], data["firstDraws"]), (True, None, ["/api/scores?from=0&to=1000"], 1))
|
||||
self.assertEqual((data["chips"]["path"], data["chips"]["mode"]), ("burn.overlays", "shown"))
|
||||
self.assertEqual(data["chips"]["series"], [
|
||||
{"id": key, "label": label, "tone": f"k-{key}"}
|
||||
for key, label in (("fix", "🎯 Fix"), ("break", "💥 Break"), ("tokens", "💸 Tokens"), ("velocity", "⏱️ Velocity"), ("security", "🛡️ Security"), ("score", "🏆 Score"))
|
||||
])
|
||||
self.assertEqual((data["off"], data["offChips"]), ({"overlays": [], "axis": []}, "legend-chips"))
|
||||
|
||||
def test_the_contract_stub_refuses_another_shape_mode_or_write(self):
|
||||
self.assertEqual(self.data["contract"], [True] * 5)
|
||||
|
||||
def test_a_chip_press_stores_its_metric_through_update(self):
|
||||
self.assertEqual(self.data["pressed"], {"stored": [True, True], "value": ["score", "fix"], "updates": 2})
|
||||
|
||||
def test_labels_of_another_shape_draw_no_overlay_and_never_throw(self):
|
||||
self.assertEqual(self.data["brokenView"], [None, None, 0, 24])
|
||||
|
||||
def test_the_right_margin_widens_only_while_an_overlay_is_on(self):
|
||||
self.assertEqual(self.data["rights"], {"inert": [False, 24], "off": 24, "on": 56})
|
||||
|
||||
def test_a_switched_on_chip_draws_its_series_on_the_right_axis(self):
|
||||
on = self.data["on"]
|
||||
self.assertEqual(on["axis"], [[850, "0 %"], [850, "50 %"], [850, "100 %"]])
|
||||
self.assertEqual(on["overlays"], [
|
||||
{"cls": "burn-overlay k-fix", "key": "fix", "points": [["300.0", "42.4"], ["600.0", "34.0"]]},
|
||||
{"cls": "burn-overlay k-score", "key": "score", "points": [["600.0", "48.0"]]},
|
||||
])
|
||||
|
||||
def test_a_view_store_change_redraws_the_burndown(self):
|
||||
self.assertEqual(self.data["listeners"], ["burn.overlays"])
|
||||
self.assertEqual(self.data["redrawn"], [1, ""])
|
||||
|
||||
|
||||
class TestOverlayTones(unittest.TestCase):
|
||||
def test_every_metric_chip_has_a_tone_colour(self):
|
||||
css = (HUB / "templates" / "style-tasks.css").read_text(encoding="utf-8")
|
||||
for key in json.loads(LABELS.read_text(encoding="utf-8"))["metrics"]:
|
||||
with self.subTest(key=key):
|
||||
self.assertRegex(css, rf"\.burn-box \.legend-chip\.k-{key}\s*\{{\s*--c: var\(--[a-z]+\); \}}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,60 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import unittest
|
||||
|
||||
from hub_case import HubCase
|
||||
from score_fixture import REPO_ROOT, Fixture, backfill
|
||||
|
||||
LABELS = REPO_ROOT / "skills" / "kvb-check-score" / "labels.json"
|
||||
T = 1_760_000_000
|
||||
|
||||
|
||||
class TestHubScores(HubCase):
|
||||
def fetch(self, query: str) -> tuple[int, bytes]:
|
||||
status, _, body = self.get(f"/api/scores?{query}")
|
||||
return status, body
|
||||
|
||||
def test_the_hub_serves_the_backfilled_journal_rows_of_a_window(self):
|
||||
fixture = Fixture(self.council.parent / "fixture")
|
||||
done = backfill(self.council, "--repo", str(fixture.a))
|
||||
self.assertEqual(done.returncode, 0, done.stderr)
|
||||
begin, end = fixture.time_of(fixture.new), fixture.time_of(fixture.estimated)
|
||||
status, body = self.fetch(f"from={begin}&to={end}")
|
||||
self.assertEqual(status, 200, body)
|
||||
found = json.loads(body)
|
||||
self.assertEqual((found["since"], found["until"]), (begin, end))
|
||||
self.assertEqual(found["labels"], json.loads(LABELS.read_text(encoding="utf-8")))
|
||||
self.assertEqual([(row["sha"], row["branch"]) for row in found["rows"]], [(fixture.new, "feature/two"), (fixture.estimated, "main")])
|
||||
self.assertEqual([row["values"]["score"] for row in found["rows"]], [81, 52])
|
||||
|
||||
def test_broken_lines_and_rows_without_values_are_left_out_in_time_order(self):
|
||||
self.council.mkdir(exist_ok=True)
|
||||
late = {"time": T, "branch": "feature/a", "values": {"score": 90}}
|
||||
early = {"time": T - 10, "branch": "feature/a", "values": {"score": 80}}
|
||||
lines = [json.dumps(late), "{broken", json.dumps({"time": T}), json.dumps({"time": "x", "values": {}}), json.dumps(early), json.dumps({**late, "time": T + 500})]
|
||||
(self.council / "scores.jsonl").write_text("\n".join(lines) + "\n", encoding="utf-8")
|
||||
status, body = self.fetch(f"from={T - 100}&to={T + 100}")
|
||||
self.assertEqual(status, 200, body)
|
||||
self.assertEqual(json.loads(body)["rows"], [early, late])
|
||||
self.assertEqual(self.fetch(f"from={T + 10}&to={T}")[0], 400)
|
||||
|
||||
def test_a_label_file_of_another_shape_is_served_as_null(self):
|
||||
skills = self.council.parent / "skills"
|
||||
labels = skills / "kvb-check-score" / "labels.json"
|
||||
labels.parent.mkdir(parents=True)
|
||||
self.scripts.skills = skills
|
||||
good = {"metrics": {"fix": {"emoji": "🎯", "label": "Fix", "former": []}}}
|
||||
for text, served in (("{}", None), ('{"metrics": []}', None), ('{"metrics": {}}', None), ('{"metrics": {"fix": {"emoji": 1, "label": "Fix"}}}', None), ("{broken", None), (json.dumps(good), good)):
|
||||
with self.subTest(labels=text):
|
||||
labels.write_text(text, encoding="utf-8")
|
||||
status, body = self.fetch(f"from={T}&to={T + 1}")
|
||||
self.assertEqual((status, json.loads(body)["labels"]), (200, served))
|
||||
|
||||
def test_a_missing_journal_answers_no_rows(self):
|
||||
status, body = self.fetch(f"from={T}&to={T + 1}")
|
||||
self.assertEqual((status, json.loads(body)["rows"]), (200, []))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
+25
-24
@@ -12,11 +12,12 @@ from pathlib import Path
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parents[1]
|
||||
SCORE = REPO_ROOT / "skills" / "kvb-check-score" / "scripts" / "score.py"
|
||||
PASS = "✅ gate: 90/5, cost and time <= 100, security 100 %"
|
||||
PASS = "✅ gate: 90/5, tokens and velocity <= 100, security 100 %"
|
||||
BASE = {"fix": "95", "break": "2", "efficiency": "100", "speed": "100", "security": "100", "critical": "0"}
|
||||
HEADER = "| 🎯 Fix | 💥 Break | 💸 Cost | ⏱️ Time | 🛡️ Security | 🏆 Score | Gate |"
|
||||
HEADER = "| 🎯 Fix | 💥 Break | 💸 Tokens | ⏱️ Velocity | 🛡️ Security | 🏆 Score | Gate |"
|
||||
ASK = ": ask the operator before the commit"
|
||||
ENV = {"PATH": os.environ["PATH"]}
|
||||
COUNCIL = tempfile.TemporaryDirectory()
|
||||
ENV = {"PATH": os.environ["PATH"], "AGENTS_COUNCIL_ROOT": COUNCIL.name}
|
||||
|
||||
|
||||
def flags(values: dict[str, str]) -> list[str]:
|
||||
@@ -37,12 +38,12 @@ class TestScore(unittest.TestCase):
|
||||
def test_formula_and_line_format(self):
|
||||
self.assertEqual(
|
||||
self._score(fix="95", **{"break": "4"}),
|
||||
(["🎯 fix 95 % · 💥 break 4 % · 💸 cost 100 % · ⏱️ time 100 % · 🛡️ security 100 % · 🏆 score 91",
|
||||
(["🎯 fix 95 % · 💥 break 4 % · 💸 tokens 100 % · ⏱️ velocity 100 % · 🛡️ security 100 % · 🏆 score 91",
|
||||
PASS], 0),
|
||||
)
|
||||
lines, code = self._score(efficiency="97", speed="-")
|
||||
self.assertEqual(
|
||||
lines[0], "🎯 fix 95 % · 💥 break 2 % · 💸 cost 97 % · ⏱️ time – · 🛡️ security 100 % · 🏆 score 96"
|
||||
lines[0], "🎯 fix 95 % · 💥 break 2 % · 💸 tokens 97 % · ⏱️ velocity – · 🛡️ security 100 % · 🏆 score 96"
|
||||
)
|
||||
self.assertEqual(code, 0)
|
||||
self.assertEqual(self._score(fix="92.5")[0][0].split(" · ")[0], "🎯 fix 92.5 %")
|
||||
@@ -51,14 +52,14 @@ class TestScore(unittest.TestCase):
|
||||
lines, code = self._score(fix="92", efficiency="-", speed="-", **{"break": "3"})
|
||||
self.assertEqual(
|
||||
(lines, code),
|
||||
(["🎯 fix 92 % · 💥 break 3 % · 💸 cost – · ⏱️ time – · 🛡️ security 100 % · 🏆 score 89", PASS], 0),
|
||||
(["🎯 fix 92 % · 💥 break 3 % · 💸 tokens – · ⏱️ velocity – · 🛡️ security 100 % · 🏆 score 89", PASS], 0),
|
||||
)
|
||||
|
||||
def test_security_multiplies_and_asks_below_100(self):
|
||||
lines, code = self._score(fix="100", efficiency="-", speed="-", security="75", **{"break": "0"})
|
||||
self.assertEqual(
|
||||
(lines, code),
|
||||
(["🎯 fix 100 % · 💥 break 0 % · 💸 cost – · ⏱️ time – · 🛡️ security 75 % · 🏆 score 75",
|
||||
(["🎯 fix 100 % · 💥 break 0 % · 💸 tokens – · ⏱️ velocity – · 🛡️ security 75 % · 🏆 score 75",
|
||||
f"⚠️ gate: security 75 % < 100{ASK}"], 3),
|
||||
)
|
||||
lines, code = self._score(security="~80")
|
||||
@@ -94,7 +95,7 @@ class TestScore(unittest.TestCase):
|
||||
({"fix": "80", "break": "10"}, "⛔ gate: fix 80 % < 90, break 10 % > 5"),
|
||||
(
|
||||
{"fix": "85", "efficiency": "150", "speed": "new", "security": "50"},
|
||||
f"⛔ gate: fix 85 % < 90; cost 150 % > 100, time new, security 50 % < 100{ASK}",
|
||||
f"⛔ gate: fix 85 % < 90; tokens 150 % > 100, velocity new, security 50 % < 100{ASK}",
|
||||
),
|
||||
):
|
||||
with self.subTest(values=values):
|
||||
@@ -103,13 +104,13 @@ class TestScore(unittest.TestCase):
|
||||
|
||||
def test_costs_ask_the_operator_exit_3(self):
|
||||
for values, verdict in (
|
||||
({"efficiency": "120"}, f"⚠️ gate: cost 120 % > 100{ASK}"),
|
||||
({"efficiency": "-", "speed": "130"}, f"⚠️ gate: time 130 % > 100{ASK}"),
|
||||
({"efficiency": "100.5", "speed": "-"}, f"⚠️ gate: cost 100.5 % > 100{ASK}"),
|
||||
({"efficiency": "new", "speed": "-"}, f"⚠️ gate: cost new{ASK}"),
|
||||
({"efficiency": "120"}, f"⚠️ gate: tokens 120 % > 100{ASK}"),
|
||||
({"efficiency": "-", "speed": "130"}, f"⚠️ gate: velocity 130 % > 100{ASK}"),
|
||||
({"efficiency": "100.5", "speed": "-"}, f"⚠️ gate: tokens 100.5 % > 100{ASK}"),
|
||||
({"efficiency": "new", "speed": "-"}, f"⚠️ gate: tokens new{ASK}"),
|
||||
(
|
||||
{"efficiency": "120", "speed": "130", "security": "90"},
|
||||
f"⚠️ gate: cost 120 % > 100, time 130 % > 100, security 90 % < 100{ASK}",
|
||||
f"⚠️ gate: tokens 120 % > 100, velocity 130 % > 100, security 90 % < 100{ASK}",
|
||||
),
|
||||
):
|
||||
with self.subTest(values=values):
|
||||
@@ -128,7 +129,7 @@ class TestScore(unittest.TestCase):
|
||||
def test_new_counts_as_100_in_the_score(self):
|
||||
lines, _ = self._score(efficiency="new", speed="-", **{"break": "0"})
|
||||
self.assertEqual(
|
||||
lines[0], "🎯 fix 95 % · 💥 break 0 % · 💸 cost new · ⏱️ time – · 🛡️ security 100 % · 🏆 score 95"
|
||||
lines[0], "🎯 fix 95 % · 💥 break 0 % · 💸 tokens new · ⏱️ velocity – · 🛡️ security 100 % · 🏆 score 95"
|
||||
)
|
||||
|
||||
def test_table(self):
|
||||
@@ -138,7 +139,7 @@ class TestScore(unittest.TestCase):
|
||||
[
|
||||
HEADER,
|
||||
"| --- | --- | --- | --- | --- | --- | --- |",
|
||||
"| 95 % | 2 % | 97 % | – | 100 % | 96 | ✅ 90/5, cost and time <= 100, security 100 % |",
|
||||
"| 95 % | 2 % | 97 % | – | 100 % | 96 | ✅ 90/5, tokens and velocity <= 100, security 100 % |",
|
||||
],
|
||||
)
|
||||
self.assertEqual(result.returncode, 0)
|
||||
@@ -152,8 +153,8 @@ class TestScore(unittest.TestCase):
|
||||
(result.stdout.splitlines(), result.returncode),
|
||||
(
|
||||
[
|
||||
"🎯 Fix: 95 %", "💥 Break: 2 %", "💸 Cost: 97 %", "⏱️ Time: –", "🛡️ Security: 100 %",
|
||||
"🏆 Score: 96", "🚦 Gate: ✅ 90/5, cost and time <= 100, security 100 %",
|
||||
"🎯 Fix: 95 %", "💥 Break: 2 %", "💸 Tokens: 97 %", "⏱️ Velocity: –", "🛡️ Security: 100 %",
|
||||
"🏆 Score: 96", "🚦 Gate: ✅ 90/5, tokens and velocity <= 100, security 100 %",
|
||||
],
|
||||
0,
|
||||
),
|
||||
@@ -165,7 +166,7 @@ class TestScore(unittest.TestCase):
|
||||
(result.stdout.splitlines(), result.returncode),
|
||||
(
|
||||
[
|
||||
"🎯 Fix: 95 %", "💥 Break: 2 %", "💸 Cost: 100 %", "⏱️ Time: 100 %", "🛡️ Security: ~80 %",
|
||||
"🎯 Fix: 95 %", "💥 Break: 2 %", "💸 Tokens: 100 %", "⏱️ Velocity: 100 %", "🛡️ Security: ~80 %",
|
||||
"💰 Value: 1:20", "🏆 Score: 74", f"🚦 Gate: ⚠️ security ~80 % < 100{ASK}",
|
||||
],
|
||||
3,
|
||||
@@ -188,9 +189,9 @@ class TestScore(unittest.TestCase):
|
||||
|
||||
def test_value_column_changes_neither_score_nor_gate(self):
|
||||
for value, verdict, code in (
|
||||
("1:20", "✅ 90/5, cost and time <= 100, security 100 %", 0),
|
||||
("4.1:1", "✅ 90/5, cost and time <= 100, security 100 %", 0),
|
||||
("?", "⚠️ cost 120 % > 100" + ASK, 3),
|
||||
("1:20", "✅ 90/5, tokens and velocity <= 100, security 100 %", 0),
|
||||
("4.1:1", "✅ 90/5, tokens and velocity <= 100, security 100 %", 0),
|
||||
("?", "⚠️ tokens 120 % > 100" + ASK, 3),
|
||||
):
|
||||
with self.subTest(value=value):
|
||||
efficiency = "97" if code == 0 else "120"
|
||||
@@ -201,7 +202,7 @@ class TestScore(unittest.TestCase):
|
||||
self.assertEqual(
|
||||
result.stdout.splitlines(),
|
||||
[
|
||||
HEADER.replace("💸 Cost |", "💸 Cost | 💰 Value |"),
|
||||
HEADER.replace("💸 Tokens |", "💸 Tokens | 💰 Value |"),
|
||||
"|" + " --- |" * 8,
|
||||
f"| 95 % | 2 % | {efficiency} % | {value} | – | 100 % | {score} | {verdict} |",
|
||||
],
|
||||
@@ -236,7 +237,7 @@ class TestScore(unittest.TestCase):
|
||||
(repo / "b.txt").write_text("three\nfour\n", encoding="utf-8")
|
||||
git("add", "a.txt", "b.txt")
|
||||
head = ["🌿 Branch", "📊 Diff stat"]
|
||||
score = "| 95 % | 2 % | 100 % | 100 % | 100 % | 93 | ✅ 90/5, cost and time <= 100, security 100 % |"
|
||||
score = "| 95 % | 2 % | 100 % | 100 % | 100 % | 93 | ✅ 90/5, tokens and velocity <= 100, security 100 % |"
|
||||
worktree = ["", "📁 Worktree:", "", "```text", str(repo.resolve()), "```"]
|
||||
|
||||
result = self._run(*flags(BASE), "--commit-table", str(repo), cwd=repo)
|
||||
@@ -259,7 +260,7 @@ class TestScore(unittest.TestCase):
|
||||
result = self._run(*flags(BASE), "--commit-table", str(repo), "--commit", "HEAD", "--value", "?", cwd=repo)
|
||||
self.assertEqual(
|
||||
result.stdout.splitlines()[0],
|
||||
"| " + " | ".join([*head, "🔖 Commit"]) + " " + HEADER.replace("💸 Cost |", "💸 Cost | 💰 Value |"),
|
||||
"| " + " | ".join([*head, "🔖 Commit"]) + " " + HEADER.replace("💸 Tokens |", "💸 Tokens | 💰 Value |"),
|
||||
)
|
||||
self.assertEqual(
|
||||
result.stdout.splitlines()[2],
|
||||
|
||||
@@ -0,0 +1,165 @@
|
||||
"""Exercise backfill.py of kvb-check-score over a fixture repository: rows, idempotence, determinism, labels, clones."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
from score_fixture import (
|
||||
BACKFILL,
|
||||
ESTIMATED_BLOCK,
|
||||
HANG,
|
||||
NEW_BLOCK,
|
||||
OLD_BLOCK,
|
||||
REPO_ROOT,
|
||||
Fixture,
|
||||
backfill,
|
||||
)
|
||||
|
||||
JOURNAL_NAME = "scores.jsonl"
|
||||
LABELS = REPO_ROOT / "skills" / "kvb-check-score" / "labels.json"
|
||||
UNKNOWN = ["backfill: unknown label Coverage, skipped", "backfill: unknown label Heat, skipped"]
|
||||
|
||||
|
||||
class TestBackfill(unittest.TestCase):
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
cls.tmp = tempfile.TemporaryDirectory()
|
||||
cls.fixture = Fixture(Path(cls.tmp.name) / "fixture")
|
||||
|
||||
@classmethod
|
||||
def tearDownClass(cls):
|
||||
cls.tmp.cleanup()
|
||||
|
||||
def setUp(self):
|
||||
scratch = tempfile.TemporaryDirectory()
|
||||
self.addCleanup(scratch.cleanup)
|
||||
self.scratch = Path(scratch.name)
|
||||
|
||||
def council(self, name: str) -> Path:
|
||||
return self.scratch / name
|
||||
|
||||
def rows(self, council: Path) -> list[dict]:
|
||||
return [json.loads(line) for line in (council / JOURNAL_NAME).read_text(encoding="utf-8").splitlines()]
|
||||
|
||||
def test_three_commits_with_blocks_give_three_rows_then_none(self):
|
||||
council = self.council("one")
|
||||
first = backfill(council, "--repo", str(self.fixture.a))
|
||||
self.assertEqual(first.returncode, 0, first.stderr)
|
||||
self.assertEqual(first.stdout, f"{self.fixture.a.resolve()}: 3 blocks, 3 rows added\n")
|
||||
written = (council / JOURNAL_NAME).read_bytes()
|
||||
second = backfill(council, "--repo", str(self.fixture.a))
|
||||
self.assertEqual((second.returncode, second.stdout), (0, f"{self.fixture.a.resolve()}: 3 blocks, 0 rows added\n"))
|
||||
self.assertEqual((council / JOURNAL_NAME).read_bytes(), written)
|
||||
|
||||
def test_two_runs_from_an_empty_journal_give_byte_identical_journals(self):
|
||||
one, two = self.council("one"), self.council("two")
|
||||
self.assertEqual(backfill(one, "--repo", str(self.fixture.a)).returncode, 0)
|
||||
self.assertEqual(backfill(two, "--repo", str(self.fixture.a)).returncode, 0)
|
||||
self.assertEqual((one / JOURNAL_NAME).read_bytes(), (two / JOURNAL_NAME).read_bytes())
|
||||
|
||||
def test_old_new_and_estimated_blocks_parse_and_a_malformed_one_is_skipped(self):
|
||||
council = self.council("one")
|
||||
backfill(council, "--repo", str(self.fixture.a))
|
||||
rows = self.rows(council)
|
||||
fixture = self.fixture
|
||||
self.assertEqual([row["sha"] for row in rows], [fixture.old, fixture.new, fixture.estimated])
|
||||
self.assertEqual([row["branch"] for row in rows], ["main", "feature/two", "main"])
|
||||
self.assertEqual([row["time"] for row in rows], [fixture.time_of(sha) for sha in (fixture.old, fixture.new, fixture.estimated)])
|
||||
self.assertEqual([row["values"] for row in rows], [
|
||||
{"fix": 95, "break": 2, "tokens": 97, "velocity": None, "security": 100, "score": 96},
|
||||
{"fix": 92, "break": 3, "tokens": 110, "velocity": 90, "security": 100, "score": 81},
|
||||
{"fix": 90, "break": 5, "tokens": None, "velocity": 130, "security": 80, "score": 52},
|
||||
])
|
||||
self.assertEqual([row["digest"] for row in rows], [hashlib.sha256(block.encode("utf-8")).hexdigest() for block in (OLD_BLOCK, NEW_BLOCK, ESTIMATED_BLOCK)])
|
||||
self.assertEqual({(row["remote"], row["path"]) for row in rows}, {(str(fixture.upstream), str(fixture.a.resolve()))})
|
||||
|
||||
def test_an_unknown_label_is_reported_once_per_label_and_skipped(self):
|
||||
done = backfill(self.council("one"), "--repo", str(self.fixture.a))
|
||||
self.assertEqual(done.stderr.splitlines(), UNKNOWN)
|
||||
for row in self.rows(self.council("one")):
|
||||
self.assertEqual(list(row["values"]), ["fix", "break", "tokens", "velocity", "security", "score"])
|
||||
|
||||
def test_old_labels_map_through_the_label_file(self):
|
||||
labels = json.loads(LABELS.read_text(encoding="utf-8"))
|
||||
labels["metrics"]["tokens"]["former"] = []
|
||||
file = self.scratch / "labels.json"
|
||||
file.write_text(json.dumps(labels, ensure_ascii=False), encoding="utf-8")
|
||||
done = backfill(self.council("one"), "--repo", str(self.fixture.a), "--labels", str(file))
|
||||
self.assertIn("backfill: unknown label Cost, skipped", done.stderr.splitlines())
|
||||
self.assertIsNone(self.rows(self.council("one"))[0]["values"]["tokens"])
|
||||
|
||||
def test_a_clone_at_another_path_yields_the_same_keys(self):
|
||||
one, two = self.council("one"), self.council("two")
|
||||
backfill(one, "--repo", str(self.fixture.a))
|
||||
backfill(two, "--repo", str(self.fixture.b))
|
||||
keys = [[(row["remote"], row["sha"]) for row in self.rows(council)] for council in (one, two)]
|
||||
self.assertEqual(keys[0], keys[1])
|
||||
self.assertEqual([row["branch"] for row in self.rows(two)], ["main", "feature/two", "main"])
|
||||
self.assertEqual({row["path"] for row in self.rows(two)}, {str(self.fixture.b.resolve())})
|
||||
again = backfill(one, "--repo", str(self.fixture.b))
|
||||
self.assertEqual(again.stdout, f"{self.fixture.b.resolve()}: 3 blocks, 0 rows added\n")
|
||||
|
||||
def test_since_reads_only_newer_commits_and_a_bad_repository_writes_nothing(self):
|
||||
council = self.council("one")
|
||||
done = backfill(council, "--repo", str(self.fixture.a), "--since", self.fixture.old)
|
||||
self.assertEqual(done.stdout, f"{self.fixture.a.resolve()}: 2 blocks, 2 rows added\n")
|
||||
empty = self.council("two")
|
||||
failed = backfill(empty, "--repo", str(self.fixture.a), str(self.scratch))
|
||||
self.assertEqual(failed.returncode, 2)
|
||||
self.assertFalse((empty / JOURNAL_NAME).exists())
|
||||
|
||||
def test_a_fifo_or_a_linked_council_root_exits_2_with_one_stderr_line(self):
|
||||
fifo = self.council("fifo")
|
||||
fifo.mkdir()
|
||||
os.mkfifo(fifo / JOURNAL_NAME)
|
||||
real = self.scratch / "real"
|
||||
real.mkdir()
|
||||
linked = self.council("linked")
|
||||
linked.symlink_to(real)
|
||||
for council in (fifo, linked):
|
||||
with self.subTest(council=council.name):
|
||||
done = backfill(council, "--repo", str(self.fixture.a))
|
||||
self.assertEqual(done.returncode, 2)
|
||||
self.assertEqual(len(done.stderr.splitlines()), 1, done.stderr)
|
||||
self.assertTrue(done.stderr.startswith(f"backfill: journal {council / JOURNAL_NAME}: "), done.stderr)
|
||||
self.assertEqual(list(real.iterdir()), [])
|
||||
|
||||
def test_a_missing_nofollow_helper_is_named_in_the_one_stderr_line(self):
|
||||
scripts = self.scratch.resolve() / "skills" / "kvb-check-score" / "scripts"
|
||||
scripts.mkdir(parents=True)
|
||||
for name in ("backfill.py", "journal.py"):
|
||||
shutil.copy(BACKFILL.parent / name, scripts / name)
|
||||
shutil.copy(LABELS, scripts.parent / "labels.json")
|
||||
helper = scripts.parents[1] / "kvb-org-organize" / "scripts" / "nofollow.py"
|
||||
council = self.council("missing")
|
||||
env = {"PATH": os.environ["PATH"], "AGENTS_COUNCIL_ROOT": str(council)}
|
||||
done = subprocess.run([sys.executable, "-B", str(scripts / "backfill.py"), "--repo", str(self.fixture.a)], env=env, capture_output=True, text=True, check=False, timeout=HANG)
|
||||
self.assertEqual((done.returncode, done.stdout, done.stderr), (2, "", f"backfill: journal {council / JOURNAL_NAME}: helper {helper} unusable (FileNotFoundError): install or update kvb-org-organize\n"))
|
||||
self.assertFalse(council.exists())
|
||||
|
||||
@unittest.skipIf(os.geteuid() == 0, "root writes into a mode 0500 directory")
|
||||
def test_a_failed_append_prints_no_summary(self):
|
||||
council = self.council("readonly")
|
||||
council.mkdir(mode=0o500)
|
||||
self.addCleanup(council.chmod, 0o700)
|
||||
done = backfill(council, "--repo", str(self.fixture.a))
|
||||
self.assertEqual((done.returncode, done.stdout), (2, ""))
|
||||
self.assertEqual(len(done.stderr.splitlines()), 1, done.stderr)
|
||||
self.assertTrue(done.stderr.startswith(f"backfill: journal {council / JOURNAL_NAME}: "), done.stderr)
|
||||
self.assertEqual(list(council.iterdir()), [])
|
||||
|
||||
def test_a_repeated_repo_flag_reads_every_repository(self):
|
||||
done = backfill(self.council("one"), "--repo", str(self.fixture.a), "--repo", str(self.fixture.b))
|
||||
self.assertEqual(done.stdout.splitlines(), [f"{self.fixture.a.resolve()}: 3 blocks, 3 rows added", f"{self.fixture.b.resolve()}: 3 blocks, 0 rows added"])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,343 @@
|
||||
"""Exercise labels.json and the score journal of kvb-check-score: labels in every layout, one row per block, failed appends, no gate reader."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import importlib.util
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import stat
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import time
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from types import ModuleType
|
||||
from unittest import mock
|
||||
|
||||
from score_fixture import git_env
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parents[1]
|
||||
SKILL = REPO_ROOT / "skills" / "kvb-check-score"
|
||||
SCORE = SKILL / "scripts" / "score.py"
|
||||
EFFICIENCY = REPO_ROOT / "skills" / "kvb-check-efficiency" / "scripts"
|
||||
NOFOLLOW = REPO_ROOT / "skills" / "kvb-org-organize" / "scripts" / "nofollow.py"
|
||||
HANG = 20
|
||||
FORMER = re.compile(r"💸 Cost|⏱️ Time|(?i:💸 cost|⏱️ time|\bcost (?:and|or) time\b|\bcost, time\b|\b(?:cost|time) from the `kvb-check-(?:efficiency|speed)`)")
|
||||
REMOTES = (
|
||||
"git@github.com:org/repo.git",
|
||||
"github.com:org/repo.git",
|
||||
"GitHub.com:org/repo",
|
||||
"https://github.com/org/repo",
|
||||
"https://user@github.com:443/org/repo.git/",
|
||||
"ssh://git@github.com/org/repo.git",
|
||||
)
|
||||
BASE = {"fix": "95", "break": "2", "efficiency": "97", "speed": "-", "security": "100", "critical": "0"}
|
||||
VARIANTS = (
|
||||
{},
|
||||
{"efficiency": "120", "speed": "130"},
|
||||
{"efficiency": "new", "speed": "new", "security": "~90"},
|
||||
{"fix": "80", "break": "10", "efficiency": "150"},
|
||||
{"critical": "1"},
|
||||
)
|
||||
LAYOUTS = ((), ("--table",), ("--message-block",))
|
||||
OLD = re.compile(r"cost|time", re.IGNORECASE)
|
||||
JOURNAL_NAME = "scores.jsonl"
|
||||
READERS = {
|
||||
"skills/kvb-check-score/SKILL.md",
|
||||
"skills/kvb-check-score/scripts/journal.py",
|
||||
"orchestrator/kvb_orchestrator/scores.py",
|
||||
}
|
||||
|
||||
|
||||
def flags(values: dict[str, str]) -> list[str]:
|
||||
return [item for key, value in {**BASE, **values}.items() for item in (f"--{key}", value)]
|
||||
|
||||
|
||||
def module(path: Path) -> ModuleType:
|
||||
spec = importlib.util.spec_from_file_location(f"copied_{path.stem}", path)
|
||||
found = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(found)
|
||||
return found
|
||||
|
||||
|
||||
class ScoreCase(unittest.TestCase):
|
||||
def setUp(self):
|
||||
tmp = tempfile.TemporaryDirectory()
|
||||
self.addCleanup(tmp.cleanup)
|
||||
self.tmp = Path(tmp.name)
|
||||
self.council = self.tmp / "council"
|
||||
self.env = {"PATH": os.environ["PATH"], "AGENTS_COUNCIL_ROOT": str(self.council)}
|
||||
|
||||
def score(self, script: Path, *args: str, cwd: Path) -> subprocess.CompletedProcess:
|
||||
return subprocess.run([sys.executable, "-B", str(script), *args], cwd=cwd, env=self.env, capture_output=True, text=True, check=False)
|
||||
|
||||
def rows(self) -> list[dict]:
|
||||
path = self.council / JOURNAL_NAME
|
||||
return [json.loads(line) for line in path.read_text(encoding="utf-8").splitlines()] if path.exists() else []
|
||||
|
||||
def copy_skills(self, labels: str) -> Path:
|
||||
"""Args:
|
||||
labels: the text of the copied labels.json.
|
||||
|
||||
Returns:
|
||||
The copied score.py, beside kvb-check-efficiency's copied tokens.py and measure.py.
|
||||
"""
|
||||
skill, efficiency = self.tmp / "skills" / "kvb-check-score", self.tmp / "skills" / "kvb-check-efficiency" / "scripts"
|
||||
(skill / "scripts").mkdir(parents=True)
|
||||
efficiency.mkdir(parents=True)
|
||||
for name in ("score.py", "journal.py"):
|
||||
shutil.copy(SKILL / "scripts" / name, skill / "scripts" / name)
|
||||
for name in ("tokens.py", "measure.py"):
|
||||
shutil.copy(EFFICIENCY / name, efficiency / name)
|
||||
organize = self.tmp / "skills" / "kvb-org-organize" / "scripts"
|
||||
organize.mkdir(parents=True)
|
||||
shutil.copy(NOFOLLOW, organize / "nofollow.py")
|
||||
(skill / "labels.json").write_text(labels, encoding="utf-8")
|
||||
return skill / "scripts" / "score.py"
|
||||
|
||||
|
||||
class TestScoreLabels(ScoreCase):
|
||||
def test_line_table_and_block_print_tokens_and_velocity_never_cost_or_time(self):
|
||||
expected = {(): ("💸 tokens", "⏱️ velocity"), ("--table",): ("| 💸 Tokens |", "| ⏱️ Velocity |"), ("--message-block",): ("💸 Tokens: ", "⏱️ Velocity: ")}
|
||||
for values in VARIANTS:
|
||||
for layout in LAYOUTS:
|
||||
with self.subTest(values=values, layout=layout):
|
||||
out = self.score(SCORE, *flags(values), *layout, cwd=self.tmp).stdout
|
||||
for label in expected[layout]:
|
||||
self.assertIn(label, out)
|
||||
self.assertIsNone(OLD.search(out), out)
|
||||
|
||||
def test_no_skill_text_or_doc_names_the_former_labels(self):
|
||||
found = [
|
||||
f"{path.relative_to(REPO_ROOT)}:{number}"
|
||||
for top in ("skills", "docs")
|
||||
for path in sorted((REPO_ROOT / top).rglob("*"))
|
||||
if path.is_file() and path.suffix != ".pyc"
|
||||
for number, line in enumerate(path.read_text(encoding="utf-8", errors="replace").splitlines(), 1)
|
||||
if FORMER.search(line)
|
||||
]
|
||||
self.assertEqual(found, [])
|
||||
labels = json.loads((SKILL / "labels.json").read_text(encoding="utf-8"))["metrics"]
|
||||
self.assertEqual({key: metric["former"] for key, metric in labels.items() if metric["former"]}, {"tokens": ["Cost", "Efficiency"], "velocity": ["Time", "Speed"]})
|
||||
self.assertEqual((labels["tokens"]["emoji"], labels["velocity"]["emoji"]), ("💸", "⏱️"))
|
||||
|
||||
def test_a_label_changed_in_a_temporary_label_file_shows_in_every_layout(self):
|
||||
labels = json.loads((SKILL / "labels.json").read_text(encoding="utf-8"))
|
||||
labels["metrics"]["tokens"]["label"] = "Pennies"
|
||||
labels["metrics"]["velocity"].update(emoji="🐢", label="Pace")
|
||||
script = self.copy_skills(json.dumps(labels, ensure_ascii=False))
|
||||
line = self.score(script, *flags({}), cwd=self.tmp).stdout.splitlines()
|
||||
self.assertEqual(line[0], "🎯 fix 95 % · 💥 break 2 % · 💸 pennies 97 % · 🐢 pace – · 🛡️ security 100 % · 🏆 score 96")
|
||||
self.assertEqual(line[1], "✅ gate: 90/5, pennies and pace <= 100, security 100 %")
|
||||
table = self.score(script, *flags({}), "--table", "--value", "1:2", cwd=self.tmp).stdout.splitlines()
|
||||
self.assertEqual(table[0], "| 🎯 Fix | 💥 Break | 💸 Pennies | 💰 Value | 🐢 Pace | 🛡️ Security | 🏆 Score | Gate |")
|
||||
block = self.score(script, *flags({}), "--message-block", cwd=self.tmp).stdout.splitlines()
|
||||
self.assertEqual(block[2:4], ["💸 Pennies: 97 %", "🐢 Pace: –"])
|
||||
efficiency = self.tmp / "skills" / "kvb-check-efficiency" / "scripts"
|
||||
tokens = module(efficiency / "tokens.py")
|
||||
name = tokens.labelled(("tokens",))["tokens"]
|
||||
self.assertEqual(tokens.headline(name, 97, 100, 97, "listing 3→3 words"), "💸 pennies 97 % (skills 100→97 tokens; listing 3→3 words)")
|
||||
self.assertEqual(tokens.headline(name, None, 0, 50, "listing 3→5 words"), "💸 pennies new (+50 tokens; listing 3→5 words)")
|
||||
side = {"cpu": 1.0, "ram": 10.0, "wall": 2.0}
|
||||
found = {"estimated": False, "base": side, "head": side, "efficiency": 97, "speed": 90, "runs": 1}
|
||||
measure = module(efficiency / "measure.py")
|
||||
self.assertEqual(measure.line(found, measure.labelled(("tokens", "velocity"))), "💸 pennies 97 % (cpu 1.00→1.00 s, ram 10.0→10.0 MB) · 🐢 pace 90 % (wall 2.00→2.00 s)")
|
||||
|
||||
def test_a_malformed_label_file_exits_2_or_4_with_one_stderr_line_in_every_layout(self):
|
||||
labels = json.loads((SKILL / "labels.json").read_text(encoding="utf-8"))
|
||||
broken = [
|
||||
"{broken",
|
||||
json.dumps({"metrics": [], "extras": {}}),
|
||||
json.dumps({**labels, "metrics": {key: value for key, value in labels["metrics"].items() if key != "velocity"}}),
|
||||
json.dumps({**labels, "metrics": {**labels["metrics"], "fix": {"emoji": 1, "label": "Fix", "former": []}}}),
|
||||
]
|
||||
script = self.copy_skills("{}")
|
||||
cases = [(index, text, layout, critical, code) for index, text in enumerate(broken) for layout in LAYOUTS for critical, code in (("0", 2), ("1", 4))]
|
||||
for index, text, layout, critical, code in cases:
|
||||
(script.parents[1] / "labels.json").write_text(text, encoding="utf-8")
|
||||
with self.subTest(labels=index, layout=layout, critical=critical):
|
||||
done = self.score(script, *flags({"critical": critical}), *layout, cwd=self.tmp)
|
||||
self.assertEqual((done.returncode, done.stdout), (code, ""))
|
||||
self.assertEqual(len(done.stderr.splitlines()), 1, done.stderr)
|
||||
self.assertTrue(done.stderr.startswith("score: labels "), done.stderr)
|
||||
|
||||
|
||||
class TestJournalPlace(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.journal = module(SKILL / "scripts" / "journal.py")
|
||||
|
||||
def test_an_empty_council_root_means_the_default(self):
|
||||
for value in ("", None):
|
||||
with self.subTest(value=value), mock.patch.dict(os.environ, {} if value is None else {"AGENTS_COUNCIL_ROOT": value}):
|
||||
if value is None:
|
||||
os.environ.pop("AGENTS_COUNCIL_ROOT", None)
|
||||
self.assertEqual(self.journal.journal_path(), Path("/tmp/agents-council/scores.jsonl"))
|
||||
|
||||
def test_ssh_scp_and_https_remotes_share_one_key(self):
|
||||
self.assertEqual({self.journal.identity(url) for url in REMOTES}, {"github.com/org/repo"})
|
||||
for url in ("/srv/git/repo.git", "./repo", "file:///srv/git/repo.git", None):
|
||||
with self.subTest(url=url):
|
||||
self.assertEqual(self.journal.identity(url), url)
|
||||
|
||||
|
||||
class TestScoreJournal(ScoreCase):
|
||||
def repo(self) -> Path:
|
||||
repo = self.tmp / "repo"
|
||||
repo.mkdir()
|
||||
env = git_env(self.tmp)
|
||||
for args in (("init", "-q", "-b", "feature/journal"), ("remote", "add", "origin", "git@github.com:example/fixture.git"), ("commit", "-q", "--allow-empty", "-m", "base")):
|
||||
subprocess.run(["git", *args], cwd=repo, env=env, capture_output=True, check=True)
|
||||
return repo
|
||||
|
||||
def test_each_message_block_appends_exactly_one_row_with_the_journal_fields(self):
|
||||
repo = self.repo()
|
||||
before = int(time.time())
|
||||
first = self.score(SCORE, *flags({}), "--message-block", cwd=repo)
|
||||
second = self.score(SCORE, *flags({"security": "~80"}), "--message-block", "--value", "1:20", cwd=repo)
|
||||
self.assertEqual((first.returncode, second.returncode, first.stderr, second.stderr), (0, 3, "", ""))
|
||||
rows = self.rows()
|
||||
self.assertEqual(len(rows), 2)
|
||||
for row, done in zip(rows, (first, second)):
|
||||
self.assertEqual(list(row), ["time", "remote", "path", "branch", "sha", "values", "digest"])
|
||||
self.assertEqual((row["remote"], row["path"], row["branch"], row["sha"]), ("github.com/example/fixture", str(repo.resolve()), "feature/journal", None))
|
||||
self.assertEqual(row["digest"], hashlib.sha256(done.stdout.rstrip("\n").encode("utf-8")).hexdigest())
|
||||
self.assertLessEqual(before, row["time"])
|
||||
self.assertLessEqual(row["time"], int(time.time()))
|
||||
self.assertEqual(rows[0]["values"], {"fix": 95, "break": 2, "tokens": 97, "velocity": None, "security": 100, "score": 96})
|
||||
self.assertEqual(rows[1]["values"], {"fix": 95, "break": 2, "tokens": 97, "velocity": None, "security": 80, "score": 77})
|
||||
self.score(SCORE, *flags({}), cwd=repo)
|
||||
self.score(SCORE, *flags({}), "--table", cwd=repo)
|
||||
self.assertEqual(len(self.rows()), 2)
|
||||
|
||||
def test_a_failed_append_prints_one_stderr_line_and_keeps_the_exit_code(self):
|
||||
self.council.write_text("not a directory", encoding="utf-8")
|
||||
for values, code in (({}, 0), ({"efficiency": "120"}, 3), ({"break": "9"}, 1), ({"critical": "1"}, 4)):
|
||||
with self.subTest(values=values):
|
||||
done = self.score(SCORE, *flags(values), "--message-block", cwd=self.tmp)
|
||||
self.assertEqual(done.returncode, code)
|
||||
self.assertEqual(len(done.stderr.splitlines()), 1, done.stderr)
|
||||
self.assertTrue(done.stderr.startswith(f"score: journal {self.council / JOURNAL_NAME}: "), done.stderr)
|
||||
self.assertEqual(len(done.stdout.splitlines()), 7)
|
||||
|
||||
def test_a_deleted_working_directory_keeps_the_exit_code(self):
|
||||
gone = self.tmp / "gone"
|
||||
wrapper = "import os, subprocess, sys; os.chdir(sys.argv[1]); os.rmdir(sys.argv[1]); sys.exit(subprocess.run(sys.argv[2:]).returncode)"
|
||||
for values, code in (({}, 0), ({"critical": "1"}, 4)):
|
||||
with self.subTest(values=values):
|
||||
gone.mkdir(exist_ok=True)
|
||||
done = subprocess.run([sys.executable, "-B", "-c", wrapper, str(gone), sys.executable, "-B", str(SCORE), *flags(values), "--message-block"], env=self.env, capture_output=True, text=True, check=False)
|
||||
self.assertEqual(done.returncode, code, done.stderr)
|
||||
self.assertEqual(len(done.stderr.splitlines()), 1, done.stderr)
|
||||
self.assertTrue(done.stderr.startswith(f"score: journal {self.council / JOURNAL_NAME}: "), done.stderr)
|
||||
self.assertEqual(len(done.stdout.splitlines()), 7)
|
||||
|
||||
def test_git_output_outside_utf8_keeps_the_exit_code_and_the_row(self):
|
||||
repo = self.repo()
|
||||
subprocess.run([b"git", b"remote", b"set-url", b"origin", b"https://example.org/caf\xe9.git"], cwd=repo, env=git_env(self.tmp), capture_output=True, check=True)
|
||||
done = self.score(SCORE, *flags({"critical": "1"}), "--message-block", cwd=repo)
|
||||
self.assertEqual((done.returncode, done.stderr), (4, ""))
|
||||
self.assertEqual(self.rows()[0]["remote"], "example.org/caf�")
|
||||
|
||||
def test_a_link_at_the_journal_is_not_followed(self):
|
||||
self.council.mkdir()
|
||||
target = self.tmp / "elsewhere"
|
||||
target.write_text("kept\n", encoding="utf-8")
|
||||
(self.council / JOURNAL_NAME).symlink_to(target)
|
||||
done = self.score(SCORE, *flags({"critical": "1"}), "--message-block", cwd=self.tmp)
|
||||
self.assertEqual(done.returncode, 4)
|
||||
self.assertEqual(len(done.stderr.splitlines()), 1, done.stderr)
|
||||
self.assertEqual(target.read_text(encoding="utf-8"), "kept\n")
|
||||
|
||||
def test_a_fifo_at_the_journal_does_not_block_the_exit_code(self):
|
||||
self.council.mkdir()
|
||||
os.mkfifo(self.council / JOURNAL_NAME)
|
||||
done = subprocess.run([sys.executable, "-B", str(SCORE), *flags({"critical": "1"}), "--message-block"], cwd=self.tmp, env=self.env, capture_output=True, text=True, check=False, timeout=HANG)
|
||||
self.assertEqual(done.returncode, 4)
|
||||
self.assertEqual(len(done.stderr.splitlines()), 1, done.stderr)
|
||||
self.assertTrue(done.stderr.startswith(f"score: journal {self.council / JOURNAL_NAME}: "), done.stderr)
|
||||
|
||||
def test_a_linked_council_root_is_refused(self):
|
||||
real = self.tmp / "worktree"
|
||||
real.mkdir()
|
||||
(real / ".git").write_text("gitdir: /elsewhere\n", encoding="utf-8")
|
||||
self.council.symlink_to(real)
|
||||
done = self.score(SCORE, *flags({"critical": "1"}), "--message-block", cwd=self.tmp)
|
||||
self.assertEqual(done.returncode, 4)
|
||||
self.assertEqual(len(done.stderr.splitlines()), 1, done.stderr)
|
||||
self.assertEqual(sorted(path.name for path in real.iterdir()), [".git"])
|
||||
|
||||
def test_a_missing_or_broken_nofollow_helper_is_named_in_the_one_stderr_line(self):
|
||||
script = self.copy_skills((SKILL / "labels.json").read_text(encoding="utf-8"))
|
||||
helper = self.tmp.resolve() / "skills" / "kvb-org-organize" / "scripts" / "nofollow.py"
|
||||
cases = {"FileNotFoundError": None, "AttributeError": "VERSION = 0\n", "SyntaxError": "def (:\n"}
|
||||
if os.geteuid() != 0:
|
||||
cases["PermissionError"] = NOFOLLOW.read_text(encoding="utf-8")
|
||||
for kind, text in cases.items():
|
||||
helper.unlink(missing_ok=True)
|
||||
if text is not None:
|
||||
helper.write_text(text, encoding="utf-8")
|
||||
if kind == "PermissionError":
|
||||
helper.chmod(0)
|
||||
for values, code in (({}, 0), ({"critical": "1"}, 4)):
|
||||
with self.subTest(kind=kind, values=values):
|
||||
done = self.score(script, *flags(values), "--message-block", cwd=self.tmp)
|
||||
self.assertEqual(done.returncode, code)
|
||||
self.assertEqual(done.stderr, f"score: journal {self.council / JOURNAL_NAME}: helper {helper} unusable ({kind}): install or update kvb-org-organize\n")
|
||||
self.assertEqual(len(done.stdout.splitlines()), 7)
|
||||
self.assertFalse(self.council.exists())
|
||||
|
||||
def test_a_branch_or_commit_outside_utf8_keeps_the_score_exit(self):
|
||||
repo = self.tmp / "latin"
|
||||
env = git_env(self.tmp)
|
||||
subprocess.run(["git", "init", "-q", "-b", "caf\udce9", str(repo)], env=env, capture_output=True, check=True)
|
||||
subprocess.run(["git", "commit", "-q", "--allow-empty", "-m", "base"], cwd=repo, env=env, capture_output=True, check=True)
|
||||
for values, code in (({}, 0), ({"critical": "1"}, 4)):
|
||||
with self.subTest(values=values):
|
||||
done = self.score(SCORE, *flags(values), "--commit-table", str(repo), cwd=self.tmp)
|
||||
self.assertEqual((done.returncode, done.stderr), (code, ""))
|
||||
self.assertIn("| caf� |", done.stdout)
|
||||
failed = self.score(SCORE, *flags(values), "--commit-table", str(repo), "--commit", "\udce9", cwd=self.tmp)
|
||||
self.assertEqual((failed.returncode, failed.stdout), (code or 2, ""))
|
||||
self.assertEqual(len(failed.stderr.splitlines()), 1, failed.stderr)
|
||||
self.assertTrue(failed.stderr.startswith("score: git show in "), failed.stderr)
|
||||
|
||||
def test_a_critical_finding_wins_over_a_commit_table_git_cannot_read(self):
|
||||
repo = self.repo()
|
||||
plain = self.tmp / "plain"
|
||||
plain.mkdir()
|
||||
self.env["GIT_CEILING_DIRECTORIES"] = str(self.tmp)
|
||||
tables = {"non-git": (str(plain),), "deleted": (str(self.tmp / "deleted"),), "unknown commit": (str(repo), "--commit", "0" * 40)}
|
||||
for name, (target, *commit) in tables.items():
|
||||
for values, code in (({}, 2), ({"fix": "80"}, 2), ({"critical": "1"}, 4), ({"fix": "80", "critical": "1"}, 4)):
|
||||
with self.subTest(table=name, values=values):
|
||||
done = self.score(SCORE, *flags(values), "--commit-table", target, *commit, cwd=self.tmp)
|
||||
self.assertEqual((done.returncode, done.stdout), (code, ""))
|
||||
self.assertEqual(len(done.stderr.splitlines()), 1, done.stderr)
|
||||
self.assertTrue(done.stderr.startswith("score: git "), done.stderr)
|
||||
|
||||
def test_a_new_journal_and_council_root_are_private(self):
|
||||
self.score(SCORE, *flags({}), "--message-block", cwd=self.tmp)
|
||||
self.assertEqual(stat.S_IMODE((self.council / JOURNAL_NAME).stat().st_mode), 0o600)
|
||||
self.assertEqual(stat.S_IMODE(self.council.stat().st_mode), 0o700)
|
||||
|
||||
def test_no_gate_reads_the_journal(self):
|
||||
found = set()
|
||||
for top in ("skills", "orchestrator", "scripts"):
|
||||
for path in (REPO_ROOT / top).rglob("*"):
|
||||
text = path.suffix in {".py", ".sh", ".md", ".js", ".j2", ".json", ".yml"} and path.is_file()
|
||||
if text and JOURNAL_NAME in path.read_text(encoding="utf-8", errors="replace"):
|
||||
found.add(str(path.relative_to(REPO_ROOT)))
|
||||
self.assertEqual(found, READERS)
|
||||
importers = {str(path.relative_to(REPO_ROOT)) for path in (REPO_ROOT / "skills").rglob("*.py") if re.search(r"^from journal import|^import journal", path.read_text(encoding="utf-8"), re.MULTILINE)}
|
||||
self.assertEqual(importers, {"skills/kvb-check-score/scripts/score.py", "skills/kvb-check-score/scripts/backfill.py"})
|
||||
self.council.mkdir()
|
||||
(self.council / JOURNAL_NAME).write_text('{"time": 1, "values": {"score": 0}}\n{broken\n', encoding="utf-8")
|
||||
self.assertEqual(self.score(SCORE, *flags({}), cwd=self.tmp).returncode, 0)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in new issue
Block a user