Files
skills/tests/test_hub_match.py
T
Kevin Veen-BirkenbachandClaude Opus 5.5 78d4e4a6aa feat(orchestrator): Match cards show the score line as metric chips
scores.metric_chips() reads each metric's value from an agent's free score line by emoji, label or former label from labels.json, and Match cards render them as chips under the name instead of the raw score pill. New e2e: real mouse swipes send later, reject, refuse a stale card and accept at three widths; the release e2e gains its negative control. Fix stays below 90: the swipe e2e ended in ERROR in the aborted final run of 2026-11-17 without a traceback; the master gate reruns it.

🎯 Fix: 88 %
💥 Break: 3 %
💸 Tokens: –
⏱️ Velocity: –
🛡️ Security: 100 %
🏆 Score: 85
🚦 Gate: ⛔ fix 88 % < 90

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-17 19:02:17 +00:00

372 lines
22 KiB
Python

from __future__ import annotations
import json
import os
import secrets
import subprocess
import sys
import tempfile
import time
import unittest
from pathlib import Path
from unittest import mock
from hub_case import HubCase
from hub_support import APPROVAL, GIT_ENV, HUB, REPO_ROOT, key, signing
from kvb_orchestrator import batches, payloads, scores, skill_scripts
MATCH = REPO_ROOT / "skills" / "kvb-match" / "scripts" / "match.py"
CLIENT = HUB / "client"
FEEDBACK = "app.py:2: name it after what it holds\nnotes.md:3: say why"
TOLD = "app.py:2: name it after what it holds notes.md:3: say why"
def git(cwd: Path, *args: str) -> str:
done = subprocess.run(["git", *args], cwd=cwd, env={**os.environ, **GIT_ENV}, check=True, capture_output=True, text=True)
return done.stdout
def staged_repo(root: Path) -> Path:
repo = root / "demo"
repo.mkdir()
git(repo, "init", "-q", "-b", "feature/match")
(repo / "app.py").write_text("keep = 1\nold = 2\n", encoding="utf-8")
git(repo, "add", "-A")
git(repo, "commit", "-q", "-m", "base")
(repo / "app.py").write_text("keep = 1\nnew = 3\nmore = 4\n", encoding="utf-8")
(repo / "notes.md").write_text("# Notes\n\nfresh\n", encoding="utf-8")
git(repo, "add", "-A")
return repo
class MatchCase(HubCase):
def setUp(self):
super().setUp()
work = tempfile.TemporaryDirectory()
self.addCleanup(work.cleanup)
self.work = Path(work.name)
self.sandbox_view([])
def sandbox_view(self, denied: list[str]) -> None:
folder = self.state / "firewall"
folder.mkdir(parents=True, exist_ok=True)
settings = {"sandbox": {"filesystem": {"denyRead": denied}}}
files = [{"path": "/home/op/.claude/settings.json", "exists": True, "sha256": "", "settings": settings}]
(folder / "view.json").write_text(json.dumps({"written": "2026-11-14T12:00:00+00:00", "files": files}), encoding="utf-8")
def match(self, cwd: Path, *args: str) -> subprocess.CompletedProcess:
command = [sys.executable, str(MATCH), *args, "--port", str(self.port), "--client", str(CLIENT)]
return subprocess.run(command, cwd=cwd, env={**os.environ, **GIT_ENV}, capture_output=True, text=True, timeout=60, check=False)
def add(self, repo: Path) -> str:
done = self.match(repo, "add", "--repo", str(repo), "--title", "feat: match", "--score", "🎯 95 💥 3", "--gate", "pass", "--agent", "Skills", "--pages", str(self.pages))
self.assertEqual(done.returncode, 0, done.stderr)
return json.loads(done.stdout)["id"]
def cards(self) -> list[dict]:
status, _, body = self.get("/api/match")
self.assertEqual(status, 200, body)
return json.loads(body)["cards"]
def card(self, card_id: str) -> dict:
return next(card for card in self.cards() if card["id"] == card_id)
def operator_row(self, task: str, command: str, directory: str) -> None:
register = {task: {"title": command, "workdir": directory, "description": "ship the release notes", "check": "done", "roles": {"Skills": "A", "Operator": "R"}}}
self.write_council({"rasci.json": register})
class TestMatchSkill(MatchCase):
def test_add_takes_the_staged_patch_and_tree_and_list_and_show_print_them(self):
repo = staged_repo(self.work)
review = self.add(repo)
card = self.card(f"review:{review}")
self.assertEqual((card["kind"], card["hash"], card["stale"]), ("commit", git(repo, "write-tree").strip(), False))
self.assertEqual((card["branch"], card["worktree"], card["agent"], card["title"]), ("feature/match", str(repo), "Skills", "feat: match"))
self.assertEqual([entry["path"] for entry in card["files"]], ["app.py", "notes.md"])
shown = self.match(repo, "show", f"review:{review}")
self.assertEqual(shown.returncode, 0, shown.stderr)
self.assertEqual(shown.stdout, git(repo, "diff", "--cached"))
listed = self.match(repo, "list")
self.assertEqual(listed.returncode, 0, listed.stderr)
self.assertIn(f"review:{review}", listed.stdout)
self.assertIn("feat: match", listed.stdout)
def test_add_refuses_an_empty_stage(self):
repo = staged_repo(self.work)
git(repo, "commit", "-q", "-m", "all")
done = self.match(repo, "add", "--repo", str(repo), "--title", "feat: none", "--score", "🎯 1", "--gate", "pass", "--agent", "Skills", "--pages", str(self.pages))
self.assertEqual(done.returncode, 1)
self.assertIn("nothing staged", done.stderr)
self.assertEqual(self.cards(), [])
def test_a_changed_stage_turns_the_card_stale_and_adding_it_again_gives_a_fresh_card(self):
repo = staged_repo(self.work)
first = self.add(repo)
(repo / "notes.md").write_text("# Notes\n\nchanged\n", encoding="utf-8")
git(repo, "add", "-A")
listed = self.match(repo, "list")
self.assertEqual(listed.returncode, 0, listed.stderr)
self.assertIn("stale", listed.stdout)
old = self.card(f"review:{first}")
self.assertTrue(old["stale"], old)
self.assertIn("changed", old["stale_reason"])
second = self.add(repo)
fresh = self.card(f"review:{second}")
self.assertEqual((fresh["stale"], fresh["hash"]), (False, git(repo, "write-tree").strip()))
self.assertTrue(self.card(f"review:{first}")["stale"])
def test_a_stale_report_needs_the_local_guard_and_a_tree(self):
repo = staged_repo(self.work)
review = self.add(repo)
status, _, _ = self.post_json("/api/match/stale", {"id": review, "tree": "f" * 40}, {"Host": self.host})
self.assertEqual(status, 403)
status, _, _ = self.post_json("/api/match/stale", {"id": review, "tree": "nope"}, self.local())
self.assertEqual(status, 400)
self.assertFalse(self.card(f"review:{review}")["stale"])
def test_a_gone_review_or_operator_row_leaves_the_cards(self):
repo = staged_repo(self.work)
review = self.add(repo)
self.operator_row("operator/ship", "make ship", str(self.work))
self.assertEqual(sorted(card["id"] for card in self.cards()), sorted([f"review:{review}", "operator:operator/ship"]))
status, _, body = self.post_json(f"/reviews/{review}/reject", {}, self.guard(self.host))
self.assertEqual(status, 200, body)
self.write_council({"rasci.json": {}})
self.assertEqual(self.cards(), [])
class TestMatchRelease(MatchCase):
def radio(self) -> list[list[str]]:
return [args for script, args in self.scripts.calls if script == skill_scripts.RADIO]
def stack(self) -> tuple[Path, str]:
self.claim("s1", "Skills session")
repo = staged_repo(self.work)
review = self.add(repo)
self.operator_row("operator/ship", "make ship", str(self.work))
return repo, review
def decisions(self, review: str, verdict: str) -> list[dict]:
"""Args:
review: the review id of the commit card.
verdict: accept or reject for the commit card; the command card
is always rejected with FEEDBACK.
Returns:
The entries the Match page releases, built as matchEntry builds
them from the cards.
"""
commit, command = self.card(f"review:{review}"), self.card("operator:operator/ship")
rejected = {"verb": "reject", "channel": "call", "reason": FEEDBACK}
first = {"kind": "review", "id": commit["ref"], "verb": "approve", "label": "", "sha256": ""}
second = {"kind": "operator", "id": command["ref"], "label": "", "sha256": command["sha256"], **rejected, "to": command["to"]}
return [first if verdict == "accept" else {**first, **rejected, "to": commit["to"]}, second]
def verify(self, repo: Path, review: str) -> subprocess.CompletedProcess:
argv = ["verify", "--port", str(self.port), "--approvers", str(self.approvers), "--replay", str(self.replay / "commit.json"), "--review", review, "--gate", "pass"]
return subprocess.run([sys.executable, str(APPROVAL), *argv], cwd=repo, env={**os.environ, **GIT_ENV}, capture_output=True, text=True, timeout=60, check=False)
def test_a_commit_card_lists_its_score_line_as_metric_chips_and_a_command_card_none(self):
_, review = self.stack()
chips = {card["ref"]: [(metric["id"], metric["value"]) for metric in card["metrics"]] for card in self.cards()}
self.assertEqual(chips, {review: [("fix", "95"), ("break", "3")], "operator/ship": []})
def test_every_card_names_the_addressee_its_inbox_entry_shows(self):
_, review = self.stack()
shown = {item["id"]: item["to"] for item in self.snapshot()["inbox"]}
self.assertEqual({card["ref"]: card["to"] for card in self.cards()}, {review: "Skills", "operator/ship": "Skills"})
self.assertEqual((shown[review], shown["operator/ship"]), ("Skills", "Skills"))
def test_one_signature_releases_an_accepted_commit_the_commit_check_verifies_and_a_rejection_told_by_call(self):
repo, review = self.stack()
entries = self.decisions(review, "accept")
signer = self.pin()
proof = self.signed(entries, signer)
status, _, body = self.approve([entries[0], {**entries[1], "reason": "push it anyway"}], proof)
self.assertEqual(status, 400, body)
self.assertIn("list every approved item", body.decode())
status, _, body = self.approve(entries, key(-8, "stranger").proof(proof["payload"], self.origin))
self.assertEqual(status, 400, body)
self.assertIn("not signed", body.decode())
with mock.patch.object(batches, "UNDO", 0.4):
status, _, body = self.approve(entries, proof)
self.assertEqual(status, 202, body)
self.assertEqual(self.call("POST", f"/api/inbox/undo/{json.loads(body)['id']}", b"", self.guard(self.host))[0], 200)
self.assertIn("already used", self.approve(entries, proof)[2].decode())
self.assertEqual(self.approve(entries, self.signed(entries, signer))[0], 202)
self.settle(lambda: self.status(review) == "approved")
self.settle(lambda: (skill_scripts.RASCI, ["set", "operator/ship", "--status", "stopped"]) in self.scripts.calls)
self.settle(lambda: self.radio() != [])
self.assertEqual(len(self.radio()), 1, self.radio())
sent = self.radio()[0]
self.assertEqual(sent[:6], ["send", "call", "--to", "Skills session", "--session", "Operator"])
self.assertTrue(sent[6].endswith(f"Denied by the operator: operator command make ship in {self.work} (from Skills). Reason: {TOLD}"), sent)
done = self.verify(repo, review)
self.assertEqual(done.returncode, 0, done.stderr)
self.assertIn(f"tree {git(repo, 'write-tree').strip()} on feature/match", done.stdout)
self.assertNotIn(f"review:{review}", [card["id"] for card in self.cards()])
def test_a_rejected_commit_carries_its_feedback_by_call_and_the_commit_check_refuses_it(self):
repo, review = self.stack()
entries = self.decisions(review, "reject")
with mock.patch.object(batches, "UNDO", 0.05):
self.assertEqual(self.approve(entries, self.signed(entries, self.pin()))[0], 202)
self.settle(lambda: self.status(review) == "rejected")
self.settle(lambda: len(self.radio()) == 2)
told = sorted(sent[6].split("] ", 1)[1] for sent in self.radio())
self.assertEqual(told[0], f"Denied by the operator: commit feat: match on feature/match (from Skills). Reason: {TOLD}")
self.assertEqual({tuple(sent[:4]) for sent in self.radio()}, {("send", "call", "--to", "Skills session")})
stored = self.httpd.hub.reviews.get(review)
self.assertEqual((stored["channel"], stored["reason"], stored["to"]), ("call", TOLD, "Skills"))
done = self.verify(repo, review)
self.assertEqual(done.returncode, 1, done.stdout)
self.assertIn("rejected", done.stderr)
def test_a_command_release_binds_the_payload_hash_its_card_showed(self):
body = self.work / "body.md"
body.write_text("# Release\n", encoding="utf-8")
command = f"gh issue create --body-file {body}"
self.operator_row("operator/ship", command, str(self.work))
card = self.card("operator:operator/ship")
inbox = {"kind": "operator", "id": "operator/ship", "verb": "approve", "label": "", "sha256": card["sha256"]}
entry = {**inbox, "payload": card["hash"]}
row = signing.item_hash(signing.operator_item("operator/ship", command, str(self.work)))
bound = signing.match_item("operator:operator/ship", card["hash"])
self.assertEqual(self.prepare([inbox])["payload"]["items"], [row])
self.assertEqual(sorted(self.prepare([entry])["payload"]["items"]), sorted([row, signing.item_hash(bound)]))
signer = self.pin()
bare = signer.proof(signing.approval_payload([row], secrets.token_hex(16), int(time.time()) + 60), self.origin)
status, _, answer = self.approve([entry], bare)
self.assertEqual(status, 400, answer)
self.assertIn("list every approved item", answer.decode())
for bad, code in (({**entry, "payload": "x"}, 400), ({**entry, "payload": "0" * 64}, 409), ({**entry, "verb": "reject", "channel": "call", "reason": "", "to": "Skills"}, 400)):
with self.subTest(bad=bad):
self.assertEqual(self.post_json("/api/sign", {"items": [bad]}, self.guard(self.host))[0], code)
body.write_text("# Release, changed\n", encoding="utf-8")
status, _, answer = self.post_json("/api/sign", {"items": [entry]}, self.guard(self.host))
self.assertEqual(status, 409, answer)
self.assertIn("stale", answer.decode())
status, _, answer = self.post_json("/api/match/snapshot", {"task": "operator/ship"}, self.local())
self.assertEqual(status, 200, answer)
self.assertNotEqual(json.loads(answer)["hash"], card["hash"])
self.assertEqual(self.post_json("/api/sign", {"items": [entry]}, self.guard(self.host))[0], 409)
body.write_text("# Release\n", encoding="utf-8")
status, _, answer = self.post_json("/api/match/snapshot", {"task": "operator/ship"}, self.local())
self.assertEqual(json.loads(answer)["hash"], card["hash"], answer)
with mock.patch.object(batches, "UNDO", 0.4):
self.assertEqual(self.approve([entry], self.signed([entry], signer))[0], 202)
body.write_text("# Release, changed after the release\n", encoding="utf-8")
self.settle(lambda: any("payload file changed" in row["text"] for row in self.events("kinds=approval")["events"]))
self.assertEqual(self.httpd.hub.jobs.listed(), [])
body.write_text("# Release\n", encoding="utf-8")
with mock.patch.object(batches, "UNDO", 0.05):
self.assertEqual(self.approve([entry], self.signed([entry], signer))[0], 202)
self.settle(lambda: self.httpd.hub.jobs.listed() != [])
self.assertEqual([(job["task"], job["status"]) for job in self.httpd.hub.jobs.listed()], [("operator/ship", "queued")])
def test_a_commit_rejected_without_feedback_stays_a_plain_rejection(self):
_, review = self.stack()
entry = {"kind": "review", "id": review, "verb": "reject", "label": "", "sha256": ""}
for bad in ({**entry, "channel": "call", "reason": "x"}, {**entry, "channel": "mayday", "reason": "x", "to": "Skills"}, {**entry, "reason": "x", "to": "Skills"}):
with self.subTest(bad=bad):
self.assertEqual(self.post_json("/api/sign", {"items": [bad]}, self.guard(self.host))[0], 400)
with mock.patch.object(batches, "UNDO", 0.05):
self.assertEqual(self.approve([entry], None)[0], 202)
self.settle(lambda: self.status(review) == "rejected")
self.assertEqual(self.radio(), [])
class TestCommandCards(MatchCase):
def payload_command(self) -> str:
body = self.work / "body.md"
body.write_text("# Release\n\nline two\n", encoding="utf-8")
(self.work / "link.md").symlink_to(body)
(self.work / "big.txt").write_text("x" * (payloads.PAYLOAD_LIMIT + 1), encoding="utf-8")
(self.work / "secret.txt").write_text("hidden\n", encoding="utf-8")
return f"gh issue create --body-file {body} --list link.md --big=big.txt --key {self.work / 'secret.txt'} --base origin/main"
def test_an_operator_row_is_a_card_with_command_directory_reason_and_payload_files(self):
self.sandbox_view([str(self.work / "secret.txt")])
command = self.payload_command()
self.operator_row("operator/ship", command, str(self.work))
card = self.card("operator:operator/ship")
self.assertEqual((card["kind"], card["command"], card["directory"], card["reason"], card["agent"], card["stale"]), ("command", command, str(self.work), "ship the release notes", "Skills", False))
files = {Path(entry["path"]).name: entry for entry in card["files"]}
self.assertEqual(sorted(files), ["big.txt", "body.md", "link.md", "secret.txt"])
self.assertEqual((files["body.md"]["state"], files["body.md"]["content"]), ("read", "# Release\n\nline two\n"))
for name, words in (("link.md", "symlink"), ("big.txt", "bytes"), ("secret.txt", "sandbox")):
with self.subTest(name=name):
self.assertEqual((files[name]["state"], files[name]["content"]), ("refused", ""))
self.assertIn(words, files[name]["reason"])
def test_without_a_sandbox_view_every_payload_file_is_refused(self):
(self.state / "firewall" / "view.json").unlink()
command = self.payload_command()
self.operator_row("operator/ship", command, str(self.work))
states = {entry["state"] for entry in self.card("operator:operator/ship")["files"]}
self.assertEqual(states, {"refused"})
def test_a_changed_payload_file_turns_the_card_stale_until_it_is_added_again(self):
command = self.payload_command()
self.operator_row("operator/ship", command, str(self.work))
before = self.card("operator:operator/ship")
(self.work / "body.md").write_text("# Release\n\nswapped\n", encoding="utf-8")
after = self.card("operator:operator/ship")
self.assertTrue(after["stale"], after)
self.assertEqual(after["files"], before["files"])
self.assertNotEqual(after["live_hash"], after["hash"])
done = self.match(self.work, "add", "--task", "operator/ship")
self.assertEqual(done.returncode, 0, done.stderr)
fresh = self.card("operator:operator/ship")
self.assertFalse(fresh["stale"], fresh)
self.assertIn("swapped", next(entry["content"] for entry in fresh["files"] if entry["path"].endswith("body.md")))
def test_decision_rows_and_rows_at_the_runner_are_no_cards(self):
self.write_council({"rasci.json": {"operator/pick": {"title": "-", "workdir": str(self.work), "check": "decision", "roles": {"Operator": "R"}}}})
self.assertEqual(self.cards(), [])
def test_show_prints_the_command_and_its_payload(self):
command = self.payload_command()
self.operator_row("operator/ship", command, str(self.work))
shown = self.match(self.work, "show", "operator:operator/ship")
self.assertEqual(shown.returncode, 0, shown.stderr)
for words in (command, f"in {self.work}", "line two", "refused"):
with self.subTest(words=words):
self.assertIn(words, shown.stdout)
class TestPayloadHash(unittest.TestCase):
def test_the_hash_binds_command_directory_and_every_file_state(self):
files = [{"path": "/tmp/a", "state": "read", "reason": "", "sha256": "1" * 64, "size": 3, "content": "abc"}]
base = payloads.payload_hash("make ship", "/repo", files)
self.assertEqual(base, payloads.payload_hash("make ship", "/repo", [dict(files[0])]))
self.assertRegex(base, "^[0-9a-f]{64}$")
for changed in (payloads.payload_hash("make ship ", "/repo", files), payloads.payload_hash("make ship", "/other", files), payloads.payload_hash("make ship", "/repo", [{**files[0], "sha256": "2" * 64}]), payloads.payload_hash("make ship", "/repo", [{**files[0], "state": "refused"}])):
self.assertNotEqual(changed, base)
def test_named_paths_come_from_words_and_option_values_resolved_in_the_directory(self):
named = payloads.named_paths("gh issue create --body-file /tmp/x/body.md --list=list.txt scripts/run.sh 'quoted file.md' https://example.org/a.md", "/repo")
self.assertEqual(named, ["/tmp/x/body.md", "/repo/list.txt", "/repo/scripts/run.sh", "/repo/quoted file.md"])
class TestMetricChips(unittest.TestCase):
def test_score_lines_of_every_shape_give_one_chip_per_metric_in_label_order(self):
labels = scores.score_labels(REPO_ROOT / "skills")
line = scores.metric_chips("🎯 fix 94 % · 💥 break 4 % · 💸 tokens 100 % · ⏱️ velocity 100 % · 🛡️ security 100 % · 🏆 score 90", labels)
self.assertEqual([(chip["id"], chip["emoji"], chip["label"], chip["value"]) for chip in line], [
("fix", "🎯", "Fix", "94 %"), ("break", "💥", "Break", "4 %"), ("tokens", "💸", "Tokens", "100 %"),
("velocity", "⏱️", "Velocity", "100 %"), ("security", "🛡️", "Security", "100 %"), ("score", "🏆", "Score", "90"),
])
block = scores.metric_chips("🎯 Fix: 85 %\n💥 Break: 5 %\n⏱ Speed: new\n🏆 Score: 81\n🚦 Gate: ⛔ fix 85 % < 90", labels)
self.assertEqual([(chip["id"], chip["value"]) for chip in block], [("fix", "85 %"), ("break", "5 %"), ("velocity", "new"), ("score", "81")])
self.assertEqual([(chip["id"], chip["value"]) for chip in scores.metric_chips("Fix 92% Break 2.5%", labels)], [("fix", "92%"), ("break", "2.5%")])
for text in ("", "looks good", "🎯 high", "prefix 90 %"):
with self.subTest(text=text):
self.assertEqual(scores.metric_chips(text, labels), [])
self.assertEqual(scores.metric_chips("🎯 95 💥 3", None), [])
if __name__ == "__main__":
unittest.main()