Kevin Veen-BirkenbachandClaude Opus 5.5
6f940b4415
Surface (requirement 001 phase 2): navigation Stream (Inbox, Regeln) / Agents / Artifacts with keys 1-3 and hash routes (#graph and #tree redirect), header with title, command palette and settings only, server-side badges (unsnoozed inbox, beats within 900 s, open jobs), a Mayday banner with "aufheben", four stream chips with agent:/art:/seit: search, Filter and time menus, the inbox in main, an agents toolbar (Funk an alle with Newsletter, Broadcast, Pan-Pan, Security, Mayday; Priorität; Tag/Nacht), Post or Call in the agent panel, Ambiente in the settings and read-aloud over the seven classes with Mayday locked. Rename (decision 9): hub/ is orchestrator/, the package kvb_orchestrator, the unit kvb-orchestrator-runner.service, scripts/orchestrator.sh, environment KVB_ORCHESTRATOR_*, make targets orchestrator-*, the MCP server, image and compose project kvb-orchestrator, the guard header X-Kvb-Orchestrator and the message marker [kvb-orchestrator msg <id>]. install runs migrate first: it stops the old container, removes its image and code, moves state, approvals and pinned keys, swaps the MCP registration, the PermissionRequest hook and the runner unit, and changes nothing on a second run (scratch-HOME test with stub docker, claude and systemctl). Saved views fall back to the old localStorage key once. MEDUSA acceptances carried to the renamed paths: 82 rows whose lines are unchanged (reason prefixed "renamed from <old path>, line unchanged"), and 7 rows whose text changed only in the product name, the package name, a temp prefix or a new class argument, each upheld by one kvb-think-criticize skeptic (approval.py:60, messaging.py:53, server.py:56, tests/hub_support.py:64, passkey.py:205, page-agent.js:75, language.py:46). Cost 101 %: the longer name adds 15 tokens of skill text; the operator approved it with the commit. 🎯 Fix: 92 % 💥 Break: 4 % 💸 Cost: 101 % ⏱️ Time: – 🛡️ Security: 100 % 💰 Value: ? 🏆 Score: 87 🚦 Gate: ⚠️ cost 101 % > 100: ask the operator before the commit Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
251 lines
15 KiB
Python
251 lines
15 KiB
Python
from __future__ import annotations
|
|
|
|
import hashlib
|
|
import json
|
|
import os
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
import unittest
|
|
from unittest import mock
|
|
|
|
import hub_support
|
|
from hub_case import HubCase
|
|
from hub_support import (
|
|
APPROVAL,
|
|
ASK,
|
|
GIT_ENV,
|
|
PAGE,
|
|
PRESENT,
|
|
RUNNER,
|
|
TREE,
|
|
runner,
|
|
signing,
|
|
verifier,
|
|
)
|
|
from kvb_orchestrator import batches, rules, skill_scripts
|
|
|
|
# An import named key trips F402: test_rules_are_signed_once_and_checked_by_the_executor rebinds it in a comprehension.
|
|
key = hub_support.key
|
|
|
|
|
|
class TestHubExecutors(HubCase):
|
|
def trust(self, name: str) -> verifier.Trust:
|
|
return verifier.Trust(approvers=self.approvers, replay=self.replay / f"{name}.json", origin=self.origin)
|
|
|
|
def runner_jobs(self) -> list[dict]:
|
|
status, _, body = self.call("GET", "/api/runner/jobs", None, self.client())
|
|
self.assertEqual(status, 200, body)
|
|
return json.loads(body)["jobs"]
|
|
|
|
def run_runner(self) -> None:
|
|
trust = ["--approvers", str(self.approvers), "--replay", str(self.replay / "runner.json")]
|
|
command = [sys.executable, str(RUNNER), "--port", str(self.port), "--interval", "0.1", *trust, "--once"]
|
|
done = subprocess.run(command, capture_output=True, text=True, timeout=60, check=False)
|
|
self.assertEqual(done.returncode, 0, done.stderr)
|
|
|
|
def test_the_runner_runs_only_signed_commands_once(self):
|
|
command = "echo hello && echo world >&2"
|
|
register = {
|
|
"operator/greet": {"title": command, "workdir": str(self.outside), "description": "say hi", "check": "manual", "status": "queued", "roles": {}},
|
|
"operator/decide": {"title": "pick a name", "workdir": str(self.outside), "check": "decision", "status": "queued", "roles": {}},
|
|
}
|
|
self.write_council({"rasci.json": register})
|
|
items = {item["id"]: item for item in self.snapshot()["inbox"]}
|
|
greet = items["operator/greet"]
|
|
digest = hashlib.sha256(command.encode()).hexdigest()
|
|
self.assertEqual((greet["runnable"], greet["sha256"], items["operator/decide"]["runnable"]), (True, digest, False))
|
|
self.assertEqual(self.approve([self.entry("operator", "operator/greet", "", "0" * 64)], None)[0], 409)
|
|
self.assertEqual(self.approve([self.entry("operator", "operator/decide", "", "")], None)[0], 400)
|
|
self.assertEqual(self.approve([self.entry("operator", "operator/greet", "", digest)], None)[0], 400)
|
|
self.assertEqual(self.call("GET", "/api/runner/jobs", None, {"Host": self.host})[0], 403)
|
|
self.assertEqual(self.call("GET", "/api/runner/jobs", None, self.guard(self.host))[0], 403)
|
|
signer = self.pin()
|
|
greeting = [self.entry("operator", "operator/greet", "", digest)]
|
|
with mock.patch.object(batches, "UNDO", 0.05):
|
|
self.assertEqual(self.approve(greeting, self.signed(greeting, signer))[0], 202)
|
|
self.settle(lambda: self.runner_jobs() != [])
|
|
queued = self.runner_jobs()
|
|
self.assertEqual([(job["task"], job["command"], job["directory"], job["sha256"]) for job in queued], [("operator/greet", command, str(self.outside), digest)])
|
|
proof = queued[0]["proof"]
|
|
self.assertIn(signing.item_hash(signing.operator_item("operator/greet", command, str(self.outside))), proof["payload"]["items"])
|
|
self.assertNotIn("operator/greet", [item["id"] for item in self.snapshot()["inbox"]])
|
|
self.run_runner()
|
|
job = self.snapshot()["jobs"][-1]
|
|
self.assertEqual((job["status"], job["code"], job["output"]), ("done", 0, "hello\nworld\n"))
|
|
self.assertNotIn("proof", job)
|
|
self.assertEqual(self.scripts.calls[-1], (skill_scripts.OPERATOR_SCRIPT, ["done", "greet"]))
|
|
claim = {**self.client(), "Content-Type": "application/json"}
|
|
self.assertEqual(self.call("POST", f"/api/runner/{job['id']}/claim", b"{}", claim)[0], 409)
|
|
self.assertEqual(self.call("POST", f"/api/runner/{job['id']}/claim", b"{}", {**self.guard(self.host), "Content-Type": "application/json"})[0], 403)
|
|
stranger = key(-8, "stranger").proof(proof["payload"], self.origin)
|
|
forged = (
|
|
("echo evil", str(self.outside), proof, "does not list this item"),
|
|
(command, "/", proof, "does not list this item"),
|
|
(command, str(self.outside), proof, "already used"),
|
|
(command, str(self.outside), None, "no signature"),
|
|
(command, str(self.outside), stranger, "not pinned"),
|
|
)
|
|
for text, directory, relayed, reason in forged:
|
|
with self.subTest(command=text, directory=directory, reason=reason):
|
|
fields = {"task": "operator/greet", "command": text, "directory": directory, "sha256": digest, "status": "queued", "proof": relayed}
|
|
tampered = self.httpd.hub.jobs.add({**fields, "code": None, "output": "", "created_ts": time.time(), "started": None, "finished": None})
|
|
self.run_runner()
|
|
record = self.httpd.hub.jobs.get(tampered["id"])
|
|
self.assertEqual(record["status"], "refused")
|
|
self.assertTrue(record["output"].startswith(runner.REFUSED), record["output"])
|
|
self.assertIn(reason, record["output"])
|
|
self.assertEqual(len([call for call in self.scripts.calls if call[0] == skill_scripts.OPERATOR_SCRIPT]), 1)
|
|
self.approvers.unlink()
|
|
job = {"task": "operator/greet", "command": command, "directory": str(self.outside), "proof": proof}
|
|
self.assertIn("no key is pinned", runner.refusal(job, self.trust("other"), time.time()))
|
|
reports = []
|
|
self.assertEqual(runner.execute({"command": "exit 3", "directory": str(self.outside)}, reports.append), 3)
|
|
self.assertEqual(reports[-1], {"output": "", "code": 3, "refused": False})
|
|
|
|
def test_the_runner_closes_the_output_pipe_on_every_path(self):
|
|
popen, spawned = subprocess.Popen, []
|
|
job = {"command": "echo hi; sleep 0.3", "directory": str(self.outside)}
|
|
|
|
def spawn(*args: object, **kwargs: object) -> subprocess.Popen:
|
|
spawned.append(popen(*args, **kwargs))
|
|
return spawned[-1]
|
|
|
|
with mock.patch.object(subprocess, "Popen", spawn), mock.patch.object(runner, "BEAT", 0.05):
|
|
self.assertEqual(runner.execute(job, [].append), 0)
|
|
self.assertTrue(spawned[0].stdout.closed)
|
|
self.assertRaises(RuntimeError, runner.execute, job, mock.Mock(side_effect=RuntimeError("the hub is gone")))
|
|
self.settle(lambda: spawned[1].stdout.closed)
|
|
self.assertEqual(spawned[1].wait(10), 0)
|
|
|
|
def test_rules_are_signed_once_and_checked_by_the_executor(self):
|
|
now = time.time()
|
|
rule = {"kind": "ask", "tool": "Bash", "pattern": "make *", "scope": "/repo*", "expires": now + 3600}
|
|
for payload in (
|
|
{**rule, "expires": now - 1},
|
|
{**rule, "expires": now + 400 * 86400},
|
|
{**rule, "tool": ""},
|
|
{**rule, "kind": "review"},
|
|
{**rule, "pattern": " "},
|
|
{**rule, "scope": "two\nlines"},
|
|
{key: value for key, value in rule.items() if key != "scope"},
|
|
):
|
|
with self.subTest(payload=payload):
|
|
self.assertEqual(self.post_json("/api/rules/sign", payload, self.guard(self.host))[0], 400)
|
|
self.assertEqual(self.post_json("/api/rules/sign", rule, self.client())[0], 403)
|
|
signer = self.pin()
|
|
|
|
def prepared(fields: dict) -> dict:
|
|
status, _, body = self.post_json("/api/rules/sign", fields, self.guard(self.host))
|
|
self.assertEqual(status, 200, body)
|
|
return json.loads(body)
|
|
|
|
def created(payload: dict, assertion: dict | None) -> tuple[int, dict, bytes]:
|
|
return self.post_json("/api/rules", {"rule": payload, "assertion": assertion, "enabled": True}, self.guard(self.host))
|
|
|
|
ask_rule = prepared(rule)
|
|
self.assertEqual((ask_rule["sign"], ask_rule["challenge"], ask_rule["credentials"]), (True, signing.challenge(ask_rule["payload"]), [signer.id]))
|
|
signed = signer.assertion(ask_rule["payload"], self.origin, "localhost", PRESENT, 0)
|
|
for payload, assertion in (
|
|
(ask_rule["payload"], None),
|
|
({**ask_rule["payload"], "kinds": ["ask", "review"]}, signed),
|
|
({**ask_rule["payload"], "expires": now + 400 * 86400}, signed),
|
|
({**ask_rule["payload"], "id": "x"}, signed),
|
|
):
|
|
with self.subTest(payload=payload):
|
|
self.assertEqual(created(payload, assertion)[0], 400)
|
|
self.assertEqual(self.post_json("/api/rules", {"rule": ask_rule["payload"], "assertion": signed, "enabled": True}, self.client())[0], 403)
|
|
asks = json.loads(created(ask_rule["payload"], signed)[2])
|
|
self.assertEqual(created(ask_rule["payload"], signed)[0], 409)
|
|
review_rule = prepared({"kind": "review", "tool": "", "pattern": "feature/*", "scope": "/repo/*", "expires": now + 3600})
|
|
self.assertEqual(created(review_rule["payload"], signer.assertion(review_rule["payload"], self.origin, "localhost", PRESENT, 0))[0], 201)
|
|
decision_rule = prepared({"kind": "decision", "tool": "", "pattern": "*", "scope": "*", "expires": now + 3600})
|
|
self.assertFalse(decision_rule["sign"])
|
|
self.assertEqual(created(decision_rule["payload"], signed)[0], 400)
|
|
self.assertEqual(created(decision_rule["payload"], None)[0], 201)
|
|
forged_rule = prepared({**rule, "pattern": "cargo *"})
|
|
self.assertEqual(created(forged_rule["payload"], key(-8, "stranger").assertion(forged_rule["payload"], self.origin, "localhost", PRESENT, 0))[0], 201)
|
|
self.assertEqual(sorted((entry["kind"], entry["signed"]) for entry in self.snapshot()["rules"]), [("ask", True), ("ask", True), ("decision", False), ("review", True)])
|
|
with mock.patch.object(batches, "UNDO", 0.05):
|
|
matched = self.create_ask(ASK)
|
|
risky = self.create_ask({**ASK, "command": "make deploy && git push"})
|
|
other = self.create_ask({**ASK, "tool": "Edit"})
|
|
outside = self.create_ask({**ASK, "cwd": "/elsewhere"})
|
|
cargo = self.create_ask({**ASK, "command": "cargo build"})
|
|
green = self.request("feat: auto", "")
|
|
red = self.tool("review_request", {**self.arguments("feat: red", "", self.page("red.html", PAGE)), "gate": "fail"})["structuredContent"]["id"]
|
|
chosen = self.decision("Wait")
|
|
plain = self.decision("")
|
|
self.settle(lambda: self.ask_status(matched, "0") == "allowed")
|
|
self.settle(lambda: self.ask_status(cargo, "0") == "allowed")
|
|
self.settle(lambda: self.status(green) == "approved")
|
|
self.settle(lambda: self.choice(chosen) == "Wait")
|
|
time.sleep(0.3)
|
|
self.assertEqual([self.ask_status(ask, "0") for ask in (risky, other, outside)], ["pending", "pending", "pending"])
|
|
self.assertEqual((self.status(red), self.choice(plain)), ("pending", None))
|
|
self.assertEqual(len(self.events("kinds=rule")["events"]), 4)
|
|
hook = self.trust("hook")
|
|
verifier.check(json.loads(self.get(f"/api/asks/{matched}?wait=0")[2])["proof"], signing.ask_item(matched, "Bash", "make deploy", "/repo", "s1"), "", hook, time.time())
|
|
with self.assertRaisesRegex(signing.Refused, "not pinned"):
|
|
verifier.check(json.loads(self.get(f"/api/asks/{cargo}?wait=0")[2])["proof"], signing.ask_item(cargo, "Bash", "cargo build", "/repo", "s1"), "", hook, time.time())
|
|
relayed = json.loads(self.get(f"/api/reviews/{green}")[2])["proof"]
|
|
commit = signing.review_item(green, "/repo/.git", "/repo/.worktrees/feature/x", "feature/x", TREE)
|
|
verifier.check(relayed, commit, "pass", self.trust("commit"), time.time())
|
|
with self.assertRaisesRegex(signing.Refused, "does not match"):
|
|
verifier.check(relayed, commit, "fail", self.trust("commit"), time.time())
|
|
self.assertEqual(self.call("POST", f"/api/rules/{asks['id']}/disable", b"", self.guard(self.host))[0], 200)
|
|
with mock.patch.object(batches, "UNDO", 0.05):
|
|
later = self.create_ask(ASK)
|
|
time.sleep(0.3)
|
|
self.assertEqual(self.ask_status(later, "0"), "pending")
|
|
self.assertEqual(self.call("POST", f"/api/rules/{asks['id']}/delete", b"", self.guard(self.host))[0], 200)
|
|
self.assertEqual(self.call("POST", f"/api/rules/{asks['id']}/delete", b"", self.guard(self.host))[0], 404)
|
|
stored = {**asks, "enabled": True}
|
|
subject = {"item": signing.ask_item(later, "Bash", "make deploy", "/repo", "s1"), "gate": ""}
|
|
self.assertTrue(rules.rule_matches(stored, "ask", subject, now))
|
|
self.assertFalse(rules.rule_matches({**stored, "expires": now - 1}, "ask", subject, now))
|
|
self.assertFalse(rules.rule_matches({**stored, "proof": None}, "ask", subject, now))
|
|
self.assertFalse(rules.rule_matches(stored, "ask", subject, stored["proof"]["payload"]["expires"] + 1))
|
|
|
|
def test_the_commit_check_verifies_the_staged_tree_before_a_commit(self):
|
|
repo = self.outside / "repo"
|
|
repo.mkdir()
|
|
self.git(repo, "init", "--quiet", "--initial-branch", "feature/x")
|
|
(repo / "a.txt").write_text("one\n", encoding="utf-8")
|
|
self.git(repo, "add", "a.txt")
|
|
tree = self.git(repo, "write-tree").strip()
|
|
common = self.git(repo, "rev-parse", "--path-format=absolute", "--git-common-dir").strip()
|
|
top = self.git(repo, "rev-parse", "--show-toplevel").strip()
|
|
arguments = {**self.arguments("feat: signed", "", self.page("signed.html", PAGE)), "repo": common, "worktree": top, "tree": tree}
|
|
review_id = self.tool("review_request", arguments)["structuredContent"]["id"]
|
|
|
|
def verify(gate: str) -> subprocess.CompletedProcess:
|
|
argv = ["verify", "--port", str(self.port), "--approvers", str(self.approvers), "--replay", str(self.replay / "commit.json"), "--review", review_id, "--gate", gate]
|
|
return subprocess.run([sys.executable, str(APPROVAL), *argv], cwd=repo, env={**os.environ, **GIT_ENV}, capture_output=True, text=True, timeout=60, check=False)
|
|
|
|
done = verify("pass")
|
|
self.assertEqual(done.returncode, 1, done.stdout)
|
|
self.assertIn("the review is pending", done.stderr)
|
|
signer = self.pin()
|
|
entries = [self.entry("review", review_id, "", "")]
|
|
with mock.patch.object(batches, "UNDO", 0.05):
|
|
self.assertEqual(self.approve(entries, self.signed(entries, signer))[0], 202)
|
|
self.settle(lambda: self.status(review_id) == "approved")
|
|
(repo / "a.txt").write_text("two\n", encoding="utf-8")
|
|
self.git(repo, "add", "a.txt")
|
|
done = verify("pass")
|
|
self.assertEqual(done.returncode, 1, done.stdout)
|
|
self.assertIn("does not list this item", done.stderr)
|
|
(repo / "a.txt").write_text("one\n", encoding="utf-8")
|
|
self.git(repo, "add", "a.txt")
|
|
done = verify("pass")
|
|
self.assertEqual(done.returncode, 0, done.stderr)
|
|
self.assertIn(f"tree {tree} on feature/x", done.stdout)
|
|
done = verify("pass")
|
|
self.assertEqual(done.returncode, 1, done.stdout)
|
|
self.assertIn("already used", done.stderr)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|