feat(orchestrator): verify hub approvals from an MCP record

Sandboxed agents cannot reach the hub on localhost:8737 (refused directly, timeout through the sandbox proxy), so approval.py verify failed and every kvb-git-diff approval fell back to the chat question (Quality: 3 reviews on 2026-11-17). The MCP tools review_status and review_wait, which run outside the sandbox, now also return the review's passkey proof, the record the HTTP proof route already served; approval.py verify --record <file> reads that object instead of fetching it and checks the signature against the pinned approvers, the replay store and the local tree as before, so trust stays with the operator's passkey, not the transport. kvb-git-diff names the sandbox path.

🎯 Fix: 93 %
💥 Break: 3 %
💸 Tokens: 100 %
⏱️ Velocity: 100 %
🛡️ Security: 100 %
🏆 Score: 90
🚦 Gate: ✅ 90/5, tokens and velocity <= 100, security 100 %

Created-with: https://s.veen.world/skills
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Kevin Veen-BirkenbachandClaude Opus 5.5 committed 2026-11-17 21:13:45 +00:00
1 parent d768851b8b
commit b95af2a8e0
6 files changed
+76 -26

No files matched your search

+18 -5
View File
@@ -5,10 +5,12 @@
<file>, written with mode 0600. Run it on the host, outside every
sandbox, e.g. --approvers ~/.config/kvb-orchestrator/approvers.json.
verify --port <port> --approvers <file> --replay <file> --review <id>
--gate <pass|fail>: run in the worktree right before the commit;
fetches the review's signature from the Agent Orchestrator on 127.0.0.1:<port>
and checks it against the repository, worktree, branch and the tree
git write-tree prints, as this process sees them.
--gate <pass|fail> [--record <file>]: run in the worktree right before
the commit; fetches the review's signature from the Agent Orchestrator
on 127.0.0.1:<port>, or reads it from <file>, the JSON object the MCP
tool review_wait or review_status answered, where a sandbox blocks
that port; checks it against the repository, worktree, branch and
the tree git write-tree prints, as this process sees them.
request --port <port> --title <subject> --page <file> --notes <file>
--gate <pass|fail> --score <line> --agent <name> [--rules <dir>]:
run in the worktree; asks the operator to approve the staged tree,
@@ -71,8 +73,18 @@ def fetch(port: int, path: str) -> dict:
return data
def recorded(path: Path, review_id: str) -> dict:
try:
data = json.loads(path.read_text(encoding="utf-8"))
except (OSError, ValueError, RecursionError) as error:
raise Refused(f"the record {path} cannot be read as JSON: {error}") from error
if not isinstance(data, dict) or data.get("id") != review_id:
raise Refused(f"the record {path} is not review {review_id}")
return data
def verify_commit(args: argparse.Namespace) -> str:
review = fetch(args.port, f"/api/reviews/{args.review}")
review = recorded(args.record, args.review) if args.record else fetch(args.port, f"/api/reviews/{args.review}")
if review.get("status") != "approved":
raise Refused(f"the review is {review.get('status')}, not approved")
item = staged(Path.cwd(), args.review)
@@ -140,6 +152,7 @@ def main() -> int:
verifying.add_argument("--replay", type=Path, required=True)
verifying.add_argument("--review", type=review_id, required=True)
verifying.add_argument("--gate", choices=("pass", "fail"), required=True)
verifying.add_argument("--record", type=Path)
requesting = commands.add_parser("request")
requesting.add_argument("--port", type=int, required=True)
for name in ("--title", "--score", "--agent"):
+1 -1
View File
@@ -55,7 +55,7 @@ class Mcp:
"score": text_property("The kvb-check-score line of the commit, one line."),
},
),
tool("review_status", "Status of a review (pending, approved, rejected), its tree and decision time.", {"id": review_id}),
tool("review_status", "Status of a review (pending, approved, rejected), its tree, decision time and the passkey proof that approval.py verify --record checks.", {"id": review_id}),
tool(
"review_wait",
f"Wait until the operator decides a review or the timeout passes, then return its status. "
+7 -3
View File
@@ -40,6 +40,10 @@ def state_of(review: dict) -> dict:
return {key: review[key] for key in STATE}
def proven(review: dict) -> dict:
return {**state_of(review), "proof": review.get("proof")}
def review_row(review: dict) -> dict:
created = datetime.datetime.fromisoformat(review["created"]).timestamp()
return {**{key: review[key] for key in LISTED}, "notes": review["notes"], "created_ts": created, "files": review.get("files")}
@@ -92,7 +96,7 @@ class Reviewing:
review = self.reviews.get(review_id)
if review is None:
raise Missing(review_id)
return {**state_of(review), "proof": review.get("proof")}
return proven(review)
def verdict(self, review_id: str, status: str, body: bytes) -> dict:
review = self.reviews.get(review_id)
@@ -111,13 +115,13 @@ class Reviewing:
def status(self, arguments: dict) -> dict:
expect(arguments, ("id",))
review_id = field(arguments, "id")
return state_of(found(self.reviews.get(review_id), review_id))
return proven(found(self.reviews.get(review_id), review_id))
def wait(self, arguments: dict) -> dict:
expect(arguments, ("id", "timeout_seconds"))
review_id = field(arguments, "id")
timeout = seconds(arguments, "timeout_seconds", WAIT_CAP)
return state_of(found(self.reviews.wait(review_id, timeout), review_id))
return proven(found(self.reviews.wait(review_id, timeout), review_id))
def pending(self, arguments: dict) -> dict:
expect(arguments, ())
+7 -7
View File
@@ -30,10 +30,9 @@ once the operator's pinned passkey signature over it verifies here.
```
Only a `pass` review can be approved in bulk or by a rule. Tell the
operator in one line that the review waits in the hub at the `url` it
prints, written out as plain text: chat links do not open for this
operator. Without the script, the MCP tool `review_request` takes the
same fields without the diff.
operator the `url` it prints in one plain-text line; chat links do not
open for this operator. Without the script or port 8737, the MCP tool
`review_request` takes the same fields without the diff.
3. Call `review_wait` with its `id` and `timeout_seconds` 240 until the
status is no longer `pending`.
4. `approved`: in the worktree, verify the operator's passkey signature
@@ -43,9 +42,10 @@ once the operator's pinned passkey signature over it verifies here.
python3 ~/.local/share/kvb-orchestrator/client/approval.py verify --port 8737 --approvers ~/.config/kvb-orchestrator/approvers.json --replay <approvals>/commit.json --review <id> --gate <gate>
```
Sandboxed without port 8737: save `review_wait`'s answer under `/tmp`,
add `--record <file>`.
Exit 0 only: commit as the calling skill says. Any other exit,
`rejected`, or the tree changed: no commit; report the printed reason.
The hub is unreachable (no kvb-orchestrator tools, or a call cannot connect), or the
verify step refused: ask the calling skill's normal approval question in
chat instead.
No kvb-orchestrator tools, or the verify step refused: ask the calling
skill's approval question in chat.
+42 -9
View File
@@ -7,6 +7,7 @@ import subprocess
import sys
import time
import unittest
from pathlib import Path
from unittest import mock
import hub_support
@@ -214,7 +215,7 @@ class TestHubExecutors(HubCase):
self.assertFalse(rules.rule_matches({**stored, "proof": None}, "ask", subject, now))
self.assertFalse(rules.rule_matches(stored, "ask", subject, stored["proof"]["payload"]["expires"] + 1))
def test_the_commit_check_verifies_the_staged_tree_before_a_commit(self):
def staged_review(self) -> tuple[Path, str, str]:
repo = self.outside / "repo"
repo.mkdir()
self.git(repo, "init", "--quiet", "--initial-branch", "feature/x")
@@ -224,20 +225,29 @@ class TestHubExecutors(HubCase):
common = self.git(repo, "rev-parse", "--path-format=absolute", "--git-common-dir").strip()
top = self.git(repo, "rev-parse", "--show-toplevel").strip()
arguments = {**self.arguments("feat: signed", "", self.page("signed.html", PAGE)), "repo": common, "worktree": top, "tree": tree}
review_id = self.tool("review_request", arguments)["structuredContent"]["id"]
return repo, self.tool("review_request", arguments)["structuredContent"]["id"], tree
def verify(gate: str) -> subprocess.CompletedProcess:
argv = ["verify", "--port", str(self.port), "--approvers", str(self.approvers), "--replay", str(self.replay / "commit.json"), "--review", review_id, "--gate", gate]
return subprocess.run([sys.executable, str(APPROVAL), *argv], cwd=repo, env={**os.environ, **GIT_ENV}, capture_output=True, text=True, timeout=60, check=False)
done = verify("pass")
self.assertEqual(done.returncode, 1, done.stdout)
self.assertIn("the review is pending", done.stderr)
def approve_review(self, review_id: str) -> None:
signer = self.pin()
entries = [self.entry("review", review_id, "", "")]
with mock.patch.object(batches, "UNDO", 0.05):
self.assertEqual(self.approve(entries, self.signed(entries, signer))[0], 202)
self.settle(lambda: self.status(review_id) == "approved")
def verify(self, repo: Path, review_id: str, *extra: str) -> subprocess.CompletedProcess:
argv = ["verify", "--port", str(self.port), "--approvers", str(self.approvers), "--review", review_id, *extra]
return subprocess.run([sys.executable, str(APPROVAL), *argv], cwd=repo, env={**os.environ, **GIT_ENV}, capture_output=True, text=True, timeout=60, check=False)
def test_the_commit_check_verifies_the_staged_tree_before_a_commit(self):
repo, review_id, tree = self.staged_review()
def verify(gate: str) -> subprocess.CompletedProcess:
return self.verify(repo, review_id, "--replay", str(self.replay / "commit.json"), "--gate", gate)
done = verify("pass")
self.assertEqual(done.returncode, 1, done.stdout)
self.assertIn("the review is pending", done.stderr)
self.approve_review(review_id)
(repo / "a.txt").write_text("two\n", encoding="utf-8")
self.git(repo, "add", "a.txt")
done = verify("pass")
@@ -252,6 +262,29 @@ class TestHubExecutors(HubCase):
self.assertEqual(done.returncode, 1, done.stdout)
self.assertIn("already used", done.stderr)
def test_a_sandboxed_agent_verifies_the_record_the_mcp_tool_answered(self):
repo, review_id, tree = self.staged_review()
record = self.outside / "record.json"
replay = ("--replay", str(self.replay / "record.json"), "--gate", "pass", "--record", str(record))
self.assertIsNone(self.tool("review_status", {"id": review_id})["structuredContent"]["proof"])
self.approve_review(review_id)
answered = self.tool("review_wait", {"id": review_id, "timeout_seconds": 1})["structuredContent"]
self.assertEqual((answered["status"], answered["tree"]), ("approved", tree))
for content, reason in (
("{not json", "cannot be read as JSON"),
(json.dumps({**answered, "id": "0" * 16}), f"is not review {review_id}"),
(json.dumps({**answered, "proof": None}), "no signature"),
):
with self.subTest(reason=reason):
record.write_text(content, encoding="utf-8")
done = self.verify(repo, review_id, *replay)
self.assertEqual(done.returncode, 1, done.stdout)
self.assertIn(reason, done.stderr)
record.write_text(json.dumps(answered), encoding="utf-8")
done = self.verify(repo, review_id, *replay)
self.assertEqual(done.returncode, 0, done.stderr)
self.assertIn(f"tree {tree} on feature/x", done.stdout)
if __name__ == "__main__":
unittest.main()
+1 -1
View File
@@ -32,7 +32,7 @@ class TestHubMcp(HubCase):
self.assertRegex(review_id, r"^[0-9a-f]{16}$")
state = self.tool("review_status", {"id": review_id})
self.assertEqual(json.loads(state["content"][0]["text"]), state["structuredContent"])
self.assertEqual(state["structuredContent"], {"id": review_id, "status": "pending", "tree": TREE, "decided": None})
self.assertEqual(state["structuredContent"], {"id": review_id, "status": "pending", "tree": TREE, "decided": None, "proof": None})
listed = self.tool("review_list", {})["structuredContent"]["reviews"]
self.assertEqual([(r["id"], r["title"], r["tree"]) for r in listed], [(review_id, "feat: hub", TREE)])
self.assertEqual(self.rpc("resources/list", {})["error"]["code"], -32601)