feat(orchestrator): verify hub approvals from an MCP record
Sandboxed agents cannot reach the hub on localhost:8737 (refused directly, timeout through the sandbox proxy), so approval.py verify failed and every kvb-git-diff approval fell back to the chat question (Quality: 3 reviews on 2026-11-17). The MCP tools review_status and review_wait, which run outside the sandbox, now also return the review's passkey proof, the record the HTTP proof route already served; approval.py verify --record <file> reads that object instead of fetching it and checks the signature against the pinned approvers, the replay store and the local tree as before, so trust stays with the operator's passkey, not the transport. kvb-git-diff names the sandbox path. 🎯 Fix: 93 % 💥 Break: 3 % 💸 Tokens: 100 % ⏱️ Velocity: 100 % 🛡️ Security: 100 % 🏆 Score: 90 🚦 Gate: ✅ 90/5, tokens and velocity <= 100, security 100 % Created-with: https://s.veen.world/skills Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
d768851b8b
commit
b95af2a8e0
6 files changed
+76
-26
No files matched your search
@@ -5,10 +5,12 @@
|
|||||||
<file>, written with mode 0600. Run it on the host, outside every
|
<file>, written with mode 0600. Run it on the host, outside every
|
||||||
sandbox, e.g. --approvers ~/.config/kvb-orchestrator/approvers.json.
|
sandbox, e.g. --approvers ~/.config/kvb-orchestrator/approvers.json.
|
||||||
verify --port <port> --approvers <file> --replay <file> --review <id>
|
verify --port <port> --approvers <file> --replay <file> --review <id>
|
||||||
--gate <pass|fail>: run in the worktree right before the commit;
|
--gate <pass|fail> [--record <file>]: run in the worktree right before
|
||||||
fetches the review's signature from the Agent Orchestrator on 127.0.0.1:<port>
|
the commit; fetches the review's signature from the Agent Orchestrator
|
||||||
and checks it against the repository, worktree, branch and the tree
|
on 127.0.0.1:<port>, or reads it from <file>, the JSON object the MCP
|
||||||
git write-tree prints, as this process sees them.
|
tool review_wait or review_status answered, where a sandbox blocks
|
||||||
|
that port; checks it against the repository, worktree, branch and
|
||||||
|
the tree git write-tree prints, as this process sees them.
|
||||||
request --port <port> --title <subject> --page <file> --notes <file>
|
request --port <port> --title <subject> --page <file> --notes <file>
|
||||||
--gate <pass|fail> --score <line> --agent <name> [--rules <dir>]:
|
--gate <pass|fail> --score <line> --agent <name> [--rules <dir>]:
|
||||||
run in the worktree; asks the operator to approve the staged tree,
|
run in the worktree; asks the operator to approve the staged tree,
|
||||||
@@ -71,8 +73,18 @@ def fetch(port: int, path: str) -> dict:
|
|||||||
return data
|
return data
|
||||||
|
|
||||||
|
|
||||||
|
def recorded(path: Path, review_id: str) -> dict:
|
||||||
|
try:
|
||||||
|
data = json.loads(path.read_text(encoding="utf-8"))
|
||||||
|
except (OSError, ValueError, RecursionError) as error:
|
||||||
|
raise Refused(f"the record {path} cannot be read as JSON: {error}") from error
|
||||||
|
if not isinstance(data, dict) or data.get("id") != review_id:
|
||||||
|
raise Refused(f"the record {path} is not review {review_id}")
|
||||||
|
return data
|
||||||
|
|
||||||
|
|
||||||
def verify_commit(args: argparse.Namespace) -> str:
|
def verify_commit(args: argparse.Namespace) -> str:
|
||||||
review = fetch(args.port, f"/api/reviews/{args.review}")
|
review = recorded(args.record, args.review) if args.record else fetch(args.port, f"/api/reviews/{args.review}")
|
||||||
if review.get("status") != "approved":
|
if review.get("status") != "approved":
|
||||||
raise Refused(f"the review is {review.get('status')}, not approved")
|
raise Refused(f"the review is {review.get('status')}, not approved")
|
||||||
item = staged(Path.cwd(), args.review)
|
item = staged(Path.cwd(), args.review)
|
||||||
@@ -140,6 +152,7 @@ def main() -> int:
|
|||||||
verifying.add_argument("--replay", type=Path, required=True)
|
verifying.add_argument("--replay", type=Path, required=True)
|
||||||
verifying.add_argument("--review", type=review_id, required=True)
|
verifying.add_argument("--review", type=review_id, required=True)
|
||||||
verifying.add_argument("--gate", choices=("pass", "fail"), required=True)
|
verifying.add_argument("--gate", choices=("pass", "fail"), required=True)
|
||||||
|
verifying.add_argument("--record", type=Path)
|
||||||
requesting = commands.add_parser("request")
|
requesting = commands.add_parser("request")
|
||||||
requesting.add_argument("--port", type=int, required=True)
|
requesting.add_argument("--port", type=int, required=True)
|
||||||
for name in ("--title", "--score", "--agent"):
|
for name in ("--title", "--score", "--agent"):
|
||||||
|
|||||||
@@ -55,7 +55,7 @@ class Mcp:
|
|||||||
"score": text_property("The kvb-check-score line of the commit, one line."),
|
"score": text_property("The kvb-check-score line of the commit, one line."),
|
||||||
},
|
},
|
||||||
),
|
),
|
||||||
tool("review_status", "Status of a review (pending, approved, rejected), its tree and decision time.", {"id": review_id}),
|
tool("review_status", "Status of a review (pending, approved, rejected), its tree, decision time and the passkey proof that approval.py verify --record checks.", {"id": review_id}),
|
||||||
tool(
|
tool(
|
||||||
"review_wait",
|
"review_wait",
|
||||||
f"Wait until the operator decides a review or the timeout passes, then return its status. "
|
f"Wait until the operator decides a review or the timeout passes, then return its status. "
|
||||||
|
|||||||
@@ -40,6 +40,10 @@ def state_of(review: dict) -> dict:
|
|||||||
return {key: review[key] for key in STATE}
|
return {key: review[key] for key in STATE}
|
||||||
|
|
||||||
|
|
||||||
|
def proven(review: dict) -> dict:
|
||||||
|
return {**state_of(review), "proof": review.get("proof")}
|
||||||
|
|
||||||
|
|
||||||
def review_row(review: dict) -> dict:
|
def review_row(review: dict) -> dict:
|
||||||
created = datetime.datetime.fromisoformat(review["created"]).timestamp()
|
created = datetime.datetime.fromisoformat(review["created"]).timestamp()
|
||||||
return {**{key: review[key] for key in LISTED}, "notes": review["notes"], "created_ts": created, "files": review.get("files")}
|
return {**{key: review[key] for key in LISTED}, "notes": review["notes"], "created_ts": created, "files": review.get("files")}
|
||||||
@@ -92,7 +96,7 @@ class Reviewing:
|
|||||||
review = self.reviews.get(review_id)
|
review = self.reviews.get(review_id)
|
||||||
if review is None:
|
if review is None:
|
||||||
raise Missing(review_id)
|
raise Missing(review_id)
|
||||||
return {**state_of(review), "proof": review.get("proof")}
|
return proven(review)
|
||||||
|
|
||||||
def verdict(self, review_id: str, status: str, body: bytes) -> dict:
|
def verdict(self, review_id: str, status: str, body: bytes) -> dict:
|
||||||
review = self.reviews.get(review_id)
|
review = self.reviews.get(review_id)
|
||||||
@@ -111,13 +115,13 @@ class Reviewing:
|
|||||||
def status(self, arguments: dict) -> dict:
|
def status(self, arguments: dict) -> dict:
|
||||||
expect(arguments, ("id",))
|
expect(arguments, ("id",))
|
||||||
review_id = field(arguments, "id")
|
review_id = field(arguments, "id")
|
||||||
return state_of(found(self.reviews.get(review_id), review_id))
|
return proven(found(self.reviews.get(review_id), review_id))
|
||||||
|
|
||||||
def wait(self, arguments: dict) -> dict:
|
def wait(self, arguments: dict) -> dict:
|
||||||
expect(arguments, ("id", "timeout_seconds"))
|
expect(arguments, ("id", "timeout_seconds"))
|
||||||
review_id = field(arguments, "id")
|
review_id = field(arguments, "id")
|
||||||
timeout = seconds(arguments, "timeout_seconds", WAIT_CAP)
|
timeout = seconds(arguments, "timeout_seconds", WAIT_CAP)
|
||||||
return state_of(found(self.reviews.wait(review_id, timeout), review_id))
|
return proven(found(self.reviews.wait(review_id, timeout), review_id))
|
||||||
|
|
||||||
def pending(self, arguments: dict) -> dict:
|
def pending(self, arguments: dict) -> dict:
|
||||||
expect(arguments, ())
|
expect(arguments, ())
|
||||||
|
|||||||
@@ -30,10 +30,9 @@ once the operator's pinned passkey signature over it verifies here.
|
|||||||
```
|
```
|
||||||
|
|
||||||
Only a `pass` review can be approved in bulk or by a rule. Tell the
|
Only a `pass` review can be approved in bulk or by a rule. Tell the
|
||||||
operator in one line that the review waits in the hub at the `url` it
|
operator the `url` it prints in one plain-text line; chat links do not
|
||||||
prints, written out as plain text: chat links do not open for this
|
open for this operator. Without the script or port 8737, the MCP tool
|
||||||
operator. Without the script, the MCP tool `review_request` takes the
|
`review_request` takes the same fields without the diff.
|
||||||
same fields without the diff.
|
|
||||||
3. Call `review_wait` with its `id` and `timeout_seconds` 240 until the
|
3. Call `review_wait` with its `id` and `timeout_seconds` 240 until the
|
||||||
status is no longer `pending`.
|
status is no longer `pending`.
|
||||||
4. `approved`: in the worktree, verify the operator's passkey signature
|
4. `approved`: in the worktree, verify the operator's passkey signature
|
||||||
@@ -43,9 +42,10 @@ once the operator's pinned passkey signature over it verifies here.
|
|||||||
python3 ~/.local/share/kvb-orchestrator/client/approval.py verify --port 8737 --approvers ~/.config/kvb-orchestrator/approvers.json --replay <approvals>/commit.json --review <id> --gate <gate>
|
python3 ~/.local/share/kvb-orchestrator/client/approval.py verify --port 8737 --approvers ~/.config/kvb-orchestrator/approvers.json --replay <approvals>/commit.json --review <id> --gate <gate>
|
||||||
```
|
```
|
||||||
|
|
||||||
|
Sandboxed without port 8737: save `review_wait`'s answer under `/tmp`,
|
||||||
|
add `--record <file>`.
|
||||||
Exit 0 only: commit as the calling skill says. Any other exit,
|
Exit 0 only: commit as the calling skill says. Any other exit,
|
||||||
`rejected`, or the tree changed: no commit; report the printed reason.
|
`rejected`, or the tree changed: no commit; report the printed reason.
|
||||||
|
|
||||||
The hub is unreachable (no kvb-orchestrator tools, or a call cannot connect), or the
|
No kvb-orchestrator tools, or the verify step refused: ask the calling
|
||||||
verify step refused: ask the calling skill's normal approval question in
|
skill's approval question in chat.
|
||||||
chat instead.
|
|
||||||
@@ -7,6 +7,7 @@ import subprocess
|
|||||||
import sys
|
import sys
|
||||||
import time
|
import time
|
||||||
import unittest
|
import unittest
|
||||||
|
from pathlib import Path
|
||||||
from unittest import mock
|
from unittest import mock
|
||||||
|
|
||||||
import hub_support
|
import hub_support
|
||||||
@@ -214,7 +215,7 @@ class TestHubExecutors(HubCase):
|
|||||||
self.assertFalse(rules.rule_matches({**stored, "proof": None}, "ask", subject, now))
|
self.assertFalse(rules.rule_matches({**stored, "proof": None}, "ask", subject, now))
|
||||||
self.assertFalse(rules.rule_matches(stored, "ask", subject, stored["proof"]["payload"]["expires"] + 1))
|
self.assertFalse(rules.rule_matches(stored, "ask", subject, stored["proof"]["payload"]["expires"] + 1))
|
||||||
|
|
||||||
def test_the_commit_check_verifies_the_staged_tree_before_a_commit(self):
|
def staged_review(self) -> tuple[Path, str, str]:
|
||||||
repo = self.outside / "repo"
|
repo = self.outside / "repo"
|
||||||
repo.mkdir()
|
repo.mkdir()
|
||||||
self.git(repo, "init", "--quiet", "--initial-branch", "feature/x")
|
self.git(repo, "init", "--quiet", "--initial-branch", "feature/x")
|
||||||
@@ -224,20 +225,29 @@ class TestHubExecutors(HubCase):
|
|||||||
common = self.git(repo, "rev-parse", "--path-format=absolute", "--git-common-dir").strip()
|
common = self.git(repo, "rev-parse", "--path-format=absolute", "--git-common-dir").strip()
|
||||||
top = self.git(repo, "rev-parse", "--show-toplevel").strip()
|
top = self.git(repo, "rev-parse", "--show-toplevel").strip()
|
||||||
arguments = {**self.arguments("feat: signed", "", self.page("signed.html", PAGE)), "repo": common, "worktree": top, "tree": tree}
|
arguments = {**self.arguments("feat: signed", "", self.page("signed.html", PAGE)), "repo": common, "worktree": top, "tree": tree}
|
||||||
review_id = self.tool("review_request", arguments)["structuredContent"]["id"]
|
return repo, self.tool("review_request", arguments)["structuredContent"]["id"], tree
|
||||||
|
|
||||||
def verify(gate: str) -> subprocess.CompletedProcess:
|
def approve_review(self, review_id: str) -> None:
|
||||||
argv = ["verify", "--port", str(self.port), "--approvers", str(self.approvers), "--replay", str(self.replay / "commit.json"), "--review", review_id, "--gate", gate]
|
|
||||||
return subprocess.run([sys.executable, str(APPROVAL), *argv], cwd=repo, env={**os.environ, **GIT_ENV}, capture_output=True, text=True, timeout=60, check=False)
|
|
||||||
|
|
||||||
done = verify("pass")
|
|
||||||
self.assertEqual(done.returncode, 1, done.stdout)
|
|
||||||
self.assertIn("the review is pending", done.stderr)
|
|
||||||
signer = self.pin()
|
signer = self.pin()
|
||||||
entries = [self.entry("review", review_id, "", "")]
|
entries = [self.entry("review", review_id, "", "")]
|
||||||
with mock.patch.object(batches, "UNDO", 0.05):
|
with mock.patch.object(batches, "UNDO", 0.05):
|
||||||
self.assertEqual(self.approve(entries, self.signed(entries, signer))[0], 202)
|
self.assertEqual(self.approve(entries, self.signed(entries, signer))[0], 202)
|
||||||
self.settle(lambda: self.status(review_id) == "approved")
|
self.settle(lambda: self.status(review_id) == "approved")
|
||||||
|
|
||||||
|
def verify(self, repo: Path, review_id: str, *extra: str) -> subprocess.CompletedProcess:
|
||||||
|
argv = ["verify", "--port", str(self.port), "--approvers", str(self.approvers), "--review", review_id, *extra]
|
||||||
|
return subprocess.run([sys.executable, str(APPROVAL), *argv], cwd=repo, env={**os.environ, **GIT_ENV}, capture_output=True, text=True, timeout=60, check=False)
|
||||||
|
|
||||||
|
def test_the_commit_check_verifies_the_staged_tree_before_a_commit(self):
|
||||||
|
repo, review_id, tree = self.staged_review()
|
||||||
|
|
||||||
|
def verify(gate: str) -> subprocess.CompletedProcess:
|
||||||
|
return self.verify(repo, review_id, "--replay", str(self.replay / "commit.json"), "--gate", gate)
|
||||||
|
|
||||||
|
done = verify("pass")
|
||||||
|
self.assertEqual(done.returncode, 1, done.stdout)
|
||||||
|
self.assertIn("the review is pending", done.stderr)
|
||||||
|
self.approve_review(review_id)
|
||||||
(repo / "a.txt").write_text("two\n", encoding="utf-8")
|
(repo / "a.txt").write_text("two\n", encoding="utf-8")
|
||||||
self.git(repo, "add", "a.txt")
|
self.git(repo, "add", "a.txt")
|
||||||
done = verify("pass")
|
done = verify("pass")
|
||||||
@@ -252,6 +262,29 @@ class TestHubExecutors(HubCase):
|
|||||||
self.assertEqual(done.returncode, 1, done.stdout)
|
self.assertEqual(done.returncode, 1, done.stdout)
|
||||||
self.assertIn("already used", done.stderr)
|
self.assertIn("already used", done.stderr)
|
||||||
|
|
||||||
|
def test_a_sandboxed_agent_verifies_the_record_the_mcp_tool_answered(self):
|
||||||
|
repo, review_id, tree = self.staged_review()
|
||||||
|
record = self.outside / "record.json"
|
||||||
|
replay = ("--replay", str(self.replay / "record.json"), "--gate", "pass", "--record", str(record))
|
||||||
|
self.assertIsNone(self.tool("review_status", {"id": review_id})["structuredContent"]["proof"])
|
||||||
|
self.approve_review(review_id)
|
||||||
|
answered = self.tool("review_wait", {"id": review_id, "timeout_seconds": 1})["structuredContent"]
|
||||||
|
self.assertEqual((answered["status"], answered["tree"]), ("approved", tree))
|
||||||
|
for content, reason in (
|
||||||
|
("{not json", "cannot be read as JSON"),
|
||||||
|
(json.dumps({**answered, "id": "0" * 16}), f"is not review {review_id}"),
|
||||||
|
(json.dumps({**answered, "proof": None}), "no signature"),
|
||||||
|
):
|
||||||
|
with self.subTest(reason=reason):
|
||||||
|
record.write_text(content, encoding="utf-8")
|
||||||
|
done = self.verify(repo, review_id, *replay)
|
||||||
|
self.assertEqual(done.returncode, 1, done.stdout)
|
||||||
|
self.assertIn(reason, done.stderr)
|
||||||
|
record.write_text(json.dumps(answered), encoding="utf-8")
|
||||||
|
done = self.verify(repo, review_id, *replay)
|
||||||
|
self.assertEqual(done.returncode, 0, done.stderr)
|
||||||
|
self.assertIn(f"tree {tree} on feature/x", done.stdout)
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
unittest.main()
|
unittest.main()
|
||||||
@@ -32,7 +32,7 @@ class TestHubMcp(HubCase):
|
|||||||
self.assertRegex(review_id, r"^[0-9a-f]{16}$")
|
self.assertRegex(review_id, r"^[0-9a-f]{16}$")
|
||||||
state = self.tool("review_status", {"id": review_id})
|
state = self.tool("review_status", {"id": review_id})
|
||||||
self.assertEqual(json.loads(state["content"][0]["text"]), state["structuredContent"])
|
self.assertEqual(json.loads(state["content"][0]["text"]), state["structuredContent"])
|
||||||
self.assertEqual(state["structuredContent"], {"id": review_id, "status": "pending", "tree": TREE, "decided": None})
|
self.assertEqual(state["structuredContent"], {"id": review_id, "status": "pending", "tree": TREE, "decided": None, "proof": None})
|
||||||
listed = self.tool("review_list", {})["structuredContent"]["reviews"]
|
listed = self.tool("review_list", {})["structuredContent"]["reviews"]
|
||||||
self.assertEqual([(r["id"], r["title"], r["tree"]) for r in listed], [(review_id, "feat: hub", TREE)])
|
self.assertEqual([(r["id"], r["title"], r["tree"]) for r in listed], [(review_id, "feat: hub", TREE)])
|
||||||
self.assertEqual(self.rpc("resources/list", {})["error"]["code"], -32601)
|
self.assertEqual(self.rpc("resources/list", {})["error"]["code"], -32601)
|
||||||
|
|||||||
Reference in new issue
Block a user