diff --git a/orchestrator/client/approval.py b/orchestrator/client/approval.py index cb7b722..0d67882 100755 --- a/orchestrator/client/approval.py +++ b/orchestrator/client/approval.py @@ -5,10 +5,12 @@ , written with mode 0600. Run it on the host, outside every sandbox, e.g. --approvers ~/.config/kvb-orchestrator/approvers.json. verify --port --approvers --replay --review - --gate : run in the worktree right before the commit; - fetches the review's signature from the Agent Orchestrator on 127.0.0.1: - and checks it against the repository, worktree, branch and the tree - git write-tree prints, as this process sees them. + --gate [--record ]: run in the worktree right before + the commit; fetches the review's signature from the Agent Orchestrator + on 127.0.0.1:, or reads it from , the JSON object the MCP + tool review_wait or review_status answered, where a sandbox blocks + that port; checks it against the repository, worktree, branch and + the tree git write-tree prints, as this process sees them. request --port --title --page --notes --gate --score --agent [--rules ]: run in the worktree; asks the operator to approve the staged tree, @@ -71,8 +73,18 @@ def fetch(port: int, path: str) -> dict: return data +def recorded(path: Path, review_id: str) -> dict: + try: + data = json.loads(path.read_text(encoding="utf-8")) + except (OSError, ValueError, RecursionError) as error: + raise Refused(f"the record {path} cannot be read as JSON: {error}") from error + if not isinstance(data, dict) or data.get("id") != review_id: + raise Refused(f"the record {path} is not review {review_id}") + return data + + def verify_commit(args: argparse.Namespace) -> str: - review = fetch(args.port, f"/api/reviews/{args.review}") + review = recorded(args.record, args.review) if args.record else fetch(args.port, f"/api/reviews/{args.review}") if review.get("status") != "approved": raise Refused(f"the review is {review.get('status')}, not approved") item = staged(Path.cwd(), args.review) @@ -140,6 +152,7 @@ def main() -> int: verifying.add_argument("--replay", type=Path, required=True) verifying.add_argument("--review", type=review_id, required=True) verifying.add_argument("--gate", choices=("pass", "fail"), required=True) + verifying.add_argument("--record", type=Path) requesting = commands.add_parser("request") requesting.add_argument("--port", type=int, required=True) for name in ("--title", "--score", "--agent"): diff --git a/orchestrator/kvb_orchestrator/mcp.py b/orchestrator/kvb_orchestrator/mcp.py index b90b5ac..4448608 100644 --- a/orchestrator/kvb_orchestrator/mcp.py +++ b/orchestrator/kvb_orchestrator/mcp.py @@ -55,7 +55,7 @@ class Mcp: "score": text_property("The kvb-check-score line of the commit, one line."), }, ), - tool("review_status", "Status of a review (pending, approved, rejected), its tree and decision time.", {"id": review_id}), + tool("review_status", "Status of a review (pending, approved, rejected), its tree, decision time and the passkey proof that approval.py verify --record checks.", {"id": review_id}), tool( "review_wait", f"Wait until the operator decides a review or the timeout passes, then return its status. " diff --git a/orchestrator/kvb_orchestrator/reviewing.py b/orchestrator/kvb_orchestrator/reviewing.py index 7399d00..22ab9dc 100644 --- a/orchestrator/kvb_orchestrator/reviewing.py +++ b/orchestrator/kvb_orchestrator/reviewing.py @@ -40,6 +40,10 @@ def state_of(review: dict) -> dict: return {key: review[key] for key in STATE} +def proven(review: dict) -> dict: + return {**state_of(review), "proof": review.get("proof")} + + def review_row(review: dict) -> dict: created = datetime.datetime.fromisoformat(review["created"]).timestamp() return {**{key: review[key] for key in LISTED}, "notes": review["notes"], "created_ts": created, "files": review.get("files")} @@ -92,7 +96,7 @@ class Reviewing: review = self.reviews.get(review_id) if review is None: raise Missing(review_id) - return {**state_of(review), "proof": review.get("proof")} + return proven(review) def verdict(self, review_id: str, status: str, body: bytes) -> dict: review = self.reviews.get(review_id) @@ -111,13 +115,13 @@ class Reviewing: def status(self, arguments: dict) -> dict: expect(arguments, ("id",)) review_id = field(arguments, "id") - return state_of(found(self.reviews.get(review_id), review_id)) + return proven(found(self.reviews.get(review_id), review_id)) def wait(self, arguments: dict) -> dict: expect(arguments, ("id", "timeout_seconds")) review_id = field(arguments, "id") timeout = seconds(arguments, "timeout_seconds", WAIT_CAP) - return state_of(found(self.reviews.wait(review_id, timeout), review_id)) + return proven(found(self.reviews.wait(review_id, timeout), review_id)) def pending(self, arguments: dict) -> dict: expect(arguments, ()) diff --git a/skills/kvb-git-diff/SKILL.md b/skills/kvb-git-diff/SKILL.md index 887aa87..2a11908 100644 --- a/skills/kvb-git-diff/SKILL.md +++ b/skills/kvb-git-diff/SKILL.md @@ -30,10 +30,9 @@ once the operator's pinned passkey signature over it verifies here. ``` Only a `pass` review can be approved in bulk or by a rule. Tell the - operator in one line that the review waits in the hub at the `url` it - prints, written out as plain text: chat links do not open for this - operator. Without the script, the MCP tool `review_request` takes the - same fields without the diff. + operator the `url` it prints in one plain-text line; chat links do not + open for this operator. Without the script or port 8737, the MCP tool + `review_request` takes the same fields without the diff. 3. Call `review_wait` with its `id` and `timeout_seconds` 240 until the status is no longer `pending`. 4. `approved`: in the worktree, verify the operator's passkey signature @@ -43,9 +42,10 @@ once the operator's pinned passkey signature over it verifies here. python3 ~/.local/share/kvb-orchestrator/client/approval.py verify --port 8737 --approvers ~/.config/kvb-orchestrator/approvers.json --replay /commit.json --review --gate ``` + Sandboxed without port 8737: save `review_wait`'s answer under `/tmp`, + add `--record `. Exit 0 only: commit as the calling skill says. Any other exit, `rejected`, or the tree changed: no commit; report the printed reason. -The hub is unreachable (no kvb-orchestrator tools, or a call cannot connect), or the -verify step refused: ask the calling skill's normal approval question in -chat instead. +No kvb-orchestrator tools, or the verify step refused: ask the calling +skill's approval question in chat. diff --git a/tests/test_hub_executors.py b/tests/test_hub_executors.py index 979f6d6..96f8a82 100644 --- a/tests/test_hub_executors.py +++ b/tests/test_hub_executors.py @@ -7,6 +7,7 @@ import subprocess import sys import time import unittest +from pathlib import Path from unittest import mock import hub_support @@ -214,7 +215,7 @@ class TestHubExecutors(HubCase): self.assertFalse(rules.rule_matches({**stored, "proof": None}, "ask", subject, now)) self.assertFalse(rules.rule_matches(stored, "ask", subject, stored["proof"]["payload"]["expires"] + 1)) - def test_the_commit_check_verifies_the_staged_tree_before_a_commit(self): + def staged_review(self) -> tuple[Path, str, str]: repo = self.outside / "repo" repo.mkdir() self.git(repo, "init", "--quiet", "--initial-branch", "feature/x") @@ -224,20 +225,29 @@ class TestHubExecutors(HubCase): common = self.git(repo, "rev-parse", "--path-format=absolute", "--git-common-dir").strip() top = self.git(repo, "rev-parse", "--show-toplevel").strip() arguments = {**self.arguments("feat: signed", "", self.page("signed.html", PAGE)), "repo": common, "worktree": top, "tree": tree} - review_id = self.tool("review_request", arguments)["structuredContent"]["id"] + return repo, self.tool("review_request", arguments)["structuredContent"]["id"], tree - def verify(gate: str) -> subprocess.CompletedProcess: - argv = ["verify", "--port", str(self.port), "--approvers", str(self.approvers), "--replay", str(self.replay / "commit.json"), "--review", review_id, "--gate", gate] - return subprocess.run([sys.executable, str(APPROVAL), *argv], cwd=repo, env={**os.environ, **GIT_ENV}, capture_output=True, text=True, timeout=60, check=False) - - done = verify("pass") - self.assertEqual(done.returncode, 1, done.stdout) - self.assertIn("the review is pending", done.stderr) + def approve_review(self, review_id: str) -> None: signer = self.pin() entries = [self.entry("review", review_id, "", "")] with mock.patch.object(batches, "UNDO", 0.05): self.assertEqual(self.approve(entries, self.signed(entries, signer))[0], 202) self.settle(lambda: self.status(review_id) == "approved") + + def verify(self, repo: Path, review_id: str, *extra: str) -> subprocess.CompletedProcess: + argv = ["verify", "--port", str(self.port), "--approvers", str(self.approvers), "--review", review_id, *extra] + return subprocess.run([sys.executable, str(APPROVAL), *argv], cwd=repo, env={**os.environ, **GIT_ENV}, capture_output=True, text=True, timeout=60, check=False) + + def test_the_commit_check_verifies_the_staged_tree_before_a_commit(self): + repo, review_id, tree = self.staged_review() + + def verify(gate: str) -> subprocess.CompletedProcess: + return self.verify(repo, review_id, "--replay", str(self.replay / "commit.json"), "--gate", gate) + + done = verify("pass") + self.assertEqual(done.returncode, 1, done.stdout) + self.assertIn("the review is pending", done.stderr) + self.approve_review(review_id) (repo / "a.txt").write_text("two\n", encoding="utf-8") self.git(repo, "add", "a.txt") done = verify("pass") @@ -252,6 +262,29 @@ class TestHubExecutors(HubCase): self.assertEqual(done.returncode, 1, done.stdout) self.assertIn("already used", done.stderr) + def test_a_sandboxed_agent_verifies_the_record_the_mcp_tool_answered(self): + repo, review_id, tree = self.staged_review() + record = self.outside / "record.json" + replay = ("--replay", str(self.replay / "record.json"), "--gate", "pass", "--record", str(record)) + self.assertIsNone(self.tool("review_status", {"id": review_id})["structuredContent"]["proof"]) + self.approve_review(review_id) + answered = self.tool("review_wait", {"id": review_id, "timeout_seconds": 1})["structuredContent"] + self.assertEqual((answered["status"], answered["tree"]), ("approved", tree)) + for content, reason in ( + ("{not json", "cannot be read as JSON"), + (json.dumps({**answered, "id": "0" * 16}), f"is not review {review_id}"), + (json.dumps({**answered, "proof": None}), "no signature"), + ): + with self.subTest(reason=reason): + record.write_text(content, encoding="utf-8") + done = self.verify(repo, review_id, *replay) + self.assertEqual(done.returncode, 1, done.stdout) + self.assertIn(reason, done.stderr) + record.write_text(json.dumps(answered), encoding="utf-8") + done = self.verify(repo, review_id, *replay) + self.assertEqual(done.returncode, 0, done.stderr) + self.assertIn(f"tree {tree} on feature/x", done.stdout) + if __name__ == "__main__": unittest.main() diff --git a/tests/test_hub_mcp.py b/tests/test_hub_mcp.py index c307f73..b36ab99 100644 --- a/tests/test_hub_mcp.py +++ b/tests/test_hub_mcp.py @@ -32,7 +32,7 @@ class TestHubMcp(HubCase): self.assertRegex(review_id, r"^[0-9a-f]{16}$") state = self.tool("review_status", {"id": review_id}) self.assertEqual(json.loads(state["content"][0]["text"]), state["structuredContent"]) - self.assertEqual(state["structuredContent"], {"id": review_id, "status": "pending", "tree": TREE, "decided": None}) + self.assertEqual(state["structuredContent"], {"id": review_id, "status": "pending", "tree": TREE, "decided": None, "proof": None}) listed = self.tool("review_list", {})["structuredContent"]["reviews"] self.assertEqual([(r["id"], r["title"], r["tree"]) for r in listed], [(review_id, "feat: hub", TREE)]) self.assertEqual(self.rpc("resources/list", {})["error"]["code"], -32601)