Merge branch 'feature/hub-match-review' into unify/skills-20261114
# Conflicts: # docs/requirements/README.md
This commit is contained in:
commit
7241e53a98
23 files changed
+1407
-15
No files matched your search
@@ -26,7 +26,9 @@ ITEM = re.compile(r"/api/(asks|decisions)/([0-9a-f]{16})(?:/(allow|deny))?")
|
||||
ENTRY = re.compile(r"/api/stream/([0-9]{1,12})")
|
||||
BOARD = re.compile(r"/api/board/([a-z-]{1,32})")
|
||||
UPLOADS = "/api/uploads"
|
||||
POSTS = ("/api/asks", UPLOADS, "/api/messages")
|
||||
MATCH_ROUTE = "/api/match"
|
||||
MATCH_POSTS = {"/api/match/stale": "match_stale", "/api/match/snapshot": "match_snapshot"}
|
||||
POSTS = ("/api/asks", UPLOADS, "/api/messages", *MATCH_POSTS)
|
||||
STORED = {
|
||||
"page": (".html", "text/html; charset=utf-8", PAGE_CSP),
|
||||
"files": (".json", "application/json", PLAIN_CSP),
|
||||
@@ -128,6 +130,8 @@ class Handler(Exchange):
|
||||
self.answer(lambda: hub.events(self.query()), HTTPStatus.OK)
|
||||
elif path == "/api/tree":
|
||||
self.answer(hub.repositories, HTTPStatus.OK)
|
||||
elif path == MATCH_ROUTE:
|
||||
self.answer(hub.match_cards, HTTPStatus.OK)
|
||||
elif path == "/api/resources":
|
||||
self.answer(lambda: sampled(hub.kept / RESOURCES, hub.span(self.query(), time.time())), HTTPStatus.OK)
|
||||
elif path == "/api/scores":
|
||||
@@ -265,9 +269,11 @@ class Handler(Exchange):
|
||||
"""
|
||||
if path == "/mcp":
|
||||
return self.mcp_receiver(host)
|
||||
if path == "/api/asks":
|
||||
if path == "/api/asks" or path in MATCH_POSTS:
|
||||
if self.local(host):
|
||||
return lambda body: self.answer(lambda: self.server.hub.ask(body), HTTPStatus.CREATED)
|
||||
hub = self.server.hub
|
||||
receive = getattr(hub, MATCH_POSTS[path]) if path in MATCH_POSTS else hub.ask
|
||||
return lambda body: self.answer(lambda: receive(body), HTTPStatus.OK if path in MATCH_POSTS else HTTPStatus.CREATED)
|
||||
self.send_text(HTTPStatus.FORBIDDEN, "forbidden: local POST with the X-Kvb-Orchestrator header and no foreign origin only")
|
||||
return None
|
||||
if RUNNER_ROUTE.fullmatch(path):
|
||||
|
||||
@@ -26,6 +26,7 @@ from .inbox import Inbox
|
||||
from .jobs import Jobs
|
||||
from .language import Language, Model
|
||||
from .ledger import Ledger, Marks, Reviews
|
||||
from .matching import Matching
|
||||
from .mcp import Mcp
|
||||
from .messaging import Messaging
|
||||
from .names import Names
|
||||
@@ -83,7 +84,7 @@ def environment(templates: Path) -> jinja2.Environment:
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Hub(Mcp, Reviewing, Asking, Messaging, Choosing, Approving, Denying, Requesting, Cleaning, Offices, Tasking, Gating, Firewall, Inbox, Rules, Jobs, Sources, Timeline, View, Language):
|
||||
class Hub(Mcp, Reviewing, Asking, Messaging, Choosing, Approving, Denying, Requesting, Cleaning, Offices, Tasking, Gating, Firewall, Inbox, Matching, Rules, Jobs, Sources, Timeline, View, Language):
|
||||
council: Path
|
||||
pages: Path
|
||||
public_port: int
|
||||
@@ -130,6 +131,7 @@ class Hub(Mcp, Reviewing, Asking, Messaging, Choosing, Approving, Denying, Reque
|
||||
history: History
|
||||
queue_order: QueueOrder
|
||||
view_store: ViewStore
|
||||
match_snapshots: Ledger
|
||||
|
||||
def hosts(self) -> tuple[str, str]:
|
||||
return f"127.0.0.1:{self.public_port}", f"localhost:{self.public_port}"
|
||||
@@ -176,6 +178,7 @@ def open_hub(council: Path, pages: Path, public_port: int, state: Path, host_sta
|
||||
voice_notes=VoiceNotes(state / "voice", host_state / "voice", speech, lambda: {message["voice"] for message in messages.listed() if message.get("voice")}),
|
||||
replay=state / "replay" / "messages.json",
|
||||
view_store=ViewStore(state / "view.json"),
|
||||
match_snapshots=Ledger(state / "match.json"),
|
||||
rules=Ledger(state / "rules.json"),
|
||||
offers=Offers(),
|
||||
jobs=Ledger(state / "jobs.json"),
|
||||
|
||||
@@ -0,0 +1,124 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import time
|
||||
|
||||
from .council import read_claims
|
||||
from .diffs import stored_diff
|
||||
from .errors import Missing, ToolError
|
||||
from .fields import decode, expect, field
|
||||
from .payloads import denied_roots, payload_files, payload_hash
|
||||
from .records import digest, iso_epoch
|
||||
from .register import REGISTER, adder, operator_rows, read_register
|
||||
from .signed import signing
|
||||
from .tree import inside
|
||||
|
||||
OPERATOR = "operator/"
|
||||
|
||||
|
||||
def repository_of(common_dir: str) -> str:
|
||||
return common_dir.removesuffix("/.git") if common_dir.endswith("/.git") else common_dir
|
||||
|
||||
|
||||
def born_of(agent: str, names: list[dict], claims: list[dict]) -> float | None:
|
||||
"""Returns:
|
||||
When the agent was first seen: its earliest name sighting, else its
|
||||
earliest claim; None when neither knows it.
|
||||
"""
|
||||
seen = [entry["first_seen"] for entry in names if entry["name"] == agent and isinstance(entry.get("first_seen"), int | float)]
|
||||
claimed = [iso_epoch(claim["since"]) for claim in claims if agent and agent in (claim["agent"], claim["session"])]
|
||||
found = seen or [moment for moment in claimed if moment is not None]
|
||||
return min(found) if found else None
|
||||
|
||||
|
||||
def place_of(directory: str, scan: dict) -> tuple[str, str, str]:
|
||||
"""Returns:
|
||||
The repository, branch and worktree of the deepest worktree of the
|
||||
branch tree that holds directory; empty texts outside every one.
|
||||
"""
|
||||
found = [(repo["path"], branch["name"], tree["path"]) for repo in scan.get("repos", []) for branch in repo["branches"] for tree in branch["worktrees"] if tree["path"] and inside(directory, tree["path"])]
|
||||
return max(found, key=lambda place: len(place[2])) if found else ("", "", directory)
|
||||
|
||||
|
||||
class Matching:
|
||||
def match_cards(self) -> dict:
|
||||
now = time.time()
|
||||
claims = read_claims(self.council)
|
||||
names = self.names.listed()
|
||||
reviews = self.reviews.pending()
|
||||
cards = [self.commit_card(review, reviews, born_of(review.get("agent", ""), names, claims)) for review in reviews]
|
||||
tasks = read_register(self.council / REGISTER)
|
||||
held = self.open_jobs()
|
||||
rows = [row for row in operator_rows(tasks) if row["check"] != "decision" and row["task"] not in held]
|
||||
roots = denied_roots(self.settings_files()[0])
|
||||
scan = self.tree.get()
|
||||
for row in rows:
|
||||
agent = adder(tasks[row["task"]])
|
||||
cards.append(self.command_card(row, agent, born_of(agent, names, claims), roots, scan))
|
||||
listed = {card["id"] for card in cards}
|
||||
for snapshot in self.match_snapshots.listed():
|
||||
if snapshot["id"] not in listed:
|
||||
self.match_snapshots.remove(snapshot["id"])
|
||||
return {"now": now, "cards": cards}
|
||||
|
||||
def commit_card(self, review: dict, reviews: list[dict], born: float | None) -> dict:
|
||||
newer = [other["id"] for other in reviews if other["created"] > review["created"] and (other["worktree"], other["branch"]) == (review["worktree"], review["branch"])]
|
||||
live = review.get("live_tree")
|
||||
reason = f"added again as review:{newer[-1]}" if newer else "the staged tree changed since it was added" if live not in (None, review["tree"]) else ""
|
||||
try:
|
||||
files = stored_diff(self.reviews.pages, "review", review["id"])["diff"]
|
||||
except Missing:
|
||||
files = None
|
||||
place = {"repo": repository_of(review.get("repo", "")), "branch": review["branch"], "worktree": review["worktree"]}
|
||||
return {
|
||||
"id": f"review:{review['id']}", "kind": "commit", "ref": review["id"], "agent": review.get("agent", ""), "born": born, "created_ts": iso_epoch(review["created"]),
|
||||
**place, "title": review["title"], "score": review.get("score", ""), "gate": review.get("gate", ""), "hash": review["tree"], "live_hash": live or review["tree"],
|
||||
"stale": bool(reason), "stale_reason": reason, "files": files, "command": "", "directory": review["worktree"], "reason": "",
|
||||
}
|
||||
|
||||
def command_card(self, row: dict, agent: str, born: float | None, roots: list[str] | None, scan: dict) -> dict:
|
||||
card_id = f"operator:{row['task']}"
|
||||
files = payload_files(row["command"], row["directory"], roots)
|
||||
live = payload_hash(row["command"], row["directory"], files)
|
||||
snapshot = self.match_snapshots.get(card_id)
|
||||
if snapshot is None or snapshot["command_sha256"] != digest(row["command"]):
|
||||
self.match_snapshots.remove(card_id)
|
||||
snapshot = self.match_snapshots.insert(card_id, {"command_sha256": digest(row["command"]), "hash": live, "files": files, "taken": time.time()})
|
||||
stale = snapshot["hash"] != live
|
||||
repo, branch, worktree = place_of(row["directory"], scan)
|
||||
return {
|
||||
"id": card_id, "kind": "command", "ref": row["task"], "agent": agent, "born": born, "created_ts": snapshot["taken"], "repo": repo, "branch": branch, "worktree": worktree,
|
||||
"title": row["command"], "score": "", "gate": "", "hash": snapshot["hash"], "live_hash": live, "stale": stale,
|
||||
"stale_reason": "a payload file changed since it was added" if stale else "", "files": snapshot["files"], "command": row["command"], "directory": row["directory"],
|
||||
"reason": row["why"], "sha256": digest(row["command"]),
|
||||
}
|
||||
|
||||
def match_stale(self, body: bytes) -> dict:
|
||||
"""Args:
|
||||
body: {"id", "tree"}: a pending review and the tree git
|
||||
write-tree prints in its worktree now; a tree other than the
|
||||
review's marks its card stale.
|
||||
"""
|
||||
payload = decode(body)
|
||||
expect(payload, ("id", "tree"))
|
||||
review_id, tree = field(payload, "id"), field(payload, "tree")
|
||||
review = self.reviews.get(review_id)
|
||||
if review is None or review["status"] != "pending":
|
||||
raise Missing(review_id)
|
||||
self.reviews.change(review_id, {"live_tree": tree})
|
||||
return {"id": review_id, "stale": tree != review["tree"]}
|
||||
|
||||
def match_snapshot(self, body: bytes) -> dict:
|
||||
"""Args:
|
||||
body: {"task"}: an open operator row whose card takes its payload
|
||||
files as they are now.
|
||||
"""
|
||||
payload = decode(body)
|
||||
expect(payload, ("task",))
|
||||
task = payload["task"]
|
||||
if not isinstance(task, str) or not task.startswith(OPERATOR) or not signing.TASK.fullmatch(task):
|
||||
raise ToolError("task must be an operator row")
|
||||
self.match_snapshots.remove(f"operator:{task}")
|
||||
card = next((card for card in self.match_cards()["cards"] if card["id"] == f"operator:{task}"), None)
|
||||
if card is None:
|
||||
raise Missing(task)
|
||||
return {"id": card["id"], "hash": card["hash"], "stale": card["stale"]}
|
||||
@@ -0,0 +1,121 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import os
|
||||
import re
|
||||
import shlex
|
||||
import stat
|
||||
|
||||
from .signed import client_module, signing
|
||||
|
||||
sandbox_paths = client_module("sandbox_paths", {})
|
||||
|
||||
PAYLOAD_LIMIT = 64 << 10
|
||||
PAYLOAD_FILES = 8
|
||||
PAYLOAD_ROOT = "/tmp"
|
||||
BINARY_PROBE = 8000
|
||||
EXTENSION = re.compile(r"[^/]+\.[A-Za-z0-9]{1,8}")
|
||||
URL = re.compile(r"[A-Za-z][A-Za-z0-9+.-]*://")
|
||||
|
||||
|
||||
def words_of(command: str) -> list[str]:
|
||||
try:
|
||||
return shlex.split(command)
|
||||
except ValueError:
|
||||
return command.split()
|
||||
|
||||
|
||||
def named_paths(command: str, directory: str) -> list[str]:
|
||||
"""Returns:
|
||||
Every word of the command, or value of a --name=value option, that
|
||||
names a path: one holding a / or ending in a file extension, no
|
||||
URL; relative ones joined to directory, each once, in order.
|
||||
"""
|
||||
found: list[str] = []
|
||||
for word in words_of(command):
|
||||
value = word.split("=", 1)[1] if word.startswith("-") and "=" in word else word
|
||||
if not value or value.startswith("-") or URL.match(value) or not ("/" in value or EXTENSION.fullmatch(value)):
|
||||
continue
|
||||
path = os.path.normpath(os.path.join(directory, os.path.expanduser(value)))
|
||||
if path not in found:
|
||||
found.append(path)
|
||||
return found
|
||||
|
||||
|
||||
def refused(path: str, reason: str, size: int) -> dict:
|
||||
return {"path": path, "state": "refused", "reason": reason, "sha256": "", "size": size, "content": ""}
|
||||
|
||||
|
||||
def read_payload(path: str, roots: list[str] | None) -> dict | None:
|
||||
"""Args:
|
||||
path: an absolute, normalized path a command names.
|
||||
roots: the sandbox denyRead roots, None when the hub knows none yet.
|
||||
|
||||
Returns:
|
||||
The file as shown on a card: read with its content and sha256, or
|
||||
refused with the reason and no content; None when nothing lies at
|
||||
the path or it is no file.
|
||||
"""
|
||||
try:
|
||||
info = os.lstat(path)
|
||||
except OSError:
|
||||
return None
|
||||
if stat.S_ISLNK(info.st_mode):
|
||||
return refused(path, "a symlink, never followed", 0)
|
||||
if not stat.S_ISREG(info.st_mode):
|
||||
return None
|
||||
if not path.startswith(f"{PAYLOAD_ROOT}/") or os.path.realpath(path) != path:
|
||||
return refused(path, f"outside {PAYLOAD_ROOT} or behind a symlinked folder", info.st_size)
|
||||
if roots is None:
|
||||
return refused(path, "no sandbox view yet, so no file is read", info.st_size)
|
||||
if sandbox_paths.denied(path, roots):
|
||||
return refused(path, "the sandbox denies reading it", info.st_size)
|
||||
if info.st_size > PAYLOAD_LIMIT:
|
||||
return refused(path, f"{info.st_size} bytes, over the {PAYLOAD_LIMIT} bytes a card shows", info.st_size)
|
||||
try:
|
||||
descriptor = os.open(path, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK)
|
||||
except OSError as error:
|
||||
return refused(path, f"unreadable: {error.strerror}", info.st_size)
|
||||
with os.fdopen(descriptor, "rb") as handle:
|
||||
if not stat.S_ISREG(os.fstat(handle.fileno()).st_mode):
|
||||
return refused(path, "no regular file", info.st_size)
|
||||
data = handle.read(PAYLOAD_LIMIT + 1)
|
||||
if len(data) > PAYLOAD_LIMIT:
|
||||
return refused(path, f"over the {PAYLOAD_LIMIT} bytes a card shows", len(data))
|
||||
if b"\0" in data[:BINARY_PROBE]:
|
||||
return refused(path, "a binary file", len(data))
|
||||
return {"path": path, "state": "read", "reason": "", "sha256": hashlib.sha256(data).hexdigest(), "size": len(data), "content": data.decode("utf-8", "replace")}
|
||||
|
||||
|
||||
def payload_files(command: str, directory: str, roots: list[str] | None) -> list[dict]:
|
||||
found = (read_payload(path, roots) for path in named_paths(command, directory) if directory.startswith("/"))
|
||||
return [entry for entry in found if entry is not None][:PAYLOAD_FILES]
|
||||
|
||||
|
||||
def payload_hash(command: str, directory: str, files: list[dict]) -> str:
|
||||
"""Returns:
|
||||
The sha256 a command card is bound to: its command, directory and
|
||||
each named file's path, state, reason and content digest.
|
||||
"""
|
||||
listed = [[entry["path"], entry["state"], entry["reason"], entry["sha256"]] for entry in files]
|
||||
return signing.item_hash({"kind": "payload", "command": command, "directory": directory, "files": listed})
|
||||
|
||||
|
||||
def denied_roots(settings_files: list[dict]) -> list[str] | None:
|
||||
"""Args:
|
||||
settings_files: the settings files of the hub's sandbox view, each
|
||||
with its permissions and sandbox sections.
|
||||
|
||||
Returns:
|
||||
Every absolute sandbox.filesystem.denyRead entry of them, None when
|
||||
the view lists no file.
|
||||
"""
|
||||
if not settings_files:
|
||||
return None
|
||||
roots: list[str] = []
|
||||
for entry in settings_files:
|
||||
sandbox = entry["settings"].get("sandbox")
|
||||
filesystem = sandbox.get("filesystem") if isinstance(sandbox, dict) else None
|
||||
listed = filesystem.get("denyRead") if isinstance(filesystem, dict) else None
|
||||
roots += [path.rstrip("/") or "/" for path in listed or [] if isinstance(path, str) and path.startswith("/")]
|
||||
return roots
|
||||
Reference in new issue
Block a user