From 23a7d2dc1d1d05de0bef87b6404a6b95e3e95863 Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Sat, 14 Nov 2026 21:29:42 +0000 Subject: [PATCH] feat(orchestrator): show firewall and hub rules in one table with hits MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit One rule table replaces the two lists: each allow, ask and deny rule shows its hits from the extracted dry-run matcher, ask patterns the matcher never grants read never granted instead of unused, and the table scrolls in its own container below 600 px. Tests run once on unify/skills-20261114 after the merge (operator order 2026-11-14). 🎯 Fix: 85 % 💥 Break: 10 % 💸 Tokens: 100 % ⏱️ Velocity: 103 % 🛡️ Security: 100 % 🏆 Score: 74 🚦 Gate: ⛔ fix 85 % < 90, break 10 % > 5; velocity 103 % > 100: ask the operator before the commit Co-Authored-By: Claude Opus 5.5 --- docs/requirements/001-agent-orchestrator.md | 8 +- orchestrator/kvb_orchestrator/firewall.py | 81 ++++++++++- orchestrator/kvb_orchestrator/requesting.py | 36 ++++- orchestrator/templates/page-firewall.js | 146 +++++++++++++++++--- orchestrator/templates/page-rules.js | 39 ++---- orchestrator/templates/style-firewall.css | 17 ++- tests/e2e/test_firewall_e2e.py | 75 +++++++--- tests/test_hub_firewall.py | 47 ++++++- tests/test_page_reveal.py | 13 +- 9 files changed, 377 insertions(+), 85 deletions(-) diff --git a/docs/requirements/001-agent-orchestrator.md b/docs/requirements/001-agent-orchestrator.md index 3288a39..aa0b4ca 100644 --- a/docs/requirements/001-agent-orchestrator.md +++ b/docs/requirements/001-agent-orchestrator.md @@ -748,8 +748,12 @@ flowchart TD ### Phase 6: Firewall (S6) - [x] The tab is called Firewall and lists hub rules, allow and deny - patterns and the settings permissions, grouped by tool and prefix as - read from the settings files. + patterns and the settings permissions in one rule table (Action, Tool, + Pattern, Scope, Source, Hits over 7 days) with Allow, Ask, Deny, + Sandbox and Signed filter chips and one folded group per source; hits + come from the ask ledger through the rule request dry-run matcher, + allow and deny rows show not seen, an ask row without hits is flagged + unused. - [x] A rule or rule set applies to one agent, a named group or all agents. - [x] A settings change is written only by the host executor after the passkey check, only to a listed file, with a backup beside it; a forged, diff --git a/orchestrator/kvb_orchestrator/firewall.py b/orchestrator/kvb_orchestrator/firewall.py index ab5ffeb..386d8c3 100644 --- a/orchestrator/kvb_orchestrator/firewall.py +++ b/orchestrator/kvb_orchestrator/firewall.py @@ -10,6 +10,7 @@ from .errors import Conflict, Missing, ToolError from .fields import decode, expect, visible from .ledger import Ledger from .records import read_json, stamp +from .requesting import RECENT_DAYS, matched from .rules import EVERY_AGENT, RULE_KINDS from .signed import signing @@ -25,21 +26,86 @@ HASH = re.compile(r"[0-9a-f]{64}") PERMISSION_LISTS = ("allow", "ask", "deny") SET_KEYS = ("kind", "tool", "pattern", "scope") CHAINED = "chains, pipes, redirects or substitutions" +COUNT_ID = "0" * 16 +UNSEEN = {"hits": None, "never": ""} def settings_request(raw: object) -> bool: return isinstance(raw, list) and bool(raw) and all(isinstance(entry, dict) and entry.get("kind") == "settings" for entry in raw) +def tool_and_spec(entry: str) -> tuple[str, str]: + """Returns: + The tool of a permission rule such as Bash(git push:*) and its + specifier, git push:*; "" without specifier. + """ + found = RULE_TEXT.fullmatch(entry) + if found is None: + return entry.strip(), "" + return found.group("tool").strip(), (found.group("spec") or "").strip() + + def tool_and_prefix(entry: str) -> tuple[str, str]: """Returns: The tool of a permission rule such as Bash(git push:*) and the first word of its specifier, for Bash the command; "" without specifier. """ - found = RULE_TEXT.fullmatch(entry) - if found is None: - return entry.strip(), "" - return found.group("tool").strip(), WORD.split((found.group("spec") or "").strip(), maxsplit=1)[0] + tool, spec = tool_and_spec(entry) + return tool, WORD.split(spec, maxsplit=1)[0] + + +def entry_rule(entry: str) -> dict: + """Returns: + A permissions.ask entry as the fields of an ask rule for every agent + in every directory: its tool, and its specifier as pattern, where a + trailing :* or no specifier matches any rest. + """ + tool, spec = tool_and_spec(entry) + pattern = f"{spec.removesuffix(':*')}*" if spec.endswith(":*") or not spec else spec + return {"kind": "ask", "tool": tool, "pattern": pattern, "scope": "*"} + + +def hits(rule: dict, asks: list[dict], now: float) -> dict: + """Args: + rule: kind ask, tool, pattern and scope of a rule, with agents when + it applies to one agent or group only. + asks: the asks of the last RECENT_DAYS as recent_subjects() lists + them. + now: the current time in epoch seconds. + + Returns: + {"hits": how many of the asks the rule matches, on or off, expired or + not, "never": ""}; hits None and never the warning when the pattern + itself names a command the matcher never grants. + """ + pattern = rule["pattern"] + never = [*signing.warnings(pattern), *([CHAINED] if signing.CHAINED.search(pattern) else [])] + if never: + return {"hits": None, "never": never[0]} + return {"hits": matched({**rule, "id": COUNT_ID, "expires": now + RECENT_DAYS * 86400}, asks, now), "never": ""} + + +def counted(groups: list[dict], asks: list[dict], now: float) -> list[dict]: + """Returns: + groups with each entry's specifier and hits: counted for an ask + entry, UNSEEN for an allow or deny entry. + """ + def shown(entry: dict) -> dict: + found = hits(entry_rule(entry["entry"]), asks, now) if entry["list"] == "ask" else UNSEEN + return {**entry, "spec": tool_and_spec(entry["entry"])[1], **found} + + return [{**group, "entries": [shown(entry) for entry in group["entries"]]} for group in groups] + + +def stored_hits(rule: dict, asks: list[dict], now: float) -> dict: + """Returns: + hits() of a stored ask rule for the agents it is signed for, UNSEEN + for a review or decision rule. + """ + if rule["kind"] != "ask": + return UNSEEN + fields = {key: rule[key] for key in ("kind", "tool", "pattern", "scope")} + return hits({**fields, "agents": rule["agents"]} if "agents" in rule else fields, asks, now) def grouped(permissions: object) -> list[dict]: @@ -130,8 +196,11 @@ class Firewall: def firewall(self) -> dict: files, meta = self.settings_files() - listed = [{key: entry[key] for key in ("path", "exists", "sha256", "error")} | {"groups": grouped(entry["settings"].get("permissions")), "sandbox": sandbox_entries(entry["settings"].get("sandbox"), "sandbox")} for entry in files] - return {**meta, "files": listed, "lists": list(signing.SETTINGS_LISTS), "patterns": deny_patterns(), "groups": self.groups.listed(), "sets": self.rule_sets.listed()} + now = time.time() + asks = self.recent_subjects("ask", now) + listed = [{key: entry[key] for key in ("path", "exists", "sha256", "error")} | {"groups": counted(grouped(entry["settings"].get("permissions")), asks, now), "sandbox": sandbox_entries(entry["settings"].get("sandbox"), "sandbox")} for entry in files] + rules = {rule["id"]: stored_hits(rule, asks, now) for rule in self.rules.listed()} + return {**meta, "files": listed, "lists": list(signing.SETTINGS_LISTS), "patterns": deny_patterns(), "groups": self.groups.listed(), "sets": self.rule_sets.listed(), "days": RECENT_DAYS, "rules": rules} def settings_changes(self, raw: object) -> list[dict]: """Args: diff --git a/orchestrator/kvb_orchestrator/requesting.py b/orchestrator/kvb_orchestrator/requesting.py index 496f4ea..4a19927 100644 --- a/orchestrator/kvb_orchestrator/requesting.py +++ b/orchestrator/kvb_orchestrator/requesting.py @@ -26,6 +26,24 @@ def created_of(record: dict) -> float: return record["created_ts"] if "created_ts" in record else iso_epoch(record["created"]) or 0.0 +def matched(record: dict, subjects: list[dict], now: float) -> int: + """Args: + record: id, kind, tool, pattern, scope and expires of a rule, with + agents when it applies to one agent or group only. + subjects: what recent_subjects() lists for the rule's kind. + now: the current time in epoch seconds. + + Returns: + How many subjects the rule would have approved. + """ + kind_name = record["kind"] + payload = signing.rule_payload(record["id"], [kind_name], record["tool"], record["pattern"], record["scope"], int(record["expires"])) + if "agents" in record: + payload = signing.targeted(payload, record["agents"]) + rule = {**{key: record[key] for key in RULE_FIELDS}, "enabled": True, "proof": {"payload": payload, "assertion": None}} + return sum(rule_matches(rule, kind_name, subject, now) for subject in subjects) + + class Requesting: def open_requests(self, now: float) -> list[dict]: return [record for record in self.rule_requests.pending() if request_status(record, now) == "pending"] @@ -60,19 +78,23 @@ class Requesting: raise ToolError(f"no rule request {request_id}") return request_state(record, time.time()) + def recent_subjects(self, kind_name: str, now: float) -> list[dict]: + """Returns: + The asks, reviews or decisions of the last RECENT_DAYS as a rule + of that kind is matched against them. + """ + ledger = {"ask": self.asks, "review": self.reviews, "decision": self.decisions}[kind_name] + since = now - RECENT_DAYS * 86400 + return [subject_of(kind_name, entry, self.deciders(kind_name, entry)) for entry in ledger.listed() if created_of(entry) >= since] + def dry_run(self, record: dict, now: float) -> dict: """Returns: How many of the asks, reviews or decisions of the last RECENT_DAYS the requested rule would have approved, matched by the same code that applies a stored rule. """ - kind_name = record["kind"] - payload = signing.rule_payload(record["id"], [kind_name], record["tool"], record["pattern"], record["scope"], int(record["expires"])) - rule = {**{key: record[key] for key in RULE_FIELDS}, "enabled": True, "proof": {"payload": payload, "assertion": None}} - ledger = {"ask": self.asks, "review": self.reviews, "decision": self.decisions}[kind_name] - since = now - RECENT_DAYS * 86400 - subjects = [subject_of(kind_name, entry, self.deciders(kind_name, entry)) for entry in ledger.listed() if created_of(entry) >= since] - return {"matched": sum(rule_matches(rule, kind_name, subject, now) for subject in subjects), "total": len(subjects), "days": RECENT_DAYS} + subjects = self.recent_subjects(record["kind"], now) + return {"matched": matched(record, subjects, now), "total": len(subjects), "days": RECENT_DAYS} def request_items(self, base: dict, now: float, claims: list[dict]) -> list[dict]: items = [] diff --git a/orchestrator/templates/page-firewall.js b/orchestrator/templates/page-firewall.js index 4c3b99f..bda8134 100644 --- a/orchestrator/templates/page-firewall.js +++ b/orchestrator/templates/page-firewall.js @@ -1,11 +1,20 @@ const FIREWALL_LISTS = {"permissions.allow": "Allow", "permissions.ask": "Ask", "permissions.deny": "Deny", "sandbox.filesystem.allowWrite": "Sandbox: write", "sandbox.filesystem.denyRead": "Sandbox: no read", "sandbox.network.allowedDomains": "Sandbox: domain"}; const LIST_TONES = {allow: "ok", ask: "warn", deny: "bad"}; + const ROW_FILTERS = [["allow", "Allow"], ["ask", "Ask"], ["deny", "Deny"], ["sandbox", "Sandbox"], ["signed", "Signed"]]; + const ACTION_ORDER = ["deny", "ask", "allow", "sandbox", "signed"]; + const ENTRY_HINTS = { + allow: "Claude Code runs a matching call without asking, so it never reaches the ask ledger.", + ask: "Claude Code asks before a matching call; the ask reaches the Agent Orchestrator.", + deny: "Claude Code refuses a matching call, so it never reaches the ask ledger.", + sandbox: "Bounds what a sandboxed command may write, read or reach.", + }; + const HUB_SOURCE = "hub"; const SETTINGS_RESULTS = {done: ["written", "ok"], refused: ["refused", "bad"], failed: ["failed", "bad"]}; const EVERY_AGENT = "*"; const TARGET_GONE = "gone"; const TARGET_PICK = ""; const SESSION_KEY = /^(?!-)[A-Za-z0-9_-]{1,128}$/; - const firewall = {data: null, busy: false, staged: {}, dom: null, targets: new Set(), picks: {}}; + const firewall = {data: null, busy: false, staged: {}, dom: null, targets: new Set(), picks: {}, shown: new Set(ROW_FILTERS.map(([id]) => id)), open: new Set()}; const knownAgents = () => Object.keys(people()).filter((key) => key !== HUMAN && SESSION_KEY.test(key)).sort(); const firewallGroups = () => (firewall.data ? firewall.data.groups : []); @@ -270,24 +279,121 @@ drawFirewall(); } - function entryRow(path, list, entry) { - const last = list.split(".").pop(); - const tone = Object.hasOwn(LIST_TONES, last) ? LIST_TONES[last] : "neutral"; - const editable = Object.hasOwn(FIREWALL_LISTS, list); - return h("li", {"data-entry": entry, "data-list": list}, h("div", {class: "row"}, pill(FIREWALL_LISTS[list] || list, tone), h("code", {}, revealed(entry)), h("span", {class: "spacer"}), - editable ? iconBtn("bi-dash-circle", "Remove", "btn small ghost", () => stageChange(path, {list, op: "remove", entry}), `Remove: stage the removal of ${revealed(entry)} from ${list}`) : null)); + /** + * @param {Object} file a settings file as /api/firewall lists it. + * @returns {Object[]} one row per permissions entry and sandbox value: filter, action, tone, tool, pattern, list, entry (null for a sandbox value no list holds), hits and never. + */ + function fileRows(file) { + const permissions = file.groups.flatMap((group) => group.entries.map((entry) => ({filter: entry.list, action: FIREWALL_LISTS[`permissions.${entry.list}`], tone: LIST_TONES[entry.list], tool: group.tool, pattern: entry.spec, list: `permissions.${entry.list}`, entry: entry.entry, hits: entry.hits, never: entry.never}))); + const sandbox = file.sandbox.flatMap((entry) => { + const base = {filter: "sandbox", action: FIREWALL_LISTS[entry.key] || "Sandbox", tone: "info", tool: entry.key.replace(/^sandbox\./, ""), list: entry.key, hits: null, never: ""}; + return entry.values ? entry.values.map((value) => ({...base, pattern: value, entry: value})) : [{...base, pattern: JSON.stringify(entry.value), entry: null}]; + }); + return [...permissions, ...sandbox]; } - function fileCard(file) { - const groups = file.groups.map((group) => h("details", {class: "fw-group", open: true, "data-tool": group.tool, "data-prefix": group.prefix}, - h("summary", {}, h("strong", {}, group.tool), group.prefix ? h("code", {}, group.prefix) : null, counter(group.entries.length)), - h("ul", {class: "mini"}, group.entries.map((entry) => entryRow(file.path, `permissions.${entry.list}`, entry.entry))))); - const sandbox = file.sandbox.map((entry) => (entry.values - ? h("details", {class: "fw-group", "data-key": entry.key}, h("summary", {}, h("strong", {}, entry.key), counter(entry.values.length)), h("ul", {class: "mini"}, entry.values.map((value) => entryRow(file.path, entry.key, value)))) - : h("div", {class: "rule-row", "data-key": entry.key}, h("strong", {}, entry.key), h("code", {}, JSON.stringify(entry.value))))); - const status = file.error ? pill(file.error, "bad") : file.exists ? pill(`sha256 ${file.sha256.slice(0, 12)}`, "neutral") : pill("missing: a change creates it", "warn"); - return h("section", {class: "fw-file", "data-file": file.path, "aria-label": file.path}, h("div", {class: "row"}, h("code", {class: "fw-path"}, file.path), status), - groups.length ? groups : h("p", {class: "muted"}, "No permission entries."), sandbox.length ? h("h4", {}, "Sandbox") : null, sandbox); + /** + * @returns {Object[]} one row per stored hub rule, with the hits /api/firewall counted for it; pending until the next fetch counts a new rule. + */ + function signedRows() { + const counted = firewall.data.rules; + return state.rules.map((rule) => ({filter: "signed", action: `Grant ${rule.kind}`, tone: "ok", tool: rule.tool, pattern: rule.pattern, rule, ...(Object.hasOwn(counted, rule.id) ? counted[rule.id] : {hits: null, never: "", pending: true})})); + } + + const fileStatus = (file) => (file.error ? pill(file.error, "bad") : file.exists ? pill(`sha256 ${file.sha256.slice(0, 12)}`, "neutral") : pill("missing: a change creates it", "warn")); + const byAction = (one, two) => ACTION_ORDER.indexOf(one.filter) - ACTION_ORDER.indexOf(two.filter); + + function sourceGroups() { + return [{id: HUB_SOURCE, label: "Hub rules", file: false, status: null, rows: signedRows()}, ...firewall.data.files.map((file) => ({id: file.path, label: file.path, file: true, status: fileStatus(file), rows: fileRows(file)}))]; + } + + function hitsCell(row) { + if (row.pending) { + return h("span", {class: "muted", title: "Counted with the next refresh"}, "…"); + } + if (row.never) { + return h("span", {class: "muted", title: `The matcher never grants a command that ${row.never}`}, `never granted: ${row.never}`); + } + if (row.hits === null) { + return h("span", {class: "muted", title: "Only asks reach the ledger the hits come from"}, "not seen"); + } + return row.hits ? h("strong", {}, String(row.hits)) : ["0 ", h("span", {class: "pill", "data-tone": "warn", title: "No ask of the counted days matched this entry"}, "unused")]; + } + + function actionCell(row) { + const rule = row.rule; + const action = h("span", {class: "pill", "data-tone": row.tone, title: rule ? RULE_HINTS[rule.kind] : ENTRY_HINTS[row.filter]}, row.action); + if (!rule) { + return action; + } + const until = `until ${absolute.format(stampOf(rule.expires))}`; + return [action, rule.expires <= nowSeconds() ? pill("expired", "bad") : null, h("span", {class: "pill", "data-tone": rule.enabled ? "ok" : "neutral", title: until}, rule.enabled ? "active" : "off")]; + } + + function changeCell(group, row) { + const rule = row.rule; + if (rule) { + return [btn(rule.enabled ? "Turn off" : "Turn on", "btn small", () => changeRule(rule, rule.enabled ? "disable" : "enable")), btn("Delete", "btn small danger", () => { + if (window.confirm("Delete rule?")) { + changeRule(rule, "delete"); + } + })]; + } + const editable = row.entry !== null && Object.hasOwn(FIREWALL_LISTS, row.list); + return editable ? iconBtn("bi-dash-circle", "Remove", "btn small ghost", () => stageChange(group.id, {list: row.list, op: "remove", entry: row.entry}), `Remove: stage the removal of ${revealed(row.entry)} from ${row.list}`) : null; + } + + /** + * @param {Object} group the source the row belongs to: id, label and file, false for the hub rules. + * @param {Object} row a row of fileRows() or signedRows(). + * @returns {HTMLTableRowElement} the row: action, tool, pattern, scope, source, hits and its change buttons. + */ + function ruleRow(group, row) { + const rule = row.rule; + const pattern = [h("code", {}, revealed(row.pattern)), rule && rule.kind !== "decision" ? h("span", {class: "muted"}, " in ", h("code", {}, rule.scope ? revealed(rule.scope) : "–")) : null]; + const source = rule ? h("span", {class: "muted"}, rule.signed ? "Hub, passkey-signed" : "Hub, unsigned: has no effect") : h("code", {class: "fw-path"}, revealed(group.label)); + return h("tr", {"data-filter": row.filter, "data-list": row.list, "data-entry": row.entry, "data-rule": rule ? rule.id : null, "data-unused": row.hits === 0}, + h("td", {}, actionCell(row)), h("td", {}, row.tool ? h("code", {}, revealed(row.tool)) : "–"), h("td", {}, pattern), + h("td", {}, rule ? targetPill(rule.agents) : h("span", {class: "muted"}, "file")), h("td", {}, source), + h("td", {class: "fw-hits"}, hitsCell(row)), h("td", {class: "fw-change"}, changeCell(group, row))); + } + + function sourceBody(group) { + const rows = group.rows.filter((row) => firewall.shown.has(row.filter)).sort(byAction); + if (!rows.length && group.rows.length) { + return null; + } + const open = firewall.open.has(group.id); + const unused = rows.filter((row) => row.hits === 0).length; + const fold = h("button", {type: "button", class: "fw-fold", "aria-expanded": String(open), on: {click: () => { + if (open) { + firewall.open.delete(group.id); + } else { + firewall.open.add(group.id); + } + drawFirewall(); + }}}, icon(open ? "bi-chevron-down" : "bi-chevron-right"), group.file ? h("code", {class: "fw-path"}, revealed(group.label)) : h("strong", {}, group.label), counter(rows.length), unused ? pill(`${unused} unused`, "warn") : null); + return h("tbody", {"data-source": group.id}, h("tr", {class: "fw-source"}, h("th", {colspan: 7, scope: "rowgroup"}, h("div", {class: "row"}, fold, group.status))), open ? rows.map((row) => ruleRow(group, row)) : []); + } + + function filterChips() { + return h("div", {class: "chips", role: "group", "aria-label": "Show rules"}, ROW_FILTERS.map(([id, label]) => h("button", {type: "button", class: "chip", "data-filter": id, "aria-pressed": String(firewall.shown.has(id)), on: {click: () => { + if (firewall.shown.has(id)) { + firewall.shown.delete(id); + } else { + firewall.shown.add(id); + } + drawFirewall(); + }}}, label))); + } + + function ruleTable() { + const bodies = sourceGroups().map(sourceBody).filter(Boolean); + if (!bodies.length) { + return h("p", {class: "muted"}, "No rule matches the chosen chips."); + } + const head = h("thead", {}, h("tr", {}, ["Action", "Tool", "Pattern", "Scope", "Source", `Hits (${firewall.data.days} days)`].map((label) => h("th", {scope: "col"}, label)), h("th", {scope: "col", "aria-label": "Change"}))); + return h("div", {class: "fw-table-wrap"}, h("table", {class: "fw-table", "aria-label": "Firewall rules"}, head, bodies)); } function addForm(files) { @@ -325,10 +431,10 @@ return h("section", {class: "card"}, h("p", {class: "muted"}, "Loading the settings …")); } const files = data.files; - return h("section", {class: "card stack-tight", "aria-label": "Claude settings"}, h("h3", {}, "Claude settings"), - h("p", {class: "muted"}, "Permissions and sandbox entries of the listed settings files, grouped by tool and prefix as the host read them. A change shows as a diff, is signed with your passkey and is written by the host, never by an agent."), + return h("section", {class: "card stack-tight fw-rules", "aria-label": "Firewall rules"}, h("h3", {}, "Firewall rules"), + h("p", {class: "muted"}, `Every rule in one table: the hub rules and the permissions and sandbox entries of the listed settings files as the host read them, one folded group per source. Hits count the asks of the last ${data.days} days a rule matches, with the matcher of the rule request dry run; allow, deny and sandbox entries never reach the ask ledger and show not seen. A settings change shows as a diff, is signed with your passkey and is written by the host, never by an agent.`), data.error ? h("p", {class: "muted"}, data.error) : null, - files.map(fileCard), files.length ? addForm(files) : null, stagedList()); + filterChips(), ruleTable(), files.length ? addForm(files) : null, stagedList()); } function diffLines(text) { diff --git a/orchestrator/templates/page-rules.js b/orchestrator/templates/page-rules.js index 9a1ccf9..7d529dd 100644 --- a/orchestrator/templates/page-rules.js +++ b/orchestrator/templates/page-rules.js @@ -14,6 +14,18 @@ return signed ? sendJson(`/api/rules/${encodeURIComponent(rule.id)}/enable`, {proof: signed.proof}) : null; } + /** + * @param {Object} rule the stored rule. + * @param {string} verb enable, disable or delete. + */ + async function changeRule(rule, verb) { + const result = verb === "enable" ? await turnOn(rule) : await send(`/api/rules/${encodeURIComponent(rule.id)}/${verb}`, "", "application/json"); + if (result && !result.ok) { + toast("Rule not changed", fail(result), false); + } + await load(); + } + function rulesScreen() { const kind = h("select", {class: "field", "aria-label": "Kind"}, Object.entries(RULE_KINDS).map(([value, label]) => h("option", {value}, label))); const target = targetSelect("Applies to", null); @@ -63,29 +75,6 @@ toast(result.ok ? "Rule created" : "Rule not created", result.ok ? [] : fail(result), result.ok); await load(); }); - const change = async (rule, verb) => { - const result = verb === "enable" ? await turnOn(rule) : await send(`/api/rules/${encodeURIComponent(rule.id)}/${verb}`, "", "application/json"); - if (result && !result.ok) { - toast("Rule not changed", fail(result), false); - } - await load(); - }; - const now = nowSeconds(); - const list = state.rules.length ? h("div", {class: "mini"}, state.rules.map((rule) => h("div", {class: "rule-row"}, - pill(rule.kind, KIND_TONES[rule.kind]), - targetPill(rule.agents), - rule.tool ? h("code", {}, revealed(rule.tool)) : null, - h("code", {}, revealed(rule.pattern)), - rule.kind === "decision" ? null : h("code", {title: "Directory"}, rule.scope ? revealed(rule.scope) : "–"), - pill(rule.signed ? "signed" : "unsigned: has no effect", rule.signed ? "ok" : "bad"), - rule.expires <= now ? pill("expired", "bad") : h("span", {class: "muted"}, `until ${absolute.format(stampOf(rule.expires))}`), - rule.enabled ? pill("active", "ok") : pill("off", "neutral"), - h("span", {class: "spacer"}), - btn(rule.enabled ? "Turn off" : "Turn on", "btn small", () => change(rule, rule.enabled ? "disable" : "enable")), - btn("Delete", "btn small danger", () => { - if (window.confirm("Delete rule?")) { - change(rule, "delete"); - } - })))) : h("p", {class: "muted"}, "No rule yet. Every automatic grant shows up in the stream and can be undone for 3 s."); - return h("div", {class: "stack"}, h("h2", {class: "title"}, "Firewall"), h("h3", {}, "Hub rules for automatic grants"), list, h("div", {class: "card"}, hint, form), ...firewallSections()); + const [table, ...rest] = firewallSections(); + return h("div", {class: "stack"}, h("h2", {class: "title"}, "Firewall"), table, h("div", {class: "card"}, h("h3", {}, "New hub rule for automatic grants"), hint, form), ...rest); } diff --git a/orchestrator/templates/style-firewall.css b/orchestrator/templates/style-firewall.css index 9a8bea7..b8abc85 100644 --- a/orchestrator/templates/style-firewall.css +++ b/orchestrator/templates/style-firewall.css @@ -2,11 +2,20 @@ .fw-form { display: grid; grid-template-columns: repeat(auto-fill, minmax(min(100%, 200px), 1fr)); gap: 10px; align-items: end; } .fw-form label { display: grid; gap: 4px; min-width: 0; color: var(--muted); font-size: var(--fs-sm); } .fw-form .field { width: 100%; min-width: 0; } -.fw-file { display: grid; gap: 8px; min-width: 0; padding: 10px; border: 1px solid var(--line); border-radius: 10px; } -.fw-file > .row { display: flex; flex-wrap: wrap; align-items: center; gap: 8px; } .fw-path { overflow-wrap: anywhere; } -.fw-group > summary { display: flex; flex-wrap: wrap; align-items: center; gap: 8px; cursor: pointer; } -.fw-group code, .fw-patterns code { overflow-wrap: anywhere; } +.fw-patterns code { overflow-wrap: anywhere; } +.fw-rules { grid-template-columns: minmax(0, 1fr); } +.fw-table-wrap { min-width: 0; max-width: 100%; overflow-x: auto; border: 1px solid var(--line); border-radius: 10px; } +.fw-table { width: 100%; min-width: 600px; border-collapse: collapse; font-size: var(--fs-sm); } +.fw-table th, .fw-table td { padding: 6px 8px; border-bottom: 1px solid var(--line); text-align: left; vertical-align: top; } +.fw-table thead th { background: var(--sunken); color: var(--muted); font-weight: 650; white-space: nowrap; } +.fw-table td { overflow-wrap: anywhere; } +.fw-table td .pill { margin: 0 4px 2px 0; } +.fw-source > th { background: var(--sunken); font-weight: 560; } +.fw-source .row { display: flex; flex-wrap: wrap; align-items: center; gap: 8px; } +.fw-fold { display: inline-flex; flex-wrap: wrap; align-items: center; gap: 8px; min-width: 0; padding: 0; border: 0; background: none; color: var(--text); font: inherit; text-align: left; cursor: pointer; } +.fw-hits { white-space: nowrap; font-variant-numeric: tabular-nums; } +.fw-change { white-space: nowrap; text-align: right; } .fw-diff { max-height: 50vh; margin: 4px 0 0; padding: 8px; overflow: auto; border-radius: 8px; background: var(--sunken); font-size: var(--fs-sm); white-space: pre-wrap; overflow-wrap: anywhere; } .fw-diff .add { color: var(--ok); } .fw-diff .del { color: var(--bad); } diff --git a/tests/e2e/test_firewall_e2e.py b/tests/e2e/test_firewall_e2e.py index 2eed4ff..b052049 100644 --- a/tests/e2e/test_firewall_e2e.py +++ b/tests/e2e/test_firewall_e2e.py @@ -14,35 +14,56 @@ sys.path[:0] = [str(HERE.parent), str(HERE)] import test_orchestrator_e2e as base from hub_case import HubCase -from hub_support import verifier +from hub_support import ASK, verifier from test_settings_writer import ORIGINAL from test_settings_writer import host as writer_host OUT = Path(os.environ.get("E2E_OUT", "/tmp/firewall-shots")) SECRET = ORIGINAL["env"]["EXAMPLE_FLAG"] +ASKED = ["Bash(make deploy:*)", "Bash(make clean:*)", "Bash(git push:*)"] SHOW = """(async () => {@HELPERS@ await screenShown(8000); - const ready = await until(() => document.querySelector("#main [data-file] .fw-group"), 10000); + const ready = await until(() => all("#main table.fw-table tbody[data-source]").length > 1, 10000); if (!ready) { return {error: text(document.getElementById("main"))}; } const owned = (svg) => svg.closest("button, a, [title]"); + const rows = () => all("#main table.fw-table tr[data-filter]"); + const chip = (id) => document.querySelector(`#main [aria-label="Show rules"] button[data-filter="${id}"]`); + const sources = all("#main tbody[data-source]").map((node) => node.dataset.source); + const folded = {expanded: all("#main .fw-fold").map((node) => node.getAttribute("aria-expanded")), rows: rows().length}; + for (const source of sources) { + all("#main tbody[data-source]").find((node) => node.dataset.source === source).querySelector(".fw-fold").click(); + } + const opened = all("#main .fw-fold").map((node) => node.getAttribute("aria-expanded")); + const shown = rows().map((node) => [node.dataset.filter, node.dataset.list || null, node.dataset.entry || null, text(node.querySelector(".fw-hits")), node.hasAttribute("data-unused")]); + chip("deny").click(); + const withoutDeny = {pressed: chip("deny").getAttribute("aria-pressed"), deny: rows().filter((node) => node.dataset.filter === "deny").length, rows: rows().length}; + chip("deny").click(); + const wrap = document.querySelector("#main .fw-table-wrap"); + wrap.scrollIntoView({block: "start"}); + await sleep(100); return { title: text(document.querySelector("#main h2.title")), subs: all("#tabs .subtabs button").map(name), - files: all("#main [data-file]").map((node) => node.dataset.file), - groups: all("#main [data-file] details.fw-group[data-tool]").map((node) => [node.dataset.tool, node.dataset.prefix, all("li", node).length]), - lists: all("#main [data-file] li[data-list]").map((node) => [node.dataset.list, node.dataset.entry]), - sandbox: all("#main [data-file] [data-key]").map((node) => node.dataset.key), + sources, + folded, + opened, + rows: shown, + withoutDeny, + restored: rows().length, + heads: all("#main table.fw-table thead th").map(text), + chips: all('#main [aria-label="Show rules"] button').map((node) => [node.dataset.filter, node.getAttribute("aria-pressed")]), patterns: all('#main [aria-label="Never granted"] li').length, targets: all('#main .rule-form select[aria-label="Applies to"] option').map(text), unlabelled: all("#main svg.bi").map(owned).filter((owner) => !owner || (!text(owner) && !(owner.getAttribute("title") && owner.getAttribute("aria-label")))).length, secret: document.documentElement.innerHTML.includes("@SECRET@"), wide: document.documentElement.scrollWidth > innerWidth + 1, + scrolls: wrap.scrollWidth > wrap.clientWidth, }; })()""" SIGN = """(async () => {@HELPERS@ - const entryNode = (entry) => all("#main li[data-entry]").find((node) => node.dataset.entry === entry); + const entryNode = (entry) => all("#main tr[data-entry]").find((node) => node.dataset.entry === entry); if (@REMOVE@) { entryNode("Bash(git status:*)").querySelector('button[aria-label="Remove"]').click(); await sleep(100); @@ -93,6 +114,7 @@ GROUP_RULE = """(async () => {@HELPERS@ }; await screenShown(8000); await until(() => document.querySelector('#main form[aria-label="New agent group"]'), 10000); + (await until(() => document.querySelector('#main tbody[data-source="hub"] .fw-fold'), 10000)).click(); const group = document.querySelector('#main form[aria-label="New agent group"]'); group.querySelector('input[aria-label="Group name"]').value = "builders"; for (const option of group.querySelector('select[aria-label="Members"]').options) { @@ -109,8 +131,10 @@ GROUP_RULE = """(async () => {@HELPERS@ target.value = choice.value; rules.querySelector('input[placeholder="make test*"]').value = "make e2e"; rules.requestSubmit(); - const pill = await until(() => all("#main .rule-row [data-agents]").find((node) => text(node) === "group builders"), 15000); - const found = {group: text(row.querySelector("strong")), members: text(row.querySelector(".muted")), pills: all("#main .rule-row [data-agents]").map(text), found: Boolean(pill), toasts: all("#toasts .toast").map(text)}; + const ruleRow = () => all("#main tr[data-rule]").find((node) => all("code", node).some((code) => text(code) === "make e2e")); + const pill = await until(() => all("#main tr[data-rule] [data-agents]").find((node) => text(node) === "group builders"), 15000); + const hits = await until(() => ruleRow() && text(ruleRow().querySelector(".fw-hits")) !== "…" && text(ruleRow().querySelector(".fw-hits")), 8000); + const found = {group: text(row.querySelector("strong")), members: text(row.querySelector(".muted")), pills: all("#main tr[data-rule] [data-agents]").map(text), found: Boolean(pill), hits, toasts: all("#toasts .toast").map(text)}; const sets = document.querySelector('#main form[aria-label="New rule set"]'); sets.querySelector('input[aria-label="Set name"]').value = "ci"; for (const option of sets.querySelector('select[aria-label="Rules"]').options) { @@ -148,7 +172,6 @@ GROUP_RULE = """(async () => {@HELPERS@ found.sent = sent.filter((url) => url.startsWith("/api/rules")); choose(picker, "all"); found.again = {value: picker.value, valid: picker.checkValidity(), blocked: sign.disabled, options: [...picker.options].map((option) => option.value).includes("gone")}; - const ruleRow = () => all("#main .rule-row").find((node) => node.querySelector("[data-agents]") && all("code", node).some((code) => text(code) === "make e2e")); const toggle = (label) => ruleRow() && [...ruleRow().querySelectorAll("button")].find((button) => text(button) === label); toggle("Turn off").click(); found.off = Boolean(await until(() => toggle("Turn on"), 8000)); @@ -189,7 +212,7 @@ class TestFirewallE2E(HubCase): home = self.outside / "home" self.settings = home / ".claude" / "settings.json" self.settings.parent.mkdir(parents=True) - self.settings.write_text(json.dumps(ORIGINAL, indent=2) + "\n", encoding="utf-8") + self.settings.write_text(json.dumps({**ORIGINAL, "permissions": {**ORIGINAL["permissions"], "ask": ASKED}}, indent=2) + "\n", encoding="utf-8") self.settings.chmod(0o640) tree = self.outside / "writer-tree.json" tree.write_text(json.dumps({"repos": [{"path": str(self.outside / "demo"), "known": True, "default": "main"}]}), encoding="utf-8") @@ -206,6 +229,8 @@ class TestFirewallE2E(HubCase): def test_chromium_shows_the_settings_and_signs_a_settings_diff_at_three_widths(self): authenticator = self.authenticator() + for command in ("make deploy", "make deploy --dry", "git push origin main"): + self.create_ask({**ASK, "command": command}) self.beside(self.host_unit) steps = [] for index, (width, height) in enumerate(base.WIDTHS): @@ -220,14 +245,30 @@ class TestFirewallE2E(HubCase): with self.subTest(width=width): self.assertNotIn("error", shown, shown) self.assertEqual((shown["title"], shown["subs"]), ("Firewall", base.SUBS)) - self.assertEqual(shown["files"], [str(self.settings), str(self.outside / "demo" / ".claude" / "settings.json")]) - self.assertIn(["Bash", "make", 1 + index], shown["groups"]) - self.assertIn(["Read", "~/.ssh/**", 1], shown["groups"]) - self.assertIn(["sandbox.filesystem.allowWrite", "/tmp"], shown["lists"]) - self.assertIn("sandbox.enabled", shown["sandbox"]) + self.assertEqual(shown["sources"], ["hub", str(self.settings), str(self.outside / "demo" / ".claude" / "settings.json")]) + self.assertEqual(shown["folded"], {"expanded": ["false"] * 3, "rows": 0}) + self.assertEqual(shown["opened"], ["true"] * 3) + self.assertEqual(shown["heads"], ["Action", "Tool", "Pattern", "Scope", "Source", "Hits (7 days)", ""]) + self.assertEqual(shown["chips"], [[chip, "true"] for chip in ("allow", "ask", "deny", "sandbox", "signed")]) + rows = shown["rows"] + self.assertEqual([row[0] for row in rows], sorted((row[0] for row in rows), key=["deny", "ask", "allow", "sandbox"].index)) + for row in ( + ["deny", "permissions.deny", "Read(~/.ssh/**)", "not seen", False], + ["ask", "permissions.ask", "Bash(make deploy:*)", "2", False], + ["ask", "permissions.ask", "Bash(make clean:*)", "0 unused", True], + ["ask", "permissions.ask", "Bash(git push:*)", "never granted: pushes", False], + ["allow", "permissions.allow", "Bash(make test:*)", "not seen", False], + ["sandbox", "sandbox.filesystem.allowWrite", "/tmp", "not seen", False], + ["sandbox", "sandbox.enabled", None, "not seen", False], + ): + self.assertIn(row, rows) + self.assertEqual(sum(1 for row in rows if row[0] == "allow" and row[2].startswith("Bash(make")), 1 + index) + self.assertEqual(shown["withoutDeny"], {"pressed": "false", "deny": 0, "rows": len(rows) - 1}) + self.assertEqual(shown["restored"], len(rows)) self.assertGreater(shown["patterns"], 5) self.assertEqual(shown["targets"], ["All agents", "Agent Skills", "Agent Stage"]) self.assertEqual((shown["unlabelled"], shown["secret"], shown["wide"]), (0, False, False)) + self.assertEqual(shown["scrolls"], width < 600) self.assertNotIn("error", signed, signed) self.assertTrue(any(line.startswith("+") and f'"Bash(make lint-{width}:*)"' in line for line in signed["added"]), signed["added"]) self.assertEqual(any("git status" in line for line in signed["removed"]), index == 0) @@ -247,7 +288,7 @@ class TestFirewallE2E(HubCase): self.assertEqual(data["errors"], []) found = data["results"][0] self.assertNotIn("error", found, found) - self.assertEqual((found["group"], found["members"], found["found"]), ("builders", "Skills, Stage", True), found) + self.assertEqual((found["group"], found["members"], found["found"], found["hits"]), ("builders", "Skills, Stage", True, "0 unused"), found) stored = [rule for rule in self.httpd.hub.rules.listed() if rule["pattern"] == "make e2e"] self.assertEqual([(rule["agents"], rule["proof"]["payload"]["agents"]) for rule in stored], [(["s1", "s2"], ["s1", "s2"])] * 2) for rule in stored: diff --git a/tests/test_hub_firewall.py b/tests/test_hub_firewall.py index 8c4fcb9..e013467 100644 --- a/tests/test_hub_firewall.py +++ b/tests/test_hub_firewall.py @@ -7,7 +7,7 @@ from unittest import mock from hub_case import HubCase from hub_support import ASK, PRESENT, key, signing, verifier -from kvb_orchestrator import batches, rules +from kvb_orchestrator import batches, firewall, requesting, rules HOME_FILE = "/home/op/.claude/settings.json" PROJECT_FILE = "/home/op/Repositories/demo/.claude/settings.json" @@ -66,6 +66,51 @@ class TestHubFirewall(HubCase): self.assertEqual(self.get(path)[0], 405) self.assertEqual(self.get("/api/firewall/settings/0123456789abcdef")[0], 404) + def test_the_rule_table_counts_hits_of_the_last_seven_days_from_the_ask_ledger_with_the_dry_run_matcher(self): + permissions = {"allow": ["Bash(make deploy:*)"], "ask": ["Bash(make deploy:*)", "Bash(git push:*)", "Bash(make clean:*)", "Bash"], "deny": ["Bash(make deploy --prod:*)"]} + self.view([{**self.home_view(), "settings": {"permissions": permissions, "sandbox": SANDBOX}}]) + for command, session in (("make deploy", "s1"), ("make deploy --dry", "s1"), ("make deploy", "s2"), ("make deploy && rm -rf /", "s1"), ("git push origin main", "s1"), ("make test", "s1")): + self.create_ask({**ASK, "command": command, "session": session}) + old = self.create_ask({**ASK, "command": "make deploy"}) + self.httpd.hub.asks.change(old, {"created_ts": time.time() - 8 * 86400}) + signer = self.pin() + later = time.time() + 3600 + mine = self.signed_rule(signer, {"kind": "ask", "tool": "Bash", "pattern": "make deploy*", "scope": "/repo*", "expires": later, "agents": ["s1"]}) + elsewhere = self.signed_rule(signer, {"kind": "ask", "tool": "Bash", "pattern": "make deploy*", "scope": "/elsewhere*", "expires": later}) + pushing = self.signed_rule(signer, {"kind": "ask", "tool": "Bash", "pattern": "git push *", "scope": "*", "expires": later}) + review = self.signed_rule(signer, {"kind": "review", "tool": "", "pattern": "feature/*", "scope": "/repo/*", "expires": later}) + self.assertEqual(self.call("POST", f"/api/rules/{mine['id']}/disable", b"", self.guard(self.host))[0], 200) + found = self.firewall() + entries = {(entry["list"], entry["entry"]): (entry["spec"], entry["hits"], entry["never"]) for group in found["files"][0]["groups"] for entry in group["entries"]} + self.assertEqual(entries, { + ("allow", "Bash(make deploy:*)"): ("make deploy:*", None, ""), + ("ask", "Bash(make deploy:*)"): ("make deploy:*", 3, ""), + ("ask", "Bash(git push:*)"): ("git push:*", None, "pushes"), + ("ask", "Bash(make clean:*)"): ("make clean:*", 0, ""), + ("ask", "Bash"): ("", 4, ""), + ("deny", "Bash(make deploy --prod:*)"): ("make deploy --prod:*", None, ""), + }) + self.assertEqual(found["days"], requesting.RECENT_DAYS) + self.assertEqual(found["rules"], {mine["id"]: {"hits": 2, "never": ""}, elsewhere["id"]: {"hits": 0, "never": ""}, pushing["id"]: {"hits": None, "never": "pushes"}, review["id"]: {"hits": None, "never": ""}}) + + def test_an_entry_becomes_the_rule_the_dry_run_matcher_counts_and_a_hit_needs_the_rule_to_match(self): + now = time.time() + for entry, rule in ( + ("Bash(make test:*)", {"kind": "ask", "tool": "Bash", "pattern": "make test*", "scope": "*"}), + ("Bash(npm run *)", {"kind": "ask", "tool": "Bash", "pattern": "npm run *", "scope": "*"}), + ("Bash", {"kind": "ask", "tool": "Bash", "pattern": "*", "scope": "*"}), + ("mcp__memory__search", {"kind": "ask", "tool": "mcp__memory__search", "pattern": "*", "scope": "*"}), + ): + with self.subTest(entry=entry): + self.assertEqual(firewall.entry_rule(entry), rule) + asks = [rules.subject_of("ask", {"id": "0123456789abcdef", "tool": "Bash", "command": command, "cwd": "/repo", "session": "s1"}, set()) for command in ("make test", "make test-unit", "make lint")] + self.assertEqual(firewall.hits(firewall.entry_rule("Bash(make test:*)"), asks, now), {"hits": 2, "never": ""}) + self.assertEqual(firewall.hits(firewall.entry_rule("Bash(make build:*)"), asks, now), {"hits": 0, "never": ""}) + self.assertEqual(firewall.hits(firewall.entry_rule("Bash(make test | sh:*)"), asks, now), {"hits": None, "never": "pipes into a shell"}) + self.assertEqual(firewall.hits(firewall.entry_rule("Bash(make test; make lint)"), asks, now), {"hits": None, "never": firewall.CHAINED}) + with mock.patch.object(requesting, "rule_matches", return_value=False): + self.assertEqual(firewall.hits(firewall.entry_rule("Bash(make test:*)"), asks, now), {"hits": 0, "never": ""}) + def test_a_settings_change_is_previewed_signed_and_queued_for_the_host(self): self.view([self.home_view()]) item = {"kind": "settings", "file": HOME_FILE, "base": SHA, "changes": [ADD, {"list": "permissions.ask", "op": "remove", "entry": "Bash(git push:*)"}]} diff --git a/tests/test_page_reveal.py b/tests/test_page_reveal.py index ca0c865..849bf8e 100644 --- a/tests/test_page_reveal.py +++ b/tests/test_page_reveal.py @@ -173,9 +173,16 @@ class TestPageReveal(unittest.TestCase): def test_a_settings_entry_staged_for_removal_and_its_diff_show_every_hidden_character_as_a_marker(self): entry = f"Read(~/.ssh/**{RLO})" - found = shown(TEMPLATES, [["entryRow", ["/home/op/.claude/settings.json", "permissions.deny", entry]], ["diffLines", [f'- "{entry}",\n+ "Bash(make{ZWSP} build:*)"']]]) - self.assertRevealed(found[0], ["202E"]) - self.assertRevealed(found[1], ["202E", "200B"]) + path = f"/home/op{FILLER}/.claude/settings.json" + group = {"id": path, "label": path, "file": True} + row = {"filter": "deny", "action": "Deny", "tone": "bad", "tool": f"Read{CGJ}", "pattern": f"~/.ssh/**{RLO}", "list": "permissions.deny", "entry": entry, "hits": None, "never": ""} + rule = {"id": "0123456789abcdef", "kind": "ask", "tool": "Bash", "pattern": f"make test{RLO}*", "scope": f"/repo{ZWSP}*", "agents": ["*"], "expires": 1_800_000_000, "enabled": True, "signed": True} + signed = {"filter": "signed", "action": "Grant ask", "tone": "ok", "tool": f"Bash{NBSP}", "pattern": rule["pattern"], "rule": rule, "hits": 0, "never": ""} + found = shown(TEMPLATES, [["ruleRow", [group, row]], ["ruleRow", [{"id": "hub", "label": "Hub rules", "file": False}, signed]], ["diffLines", [f'- "{entry}",\n+ "Bash(make{ZWSP} build:*)"']]]) + self.assertRevealed(found[0], ["202E", "3164", "034F"]) + self.assertRevealed(found[1], ["202E", "200B", "00A0"]) + self.assertIn("unused", found[1]) + self.assertRevealed(found[2], ["202E", "200B"]) if __name__ == "__main__":