Files
skills/tests/e2e/test_firewall_e2e.py
T
Kevin Veen-BirkenbachandClaude Opus 5.5 9742884308 feat(orchestrator): firewall view with signed settings changes (001 S6)
Requirement 001 S6: the hub gets a Firewall view of the Claude Code permission and sandbox settings. The operator signs each change with the passkey; a host settings writer (systemd path unit) applies only exact, signed changes to allowlisted settings files through O_NOFOLLOW directory and file descriptors, and a timer refreshes the view. Every signed text and every page sink refuses or marks hidden characters: Cc, Cf, Zl, Zp, every space but U+0020 and the Default_Ignorable set, with U+FE0E/U+FE0F allowed only after one of the 359 non-ASCII emoji bases of Unicode 16 (shared table, validated at import, the same pair rule on the page). Removal of a deny, ask or denyRead entry is exempt only as an exact single remove. The permission hook prints the hub's refusal reason, cleaned of hidden characters. Accepted MEDUSA false positives (independent review, operator pick): 38 this round, mainly the Unicode defence tables read as attack patterns, generic MM-SCAN-058 heuristics, WEB-FLASK-004 at view.py:213 (ints from the validated table via tojson), EXF26-TOOL-004 on the non-secret WebAuthn challenge, PRIV-SCAN-060 on the office-action shape check and PQC007 on a picker key.

🎯 Fix: 95 %
💥 Break: 3 %
💸 Cost: –
⏱️ Time: –
🛡️ Security: 100 %
🏆 Score: 92
🚦 Gate: ✅ 90/5, cost and time <= 100, security 100 %

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-13 18:14:17 +00:00

276 lines
17 KiB
Python

from __future__ import annotations
import json
import os
import re
import shutil
import sys
import time
import unittest
from pathlib import Path
HERE = Path(__file__).resolve().parent
sys.path[:0] = [str(HERE.parent), str(HERE)]
import test_orchestrator_e2e as base
from hub_case import HubCase
from hub_support import verifier
from test_settings_writer import ORIGINAL
from test_settings_writer import host as writer_host
OUT = Path(os.environ.get("E2E_OUT", "/tmp/firewall-shots"))
SECRET = ORIGINAL["env"]["EXAMPLE_FLAG"]
SHOW = """(async () => {@HELPERS@
await screenShown(8000);
const ready = await until(() => document.querySelector("#main [data-file] .fw-group"), 10000);
if (!ready) {
return {error: text(document.getElementById("main"))};
}
const owned = (svg) => svg.closest("button, a, [title]");
return {
title: text(document.querySelector("#main h2.title")),
subs: all("#tabs .subtabs button").map(name),
files: all("#main [data-file]").map((node) => node.dataset.file),
groups: all("#main [data-file] details.fw-group[data-tool]").map((node) => [node.dataset.tool, node.dataset.prefix, all("li", node).length]),
lists: all("#main [data-file] li[data-list]").map((node) => [node.dataset.list, node.dataset.entry]),
sandbox: all("#main [data-file] [data-key]").map((node) => node.dataset.key),
patterns: all('#main [aria-label="Never granted"] li').length,
targets: all('#main .rule-form select[aria-label="Applies to"] option').map(text),
unlabelled: all("#main svg.bi").map(owned).filter((owner) => !owner || (!text(owner) && !(owner.getAttribute("title") && owner.getAttribute("aria-label")))).length,
secret: document.documentElement.innerHTML.includes("@SECRET@"),
wide: document.documentElement.scrollWidth > innerWidth + 1,
};
})()"""
SIGN = """(async () => {@HELPERS@
const entryNode = (entry) => all("#main li[data-entry]").find((node) => node.dataset.entry === entry);
if (@REMOVE@) {
entryNode("Bash(git status:*)").querySelector('button[aria-label="Remove"]').click();
await sleep(100);
}
const form = document.querySelector('#main form[aria-label="Stage a settings change"]');
form.querySelector('select[aria-label="Settings file"]').value = "@FILE@";
form.querySelector('select[aria-label="List"]').value = "permissions.allow";
form.querySelector('input[aria-label="Entry"]').value = "@ENTRY@";
form.requestSubmit();
await sleep(100);
const staged = all('#main [aria-label="Staged changes"] li').map((node) => node.dataset.staged);
document.querySelector('#main button[aria-label="Review and sign"]').click();
const diff = await until(() => document.querySelector('.sheet-box [aria-label="Diff"] pre.fw-diff'), 8000);
if (!diff) {
return {error: "no diff", toasts: all("#toasts .toast").map(text)};
}
const lines = (cls) => all(`span.${cls}`, diff).map((node) => node.textContent.trim());
const shown = {added: lines("add"), removed: lines("del"), staged, labelled: all(".sheet-box button").every((button) => text(button) || (button.title && button.getAttribute("aria-label")))};
document.querySelector('.sheet-box button[aria-label="Sign and apply"]').click();
const done = await until(() => document.querySelector(".sheet-box [data-result]"), 30000);
shown.result = done ? done.dataset.result : null;
shown.text = text(done);
shown.toasts = all("#toasts .toast").map(text);
shown.wide = document.documentElement.scrollWidth > innerWidth + 1;
return shown;
})()"""
GROUP_RULE = """(async () => {@HELPERS@
const prompts = [];
new MutationObserver((records) => records.forEach((record) => record.addedNodes.forEach((node) => {
if (node.id === "passkey-prompt") {
prompts.push(text(node));
}
}))).observe(document.body, {childList: true});
const choose = (select, value) => {
select.value = value;
select.dispatchEvent(new Event("change"));
};
const groupValue = (select) => [...select.options].map((option) => option.value).find((value) => value.startsWith("group:"));
const setRow = () => document.querySelector('#main [data-set="ci"]');
const setTarget = () => setRow().querySelector("select");
const setTake = () => setRow().querySelector('button[aria-label="Take over"]');
const setState = () => ({label: text(setTarget().selectedOptions[0]), value: setTarget().value.split(":")[0], blocked: setTake().disabled, valid: setTarget().checkValidity()});
const stage = async (entry) => {
const form = document.querySelector('#main form[aria-label="Stage a settings change"]');
form.querySelector('input[aria-label="Entry"]').value = entry;
form.requestSubmit();
return until(() => all('#main [aria-label="Staged changes"] li').find((node) => node.dataset.staged === `add ${entry}`), 8000);
};
await screenShown(8000);
await until(() => document.querySelector('#main form[aria-label="New agent group"]'), 10000);
const group = document.querySelector('#main form[aria-label="New agent group"]');
group.querySelector('input[aria-label="Group name"]').value = "builders";
for (const option of group.querySelector('select[aria-label="Members"]').options) {
option.selected = true;
}
group.requestSubmit();
const row = await until(() => document.querySelector('#main [data-group="builders"]'), 8000);
const rules = document.querySelector("#main .rule-form");
const target = rules.querySelector('select[aria-label="Applies to"]');
const choice = await until(() => [...target.options].find((option) => option.value.startsWith("group:")), 8000);
if (!row || !choice) {
return {error: "no group", toasts: all("#toasts .toast").map(text)};
}
target.value = choice.value;
rules.querySelector('input[placeholder="make test*"]').value = "make e2e";
rules.requestSubmit();
const pill = await until(() => all("#main .rule-row [data-agents]").find((node) => text(node) === "group builders"), 15000);
const found = {group: text(row.querySelector("strong")), members: text(row.querySelector(".muted")), pills: all("#main .rule-row [data-agents]").map(text), found: Boolean(pill), toasts: all("#toasts .toast").map(text)};
const sets = document.querySelector('#main form[aria-label="New rule set"]');
sets.querySelector('input[aria-label="Set name"]').value = "ci";
for (const option of sets.querySelector('select[aria-label="Rules"]').options) {
option.selected = option.textContent.includes("make e2e");
}
sets.requestSubmit();
if (!await until(setRow, 8000)) {
return {...found, error: "no rule set", toasts: all("#toasts .toast").map(text)};
}
found.fresh = setState();
choose(setTarget(), groupValue(setTarget()));
await stage("Bash(make e2e:*)");
found.kept = setState();
setTake().click();
found.taken = await until(() => all("#toasts .toast").map(text).find((line) => line.startsWith("ci:")), 15000);
found.reloaded = await until(() => setRow() && setTarget().value.startsWith("group:") && document.querySelector("#main .rule-form") && setState(), 8000);
const form = document.querySelector("#main .rule-form");
const picker = form.querySelector('select[aria-label="Applies to"]');
const sign = form.querySelector("[data-sign]");
choose(picker, groupValue(picker));
found.picked = {value: picker.value.split(":")[0], blocked: sign.disabled};
const sent = [];
const fetched = window.fetch;
window.fetch = (url, options) => {
sent.push(String(url));
return fetched(url, options);
};
document.querySelector('#main [data-group="builders"] button[aria-label="Delete group"]').click();
await until(() => picker.value === "gone" || !picker.isConnected, 8000);
found.gone = {connected: picker.isConnected, value: picker.value, label: text(picker.selectedOptions[0]), valid: picker.checkValidity(), blocked: sign.disabled, groups: [...picker.options].filter((option) => option.value.startsWith("group:")).length};
await stage("Bash(make e2e-again:*)");
found.setGone = setState();
setTake().click();
await sleep(500);
found.sent = sent.filter((url) => url.startsWith("/api/rules"));
choose(picker, "all");
found.again = {value: picker.value, valid: picker.checkValidity(), blocked: sign.disabled, options: [...picker.options].map((option) => option.value).includes("gone")};
const ruleRow = () => all("#main .rule-row").find((node) => node.querySelector("[data-agents]") && all("code", node).some((code) => text(code) === "make e2e"));
const toggle = (label) => ruleRow() && [...ruleRow().querySelectorAll("button")].find((button) => text(button) === label);
toggle("Turn off").click();
found.off = Boolean(await until(() => toggle("Turn on"), 8000));
toggle("Turn on").click();
found.on = Boolean(await until(() => toggle("Turn off"), 15000));
found.prompts = prompts;
return found;
})()"""
def signed_lines(rule: dict) -> list[str]:
"""Returns:
The lines the passkey prompt shows for a stored make e2e rule before
its target, its lifetime in days as N.
"""
return ["ask Bash: make e2e", "in *", f"until {time.strftime('%Y-%m-%d %H:%M', time.gmtime(rule['expires']))} UTC, in N days"]
def probe(template: str, values: dict[str, str]) -> str:
text = template.replace("@HELPERS@", base.HELPERS)
for name, value in values.items():
text = text.replace(f"@{name}@", value)
return text
@unittest.skipUnless(shutil.which(os.environ.get("CHROMIUM", "chromium")), "no Chromium")
class TestFirewallE2E(HubCase):
drive = base.TestOrchestratorE2E.drive
beside = base.TestOrchestratorE2E.beside
authenticator = base.TestOrchestratorE2E.authenticator
def setUp(self):
super().setUp()
OUT.mkdir(parents=True, exist_ok=True)
self.claim("s1", "Skills")
self.claim("s2", "Stage")
self.snapshot()
home = self.outside / "home"
self.settings = home / ".claude" / "settings.json"
self.settings.parent.mkdir(parents=True)
self.settings.write_text(json.dumps(ORIGINAL, indent=2) + "\n", encoding="utf-8")
self.settings.chmod(0o640)
tree = self.outside / "writer-tree.json"
tree.write_text(json.dumps({"repos": [{"path": str(self.outside / "demo"), "known": True, "default": "main"}]}), encoding="utf-8")
trust = verifier.Trust(approvers=self.approvers, replay=self.replay / "firewall.json", origin=self.origin)
self.writer = writer_host.Writer(trust, home, tree)
self.spool = self.state / "firewall"
verifier.write_private(self.spool / "view.json", self.writer.view())
self.local = f"http://localhost:{self.port}/"
def host_unit(self) -> None:
if any((self.spool / "jobs").iterdir()):
writer_host.cycle(self.spool / "jobs", self.spool / "results", self.writer)
verifier.write_private(self.spool / "view.json", self.writer.view())
def test_chromium_shows_the_settings_and_signs_a_settings_diff_at_three_widths(self):
authenticator = self.authenticator()
self.beside(self.host_unit)
steps = []
for index, (width, height) in enumerate(base.WIDTHS):
values = {"SECRET": SECRET, "FILE": str(self.settings), "ENTRY": f"Bash(make lint-{width}:*)", "REMOVE": "true" if index == 0 else "false"}
steps.append({"width": width, "height": height, "url": f"{self.local}?{width}#stream/rules", "wait": 1500, "probe": probe(SHOW, values), "shot": str(OUT / f"chromium-{width}-firewall.png")})
steps.append({"width": width, "height": height, "url": None, "wait": 0, "probe": probe(SIGN, values), "shot": str(OUT / f"chromium-{width}-firewall-signed.png")})
data = self.drive(steps, authenticator)
(OUT / "chromium-firewall.json").write_text(json.dumps(data, indent=1), encoding="utf-8")
self.assertEqual(data["errors"], [])
for index, (width, _) in enumerate(base.WIDTHS):
shown, signed = data["results"][2 * index : 2 * index + 2]
with self.subTest(width=width):
self.assertNotIn("error", shown, shown)
self.assertEqual((shown["title"], shown["subs"]), ("Firewall", base.SUBS))
self.assertEqual(shown["files"], [str(self.settings), str(self.outside / "demo" / ".claude" / "settings.json")])
self.assertIn(["Bash", "make", 1 + index], shown["groups"])
self.assertIn(["Read", "~/.ssh/**", 1], shown["groups"])
self.assertIn(["sandbox.filesystem.allowWrite", "/tmp"], shown["lists"])
self.assertIn("sandbox.enabled", shown["sandbox"])
self.assertGreater(shown["patterns"], 5)
self.assertEqual(shown["targets"], ["All agents", "Agent Skills", "Agent Stage"])
self.assertEqual((shown["unlabelled"], shown["secret"], shown["wide"]), (0, False, False))
self.assertNotIn("error", signed, signed)
self.assertTrue(any(line.startswith("+") and f'"Bash(make lint-{width}:*)"' in line for line in signed["added"]), signed["added"])
self.assertEqual(any("git status" in line for line in signed["removed"]), index == 0)
self.assertEqual((signed["result"], signed["labelled"], signed["wide"]), ("done", True, False), signed)
self.assertIn("Settings: written", " ".join(signed["toasts"]))
written = json.loads(self.settings.read_text(encoding="utf-8"))
self.assertEqual(written["permissions"]["allow"], ["Bash(make test:*)", *(f"Bash(make lint-{width}:*)" for width, _ in base.WIDTHS)])
self.assertEqual({key: written[key] for key in ("model", "env", "hooks", "sandbox")}, {key: ORIGINAL[key] for key in ("model", "env", "hooks", "sandbox")})
backups = sorted(self.settings.parent.glob("settings.json.before-firewall-*"))
self.assertEqual((len(backups), self.settings.stat().st_mode & 0o777), (3, 0o640))
def test_chromium_names_a_group_and_signs_a_rule_for_it(self):
authenticator = self.authenticator()
steps = [{"width": 1366, "height": 768, "url": f"{self.local}#stream/rules", "wait": 1500, "probe": probe(GROUP_RULE, {}), "shot": str(OUT / "chromium-1366-firewall-group-rule.png")}]
data = self.drive(steps, authenticator)
(OUT / "chromium-firewall-group.json").write_text(json.dumps(data, indent=1), encoding="utf-8")
self.assertEqual(data["errors"], [])
found = data["results"][0]
self.assertNotIn("error", found, found)
self.assertEqual((found["group"], found["members"], found["found"]), ("builders", "Skills, Stage", True), found)
stored = [rule for rule in self.httpd.hub.rules.listed() if rule["pattern"] == "make e2e"]
self.assertEqual([(rule["agents"], rule["proof"]["payload"]["agents"]) for rule in stored], [(["s1", "s2"], ["s1", "s2"])] * 2)
for rule in stored:
self.assertLess(abs(rule["expires"] - (time.time() + 7 * 86400)), 2 * 86400)
self.assertEqual(found["fresh"], {"label": "Pick a target", "value": "", "blocked": True, "valid": False}, found)
self.assertEqual(found["kept"], {"label": "Group builders", "value": "group", "blocked": False, "valid": True}, found)
self.assertEqual(found["taken"], "ci: 1 of 1 rules signed for group builders")
self.assertEqual(found["reloaded"], found["kept"])
self.assertEqual(found["setGone"], {"label": "Target gone: pick again", "value": "gone", "blocked": True, "valid": False}, found)
self.assertEqual(found["sent"], [])
members = "Skills (s1), Stage (s2)"
shown = [re.sub(r"in \d+\.\d days$", "in N days", prompt.removeprefix("Passkey signs"), flags=re.MULTILINE).split("\n") for prompt in found["prompts"]]
made = sorted(stored, key=lambda rule: rule["created_ts"])
self.assertEqual(shown[:2], [["Rule", *signed_lines(made[0]), f"for group builders: {members}"], ["Rule set ci", *signed_lines(made[1]), f"for group builders: {members}"]], found)
self.assertIn(shown[2:], [[["Turn on rule", *signed_lines(rule), f"for agents {members}"]] for rule in made], found)
self.assertTrue(found["prompts"][1].split("\n")[3].endswith("in 7.0 days"), found["prompts"])
self.assertEqual((found["off"], found["on"], [rule["enabled"] for rule in self.httpd.hub.rules.listed() if rule["pattern"] == "make e2e"]), (True, True, [True, True]))
self.assertEqual(found["picked"], {"value": "group", "blocked": False})
self.assertEqual(found["gone"], {"connected": True, "value": "gone", "label": "Target gone: pick again", "valid": False, "blocked": True, "groups": 0}, found)
self.assertEqual(found["again"], {"value": "all", "valid": True, "blocked": False, "options": False}, found)
self.assertEqual(self.httpd.hub.groups.listed(), [])
if __name__ == "__main__":
unittest.main()