Files
skills/tests/e2e/test_firewall_e2e.py
T
Kevin Veen-BirkenbachandClaude Opus 5.5 550f6962db fix(e2e): repair the e2e tests the skills unify merges made stale
After about 27 merged branches 28 e2e items failed on 9 causes: tests clicked the new voice-input mic instead of Record, the #tasks/match screen was missing from SCREENS, a test patched render on a frozen dataclass, a cleanup row had no local plan, live rows were stamped in the future, and a renamed helper shadowed self.local. The composer now keeps its mic in the action row like the agent form, so the Home radio tile shows two history rows at 960 px.

🎯 Fix: 85 %
💥 Break: 3 %
💸 Tokens: 100 %
⏱️ Velocity: 100 %
🛡️ Security: 95 %
🏆 Score: 78
🚦 Gate: ⛔ fix 85 % < 90; security 95 % < 100: ask the operator before the commit

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-15 09:46:37 +00:00

317 lines
19 KiB
Python

from __future__ import annotations
import json
import os
import re
import shutil
import sys
import time
import unittest
from pathlib import Path
HERE = Path(__file__).resolve().parent
sys.path[:0] = [str(HERE.parent), str(HERE)]
import test_orchestrator_e2e as base
from hub_case import HubCase
from hub_support import ASK, verifier
from test_settings_writer import ORIGINAL
from test_settings_writer import host as writer_host
OUT = Path(os.environ.get("E2E_OUT", "/tmp/firewall-shots"))
SECRET = ORIGINAL["env"]["EXAMPLE_FLAG"]
ASKED = ["Bash(make deploy:*)", "Bash(make clean:*)", "Bash(git push:*)"]
SHOW = """(async () => {@HELPERS@
await screenShown(8000);
const ready = await until(() => all("#main table.fw-table tbody[data-source]").length > 1, 10000);
if (!ready) {
return {error: text(document.getElementById("main"))};
}
const owned = (svg) => svg.closest("button, a, [title]");
const rows = () => all("#main table.fw-table tr[data-filter]");
const chip = (id) => document.querySelector(`#main [aria-label="Show rules"] button[data-filter="${id}"]`);
const sources = all("#main tbody[data-source]").map((node) => node.dataset.source);
const folded = {expanded: all("#main .fw-fold").map((node) => node.getAttribute("aria-expanded")), rows: rows().length};
for (const source of sources) {
all("#main tbody[data-source]").find((node) => node.dataset.source === source).querySelector(".fw-fold").click();
}
const opened = all("#main .fw-fold").map((node) => node.getAttribute("aria-expanded"));
const shown = rows().map((node) => [node.dataset.filter, node.dataset.list || null, node.dataset.entry || null, text(node.querySelector(".fw-hits")), node.hasAttribute("data-unused")]);
chip("deny").click();
const withoutDeny = {pressed: chip("deny").getAttribute("aria-pressed"), deny: rows().filter((node) => node.dataset.filter === "deny").length, rows: rows().length};
chip("deny").click();
const wrap = document.querySelector("#main .fw-table-wrap");
wrap.scrollIntoView({block: "start"});
await sleep(100);
return {
title: text(document.querySelector("#main h2.title")),
subs: all("#tabs .subtabs button").map(name),
sources,
folded,
opened,
rows: shown,
withoutDeny,
restored: rows().length,
heads: all("#main table.fw-table thead th").map(text),
chips: all('#main [aria-label="Show rules"] button').map((node) => [node.dataset.filter, node.getAttribute("aria-pressed")]),
patterns: all('#main [aria-label="Never granted"] li').length,
targets: all('#main .rule-form select[aria-label="Applies to"] option').map(text),
unlabelled: all("#main svg.bi").map(owned).filter((owner) => !owner || (!text(owner) && !(owner.getAttribute("title") && owner.getAttribute("aria-label")))).length,
secret: document.documentElement.innerHTML.includes("@SECRET@"),
wide: document.documentElement.scrollWidth > innerWidth + 1,
scrolls: wrap.scrollWidth > wrap.clientWidth,
};
})()"""
SIGN = """(async () => {@HELPERS@
const entryNode = (entry) => all("#main tr[data-entry]").find((node) => node.dataset.entry === entry);
if (@REMOVE@) {
entryNode("Bash(git status:*)").querySelector('button[aria-label="Remove"]').click();
await sleep(100);
}
const form = document.querySelector('#main form[aria-label="Stage a settings change"]');
form.querySelector('select[aria-label="Settings file"]').value = "@FILE@";
form.querySelector('select[aria-label="List"]').value = "permissions.allow";
form.querySelector('input[aria-label="Entry"]').value = "@ENTRY@";
form.requestSubmit();
await sleep(100);
const staged = all('#main [aria-label="Staged changes"] li').map((node) => node.dataset.staged);
document.querySelector('#main button[aria-label="Review and sign"]').click();
const diff = await until(() => document.querySelector('.sheet-box [aria-label="Diff"] pre.fw-diff'), 8000);
if (!diff) {
return {error: "no diff", toasts: all("#toasts .toast").map(text)};
}
const lines = (cls) => all(`span.${cls}`, diff).map((node) => node.textContent.trim());
const shown = {added: lines("add"), removed: lines("del"), staged, labelled: all(".sheet-box button").every((button) => text(button) || (button.title && button.getAttribute("aria-label")))};
document.querySelector('.sheet-box button[aria-label="Sign and apply"]').click();
const done = await until(() => document.querySelector(".sheet-box [data-result]"), 30000);
shown.result = done ? done.dataset.result : null;
shown.text = text(done);
shown.toasts = all("#toasts .toast").map(text);
shown.wide = document.documentElement.scrollWidth > innerWidth + 1;
return shown;
})()"""
GROUP_RULE = """(async () => {@HELPERS@
const prompts = [];
new MutationObserver((records) => records.forEach((record) => record.addedNodes.forEach((node) => {
if (node.id === "passkey-prompt") {
prompts.push(text(node));
}
}))).observe(document.body, {childList: true});
const choose = (select, value) => {
select.value = value;
select.dispatchEvent(new Event("change"));
};
const groupValue = (select) => [...select.options].map((option) => option.value).find((value) => value.startsWith("group:"));
const setRow = () => document.querySelector('#main [data-set="ci"]');
const setTarget = () => setRow().querySelector("select");
const setTake = () => setRow().querySelector('button[aria-label="Take over"]');
const setState = () => ({label: text(setTarget().selectedOptions[0]), value: setTarget().value.split(":")[0], blocked: setTake().disabled, valid: setTarget().checkValidity()});
const stage = async (entry) => {
const form = document.querySelector('#main form[aria-label="Stage a settings change"]');
form.querySelector('input[aria-label="Entry"]').value = entry;
form.requestSubmit();
return until(() => all('#main [aria-label="Staged changes"] li').find((node) => node.dataset.staged === `add ${entry}`), 8000);
};
await screenShown(8000);
await until(() => document.querySelector('#main form[aria-label="New agent group"]'), 10000);
(await until(() => document.querySelector('#main tbody[data-source="hub"] .fw-fold'), 10000)).click();
const group = document.querySelector('#main form[aria-label="New agent group"]');
group.querySelector('input[aria-label="Group name"]').value = "builders";
for (const option of group.querySelector('select[aria-label="Members"]').options) {
option.selected = true;
}
group.requestSubmit();
const row = await until(() => document.querySelector('#main [data-group="builders"]'), 8000);
const rules = document.querySelector("#main .rule-form");
const target = rules.querySelector('select[aria-label="Applies to"]');
const choice = await until(() => [...target.options].find((option) => option.value.startsWith("group:")), 8000);
if (!row || !choice) {
return {error: "no group", toasts: all("#toasts .toast").map(text)};
}
target.value = choice.value;
rules.querySelector('input[placeholder="make test*"]').value = "make e2e";
rules.requestSubmit();
const ruleRow = () => all("#main tr[data-rule]").find((node) => all("code", node).some((code) => text(code) === "make e2e"));
const pill = await until(() => all("#main tr[data-rule] [data-agents]").find((node) => text(node) === "group builders"), 15000);
const hits = await until(() => ruleRow() && text(ruleRow().querySelector(".fw-hits")) !== "…" && text(ruleRow().querySelector(".fw-hits")), 8000);
const found = {group: text(row.querySelector("strong")), members: text(row.querySelector(".muted")), pills: all("#main tr[data-rule] [data-agents]").map(text), found: Boolean(pill), hits, toasts: all("#toasts .toast").map(text)};
const sets = document.querySelector('#main form[aria-label="New rule set"]');
sets.querySelector('input[aria-label="Set name"]').value = "ci";
for (const option of sets.querySelector('select[aria-label="Rules"]').options) {
option.selected = option.textContent.includes("make e2e");
}
sets.requestSubmit();
if (!await until(setRow, 8000)) {
return {...found, error: "no rule set", toasts: all("#toasts .toast").map(text)};
}
found.fresh = setState();
choose(setTarget(), groupValue(setTarget()));
await stage("Bash(make e2e:*)");
found.kept = setState();
setTake().click();
found.taken = await until(() => all("#toasts .toast").map(text).find((line) => line.startsWith("ci:")), 15000);
found.reloaded = await until(() => setRow() && setTarget().value.startsWith("group:") && document.querySelector("#main .rule-form") && setState(), 8000);
const form = document.querySelector("#main .rule-form");
const picker = form.querySelector('select[aria-label="Applies to"]');
const sign = form.querySelector("[data-sign]");
choose(picker, groupValue(picker));
found.picked = {value: picker.value.split(":")[0], blocked: sign.disabled};
const sent = [];
const fetched = window.fetch;
window.fetch = (url, options) => {
sent.push(String(url));
return fetched(url, options);
};
document.querySelector('#main [data-group="builders"] button[aria-label="Delete group"]').click();
await until(() => picker.value === "gone" || !picker.isConnected, 8000);
found.gone = {connected: picker.isConnected, value: picker.value, label: text(picker.selectedOptions[0]), valid: picker.checkValidity(), blocked: sign.disabled, groups: [...picker.options].filter((option) => option.value.startsWith("group:")).length};
await stage("Bash(make e2e-again:*)");
found.setGone = setState();
setTake().click();
await sleep(500);
found.sent = sent.filter((url) => url.startsWith("/api/rules"));
choose(picker, "all");
found.again = {value: picker.value, valid: picker.checkValidity(), blocked: sign.disabled, options: [...picker.options].map((option) => option.value).includes("gone")};
const toggle = (label) => ruleRow() && [...ruleRow().querySelectorAll("button")].find((button) => text(button) === label);
toggle("Turn off").click();
found.off = Boolean(await until(() => toggle("Turn on"), 8000));
toggle("Turn on").click();
found.on = Boolean(await until(() => toggle("Turn off"), 15000));
found.prompts = prompts;
return found;
})()"""
def signed_lines(rule: dict) -> list[str]:
"""Returns:
The lines the passkey prompt shows for a stored make e2e rule before
its target, its lifetime in days as N.
"""
return ["ask Bash: make e2e", "in *", f"until {time.strftime('%Y-%m-%d %H:%M', time.gmtime(rule['expires']))} UTC, in N days"]
def probe(template: str, values: dict[str, str]) -> str:
text = template.replace("@HELPERS@", base.HELPERS)
for name, value in values.items():
text = text.replace(f"@{name}@", value)
return text
@unittest.skipUnless(shutil.which(os.environ.get("CHROMIUM", "chromium")), "no Chromium")
class TestFirewallE2E(HubCase):
drive = base.TestOrchestratorE2E.drive
beside = base.TestOrchestratorE2E.beside
authenticator = base.TestOrchestratorE2E.authenticator
def setUp(self):
super().setUp()
OUT.mkdir(parents=True, exist_ok=True)
self.claim("s1", "Skills")
self.claim("s2", "Stage")
self.snapshot()
home = self.outside / "home"
self.settings = home / ".claude" / "settings.json"
self.settings.parent.mkdir(parents=True)
self.settings.write_text(json.dumps({**ORIGINAL, "permissions": {**ORIGINAL["permissions"], "ask": ASKED}}, indent=2) + "\n", encoding="utf-8")
self.settings.chmod(0o640)
tree = self.outside / "writer-tree.json"
tree.write_text(json.dumps({"repos": [{"path": str(self.outside / "demo"), "known": True, "default": "main"}]}), encoding="utf-8")
trust = verifier.Trust(approvers=self.approvers, replay=self.replay / "firewall.json", origin=self.origin)
self.writer = writer_host.Writer(trust, home, tree)
self.spool = self.state / "firewall"
verifier.write_private(self.spool / "view.json", self.writer.view())
self.page_url = f"http://localhost:{self.port}/"
def host_unit(self) -> None:
if any((self.spool / "jobs").iterdir()):
writer_host.cycle(self.spool / "jobs", self.spool / "results", self.writer)
verifier.write_private(self.spool / "view.json", self.writer.view())
def test_chromium_shows_the_settings_and_signs_a_settings_diff_at_three_widths(self):
authenticator = self.authenticator()
for command in ("make deploy", "make deploy --dry", "git push origin main"):
self.create_ask({**ASK, "command": command})
self.beside(self.host_unit)
steps = []
for index, (width, height) in enumerate(base.WIDTHS):
values = {"SECRET": SECRET, "FILE": str(self.settings), "ENTRY": f"Bash(make lint-{width}:*)", "REMOVE": "true" if index == 0 else "false"}
steps.append({"width": width, "height": height, "url": f"{self.page_url}?{width}#stream/rules", "wait": 1500, "probe": probe(SHOW, values), "shot": str(OUT / f"chromium-{width}-firewall.png")})
steps.append({"width": width, "height": height, "url": None, "wait": 0, "probe": probe(SIGN, values), "shot": str(OUT / f"chromium-{width}-firewall-signed.png")})
data = self.drive(steps, authenticator)
(OUT / "chromium-firewall.json").write_text(json.dumps(data, indent=1), encoding="utf-8")
self.assertEqual(data["errors"], [])
for index, (width, _) in enumerate(base.WIDTHS):
shown, signed = data["results"][2 * index : 2 * index + 2]
with self.subTest(width=width):
self.assertNotIn("error", shown, shown)
self.assertEqual((shown["title"], shown["subs"]), ("Firewall", base.SUBS))
self.assertEqual(shown["sources"], ["hub", str(self.settings), str(self.outside / "demo" / ".claude" / "settings.json")])
self.assertEqual(shown["folded"], {"expanded": ["false"] * 3, "rows": 0})
self.assertEqual(shown["opened"], ["true"] * 3)
self.assertEqual(shown["heads"], ["Action", "Tool", "Pattern", "Scope", "Source", "Hits (7 days)", ""])
self.assertEqual(shown["chips"], [[chip, "true"] for chip in ("allow", "ask", "deny", "sandbox", "signed")])
rows = shown["rows"]
self.assertEqual([row[0] for row in rows], sorted((row[0] for row in rows), key=["deny", "ask", "allow", "sandbox"].index))
for row in (
["deny", "permissions.deny", "Read(~/.ssh/**)", "not seen", False],
["ask", "permissions.ask", "Bash(make deploy:*)", "2", False],
["ask", "permissions.ask", "Bash(make clean:*)", "0 unused", True],
["ask", "permissions.ask", "Bash(git push:*)", "never granted: pushes", False],
["allow", "permissions.allow", "Bash(make test:*)", "not seen", False],
["sandbox", "sandbox.filesystem.allowWrite", "/tmp", "not seen", False],
["sandbox", "sandbox.enabled", None, "not seen", False],
):
self.assertIn(row, rows)
self.assertEqual(sum(1 for row in rows if row[0] == "allow" and row[2].startswith("Bash(make")), 1 + index)
self.assertEqual(shown["withoutDeny"], {"pressed": "false", "deny": 0, "rows": len(rows) - 1})
self.assertEqual(shown["restored"], len(rows))
self.assertGreater(shown["patterns"], 5)
self.assertEqual(shown["targets"], ["All agents", "Agent Skills", "Agent Stage"])
self.assertEqual((shown["unlabelled"], shown["secret"], shown["wide"]), (0, False, False))
self.assertEqual(shown["scrolls"], width < 600)
self.assertNotIn("error", signed, signed)
self.assertTrue(any(line.startswith("+") and f'"Bash(make lint-{width}:*)"' in line for line in signed["added"]), signed["added"])
self.assertEqual(any("git status" in line for line in signed["removed"]), index == 0)
self.assertEqual((signed["result"], signed["labelled"], signed["wide"]), ("done", True, False), signed)
self.assertIn("Settings: written", " ".join(signed["toasts"]))
written = json.loads(self.settings.read_text(encoding="utf-8"))
self.assertEqual(written["permissions"]["allow"], ["Bash(make test:*)", *(f"Bash(make lint-{width}:*)" for width, _ in base.WIDTHS)])
self.assertEqual({key: written[key] for key in ("model", "env", "hooks", "sandbox")}, {key: ORIGINAL[key] for key in ("model", "env", "hooks", "sandbox")})
backups = sorted(self.settings.parent.glob("settings.json.before-firewall-*"))
self.assertEqual((len(backups), self.settings.stat().st_mode & 0o777), (3, 0o640))
def test_chromium_names_a_group_and_signs_a_rule_for_it(self):
authenticator = self.authenticator()
steps = [{"width": 1366, "height": 768, "url": f"{self.page_url}#stream/rules", "wait": 1500, "probe": probe(GROUP_RULE, {}), "shot": str(OUT / "chromium-1366-firewall-group-rule.png")}]
data = self.drive(steps, authenticator)
(OUT / "chromium-firewall-group.json").write_text(json.dumps(data, indent=1), encoding="utf-8")
self.assertEqual(data["errors"], [])
found = data["results"][0]
self.assertNotIn("error", found, found)
self.assertEqual((found["group"], found["members"], found["found"], found["hits"]), ("builders", "Skills, Stage", True, "0 unused"), found)
stored = [rule for rule in self.httpd.hub.rules.listed() if rule["pattern"] == "make e2e"]
self.assertEqual([(rule["agents"], rule["proof"]["payload"]["agents"]) for rule in stored], [(["s1", "s2"], ["s1", "s2"])] * 2)
for rule in stored:
self.assertLess(abs(rule["expires"] - (time.time() + 7 * 86400)), 2 * 86400)
self.assertEqual(found["fresh"], {"label": "Pick a target", "value": "", "blocked": True, "valid": False}, found)
self.assertEqual(found["kept"], {"label": "Group builders", "value": "group", "blocked": False, "valid": True}, found)
self.assertEqual(found["taken"], "ci: 1 of 1 rules signed for group builders")
self.assertEqual(found["reloaded"], found["kept"])
self.assertEqual(found["setGone"], {"label": "Target gone: pick again", "value": "gone", "blocked": True, "valid": False}, found)
self.assertEqual(found["sent"], [])
members = "Skills (s1), Stage (s2)"
shown = [re.sub(r"in \d+\.\d days$", "in N days", prompt.removeprefix("Passkey signs"), flags=re.MULTILINE).split("\n") for prompt in found["prompts"]]
made = sorted(stored, key=lambda rule: rule["created_ts"])
self.assertEqual(shown[:2], [["Rule", *signed_lines(made[0]), f"for group builders: {members}"], ["Rule set ci", *signed_lines(made[1]), f"for group builders: {members}"]], found)
self.assertIn(shown[2:], [[["Turn on rule", *signed_lines(rule), f"for agents {members}"]] for rule in made], found)
self.assertTrue(found["prompts"][1].split("\n")[3].endswith("in 7.0 days"), found["prompts"])
self.assertEqual((found["off"], found["on"], [rule["enabled"] for rule in self.httpd.hub.rules.listed() if rule["pattern"] == "make e2e"]), (True, True, [True, True]))
self.assertEqual(found["picked"], {"value": "group", "blocked": False})
self.assertEqual(found["gone"], {"connected": True, "value": "gone", "label": "Target gone: pick again", "valid": False, "blocked": True, "groups": 0}, found)
self.assertEqual(found["again"], {"value": "all", "valid": True, "blocked": False, "options": False}, found)
self.assertEqual(self.httpd.hub.groups.listed(), [])
if __name__ == "__main__":
unittest.main()