🧪 Test / 🧪 Lock + lint (push) Canceled after 0s
The e2e repair 550f696 on top of the e2e split: the mic exclusion of the Record lookups and the mic-field id filter move into hub_e2e_radio.py, radio_layout_probes.py and radio_layout_focus_probes.py, and tasks/match joins SCREENS in hub_e2e_kit.py, where the split put that code.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
317 lines
19 KiB
Python
317 lines
19 KiB
Python
from __future__ import annotations
|
|
|
|
import json
|
|
import os
|
|
import re
|
|
import shutil
|
|
import sys
|
|
import time
|
|
import unittest
|
|
from pathlib import Path
|
|
|
|
HERE = Path(__file__).resolve().parent
|
|
sys.path[:0] = [str(HERE.parent), str(HERE)]
|
|
|
|
import hub_e2e_kit as base
|
|
from hub_case import HubCase
|
|
from hub_support import ASK, verifier
|
|
from test_settings_writer import ORIGINAL
|
|
from test_settings_writer import host as writer_host
|
|
|
|
OUT = Path(os.environ.get("E2E_OUT", "/tmp/firewall-shots"))
|
|
SECRET = ORIGINAL["env"]["EXAMPLE_FLAG"]
|
|
ASKED = ["Bash(make deploy:*)", "Bash(make clean:*)", "Bash(git push:*)"]
|
|
SHOW = """(async () => {@HELPERS@
|
|
await screenShown(8000);
|
|
const ready = await until(() => all("#main table.fw-table tbody[data-source]").length > 1, 10000);
|
|
if (!ready) {
|
|
return {error: text(document.getElementById("main"))};
|
|
}
|
|
const owned = (svg) => svg.closest("button, a, [title]");
|
|
const rows = () => all("#main table.fw-table tr[data-filter]");
|
|
const chip = (id) => document.querySelector(`#main [aria-label="Show rules"] button[data-filter="${id}"]`);
|
|
const sources = all("#main tbody[data-source]").map((node) => node.dataset.source);
|
|
const folded = {expanded: all("#main .fw-fold").map((node) => node.getAttribute("aria-expanded")), rows: rows().length};
|
|
for (const source of sources) {
|
|
all("#main tbody[data-source]").find((node) => node.dataset.source === source).querySelector(".fw-fold").click();
|
|
}
|
|
const opened = all("#main .fw-fold").map((node) => node.getAttribute("aria-expanded"));
|
|
const shown = rows().map((node) => [node.dataset.filter, node.dataset.list || null, node.dataset.entry || null, text(node.querySelector(".fw-hits")), node.hasAttribute("data-unused")]);
|
|
chip("deny").click();
|
|
const withoutDeny = {pressed: chip("deny").getAttribute("aria-pressed"), deny: rows().filter((node) => node.dataset.filter === "deny").length, rows: rows().length};
|
|
chip("deny").click();
|
|
const wrap = document.querySelector("#main .fw-table-wrap");
|
|
wrap.scrollIntoView({block: "start"});
|
|
await sleep(100);
|
|
return {
|
|
title: text(document.querySelector("#main h2.title")),
|
|
subs: all("#tabs .subtabs button").map(name),
|
|
sources,
|
|
folded,
|
|
opened,
|
|
rows: shown,
|
|
withoutDeny,
|
|
restored: rows().length,
|
|
heads: all("#main table.fw-table thead th").map(text),
|
|
chips: all('#main [aria-label="Show rules"] button').map((node) => [node.dataset.filter, node.getAttribute("aria-pressed")]),
|
|
patterns: all('#main [aria-label="Never granted"] li').length,
|
|
targets: all('#main .rule-form select[aria-label="Applies to"] option').map(text),
|
|
unlabelled: all("#main svg.bi").map(owned).filter((owner) => !owner || (!text(owner) && !(owner.getAttribute("title") && owner.getAttribute("aria-label")))).length,
|
|
secret: document.documentElement.innerHTML.includes("@SECRET@"),
|
|
wide: document.documentElement.scrollWidth > innerWidth + 1,
|
|
scrolls: wrap.scrollWidth > wrap.clientWidth,
|
|
};
|
|
})()"""
|
|
SIGN = """(async () => {@HELPERS@
|
|
const entryNode = (entry) => all("#main tr[data-entry]").find((node) => node.dataset.entry === entry);
|
|
if (@REMOVE@) {
|
|
entryNode("Bash(git status:*)").querySelector('button[aria-label="Remove"]').click();
|
|
await sleep(100);
|
|
}
|
|
const form = document.querySelector('#main form[aria-label="Stage a settings change"]');
|
|
form.querySelector('select[aria-label="Settings file"]').value = "@FILE@";
|
|
form.querySelector('select[aria-label="List"]').value = "permissions.allow";
|
|
form.querySelector('input[aria-label="Entry"]').value = "@ENTRY@";
|
|
form.requestSubmit();
|
|
await sleep(100);
|
|
const staged = all('#main [aria-label="Staged changes"] li').map((node) => node.dataset.staged);
|
|
document.querySelector('#main button[aria-label="Review and sign"]').click();
|
|
const diff = await until(() => document.querySelector('.sheet-box [aria-label="Diff"] pre.fw-diff'), 8000);
|
|
if (!diff) {
|
|
return {error: "no diff", toasts: all("#toasts .toast").map(text)};
|
|
}
|
|
const lines = (cls) => all(`span.${cls}`, diff).map((node) => node.textContent.trim());
|
|
const shown = {added: lines("add"), removed: lines("del"), staged, labelled: all(".sheet-box button").every((button) => text(button) || (button.title && button.getAttribute("aria-label")))};
|
|
document.querySelector('.sheet-box button[aria-label="Sign and apply"]').click();
|
|
const done = await until(() => document.querySelector(".sheet-box [data-result]"), 30000);
|
|
shown.result = done ? done.dataset.result : null;
|
|
shown.text = text(done);
|
|
shown.toasts = all("#toasts .toast").map(text);
|
|
shown.wide = document.documentElement.scrollWidth > innerWidth + 1;
|
|
return shown;
|
|
})()"""
|
|
GROUP_RULE = """(async () => {@HELPERS@
|
|
const prompts = [];
|
|
new MutationObserver((records) => records.forEach((record) => record.addedNodes.forEach((node) => {
|
|
if (node.id === "passkey-prompt") {
|
|
prompts.push(text(node));
|
|
}
|
|
}))).observe(document.body, {childList: true});
|
|
const choose = (select, value) => {
|
|
select.value = value;
|
|
select.dispatchEvent(new Event("change"));
|
|
};
|
|
const groupValue = (select) => [...select.options].map((option) => option.value).find((value) => value.startsWith("group:"));
|
|
const setRow = () => document.querySelector('#main [data-set="ci"]');
|
|
const setTarget = () => setRow().querySelector("select");
|
|
const setTake = () => setRow().querySelector('button[aria-label="Take over"]');
|
|
const setState = () => ({label: text(setTarget().selectedOptions[0]), value: setTarget().value.split(":")[0], blocked: setTake().disabled, valid: setTarget().checkValidity()});
|
|
const stage = async (entry) => {
|
|
const form = document.querySelector('#main form[aria-label="Stage a settings change"]');
|
|
form.querySelector('input[aria-label="Entry"]').value = entry;
|
|
form.requestSubmit();
|
|
return until(() => all('#main [aria-label="Staged changes"] li').find((node) => node.dataset.staged === `add ${entry}`), 8000);
|
|
};
|
|
await screenShown(8000);
|
|
await until(() => document.querySelector('#main form[aria-label="New agent group"]'), 10000);
|
|
(await until(() => document.querySelector('#main tbody[data-source="hub"] .fw-fold'), 10000)).click();
|
|
const group = document.querySelector('#main form[aria-label="New agent group"]');
|
|
group.querySelector('input[aria-label="Group name"]').value = "builders";
|
|
for (const option of group.querySelector('select[aria-label="Members"]').options) {
|
|
option.selected = true;
|
|
}
|
|
group.requestSubmit();
|
|
const row = await until(() => document.querySelector('#main [data-group="builders"]'), 8000);
|
|
const rules = document.querySelector("#main .rule-form");
|
|
const target = rules.querySelector('select[aria-label="Applies to"]');
|
|
const choice = await until(() => [...target.options].find((option) => option.value.startsWith("group:")), 8000);
|
|
if (!row || !choice) {
|
|
return {error: "no group", toasts: all("#toasts .toast").map(text)};
|
|
}
|
|
target.value = choice.value;
|
|
rules.querySelector('input[placeholder="make test*"]').value = "make e2e";
|
|
rules.requestSubmit();
|
|
const ruleRow = () => all("#main tr[data-rule]").find((node) => all("code", node).some((code) => text(code) === "make e2e"));
|
|
const pill = await until(() => all("#main tr[data-rule] [data-agents]").find((node) => text(node) === "group builders"), 15000);
|
|
const hits = await until(() => ruleRow() && text(ruleRow().querySelector(".fw-hits")) !== "…" && text(ruleRow().querySelector(".fw-hits")), 8000);
|
|
const found = {group: text(row.querySelector("strong")), members: text(row.querySelector(".muted")), pills: all("#main tr[data-rule] [data-agents]").map(text), found: Boolean(pill), hits, toasts: all("#toasts .toast").map(text)};
|
|
const sets = document.querySelector('#main form[aria-label="New rule set"]');
|
|
sets.querySelector('input[aria-label="Set name"]').value = "ci";
|
|
for (const option of sets.querySelector('select[aria-label="Rules"]').options) {
|
|
option.selected = option.textContent.includes("make e2e");
|
|
}
|
|
sets.requestSubmit();
|
|
if (!await until(setRow, 8000)) {
|
|
return {...found, error: "no rule set", toasts: all("#toasts .toast").map(text)};
|
|
}
|
|
found.fresh = setState();
|
|
choose(setTarget(), groupValue(setTarget()));
|
|
await stage("Bash(make e2e:*)");
|
|
found.kept = setState();
|
|
setTake().click();
|
|
found.taken = await until(() => all("#toasts .toast").map(text).find((line) => line.startsWith("ci:")), 15000);
|
|
found.reloaded = await until(() => setRow() && setTarget().value.startsWith("group:") && document.querySelector("#main .rule-form") && setState(), 8000);
|
|
const form = document.querySelector("#main .rule-form");
|
|
const picker = form.querySelector('select[aria-label="Applies to"]');
|
|
const sign = form.querySelector("[data-sign]");
|
|
choose(picker, groupValue(picker));
|
|
found.picked = {value: picker.value.split(":")[0], blocked: sign.disabled};
|
|
const sent = [];
|
|
const fetched = window.fetch;
|
|
window.fetch = (url, options) => {
|
|
sent.push(String(url));
|
|
return fetched(url, options);
|
|
};
|
|
document.querySelector('#main [data-group="builders"] button[aria-label="Delete group"]').click();
|
|
await until(() => picker.value === "gone" || !picker.isConnected, 8000);
|
|
found.gone = {connected: picker.isConnected, value: picker.value, label: text(picker.selectedOptions[0]), valid: picker.checkValidity(), blocked: sign.disabled, groups: [...picker.options].filter((option) => option.value.startsWith("group:")).length};
|
|
await stage("Bash(make e2e-again:*)");
|
|
found.setGone = setState();
|
|
setTake().click();
|
|
await sleep(500);
|
|
found.sent = sent.filter((url) => url.startsWith("/api/rules"));
|
|
choose(picker, "all");
|
|
found.again = {value: picker.value, valid: picker.checkValidity(), blocked: sign.disabled, options: [...picker.options].map((option) => option.value).includes("gone")};
|
|
const toggle = (label) => ruleRow() && [...ruleRow().querySelectorAll("button")].find((button) => text(button) === label);
|
|
toggle("Turn off").click();
|
|
found.off = Boolean(await until(() => toggle("Turn on"), 8000));
|
|
toggle("Turn on").click();
|
|
found.on = Boolean(await until(() => toggle("Turn off"), 15000));
|
|
found.prompts = prompts;
|
|
return found;
|
|
})()"""
|
|
|
|
|
|
def signed_lines(rule: dict) -> list[str]:
|
|
"""Returns:
|
|
The lines the passkey prompt shows for a stored make e2e rule before
|
|
its target, its lifetime in days as N.
|
|
"""
|
|
return ["ask Bash: make e2e", "in *", f"until {time.strftime('%Y-%m-%d %H:%M', time.gmtime(rule['expires']))} UTC, in N days"]
|
|
|
|
|
|
def probe(template: str, values: dict[str, str]) -> str:
|
|
text = template.replace("@HELPERS@", base.HELPERS)
|
|
for name, value in values.items():
|
|
text = text.replace(f"@{name}@", value)
|
|
return text
|
|
|
|
|
|
@unittest.skipUnless(shutil.which(os.environ.get("CHROMIUM", "chromium")), "no Chromium")
|
|
class TestFirewallE2E(HubCase):
|
|
drive = base.OrchestratorE2ECase.drive
|
|
beside = base.OrchestratorE2ECase.beside
|
|
authenticator = base.OrchestratorE2ECase.authenticator
|
|
|
|
def setUp(self):
|
|
super().setUp()
|
|
OUT.mkdir(parents=True, exist_ok=True)
|
|
self.claim("s1", "Skills")
|
|
self.claim("s2", "Stage")
|
|
self.snapshot()
|
|
home = self.outside / "home"
|
|
self.settings = home / ".claude" / "settings.json"
|
|
self.settings.parent.mkdir(parents=True)
|
|
self.settings.write_text(json.dumps({**ORIGINAL, "permissions": {**ORIGINAL["permissions"], "ask": ASKED}}, indent=2) + "\n", encoding="utf-8")
|
|
self.settings.chmod(0o640)
|
|
tree = self.outside / "writer-tree.json"
|
|
tree.write_text(json.dumps({"repos": [{"path": str(self.outside / "demo"), "known": True, "default": "main"}]}), encoding="utf-8")
|
|
trust = verifier.Trust(approvers=self.approvers, replay=self.replay / "firewall.json", origin=self.origin)
|
|
self.writer = writer_host.Writer(trust, home, tree)
|
|
self.spool = self.state / "firewall"
|
|
verifier.write_private(self.spool / "view.json", self.writer.view())
|
|
self.page_url = f"http://localhost:{self.port}/"
|
|
|
|
def host_unit(self) -> None:
|
|
if any((self.spool / "jobs").iterdir()):
|
|
writer_host.cycle(self.spool / "jobs", self.spool / "results", self.writer)
|
|
verifier.write_private(self.spool / "view.json", self.writer.view())
|
|
|
|
def test_chromium_shows_the_settings_and_signs_a_settings_diff_at_three_widths(self):
|
|
authenticator = self.authenticator()
|
|
for command in ("make deploy", "make deploy --dry", "git push origin main"):
|
|
self.create_ask({**ASK, "command": command})
|
|
self.beside(self.host_unit)
|
|
steps = []
|
|
for index, (width, height) in enumerate(base.WIDTHS):
|
|
values = {"SECRET": SECRET, "FILE": str(self.settings), "ENTRY": f"Bash(make lint-{width}:*)", "REMOVE": "true" if index == 0 else "false"}
|
|
steps.append({"width": width, "height": height, "url": f"{self.page_url}?{width}#stream/rules", "wait": 1500, "probe": probe(SHOW, values), "shot": str(OUT / f"chromium-{width}-firewall.png")})
|
|
steps.append({"width": width, "height": height, "url": None, "wait": 0, "probe": probe(SIGN, values), "shot": str(OUT / f"chromium-{width}-firewall-signed.png")})
|
|
data = self.drive(steps, authenticator)
|
|
(OUT / "chromium-firewall.json").write_text(json.dumps(data, indent=1), encoding="utf-8")
|
|
self.assertEqual(data["errors"], [])
|
|
for index, (width, _) in enumerate(base.WIDTHS):
|
|
shown, signed = data["results"][2 * index : 2 * index + 2]
|
|
with self.subTest(width=width):
|
|
self.assertNotIn("error", shown, shown)
|
|
self.assertEqual((shown["title"], shown["subs"]), ("Firewall", base.SUBS))
|
|
self.assertEqual(shown["sources"], ["hub", str(self.settings), str(self.outside / "demo" / ".claude" / "settings.json")])
|
|
self.assertEqual(shown["folded"], {"expanded": ["false"] * 3, "rows": 0})
|
|
self.assertEqual(shown["opened"], ["true"] * 3)
|
|
self.assertEqual(shown["heads"], ["Action", "Tool", "Pattern", "Scope", "Source", "Hits (7 days)", ""])
|
|
self.assertEqual(shown["chips"], [[chip, "true"] for chip in ("allow", "ask", "deny", "sandbox", "signed")])
|
|
rows = shown["rows"]
|
|
self.assertEqual([row[0] for row in rows], sorted((row[0] for row in rows), key=["deny", "ask", "allow", "sandbox"].index))
|
|
for row in (
|
|
["deny", "permissions.deny", "Read(~/.ssh/**)", "not seen", False],
|
|
["ask", "permissions.ask", "Bash(make deploy:*)", "2", False],
|
|
["ask", "permissions.ask", "Bash(make clean:*)", "0 unused", True],
|
|
["ask", "permissions.ask", "Bash(git push:*)", "never granted: pushes", False],
|
|
["allow", "permissions.allow", "Bash(make test:*)", "not seen", False],
|
|
["sandbox", "sandbox.filesystem.allowWrite", "/tmp", "not seen", False],
|
|
["sandbox", "sandbox.enabled", None, "not seen", False],
|
|
):
|
|
self.assertIn(row, rows)
|
|
self.assertEqual(sum(1 for row in rows if row[0] == "allow" and row[2].startswith("Bash(make")), 1 + index)
|
|
self.assertEqual(shown["withoutDeny"], {"pressed": "false", "deny": 0, "rows": len(rows) - 1})
|
|
self.assertEqual(shown["restored"], len(rows))
|
|
self.assertGreater(shown["patterns"], 5)
|
|
self.assertEqual(shown["targets"], ["All agents", "Agent Skills", "Agent Stage"])
|
|
self.assertEqual((shown["unlabelled"], shown["secret"], shown["wide"]), (0, False, False))
|
|
self.assertEqual(shown["scrolls"], width < 600)
|
|
self.assertNotIn("error", signed, signed)
|
|
self.assertTrue(any(line.startswith("+") and f'"Bash(make lint-{width}:*)"' in line for line in signed["added"]), signed["added"])
|
|
self.assertEqual(any("git status" in line for line in signed["removed"]), index == 0)
|
|
self.assertEqual((signed["result"], signed["labelled"], signed["wide"]), ("done", True, False), signed)
|
|
self.assertIn("Settings: written", " ".join(signed["toasts"]))
|
|
written = json.loads(self.settings.read_text(encoding="utf-8"))
|
|
self.assertEqual(written["permissions"]["allow"], ["Bash(make test:*)", *(f"Bash(make lint-{width}:*)" for width, _ in base.WIDTHS)])
|
|
self.assertEqual({key: written[key] for key in ("model", "env", "hooks", "sandbox")}, {key: ORIGINAL[key] for key in ("model", "env", "hooks", "sandbox")})
|
|
backups = sorted(self.settings.parent.glob("settings.json.before-firewall-*"))
|
|
self.assertEqual((len(backups), self.settings.stat().st_mode & 0o777), (3, 0o640))
|
|
|
|
def test_chromium_names_a_group_and_signs_a_rule_for_it(self):
|
|
authenticator = self.authenticator()
|
|
steps = [{"width": 1366, "height": 768, "url": f"{self.page_url}#stream/rules", "wait": 1500, "probe": probe(GROUP_RULE, {}), "shot": str(OUT / "chromium-1366-firewall-group-rule.png")}]
|
|
data = self.drive(steps, authenticator)
|
|
(OUT / "chromium-firewall-group.json").write_text(json.dumps(data, indent=1), encoding="utf-8")
|
|
self.assertEqual(data["errors"], [])
|
|
found = data["results"][0]
|
|
self.assertNotIn("error", found, found)
|
|
self.assertEqual((found["group"], found["members"], found["found"], found["hits"]), ("builders", "Skills, Stage", True, "0 unused"), found)
|
|
stored = [rule for rule in self.httpd.hub.rules.listed() if rule["pattern"] == "make e2e"]
|
|
self.assertEqual([(rule["agents"], rule["proof"]["payload"]["agents"]) for rule in stored], [(["s1", "s2"], ["s1", "s2"])] * 2)
|
|
for rule in stored:
|
|
self.assertLess(abs(rule["expires"] - (time.time() + 7 * 86400)), 2 * 86400)
|
|
self.assertEqual(found["fresh"], {"label": "Pick a target", "value": "", "blocked": True, "valid": False}, found)
|
|
self.assertEqual(found["kept"], {"label": "Group builders", "value": "group", "blocked": False, "valid": True}, found)
|
|
self.assertEqual(found["taken"], "ci: 1 of 1 rules signed for group builders")
|
|
self.assertEqual(found["reloaded"], found["kept"])
|
|
self.assertEqual(found["setGone"], {"label": "Target gone: pick again", "value": "gone", "blocked": True, "valid": False}, found)
|
|
self.assertEqual(found["sent"], [])
|
|
members = "Skills (s1), Stage (s2)"
|
|
shown = [re.sub(r"in \d+\.\d days$", "in N days", prompt.removeprefix("Passkey signs"), flags=re.MULTILINE).split("\n") for prompt in found["prompts"]]
|
|
made = sorted(stored, key=lambda rule: rule["created_ts"])
|
|
self.assertEqual(shown[:2], [["Rule", *signed_lines(made[0]), f"for group builders: {members}"], ["Rule set ci", *signed_lines(made[1]), f"for group builders: {members}"]], found)
|
|
self.assertIn(shown[2:], [[["Turn on rule", *signed_lines(rule), f"for agents {members}"]] for rule in made], found)
|
|
self.assertTrue(found["prompts"][1].split("\n")[3].endswith("in 7.0 days"), found["prompts"])
|
|
self.assertEqual((found["off"], found["on"], [rule["enabled"] for rule in self.httpd.hub.rules.listed() if rule["pattern"] == "make e2e"]), (True, True, [True, True]))
|
|
self.assertEqual(found["picked"], {"value": "group", "blocked": False})
|
|
self.assertEqual(found["gone"], {"connected": True, "value": "gone", "label": "Target gone: pick again", "valid": False, "blocked": True, "groups": 0}, found)
|
|
self.assertEqual(found["again"], {"value": "all", "valid": True, "blocked": False, "options": False}, found)
|
|
self.assertEqual(self.httpd.hub.groups.listed(), [])
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|