fix(orchestrator): close operator rows the server no longer has

A 404 on Run, Close, a passkey signature or a rule accept closes the card with the notice "already done or removed elsewhere" instead of a lasting error; a selection reloads /api/state once and closes only the cards that are gone, other failures keep the old error. Tests run once on unify/skills-20261114 after the merge (operator order 2026-11-14).

🎯 Fix: 90 %
💥 Break: 3 %
💸 Tokens: –
⏱️ Velocity: –
🛡️ Security: 100 %
🏆 Score: 87
🚦 Gate: ✅ 90/5, tokens and velocity <= 100, security 100 %

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Kevin Veen-BirkenbachandClaude Opus 5.5 committed 2026-11-14 21:21:07 +00:00
1 parent a2a03995cb
commit ec30d8c94e
6 files changed
+372 -13

No files matched your search

+134
View File
@@ -0,0 +1,134 @@
from __future__ import annotations
import os
import shutil
import sys
import unittest
from pathlib import Path
from unittest import mock
HERE = Path(__file__).resolve().parent
sys.path[:0] = [str(HERE.parent), str(HERE)]
import test_orchestrator_e2e as base
from hub_case import HubCase
OUT = Path(os.environ.get("E2E_OUT", "/tmp/inbox-gone-rows-shots"))
WIDTHS = ((1920, 1080), (390, 844))
GONE = "git merge --no-ff --no-verify -F msg"
NOTICE = "already done or removed elsewhere"
FILL = {"workdir": "/repo", "check": "manual", "status": "queued", "roles": {"Operator": "R", "Store": "A"}}
SHOWN = {
"operator/gui-agentbox-merge-push": {**FILL, "title": GONE, "description": "merge the agentbox", "seq": 1},
"operator/later": {**FILL, "title": "echo later", "description": "removed while the page shows it", "seq": 2},
"operator/kept": {**FILL, "title": "echo kept", "description": "changed while the page shows it", "seq": 3},
}
LEFT = {"operator/kept": {**SHOWN["operator/kept"], "title": "echo kept changed"}}
HOLD = """() => {
const send = window.fetch.bind(window);
const held = [];
let holding = true;
window.__release = () => {
holding = false;
held.splice(0).forEach((go) => go());
};
window.fetch = (path, init) => (holding && String(path).startsWith("/api/state") ? new Promise((resolve, reject) => held.push(() => send(path, init).then(resolve, reject))) : send(path, init));
}"""
CLICK = """
const click = async (words) => {
const row = rowOf(words);
if (!row) {
return {error: text(document.getElementById("main"))};
}
const shown = new Set(all("#toasts .toast"));
press(row, "Run");
const toast = await until(() => all("#toasts .toast").find((box) => !shown.has(box)), 8000);
await sleep(300);
return {toast: text(toast && toast.querySelector("strong")), lines: text(toast && toast.querySelector("pre")), bad: toast ? toast.classList.contains("bad") : null, open: Boolean(rowOf(words)), wide: wide()};
};
"""
BEFORE = """(async () => {@HELPERS@@LABELS@
window.__page = "same page";
await until(() => all("#main .inbox-list > li").length === 3, 8000);
return {rows: all("#main .inbox-list > li .command code").map(text), wide: wide()};
})()"""
RUN_GONE = """(async () => {@HELPERS@@LABELS@@CLICK@
return click("git merge");
})()"""
RUN_KEPT = """(async () => {@HELPERS@@LABELS@@CLICK@
return click("echo kept");
})()"""
REFRESH = """(async () => {@HELPERS@@LABELS@
window.__release();
window.dispatchEvent(new Event("focus"));
const settled = await until(() => !rowOf("echo later") && rowOf("echo kept changed"), 15000);
return {settled: Boolean(settled), samePage: window.__page === "same page", rows: all("#main .inbox-list > li .command code").map(text)};
})()"""
def probe(template: str) -> str:
return template.replace("@CLICK@", CLICK).replace("@HELPERS@", base.HELPERS).replace("@LABELS@", base.LABELS)
@unittest.skipUnless(shutil.which(os.environ.get("CHROMIUM", "chromium")), "no Chromium")
class TestInboxGoneRows(HubCase):
drive = base.TestOrchestratorE2E.drive
def setUp(self):
super().setUp()
OUT.mkdir(parents=True, exist_ok=True)
self.write_tree()
hub = self.httpd.hub
real = hub.render
def render(nonce: str) -> tuple[str, str]:
self.write_council({"rasci.json": SHOWN})
page = real(nonce)
self.write_council({"rasci.json": LEFT})
return page
patch = mock.patch.object(hub, "render", render)
patch.start()
self.addCleanup(patch.stop)
def steps(self, browser: str) -> list[dict]:
steps = []
for width, height in WIDTHS:
shot = OUT / f"{browser}-{width}"
url = f"http://localhost:{self.port}/?{width}#stream/inbox"
steps += [
{"width": width, "height": height, "url": url, "wait": 1200, "probe": probe(BEFORE), "shot": f"{shot}-1-stale-row.png"},
{"width": width, "height": height, "url": None, "wait": 0, "probe": probe(RUN_GONE), "shot": f"{shot}-2-run-removed-row.png"},
{"width": width, "height": height, "url": None, "wait": 0, "probe": probe(RUN_KEPT), "shot": f"{shot}-3-run-changed-row.png"},
{"width": width, "height": height, "url": None, "wait": 0, "probe": probe(REFRESH), "shot": f"{shot}-4-refreshed.png"},
]
return steps
def check(self, browser: str, driver: str, binary: str) -> None:
data = self.drive(self.steps(browser), None, driver, binary, HOLD)
self.assertEqual([error for error in data["errors"] if "(style-src-elem)" not in error and "/favicon.ico" not in error], [])
results = iter(data["results"])
for width, _ in WIDTHS:
before, gone, kept, refreshed = next(results), next(results), next(results), next(results)
with self.subTest(browser=browser, width=width, part="the page draws the rows it was rendered with"):
self.assertEqual((sorted(before["rows"]), before["wide"]), (sorted([GONE, "echo later", "echo kept"]), False), before)
with self.subTest(browser=browser, width=width, part="a run the hub answers 404 for closes the card at once with a notice"):
self.assertNotIn("error", gone, gone)
self.assertEqual((gone["toast"], gone["lines"], gone["bad"]), (f"Run: {GONE}: {NOTICE}", None, False), gone)
self.assertEqual((gone["open"], gone["wide"]), (False, False), gone)
with self.subTest(browser=browser, width=width, part="a present row whose run fails otherwise stays with its error"):
self.assertNotIn("error", kept, kept)
self.assertEqual((kept["toast"], kept["lines"], kept["bad"], kept["open"]), ("Run: echo kept: not possible", '409 {"id": "operator/kept", "status": "changed"}', True, True), kept)
with self.subTest(browser=browser, width=width, part="the next refresh drops every row the hub no longer returns, without a reload"):
self.assertEqual((refreshed["settled"], refreshed["samePage"], refreshed["rows"]), (True, True, ["echo kept changed"]), refreshed)
def test_chromium_closes_removed_rows_and_keeps_failed_ones_at_1920_and_390(self):
self.check("chromium", "cdp.mjs", os.environ.get("CHROMIUM", "chromium"))
@unittest.skipUnless(shutil.which(os.environ.get("FIREFOX", "firefox")), "no Firefox")
def test_firefox_closes_removed_rows_and_keeps_failed_ones_at_1920_and_390(self):
self.check("firefox", "bidi.mjs", os.environ.get("FIREFOX", "firefox"))
if __name__ == "__main__":
unittest.main()
+186
View File
@@ -0,0 +1,186 @@
from __future__ import annotations
import json
import subprocess
import unittest
from hub_support import HUB
TEMPLATES = HUB / "templates"
MODULES = ["page-constants.js", "page-reveal.js", "page-dom.js", "page-api.js", "page-passkey.js", "page-inbox.js", "page-console.js", "page-rule-request.js", "page-load.js"]
GONE = "operator/gui-agentbox-merge-push"
NOTICE = "already done or removed elsewhere"
PROBE = r"""
const fs = require("fs");
const vm = require("vm");
const [dir, names, GONE] = [process.argv[1], JSON.parse(process.argv[2]), process.argv[3]];
class El {
constructor() { Object.assign(this, {children: [], attrs: {}, dataset: {}, style: {}, textContent: ""}); }
setAttribute(key, value) { this.attrs[key] = String(value); }
append(...nodes) { this.children.push(...nodes); }
addEventListener() {}
remove() {}
}
const document = {
currentScript: null, body: new El(), hidden: false,
getElementById: (id) => Object.assign(new El(), {textContent: id === "emoji-bases" ? "[]" : ""}),
createElement: () => new El(), createElementNS: () => new El(),
};
const now = 1_800_000_000;
const base = {agent: "Store", to: "Store", detail: "", green: false, gate: "", score: "", warnings: [], options: [], recommended: "", command: "", directory: "/repo", sha256: "s", runnable: true, deniable: true, job: null, rule: null, dry_run: null, diff: null, pinned: false, until: 0, held: false, ts: now};
const row = (id, command, extra) => ({...base, kind: "operator", id, title: command, command, ...extra});
const items = {
gone: row(GONE, "git merge --no-ff --no-verify -F msg"),
kept: row("operator/kept", "echo kept"),
later: row("operator/later", "echo later"),
closed: row("operator/closed", "echo closed", {runnable: false, deniable: false, job: {id: "0123456789abcdef", status: "done", code: 0, output: "", created_ts: now, started: now, finished: now}}),
rule: {...base, kind: "rule", id: "1123456789abcdef", title: "Rule ask Bash make deploy", runnable: false},
};
const context = {
console, document, JSON, Math, Date, Intl, Promise, Set, Map, Number, String, Array, Object, RegExp, Boolean, Error, AbortController, setTimeout, setImmediate,
window: {location: {hash: "", hostname: "localhost", port: "8737", pathname: "/"}, setTimeout: () => 0, clearTimeout: () => {}, setInterval: () => 0, clearInterval: () => {}, getSelection: () => "", matchMedia: () => ({matches: false})},
ResizeObserver: class { observe() {} unobserve() {} },
state: {inbox: [], radio: [], settings: {}, now},
ui: {screen: "inbox", busy: false, selected: new Set(), cursor: 0, snoozed: false, asOf: null, playing: false, offline: false},
known: {batches: new Set()}, motion: {current: new Map([["x", {}]]), asOf: null}, settingsSig: "{}",
windowQuery: () => "", adoptSettings: () => {}, tuneMusic: () => {}, drawSky: () => {}, announce: () => {}, drawTime: () => {}, drawAgent: () => {}, relayout: () => {}, pulses: () => {}, tick: () => {},
GONE, items, now,
};
const source = names.map((name) => fs.readFileSync(`${dir}/${name}`, "utf8")).join("\n");
const probe = `
const fresh = load;
const toasts = [];
const posts = [];
let renders = [];
let atLoad = null;
let loads = 0;
let reloaded = null;
let answers = {};
toast = (title, lines, good) => { toasts.push([title, lines, good]); return {remove() {}}; };
render = (force) => { renders.push(force); };
drawStream = () => {};
load = async () => {
loads += 1;
atLoad = atLoad || state.inbox.map((item) => item.id);
if (reloaded) {
state.inbox = reloaded.map((key) => ({...items[key]}));
}
};
fetch = async (path) => {
posts.push(path);
const answer = answers[path] || [500, "unexpected " + path];
if (answer === "offline") {
throw new TypeError("Failed to fetch");
}
const [status, text] = answer;
return {ok: status >= 200 && status < 300, status, text: async () => text, json: async () => JSON.parse(text)};
};
const reply = (status, text) => [status, text];
const missing = reply(404, "not found\\n");
const results = {};
async function scenario(name, shown, given, act, after) {
state.inbox = shown.map((key) => ({...items[key]}));
ui.busy = false;
toasts.length = 0;
posts.length = 0;
renders = [];
atLoad = null;
loads = 0;
reloaded = after || null;
answers = given;
await act();
results[name] = {inbox: state.inbox.map((item) => item.id), atLoad, loads, renders: renders.length, toasts: [...toasts], posts: [...posts], busy: ui.busy};
}
const grantBoth = () => submit(state.inbox.map((item) => entryOf(item, "approve", "")), "Grant selection: 2 entries");
(async () => {
await scenario("run_sign_404", ["gone", "kept"], {"/api/sign": missing}, () => approveOne(state.inbox[0]));
await scenario("run_approve_404", ["gone", "kept"], {"/api/sign": reply(200, '{"payload": null}'), "/api/inbox/approve": missing}, () => approveOne(state.inbox[0]));
await scenario("deny_404", ["gone", "kept"], {"/api/sign": missing}, () => rejectOne(state.inbox[0]));
await scenario("dismiss_404", ["closed", "kept"], {"/api/inbox/dismiss": missing}, () => dismissJob(state.inbox[0].job));
await scenario("rule_404", ["rule", "kept"], {["/api/rule-requests/" + items.rule.id + "/sign"]: missing}, () => acceptRule(state.inbox[0]));
await scenario("run_409", ["gone", "kept"], {"/api/sign": reply(409, '{"id": "operator/kept", "status": "changed"}')}, () => approveOne(state.inbox[1]));
await scenario("run_offline", ["gone", "kept"], {"/api/sign": "offline"}, () => approveOne(state.inbox[1]));
await scenario("bulk_404", ["gone", "kept"], {"/api/sign": missing}, grantBoth, ["kept"]);
await scenario("bulk_approve_404", ["gone", "kept"], {"/api/sign": reply(200, '{"payload": null}'), "/api/inbox/approve": missing}, grantBoth, ["kept"]);
await scenario("bulk_404_none_gone", ["gone", "kept"], {"/api/sign": missing}, grantBoth, ["gone", "kept"]);
await scenario("bulk_409", ["gone", "kept"], {"/api/sign": reply(409, '{"status": "held"}')}, grantBoth, ["kept"]);
state.inbox = [items.gone, items.kept, items.later].map((item) => ({...item}));
renders = [];
answers = {"/api/state?": reply(200, JSON.stringify({inbox: [items.kept], radio: [], settings: {}, now: now + 10}))};
await fresh();
results.refresh = {inbox: state.inbox.map((item) => item.id), renders, offline: ui.offline};
console.log(JSON.stringify(results));
})().catch((error) => { console.error(error && error.stack || error); process.exit(1); });
`;
vm.runInNewContext(`${source}\n${probe}`, {...context, process});
"""
class TestInboxGoneRows(unittest.TestCase):
@classmethod
def setUpClass(cls):
done = subprocess.run(["node", "-e", PROBE, str(TEMPLATES), json.dumps(MODULES), GONE], capture_output=True, text=True, timeout=60, check=False)
if done.returncode != 0:
raise AssertionError(done.stderr[-4000:])
cls.data = json.loads(done.stdout)
def assert_closed(self, name: str, kept: list[str], label: str) -> None:
"""Args:
name: the scenario whose card the hub answered 404 for.
kept: the ids the inbox still holds after the action.
label: the start of the notice the page shows instead of an error.
"""
found = self.data[name]
self.assertEqual(found["inbox"], kept, found)
self.assertEqual(found["atLoad"], kept, "the card must go before the page reloads its state")
self.assertGreaterEqual(found["renders"], 1, found)
self.assertEqual(found["toasts"], [[f"{label}: {NOTICE}", [], True]], found)
self.assertFalse(found["busy"], found)
def test_a_run_the_hub_answers_404_at_signing_closes_the_card_with_a_notice(self):
self.assert_closed("run_sign_404", ["operator/kept"], "Run: git merge --no-ff --no-verify -F msg")
def test_a_run_the_hub_answers_404_at_the_grant_closes_the_card_with_a_notice(self):
self.assert_closed("run_approve_404", ["operator/kept"], "Run: git merge --no-ff --no-verify -F msg")
self.assertEqual(self.data["run_approve_404"]["posts"], ["/api/sign", "/api/inbox/approve"])
def test_a_denial_of_a_removed_row_closes_its_card(self):
self.assert_closed("deny_404", ["operator/kept"], "Denial: git merge --no-ff --no-verify -F msg")
def test_closing_the_console_of_a_removed_job_closes_its_card(self):
self.assert_closed("dismiss_404", ["operator/kept"], "Close")
def test_accepting_a_removed_rule_request_closes_its_card(self):
self.assert_closed("rule_404", ["operator/kept"], "Rule: Rule ask Bash make deploy")
def test_a_present_row_whose_run_fails_otherwise_stays_with_its_error(self):
for name, line in (("run_409", '409 {"id": "operator/kept", "status": "changed"}'), ("run_offline", "0 TypeError: Failed to fetch")):
with self.subTest(name=name):
found = self.data[name]
self.assertEqual(found["inbox"], [GONE, "operator/kept"], found)
self.assertEqual(found["toasts"], [["Run: echo kept: not possible", [line], False]], found)
def test_a_404_for_a_selection_reloads_once_and_closes_the_cards_that_are_gone_with_the_notice(self):
for name in ("bulk_404", "bulk_approve_404"):
with self.subTest(name=name):
found = self.data[name]
self.assertEqual((found["inbox"], found["loads"]), (["operator/kept"], 1), found)
self.assertEqual(found["toasts"], [[f"Grant selection: 2 entries: {NOTICE}", [], True]], found)
def test_a_404_for_a_selection_whose_reload_still_lists_every_card_keeps_the_error(self):
found = self.data["bulk_404_none_gone"]
self.assertEqual((found["inbox"], found["loads"]), ([GONE, "operator/kept"], 1), found)
self.assertEqual(found["toasts"], [["Grant selection: 2 entries: not possible", ["404 not found"], False]], found)
def test_a_selection_refused_for_another_reason_keeps_its_error_and_no_notice(self):
found = self.data["bulk_409"]
self.assertEqual(found["toasts"], [["Grant selection: 2 entries: not possible", ['409 {"status": "held"}'], False]], found)
def test_the_next_refresh_drops_every_row_the_hub_no_longer_returns_without_a_reload(self):
found = self.data["refresh"]
self.assertEqual(found["inbox"], ["operator/kept"], found)
self.assertEqual((found["renders"], found["offline"]), ([False], False), found)
if __name__ == "__main__":
unittest.main()