fix(orchestrator): settle queue jobs whose holder is gone instead of showing them running

The hub read queue.json alone, so an active entry whose hold had exited kept showing as running. client/queue_live.py probes each hold lock as queue.py's held() does (shared non-blocking flock, hard links probed too) and reads the whole queue-history.jsonl, cached by inode, size and mtime; an active entry with a history row started after its enqueue ends with that row's exit, one whose holder is gone becomes gone. The outcome lives in the hub-owned field hub_outcome; queue.py status is never rewritten. resources.py uses the same settled(), plan medians skip the queue kind, and a guard test keeps every client module in the Dockerfile COPY line. Also closes fd leaks in read_history and records.read_regular.

MEDUSA: 11 findings in resources.py (lines 73, 81, 262, 445 x2, 576, 227, 479, 524, 578) and crons.py:125 accepted as false positives after an independent review, operator pick 2026-11-12.

🎯 Fix: 92 %
💥 Break: 5 %
💸 Cost: 100 %
⏱️ Time: 100 %
🛡️ Security: 100 %
🏆 Score: 87
🚦 Gate: ✅ 90/5, cost and time <= 100, security 100 %

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Kevin Veen-BirkenbachandClaude Opus 5.5 committed 2026-11-12 23:09:32 +00:00
1 parent 3418738993
commit 85b79de0d8
15 files changed
+534 -39

No files matched your search

+69 -10
View File
@@ -1,15 +1,23 @@
from __future__ import annotations
import os
import threading
from collections.abc import Callable
from dataclasses import dataclass, field
from pathlib import Path
from .council import agent
from .events import event
from .names import Directory
from .records import epoch, read_json, shown
from .signed import client_module
queue_live = client_module("queue_live", {})
CRON_ACTIVE = 900
TIMER_KEYS = ("OnCalendar", "OnActiveSec", "OnBootSec", "OnStartupSec", "OnUnitActiveSec", "OnUnitInactiveSec")
CRON_LABELS = {"cron": "Cron", "wakeup": "Wakeup", "background": "Background", "timer": "Timer", "queue": "Queue"}
OUTCOME = "hub_outcome"
def cron_state(cron: dict, beats: dict[str, float], now: float) -> dict:
@@ -41,10 +49,52 @@ def timer_rows(folder: Path) -> list[dict]:
return rows
def queue_entries(council: Path) -> list[tuple[dict, float]]:
"""Returns:
Each kvb-org-queue entry of queue.json with its enqueue time; an
entry without one is no entry.
@dataclass
class History:
path: Path
lock: threading.Lock = field(default_factory=threading.Lock)
cached: list[dict] = field(default_factory=list)
stamp: tuple[int, int, int] | None = None
def rows(self) -> list[dict]:
"""Returns:
queue_live.read_history() of the whole file, parsed again only
when its inode, size or mtime changed; after a failed read the
rows read before, and the next call reads again. The list is
shared, read only.
"""
with self.lock:
try:
info = os.lstat(self.path)
except FileNotFoundError:
self.cached, self.stamp = [], None
return self.cached
except OSError:
self.stamp = None
return self.cached
stamp = (info.st_ino, info.st_size, info.st_mtime_ns)
if stamp != self.stamp:
found = queue_live.read_history(self.path)
if found is None:
self.stamp = None
else:
self.cached, self.stamp = found, stamp
return self.cached
def outcome_of(entry: dict) -> dict | None:
found = entry.get(OUTCOME)
return found if isinstance(found, dict) else None
def queue_entries(council: Path, history: Callable[[], list[dict]]) -> list[tuple[dict, float]]:
"""Args:
council: the council directory.
history: returns the queue-history.jsonl rows, History.rows.
Returns:
Each entry of queue.json with an enqueue time that queue_live.settled()
keeps, with that time, as a copy whose OUTCOME is its outcome.
"""
data = read_json(council / "queue.json")
entries = data.get("entries") if isinstance(data, dict) else None
@@ -53,17 +103,26 @@ def queue_entries(council: Path) -> list[tuple[dict, float]]:
created = epoch(entry.get("enqueued")) if isinstance(entry, dict) else None
if created is not None:
found.append((entry, created))
return found
return [({**entry, OUTCOME: result}, created) for entry, created, result in queue_live.settled(council, found, history)]
def queue_rows(council: Path) -> list[dict]:
def queue_state(entry: dict) -> str:
found = outcome_of(entry)
if found is None:
return "running" if entry.get("status") == "active" else "waiting"
if found.get("state") == "gone":
return "holder gone without an exit code"
code = found.get("exit")
return "ended" if code is None else f"exit {code}"
def queue_rows(council: Path, history: Callable[[], list[dict]]) -> list[dict]:
rows = []
for entry, created in queue_entries(council):
running = "running" if entry.get("status") == "active" else "waiting"
for entry, created in queue_entries(council, history):
rows.append({
"key": f"queue:{shown(entry.get('id'))}", "type": "queue", "session": shown(entry.get("session")), "sub": "", "created": created,
"deleted": None, "until": None, "job": shown(entry.get("id")), "expression": f"{shown(entry.get('kind'))} · {running}",
"prompt": shown(entry.get("cmd")), "reason": shown(entry.get("task")), "active": True, "agent": agent(entry.get("name")),
"deleted": None, "until": None, "job": shown(entry.get("id")), "expression": f"{shown(entry.get('kind'))} · {queue_state(entry)}",
"prompt": shown(entry.get("cmd")), "reason": shown(entry.get("task")), "active": outcome_of(entry) is None, "agent": agent(entry.get("name")),
})
return rows
+3
View File
@@ -13,6 +13,7 @@ from .asking import Asking
from .batches import Pending
from .choosing import Choosing
from .cleanup import Cleaning, Spool
from .crons import History, queue_live
from .denying import Denying
from .engines import Engines, Speech
from .events import Log, Watch
@@ -103,6 +104,7 @@ class Hub(Mcp, Reviewing, Asking, Messaging, Choosing, Approving, Denying, Reque
touched: Touched
voice_notes: VoiceNotes
replay: Path
history: History
def hosts(self) -> tuple[str, str]:
return f"127.0.0.1:{self.public_port}", f"localhost:{self.public_port}"
@@ -142,6 +144,7 @@ def open_hub(council: Path, pages: Path, public_port: int, state: Path, host_sta
timers=places.timers,
kept=places.kept,
council=council,
history=History(council / queue_live.HISTORY),
pages=pages,
public_port=public_port,
reviews=reviews,
+7 -3
View File
@@ -34,10 +34,14 @@ def read_regular(path: Path, limit: int | None) -> str:
Returns:
The file's text, UTF-8 decoded.
"""
with open(os.open(path, READ_FLAGS), "rb") as handle:
if not stat.S_ISREG(os.fstat(handle.fileno()).st_mode):
fd = os.open(path, READ_FLAGS)
try:
if not stat.S_ISREG(os.fstat(fd).st_mode):
raise OSError(errno.EINVAL, "not a regular file", str(path))
data = handle.read() if limit is None else handle.read(limit + 1)
with open(fd, "rb", closefd=False) as handle:
data = handle.read() if limit is None else handle.read(limit + 1)
finally:
os.close(fd)
if limit is not None and len(data) > limit:
raise OSError(errno.EFBIG, f"larger than {limit} bytes", str(path))
return data.decode("utf-8")
+3 -3
View File
@@ -9,7 +9,7 @@ from .errors import ToolError
from .fields import single
from .lifetimes import edge_times, lifetime, session_gone, subagent_rows
from .names import PSEUDO, SESSION_ID, Directory, title_sightings
from .records import epoch, iso_epoch, read_jsonl
from .records import epoch, iso_epoch
from .register import REGISTER, read_register
from .tree import assign
@@ -48,10 +48,10 @@ class Sources:
tasks = read_register(self.council / REGISTER)
reviews = self.reviews.listed()
scan = assign(self.tree.get(), tasks, read_claims(self.council), reviews)
history = read_jsonl(self.council / "queue-history.jsonl")
history = self.history.rows()
return {**enrich(scan, tasks, reviews, self.touched, history, self.jobs.listed()), "cleanups": self.cleanup_results(), "gates": self.gate_view(scan)}
def schedules(self, claims: list[dict], now: float) -> list[dict]:
beats = {claim["session"]: claim["beat"] for claim in claims if claim["beat"] is not None}
rows = [cron_state(cron, beats, now) for cron in self.transcripts.schedules()]
return rows + timer_rows(self.timers) + queue_rows(self.council)
return rows + timer_rows(self.timers) + queue_rows(self.council, self.history.rows)
+16 -6
View File
@@ -4,6 +4,7 @@ import itertools
import statistics
from .council import HUMAN, agent
from .crons import outcome_of
from .records import epoch, iso_epoch, shown
from .register import ENDED, OPERATOR, parties, register_row, status_of
from .signed import signing
@@ -152,12 +153,21 @@ def task_row(task: str, row: dict, trees: list[dict]) -> dict:
}
def queue_status(entry: dict, found: dict | None) -> str:
if found is not None:
return "done" if found.get("state") == "ended" and found.get("exit") == 0 else "stopped"
return "running" if entry.get("status") == "active" else "awaiting"
def queue_row(entry: dict, created: float, trees: list[dict]) -> dict:
text = {name: shown(entry.get(name))[:TEXT_SHOWN] for name in ("id", "cmd", "kind", "task", "workdir")}
found = outcome_of(entry)
ended = found is not None
granted = entry.get("status") == "active" or ended
return {
"task": f"{QUEUE}{text['id']}",
"title": text["cmd"],
"status": "running" if entry.get("status") == "active" else "awaiting",
"status": queue_status(entry, found),
"worker": agent(entry.get("name"))[:TEXT_SHOWN] or "-",
"responsible": "-",
"accountable": "-",
@@ -167,7 +177,7 @@ def queue_row(entry: dict, created: float, trees: list[dict]) -> dict:
"met": 0,
"total": 0,
"kind": "queue",
"ended": False,
"ended": ended,
"editable": False,
"movable": False,
"prio": None,
@@ -175,13 +185,13 @@ def queue_row(entry: dict, created: float, trees: list[dict]) -> dict:
"wsjf": None,
"value": None,
"points": None,
"start": (epoch(entry.get("granted")) or created) if entry.get("status") == "active" else None,
"end": None,
"start": (epoch(entry.get("granted")) or created) if granted else None,
"end": epoch(found.get("end")) if found is not None else None,
"updated": None,
"after": [],
"base": "-",
"roles": {},
"queue": {"kind": text["kind"], "parent": text["task"], "enqueued": created},
"queue": {"kind": text["kind"], "parent": text["task"], "enqueued": created, "exit": found.get("exit") if found is not None else None},
**located(entry, trees, False),
}
@@ -253,7 +263,7 @@ def queue_chain(queue: list[tuple[dict, float]], tasks: dict[str, dict], agents:
holder = order[name] if isinstance(name, str) and name in order else len(order)
return (not entry.get("hotfix"), place - hours, holder, seq if isinstance(seq, int) and not isinstance(seq, bool) else 0)
active = sorted((pair for pair in queue if pair[0].get("status") == "active"), key=lambda pair: epoch(pair[0].get("granted")) or pair[1])
active = sorted((pair for pair in queue if pair[0].get("status") == "active" and outcome_of(pair[0]) is None), key=lambda pair: epoch(pair[0].get("granted")) or pair[1])
waiting = sorted((pair for pair in queue if pair[0].get("status") == "waiting"), key=waiting_key)
return [(shown(entry.get("id"))[:TEXT_SHOWN], entry.get("status") == "waiting") for entry, _ in [*active, *waiting]]
+2 -2
View File
@@ -85,7 +85,7 @@ class View:
def state(self, begin: float, end: float, now: float, steps: int) -> dict:
tasks = read_register(self.council / REGISTER)
news = read_jsonl(self.council / NEWS)
history = read_jsonl(self.council / "queue-history.jsonl")
history = self.history.rows()
claims = read_claims(self.council)
names = sessions(claims)
directory = self.directory(claims, news, now)
@@ -105,7 +105,7 @@ class View:
people = {node["id"]: {"name": node["label"], "now": directory.now(node["id"]) or node["label"], "names": node["names"]} for node in drawn["nodes"] if node["kind"] != "subagent"}
settings = self.settings.get()
inbox = self.inbox(now, tasks, directory)
queue = queue_entries(self.council)
queue = queue_entries(self.council, self.history.rows)
order = stored_order(self.council / AGENT_ORDER)
rows = task_rows(tasks, queue, self.tree.get())
chain = queue_chain(queue, tasks, order, now)