fix(orchestrator): settle queue jobs whose holder is gone instead of showing them running

The hub read queue.json alone, so an active entry whose hold had exited kept showing as running. client/queue_live.py probes each hold lock as queue.py's held() does (shared non-blocking flock, hard links probed too) and reads the whole queue-history.jsonl, cached by inode, size and mtime; an active entry with a history row started after its enqueue ends with that row's exit, one whose holder is gone becomes gone. The outcome lives in the hub-owned field hub_outcome; queue.py status is never rewritten. resources.py uses the same settled(), plan medians skip the queue kind, and a guard test keeps every client module in the Dockerfile COPY line. Also closes fd leaks in read_history and records.read_regular.

MEDUSA: 11 findings in resources.py (lines 73, 81, 262, 445 x2, 576, 227, 479, 524, 578) and crons.py:125 accepted as false positives after an independent review, operator pick 2026-11-12.

🎯 Fix: 92 %
💥 Break: 5 %
💸 Cost: 100 %
⏱️ Time: 100 %
🛡️ Security: 100 %
🏆 Score: 87
🚦 Gate: ✅ 90/5, cost and time <= 100, security 100 %

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Kevin Veen-BirkenbachandClaude Opus 5.5 committed 2026-11-12 23:09:32 +00:00
1 parent 3418738993
commit 85b79de0d8
15 files changed
+534 -39

No files matched your search

+16 -6
View File
@@ -4,6 +4,7 @@ import itertools
import statistics
from .council import HUMAN, agent
from .crons import outcome_of
from .records import epoch, iso_epoch, shown
from .register import ENDED, OPERATOR, parties, register_row, status_of
from .signed import signing
@@ -152,12 +153,21 @@ def task_row(task: str, row: dict, trees: list[dict]) -> dict:
}
def queue_status(entry: dict, found: dict | None) -> str:
if found is not None:
return "done" if found.get("state") == "ended" and found.get("exit") == 0 else "stopped"
return "running" if entry.get("status") == "active" else "awaiting"
def queue_row(entry: dict, created: float, trees: list[dict]) -> dict:
text = {name: shown(entry.get(name))[:TEXT_SHOWN] for name in ("id", "cmd", "kind", "task", "workdir")}
found = outcome_of(entry)
ended = found is not None
granted = entry.get("status") == "active" or ended
return {
"task": f"{QUEUE}{text['id']}",
"title": text["cmd"],
"status": "running" if entry.get("status") == "active" else "awaiting",
"status": queue_status(entry, found),
"worker": agent(entry.get("name"))[:TEXT_SHOWN] or "-",
"responsible": "-",
"accountable": "-",
@@ -167,7 +177,7 @@ def queue_row(entry: dict, created: float, trees: list[dict]) -> dict:
"met": 0,
"total": 0,
"kind": "queue",
"ended": False,
"ended": ended,
"editable": False,
"movable": False,
"prio": None,
@@ -175,13 +185,13 @@ def queue_row(entry: dict, created: float, trees: list[dict]) -> dict:
"wsjf": None,
"value": None,
"points": None,
"start": (epoch(entry.get("granted")) or created) if entry.get("status") == "active" else None,
"end": None,
"start": (epoch(entry.get("granted")) or created) if granted else None,
"end": epoch(found.get("end")) if found is not None else None,
"updated": None,
"after": [],
"base": "-",
"roles": {},
"queue": {"kind": text["kind"], "parent": text["task"], "enqueued": created},
"queue": {"kind": text["kind"], "parent": text["task"], "enqueued": created, "exit": found.get("exit") if found is not None else None},
**located(entry, trees, False),
}
@@ -253,7 +263,7 @@ def queue_chain(queue: list[tuple[dict, float]], tasks: dict[str, dict], agents:
holder = order[name] if isinstance(name, str) and name in order else len(order)
return (not entry.get("hotfix"), place - hours, holder, seq if isinstance(seq, int) and not isinstance(seq, bool) else 0)
active = sorted((pair for pair in queue if pair[0].get("status") == "active"), key=lambda pair: epoch(pair[0].get("granted")) or pair[1])
active = sorted((pair for pair in queue if pair[0].get("status") == "active" and outcome_of(pair[0]) is None), key=lambda pair: epoch(pair[0].get("granted")) or pair[1])
waiting = sorted((pair for pair in queue if pair[0].get("status") == "waiting"), key=waiting_key)
return [(shown(entry.get("id"))[:TEXT_SHOWN], entry.get("status") == "waiting") for entry, _ in [*active, *waiting]]