fix(orchestrator): settle queue jobs whose holder is gone instead of showing them running

The hub read queue.json alone, so an active entry whose hold had exited kept showing as running. client/queue_live.py probes each hold lock as queue.py's held() does (shared non-blocking flock, hard links probed too) and reads the whole queue-history.jsonl, cached by inode, size and mtime; an active entry with a history row started after its enqueue ends with that row's exit, one whose holder is gone becomes gone. The outcome lives in the hub-owned field hub_outcome; queue.py status is never rewritten. resources.py uses the same settled(), plan medians skip the queue kind, and a guard test keeps every client module in the Dockerfile COPY line. Also closes fd leaks in read_history and records.read_regular.

MEDUSA: 11 findings in resources.py (lines 73, 81, 262, 445 x2, 576, 227, 479, 524, 578) and crons.py:125 accepted as false positives after an independent review, operator pick 2026-11-12.

🎯 Fix: 92 %
💥 Break: 5 %
💸 Cost: 100 %
⏱️ Time: 100 %
🛡️ Security: 100 %
🏆 Score: 87
🚦 Gate: ✅ 90/5, cost and time <= 100, security 100 %

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Kevin Veen-BirkenbachandClaude Opus 5.5 committed 2026-11-12 23:09:32 +00:00
1 parent 3418738993
commit 85b79de0d8
15 files changed
+534 -39

No files matched your search

+69 -10
View File
@@ -1,15 +1,23 @@
from __future__ import annotations
import os
import threading
from collections.abc import Callable
from dataclasses import dataclass, field
from pathlib import Path
from .council import agent
from .events import event
from .names import Directory
from .records import epoch, read_json, shown
from .signed import client_module
queue_live = client_module("queue_live", {})
CRON_ACTIVE = 900
TIMER_KEYS = ("OnCalendar", "OnActiveSec", "OnBootSec", "OnStartupSec", "OnUnitActiveSec", "OnUnitInactiveSec")
CRON_LABELS = {"cron": "Cron", "wakeup": "Wakeup", "background": "Background", "timer": "Timer", "queue": "Queue"}
OUTCOME = "hub_outcome"
def cron_state(cron: dict, beats: dict[str, float], now: float) -> dict:
@@ -41,10 +49,52 @@ def timer_rows(folder: Path) -> list[dict]:
return rows
def queue_entries(council: Path) -> list[tuple[dict, float]]:
"""Returns:
Each kvb-org-queue entry of queue.json with its enqueue time; an
entry without one is no entry.
@dataclass
class History:
path: Path
lock: threading.Lock = field(default_factory=threading.Lock)
cached: list[dict] = field(default_factory=list)
stamp: tuple[int, int, int] | None = None
def rows(self) -> list[dict]:
"""Returns:
queue_live.read_history() of the whole file, parsed again only
when its inode, size or mtime changed; after a failed read the
rows read before, and the next call reads again. The list is
shared, read only.
"""
with self.lock:
try:
info = os.lstat(self.path)
except FileNotFoundError:
self.cached, self.stamp = [], None
return self.cached
except OSError:
self.stamp = None
return self.cached
stamp = (info.st_ino, info.st_size, info.st_mtime_ns)
if stamp != self.stamp:
found = queue_live.read_history(self.path)
if found is None:
self.stamp = None
else:
self.cached, self.stamp = found, stamp
return self.cached
def outcome_of(entry: dict) -> dict | None:
found = entry.get(OUTCOME)
return found if isinstance(found, dict) else None
def queue_entries(council: Path, history: Callable[[], list[dict]]) -> list[tuple[dict, float]]:
"""Args:
council: the council directory.
history: returns the queue-history.jsonl rows, History.rows.
Returns:
Each entry of queue.json with an enqueue time that queue_live.settled()
keeps, with that time, as a copy whose OUTCOME is its outcome.
"""
data = read_json(council / "queue.json")
entries = data.get("entries") if isinstance(data, dict) else None
@@ -53,17 +103,26 @@ def queue_entries(council: Path) -> list[tuple[dict, float]]:
created = epoch(entry.get("enqueued")) if isinstance(entry, dict) else None
if created is not None:
found.append((entry, created))
return found
return [({**entry, OUTCOME: result}, created) for entry, created, result in queue_live.settled(council, found, history)]
def queue_rows(council: Path) -> list[dict]:
def queue_state(entry: dict) -> str:
found = outcome_of(entry)
if found is None:
return "running" if entry.get("status") == "active" else "waiting"
if found.get("state") == "gone":
return "holder gone without an exit code"
code = found.get("exit")
return "ended" if code is None else f"exit {code}"
def queue_rows(council: Path, history: Callable[[], list[dict]]) -> list[dict]:
rows = []
for entry, created in queue_entries(council):
running = "running" if entry.get("status") == "active" else "waiting"
for entry, created in queue_entries(council, history):
rows.append({
"key": f"queue:{shown(entry.get('id'))}", "type": "queue", "session": shown(entry.get("session")), "sub": "", "created": created,
"deleted": None, "until": None, "job": shown(entry.get("id")), "expression": f"{shown(entry.get('kind'))} · {running}",
"prompt": shown(entry.get("cmd")), "reason": shown(entry.get("task")), "active": True, "agent": agent(entry.get("name")),
"deleted": None, "until": None, "job": shown(entry.get("id")), "expression": f"{shown(entry.get('kind'))} · {queue_state(entry)}",
"prompt": shown(entry.get("cmd")), "reason": shown(entry.get("task")), "active": outcome_of(entry) is None, "agent": agent(entry.get("name")),
})
return rows