fix(orchestrator): settle queue jobs whose holder is gone instead of showing them running
The hub read queue.json alone, so an active entry whose hold had exited kept showing as running. client/queue_live.py probes each hold lock as queue.py's held() does (shared non-blocking flock, hard links probed too) and reads the whole queue-history.jsonl, cached by inode, size and mtime; an active entry with a history row started after its enqueue ends with that row's exit, one whose holder is gone becomes gone. The outcome lives in the hub-owned field hub_outcome; queue.py status is never rewritten. resources.py uses the same settled(), plan medians skip the queue kind, and a guard test keeps every client module in the Dockerfile COPY line. Also closes fd leaks in read_history and records.read_regular. MEDUSA: 11 findings in resources.py (lines 73, 81, 262, 445 x2, 576, 227, 479, 524, 578) and crons.py:125 accepted as false positives after an independent review, operator pick 2026-11-12. 🎯 Fix: 92 % 💥 Break: 5 % 💸 Cost: 100 % ⏱️ Time: 100 % 🛡️ Security: 100 % 🏆 Score: 87 🚦 Gate: ✅ 90/5, cost and time <= 100, security 100 % Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
3418738993
commit
85b79de0d8
15 files changed
+534
-39
No files matched your search
@@ -1,15 +1,23 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import threading
|
||||
from collections.abc import Callable
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
|
||||
from .council import agent
|
||||
from .events import event
|
||||
from .names import Directory
|
||||
from .records import epoch, read_json, shown
|
||||
from .signed import client_module
|
||||
|
||||
queue_live = client_module("queue_live", {})
|
||||
|
||||
CRON_ACTIVE = 900
|
||||
TIMER_KEYS = ("OnCalendar", "OnActiveSec", "OnBootSec", "OnStartupSec", "OnUnitActiveSec", "OnUnitInactiveSec")
|
||||
CRON_LABELS = {"cron": "Cron", "wakeup": "Wakeup", "background": "Background", "timer": "Timer", "queue": "Queue"}
|
||||
OUTCOME = "hub_outcome"
|
||||
|
||||
|
||||
def cron_state(cron: dict, beats: dict[str, float], now: float) -> dict:
|
||||
@@ -41,10 +49,52 @@ def timer_rows(folder: Path) -> list[dict]:
|
||||
return rows
|
||||
|
||||
|
||||
def queue_entries(council: Path) -> list[tuple[dict, float]]:
|
||||
"""Returns:
|
||||
Each kvb-org-queue entry of queue.json with its enqueue time; an
|
||||
entry without one is no entry.
|
||||
@dataclass
|
||||
class History:
|
||||
path: Path
|
||||
lock: threading.Lock = field(default_factory=threading.Lock)
|
||||
cached: list[dict] = field(default_factory=list)
|
||||
stamp: tuple[int, int, int] | None = None
|
||||
|
||||
def rows(self) -> list[dict]:
|
||||
"""Returns:
|
||||
queue_live.read_history() of the whole file, parsed again only
|
||||
when its inode, size or mtime changed; after a failed read the
|
||||
rows read before, and the next call reads again. The list is
|
||||
shared, read only.
|
||||
"""
|
||||
with self.lock:
|
||||
try:
|
||||
info = os.lstat(self.path)
|
||||
except FileNotFoundError:
|
||||
self.cached, self.stamp = [], None
|
||||
return self.cached
|
||||
except OSError:
|
||||
self.stamp = None
|
||||
return self.cached
|
||||
stamp = (info.st_ino, info.st_size, info.st_mtime_ns)
|
||||
if stamp != self.stamp:
|
||||
found = queue_live.read_history(self.path)
|
||||
if found is None:
|
||||
self.stamp = None
|
||||
else:
|
||||
self.cached, self.stamp = found, stamp
|
||||
return self.cached
|
||||
|
||||
|
||||
def outcome_of(entry: dict) -> dict | None:
|
||||
found = entry.get(OUTCOME)
|
||||
return found if isinstance(found, dict) else None
|
||||
|
||||
|
||||
def queue_entries(council: Path, history: Callable[[], list[dict]]) -> list[tuple[dict, float]]:
|
||||
"""Args:
|
||||
council: the council directory.
|
||||
history: returns the queue-history.jsonl rows, History.rows.
|
||||
|
||||
Returns:
|
||||
Each entry of queue.json with an enqueue time that queue_live.settled()
|
||||
keeps, with that time, as a copy whose OUTCOME is its outcome.
|
||||
"""
|
||||
data = read_json(council / "queue.json")
|
||||
entries = data.get("entries") if isinstance(data, dict) else None
|
||||
@@ -53,17 +103,26 @@ def queue_entries(council: Path) -> list[tuple[dict, float]]:
|
||||
created = epoch(entry.get("enqueued")) if isinstance(entry, dict) else None
|
||||
if created is not None:
|
||||
found.append((entry, created))
|
||||
return found
|
||||
return [({**entry, OUTCOME: result}, created) for entry, created, result in queue_live.settled(council, found, history)]
|
||||
|
||||
|
||||
def queue_rows(council: Path) -> list[dict]:
|
||||
def queue_state(entry: dict) -> str:
|
||||
found = outcome_of(entry)
|
||||
if found is None:
|
||||
return "running" if entry.get("status") == "active" else "waiting"
|
||||
if found.get("state") == "gone":
|
||||
return "holder gone without an exit code"
|
||||
code = found.get("exit")
|
||||
return "ended" if code is None else f"exit {code}"
|
||||
|
||||
|
||||
def queue_rows(council: Path, history: Callable[[], list[dict]]) -> list[dict]:
|
||||
rows = []
|
||||
for entry, created in queue_entries(council):
|
||||
running = "running" if entry.get("status") == "active" else "waiting"
|
||||
for entry, created in queue_entries(council, history):
|
||||
rows.append({
|
||||
"key": f"queue:{shown(entry.get('id'))}", "type": "queue", "session": shown(entry.get("session")), "sub": "", "created": created,
|
||||
"deleted": None, "until": None, "job": shown(entry.get("id")), "expression": f"{shown(entry.get('kind'))} · {running}",
|
||||
"prompt": shown(entry.get("cmd")), "reason": shown(entry.get("task")), "active": True, "agent": agent(entry.get("name")),
|
||||
"deleted": None, "until": None, "job": shown(entry.get("id")), "expression": f"{shown(entry.get('kind'))} · {queue_state(entry)}",
|
||||
"prompt": shown(entry.get("cmd")), "reason": shown(entry.get("task")), "active": outcome_of(entry) is None, "agent": agent(entry.get("name")),
|
||||
})
|
||||
return rows
|
||||
|
||||
|
||||
Reference in new issue
Block a user