fix(orchestrator): settle queue jobs whose holder is gone instead of showing them running
The hub read queue.json alone, so an active entry whose hold had exited kept showing as running. client/queue_live.py probes each hold lock as queue.py's held() does (shared non-blocking flock, hard links probed too) and reads the whole queue-history.jsonl, cached by inode, size and mtime; an active entry with a history row started after its enqueue ends with that row's exit, one whose holder is gone becomes gone. The outcome lives in the hub-owned field hub_outcome; queue.py status is never rewritten. resources.py uses the same settled(), plan medians skip the queue kind, and a guard test keeps every client module in the Dockerfile COPY line. Also closes fd leaks in read_history and records.read_regular. MEDUSA: 11 findings in resources.py (lines 73, 81, 262, 445 x2, 576, 227, 479, 524, 578) and crons.py:125 accepted as false positives after an independent review, operator pick 2026-11-12. 🎯 Fix: 92 % 💥 Break: 5 % 💸 Cost: 100 % ⏱️ Time: 100 % 🛡️ Security: 100 % 🏆 Score: 87 🚦 Gate: ✅ 90/5, cost and time <= 100, security 100 % Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
3418738993
commit
85b79de0d8
15 files changed
+534
-39
No files matched your search
@@ -1,15 +1,23 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import threading
|
||||
from collections.abc import Callable
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
|
||||
from .council import agent
|
||||
from .events import event
|
||||
from .names import Directory
|
||||
from .records import epoch, read_json, shown
|
||||
from .signed import client_module
|
||||
|
||||
queue_live = client_module("queue_live", {})
|
||||
|
||||
CRON_ACTIVE = 900
|
||||
TIMER_KEYS = ("OnCalendar", "OnActiveSec", "OnBootSec", "OnStartupSec", "OnUnitActiveSec", "OnUnitInactiveSec")
|
||||
CRON_LABELS = {"cron": "Cron", "wakeup": "Wakeup", "background": "Background", "timer": "Timer", "queue": "Queue"}
|
||||
OUTCOME = "hub_outcome"
|
||||
|
||||
|
||||
def cron_state(cron: dict, beats: dict[str, float], now: float) -> dict:
|
||||
@@ -41,10 +49,52 @@ def timer_rows(folder: Path) -> list[dict]:
|
||||
return rows
|
||||
|
||||
|
||||
def queue_entries(council: Path) -> list[tuple[dict, float]]:
|
||||
"""Returns:
|
||||
Each kvb-org-queue entry of queue.json with its enqueue time; an
|
||||
entry without one is no entry.
|
||||
@dataclass
|
||||
class History:
|
||||
path: Path
|
||||
lock: threading.Lock = field(default_factory=threading.Lock)
|
||||
cached: list[dict] = field(default_factory=list)
|
||||
stamp: tuple[int, int, int] | None = None
|
||||
|
||||
def rows(self) -> list[dict]:
|
||||
"""Returns:
|
||||
queue_live.read_history() of the whole file, parsed again only
|
||||
when its inode, size or mtime changed; after a failed read the
|
||||
rows read before, and the next call reads again. The list is
|
||||
shared, read only.
|
||||
"""
|
||||
with self.lock:
|
||||
try:
|
||||
info = os.lstat(self.path)
|
||||
except FileNotFoundError:
|
||||
self.cached, self.stamp = [], None
|
||||
return self.cached
|
||||
except OSError:
|
||||
self.stamp = None
|
||||
return self.cached
|
||||
stamp = (info.st_ino, info.st_size, info.st_mtime_ns)
|
||||
if stamp != self.stamp:
|
||||
found = queue_live.read_history(self.path)
|
||||
if found is None:
|
||||
self.stamp = None
|
||||
else:
|
||||
self.cached, self.stamp = found, stamp
|
||||
return self.cached
|
||||
|
||||
|
||||
def outcome_of(entry: dict) -> dict | None:
|
||||
found = entry.get(OUTCOME)
|
||||
return found if isinstance(found, dict) else None
|
||||
|
||||
|
||||
def queue_entries(council: Path, history: Callable[[], list[dict]]) -> list[tuple[dict, float]]:
|
||||
"""Args:
|
||||
council: the council directory.
|
||||
history: returns the queue-history.jsonl rows, History.rows.
|
||||
|
||||
Returns:
|
||||
Each entry of queue.json with an enqueue time that queue_live.settled()
|
||||
keeps, with that time, as a copy whose OUTCOME is its outcome.
|
||||
"""
|
||||
data = read_json(council / "queue.json")
|
||||
entries = data.get("entries") if isinstance(data, dict) else None
|
||||
@@ -53,17 +103,26 @@ def queue_entries(council: Path) -> list[tuple[dict, float]]:
|
||||
created = epoch(entry.get("enqueued")) if isinstance(entry, dict) else None
|
||||
if created is not None:
|
||||
found.append((entry, created))
|
||||
return found
|
||||
return [({**entry, OUTCOME: result}, created) for entry, created, result in queue_live.settled(council, found, history)]
|
||||
|
||||
|
||||
def queue_rows(council: Path) -> list[dict]:
|
||||
def queue_state(entry: dict) -> str:
|
||||
found = outcome_of(entry)
|
||||
if found is None:
|
||||
return "running" if entry.get("status") == "active" else "waiting"
|
||||
if found.get("state") == "gone":
|
||||
return "holder gone without an exit code"
|
||||
code = found.get("exit")
|
||||
return "ended" if code is None else f"exit {code}"
|
||||
|
||||
|
||||
def queue_rows(council: Path, history: Callable[[], list[dict]]) -> list[dict]:
|
||||
rows = []
|
||||
for entry, created in queue_entries(council):
|
||||
running = "running" if entry.get("status") == "active" else "waiting"
|
||||
for entry, created in queue_entries(council, history):
|
||||
rows.append({
|
||||
"key": f"queue:{shown(entry.get('id'))}", "type": "queue", "session": shown(entry.get("session")), "sub": "", "created": created,
|
||||
"deleted": None, "until": None, "job": shown(entry.get("id")), "expression": f"{shown(entry.get('kind'))} · {running}",
|
||||
"prompt": shown(entry.get("cmd")), "reason": shown(entry.get("task")), "active": True, "agent": agent(entry.get("name")),
|
||||
"deleted": None, "until": None, "job": shown(entry.get("id")), "expression": f"{shown(entry.get('kind'))} · {queue_state(entry)}",
|
||||
"prompt": shown(entry.get("cmd")), "reason": shown(entry.get("task")), "active": outcome_of(entry) is None, "agent": agent(entry.get("name")),
|
||||
})
|
||||
return rows
|
||||
|
||||
|
||||
@@ -13,6 +13,7 @@ from .asking import Asking
|
||||
from .batches import Pending
|
||||
from .choosing import Choosing
|
||||
from .cleanup import Cleaning, Spool
|
||||
from .crons import History, queue_live
|
||||
from .denying import Denying
|
||||
from .engines import Engines, Speech
|
||||
from .events import Log, Watch
|
||||
@@ -103,6 +104,7 @@ class Hub(Mcp, Reviewing, Asking, Messaging, Choosing, Approving, Denying, Reque
|
||||
touched: Touched
|
||||
voice_notes: VoiceNotes
|
||||
replay: Path
|
||||
history: History
|
||||
|
||||
def hosts(self) -> tuple[str, str]:
|
||||
return f"127.0.0.1:{self.public_port}", f"localhost:{self.public_port}"
|
||||
@@ -142,6 +144,7 @@ def open_hub(council: Path, pages: Path, public_port: int, state: Path, host_sta
|
||||
timers=places.timers,
|
||||
kept=places.kept,
|
||||
council=council,
|
||||
history=History(council / queue_live.HISTORY),
|
||||
pages=pages,
|
||||
public_port=public_port,
|
||||
reviews=reviews,
|
||||
|
||||
@@ -34,10 +34,14 @@ def read_regular(path: Path, limit: int | None) -> str:
|
||||
Returns:
|
||||
The file's text, UTF-8 decoded.
|
||||
"""
|
||||
with open(os.open(path, READ_FLAGS), "rb") as handle:
|
||||
if not stat.S_ISREG(os.fstat(handle.fileno()).st_mode):
|
||||
fd = os.open(path, READ_FLAGS)
|
||||
try:
|
||||
if not stat.S_ISREG(os.fstat(fd).st_mode):
|
||||
raise OSError(errno.EINVAL, "not a regular file", str(path))
|
||||
data = handle.read() if limit is None else handle.read(limit + 1)
|
||||
with open(fd, "rb", closefd=False) as handle:
|
||||
data = handle.read() if limit is None else handle.read(limit + 1)
|
||||
finally:
|
||||
os.close(fd)
|
||||
if limit is not None and len(data) > limit:
|
||||
raise OSError(errno.EFBIG, f"larger than {limit} bytes", str(path))
|
||||
return data.decode("utf-8")
|
||||
|
||||
@@ -9,7 +9,7 @@ from .errors import ToolError
|
||||
from .fields import single
|
||||
from .lifetimes import edge_times, lifetime, session_gone, subagent_rows
|
||||
from .names import PSEUDO, SESSION_ID, Directory, title_sightings
|
||||
from .records import epoch, iso_epoch, read_jsonl
|
||||
from .records import epoch, iso_epoch
|
||||
from .register import REGISTER, read_register
|
||||
from .tree import assign
|
||||
|
||||
@@ -48,10 +48,10 @@ class Sources:
|
||||
tasks = read_register(self.council / REGISTER)
|
||||
reviews = self.reviews.listed()
|
||||
scan = assign(self.tree.get(), tasks, read_claims(self.council), reviews)
|
||||
history = read_jsonl(self.council / "queue-history.jsonl")
|
||||
history = self.history.rows()
|
||||
return {**enrich(scan, tasks, reviews, self.touched, history, self.jobs.listed()), "cleanups": self.cleanup_results(), "gates": self.gate_view(scan)}
|
||||
|
||||
def schedules(self, claims: list[dict], now: float) -> list[dict]:
|
||||
beats = {claim["session"]: claim["beat"] for claim in claims if claim["beat"] is not None}
|
||||
rows = [cron_state(cron, beats, now) for cron in self.transcripts.schedules()]
|
||||
return rows + timer_rows(self.timers) + queue_rows(self.council)
|
||||
return rows + timer_rows(self.timers) + queue_rows(self.council, self.history.rows)
|
||||
@@ -4,6 +4,7 @@ import itertools
|
||||
import statistics
|
||||
|
||||
from .council import HUMAN, agent
|
||||
from .crons import outcome_of
|
||||
from .records import epoch, iso_epoch, shown
|
||||
from .register import ENDED, OPERATOR, parties, register_row, status_of
|
||||
from .signed import signing
|
||||
@@ -152,12 +153,21 @@ def task_row(task: str, row: dict, trees: list[dict]) -> dict:
|
||||
}
|
||||
|
||||
|
||||
def queue_status(entry: dict, found: dict | None) -> str:
|
||||
if found is not None:
|
||||
return "done" if found.get("state") == "ended" and found.get("exit") == 0 else "stopped"
|
||||
return "running" if entry.get("status") == "active" else "awaiting"
|
||||
|
||||
|
||||
def queue_row(entry: dict, created: float, trees: list[dict]) -> dict:
|
||||
text = {name: shown(entry.get(name))[:TEXT_SHOWN] for name in ("id", "cmd", "kind", "task", "workdir")}
|
||||
found = outcome_of(entry)
|
||||
ended = found is not None
|
||||
granted = entry.get("status") == "active" or ended
|
||||
return {
|
||||
"task": f"{QUEUE}{text['id']}",
|
||||
"title": text["cmd"],
|
||||
"status": "running" if entry.get("status") == "active" else "awaiting",
|
||||
"status": queue_status(entry, found),
|
||||
"worker": agent(entry.get("name"))[:TEXT_SHOWN] or "-",
|
||||
"responsible": "-",
|
||||
"accountable": "-",
|
||||
@@ -167,7 +177,7 @@ def queue_row(entry: dict, created: float, trees: list[dict]) -> dict:
|
||||
"met": 0,
|
||||
"total": 0,
|
||||
"kind": "queue",
|
||||
"ended": False,
|
||||
"ended": ended,
|
||||
"editable": False,
|
||||
"movable": False,
|
||||
"prio": None,
|
||||
@@ -175,13 +185,13 @@ def queue_row(entry: dict, created: float, trees: list[dict]) -> dict:
|
||||
"wsjf": None,
|
||||
"value": None,
|
||||
"points": None,
|
||||
"start": (epoch(entry.get("granted")) or created) if entry.get("status") == "active" else None,
|
||||
"end": None,
|
||||
"start": (epoch(entry.get("granted")) or created) if granted else None,
|
||||
"end": epoch(found.get("end")) if found is not None else None,
|
||||
"updated": None,
|
||||
"after": [],
|
||||
"base": "-",
|
||||
"roles": {},
|
||||
"queue": {"kind": text["kind"], "parent": text["task"], "enqueued": created},
|
||||
"queue": {"kind": text["kind"], "parent": text["task"], "enqueued": created, "exit": found.get("exit") if found is not None else None},
|
||||
**located(entry, trees, False),
|
||||
}
|
||||
|
||||
@@ -253,7 +263,7 @@ def queue_chain(queue: list[tuple[dict, float]], tasks: dict[str, dict], agents:
|
||||
holder = order[name] if isinstance(name, str) and name in order else len(order)
|
||||
return (not entry.get("hotfix"), place - hours, holder, seq if isinstance(seq, int) and not isinstance(seq, bool) else 0)
|
||||
|
||||
active = sorted((pair for pair in queue if pair[0].get("status") == "active"), key=lambda pair: epoch(pair[0].get("granted")) or pair[1])
|
||||
active = sorted((pair for pair in queue if pair[0].get("status") == "active" and outcome_of(pair[0]) is None), key=lambda pair: epoch(pair[0].get("granted")) or pair[1])
|
||||
waiting = sorted((pair for pair in queue if pair[0].get("status") == "waiting"), key=waiting_key)
|
||||
return [(shown(entry.get("id"))[:TEXT_SHOWN], entry.get("status") == "waiting") for entry, _ in [*active, *waiting]]
|
||||
|
||||
|
||||
@@ -85,7 +85,7 @@ class View:
|
||||
def state(self, begin: float, end: float, now: float, steps: int) -> dict:
|
||||
tasks = read_register(self.council / REGISTER)
|
||||
news = read_jsonl(self.council / NEWS)
|
||||
history = read_jsonl(self.council / "queue-history.jsonl")
|
||||
history = self.history.rows()
|
||||
claims = read_claims(self.council)
|
||||
names = sessions(claims)
|
||||
directory = self.directory(claims, news, now)
|
||||
@@ -105,7 +105,7 @@ class View:
|
||||
people = {node["id"]: {"name": node["label"], "now": directory.now(node["id"]) or node["label"], "names": node["names"]} for node in drawn["nodes"] if node["kind"] != "subagent"}
|
||||
settings = self.settings.get()
|
||||
inbox = self.inbox(now, tasks, directory)
|
||||
queue = queue_entries(self.council)
|
||||
queue = queue_entries(self.council, self.history.rows)
|
||||
order = stored_order(self.council / AGENT_ORDER)
|
||||
rows = task_rows(tasks, queue, self.tree.get())
|
||||
chain = queue_chain(queue, tasks, order, now)
|
||||
|
||||
Reference in new issue
Block a user