fix(orchestrator): settle queue jobs whose holder is gone instead of showing them running

The hub read queue.json alone, so an active entry whose hold had exited kept showing as running. client/queue_live.py probes each hold lock as queue.py's held() does (shared non-blocking flock, hard links probed too) and reads the whole queue-history.jsonl, cached by inode, size and mtime; an active entry with a history row started after its enqueue ends with that row's exit, one whose holder is gone becomes gone. The outcome lives in the hub-owned field hub_outcome; queue.py status is never rewritten. resources.py uses the same settled(), plan medians skip the queue kind, and a guard test keeps every client module in the Dockerfile COPY line. Also closes fd leaks in read_history and records.read_regular.

MEDUSA: 11 findings in resources.py (lines 73, 81, 262, 445 x2, 576, 227, 479, 524, 578) and crons.py:125 accepted as false positives after an independent review, operator pick 2026-11-12.

🎯 Fix: 92 %
💥 Break: 5 %
💸 Cost: 100 %
⏱️ Time: 100 %
🛡️ Security: 100 %
🏆 Score: 87
🚦 Gate: ✅ 90/5, cost and time <= 100, security 100 %

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Kevin Veen-BirkenbachandClaude Opus 5.5 committed 2026-11-12 23:09:32 +00:00
1 parent 3418738993
commit 85b79de0d8
15 files changed
+534 -39

No files matched your search

+1 -1
View File
@@ -36,7 +36,7 @@ RUN chmod 0755 /opt/voices /opt/whisper
COPY server.py /app/server.py
COPY kvb_orchestrator /app/kvb_orchestrator
COPY client/hub_limits.py client/signing.py client/passkey.py client/verifier.py /app/client/
COPY client/hub_limits.py client/signing.py client/passkey.py client/verifier.py client/queue_live.py /app/client/
COPY templates /app/templates
COPY vendor /app/vendor
+181
View File
@@ -0,0 +1,181 @@
from __future__ import annotations
import errno
import fcntl
import json
import os
import re
import stat
from collections.abc import Callable
from pathlib import Path
HISTORY = "queue-history.jsonl"
HOLDS = "holds"
HOLD_TAG = re.compile(r"\d+-\d+")
JOB_SUFFIX = re.compile(r"\d+")
READ_FLAGS = os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK | os.O_CLOEXEC
DIRECTORY = os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW | os.O_CLOEXEC
REFUSED = frozenset({errno.ELOOP, errno.ENOTDIR})
NO_HISTORY = frozenset({errno.ENOENT, errno.ELOOP, errno.ENOTDIR})
def moment(value: object) -> float | None:
if isinstance(value, bool) or not isinstance(value, int | float) or not 0 <= value < 1e11:
return None
return float(value)
def opened(council: Path, tag: str) -> tuple[int | None, bool]:
"""Returns:
(descriptor, gone): the descriptor of holds/<tag>.lock, opened as
queue.py's open_below walks to it, and False; else None and True
where queue.py's held() finds no lock (a missing directory or file)
or open_below refuses one (a link or a non-directory on the way),
None and False where the open fails otherwise.
"""
try:
folder = os.open(council / HOLDS, DIRECTORY)
except FileNotFoundError:
return None, True
except OSError as error:
return None, error.errno in REFUSED
try:
return os.open(f"{tag}.lock", READ_FLAGS, dir_fd=folder), False
except FileNotFoundError:
return None, True
except OSError as error:
return None, error.errno in REFUSED
finally:
os.close(folder)
def holder_gone(council: Path, tag: object) -> bool:
"""Args:
council: the council directory.
tag: the lock tag of an entry that has one; queue.py's living()
decides by held(tag) alone then.
Returns:
not held(tag) for every lock queue.py's held() decides: True for a
tag queue.py would not write, a missing lock and a lock a
non-blocking shared flock gets, False for a held one. A hard-linked
lock gets the same probe. A link or a non-regular file, which no
hold can create, is True; a lock that cannot be opened or flocked
otherwise (EACCES) is False.
"""
if not isinstance(tag, str) or not HOLD_TAG.fullmatch(tag):
return True
fd, gone = opened(council, tag)
if fd is None:
return gone
try:
if not stat.S_ISREG(os.fstat(fd).st_mode):
return True
fcntl.flock(fd, fcntl.LOCK_SH | fcntl.LOCK_NB)
except OSError:
return False
finally:
os.close(fd)
return True
def read_history(path: Path) -> list[dict] | None:
"""Args:
path: a queue-history.jsonl; a missing file, a link or another
non-regular file reads as no rows.
Returns:
The JSON object rows of the whole file, in file order; an undecodable
byte or a bad line costs that line only. None when the read fails
otherwise (EACCES, EIO, EMFILE).
"""
try:
fd = os.open(path, READ_FLAGS)
except OSError as error:
return [] if error.errno in NO_HISTORY else None
try:
if not stat.S_ISREG(os.fstat(fd).st_mode):
return []
with open(fd, "rb", closefd=False) as handle:
data = handle.read()
except OSError:
return None
finally:
os.close(fd)
rows = []
for line in data.decode("utf-8", errors="replace").splitlines():
try:
row = json.loads(line)
except (ValueError, RecursionError):
continue
if isinstance(row, dict):
rows.append(row)
return rows
def history_jobs(rows: list[dict]) -> dict[str, list[dict]]:
"""Returns:
The history rows with a start by entry id, the row id without its
last dash and digits.
"""
jobs: dict[str, list[dict]] = {}
for row in rows:
entry_id, _, suffix = str(row.get("id", "")).rpartition("-")
if entry_id and JOB_SUFFIX.fullmatch(suffix) and moment(row.get("start")) is not None:
jobs.setdefault(entry_id, []).append(row)
return jobs
def exit_of(row: dict) -> int | None:
code = row.get("exit")
return code if isinstance(code, int) and not isinstance(code, bool) else None
def outcome(entry: dict, created: float, gone: bool, jobs: dict[str, list[dict]]) -> tuple[bool, dict | None]:
"""Args:
entry: a queue.json entry.
created: its enqueue time.
gone: holder_gone() of its lock, False without a lock key.
jobs: history_jobs().
Returns:
(kept, outcome): an active entry whose job has a history row started
after the enqueue is kept with {"state": "ended", "exit", "end"} of
the newest such row by end; else an active one whose holder is gone
is kept with {"state": "gone", "exit": None, "end": None}; else any
other entry whose holder is gone is not kept; else the entry is kept
live with outcome None.
"""
active = entry.get("status") == "active"
ident = entry.get("id")
rows = [row for row in jobs.get(ident, []) if (moment(row.get("start")) or 0.0) >= created] if active and isinstance(ident, str) else []
if rows:
row = max(rows, key=lambda found: moment(found.get("end")) or 0.0)
return True, {"state": "ended", "exit": exit_of(row), "end": moment(row.get("end"))}
if gone:
return active, {"state": "gone", "exit": None, "end": None} if active else None
return True, None
def settled(council: Path, found: list[tuple[dict, float]], history: Callable[[], list[dict]]) -> list[tuple[dict, float, dict | None]]:
"""Args:
council: the council directory.
found: the queue.json entries with their enqueue time.
history: returns the queue-history.jsonl rows, None for an
unreadable file; called once, after every hold lock is probed,
and only while an entry is active.
Returns:
Each kept entry with its enqueue time and outcome, as outcome()
reads it, in the order of found; an entry without a lock key, which
queue.py's living() judges by pids of other PID namespaces, is never
gone and ends only by its history row.
"""
gone = ["lock" in entry and holder_gone(council, entry["lock"]) for entry, _ in found]
jobs = history_jobs(history() or []) if any(entry.get("status") == "active" for entry, _ in found) else {}
kept = []
for (entry, created), dead in zip(found, gone, strict=True):
keep, result = outcome(entry, created, dead, jobs)
if keep:
kept.append((entry, created, result))
return kept
+14 -1
View File
@@ -44,6 +44,7 @@ from __future__ import annotations
import argparse
import base64
import datetime
import functools
import hashlib
import json
import os
@@ -55,6 +56,8 @@ import time
from collections.abc import Callable
from pathlib import Path
import queue_live
INTERVAL = 60
BUCKET = 900
HOUR = 3600
@@ -226,9 +229,19 @@ def gpus(sys_root: Path, run: Callable[[list[str]], str | None]) -> list[dict]:
def queue(council: Path) -> dict:
"""Returns:
{waiting, active}: the queue.json entries with an enqueue time that
queue_live.settled() keeps without an outcome, as the hub counts them.
"""
data = read_json(council / "queue.json")
entries = data.get("entries") if isinstance(data, dict) else None
statuses = [entry.get("status") for entry in entries if isinstance(entry, dict)] if isinstance(entries, list) else []
found = []
for entry in entries if isinstance(entries, list) else []:
created = queue_live.moment(entry.get("enqueued")) if isinstance(entry, dict) else None
if created is not None:
found.append((entry, created))
history = functools.partial(queue_live.read_history, council / queue_live.HISTORY)
statuses = [entry.get("status") for entry, _, result in queue_live.settled(council, found, history) if result is None]
return {"waiting": statuses.count("waiting"), "active": statuses.count("active")}
+69 -10
View File
@@ -1,15 +1,23 @@
from __future__ import annotations
import os
import threading
from collections.abc import Callable
from dataclasses import dataclass, field
from pathlib import Path
from .council import agent
from .events import event
from .names import Directory
from .records import epoch, read_json, shown
from .signed import client_module
queue_live = client_module("queue_live", {})
CRON_ACTIVE = 900
TIMER_KEYS = ("OnCalendar", "OnActiveSec", "OnBootSec", "OnStartupSec", "OnUnitActiveSec", "OnUnitInactiveSec")
CRON_LABELS = {"cron": "Cron", "wakeup": "Wakeup", "background": "Background", "timer": "Timer", "queue": "Queue"}
OUTCOME = "hub_outcome"
def cron_state(cron: dict, beats: dict[str, float], now: float) -> dict:
@@ -41,10 +49,52 @@ def timer_rows(folder: Path) -> list[dict]:
return rows
def queue_entries(council: Path) -> list[tuple[dict, float]]:
"""Returns:
Each kvb-org-queue entry of queue.json with its enqueue time; an
entry without one is no entry.
@dataclass
class History:
path: Path
lock: threading.Lock = field(default_factory=threading.Lock)
cached: list[dict] = field(default_factory=list)
stamp: tuple[int, int, int] | None = None
def rows(self) -> list[dict]:
"""Returns:
queue_live.read_history() of the whole file, parsed again only
when its inode, size or mtime changed; after a failed read the
rows read before, and the next call reads again. The list is
shared, read only.
"""
with self.lock:
try:
info = os.lstat(self.path)
except FileNotFoundError:
self.cached, self.stamp = [], None
return self.cached
except OSError:
self.stamp = None
return self.cached
stamp = (info.st_ino, info.st_size, info.st_mtime_ns)
if stamp != self.stamp:
found = queue_live.read_history(self.path)
if found is None:
self.stamp = None
else:
self.cached, self.stamp = found, stamp
return self.cached
def outcome_of(entry: dict) -> dict | None:
found = entry.get(OUTCOME)
return found if isinstance(found, dict) else None
def queue_entries(council: Path, history: Callable[[], list[dict]]) -> list[tuple[dict, float]]:
"""Args:
council: the council directory.
history: returns the queue-history.jsonl rows, History.rows.
Returns:
Each entry of queue.json with an enqueue time that queue_live.settled()
keeps, with that time, as a copy whose OUTCOME is its outcome.
"""
data = read_json(council / "queue.json")
entries = data.get("entries") if isinstance(data, dict) else None
@@ -53,17 +103,26 @@ def queue_entries(council: Path) -> list[tuple[dict, float]]:
created = epoch(entry.get("enqueued")) if isinstance(entry, dict) else None
if created is not None:
found.append((entry, created))
return found
return [({**entry, OUTCOME: result}, created) for entry, created, result in queue_live.settled(council, found, history)]
def queue_rows(council: Path) -> list[dict]:
def queue_state(entry: dict) -> str:
found = outcome_of(entry)
if found is None:
return "running" if entry.get("status") == "active" else "waiting"
if found.get("state") == "gone":
return "holder gone without an exit code"
code = found.get("exit")
return "ended" if code is None else f"exit {code}"
def queue_rows(council: Path, history: Callable[[], list[dict]]) -> list[dict]:
rows = []
for entry, created in queue_entries(council):
running = "running" if entry.get("status") == "active" else "waiting"
for entry, created in queue_entries(council, history):
rows.append({
"key": f"queue:{shown(entry.get('id'))}", "type": "queue", "session": shown(entry.get("session")), "sub": "", "created": created,
"deleted": None, "until": None, "job": shown(entry.get("id")), "expression": f"{shown(entry.get('kind'))} · {running}",
"prompt": shown(entry.get("cmd")), "reason": shown(entry.get("task")), "active": True, "agent": agent(entry.get("name")),
"deleted": None, "until": None, "job": shown(entry.get("id")), "expression": f"{shown(entry.get('kind'))} · {queue_state(entry)}",
"prompt": shown(entry.get("cmd")), "reason": shown(entry.get("task")), "active": outcome_of(entry) is None, "agent": agent(entry.get("name")),
})
return rows
+3
View File
@@ -13,6 +13,7 @@ from .asking import Asking
from .batches import Pending
from .choosing import Choosing
from .cleanup import Cleaning, Spool
from .crons import History, queue_live
from .denying import Denying
from .engines import Engines, Speech
from .events import Log, Watch
@@ -103,6 +104,7 @@ class Hub(Mcp, Reviewing, Asking, Messaging, Choosing, Approving, Denying, Reque
touched: Touched
voice_notes: VoiceNotes
replay: Path
history: History
def hosts(self) -> tuple[str, str]:
return f"127.0.0.1:{self.public_port}", f"localhost:{self.public_port}"
@@ -142,6 +144,7 @@ def open_hub(council: Path, pages: Path, public_port: int, state: Path, host_sta
timers=places.timers,
kept=places.kept,
council=council,
history=History(council / queue_live.HISTORY),
pages=pages,
public_port=public_port,
reviews=reviews,
+7 -3
View File
@@ -34,10 +34,14 @@ def read_regular(path: Path, limit: int | None) -> str:
Returns:
The file's text, UTF-8 decoded.
"""
with open(os.open(path, READ_FLAGS), "rb") as handle:
if not stat.S_ISREG(os.fstat(handle.fileno()).st_mode):
fd = os.open(path, READ_FLAGS)
try:
if not stat.S_ISREG(os.fstat(fd).st_mode):
raise OSError(errno.EINVAL, "not a regular file", str(path))
data = handle.read() if limit is None else handle.read(limit + 1)
with open(fd, "rb", closefd=False) as handle:
data = handle.read() if limit is None else handle.read(limit + 1)
finally:
os.close(fd)
if limit is not None and len(data) > limit:
raise OSError(errno.EFBIG, f"larger than {limit} bytes", str(path))
return data.decode("utf-8")
+3 -3
View File
@@ -9,7 +9,7 @@ from .errors import ToolError
from .fields import single
from .lifetimes import edge_times, lifetime, session_gone, subagent_rows
from .names import PSEUDO, SESSION_ID, Directory, title_sightings
from .records import epoch, iso_epoch, read_jsonl
from .records import epoch, iso_epoch
from .register import REGISTER, read_register
from .tree import assign
@@ -48,10 +48,10 @@ class Sources:
tasks = read_register(self.council / REGISTER)
reviews = self.reviews.listed()
scan = assign(self.tree.get(), tasks, read_claims(self.council), reviews)
history = read_jsonl(self.council / "queue-history.jsonl")
history = self.history.rows()
return {**enrich(scan, tasks, reviews, self.touched, history, self.jobs.listed()), "cleanups": self.cleanup_results(), "gates": self.gate_view(scan)}
def schedules(self, claims: list[dict], now: float) -> list[dict]:
beats = {claim["session"]: claim["beat"] for claim in claims if claim["beat"] is not None}
rows = [cron_state(cron, beats, now) for cron in self.transcripts.schedules()]
return rows + timer_rows(self.timers) + queue_rows(self.council)
return rows + timer_rows(self.timers) + queue_rows(self.council, self.history.rows)
+16 -6
View File
@@ -4,6 +4,7 @@ import itertools
import statistics
from .council import HUMAN, agent
from .crons import outcome_of
from .records import epoch, iso_epoch, shown
from .register import ENDED, OPERATOR, parties, register_row, status_of
from .signed import signing
@@ -152,12 +153,21 @@ def task_row(task: str, row: dict, trees: list[dict]) -> dict:
}
def queue_status(entry: dict, found: dict | None) -> str:
if found is not None:
return "done" if found.get("state") == "ended" and found.get("exit") == 0 else "stopped"
return "running" if entry.get("status") == "active" else "awaiting"
def queue_row(entry: dict, created: float, trees: list[dict]) -> dict:
text = {name: shown(entry.get(name))[:TEXT_SHOWN] for name in ("id", "cmd", "kind", "task", "workdir")}
found = outcome_of(entry)
ended = found is not None
granted = entry.get("status") == "active" or ended
return {
"task": f"{QUEUE}{text['id']}",
"title": text["cmd"],
"status": "running" if entry.get("status") == "active" else "awaiting",
"status": queue_status(entry, found),
"worker": agent(entry.get("name"))[:TEXT_SHOWN] or "-",
"responsible": "-",
"accountable": "-",
@@ -167,7 +177,7 @@ def queue_row(entry: dict, created: float, trees: list[dict]) -> dict:
"met": 0,
"total": 0,
"kind": "queue",
"ended": False,
"ended": ended,
"editable": False,
"movable": False,
"prio": None,
@@ -175,13 +185,13 @@ def queue_row(entry: dict, created: float, trees: list[dict]) -> dict:
"wsjf": None,
"value": None,
"points": None,
"start": (epoch(entry.get("granted")) or created) if entry.get("status") == "active" else None,
"end": None,
"start": (epoch(entry.get("granted")) or created) if granted else None,
"end": epoch(found.get("end")) if found is not None else None,
"updated": None,
"after": [],
"base": "-",
"roles": {},
"queue": {"kind": text["kind"], "parent": text["task"], "enqueued": created},
"queue": {"kind": text["kind"], "parent": text["task"], "enqueued": created, "exit": found.get("exit") if found is not None else None},
**located(entry, trees, False),
}
@@ -253,7 +263,7 @@ def queue_chain(queue: list[tuple[dict, float]], tasks: dict[str, dict], agents:
holder = order[name] if isinstance(name, str) and name in order else len(order)
return (not entry.get("hotfix"), place - hours, holder, seq if isinstance(seq, int) and not isinstance(seq, bool) else 0)
active = sorted((pair for pair in queue if pair[0].get("status") == "active"), key=lambda pair: epoch(pair[0].get("granted")) or pair[1])
active = sorted((pair for pair in queue if pair[0].get("status") == "active" and outcome_of(pair[0]) is None), key=lambda pair: epoch(pair[0].get("granted")) or pair[1])
waiting = sorted((pair for pair in queue if pair[0].get("status") == "waiting"), key=waiting_key)
return [(shown(entry.get("id"))[:TEXT_SHOWN], entry.get("status") == "waiting") for entry, _ in [*active, *waiting]]
+2 -2
View File
@@ -85,7 +85,7 @@ class View:
def state(self, begin: float, end: float, now: float, steps: int) -> dict:
tasks = read_register(self.council / REGISTER)
news = read_jsonl(self.council / NEWS)
history = read_jsonl(self.council / "queue-history.jsonl")
history = self.history.rows()
claims = read_claims(self.council)
names = sessions(claims)
directory = self.directory(claims, news, now)
@@ -105,7 +105,7 @@ class View:
people = {node["id"]: {"name": node["label"], "now": directory.now(node["id"]) or node["label"], "names": node["names"]} for node in drawn["nodes"] if node["kind"] != "subagent"}
settings = self.settings.get()
inbox = self.inbox(now, tasks, directory)
queue = queue_entries(self.council)
queue = queue_entries(self.council, self.history.rows)
order = stored_order(self.council / AGENT_ORDER)
rows = task_rows(tasks, queue, self.tree.get())
chain = queue_chain(queue, tasks, order, now)
+2 -1
View File
@@ -47,7 +47,8 @@
function taskPlace(row) {
if (row.kind === "queue") {
return h("span", {}, `${row.queue.kind} for ${row.queue.parent} · queued `, when(row.queue.enqueued));
const outcome = !row.ended ? null : row.queue.exit !== null ? ` · exit ${row.queue.exit}` : " · ended without an exit code";
return h("span", {}, `${row.queue.kind} for ${row.queue.parent} · queued `, when(row.queue.enqueued), outcome);
}
const place = row.worktree || (row.workdir !== "-" ? row.workdir : "");
return place ? h("code", {title: row.repo || place}, place) : null;
+4 -4
View File
@@ -23,12 +23,12 @@ const Plan = (() => {
}
/**
* @param rows - task rows with status, start, end (epoch seconds or null) and points (median complexity or null).
* @returns {duration, perPoint}: the median seconds a done task took from start to end, and the median seconds
* per point of the done tasks with points; null where no done task gives one.
* @param rows - task rows with kind, status, start, end (epoch seconds or null) and points (median complexity or null).
* @returns {duration, perPoint}: the median seconds a done task, never a queue job, took from start to end, and the
* median seconds per point of the done tasks with points; null where no done task gives one.
*/
function medians(rows) {
const done = rows.filter((row) => row.status === "done" && row.start !== null && row.end !== null && row.end >= row.start);
const done = rows.filter((row) => row.kind !== "queue" && row.status === "done" && row.start !== null && row.end !== null && row.end >= row.start);
return {
duration: median(done.map((row) => row.end - row.start)),
perPoint: median(done.filter((row) => row.points).map((row) => (row.end - row.start) / row.points)),