fix(orchestrator): settle queue jobs whose holder is gone instead of showing them running

The hub read queue.json alone, so an active entry whose hold had exited kept showing as running. client/queue_live.py probes each hold lock as queue.py's held() does (shared non-blocking flock, hard links probed too) and reads the whole queue-history.jsonl, cached by inode, size and mtime; an active entry with a history row started after its enqueue ends with that row's exit, one whose holder is gone becomes gone. The outcome lives in the hub-owned field hub_outcome; queue.py status is never rewritten. resources.py uses the same settled(), plan medians skip the queue kind, and a guard test keeps every client module in the Dockerfile COPY line. Also closes fd leaks in read_history and records.read_regular.

MEDUSA: 11 findings in resources.py (lines 73, 81, 262, 445 x2, 576, 227, 479, 524, 578) and crons.py:125 accepted as false positives after an independent review, operator pick 2026-11-12.

🎯 Fix: 92 %
💥 Break: 5 %
💸 Cost: 100 %
⏱️ Time: 100 %
🛡️ Security: 100 %
🏆 Score: 87
🚦 Gate: ✅ 90/5, cost and time <= 100, security 100 %

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Kevin Veen-BirkenbachandClaude Opus 5.5 committed 2026-11-12 23:09:32 +00:00
1 parent 3418738993
commit 85b79de0d8
15 files changed
+534 -39

No files matched your search

+1 -1
View File
@@ -36,7 +36,7 @@ RUN chmod 0755 /opt/voices /opt/whisper
COPY server.py /app/server.py COPY server.py /app/server.py
COPY kvb_orchestrator /app/kvb_orchestrator COPY kvb_orchestrator /app/kvb_orchestrator
COPY client/hub_limits.py client/signing.py client/passkey.py client/verifier.py /app/client/ COPY client/hub_limits.py client/signing.py client/passkey.py client/verifier.py client/queue_live.py /app/client/
COPY templates /app/templates COPY templates /app/templates
COPY vendor /app/vendor COPY vendor /app/vendor
+181
View File
@@ -0,0 +1,181 @@
from __future__ import annotations
import errno
import fcntl
import json
import os
import re
import stat
from collections.abc import Callable
from pathlib import Path
HISTORY = "queue-history.jsonl"
HOLDS = "holds"
HOLD_TAG = re.compile(r"\d+-\d+")
JOB_SUFFIX = re.compile(r"\d+")
READ_FLAGS = os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK | os.O_CLOEXEC
DIRECTORY = os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW | os.O_CLOEXEC
REFUSED = frozenset({errno.ELOOP, errno.ENOTDIR})
NO_HISTORY = frozenset({errno.ENOENT, errno.ELOOP, errno.ENOTDIR})
def moment(value: object) -> float | None:
if isinstance(value, bool) or not isinstance(value, int | float) or not 0 <= value < 1e11:
return None
return float(value)
def opened(council: Path, tag: str) -> tuple[int | None, bool]:
"""Returns:
(descriptor, gone): the descriptor of holds/<tag>.lock, opened as
queue.py's open_below walks to it, and False; else None and True
where queue.py's held() finds no lock (a missing directory or file)
or open_below refuses one (a link or a non-directory on the way),
None and False where the open fails otherwise.
"""
try:
folder = os.open(council / HOLDS, DIRECTORY)
except FileNotFoundError:
return None, True
except OSError as error:
return None, error.errno in REFUSED
try:
return os.open(f"{tag}.lock", READ_FLAGS, dir_fd=folder), False
except FileNotFoundError:
return None, True
except OSError as error:
return None, error.errno in REFUSED
finally:
os.close(folder)
def holder_gone(council: Path, tag: object) -> bool:
"""Args:
council: the council directory.
tag: the lock tag of an entry that has one; queue.py's living()
decides by held(tag) alone then.
Returns:
not held(tag) for every lock queue.py's held() decides: True for a
tag queue.py would not write, a missing lock and a lock a
non-blocking shared flock gets, False for a held one. A hard-linked
lock gets the same probe. A link or a non-regular file, which no
hold can create, is True; a lock that cannot be opened or flocked
otherwise (EACCES) is False.
"""
if not isinstance(tag, str) or not HOLD_TAG.fullmatch(tag):
return True
fd, gone = opened(council, tag)
if fd is None:
return gone
try:
if not stat.S_ISREG(os.fstat(fd).st_mode):
return True
fcntl.flock(fd, fcntl.LOCK_SH | fcntl.LOCK_NB)
except OSError:
return False
finally:
os.close(fd)
return True
def read_history(path: Path) -> list[dict] | None:
"""Args:
path: a queue-history.jsonl; a missing file, a link or another
non-regular file reads as no rows.
Returns:
The JSON object rows of the whole file, in file order; an undecodable
byte or a bad line costs that line only. None when the read fails
otherwise (EACCES, EIO, EMFILE).
"""
try:
fd = os.open(path, READ_FLAGS)
except OSError as error:
return [] if error.errno in NO_HISTORY else None
try:
if not stat.S_ISREG(os.fstat(fd).st_mode):
return []
with open(fd, "rb", closefd=False) as handle:
data = handle.read()
except OSError:
return None
finally:
os.close(fd)
rows = []
for line in data.decode("utf-8", errors="replace").splitlines():
try:
row = json.loads(line)
except (ValueError, RecursionError):
continue
if isinstance(row, dict):
rows.append(row)
return rows
def history_jobs(rows: list[dict]) -> dict[str, list[dict]]:
"""Returns:
The history rows with a start by entry id, the row id without its
last dash and digits.
"""
jobs: dict[str, list[dict]] = {}
for row in rows:
entry_id, _, suffix = str(row.get("id", "")).rpartition("-")
if entry_id and JOB_SUFFIX.fullmatch(suffix) and moment(row.get("start")) is not None:
jobs.setdefault(entry_id, []).append(row)
return jobs
def exit_of(row: dict) -> int | None:
code = row.get("exit")
return code if isinstance(code, int) and not isinstance(code, bool) else None
def outcome(entry: dict, created: float, gone: bool, jobs: dict[str, list[dict]]) -> tuple[bool, dict | None]:
"""Args:
entry: a queue.json entry.
created: its enqueue time.
gone: holder_gone() of its lock, False without a lock key.
jobs: history_jobs().
Returns:
(kept, outcome): an active entry whose job has a history row started
after the enqueue is kept with {"state": "ended", "exit", "end"} of
the newest such row by end; else an active one whose holder is gone
is kept with {"state": "gone", "exit": None, "end": None}; else any
other entry whose holder is gone is not kept; else the entry is kept
live with outcome None.
"""
active = entry.get("status") == "active"
ident = entry.get("id")
rows = [row for row in jobs.get(ident, []) if (moment(row.get("start")) or 0.0) >= created] if active and isinstance(ident, str) else []
if rows:
row = max(rows, key=lambda found: moment(found.get("end")) or 0.0)
return True, {"state": "ended", "exit": exit_of(row), "end": moment(row.get("end"))}
if gone:
return active, {"state": "gone", "exit": None, "end": None} if active else None
return True, None
def settled(council: Path, found: list[tuple[dict, float]], history: Callable[[], list[dict]]) -> list[tuple[dict, float, dict | None]]:
"""Args:
council: the council directory.
found: the queue.json entries with their enqueue time.
history: returns the queue-history.jsonl rows, None for an
unreadable file; called once, after every hold lock is probed,
and only while an entry is active.
Returns:
Each kept entry with its enqueue time and outcome, as outcome()
reads it, in the order of found; an entry without a lock key, which
queue.py's living() judges by pids of other PID namespaces, is never
gone and ends only by its history row.
"""
gone = ["lock" in entry and holder_gone(council, entry["lock"]) for entry, _ in found]
jobs = history_jobs(history() or []) if any(entry.get("status") == "active" for entry, _ in found) else {}
kept = []
for (entry, created), dead in zip(found, gone, strict=True):
keep, result = outcome(entry, created, dead, jobs)
if keep:
kept.append((entry, created, result))
return kept
+14 -1
View File
@@ -44,6 +44,7 @@ from __future__ import annotations
import argparse import argparse
import base64 import base64
import datetime import datetime
import functools
import hashlib import hashlib
import json import json
import os import os
@@ -55,6 +56,8 @@ import time
from collections.abc import Callable from collections.abc import Callable
from pathlib import Path from pathlib import Path
import queue_live
INTERVAL = 60 INTERVAL = 60
BUCKET = 900 BUCKET = 900
HOUR = 3600 HOUR = 3600
@@ -226,9 +229,19 @@ def gpus(sys_root: Path, run: Callable[[list[str]], str | None]) -> list[dict]:
def queue(council: Path) -> dict: def queue(council: Path) -> dict:
"""Returns:
{waiting, active}: the queue.json entries with an enqueue time that
queue_live.settled() keeps without an outcome, as the hub counts them.
"""
data = read_json(council / "queue.json") data = read_json(council / "queue.json")
entries = data.get("entries") if isinstance(data, dict) else None entries = data.get("entries") if isinstance(data, dict) else None
statuses = [entry.get("status") for entry in entries if isinstance(entry, dict)] if isinstance(entries, list) else [] found = []
for entry in entries if isinstance(entries, list) else []:
created = queue_live.moment(entry.get("enqueued")) if isinstance(entry, dict) else None
if created is not None:
found.append((entry, created))
history = functools.partial(queue_live.read_history, council / queue_live.HISTORY)
statuses = [entry.get("status") for entry, _, result in queue_live.settled(council, found, history) if result is None]
return {"waiting": statuses.count("waiting"), "active": statuses.count("active")} return {"waiting": statuses.count("waiting"), "active": statuses.count("active")}
+69 -10
View File
@@ -1,15 +1,23 @@
from __future__ import annotations from __future__ import annotations
import os
import threading
from collections.abc import Callable
from dataclasses import dataclass, field
from pathlib import Path from pathlib import Path
from .council import agent from .council import agent
from .events import event from .events import event
from .names import Directory from .names import Directory
from .records import epoch, read_json, shown from .records import epoch, read_json, shown
from .signed import client_module
queue_live = client_module("queue_live", {})
CRON_ACTIVE = 900 CRON_ACTIVE = 900
TIMER_KEYS = ("OnCalendar", "OnActiveSec", "OnBootSec", "OnStartupSec", "OnUnitActiveSec", "OnUnitInactiveSec") TIMER_KEYS = ("OnCalendar", "OnActiveSec", "OnBootSec", "OnStartupSec", "OnUnitActiveSec", "OnUnitInactiveSec")
CRON_LABELS = {"cron": "Cron", "wakeup": "Wakeup", "background": "Background", "timer": "Timer", "queue": "Queue"} CRON_LABELS = {"cron": "Cron", "wakeup": "Wakeup", "background": "Background", "timer": "Timer", "queue": "Queue"}
OUTCOME = "hub_outcome"
def cron_state(cron: dict, beats: dict[str, float], now: float) -> dict: def cron_state(cron: dict, beats: dict[str, float], now: float) -> dict:
@@ -41,10 +49,52 @@ def timer_rows(folder: Path) -> list[dict]:
return rows return rows
def queue_entries(council: Path) -> list[tuple[dict, float]]: @dataclass
"""Returns: class History:
Each kvb-org-queue entry of queue.json with its enqueue time; an path: Path
entry without one is no entry. lock: threading.Lock = field(default_factory=threading.Lock)
cached: list[dict] = field(default_factory=list)
stamp: tuple[int, int, int] | None = None
def rows(self) -> list[dict]:
"""Returns:
queue_live.read_history() of the whole file, parsed again only
when its inode, size or mtime changed; after a failed read the
rows read before, and the next call reads again. The list is
shared, read only.
"""
with self.lock:
try:
info = os.lstat(self.path)
except FileNotFoundError:
self.cached, self.stamp = [], None
return self.cached
except OSError:
self.stamp = None
return self.cached
stamp = (info.st_ino, info.st_size, info.st_mtime_ns)
if stamp != self.stamp:
found = queue_live.read_history(self.path)
if found is None:
self.stamp = None
else:
self.cached, self.stamp = found, stamp
return self.cached
def outcome_of(entry: dict) -> dict | None:
found = entry.get(OUTCOME)
return found if isinstance(found, dict) else None
def queue_entries(council: Path, history: Callable[[], list[dict]]) -> list[tuple[dict, float]]:
"""Args:
council: the council directory.
history: returns the queue-history.jsonl rows, History.rows.
Returns:
Each entry of queue.json with an enqueue time that queue_live.settled()
keeps, with that time, as a copy whose OUTCOME is its outcome.
""" """
data = read_json(council / "queue.json") data = read_json(council / "queue.json")
entries = data.get("entries") if isinstance(data, dict) else None entries = data.get("entries") if isinstance(data, dict) else None
@@ -53,17 +103,26 @@ def queue_entries(council: Path) -> list[tuple[dict, float]]:
created = epoch(entry.get("enqueued")) if isinstance(entry, dict) else None created = epoch(entry.get("enqueued")) if isinstance(entry, dict) else None
if created is not None: if created is not None:
found.append((entry, created)) found.append((entry, created))
return found return [({**entry, OUTCOME: result}, created) for entry, created, result in queue_live.settled(council, found, history)]
def queue_rows(council: Path) -> list[dict]: def queue_state(entry: dict) -> str:
found = outcome_of(entry)
if found is None:
return "running" if entry.get("status") == "active" else "waiting"
if found.get("state") == "gone":
return "holder gone without an exit code"
code = found.get("exit")
return "ended" if code is None else f"exit {code}"
def queue_rows(council: Path, history: Callable[[], list[dict]]) -> list[dict]:
rows = [] rows = []
for entry, created in queue_entries(council): for entry, created in queue_entries(council, history):
running = "running" if entry.get("status") == "active" else "waiting"
rows.append({ rows.append({
"key": f"queue:{shown(entry.get('id'))}", "type": "queue", "session": shown(entry.get("session")), "sub": "", "created": created, "key": f"queue:{shown(entry.get('id'))}", "type": "queue", "session": shown(entry.get("session")), "sub": "", "created": created,
"deleted": None, "until": None, "job": shown(entry.get("id")), "expression": f"{shown(entry.get('kind'))} · {running}", "deleted": None, "until": None, "job": shown(entry.get("id")), "expression": f"{shown(entry.get('kind'))} · {queue_state(entry)}",
"prompt": shown(entry.get("cmd")), "reason": shown(entry.get("task")), "active": True, "agent": agent(entry.get("name")), "prompt": shown(entry.get("cmd")), "reason": shown(entry.get("task")), "active": outcome_of(entry) is None, "agent": agent(entry.get("name")),
}) })
return rows return rows
+3
View File
@@ -13,6 +13,7 @@ from .asking import Asking
from .batches import Pending from .batches import Pending
from .choosing import Choosing from .choosing import Choosing
from .cleanup import Cleaning, Spool from .cleanup import Cleaning, Spool
from .crons import History, queue_live
from .denying import Denying from .denying import Denying
from .engines import Engines, Speech from .engines import Engines, Speech
from .events import Log, Watch from .events import Log, Watch
@@ -103,6 +104,7 @@ class Hub(Mcp, Reviewing, Asking, Messaging, Choosing, Approving, Denying, Reque
touched: Touched touched: Touched
voice_notes: VoiceNotes voice_notes: VoiceNotes
replay: Path replay: Path
history: History
def hosts(self) -> tuple[str, str]: def hosts(self) -> tuple[str, str]:
return f"127.0.0.1:{self.public_port}", f"localhost:{self.public_port}" return f"127.0.0.1:{self.public_port}", f"localhost:{self.public_port}"
@@ -142,6 +144,7 @@ def open_hub(council: Path, pages: Path, public_port: int, state: Path, host_sta
timers=places.timers, timers=places.timers,
kept=places.kept, kept=places.kept,
council=council, council=council,
history=History(council / queue_live.HISTORY),
pages=pages, pages=pages,
public_port=public_port, public_port=public_port,
reviews=reviews, reviews=reviews,
+7 -3
View File
@@ -34,10 +34,14 @@ def read_regular(path: Path, limit: int | None) -> str:
Returns: Returns:
The file's text, UTF-8 decoded. The file's text, UTF-8 decoded.
""" """
with open(os.open(path, READ_FLAGS), "rb") as handle: fd = os.open(path, READ_FLAGS)
if not stat.S_ISREG(os.fstat(handle.fileno()).st_mode): try:
if not stat.S_ISREG(os.fstat(fd).st_mode):
raise OSError(errno.EINVAL, "not a regular file", str(path)) raise OSError(errno.EINVAL, "not a regular file", str(path))
data = handle.read() if limit is None else handle.read(limit + 1) with open(fd, "rb", closefd=False) as handle:
data = handle.read() if limit is None else handle.read(limit + 1)
finally:
os.close(fd)
if limit is not None and len(data) > limit: if limit is not None and len(data) > limit:
raise OSError(errno.EFBIG, f"larger than {limit} bytes", str(path)) raise OSError(errno.EFBIG, f"larger than {limit} bytes", str(path))
return data.decode("utf-8") return data.decode("utf-8")
+3 -3
View File
@@ -9,7 +9,7 @@ from .errors import ToolError
from .fields import single from .fields import single
from .lifetimes import edge_times, lifetime, session_gone, subagent_rows from .lifetimes import edge_times, lifetime, session_gone, subagent_rows
from .names import PSEUDO, SESSION_ID, Directory, title_sightings from .names import PSEUDO, SESSION_ID, Directory, title_sightings
from .records import epoch, iso_epoch, read_jsonl from .records import epoch, iso_epoch
from .register import REGISTER, read_register from .register import REGISTER, read_register
from .tree import assign from .tree import assign
@@ -48,10 +48,10 @@ class Sources:
tasks = read_register(self.council / REGISTER) tasks = read_register(self.council / REGISTER)
reviews = self.reviews.listed() reviews = self.reviews.listed()
scan = assign(self.tree.get(), tasks, read_claims(self.council), reviews) scan = assign(self.tree.get(), tasks, read_claims(self.council), reviews)
history = read_jsonl(self.council / "queue-history.jsonl") history = self.history.rows()
return {**enrich(scan, tasks, reviews, self.touched, history, self.jobs.listed()), "cleanups": self.cleanup_results(), "gates": self.gate_view(scan)} return {**enrich(scan, tasks, reviews, self.touched, history, self.jobs.listed()), "cleanups": self.cleanup_results(), "gates": self.gate_view(scan)}
def schedules(self, claims: list[dict], now: float) -> list[dict]: def schedules(self, claims: list[dict], now: float) -> list[dict]:
beats = {claim["session"]: claim["beat"] for claim in claims if claim["beat"] is not None} beats = {claim["session"]: claim["beat"] for claim in claims if claim["beat"] is not None}
rows = [cron_state(cron, beats, now) for cron in self.transcripts.schedules()] rows = [cron_state(cron, beats, now) for cron in self.transcripts.schedules()]
return rows + timer_rows(self.timers) + queue_rows(self.council) return rows + timer_rows(self.timers) + queue_rows(self.council, self.history.rows)
+16 -6
View File
@@ -4,6 +4,7 @@ import itertools
import statistics import statistics
from .council import HUMAN, agent from .council import HUMAN, agent
from .crons import outcome_of
from .records import epoch, iso_epoch, shown from .records import epoch, iso_epoch, shown
from .register import ENDED, OPERATOR, parties, register_row, status_of from .register import ENDED, OPERATOR, parties, register_row, status_of
from .signed import signing from .signed import signing
@@ -152,12 +153,21 @@ def task_row(task: str, row: dict, trees: list[dict]) -> dict:
} }
def queue_status(entry: dict, found: dict | None) -> str:
if found is not None:
return "done" if found.get("state") == "ended" and found.get("exit") == 0 else "stopped"
return "running" if entry.get("status") == "active" else "awaiting"
def queue_row(entry: dict, created: float, trees: list[dict]) -> dict: def queue_row(entry: dict, created: float, trees: list[dict]) -> dict:
text = {name: shown(entry.get(name))[:TEXT_SHOWN] for name in ("id", "cmd", "kind", "task", "workdir")} text = {name: shown(entry.get(name))[:TEXT_SHOWN] for name in ("id", "cmd", "kind", "task", "workdir")}
found = outcome_of(entry)
ended = found is not None
granted = entry.get("status") == "active" or ended
return { return {
"task": f"{QUEUE}{text['id']}", "task": f"{QUEUE}{text['id']}",
"title": text["cmd"], "title": text["cmd"],
"status": "running" if entry.get("status") == "active" else "awaiting", "status": queue_status(entry, found),
"worker": agent(entry.get("name"))[:TEXT_SHOWN] or "-", "worker": agent(entry.get("name"))[:TEXT_SHOWN] or "-",
"responsible": "-", "responsible": "-",
"accountable": "-", "accountable": "-",
@@ -167,7 +177,7 @@ def queue_row(entry: dict, created: float, trees: list[dict]) -> dict:
"met": 0, "met": 0,
"total": 0, "total": 0,
"kind": "queue", "kind": "queue",
"ended": False, "ended": ended,
"editable": False, "editable": False,
"movable": False, "movable": False,
"prio": None, "prio": None,
@@ -175,13 +185,13 @@ def queue_row(entry: dict, created: float, trees: list[dict]) -> dict:
"wsjf": None, "wsjf": None,
"value": None, "value": None,
"points": None, "points": None,
"start": (epoch(entry.get("granted")) or created) if entry.get("status") == "active" else None, "start": (epoch(entry.get("granted")) or created) if granted else None,
"end": None, "end": epoch(found.get("end")) if found is not None else None,
"updated": None, "updated": None,
"after": [], "after": [],
"base": "-", "base": "-",
"roles": {}, "roles": {},
"queue": {"kind": text["kind"], "parent": text["task"], "enqueued": created}, "queue": {"kind": text["kind"], "parent": text["task"], "enqueued": created, "exit": found.get("exit") if found is not None else None},
**located(entry, trees, False), **located(entry, trees, False),
} }
@@ -253,7 +263,7 @@ def queue_chain(queue: list[tuple[dict, float]], tasks: dict[str, dict], agents:
holder = order[name] if isinstance(name, str) and name in order else len(order) holder = order[name] if isinstance(name, str) and name in order else len(order)
return (not entry.get("hotfix"), place - hours, holder, seq if isinstance(seq, int) and not isinstance(seq, bool) else 0) return (not entry.get("hotfix"), place - hours, holder, seq if isinstance(seq, int) and not isinstance(seq, bool) else 0)
active = sorted((pair for pair in queue if pair[0].get("status") == "active"), key=lambda pair: epoch(pair[0].get("granted")) or pair[1]) active = sorted((pair for pair in queue if pair[0].get("status") == "active" and outcome_of(pair[0]) is None), key=lambda pair: epoch(pair[0].get("granted")) or pair[1])
waiting = sorted((pair for pair in queue if pair[0].get("status") == "waiting"), key=waiting_key) waiting = sorted((pair for pair in queue if pair[0].get("status") == "waiting"), key=waiting_key)
return [(shown(entry.get("id"))[:TEXT_SHOWN], entry.get("status") == "waiting") for entry, _ in [*active, *waiting]] return [(shown(entry.get("id"))[:TEXT_SHOWN], entry.get("status") == "waiting") for entry, _ in [*active, *waiting]]
+2 -2
View File
@@ -85,7 +85,7 @@ class View:
def state(self, begin: float, end: float, now: float, steps: int) -> dict: def state(self, begin: float, end: float, now: float, steps: int) -> dict:
tasks = read_register(self.council / REGISTER) tasks = read_register(self.council / REGISTER)
news = read_jsonl(self.council / NEWS) news = read_jsonl(self.council / NEWS)
history = read_jsonl(self.council / "queue-history.jsonl") history = self.history.rows()
claims = read_claims(self.council) claims = read_claims(self.council)
names = sessions(claims) names = sessions(claims)
directory = self.directory(claims, news, now) directory = self.directory(claims, news, now)
@@ -105,7 +105,7 @@ class View:
people = {node["id"]: {"name": node["label"], "now": directory.now(node["id"]) or node["label"], "names": node["names"]} for node in drawn["nodes"] if node["kind"] != "subagent"} people = {node["id"]: {"name": node["label"], "now": directory.now(node["id"]) or node["label"], "names": node["names"]} for node in drawn["nodes"] if node["kind"] != "subagent"}
settings = self.settings.get() settings = self.settings.get()
inbox = self.inbox(now, tasks, directory) inbox = self.inbox(now, tasks, directory)
queue = queue_entries(self.council) queue = queue_entries(self.council, self.history.rows)
order = stored_order(self.council / AGENT_ORDER) order = stored_order(self.council / AGENT_ORDER)
rows = task_rows(tasks, queue, self.tree.get()) rows = task_rows(tasks, queue, self.tree.get())
chain = queue_chain(queue, tasks, order, now) chain = queue_chain(queue, tasks, order, now)
+2 -1
View File
@@ -47,7 +47,8 @@
function taskPlace(row) { function taskPlace(row) {
if (row.kind === "queue") { if (row.kind === "queue") {
return h("span", {}, `${row.queue.kind} for ${row.queue.parent} · queued `, when(row.queue.enqueued)); const outcome = !row.ended ? null : row.queue.exit !== null ? ` · exit ${row.queue.exit}` : " · ended without an exit code";
return h("span", {}, `${row.queue.kind} for ${row.queue.parent} · queued `, when(row.queue.enqueued), outcome);
} }
const place = row.worktree || (row.workdir !== "-" ? row.workdir : ""); const place = row.worktree || (row.workdir !== "-" ? row.workdir : "");
return place ? h("code", {title: row.repo || place}, place) : null; return place ? h("code", {title: row.repo || place}, place) : null;
+4 -4
View File
@@ -23,12 +23,12 @@ const Plan = (() => {
} }
/** /**
* @param rows - task rows with status, start, end (epoch seconds or null) and points (median complexity or null). * @param rows - task rows with kind, status, start, end (epoch seconds or null) and points (median complexity or null).
* @returns {duration, perPoint}: the median seconds a done task took from start to end, and the median seconds * @returns {duration, perPoint}: the median seconds a done task, never a queue job, took from start to end, and the
* per point of the done tasks with points; null where no done task gives one. * median seconds per point of the done tasks with points; null where no done task gives one.
*/ */
function medians(rows) { function medians(rows) {
const done = rows.filter((row) => row.status === "done" && row.start !== null && row.end !== null && row.end >= row.start); const done = rows.filter((row) => row.kind !== "queue" && row.status === "done" && row.start !== null && row.end !== null && row.end >= row.start);
return { return {
duration: median(done.map((row) => row.end - row.start)), duration: median(done.map((row) => row.end - row.start)),
perPoint: median(done.filter((row) => row.points).map((row) => (row.end - row.start) / row.points)), perPoint: median(done.filter((row) => row.points).map((row) => (row.end - row.start) / row.points)),
+14 -4
View File
@@ -241,6 +241,10 @@ def rules_dir(root: Path) -> Path:
return root / "rules" return root / "rules"
def client_calls(tree: ast.AST) -> list[ast.Call]:
return [node for node in ast.walk(tree) if isinstance(node, ast.Call) and "client_module" in (getattr(node.func, "id", ""), getattr(node.func, "attr", ""))]
class TestDiffsModule(unittest.TestCase): class TestDiffsModule(unittest.TestCase):
@classmethod @classmethod
def setUpClass(cls): def setUpClass(cls):
@@ -825,12 +829,18 @@ class TestSharedLimits(unittest.TestCase):
self.assertLessEqual(len(json.dumps(patch, ensure_ascii=False).encode("utf-8")), hub_limits.DIFF_LIMIT) self.assertLessEqual(len(json.dumps(patch, ensure_ascii=False).encode("utf-8")), hub_limits.DIFF_LIMIT)
self.assertGreater(len(permission_hook.encoded({"diff": [{"patch": patch}]})), hub_limits.DIFF_BODY_LIMIT) self.assertGreater(len(permission_hook.encoded({"diff": [{"patch": patch}]})), hub_limits.DIFF_BODY_LIMIT)
def test_the_hub_image_copies_every_client_module_signed_loads(self): def test_client_module_calls_are_found_bare_and_as_attributes(self):
tree = ast.parse((HUB / "kvb_orchestrator" / "signed.py").read_text(encoding="utf-8")) found = client_calls(ast.parse('signed.client_module("a", {})\nclient_module("b", {})\nother("c")\nx.other("d")'))
loaded = {f"{node.args[0].value}.py" for node in ast.walk(tree) if isinstance(node, ast.Call) and getattr(node.func, "id", "") == "client_module"} self.assertEqual([node.args[0].value for node in found], ["a", "b"])
def test_the_hub_image_copies_every_client_module_the_hub_loads(self):
trees = [ast.parse(path.read_text(encoding="utf-8")) for path in sorted((HUB / "kvb_orchestrator").glob("*.py"))]
calls = [node for tree in trees for node in client_calls(tree)]
self.assertTrue(all(node.args and isinstance(node.args[0], ast.Constant) for node in calls))
loaded = {f"{node.args[0].value}.py" for node in calls}
copies = [line.split()[1:] for line in (HUB / "Dockerfile").read_text(encoding="utf-8").splitlines() if line.startswith("COPY ")] copies = [line.split()[1:] for line in (HUB / "Dockerfile").read_text(encoding="utf-8").splitlines() if line.startswith("COPY ")]
copied = {Path(source).name for words in copies if words[-1] == "/app/client/" for source in words[:-1]} copied = {Path(source).name for words in copies if words[-1] == "/app/client/" for source in words[:-1]}
self.assertIn("hub_limits.py", loaded) self.assertLessEqual({"hub_limits.py", "queue_live.py"}, loaded)
self.assertLessEqual(loaded, copied) self.assertLessEqual(loaded, copied)
+4
View File
@@ -74,6 +74,7 @@ console.log(JSON.stringify({
steps: {whole: stepsOf(70 * H, now), window: stepsOf(90 * H, 99 * H), burnt: P.steps(burnt, burnPoints, 0)}, steps: {whole: stepsOf(70 * H, now), window: stepsOf(90 * H, 99 * H), burnt: P.steps(burnt, burnPoints, 0)},
etas: {none: eta("none"), points: eta("points"), loop: [...P.etas(loop, looped, {now, estimation: "none"}).keys()].sort()}, etas: {none: eta("none"), points: eta("points"), loop: [...P.etas(loop, looped, {now, estimation: "none"}).keys()].sort()},
medians: P.medians(rows), medians: P.medians(rows),
queueMedians: P.medians([...rows, row("queue/done", {kind: "queue", status: "done", ended: true, start: 0, end: 50 * H, points: 1})]),
median: [P.median([]), P.median([3, 1, 2]), P.median([4, 1, 3, 2])], median: [P.median([]), P.median([3, 1, 2]), P.median([4, 1, 3, 2])],
bars: {noneAlways: lay("none", "always"), pointsAlways: lay("points", "always"), pointsOnly: lay("points", "points"), noneOnly: lay("none", "points"), real: lay("points", "real")}, bars: {noneAlways: lay("none", "always"), pointsAlways: lay("points", "always"), pointsOnly: lay("points", "points"), noneOnly: lay("none", "points"), real: lay("points", "real")},
loop: P.bars(loop, looped, {now, estimation: "none", forecast: "always"}).map((bar) => bar.task), loop: P.bars(loop, looped, {now, estimation: "none", forecast: "always"}).map((bar) => bar.task),
@@ -157,6 +158,9 @@ class TestPlanModule(unittest.TestCase):
self.assertEqual(self.data["medians"], {"duration": 2 * HOUR, "perPoint": 1.5 * HOUR}) self.assertEqual(self.data["medians"], {"duration": 2 * HOUR, "perPoint": 1.5 * HOUR})
self.assertEqual(self.data["median"], [None, 2, 2.5]) self.assertEqual(self.data["median"], [None, 2, 2.5])
def test_a_done_queue_job_never_moves_the_medians(self):
self.assertEqual(self.data["queueMedians"], self.data["medians"])
def test_without_estimate_a_forecast_is_as_long_as_the_median_duration_behind_its_predecessors(self): def test_without_estimate_a_forecast_is_as_long_as_the_median_duration_behind_its_predecessors(self):
self.assertEqual(self.data["bars"]["noneAlways"], [ self.assertEqual(self.data["bars"]["noneAlways"], [
["work/a", 80, 82, False, False], ["work/a", 80, 82, False, False],
+192 -2
View File
@@ -1,5 +1,7 @@
from __future__ import annotations from __future__ import annotations
import fcntl
import functools
import importlib.util import importlib.util
import json import json
import os import os
@@ -12,13 +14,14 @@ from unittest import mock
from hub_case import HubCase from hub_case import HubCase
from hub_support import REPO_ROOT, iso, key, runner, signing, verifier from hub_support import REPO_ROOT, iso, key, runner, signing, verifier
from kvb_orchestrator import settings, skill_scripts, tasks, tree from kvb_orchestrator import crons, records, settings, skill_scripts, tasks, tree
BROKEN = {"pull": [], "push": [], "source": "invalid"} BROKEN = {"pull": [], "push": [], "source": "invalid"}
FLOW = {"pull": ["origin/main", "store-i18n"], "push": ["main", "production"], "source": "file"} FLOW = {"pull": ["origin/main", "store-i18n"], "push": ["main", "production"], "source": "file"}
PRIO = ["set", "work/feature", "--prio", "3"] PRIO = ["set", "work/feature", "--prio", "3"]
ESTIMATE = ["estimate", "work/feature", "Operator", "--value", "8", "--complexity", "3"] ESTIMATE = ["estimate", "work/feature", "Operator", "--value", "8", "--complexity", "3"]
QUEUE_SCRIPT = REPO_ROOT / "skills" / "kvb-org-queue" / "scripts" / "queue.py" QUEUE_SCRIPT = REPO_ROOT / "skills" / "kvb-org-queue" / "scripts" / "queue.py"
PROOF = "/home/me/worktrees/github.com/infinito-nexus/core/proof/karakalpak-archive-role"
def change(argv: list, task: str = "work/feature") -> dict: def change(argv: list, task: str = "work/feature") -> dict:
@@ -136,11 +139,198 @@ class TestHubTasks(HubCase):
self.assertEqual((command["kind"], command["title"], command["editable"], command["open_work"]), ("command", "git push", False, ["task"])) self.assertEqual((command["kind"], command["title"], command["editable"], command["open_work"]), ("command", "git push", False, ["task"]))
gate, deploy = rows["queue/gate-7"], rows["queue/deploy-8"] gate, deploy = rows["queue/gate-7"], rows["queue/deploy-8"]
self.assertEqual((gate["kind"], gate["status"], gate["title"], gate["worker"], gate["editable"], gate["flow"]), ("queue", "awaiting", "make test", "Stage", False, FLOW)) self.assertEqual((gate["kind"], gate["status"], gate["title"], gate["worker"], gate["editable"], gate["flow"]), ("queue", "awaiting", "make test", "Stage", False, FLOW))
self.assertEqual(gate["queue"], {"kind": "gate", "parent": "work/feature", "enqueued": now - 60}) self.assertEqual(gate["queue"], {"kind": "gate", "parent": "work/feature", "enqueued": now - 60, "exit": None})
self.assertEqual((deploy["status"], deploy["flow"], deploy["queue"]["parent"]), ("running", None, "work/pinned")) self.assertEqual((deploy["status"], deploy["flow"], deploy["queue"]["parent"]), ("running", None, "work/pinned"))
done = rows["work/done"] done = rows["work/done"]
self.assertEqual((done["ended"], done["editable"], done["open_work"], done["worktree"]), (True, False, [], "")) self.assertEqual((done["ended"], done["editable"], done["open_work"], done["worktree"]), (True, False, [], ""))
def held_lock(self, tag: str, mode: int = fcntl.LOCK_EX) -> None:
holds = self.council / "holds"
holds.mkdir(parents=True, exist_ok=True)
handle = (holds / f"{tag}.lock").open("w", encoding="utf-8")
self.addCleanup(handle.close)
fcntl.flock(handle, mode)
def queue_job(self, now: float, ident: str, status: str, task: str, lock: str, age: float) -> dict:
granted = now - age + 10 if status == "active" else None
return {"id": ident, "kind": ident.rpartition("-")[0], "task": task, "workdir": PROOF, "session": "s1", "name": "Production", "lock": lock, "pid": 7,
"status": status, "enqueued": now - age, "granted": granted, "cmd": "make quality-high"}
def test_a_queue_job_with_a_history_row_or_a_gone_holder_is_no_longer_running(self):
now = time.time()
for tag in ("7-1410", "7-1451", "7-5", "7-1387", "7-1500", "7-1501", "7-1502"):
self.held_lock(tag)
self.held_lock("7-1481", fcntl.LOCK_SH)
(self.council / "holds" / "7-1465.lock").write_text("", encoding="utf-8")
(self.council / "holds" / "7-1480.lock").symlink_to(self.council / "holds" / "7-1451.lock")
job = functools.partial(self.queue_job, now)
queue = {"seq": 1502, "entries": [
job("quality-high-1410", "active", "karakalpak-archive", "7-1410", 1200),
job("test-nostack-1464", "active", "karakalpak-archive", "7-1464", 300),
job("test-nostack-1465", "active", "iam/final-066-backchannel", "7-1465", 200),
job("test-nostack-1466", "active", "iam/final-066-backchannel", "7-1466", 100),
job("quality-high-1451", "active", "work/gitea-oidc-auto-link", "7-1451", 900),
job("test-nostack-1480", "active", "work/symlinked", "7-1480", 80),
job("test-nostack-1481", "active", "iam/final-066-backchannel", "7-1481", 70),
job("gate-5", "active", "work/reused", "7-5", 60),
job("test-1387", "waiting", "work/security-page-optimize", "7-1387", 50),
job("test-1388", "waiting", "work/security-page-optimize", "7-1388", 40),
job("test-nostack-1471", "waiting", "work/orchestrator-firewall", "../holds/7-1471", 30),
job("test-1500", "ended", "work/foreign", "7-1500", 25),
job("test-1501", "gone", "work/foreign", "7-1501", 24),
{**job("test-1502", "active", "work/foreign", "7-1502", 20), "hub_outcome": {"state": "ended", "exit": 9}, "exit": 9, "end": now},
]}
history = [
{"end": now - 600, "exit": 1, "id": "quality-high-1410-1794513000000000000", "kind": "quality-high", "start": now - 1180, "workdir": PROOF},
{"concurrent": 2, "cpu_before": 53.9, "cpu_max": 100.0, "disk_gb": 1.91, "end": now - 5, "exit": 2, "free_before_gb": 113.12,
"id": "quality-high-1410-1794514233361361126", "kind": "quality-high", "mem_before": 43.2, "mem_max": 84.3, "min_free_gb": 111.21,
"minutes": 18.13, "signature": "infinito-nexus-core: make quality-high", "start": now - 1185, "workdir": PROOF},
{"end": now - 20, "exit": 0, "id": "test-nostack-1464-1794514708907109250", "kind": "test-nostack", "start": now - 290, "workdir": PROOF},
{"end": now - 3, "exit": 0, "id": "test-nostack-1466-retry", "kind": "test-nostack", "start": now - 90, "workdir": PROOF},
{"end": now - 7200, "exit": 1, "id": "gate-5-1794500000000000000", "kind": "gate", "start": now - 7300, "workdir": PROOF},
{"end": now - 1, "exit": 0, "id": "gate-5x-1794500000000000000", "kind": "gate", "start": now - 30, "workdir": PROOF},
]
self.write_council({"queue.json": queue, "queue-history.jsonl": self.jsonl(history)})
state = self.snapshot()
rows = {row["task"]: row for row in state["tasks"] if row["kind"] == "queue"}
def seen(ident: str) -> tuple:
row = rows[f"queue/{ident}"]
return row["status"], row["ended"], row["end"], row["queue"].get("exit")
self.assertEqual(seen("quality-high-1410"), ("stopped", True, now - 5, 2))
self.assertEqual(rows["queue/quality-high-1410"]["start"], now - 1190)
self.assertEqual(seen("test-nostack-1464"), ("done", True, now - 20, 0))
self.assertEqual(seen("test-nostack-1465"), ("stopped", True, None, None))
self.assertEqual(seen("test-nostack-1466"), ("stopped", True, None, None))
self.assertEqual(seen("test-nostack-1481"), ("stopped", True, None, None))
self.assertEqual(seen("test-nostack-1480"), ("stopped", True, None, None))
live = ("quality-high-1451", "gate-5", "test-1502")
shown = ("quality-high-1410", "test-nostack-1464", "test-nostack-1465", "test-nostack-1466", "test-nostack-1480", "test-nostack-1481", "test-1387", "test-1500", "test-1501", *live)
self.assertEqual(sorted(rows), sorted(f"queue/{ident}" for ident in shown))
self.assertEqual([seen(ident) for ident in live], [("running", False, None, None)] * 3)
self.assertEqual([seen(ident) for ident in ("test-1387", "test-1500", "test-1501")], [("awaiting", False, None, None)] * 3)
self.assertEqual(state["queue_line"], ["work/gitea-oidc-auto-link", "work/reused", "work/foreign", "work/security-page-optimize"])
schedule = {row["job"]: (row["expression"], row["active"]) for row in crons.queue_rows(self.council, self.httpd.hub.history.rows)}
self.assertEqual(schedule["quality-high-1410"], ("quality-high · exit 2", False))
self.assertEqual(schedule["test-nostack-1465"], ("test-nostack · holder gone without an exit code", False))
self.assertEqual(schedule["quality-high-1451"], ("quality-high · running", True))
self.assertEqual(schedule["test-1500"], ("test · waiting", True))
def test_the_history_is_read_after_the_hold_locks_and_parsed_once_per_change(self):
now = time.time()
self.held_lock("7-1")
queue = {"seq": 2, "entries": [self.queue_job(now, "test-1", "active", "work/a", "7-1", 100), self.queue_job(now, "test-2", "active", "work/b", "7-2", 50)]}
self.write_council({"queue.json": queue, "queue-history.jsonl": self.jsonl([{"id": "other-9-1", "start": now - 500, "end": now - 400, "exit": 0}])})
probe, read = crons.queue_live.holder_gone, crons.queue_live.read_history
reads = []
def leaving(council: Path, tag: object) -> bool:
if tag == "7-2":
with (self.council / "queue-history.jsonl").open("a", encoding="utf-8") as handle:
handle.write(json.dumps({"id": "test-2-1794514233361361126", "start": now - 40, "end": now - 1, "exit": 3}) + "\n")
return probe(council, tag)
def counted(path: Path) -> list[dict]:
reads.append(path.name)
return read(path)
with mock.patch.object(crons.queue_live, "holder_gone", leaving), mock.patch.object(crons.queue_live, "read_history", counted):
rows = {row["task"]: row for row in self.snapshot()["tasks"] if row["kind"] == "queue"}
self.assertEqual((rows["queue/test-2"]["status"], rows["queue/test-2"]["queue"]["exit"]), ("stopped", 3))
self.assertEqual(rows["queue/test-1"]["status"], "running")
self.assertEqual(len(reads), 2)
self.httpd.hub.history.rows()
self.assertEqual(len(reads), 2)
def test_the_whole_history_is_read_and_a_bad_byte_costs_its_line_only(self):
path = self.council / "queue-history.jsonl"
self.council.mkdir(parents=True, exist_ok=True)
rows = b"".join(json.dumps({"id": f"gate-{n}-1", "start": n, "pad": "x" * 1000}).encode() + b"\n" for n in range(6000))
path.write_bytes(b'{"id": "bad-\xff-1"}\n' + rows + b"[[[\n" + b'{"id": "last-1"}')
found = crons.queue_live.read_history(path)
self.assertGreater(path.stat().st_size, 1 << 22)
self.assertEqual((len(found), found[1]["id"], found[-2]["id"], found[-1]["id"]), (6002, "gate-0-1", "gate-5999-1", "last-1"))
def test_reading_a_directory_as_history_or_json_leaks_no_descriptor(self):
folder = self.council / "queue-history.jsonl"
folder.mkdir(parents=True)
before = len(os.listdir("/proc/self/fd"))
for _ in range(50):
self.assertEqual(crons.queue_live.read_history(folder), [])
self.assertIsNone(records.read_json(folder))
self.assertEqual(records.read_jsonl(folder), [])
self.assertLessEqual(len(os.listdir("/proc/self/fd")), before + 2)
def test_the_history_cache_reparses_on_a_new_inode_mtime_or_size_alone(self):
path = self.council / "queue-history.jsonl"
self.council.mkdir(parents=True, exist_ok=True)
history = crons.History(path)
def write(text: str, ns: int) -> None:
path.write_text(text, encoding="utf-8")
os.utime(path, ns=(ns, ns))
write('{"id": "a-1"}\n', 10**18)
self.assertEqual([row["id"] for row in history.rows()], ["a-1"])
replacement = self.council / "replacement"
replacement.write_text('{"id": "b-1"}\n', encoding="utf-8")
os.utime(replacement, ns=(10**18, 10**18))
os.replace(replacement, path)
self.assertEqual([row["id"] for row in history.rows()], ["b-1"])
write('{"id": "c-1"}\n', 10**18 + 1)
self.assertEqual([row["id"] for row in history.rows()], ["c-1"])
write('{"id": "c-1"}\n{"id": "d-1"}\n', 10**18 + 1)
self.assertEqual([row["id"] for row in history.rows()], ["c-1", "d-1"])
def test_a_hard_link_to_a_live_hold_lock_keeps_the_hold_alive(self):
self.held_lock("7-20")
holds = self.council / "holds"
os.link(holds / "7-20.lock", holds / "7-21.lock")
self.assertEqual((holds / "7-20.lock").stat().st_nlink, 2)
self.assertEqual([crons.queue_live.holder_gone(self.council, tag) for tag in ("7-20", "7-21")], [False, False])
@unittest.skipIf(os.geteuid() == 0, "root reads a file of mode 0")
def test_a_failed_history_read_keeps_the_rows_before_and_retries_next_time(self):
path = self.council / "queue-history.jsonl"
self.write_council({"queue-history.jsonl": '{"id": "a-1"}\n'})
history = crons.History(path)
self.assertEqual([row["id"] for row in history.rows()], ["a-1"])
with path.open("a", encoding="utf-8") as handle:
handle.write('{"id": "b-1"}\n')
path.chmod(0)
self.addCleanup(path.chmod, 0o600)
self.assertEqual([row["id"] for row in history.rows()], ["a-1"])
self.assertIsNone(crons.queue_live.read_history(path))
path.chmod(0o600)
self.assertEqual([row["id"] for row in history.rows()], ["a-1", "b-1"])
def test_the_hub_judges_every_hold_lock_as_queue_py_held_does(self):
queue_py = load_queue()
holds = self.council / "holds"
self.held_lock("7-1")
(holds / "7-2.lock").write_text("", encoding="utf-8")
(holds / "7-3.lock").symlink_to(holds / "7-1.lock")
(holds / "7-7.lock").write_text("", encoding="utf-8")
os.link(holds / "7-7.lock", holds / "7-6.lock")
decided = {"held": "7-1", "free": "7-2", "missing": "7-404", "malformed": "x-1", "traversal": "../holds/7-1", "none": None}
refused = {"symlink": "7-3", "hard link": "7-6"}
hub = {name: crons.queue_live.holder_gone(self.council, tag) for name, tag in {**decided, **refused}.items()}
with mock.patch.object(queue_py, "HOLDS", holds), mock.patch.object(queue_py.watch, "ROOT", self.council):
for name, tag in refused.items():
with self.subTest(refused=name), self.assertRaises(OSError):
queue_py.held(tag)
with self.subTest(refused=name), self.assertRaises(OSError), queue_py.nofollow.locked(self.council, holds / f"{tag}.lock", 0o600, None):
pass
verdicts = {name: queue_py.held(tag) for name, tag in decided.items()}
self.assertEqual({name: not hub[name] for name in decided}, verdicts)
self.assertEqual(verdicts, {"held": True, "free": False, "missing": False, "malformed": False, "traversal": False, "none": False})
self.assertEqual({name: hub[name] for name in refused}, {"symlink": True, "hard link": True})
if os.geteuid() != 0:
(holds / "7-9.lock").write_text("", encoding="utf-8")
(holds / "7-9.lock").chmod(0)
self.assertFalse(crons.queue_live.holder_gone(self.council, "7-9"))
def test_only_the_newest_ended_tasks_are_listed(self): def test_only_the_newest_ended_tasks_are_listed(self):
register = {f"work/old-{n:03d}": {"status": "done", "end": f"2026-11-{1 + n // 24:02d}T{n % 24:02d}:00:00+00:00"} for n in range(30)} register = {f"work/old-{n:03d}": {"status": "done", "end": f"2026-11-{1 + n // 24:02d}T{n % 24:02d}:00:00+00:00"} for n in range(30)}
self.write_council({"rasci.json": {**register, "work/open": {"status": "queued"}}}) self.write_council({"rasci.json": {**register, "work/open": {"status": "queued"}}})
+22 -2
View File
@@ -1,5 +1,6 @@
from __future__ import annotations from __future__ import annotations
import fcntl
import json import json
import os import os
import subprocess import subprocess
@@ -108,10 +109,29 @@ class TestParsing(unittest.TestCase):
with tempfile.TemporaryDirectory() as tmp: with tempfile.TemporaryDirectory() as tmp:
council = Path(tmp) council = Path(tmp)
self.assertEqual(sampler.queue(council), {"waiting": 0, "active": 0}) self.assertEqual(sampler.queue(council), {"waiting": 0, "active": 0})
entries = [{"status": "waiting"}, {"status": "waiting"}, {"status": "active"}, {"status": "done"}, "junk"] entries = [{"status": status, "enqueued": NOW} for status in ("waiting", "waiting", "active", "done")] + [{"status": "active"}, "junk"]
(council / "queue.json").write_text(json.dumps({"entries": entries}), encoding="utf-8") (council / "queue.json").write_text(json.dumps({"entries": entries}), encoding="utf-8")
self.assertEqual(sampler.queue(council), {"waiting": 2, "active": 1}) self.assertEqual(sampler.queue(council), {"waiting": 2, "active": 1})
def test_the_queue_counts_only_live_entries_as_the_hub_settles_them(self):
with tempfile.TemporaryDirectory() as tmp:
council = Path(tmp)
(council / "holds").mkdir()
handles = [(council / "holds" / f"7-{n}.lock").open("w", encoding="utf-8") for n in (1, 2, 3)]
for handle in handles:
self.addCleanup(handle.close)
fcntl.flock(handle, fcntl.LOCK_EX)
entries = [
{"id": "test-1", "status": "waiting", "lock": "7-1", "enqueued": NOW},
{"id": "test-2", "status": "active", "lock": "7-2", "enqueued": NOW},
{"id": "quality-high-3", "status": "active", "lock": "7-3", "enqueued": NOW},
{"id": "test-4", "status": "waiting", "lock": "7-4", "enqueued": NOW},
{"id": "test-5", "status": "active", "lock": "7-5", "enqueued": NOW},
]
(council / "queue.json").write_text(json.dumps({"entries": entries}), encoding="utf-8")
(council / "queue-history.jsonl").write_text(json.dumps({"id": "quality-high-3-1794514233361361126", "start": NOW + 10, "end": NOW + 20, "exit": 2}) + "\n", encoding="utf-8")
self.assertEqual(sampler.queue(council), {"waiting": 1, "active": 1})
class TestCpu(unittest.TestCase): class TestCpu(unittest.TestCase):
def test_cpu_is_the_busy_share_since_the_ticks_the_run_before_kept_and_the_sampler_never_sleeps(self): def test_cpu_is_the_busy_share_since_the_ticks_the_run_before_kept_and_the_sampler_never_sleeps(self):
@@ -233,7 +253,7 @@ class TestSamplerInScratchHome(unittest.TestCase):
(stubs / tool).chmod(0o755) (stubs / tool).chmod(0o755)
self.sys = host_sys(base / "sys") self.sys = host_sys(base / "sys")
self.log = base / "calls.jsonl" self.log = base / "calls.jsonl"
(self.council / "queue.json").write_text(json.dumps({"entries": [{"status": "waiting"}, {"status": "active"}, {"status": "active"}]}), encoding="utf-8") (self.council / "queue.json").write_text(json.dumps({"entries": [{"status": status, "enqueued": NOW} for status in ("waiting", "active", "active")]}), encoding="utf-8")
self.env = { self.env = {
**os.environ, "HOME": str(self.home), "PATH": f"{stubs}:/usr/bin:/bin", **os.environ, "HOME": str(self.home), "PATH": f"{stubs}:/usr/bin:/bin",
"STUB_LOG": str(self.log), "STUB_DF": DF, "STUB_NVIDIA_SMI": NVIDIA_SMI.splitlines()[0] + "\n", "STUB_LOG": str(self.log), "STUB_DF": DF, "STUB_NVIDIA_SMI": NVIDIA_SMI.splitlines()[0] + "\n",