Merge branch 'fix/kvb-org-queue-hung-guard'
🧪 Test / 🧪 Lock + lint (push) Waiting to run

This commit is contained in:
Kevin Veen-Birkenbach committed 2026-11-15 11:11:11 +00:00
commit b88f187ffd
7 files changed
+991 -25

No files matched your search

+5 -1
View File
@@ -34,7 +34,11 @@ step inside it, or a process its own ended session left behind, once at least
child process, no network traffic, CPU time not moving). It secures the live
state first, then reports the timestamp, the step, the probe and the command.
A peer's hung job goes to its owner, and to the operator when the owner does
not answer.
not answer. `queue.py hold` does this itself for a queued job: after 60
minutes without CPU, I/O or new process in its tree, no stopped member and no
docker, podman or ssh client, it records the stuck processes, ends the tree
and exits 124; `queue.py list` marks a run past three times its longest
learned one `suspect hung`.
## Resource watch
+4 -4
View File
@@ -77,12 +77,12 @@ queue.py cancel <id>
- Never ask the chair for a heavy slot or announce a grant; the chair handles
only a stuck queue or hung job (`capacity.md` of `kvb-org-organize`).
- A running job is never cut short for the queue; only the operator orders
one down, and `kvb-git-hotfix` may void a gate. `kvb-org-queue-cleanup`
- Only the operator, a `kvb-git-hotfix` gate void, or hold after 60 minutes
of tree standstill (exit 124) cut a job short. `kvb-org-queue-cleanup`
deletes only entries whose lock no living process holds.
- With `queue_enforce: true`, the `block` hook (PreToolUse on Bash) refuses
`heavy_commands`, image builds, `make install|test*|quality*|verify-*`,
tests beyond one unit file and `gate.py run|hold` outside `queue.py hold`.
- Without the config, hold exits 2: tell the operator, ask the chair
(`kvb-org-organize`). A waiting hold rereads it every poll
(a broken file keeps the last good one).
(`kvb-org-organize`). Waiting holds reread it every poll, keeping the
last good one.
+2 -1
View File
@@ -2,7 +2,8 @@
Every finished job appends a row (signature such as `<repo>: make <target>`,
kind, minutes, disk, RAM and CPU) to `queue-history.jsonl`; the
`kvb-org-organize` watch hook keeps it across reboots.
`kvb-org-organize` watch hook keeps it across reboots. The row of a run
hold ended for standing still (exit 124, a `hung` probe) teaches nothing.
From the last 30 days, once 5 runs exist:
+13 -7
View File
@@ -6,7 +6,9 @@ and the copy the kvb-org-organize watch hook keeps in ~/.local/state.
A row: id, kind, signature, workdir, start, end (epoch seconds), minutes, exit,
free_before_gb, min_free_gb, disk_gb (free_before_gb - min_free_gb, at least
0), mem_before, mem_max, cpu_before, cpu_max (percent), concurrent (other
active jobs at the grant).
active jobs at the grant), and HUNG ({cpu_s, read_bytes, written_bytes,
still_min, members, tree}, the report of standstill.py) only on a run hold
ended for standing still, whose exit is 124.
"""
from __future__ import annotations
@@ -33,6 +35,7 @@ NAME_KEY = operator.itemgetter("signature")
KIND_KEY = operator.itemgetter("kind")
ANY_KIND = "*"
RAM_CAP = "ram_cap_percent"
HUNG = "hung"
SIGNATURE_HEADER = ("Signature", "Kind", "Runs", "Median min", "p90 disk GB", "p90 RAM %", "Need GB")
KIND_HEADER = ("Kind", "Runs", "p90 RAM share %", "Median CPU share %", "Slot limit", "Source")
@@ -156,11 +159,12 @@ def grouped(rows: list[dict], key: Callable[[dict], Hashable]) -> dict[Hashable,
def command_stats(rows: list[dict]) -> dict:
stats = {"runs": len(rows), "minutes": None, "disk_gb": None, "ram_share": None, "cpu_share": None, "need_gb": None}
stats = {"runs": len(rows), "minutes": None, "longest": None, "disk_gb": None, "ram_share": None, "cpu_share": None,
"need_gb": None}
if len(rows) >= MIN_RUNS:
disk = p90([r["disk_gb"] for r in rows])
stats.update(
minutes=statistics.median(r["minutes"] for r in rows), disk_gb=disk,
minutes=statistics.median(r["minutes"] for r in rows), longest=max(r["minutes"] for r in rows), disk_gb=disk,
ram_share=p90(shares(rows, "mem")), cpu_share=statistics.median(shares(rows, "cpu")),
need_gb=math.ceil(disk + DISK_MARGIN_GB),
)
@@ -186,7 +190,8 @@ def learn(rows: list[dict], now: float, cfg: dict) -> tuple[dict[tuple[str, str]
"""Derive disk need per command and slot limit per kind from recent rows.
Args:
rows: history rows, as read() returns them.
rows: history rows, as read() returns them; HUNG rows are ignored, as
their standstill would stretch the minutes and the shares.
now: epoch seconds; rows that ended more than WINDOW_SECONDS earlier are
ignored.
cfg: the config; load_limits.mem_percent and cpu_percent bound a limit,
@@ -195,19 +200,20 @@ def learn(rows: list[dict], now: float, cfg: dict) -> tuple[dict[tuple[str, str]
Returns:
(signatures, kinds). signatures: {(kind, signature): {runs, minutes,
disk_gb, ram_share, cpu_share, need_gb}}, and {(ANY_KIND, signature):
longest, disk_gb, ram_share, cpu_share, need_gb}}, and {(ANY_KIND, signature):
...} over the rows of every kind with that signature, which is what a
queue before the per-kind learning booked; kinds: {kind: {runs,
ram_share, cpu_share, limit, light, ram_cap}}, ram_cap None without a
cap. With fewer than MIN_RUNS rows every value but runs, light and
ram_cap is None.
minutes is the median, disk_gb and ram_share the p90 of the disk taken
minutes is the median and longest the largest minutes, disk_gb and
ram_share the p90 of the disk taken
and of max minus before in percentage points, cpu_share the median of
max minus before; need_gb is ceil(disk_gb + DISK_MARGIN_GB); limit is
max(1, min over RAM and CPU of floor((load limit - median before) /
max(share, MIN_SHARE_PERCENT))).
"""
recent = [r for r in rows if now - r["end"] <= WINDOW_SECONDS]
recent = [r for r in rows if now - r["end"] <= WINDOW_SECONDS and HUNG not in r]
light = set(cfg.get("light_kinds", []))
caps = cfg.get(RAM_CAP, {})
signatures = {(ANY_KIND, name): command_stats(group) for name, group in grouped(recent, NAME_KEY).items()}
+64 -12
View File
@@ -41,9 +41,10 @@ Args:
proposing it (latched()); measure disk, RAM and CPU, run
CMD in the current directory with inherited stdio in a session of its
own, forward SIGINT and SIGTERM to its process group, measure every
POLL_SECONDS and at its end, record its history row and leave the
queue. The entry carries its code (a hash of this file and the
SIBLINGS); a poll that finds other code installed in
POLL_SECONDS and at its end, end its process tree once the tree stood
still for standstill.HUNG_SECONDS (supervise()), record its history
row and leave the queue. The entry carries its code (a hash of this
file and the SIBLINGS); a poll that finds other code installed in
~/.agents/skills leaves the queue ungranted and restarts the same hold
with the installed queue.py, which keeps its entry id, place and
enqueue time (hand_over()).
@@ -74,7 +75,9 @@ Args:
poll, host load included, such as `waits in line behind <id>` or
`drained for <id>`, the head's after `escalated <step>: ` once the
stall reached it; `backfilled` for a job started past a blocked one;
prefixed when the entry runs other code than the installed one).
led by `suspect hung (<runtime> min)` for an active job past
standstill.suspect_seconds() (suspected()); prefixed when the entry
runs other code than the installed one).
Entries of dead holders are dropped first.
--json: print the same rows as a JSON list (listed()), each with
the trace of the evaluate() pass that set its reason.
@@ -109,7 +112,8 @@ watch.py documents them. A signature is `<repository>: <command signature>`
(aliases()).
Exit codes:
hold: the command's exit code unchanged, 128+N when signal N ended it;
hold: the command's exit code unchanged, 128+N when signal N ended it,
standstill.HUNG_EXIT (124) when hold ended it for standing still;
a command exiting 75 (EX_TEMPFAIL: it did no work, such as a declined
`operator.py granted`) leaves no history row;
before the command started 2 on a usage or config error or when it
@@ -154,6 +158,7 @@ SIBLINGS = {
"council": SKILLS / "kvb-org-organize" / "scripts" / "council.py",
"rasci": SKILLS / "kvb-org-rasci" / "scripts" / "rasci.py",
"history": Path(__file__).resolve().parent / "history.py",
"standstill": Path(__file__).resolve().parent / "standstill.py",
"nofollow": SKILLS / "kvb-org-organize" / "scripts" / "nofollow.py",
"councilroot": SKILLS / "kvb-org-organize" / "scripts" / "councilroot.py",
}
@@ -228,6 +233,7 @@ def sibling(name: str) -> ModuleType:
watch = sibling("watch")
history = sibling("history")
standstill = sibling("standstill")
nofollow = sibling("nofollow")
STORE = watch.ROOT / "queue.json"
LOCK = watch.ROOT / "queue.lock"
@@ -1444,23 +1450,54 @@ def measure(cfg: dict) -> dict[str, float]:
return {"free_gb": free, "mem": mem, "cpu": cpu}
def supervise(cfg: dict, child: subprocess.Popen, samples: list[dict[str, float]]) -> int:
def supervise(cfg: dict, child: subprocess.Popen, samples: list[dict[str, float]]) -> tuple[int, dict | None]:
"""Wait for the child, measuring every POLL_SECONDS and once at its end.
Args:
cfg: the config.
child: the running command.
child: the running command, leading a session of its own.
samples: the measurements so far; each new one is appended.
Returns:
The child's return code.
(the child's return code, None); (standstill.HUNG_EXIT, the probe of
standstill.Clock.report()) once its process tree stood still for
standstill.HUNG_SECONDS and stop() ended it.
"""
clock = standstill.Clock(child.pid)
code = None
while code is None:
with contextlib.suppress(subprocess.TimeoutExpired):
code = child.wait(timeout=POLL_SECONDS)
samples.append(measure(cfg))
return code
if code is None and clock.hung() and child.poll() is None:
probe = clock.report()
print(f"queue: {standstill.told(probe)}; ending the tree", file=sys.stderr, flush=True)
return stop(child), probe
return code, None
def stop(child: subprocess.Popen) -> int:
"""SIGTERM every process of the child's tree (standstill.members()), of
the sessions its members lead and the descendants those have now,
SIGKILL the ones still there after standstill.KILL_GRACE_SECONDS and
those standstill.settle() found during it, again at each pass of the
SIGKILL grace, a recorded member only while its start time matches; the
child's process group (send()) when /proc does not show the child. A
process that leaves the tree and every recorded session between two
passes (setsid, then its parent dies) escapes; only a subreaper or a
cgroup per job would hold it.
Returns:
standstill.HUNG_EXIT, also when a process outlives SIGKILL by another
grace (uninterruptible sleep), so hold never waits on it.
"""
targets = standstill.members(child.pid)
for signum, repeat in ((signal.SIGTERM, None), (signal.SIGKILL, signal.SIGKILL)):
living = standstill.signal_all(targets, signum)
if child.pid not in living:
send(child, signum)
targets = standstill.settle(targets | living, child.poll, repeat)
return standstill.HUNG_EXIT
def keep(row: dict) -> None:
@@ -1582,10 +1619,10 @@ def cmd_hold(args: argparse.Namespace) -> int:
for signum in caught:
send(child, signum)
update(entry["id"], child=child.pid, child_start=identity(child.pid))
code = supervise(cfg, child, samples)
code, hung = supervise(cfg, child, samples)
code = code if code >= 0 else 128 - code
job = {"id": f"{entry['id']}-{time.time_ns()}", "kind": args.kind, "signature": names[0],
"workdir": entry["workdir"]}
"workdir": entry["workdir"], **({} if hung is None else {history.HUNG: hung})}
record(job, start, code, samples, concurrent)
keep_receipt(gates, args, here, before, code)
return code
@@ -1607,7 +1644,8 @@ def cmd_list(args: argparse.Namespace) -> int:
by_id = {e["id"]: e for e in entries}
running = [(key_of(e), e["granted"]) for e in entries if e["status"] == "active"]
rows = [
(None, e, history.eta(signatures, [(key_of(e), e["granted"])], [], now), e.get("reason") or "-", e.get("trace", []))
(None, e, history.eta(signatures, [(key_of(e), e["granted"])], [], now), suspected(e, e.get("reason"), signatures, now) or "-",
e.get("trace", []))
for e in entries if e["status"] == "active"
]
ahead: list[tuple[str, str]] = []
@@ -1633,6 +1671,20 @@ def cmd_list(args: argparse.Namespace) -> int:
return 0
def suspected(entry: dict, reason: str | None, signatures: dict[tuple[str, str], dict], now: float) -> str | None:
"""Returns:
reason led by `suspect hung (<runtime> min)` once the active entry
runs past standstill.suspect_seconds() of the longest run its
signature learned (history.learned_value()), else reason.
"""
longest = history.learned_value(signatures, entry["kind"], aliases_of(entry), "longest")
runtime = now - entry["granted"]
if runtime < standstill.suspect_seconds(longest):
return reason
mark = f"suspect hung ({runtime / 60:.0f} min)"
return mark if reason is None else f"{mark}; {reason}"
def stamped(entry: dict, reason: str, host: str) -> str:
if entry.get("code") == host:
return reason
+343
View File
@@ -0,0 +1,343 @@
"""Tell a held command that stands still from one that works, from /proc alone.
queue.py's supervise() asks a Clock at every poll; once the process tree of
the command (tree()) stood still for HUNG_SECONDS, hold signals every member
(members(), signal_all(), settle()) and exits HUNG_EXIT, and list marks an
active job that runs SUSPECT_FACTOR times its longest learned run, at least
SUSPECT_FLOOR_SECONDS, as suspect hung.
"""
from __future__ import annotations
import contextlib
import os
import sys
import time
from collections.abc import Callable, Iterable
from pathlib import Path
from typing import NamedTuple
PROC = Path("/proc")
HUNG_SECONDS = 3600
KILL_GRACE_SECONDS = 30
SETTLE_SECONDS = 0.2
HUNG_EXIT = 124
SUSPECT_FACTOR = 3
SUSPECT_FLOOR_SECONDS = 1800
TRACE_LIMIT = 20
STATE, PARENT, SESSION, START = 0, 1, 3, 19
TIMES = slice(11, 15)
IO_KEYS = (b"rchar", b"wchar", b"read_bytes", b"write_bytes")
PAUSED = frozenset({b"T", b"t"})
GONE = frozenset({b"Z", b"X"})
ELSEWHERE = frozenset({b"docker", b"docker-compose", b"docker-buildx", b"podman", b"podman-compose", b"ssh"})
TICKS = os.sysconf("SC_CLK_TCK")
class Process(NamedTuple):
pid: int
comm: bytes
state: bytes
parent: int
session: int
ticks: int
start: int
class Sample(NamedTuple):
members: frozenset[tuple[int, int]]
ticks: int
io: tuple[int, ...]
elsewhere: tuple[bytes, ...]
paused: bool
@property
def held(self) -> bool:
return bool(self.elsewhere) or self.paused
def parse_stat(data: bytes) -> Process:
"""Args:
data: the bytes of /proc/<pid>/stat; field 2, the command name in
parentheses, may hold spaces and parentheses itself, so the
fields are split after the last `)`.
Returns:
The process: pid (field 1), comm (2), state (3), parent (4), session
(6), utime + stime + cutime + cstime (14 to 17, in clock ticks) and
start time (22, in clock ticks after boot).
"""
head, tail = data.rsplit(b")", 1)
pid, comm = head.split(b"(", 1)
fields = tail.split()
return Process(int(pid), comm, fields[STATE], int(fields[PARENT]), int(fields[SESSION]),
sum(int(ticks) for ticks in fields[TIMES]), int(fields[START]))
def read(name: str) -> Process | None:
try:
return parse_stat((PROC / name / "stat").read_bytes())
except (OSError, ValueError, IndexError):
return None
def table() -> dict[int, Process] | None:
"""Returns:
Every process /proc lists, by pid, skipping one that vanished
between the listing and the read; None when /proc cannot be listed.
"""
try:
names = [name for name in os.listdir(PROC) if name.isdigit()]
except OSError:
return None
return {found.pid: found for found in map(read, names) if found is not None}
def family(processes: dict[int, Process], roots: Iterable[int]) -> set[int]:
"""Returns:
The roots processes shows and all their descendants, whatever
session or process group each one moved to.
"""
children: dict[int, list[int]] = {}
for process in processes.values():
children.setdefault(process.parent, []).append(process.pid)
found: set[int] = set()
todo = [pid for pid in roots if pid in processes]
while todo:
pid = todo.pop()
if pid not in found:
found.add(pid)
todo.extend(children.get(pid, ()))
return found
def tree(processes: dict[int, Process], root: int) -> list[Process]:
"""Returns:
root, its descendants (family()) and every process still in the
session root leads, as an orphan handed to init keeps it.
"""
session = {process.pid for process in processes.values() if process.session == root}
return [processes[pid] for pid in family(processes, [root]) | session]
def io_of(pid: int) -> tuple[int, ...]:
"""Returns:
The IO_KEYS counters of /proc/<pid>/io; zeros when that file cannot
be read (a setuid member's, or one that vanished after its stat),
which leaves its CPU time counted.
"""
try:
pairs = dict(line.split(b":", 1) for line in (PROC / str(pid) / "io").read_bytes().splitlines())
return tuple(int(pairs[key]) for key in IO_KEYS)
except (OSError, ValueError, KeyError):
return (0,) * len(IO_KEYS)
def wchan_of(pid: int) -> str:
"""Returns:
The kernel function /proc/<pid>/wchan names as the one the process
sleeps in; empty while it runs and when that file cannot be read.
"""
try:
name = (PROC / str(pid) / "wchan").read_text(encoding="ascii", errors="replace")
except OSError:
return ""
return "" if name == "0" else name
def trace(root: int) -> list[str]:
"""Returns:
`<pid> <comm> <state> [<wchan>]` of the tree() members of root, the
ones without a child in the tree first (the stuck one, not the
ancestors waiting on it), then by pid, at most TRACE_LIMIT; comm with
control bytes escaped, as a process sets its own. Never a command
line, which can carry credentials. Empty when /proc cannot be listed
or does not show root.
"""
processes = table()
if processes is None or root not in processes:
return []
in_tree = tree(processes, root)
parents = {process.parent for process in in_tree}
return [" ".join(filter(None, (str(process.pid), process.comm.decode(errors="replace").encode("unicode_escape").decode(),
process.state.decode(), wchan_of(process.pid))))
for process in sorted(in_tree, key=lambda process: (process.pid in parents, process.pid))[:TRACE_LIMIT]]
def probe(root: int) -> Sample | None:
"""Args:
root: the pid of the held command.
Returns:
The tree() of root: its members as (pid, start time), their CPU
ticks and io_of() counters summed, the ELSEWHERE commands among them
and whether one is stopped or traced; None when /proc cannot be
listed or does not show root, as a /proc of another PID namespace
does not.
"""
processes = table()
if processes is None or root not in processes:
return None
in_tree = tree(processes, root)
return Sample(
frozenset((process.pid, process.start) for process in in_tree),
sum(process.ticks for process in in_tree),
tuple(map(sum, zip(*(io_of(process.pid) for process in in_tree)))),
tuple(sorted({process.comm for process in in_tree} & ELSEWHERE)),
any(process.state in PAUSED for process in in_tree),
)
def say(text: str) -> None:
print(f"queue: {text}", file=sys.stderr, flush=True)
class Clock:
"""Clock how long the process tree of a held command stood still.
Args:
root: the command's pid.
"""
def __init__(self, root: int) -> None:
self.root = root
self.seen: Sample | None = None
self.since = time.monotonic()
self.blind = False
self.away = False
def hung(self) -> bool:
"""Returns:
Whether probe() of the tree gave the same sample for
HUNG_SECONDS. A probe that sees nothing, a tree running an
ELSEWHERE client, whose work runs outside the tree, and a tree
with a stopped or traced member restart the clock and never fire.
"""
now = time.monotonic()
seen = probe(self.root)
self.warn(seen)
if seen is None or seen.held or seen != self.seen:
self.since = now
self.seen = seen
return seen is not None and not seen.held and now - self.since >= HUNG_SECONDS
def warn(self, seen: Sample | None) -> None:
"""Args:
seen: the probe() of this poll; the first None of the run says
on stderr that the guard is off, the first naming an
ELSEWHERE client that it holds off.
"""
if seen is None and not self.blind:
say(f"/proc does not show process {self.root}; the hung-job guard is off for it")
self.blind = True
if seen is not None and seen.elsewhere and not self.away:
names = ", ".join(name.decode(errors="replace") for name in seen.elsewhere)
say(f"the command runs {names}, whose work /proc does not show; the hung-job guard holds off while one runs")
self.away = True
def report(self) -> dict[str, float | int | list[str]]:
"""Returns:
The sample of the last hung() that saw the tree, for the history
row: cpu_s (CPU seconds), read_bytes and written_bytes (rchar and
wchar), still_min (minutes without a change), members (the
number of processes) and tree (their trace(), taken now, before
any signal).
"""
rchar, wchar = self.seen.io[:2]
return {"cpu_s": round(self.seen.ticks / TICKS, 2), "read_bytes": rchar, "written_bytes": wchar,
"still_min": round((time.monotonic() - self.since) / 60, 1), "members": len(self.seen.members),
"tree": trace(self.root)}
def told(report: dict) -> str:
return (f"the command's process tree stood still for {report['still_min']:g} min (CPU {report['cpu_s']:g} s, "
f"{report['read_bytes']} bytes read, {report['written_bytes']} written, all unchanged); "
f"tree of {report['members']}: {', '.join(report['tree'])}")
def members(root: int) -> dict[int, int]:
"""Returns:
The tree() of root as {pid: start time}; empty when /proc cannot be
listed or does not show root.
"""
processes = table()
if processes is None or root not in processes:
return {}
return {process.pid: process.start for process in tree(processes, root)}
def alive(targets: dict[int, int]) -> dict[int, int]:
"""Args:
targets: {pid: start time} as members() recorded them, the command
among them (a zombie still shows in /proc).
Returns:
The targets still running with their recorded start time, so a
reused pid never counts, the members of every session such a target
or a target gone from /proc leads, as a process a member starts
during the grace keeps its session when its parent exits (Linux
keeps a session id allocated while a member lives, so no reused pid
matches it), and the descendants all of them have now, as
{pid: start time}; zombies are left out, and nothing when /proc
cannot be listed.
"""
processes = table() or {}
kept = [pid for pid, start in targets.items() if pid in processes and processes[pid].start == start]
leaders = {pid for pid, start in targets.items() if pid not in processes or processes[pid].start == start}
kept += [process.pid for process in processes.values() if process.session in leaders]
return {pid: processes[pid].start for pid in family(processes, kept) if processes[pid].state not in GONE}
def signal_all(targets: dict[int, int], signum: int) -> dict[int, int]:
"""Send signum to every process alive() finds of targets.
Returns:
Those processes as {pid: start time}.
"""
living = alive(targets)
for pid in living:
with contextlib.suppress(ProcessLookupError, PermissionError):
os.kill(pid, signum)
return living
def settle(targets: dict[int, int], reap: Callable[[], int | None], signum: int | None) -> dict[int, int]:
"""Wait up to KILL_GRACE_SECONDS until the command is reaped and alive()
finds none of targets, recording every process it finds on the way, so
one a member starts in a session of its own during the grace stays a
target after its parent exits; one started and orphaned between two
passes, SETTLE_SECONDS and a /proc scan apart, is not seen.
Args:
targets: {pid: start time} of the tree.
reap: Popen.poll of the command, None while it runs.
signum: sent to everything each pass finds (signal_all()), so a
process forked while the last pass signalled gets it too; None
only waits, letting a member's cleanup run out the grace.
Returns:
targets and every process found, as {pid: start time}.
"""
seen = dict(targets)
deadline = time.monotonic() + KILL_GRACE_SECONDS
while time.monotonic() < deadline:
living = alive(seen) if signum is None else signal_all(seen, signum)
seen |= living
if reap() is not None and not living:
break
time.sleep(SETTLE_SECONDS)
return seen
def suspect_seconds(longest: float | None) -> float:
"""Args:
longest: the longest learned run of a job's signature in minutes,
None while nothing is learned.
Returns:
The runtime past which list marks the job suspect hung:
SUSPECT_FACTOR times longest, at least SUSPECT_FLOOR_SECONDS.
"""
if longest is None:
return SUSPECT_FLOOR_SECONDS
return max(SUSPECT_FLOOR_SECONDS, SUSPECT_FACTOR * longest * 60)
+560
View File
@@ -0,0 +1,560 @@
"""Exercise the kvb-org-queue hung-job guard: the process-tree probe, hold ending a tree that stands still and sparing one that works, the list mark and the learning that skips hung runs."""
from __future__ import annotations
import contextlib
import importlib.util
import io
import json
import os
import shutil
import signal
import subprocess
import sys
import tempfile
import time
import unittest
from pathlib import Path
from unittest import mock
REPO_ROOT = Path(__file__).resolve().parents[1]
SCRIPTS = REPO_ROOT / "skills" / "kvb-org-queue" / "scripts"
QUEUE = SCRIPTS / "queue.py"
HISTORY = SCRIPTS / "history.py"
STANDSTILL = SCRIPTS / "standstill.py"
NOW = 1_800_000_000.0
POLL = 0.1
HUNG = 2.0
GRACE = 0.5
OUTLAST = HUNG + 0.5
SAMPLE = {"free_gb": 50.0, "mem": 20.0, "cpu": 10.0}
PROBE_KEYS = {"cpu_s", "read_bytes", "written_bytes", "still_min", "members", "tree"}
OPEN_MEMORY = {
"available": -1, "swap_free": -1, "psi_some": 1000, "psi_full": 1000, "swap_out_mb_min": 1e9, "container_swap_gb": 1e9,
}
SLEEP = "import time\ntime.sleep(60)\n"
PRINT = (
f"import sys, time\nend = time.monotonic() + {OUTLAST}\n"
"while time.monotonic() < end:\n sys.stdout.write('x\\n')\n sys.stdout.flush()\n time.sleep(0.05)\n"
)
BURN = f"import time\nend = time.monotonic() + {OUTLAST}\nwhile time.monotonic() < end:\n pass\n"
STUBBORN = (
"import os, signal, sys, time\n"
"signal.signal(signal.SIGTERM, signal.SIG_IGN)\n"
"with open(sys.argv[1] + '.part', 'w') as out:\n"
" out.write(str(os.getpid()))\n"
"os.replace(sys.argv[1] + '.part', sys.argv[1])\n"
"time.sleep(60)\n"
)
MEMBER = (
"import os, signal, sys, time\n"
"def ended(*_):\n"
" with open(sys.argv[1], 'w') as out:\n"
" out.write('term')\n"
" os._exit(0)\n"
"signal.signal(signal.SIGTERM, ended)\n"
"time.sleep(60)\n"
)
SAME_SESSION = f"import subprocess, sys\nsubprocess.run([sys.executable, '-c', {BURN!r}], check=True)\n"
NEW_SESSION = f"import subprocess, sys\nsubprocess.run([sys.executable, '-c', {BURN!r}], check=True, start_new_session=True)\n"
FAMILY = (
"import subprocess, sys, time\n"
f"subprocess.Popen([sys.executable, '-c', {MEMBER!r}, sys.argv[1]])\n"
f"subprocess.Popen([sys.executable, '-c', {MEMBER!r}, sys.argv[2]], process_group=0)\n"
"time.sleep(60)\n"
)
ABANDONED = (
"import subprocess, sys, time\n"
f"subprocess.Popen([sys.executable, '-c', {STUBBORN!r}, sys.argv[1]], process_group=0)\n"
"time.sleep(60)\n"
)
CLEANUP_WORK = 0.5
def cleanup(own_session: bool, work: float) -> str:
"""Args:
own_session: whether the process the SIGTERM handler starts leads a
session of its own.
work: seconds the handler works once that process is ready, 0 to
exit at once; the session sweep alone, deterministically, is
test_signals_reach_members_whose_start_time_still_matches_the_session_and_their_new_descendants.
Returns:
A command whose SIGTERM handler starts STUBBORN, works, and exits.
"""
return (
"import os, signal, subprocess, sys, time\n"
"def ended(*_):\n"
f" subprocess.Popen([sys.executable, '-c', {STUBBORN!r}, sys.argv[1]], start_new_session={own_session})\n"
" while not os.path.exists(sys.argv[1]):\n"
" time.sleep(0.02)\n"
f" time.sleep({work})\n"
" os._exit(0)\n"
"signal.signal(signal.SIGTERM, ended)\n"
"time.sleep(60)\n"
)
CLEANUP = cleanup(False, 0)
SETSID_CLEANUP = cleanup(True, CLEANUP_WORK)
SESSION_CLEANUP = (
"import subprocess, sys, time\n"
f"subprocess.Popen([sys.executable, '-c', {CLEANUP!r}, sys.argv[1]], start_new_session=True)\n"
"time.sleep(60)\n"
)
CLEANUP_GRACE = 3.0
CLIENT = f"import subprocess, sys\nsubprocess.run([sys.argv[1], '{OUTLAST}'], check=True)\n"
STOPPED = (
"import signal, subprocess, sys, time\n"
"member = subprocess.Popen([sys.executable, '-c', 'import time; time.sleep(60)'])\n"
"member.send_signal(signal.SIGSTOP)\n"
f"time.sleep({OUTLAST})\n"
"member.kill()\n"
"member.wait()\n"
)
HOLD = (
"import importlib.util, sys\n"
"spec = importlib.util.spec_from_file_location('kvb_queue_hung', sys.argv[1])\n"
"queue = importlib.util.module_from_spec(spec)\n"
"spec.loader.exec_module(queue)\n"
f"queue.POLL_SECONDS = {POLL}\n"
f"queue.standstill.HUNG_SECONDS = {HUNG}\n"
f"queue.standstill.KILL_GRACE_SECONDS = {GRACE}\n"
"sys.argv = sys.argv[1:]\n"
"sys.exit(queue.main())\n"
)
def _load(path: Path):
spec = importlib.util.spec_from_file_location(f"{path.stem}_hung_under_test", path)
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
return module
def history_row(seq: int, **fields: object) -> dict:
return {
"id": f"r{seq}", "kind": "test", "signature": "w: make test", "workdir": "/w", "start": NOW - 600, "end": NOW,
"minutes": 10.0, "exit": 0, "free_before_gb": 50.0, "min_free_gb": 49.0, "disk_gb": 1.0, "mem_before": 20.0,
"mem_max": 30.0, "cpu_before": 10.0, "cpu_max": 40.0, "concurrent": 0, **fields,
}
def stat_line(pid: int, comm: str, parent: int, session: int, times: tuple[int, int, int, int], state: str, start: int) -> bytes:
after = [state, str(parent), str(pid), str(session), "0", "-1", "4194560", "100", "0", "0", "0",
*map(str, times), "20", "0", "1", "0", str(start), "0", "0"]
return f"{pid} ({comm}) {' '.join(after)}\n".encode()
def plain(pid: int, parent: int, session: int, utime: int) -> bytes:
return stat_line(pid, "make", parent, session, (utime, 0, 0, 0), "S", 1000 + pid)
def io_lines(rchar: int, wchar: int, read_bytes: int, write_bytes: int) -> bytes:
return (f"rchar: {rchar}\nwchar: {wchar}\nsyscr: 1\nsyscw: 1\nread_bytes: {read_bytes}\n"
f"write_bytes: {write_bytes}\ncancelled_write_bytes: 0\n").encode()
def awaited(path: Path) -> int:
deadline = time.monotonic() + 30
while not path.exists():
if time.monotonic() > deadline:
raise AssertionError(f"{path} never appeared")
time.sleep(0.02)
return int(path.read_text(encoding="utf-8"))
def ended(pid: int) -> bool:
try:
stat = Path(f"/proc/{pid}/stat").read_bytes()
except OSError:
return True
return stat.rsplit(b")", 1)[1].split()[0] in (b"Z", b"X")
def kill_quietly(pid: int) -> None:
with contextlib.suppress(ProcessLookupError):
os.kill(pid, signal.SIGKILL)
class TestProbe(unittest.TestCase):
@classmethod
def setUpClass(cls):
cls.standstill = _load(STANDSTILL)
def setUp(self):
tmp = tempfile.TemporaryDirectory()
self.addCleanup(tmp.cleanup)
self.proc = Path(tmp.name) / "proc"
self.proc.mkdir()
patched = mock.patch.object(self.standstill, "PROC", self.proc)
patched.start()
self.addCleanup(patched.stop)
def process(self, pid: int, stat: bytes | None, io: bytes | None) -> None:
folder = self.proc / str(pid)
folder.mkdir(exist_ok=True)
if stat is not None:
(folder / "stat").write_bytes(stat)
if io is not None:
(folder / "io").write_bytes(io)
def test_stat_splits_after_the_last_parenthesis_so_a_command_name_with_parentheses_reads_right(self):
line = stat_line(4242, "a) (b c) 9 9", 7, 4242, (7, 3, 2, 1), "T", 999)
self.assertEqual(self.standstill.parse_stat(line), (4242, b"a) (b c) 9 9", b"T", 7, 4242, 13, 999))
self.assertNotEqual(line.split(b")", 1)[1].split()[3], b"4242")
def test_the_probe_walks_the_whole_tree_whatever_its_session_and_skips_vanished_and_foreign_processes(self):
self.process(4242, plain(4242, 100, 4242, 10), io_lines(0, 100, 0, 0))
self.process(4243, plain(4243, 4242, 4243, 5), None)
self.process(4244, plain(4244, 4243, 4244, 7), io_lines(30, 0, 0, 0))
self.process(4245, plain(4245, 1, 4242, 1), None)
self.process(4246, plain(4246, 1, 1, 100), io_lines(9000, 9000, 0, 0))
self.process(4247, None, None)
(self.proc / "self").mkdir()
seen = self.standstill.probe(4242)
self.assertEqual(seen.members, {(pid, 1000 + pid) for pid in (4242, 4243, 4244, 4245)})
self.assertEqual((seen.ticks, seen.io, seen.elsewhere, seen.paused), (23, (30, 100, 0, 0), (), False))
self.assertEqual(self.standstill.probe(4246).ticks, 100)
self.assertIsNone(self.standstill.probe(4999))
def test_cpu_time_reads_writes_and_a_new_process_each_count_as_progress(self):
self.process(4242, plain(4242, 1, 4242, 1), io_lines(5, 10, 0, 0))
clock = self.standstill.Clock(4242)
changes = (
(4242, plain(4242, 1, 4242, 2), None),
(4242, None, io_lines(6, 10, 0, 0)),
(4242, None, io_lines(6, 11, 0, 0)),
(4242, None, io_lines(6, 11, 1, 0)),
(4242, None, io_lines(6, 11, 1, 1)),
(4243, plain(4243, 4242, 4243, 0), None),
)
with mock.patch.object(self.standstill, "HUNG_SECONDS", 0.2):
self.assertFalse(clock.hung())
for pid, stat, counters in changes:
time.sleep(0.25)
self.assertTrue(clock.hung())
self.process(pid, stat, counters)
self.assertFalse(clock.hung(), (pid, stat, counters))
time.sleep(0.25)
self.assertTrue(clock.hung())
report = clock.report()
self.assertEqual((report["cpu_s"], report["read_bytes"], report["written_bytes"]), (round(2 / self.standstill.TICKS, 2), 6, 11))
self.assertGreaterEqual(report["still_min"], 0)
self.assertEqual((report["members"], report["tree"]), (2, ["4243 make S", "4242 make S"]))
def test_the_trace_names_the_leaves_first_with_state_and_wait_channel_but_never_a_command_line(self):
self.process(4242, plain(4242, 1, 4242, 0), None)
self.process(4243, stat_line(4243, "sleep", 4242, 4242, (0, 0, 0, 0), "D", 5243), None)
self.process(4244, plain(4244, 4242, 4242, 0), None)
self.process(4245, stat_line(4245, "x\x1b[2J", 4242, 4242, (0, 0, 0, 0), "S", 5245), None)
for pid, wchan in ((4242, "do_wait"), (4243, "folio_wait_bit"), (4244, "0")):
(self.proc / str(pid) / "wchan").write_text(wchan, encoding="ascii")
(self.proc / "4243" / "cmdline").write_bytes(b"sleep\0--token=secret\0")
leaves = ["4243 sleep D folio_wait_bit", "4244 make S", "4245 x\\x1b[2J S"]
self.assertEqual(self.standstill.trace(4242), [*leaves, "4242 make S do_wait"])
with mock.patch.object(self.standstill, "TRACE_LIMIT", 2):
self.assertEqual(self.standstill.trace(4242), leaves[:2])
self.assertEqual(self.standstill.trace(4999), [])
def test_a_stopped_member_or_a_client_whose_work_runs_elsewhere_never_fires_and_the_client_is_told_once(self):
self.process(4242, plain(4242, 1, 4242, 1), None)
held = (("docker", "S"), ("docker-compose", "S"), ("podman", "S"), ("ssh", "S"), ("sleep", "T"), ("python3", "t"))
with mock.patch.object(self.standstill, "HUNG_SECONDS", 0), contextlib.redirect_stderr(io.StringIO()) as err:
self.assertTrue(self.standstill.Clock(4242).hung())
for comm, state in held:
self.process(4243, stat_line(4243, comm, 4242, 4243, (0, 0, 0, 0), state, 5000), None)
clock = self.standstill.Clock(4242)
self.assertEqual([clock.hung() for _ in range(3)], [False, False, False], comm)
self.assertEqual(err.getvalue().count("holds off"), 4)
def test_an_unreadable_proc_never_fires_and_says_so_once(self):
self.process(4242, plain(4242, 1, 4242, 1), io_lines(0, 10, 0, 0))
with mock.patch.object(self.standstill, "HUNG_SECONDS", 0):
self.assertTrue(self.standstill.Clock(4242).hung())
clock = self.standstill.Clock(4242)
with mock.patch.object(self.standstill, "PROC", self.proc / "missing"), \
contextlib.redirect_stderr(io.StringIO()) as err:
self.assertEqual([clock.hung() for _ in range(3)], [False, False, False])
self.assertEqual(err.getvalue().count("hung-job guard is off"), 1)
def test_signals_reach_members_whose_start_time_still_matches_the_session_and_their_new_descendants(self):
self.process(4242, plain(4242, 1, 4242, 0), None)
self.process(4243, plain(4243, 4242, 4242, 0), None)
self.process(4244, stat_line(4244, "make", 4242, 4242, (0, 0, 0, 0), "Z", 5244), None)
self.process(4245, plain(4245, 1, 4242, 0), None)
self.process(4250, plain(4250, 4243, 4250, 0), None)
self.process(4260, plain(4260, 1, 1, 0), None)
self.process(4270, plain(4270, 1, 4270, 0), None)
self.process(4271, plain(4271, 4270, 4271, 0), None)
self.process(4281, plain(4281, 1, 4280, 0), None)
self.assertEqual(self.standstill.members(4242), {pid: 1000 + pid for pid in (4242, 4243, 4244, 4245, 4250)})
self.assertEqual(self.standstill.members(4999), {})
with mock.patch.object(self.standstill.os, "kill") as kill:
living = self.standstill.signal_all({4242: 5242, 4244: 5244, 4270: 1, 4280: 5280}, signal.SIGTERM)
self.assertEqual(living, {4242: 5242, 4243: 5243, 4245: 5245, 4250: 5250, 4281: 5281})
self.assertEqual(sorted(call.args for call in kill.call_args_list),
[(pid, signal.SIGTERM) for pid in (4242, 4243, 4245, 4250, 4281)])
def test_the_suspect_runtime_is_three_times_the_longest_run_with_a_floor(self):
self.assertEqual(self.standstill.suspect_seconds(20), 3600)
self.assertEqual(self.standstill.suspect_seconds(5), self.standstill.SUSPECT_FLOOR_SECONDS)
self.assertEqual(self.standstill.suspect_seconds(None), self.standstill.SUSPECT_FLOOR_SECONDS)
class TestSupervise(unittest.TestCase):
@classmethod
def setUpClass(cls):
cls.queue = _load(QUEUE)
def start(self, code: str, *args: str) -> subprocess.Popen:
child = subprocess.Popen([sys.executable, "-c", code, *args], start_new_session=True,
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
self.addCleanup(child.wait, 30)
self.addCleanup(self.queue.send, child, signal.SIGKILL)
return child
def scratch(self) -> Path:
tmp = tempfile.TemporaryDirectory()
self.addCleanup(tmp.cleanup)
return Path(tmp.name)
def supervised(self, child: subprocess.Popen, *extra) -> tuple[tuple[int, dict | None], float, str]:
"""Args:
child: the started command.
extra: patches entered after the defaults, so they win.
"""
patches = (
mock.patch.object(self.queue, "POLL_SECONDS", POLL),
mock.patch.object(self.queue.standstill, "HUNG_SECONDS", HUNG),
mock.patch.object(self.queue.standstill, "KILL_GRACE_SECONDS", GRACE),
mock.patch.object(self.queue, "measure", return_value=SAMPLE),
*extra,
)
with contextlib.ExitStack() as stack, contextlib.redirect_stderr(io.StringIO()) as err:
for patch in patches:
stack.enter_context(patch)
started = time.monotonic()
result = self.queue.supervise({}, child, [])
return result, time.monotonic() - started, err.getvalue()
def test_a_sleeping_command_is_ended_with_124_and_its_probe(self):
child = self.start(SLEEP)
(code, probe), took, err = self.supervised(child)
self.assertEqual(code, 124)
self.assertEqual(set(probe), PROBE_KEYS)
self.assertEqual(probe["members"], 1)
self.assertRegex(probe["tree"][0], rf"^{child.pid} \S+ S")
self.assertEqual(child.returncode, -signal.SIGTERM)
self.assertLess(took, HUNG + 10)
self.assertIn("stood still", err)
self.assertIn(f"tree of 1: {child.pid} ", err)
def test_a_command_that_keeps_printing_is_not_ended(self):
self.assertEqual(self.supervised(self.start(PRINT))[0], (0, None))
def test_a_command_that_burns_cpu_without_printing_is_not_ended(self):
self.assertEqual(self.supervised(self.start(BURN))[0], (0, None))
def test_a_command_ignoring_sigterm_is_killed_after_the_grace(self):
ready = self.scratch() / "ready"
child = self.start(STUBBORN, str(ready))
awaited(ready)
(code, probe), took, _ = self.supervised(child)
self.assertEqual((code, child.returncode, set(probe)), (124, -signal.SIGKILL, PROBE_KEYS))
self.assertGreaterEqual(took, HUNG + GRACE)
def test_a_working_grandchild_counts_in_the_same_session_and_in_a_new_one(self):
self.assertEqual(self.supervised(self.start(SAME_SESSION))[0], (0, None))
self.assertEqual(self.supervised(self.start(NEW_SESSION))[0], (0, None))
def test_a_sleeping_tree_is_ended_and_every_member_gets_sigterm_whatever_its_process_group(self):
folder = self.scratch()
marks = [folder / "same-group", folder / "own-group"]
child = self.start(FAMILY, *map(str, marks))
(code, _), _, _ = self.supervised(child)
self.assertEqual((code, child.returncode), (124, -signal.SIGTERM))
self.assertEqual([mark.read_text(encoding="utf-8") for mark in marks], ["term", "term"])
def test_a_member_ignoring_sigterm_is_killed_after_the_grace_although_the_leader_already_exited(self):
ready = self.scratch() / "ready"
child = self.start(ABANDONED, str(ready))
member = awaited(ready)
(code, _), took, _ = self.supervised(child)
self.assertEqual((code, child.returncode), (124, -signal.SIGTERM))
self.assertTrue(ended(member))
self.assertGreaterEqual(took, HUNG + GRACE)
def test_a_process_a_member_starts_during_the_grace_is_killed_although_its_parent_exited(self):
cases = (("same session", CLEANUP, 0), ("member in own session", SESSION_CLEANUP, -signal.SIGTERM),
("orphan in own session", SETSID_CLEANUP, 0))
for case, script, leader_exit in cases:
with self.subTest(case):
ready = self.scratch() / "ready"
child = self.start(script, str(ready))
grace = mock.patch.object(self.queue.standstill, "KILL_GRACE_SECONDS", CLEANUP_GRACE)
(code, _), _, _ = self.supervised(child, grace)
orphan = awaited(ready)
self.addCleanup(kill_quietly, orphan)
self.assertEqual((code, child.returncode), (124, leader_exit))
self.assertTrue(ended(orphan))
def test_a_process_forked_while_sigkill_goes_out_is_killed_on_the_next_pass(self):
proc = self.scratch() / "proc"
def spawn(pid: int, parent: int) -> None:
(proc / str(pid)).mkdir(parents=True)
(proc / str(pid) / "stat").write_bytes(plain(pid, parent, 4242, 0))
def kill(pid: int, signum: int) -> None:
if signum == signal.SIGKILL:
shutil.rmtree(proc / str(pid))
if pid == 4243:
spawn(4250, 1)
spawn(4242, 1)
spawn(4243, 4242)
child = mock.Mock(pid=4242, returncode=None, poll=lambda: None if (proc / "4242").exists() else 0)
with mock.patch.object(self.queue.standstill, "PROC", proc), \
mock.patch.object(self.queue.standstill, "KILL_GRACE_SECONDS", GRACE), \
mock.patch.object(self.queue.standstill.os, "kill", side_effect=kill) as sent, \
mock.patch.object(self.queue.os, "killpg") as group:
self.assertEqual(self.queue.stop(child), 124)
self.assertIn(mock.call(4250, signal.SIGKILL), sent.call_args_list)
self.assertEqual(sorted(path.name for path in proc.iterdir()), [])
group.assert_not_called()
def test_a_tree_running_a_container_client_never_fires_and_says_so_once(self):
sleep = shutil.which("sleep")
if sleep is None:
self.skipTest("no sleep binary on PATH")
docker = self.scratch() / "docker"
docker.symlink_to(sleep)
result, _, err = self.supervised(self.start(CLIENT, str(docker)))
self.assertEqual(result, (0, None))
self.assertEqual(err.count("the command runs docker,"), 1)
def test_a_tree_with_a_stopped_member_never_fires(self):
self.assertEqual(self.supervised(self.start(STOPPED))[0], (0, None))
def test_stop_falls_back_to_the_process_group_when_proc_shows_nothing(self):
child = self.start(SLEEP)
with mock.patch.object(self.queue.standstill, "PROC", self.scratch() / "missing"), \
mock.patch.object(self.queue.standstill, "KILL_GRACE_SECONDS", GRACE):
self.assertEqual(self.queue.stop(child), 124)
self.assertEqual(child.returncode, -signal.SIGTERM)
class TestHungHistory(unittest.TestCase):
@classmethod
def setUpClass(cls):
cls.history = _load(HISTORY)
cls.queue = _load(QUEUE)
def learn(self, rows: list[dict]) -> dict:
return self.history.learn(rows, NOW, {"load_limits": {"mem_percent": 80, "cpu_percent": 90}})[0]
def test_a_hung_row_stays_out_of_the_learned_durations_and_shares(self):
rows = [history_row(n) for n in range(5)]
probe = {"cpu_s": 1.0, "read_bytes": 0, "written_bytes": 0, "still_min": 60.0}
stuck = history_row(5, minutes=500.0, mem_max=95.0, exit=124)
stats = self.learn([*rows, {**stuck, self.history.HUNG: probe}])["test", "w: make test"]
self.assertEqual((stats["runs"], stats["minutes"], stats["longest"], stats["ram_share"]), (5, 10.0, 10.0, 10.0))
counted = self.learn([*rows, stuck])["test", "w: make test"]
self.assertEqual((counted["runs"], counted["longest"]), (6, 500.0))
def test_list_marks_an_active_job_past_three_times_its_longest_run_and_past_the_floor(self):
signatures = self.learn([history_row(n, minutes=20.0 - n) for n in range(5)])
entry = {"kind": "test", "aliases": ["w: make test"], "granted": NOW}
suspected = self.queue.suspected
self.assertIsNone(suspected({**entry, "granted": NOW - 59 * 60}, None, signatures, NOW))
self.assertEqual(suspected({**entry, "granted": NOW - 61 * 60}, None, signatures, NOW), "suspect hung (61 min)")
self.assertEqual(suspected({**entry, "granted": NOW - 61 * 60}, "backfilled", signatures, NOW),
"suspect hung (61 min); backfilled")
unknown = {**entry, "aliases": ["w: make lint"]}
self.assertEqual(suspected({**unknown, "granted": NOW - 29 * 60}, "backfilled", signatures, NOW), "backfilled")
self.assertEqual(suspected({**unknown, "granted": NOW - 31 * 60}, None, signatures, NOW), "suspect hung (31 min)")
short = self.learn([history_row(n, minutes=5.0) for n in range(5)])
self.assertIsNone(suspected({**entry, "granted": NOW - 20 * 60}, None, short, NOW))
class TestHungHold(unittest.TestCase):
def setUp(self):
tmp = tempfile.TemporaryDirectory()
self.addCleanup(tmp.cleanup)
self.base = Path(tmp.name).resolve()
self.root = self.base / "council"
self.root.mkdir()
home = self.base / "home"
home.mkdir()
self.history_file = self.root / "queue-history.jsonl"
self.config = self.base / "config.json"
self.config.write_text(json.dumps({
"disk": str(self.base),
"thresholds_gb": {"warn": 0, "stop": 0, "critical": 0},
"projection_minutes": 30,
"slot_need_gb": {"test": 0.001},
"slot_limits": {"test": 1},
"load_limits": {"cpu_percent": 101, "mem_percent": 101, "sample_seconds": 0.1},
"budget_percent": {"ram": 75, "cpu": 100},
"memory_limits": {stage: OPEN_MEMORY for stage in ("warn", "stop")},
"heavy_commands": ["^make compose-deploy"],
"never_block": ["^make compose-down"],
}), encoding="utf-8")
(self.root / "rasci.json").write_text(json.dumps({"work/a": {"roles": {}, "status": "running"}}), encoding="utf-8")
self.env = {
"PATH": os.environ["PATH"],
"HOME": str(home),
"GIT_CEILING_DIRECTORIES": str(self.base.parent),
"AGENTS_COUNCIL_ROOT": str(self.root),
"AGENTS_COUNCIL_CONFIG": str(self.config),
"CLAUDE_PROJECT_DIR": str(self.base / "project"),
}
def run_queue(self, *args: str) -> subprocess.CompletedProcess:
return subprocess.run(
[sys.executable, "-c", HOLD, str(QUEUE), *args], cwd=self.base, env=self.env, stdin=subprocess.DEVNULL,
capture_output=True, text=True, timeout=60, check=False,
)
def hold(self, code: str) -> subprocess.CompletedProcess:
return self.run_queue("hold", "--kind", "test", "--task", "work/a", "--session", "s1", "--", sys.executable, "-c", code)
def rows(self) -> list[dict]:
return [json.loads(line) for line in self.history_file.read_text(encoding="utf-8").splitlines()]
def test_a_hold_whose_command_sleeps_exits_124_and_its_history_row_marks_it_hung(self):
result = self.hold(SLEEP)
self.assertEqual(result.returncode, 124, result.stderr)
self.assertIn("stood still", result.stderr)
[row] = self.rows()
self.assertEqual((row["exit"], set(row["hung"])), (124, PROBE_KEYS))
self.assertEqual(json.loads((self.root / "queue.json").read_text(encoding="utf-8"))["entries"], [])
def test_a_hold_whose_command_works_keeps_its_exit_code_and_an_unmarked_row(self):
result = self.hold(BURN + "raise SystemExit(5)\n")
self.assertEqual((result.returncode, result.stderr), (5, ""))
[row] = self.rows()
self.assertEqual(row["exit"], 5)
self.assertNotIn("hung", row)
def test_list_marks_an_active_job_past_the_floor_when_nothing_is_learned(self):
queue = _load(QUEUE)
me, mine = os.getpid(), queue.identity(os.getpid())
now = time.time()
floor = queue.standstill.SUSPECT_FLOOR_SECONDS
entry = {
"seq": 1, "kind": "test", "task": "work/a", "need_gb": 0, "hotfix": None, "workdir": "/w", "session": "s1",
"name": "s1", "pid": me, "pid_start": mine, "child": me, "child_start": mine, "status": "active",
"enqueued": now - floor - 120, "cmd": "make test", "code": queue.CODE,
}
entries = [{**entry, "id": "test-1", "granted": now - floor - 60}, {**entry, "id": "test-2", "seq": 2, "granted": now - floor + 60}]
(self.root / "queue.json").write_text(json.dumps({"seq": 2, "entries": entries}), encoding="utf-8")
listing = self.run_queue("list")
self.assertEqual(listing.returncode, 0, listing.stderr)
reasons = [line.split("\t")[-1] for line in listing.stdout.splitlines()[1:]]
self.assertEqual(reasons, [f"suspect hung ({(floor + 60) // 60} min)", "-"])
if __name__ == "__main__":
unittest.main()