diff --git a/skills/kvb-org-organize/capacity.md b/skills/kvb-org-organize/capacity.md index 076932f..fd5f97e 100644 --- a/skills/kvb-org-organize/capacity.md +++ b/skills/kvb-org-organize/capacity.md @@ -34,7 +34,11 @@ step inside it, or a process its own ended session left behind, once at least child process, no network traffic, CPU time not moving). It secures the live state first, then reports the timestamp, the step, the probe and the command. A peer's hung job goes to its owner, and to the operator when the owner does -not answer. +not answer. `queue.py hold` does this itself for a queued job: after 60 +minutes without CPU, I/O or new process in its tree, no stopped member and no +docker, podman or ssh client, it records the stuck processes, ends the tree +and exits 124; `queue.py list` marks a run past three times its longest +learned one `suspect hung`. ## Resource watch diff --git a/skills/kvb-org-queue/SKILL.md b/skills/kvb-org-queue/SKILL.md index e341fd7..5608f8d 100644 --- a/skills/kvb-org-queue/SKILL.md +++ b/skills/kvb-org-queue/SKILL.md @@ -77,12 +77,12 @@ queue.py cancel - Never ask the chair for a heavy slot or announce a grant; the chair handles only a stuck queue or hung job (`capacity.md` of `kvb-org-organize`). -- A running job is never cut short for the queue; only the operator orders - one down, and `kvb-git-hotfix` may void a gate. `kvb-org-queue-cleanup` +- Only the operator, a `kvb-git-hotfix` gate void, or hold after 60 minutes + of tree standstill (exit 124) cut a job short. `kvb-org-queue-cleanup` deletes only entries whose lock no living process holds. - With `queue_enforce: true`, the `block` hook (PreToolUse on Bash) refuses `heavy_commands`, image builds, `make install|test*|quality*|verify-*`, tests beyond one unit file and `gate.py run|hold` outside `queue.py hold`. - Without the config, hold exits 2: tell the operator, ask the chair - (`kvb-org-organize`). A waiting hold rereads it every poll - (a broken file keeps the last good one). + (`kvb-org-organize`). Waiting holds reread it every poll, keeping the + last good one. diff --git a/skills/kvb-org-queue/learning.md b/skills/kvb-org-queue/learning.md index 327ec59..2aa56d1 100644 --- a/skills/kvb-org-queue/learning.md +++ b/skills/kvb-org-queue/learning.md @@ -2,7 +2,8 @@ Every finished job appends a row (signature such as `: make `, kind, minutes, disk, RAM and CPU) to `queue-history.jsonl`; the -`kvb-org-organize` watch hook keeps it across reboots. +`kvb-org-organize` watch hook keeps it across reboots. The row of a run +hold ended for standing still (exit 124, a `hung` probe) teaches nothing. From the last 30 days, once 5 runs exist: diff --git a/skills/kvb-org-queue/scripts/history.py b/skills/kvb-org-queue/scripts/history.py index 3276c2e..95421b2 100644 --- a/skills/kvb-org-queue/scripts/history.py +++ b/skills/kvb-org-queue/scripts/history.py @@ -6,7 +6,9 @@ and the copy the kvb-org-organize watch hook keeps in ~/.local/state. A row: id, kind, signature, workdir, start, end (epoch seconds), minutes, exit, free_before_gb, min_free_gb, disk_gb (free_before_gb - min_free_gb, at least 0), mem_before, mem_max, cpu_before, cpu_max (percent), concurrent (other -active jobs at the grant). +active jobs at the grant), and HUNG ({cpu_s, read_bytes, written_bytes, +still_min, members, tree}, the report of standstill.py) only on a run hold +ended for standing still, whose exit is 124. """ from __future__ import annotations @@ -33,6 +35,7 @@ NAME_KEY = operator.itemgetter("signature") KIND_KEY = operator.itemgetter("kind") ANY_KIND = "*" RAM_CAP = "ram_cap_percent" +HUNG = "hung" SIGNATURE_HEADER = ("Signature", "Kind", "Runs", "Median min", "p90 disk GB", "p90 RAM %", "Need GB") KIND_HEADER = ("Kind", "Runs", "p90 RAM share %", "Median CPU share %", "Slot limit", "Source") @@ -156,11 +159,12 @@ def grouped(rows: list[dict], key: Callable[[dict], Hashable]) -> dict[Hashable, def command_stats(rows: list[dict]) -> dict: - stats = {"runs": len(rows), "minutes": None, "disk_gb": None, "ram_share": None, "cpu_share": None, "need_gb": None} + stats = {"runs": len(rows), "minutes": None, "longest": None, "disk_gb": None, "ram_share": None, "cpu_share": None, + "need_gb": None} if len(rows) >= MIN_RUNS: disk = p90([r["disk_gb"] for r in rows]) stats.update( - minutes=statistics.median(r["minutes"] for r in rows), disk_gb=disk, + minutes=statistics.median(r["minutes"] for r in rows), longest=max(r["minutes"] for r in rows), disk_gb=disk, ram_share=p90(shares(rows, "mem")), cpu_share=statistics.median(shares(rows, "cpu")), need_gb=math.ceil(disk + DISK_MARGIN_GB), ) @@ -186,7 +190,8 @@ def learn(rows: list[dict], now: float, cfg: dict) -> tuple[dict[tuple[str, str] """Derive disk need per command and slot limit per kind from recent rows. Args: - rows: history rows, as read() returns them. + rows: history rows, as read() returns them; HUNG rows are ignored, as + their standstill would stretch the minutes and the shares. now: epoch seconds; rows that ended more than WINDOW_SECONDS earlier are ignored. cfg: the config; load_limits.mem_percent and cpu_percent bound a limit, @@ -195,19 +200,20 @@ def learn(rows: list[dict], now: float, cfg: dict) -> tuple[dict[tuple[str, str] Returns: (signatures, kinds). signatures: {(kind, signature): {runs, minutes, - disk_gb, ram_share, cpu_share, need_gb}}, and {(ANY_KIND, signature): + longest, disk_gb, ram_share, cpu_share, need_gb}}, and {(ANY_KIND, signature): ...} over the rows of every kind with that signature, which is what a queue before the per-kind learning booked; kinds: {kind: {runs, ram_share, cpu_share, limit, light, ram_cap}}, ram_cap None without a cap. With fewer than MIN_RUNS rows every value but runs, light and ram_cap is None. - minutes is the median, disk_gb and ram_share the p90 of the disk taken + minutes is the median and longest the largest minutes, disk_gb and + ram_share the p90 of the disk taken and of max minus before in percentage points, cpu_share the median of max minus before; need_gb is ceil(disk_gb + DISK_MARGIN_GB); limit is max(1, min over RAM and CPU of floor((load limit - median before) / max(share, MIN_SHARE_PERCENT))). """ - recent = [r for r in rows if now - r["end"] <= WINDOW_SECONDS] + recent = [r for r in rows if now - r["end"] <= WINDOW_SECONDS and HUNG not in r] light = set(cfg.get("light_kinds", [])) caps = cfg.get(RAM_CAP, {}) signatures = {(ANY_KIND, name): command_stats(group) for name, group in grouped(recent, NAME_KEY).items()} diff --git a/skills/kvb-org-queue/scripts/queue.py b/skills/kvb-org-queue/scripts/queue.py index 140f922..fec494f 100755 --- a/skills/kvb-org-queue/scripts/queue.py +++ b/skills/kvb-org-queue/scripts/queue.py @@ -41,9 +41,10 @@ Args: proposing it (latched()); measure disk, RAM and CPU, run CMD in the current directory with inherited stdio in a session of its own, forward SIGINT and SIGTERM to its process group, measure every - POLL_SECONDS and at its end, record its history row and leave the - queue. The entry carries its code (a hash of this file and the - SIBLINGS); a poll that finds other code installed in + POLL_SECONDS and at its end, end its process tree once the tree stood + still for standstill.HUNG_SECONDS (supervise()), record its history + row and leave the queue. The entry carries its code (a hash of this + file and the SIBLINGS); a poll that finds other code installed in ~/.agents/skills leaves the queue ungranted and restarts the same hold with the installed queue.py, which keeps its entry id, place and enqueue time (hand_over()). @@ -74,7 +75,9 @@ Args: poll, host load included, such as `waits in line behind ` or `drained for `, the head's after `escalated : ` once the stall reached it; `backfilled` for a job started past a blocked one; - prefixed when the entry runs other code than the installed one). + led by `suspect hung ( min)` for an active job past + standstill.suspect_seconds() (suspected()); prefixed when the entry + runs other code than the installed one). Entries of dead holders are dropped first. --json: print the same rows as a JSON list (listed()), each with the trace of the evaluate() pass that set its reason. @@ -109,7 +112,8 @@ watch.py documents them. A signature is `: ` (aliases()). Exit codes: - hold: the command's exit code unchanged, 128+N when signal N ended it; + hold: the command's exit code unchanged, 128+N when signal N ended it, + standstill.HUNG_EXIT (124) when hold ended it for standing still; a command exiting 75 (EX_TEMPFAIL: it did no work, such as a declined `operator.py granted`) leaves no history row; before the command started 2 on a usage or config error or when it @@ -154,6 +158,7 @@ SIBLINGS = { "council": SKILLS / "kvb-org-organize" / "scripts" / "council.py", "rasci": SKILLS / "kvb-org-rasci" / "scripts" / "rasci.py", "history": Path(__file__).resolve().parent / "history.py", + "standstill": Path(__file__).resolve().parent / "standstill.py", "nofollow": SKILLS / "kvb-org-organize" / "scripts" / "nofollow.py", "councilroot": SKILLS / "kvb-org-organize" / "scripts" / "councilroot.py", } @@ -228,6 +233,7 @@ def sibling(name: str) -> ModuleType: watch = sibling("watch") history = sibling("history") +standstill = sibling("standstill") nofollow = sibling("nofollow") STORE = watch.ROOT / "queue.json" LOCK = watch.ROOT / "queue.lock" @@ -1444,23 +1450,54 @@ def measure(cfg: dict) -> dict[str, float]: return {"free_gb": free, "mem": mem, "cpu": cpu} -def supervise(cfg: dict, child: subprocess.Popen, samples: list[dict[str, float]]) -> int: +def supervise(cfg: dict, child: subprocess.Popen, samples: list[dict[str, float]]) -> tuple[int, dict | None]: """Wait for the child, measuring every POLL_SECONDS and once at its end. Args: cfg: the config. - child: the running command. + child: the running command, leading a session of its own. samples: the measurements so far; each new one is appended. Returns: - The child's return code. + (the child's return code, None); (standstill.HUNG_EXIT, the probe of + standstill.Clock.report()) once its process tree stood still for + standstill.HUNG_SECONDS and stop() ended it. """ + clock = standstill.Clock(child.pid) code = None while code is None: with contextlib.suppress(subprocess.TimeoutExpired): code = child.wait(timeout=POLL_SECONDS) samples.append(measure(cfg)) - return code + if code is None and clock.hung() and child.poll() is None: + probe = clock.report() + print(f"queue: {standstill.told(probe)}; ending the tree", file=sys.stderr, flush=True) + return stop(child), probe + return code, None + + +def stop(child: subprocess.Popen) -> int: + """SIGTERM every process of the child's tree (standstill.members()), of + the sessions its members lead and the descendants those have now, + SIGKILL the ones still there after standstill.KILL_GRACE_SECONDS and + those standstill.settle() found during it, again at each pass of the + SIGKILL grace, a recorded member only while its start time matches; the + child's process group (send()) when /proc does not show the child. A + process that leaves the tree and every recorded session between two + passes (setsid, then its parent dies) escapes; only a subreaper or a + cgroup per job would hold it. + + Returns: + standstill.HUNG_EXIT, also when a process outlives SIGKILL by another + grace (uninterruptible sleep), so hold never waits on it. + """ + targets = standstill.members(child.pid) + for signum, repeat in ((signal.SIGTERM, None), (signal.SIGKILL, signal.SIGKILL)): + living = standstill.signal_all(targets, signum) + if child.pid not in living: + send(child, signum) + targets = standstill.settle(targets | living, child.poll, repeat) + return standstill.HUNG_EXIT def keep(row: dict) -> None: @@ -1582,10 +1619,10 @@ def cmd_hold(args: argparse.Namespace) -> int: for signum in caught: send(child, signum) update(entry["id"], child=child.pid, child_start=identity(child.pid)) - code = supervise(cfg, child, samples) + code, hung = supervise(cfg, child, samples) code = code if code >= 0 else 128 - code job = {"id": f"{entry['id']}-{time.time_ns()}", "kind": args.kind, "signature": names[0], - "workdir": entry["workdir"]} + "workdir": entry["workdir"], **({} if hung is None else {history.HUNG: hung})} record(job, start, code, samples, concurrent) keep_receipt(gates, args, here, before, code) return code @@ -1607,7 +1644,8 @@ def cmd_list(args: argparse.Namespace) -> int: by_id = {e["id"]: e for e in entries} running = [(key_of(e), e["granted"]) for e in entries if e["status"] == "active"] rows = [ - (None, e, history.eta(signatures, [(key_of(e), e["granted"])], [], now), e.get("reason") or "-", e.get("trace", [])) + (None, e, history.eta(signatures, [(key_of(e), e["granted"])], [], now), suspected(e, e.get("reason"), signatures, now) or "-", + e.get("trace", [])) for e in entries if e["status"] == "active" ] ahead: list[tuple[str, str]] = [] @@ -1633,6 +1671,20 @@ def cmd_list(args: argparse.Namespace) -> int: return 0 +def suspected(entry: dict, reason: str | None, signatures: dict[tuple[str, str], dict], now: float) -> str | None: + """Returns: + reason led by `suspect hung ( min)` once the active entry + runs past standstill.suspect_seconds() of the longest run its + signature learned (history.learned_value()), else reason. + """ + longest = history.learned_value(signatures, entry["kind"], aliases_of(entry), "longest") + runtime = now - entry["granted"] + if runtime < standstill.suspect_seconds(longest): + return reason + mark = f"suspect hung ({runtime / 60:.0f} min)" + return mark if reason is None else f"{mark}; {reason}" + + def stamped(entry: dict, reason: str, host: str) -> str: if entry.get("code") == host: return reason diff --git a/skills/kvb-org-queue/scripts/standstill.py b/skills/kvb-org-queue/scripts/standstill.py new file mode 100644 index 0000000..0b87f40 --- /dev/null +++ b/skills/kvb-org-queue/scripts/standstill.py @@ -0,0 +1,343 @@ +"""Tell a held command that stands still from one that works, from /proc alone. + +queue.py's supervise() asks a Clock at every poll; once the process tree of +the command (tree()) stood still for HUNG_SECONDS, hold signals every member +(members(), signal_all(), settle()) and exits HUNG_EXIT, and list marks an +active job that runs SUSPECT_FACTOR times its longest learned run, at least +SUSPECT_FLOOR_SECONDS, as suspect hung. +""" + +from __future__ import annotations + +import contextlib +import os +import sys +import time +from collections.abc import Callable, Iterable +from pathlib import Path +from typing import NamedTuple + +PROC = Path("/proc") +HUNG_SECONDS = 3600 +KILL_GRACE_SECONDS = 30 +SETTLE_SECONDS = 0.2 +HUNG_EXIT = 124 +SUSPECT_FACTOR = 3 +SUSPECT_FLOOR_SECONDS = 1800 +TRACE_LIMIT = 20 +STATE, PARENT, SESSION, START = 0, 1, 3, 19 +TIMES = slice(11, 15) +IO_KEYS = (b"rchar", b"wchar", b"read_bytes", b"write_bytes") +PAUSED = frozenset({b"T", b"t"}) +GONE = frozenset({b"Z", b"X"}) +ELSEWHERE = frozenset({b"docker", b"docker-compose", b"docker-buildx", b"podman", b"podman-compose", b"ssh"}) +TICKS = os.sysconf("SC_CLK_TCK") + + +class Process(NamedTuple): + pid: int + comm: bytes + state: bytes + parent: int + session: int + ticks: int + start: int + + +class Sample(NamedTuple): + members: frozenset[tuple[int, int]] + ticks: int + io: tuple[int, ...] + elsewhere: tuple[bytes, ...] + paused: bool + + @property + def held(self) -> bool: + return bool(self.elsewhere) or self.paused + + +def parse_stat(data: bytes) -> Process: + """Args: + data: the bytes of /proc//stat; field 2, the command name in + parentheses, may hold spaces and parentheses itself, so the + fields are split after the last `)`. + + Returns: + The process: pid (field 1), comm (2), state (3), parent (4), session + (6), utime + stime + cutime + cstime (14 to 17, in clock ticks) and + start time (22, in clock ticks after boot). + """ + head, tail = data.rsplit(b")", 1) + pid, comm = head.split(b"(", 1) + fields = tail.split() + return Process(int(pid), comm, fields[STATE], int(fields[PARENT]), int(fields[SESSION]), + sum(int(ticks) for ticks in fields[TIMES]), int(fields[START])) + + +def read(name: str) -> Process | None: + try: + return parse_stat((PROC / name / "stat").read_bytes()) + except (OSError, ValueError, IndexError): + return None + + +def table() -> dict[int, Process] | None: + """Returns: + Every process /proc lists, by pid, skipping one that vanished + between the listing and the read; None when /proc cannot be listed. + """ + try: + names = [name for name in os.listdir(PROC) if name.isdigit()] + except OSError: + return None + return {found.pid: found for found in map(read, names) if found is not None} + + +def family(processes: dict[int, Process], roots: Iterable[int]) -> set[int]: + """Returns: + The roots processes shows and all their descendants, whatever + session or process group each one moved to. + """ + children: dict[int, list[int]] = {} + for process in processes.values(): + children.setdefault(process.parent, []).append(process.pid) + found: set[int] = set() + todo = [pid for pid in roots if pid in processes] + while todo: + pid = todo.pop() + if pid not in found: + found.add(pid) + todo.extend(children.get(pid, ())) + return found + + +def tree(processes: dict[int, Process], root: int) -> list[Process]: + """Returns: + root, its descendants (family()) and every process still in the + session root leads, as an orphan handed to init keeps it. + """ + session = {process.pid for process in processes.values() if process.session == root} + return [processes[pid] for pid in family(processes, [root]) | session] + + +def io_of(pid: int) -> tuple[int, ...]: + """Returns: + The IO_KEYS counters of /proc//io; zeros when that file cannot + be read (a setuid member's, or one that vanished after its stat), + which leaves its CPU time counted. + """ + try: + pairs = dict(line.split(b":", 1) for line in (PROC / str(pid) / "io").read_bytes().splitlines()) + return tuple(int(pairs[key]) for key in IO_KEYS) + except (OSError, ValueError, KeyError): + return (0,) * len(IO_KEYS) + + +def wchan_of(pid: int) -> str: + """Returns: + The kernel function /proc//wchan names as the one the process + sleeps in; empty while it runs and when that file cannot be read. + """ + try: + name = (PROC / str(pid) / "wchan").read_text(encoding="ascii", errors="replace") + except OSError: + return "" + return "" if name == "0" else name + + +def trace(root: int) -> list[str]: + """Returns: + ` []` of the tree() members of root, the + ones without a child in the tree first (the stuck one, not the + ancestors waiting on it), then by pid, at most TRACE_LIMIT; comm with + control bytes escaped, as a process sets its own. Never a command + line, which can carry credentials. Empty when /proc cannot be listed + or does not show root. + """ + processes = table() + if processes is None or root not in processes: + return [] + in_tree = tree(processes, root) + parents = {process.parent for process in in_tree} + return [" ".join(filter(None, (str(process.pid), process.comm.decode(errors="replace").encode("unicode_escape").decode(), + process.state.decode(), wchan_of(process.pid)))) + for process in sorted(in_tree, key=lambda process: (process.pid in parents, process.pid))[:TRACE_LIMIT]] + + +def probe(root: int) -> Sample | None: + """Args: + root: the pid of the held command. + + Returns: + The tree() of root: its members as (pid, start time), their CPU + ticks and io_of() counters summed, the ELSEWHERE commands among them + and whether one is stopped or traced; None when /proc cannot be + listed or does not show root, as a /proc of another PID namespace + does not. + """ + processes = table() + if processes is None or root not in processes: + return None + in_tree = tree(processes, root) + return Sample( + frozenset((process.pid, process.start) for process in in_tree), + sum(process.ticks for process in in_tree), + tuple(map(sum, zip(*(io_of(process.pid) for process in in_tree)))), + tuple(sorted({process.comm for process in in_tree} & ELSEWHERE)), + any(process.state in PAUSED for process in in_tree), + ) + + +def say(text: str) -> None: + print(f"queue: {text}", file=sys.stderr, flush=True) + + +class Clock: + """Clock how long the process tree of a held command stood still. + + Args: + root: the command's pid. + """ + + def __init__(self, root: int) -> None: + self.root = root + self.seen: Sample | None = None + self.since = time.monotonic() + self.blind = False + self.away = False + + def hung(self) -> bool: + """Returns: + Whether probe() of the tree gave the same sample for + HUNG_SECONDS. A probe that sees nothing, a tree running an + ELSEWHERE client, whose work runs outside the tree, and a tree + with a stopped or traced member restart the clock and never fire. + """ + now = time.monotonic() + seen = probe(self.root) + self.warn(seen) + if seen is None or seen.held or seen != self.seen: + self.since = now + self.seen = seen + return seen is not None and not seen.held and now - self.since >= HUNG_SECONDS + + def warn(self, seen: Sample | None) -> None: + """Args: + seen: the probe() of this poll; the first None of the run says + on stderr that the guard is off, the first naming an + ELSEWHERE client that it holds off. + """ + if seen is None and not self.blind: + say(f"/proc does not show process {self.root}; the hung-job guard is off for it") + self.blind = True + if seen is not None and seen.elsewhere and not self.away: + names = ", ".join(name.decode(errors="replace") for name in seen.elsewhere) + say(f"the command runs {names}, whose work /proc does not show; the hung-job guard holds off while one runs") + self.away = True + + def report(self) -> dict[str, float | int | list[str]]: + """Returns: + The sample of the last hung() that saw the tree, for the history + row: cpu_s (CPU seconds), read_bytes and written_bytes (rchar and + wchar), still_min (minutes without a change), members (the + number of processes) and tree (their trace(), taken now, before + any signal). + """ + rchar, wchar = self.seen.io[:2] + return {"cpu_s": round(self.seen.ticks / TICKS, 2), "read_bytes": rchar, "written_bytes": wchar, + "still_min": round((time.monotonic() - self.since) / 60, 1), "members": len(self.seen.members), + "tree": trace(self.root)} + + +def told(report: dict) -> str: + return (f"the command's process tree stood still for {report['still_min']:g} min (CPU {report['cpu_s']:g} s, " + f"{report['read_bytes']} bytes read, {report['written_bytes']} written, all unchanged); " + f"tree of {report['members']}: {', '.join(report['tree'])}") + + +def members(root: int) -> dict[int, int]: + """Returns: + The tree() of root as {pid: start time}; empty when /proc cannot be + listed or does not show root. + """ + processes = table() + if processes is None or root not in processes: + return {} + return {process.pid: process.start for process in tree(processes, root)} + + +def alive(targets: dict[int, int]) -> dict[int, int]: + """Args: + targets: {pid: start time} as members() recorded them, the command + among them (a zombie still shows in /proc). + + Returns: + The targets still running with their recorded start time, so a + reused pid never counts, the members of every session such a target + or a target gone from /proc leads, as a process a member starts + during the grace keeps its session when its parent exits (Linux + keeps a session id allocated while a member lives, so no reused pid + matches it), and the descendants all of them have now, as + {pid: start time}; zombies are left out, and nothing when /proc + cannot be listed. + """ + processes = table() or {} + kept = [pid for pid, start in targets.items() if pid in processes and processes[pid].start == start] + leaders = {pid for pid, start in targets.items() if pid not in processes or processes[pid].start == start} + kept += [process.pid for process in processes.values() if process.session in leaders] + return {pid: processes[pid].start for pid in family(processes, kept) if processes[pid].state not in GONE} + + +def signal_all(targets: dict[int, int], signum: int) -> dict[int, int]: + """Send signum to every process alive() finds of targets. + + Returns: + Those processes as {pid: start time}. + """ + living = alive(targets) + for pid in living: + with contextlib.suppress(ProcessLookupError, PermissionError): + os.kill(pid, signum) + return living + + +def settle(targets: dict[int, int], reap: Callable[[], int | None], signum: int | None) -> dict[int, int]: + """Wait up to KILL_GRACE_SECONDS until the command is reaped and alive() + finds none of targets, recording every process it finds on the way, so + one a member starts in a session of its own during the grace stays a + target after its parent exits; one started and orphaned between two + passes, SETTLE_SECONDS and a /proc scan apart, is not seen. + + Args: + targets: {pid: start time} of the tree. + reap: Popen.poll of the command, None while it runs. + signum: sent to everything each pass finds (signal_all()), so a + process forked while the last pass signalled gets it too; None + only waits, letting a member's cleanup run out the grace. + + Returns: + targets and every process found, as {pid: start time}. + """ + seen = dict(targets) + deadline = time.monotonic() + KILL_GRACE_SECONDS + while time.monotonic() < deadline: + living = alive(seen) if signum is None else signal_all(seen, signum) + seen |= living + if reap() is not None and not living: + break + time.sleep(SETTLE_SECONDS) + return seen + + +def suspect_seconds(longest: float | None) -> float: + """Args: + longest: the longest learned run of a job's signature in minutes, + None while nothing is learned. + + Returns: + The runtime past which list marks the job suspect hung: + SUSPECT_FACTOR times longest, at least SUSPECT_FLOOR_SECONDS. + """ + if longest is None: + return SUSPECT_FLOOR_SECONDS + return max(SUSPECT_FLOOR_SECONDS, SUSPECT_FACTOR * longest * 60) diff --git a/tests/test_queue_hung.py b/tests/test_queue_hung.py new file mode 100644 index 0000000..42af7ab --- /dev/null +++ b/tests/test_queue_hung.py @@ -0,0 +1,560 @@ +"""Exercise the kvb-org-queue hung-job guard: the process-tree probe, hold ending a tree that stands still and sparing one that works, the list mark and the learning that skips hung runs.""" + +from __future__ import annotations + +import contextlib +import importlib.util +import io +import json +import os +import shutil +import signal +import subprocess +import sys +import tempfile +import time +import unittest +from pathlib import Path +from unittest import mock + +REPO_ROOT = Path(__file__).resolve().parents[1] +SCRIPTS = REPO_ROOT / "skills" / "kvb-org-queue" / "scripts" +QUEUE = SCRIPTS / "queue.py" +HISTORY = SCRIPTS / "history.py" +STANDSTILL = SCRIPTS / "standstill.py" +NOW = 1_800_000_000.0 +POLL = 0.1 +HUNG = 2.0 +GRACE = 0.5 +OUTLAST = HUNG + 0.5 +SAMPLE = {"free_gb": 50.0, "mem": 20.0, "cpu": 10.0} +PROBE_KEYS = {"cpu_s", "read_bytes", "written_bytes", "still_min", "members", "tree"} +OPEN_MEMORY = { + "available": -1, "swap_free": -1, "psi_some": 1000, "psi_full": 1000, "swap_out_mb_min": 1e9, "container_swap_gb": 1e9, +} +SLEEP = "import time\ntime.sleep(60)\n" +PRINT = ( + f"import sys, time\nend = time.monotonic() + {OUTLAST}\n" + "while time.monotonic() < end:\n sys.stdout.write('x\\n')\n sys.stdout.flush()\n time.sleep(0.05)\n" +) +BURN = f"import time\nend = time.monotonic() + {OUTLAST}\nwhile time.monotonic() < end:\n pass\n" +STUBBORN = ( + "import os, signal, sys, time\n" + "signal.signal(signal.SIGTERM, signal.SIG_IGN)\n" + "with open(sys.argv[1] + '.part', 'w') as out:\n" + " out.write(str(os.getpid()))\n" + "os.replace(sys.argv[1] + '.part', sys.argv[1])\n" + "time.sleep(60)\n" +) +MEMBER = ( + "import os, signal, sys, time\n" + "def ended(*_):\n" + " with open(sys.argv[1], 'w') as out:\n" + " out.write('term')\n" + " os._exit(0)\n" + "signal.signal(signal.SIGTERM, ended)\n" + "time.sleep(60)\n" +) +SAME_SESSION = f"import subprocess, sys\nsubprocess.run([sys.executable, '-c', {BURN!r}], check=True)\n" +NEW_SESSION = f"import subprocess, sys\nsubprocess.run([sys.executable, '-c', {BURN!r}], check=True, start_new_session=True)\n" +FAMILY = ( + "import subprocess, sys, time\n" + f"subprocess.Popen([sys.executable, '-c', {MEMBER!r}, sys.argv[1]])\n" + f"subprocess.Popen([sys.executable, '-c', {MEMBER!r}, sys.argv[2]], process_group=0)\n" + "time.sleep(60)\n" +) +ABANDONED = ( + "import subprocess, sys, time\n" + f"subprocess.Popen([sys.executable, '-c', {STUBBORN!r}, sys.argv[1]], process_group=0)\n" + "time.sleep(60)\n" +) +CLEANUP_WORK = 0.5 + + +def cleanup(own_session: bool, work: float) -> str: + """Args: + own_session: whether the process the SIGTERM handler starts leads a + session of its own. + work: seconds the handler works once that process is ready, 0 to + exit at once; the session sweep alone, deterministically, is + test_signals_reach_members_whose_start_time_still_matches_the_session_and_their_new_descendants. + + Returns: + A command whose SIGTERM handler starts STUBBORN, works, and exits. + """ + return ( + "import os, signal, subprocess, sys, time\n" + "def ended(*_):\n" + f" subprocess.Popen([sys.executable, '-c', {STUBBORN!r}, sys.argv[1]], start_new_session={own_session})\n" + " while not os.path.exists(sys.argv[1]):\n" + " time.sleep(0.02)\n" + f" time.sleep({work})\n" + " os._exit(0)\n" + "signal.signal(signal.SIGTERM, ended)\n" + "time.sleep(60)\n" + ) + + +CLEANUP = cleanup(False, 0) +SETSID_CLEANUP = cleanup(True, CLEANUP_WORK) +SESSION_CLEANUP = ( + "import subprocess, sys, time\n" + f"subprocess.Popen([sys.executable, '-c', {CLEANUP!r}, sys.argv[1]], start_new_session=True)\n" + "time.sleep(60)\n" +) +CLEANUP_GRACE = 3.0 +CLIENT = f"import subprocess, sys\nsubprocess.run([sys.argv[1], '{OUTLAST}'], check=True)\n" +STOPPED = ( + "import signal, subprocess, sys, time\n" + "member = subprocess.Popen([sys.executable, '-c', 'import time; time.sleep(60)'])\n" + "member.send_signal(signal.SIGSTOP)\n" + f"time.sleep({OUTLAST})\n" + "member.kill()\n" + "member.wait()\n" +) +HOLD = ( + "import importlib.util, sys\n" + "spec = importlib.util.spec_from_file_location('kvb_queue_hung', sys.argv[1])\n" + "queue = importlib.util.module_from_spec(spec)\n" + "spec.loader.exec_module(queue)\n" + f"queue.POLL_SECONDS = {POLL}\n" + f"queue.standstill.HUNG_SECONDS = {HUNG}\n" + f"queue.standstill.KILL_GRACE_SECONDS = {GRACE}\n" + "sys.argv = sys.argv[1:]\n" + "sys.exit(queue.main())\n" +) + + +def _load(path: Path): + spec = importlib.util.spec_from_file_location(f"{path.stem}_hung_under_test", path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def history_row(seq: int, **fields: object) -> dict: + return { + "id": f"r{seq}", "kind": "test", "signature": "w: make test", "workdir": "/w", "start": NOW - 600, "end": NOW, + "minutes": 10.0, "exit": 0, "free_before_gb": 50.0, "min_free_gb": 49.0, "disk_gb": 1.0, "mem_before": 20.0, + "mem_max": 30.0, "cpu_before": 10.0, "cpu_max": 40.0, "concurrent": 0, **fields, + } + + +def stat_line(pid: int, comm: str, parent: int, session: int, times: tuple[int, int, int, int], state: str, start: int) -> bytes: + after = [state, str(parent), str(pid), str(session), "0", "-1", "4194560", "100", "0", "0", "0", + *map(str, times), "20", "0", "1", "0", str(start), "0", "0"] + return f"{pid} ({comm}) {' '.join(after)}\n".encode() + + +def plain(pid: int, parent: int, session: int, utime: int) -> bytes: + return stat_line(pid, "make", parent, session, (utime, 0, 0, 0), "S", 1000 + pid) + + +def io_lines(rchar: int, wchar: int, read_bytes: int, write_bytes: int) -> bytes: + return (f"rchar: {rchar}\nwchar: {wchar}\nsyscr: 1\nsyscw: 1\nread_bytes: {read_bytes}\n" + f"write_bytes: {write_bytes}\ncancelled_write_bytes: 0\n").encode() + + +def awaited(path: Path) -> int: + deadline = time.monotonic() + 30 + while not path.exists(): + if time.monotonic() > deadline: + raise AssertionError(f"{path} never appeared") + time.sleep(0.02) + return int(path.read_text(encoding="utf-8")) + + +def ended(pid: int) -> bool: + try: + stat = Path(f"/proc/{pid}/stat").read_bytes() + except OSError: + return True + return stat.rsplit(b")", 1)[1].split()[0] in (b"Z", b"X") + + +def kill_quietly(pid: int) -> None: + with contextlib.suppress(ProcessLookupError): + os.kill(pid, signal.SIGKILL) + + +class TestProbe(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.standstill = _load(STANDSTILL) + + def setUp(self): + tmp = tempfile.TemporaryDirectory() + self.addCleanup(tmp.cleanup) + self.proc = Path(tmp.name) / "proc" + self.proc.mkdir() + patched = mock.patch.object(self.standstill, "PROC", self.proc) + patched.start() + self.addCleanup(patched.stop) + + def process(self, pid: int, stat: bytes | None, io: bytes | None) -> None: + folder = self.proc / str(pid) + folder.mkdir(exist_ok=True) + if stat is not None: + (folder / "stat").write_bytes(stat) + if io is not None: + (folder / "io").write_bytes(io) + + def test_stat_splits_after_the_last_parenthesis_so_a_command_name_with_parentheses_reads_right(self): + line = stat_line(4242, "a) (b c) 9 9", 7, 4242, (7, 3, 2, 1), "T", 999) + self.assertEqual(self.standstill.parse_stat(line), (4242, b"a) (b c) 9 9", b"T", 7, 4242, 13, 999)) + self.assertNotEqual(line.split(b")", 1)[1].split()[3], b"4242") + + def test_the_probe_walks_the_whole_tree_whatever_its_session_and_skips_vanished_and_foreign_processes(self): + self.process(4242, plain(4242, 100, 4242, 10), io_lines(0, 100, 0, 0)) + self.process(4243, plain(4243, 4242, 4243, 5), None) + self.process(4244, plain(4244, 4243, 4244, 7), io_lines(30, 0, 0, 0)) + self.process(4245, plain(4245, 1, 4242, 1), None) + self.process(4246, plain(4246, 1, 1, 100), io_lines(9000, 9000, 0, 0)) + self.process(4247, None, None) + (self.proc / "self").mkdir() + seen = self.standstill.probe(4242) + self.assertEqual(seen.members, {(pid, 1000 + pid) for pid in (4242, 4243, 4244, 4245)}) + self.assertEqual((seen.ticks, seen.io, seen.elsewhere, seen.paused), (23, (30, 100, 0, 0), (), False)) + self.assertEqual(self.standstill.probe(4246).ticks, 100) + self.assertIsNone(self.standstill.probe(4999)) + + def test_cpu_time_reads_writes_and_a_new_process_each_count_as_progress(self): + self.process(4242, plain(4242, 1, 4242, 1), io_lines(5, 10, 0, 0)) + clock = self.standstill.Clock(4242) + changes = ( + (4242, plain(4242, 1, 4242, 2), None), + (4242, None, io_lines(6, 10, 0, 0)), + (4242, None, io_lines(6, 11, 0, 0)), + (4242, None, io_lines(6, 11, 1, 0)), + (4242, None, io_lines(6, 11, 1, 1)), + (4243, plain(4243, 4242, 4243, 0), None), + ) + with mock.patch.object(self.standstill, "HUNG_SECONDS", 0.2): + self.assertFalse(clock.hung()) + for pid, stat, counters in changes: + time.sleep(0.25) + self.assertTrue(clock.hung()) + self.process(pid, stat, counters) + self.assertFalse(clock.hung(), (pid, stat, counters)) + time.sleep(0.25) + self.assertTrue(clock.hung()) + report = clock.report() + self.assertEqual((report["cpu_s"], report["read_bytes"], report["written_bytes"]), (round(2 / self.standstill.TICKS, 2), 6, 11)) + self.assertGreaterEqual(report["still_min"], 0) + self.assertEqual((report["members"], report["tree"]), (2, ["4243 make S", "4242 make S"])) + + def test_the_trace_names_the_leaves_first_with_state_and_wait_channel_but_never_a_command_line(self): + self.process(4242, plain(4242, 1, 4242, 0), None) + self.process(4243, stat_line(4243, "sleep", 4242, 4242, (0, 0, 0, 0), "D", 5243), None) + self.process(4244, plain(4244, 4242, 4242, 0), None) + self.process(4245, stat_line(4245, "x\x1b[2J", 4242, 4242, (0, 0, 0, 0), "S", 5245), None) + for pid, wchan in ((4242, "do_wait"), (4243, "folio_wait_bit"), (4244, "0")): + (self.proc / str(pid) / "wchan").write_text(wchan, encoding="ascii") + (self.proc / "4243" / "cmdline").write_bytes(b"sleep\0--token=secret\0") + leaves = ["4243 sleep D folio_wait_bit", "4244 make S", "4245 x\\x1b[2J S"] + self.assertEqual(self.standstill.trace(4242), [*leaves, "4242 make S do_wait"]) + with mock.patch.object(self.standstill, "TRACE_LIMIT", 2): + self.assertEqual(self.standstill.trace(4242), leaves[:2]) + self.assertEqual(self.standstill.trace(4999), []) + + def test_a_stopped_member_or_a_client_whose_work_runs_elsewhere_never_fires_and_the_client_is_told_once(self): + self.process(4242, plain(4242, 1, 4242, 1), None) + held = (("docker", "S"), ("docker-compose", "S"), ("podman", "S"), ("ssh", "S"), ("sleep", "T"), ("python3", "t")) + with mock.patch.object(self.standstill, "HUNG_SECONDS", 0), contextlib.redirect_stderr(io.StringIO()) as err: + self.assertTrue(self.standstill.Clock(4242).hung()) + for comm, state in held: + self.process(4243, stat_line(4243, comm, 4242, 4243, (0, 0, 0, 0), state, 5000), None) + clock = self.standstill.Clock(4242) + self.assertEqual([clock.hung() for _ in range(3)], [False, False, False], comm) + self.assertEqual(err.getvalue().count("holds off"), 4) + + def test_an_unreadable_proc_never_fires_and_says_so_once(self): + self.process(4242, plain(4242, 1, 4242, 1), io_lines(0, 10, 0, 0)) + with mock.patch.object(self.standstill, "HUNG_SECONDS", 0): + self.assertTrue(self.standstill.Clock(4242).hung()) + clock = self.standstill.Clock(4242) + with mock.patch.object(self.standstill, "PROC", self.proc / "missing"), \ + contextlib.redirect_stderr(io.StringIO()) as err: + self.assertEqual([clock.hung() for _ in range(3)], [False, False, False]) + self.assertEqual(err.getvalue().count("hung-job guard is off"), 1) + + def test_signals_reach_members_whose_start_time_still_matches_the_session_and_their_new_descendants(self): + self.process(4242, plain(4242, 1, 4242, 0), None) + self.process(4243, plain(4243, 4242, 4242, 0), None) + self.process(4244, stat_line(4244, "make", 4242, 4242, (0, 0, 0, 0), "Z", 5244), None) + self.process(4245, plain(4245, 1, 4242, 0), None) + self.process(4250, plain(4250, 4243, 4250, 0), None) + self.process(4260, plain(4260, 1, 1, 0), None) + self.process(4270, plain(4270, 1, 4270, 0), None) + self.process(4271, plain(4271, 4270, 4271, 0), None) + self.process(4281, plain(4281, 1, 4280, 0), None) + self.assertEqual(self.standstill.members(4242), {pid: 1000 + pid for pid in (4242, 4243, 4244, 4245, 4250)}) + self.assertEqual(self.standstill.members(4999), {}) + with mock.patch.object(self.standstill.os, "kill") as kill: + living = self.standstill.signal_all({4242: 5242, 4244: 5244, 4270: 1, 4280: 5280}, signal.SIGTERM) + self.assertEqual(living, {4242: 5242, 4243: 5243, 4245: 5245, 4250: 5250, 4281: 5281}) + self.assertEqual(sorted(call.args for call in kill.call_args_list), + [(pid, signal.SIGTERM) for pid in (4242, 4243, 4245, 4250, 4281)]) + + def test_the_suspect_runtime_is_three_times_the_longest_run_with_a_floor(self): + self.assertEqual(self.standstill.suspect_seconds(20), 3600) + self.assertEqual(self.standstill.suspect_seconds(5), self.standstill.SUSPECT_FLOOR_SECONDS) + self.assertEqual(self.standstill.suspect_seconds(None), self.standstill.SUSPECT_FLOOR_SECONDS) + + +class TestSupervise(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.queue = _load(QUEUE) + + def start(self, code: str, *args: str) -> subprocess.Popen: + child = subprocess.Popen([sys.executable, "-c", code, *args], start_new_session=True, + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + self.addCleanup(child.wait, 30) + self.addCleanup(self.queue.send, child, signal.SIGKILL) + return child + + def scratch(self) -> Path: + tmp = tempfile.TemporaryDirectory() + self.addCleanup(tmp.cleanup) + return Path(tmp.name) + + def supervised(self, child: subprocess.Popen, *extra) -> tuple[tuple[int, dict | None], float, str]: + """Args: + child: the started command. + extra: patches entered after the defaults, so they win. + """ + patches = ( + mock.patch.object(self.queue, "POLL_SECONDS", POLL), + mock.patch.object(self.queue.standstill, "HUNG_SECONDS", HUNG), + mock.patch.object(self.queue.standstill, "KILL_GRACE_SECONDS", GRACE), + mock.patch.object(self.queue, "measure", return_value=SAMPLE), + *extra, + ) + with contextlib.ExitStack() as stack, contextlib.redirect_stderr(io.StringIO()) as err: + for patch in patches: + stack.enter_context(patch) + started = time.monotonic() + result = self.queue.supervise({}, child, []) + return result, time.monotonic() - started, err.getvalue() + + def test_a_sleeping_command_is_ended_with_124_and_its_probe(self): + child = self.start(SLEEP) + (code, probe), took, err = self.supervised(child) + self.assertEqual(code, 124) + self.assertEqual(set(probe), PROBE_KEYS) + self.assertEqual(probe["members"], 1) + self.assertRegex(probe["tree"][0], rf"^{child.pid} \S+ S") + self.assertEqual(child.returncode, -signal.SIGTERM) + self.assertLess(took, HUNG + 10) + self.assertIn("stood still", err) + self.assertIn(f"tree of 1: {child.pid} ", err) + + def test_a_command_that_keeps_printing_is_not_ended(self): + self.assertEqual(self.supervised(self.start(PRINT))[0], (0, None)) + + def test_a_command_that_burns_cpu_without_printing_is_not_ended(self): + self.assertEqual(self.supervised(self.start(BURN))[0], (0, None)) + + def test_a_command_ignoring_sigterm_is_killed_after_the_grace(self): + ready = self.scratch() / "ready" + child = self.start(STUBBORN, str(ready)) + awaited(ready) + (code, probe), took, _ = self.supervised(child) + self.assertEqual((code, child.returncode, set(probe)), (124, -signal.SIGKILL, PROBE_KEYS)) + self.assertGreaterEqual(took, HUNG + GRACE) + + def test_a_working_grandchild_counts_in_the_same_session_and_in_a_new_one(self): + self.assertEqual(self.supervised(self.start(SAME_SESSION))[0], (0, None)) + self.assertEqual(self.supervised(self.start(NEW_SESSION))[0], (0, None)) + + def test_a_sleeping_tree_is_ended_and_every_member_gets_sigterm_whatever_its_process_group(self): + folder = self.scratch() + marks = [folder / "same-group", folder / "own-group"] + child = self.start(FAMILY, *map(str, marks)) + (code, _), _, _ = self.supervised(child) + self.assertEqual((code, child.returncode), (124, -signal.SIGTERM)) + self.assertEqual([mark.read_text(encoding="utf-8") for mark in marks], ["term", "term"]) + + def test_a_member_ignoring_sigterm_is_killed_after_the_grace_although_the_leader_already_exited(self): + ready = self.scratch() / "ready" + child = self.start(ABANDONED, str(ready)) + member = awaited(ready) + (code, _), took, _ = self.supervised(child) + self.assertEqual((code, child.returncode), (124, -signal.SIGTERM)) + self.assertTrue(ended(member)) + self.assertGreaterEqual(took, HUNG + GRACE) + + def test_a_process_a_member_starts_during_the_grace_is_killed_although_its_parent_exited(self): + cases = (("same session", CLEANUP, 0), ("member in own session", SESSION_CLEANUP, -signal.SIGTERM), + ("orphan in own session", SETSID_CLEANUP, 0)) + for case, script, leader_exit in cases: + with self.subTest(case): + ready = self.scratch() / "ready" + child = self.start(script, str(ready)) + grace = mock.patch.object(self.queue.standstill, "KILL_GRACE_SECONDS", CLEANUP_GRACE) + (code, _), _, _ = self.supervised(child, grace) + orphan = awaited(ready) + self.addCleanup(kill_quietly, orphan) + self.assertEqual((code, child.returncode), (124, leader_exit)) + self.assertTrue(ended(orphan)) + + def test_a_process_forked_while_sigkill_goes_out_is_killed_on_the_next_pass(self): + proc = self.scratch() / "proc" + + def spawn(pid: int, parent: int) -> None: + (proc / str(pid)).mkdir(parents=True) + (proc / str(pid) / "stat").write_bytes(plain(pid, parent, 4242, 0)) + + def kill(pid: int, signum: int) -> None: + if signum == signal.SIGKILL: + shutil.rmtree(proc / str(pid)) + if pid == 4243: + spawn(4250, 1) + + spawn(4242, 1) + spawn(4243, 4242) + child = mock.Mock(pid=4242, returncode=None, poll=lambda: None if (proc / "4242").exists() else 0) + with mock.patch.object(self.queue.standstill, "PROC", proc), \ + mock.patch.object(self.queue.standstill, "KILL_GRACE_SECONDS", GRACE), \ + mock.patch.object(self.queue.standstill.os, "kill", side_effect=kill) as sent, \ + mock.patch.object(self.queue.os, "killpg") as group: + self.assertEqual(self.queue.stop(child), 124) + self.assertIn(mock.call(4250, signal.SIGKILL), sent.call_args_list) + self.assertEqual(sorted(path.name for path in proc.iterdir()), []) + group.assert_not_called() + + def test_a_tree_running_a_container_client_never_fires_and_says_so_once(self): + sleep = shutil.which("sleep") + if sleep is None: + self.skipTest("no sleep binary on PATH") + docker = self.scratch() / "docker" + docker.symlink_to(sleep) + result, _, err = self.supervised(self.start(CLIENT, str(docker))) + self.assertEqual(result, (0, None)) + self.assertEqual(err.count("the command runs docker,"), 1) + + def test_a_tree_with_a_stopped_member_never_fires(self): + self.assertEqual(self.supervised(self.start(STOPPED))[0], (0, None)) + + def test_stop_falls_back_to_the_process_group_when_proc_shows_nothing(self): + child = self.start(SLEEP) + with mock.patch.object(self.queue.standstill, "PROC", self.scratch() / "missing"), \ + mock.patch.object(self.queue.standstill, "KILL_GRACE_SECONDS", GRACE): + self.assertEqual(self.queue.stop(child), 124) + self.assertEqual(child.returncode, -signal.SIGTERM) + + +class TestHungHistory(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.history = _load(HISTORY) + cls.queue = _load(QUEUE) + + def learn(self, rows: list[dict]) -> dict: + return self.history.learn(rows, NOW, {"load_limits": {"mem_percent": 80, "cpu_percent": 90}})[0] + + def test_a_hung_row_stays_out_of_the_learned_durations_and_shares(self): + rows = [history_row(n) for n in range(5)] + probe = {"cpu_s": 1.0, "read_bytes": 0, "written_bytes": 0, "still_min": 60.0} + stuck = history_row(5, minutes=500.0, mem_max=95.0, exit=124) + stats = self.learn([*rows, {**stuck, self.history.HUNG: probe}])["test", "w: make test"] + self.assertEqual((stats["runs"], stats["minutes"], stats["longest"], stats["ram_share"]), (5, 10.0, 10.0, 10.0)) + counted = self.learn([*rows, stuck])["test", "w: make test"] + self.assertEqual((counted["runs"], counted["longest"]), (6, 500.0)) + + def test_list_marks_an_active_job_past_three_times_its_longest_run_and_past_the_floor(self): + signatures = self.learn([history_row(n, minutes=20.0 - n) for n in range(5)]) + entry = {"kind": "test", "aliases": ["w: make test"], "granted": NOW} + suspected = self.queue.suspected + self.assertIsNone(suspected({**entry, "granted": NOW - 59 * 60}, None, signatures, NOW)) + self.assertEqual(suspected({**entry, "granted": NOW - 61 * 60}, None, signatures, NOW), "suspect hung (61 min)") + self.assertEqual(suspected({**entry, "granted": NOW - 61 * 60}, "backfilled", signatures, NOW), + "suspect hung (61 min); backfilled") + unknown = {**entry, "aliases": ["w: make lint"]} + self.assertEqual(suspected({**unknown, "granted": NOW - 29 * 60}, "backfilled", signatures, NOW), "backfilled") + self.assertEqual(suspected({**unknown, "granted": NOW - 31 * 60}, None, signatures, NOW), "suspect hung (31 min)") + short = self.learn([history_row(n, minutes=5.0) for n in range(5)]) + self.assertIsNone(suspected({**entry, "granted": NOW - 20 * 60}, None, short, NOW)) + + +class TestHungHold(unittest.TestCase): + def setUp(self): + tmp = tempfile.TemporaryDirectory() + self.addCleanup(tmp.cleanup) + self.base = Path(tmp.name).resolve() + self.root = self.base / "council" + self.root.mkdir() + home = self.base / "home" + home.mkdir() + self.history_file = self.root / "queue-history.jsonl" + self.config = self.base / "config.json" + self.config.write_text(json.dumps({ + "disk": str(self.base), + "thresholds_gb": {"warn": 0, "stop": 0, "critical": 0}, + "projection_minutes": 30, + "slot_need_gb": {"test": 0.001}, + "slot_limits": {"test": 1}, + "load_limits": {"cpu_percent": 101, "mem_percent": 101, "sample_seconds": 0.1}, + "budget_percent": {"ram": 75, "cpu": 100}, + "memory_limits": {stage: OPEN_MEMORY for stage in ("warn", "stop")}, + "heavy_commands": ["^make compose-deploy"], + "never_block": ["^make compose-down"], + }), encoding="utf-8") + (self.root / "rasci.json").write_text(json.dumps({"work/a": {"roles": {}, "status": "running"}}), encoding="utf-8") + self.env = { + "PATH": os.environ["PATH"], + "HOME": str(home), + "GIT_CEILING_DIRECTORIES": str(self.base.parent), + "AGENTS_COUNCIL_ROOT": str(self.root), + "AGENTS_COUNCIL_CONFIG": str(self.config), + "CLAUDE_PROJECT_DIR": str(self.base / "project"), + } + + def run_queue(self, *args: str) -> subprocess.CompletedProcess: + return subprocess.run( + [sys.executable, "-c", HOLD, str(QUEUE), *args], cwd=self.base, env=self.env, stdin=subprocess.DEVNULL, + capture_output=True, text=True, timeout=60, check=False, + ) + + def hold(self, code: str) -> subprocess.CompletedProcess: + return self.run_queue("hold", "--kind", "test", "--task", "work/a", "--session", "s1", "--", sys.executable, "-c", code) + + def rows(self) -> list[dict]: + return [json.loads(line) for line in self.history_file.read_text(encoding="utf-8").splitlines()] + + def test_a_hold_whose_command_sleeps_exits_124_and_its_history_row_marks_it_hung(self): + result = self.hold(SLEEP) + self.assertEqual(result.returncode, 124, result.stderr) + self.assertIn("stood still", result.stderr) + [row] = self.rows() + self.assertEqual((row["exit"], set(row["hung"])), (124, PROBE_KEYS)) + self.assertEqual(json.loads((self.root / "queue.json").read_text(encoding="utf-8"))["entries"], []) + + def test_a_hold_whose_command_works_keeps_its_exit_code_and_an_unmarked_row(self): + result = self.hold(BURN + "raise SystemExit(5)\n") + self.assertEqual((result.returncode, result.stderr), (5, "")) + [row] = self.rows() + self.assertEqual(row["exit"], 5) + self.assertNotIn("hung", row) + + def test_list_marks_an_active_job_past_the_floor_when_nothing_is_learned(self): + queue = _load(QUEUE) + me, mine = os.getpid(), queue.identity(os.getpid()) + now = time.time() + floor = queue.standstill.SUSPECT_FLOOR_SECONDS + entry = { + "seq": 1, "kind": "test", "task": "work/a", "need_gb": 0, "hotfix": None, "workdir": "/w", "session": "s1", + "name": "s1", "pid": me, "pid_start": mine, "child": me, "child_start": mine, "status": "active", + "enqueued": now - floor - 120, "cmd": "make test", "code": queue.CODE, + } + entries = [{**entry, "id": "test-1", "granted": now - floor - 60}, {**entry, "id": "test-2", "seq": 2, "granted": now - floor + 60}] + (self.root / "queue.json").write_text(json.dumps({"seq": 2, "entries": entries}), encoding="utf-8") + listing = self.run_queue("list") + self.assertEqual(listing.returncode, 0, listing.stderr) + reasons = [line.split("\t")[-1] for line in listing.stdout.splitlines()[1:]] + self.assertEqual(reasons, [f"suspect hung ({(floor + 60) // 60} min)", "-"]) + + +if __name__ == "__main__": + unittest.main()