This commit is contained in:
7 files changed
+991
-25
No files matched your search
@@ -34,7 +34,11 @@ step inside it, or a process its own ended session left behind, once at least
|
||||
child process, no network traffic, CPU time not moving). It secures the live
|
||||
state first, then reports the timestamp, the step, the probe and the command.
|
||||
A peer's hung job goes to its owner, and to the operator when the owner does
|
||||
not answer.
|
||||
not answer. `queue.py hold` does this itself for a queued job: after 60
|
||||
minutes without CPU, I/O or new process in its tree, no stopped member and no
|
||||
docker, podman or ssh client, it records the stuck processes, ends the tree
|
||||
and exits 124; `queue.py list` marks a run past three times its longest
|
||||
learned one `suspect hung`.
|
||||
|
||||
## Resource watch
|
||||
|
||||
|
||||
@@ -77,12 +77,12 @@ queue.py cancel <id>
|
||||
|
||||
- Never ask the chair for a heavy slot or announce a grant; the chair handles
|
||||
only a stuck queue or hung job (`capacity.md` of `kvb-org-organize`).
|
||||
- A running job is never cut short for the queue; only the operator orders
|
||||
one down, and `kvb-git-hotfix` may void a gate. `kvb-org-queue-cleanup`
|
||||
- Only the operator, a `kvb-git-hotfix` gate void, or hold after 60 minutes
|
||||
of tree standstill (exit 124) cut a job short. `kvb-org-queue-cleanup`
|
||||
deletes only entries whose lock no living process holds.
|
||||
- With `queue_enforce: true`, the `block` hook (PreToolUse on Bash) refuses
|
||||
`heavy_commands`, image builds, `make install|test*|quality*|verify-*`,
|
||||
tests beyond one unit file and `gate.py run|hold` outside `queue.py hold`.
|
||||
- Without the config, hold exits 2: tell the operator, ask the chair
|
||||
(`kvb-org-organize`). A waiting hold rereads it every poll
|
||||
(a broken file keeps the last good one).
|
||||
(`kvb-org-organize`). Waiting holds reread it every poll, keeping the
|
||||
last good one.
|
||||
@@ -2,7 +2,8 @@
|
||||
|
||||
Every finished job appends a row (signature such as `<repo>: make <target>`,
|
||||
kind, minutes, disk, RAM and CPU) to `queue-history.jsonl`; the
|
||||
`kvb-org-organize` watch hook keeps it across reboots.
|
||||
`kvb-org-organize` watch hook keeps it across reboots. The row of a run
|
||||
hold ended for standing still (exit 124, a `hung` probe) teaches nothing.
|
||||
|
||||
From the last 30 days, once 5 runs exist:
|
||||
|
||||
|
||||
@@ -6,7 +6,9 @@ and the copy the kvb-org-organize watch hook keeps in ~/.local/state.
|
||||
A row: id, kind, signature, workdir, start, end (epoch seconds), minutes, exit,
|
||||
free_before_gb, min_free_gb, disk_gb (free_before_gb - min_free_gb, at least
|
||||
0), mem_before, mem_max, cpu_before, cpu_max (percent), concurrent (other
|
||||
active jobs at the grant).
|
||||
active jobs at the grant), and HUNG ({cpu_s, read_bytes, written_bytes,
|
||||
still_min, members, tree}, the report of standstill.py) only on a run hold
|
||||
ended for standing still, whose exit is 124.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -33,6 +35,7 @@ NAME_KEY = operator.itemgetter("signature")
|
||||
KIND_KEY = operator.itemgetter("kind")
|
||||
ANY_KIND = "*"
|
||||
RAM_CAP = "ram_cap_percent"
|
||||
HUNG = "hung"
|
||||
SIGNATURE_HEADER = ("Signature", "Kind", "Runs", "Median min", "p90 disk GB", "p90 RAM %", "Need GB")
|
||||
KIND_HEADER = ("Kind", "Runs", "p90 RAM share %", "Median CPU share %", "Slot limit", "Source")
|
||||
|
||||
@@ -156,11 +159,12 @@ def grouped(rows: list[dict], key: Callable[[dict], Hashable]) -> dict[Hashable,
|
||||
|
||||
|
||||
def command_stats(rows: list[dict]) -> dict:
|
||||
stats = {"runs": len(rows), "minutes": None, "disk_gb": None, "ram_share": None, "cpu_share": None, "need_gb": None}
|
||||
stats = {"runs": len(rows), "minutes": None, "longest": None, "disk_gb": None, "ram_share": None, "cpu_share": None,
|
||||
"need_gb": None}
|
||||
if len(rows) >= MIN_RUNS:
|
||||
disk = p90([r["disk_gb"] for r in rows])
|
||||
stats.update(
|
||||
minutes=statistics.median(r["minutes"] for r in rows), disk_gb=disk,
|
||||
minutes=statistics.median(r["minutes"] for r in rows), longest=max(r["minutes"] for r in rows), disk_gb=disk,
|
||||
ram_share=p90(shares(rows, "mem")), cpu_share=statistics.median(shares(rows, "cpu")),
|
||||
need_gb=math.ceil(disk + DISK_MARGIN_GB),
|
||||
)
|
||||
@@ -186,7 +190,8 @@ def learn(rows: list[dict], now: float, cfg: dict) -> tuple[dict[tuple[str, str]
|
||||
"""Derive disk need per command and slot limit per kind from recent rows.
|
||||
|
||||
Args:
|
||||
rows: history rows, as read() returns them.
|
||||
rows: history rows, as read() returns them; HUNG rows are ignored, as
|
||||
their standstill would stretch the minutes and the shares.
|
||||
now: epoch seconds; rows that ended more than WINDOW_SECONDS earlier are
|
||||
ignored.
|
||||
cfg: the config; load_limits.mem_percent and cpu_percent bound a limit,
|
||||
@@ -195,19 +200,20 @@ def learn(rows: list[dict], now: float, cfg: dict) -> tuple[dict[tuple[str, str]
|
||||
|
||||
Returns:
|
||||
(signatures, kinds). signatures: {(kind, signature): {runs, minutes,
|
||||
disk_gb, ram_share, cpu_share, need_gb}}, and {(ANY_KIND, signature):
|
||||
longest, disk_gb, ram_share, cpu_share, need_gb}}, and {(ANY_KIND, signature):
|
||||
...} over the rows of every kind with that signature, which is what a
|
||||
queue before the per-kind learning booked; kinds: {kind: {runs,
|
||||
ram_share, cpu_share, limit, light, ram_cap}}, ram_cap None without a
|
||||
cap. With fewer than MIN_RUNS rows every value but runs, light and
|
||||
ram_cap is None.
|
||||
minutes is the median, disk_gb and ram_share the p90 of the disk taken
|
||||
minutes is the median and longest the largest minutes, disk_gb and
|
||||
ram_share the p90 of the disk taken
|
||||
and of max minus before in percentage points, cpu_share the median of
|
||||
max minus before; need_gb is ceil(disk_gb + DISK_MARGIN_GB); limit is
|
||||
max(1, min over RAM and CPU of floor((load limit - median before) /
|
||||
max(share, MIN_SHARE_PERCENT))).
|
||||
"""
|
||||
recent = [r for r in rows if now - r["end"] <= WINDOW_SECONDS]
|
||||
recent = [r for r in rows if now - r["end"] <= WINDOW_SECONDS and HUNG not in r]
|
||||
light = set(cfg.get("light_kinds", []))
|
||||
caps = cfg.get(RAM_CAP, {})
|
||||
signatures = {(ANY_KIND, name): command_stats(group) for name, group in grouped(recent, NAME_KEY).items()}
|
||||
|
||||
@@ -41,9 +41,10 @@ Args:
|
||||
proposing it (latched()); measure disk, RAM and CPU, run
|
||||
CMD in the current directory with inherited stdio in a session of its
|
||||
own, forward SIGINT and SIGTERM to its process group, measure every
|
||||
POLL_SECONDS and at its end, record its history row and leave the
|
||||
queue. The entry carries its code (a hash of this file and the
|
||||
SIBLINGS); a poll that finds other code installed in
|
||||
POLL_SECONDS and at its end, end its process tree once the tree stood
|
||||
still for standstill.HUNG_SECONDS (supervise()), record its history
|
||||
row and leave the queue. The entry carries its code (a hash of this
|
||||
file and the SIBLINGS); a poll that finds other code installed in
|
||||
~/.agents/skills leaves the queue ungranted and restarts the same hold
|
||||
with the installed queue.py, which keeps its entry id, place and
|
||||
enqueue time (hand_over()).
|
||||
@@ -74,7 +75,9 @@ Args:
|
||||
poll, host load included, such as `waits in line behind <id>` or
|
||||
`drained for <id>`, the head's after `escalated <step>: ` once the
|
||||
stall reached it; `backfilled` for a job started past a blocked one;
|
||||
prefixed when the entry runs other code than the installed one).
|
||||
led by `suspect hung (<runtime> min)` for an active job past
|
||||
standstill.suspect_seconds() (suspected()); prefixed when the entry
|
||||
runs other code than the installed one).
|
||||
Entries of dead holders are dropped first.
|
||||
--json: print the same rows as a JSON list (listed()), each with
|
||||
the trace of the evaluate() pass that set its reason.
|
||||
@@ -109,7 +112,8 @@ watch.py documents them. A signature is `<repository>: <command signature>`
|
||||
(aliases()).
|
||||
|
||||
Exit codes:
|
||||
hold: the command's exit code unchanged, 128+N when signal N ended it;
|
||||
hold: the command's exit code unchanged, 128+N when signal N ended it,
|
||||
standstill.HUNG_EXIT (124) when hold ended it for standing still;
|
||||
a command exiting 75 (EX_TEMPFAIL: it did no work, such as a declined
|
||||
`operator.py granted`) leaves no history row;
|
||||
before the command started 2 on a usage or config error or when it
|
||||
@@ -154,6 +158,7 @@ SIBLINGS = {
|
||||
"council": SKILLS / "kvb-org-organize" / "scripts" / "council.py",
|
||||
"rasci": SKILLS / "kvb-org-rasci" / "scripts" / "rasci.py",
|
||||
"history": Path(__file__).resolve().parent / "history.py",
|
||||
"standstill": Path(__file__).resolve().parent / "standstill.py",
|
||||
"nofollow": SKILLS / "kvb-org-organize" / "scripts" / "nofollow.py",
|
||||
"councilroot": SKILLS / "kvb-org-organize" / "scripts" / "councilroot.py",
|
||||
}
|
||||
@@ -228,6 +233,7 @@ def sibling(name: str) -> ModuleType:
|
||||
|
||||
watch = sibling("watch")
|
||||
history = sibling("history")
|
||||
standstill = sibling("standstill")
|
||||
nofollow = sibling("nofollow")
|
||||
STORE = watch.ROOT / "queue.json"
|
||||
LOCK = watch.ROOT / "queue.lock"
|
||||
@@ -1444,23 +1450,54 @@ def measure(cfg: dict) -> dict[str, float]:
|
||||
return {"free_gb": free, "mem": mem, "cpu": cpu}
|
||||
|
||||
|
||||
def supervise(cfg: dict, child: subprocess.Popen, samples: list[dict[str, float]]) -> int:
|
||||
def supervise(cfg: dict, child: subprocess.Popen, samples: list[dict[str, float]]) -> tuple[int, dict | None]:
|
||||
"""Wait for the child, measuring every POLL_SECONDS and once at its end.
|
||||
|
||||
Args:
|
||||
cfg: the config.
|
||||
child: the running command.
|
||||
child: the running command, leading a session of its own.
|
||||
samples: the measurements so far; each new one is appended.
|
||||
|
||||
Returns:
|
||||
The child's return code.
|
||||
(the child's return code, None); (standstill.HUNG_EXIT, the probe of
|
||||
standstill.Clock.report()) once its process tree stood still for
|
||||
standstill.HUNG_SECONDS and stop() ended it.
|
||||
"""
|
||||
clock = standstill.Clock(child.pid)
|
||||
code = None
|
||||
while code is None:
|
||||
with contextlib.suppress(subprocess.TimeoutExpired):
|
||||
code = child.wait(timeout=POLL_SECONDS)
|
||||
samples.append(measure(cfg))
|
||||
return code
|
||||
if code is None and clock.hung() and child.poll() is None:
|
||||
probe = clock.report()
|
||||
print(f"queue: {standstill.told(probe)}; ending the tree", file=sys.stderr, flush=True)
|
||||
return stop(child), probe
|
||||
return code, None
|
||||
|
||||
|
||||
def stop(child: subprocess.Popen) -> int:
|
||||
"""SIGTERM every process of the child's tree (standstill.members()), of
|
||||
the sessions its members lead and the descendants those have now,
|
||||
SIGKILL the ones still there after standstill.KILL_GRACE_SECONDS and
|
||||
those standstill.settle() found during it, again at each pass of the
|
||||
SIGKILL grace, a recorded member only while its start time matches; the
|
||||
child's process group (send()) when /proc does not show the child. A
|
||||
process that leaves the tree and every recorded session between two
|
||||
passes (setsid, then its parent dies) escapes; only a subreaper or a
|
||||
cgroup per job would hold it.
|
||||
|
||||
Returns:
|
||||
standstill.HUNG_EXIT, also when a process outlives SIGKILL by another
|
||||
grace (uninterruptible sleep), so hold never waits on it.
|
||||
"""
|
||||
targets = standstill.members(child.pid)
|
||||
for signum, repeat in ((signal.SIGTERM, None), (signal.SIGKILL, signal.SIGKILL)):
|
||||
living = standstill.signal_all(targets, signum)
|
||||
if child.pid not in living:
|
||||
send(child, signum)
|
||||
targets = standstill.settle(targets | living, child.poll, repeat)
|
||||
return standstill.HUNG_EXIT
|
||||
|
||||
|
||||
def keep(row: dict) -> None:
|
||||
@@ -1582,10 +1619,10 @@ def cmd_hold(args: argparse.Namespace) -> int:
|
||||
for signum in caught:
|
||||
send(child, signum)
|
||||
update(entry["id"], child=child.pid, child_start=identity(child.pid))
|
||||
code = supervise(cfg, child, samples)
|
||||
code, hung = supervise(cfg, child, samples)
|
||||
code = code if code >= 0 else 128 - code
|
||||
job = {"id": f"{entry['id']}-{time.time_ns()}", "kind": args.kind, "signature": names[0],
|
||||
"workdir": entry["workdir"]}
|
||||
"workdir": entry["workdir"], **({} if hung is None else {history.HUNG: hung})}
|
||||
record(job, start, code, samples, concurrent)
|
||||
keep_receipt(gates, args, here, before, code)
|
||||
return code
|
||||
@@ -1607,7 +1644,8 @@ def cmd_list(args: argparse.Namespace) -> int:
|
||||
by_id = {e["id"]: e for e in entries}
|
||||
running = [(key_of(e), e["granted"]) for e in entries if e["status"] == "active"]
|
||||
rows = [
|
||||
(None, e, history.eta(signatures, [(key_of(e), e["granted"])], [], now), e.get("reason") or "-", e.get("trace", []))
|
||||
(None, e, history.eta(signatures, [(key_of(e), e["granted"])], [], now), suspected(e, e.get("reason"), signatures, now) or "-",
|
||||
e.get("trace", []))
|
||||
for e in entries if e["status"] == "active"
|
||||
]
|
||||
ahead: list[tuple[str, str]] = []
|
||||
@@ -1633,6 +1671,20 @@ def cmd_list(args: argparse.Namespace) -> int:
|
||||
return 0
|
||||
|
||||
|
||||
def suspected(entry: dict, reason: str | None, signatures: dict[tuple[str, str], dict], now: float) -> str | None:
|
||||
"""Returns:
|
||||
reason led by `suspect hung (<runtime> min)` once the active entry
|
||||
runs past standstill.suspect_seconds() of the longest run its
|
||||
signature learned (history.learned_value()), else reason.
|
||||
"""
|
||||
longest = history.learned_value(signatures, entry["kind"], aliases_of(entry), "longest")
|
||||
runtime = now - entry["granted"]
|
||||
if runtime < standstill.suspect_seconds(longest):
|
||||
return reason
|
||||
mark = f"suspect hung ({runtime / 60:.0f} min)"
|
||||
return mark if reason is None else f"{mark}; {reason}"
|
||||
|
||||
|
||||
def stamped(entry: dict, reason: str, host: str) -> str:
|
||||
if entry.get("code") == host:
|
||||
return reason
|
||||
|
||||
@@ -0,0 +1,343 @@
|
||||
"""Tell a held command that stands still from one that works, from /proc alone.
|
||||
|
||||
queue.py's supervise() asks a Clock at every poll; once the process tree of
|
||||
the command (tree()) stood still for HUNG_SECONDS, hold signals every member
|
||||
(members(), signal_all(), settle()) and exits HUNG_EXIT, and list marks an
|
||||
active job that runs SUSPECT_FACTOR times its longest learned run, at least
|
||||
SUSPECT_FLOOR_SECONDS, as suspect hung.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import contextlib
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
from collections.abc import Callable, Iterable
|
||||
from pathlib import Path
|
||||
from typing import NamedTuple
|
||||
|
||||
PROC = Path("/proc")
|
||||
HUNG_SECONDS = 3600
|
||||
KILL_GRACE_SECONDS = 30
|
||||
SETTLE_SECONDS = 0.2
|
||||
HUNG_EXIT = 124
|
||||
SUSPECT_FACTOR = 3
|
||||
SUSPECT_FLOOR_SECONDS = 1800
|
||||
TRACE_LIMIT = 20
|
||||
STATE, PARENT, SESSION, START = 0, 1, 3, 19
|
||||
TIMES = slice(11, 15)
|
||||
IO_KEYS = (b"rchar", b"wchar", b"read_bytes", b"write_bytes")
|
||||
PAUSED = frozenset({b"T", b"t"})
|
||||
GONE = frozenset({b"Z", b"X"})
|
||||
ELSEWHERE = frozenset({b"docker", b"docker-compose", b"docker-buildx", b"podman", b"podman-compose", b"ssh"})
|
||||
TICKS = os.sysconf("SC_CLK_TCK")
|
||||
|
||||
|
||||
class Process(NamedTuple):
|
||||
pid: int
|
||||
comm: bytes
|
||||
state: bytes
|
||||
parent: int
|
||||
session: int
|
||||
ticks: int
|
||||
start: int
|
||||
|
||||
|
||||
class Sample(NamedTuple):
|
||||
members: frozenset[tuple[int, int]]
|
||||
ticks: int
|
||||
io: tuple[int, ...]
|
||||
elsewhere: tuple[bytes, ...]
|
||||
paused: bool
|
||||
|
||||
@property
|
||||
def held(self) -> bool:
|
||||
return bool(self.elsewhere) or self.paused
|
||||
|
||||
|
||||
def parse_stat(data: bytes) -> Process:
|
||||
"""Args:
|
||||
data: the bytes of /proc/<pid>/stat; field 2, the command name in
|
||||
parentheses, may hold spaces and parentheses itself, so the
|
||||
fields are split after the last `)`.
|
||||
|
||||
Returns:
|
||||
The process: pid (field 1), comm (2), state (3), parent (4), session
|
||||
(6), utime + stime + cutime + cstime (14 to 17, in clock ticks) and
|
||||
start time (22, in clock ticks after boot).
|
||||
"""
|
||||
head, tail = data.rsplit(b")", 1)
|
||||
pid, comm = head.split(b"(", 1)
|
||||
fields = tail.split()
|
||||
return Process(int(pid), comm, fields[STATE], int(fields[PARENT]), int(fields[SESSION]),
|
||||
sum(int(ticks) for ticks in fields[TIMES]), int(fields[START]))
|
||||
|
||||
|
||||
def read(name: str) -> Process | None:
|
||||
try:
|
||||
return parse_stat((PROC / name / "stat").read_bytes())
|
||||
except (OSError, ValueError, IndexError):
|
||||
return None
|
||||
|
||||
|
||||
def table() -> dict[int, Process] | None:
|
||||
"""Returns:
|
||||
Every process /proc lists, by pid, skipping one that vanished
|
||||
between the listing and the read; None when /proc cannot be listed.
|
||||
"""
|
||||
try:
|
||||
names = [name for name in os.listdir(PROC) if name.isdigit()]
|
||||
except OSError:
|
||||
return None
|
||||
return {found.pid: found for found in map(read, names) if found is not None}
|
||||
|
||||
|
||||
def family(processes: dict[int, Process], roots: Iterable[int]) -> set[int]:
|
||||
"""Returns:
|
||||
The roots processes shows and all their descendants, whatever
|
||||
session or process group each one moved to.
|
||||
"""
|
||||
children: dict[int, list[int]] = {}
|
||||
for process in processes.values():
|
||||
children.setdefault(process.parent, []).append(process.pid)
|
||||
found: set[int] = set()
|
||||
todo = [pid for pid in roots if pid in processes]
|
||||
while todo:
|
||||
pid = todo.pop()
|
||||
if pid not in found:
|
||||
found.add(pid)
|
||||
todo.extend(children.get(pid, ()))
|
||||
return found
|
||||
|
||||
|
||||
def tree(processes: dict[int, Process], root: int) -> list[Process]:
|
||||
"""Returns:
|
||||
root, its descendants (family()) and every process still in the
|
||||
session root leads, as an orphan handed to init keeps it.
|
||||
"""
|
||||
session = {process.pid for process in processes.values() if process.session == root}
|
||||
return [processes[pid] for pid in family(processes, [root]) | session]
|
||||
|
||||
|
||||
def io_of(pid: int) -> tuple[int, ...]:
|
||||
"""Returns:
|
||||
The IO_KEYS counters of /proc/<pid>/io; zeros when that file cannot
|
||||
be read (a setuid member's, or one that vanished after its stat),
|
||||
which leaves its CPU time counted.
|
||||
"""
|
||||
try:
|
||||
pairs = dict(line.split(b":", 1) for line in (PROC / str(pid) / "io").read_bytes().splitlines())
|
||||
return tuple(int(pairs[key]) for key in IO_KEYS)
|
||||
except (OSError, ValueError, KeyError):
|
||||
return (0,) * len(IO_KEYS)
|
||||
|
||||
|
||||
def wchan_of(pid: int) -> str:
|
||||
"""Returns:
|
||||
The kernel function /proc/<pid>/wchan names as the one the process
|
||||
sleeps in; empty while it runs and when that file cannot be read.
|
||||
"""
|
||||
try:
|
||||
name = (PROC / str(pid) / "wchan").read_text(encoding="ascii", errors="replace")
|
||||
except OSError:
|
||||
return ""
|
||||
return "" if name == "0" else name
|
||||
|
||||
|
||||
def trace(root: int) -> list[str]:
|
||||
"""Returns:
|
||||
`<pid> <comm> <state> [<wchan>]` of the tree() members of root, the
|
||||
ones without a child in the tree first (the stuck one, not the
|
||||
ancestors waiting on it), then by pid, at most TRACE_LIMIT; comm with
|
||||
control bytes escaped, as a process sets its own. Never a command
|
||||
line, which can carry credentials. Empty when /proc cannot be listed
|
||||
or does not show root.
|
||||
"""
|
||||
processes = table()
|
||||
if processes is None or root not in processes:
|
||||
return []
|
||||
in_tree = tree(processes, root)
|
||||
parents = {process.parent for process in in_tree}
|
||||
return [" ".join(filter(None, (str(process.pid), process.comm.decode(errors="replace").encode("unicode_escape").decode(),
|
||||
process.state.decode(), wchan_of(process.pid))))
|
||||
for process in sorted(in_tree, key=lambda process: (process.pid in parents, process.pid))[:TRACE_LIMIT]]
|
||||
|
||||
|
||||
def probe(root: int) -> Sample | None:
|
||||
"""Args:
|
||||
root: the pid of the held command.
|
||||
|
||||
Returns:
|
||||
The tree() of root: its members as (pid, start time), their CPU
|
||||
ticks and io_of() counters summed, the ELSEWHERE commands among them
|
||||
and whether one is stopped or traced; None when /proc cannot be
|
||||
listed or does not show root, as a /proc of another PID namespace
|
||||
does not.
|
||||
"""
|
||||
processes = table()
|
||||
if processes is None or root not in processes:
|
||||
return None
|
||||
in_tree = tree(processes, root)
|
||||
return Sample(
|
||||
frozenset((process.pid, process.start) for process in in_tree),
|
||||
sum(process.ticks for process in in_tree),
|
||||
tuple(map(sum, zip(*(io_of(process.pid) for process in in_tree)))),
|
||||
tuple(sorted({process.comm for process in in_tree} & ELSEWHERE)),
|
||||
any(process.state in PAUSED for process in in_tree),
|
||||
)
|
||||
|
||||
|
||||
def say(text: str) -> None:
|
||||
print(f"queue: {text}", file=sys.stderr, flush=True)
|
||||
|
||||
|
||||
class Clock:
|
||||
"""Clock how long the process tree of a held command stood still.
|
||||
|
||||
Args:
|
||||
root: the command's pid.
|
||||
"""
|
||||
|
||||
def __init__(self, root: int) -> None:
|
||||
self.root = root
|
||||
self.seen: Sample | None = None
|
||||
self.since = time.monotonic()
|
||||
self.blind = False
|
||||
self.away = False
|
||||
|
||||
def hung(self) -> bool:
|
||||
"""Returns:
|
||||
Whether probe() of the tree gave the same sample for
|
||||
HUNG_SECONDS. A probe that sees nothing, a tree running an
|
||||
ELSEWHERE client, whose work runs outside the tree, and a tree
|
||||
with a stopped or traced member restart the clock and never fire.
|
||||
"""
|
||||
now = time.monotonic()
|
||||
seen = probe(self.root)
|
||||
self.warn(seen)
|
||||
if seen is None or seen.held or seen != self.seen:
|
||||
self.since = now
|
||||
self.seen = seen
|
||||
return seen is not None and not seen.held and now - self.since >= HUNG_SECONDS
|
||||
|
||||
def warn(self, seen: Sample | None) -> None:
|
||||
"""Args:
|
||||
seen: the probe() of this poll; the first None of the run says
|
||||
on stderr that the guard is off, the first naming an
|
||||
ELSEWHERE client that it holds off.
|
||||
"""
|
||||
if seen is None and not self.blind:
|
||||
say(f"/proc does not show process {self.root}; the hung-job guard is off for it")
|
||||
self.blind = True
|
||||
if seen is not None and seen.elsewhere and not self.away:
|
||||
names = ", ".join(name.decode(errors="replace") for name in seen.elsewhere)
|
||||
say(f"the command runs {names}, whose work /proc does not show; the hung-job guard holds off while one runs")
|
||||
self.away = True
|
||||
|
||||
def report(self) -> dict[str, float | int | list[str]]:
|
||||
"""Returns:
|
||||
The sample of the last hung() that saw the tree, for the history
|
||||
row: cpu_s (CPU seconds), read_bytes and written_bytes (rchar and
|
||||
wchar), still_min (minutes without a change), members (the
|
||||
number of processes) and tree (their trace(), taken now, before
|
||||
any signal).
|
||||
"""
|
||||
rchar, wchar = self.seen.io[:2]
|
||||
return {"cpu_s": round(self.seen.ticks / TICKS, 2), "read_bytes": rchar, "written_bytes": wchar,
|
||||
"still_min": round((time.monotonic() - self.since) / 60, 1), "members": len(self.seen.members),
|
||||
"tree": trace(self.root)}
|
||||
|
||||
|
||||
def told(report: dict) -> str:
|
||||
return (f"the command's process tree stood still for {report['still_min']:g} min (CPU {report['cpu_s']:g} s, "
|
||||
f"{report['read_bytes']} bytes read, {report['written_bytes']} written, all unchanged); "
|
||||
f"tree of {report['members']}: {', '.join(report['tree'])}")
|
||||
|
||||
|
||||
def members(root: int) -> dict[int, int]:
|
||||
"""Returns:
|
||||
The tree() of root as {pid: start time}; empty when /proc cannot be
|
||||
listed or does not show root.
|
||||
"""
|
||||
processes = table()
|
||||
if processes is None or root not in processes:
|
||||
return {}
|
||||
return {process.pid: process.start for process in tree(processes, root)}
|
||||
|
||||
|
||||
def alive(targets: dict[int, int]) -> dict[int, int]:
|
||||
"""Args:
|
||||
targets: {pid: start time} as members() recorded them, the command
|
||||
among them (a zombie still shows in /proc).
|
||||
|
||||
Returns:
|
||||
The targets still running with their recorded start time, so a
|
||||
reused pid never counts, the members of every session such a target
|
||||
or a target gone from /proc leads, as a process a member starts
|
||||
during the grace keeps its session when its parent exits (Linux
|
||||
keeps a session id allocated while a member lives, so no reused pid
|
||||
matches it), and the descendants all of them have now, as
|
||||
{pid: start time}; zombies are left out, and nothing when /proc
|
||||
cannot be listed.
|
||||
"""
|
||||
processes = table() or {}
|
||||
kept = [pid for pid, start in targets.items() if pid in processes and processes[pid].start == start]
|
||||
leaders = {pid for pid, start in targets.items() if pid not in processes or processes[pid].start == start}
|
||||
kept += [process.pid for process in processes.values() if process.session in leaders]
|
||||
return {pid: processes[pid].start for pid in family(processes, kept) if processes[pid].state not in GONE}
|
||||
|
||||
|
||||
def signal_all(targets: dict[int, int], signum: int) -> dict[int, int]:
|
||||
"""Send signum to every process alive() finds of targets.
|
||||
|
||||
Returns:
|
||||
Those processes as {pid: start time}.
|
||||
"""
|
||||
living = alive(targets)
|
||||
for pid in living:
|
||||
with contextlib.suppress(ProcessLookupError, PermissionError):
|
||||
os.kill(pid, signum)
|
||||
return living
|
||||
|
||||
|
||||
def settle(targets: dict[int, int], reap: Callable[[], int | None], signum: int | None) -> dict[int, int]:
|
||||
"""Wait up to KILL_GRACE_SECONDS until the command is reaped and alive()
|
||||
finds none of targets, recording every process it finds on the way, so
|
||||
one a member starts in a session of its own during the grace stays a
|
||||
target after its parent exits; one started and orphaned between two
|
||||
passes, SETTLE_SECONDS and a /proc scan apart, is not seen.
|
||||
|
||||
Args:
|
||||
targets: {pid: start time} of the tree.
|
||||
reap: Popen.poll of the command, None while it runs.
|
||||
signum: sent to everything each pass finds (signal_all()), so a
|
||||
process forked while the last pass signalled gets it too; None
|
||||
only waits, letting a member's cleanup run out the grace.
|
||||
|
||||
Returns:
|
||||
targets and every process found, as {pid: start time}.
|
||||
"""
|
||||
seen = dict(targets)
|
||||
deadline = time.monotonic() + KILL_GRACE_SECONDS
|
||||
while time.monotonic() < deadline:
|
||||
living = alive(seen) if signum is None else signal_all(seen, signum)
|
||||
seen |= living
|
||||
if reap() is not None and not living:
|
||||
break
|
||||
time.sleep(SETTLE_SECONDS)
|
||||
return seen
|
||||
|
||||
|
||||
def suspect_seconds(longest: float | None) -> float:
|
||||
"""Args:
|
||||
longest: the longest learned run of a job's signature in minutes,
|
||||
None while nothing is learned.
|
||||
|
||||
Returns:
|
||||
The runtime past which list marks the job suspect hung:
|
||||
SUSPECT_FACTOR times longest, at least SUSPECT_FLOOR_SECONDS.
|
||||
"""
|
||||
if longest is None:
|
||||
return SUSPECT_FLOOR_SECONDS
|
||||
return max(SUSPECT_FLOOR_SECONDS, SUSPECT_FACTOR * longest * 60)
|
||||
@@ -0,0 +1,560 @@
|
||||
"""Exercise the kvb-org-queue hung-job guard: the process-tree probe, hold ending a tree that stands still and sparing one that works, the list mark and the learning that skips hung runs."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import contextlib
|
||||
import importlib.util
|
||||
import io
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import signal
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import time
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from unittest import mock
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parents[1]
|
||||
SCRIPTS = REPO_ROOT / "skills" / "kvb-org-queue" / "scripts"
|
||||
QUEUE = SCRIPTS / "queue.py"
|
||||
HISTORY = SCRIPTS / "history.py"
|
||||
STANDSTILL = SCRIPTS / "standstill.py"
|
||||
NOW = 1_800_000_000.0
|
||||
POLL = 0.1
|
||||
HUNG = 2.0
|
||||
GRACE = 0.5
|
||||
OUTLAST = HUNG + 0.5
|
||||
SAMPLE = {"free_gb": 50.0, "mem": 20.0, "cpu": 10.0}
|
||||
PROBE_KEYS = {"cpu_s", "read_bytes", "written_bytes", "still_min", "members", "tree"}
|
||||
OPEN_MEMORY = {
|
||||
"available": -1, "swap_free": -1, "psi_some": 1000, "psi_full": 1000, "swap_out_mb_min": 1e9, "container_swap_gb": 1e9,
|
||||
}
|
||||
SLEEP = "import time\ntime.sleep(60)\n"
|
||||
PRINT = (
|
||||
f"import sys, time\nend = time.monotonic() + {OUTLAST}\n"
|
||||
"while time.monotonic() < end:\n sys.stdout.write('x\\n')\n sys.stdout.flush()\n time.sleep(0.05)\n"
|
||||
)
|
||||
BURN = f"import time\nend = time.monotonic() + {OUTLAST}\nwhile time.monotonic() < end:\n pass\n"
|
||||
STUBBORN = (
|
||||
"import os, signal, sys, time\n"
|
||||
"signal.signal(signal.SIGTERM, signal.SIG_IGN)\n"
|
||||
"with open(sys.argv[1] + '.part', 'w') as out:\n"
|
||||
" out.write(str(os.getpid()))\n"
|
||||
"os.replace(sys.argv[1] + '.part', sys.argv[1])\n"
|
||||
"time.sleep(60)\n"
|
||||
)
|
||||
MEMBER = (
|
||||
"import os, signal, sys, time\n"
|
||||
"def ended(*_):\n"
|
||||
" with open(sys.argv[1], 'w') as out:\n"
|
||||
" out.write('term')\n"
|
||||
" os._exit(0)\n"
|
||||
"signal.signal(signal.SIGTERM, ended)\n"
|
||||
"time.sleep(60)\n"
|
||||
)
|
||||
SAME_SESSION = f"import subprocess, sys\nsubprocess.run([sys.executable, '-c', {BURN!r}], check=True)\n"
|
||||
NEW_SESSION = f"import subprocess, sys\nsubprocess.run([sys.executable, '-c', {BURN!r}], check=True, start_new_session=True)\n"
|
||||
FAMILY = (
|
||||
"import subprocess, sys, time\n"
|
||||
f"subprocess.Popen([sys.executable, '-c', {MEMBER!r}, sys.argv[1]])\n"
|
||||
f"subprocess.Popen([sys.executable, '-c', {MEMBER!r}, sys.argv[2]], process_group=0)\n"
|
||||
"time.sleep(60)\n"
|
||||
)
|
||||
ABANDONED = (
|
||||
"import subprocess, sys, time\n"
|
||||
f"subprocess.Popen([sys.executable, '-c', {STUBBORN!r}, sys.argv[1]], process_group=0)\n"
|
||||
"time.sleep(60)\n"
|
||||
)
|
||||
CLEANUP_WORK = 0.5
|
||||
|
||||
|
||||
def cleanup(own_session: bool, work: float) -> str:
|
||||
"""Args:
|
||||
own_session: whether the process the SIGTERM handler starts leads a
|
||||
session of its own.
|
||||
work: seconds the handler works once that process is ready, 0 to
|
||||
exit at once; the session sweep alone, deterministically, is
|
||||
test_signals_reach_members_whose_start_time_still_matches_the_session_and_their_new_descendants.
|
||||
|
||||
Returns:
|
||||
A command whose SIGTERM handler starts STUBBORN, works, and exits.
|
||||
"""
|
||||
return (
|
||||
"import os, signal, subprocess, sys, time\n"
|
||||
"def ended(*_):\n"
|
||||
f" subprocess.Popen([sys.executable, '-c', {STUBBORN!r}, sys.argv[1]], start_new_session={own_session})\n"
|
||||
" while not os.path.exists(sys.argv[1]):\n"
|
||||
" time.sleep(0.02)\n"
|
||||
f" time.sleep({work})\n"
|
||||
" os._exit(0)\n"
|
||||
"signal.signal(signal.SIGTERM, ended)\n"
|
||||
"time.sleep(60)\n"
|
||||
)
|
||||
|
||||
|
||||
CLEANUP = cleanup(False, 0)
|
||||
SETSID_CLEANUP = cleanup(True, CLEANUP_WORK)
|
||||
SESSION_CLEANUP = (
|
||||
"import subprocess, sys, time\n"
|
||||
f"subprocess.Popen([sys.executable, '-c', {CLEANUP!r}, sys.argv[1]], start_new_session=True)\n"
|
||||
"time.sleep(60)\n"
|
||||
)
|
||||
CLEANUP_GRACE = 3.0
|
||||
CLIENT = f"import subprocess, sys\nsubprocess.run([sys.argv[1], '{OUTLAST}'], check=True)\n"
|
||||
STOPPED = (
|
||||
"import signal, subprocess, sys, time\n"
|
||||
"member = subprocess.Popen([sys.executable, '-c', 'import time; time.sleep(60)'])\n"
|
||||
"member.send_signal(signal.SIGSTOP)\n"
|
||||
f"time.sleep({OUTLAST})\n"
|
||||
"member.kill()\n"
|
||||
"member.wait()\n"
|
||||
)
|
||||
HOLD = (
|
||||
"import importlib.util, sys\n"
|
||||
"spec = importlib.util.spec_from_file_location('kvb_queue_hung', sys.argv[1])\n"
|
||||
"queue = importlib.util.module_from_spec(spec)\n"
|
||||
"spec.loader.exec_module(queue)\n"
|
||||
f"queue.POLL_SECONDS = {POLL}\n"
|
||||
f"queue.standstill.HUNG_SECONDS = {HUNG}\n"
|
||||
f"queue.standstill.KILL_GRACE_SECONDS = {GRACE}\n"
|
||||
"sys.argv = sys.argv[1:]\n"
|
||||
"sys.exit(queue.main())\n"
|
||||
)
|
||||
|
||||
|
||||
def _load(path: Path):
|
||||
spec = importlib.util.spec_from_file_location(f"{path.stem}_hung_under_test", path)
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(module)
|
||||
return module
|
||||
|
||||
|
||||
def history_row(seq: int, **fields: object) -> dict:
|
||||
return {
|
||||
"id": f"r{seq}", "kind": "test", "signature": "w: make test", "workdir": "/w", "start": NOW - 600, "end": NOW,
|
||||
"minutes": 10.0, "exit": 0, "free_before_gb": 50.0, "min_free_gb": 49.0, "disk_gb": 1.0, "mem_before": 20.0,
|
||||
"mem_max": 30.0, "cpu_before": 10.0, "cpu_max": 40.0, "concurrent": 0, **fields,
|
||||
}
|
||||
|
||||
|
||||
def stat_line(pid: int, comm: str, parent: int, session: int, times: tuple[int, int, int, int], state: str, start: int) -> bytes:
|
||||
after = [state, str(parent), str(pid), str(session), "0", "-1", "4194560", "100", "0", "0", "0",
|
||||
*map(str, times), "20", "0", "1", "0", str(start), "0", "0"]
|
||||
return f"{pid} ({comm}) {' '.join(after)}\n".encode()
|
||||
|
||||
|
||||
def plain(pid: int, parent: int, session: int, utime: int) -> bytes:
|
||||
return stat_line(pid, "make", parent, session, (utime, 0, 0, 0), "S", 1000 + pid)
|
||||
|
||||
|
||||
def io_lines(rchar: int, wchar: int, read_bytes: int, write_bytes: int) -> bytes:
|
||||
return (f"rchar: {rchar}\nwchar: {wchar}\nsyscr: 1\nsyscw: 1\nread_bytes: {read_bytes}\n"
|
||||
f"write_bytes: {write_bytes}\ncancelled_write_bytes: 0\n").encode()
|
||||
|
||||
|
||||
def awaited(path: Path) -> int:
|
||||
deadline = time.monotonic() + 30
|
||||
while not path.exists():
|
||||
if time.monotonic() > deadline:
|
||||
raise AssertionError(f"{path} never appeared")
|
||||
time.sleep(0.02)
|
||||
return int(path.read_text(encoding="utf-8"))
|
||||
|
||||
|
||||
def ended(pid: int) -> bool:
|
||||
try:
|
||||
stat = Path(f"/proc/{pid}/stat").read_bytes()
|
||||
except OSError:
|
||||
return True
|
||||
return stat.rsplit(b")", 1)[1].split()[0] in (b"Z", b"X")
|
||||
|
||||
|
||||
def kill_quietly(pid: int) -> None:
|
||||
with contextlib.suppress(ProcessLookupError):
|
||||
os.kill(pid, signal.SIGKILL)
|
||||
|
||||
|
||||
class TestProbe(unittest.TestCase):
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
cls.standstill = _load(STANDSTILL)
|
||||
|
||||
def setUp(self):
|
||||
tmp = tempfile.TemporaryDirectory()
|
||||
self.addCleanup(tmp.cleanup)
|
||||
self.proc = Path(tmp.name) / "proc"
|
||||
self.proc.mkdir()
|
||||
patched = mock.patch.object(self.standstill, "PROC", self.proc)
|
||||
patched.start()
|
||||
self.addCleanup(patched.stop)
|
||||
|
||||
def process(self, pid: int, stat: bytes | None, io: bytes | None) -> None:
|
||||
folder = self.proc / str(pid)
|
||||
folder.mkdir(exist_ok=True)
|
||||
if stat is not None:
|
||||
(folder / "stat").write_bytes(stat)
|
||||
if io is not None:
|
||||
(folder / "io").write_bytes(io)
|
||||
|
||||
def test_stat_splits_after_the_last_parenthesis_so_a_command_name_with_parentheses_reads_right(self):
|
||||
line = stat_line(4242, "a) (b c) 9 9", 7, 4242, (7, 3, 2, 1), "T", 999)
|
||||
self.assertEqual(self.standstill.parse_stat(line), (4242, b"a) (b c) 9 9", b"T", 7, 4242, 13, 999))
|
||||
self.assertNotEqual(line.split(b")", 1)[1].split()[3], b"4242")
|
||||
|
||||
def test_the_probe_walks_the_whole_tree_whatever_its_session_and_skips_vanished_and_foreign_processes(self):
|
||||
self.process(4242, plain(4242, 100, 4242, 10), io_lines(0, 100, 0, 0))
|
||||
self.process(4243, plain(4243, 4242, 4243, 5), None)
|
||||
self.process(4244, plain(4244, 4243, 4244, 7), io_lines(30, 0, 0, 0))
|
||||
self.process(4245, plain(4245, 1, 4242, 1), None)
|
||||
self.process(4246, plain(4246, 1, 1, 100), io_lines(9000, 9000, 0, 0))
|
||||
self.process(4247, None, None)
|
||||
(self.proc / "self").mkdir()
|
||||
seen = self.standstill.probe(4242)
|
||||
self.assertEqual(seen.members, {(pid, 1000 + pid) for pid in (4242, 4243, 4244, 4245)})
|
||||
self.assertEqual((seen.ticks, seen.io, seen.elsewhere, seen.paused), (23, (30, 100, 0, 0), (), False))
|
||||
self.assertEqual(self.standstill.probe(4246).ticks, 100)
|
||||
self.assertIsNone(self.standstill.probe(4999))
|
||||
|
||||
def test_cpu_time_reads_writes_and_a_new_process_each_count_as_progress(self):
|
||||
self.process(4242, plain(4242, 1, 4242, 1), io_lines(5, 10, 0, 0))
|
||||
clock = self.standstill.Clock(4242)
|
||||
changes = (
|
||||
(4242, plain(4242, 1, 4242, 2), None),
|
||||
(4242, None, io_lines(6, 10, 0, 0)),
|
||||
(4242, None, io_lines(6, 11, 0, 0)),
|
||||
(4242, None, io_lines(6, 11, 1, 0)),
|
||||
(4242, None, io_lines(6, 11, 1, 1)),
|
||||
(4243, plain(4243, 4242, 4243, 0), None),
|
||||
)
|
||||
with mock.patch.object(self.standstill, "HUNG_SECONDS", 0.2):
|
||||
self.assertFalse(clock.hung())
|
||||
for pid, stat, counters in changes:
|
||||
time.sleep(0.25)
|
||||
self.assertTrue(clock.hung())
|
||||
self.process(pid, stat, counters)
|
||||
self.assertFalse(clock.hung(), (pid, stat, counters))
|
||||
time.sleep(0.25)
|
||||
self.assertTrue(clock.hung())
|
||||
report = clock.report()
|
||||
self.assertEqual((report["cpu_s"], report["read_bytes"], report["written_bytes"]), (round(2 / self.standstill.TICKS, 2), 6, 11))
|
||||
self.assertGreaterEqual(report["still_min"], 0)
|
||||
self.assertEqual((report["members"], report["tree"]), (2, ["4243 make S", "4242 make S"]))
|
||||
|
||||
def test_the_trace_names_the_leaves_first_with_state_and_wait_channel_but_never_a_command_line(self):
|
||||
self.process(4242, plain(4242, 1, 4242, 0), None)
|
||||
self.process(4243, stat_line(4243, "sleep", 4242, 4242, (0, 0, 0, 0), "D", 5243), None)
|
||||
self.process(4244, plain(4244, 4242, 4242, 0), None)
|
||||
self.process(4245, stat_line(4245, "x\x1b[2J", 4242, 4242, (0, 0, 0, 0), "S", 5245), None)
|
||||
for pid, wchan in ((4242, "do_wait"), (4243, "folio_wait_bit"), (4244, "0")):
|
||||
(self.proc / str(pid) / "wchan").write_text(wchan, encoding="ascii")
|
||||
(self.proc / "4243" / "cmdline").write_bytes(b"sleep\0--token=secret\0")
|
||||
leaves = ["4243 sleep D folio_wait_bit", "4244 make S", "4245 x\\x1b[2J S"]
|
||||
self.assertEqual(self.standstill.trace(4242), [*leaves, "4242 make S do_wait"])
|
||||
with mock.patch.object(self.standstill, "TRACE_LIMIT", 2):
|
||||
self.assertEqual(self.standstill.trace(4242), leaves[:2])
|
||||
self.assertEqual(self.standstill.trace(4999), [])
|
||||
|
||||
def test_a_stopped_member_or_a_client_whose_work_runs_elsewhere_never_fires_and_the_client_is_told_once(self):
|
||||
self.process(4242, plain(4242, 1, 4242, 1), None)
|
||||
held = (("docker", "S"), ("docker-compose", "S"), ("podman", "S"), ("ssh", "S"), ("sleep", "T"), ("python3", "t"))
|
||||
with mock.patch.object(self.standstill, "HUNG_SECONDS", 0), contextlib.redirect_stderr(io.StringIO()) as err:
|
||||
self.assertTrue(self.standstill.Clock(4242).hung())
|
||||
for comm, state in held:
|
||||
self.process(4243, stat_line(4243, comm, 4242, 4243, (0, 0, 0, 0), state, 5000), None)
|
||||
clock = self.standstill.Clock(4242)
|
||||
self.assertEqual([clock.hung() for _ in range(3)], [False, False, False], comm)
|
||||
self.assertEqual(err.getvalue().count("holds off"), 4)
|
||||
|
||||
def test_an_unreadable_proc_never_fires_and_says_so_once(self):
|
||||
self.process(4242, plain(4242, 1, 4242, 1), io_lines(0, 10, 0, 0))
|
||||
with mock.patch.object(self.standstill, "HUNG_SECONDS", 0):
|
||||
self.assertTrue(self.standstill.Clock(4242).hung())
|
||||
clock = self.standstill.Clock(4242)
|
||||
with mock.patch.object(self.standstill, "PROC", self.proc / "missing"), \
|
||||
contextlib.redirect_stderr(io.StringIO()) as err:
|
||||
self.assertEqual([clock.hung() for _ in range(3)], [False, False, False])
|
||||
self.assertEqual(err.getvalue().count("hung-job guard is off"), 1)
|
||||
|
||||
def test_signals_reach_members_whose_start_time_still_matches_the_session_and_their_new_descendants(self):
|
||||
self.process(4242, plain(4242, 1, 4242, 0), None)
|
||||
self.process(4243, plain(4243, 4242, 4242, 0), None)
|
||||
self.process(4244, stat_line(4244, "make", 4242, 4242, (0, 0, 0, 0), "Z", 5244), None)
|
||||
self.process(4245, plain(4245, 1, 4242, 0), None)
|
||||
self.process(4250, plain(4250, 4243, 4250, 0), None)
|
||||
self.process(4260, plain(4260, 1, 1, 0), None)
|
||||
self.process(4270, plain(4270, 1, 4270, 0), None)
|
||||
self.process(4271, plain(4271, 4270, 4271, 0), None)
|
||||
self.process(4281, plain(4281, 1, 4280, 0), None)
|
||||
self.assertEqual(self.standstill.members(4242), {pid: 1000 + pid for pid in (4242, 4243, 4244, 4245, 4250)})
|
||||
self.assertEqual(self.standstill.members(4999), {})
|
||||
with mock.patch.object(self.standstill.os, "kill") as kill:
|
||||
living = self.standstill.signal_all({4242: 5242, 4244: 5244, 4270: 1, 4280: 5280}, signal.SIGTERM)
|
||||
self.assertEqual(living, {4242: 5242, 4243: 5243, 4245: 5245, 4250: 5250, 4281: 5281})
|
||||
self.assertEqual(sorted(call.args for call in kill.call_args_list),
|
||||
[(pid, signal.SIGTERM) for pid in (4242, 4243, 4245, 4250, 4281)])
|
||||
|
||||
def test_the_suspect_runtime_is_three_times_the_longest_run_with_a_floor(self):
|
||||
self.assertEqual(self.standstill.suspect_seconds(20), 3600)
|
||||
self.assertEqual(self.standstill.suspect_seconds(5), self.standstill.SUSPECT_FLOOR_SECONDS)
|
||||
self.assertEqual(self.standstill.suspect_seconds(None), self.standstill.SUSPECT_FLOOR_SECONDS)
|
||||
|
||||
|
||||
class TestSupervise(unittest.TestCase):
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
cls.queue = _load(QUEUE)
|
||||
|
||||
def start(self, code: str, *args: str) -> subprocess.Popen:
|
||||
child = subprocess.Popen([sys.executable, "-c", code, *args], start_new_session=True,
|
||||
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
|
||||
self.addCleanup(child.wait, 30)
|
||||
self.addCleanup(self.queue.send, child, signal.SIGKILL)
|
||||
return child
|
||||
|
||||
def scratch(self) -> Path:
|
||||
tmp = tempfile.TemporaryDirectory()
|
||||
self.addCleanup(tmp.cleanup)
|
||||
return Path(tmp.name)
|
||||
|
||||
def supervised(self, child: subprocess.Popen, *extra) -> tuple[tuple[int, dict | None], float, str]:
|
||||
"""Args:
|
||||
child: the started command.
|
||||
extra: patches entered after the defaults, so they win.
|
||||
"""
|
||||
patches = (
|
||||
mock.patch.object(self.queue, "POLL_SECONDS", POLL),
|
||||
mock.patch.object(self.queue.standstill, "HUNG_SECONDS", HUNG),
|
||||
mock.patch.object(self.queue.standstill, "KILL_GRACE_SECONDS", GRACE),
|
||||
mock.patch.object(self.queue, "measure", return_value=SAMPLE),
|
||||
*extra,
|
||||
)
|
||||
with contextlib.ExitStack() as stack, contextlib.redirect_stderr(io.StringIO()) as err:
|
||||
for patch in patches:
|
||||
stack.enter_context(patch)
|
||||
started = time.monotonic()
|
||||
result = self.queue.supervise({}, child, [])
|
||||
return result, time.monotonic() - started, err.getvalue()
|
||||
|
||||
def test_a_sleeping_command_is_ended_with_124_and_its_probe(self):
|
||||
child = self.start(SLEEP)
|
||||
(code, probe), took, err = self.supervised(child)
|
||||
self.assertEqual(code, 124)
|
||||
self.assertEqual(set(probe), PROBE_KEYS)
|
||||
self.assertEqual(probe["members"], 1)
|
||||
self.assertRegex(probe["tree"][0], rf"^{child.pid} \S+ S")
|
||||
self.assertEqual(child.returncode, -signal.SIGTERM)
|
||||
self.assertLess(took, HUNG + 10)
|
||||
self.assertIn("stood still", err)
|
||||
self.assertIn(f"tree of 1: {child.pid} ", err)
|
||||
|
||||
def test_a_command_that_keeps_printing_is_not_ended(self):
|
||||
self.assertEqual(self.supervised(self.start(PRINT))[0], (0, None))
|
||||
|
||||
def test_a_command_that_burns_cpu_without_printing_is_not_ended(self):
|
||||
self.assertEqual(self.supervised(self.start(BURN))[0], (0, None))
|
||||
|
||||
def test_a_command_ignoring_sigterm_is_killed_after_the_grace(self):
|
||||
ready = self.scratch() / "ready"
|
||||
child = self.start(STUBBORN, str(ready))
|
||||
awaited(ready)
|
||||
(code, probe), took, _ = self.supervised(child)
|
||||
self.assertEqual((code, child.returncode, set(probe)), (124, -signal.SIGKILL, PROBE_KEYS))
|
||||
self.assertGreaterEqual(took, HUNG + GRACE)
|
||||
|
||||
def test_a_working_grandchild_counts_in_the_same_session_and_in_a_new_one(self):
|
||||
self.assertEqual(self.supervised(self.start(SAME_SESSION))[0], (0, None))
|
||||
self.assertEqual(self.supervised(self.start(NEW_SESSION))[0], (0, None))
|
||||
|
||||
def test_a_sleeping_tree_is_ended_and_every_member_gets_sigterm_whatever_its_process_group(self):
|
||||
folder = self.scratch()
|
||||
marks = [folder / "same-group", folder / "own-group"]
|
||||
child = self.start(FAMILY, *map(str, marks))
|
||||
(code, _), _, _ = self.supervised(child)
|
||||
self.assertEqual((code, child.returncode), (124, -signal.SIGTERM))
|
||||
self.assertEqual([mark.read_text(encoding="utf-8") for mark in marks], ["term", "term"])
|
||||
|
||||
def test_a_member_ignoring_sigterm_is_killed_after_the_grace_although_the_leader_already_exited(self):
|
||||
ready = self.scratch() / "ready"
|
||||
child = self.start(ABANDONED, str(ready))
|
||||
member = awaited(ready)
|
||||
(code, _), took, _ = self.supervised(child)
|
||||
self.assertEqual((code, child.returncode), (124, -signal.SIGTERM))
|
||||
self.assertTrue(ended(member))
|
||||
self.assertGreaterEqual(took, HUNG + GRACE)
|
||||
|
||||
def test_a_process_a_member_starts_during_the_grace_is_killed_although_its_parent_exited(self):
|
||||
cases = (("same session", CLEANUP, 0), ("member in own session", SESSION_CLEANUP, -signal.SIGTERM),
|
||||
("orphan in own session", SETSID_CLEANUP, 0))
|
||||
for case, script, leader_exit in cases:
|
||||
with self.subTest(case):
|
||||
ready = self.scratch() / "ready"
|
||||
child = self.start(script, str(ready))
|
||||
grace = mock.patch.object(self.queue.standstill, "KILL_GRACE_SECONDS", CLEANUP_GRACE)
|
||||
(code, _), _, _ = self.supervised(child, grace)
|
||||
orphan = awaited(ready)
|
||||
self.addCleanup(kill_quietly, orphan)
|
||||
self.assertEqual((code, child.returncode), (124, leader_exit))
|
||||
self.assertTrue(ended(orphan))
|
||||
|
||||
def test_a_process_forked_while_sigkill_goes_out_is_killed_on_the_next_pass(self):
|
||||
proc = self.scratch() / "proc"
|
||||
|
||||
def spawn(pid: int, parent: int) -> None:
|
||||
(proc / str(pid)).mkdir(parents=True)
|
||||
(proc / str(pid) / "stat").write_bytes(plain(pid, parent, 4242, 0))
|
||||
|
||||
def kill(pid: int, signum: int) -> None:
|
||||
if signum == signal.SIGKILL:
|
||||
shutil.rmtree(proc / str(pid))
|
||||
if pid == 4243:
|
||||
spawn(4250, 1)
|
||||
|
||||
spawn(4242, 1)
|
||||
spawn(4243, 4242)
|
||||
child = mock.Mock(pid=4242, returncode=None, poll=lambda: None if (proc / "4242").exists() else 0)
|
||||
with mock.patch.object(self.queue.standstill, "PROC", proc), \
|
||||
mock.patch.object(self.queue.standstill, "KILL_GRACE_SECONDS", GRACE), \
|
||||
mock.patch.object(self.queue.standstill.os, "kill", side_effect=kill) as sent, \
|
||||
mock.patch.object(self.queue.os, "killpg") as group:
|
||||
self.assertEqual(self.queue.stop(child), 124)
|
||||
self.assertIn(mock.call(4250, signal.SIGKILL), sent.call_args_list)
|
||||
self.assertEqual(sorted(path.name for path in proc.iterdir()), [])
|
||||
group.assert_not_called()
|
||||
|
||||
def test_a_tree_running_a_container_client_never_fires_and_says_so_once(self):
|
||||
sleep = shutil.which("sleep")
|
||||
if sleep is None:
|
||||
self.skipTest("no sleep binary on PATH")
|
||||
docker = self.scratch() / "docker"
|
||||
docker.symlink_to(sleep)
|
||||
result, _, err = self.supervised(self.start(CLIENT, str(docker)))
|
||||
self.assertEqual(result, (0, None))
|
||||
self.assertEqual(err.count("the command runs docker,"), 1)
|
||||
|
||||
def test_a_tree_with_a_stopped_member_never_fires(self):
|
||||
self.assertEqual(self.supervised(self.start(STOPPED))[0], (0, None))
|
||||
|
||||
def test_stop_falls_back_to_the_process_group_when_proc_shows_nothing(self):
|
||||
child = self.start(SLEEP)
|
||||
with mock.patch.object(self.queue.standstill, "PROC", self.scratch() / "missing"), \
|
||||
mock.patch.object(self.queue.standstill, "KILL_GRACE_SECONDS", GRACE):
|
||||
self.assertEqual(self.queue.stop(child), 124)
|
||||
self.assertEqual(child.returncode, -signal.SIGTERM)
|
||||
|
||||
|
||||
class TestHungHistory(unittest.TestCase):
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
cls.history = _load(HISTORY)
|
||||
cls.queue = _load(QUEUE)
|
||||
|
||||
def learn(self, rows: list[dict]) -> dict:
|
||||
return self.history.learn(rows, NOW, {"load_limits": {"mem_percent": 80, "cpu_percent": 90}})[0]
|
||||
|
||||
def test_a_hung_row_stays_out_of_the_learned_durations_and_shares(self):
|
||||
rows = [history_row(n) for n in range(5)]
|
||||
probe = {"cpu_s": 1.0, "read_bytes": 0, "written_bytes": 0, "still_min": 60.0}
|
||||
stuck = history_row(5, minutes=500.0, mem_max=95.0, exit=124)
|
||||
stats = self.learn([*rows, {**stuck, self.history.HUNG: probe}])["test", "w: make test"]
|
||||
self.assertEqual((stats["runs"], stats["minutes"], stats["longest"], stats["ram_share"]), (5, 10.0, 10.0, 10.0))
|
||||
counted = self.learn([*rows, stuck])["test", "w: make test"]
|
||||
self.assertEqual((counted["runs"], counted["longest"]), (6, 500.0))
|
||||
|
||||
def test_list_marks_an_active_job_past_three_times_its_longest_run_and_past_the_floor(self):
|
||||
signatures = self.learn([history_row(n, minutes=20.0 - n) for n in range(5)])
|
||||
entry = {"kind": "test", "aliases": ["w: make test"], "granted": NOW}
|
||||
suspected = self.queue.suspected
|
||||
self.assertIsNone(suspected({**entry, "granted": NOW - 59 * 60}, None, signatures, NOW))
|
||||
self.assertEqual(suspected({**entry, "granted": NOW - 61 * 60}, None, signatures, NOW), "suspect hung (61 min)")
|
||||
self.assertEqual(suspected({**entry, "granted": NOW - 61 * 60}, "backfilled", signatures, NOW),
|
||||
"suspect hung (61 min); backfilled")
|
||||
unknown = {**entry, "aliases": ["w: make lint"]}
|
||||
self.assertEqual(suspected({**unknown, "granted": NOW - 29 * 60}, "backfilled", signatures, NOW), "backfilled")
|
||||
self.assertEqual(suspected({**unknown, "granted": NOW - 31 * 60}, None, signatures, NOW), "suspect hung (31 min)")
|
||||
short = self.learn([history_row(n, minutes=5.0) for n in range(5)])
|
||||
self.assertIsNone(suspected({**entry, "granted": NOW - 20 * 60}, None, short, NOW))
|
||||
|
||||
|
||||
class TestHungHold(unittest.TestCase):
|
||||
def setUp(self):
|
||||
tmp = tempfile.TemporaryDirectory()
|
||||
self.addCleanup(tmp.cleanup)
|
||||
self.base = Path(tmp.name).resolve()
|
||||
self.root = self.base / "council"
|
||||
self.root.mkdir()
|
||||
home = self.base / "home"
|
||||
home.mkdir()
|
||||
self.history_file = self.root / "queue-history.jsonl"
|
||||
self.config = self.base / "config.json"
|
||||
self.config.write_text(json.dumps({
|
||||
"disk": str(self.base),
|
||||
"thresholds_gb": {"warn": 0, "stop": 0, "critical": 0},
|
||||
"projection_minutes": 30,
|
||||
"slot_need_gb": {"test": 0.001},
|
||||
"slot_limits": {"test": 1},
|
||||
"load_limits": {"cpu_percent": 101, "mem_percent": 101, "sample_seconds": 0.1},
|
||||
"budget_percent": {"ram": 75, "cpu": 100},
|
||||
"memory_limits": {stage: OPEN_MEMORY for stage in ("warn", "stop")},
|
||||
"heavy_commands": ["^make compose-deploy"],
|
||||
"never_block": ["^make compose-down"],
|
||||
}), encoding="utf-8")
|
||||
(self.root / "rasci.json").write_text(json.dumps({"work/a": {"roles": {}, "status": "running"}}), encoding="utf-8")
|
||||
self.env = {
|
||||
"PATH": os.environ["PATH"],
|
||||
"HOME": str(home),
|
||||
"GIT_CEILING_DIRECTORIES": str(self.base.parent),
|
||||
"AGENTS_COUNCIL_ROOT": str(self.root),
|
||||
"AGENTS_COUNCIL_CONFIG": str(self.config),
|
||||
"CLAUDE_PROJECT_DIR": str(self.base / "project"),
|
||||
}
|
||||
|
||||
def run_queue(self, *args: str) -> subprocess.CompletedProcess:
|
||||
return subprocess.run(
|
||||
[sys.executable, "-c", HOLD, str(QUEUE), *args], cwd=self.base, env=self.env, stdin=subprocess.DEVNULL,
|
||||
capture_output=True, text=True, timeout=60, check=False,
|
||||
)
|
||||
|
||||
def hold(self, code: str) -> subprocess.CompletedProcess:
|
||||
return self.run_queue("hold", "--kind", "test", "--task", "work/a", "--session", "s1", "--", sys.executable, "-c", code)
|
||||
|
||||
def rows(self) -> list[dict]:
|
||||
return [json.loads(line) for line in self.history_file.read_text(encoding="utf-8").splitlines()]
|
||||
|
||||
def test_a_hold_whose_command_sleeps_exits_124_and_its_history_row_marks_it_hung(self):
|
||||
result = self.hold(SLEEP)
|
||||
self.assertEqual(result.returncode, 124, result.stderr)
|
||||
self.assertIn("stood still", result.stderr)
|
||||
[row] = self.rows()
|
||||
self.assertEqual((row["exit"], set(row["hung"])), (124, PROBE_KEYS))
|
||||
self.assertEqual(json.loads((self.root / "queue.json").read_text(encoding="utf-8"))["entries"], [])
|
||||
|
||||
def test_a_hold_whose_command_works_keeps_its_exit_code_and_an_unmarked_row(self):
|
||||
result = self.hold(BURN + "raise SystemExit(5)\n")
|
||||
self.assertEqual((result.returncode, result.stderr), (5, ""))
|
||||
[row] = self.rows()
|
||||
self.assertEqual(row["exit"], 5)
|
||||
self.assertNotIn("hung", row)
|
||||
|
||||
def test_list_marks_an_active_job_past_the_floor_when_nothing_is_learned(self):
|
||||
queue = _load(QUEUE)
|
||||
me, mine = os.getpid(), queue.identity(os.getpid())
|
||||
now = time.time()
|
||||
floor = queue.standstill.SUSPECT_FLOOR_SECONDS
|
||||
entry = {
|
||||
"seq": 1, "kind": "test", "task": "work/a", "need_gb": 0, "hotfix": None, "workdir": "/w", "session": "s1",
|
||||
"name": "s1", "pid": me, "pid_start": mine, "child": me, "child_start": mine, "status": "active",
|
||||
"enqueued": now - floor - 120, "cmd": "make test", "code": queue.CODE,
|
||||
}
|
||||
entries = [{**entry, "id": "test-1", "granted": now - floor - 60}, {**entry, "id": "test-2", "seq": 2, "granted": now - floor + 60}]
|
||||
(self.root / "queue.json").write_text(json.dumps({"seq": 2, "entries": entries}), encoding="utf-8")
|
||||
listing = self.run_queue("list")
|
||||
self.assertEqual(listing.returncode, 0, listing.stderr)
|
||||
reasons = [line.split("\t")[-1] for line in listing.stdout.splitlines()[1:]]
|
||||
self.assertEqual(reasons, [f"suspect hung ({(floor + 60) // 60} min)", "-"])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in new issue
Block a user