fix(kvb-org-queue): global budget so no queue mix can OOM the host
Root cause of the 2026-11-07 20:14 OOM: the queue granted jobs per kind only, so several heavy kinds and direct heavy commands outside the queue ran side by side until the kernel killed the editor. - queue.py: every grant checks the sum of the learned RAM and CPU footprints of active and granted jobs against budget_percent; an unlearned kind takes the whole budget and runs alone. RAM used plus what young jobs (RAMP_SECONDS) may still take must stay below load_limits. Memory stage stop blocks every grant. - watch.py: memory stage from MemAvailable, swap, PSI full and the swap of the container holding most of it; the block hook classifies builds, make install/verify-*, and pytest or unittest beyond one file as heavy through wrappers, sh -c, command substitutions and heredocs a shell reads. - history.py: footprints per kind and signature, seed RAM for pending shares. - operator.py: [h]old queues a heavy row through queue.py hold and asks again on its grant, with a flushed tty and a timeout; sudo rows, which need the terminal, stay direct. Residuals: a shell reading code from a pipe or here-string, its own script, a program named by a substitution, quotes inside an expanding heredoc, an escaped backslash before a substitution; a lone job waits while the host is over load_limits. 🎯 Fix: 92 % 💥 Break: 4 % 💸 Cost: 100 % ⏱️ Time: – 🛡️ Security: 100 % 🏆 Score: 88 🚦 Gate: ✅ 90/5, cost and time <= 100, security 100 % Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Kevin Veen-BirkenbachandClaude Opus 5.5 committed
1 parent
09f68dd75b
commit
4321b97529
8 files changed
+1665
-218
No files matched your search
@@ -32,8 +32,19 @@ Args:
|
||||
visible), reason and check, then a preview of what the command
|
||||
changes, and asks [y]es / [s]kip / [d]iff / [q]uit; d shows the full
|
||||
diff. y runs the command in its directory, then sets the row done
|
||||
when its check passes (manual: when the command exits 0). End of
|
||||
input quits. There is no flag that answers for the operator.
|
||||
when its check passes (manual: when the command exits 0). A command
|
||||
the kvb-org-organize watch config calls heavy also shows the memory
|
||||
stage and `queue.py list`; once the config holds the queue keys and
|
||||
slot_need_gb.operator, and the command runs no sudo, [h]old runs
|
||||
`granted` through `queue.py hold --kind operator --task <row>
|
||||
--session Operator`. End of input quits. There is no flag that
|
||||
answers for the operator.
|
||||
granted <task> --dir <dir> --check <check> -- $SHELL -ic <cmd>: what
|
||||
[h]old queues. On the grant it shows the row and its preview again
|
||||
and asks [y]es / [s]kip / [d]iff / [q]uit, then runs the command in
|
||||
the row's directory; a row no longer open with exactly that
|
||||
directory, check and command runs nothing, nor does a question left
|
||||
unanswered for GRANT_SECONDS.
|
||||
done <slug>: mark a row done, whatever its check.
|
||||
|
||||
Output:
|
||||
@@ -45,7 +56,8 @@ Output:
|
||||
prints the same rows as a JSON list.
|
||||
|
||||
Exit codes:
|
||||
0 done, 2 usage error.
|
||||
0 done, 2 usage error. granted: the command's exit code, or 75
|
||||
(EX_TEMPFAIL, which queue.py hold does not learn from) when nothing ran.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -62,13 +74,25 @@ import hashlib
|
||||
import importlib.util
|
||||
import json
|
||||
import re
|
||||
import select
|
||||
import shlex
|
||||
import shutil
|
||||
import subprocess
|
||||
import termios
|
||||
import time
|
||||
from collections.abc import Callable
|
||||
from pathlib import Path
|
||||
from types import ModuleType
|
||||
|
||||
RASCI = Path(__file__).resolve().parents[2] / "kvb-org-rasci" / "scripts" / "rasci.py"
|
||||
SKILLS = Path(__file__).resolve().parents[2]
|
||||
RASCI = SKILLS / "kvb-org-rasci" / "scripts" / "rasci.py"
|
||||
WATCH = SKILLS / "kvb-org-organize" / "scripts" / "watch.py"
|
||||
QUEUE = SKILLS / "kvb-org-queue" / "scripts" / "queue.py"
|
||||
HOLD_KIND = "operator"
|
||||
HOLD_SESSION = "Operator"
|
||||
NOT_RUN = 75
|
||||
GRANT_SECONDS = 600
|
||||
SUDO = re.compile(r"\bsudo\b")
|
||||
PREFIX = "operator/"
|
||||
KINDS = ("pushed", "installed", "binary", "absent", "manual", "decision")
|
||||
OPEN = ("queued", "blocked")
|
||||
@@ -264,8 +288,8 @@ def runnable(row: dict) -> bool:
|
||||
return bool(row["directory"] and row["command"]) and row["check"] != "decision"
|
||||
|
||||
|
||||
def visible(text: str) -> str:
|
||||
return "".join(c if c.isprintable() else c.encode("unicode_escape").decode() for c in text)
|
||||
def visible(text: str, keep: str) -> str:
|
||||
return "".join(c if c.isprintable() or c in keep else c.encode("unicode_escape").decode() for c in text)
|
||||
|
||||
|
||||
def git_tree(directory: str) -> bool:
|
||||
@@ -312,14 +336,113 @@ def show(commands: list[tuple[list[str], str]]) -> None:
|
||||
print(f" {error}", flush=True)
|
||||
|
||||
|
||||
def ask() -> str:
|
||||
def ask(holdable: bool, read: Callable[[str], str | None]) -> str:
|
||||
choices = ("y", "h", "s", "d", "q") if holdable else ("y", "s", "d", "q")
|
||||
prompt = "Run it? [y]es / [h]old in the queue / [s]kip / [d]iff / [q]uit: " if holdable else "Run it? [y]es / [s]kip / [d]iff / [q]uit: "
|
||||
while True:
|
||||
try:
|
||||
answer = input("Run it? [y]es / [s]kip / [d]iff / [q]uit: ").strip().lower()
|
||||
except EOFError:
|
||||
answer = read(prompt)
|
||||
if answer is None:
|
||||
return "q"
|
||||
if answer in ("y", "s", "d", "q"):
|
||||
return answer
|
||||
if answer.strip().lower() in choices:
|
||||
return answer.strip().lower()
|
||||
|
||||
|
||||
def typed(prompt: str) -> str | None:
|
||||
try:
|
||||
return input(prompt)
|
||||
except EOFError:
|
||||
return None
|
||||
|
||||
|
||||
def timed(prompt: str) -> str | None:
|
||||
"""Returns:
|
||||
One line read unbuffered from stdin, None at its end or when no line
|
||||
came within GRANT_SECONDS, so an unanswered grant frees its slot;
|
||||
what was typed while the job waited is dropped first.
|
||||
"""
|
||||
if sys.stdin.isatty():
|
||||
termios.tcflush(sys.stdin.fileno(), termios.TCIFLUSH)
|
||||
print(prompt, end="", flush=True)
|
||||
deadline = time.monotonic() + GRANT_SECONDS
|
||||
line = b""
|
||||
while not line.endswith(b"\n"):
|
||||
ready, _, _ = select.select([sys.stdin.fileno()], [], [], max(0.0, deadline - time.monotonic()))
|
||||
byte = os.read(sys.stdin.fileno(), 1) if ready else b""
|
||||
if not byte:
|
||||
print(flush=True)
|
||||
return None
|
||||
line += byte
|
||||
return line.decode(errors="replace")
|
||||
|
||||
|
||||
def needs_terminal(command: str) -> bool:
|
||||
return SUDO.search(command) is not None
|
||||
|
||||
|
||||
def heavy_row(watch: ModuleType, cfg: dict | None, row: dict) -> bool:
|
||||
"""Returns:
|
||||
Whether [h]old can queue the row's command: a heavy job by the
|
||||
kvb-org-organize watch config, which holds every queue key and the
|
||||
kind operator, and no sudo, whose prompt needs the terminal a queued
|
||||
command lacks; prints the memory stage and `queue.py list` for a
|
||||
heavy one, its output control characters made visible.
|
||||
"""
|
||||
if cfg is None or not watch.heavy(row["command"], cfg, tuple(cfg["never_block"])):
|
||||
return False
|
||||
problem = watch.missing(cfg, (*watch.MEMORY_KEYS, "budget_percent.ram", "budget_percent.cpu", f"slot_need_gb.{HOLD_KIND}"))
|
||||
if problem is None and needs_terminal(row["command"]):
|
||||
problem = "sudo asks on the terminal, which a queued command lacks"
|
||||
if problem is None:
|
||||
sample = watch.memory()
|
||||
print(f" heavy: memory stage {watch.memory_stage(sample, cfg)} ({watch.memory_text(sample)}); the queue:", flush=True)
|
||||
else:
|
||||
print(f" heavy: {problem}; no [h]old until it does. The queue:", flush=True)
|
||||
listing = subprocess.run([sys.executable, str(QUEUE), "list"], cwd=row["directory"], capture_output=True, text=True, check=False)
|
||||
print(visible(listing.stdout + listing.stderr, "\t\n"), end="", flush=True)
|
||||
return problem is None
|
||||
|
||||
|
||||
def held(row: dict, shell: str) -> list[str]:
|
||||
return [
|
||||
sys.executable, str(QUEUE), "hold", "--kind", HOLD_KIND, "--task", row["task"], "--session", HOLD_SESSION,
|
||||
"--", sys.executable, str(Path(__file__).resolve()), "granted", row["task"], "--dir", row["directory"],
|
||||
"--check", row["check"], "--", shell, "-ic", row["command"],
|
||||
]
|
||||
|
||||
|
||||
def review(row: dict, holdable: bool, read: Callable[[str], str | None]) -> str:
|
||||
check = "manual" if problem(row["check"]) else row["check"]
|
||||
show(evidence(check, row["directory"], full=False))
|
||||
answer = ask(holdable, read)
|
||||
while answer == "d":
|
||||
show(evidence(check, row["directory"], full=True))
|
||||
answer = ask(holdable, read)
|
||||
return answer
|
||||
|
||||
|
||||
def cmd_granted(rasci: ModuleType, args: argparse.Namespace) -> int:
|
||||
shell = os.environ.get("SHELL")
|
||||
approved = [
|
||||
row for row in open_rows(rasci)
|
||||
if runnable(row) and (row["task"], row["directory"], row["check"]) == (args.task, args.dir, args.check)
|
||||
and args.argv == [shell, "-ic", row["command"]]
|
||||
]
|
||||
if not shell or not approved:
|
||||
print(f"operator: {args.task} is no longer open with the approved directory, check and command; nothing ran",
|
||||
file=sys.stderr)
|
||||
return NOT_RUN
|
||||
row = approved[0]
|
||||
print(f"\ngranted {row['task']}: the queue lets it run now; its tree may have changed while it waited")
|
||||
print(f" directory: {visible(row['directory'], '')}")
|
||||
print(f" command: {visible(row['command'], '')}", flush=True)
|
||||
try:
|
||||
answer = review(row, False, timed)
|
||||
except KeyboardInterrupt:
|
||||
answer = "q"
|
||||
if answer != "y":
|
||||
print(f"not run {row['task']}")
|
||||
return NOT_RUN
|
||||
return subprocess.run(args.argv, cwd=row["directory"], check=False).returncode
|
||||
|
||||
|
||||
def close(rasci: ModuleType, task: str) -> None:
|
||||
@@ -337,26 +460,26 @@ def cmd_run(rasci: ModuleType, args: argparse.Namespace) -> int:
|
||||
if not steps:
|
||||
print("operator: nothing to run")
|
||||
return 0
|
||||
watch = module("kvb_operator_watch", WATCH)
|
||||
cfg = watch.load_config()
|
||||
for number, row in enumerate(steps, 1):
|
||||
check = "manual" if problem(row["check"]) else row["check"]
|
||||
print(f"\n[{number}/{len(steps)}] {row['task']}")
|
||||
print(f" directory: {visible(row['directory'])}")
|
||||
print(f" command: {visible(row['command'])}")
|
||||
print(f" why: {visible(row['why'] or '-')}")
|
||||
print(f" directory: {visible(row['directory'], '')}")
|
||||
print(f" command: {visible(row['command'], '')}")
|
||||
print(f" why: {visible(row['why'] or '-', '')}")
|
||||
print(f" check: {check}", flush=True)
|
||||
show(evidence(check, row["directory"], full=False))
|
||||
answer = ask()
|
||||
while answer == "d":
|
||||
show(evidence(check, row["directory"], full=True))
|
||||
answer = ask()
|
||||
holdable = heavy_row(watch, cfg, row)
|
||||
answer = review(row, holdable, typed)
|
||||
if answer == "q":
|
||||
print("operator: stopped")
|
||||
return 0
|
||||
if answer == "s":
|
||||
print(f"skipped {row['task']}")
|
||||
continue
|
||||
argv = held(row, shell) if answer == "h" else [shell, "-ic", row["command"]]
|
||||
try:
|
||||
code = subprocess.run([shell, "-ic", row["command"]], cwd=row["directory"], check=False).returncode
|
||||
code = subprocess.run(argv, cwd=row["directory"], check=False).returncode
|
||||
except OSError as error:
|
||||
print(f"failed {row['task']}: {error}")
|
||||
continue
|
||||
@@ -394,6 +517,12 @@ def main() -> int:
|
||||
p.set_defaults(func=cmd_list)
|
||||
p = sub.add_parser("run")
|
||||
p.set_defaults(func=cmd_run)
|
||||
p = sub.add_parser("granted")
|
||||
p.add_argument("task")
|
||||
p.add_argument("--dir", required=True)
|
||||
p.add_argument("--check", required=True)
|
||||
p.add_argument("argv", nargs="+")
|
||||
p.set_defaults(func=cmd_granted)
|
||||
p = sub.add_parser("done")
|
||||
p.add_argument("slug")
|
||||
p.set_defaults(func=cmd_done)
|
||||
|
||||
@@ -9,11 +9,14 @@ Usage:
|
||||
(kvb-check-security) whose id it lacks to the same name
|
||||
under ~/.local/state/agents-council/
|
||||
watch.py block hook (PreToolUse on Bash): exit 2 for a heavy job while the
|
||||
stage is stop or critical, or while a kvb-org-radio mayday
|
||||
holds radio silence (<root>/radio-silence.json, until > now)
|
||||
disk stage is stop or critical, the memory stage is stop
|
||||
or cannot be judged without memory_limits, or a
|
||||
kvb-org-radio mayday holds radio silence
|
||||
(<root>/radio-silence.json, until > now)
|
||||
watch.py admit (--need-gb N | --kind KIND)
|
||||
exit 0 when the grant leaves at least the stop threshold
|
||||
free, 3 otherwise
|
||||
free, CPU busy and RAM used stay below load_limits and
|
||||
the memory stage is not stop, 3 otherwise
|
||||
watch.py cleanup (--caches | --full)
|
||||
trim caches to their budget; --full also runs the
|
||||
configured low-risk cleanup commands
|
||||
@@ -45,8 +48,15 @@ compose project name is skipped), worktree_marker (path relative
|
||||
to a worktree that must exist), busy_processes (process names; a worktree
|
||||
where one of them runs is skipped), cache_dirs, cache_budget_percent,
|
||||
slot_need_gb {kind: GB}, notify (argv the alert text is appended to),
|
||||
load_limits {cpu_percent, mem_percent, sample_seconds, kinds}: admit for a
|
||||
kind listed in kinds also needs CPU busy and RAM used below the limits.
|
||||
load_limits {cpu_percent, mem_percent, sample_seconds}: every admit and every
|
||||
queue grant needs CPU busy and RAM used below them, memory_limits {warn, stop},
|
||||
each {available, swap_free, psi_full, container_swap_gb}: the memory stage is
|
||||
the first of stop and warn whose MemAvailable or SwapFree percentage lies below
|
||||
its bound, whose /proc/pressure/memory full avg10 reaches its bound, or where
|
||||
one container (a docker-*.scope cgroup under system.slice, nested groups
|
||||
counted) holds at least container_swap_gb GB of swap, else ok; a host without
|
||||
swap or PSI skips that bound. heavy_commands extend the built-in rules of
|
||||
heavy().
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -56,6 +66,7 @@ import contextlib
|
||||
import fcntl
|
||||
import importlib.util
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
import re
|
||||
import shlex
|
||||
@@ -99,6 +110,86 @@ CLIP = 1000
|
||||
CLEANUP_LOCK = "cleanup.lock"
|
||||
LOCK_TTL = 1800
|
||||
GB = 1e9
|
||||
MEMINFO = Path("/proc/meminfo")
|
||||
PRESSURE = Path("/proc/pressure/memory")
|
||||
CGROUPS = Path("/sys/fs/cgroup/system.slice")
|
||||
SWAP_HOLDERS = 3
|
||||
NEST_DEPTH = 8
|
||||
MEMORY_STAGES = ("ok", "warn", "stop")
|
||||
MEMORY_BOUNDS = ("available", "swap_free", "psi_full", "container_swap_gb")
|
||||
MEMORY_KEYS = tuple(f"memory_limits.{stage}.{bound}" for stage in MEMORY_STAGES[1:] for bound in MEMORY_BOUNDS)
|
||||
CONTAINER_ID = re.compile(r"[0-9a-f]{64}")
|
||||
COMPOSE_VALUES = frozenset({
|
||||
"-f", "--file", "-p", "--project-name", "--project-directory", "--env-file", "--profile", "--ansi", "--parallel",
|
||||
"--progress",
|
||||
})
|
||||
BUILDING = frozenset({"up", "run", "create"})
|
||||
SEPARATOR = re.compile(r"[();<>|&\n]+")
|
||||
PUNCTUATION = "();<>|&\n"
|
||||
WORD_BREAK = " \t\n;&|()<>"
|
||||
ASSIGNMENT = re.compile(r"[A-Za-z_]\w*=.*", re.DOTALL)
|
||||
WRAPPERS = frozenset({
|
||||
"env", "sudo", "nice", "nohup", "timeout", "exec", "xargs", "stdbuf", "ionice", "uv", "uvx", "poetry", "pdm",
|
||||
"pipenv", "coverage", "time", "command",
|
||||
})
|
||||
UV_VALUES = frozenset({
|
||||
"--with", "--extra", "--group", "--python", "-p", "--project", "--directory", "--package", "--env-file", "--from",
|
||||
})
|
||||
WRAPPER_VALUES = {
|
||||
"sudo": frozenset({
|
||||
"-u", "-g", "-h", "-p", "-C", "-D", "-r", "-t", "-U", "-T", "--user", "--group", "--host", "--chdir", "--role",
|
||||
"--type", "--prompt", "--other-user", "--command-timeout", "--close-from",
|
||||
}),
|
||||
"timeout": frozenset({"-s", "-k", "--signal", "--kill-after"}),
|
||||
"env": frozenset({"-u", "-C", "--unset", "--chdir"}),
|
||||
"nice": frozenset({"-n", "--adjustment"}),
|
||||
"ionice": frozenset({"-c", "-n", "-p", "--class", "--classdata"}),
|
||||
"xargs": frozenset({"-a", "-d", "-E", "-I", "-L", "-n", "-P", "-s"}),
|
||||
"stdbuf": frozenset({"-i", "-o", "-e"}),
|
||||
"time": frozenset({"-f", "-o", "--format", "--output"}),
|
||||
"exec": frozenset({"-a"}),
|
||||
"uv": UV_VALUES,
|
||||
"uvx": UV_VALUES,
|
||||
"poetry": frozenset({"-C", "-P", "--directory", "--project"}),
|
||||
"pdm": frozenset({"-p", "--project"}),
|
||||
"coverage": frozenset({"--source", "--include", "--omit", "--rcfile", "--context", "--data-file"}),
|
||||
}
|
||||
WRAPPER_WORDS = frozenset({"run", "tool"})
|
||||
RESERVED = frozenset({"!", "{", "if", "then", "else", "elif", "do", "while", "until"})
|
||||
WRAPPER_ARGUMENT = re.compile(r"-.*|\d+(\.\d+)?[smhd]?", re.DOTALL)
|
||||
SHELLS = frozenset({"bash", "sh", "zsh", "dash"})
|
||||
INLINE = re.compile(r"-[A-Za-z]*c[A-Za-z]*")
|
||||
REDIRECT = re.compile(r"&?[<>]+&?")
|
||||
TRAILING_REDIRECT = re.compile(r"[<>]+&?$")
|
||||
PAREN = {"(": 1, ")": -1}
|
||||
PYTHON = re.compile(r"python[\d.]*")
|
||||
PYTHON_VALUE_OPTIONS = frozenset({"-X", "-W", "-Q"})
|
||||
DRY_RUN = frozenset({"-n", "--dry-run", "--just-print", "--recon", "-q", "--question"})
|
||||
SHORT_DRY = re.compile(r"-(?:(?![CfIoOWjl])[A-Za-z])*[nq][A-Za-z]*")
|
||||
SHELL_VALUE = re.compile(r"[-+][A-Za-z]*[oO]|--rcfile|--init-file")
|
||||
STDIN_FLAG = re.compile(r"-[A-Za-z]*s[A-Za-z]*")
|
||||
SPACE = re.compile(r"\s")
|
||||
ESCAPED = re.compile(r"\\[`$]")
|
||||
OPENER = re.compile(r"`|\$\(")
|
||||
PLACEHOLDER = "-_"
|
||||
HEREDOC_MARK = "kvbheredoc"
|
||||
MARK_WORD = re.compile(rf"{HEREDOC_MARK}((?:[0-9a-f]{{2}})*)")
|
||||
HELP = frozenset({"-h", "--help", "--version"})
|
||||
PYTEST_LIGHT = HELP | {"--collect-only", "--co"}
|
||||
TEST_FILE = re.compile(r"[^*?\[\]]+\.py(::.*)?", re.DOTALL)
|
||||
PYTEST_VALUE_OPTIONS = frozenset({
|
||||
"-p", "-k", "-m", "-c", "-o", "-W", "-n", "--numprocesses", "-r", "--maxfail", "--tb", "--rootdir", "--basetemp", "--deselect",
|
||||
"--ignore", "--ignore-glob", "--confcutdir", "--durations", "--junitxml", "--log-level", "--dist", "--cov",
|
||||
"--cov-report", "--cov-config", "--timeout", "--import-mode", "--capture", "--color", "--log-cli-level",
|
||||
"--log-file", "--reruns", "--tx", "--max-worker-restart",
|
||||
})
|
||||
UNITTEST_VALUE_OPTIONS = frozenset({"-k"})
|
||||
DISCOVER_VALUE_OPTIONS = frozenset({"-s", "-p", "-t", "-k", "--start-directory", "--pattern", "--top-level-directory"})
|
||||
PATTERN_OPTIONS = frozenset({"-p", "--pattern"})
|
||||
MAKE_VALUE_OPTIONS = frozenset({
|
||||
"-C", "-f", "-I", "-o", "-W", "--directory", "--file", "--makefile", "--include-dir", "--old-file", "--assume-old",
|
||||
"--what-if", "--new-file", "--assume-new",
|
||||
})
|
||||
|
||||
|
||||
def guard_root(hook: bool) -> int | None:
|
||||
@@ -123,6 +214,35 @@ def load_config() -> dict | None:
|
||||
return json.loads(CONFIG.read_text(encoding="utf-8"))
|
||||
|
||||
|
||||
def absent_part(cfg: dict, key: str) -> str | None:
|
||||
node: object = cfg
|
||||
parts = key.split(".")
|
||||
for depth, part in enumerate(parts, 1):
|
||||
if not isinstance(node, dict) or part not in node:
|
||||
return ".".join(parts[:depth])
|
||||
node = node[part]
|
||||
return None if isinstance(node, (int, float)) and not isinstance(node, bool) and math.isfinite(node) else key
|
||||
|
||||
|
||||
def missing(cfg: dict, keys: tuple[str, ...]) -> str | None:
|
||||
"""Args:
|
||||
cfg: the config.
|
||||
keys: required number keys, nested ones dotted
|
||||
(`memory_limits.stop.psi_full`).
|
||||
|
||||
Returns:
|
||||
None when every key holds a number, else the error naming the
|
||||
shortest missing part of each, or the key whose value is no number.
|
||||
"""
|
||||
absent = list(dict.fromkeys(part for key in keys if (part := absent_part(cfg, key)) is not None))
|
||||
if not absent:
|
||||
return None
|
||||
return (
|
||||
f"agents council: {CONFIG} lacks {', '.join(absent)}; add them as numbers as the watch.py and queue.py "
|
||||
"docstrings document"
|
||||
)
|
||||
|
||||
|
||||
def disk(cfg: dict) -> tuple[float, float]:
|
||||
st = os.statvfs(cfg["disk"])
|
||||
return st.f_bavail * st.f_frsize / GB, st.f_blocks * st.f_frsize / GB
|
||||
@@ -237,6 +357,19 @@ def act(cfg: dict, stage: str, previous: str, free: float) -> None:
|
||||
notify(cfg, f"Agents council: {head}. Heavy jobs blocked; a running stack needs the operator's OK to go.")
|
||||
|
||||
|
||||
def act_memory(cfg: dict, stage: str, previous: str, sample: dict[str, float | None]) -> None:
|
||||
head = memory_text(sample)
|
||||
if MEMORY_STAGES.index(stage) < MEMORY_STAGES.index(previous):
|
||||
alert(stage, f"RESOURCE MEMORY {stage.upper()}: recovered, {head}.")
|
||||
return
|
||||
holders = "; ".join(swap_holders(CGROUPS, container_names())) or "no container holds swap"
|
||||
if stage == "warn":
|
||||
alert(stage, f"RESOURCE MEMORY WARN: {head}. Top swap holders: {holders}. Chair: find the holder; peers: start nothing heavy.")
|
||||
else:
|
||||
alert(stage, f"RESOURCE MEMORY STOP: {head}. New heavy jobs are blocked. Top swap holders: {holders}.")
|
||||
notify(cfg, f"Agents council: memory nearly exhausted ({head}). Heavy jobs blocked; top swap holders: {holders}.")
|
||||
|
||||
|
||||
def history_rows(text: str) -> list[tuple[str, str]]:
|
||||
rows = []
|
||||
for line in text.splitlines():
|
||||
@@ -280,27 +413,42 @@ def stale(state: dict, cfg: dict, now: float) -> bool:
|
||||
return now - state.get("checked", 0) >= cfg["interval_seconds"]
|
||||
|
||||
|
||||
def cmd_watch(cfg: dict) -> int:
|
||||
def measure_and_act(cfg: dict, state: dict, now: float, problem: str | None) -> None:
|
||||
free, _ = disk(cfg)
|
||||
samples = [s for s in state.get("samples", []) if now - s[0] <= cfg["projection_minutes"] * 60]
|
||||
stage = stage_for(free, samples, cfg, now)
|
||||
previous = state.get("stage", "ok")
|
||||
samples.append([now, free])
|
||||
state.update(checked=now, free=free, stage=stage, samples=samples)
|
||||
sample = None if problem else memory()
|
||||
was_short = state.get("memory_stage", "ok")
|
||||
if sample is not None:
|
||||
state["memory_stage"] = memory_stage(sample, cfg)
|
||||
due = now - state.get("caches_checked", 0) >= cfg["cache_interval_seconds"]
|
||||
if due:
|
||||
state["caches_checked"] = now
|
||||
write_json(STATE, state)
|
||||
if stage != previous:
|
||||
act(cfg, stage, previous, free)
|
||||
elif due:
|
||||
spawn_cleanup("--caches")
|
||||
if sample is not None and state["memory_stage"] != was_short:
|
||||
act_memory(cfg, state["memory_stage"], was_short, sample)
|
||||
|
||||
|
||||
def cmd_watch(cfg: dict, problem: str | None) -> int:
|
||||
"""Args:
|
||||
cfg: the config.
|
||||
problem: why the memory stage cannot be judged, None when it can; the
|
||||
disk part still runs and every session sees the problem.
|
||||
"""
|
||||
payload = json.load(sys.stdin)
|
||||
now = time.time()
|
||||
state = read_json(STATE)
|
||||
if stale(state, cfg, now):
|
||||
free, _ = disk(cfg)
|
||||
samples = [s for s in state.get("samples", []) if now - s[0] <= cfg["projection_minutes"] * 60]
|
||||
stage = stage_for(free, samples, cfg, now)
|
||||
previous = state.get("stage", "ok")
|
||||
samples.append([now, free])
|
||||
state.update(checked=now, free=free, stage=stage, samples=samples)
|
||||
due = now - state.get("caches_checked", 0) >= cfg["cache_interval_seconds"]
|
||||
if due:
|
||||
state["caches_checked"] = now
|
||||
write_json(STATE, state)
|
||||
if stage != previous:
|
||||
act(cfg, stage, previous, free)
|
||||
elif due:
|
||||
spawn_cleanup("--caches")
|
||||
measure_and_act(cfg, state, now, problem)
|
||||
keep_history()
|
||||
texts = unseen_alerts(payload["session_id"])
|
||||
texts = unseen_alerts(payload["session_id"]) + ([] if problem is None else [problem])
|
||||
if texts:
|
||||
print(json.dumps({"hookSpecificOutput": {
|
||||
"hookEventName": payload["hook_event_name"],
|
||||
@@ -309,7 +457,10 @@ def cmd_watch(cfg: dict) -> int:
|
||||
return 0
|
||||
|
||||
|
||||
HEREDOC = re.compile(r"<<-?\s*(['\"]?)(\w+)\1[^\n]*\n.*?\n\s*\2(?=\s|\)|$)", re.DOTALL)
|
||||
HEREDOC = re.compile(
|
||||
r"<<-?\s*(?P<quote>['\"]?)(?P<escape>\\?)(?P<tag>[\w.-]+)(?P=quote)(?P<rest>[^\n]*)\n(?P<body>.*?)\n\s*(?P=tag)(?=\s|\)|$)",
|
||||
re.DOTALL,
|
||||
)
|
||||
QUOTED = re.compile(r"'[^']*'|\"(?:\\.|[^\"\\])*\"")
|
||||
|
||||
|
||||
@@ -322,32 +473,369 @@ def silence() -> dict | None:
|
||||
return state if state.get("until", 0) > time.time() else None
|
||||
|
||||
|
||||
def cmd_block(cfg: dict) -> int:
|
||||
def uncommented(command: str) -> str:
|
||||
kept: list[str] = []
|
||||
quote = ""
|
||||
escaped = skipping = False
|
||||
word_start = True
|
||||
for char in command:
|
||||
if skipping and char != "\n":
|
||||
continue
|
||||
skipping = False
|
||||
if escaped:
|
||||
escaped = False
|
||||
elif quote == "'":
|
||||
quote = "" if char == "'" else quote
|
||||
elif char == "\\":
|
||||
escaped = True
|
||||
elif quote == '"':
|
||||
quote = "" if char == '"' else quote
|
||||
elif char in "'\"":
|
||||
quote = char
|
||||
elif char == "#" and word_start:
|
||||
skipping = True
|
||||
continue
|
||||
kept.append(char)
|
||||
word_start = not quote and not escaped and char in WORD_BREAK
|
||||
return "".join(kept)
|
||||
|
||||
|
||||
def segments(command: str) -> list[list[str]]:
|
||||
"""Args:
|
||||
command: a shell command line.
|
||||
|
||||
Returns:
|
||||
The words of each simple command, split at newlines, `;`, `&`, `|`
|
||||
and parentheses; a redirection, its target and a file descriptor
|
||||
number in front of it are dropped and the command goes on.
|
||||
|
||||
Raises:
|
||||
ValueError: on an unbalanced quote.
|
||||
"""
|
||||
lexer = shlex.shlex(uncommented(command.replace("\\\n", "")), posix=True, punctuation_chars=PUNCTUATION)
|
||||
lexer.whitespace = " \t\r"
|
||||
lexer.whitespace_split = True
|
||||
lexer.commenters = ""
|
||||
parts: list[list[str]] = [[]]
|
||||
target = False
|
||||
for token in lexer:
|
||||
if REDIRECT.fullmatch(token):
|
||||
if parts[-1] and parts[-1][-1].isdigit():
|
||||
parts[-1].pop()
|
||||
target = True
|
||||
elif SEPARATOR.fullmatch(token):
|
||||
parts.append([])
|
||||
target = TRAILING_REDIRECT.search(token) is not None
|
||||
elif target:
|
||||
target = False
|
||||
else:
|
||||
parts[-1].append(token)
|
||||
return parts
|
||||
|
||||
|
||||
def program(words: list[str]) -> list[str]:
|
||||
"""Returns:
|
||||
The words from the program on: shell reserved words, VAR=value,
|
||||
wrappers such as env, sudo, time, timeout or `uv run`, their options,
|
||||
the values of their options and their numbers dropped; the split
|
||||
string of `env -S`, nothing for the lookup `command -v`.
|
||||
"""
|
||||
at, wrapper = 0, ""
|
||||
while at < len(words):
|
||||
word = words[at]
|
||||
if os.path.basename(word) in WRAPPERS:
|
||||
wrapper = os.path.basename(word)
|
||||
elif word in WRAPPER_VALUES.get(wrapper, ()):
|
||||
at += 1
|
||||
elif word in WRAPPER_TURNS.get(wrapper, {}):
|
||||
return WRAPPER_TURNS[wrapper][word](words[at + 1 :])
|
||||
elif not (word in RESERVED or ASSIGNMENT.fullmatch(word) or (wrapper and wrapped_word(word))):
|
||||
break
|
||||
at += 1
|
||||
return words[at:]
|
||||
|
||||
|
||||
def lookup(rest: list[str]) -> list[str]:
|
||||
return []
|
||||
|
||||
|
||||
def split_string(rest: list[str]) -> list[str]:
|
||||
return program(" ".join(rest[:1]).split() + rest[1:])
|
||||
|
||||
|
||||
WRAPPER_TURNS = {
|
||||
"command": {"-v": lookup, "-V": lookup},
|
||||
"env": {"-S": split_string, "--split-string": split_string},
|
||||
}
|
||||
|
||||
|
||||
def wrapped_word(word: str) -> bool:
|
||||
return word in WRAPPER_WORDS or WRAPPER_ARGUMENT.fullmatch(word) is not None
|
||||
|
||||
|
||||
def inline_code(words: list[str]) -> str | None:
|
||||
"""Args:
|
||||
words: a simple command from its program on.
|
||||
|
||||
Returns:
|
||||
The code a shell runs with -c (also clustered, such as -lc or -ic,
|
||||
and after a `--`), else None.
|
||||
"""
|
||||
if not words or os.path.basename(words[0]) not in SHELLS:
|
||||
return None
|
||||
rest = words[1:]
|
||||
for at, word in enumerate(rest):
|
||||
if INLINE.fullmatch(word):
|
||||
code = [later for later in rest[at + 1 :] if later != "--"][:1]
|
||||
return code[0] if code else None
|
||||
return None
|
||||
|
||||
|
||||
def operands(args: list[str], value_options: frozenset[str]) -> list[str]:
|
||||
found, skip = [], False
|
||||
for word in args:
|
||||
if skip:
|
||||
skip = False
|
||||
elif word in value_options:
|
||||
skip = True
|
||||
elif not word.startswith("-"):
|
||||
found.append(word)
|
||||
return found
|
||||
|
||||
|
||||
def python_module(args: list[str]) -> tuple[str, list[str]]:
|
||||
skip = False
|
||||
for at, word in enumerate(args):
|
||||
if skip:
|
||||
skip = False
|
||||
elif word == "-m":
|
||||
return (args[at + 1], args[at + 2 :]) if at + 1 < len(args) else ("", [])
|
||||
elif word.startswith("-m"):
|
||||
return word[2:], args[at + 1 :]
|
||||
elif word in PYTHON_VALUE_OPTIONS:
|
||||
skip = True
|
||||
elif not word.startswith("-"):
|
||||
return os.path.basename(word), args[at + 1 :]
|
||||
return "", []
|
||||
|
||||
|
||||
def light_unittest(args: list[str]) -> bool:
|
||||
if args[:1] != ["discover"]:
|
||||
return len(operands(args, UNITTEST_VALUE_OPTIONS)) == 1
|
||||
rest = args[1:]
|
||||
patterns = [rest[at + 1] for at, word in enumerate(rest[:-1]) if word in PATTERN_OPTIONS]
|
||||
patterns = patterns or operands(rest, DISCOVER_VALUE_OPTIONS)[1:2]
|
||||
return bool(patterns) and TEST_FILE.fullmatch(patterns[-1]) is not None
|
||||
|
||||
|
||||
def heavy_pytest(args: list[str]) -> bool:
|
||||
files = operands(args, PYTEST_VALUE_OPTIONS)
|
||||
return not (PYTEST_LIGHT.intersection(args) or (len(files) == 1 and TEST_FILE.fullmatch(files[0])))
|
||||
|
||||
|
||||
def heavy_unittest(args: list[str]) -> bool:
|
||||
return not (HELP.intersection(args) or light_unittest(args))
|
||||
|
||||
|
||||
def make_targets(args: list[str]) -> list[str]:
|
||||
return [word for word in operands(args, MAKE_VALUE_OPTIONS) if "=" not in word and not word.isdigit()]
|
||||
|
||||
|
||||
def dry_run(args: list[str]) -> bool:
|
||||
return bool(DRY_RUN.intersection(args)) or any(SHORT_DRY.fullmatch(word) for word in args)
|
||||
|
||||
|
||||
def heavy_make(args: list[str]) -> bool:
|
||||
return not dry_run(args) and any(target == "install" or target.startswith("verify-") for target in make_targets(args))
|
||||
|
||||
|
||||
def heavy_compose(args: list[str]) -> bool:
|
||||
words = operands(args, COMPOSE_VALUES)
|
||||
command = words[0] if words else ""
|
||||
return command == "build" or (command in BUILDING and "--build" in args)
|
||||
|
||||
|
||||
def heavy_docker(args: list[str]) -> bool:
|
||||
return args[:1] == ["compose"] and heavy_compose(args[1:])
|
||||
|
||||
|
||||
HEAVY_TOOLS = {
|
||||
"pytest": heavy_pytest, "py.test": heavy_pytest, "unittest": heavy_unittest, "make": heavy_make,
|
||||
"docker": heavy_docker, "docker-compose": heavy_compose,
|
||||
}
|
||||
|
||||
|
||||
def heavy_words(run: list[str], cfg: dict, exempt: tuple[str, ...]) -> bool:
|
||||
if not run:
|
||||
return False
|
||||
code = inline_code(run)
|
||||
if code is not None:
|
||||
return heavy(code, cfg, exempt)
|
||||
name, args = os.path.basename(run[0]), run[1:]
|
||||
if PYTHON.fullmatch(name):
|
||||
name, args = python_module(args)
|
||||
if name in WRAPPERS:
|
||||
return heavy_words(program([name, *args]), cfg, exempt)
|
||||
return name in HEAVY_TOOLS and HEAVY_TOOLS[name](args)
|
||||
|
||||
|
||||
def segment_texts(words: list[str], run: list[str]) -> list[str]:
|
||||
return list(dict.fromkeys(" ".join("''" if SPACE.search(word) else word for word in part) for part in (words, run)))
|
||||
|
||||
|
||||
def judged(texts: list[str], run: list[str], cfg: dict, exempt: tuple[str, ...]) -> bool:
|
||||
if any(re.search(pattern, text) for pattern in exempt for text in texts):
|
||||
return False
|
||||
if any(re.search(pattern, text) for pattern in cfg["heavy_commands"] for text in texts):
|
||||
return True
|
||||
return heavy_words(run, cfg, exempt)
|
||||
|
||||
|
||||
def heavy_part(words: list[str], cfg: dict, exempt: tuple[str, ...]) -> bool:
|
||||
"""Returns:
|
||||
Whether one simple command is heavy; a make call is judged per target
|
||||
as `make <target>`, so an exempt target never covers the next one.
|
||||
"""
|
||||
run = program(words)
|
||||
if run and os.path.basename(run[0]) == "make" and not dry_run(run[1:]):
|
||||
return any(judged([f"make {target}"], ["make", target], cfg, exempt) for target in make_targets(run[1:]))
|
||||
return judged(segment_texts(words, run), run, cfg, exempt)
|
||||
|
||||
|
||||
def marked_heredocs(command: str) -> tuple[str, list[re.Match[str]]]:
|
||||
"""Returns:
|
||||
The command with each heredoc replaced by the word HEREDOC_MARK
|
||||
followed by its body in hex, so a nested call still reads it, the
|
||||
rest of its first line kept, and the heredoc matches.
|
||||
"""
|
||||
found: list[re.Match[str]] = []
|
||||
|
||||
def mark(match: re.Match[str]) -> str:
|
||||
found.append(match)
|
||||
return f" {HEREDOC_MARK}{match['body'].encode(errors='surrogatepass').hex()} {match['rest']}"
|
||||
|
||||
return HEREDOC.sub(mark, command), found
|
||||
|
||||
|
||||
def shell_operands(args: list[str]) -> list[str]:
|
||||
found, skip = [], False
|
||||
for word in args:
|
||||
if skip:
|
||||
skip = False
|
||||
elif word.startswith(("-", "+")):
|
||||
skip = SHELL_VALUE.fullmatch(word) is not None
|
||||
else:
|
||||
found.append(word)
|
||||
return found
|
||||
|
||||
|
||||
def reads_stdin(run: list[str]) -> bool:
|
||||
"""Returns:
|
||||
Whether a simple command from its program on is a shell that runs the
|
||||
code on its stdin: no -c code, and -s or no operand but heredoc marks.
|
||||
"""
|
||||
if not run or os.path.basename(run[0]) not in SHELLS or inline_code(run) is not None:
|
||||
return False
|
||||
args = run[1:]
|
||||
return any(STDIN_FLAG.fullmatch(word) for word in args) or all(MARK_WORD.fullmatch(word) for word in shell_operands(args))
|
||||
|
||||
|
||||
def expanded_codes(heredocs: list[re.Match[str]]) -> list[str]:
|
||||
"""Returns:
|
||||
The command substitutions in the bodies of heredocs whose delimiter
|
||||
is unquoted, which the shell expands.
|
||||
"""
|
||||
return [code for match in heredocs if not (match["quote"] or match["escape"]) for code in substituted(match["body"])[1]]
|
||||
|
||||
|
||||
def fed_bodies(parts: list[list[str]]) -> list[str]:
|
||||
marks = [MARK_WORD.fullmatch(word) for words in parts if reads_stdin(program(words)) for word in words]
|
||||
return [bytes.fromhex(mark[1]).decode(errors="ignore") for mark in marks if mark]
|
||||
|
||||
|
||||
def closing(text: str, start: int) -> int:
|
||||
depth = 1
|
||||
for at in range(start, len(text)):
|
||||
depth += PAREN.get(text[at], 0)
|
||||
if depth == 0:
|
||||
return at
|
||||
return len(text)
|
||||
|
||||
|
||||
def blanked_single(quoted: re.Match[str]) -> str:
|
||||
text = quoted[0]
|
||||
return text if text.startswith('"') else "'" + " " * (len(text) - 2) + "'"
|
||||
|
||||
|
||||
def substitution_end(shown: str, found: re.Match[str]) -> int:
|
||||
if found[0] == "`":
|
||||
end = shown.find("`", found.end())
|
||||
return len(shown) if end < 0 else end
|
||||
return closing(shown, found.end())
|
||||
|
||||
|
||||
def substituted(text: str) -> tuple[str, list[str]]:
|
||||
"""Returns:
|
||||
The text with every `...` and $(...) outside single quotes and
|
||||
escapes replaced by the option-like word PLACEHOLDER, so a wrapper
|
||||
around it still parses, and the code of each, which the shell runs
|
||||
first; nested ones stay inside their code.
|
||||
"""
|
||||
shown = QUOTED.sub(blanked_single, ESCAPED.sub(" ", text))
|
||||
kept: list[str] = []
|
||||
codes: list[str] = []
|
||||
at = 0
|
||||
found = OPENER.search(shown)
|
||||
while found:
|
||||
end = substitution_end(shown, found)
|
||||
kept.append(text[at : found.start()] + PLACEHOLDER)
|
||||
codes.append(text[found.end() : end])
|
||||
at = end + 1
|
||||
found = OPENER.search(shown, at)
|
||||
return "".join(kept) + text[at:], codes
|
||||
|
||||
|
||||
def cmd_block(cfg: dict, problem: str | None) -> int:
|
||||
"""Args:
|
||||
cfg: the config.
|
||||
problem: why the memory stage cannot be judged, None when it can; a
|
||||
heavy job is then refused with it.
|
||||
"""
|
||||
payload = json.load(sys.stdin)
|
||||
command = executable_text((payload.get("tool_input") or {}).get("command", ""))
|
||||
raw = (payload.get("tool_input") or {}).get("command", "")
|
||||
if not heavy(raw, cfg, tuple(cfg["never_block"])):
|
||||
return 0
|
||||
reason = block_reason(cfg, problem)
|
||||
if reason is None:
|
||||
return 0
|
||||
print(f"kvb-org-organize: {reason}. Teardown commands still run. Ask the resource chair.", file=sys.stderr)
|
||||
return 2
|
||||
|
||||
|
||||
def disk_stage(cfg: dict) -> str:
|
||||
state = read_json(STATE)
|
||||
now = time.time()
|
||||
stage = state.get("stage", "ok")
|
||||
if stale(state, cfg, now):
|
||||
stage = stage_for(disk(cfg)[0], state.get("samples", []), cfg, now)
|
||||
return stage_for(disk(cfg)[0], state.get("samples", []), cfg, now)
|
||||
return state.get("stage", "ok")
|
||||
|
||||
|
||||
def block_reason(cfg: dict, problem: str | None) -> str | None:
|
||||
mayday = silence()
|
||||
if stage not in ("stop", "critical") and not mayday:
|
||||
return 0
|
||||
if any(re.search(p, command) for p in cfg["never_block"]):
|
||||
return 0
|
||||
if any(re.search(p, command) for p in cfg["heavy_commands"]):
|
||||
reason = (
|
||||
if mayday:
|
||||
return (
|
||||
f"a mayday by {clip(mayday.get('by', '?'))} holds radio silence: {clip(mayday.get('text', ''))}; "
|
||||
"it ends with radio.py clear"
|
||||
if mayday
|
||||
else f"resource stage is {stage}; new heavy jobs are blocked until the disk recovers"
|
||||
)
|
||||
print(
|
||||
f"kvb-org-organize: {reason}. Teardown commands still run. Ask the resource chair.",
|
||||
file=sys.stderr,
|
||||
)
|
||||
return 2
|
||||
return 0
|
||||
stage = disk_stage(cfg)
|
||||
if stage in ("stop", "critical"):
|
||||
return f"resource stage is {stage}; new heavy jobs are blocked until the disk recovers"
|
||||
if problem is not None:
|
||||
return f"{problem}; new heavy jobs are blocked until it does"
|
||||
sample = memory()
|
||||
if memory_stage(sample, cfg) == "stop":
|
||||
return f"memory stage is stop ({memory_text(sample)}); new heavy jobs are blocked until memory recovers"
|
||||
return None
|
||||
|
||||
|
||||
def cpu_times() -> tuple[int, int]:
|
||||
@@ -361,12 +849,17 @@ def cpu_busy_percent(before: tuple[int, int], after: tuple[int, int]) -> float:
|
||||
return 100.0 * (total - idle) / total
|
||||
|
||||
|
||||
def mem_used_percent(meminfo: str) -> float:
|
||||
def meminfo_kb(meminfo: str) -> dict[str, int]:
|
||||
values = {}
|
||||
for line in meminfo.splitlines():
|
||||
key, _, rest = line.partition(":")
|
||||
if rest:
|
||||
values[key] = int(rest.split()[0])
|
||||
return values
|
||||
|
||||
|
||||
def mem_used_percent(meminfo: str) -> float:
|
||||
values = meminfo_kb(meminfo)
|
||||
return 100.0 * (1 - values["MemAvailable"] / values["MemTotal"])
|
||||
|
||||
|
||||
@@ -374,7 +867,161 @@ def load(cfg: dict) -> tuple[float, float]:
|
||||
before = cpu_times()
|
||||
time.sleep(cfg["load_limits"]["sample_seconds"])
|
||||
cpu = cpu_busy_percent(before, cpu_times())
|
||||
return cpu, mem_used_percent(Path("/proc/meminfo").read_text(encoding="utf-8"))
|
||||
return cpu, mem_used_percent(MEMINFO.read_text(encoding="utf-8"))
|
||||
|
||||
|
||||
def psi_full(pressure: str) -> float | None:
|
||||
for line in pressure.splitlines():
|
||||
if line.startswith("full "):
|
||||
return float(dict(field.split("=", 1) for field in line.split()[1:])["avg10"])
|
||||
return None
|
||||
|
||||
|
||||
def memory_sample(meminfo: str, pressure: str | None, container_swap_gb: float) -> dict[str, float | None]:
|
||||
"""Args:
|
||||
meminfo: the text of /proc/meminfo.
|
||||
pressure: the text of /proc/pressure/memory, None without PSI.
|
||||
container_swap_gb: the swap of the container holding the most, in GB.
|
||||
|
||||
Returns:
|
||||
{available, swap_free, psi_full, container_swap_gb}: MemAvailable and
|
||||
SwapFree in percent of their totals, the full avg10 stall percentage
|
||||
and container_swap_gb unchanged; swap_free is None without swap,
|
||||
psi_full None without PSI.
|
||||
"""
|
||||
values = meminfo_kb(meminfo)
|
||||
swap = values["SwapTotal"]
|
||||
return {
|
||||
"available": 100.0 * values["MemAvailable"] / values["MemTotal"],
|
||||
"swap_free": 100.0 * values["SwapFree"] / swap if swap else None,
|
||||
"psi_full": None if pressure is None else psi_full(pressure),
|
||||
"container_swap_gb": container_swap_gb,
|
||||
}
|
||||
|
||||
|
||||
def memory() -> dict[str, float | None]:
|
||||
try:
|
||||
pressure = PRESSURE.read_text(encoding="utf-8")
|
||||
except OSError:
|
||||
pressure = None
|
||||
largest = max((size for size, _ in container_swaps(CGROUPS)), default=0)
|
||||
return memory_sample(MEMINFO.read_text(encoding="utf-8"), pressure, largest / GB)
|
||||
|
||||
|
||||
def memory_stage(sample: dict[str, float | None], cfg: dict) -> str:
|
||||
for name in ("stop", "warn"):
|
||||
bound = cfg["memory_limits"][name]
|
||||
if (
|
||||
sample["available"] < bound["available"]
|
||||
or (sample["swap_free"] is not None and sample["swap_free"] < bound["swap_free"])
|
||||
or (sample["psi_full"] is not None and sample["psi_full"] >= bound["psi_full"])
|
||||
or sample["container_swap_gb"] >= bound["container_swap_gb"]
|
||||
):
|
||||
return name
|
||||
return "ok"
|
||||
|
||||
|
||||
def memory_text(sample: dict[str, float | None]) -> str:
|
||||
swap = "no swap" if sample["swap_free"] is None else f"swap free {sample['swap_free']:.0f} %"
|
||||
psi = "no PSI" if sample["psi_full"] is None else f"PSI full avg10 {sample['psi_full']:.1f}"
|
||||
return f"MemAvailable {sample['available']:.0f} %, {swap}, {psi}, largest container swap {sample['container_swap_gb']:.1f} GB"
|
||||
|
||||
|
||||
def swap_bytes(cgroup: Path) -> int:
|
||||
try:
|
||||
return int((cgroup / "memory.swap.current").read_text(encoding="utf-8"))
|
||||
except (OSError, ValueError):
|
||||
return 0
|
||||
|
||||
|
||||
def container_swaps(base: Path) -> list[tuple[int, Path]]:
|
||||
try:
|
||||
return [(swap_bytes(scope), scope) for scope in base.glob("docker-*.scope")]
|
||||
except OSError:
|
||||
return []
|
||||
|
||||
|
||||
def cgroup_label(name: str, names: dict[str, str]) -> str:
|
||||
bare = name.removeprefix("docker-").removesuffix(".scope")
|
||||
if CONTAINER_ID.fullmatch(bare):
|
||||
return names[bare] if bare in names else bare[:12]
|
||||
return bare
|
||||
|
||||
|
||||
def container_names() -> dict[str, str]:
|
||||
try:
|
||||
out = subprocess.run(
|
||||
["docker", "ps", "--no-trunc", "--format", "{{.ID}}\t{{.Names}}"],
|
||||
capture_output=True, text=True, timeout=30, check=False,
|
||||
).stdout
|
||||
except (OSError, subprocess.TimeoutExpired):
|
||||
return {}
|
||||
return dict(line.split("\t", 1) for line in out.splitlines() if "\t" in line)
|
||||
|
||||
|
||||
def swap_chain(cgroup: Path, held: int, names: dict[str, str]) -> list[str]:
|
||||
chain = [cgroup_label(cgroup.name, names)]
|
||||
for _ in range(NEST_DEPTH):
|
||||
try:
|
||||
children = [(swap_bytes(child), child) for child in cgroup.iterdir() if child.is_dir()]
|
||||
except OSError:
|
||||
break
|
||||
if not children:
|
||||
break
|
||||
size, child = max(children)
|
||||
if size * 2 < held:
|
||||
break
|
||||
chain.append(cgroup_label(child.name, names))
|
||||
cgroup, held = child, size
|
||||
return chain
|
||||
|
||||
|
||||
def swap_holders(base: Path, names: dict[str, str]) -> list[str]:
|
||||
"""Args:
|
||||
base: the cgroup directory holding the docker-<id>.scope groups.
|
||||
names: {full container id: name} for the labels.
|
||||
|
||||
Returns:
|
||||
Up to SWAP_HOLDERS lines, the largest swap first: the container, then
|
||||
each nested group holding at least half of its parent's swap, and the
|
||||
container's memory.swap.current in GB.
|
||||
"""
|
||||
ranked = sorted((pair for pair in container_swaps(base) if pair[0] > 0), reverse=True)[:SWAP_HOLDERS]
|
||||
return [f"{' > '.join(swap_chain(scope, size, names))} {size / GB:.1f} GB" for size, scope in ranked]
|
||||
|
||||
|
||||
def heavy(command: str, cfg: dict, exempt: tuple[str, ...]) -> bool:
|
||||
"""Args:
|
||||
command: the shell command line as typed, quotes included.
|
||||
cfg: the config; its heavy_commands search each simple command with
|
||||
quoted text blanked, and a make call also as `make <targets>`.
|
||||
exempt: regexes; a simple command one of them finds is never heavy
|
||||
(never_block, the queue's own hold), the others still count.
|
||||
|
||||
Returns:
|
||||
Whether one simple command is a heavy job: an image build through
|
||||
docker compose (build, or up, run or create with --build), a
|
||||
heavy_commands match, make install or verify-* (not as a dry run), or
|
||||
pytest or unittest on anything but exactly one test file or module,
|
||||
help and version aside. Wrappers (program()), the code of `sh -c`,
|
||||
command substitutions (substituted(), also in heredocs the shell
|
||||
expands) and heredocs a shell reads as its code count the same. A
|
||||
command with an unbalanced quote, which no shell runs, counts by the
|
||||
regexes on its whole text alone. A guard against accidental heavy
|
||||
runs, not a sandbox: a shell reading code from a pipe or a
|
||||
here-string, a script of its own, a program named by a substitution,
|
||||
quotes inside an expanding heredoc and an escaped backslash in front
|
||||
of a substitution are not looked into.
|
||||
"""
|
||||
marked, heredocs = marked_heredocs(command)
|
||||
plain, codes = substituted(uncommented(marked))
|
||||
try:
|
||||
parts = segments(plain)
|
||||
except ValueError:
|
||||
text = executable_text(command)
|
||||
return not any(re.search(p, text) for p in exempt) and any(re.search(p, text) for p in cfg["heavy_commands"])
|
||||
nested = codes + expanded_codes(heredocs) + fed_bodies(parts)
|
||||
return any(heavy_part(words, cfg, exempt) for words in parts) or any(heavy(code, cfg, exempt) for code in nested)
|
||||
|
||||
|
||||
def cmd_admit(cfg: dict, args: argparse.Namespace) -> int:
|
||||
@@ -385,12 +1032,12 @@ def cmd_admit(cfg: dict, args: argparse.Namespace) -> int:
|
||||
passed = [left >= floor]
|
||||
report = [f"free {free:.1f} GB, need {need} GB, leaves {left:.1f} GB (floor {floor} GB)"]
|
||||
limits = cfg["load_limits"]
|
||||
if args.kind in limits["kinds"]:
|
||||
cpu, mem = load(cfg)
|
||||
passed += [cpu < limits["cpu_percent"], mem < limits["mem_percent"]]
|
||||
report.append(
|
||||
f"cpu {cpu:.0f} % (limit {limits['cpu_percent']}), ram {mem:.0f} % (limit {limits['mem_percent']})"
|
||||
)
|
||||
cpu, mem = load(cfg)
|
||||
sample = memory()
|
||||
stage = memory_stage(sample, cfg)
|
||||
passed += [cpu < limits["cpu_percent"], mem < limits["mem_percent"], stage != "stop"]
|
||||
report.append(f"cpu {cpu:.0f} % (limit {limits['cpu_percent']}), ram {mem:.0f} % (limit {limits['mem_percent']})")
|
||||
report.append(f"memory stage {stage} ({memory_text(sample)})")
|
||||
verdict = "grant" if all(passed) else "deny"
|
||||
print(f"{verdict}: {'; '.join(report)}")
|
||||
return 0 if verdict == "grant" else 3
|
||||
@@ -511,7 +1158,11 @@ def cmd_status(cfg: dict) -> int:
|
||||
now = time.time()
|
||||
samples = state.get("samples", [])
|
||||
age = f"{now - state['checked']:.0f}s" if "checked" in state else "unmeasured"
|
||||
print(f"stage={stage_for(free, samples, cfg, now)} free={free:.1f}GB total={total:.1f}GB samples={len(samples)} age={age}")
|
||||
sample = memory()
|
||||
print(
|
||||
f"stage={stage_for(free, samples, cfg, now)} free={free:.1f}GB total={total:.1f}GB samples={len(samples)} age={age}"
|
||||
f" memory={memory_stage(sample, cfg)} ({memory_text(sample)})"
|
||||
)
|
||||
return 0
|
||||
|
||||
|
||||
@@ -537,10 +1188,14 @@ def main() -> int:
|
||||
cfg = load_config()
|
||||
if cfg is None:
|
||||
return 0
|
||||
problem = missing(cfg, MEMORY_KEYS)
|
||||
if args.cmd == "watch":
|
||||
return cmd_watch(cfg)
|
||||
return cmd_watch(cfg, problem)
|
||||
if args.cmd == "block":
|
||||
return cmd_block(cfg)
|
||||
return cmd_block(cfg, problem)
|
||||
if problem is not None:
|
||||
print(problem, file=sys.stderr)
|
||||
return 2
|
||||
if args.cmd == "admit":
|
||||
return cmd_admit(cfg, args)
|
||||
if args.cmd == "cleanup":
|
||||
|
||||
@@ -15,7 +15,7 @@ queue.py hold --kind <kind> --task <register task> -- <command>
|
||||
```
|
||||
|
||||
Start it in the background (`run_in_background` or equivalent) and end the
|
||||
turn: hold waits, runs the command on its grant and exits with its code.
|
||||
turn: hold runs the command on its grant and exits with its code.
|
||||
|
||||
- `<kind>`: a key of `slot_need_gb` in `~/.claude/agents-council/config.json`.
|
||||
- `<register task>`: your open `kvb-org-rasci` task; its rank places you.
|
||||
@@ -37,31 +37,32 @@ Every 20 s each waiting hold evaluates the whole queue under one lock:
|
||||
1. Hotfix jobs first, then the register rank (`rasci.py show`), one place
|
||||
earlier per whole hour waited, then arrival.
|
||||
2. In that order, a job is blocked at disk stage stop or critical, at warn
|
||||
when it needs disk, during a radio-silence mayday, while its kind's slots
|
||||
are taken, or while its workdir runs another job.
|
||||
3. A job whose need would cut below the stop threshold reserves that disk
|
||||
against smaller jobs behind it.
|
||||
when it needs disk, at memory stage stop, during a radio-silence mayday,
|
||||
while its kind's slots are taken, or while its workdir runs another job.
|
||||
3. A job whose disk need would cut below the stop threshold, or whose
|
||||
footprint would push the running ones past `budget_percent`, reserves it
|
||||
against the jobs behind it; a job alone always fits.
|
||||
4. An unblocked job counts as granted for the jobs behind it: none takes its
|
||||
slot.
|
||||
5. Kinds in `load_limits.kinds` also need CPU and RAM below the limits.
|
||||
|
||||
A waiting hold prints a stderr line when its position or reason changes.
|
||||
5. Every job, alone too, needs CPU, and RAM used plus what it and the
|
||||
running jobs may still take, below `load_limits`.
|
||||
|
||||
## History and learning
|
||||
|
||||
Every finished job appends a row to the council's `queue-history.jsonl`:
|
||||
signature (`make <target>`, `<script>.py <subcommand>`), kind, minutes, exit
|
||||
code, and disk, RAM and CPU at the grant and at their 20 s extremes. The
|
||||
`watch` hook of `kvb-org-organize` keeps them in
|
||||
`~/.local/state/agents-council/` across reboots.
|
||||
Every finished job appends a row (signature such as `<repo>: make <target>`,
|
||||
kind, minutes, disk, RAM and CPU) to `queue-history.jsonl`; the
|
||||
`kvb-org-organize` watch hook keeps it across reboots.
|
||||
|
||||
From the last 30 days, once 5 runs exist:
|
||||
|
||||
- per signature, the disk need is p90 disk taken plus 1 GB;
|
||||
- per kind, the slot limit is how many p90 RAM and CPU shares fit between the
|
||||
median load at the grant and `load_limits`, at least 1.
|
||||
median load at the grant and `load_limits`, at least 1;
|
||||
- per job, the footprint is the larger p90 share of its kind and of its
|
||||
signature, else the largest of its kind.
|
||||
|
||||
Until then the seeds hold: `slot_need_gb[kind]` for disk, 1 slot per kind.
|
||||
Until then the seeds hold: `slot_need_gb[kind]` for disk, 1 slot per kind,
|
||||
the whole budget as footprint: it runs alone.
|
||||
The optional `slot_limits {kind: N}` only caps a limit from above.
|
||||
|
||||
## Inspect and cancel
|
||||
@@ -72,11 +73,8 @@ queue.py history
|
||||
queue.py cancel <id>
|
||||
```
|
||||
|
||||
`list` shows each job's position, need, wait, ETA and blocking reason; dead
|
||||
holders drop out. The ETA adds the median minutes left of running jobs
|
||||
and of the jobs ahead, `-` while one is unknown. `history` prints the learned
|
||||
values per signature and per kind, with each limit's source. `cancel` removes
|
||||
your own waiting job; its hold exits 3.
|
||||
`list` shows each job's position, ETA and blocking reason, `history` the
|
||||
learned values, `cancel` removes your own waiting job (its hold exits 3).
|
||||
|
||||
## Rules
|
||||
|
||||
@@ -86,6 +84,7 @@ your own waiting job; its hold exits 3.
|
||||
- A running job is never cut short for the queue; only the operator orders
|
||||
one down, and `kvb-git-hotfix` may void a gate.
|
||||
- With `queue_enforce: true`, the `block` hook (PreToolUse on Bash) refuses
|
||||
`heavy_commands` and `gate.py run|hold` outside `queue.py hold`.
|
||||
`heavy_commands`, image builds, `make install|verify-*`, tests beyond one
|
||||
file and `gate.py run|hold` outside `queue.py hold`.
|
||||
- Without the config, hold exits 2: tell the operator and ask the chair
|
||||
(`kvb-org-organize`) until it exists.
|
||||
@@ -26,6 +26,7 @@ MIN_SHARE_PERCENT = 1
|
||||
SEED_LIMIT = 1
|
||||
INTERPRETERS = re.compile(r"python[\d.]*|bash|sh|zsh|node|perl|ruby")
|
||||
MAKE_VALUE_OPTIONS = frozenset({"-C", "-f", "-I", "-o", "-W"})
|
||||
RESOURCES = ("ram", "cpu")
|
||||
SIGNATURE_HEADER = ("Signature", "Kind", "Runs", "Median min", "p90 disk GB", "p90 RAM %", "Need GB")
|
||||
KIND_HEADER = ("Kind", "Runs", "p90 RAM share %", "p90 CPU share %", "Slot limit", "Source")
|
||||
|
||||
@@ -149,13 +150,14 @@ def grouped(rows: list[dict], key: str) -> dict[str, list[dict]]:
|
||||
def command_stats(rows: list[dict]) -> dict:
|
||||
stats = {
|
||||
"kinds": sorted({r["kind"] for r in rows}), "runs": len(rows),
|
||||
"minutes": None, "disk_gb": None, "ram_share": None, "need_gb": None,
|
||||
"minutes": None, "disk_gb": None, "ram_share": None, "cpu_share": None, "need_gb": None,
|
||||
}
|
||||
if len(rows) >= MIN_RUNS:
|
||||
disk = p90([r["disk_gb"] for r in rows])
|
||||
stats.update(
|
||||
minutes=statistics.median(r["minutes"] for r in rows), disk_gb=disk,
|
||||
ram_share=p90(shares(rows, "mem")), need_gb=math.ceil(disk + DISK_MARGIN_GB),
|
||||
ram_share=p90(shares(rows, "mem")), cpu_share=p90(shares(rows, "cpu")),
|
||||
need_gb=math.ceil(disk + DISK_MARGIN_GB),
|
||||
)
|
||||
return stats
|
||||
|
||||
@@ -186,7 +188,7 @@ def learn(rows: list[dict], now: float, cfg: dict) -> tuple[dict[str, dict], dic
|
||||
|
||||
Returns:
|
||||
(signatures, kinds). signatures: {signature: {kinds, runs, minutes,
|
||||
disk_gb, ram_share, need_gb}}; kinds: {kind: {runs, ram_share,
|
||||
disk_gb, ram_share, cpu_share, need_gb}}; kinds: {kind: {runs, ram_share,
|
||||
cpu_share, limit}}. With fewer than MIN_RUNS rows every value but kinds
|
||||
and runs is None. minutes is the median, disk_gb, ram_share and
|
||||
cpu_share the p90 of the disk taken and of max minus before in
|
||||
@@ -200,6 +202,41 @@ def learn(rows: list[dict], now: float, cfg: dict) -> tuple[dict[str, dict], dic
|
||||
return signatures, kinds
|
||||
|
||||
|
||||
def footprint(kind: str, name: str, signatures: dict[str, dict], kinds: dict[str, dict]) -> dict[str, float] | None:
|
||||
"""Predict the RAM and CPU share a job takes, in percentage points.
|
||||
|
||||
Args:
|
||||
kind: the job's kind.
|
||||
name: its command signature.
|
||||
signatures: the signature map of learn().
|
||||
kinds: the kind map of learn().
|
||||
|
||||
Returns:
|
||||
{ram, cpu}: the larger of the kind's p90 share and the signature's, at
|
||||
least MIN_SHARE_PERCENT; a signature not learned yet takes the largest
|
||||
learned share of its kind. None while the kind is not learned yet.
|
||||
"""
|
||||
if kind not in kinds or kinds[kind]["ram_share"] is None:
|
||||
return None
|
||||
own_kind = kinds[kind]
|
||||
if name in signatures and signatures[name]["ram_share"] is not None:
|
||||
peers = [signatures[name]]
|
||||
else:
|
||||
peers = [s for s in signatures.values() if kind in s["kinds"] and s["ram_share"] is not None]
|
||||
return {
|
||||
resource: max([MIN_SHARE_PERCENT, own_kind[f"{resource}_share"], *(peer[f"{resource}_share"] for peer in peers)])
|
||||
for resource in RESOURCES
|
||||
}
|
||||
|
||||
|
||||
def seed_ram(kinds: dict[str, dict]) -> float:
|
||||
"""Returns:
|
||||
The RAM share assumed for a kind not learned yet: the largest of the
|
||||
learned kinds' p90 RAM shares, 0 while none is learned.
|
||||
"""
|
||||
return max((stats["ram_share"] for stats in kinds.values() if stats["ram_share"] is not None), default=0.0)
|
||||
|
||||
|
||||
def need_gb(signatures: dict[str, dict], name: str) -> int | None:
|
||||
return signatures[name]["need_gb"] if name in signatures else None
|
||||
|
||||
|
||||
@@ -57,19 +57,27 @@ Args:
|
||||
Bash deny rule of the settings is refused, one meeting an ask rule
|
||||
asks the operator, with or without queue_enforce; a hold that cannot
|
||||
be parsed or checked is refused. With queue_enforce true in the config,
|
||||
refuse a command matching heavy_commands or `gate.py run|hold`
|
||||
unless it matches never_block or runs `queue.py hold`.
|
||||
refuse a command that watch.heavy() calls heavy or that runs
|
||||
`gate.py run|hold`, unless it matches never_block or runs `queue.py
|
||||
hold`.
|
||||
|
||||
Config keys: slot_limits {kind: N}, optional, caps the learned slot limit of a
|
||||
kind from above (history.slot_limits); queue_enforce (bool), optional, read by
|
||||
block only. The watch.py keys disk, thresholds_gb (stop is the floor every
|
||||
grant leaves free), slot_need_gb (the disk seed per kind), load_limits (kinds
|
||||
in its kinds also need CPU and RAM below the limits; mem_percent and
|
||||
cpu_percent bound the learned slot limits), heavy_commands and never_block are
|
||||
used as watch.py documents them.
|
||||
Config keys: budget_percent {ram, cpu}, required: the active and granted jobs'
|
||||
learned footprints (history.footprint, the whole budget for a kind not learned
|
||||
yet) may sum to at most these percentage points, a job alone always fits;
|
||||
slot_limits {kind: N}, optional, caps the learned slot limit of a kind from
|
||||
above (history.slot_limits); queue_enforce (bool), optional, read by block
|
||||
only. The watch.py keys disk, thresholds_gb (stop is the floor every grant
|
||||
leaves free), slot_need_gb (the disk seed per kind), load_limits (every grant
|
||||
also needs CPU busy below cpu_percent and RAM used plus the job's learned RAM
|
||||
share below mem_percent; both bound the learned slot limits), memory_limits (no
|
||||
grant at memory stage stop), heavy_commands and never_block are used as
|
||||
watch.py documents them. A signature is `<repository>: <command signature>`
|
||||
(qualified()).
|
||||
|
||||
Exit codes:
|
||||
hold: the command's exit code unchanged, 128+N when signal N ended it;
|
||||
a command exiting 75 (EX_TEMPFAIL: it did no work, such as a declined
|
||||
`operator.py granted`) leaves no history row;
|
||||
before the command started 2 on a usage or config error or when it
|
||||
cannot start, 3 when cancel removed the entry, 128+N when signal N
|
||||
reached hold.
|
||||
@@ -115,14 +123,16 @@ POLL_SECONDS = 20
|
||||
TICK_SECONDS = 0.2
|
||||
HOUR = 3600
|
||||
CANCELLED = 3
|
||||
FORWARDED = (signal.SIGINT, signal.SIGTERM)
|
||||
NOT_RUN = 75
|
||||
RAMP_SECONDS = 900
|
||||
SETUP =frozenset({"cd", "pushd", "set", "source", ".", "export", "umask", "shopt", "trap"})
|
||||
FORWARDED =(signal.SIGINT, signal.SIGTERM)
|
||||
GATE = re.compile(r"\bgate\.py\s+(run|hold)\b")
|
||||
THROUGH_QUEUE = re.compile(r"\bqueue\.py\s+hold\b")
|
||||
BASH_RULE = re.compile(r"Bash(?:\((.*)\))?", re.DOTALL)
|
||||
RULE_PART = re.compile(r"( \*|\*)")
|
||||
SEPARATOR = re.compile(r"[();<>|&]+")
|
||||
WORD_BREAK = " \t\n;&|()<>"
|
||||
MANAGED = Path("/etc/claude-code")
|
||||
BUDGET_KEYS = ("budget_percent.ram", "budget_percent.cpu")
|
||||
HOTFIX = "hotfix/"
|
||||
|
||||
|
||||
@@ -154,6 +164,9 @@ def config() -> dict:
|
||||
cfg = watch.load_config()
|
||||
if cfg is None:
|
||||
fail(f"no config at {watch.CONFIG}; the queue reads the kvb-org-organize watch config")
|
||||
problem = watch.missing(cfg, (*BUDGET_KEYS, *watch.MEMORY_KEYS))
|
||||
if problem is not None:
|
||||
fail(problem)
|
||||
return cfg
|
||||
|
||||
|
||||
@@ -169,8 +182,59 @@ def limits(cfg: dict, kinds: dict[str, dict], entries: list[dict]) -> dict[str,
|
||||
return history.slot_limits(kinds, caps(cfg), {*cfg["slot_need_gb"], *(e["kind"] for e in entries)})
|
||||
|
||||
|
||||
@functools.cache
|
||||
def repository(path: str) -> str:
|
||||
try:
|
||||
common = subprocess.run(
|
||||
["git", "rev-parse", "--path-format=absolute", "--git-common-dir"],
|
||||
cwd=path, capture_output=True, text=True, check=False,
|
||||
)
|
||||
except OSError:
|
||||
return os.path.basename(path)
|
||||
if common.returncode != 0:
|
||||
return os.path.basename(path)
|
||||
return os.path.basename(os.path.dirname(common.stdout.strip()))
|
||||
|
||||
|
||||
def command_words(argv: list[str]) -> list[str]:
|
||||
"""Returns:
|
||||
The command a wrapper runs: watch.program() unwrapped, the first simple
|
||||
command of `sh -c` code that is no setup such as cd, set or source,
|
||||
the tail after `--` of a Python script such as gate.py hold or
|
||||
operator.py granted.
|
||||
"""
|
||||
run = watch.program(argv) or argv
|
||||
code = watch.inline_code(run)
|
||||
inner = first_command(code) if code is not None else script_tail(run)
|
||||
return run if inner is None else command_words(inner)
|
||||
|
||||
|
||||
def first_command(code: str) -> list[str] | None:
|
||||
try:
|
||||
parts = [part for part in map(watch.program, watch.segments(code)) if part and os.path.basename(part[0]) not in SETUP]
|
||||
except ValueError:
|
||||
return None
|
||||
return parts[0] if parts else None
|
||||
|
||||
|
||||
def script_tail(run: list[str]) -> list[str] | None:
|
||||
if len(run) > 2 and watch.PYTHON.fullmatch(os.path.basename(run[0])) and run[1].endswith(".py") and "--" in run[2:-1]:
|
||||
return run[run.index("--", 2) + 1 :]
|
||||
return None
|
||||
|
||||
|
||||
def qualified(workdir: str, argv: list[str]) -> str:
|
||||
"""Returns:
|
||||
`<repository>: <history.signature of command_words>`: the directory
|
||||
holding the main checkout's .git, so every worktree of one repository
|
||||
shares it, else the workdir's name; one make test of two repositories
|
||||
stays apart.
|
||||
"""
|
||||
return f"{repository(workdir)}: {history.signature(command_words(argv))}"
|
||||
|
||||
|
||||
def signature_of(entry: dict) -> str:
|
||||
return history.signature(shlex.split(entry["cmd"]))
|
||||
return qualified(entry["workdir"], shlex.split(entry["cmd"]))
|
||||
|
||||
|
||||
def identity(pid: int) -> str | None:
|
||||
@@ -270,18 +334,33 @@ def ordered(waiting: list[dict], ranks: dict[str, int], agents: dict[str, int],
|
||||
return sorted(waiting, key=key)
|
||||
|
||||
|
||||
def evaluate(entries: list[dict], free_gb: float, floor: float, stage: str, silenced: bool,
|
||||
slot_limits: dict[str, int], ranks: dict[str, int], agents: dict[str, int],
|
||||
now: float) -> dict[str, str | None]:
|
||||
def over_budget(used: dict[str, float], share: dict[str, float], budget: dict[str, float]) -> str | None:
|
||||
over = [resource for resource in budget if used[resource] + share[resource] > budget[resource]]
|
||||
if not over:
|
||||
return None
|
||||
return ", ".join(
|
||||
f"{resource.upper()} {used[resource]:.0f} % in use or reserved + {share[resource]:.0f} % > budget {budget[resource]:g} %"
|
||||
for resource in over
|
||||
) + "; reserved"
|
||||
|
||||
|
||||
def evaluate(entries: list[dict], free_gb: float, floor: float, stage: str, memory: str, silenced: bool,
|
||||
slot_limits: dict[str, int], footprints: dict[str, dict[str, float]], budget: dict[str, float],
|
||||
ranks: dict[str, int], agents: dict[str, int], now: float) -> dict[str, str | None]:
|
||||
"""Decide every waiting entry in grant order, without disk or processes.
|
||||
|
||||
Args:
|
||||
entries: the queue's entries, active and waiting.
|
||||
free_gb: free disk in GB.
|
||||
floor: GB every grant must leave free (thresholds_gb.stop).
|
||||
stage: the watch stage: ok, warn, stop or critical.
|
||||
stage: the watch disk stage: ok, warn, stop or critical.
|
||||
memory: the watch memory stage: ok, warn or stop.
|
||||
silenced: whether a radio-silence mayday holds.
|
||||
slot_limits: {kind: max concurrent jobs}.
|
||||
slot_limits: {kind: max concurrent jobs}, an upper cap beside the budget.
|
||||
footprints: {entry id: {ram, cpu}}, the learned share of every entry
|
||||
(history.footprint), in percentage points.
|
||||
budget: {ram, cpu}: the sum of the footprints of active and granted
|
||||
entries may not pass it; a job alone always fits.
|
||||
ranks: {task: index in the register's ranking}; a task missing there
|
||||
ranks after all of them.
|
||||
agents: {agent name: index in the operator's agent order}; it breaks
|
||||
@@ -292,39 +371,69 @@ def evaluate(entries: list[dict], free_gb: float, floor: float, stage: str, sile
|
||||
Returns:
|
||||
{entry id: None when granted, else the blocking reason}, in grant order.
|
||||
An unblocked entry counts as granted for the entries after it, whoever
|
||||
holds it, so a lower entry never takes a higher entry's slot.
|
||||
holds it, so a lower entry never takes a higher entry's slot; one
|
||||
blocked by disk or budget reserves its need against the entries after it.
|
||||
"""
|
||||
active = [e for e in entries if e["status"] == "active"]
|
||||
available = free_gb - sum(e["need_gb"] for e in active)
|
||||
used = {resource: sum(footprints[e["id"]][resource] for e in active) for resource in budget}
|
||||
taken = len(active)
|
||||
counts = Counter(e["kind"] for e in active)
|
||||
busy = {e["workdir"] for e in active}
|
||||
verdicts: dict[str, str | None] = {}
|
||||
for entry in ordered([e for e in entries if e["status"] == "waiting"], ranks, agents, now):
|
||||
kind, need, workdir = entry["kind"], entry["need_gb"], entry["workdir"]
|
||||
reason = None
|
||||
if stage in ("stop", "critical"):
|
||||
reason = f"disk stage {stage}"
|
||||
elif stage == "warn" and need > 0:
|
||||
reason = "disk stage warn admits only jobs that need no disk"
|
||||
elif silenced:
|
||||
reason = "radio silence"
|
||||
elif kind not in slot_limits:
|
||||
reason = f"slot_limits has no {kind}"
|
||||
elif counts[kind] >= slot_limits[kind]:
|
||||
reason = f"all {slot_limits[kind]} {kind} slots taken"
|
||||
elif workdir in busy:
|
||||
reason = f"{workdir} runs another job"
|
||||
elif available - need < floor:
|
||||
reason = f"{need:g} GB would cut into the {floor:g} GB floor; reserved"
|
||||
available -= need
|
||||
else:
|
||||
counts[kind] += 1
|
||||
busy.add(workdir)
|
||||
available -= need
|
||||
kind, need, workdir, share = entry["kind"], entry["need_gb"], entry["workdir"], footprints[entry["id"]]
|
||||
reason = host_reason(stage, memory, silenced, need) or slot_reason(kind, workdir, slot_limits, counts, busy)
|
||||
if reason is None:
|
||||
reason, used, available = resources(need, share, used, available, floor, budget, taken)
|
||||
if reason is None:
|
||||
taken += 1
|
||||
counts[kind] += 1
|
||||
busy.add(workdir)
|
||||
verdicts[entry["id"]] = reason
|
||||
return verdicts
|
||||
|
||||
|
||||
def plus(used: dict[str, float], share: dict[str, float]) -> dict[str, float]:
|
||||
return {resource: used[resource] + share[resource] for resource in used}
|
||||
|
||||
|
||||
def resources(need: float, share: dict[str, float], used: dict[str, float], available: float, floor: float,
|
||||
budget: dict[str, float], taken: int) -> tuple[str | None, dict[str, float], float]:
|
||||
"""Returns:
|
||||
(reason or None, RAM and CPU used or reserved, disk available) after
|
||||
the job: one passing the budget beside other jobs reserves its share,
|
||||
one cutting into the floor its disk, a granted one takes both.
|
||||
"""
|
||||
if taken and (reason := over_budget(used, share, budget)) is not None:
|
||||
return reason, plus(used, share), available
|
||||
if available - need < floor:
|
||||
return f"{need:g} GB would cut into the {floor:g} GB floor; reserved", used, available - need
|
||||
return None, plus(used, share), available - need
|
||||
|
||||
|
||||
def host_reason(stage: str, memory: str, silenced: bool, need: float) -> str | None:
|
||||
if stage in ("stop", "critical"):
|
||||
return f"disk stage {stage}"
|
||||
if stage == "warn" and need > 0:
|
||||
return "disk stage warn admits only jobs that need no disk"
|
||||
if memory == "stop":
|
||||
return "memory stage stop"
|
||||
if silenced:
|
||||
return "radio silence"
|
||||
return None
|
||||
|
||||
|
||||
def slot_reason(kind: str, workdir: str, slot_limits: dict[str, int], counts: Counter, busy: set[str]) -> str | None:
|
||||
if kind not in slot_limits:
|
||||
return f"slot_limits has no {kind}"
|
||||
if counts[kind] >= slot_limits[kind]:
|
||||
return f"all {slot_limits[kind]} {kind} slots taken"
|
||||
if workdir in busy:
|
||||
return f"{workdir} runs another job"
|
||||
return None
|
||||
|
||||
|
||||
def register_ranks() -> dict[str, int]:
|
||||
rasci = sibling("rasci")
|
||||
return {task: index for index, task in enumerate(rasci.ranked(rasci.merged(False)))}
|
||||
@@ -344,22 +453,62 @@ def silenced(now: float) -> bool:
|
||||
return "until" in state and state["until"] > now
|
||||
|
||||
|
||||
def decide(cfg: dict, entries: list[dict], free_gb: float, now: float, kinds: dict[str, dict]) -> dict[str, str | None]:
|
||||
def footprint(entry: dict, signatures: dict[str, dict], kinds: dict[str, dict]) -> dict[str, float] | None:
|
||||
return history.footprint(entry["kind"], signature_of(entry), signatures, kinds)
|
||||
|
||||
|
||||
def decide(cfg: dict, entries: list[dict], free_gb: float, now: float, signatures: dict[str, dict],
|
||||
kinds: dict[str, dict]) -> dict[str, str | None]:
|
||||
slots = {kind: limit for kind, (limit, _) in limits(cfg, kinds, entries).items()}
|
||||
budget = cfg["budget_percent"]
|
||||
footprints = {}
|
||||
for entry in entries:
|
||||
share = footprint(entry, signatures, kinds)
|
||||
footprints[entry["id"]] = dict(budget) if share is None else share
|
||||
return evaluate(
|
||||
entries, free_gb, cfg["thresholds_gb"]["stop"], stage(cfg, free_gb, now), silenced(now),
|
||||
slots, register_ranks(), agent_ranks(), now,
|
||||
entries, free_gb, cfg["thresholds_gb"]["stop"], stage(cfg, free_gb, now),
|
||||
watch.memory_stage(watch.memory(), cfg), silenced(now), slots, footprints, budget,
|
||||
register_ranks(), agent_ranks(), now,
|
||||
)
|
||||
|
||||
|
||||
def load_reason(cfg: dict, kind: str) -> str | None:
|
||||
limits = cfg["load_limits"]
|
||||
if kind not in limits["kinds"]:
|
||||
return None
|
||||
cpu, mem = watch.load(cfg)
|
||||
if cpu < limits["cpu_percent"] and mem < limits["mem_percent"]:
|
||||
return None
|
||||
return f"cpu or ram at the limit ({limits['cpu_percent']} %, {limits['mem_percent']} %)"
|
||||
def load_reason(limits: dict, cpu: float, mem: float, ram_share: float) -> str | None:
|
||||
"""Args:
|
||||
limits: load_limits of the config.
|
||||
cpu: CPU busy now, in percent.
|
||||
mem: RAM used now, in percent; stacks and jobs outside the queue
|
||||
included.
|
||||
ram_share: what the job and the active ones may still take, in
|
||||
percentage points (pending_ram()).
|
||||
|
||||
Returns:
|
||||
None when CPU stays below cpu_percent and RAM used plus the share
|
||||
below mem_percent, else why not.
|
||||
"""
|
||||
if cpu >= limits["cpu_percent"]:
|
||||
return f"CPU {cpu:.0f} % busy reaches the limit {limits['cpu_percent']} %"
|
||||
if mem + ram_share >= limits["mem_percent"]:
|
||||
return f"RAM {mem:.0f} % used + {ram_share:.0f} % still to come reach the limit {limits['mem_percent']} %"
|
||||
return None
|
||||
|
||||
|
||||
def ram_share(entry: dict, signatures: dict[str, dict], kinds: dict[str, dict]) -> float:
|
||||
share = footprint(entry, signatures, kinds)
|
||||
return history.seed_ram(kinds) if share is None else share["ram"]
|
||||
|
||||
|
||||
def pending_ram(entries: list[dict], entry_id: str, now: float, signatures: dict[str, dict],
|
||||
kinds: dict[str, dict]) -> float:
|
||||
"""Returns:
|
||||
The RAM share of entry_id plus the whole share of every active job
|
||||
granted less than RAMP_SECONDS ago (or without a grant time): what
|
||||
may still come once they ramp up. An older job counts as ramped, its
|
||||
RAM inside the RAM used; one that peaks later is bounded by the
|
||||
memory stage only.
|
||||
"""
|
||||
mine = [e for e in entries if e["id"] == entry_id]
|
||||
young = [e for e in entries if e["status"] == "active" and now - e.get("granted", now) < RAMP_SECONDS]
|
||||
return ram_share(mine[0], signatures, kinds) + sum(ram_share(e, signatures, kinds) for e in young)
|
||||
|
||||
|
||||
def workdir() -> str:
|
||||
@@ -399,13 +548,12 @@ def pause(caught: list[int]) -> None:
|
||||
time.sleep(TICK_SECONDS)
|
||||
|
||||
|
||||
def wait(cfg: dict, entry_id: str, kind: str, caught: list[int]) -> tuple[int | None, int]:
|
||||
def wait(cfg: dict, entry_id: str, caught: list[int]) -> tuple[int | None, int]:
|
||||
"""Poll until the grant.
|
||||
|
||||
Args:
|
||||
cfg: the config.
|
||||
entry_id: this hold's entry.
|
||||
kind: its kind, for the load sample.
|
||||
caught: signals the caller caught; the first ends the wait.
|
||||
|
||||
Returns:
|
||||
@@ -414,18 +562,19 @@ def wait(cfg: dict, entry_id: str, kind: str, caught: list[int]) -> tuple[int |
|
||||
"""
|
||||
shown = None
|
||||
while not caught:
|
||||
busy_load = load_reason(cfg, kind)
|
||||
cpu, mem = watch.load(cfg)
|
||||
free, _ = watch.disk(cfg)
|
||||
now = time.time()
|
||||
_, kinds = learned(cfg, now)
|
||||
signatures, kinds = learned(cfg, now)
|
||||
with store() as data:
|
||||
data["entries"] = prune(data["entries"])
|
||||
mine = [e for e in data["entries"] if e["id"] == entry_id]
|
||||
if not mine:
|
||||
print(f"queue: {entry_id} was cancelled", file=sys.stderr)
|
||||
return CANCELLED, 0
|
||||
verdicts = decide(cfg, data["entries"], free, now, kinds)
|
||||
reason = verdicts[entry_id] or busy_load
|
||||
verdicts = decide(cfg, data["entries"], free, now, signatures, kinds)
|
||||
pending = pending_ram(data["entries"], entry_id, now, signatures, kinds)
|
||||
reason = verdicts[entry_id] or load_reason(cfg["load_limits"], cpu, mem, pending)
|
||||
if reason is None:
|
||||
concurrent = sum(e["status"] == "active" for e in data["entries"])
|
||||
mine[0].update(status="active", granted=now)
|
||||
@@ -477,6 +626,11 @@ def keep(row: dict) -> None:
|
||||
print(f"queue: no history row for {row['id']}: {error.strerror}", file=sys.stderr)
|
||||
|
||||
|
||||
def record(job: dict, start: float, code: int, samples: list[dict[str, float]], concurrent: int) -> None:
|
||||
if code != NOT_RUN:
|
||||
keep(history.row(job, start, time.time(), code, samples, concurrent))
|
||||
|
||||
|
||||
def cmd_hold(args: argparse.Namespace) -> int:
|
||||
cfg = config()
|
||||
if args.kind not in cfg["slot_need_gb"]:
|
||||
@@ -486,7 +640,8 @@ def cmd_hold(args: argparse.Namespace) -> int:
|
||||
fail(f"{args.task} is no open task of the kvb-org-rasci register")
|
||||
if args.hotfix is not None and HOTFIX + args.hotfix not in rasci.load():
|
||||
fail(f"the register has no row {HOTFIX}{args.hotfix}")
|
||||
signature = history.signature(args.command)
|
||||
here = workdir()
|
||||
signature = qualified(here, args.command)
|
||||
signatures, _ = learned(cfg, time.time())
|
||||
need = args.need_gb
|
||||
if need is None:
|
||||
@@ -511,13 +666,13 @@ def cmd_hold(args: argparse.Namespace) -> int:
|
||||
with holding() as (tag, lock_fd):
|
||||
entry = enqueue({
|
||||
"kind": args.kind, "task": args.task, "need_gb": need, "hotfix": args.hotfix,
|
||||
"workdir": workdir(), "session": session, "name": council.live_sessions().get(session) or session,
|
||||
"workdir": here, "session": session, "name": council.live_sessions().get(session) or session,
|
||||
"lock": tag, "pid": os.getpid(), "pid_start": identity(os.getpid()), "child": None,
|
||||
"child_start": None, "status": "waiting", "enqueued": time.time(), "granted": None,
|
||||
"cmd": shlex.join(args.command),
|
||||
})
|
||||
try:
|
||||
code, concurrent = wait(cfg, entry["id"], args.kind, caught)
|
||||
code, concurrent = wait(cfg, entry["id"], caught)
|
||||
if code is not None:
|
||||
return code
|
||||
if caught:
|
||||
@@ -536,7 +691,7 @@ def cmd_hold(args: argparse.Namespace) -> int:
|
||||
code = code if code >= 0 else 128 - code
|
||||
job = {"id": f"{entry['id']}-{time.time_ns()}", "kind": args.kind, "signature": signature,
|
||||
"workdir": entry["workdir"]}
|
||||
keep(history.row(job, start, time.time(), code, samples, concurrent))
|
||||
record(job, start, code, samples, concurrent)
|
||||
return code
|
||||
finally:
|
||||
leave(entry["id"])
|
||||
@@ -550,7 +705,7 @@ def cmd_list(_: argparse.Namespace) -> int:
|
||||
with store() as data:
|
||||
data["entries"] = prune(data["entries"])
|
||||
entries = list(data["entries"])
|
||||
verdicts = decide(cfg, entries, free, now, kinds)
|
||||
verdicts = decide(cfg, entries, free, now, signatures, kinds)
|
||||
by_id = {e["id"]: e for e in entries}
|
||||
running = [(signature_of(e), e["granted"]) for e in entries if e["status"] == "active"]
|
||||
rows = [
|
||||
@@ -565,13 +720,17 @@ def cmd_list(_: argparse.Namespace) -> int:
|
||||
print("position\tid\tstatus\tkind\ttask\tneed_gb\twaited_min\teta_min\tworkdir\treason")
|
||||
for position, e, eta, reason in rows:
|
||||
waited = ((e["granted"] or now) - e["enqueued"]) / 60
|
||||
print("\t".join(
|
||||
print(tab_line(
|
||||
[position, e["id"], e["status"], e["kind"], e["task"], f"{e['need_gb']:g}", f"{waited:.0f}",
|
||||
"-" if eta is None else f"{eta:.0f}", e["workdir"], reason]
|
||||
))
|
||||
return 0
|
||||
|
||||
|
||||
def tab_line(fields: list[object]) -> str:
|
||||
return "\t".join("".join(c if c.isprintable() else c.encode("unicode_escape").decode() for c in str(f)) for f in fields)
|
||||
|
||||
|
||||
def cmd_history(_: argparse.Namespace) -> int:
|
||||
cfg = config()
|
||||
signatures, kinds = learned(cfg, time.time())
|
||||
@@ -633,50 +792,10 @@ def permission(commands: list[str], project: Path) -> str | None:
|
||||
return None
|
||||
|
||||
|
||||
def uncommented(command: str) -> str:
|
||||
kept: list[str] = []
|
||||
quote = ""
|
||||
escaped = skipping = False
|
||||
word_start = True
|
||||
for char in command:
|
||||
if skipping and char != "\n":
|
||||
continue
|
||||
skipping = False
|
||||
if escaped:
|
||||
escaped = False
|
||||
elif quote == "'":
|
||||
quote = "" if char == "'" else quote
|
||||
elif char == "\\":
|
||||
escaped = True
|
||||
elif quote == '"':
|
||||
quote = "" if char == '"' else quote
|
||||
elif char in "'\"":
|
||||
quote = char
|
||||
elif char == "#" and word_start:
|
||||
skipping = True
|
||||
continue
|
||||
kept.append(char)
|
||||
word_start = not quote and not escaped and char in WORD_BREAK
|
||||
return "".join(kept)
|
||||
|
||||
|
||||
def segments(command: str) -> list[list[str]]:
|
||||
lexer = shlex.shlex(uncommented(command.replace("\\\n", "")), posix=True, punctuation_chars=True)
|
||||
lexer.whitespace_split = True
|
||||
lexer.commenters = ""
|
||||
parts: list[list[str]] = [[]]
|
||||
for token in lexer:
|
||||
if SEPARATOR.fullmatch(token):
|
||||
parts.append([])
|
||||
else:
|
||||
parts[-1].append(token)
|
||||
return parts
|
||||
|
||||
|
||||
def held_commands(command: str) -> list[list[str]]:
|
||||
return [
|
||||
cli().parse_args(part[at + 1 :]).command
|
||||
for part in segments(command)
|
||||
for part in watch.segments(command)
|
||||
for at, token in enumerate(part)
|
||||
if token.endswith("queue.py") and part[at + 1 : at + 2] == ["hold"]
|
||||
]
|
||||
@@ -715,10 +834,8 @@ def cmd_block() -> int:
|
||||
cfg = watch.load_config()
|
||||
if cfg is None or cfg.get("queue_enforce") is not True:
|
||||
return 0
|
||||
command = watch.executable_text((payload.get("tool_input") or {}).get("command") or "")
|
||||
if THROUGH_QUEUE.search(command) or any(re.search(p, command) for p in cfg["never_block"]):
|
||||
return 0
|
||||
if GATE.search(command) or any(re.search(p, command) for p in cfg["heavy_commands"]):
|
||||
gated = {**cfg, "heavy_commands": [*cfg["heavy_commands"], GATE.pattern]}
|
||||
if watch.heavy(raw, gated, (THROUGH_QUEUE.pattern, *cfg["never_block"])):
|
||||
print(
|
||||
f"kvb-org-queue: heavy jobs run through the queue: start `python3 {Path(__file__).resolve()} hold "
|
||||
"--kind <kind> --task <register task> -- <command>` in the background; it runs the command on its grant.",
|
||||
|
||||
@@ -249,6 +249,90 @@ class TestOperator(unittest.TestCase):
|
||||
self.assertNotIn("decide", result.stdout)
|
||||
self.assertEqual(self._open(), ["operator/ask"])
|
||||
|
||||
def _queue_config(self, slot_need_gb: dict) -> None:
|
||||
config = self.base / "config.json"
|
||||
open_memory = {"available": -1, "swap_free": -1, "psi_full": 1000, "container_swap_gb": 1e9}
|
||||
config.write_text(json.dumps({
|
||||
"disk": str(self.base), "thresholds_gb": {"warn": 0, "stop": 0, "critical": 0}, "projection_minutes": 30,
|
||||
"slot_need_gb": slot_need_gb, "heavy_commands": [r"\btouch heavy\b"], "never_block": [],
|
||||
"load_limits": {"cpu_percent": 101, "mem_percent": 101, "sample_seconds": 0.1},
|
||||
"budget_percent": {"ram": 75, "cpu": 100}, "memory_limits": {"warn": open_memory, "stop": open_memory},
|
||||
}), encoding="utf-8")
|
||||
self.env["AGENTS_COUNCIL_CONFIG"] = str(config)
|
||||
|
||||
def test_run_offers_the_queue_for_a_heavy_command_and_asks_again_on_the_grant(self):
|
||||
self._queue_config({"operator": 0})
|
||||
work = self.base / "work"
|
||||
work.mkdir()
|
||||
self._add("light", work, "touch light", "plain", "manual")
|
||||
self._add("heavy", work, "touch heavy\x1b[2K", "build", "manual")
|
||||
result = self._feed("y\nh\n")
|
||||
self.assertEqual(result.returncode, 0, result.stderr)
|
||||
light, heavy = result.stdout.split("[2/2]")
|
||||
self.assertIn("[s]kip", light)
|
||||
self.assertNotIn("heavy:", light)
|
||||
self.assertIn("heavy: memory stage ok", heavy)
|
||||
self.assertIn("position\tid\tstatus", heavy)
|
||||
self.assertIn("[h]old in the queue", heavy)
|
||||
self.assertNotIn("\x1b", heavy)
|
||||
self.assertIn("granted operator/heavy: the queue lets it run now", heavy)
|
||||
self.assertIn("not run operator/heavy", heavy)
|
||||
self.assertIn("failed operator/heavy: exit 75; the row stays open", heavy)
|
||||
self.assertTrue((work / "light").exists())
|
||||
self.assertFalse((work / "heavy\x1b[2K").exists())
|
||||
self.assertEqual(self._open(), ["operator/heavy"])
|
||||
self.assertFalse((self.root / "queue-history.jsonl").exists())
|
||||
|
||||
def test_run_offers_no_hold_while_the_config_lacks_the_operator_kind(self):
|
||||
self._queue_config({"test": 0})
|
||||
work = self.base / "work"
|
||||
work.mkdir()
|
||||
self._add("heavy", work, "touch heavy", "build", "manual")
|
||||
result = self._feed("h\ns\n")
|
||||
self.assertEqual(result.returncode, 0, result.stderr)
|
||||
self.assertIn("lacks slot_need_gb.operator;", result.stdout)
|
||||
self.assertIn("no [h]old until it does", result.stdout)
|
||||
self.assertNotIn("[h]old in the queue", result.stdout)
|
||||
self.assertIn("skipped operator/heavy", result.stdout)
|
||||
|
||||
def test_run_offers_no_hold_for_sudo_whose_prompt_needs_the_terminal(self):
|
||||
self._queue_config({"operator": 0})
|
||||
work = self.base / "work"
|
||||
work.mkdir()
|
||||
self._add("heavy", work, "bash -c 'sudo -n touch heavy'", "build", "manual")
|
||||
result = self._feed("h\ns\n")
|
||||
self.assertEqual(result.returncode, 0, result.stderr)
|
||||
self.assertIn("sudo asks on the terminal, which a queued command lacks", result.stdout)
|
||||
self.assertNotIn("[h]old in the queue", result.stdout)
|
||||
|
||||
def test_granted_runs_only_the_approved_row_in_its_directory_after_a_new_yes(self):
|
||||
work = self.base / "work"
|
||||
work.mkdir()
|
||||
self._add("heavy", work, "touch heavy", "build", "manual")
|
||||
env = {**self.env, "SHELL": "/bin/sh"}
|
||||
|
||||
def granted(answers: str, directory: Path, *argv: str) -> subprocess.CompletedProcess:
|
||||
return subprocess.run(
|
||||
[sys.executable, str(OPERATOR), "granted", "operator/heavy", "--dir", str(directory), "--check",
|
||||
"manual", "--", *argv],
|
||||
input=answers, capture_output=True, text=True, env=env, cwd=self.base, check=False,
|
||||
)
|
||||
|
||||
changed = granted("y\n", work, "/bin/sh", "-ic", "touch other")
|
||||
self.assertEqual(changed.returncode, 75)
|
||||
self.assertIn("no longer open with the approved directory, check and command", changed.stderr)
|
||||
moved = granted("y\n", self.base, "/bin/sh", "-ic", "touch heavy")
|
||||
self.assertEqual((moved.returncode, (self.base / "heavy").exists()), (75, False))
|
||||
declined = granted("s\n", work, "/bin/sh", "-ic", "touch heavy")
|
||||
self.assertEqual((declined.returncode, (work / "heavy").exists()), (75, False))
|
||||
silent = granted("", work, "/bin/sh", "-ic", "touch heavy")
|
||||
self.assertEqual((silent.returncode, (work / "heavy").exists()), (75, False))
|
||||
ran = granted("d\ny\n", work, "/bin/sh", "-ic", "touch heavy")
|
||||
self.assertEqual(ran.returncode, 0, ran.stderr)
|
||||
self.assertIn("granted operator/heavy", ran.stdout)
|
||||
self.assertTrue((work / "heavy").exists())
|
||||
self.assertFalse((self.base / "heavy").exists() or (work / "other").exists())
|
||||
|
||||
def test_run_needs_shell(self):
|
||||
self._add("x", self.base, "true", "r", "manual")
|
||||
result = subprocess.run(
|
||||
|
||||
+294
-6
@@ -20,12 +20,15 @@ QUEUE = REPO_ROOT / "skills" / "kvb-org-queue" / "scripts" / "queue.py"
|
||||
HISTORY = REPO_ROOT / "skills" / "kvb-org-queue" / "scripts" / "history.py"
|
||||
NOW = 1_800_000_000.0
|
||||
HOUR = 3600
|
||||
OPEN_MEMORY = {"available": -1, "swap_free": -1, "psi_full": 1000, "container_swap_gb": 1e9}
|
||||
BASE = {
|
||||
"free_gb": 100.0,
|
||||
"floor": 10.0,
|
||||
"stage": "ok",
|
||||
"memory": "ok",
|
||||
"silenced": False,
|
||||
"slot_limits": {"test": 5, "deploy": 1},
|
||||
"budget": {"ram": 75, "cpu": 100},
|
||||
"ranks": {"work/a": 0, "work/b": 1, "work/c": 2},
|
||||
"agents": {},
|
||||
"now": NOW,
|
||||
@@ -83,7 +86,97 @@ class TestEvaluate(unittest.TestCase):
|
||||
cls.queue = _load(QUEUE)
|
||||
|
||||
def decide(self, entries: list[dict], **overrides: object) -> dict:
|
||||
return self.queue.evaluate(entries, **{**BASE, **overrides})
|
||||
footprints = {e["id"]: {"ram": 0, "cpu": 0} for e in entries} | overrides.pop("footprints", {})
|
||||
return self.queue.evaluate(entries, **{**BASE, "footprints": footprints, **overrides})
|
||||
|
||||
def test_the_budget_refuses_a_job_whose_share_would_pass_it_and_reserves_it(self):
|
||||
running = job(1, kind="test", status="active")
|
||||
candidate = job(2, kind="gui-dind", workdir="/x")
|
||||
small = job(3, task="work/b", workdir="/y")
|
||||
shares = {"j1": {"ram": 60, "cpu": 40}, "j2": {"ram": 30, "cpu": 20}, "j3": {"ram": 10, "cpu": 5}}
|
||||
verdicts = self.decide([running, candidate, small], slot_limits={"test": 5, "gui-dind": 1}, footprints=shares)
|
||||
self.assertEqual(verdicts["j2"], "RAM 60 % in use or reserved + 30 % > budget 75 %; reserved")
|
||||
self.assertEqual(verdicts["j3"], "RAM 90 % in use or reserved + 10 % > budget 75 %; reserved")
|
||||
unbounded = {"ram": 1000, "cpu": 1000}
|
||||
self.assertEqual(
|
||||
self.decide([running, candidate, small], slot_limits={"test": 5, "gui-dind": 1}, footprints=shares, budget=unbounded),
|
||||
{"j2": None, "j3": None},
|
||||
)
|
||||
|
||||
def test_a_job_alone_always_fits_and_cpu_counts_like_ram(self):
|
||||
alone = self.decide([job(1)], footprints={"j1": {"ram": 90, "cpu": 150}})
|
||||
self.assertEqual(alone, {"j1": None})
|
||||
busy = self.decide([job(1, status="active"), job(2)], footprints={"j1": {"ram": 1, "cpu": 70}, "j2": {"ram": 1, "cpu": 40}})
|
||||
self.assertEqual(busy, {"j2": "CPU 70 % in use or reserved + 40 % > budget 100 %; reserved"})
|
||||
|
||||
def test_a_share_beyond_the_budget_runs_only_alone_even_beside_a_job_of_no_share(self):
|
||||
whole = {"ram": 80, "cpu": 1}
|
||||
beside = self.decide([job(1, status="active"), job(2)], footprints={"j1": {"ram": 0, "cpu": 0}, "j2": whole})
|
||||
self.assertEqual(beside, {"j2": "RAM 0 % in use or reserved + 80 % > budget 75 %; reserved"})
|
||||
self.assertEqual(self.decide([job(2)], footprints={"j2": whole}), {"j2": None})
|
||||
|
||||
def test_load_counts_the_ram_in_use_plus_what_is_still_to_come(self):
|
||||
limits = {"cpu_percent": 90, "mem_percent": 90}
|
||||
reason = self.queue.load_reason
|
||||
self.assertIsNone(reason(limits, 50, 60, 29))
|
||||
self.assertEqual(reason(limits, 50, 60, 30), "RAM 60 % used + 30 % still to come reach the limit 90 %")
|
||||
self.assertIsNone(reason(limits, 50, 89, 0))
|
||||
self.assertEqual(reason(limits, 90, 10, 0), "CPU 90 % busy reaches the limit 90 %")
|
||||
|
||||
def test_pending_ram_adds_what_active_jobs_have_not_taken_yet_and_seeds_an_unlearned_kind(self):
|
||||
history = _load(HISTORY)
|
||||
rows = [history_row(n, kind=kind, signature=f"w: {kind}", mem_before=20, mem_max=20 + share)
|
||||
for n, (kind, share) in enumerate([("a", 30)] * 5 + [("b", 25)] * 5 + [("c", 10)] * 5)]
|
||||
signatures, kinds = history.learn(rows, NOW, {"load_limits": {"mem_percent": 90, "cpu_percent": 90}})
|
||||
self.assertEqual(history.seed_ram(kinds), 30)
|
||||
self.assertEqual(history.seed_ram({}), 0)
|
||||
cmd = {"workdir": "/w"}
|
||||
fresh = {**job(2, kind="b"), **cmd, "cmd": "b"}
|
||||
young = {**job(1, kind="a", status="active", granted=NOW - 60), **cmd, "cmd": "a"}
|
||||
overshooting = {**job(5, kind="c", status="active", granted=NOW - 120), **cmd, "cmd": "c"}
|
||||
old = {**job(4, kind="c", status="active", granted=NOW - self.queue.RAMP_SECONDS), **cmd, "cmd": "c"}
|
||||
self.assertEqual(self.queue.pending_ram([young, overshooting, old, fresh], "j2", NOW, signatures, kinds), 30 + 10 + 25)
|
||||
undated = {k: v for k, v in old.items() if k != "granted"}
|
||||
self.assertEqual(self.queue.pending_ram([undated, fresh], "j2", NOW, signatures, kinds), 10 + 25)
|
||||
unknown = {**job(3, kind="new"), **cmd, "cmd": "new"}
|
||||
self.assertEqual(self.queue.pending_ram([unknown], "j3", NOW, signatures, kinds), 30)
|
||||
|
||||
def test_a_signature_names_the_repository_every_worktree_shares(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
base = Path(tmp).resolve()
|
||||
main = base / "skills"
|
||||
main.mkdir()
|
||||
identity = {**os.environ, "GIT_AUTHOR_NAME": "T", "GIT_AUTHOR_EMAIL": "t@x", "GIT_COMMITTER_NAME": "T",
|
||||
"GIT_COMMITTER_EMAIL": "t@x", "GIT_CONFIG_NOSYSTEM": "1"}
|
||||
for args in (["init", "-q"], ["commit", "-q", "--allow-empty", "-m", "x"], ["worktree", "add", "-q", str(base / "wt")]):
|
||||
subprocess.run(["git", *args], cwd=main, env=identity, check=True, capture_output=True)
|
||||
self.assertEqual(self.queue.qualified(str(main), ["make", "test"]), "skills: make test")
|
||||
self.assertEqual(self.queue.qualified(str(base / "wt"), ["make", "test"]), "skills: make test")
|
||||
plain = base / "plain"
|
||||
plain.mkdir()
|
||||
self.assertEqual(self.queue.qualified(str(plain), ["true"]), "plain: true")
|
||||
self.assertEqual(self.queue.qualified(str(base / "gone"), ["true"]), "gone: true")
|
||||
for argv, expected in {
|
||||
("env", "TMPDIR=/t", "-i", "make", "test"): "make test",
|
||||
("nice", "-n", "10", "make", "compose-deploy"): "make compose-deploy",
|
||||
("timeout", "3600", "make", "lint"): "make lint",
|
||||
("python3", "/s/gate.py", "hold", "--repo", ".", "--", "make", "verify-test"): "make verify-test",
|
||||
("python3", "/s/operator.py", "granted", "operator/x", "--dir", "/a", "--check", "manual", "--",
|
||||
"zsh", "-ic", "make -C /a/core compose-deploy"): "make compose-deploy",
|
||||
("bash", "-c", "cd core && make test 2>&1 | tee /tmp/log"): "make test",
|
||||
("python3", "/s/x.py", "--"): "x.py",
|
||||
("bash", "-c", "set -e; make verify-test"): "make verify-test",
|
||||
("bash", "-c", "source .venv/bin/activate && make test"): "make test",
|
||||
}.items():
|
||||
with self.subTest(argv=argv):
|
||||
self.assertEqual(self.queue.qualified(str(plain), list(argv)), f"plain: {expected}")
|
||||
module = ["python3", "-m", "pytest", "--", "tests/"]
|
||||
self.assertEqual(self.queue.command_words(module), module)
|
||||
|
||||
def test_memory_stage_stop_blocks_every_job_and_warn_none(self):
|
||||
entries = [job(1, need_gb=0), job(2, task="work/b")]
|
||||
self.assertEqual(set(self.decide(entries, memory="stop").values()), {"memory stage stop"})
|
||||
self.assertEqual(self.decide(entries, memory="warn"), {"j1": None, "j2": None})
|
||||
|
||||
def test_hotfix_goes_first_then_rank_then_arrival_and_unknown_tasks_last(self):
|
||||
entries = [
|
||||
@@ -199,6 +292,23 @@ class TestHistory(unittest.TestCase):
|
||||
crowded = [history_row(n, mem_before=90, mem_max=95) for n in range(5)]
|
||||
self.assertEqual(self.learn(crowded)[1]["test"]["limit"], 1)
|
||||
|
||||
def test_footprint_takes_the_larger_of_kind_and_signature_at_least_the_minimum_and_none_for_a_new_kind(self):
|
||||
small = [history_row(n, signature="make test", mem_max=25, cpu_max=20) for n in range(5)]
|
||||
big = [
|
||||
history_row(n + 5, kind="quality-high", signature="make verify-test", mem_max=80, cpu_max=90)
|
||||
for n in range(5)
|
||||
]
|
||||
idle = [history_row(n + 10, kind="idle", signature="true", mem_max=20, cpu_max=5) for n in range(5)]
|
||||
signatures, kinds = self.learn([*small, *big, *idle])
|
||||
footprint = self.history.footprint
|
||||
self.assertEqual(footprint("test", "make test", signatures, kinds), {"ram": 5, "cpu": 10})
|
||||
self.assertEqual(footprint("test", "make verify-test", signatures, kinds), {"ram": 60, "cpu": 80})
|
||||
self.assertEqual(footprint("test", "make unseen", signatures, kinds), {"ram": 5, "cpu": 10})
|
||||
self.assertEqual(footprint("idle", "true", signatures, kinds), {"ram": 1, "cpu": 1})
|
||||
self.assertIsNone(footprint("gui-dind", "make test-up", signatures, kinds))
|
||||
signatures, kinds = self.learn(small[:4])
|
||||
self.assertIsNone(footprint("test", "make test", signatures, kinds))
|
||||
|
||||
def test_slot_limits_take_the_learned_limit_else_the_seed_and_a_smaller_cap(self):
|
||||
kinds = {"deploy": {"limit": 3}, "build": {"limit": None}}
|
||||
self.assertEqual(self.history.slot_limits(kinds, {"deploy": 2, "swarm": 0, "test": 9}, ["test", "gate"]), {
|
||||
@@ -256,7 +366,9 @@ class TestQueueCommands(unittest.TestCase):
|
||||
"projection_minutes": 30,
|
||||
"slot_need_gb": {"test": 0.001, "deploy": 0},
|
||||
"slot_limits": {"test": 1, "deploy": 1},
|
||||
"load_limits": {"cpu_percent": 101, "mem_percent": 101, "sample_seconds": 0.1, "kinds": ["test"]},
|
||||
"load_limits": {"cpu_percent": 101, "mem_percent": 101, "sample_seconds": 0.1},
|
||||
"budget_percent": {"ram": 75, "cpu": 100},
|
||||
"memory_limits": {stage: OPEN_MEMORY for stage in ("warn", "stop")},
|
||||
"heavy_commands": ["^make compose-deploy"],
|
||||
"never_block": ["^make compose-down"],
|
||||
}
|
||||
@@ -338,13 +450,23 @@ class TestQueueCommands(unittest.TestCase):
|
||||
[row] = [json.loads(line) for line in self.history_file.read_text(encoding="utf-8").splitlines()]
|
||||
self.assertEqual(
|
||||
[row[k] for k in ("kind", "signature", "workdir", "exit", "concurrent")],
|
||||
["test", f"{Path(sys.executable).name} -c", str(self.base), 7, 0],
|
||||
["test", f"{self.base.name}: {Path(sys.executable).name} -c", str(self.base), 7, 0],
|
||||
)
|
||||
self.assertTrue(row["id"].startswith("test-1-"))
|
||||
self.assertLessEqual(row["start"], row["end"])
|
||||
self.assertEqual(row["disk_gb"], round(max(0.0, row["free_before_gb"] - row["min_free_gb"]), 2))
|
||||
self.assertLessEqual(row["mem_before"], row["mem_max"])
|
||||
|
||||
def test_a_command_that_did_no_work_leaves_no_history_row(self):
|
||||
result = self.run_queue(
|
||||
"hold", "--kind", "test", "--task", "work/a", "--session", "s1", "--", sys.executable, "-c",
|
||||
"import sys; sys.exit(75)",
|
||||
)
|
||||
self.assertEqual(result.returncode, 75)
|
||||
self.assertFalse(self.history_file.exists())
|
||||
self.assertEqual(self.run_queue("hold", "--kind", "test", "--task", "work/a", "--session", "s1", "--", "false").returncode, 1)
|
||||
self.assertEqual(len(self.history_file.read_text(encoding="utf-8").splitlines()), 1)
|
||||
|
||||
def test_hold_runs_without_slot_limits_on_the_seed_limit(self):
|
||||
del self.cfg["slot_limits"]
|
||||
self.write_config()
|
||||
@@ -464,7 +586,8 @@ class TestQueueCommands(unittest.TestCase):
|
||||
self.write_config()
|
||||
now = time.time()
|
||||
rows = [
|
||||
history_row(n, kind="deploy", signature="true", end=now, disk_gb=gb) for n, gb in enumerate((4, 1, 10, 3, 2))
|
||||
history_row(n, kind="deploy", signature=f"{self.base.name}: true", end=now, disk_gb=gb)
|
||||
for n, gb in enumerate((4, 1, 10, 3, 2))
|
||||
]
|
||||
self.write_history(self.history_file, rows[:3])
|
||||
self.write_history(self.kept, [{**rows[0], "disk_gb": 100}, *rows[3:]])
|
||||
@@ -480,9 +603,9 @@ class TestQueueCommands(unittest.TestCase):
|
||||
me, mine = os.getpid(), queue.identity(os.getpid())
|
||||
now = time.time()
|
||||
self.write_history(
|
||||
self.history_file, [history_row(n, signature="make compose-deploy", end=now) for n in range(5)]
|
||||
self.history_file, [history_row(n, signature="w: make compose-deploy", end=now) for n in range(5)]
|
||||
)
|
||||
deploy = {"cmd": "make compose-deploy apps=x"}
|
||||
deploy = {"cmd": "make compose-deploy apps=x", "workdir": "/w"}
|
||||
self.write_store([
|
||||
{**stored(1, "active", me, mine, me, mine), **deploy, "granted": now - 240},
|
||||
{**stored(2, "waiting", me, mine, None, None), **deploy},
|
||||
@@ -545,6 +668,9 @@ class TestQueueCommands(unittest.TestCase):
|
||||
self.assertEqual(refused.returncode, 2)
|
||||
self.assertIn("queue.py hold --kind <kind> --task <register task> -- <command>", refused.stderr)
|
||||
self.assertEqual(self.bash("python3 skills/kvb-check-gate/scripts/gate.py run --repo . --target test"), 2)
|
||||
for command in ("make -j4 compose-deploy apps=x", "make 'compose-deploy'", "cd x && make compose-deploy"):
|
||||
with self.subTest(command=command):
|
||||
self.assertEqual(self.bash(command), 2)
|
||||
for command in (
|
||||
"python3 /x/queue.py hold --kind deploy --task work/a -- make compose-deploy",
|
||||
"make compose-down",
|
||||
@@ -557,6 +683,168 @@ class TestQueueCommands(unittest.TestCase):
|
||||
self.config.unlink()
|
||||
self.assertEqual(self.bash("make compose-deploy"), 0)
|
||||
|
||||
def test_block_with_queue_enforce_refuses_builds_installs_and_full_suites_by_itself(self):
|
||||
self.cfg["queue_enforce"] = True
|
||||
self.cfg["heavy_commands"] = []
|
||||
self.write_config()
|
||||
for command in (
|
||||
"docker compose up -d --build",
|
||||
"docker compose -f x.yml build web",
|
||||
"docker-compose up --build",
|
||||
"make install",
|
||||
"make -C core verify-test",
|
||||
"pytest tests/",
|
||||
"python3 -m pytest -q",
|
||||
".venv/bin/pytest tests/lint tests/unit",
|
||||
"pytest tests/a.py tests/b.py",
|
||||
"python3 -m unittest discover -s tests",
|
||||
"python3 -m unittest discover -s tests -p 'test_*.py'",
|
||||
"pytest tests/x.py tests/lint",
|
||||
"pytest tests/lint --ignore=tests/lint/y.py",
|
||||
"python3 -m unittest",
|
||||
"make -j4 install",
|
||||
"make VAR=x install",
|
||||
"cd core && env A=1 timeout 600 python3 -m pytest -q 2>&1 | tee /tmp/log",
|
||||
"bash -c 'pytest tests/'",
|
||||
"pytest tests/ # queue.py hold",
|
||||
"python3 /x/queue.py hold --kind test --task work/a -- true & pytest tests/",
|
||||
"make compose-down; make install",
|
||||
"uv run pytest tests/",
|
||||
"coverage run -m pytest",
|
||||
"python3 -mpytest tests/",
|
||||
"python3 -X dev -m pytest",
|
||||
"timeout -s KILL 900 pytest tests/",
|
||||
"sudo -u nobody pytest tests/",
|
||||
"env -u FOO pytest",
|
||||
"{ pytest tests/; }",
|
||||
"! pytest tests/",
|
||||
"if pytest tests/; then echo ok; fi",
|
||||
"echo `pytest tests/`",
|
||||
"bash -c -- 'pytest tests/'",
|
||||
"bash <<'EOF'\npytest tests/\nEOF",
|
||||
'OUT="$(pytest tests/ 2>&1)"',
|
||||
'echo "r: $(make install)"',
|
||||
"cat <<EOF\n`make install`\nEOF",
|
||||
"uv run --extra dev pytest tests/",
|
||||
"uv run --with pytest-xdist pytest -n 8 tests/",
|
||||
"coverage run --source pkg -m pytest tests/",
|
||||
"poetry -C core run pytest tests/",
|
||||
"python3 -m coverage run -m pytest tests/",
|
||||
"uvx pytest tests/",
|
||||
"uv tool run pytest tests/",
|
||||
"time -p pytest tests/",
|
||||
"/usr/bin/time -v pytest tests/",
|
||||
"/usr/bin/time -f %M make install",
|
||||
"make 2>&1 install",
|
||||
">/tmp/log pytest tests/",
|
||||
"2>/dev/null make install",
|
||||
"pytest tests/test_watch.py 2>/dev/null tests/",
|
||||
"make compose-down install",
|
||||
"make -C core compose-down verify-test",
|
||||
"make --directory core install",
|
||||
"command pytest tests/",
|
||||
"command -p pytest tests/",
|
||||
"command make install",
|
||||
"env -S 'pytest tests/'",
|
||||
"bash -Ec 'pytest tests/'",
|
||||
"exec -a x pytest tests/",
|
||||
"sudo --user root pytest tests/",
|
||||
"env $(printenv | grep -o -E '^INFINITO_[A-Z0-9_]+' | sed 's/^/-u /') -C /x python3 -m pytest tests/",
|
||||
"env $(cat .env) -u X make install",
|
||||
"sudo -u $(id -un) -- pytest tests/",
|
||||
"timeout --signal=$(echo KILL) 900 pytest tests/",
|
||||
"sudo --chdir /x pytest tests/",
|
||||
"sudo bash <<'EOF'\npytest tests/\nEOF",
|
||||
"bash -s <<'END-MSG'\nmake install\nEND-MSG",
|
||||
"out=$(bash <<'EOF'\nmake install\nEOF\n)",
|
||||
"bash -c 'bash <<EOF\nmake install\nEOF\n'",
|
||||
"bash <<'A'\nbash <<'B'\nmake install\nB\nA",
|
||||
"bash -euo pipefail <<'EOF'\nmake test\nmake install\nEOF",
|
||||
"bash -o pipefail <<'EOF'\npytest tests/\nEOF",
|
||||
"bash -s -- one <<'EOF'\nmake install\nEOF",
|
||||
"bash --rcfile x <<'EOF'\nmake install\nEOF",
|
||||
"make -Cinfinito install",
|
||||
"make -Oline install",
|
||||
"bash <<'EOF'\nmake install \ud800\nEOF",
|
||||
):
|
||||
with self.subTest(command=command):
|
||||
self.assertEqual(self.bash(command), 2)
|
||||
for command in (
|
||||
"docker compose up -d",
|
||||
"docker compose ps",
|
||||
"make install-skills",
|
||||
"pytest tests/unit/test_x.py",
|
||||
"python3 -m pytest tests/unit/test_x.py::TestA -q",
|
||||
"pip install pytest",
|
||||
"python3 -m unittest discover -s tests -p test_queue.py",
|
||||
"python3 -m unittest tests.test_queue",
|
||||
"python3 /x/queue.py hold --kind test --task work/a -- pytest tests/",
|
||||
'pytest "tests/unit/test_x.py"',
|
||||
"python3 -m unittest discover -s tests -p 'test_queue.py'",
|
||||
"grep -rn pytest tests/",
|
||||
"which pytest",
|
||||
"pytest --version",
|
||||
"pytest tests/unit/test_x.py -p no:cacheprovider 2>&1 | tee /tmp/log",
|
||||
"command -v pytest",
|
||||
"make -n install",
|
||||
"cat <<'EOF'\npytest tests/\nEOF",
|
||||
"pytest >/tmp/out.log tests/test_watch.py",
|
||||
"git commit -m \"$(cat <<'EOF'\nfix: run `make install` once\nEOF\n)\"",
|
||||
"gh pr create --body-file - <<'EOF'\nruns `make verify-test`\nEOF",
|
||||
"echo '$(make install)'",
|
||||
"uv pip install pytest",
|
||||
"command -V pytest",
|
||||
"time pytest tests/unit/test_x.py",
|
||||
"docker compose -f compose.yml -f build/compose/runner-mirror.override.yml config --services",
|
||||
"docker compose --project-directory build up -d",
|
||||
'echo "see \\`make install\\` and \\$(make install) docs"',
|
||||
"pytest -n auto --dist loadscope tests/x.py",
|
||||
"pytest --cov src --timeout 30 tests/x.py",
|
||||
"cat > /tmp/run.sh <<'EOF'\npytest tests/\nEOF",
|
||||
"gh pr create --title \"fix(bash): quoting\" --body \"$(cat <<'EOF'\nruns `make install`\nEOF\n)\"",
|
||||
"ls # it's $(make install)",
|
||||
"make -nC core install",
|
||||
"make -kn install",
|
||||
"python3 /x/queue.py hold --kind test --task work/a -- bash <<'EOF'\nmake install\nEOF",
|
||||
"python3 /x/queue.py hold --kind test --task work/a -- bash -c \"$(cat <<'EOF'\nmake install\nEOF\n)\"",
|
||||
"cat <<'END-MSG'\nmake install\nEND-MSG",
|
||||
"cat <<\\EOF\n$(make install)\nEOF",
|
||||
"pytest --numprocesses 4 tests/x.py",
|
||||
"pytest tests/lint --collect-only",
|
||||
"bash script.sh <<'EOF'\nmake install\nEOF",
|
||||
"out=$(cat <<'EOF'\nmake install\nEOF\n)",
|
||||
"echo kvbheredoc1 kvbheredoczz",
|
||||
):
|
||||
with self.subTest(command=command):
|
||||
self.assertEqual(self.bash(command), 0)
|
||||
|
||||
def test_block_matches_a_config_regex_on_the_wrapped_program_too(self):
|
||||
self.cfg["queue_enforce"] = True
|
||||
self.cfg["heavy_commands"] = [r"^docker compose up"]
|
||||
self.write_config()
|
||||
for command in ("timeout 900 docker compose up -d", "sudo docker compose up -d", "docker compose up -d"):
|
||||
with self.subTest(command=command):
|
||||
self.assertEqual(self.bash(command), 2)
|
||||
self.assertEqual(self.bash("docker compose ps"), 0)
|
||||
|
||||
def test_hold_refuses_a_config_without_the_budget_or_the_memory_limits(self):
|
||||
for key, cfg in (
|
||||
("budget_percent", {k: v for k, v in self.cfg.items() if k != "budget_percent"}),
|
||||
("memory_limits", {k: v for k, v in self.cfg.items() if k != "memory_limits"}),
|
||||
("budget_percent.cpu", {**self.cfg, "budget_percent": {"ram": 75}}),
|
||||
("budget_percent.ram", {**self.cfg, "budget_percent": {"ram": "75", "cpu": 100}}),
|
||||
("budget_percent.cpu", {**self.cfg, "budget_percent": {"ram": 75, "cpu": float("nan")}}),
|
||||
("memory_limits.warn.container_swap_gb", {**self.cfg, "memory_limits": {
|
||||
"warn": {k: v for k, v in OPEN_MEMORY.items() if k != "container_swap_gb"}, "stop": OPEN_MEMORY,
|
||||
}}),
|
||||
):
|
||||
with self.subTest(key=key):
|
||||
self.config.write_text(json.dumps(cfg), encoding="utf-8")
|
||||
result = self.run_queue("hold", "--kind", "test", "--task", "work/a", "--session", "s1", "--", "true")
|
||||
self.assertEqual(result.returncode, 2)
|
||||
self.assertIn(f"lacks {key};", result.stderr)
|
||||
self.assertFalse((self.root / "queue.json").exists())
|
||||
|
||||
def test_block_holds_a_held_command_to_the_bash_rules_of_the_settings(self):
|
||||
project = self.base / "project"
|
||||
(project / ".claude").mkdir(parents=True)
|
||||
|
||||
+146
-8
@@ -17,6 +17,10 @@ from pathlib import Path
|
||||
REPO_ROOT = Path(__file__).resolve().parents[1]
|
||||
WATCH = REPO_ROOT / "skills" / "kvb-org-organize" / "scripts" / "watch.py"
|
||||
LIMITS = {"warn": 40, "stop": 15, "critical": 12}
|
||||
OPEN_MEMORY = {"available": -1, "swap_free": -1, "psi_full": 1000, "container_swap_gb": 1e9}
|
||||
SHUT_MEMORY = {"available": 101, "swap_free": 101, "psi_full": 0, "container_swap_gb": 0}
|
||||
MEMINFO = "MemTotal: 1000 kB\nMemAvailable: 120 kB\nSwapTotal: 2000 kB\nSwapFree: 300 kB\n"
|
||||
PSI = "some avg10=30.00 avg60=1.00 avg300=0.50 total=1\nfull avg10=12.50 avg60=0.80 avg300=0.10 total=1\n"
|
||||
|
||||
|
||||
def _load():
|
||||
@@ -41,6 +45,66 @@ class TestStage(unittest.TestCase):
|
||||
meminfo = "MemTotal: 1000 kB\nMemFree: 100 kB\nMemAvailable: 250 kB\n"
|
||||
self.assertEqual(self.watch.mem_used_percent(meminfo), 75.0)
|
||||
|
||||
def test_memory_sample_reads_available_swap_and_psi_and_tolerates_their_absence(self):
|
||||
self.assertEqual(
|
||||
self.watch.memory_sample(MEMINFO, PSI, 2.5),
|
||||
{"available": 12.0, "swap_free": 15.0, "psi_full": 12.5, "container_swap_gb": 2.5},
|
||||
)
|
||||
bare = "MemTotal: 1000 kB\nMemAvailable: 500 kB\nSwapTotal: 0 kB\nSwapFree: 0 kB\n"
|
||||
self.assertEqual(
|
||||
self.watch.memory_sample(bare, None, 0.0),
|
||||
{"available": 50.0, "swap_free": None, "psi_full": None, "container_swap_gb": 0.0},
|
||||
)
|
||||
|
||||
def test_memory_takes_the_swap_of_the_container_holding_the_most_nested_groups_included(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
base = Path(tmp)
|
||||
for name, swap in ((f"docker-{'a' * 64}.scope", 3_000_000_000), (f"docker-{'b' * 64}.scope", 18_000_000_000),
|
||||
("other.scope", 90_000_000_000)):
|
||||
(base / name).mkdir()
|
||||
(base / name / "memory.swap.current").write_text(str(swap), encoding="utf-8")
|
||||
self.watch.CGROUPS = base
|
||||
self.assertEqual(self.watch.memory()["container_swap_gb"], 18.0)
|
||||
self.watch.CGROUPS = base / "absent"
|
||||
self.assertEqual(self.watch.memory()["container_swap_gb"], 0.0)
|
||||
|
||||
def test_memory_stage_takes_the_first_bound_crossed_stop_before_warn(self):
|
||||
cfg = {"memory_limits": {
|
||||
"warn": {"available": 15, "swap_free": 25, "psi_full": 5, "container_swap_gb": 4},
|
||||
"stop": {"available": 8, "swap_free": 10, "psi_full": 10, "container_swap_gb": 16},
|
||||
}}
|
||||
stage = self.watch.memory_stage
|
||||
healthy = {"available": 40.0, "swap_free": 80.0, "psi_full": 0.0, "container_swap_gb": 3.9}
|
||||
self.assertEqual(stage(healthy, cfg), "ok")
|
||||
for change, expected in (
|
||||
({"available": 14.0}, "warn"), ({"swap_free": 24.0}, "warn"), ({"psi_full": 5.0}, "warn"),
|
||||
({"container_swap_gb": 4.0}, "warn"),
|
||||
({"available": 7.0}, "stop"), ({"swap_free": 9.0}, "stop"), ({"psi_full": 10.0}, "stop"),
|
||||
({"container_swap_gb": 16.0}, "stop"),
|
||||
({"swap_free": None, "psi_full": None}, "ok"),
|
||||
):
|
||||
with self.subTest(change=change):
|
||||
self.assertEqual(stage(healthy | change, cfg), expected)
|
||||
|
||||
def test_swap_holders_follow_the_nested_group_holding_most_of_the_swap(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
base = Path(tmp)
|
||||
outer = "a" * 64
|
||||
|
||||
def group(path: Path, swap: int) -> None:
|
||||
path.mkdir(parents=True)
|
||||
(path / "memory.swap.current").write_text(str(swap), encoding="utf-8")
|
||||
|
||||
group(base / f"docker-{outer}.scope", 18_000_000_000)
|
||||
group(base / f"docker-{outer}.scope" / "c977e579", 17_500_000_000)
|
||||
group(base / f"docker-{outer}.scope" / "c977e579" / f"docker-{'b' * 64}.scope", 17_200_000_000)
|
||||
group(base / f"docker-{outer}.scope" / "c977e579" / "small", 100)
|
||||
group(base / f"docker-{'c' * 64}.scope", 1_000_000_000)
|
||||
group(base / f"docker-{'d' * 64}.scope", 0)
|
||||
(base / "other.scope").mkdir()
|
||||
holders = self.watch.swap_holders(base, {outer: "iam"})
|
||||
self.assertEqual(holders, [f"iam > c977e579 > {'b' * 12} 18.0 GB", f"{'c' * 12} 1.0 GB"])
|
||||
|
||||
def test_fast_fall_escalates_one_stage(self):
|
||||
now = time.time()
|
||||
self.assertEqual(self.watch.stage_for(32, [[now - 600, 48]], self.cfg, now), "stop")
|
||||
@@ -73,7 +137,8 @@ class TestWatchCommands(unittest.TestCase):
|
||||
"cache_budget_percent": 3,
|
||||
"slot_need_gb": {"swarm": 30, "test": 0.001},
|
||||
"notify": ["true"],
|
||||
"load_limits": {"cpu_percent": 101, "mem_percent": 101, "sample_seconds": 0.1, "kinds": ["test"]},
|
||||
"load_limits": {"cpu_percent": 101, "mem_percent": 101, "sample_seconds": 0.1},
|
||||
"memory_limits": {stage: dict(OPEN_MEMORY) for stage in ("warn", "stop")},
|
||||
}
|
||||
self._write_config()
|
||||
|
||||
@@ -126,17 +191,21 @@ class TestWatchCommands(unittest.TestCase):
|
||||
self.cfg["heavy_commands"] = [r"\bmake\s+(-C\s+\S+\s+)?(compose-deploy|test|swarm-\S+)(\s|$)"]
|
||||
self._write_config()
|
||||
self._stage("stop")
|
||||
message = "$(cat <<'EOF'\nfix: run \"make test\" and make swarm-zombie\nEOF\n)"
|
||||
message = "$(cat <<'EOF'\nfix: run \"make test\" and make swarm-zombie, `make test` too\nEOF\n)"
|
||||
for command in (
|
||||
'git commit --no-verify -m "fix: passes the vars to make swarm-zombie app=x"',
|
||||
"grep -n 'make test' Makefile",
|
||||
f'git commit --no-verify -m "{message}"',
|
||||
"gh pr create --body-file - <<'EOF'\nruns `make test`\nEOF",
|
||||
"echo 'not run: $(make test)'",
|
||||
):
|
||||
self.assertEqual(self._run("block", stdin=self._bash(command)).returncode, 0, command)
|
||||
for command in (
|
||||
"make -C x swarm-zombie app=y",
|
||||
"env FOO=1 make swarm-zombie",
|
||||
'make compose-deploy apps="web-app-x"',
|
||||
'OUT="$(make test 2>&1)"',
|
||||
"cat <<EOF\n$(make test)\nEOF",
|
||||
):
|
||||
self.assertEqual(self._run("block", stdin=self._bash(command)).returncode, 2, command)
|
||||
|
||||
@@ -146,16 +215,85 @@ class TestWatchCommands(unittest.TestCase):
|
||||
self.assertEqual(self._run("admit", "--need-gb", "0.001").returncode, 0)
|
||||
self.assertEqual(self._run("admit", "--need-gb", "1e9").returncode, 3)
|
||||
|
||||
def test_admit_for_a_load_kind_also_needs_cpu_and_ram_below_the_limits(self):
|
||||
def test_every_admit_also_needs_cpu_and_ram_below_the_limits_and_memory_short_of_stop(self):
|
||||
self.cfg["thresholds_gb"] = {"warn": 0, "stop": 0, "critical": 0}
|
||||
self._write_config()
|
||||
self.assertEqual(self._run("admit", "--kind", "test").returncode, 0)
|
||||
granted = self._run("admit", "--kind", "test")
|
||||
self.assertEqual(granted.returncode, 0)
|
||||
self.assertIn("memory stage ok", granted.stdout)
|
||||
self.cfg["load_limits"]["cpu_percent"] = 0
|
||||
self._write_config()
|
||||
denied = self._run("admit", "--kind", "test")
|
||||
self.assertEqual(denied.returncode, 3)
|
||||
self.assertIn("cpu", denied.stdout)
|
||||
self.assertEqual(self._run("admit", "--need-gb", "0.001").returncode, 0)
|
||||
for args in (("--kind", "test"), ("--kind", "swarm"), ("--need-gb", "0.001")):
|
||||
with self.subTest(args=args):
|
||||
denied = self._run("admit", *args)
|
||||
self.assertEqual(denied.returncode, 3)
|
||||
self.assertIn("cpu", denied.stdout)
|
||||
self.cfg["load_limits"]["cpu_percent"] = 101
|
||||
self.cfg["memory_limits"]["stop"] = dict(SHUT_MEMORY)
|
||||
self._write_config()
|
||||
short = self._run("admit", "--need-gb", "0.001")
|
||||
self.assertEqual(short.returncode, 3)
|
||||
self.assertIn("memory stage stop", short.stdout)
|
||||
|
||||
def test_block_refuses_heavy_jobs_at_memory_stop_only(self):
|
||||
self._stage("ok")
|
||||
self.cfg["memory_limits"]["warn"] = dict(SHUT_MEMORY)
|
||||
self._write_config()
|
||||
self.assertEqual(self._run("block", stdin=self._bash("make compose-deploy")).returncode, 0)
|
||||
self.cfg["memory_limits"]["stop"] = dict(SHUT_MEMORY)
|
||||
self._write_config()
|
||||
blocked = self._run("block", stdin=self._bash("make compose-deploy"))
|
||||
self.assertEqual(blocked.returncode, 2)
|
||||
self.assertIn("memory stage is stop", blocked.stderr)
|
||||
self.assertEqual(self._run("block", stdin=self._bash("docker compose up -d --build")).returncode, 2)
|
||||
self.assertEqual(self._run("block", stdin=self._bash("make swarm-down")).returncode, 0)
|
||||
self.assertEqual(self._run("block", stdin=self._bash("ls")).returncode, 0)
|
||||
|
||||
def test_memory_warn_alert_names_the_stage_once_per_session(self):
|
||||
self.cfg["thresholds_gb"] = {"warn": 0, "stop": 0, "critical": 0}
|
||||
self.cfg["memory_limits"]["warn"] = dict(SHUT_MEMORY)
|
||||
self._write_config()
|
||||
payload = json.dumps({"session_id": "s1", "hook_event_name": "UserPromptSubmit"})
|
||||
first = self._run("watch", stdin=payload).stdout
|
||||
self.assertIn("RESOURCE MEMORY WARN", first)
|
||||
self.assertIn("Top swap holders", first)
|
||||
self.assertEqual(self._run("watch", stdin=payload).stdout, "")
|
||||
|
||||
def test_a_config_without_memory_limits_blocks_heavy_jobs_keeps_the_disk_watch_and_stops_commands(self):
|
||||
del self.cfg["memory_limits"]
|
||||
self.cfg["thresholds_gb"] = {"warn": 1e9, "stop": -1, "critical": -2}
|
||||
self._write_config()
|
||||
blocked = self._run("block", stdin=self._bash("make compose-deploy"))
|
||||
self.assertEqual(blocked.returncode, 2)
|
||||
self.assertIn("lacks memory_limits", blocked.stderr)
|
||||
self.assertEqual(self._run("block", stdin=self._bash("ls")).returncode, 0)
|
||||
payload = json.dumps({"session_id": "s1", "hook_event_name": "UserPromptSubmit"})
|
||||
watched = self._run("watch", stdin=payload)
|
||||
self.assertEqual(watched.returncode, 0)
|
||||
self.assertIn("RESOURCE WARN", watched.stdout)
|
||||
self.assertIn("lacks memory_limits", watched.stdout)
|
||||
command = self._run("admit", "--need-gb", "0")
|
||||
self.assertEqual(command.returncode, 2)
|
||||
self.assertIn("lacks memory_limits", command.stderr)
|
||||
|
||||
def test_a_memory_limit_missing_one_bound_names_it_and_blocks_heavy_jobs(self):
|
||||
del self.cfg["memory_limits"]["stop"]["psi_full"]
|
||||
self.cfg["thresholds_gb"] = {"warn": 0, "stop": 0, "critical": 0}
|
||||
self._write_config()
|
||||
blocked = self._run("block", stdin=self._bash("make compose-deploy"))
|
||||
self.assertEqual(blocked.returncode, 2)
|
||||
self.assertIn("lacks memory_limits.stop.psi_full;", blocked.stderr)
|
||||
self.assertEqual(self._run("block", stdin=self._bash("ls")).returncode, 0)
|
||||
payload = json.dumps({"session_id": "s1", "hook_event_name": "UserPromptSubmit"})
|
||||
watched = self._run("watch", stdin=payload)
|
||||
self.assertEqual(watched.returncode, 0, watched.stderr)
|
||||
self.assertIn("lacks memory_limits.stop.psi_full", watched.stdout)
|
||||
|
||||
def test_segments_split_at_newlines_and_drop_a_redirection_with_its_target(self):
|
||||
self.assertEqual(
|
||||
_load().segments("pytest a.py 2>&1 | tee log\ncd x; make 'a b' # pytest\n>/tmp/o make 2>/dev/null install &>x\ndiff <(pytest t) y"),
|
||||
[["pytest", "a.py"], ["tee", "log"], ["cd", "x"], ["make", "a b"], ["make", "install"], ["diff"], ["pytest", "t"], ["y"]],
|
||||
)
|
||||
|
||||
def test_warn_alert_reaches_each_session_once(self):
|
||||
self.cfg["thresholds_gb"] = {"warn": 1e9, "stop": -1, "critical": -2}
|
||||
|
||||
Reference in new issue
Block a user