fix(kvb-org-queue): global budget so no queue mix can OOM the host
Root cause of the 2026-11-07 20:14 OOM: the queue granted jobs per kind only, so several heavy kinds and direct heavy commands outside the queue ran side by side until the kernel killed the editor. - queue.py: every grant checks the sum of the learned RAM and CPU footprints of active and granted jobs against budget_percent; an unlearned kind takes the whole budget and runs alone. RAM used plus what young jobs (RAMP_SECONDS) may still take must stay below load_limits. Memory stage stop blocks every grant. - watch.py: memory stage from MemAvailable, swap, PSI full and the swap of the container holding most of it; the block hook classifies builds, make install/verify-*, and pytest or unittest beyond one file as heavy through wrappers, sh -c, command substitutions and heredocs a shell reads. - history.py: footprints per kind and signature, seed RAM for pending shares. - operator.py: [h]old queues a heavy row through queue.py hold and asks again on its grant, with a flushed tty and a timeout; sudo rows, which need the terminal, stay direct. Residuals: a shell reading code from a pipe or here-string, its own script, a program named by a substitution, quotes inside an expanding heredoc, an escaped backslash before a substitution; a lone job waits while the host is over load_limits. 🎯 Fix: 92 % 💥 Break: 4 % 💸 Cost: 100 % ⏱️ Time: – 🛡️ Security: 100 % 🏆 Score: 88 🚦 Gate: ✅ 90/5, cost and time <= 100, security 100 % Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
09f68dd75b
commit
4321b97529
8 files changed
+1665
-218
No files matched your search
@@ -32,8 +32,19 @@ Args:
|
|||||||
visible), reason and check, then a preview of what the command
|
visible), reason and check, then a preview of what the command
|
||||||
changes, and asks [y]es / [s]kip / [d]iff / [q]uit; d shows the full
|
changes, and asks [y]es / [s]kip / [d]iff / [q]uit; d shows the full
|
||||||
diff. y runs the command in its directory, then sets the row done
|
diff. y runs the command in its directory, then sets the row done
|
||||||
when its check passes (manual: when the command exits 0). End of
|
when its check passes (manual: when the command exits 0). A command
|
||||||
input quits. There is no flag that answers for the operator.
|
the kvb-org-organize watch config calls heavy also shows the memory
|
||||||
|
stage and `queue.py list`; once the config holds the queue keys and
|
||||||
|
slot_need_gb.operator, and the command runs no sudo, [h]old runs
|
||||||
|
`granted` through `queue.py hold --kind operator --task <row>
|
||||||
|
--session Operator`. End of input quits. There is no flag that
|
||||||
|
answers for the operator.
|
||||||
|
granted <task> --dir <dir> --check <check> -- $SHELL -ic <cmd>: what
|
||||||
|
[h]old queues. On the grant it shows the row and its preview again
|
||||||
|
and asks [y]es / [s]kip / [d]iff / [q]uit, then runs the command in
|
||||||
|
the row's directory; a row no longer open with exactly that
|
||||||
|
directory, check and command runs nothing, nor does a question left
|
||||||
|
unanswered for GRANT_SECONDS.
|
||||||
done <slug>: mark a row done, whatever its check.
|
done <slug>: mark a row done, whatever its check.
|
||||||
|
|
||||||
Output:
|
Output:
|
||||||
@@ -45,7 +56,8 @@ Output:
|
|||||||
prints the same rows as a JSON list.
|
prints the same rows as a JSON list.
|
||||||
|
|
||||||
Exit codes:
|
Exit codes:
|
||||||
0 done, 2 usage error.
|
0 done, 2 usage error. granted: the command's exit code, or 75
|
||||||
|
(EX_TEMPFAIL, which queue.py hold does not learn from) when nothing ran.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
@@ -62,13 +74,25 @@ import hashlib
|
|||||||
import importlib.util
|
import importlib.util
|
||||||
import json
|
import json
|
||||||
import re
|
import re
|
||||||
|
import select
|
||||||
import shlex
|
import shlex
|
||||||
import shutil
|
import shutil
|
||||||
import subprocess
|
import subprocess
|
||||||
|
import termios
|
||||||
|
import time
|
||||||
|
from collections.abc import Callable
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from types import ModuleType
|
from types import ModuleType
|
||||||
|
|
||||||
RASCI = Path(__file__).resolve().parents[2] / "kvb-org-rasci" / "scripts" / "rasci.py"
|
SKILLS = Path(__file__).resolve().parents[2]
|
||||||
|
RASCI = SKILLS / "kvb-org-rasci" / "scripts" / "rasci.py"
|
||||||
|
WATCH = SKILLS / "kvb-org-organize" / "scripts" / "watch.py"
|
||||||
|
QUEUE = SKILLS / "kvb-org-queue" / "scripts" / "queue.py"
|
||||||
|
HOLD_KIND = "operator"
|
||||||
|
HOLD_SESSION = "Operator"
|
||||||
|
NOT_RUN = 75
|
||||||
|
GRANT_SECONDS = 600
|
||||||
|
SUDO = re.compile(r"\bsudo\b")
|
||||||
PREFIX = "operator/"
|
PREFIX = "operator/"
|
||||||
KINDS = ("pushed", "installed", "binary", "absent", "manual", "decision")
|
KINDS = ("pushed", "installed", "binary", "absent", "manual", "decision")
|
||||||
OPEN = ("queued", "blocked")
|
OPEN = ("queued", "blocked")
|
||||||
@@ -264,8 +288,8 @@ def runnable(row: dict) -> bool:
|
|||||||
return bool(row["directory"] and row["command"]) and row["check"] != "decision"
|
return bool(row["directory"] and row["command"]) and row["check"] != "decision"
|
||||||
|
|
||||||
|
|
||||||
def visible(text: str) -> str:
|
def visible(text: str, keep: str) -> str:
|
||||||
return "".join(c if c.isprintable() else c.encode("unicode_escape").decode() for c in text)
|
return "".join(c if c.isprintable() or c in keep else c.encode("unicode_escape").decode() for c in text)
|
||||||
|
|
||||||
|
|
||||||
def git_tree(directory: str) -> bool:
|
def git_tree(directory: str) -> bool:
|
||||||
@@ -312,14 +336,113 @@ def show(commands: list[tuple[list[str], str]]) -> None:
|
|||||||
print(f" {error}", flush=True)
|
print(f" {error}", flush=True)
|
||||||
|
|
||||||
|
|
||||||
def ask() -> str:
|
def ask(holdable: bool, read: Callable[[str], str | None]) -> str:
|
||||||
|
choices = ("y", "h", "s", "d", "q") if holdable else ("y", "s", "d", "q")
|
||||||
|
prompt = "Run it? [y]es / [h]old in the queue / [s]kip / [d]iff / [q]uit: " if holdable else "Run it? [y]es / [s]kip / [d]iff / [q]uit: "
|
||||||
while True:
|
while True:
|
||||||
try:
|
answer = read(prompt)
|
||||||
answer = input("Run it? [y]es / [s]kip / [d]iff / [q]uit: ").strip().lower()
|
if answer is None:
|
||||||
except EOFError:
|
|
||||||
return "q"
|
return "q"
|
||||||
if answer in ("y", "s", "d", "q"):
|
if answer.strip().lower() in choices:
|
||||||
return answer
|
return answer.strip().lower()
|
||||||
|
|
||||||
|
|
||||||
|
def typed(prompt: str) -> str | None:
|
||||||
|
try:
|
||||||
|
return input(prompt)
|
||||||
|
except EOFError:
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def timed(prompt: str) -> str | None:
|
||||||
|
"""Returns:
|
||||||
|
One line read unbuffered from stdin, None at its end or when no line
|
||||||
|
came within GRANT_SECONDS, so an unanswered grant frees its slot;
|
||||||
|
what was typed while the job waited is dropped first.
|
||||||
|
"""
|
||||||
|
if sys.stdin.isatty():
|
||||||
|
termios.tcflush(sys.stdin.fileno(), termios.TCIFLUSH)
|
||||||
|
print(prompt, end="", flush=True)
|
||||||
|
deadline = time.monotonic() + GRANT_SECONDS
|
||||||
|
line = b""
|
||||||
|
while not line.endswith(b"\n"):
|
||||||
|
ready, _, _ = select.select([sys.stdin.fileno()], [], [], max(0.0, deadline - time.monotonic()))
|
||||||
|
byte = os.read(sys.stdin.fileno(), 1) if ready else b""
|
||||||
|
if not byte:
|
||||||
|
print(flush=True)
|
||||||
|
return None
|
||||||
|
line += byte
|
||||||
|
return line.decode(errors="replace")
|
||||||
|
|
||||||
|
|
||||||
|
def needs_terminal(command: str) -> bool:
|
||||||
|
return SUDO.search(command) is not None
|
||||||
|
|
||||||
|
|
||||||
|
def heavy_row(watch: ModuleType, cfg: dict | None, row: dict) -> bool:
|
||||||
|
"""Returns:
|
||||||
|
Whether [h]old can queue the row's command: a heavy job by the
|
||||||
|
kvb-org-organize watch config, which holds every queue key and the
|
||||||
|
kind operator, and no sudo, whose prompt needs the terminal a queued
|
||||||
|
command lacks; prints the memory stage and `queue.py list` for a
|
||||||
|
heavy one, its output control characters made visible.
|
||||||
|
"""
|
||||||
|
if cfg is None or not watch.heavy(row["command"], cfg, tuple(cfg["never_block"])):
|
||||||
|
return False
|
||||||
|
problem = watch.missing(cfg, (*watch.MEMORY_KEYS, "budget_percent.ram", "budget_percent.cpu", f"slot_need_gb.{HOLD_KIND}"))
|
||||||
|
if problem is None and needs_terminal(row["command"]):
|
||||||
|
problem = "sudo asks on the terminal, which a queued command lacks"
|
||||||
|
if problem is None:
|
||||||
|
sample = watch.memory()
|
||||||
|
print(f" heavy: memory stage {watch.memory_stage(sample, cfg)} ({watch.memory_text(sample)}); the queue:", flush=True)
|
||||||
|
else:
|
||||||
|
print(f" heavy: {problem}; no [h]old until it does. The queue:", flush=True)
|
||||||
|
listing = subprocess.run([sys.executable, str(QUEUE), "list"], cwd=row["directory"], capture_output=True, text=True, check=False)
|
||||||
|
print(visible(listing.stdout + listing.stderr, "\t\n"), end="", flush=True)
|
||||||
|
return problem is None
|
||||||
|
|
||||||
|
|
||||||
|
def held(row: dict, shell: str) -> list[str]:
|
||||||
|
return [
|
||||||
|
sys.executable, str(QUEUE), "hold", "--kind", HOLD_KIND, "--task", row["task"], "--session", HOLD_SESSION,
|
||||||
|
"--", sys.executable, str(Path(__file__).resolve()), "granted", row["task"], "--dir", row["directory"],
|
||||||
|
"--check", row["check"], "--", shell, "-ic", row["command"],
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
def review(row: dict, holdable: bool, read: Callable[[str], str | None]) -> str:
|
||||||
|
check = "manual" if problem(row["check"]) else row["check"]
|
||||||
|
show(evidence(check, row["directory"], full=False))
|
||||||
|
answer = ask(holdable, read)
|
||||||
|
while answer == "d":
|
||||||
|
show(evidence(check, row["directory"], full=True))
|
||||||
|
answer = ask(holdable, read)
|
||||||
|
return answer
|
||||||
|
|
||||||
|
|
||||||
|
def cmd_granted(rasci: ModuleType, args: argparse.Namespace) -> int:
|
||||||
|
shell = os.environ.get("SHELL")
|
||||||
|
approved = [
|
||||||
|
row for row in open_rows(rasci)
|
||||||
|
if runnable(row) and (row["task"], row["directory"], row["check"]) == (args.task, args.dir, args.check)
|
||||||
|
and args.argv == [shell, "-ic", row["command"]]
|
||||||
|
]
|
||||||
|
if not shell or not approved:
|
||||||
|
print(f"operator: {args.task} is no longer open with the approved directory, check and command; nothing ran",
|
||||||
|
file=sys.stderr)
|
||||||
|
return NOT_RUN
|
||||||
|
row = approved[0]
|
||||||
|
print(f"\ngranted {row['task']}: the queue lets it run now; its tree may have changed while it waited")
|
||||||
|
print(f" directory: {visible(row['directory'], '')}")
|
||||||
|
print(f" command: {visible(row['command'], '')}", flush=True)
|
||||||
|
try:
|
||||||
|
answer = review(row, False, timed)
|
||||||
|
except KeyboardInterrupt:
|
||||||
|
answer = "q"
|
||||||
|
if answer != "y":
|
||||||
|
print(f"not run {row['task']}")
|
||||||
|
return NOT_RUN
|
||||||
|
return subprocess.run(args.argv, cwd=row["directory"], check=False).returncode
|
||||||
|
|
||||||
|
|
||||||
def close(rasci: ModuleType, task: str) -> None:
|
def close(rasci: ModuleType, task: str) -> None:
|
||||||
@@ -337,26 +460,26 @@ def cmd_run(rasci: ModuleType, args: argparse.Namespace) -> int:
|
|||||||
if not steps:
|
if not steps:
|
||||||
print("operator: nothing to run")
|
print("operator: nothing to run")
|
||||||
return 0
|
return 0
|
||||||
|
watch = module("kvb_operator_watch", WATCH)
|
||||||
|
cfg = watch.load_config()
|
||||||
for number, row in enumerate(steps, 1):
|
for number, row in enumerate(steps, 1):
|
||||||
check = "manual" if problem(row["check"]) else row["check"]
|
check = "manual" if problem(row["check"]) else row["check"]
|
||||||
print(f"\n[{number}/{len(steps)}] {row['task']}")
|
print(f"\n[{number}/{len(steps)}] {row['task']}")
|
||||||
print(f" directory: {visible(row['directory'])}")
|
print(f" directory: {visible(row['directory'], '')}")
|
||||||
print(f" command: {visible(row['command'])}")
|
print(f" command: {visible(row['command'], '')}")
|
||||||
print(f" why: {visible(row['why'] or '-')}")
|
print(f" why: {visible(row['why'] or '-', '')}")
|
||||||
print(f" check: {check}", flush=True)
|
print(f" check: {check}", flush=True)
|
||||||
show(evidence(check, row["directory"], full=False))
|
holdable = heavy_row(watch, cfg, row)
|
||||||
answer = ask()
|
answer = review(row, holdable, typed)
|
||||||
while answer == "d":
|
|
||||||
show(evidence(check, row["directory"], full=True))
|
|
||||||
answer = ask()
|
|
||||||
if answer == "q":
|
if answer == "q":
|
||||||
print("operator: stopped")
|
print("operator: stopped")
|
||||||
return 0
|
return 0
|
||||||
if answer == "s":
|
if answer == "s":
|
||||||
print(f"skipped {row['task']}")
|
print(f"skipped {row['task']}")
|
||||||
continue
|
continue
|
||||||
|
argv = held(row, shell) if answer == "h" else [shell, "-ic", row["command"]]
|
||||||
try:
|
try:
|
||||||
code = subprocess.run([shell, "-ic", row["command"]], cwd=row["directory"], check=False).returncode
|
code = subprocess.run(argv, cwd=row["directory"], check=False).returncode
|
||||||
except OSError as error:
|
except OSError as error:
|
||||||
print(f"failed {row['task']}: {error}")
|
print(f"failed {row['task']}: {error}")
|
||||||
continue
|
continue
|
||||||
@@ -394,6 +517,12 @@ def main() -> int:
|
|||||||
p.set_defaults(func=cmd_list)
|
p.set_defaults(func=cmd_list)
|
||||||
p = sub.add_parser("run")
|
p = sub.add_parser("run")
|
||||||
p.set_defaults(func=cmd_run)
|
p.set_defaults(func=cmd_run)
|
||||||
|
p = sub.add_parser("granted")
|
||||||
|
p.add_argument("task")
|
||||||
|
p.add_argument("--dir", required=True)
|
||||||
|
p.add_argument("--check", required=True)
|
||||||
|
p.add_argument("argv", nargs="+")
|
||||||
|
p.set_defaults(func=cmd_granted)
|
||||||
p = sub.add_parser("done")
|
p = sub.add_parser("done")
|
||||||
p.add_argument("slug")
|
p.add_argument("slug")
|
||||||
p.set_defaults(func=cmd_done)
|
p.set_defaults(func=cmd_done)
|
||||||
|
|||||||
@@ -9,11 +9,14 @@ Usage:
|
|||||||
(kvb-check-security) whose id it lacks to the same name
|
(kvb-check-security) whose id it lacks to the same name
|
||||||
under ~/.local/state/agents-council/
|
under ~/.local/state/agents-council/
|
||||||
watch.py block hook (PreToolUse on Bash): exit 2 for a heavy job while the
|
watch.py block hook (PreToolUse on Bash): exit 2 for a heavy job while the
|
||||||
stage is stop or critical, or while a kvb-org-radio mayday
|
disk stage is stop or critical, the memory stage is stop
|
||||||
holds radio silence (<root>/radio-silence.json, until > now)
|
or cannot be judged without memory_limits, or a
|
||||||
|
kvb-org-radio mayday holds radio silence
|
||||||
|
(<root>/radio-silence.json, until > now)
|
||||||
watch.py admit (--need-gb N | --kind KIND)
|
watch.py admit (--need-gb N | --kind KIND)
|
||||||
exit 0 when the grant leaves at least the stop threshold
|
exit 0 when the grant leaves at least the stop threshold
|
||||||
free, 3 otherwise
|
free, CPU busy and RAM used stay below load_limits and
|
||||||
|
the memory stage is not stop, 3 otherwise
|
||||||
watch.py cleanup (--caches | --full)
|
watch.py cleanup (--caches | --full)
|
||||||
trim caches to their budget; --full also runs the
|
trim caches to their budget; --full also runs the
|
||||||
configured low-risk cleanup commands
|
configured low-risk cleanup commands
|
||||||
@@ -45,8 +48,15 @@ compose project name is skipped), worktree_marker (path relative
|
|||||||
to a worktree that must exist), busy_processes (process names; a worktree
|
to a worktree that must exist), busy_processes (process names; a worktree
|
||||||
where one of them runs is skipped), cache_dirs, cache_budget_percent,
|
where one of them runs is skipped), cache_dirs, cache_budget_percent,
|
||||||
slot_need_gb {kind: GB}, notify (argv the alert text is appended to),
|
slot_need_gb {kind: GB}, notify (argv the alert text is appended to),
|
||||||
load_limits {cpu_percent, mem_percent, sample_seconds, kinds}: admit for a
|
load_limits {cpu_percent, mem_percent, sample_seconds}: every admit and every
|
||||||
kind listed in kinds also needs CPU busy and RAM used below the limits.
|
queue grant needs CPU busy and RAM used below them, memory_limits {warn, stop},
|
||||||
|
each {available, swap_free, psi_full, container_swap_gb}: the memory stage is
|
||||||
|
the first of stop and warn whose MemAvailable or SwapFree percentage lies below
|
||||||
|
its bound, whose /proc/pressure/memory full avg10 reaches its bound, or where
|
||||||
|
one container (a docker-*.scope cgroup under system.slice, nested groups
|
||||||
|
counted) holds at least container_swap_gb GB of swap, else ok; a host without
|
||||||
|
swap or PSI skips that bound. heavy_commands extend the built-in rules of
|
||||||
|
heavy().
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
@@ -56,6 +66,7 @@ import contextlib
|
|||||||
import fcntl
|
import fcntl
|
||||||
import importlib.util
|
import importlib.util
|
||||||
import json
|
import json
|
||||||
|
import math
|
||||||
import os
|
import os
|
||||||
import re
|
import re
|
||||||
import shlex
|
import shlex
|
||||||
@@ -99,6 +110,86 @@ CLIP = 1000
|
|||||||
CLEANUP_LOCK = "cleanup.lock"
|
CLEANUP_LOCK = "cleanup.lock"
|
||||||
LOCK_TTL = 1800
|
LOCK_TTL = 1800
|
||||||
GB = 1e9
|
GB = 1e9
|
||||||
|
MEMINFO = Path("/proc/meminfo")
|
||||||
|
PRESSURE = Path("/proc/pressure/memory")
|
||||||
|
CGROUPS = Path("/sys/fs/cgroup/system.slice")
|
||||||
|
SWAP_HOLDERS = 3
|
||||||
|
NEST_DEPTH = 8
|
||||||
|
MEMORY_STAGES = ("ok", "warn", "stop")
|
||||||
|
MEMORY_BOUNDS = ("available", "swap_free", "psi_full", "container_swap_gb")
|
||||||
|
MEMORY_KEYS = tuple(f"memory_limits.{stage}.{bound}" for stage in MEMORY_STAGES[1:] for bound in MEMORY_BOUNDS)
|
||||||
|
CONTAINER_ID = re.compile(r"[0-9a-f]{64}")
|
||||||
|
COMPOSE_VALUES = frozenset({
|
||||||
|
"-f", "--file", "-p", "--project-name", "--project-directory", "--env-file", "--profile", "--ansi", "--parallel",
|
||||||
|
"--progress",
|
||||||
|
})
|
||||||
|
BUILDING = frozenset({"up", "run", "create"})
|
||||||
|
SEPARATOR = re.compile(r"[();<>|&\n]+")
|
||||||
|
PUNCTUATION = "();<>|&\n"
|
||||||
|
WORD_BREAK = " \t\n;&|()<>"
|
||||||
|
ASSIGNMENT = re.compile(r"[A-Za-z_]\w*=.*", re.DOTALL)
|
||||||
|
WRAPPERS = frozenset({
|
||||||
|
"env", "sudo", "nice", "nohup", "timeout", "exec", "xargs", "stdbuf", "ionice", "uv", "uvx", "poetry", "pdm",
|
||||||
|
"pipenv", "coverage", "time", "command",
|
||||||
|
})
|
||||||
|
UV_VALUES = frozenset({
|
||||||
|
"--with", "--extra", "--group", "--python", "-p", "--project", "--directory", "--package", "--env-file", "--from",
|
||||||
|
})
|
||||||
|
WRAPPER_VALUES = {
|
||||||
|
"sudo": frozenset({
|
||||||
|
"-u", "-g", "-h", "-p", "-C", "-D", "-r", "-t", "-U", "-T", "--user", "--group", "--host", "--chdir", "--role",
|
||||||
|
"--type", "--prompt", "--other-user", "--command-timeout", "--close-from",
|
||||||
|
}),
|
||||||
|
"timeout": frozenset({"-s", "-k", "--signal", "--kill-after"}),
|
||||||
|
"env": frozenset({"-u", "-C", "--unset", "--chdir"}),
|
||||||
|
"nice": frozenset({"-n", "--adjustment"}),
|
||||||
|
"ionice": frozenset({"-c", "-n", "-p", "--class", "--classdata"}),
|
||||||
|
"xargs": frozenset({"-a", "-d", "-E", "-I", "-L", "-n", "-P", "-s"}),
|
||||||
|
"stdbuf": frozenset({"-i", "-o", "-e"}),
|
||||||
|
"time": frozenset({"-f", "-o", "--format", "--output"}),
|
||||||
|
"exec": frozenset({"-a"}),
|
||||||
|
"uv": UV_VALUES,
|
||||||
|
"uvx": UV_VALUES,
|
||||||
|
"poetry": frozenset({"-C", "-P", "--directory", "--project"}),
|
||||||
|
"pdm": frozenset({"-p", "--project"}),
|
||||||
|
"coverage": frozenset({"--source", "--include", "--omit", "--rcfile", "--context", "--data-file"}),
|
||||||
|
}
|
||||||
|
WRAPPER_WORDS = frozenset({"run", "tool"})
|
||||||
|
RESERVED = frozenset({"!", "{", "if", "then", "else", "elif", "do", "while", "until"})
|
||||||
|
WRAPPER_ARGUMENT = re.compile(r"-.*|\d+(\.\d+)?[smhd]?", re.DOTALL)
|
||||||
|
SHELLS = frozenset({"bash", "sh", "zsh", "dash"})
|
||||||
|
INLINE = re.compile(r"-[A-Za-z]*c[A-Za-z]*")
|
||||||
|
REDIRECT = re.compile(r"&?[<>]+&?")
|
||||||
|
TRAILING_REDIRECT = re.compile(r"[<>]+&?$")
|
||||||
|
PAREN = {"(": 1, ")": -1}
|
||||||
|
PYTHON = re.compile(r"python[\d.]*")
|
||||||
|
PYTHON_VALUE_OPTIONS = frozenset({"-X", "-W", "-Q"})
|
||||||
|
DRY_RUN = frozenset({"-n", "--dry-run", "--just-print", "--recon", "-q", "--question"})
|
||||||
|
SHORT_DRY = re.compile(r"-(?:(?![CfIoOWjl])[A-Za-z])*[nq][A-Za-z]*")
|
||||||
|
SHELL_VALUE = re.compile(r"[-+][A-Za-z]*[oO]|--rcfile|--init-file")
|
||||||
|
STDIN_FLAG = re.compile(r"-[A-Za-z]*s[A-Za-z]*")
|
||||||
|
SPACE = re.compile(r"\s")
|
||||||
|
ESCAPED = re.compile(r"\\[`$]")
|
||||||
|
OPENER = re.compile(r"`|\$\(")
|
||||||
|
PLACEHOLDER = "-_"
|
||||||
|
HEREDOC_MARK = "kvbheredoc"
|
||||||
|
MARK_WORD = re.compile(rf"{HEREDOC_MARK}((?:[0-9a-f]{{2}})*)")
|
||||||
|
HELP = frozenset({"-h", "--help", "--version"})
|
||||||
|
PYTEST_LIGHT = HELP | {"--collect-only", "--co"}
|
||||||
|
TEST_FILE = re.compile(r"[^*?\[\]]+\.py(::.*)?", re.DOTALL)
|
||||||
|
PYTEST_VALUE_OPTIONS = frozenset({
|
||||||
|
"-p", "-k", "-m", "-c", "-o", "-W", "-n", "--numprocesses", "-r", "--maxfail", "--tb", "--rootdir", "--basetemp", "--deselect",
|
||||||
|
"--ignore", "--ignore-glob", "--confcutdir", "--durations", "--junitxml", "--log-level", "--dist", "--cov",
|
||||||
|
"--cov-report", "--cov-config", "--timeout", "--import-mode", "--capture", "--color", "--log-cli-level",
|
||||||
|
"--log-file", "--reruns", "--tx", "--max-worker-restart",
|
||||||
|
})
|
||||||
|
UNITTEST_VALUE_OPTIONS = frozenset({"-k"})
|
||||||
|
DISCOVER_VALUE_OPTIONS = frozenset({"-s", "-p", "-t", "-k", "--start-directory", "--pattern", "--top-level-directory"})
|
||||||
|
PATTERN_OPTIONS = frozenset({"-p", "--pattern"})
|
||||||
|
MAKE_VALUE_OPTIONS = frozenset({
|
||||||
|
"-C", "-f", "-I", "-o", "-W", "--directory", "--file", "--makefile", "--include-dir", "--old-file", "--assume-old",
|
||||||
|
"--what-if", "--new-file", "--assume-new",
|
||||||
|
})
|
||||||
|
|
||||||
|
|
||||||
def guard_root(hook: bool) -> int | None:
|
def guard_root(hook: bool) -> int | None:
|
||||||
@@ -123,6 +214,35 @@ def load_config() -> dict | None:
|
|||||||
return json.loads(CONFIG.read_text(encoding="utf-8"))
|
return json.loads(CONFIG.read_text(encoding="utf-8"))
|
||||||
|
|
||||||
|
|
||||||
|
def absent_part(cfg: dict, key: str) -> str | None:
|
||||||
|
node: object = cfg
|
||||||
|
parts = key.split(".")
|
||||||
|
for depth, part in enumerate(parts, 1):
|
||||||
|
if not isinstance(node, dict) or part not in node:
|
||||||
|
return ".".join(parts[:depth])
|
||||||
|
node = node[part]
|
||||||
|
return None if isinstance(node, (int, float)) and not isinstance(node, bool) and math.isfinite(node) else key
|
||||||
|
|
||||||
|
|
||||||
|
def missing(cfg: dict, keys: tuple[str, ...]) -> str | None:
|
||||||
|
"""Args:
|
||||||
|
cfg: the config.
|
||||||
|
keys: required number keys, nested ones dotted
|
||||||
|
(`memory_limits.stop.psi_full`).
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
None when every key holds a number, else the error naming the
|
||||||
|
shortest missing part of each, or the key whose value is no number.
|
||||||
|
"""
|
||||||
|
absent = list(dict.fromkeys(part for key in keys if (part := absent_part(cfg, key)) is not None))
|
||||||
|
if not absent:
|
||||||
|
return None
|
||||||
|
return (
|
||||||
|
f"agents council: {CONFIG} lacks {', '.join(absent)}; add them as numbers as the watch.py and queue.py "
|
||||||
|
"docstrings document"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def disk(cfg: dict) -> tuple[float, float]:
|
def disk(cfg: dict) -> tuple[float, float]:
|
||||||
st = os.statvfs(cfg["disk"])
|
st = os.statvfs(cfg["disk"])
|
||||||
return st.f_bavail * st.f_frsize / GB, st.f_blocks * st.f_frsize / GB
|
return st.f_bavail * st.f_frsize / GB, st.f_blocks * st.f_frsize / GB
|
||||||
@@ -237,6 +357,19 @@ def act(cfg: dict, stage: str, previous: str, free: float) -> None:
|
|||||||
notify(cfg, f"Agents council: {head}. Heavy jobs blocked; a running stack needs the operator's OK to go.")
|
notify(cfg, f"Agents council: {head}. Heavy jobs blocked; a running stack needs the operator's OK to go.")
|
||||||
|
|
||||||
|
|
||||||
|
def act_memory(cfg: dict, stage: str, previous: str, sample: dict[str, float | None]) -> None:
|
||||||
|
head = memory_text(sample)
|
||||||
|
if MEMORY_STAGES.index(stage) < MEMORY_STAGES.index(previous):
|
||||||
|
alert(stage, f"RESOURCE MEMORY {stage.upper()}: recovered, {head}.")
|
||||||
|
return
|
||||||
|
holders = "; ".join(swap_holders(CGROUPS, container_names())) or "no container holds swap"
|
||||||
|
if stage == "warn":
|
||||||
|
alert(stage, f"RESOURCE MEMORY WARN: {head}. Top swap holders: {holders}. Chair: find the holder; peers: start nothing heavy.")
|
||||||
|
else:
|
||||||
|
alert(stage, f"RESOURCE MEMORY STOP: {head}. New heavy jobs are blocked. Top swap holders: {holders}.")
|
||||||
|
notify(cfg, f"Agents council: memory nearly exhausted ({head}). Heavy jobs blocked; top swap holders: {holders}.")
|
||||||
|
|
||||||
|
|
||||||
def history_rows(text: str) -> list[tuple[str, str]]:
|
def history_rows(text: str) -> list[tuple[str, str]]:
|
||||||
rows = []
|
rows = []
|
||||||
for line in text.splitlines():
|
for line in text.splitlines():
|
||||||
@@ -280,27 +413,42 @@ def stale(state: dict, cfg: dict, now: float) -> bool:
|
|||||||
return now - state.get("checked", 0) >= cfg["interval_seconds"]
|
return now - state.get("checked", 0) >= cfg["interval_seconds"]
|
||||||
|
|
||||||
|
|
||||||
def cmd_watch(cfg: dict) -> int:
|
def measure_and_act(cfg: dict, state: dict, now: float, problem: str | None) -> None:
|
||||||
|
free, _ = disk(cfg)
|
||||||
|
samples = [s for s in state.get("samples", []) if now - s[0] <= cfg["projection_minutes"] * 60]
|
||||||
|
stage = stage_for(free, samples, cfg, now)
|
||||||
|
previous = state.get("stage", "ok")
|
||||||
|
samples.append([now, free])
|
||||||
|
state.update(checked=now, free=free, stage=stage, samples=samples)
|
||||||
|
sample = None if problem else memory()
|
||||||
|
was_short = state.get("memory_stage", "ok")
|
||||||
|
if sample is not None:
|
||||||
|
state["memory_stage"] = memory_stage(sample, cfg)
|
||||||
|
due = now - state.get("caches_checked", 0) >= cfg["cache_interval_seconds"]
|
||||||
|
if due:
|
||||||
|
state["caches_checked"] = now
|
||||||
|
write_json(STATE, state)
|
||||||
|
if stage != previous:
|
||||||
|
act(cfg, stage, previous, free)
|
||||||
|
elif due:
|
||||||
|
spawn_cleanup("--caches")
|
||||||
|
if sample is not None and state["memory_stage"] != was_short:
|
||||||
|
act_memory(cfg, state["memory_stage"], was_short, sample)
|
||||||
|
|
||||||
|
|
||||||
|
def cmd_watch(cfg: dict, problem: str | None) -> int:
|
||||||
|
"""Args:
|
||||||
|
cfg: the config.
|
||||||
|
problem: why the memory stage cannot be judged, None when it can; the
|
||||||
|
disk part still runs and every session sees the problem.
|
||||||
|
"""
|
||||||
payload = json.load(sys.stdin)
|
payload = json.load(sys.stdin)
|
||||||
now = time.time()
|
now = time.time()
|
||||||
state = read_json(STATE)
|
state = read_json(STATE)
|
||||||
if stale(state, cfg, now):
|
if stale(state, cfg, now):
|
||||||
free, _ = disk(cfg)
|
measure_and_act(cfg, state, now, problem)
|
||||||
samples = [s for s in state.get("samples", []) if now - s[0] <= cfg["projection_minutes"] * 60]
|
|
||||||
stage = stage_for(free, samples, cfg, now)
|
|
||||||
previous = state.get("stage", "ok")
|
|
||||||
samples.append([now, free])
|
|
||||||
state.update(checked=now, free=free, stage=stage, samples=samples)
|
|
||||||
due = now - state.get("caches_checked", 0) >= cfg["cache_interval_seconds"]
|
|
||||||
if due:
|
|
||||||
state["caches_checked"] = now
|
|
||||||
write_json(STATE, state)
|
|
||||||
if stage != previous:
|
|
||||||
act(cfg, stage, previous, free)
|
|
||||||
elif due:
|
|
||||||
spawn_cleanup("--caches")
|
|
||||||
keep_history()
|
keep_history()
|
||||||
texts = unseen_alerts(payload["session_id"])
|
texts = unseen_alerts(payload["session_id"]) + ([] if problem is None else [problem])
|
||||||
if texts:
|
if texts:
|
||||||
print(json.dumps({"hookSpecificOutput": {
|
print(json.dumps({"hookSpecificOutput": {
|
||||||
"hookEventName": payload["hook_event_name"],
|
"hookEventName": payload["hook_event_name"],
|
||||||
@@ -309,7 +457,10 @@ def cmd_watch(cfg: dict) -> int:
|
|||||||
return 0
|
return 0
|
||||||
|
|
||||||
|
|
||||||
HEREDOC = re.compile(r"<<-?\s*(['\"]?)(\w+)\1[^\n]*\n.*?\n\s*\2(?=\s|\)|$)", re.DOTALL)
|
HEREDOC = re.compile(
|
||||||
|
r"<<-?\s*(?P<quote>['\"]?)(?P<escape>\\?)(?P<tag>[\w.-]+)(?P=quote)(?P<rest>[^\n]*)\n(?P<body>.*?)\n\s*(?P=tag)(?=\s|\)|$)",
|
||||||
|
re.DOTALL,
|
||||||
|
)
|
||||||
QUOTED = re.compile(r"'[^']*'|\"(?:\\.|[^\"\\])*\"")
|
QUOTED = re.compile(r"'[^']*'|\"(?:\\.|[^\"\\])*\"")
|
||||||
|
|
||||||
|
|
||||||
@@ -322,32 +473,369 @@ def silence() -> dict | None:
|
|||||||
return state if state.get("until", 0) > time.time() else None
|
return state if state.get("until", 0) > time.time() else None
|
||||||
|
|
||||||
|
|
||||||
def cmd_block(cfg: dict) -> int:
|
def uncommented(command: str) -> str:
|
||||||
|
kept: list[str] = []
|
||||||
|
quote = ""
|
||||||
|
escaped = skipping = False
|
||||||
|
word_start = True
|
||||||
|
for char in command:
|
||||||
|
if skipping and char != "\n":
|
||||||
|
continue
|
||||||
|
skipping = False
|
||||||
|
if escaped:
|
||||||
|
escaped = False
|
||||||
|
elif quote == "'":
|
||||||
|
quote = "" if char == "'" else quote
|
||||||
|
elif char == "\\":
|
||||||
|
escaped = True
|
||||||
|
elif quote == '"':
|
||||||
|
quote = "" if char == '"' else quote
|
||||||
|
elif char in "'\"":
|
||||||
|
quote = char
|
||||||
|
elif char == "#" and word_start:
|
||||||
|
skipping = True
|
||||||
|
continue
|
||||||
|
kept.append(char)
|
||||||
|
word_start = not quote and not escaped and char in WORD_BREAK
|
||||||
|
return "".join(kept)
|
||||||
|
|
||||||
|
|
||||||
|
def segments(command: str) -> list[list[str]]:
|
||||||
|
"""Args:
|
||||||
|
command: a shell command line.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
The words of each simple command, split at newlines, `;`, `&`, `|`
|
||||||
|
and parentheses; a redirection, its target and a file descriptor
|
||||||
|
number in front of it are dropped and the command goes on.
|
||||||
|
|
||||||
|
Raises:
|
||||||
|
ValueError: on an unbalanced quote.
|
||||||
|
"""
|
||||||
|
lexer = shlex.shlex(uncommented(command.replace("\\\n", "")), posix=True, punctuation_chars=PUNCTUATION)
|
||||||
|
lexer.whitespace = " \t\r"
|
||||||
|
lexer.whitespace_split = True
|
||||||
|
lexer.commenters = ""
|
||||||
|
parts: list[list[str]] = [[]]
|
||||||
|
target = False
|
||||||
|
for token in lexer:
|
||||||
|
if REDIRECT.fullmatch(token):
|
||||||
|
if parts[-1] and parts[-1][-1].isdigit():
|
||||||
|
parts[-1].pop()
|
||||||
|
target = True
|
||||||
|
elif SEPARATOR.fullmatch(token):
|
||||||
|
parts.append([])
|
||||||
|
target = TRAILING_REDIRECT.search(token) is not None
|
||||||
|
elif target:
|
||||||
|
target = False
|
||||||
|
else:
|
||||||
|
parts[-1].append(token)
|
||||||
|
return parts
|
||||||
|
|
||||||
|
|
||||||
|
def program(words: list[str]) -> list[str]:
|
||||||
|
"""Returns:
|
||||||
|
The words from the program on: shell reserved words, VAR=value,
|
||||||
|
wrappers such as env, sudo, time, timeout or `uv run`, their options,
|
||||||
|
the values of their options and their numbers dropped; the split
|
||||||
|
string of `env -S`, nothing for the lookup `command -v`.
|
||||||
|
"""
|
||||||
|
at, wrapper = 0, ""
|
||||||
|
while at < len(words):
|
||||||
|
word = words[at]
|
||||||
|
if os.path.basename(word) in WRAPPERS:
|
||||||
|
wrapper = os.path.basename(word)
|
||||||
|
elif word in WRAPPER_VALUES.get(wrapper, ()):
|
||||||
|
at += 1
|
||||||
|
elif word in WRAPPER_TURNS.get(wrapper, {}):
|
||||||
|
return WRAPPER_TURNS[wrapper][word](words[at + 1 :])
|
||||||
|
elif not (word in RESERVED or ASSIGNMENT.fullmatch(word) or (wrapper and wrapped_word(word))):
|
||||||
|
break
|
||||||
|
at += 1
|
||||||
|
return words[at:]
|
||||||
|
|
||||||
|
|
||||||
|
def lookup(rest: list[str]) -> list[str]:
|
||||||
|
return []
|
||||||
|
|
||||||
|
|
||||||
|
def split_string(rest: list[str]) -> list[str]:
|
||||||
|
return program(" ".join(rest[:1]).split() + rest[1:])
|
||||||
|
|
||||||
|
|
||||||
|
WRAPPER_TURNS = {
|
||||||
|
"command": {"-v": lookup, "-V": lookup},
|
||||||
|
"env": {"-S": split_string, "--split-string": split_string},
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def wrapped_word(word: str) -> bool:
|
||||||
|
return word in WRAPPER_WORDS or WRAPPER_ARGUMENT.fullmatch(word) is not None
|
||||||
|
|
||||||
|
|
||||||
|
def inline_code(words: list[str]) -> str | None:
|
||||||
|
"""Args:
|
||||||
|
words: a simple command from its program on.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
The code a shell runs with -c (also clustered, such as -lc or -ic,
|
||||||
|
and after a `--`), else None.
|
||||||
|
"""
|
||||||
|
if not words or os.path.basename(words[0]) not in SHELLS:
|
||||||
|
return None
|
||||||
|
rest = words[1:]
|
||||||
|
for at, word in enumerate(rest):
|
||||||
|
if INLINE.fullmatch(word):
|
||||||
|
code = [later for later in rest[at + 1 :] if later != "--"][:1]
|
||||||
|
return code[0] if code else None
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def operands(args: list[str], value_options: frozenset[str]) -> list[str]:
|
||||||
|
found, skip = [], False
|
||||||
|
for word in args:
|
||||||
|
if skip:
|
||||||
|
skip = False
|
||||||
|
elif word in value_options:
|
||||||
|
skip = True
|
||||||
|
elif not word.startswith("-"):
|
||||||
|
found.append(word)
|
||||||
|
return found
|
||||||
|
|
||||||
|
|
||||||
|
def python_module(args: list[str]) -> tuple[str, list[str]]:
|
||||||
|
skip = False
|
||||||
|
for at, word in enumerate(args):
|
||||||
|
if skip:
|
||||||
|
skip = False
|
||||||
|
elif word == "-m":
|
||||||
|
return (args[at + 1], args[at + 2 :]) if at + 1 < len(args) else ("", [])
|
||||||
|
elif word.startswith("-m"):
|
||||||
|
return word[2:], args[at + 1 :]
|
||||||
|
elif word in PYTHON_VALUE_OPTIONS:
|
||||||
|
skip = True
|
||||||
|
elif not word.startswith("-"):
|
||||||
|
return os.path.basename(word), args[at + 1 :]
|
||||||
|
return "", []
|
||||||
|
|
||||||
|
|
||||||
|
def light_unittest(args: list[str]) -> bool:
|
||||||
|
if args[:1] != ["discover"]:
|
||||||
|
return len(operands(args, UNITTEST_VALUE_OPTIONS)) == 1
|
||||||
|
rest = args[1:]
|
||||||
|
patterns = [rest[at + 1] for at, word in enumerate(rest[:-1]) if word in PATTERN_OPTIONS]
|
||||||
|
patterns = patterns or operands(rest, DISCOVER_VALUE_OPTIONS)[1:2]
|
||||||
|
return bool(patterns) and TEST_FILE.fullmatch(patterns[-1]) is not None
|
||||||
|
|
||||||
|
|
||||||
|
def heavy_pytest(args: list[str]) -> bool:
|
||||||
|
files = operands(args, PYTEST_VALUE_OPTIONS)
|
||||||
|
return not (PYTEST_LIGHT.intersection(args) or (len(files) == 1 and TEST_FILE.fullmatch(files[0])))
|
||||||
|
|
||||||
|
|
||||||
|
def heavy_unittest(args: list[str]) -> bool:
|
||||||
|
return not (HELP.intersection(args) or light_unittest(args))
|
||||||
|
|
||||||
|
|
||||||
|
def make_targets(args: list[str]) -> list[str]:
|
||||||
|
return [word for word in operands(args, MAKE_VALUE_OPTIONS) if "=" not in word and not word.isdigit()]
|
||||||
|
|
||||||
|
|
||||||
|
def dry_run(args: list[str]) -> bool:
|
||||||
|
return bool(DRY_RUN.intersection(args)) or any(SHORT_DRY.fullmatch(word) for word in args)
|
||||||
|
|
||||||
|
|
||||||
|
def heavy_make(args: list[str]) -> bool:
|
||||||
|
return not dry_run(args) and any(target == "install" or target.startswith("verify-") for target in make_targets(args))
|
||||||
|
|
||||||
|
|
||||||
|
def heavy_compose(args: list[str]) -> bool:
|
||||||
|
words = operands(args, COMPOSE_VALUES)
|
||||||
|
command = words[0] if words else ""
|
||||||
|
return command == "build" or (command in BUILDING and "--build" in args)
|
||||||
|
|
||||||
|
|
||||||
|
def heavy_docker(args: list[str]) -> bool:
|
||||||
|
return args[:1] == ["compose"] and heavy_compose(args[1:])
|
||||||
|
|
||||||
|
|
||||||
|
HEAVY_TOOLS = {
|
||||||
|
"pytest": heavy_pytest, "py.test": heavy_pytest, "unittest": heavy_unittest, "make": heavy_make,
|
||||||
|
"docker": heavy_docker, "docker-compose": heavy_compose,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def heavy_words(run: list[str], cfg: dict, exempt: tuple[str, ...]) -> bool:
|
||||||
|
if not run:
|
||||||
|
return False
|
||||||
|
code = inline_code(run)
|
||||||
|
if code is not None:
|
||||||
|
return heavy(code, cfg, exempt)
|
||||||
|
name, args = os.path.basename(run[0]), run[1:]
|
||||||
|
if PYTHON.fullmatch(name):
|
||||||
|
name, args = python_module(args)
|
||||||
|
if name in WRAPPERS:
|
||||||
|
return heavy_words(program([name, *args]), cfg, exempt)
|
||||||
|
return name in HEAVY_TOOLS and HEAVY_TOOLS[name](args)
|
||||||
|
|
||||||
|
|
||||||
|
def segment_texts(words: list[str], run: list[str]) -> list[str]:
|
||||||
|
return list(dict.fromkeys(" ".join("''" if SPACE.search(word) else word for word in part) for part in (words, run)))
|
||||||
|
|
||||||
|
|
||||||
|
def judged(texts: list[str], run: list[str], cfg: dict, exempt: tuple[str, ...]) -> bool:
|
||||||
|
if any(re.search(pattern, text) for pattern in exempt for text in texts):
|
||||||
|
return False
|
||||||
|
if any(re.search(pattern, text) for pattern in cfg["heavy_commands"] for text in texts):
|
||||||
|
return True
|
||||||
|
return heavy_words(run, cfg, exempt)
|
||||||
|
|
||||||
|
|
||||||
|
def heavy_part(words: list[str], cfg: dict, exempt: tuple[str, ...]) -> bool:
|
||||||
|
"""Returns:
|
||||||
|
Whether one simple command is heavy; a make call is judged per target
|
||||||
|
as `make <target>`, so an exempt target never covers the next one.
|
||||||
|
"""
|
||||||
|
run = program(words)
|
||||||
|
if run and os.path.basename(run[0]) == "make" and not dry_run(run[1:]):
|
||||||
|
return any(judged([f"make {target}"], ["make", target], cfg, exempt) for target in make_targets(run[1:]))
|
||||||
|
return judged(segment_texts(words, run), run, cfg, exempt)
|
||||||
|
|
||||||
|
|
||||||
|
def marked_heredocs(command: str) -> tuple[str, list[re.Match[str]]]:
|
||||||
|
"""Returns:
|
||||||
|
The command with each heredoc replaced by the word HEREDOC_MARK
|
||||||
|
followed by its body in hex, so a nested call still reads it, the
|
||||||
|
rest of its first line kept, and the heredoc matches.
|
||||||
|
"""
|
||||||
|
found: list[re.Match[str]] = []
|
||||||
|
|
||||||
|
def mark(match: re.Match[str]) -> str:
|
||||||
|
found.append(match)
|
||||||
|
return f" {HEREDOC_MARK}{match['body'].encode(errors='surrogatepass').hex()} {match['rest']}"
|
||||||
|
|
||||||
|
return HEREDOC.sub(mark, command), found
|
||||||
|
|
||||||
|
|
||||||
|
def shell_operands(args: list[str]) -> list[str]:
|
||||||
|
found, skip = [], False
|
||||||
|
for word in args:
|
||||||
|
if skip:
|
||||||
|
skip = False
|
||||||
|
elif word.startswith(("-", "+")):
|
||||||
|
skip = SHELL_VALUE.fullmatch(word) is not None
|
||||||
|
else:
|
||||||
|
found.append(word)
|
||||||
|
return found
|
||||||
|
|
||||||
|
|
||||||
|
def reads_stdin(run: list[str]) -> bool:
|
||||||
|
"""Returns:
|
||||||
|
Whether a simple command from its program on is a shell that runs the
|
||||||
|
code on its stdin: no -c code, and -s or no operand but heredoc marks.
|
||||||
|
"""
|
||||||
|
if not run or os.path.basename(run[0]) not in SHELLS or inline_code(run) is not None:
|
||||||
|
return False
|
||||||
|
args = run[1:]
|
||||||
|
return any(STDIN_FLAG.fullmatch(word) for word in args) or all(MARK_WORD.fullmatch(word) for word in shell_operands(args))
|
||||||
|
|
||||||
|
|
||||||
|
def expanded_codes(heredocs: list[re.Match[str]]) -> list[str]:
|
||||||
|
"""Returns:
|
||||||
|
The command substitutions in the bodies of heredocs whose delimiter
|
||||||
|
is unquoted, which the shell expands.
|
||||||
|
"""
|
||||||
|
return [code for match in heredocs if not (match["quote"] or match["escape"]) for code in substituted(match["body"])[1]]
|
||||||
|
|
||||||
|
|
||||||
|
def fed_bodies(parts: list[list[str]]) -> list[str]:
|
||||||
|
marks = [MARK_WORD.fullmatch(word) for words in parts if reads_stdin(program(words)) for word in words]
|
||||||
|
return [bytes.fromhex(mark[1]).decode(errors="ignore") for mark in marks if mark]
|
||||||
|
|
||||||
|
|
||||||
|
def closing(text: str, start: int) -> int:
|
||||||
|
depth = 1
|
||||||
|
for at in range(start, len(text)):
|
||||||
|
depth += PAREN.get(text[at], 0)
|
||||||
|
if depth == 0:
|
||||||
|
return at
|
||||||
|
return len(text)
|
||||||
|
|
||||||
|
|
||||||
|
def blanked_single(quoted: re.Match[str]) -> str:
|
||||||
|
text = quoted[0]
|
||||||
|
return text if text.startswith('"') else "'" + " " * (len(text) - 2) + "'"
|
||||||
|
|
||||||
|
|
||||||
|
def substitution_end(shown: str, found: re.Match[str]) -> int:
|
||||||
|
if found[0] == "`":
|
||||||
|
end = shown.find("`", found.end())
|
||||||
|
return len(shown) if end < 0 else end
|
||||||
|
return closing(shown, found.end())
|
||||||
|
|
||||||
|
|
||||||
|
def substituted(text: str) -> tuple[str, list[str]]:
|
||||||
|
"""Returns:
|
||||||
|
The text with every `...` and $(...) outside single quotes and
|
||||||
|
escapes replaced by the option-like word PLACEHOLDER, so a wrapper
|
||||||
|
around it still parses, and the code of each, which the shell runs
|
||||||
|
first; nested ones stay inside their code.
|
||||||
|
"""
|
||||||
|
shown = QUOTED.sub(blanked_single, ESCAPED.sub(" ", text))
|
||||||
|
kept: list[str] = []
|
||||||
|
codes: list[str] = []
|
||||||
|
at = 0
|
||||||
|
found = OPENER.search(shown)
|
||||||
|
while found:
|
||||||
|
end = substitution_end(shown, found)
|
||||||
|
kept.append(text[at : found.start()] + PLACEHOLDER)
|
||||||
|
codes.append(text[found.end() : end])
|
||||||
|
at = end + 1
|
||||||
|
found = OPENER.search(shown, at)
|
||||||
|
return "".join(kept) + text[at:], codes
|
||||||
|
|
||||||
|
|
||||||
|
def cmd_block(cfg: dict, problem: str | None) -> int:
|
||||||
|
"""Args:
|
||||||
|
cfg: the config.
|
||||||
|
problem: why the memory stage cannot be judged, None when it can; a
|
||||||
|
heavy job is then refused with it.
|
||||||
|
"""
|
||||||
payload = json.load(sys.stdin)
|
payload = json.load(sys.stdin)
|
||||||
command = executable_text((payload.get("tool_input") or {}).get("command", ""))
|
raw = (payload.get("tool_input") or {}).get("command", "")
|
||||||
|
if not heavy(raw, cfg, tuple(cfg["never_block"])):
|
||||||
|
return 0
|
||||||
|
reason = block_reason(cfg, problem)
|
||||||
|
if reason is None:
|
||||||
|
return 0
|
||||||
|
print(f"kvb-org-organize: {reason}. Teardown commands still run. Ask the resource chair.", file=sys.stderr)
|
||||||
|
return 2
|
||||||
|
|
||||||
|
|
||||||
|
def disk_stage(cfg: dict) -> str:
|
||||||
state = read_json(STATE)
|
state = read_json(STATE)
|
||||||
now = time.time()
|
now = time.time()
|
||||||
stage = state.get("stage", "ok")
|
|
||||||
if stale(state, cfg, now):
|
if stale(state, cfg, now):
|
||||||
stage = stage_for(disk(cfg)[0], state.get("samples", []), cfg, now)
|
return stage_for(disk(cfg)[0], state.get("samples", []), cfg, now)
|
||||||
|
return state.get("stage", "ok")
|
||||||
|
|
||||||
|
|
||||||
|
def block_reason(cfg: dict, problem: str | None) -> str | None:
|
||||||
mayday = silence()
|
mayday = silence()
|
||||||
if stage not in ("stop", "critical") and not mayday:
|
if mayday:
|
||||||
return 0
|
return (
|
||||||
if any(re.search(p, command) for p in cfg["never_block"]):
|
|
||||||
return 0
|
|
||||||
if any(re.search(p, command) for p in cfg["heavy_commands"]):
|
|
||||||
reason = (
|
|
||||||
f"a mayday by {clip(mayday.get('by', '?'))} holds radio silence: {clip(mayday.get('text', ''))}; "
|
f"a mayday by {clip(mayday.get('by', '?'))} holds radio silence: {clip(mayday.get('text', ''))}; "
|
||||||
"it ends with radio.py clear"
|
"it ends with radio.py clear"
|
||||||
if mayday
|
|
||||||
else f"resource stage is {stage}; new heavy jobs are blocked until the disk recovers"
|
|
||||||
)
|
)
|
||||||
print(
|
stage = disk_stage(cfg)
|
||||||
f"kvb-org-organize: {reason}. Teardown commands still run. Ask the resource chair.",
|
if stage in ("stop", "critical"):
|
||||||
file=sys.stderr,
|
return f"resource stage is {stage}; new heavy jobs are blocked until the disk recovers"
|
||||||
)
|
if problem is not None:
|
||||||
return 2
|
return f"{problem}; new heavy jobs are blocked until it does"
|
||||||
return 0
|
sample = memory()
|
||||||
|
if memory_stage(sample, cfg) == "stop":
|
||||||
|
return f"memory stage is stop ({memory_text(sample)}); new heavy jobs are blocked until memory recovers"
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
def cpu_times() -> tuple[int, int]:
|
def cpu_times() -> tuple[int, int]:
|
||||||
@@ -361,12 +849,17 @@ def cpu_busy_percent(before: tuple[int, int], after: tuple[int, int]) -> float:
|
|||||||
return 100.0 * (total - idle) / total
|
return 100.0 * (total - idle) / total
|
||||||
|
|
||||||
|
|
||||||
def mem_used_percent(meminfo: str) -> float:
|
def meminfo_kb(meminfo: str) -> dict[str, int]:
|
||||||
values = {}
|
values = {}
|
||||||
for line in meminfo.splitlines():
|
for line in meminfo.splitlines():
|
||||||
key, _, rest = line.partition(":")
|
key, _, rest = line.partition(":")
|
||||||
if rest:
|
if rest:
|
||||||
values[key] = int(rest.split()[0])
|
values[key] = int(rest.split()[0])
|
||||||
|
return values
|
||||||
|
|
||||||
|
|
||||||
|
def mem_used_percent(meminfo: str) -> float:
|
||||||
|
values = meminfo_kb(meminfo)
|
||||||
return 100.0 * (1 - values["MemAvailable"] / values["MemTotal"])
|
return 100.0 * (1 - values["MemAvailable"] / values["MemTotal"])
|
||||||
|
|
||||||
|
|
||||||
@@ -374,7 +867,161 @@ def load(cfg: dict) -> tuple[float, float]:
|
|||||||
before = cpu_times()
|
before = cpu_times()
|
||||||
time.sleep(cfg["load_limits"]["sample_seconds"])
|
time.sleep(cfg["load_limits"]["sample_seconds"])
|
||||||
cpu = cpu_busy_percent(before, cpu_times())
|
cpu = cpu_busy_percent(before, cpu_times())
|
||||||
return cpu, mem_used_percent(Path("/proc/meminfo").read_text(encoding="utf-8"))
|
return cpu, mem_used_percent(MEMINFO.read_text(encoding="utf-8"))
|
||||||
|
|
||||||
|
|
||||||
|
def psi_full(pressure: str) -> float | None:
|
||||||
|
for line in pressure.splitlines():
|
||||||
|
if line.startswith("full "):
|
||||||
|
return float(dict(field.split("=", 1) for field in line.split()[1:])["avg10"])
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def memory_sample(meminfo: str, pressure: str | None, container_swap_gb: float) -> dict[str, float | None]:
|
||||||
|
"""Args:
|
||||||
|
meminfo: the text of /proc/meminfo.
|
||||||
|
pressure: the text of /proc/pressure/memory, None without PSI.
|
||||||
|
container_swap_gb: the swap of the container holding the most, in GB.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
{available, swap_free, psi_full, container_swap_gb}: MemAvailable and
|
||||||
|
SwapFree in percent of their totals, the full avg10 stall percentage
|
||||||
|
and container_swap_gb unchanged; swap_free is None without swap,
|
||||||
|
psi_full None without PSI.
|
||||||
|
"""
|
||||||
|
values = meminfo_kb(meminfo)
|
||||||
|
swap = values["SwapTotal"]
|
||||||
|
return {
|
||||||
|
"available": 100.0 * values["MemAvailable"] / values["MemTotal"],
|
||||||
|
"swap_free": 100.0 * values["SwapFree"] / swap if swap else None,
|
||||||
|
"psi_full": None if pressure is None else psi_full(pressure),
|
||||||
|
"container_swap_gb": container_swap_gb,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def memory() -> dict[str, float | None]:
|
||||||
|
try:
|
||||||
|
pressure = PRESSURE.read_text(encoding="utf-8")
|
||||||
|
except OSError:
|
||||||
|
pressure = None
|
||||||
|
largest = max((size for size, _ in container_swaps(CGROUPS)), default=0)
|
||||||
|
return memory_sample(MEMINFO.read_text(encoding="utf-8"), pressure, largest / GB)
|
||||||
|
|
||||||
|
|
||||||
|
def memory_stage(sample: dict[str, float | None], cfg: dict) -> str:
|
||||||
|
for name in ("stop", "warn"):
|
||||||
|
bound = cfg["memory_limits"][name]
|
||||||
|
if (
|
||||||
|
sample["available"] < bound["available"]
|
||||||
|
or (sample["swap_free"] is not None and sample["swap_free"] < bound["swap_free"])
|
||||||
|
or (sample["psi_full"] is not None and sample["psi_full"] >= bound["psi_full"])
|
||||||
|
or sample["container_swap_gb"] >= bound["container_swap_gb"]
|
||||||
|
):
|
||||||
|
return name
|
||||||
|
return "ok"
|
||||||
|
|
||||||
|
|
||||||
|
def memory_text(sample: dict[str, float | None]) -> str:
|
||||||
|
swap = "no swap" if sample["swap_free"] is None else f"swap free {sample['swap_free']:.0f} %"
|
||||||
|
psi = "no PSI" if sample["psi_full"] is None else f"PSI full avg10 {sample['psi_full']:.1f}"
|
||||||
|
return f"MemAvailable {sample['available']:.0f} %, {swap}, {psi}, largest container swap {sample['container_swap_gb']:.1f} GB"
|
||||||
|
|
||||||
|
|
||||||
|
def swap_bytes(cgroup: Path) -> int:
|
||||||
|
try:
|
||||||
|
return int((cgroup / "memory.swap.current").read_text(encoding="utf-8"))
|
||||||
|
except (OSError, ValueError):
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
def container_swaps(base: Path) -> list[tuple[int, Path]]:
|
||||||
|
try:
|
||||||
|
return [(swap_bytes(scope), scope) for scope in base.glob("docker-*.scope")]
|
||||||
|
except OSError:
|
||||||
|
return []
|
||||||
|
|
||||||
|
|
||||||
|
def cgroup_label(name: str, names: dict[str, str]) -> str:
|
||||||
|
bare = name.removeprefix("docker-").removesuffix(".scope")
|
||||||
|
if CONTAINER_ID.fullmatch(bare):
|
||||||
|
return names[bare] if bare in names else bare[:12]
|
||||||
|
return bare
|
||||||
|
|
||||||
|
|
||||||
|
def container_names() -> dict[str, str]:
|
||||||
|
try:
|
||||||
|
out = subprocess.run(
|
||||||
|
["docker", "ps", "--no-trunc", "--format", "{{.ID}}\t{{.Names}}"],
|
||||||
|
capture_output=True, text=True, timeout=30, check=False,
|
||||||
|
).stdout
|
||||||
|
except (OSError, subprocess.TimeoutExpired):
|
||||||
|
return {}
|
||||||
|
return dict(line.split("\t", 1) for line in out.splitlines() if "\t" in line)
|
||||||
|
|
||||||
|
|
||||||
|
def swap_chain(cgroup: Path, held: int, names: dict[str, str]) -> list[str]:
|
||||||
|
chain = [cgroup_label(cgroup.name, names)]
|
||||||
|
for _ in range(NEST_DEPTH):
|
||||||
|
try:
|
||||||
|
children = [(swap_bytes(child), child) for child in cgroup.iterdir() if child.is_dir()]
|
||||||
|
except OSError:
|
||||||
|
break
|
||||||
|
if not children:
|
||||||
|
break
|
||||||
|
size, child = max(children)
|
||||||
|
if size * 2 < held:
|
||||||
|
break
|
||||||
|
chain.append(cgroup_label(child.name, names))
|
||||||
|
cgroup, held = child, size
|
||||||
|
return chain
|
||||||
|
|
||||||
|
|
||||||
|
def swap_holders(base: Path, names: dict[str, str]) -> list[str]:
|
||||||
|
"""Args:
|
||||||
|
base: the cgroup directory holding the docker-<id>.scope groups.
|
||||||
|
names: {full container id: name} for the labels.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
Up to SWAP_HOLDERS lines, the largest swap first: the container, then
|
||||||
|
each nested group holding at least half of its parent's swap, and the
|
||||||
|
container's memory.swap.current in GB.
|
||||||
|
"""
|
||||||
|
ranked = sorted((pair for pair in container_swaps(base) if pair[0] > 0), reverse=True)[:SWAP_HOLDERS]
|
||||||
|
return [f"{' > '.join(swap_chain(scope, size, names))} {size / GB:.1f} GB" for size, scope in ranked]
|
||||||
|
|
||||||
|
|
||||||
|
def heavy(command: str, cfg: dict, exempt: tuple[str, ...]) -> bool:
|
||||||
|
"""Args:
|
||||||
|
command: the shell command line as typed, quotes included.
|
||||||
|
cfg: the config; its heavy_commands search each simple command with
|
||||||
|
quoted text blanked, and a make call also as `make <targets>`.
|
||||||
|
exempt: regexes; a simple command one of them finds is never heavy
|
||||||
|
(never_block, the queue's own hold), the others still count.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
Whether one simple command is a heavy job: an image build through
|
||||||
|
docker compose (build, or up, run or create with --build), a
|
||||||
|
heavy_commands match, make install or verify-* (not as a dry run), or
|
||||||
|
pytest or unittest on anything but exactly one test file or module,
|
||||||
|
help and version aside. Wrappers (program()), the code of `sh -c`,
|
||||||
|
command substitutions (substituted(), also in heredocs the shell
|
||||||
|
expands) and heredocs a shell reads as its code count the same. A
|
||||||
|
command with an unbalanced quote, which no shell runs, counts by the
|
||||||
|
regexes on its whole text alone. A guard against accidental heavy
|
||||||
|
runs, not a sandbox: a shell reading code from a pipe or a
|
||||||
|
here-string, a script of its own, a program named by a substitution,
|
||||||
|
quotes inside an expanding heredoc and an escaped backslash in front
|
||||||
|
of a substitution are not looked into.
|
||||||
|
"""
|
||||||
|
marked, heredocs = marked_heredocs(command)
|
||||||
|
plain, codes = substituted(uncommented(marked))
|
||||||
|
try:
|
||||||
|
parts = segments(plain)
|
||||||
|
except ValueError:
|
||||||
|
text = executable_text(command)
|
||||||
|
return not any(re.search(p, text) for p in exempt) and any(re.search(p, text) for p in cfg["heavy_commands"])
|
||||||
|
nested = codes + expanded_codes(heredocs) + fed_bodies(parts)
|
||||||
|
return any(heavy_part(words, cfg, exempt) for words in parts) or any(heavy(code, cfg, exempt) for code in nested)
|
||||||
|
|
||||||
|
|
||||||
def cmd_admit(cfg: dict, args: argparse.Namespace) -> int:
|
def cmd_admit(cfg: dict, args: argparse.Namespace) -> int:
|
||||||
@@ -385,12 +1032,12 @@ def cmd_admit(cfg: dict, args: argparse.Namespace) -> int:
|
|||||||
passed = [left >= floor]
|
passed = [left >= floor]
|
||||||
report = [f"free {free:.1f} GB, need {need} GB, leaves {left:.1f} GB (floor {floor} GB)"]
|
report = [f"free {free:.1f} GB, need {need} GB, leaves {left:.1f} GB (floor {floor} GB)"]
|
||||||
limits = cfg["load_limits"]
|
limits = cfg["load_limits"]
|
||||||
if args.kind in limits["kinds"]:
|
cpu, mem = load(cfg)
|
||||||
cpu, mem = load(cfg)
|
sample = memory()
|
||||||
passed += [cpu < limits["cpu_percent"], mem < limits["mem_percent"]]
|
stage = memory_stage(sample, cfg)
|
||||||
report.append(
|
passed += [cpu < limits["cpu_percent"], mem < limits["mem_percent"], stage != "stop"]
|
||||||
f"cpu {cpu:.0f} % (limit {limits['cpu_percent']}), ram {mem:.0f} % (limit {limits['mem_percent']})"
|
report.append(f"cpu {cpu:.0f} % (limit {limits['cpu_percent']}), ram {mem:.0f} % (limit {limits['mem_percent']})")
|
||||||
)
|
report.append(f"memory stage {stage} ({memory_text(sample)})")
|
||||||
verdict = "grant" if all(passed) else "deny"
|
verdict = "grant" if all(passed) else "deny"
|
||||||
print(f"{verdict}: {'; '.join(report)}")
|
print(f"{verdict}: {'; '.join(report)}")
|
||||||
return 0 if verdict == "grant" else 3
|
return 0 if verdict == "grant" else 3
|
||||||
@@ -511,7 +1158,11 @@ def cmd_status(cfg: dict) -> int:
|
|||||||
now = time.time()
|
now = time.time()
|
||||||
samples = state.get("samples", [])
|
samples = state.get("samples", [])
|
||||||
age = f"{now - state['checked']:.0f}s" if "checked" in state else "unmeasured"
|
age = f"{now - state['checked']:.0f}s" if "checked" in state else "unmeasured"
|
||||||
print(f"stage={stage_for(free, samples, cfg, now)} free={free:.1f}GB total={total:.1f}GB samples={len(samples)} age={age}")
|
sample = memory()
|
||||||
|
print(
|
||||||
|
f"stage={stage_for(free, samples, cfg, now)} free={free:.1f}GB total={total:.1f}GB samples={len(samples)} age={age}"
|
||||||
|
f" memory={memory_stage(sample, cfg)} ({memory_text(sample)})"
|
||||||
|
)
|
||||||
return 0
|
return 0
|
||||||
|
|
||||||
|
|
||||||
@@ -537,10 +1188,14 @@ def main() -> int:
|
|||||||
cfg = load_config()
|
cfg = load_config()
|
||||||
if cfg is None:
|
if cfg is None:
|
||||||
return 0
|
return 0
|
||||||
|
problem = missing(cfg, MEMORY_KEYS)
|
||||||
if args.cmd == "watch":
|
if args.cmd == "watch":
|
||||||
return cmd_watch(cfg)
|
return cmd_watch(cfg, problem)
|
||||||
if args.cmd == "block":
|
if args.cmd == "block":
|
||||||
return cmd_block(cfg)
|
return cmd_block(cfg, problem)
|
||||||
|
if problem is not None:
|
||||||
|
print(problem, file=sys.stderr)
|
||||||
|
return 2
|
||||||
if args.cmd == "admit":
|
if args.cmd == "admit":
|
||||||
return cmd_admit(cfg, args)
|
return cmd_admit(cfg, args)
|
||||||
if args.cmd == "cleanup":
|
if args.cmd == "cleanup":
|
||||||
|
|||||||
@@ -15,7 +15,7 @@ queue.py hold --kind <kind> --task <register task> -- <command>
|
|||||||
```
|
```
|
||||||
|
|
||||||
Start it in the background (`run_in_background` or equivalent) and end the
|
Start it in the background (`run_in_background` or equivalent) and end the
|
||||||
turn: hold waits, runs the command on its grant and exits with its code.
|
turn: hold runs the command on its grant and exits with its code.
|
||||||
|
|
||||||
- `<kind>`: a key of `slot_need_gb` in `~/.claude/agents-council/config.json`.
|
- `<kind>`: a key of `slot_need_gb` in `~/.claude/agents-council/config.json`.
|
||||||
- `<register task>`: your open `kvb-org-rasci` task; its rank places you.
|
- `<register task>`: your open `kvb-org-rasci` task; its rank places you.
|
||||||
@@ -37,31 +37,32 @@ Every 20 s each waiting hold evaluates the whole queue under one lock:
|
|||||||
1. Hotfix jobs first, then the register rank (`rasci.py show`), one place
|
1. Hotfix jobs first, then the register rank (`rasci.py show`), one place
|
||||||
earlier per whole hour waited, then arrival.
|
earlier per whole hour waited, then arrival.
|
||||||
2. In that order, a job is blocked at disk stage stop or critical, at warn
|
2. In that order, a job is blocked at disk stage stop or critical, at warn
|
||||||
when it needs disk, during a radio-silence mayday, while its kind's slots
|
when it needs disk, at memory stage stop, during a radio-silence mayday,
|
||||||
are taken, or while its workdir runs another job.
|
while its kind's slots are taken, or while its workdir runs another job.
|
||||||
3. A job whose need would cut below the stop threshold reserves that disk
|
3. A job whose disk need would cut below the stop threshold, or whose
|
||||||
against smaller jobs behind it.
|
footprint would push the running ones past `budget_percent`, reserves it
|
||||||
|
against the jobs behind it; a job alone always fits.
|
||||||
4. An unblocked job counts as granted for the jobs behind it: none takes its
|
4. An unblocked job counts as granted for the jobs behind it: none takes its
|
||||||
slot.
|
slot.
|
||||||
5. Kinds in `load_limits.kinds` also need CPU and RAM below the limits.
|
5. Every job, alone too, needs CPU, and RAM used plus what it and the
|
||||||
|
running jobs may still take, below `load_limits`.
|
||||||
A waiting hold prints a stderr line when its position or reason changes.
|
|
||||||
|
|
||||||
## History and learning
|
## History and learning
|
||||||
|
|
||||||
Every finished job appends a row to the council's `queue-history.jsonl`:
|
Every finished job appends a row (signature such as `<repo>: make <target>`,
|
||||||
signature (`make <target>`, `<script>.py <subcommand>`), kind, minutes, exit
|
kind, minutes, disk, RAM and CPU) to `queue-history.jsonl`; the
|
||||||
code, and disk, RAM and CPU at the grant and at their 20 s extremes. The
|
`kvb-org-organize` watch hook keeps it across reboots.
|
||||||
`watch` hook of `kvb-org-organize` keeps them in
|
|
||||||
`~/.local/state/agents-council/` across reboots.
|
|
||||||
|
|
||||||
From the last 30 days, once 5 runs exist:
|
From the last 30 days, once 5 runs exist:
|
||||||
|
|
||||||
- per signature, the disk need is p90 disk taken plus 1 GB;
|
- per signature, the disk need is p90 disk taken plus 1 GB;
|
||||||
- per kind, the slot limit is how many p90 RAM and CPU shares fit between the
|
- per kind, the slot limit is how many p90 RAM and CPU shares fit between the
|
||||||
median load at the grant and `load_limits`, at least 1.
|
median load at the grant and `load_limits`, at least 1;
|
||||||
|
- per job, the footprint is the larger p90 share of its kind and of its
|
||||||
|
signature, else the largest of its kind.
|
||||||
|
|
||||||
Until then the seeds hold: `slot_need_gb[kind]` for disk, 1 slot per kind.
|
Until then the seeds hold: `slot_need_gb[kind]` for disk, 1 slot per kind,
|
||||||
|
the whole budget as footprint: it runs alone.
|
||||||
The optional `slot_limits {kind: N}` only caps a limit from above.
|
The optional `slot_limits {kind: N}` only caps a limit from above.
|
||||||
|
|
||||||
## Inspect and cancel
|
## Inspect and cancel
|
||||||
@@ -72,11 +73,8 @@ queue.py history
|
|||||||
queue.py cancel <id>
|
queue.py cancel <id>
|
||||||
```
|
```
|
||||||
|
|
||||||
`list` shows each job's position, need, wait, ETA and blocking reason; dead
|
`list` shows each job's position, ETA and blocking reason, `history` the
|
||||||
holders drop out. The ETA adds the median minutes left of running jobs
|
learned values, `cancel` removes your own waiting job (its hold exits 3).
|
||||||
and of the jobs ahead, `-` while one is unknown. `history` prints the learned
|
|
||||||
values per signature and per kind, with each limit's source. `cancel` removes
|
|
||||||
your own waiting job; its hold exits 3.
|
|
||||||
|
|
||||||
## Rules
|
## Rules
|
||||||
|
|
||||||
@@ -86,6 +84,7 @@ your own waiting job; its hold exits 3.
|
|||||||
- A running job is never cut short for the queue; only the operator orders
|
- A running job is never cut short for the queue; only the operator orders
|
||||||
one down, and `kvb-git-hotfix` may void a gate.
|
one down, and `kvb-git-hotfix` may void a gate.
|
||||||
- With `queue_enforce: true`, the `block` hook (PreToolUse on Bash) refuses
|
- With `queue_enforce: true`, the `block` hook (PreToolUse on Bash) refuses
|
||||||
`heavy_commands` and `gate.py run|hold` outside `queue.py hold`.
|
`heavy_commands`, image builds, `make install|verify-*`, tests beyond one
|
||||||
|
file and `gate.py run|hold` outside `queue.py hold`.
|
||||||
- Without the config, hold exits 2: tell the operator and ask the chair
|
- Without the config, hold exits 2: tell the operator and ask the chair
|
||||||
(`kvb-org-organize`) until it exists.
|
(`kvb-org-organize`) until it exists.
|
||||||
@@ -26,6 +26,7 @@ MIN_SHARE_PERCENT = 1
|
|||||||
SEED_LIMIT = 1
|
SEED_LIMIT = 1
|
||||||
INTERPRETERS = re.compile(r"python[\d.]*|bash|sh|zsh|node|perl|ruby")
|
INTERPRETERS = re.compile(r"python[\d.]*|bash|sh|zsh|node|perl|ruby")
|
||||||
MAKE_VALUE_OPTIONS = frozenset({"-C", "-f", "-I", "-o", "-W"})
|
MAKE_VALUE_OPTIONS = frozenset({"-C", "-f", "-I", "-o", "-W"})
|
||||||
|
RESOURCES = ("ram", "cpu")
|
||||||
SIGNATURE_HEADER = ("Signature", "Kind", "Runs", "Median min", "p90 disk GB", "p90 RAM %", "Need GB")
|
SIGNATURE_HEADER = ("Signature", "Kind", "Runs", "Median min", "p90 disk GB", "p90 RAM %", "Need GB")
|
||||||
KIND_HEADER = ("Kind", "Runs", "p90 RAM share %", "p90 CPU share %", "Slot limit", "Source")
|
KIND_HEADER = ("Kind", "Runs", "p90 RAM share %", "p90 CPU share %", "Slot limit", "Source")
|
||||||
|
|
||||||
@@ -149,13 +150,14 @@ def grouped(rows: list[dict], key: str) -> dict[str, list[dict]]:
|
|||||||
def command_stats(rows: list[dict]) -> dict:
|
def command_stats(rows: list[dict]) -> dict:
|
||||||
stats = {
|
stats = {
|
||||||
"kinds": sorted({r["kind"] for r in rows}), "runs": len(rows),
|
"kinds": sorted({r["kind"] for r in rows}), "runs": len(rows),
|
||||||
"minutes": None, "disk_gb": None, "ram_share": None, "need_gb": None,
|
"minutes": None, "disk_gb": None, "ram_share": None, "cpu_share": None, "need_gb": None,
|
||||||
}
|
}
|
||||||
if len(rows) >= MIN_RUNS:
|
if len(rows) >= MIN_RUNS:
|
||||||
disk = p90([r["disk_gb"] for r in rows])
|
disk = p90([r["disk_gb"] for r in rows])
|
||||||
stats.update(
|
stats.update(
|
||||||
minutes=statistics.median(r["minutes"] for r in rows), disk_gb=disk,
|
minutes=statistics.median(r["minutes"] for r in rows), disk_gb=disk,
|
||||||
ram_share=p90(shares(rows, "mem")), need_gb=math.ceil(disk + DISK_MARGIN_GB),
|
ram_share=p90(shares(rows, "mem")), cpu_share=p90(shares(rows, "cpu")),
|
||||||
|
need_gb=math.ceil(disk + DISK_MARGIN_GB),
|
||||||
)
|
)
|
||||||
return stats
|
return stats
|
||||||
|
|
||||||
@@ -186,7 +188,7 @@ def learn(rows: list[dict], now: float, cfg: dict) -> tuple[dict[str, dict], dic
|
|||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
(signatures, kinds). signatures: {signature: {kinds, runs, minutes,
|
(signatures, kinds). signatures: {signature: {kinds, runs, minutes,
|
||||||
disk_gb, ram_share, need_gb}}; kinds: {kind: {runs, ram_share,
|
disk_gb, ram_share, cpu_share, need_gb}}; kinds: {kind: {runs, ram_share,
|
||||||
cpu_share, limit}}. With fewer than MIN_RUNS rows every value but kinds
|
cpu_share, limit}}. With fewer than MIN_RUNS rows every value but kinds
|
||||||
and runs is None. minutes is the median, disk_gb, ram_share and
|
and runs is None. minutes is the median, disk_gb, ram_share and
|
||||||
cpu_share the p90 of the disk taken and of max minus before in
|
cpu_share the p90 of the disk taken and of max minus before in
|
||||||
@@ -200,6 +202,41 @@ def learn(rows: list[dict], now: float, cfg: dict) -> tuple[dict[str, dict], dic
|
|||||||
return signatures, kinds
|
return signatures, kinds
|
||||||
|
|
||||||
|
|
||||||
|
def footprint(kind: str, name: str, signatures: dict[str, dict], kinds: dict[str, dict]) -> dict[str, float] | None:
|
||||||
|
"""Predict the RAM and CPU share a job takes, in percentage points.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
kind: the job's kind.
|
||||||
|
name: its command signature.
|
||||||
|
signatures: the signature map of learn().
|
||||||
|
kinds: the kind map of learn().
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
{ram, cpu}: the larger of the kind's p90 share and the signature's, at
|
||||||
|
least MIN_SHARE_PERCENT; a signature not learned yet takes the largest
|
||||||
|
learned share of its kind. None while the kind is not learned yet.
|
||||||
|
"""
|
||||||
|
if kind not in kinds or kinds[kind]["ram_share"] is None:
|
||||||
|
return None
|
||||||
|
own_kind = kinds[kind]
|
||||||
|
if name in signatures and signatures[name]["ram_share"] is not None:
|
||||||
|
peers = [signatures[name]]
|
||||||
|
else:
|
||||||
|
peers = [s for s in signatures.values() if kind in s["kinds"] and s["ram_share"] is not None]
|
||||||
|
return {
|
||||||
|
resource: max([MIN_SHARE_PERCENT, own_kind[f"{resource}_share"], *(peer[f"{resource}_share"] for peer in peers)])
|
||||||
|
for resource in RESOURCES
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def seed_ram(kinds: dict[str, dict]) -> float:
|
||||||
|
"""Returns:
|
||||||
|
The RAM share assumed for a kind not learned yet: the largest of the
|
||||||
|
learned kinds' p90 RAM shares, 0 while none is learned.
|
||||||
|
"""
|
||||||
|
return max((stats["ram_share"] for stats in kinds.values() if stats["ram_share"] is not None), default=0.0)
|
||||||
|
|
||||||
|
|
||||||
def need_gb(signatures: dict[str, dict], name: str) -> int | None:
|
def need_gb(signatures: dict[str, dict], name: str) -> int | None:
|
||||||
return signatures[name]["need_gb"] if name in signatures else None
|
return signatures[name]["need_gb"] if name in signatures else None
|
||||||
|
|
||||||
|
|||||||
@@ -57,19 +57,27 @@ Args:
|
|||||||
Bash deny rule of the settings is refused, one meeting an ask rule
|
Bash deny rule of the settings is refused, one meeting an ask rule
|
||||||
asks the operator, with or without queue_enforce; a hold that cannot
|
asks the operator, with or without queue_enforce; a hold that cannot
|
||||||
be parsed or checked is refused. With queue_enforce true in the config,
|
be parsed or checked is refused. With queue_enforce true in the config,
|
||||||
refuse a command matching heavy_commands or `gate.py run|hold`
|
refuse a command that watch.heavy() calls heavy or that runs
|
||||||
unless it matches never_block or runs `queue.py hold`.
|
`gate.py run|hold`, unless it matches never_block or runs `queue.py
|
||||||
|
hold`.
|
||||||
|
|
||||||
Config keys: slot_limits {kind: N}, optional, caps the learned slot limit of a
|
Config keys: budget_percent {ram, cpu}, required: the active and granted jobs'
|
||||||
kind from above (history.slot_limits); queue_enforce (bool), optional, read by
|
learned footprints (history.footprint, the whole budget for a kind not learned
|
||||||
block only. The watch.py keys disk, thresholds_gb (stop is the floor every
|
yet) may sum to at most these percentage points, a job alone always fits;
|
||||||
grant leaves free), slot_need_gb (the disk seed per kind), load_limits (kinds
|
slot_limits {kind: N}, optional, caps the learned slot limit of a kind from
|
||||||
in its kinds also need CPU and RAM below the limits; mem_percent and
|
above (history.slot_limits); queue_enforce (bool), optional, read by block
|
||||||
cpu_percent bound the learned slot limits), heavy_commands and never_block are
|
only. The watch.py keys disk, thresholds_gb (stop is the floor every grant
|
||||||
used as watch.py documents them.
|
leaves free), slot_need_gb (the disk seed per kind), load_limits (every grant
|
||||||
|
also needs CPU busy below cpu_percent and RAM used plus the job's learned RAM
|
||||||
|
share below mem_percent; both bound the learned slot limits), memory_limits (no
|
||||||
|
grant at memory stage stop), heavy_commands and never_block are used as
|
||||||
|
watch.py documents them. A signature is `<repository>: <command signature>`
|
||||||
|
(qualified()).
|
||||||
|
|
||||||
Exit codes:
|
Exit codes:
|
||||||
hold: the command's exit code unchanged, 128+N when signal N ended it;
|
hold: the command's exit code unchanged, 128+N when signal N ended it;
|
||||||
|
a command exiting 75 (EX_TEMPFAIL: it did no work, such as a declined
|
||||||
|
`operator.py granted`) leaves no history row;
|
||||||
before the command started 2 on a usage or config error or when it
|
before the command started 2 on a usage or config error or when it
|
||||||
cannot start, 3 when cancel removed the entry, 128+N when signal N
|
cannot start, 3 when cancel removed the entry, 128+N when signal N
|
||||||
reached hold.
|
reached hold.
|
||||||
@@ -115,14 +123,16 @@ POLL_SECONDS = 20
|
|||||||
TICK_SECONDS = 0.2
|
TICK_SECONDS = 0.2
|
||||||
HOUR = 3600
|
HOUR = 3600
|
||||||
CANCELLED = 3
|
CANCELLED = 3
|
||||||
FORWARDED = (signal.SIGINT, signal.SIGTERM)
|
NOT_RUN = 75
|
||||||
|
RAMP_SECONDS = 900
|
||||||
|
SETUP =frozenset({"cd", "pushd", "set", "source", ".", "export", "umask", "shopt", "trap"})
|
||||||
|
FORWARDED =(signal.SIGINT, signal.SIGTERM)
|
||||||
GATE = re.compile(r"\bgate\.py\s+(run|hold)\b")
|
GATE = re.compile(r"\bgate\.py\s+(run|hold)\b")
|
||||||
THROUGH_QUEUE = re.compile(r"\bqueue\.py\s+hold\b")
|
THROUGH_QUEUE = re.compile(r"\bqueue\.py\s+hold\b")
|
||||||
BASH_RULE = re.compile(r"Bash(?:\((.*)\))?", re.DOTALL)
|
BASH_RULE = re.compile(r"Bash(?:\((.*)\))?", re.DOTALL)
|
||||||
RULE_PART = re.compile(r"( \*|\*)")
|
RULE_PART = re.compile(r"( \*|\*)")
|
||||||
SEPARATOR = re.compile(r"[();<>|&]+")
|
|
||||||
WORD_BREAK = " \t\n;&|()<>"
|
|
||||||
MANAGED = Path("/etc/claude-code")
|
MANAGED = Path("/etc/claude-code")
|
||||||
|
BUDGET_KEYS = ("budget_percent.ram", "budget_percent.cpu")
|
||||||
HOTFIX = "hotfix/"
|
HOTFIX = "hotfix/"
|
||||||
|
|
||||||
|
|
||||||
@@ -154,6 +164,9 @@ def config() -> dict:
|
|||||||
cfg = watch.load_config()
|
cfg = watch.load_config()
|
||||||
if cfg is None:
|
if cfg is None:
|
||||||
fail(f"no config at {watch.CONFIG}; the queue reads the kvb-org-organize watch config")
|
fail(f"no config at {watch.CONFIG}; the queue reads the kvb-org-organize watch config")
|
||||||
|
problem = watch.missing(cfg, (*BUDGET_KEYS, *watch.MEMORY_KEYS))
|
||||||
|
if problem is not None:
|
||||||
|
fail(problem)
|
||||||
return cfg
|
return cfg
|
||||||
|
|
||||||
|
|
||||||
@@ -169,8 +182,59 @@ def limits(cfg: dict, kinds: dict[str, dict], entries: list[dict]) -> dict[str,
|
|||||||
return history.slot_limits(kinds, caps(cfg), {*cfg["slot_need_gb"], *(e["kind"] for e in entries)})
|
return history.slot_limits(kinds, caps(cfg), {*cfg["slot_need_gb"], *(e["kind"] for e in entries)})
|
||||||
|
|
||||||
|
|
||||||
|
@functools.cache
|
||||||
|
def repository(path: str) -> str:
|
||||||
|
try:
|
||||||
|
common = subprocess.run(
|
||||||
|
["git", "rev-parse", "--path-format=absolute", "--git-common-dir"],
|
||||||
|
cwd=path, capture_output=True, text=True, check=False,
|
||||||
|
)
|
||||||
|
except OSError:
|
||||||
|
return os.path.basename(path)
|
||||||
|
if common.returncode != 0:
|
||||||
|
return os.path.basename(path)
|
||||||
|
return os.path.basename(os.path.dirname(common.stdout.strip()))
|
||||||
|
|
||||||
|
|
||||||
|
def command_words(argv: list[str]) -> list[str]:
|
||||||
|
"""Returns:
|
||||||
|
The command a wrapper runs: watch.program() unwrapped, the first simple
|
||||||
|
command of `sh -c` code that is no setup such as cd, set or source,
|
||||||
|
the tail after `--` of a Python script such as gate.py hold or
|
||||||
|
operator.py granted.
|
||||||
|
"""
|
||||||
|
run = watch.program(argv) or argv
|
||||||
|
code = watch.inline_code(run)
|
||||||
|
inner = first_command(code) if code is not None else script_tail(run)
|
||||||
|
return run if inner is None else command_words(inner)
|
||||||
|
|
||||||
|
|
||||||
|
def first_command(code: str) -> list[str] | None:
|
||||||
|
try:
|
||||||
|
parts = [part for part in map(watch.program, watch.segments(code)) if part and os.path.basename(part[0]) not in SETUP]
|
||||||
|
except ValueError:
|
||||||
|
return None
|
||||||
|
return parts[0] if parts else None
|
||||||
|
|
||||||
|
|
||||||
|
def script_tail(run: list[str]) -> list[str] | None:
|
||||||
|
if len(run) > 2 and watch.PYTHON.fullmatch(os.path.basename(run[0])) and run[1].endswith(".py") and "--" in run[2:-1]:
|
||||||
|
return run[run.index("--", 2) + 1 :]
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def qualified(workdir: str, argv: list[str]) -> str:
|
||||||
|
"""Returns:
|
||||||
|
`<repository>: <history.signature of command_words>`: the directory
|
||||||
|
holding the main checkout's .git, so every worktree of one repository
|
||||||
|
shares it, else the workdir's name; one make test of two repositories
|
||||||
|
stays apart.
|
||||||
|
"""
|
||||||
|
return f"{repository(workdir)}: {history.signature(command_words(argv))}"
|
||||||
|
|
||||||
|
|
||||||
def signature_of(entry: dict) -> str:
|
def signature_of(entry: dict) -> str:
|
||||||
return history.signature(shlex.split(entry["cmd"]))
|
return qualified(entry["workdir"], shlex.split(entry["cmd"]))
|
||||||
|
|
||||||
|
|
||||||
def identity(pid: int) -> str | None:
|
def identity(pid: int) -> str | None:
|
||||||
@@ -270,18 +334,33 @@ def ordered(waiting: list[dict], ranks: dict[str, int], agents: dict[str, int],
|
|||||||
return sorted(waiting, key=key)
|
return sorted(waiting, key=key)
|
||||||
|
|
||||||
|
|
||||||
def evaluate(entries: list[dict], free_gb: float, floor: float, stage: str, silenced: bool,
|
def over_budget(used: dict[str, float], share: dict[str, float], budget: dict[str, float]) -> str | None:
|
||||||
slot_limits: dict[str, int], ranks: dict[str, int], agents: dict[str, int],
|
over = [resource for resource in budget if used[resource] + share[resource] > budget[resource]]
|
||||||
now: float) -> dict[str, str | None]:
|
if not over:
|
||||||
|
return None
|
||||||
|
return ", ".join(
|
||||||
|
f"{resource.upper()} {used[resource]:.0f} % in use or reserved + {share[resource]:.0f} % > budget {budget[resource]:g} %"
|
||||||
|
for resource in over
|
||||||
|
) + "; reserved"
|
||||||
|
|
||||||
|
|
||||||
|
def evaluate(entries: list[dict], free_gb: float, floor: float, stage: str, memory: str, silenced: bool,
|
||||||
|
slot_limits: dict[str, int], footprints: dict[str, dict[str, float]], budget: dict[str, float],
|
||||||
|
ranks: dict[str, int], agents: dict[str, int], now: float) -> dict[str, str | None]:
|
||||||
"""Decide every waiting entry in grant order, without disk or processes.
|
"""Decide every waiting entry in grant order, without disk or processes.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
entries: the queue's entries, active and waiting.
|
entries: the queue's entries, active and waiting.
|
||||||
free_gb: free disk in GB.
|
free_gb: free disk in GB.
|
||||||
floor: GB every grant must leave free (thresholds_gb.stop).
|
floor: GB every grant must leave free (thresholds_gb.stop).
|
||||||
stage: the watch stage: ok, warn, stop or critical.
|
stage: the watch disk stage: ok, warn, stop or critical.
|
||||||
|
memory: the watch memory stage: ok, warn or stop.
|
||||||
silenced: whether a radio-silence mayday holds.
|
silenced: whether a radio-silence mayday holds.
|
||||||
slot_limits: {kind: max concurrent jobs}.
|
slot_limits: {kind: max concurrent jobs}, an upper cap beside the budget.
|
||||||
|
footprints: {entry id: {ram, cpu}}, the learned share of every entry
|
||||||
|
(history.footprint), in percentage points.
|
||||||
|
budget: {ram, cpu}: the sum of the footprints of active and granted
|
||||||
|
entries may not pass it; a job alone always fits.
|
||||||
ranks: {task: index in the register's ranking}; a task missing there
|
ranks: {task: index in the register's ranking}; a task missing there
|
||||||
ranks after all of them.
|
ranks after all of them.
|
||||||
agents: {agent name: index in the operator's agent order}; it breaks
|
agents: {agent name: index in the operator's agent order}; it breaks
|
||||||
@@ -292,39 +371,69 @@ def evaluate(entries: list[dict], free_gb: float, floor: float, stage: str, sile
|
|||||||
Returns:
|
Returns:
|
||||||
{entry id: None when granted, else the blocking reason}, in grant order.
|
{entry id: None when granted, else the blocking reason}, in grant order.
|
||||||
An unblocked entry counts as granted for the entries after it, whoever
|
An unblocked entry counts as granted for the entries after it, whoever
|
||||||
holds it, so a lower entry never takes a higher entry's slot.
|
holds it, so a lower entry never takes a higher entry's slot; one
|
||||||
|
blocked by disk or budget reserves its need against the entries after it.
|
||||||
"""
|
"""
|
||||||
active = [e for e in entries if e["status"] == "active"]
|
active = [e for e in entries if e["status"] == "active"]
|
||||||
available = free_gb - sum(e["need_gb"] for e in active)
|
available = free_gb - sum(e["need_gb"] for e in active)
|
||||||
|
used = {resource: sum(footprints[e["id"]][resource] for e in active) for resource in budget}
|
||||||
|
taken = len(active)
|
||||||
counts = Counter(e["kind"] for e in active)
|
counts = Counter(e["kind"] for e in active)
|
||||||
busy = {e["workdir"] for e in active}
|
busy = {e["workdir"] for e in active}
|
||||||
verdicts: dict[str, str | None] = {}
|
verdicts: dict[str, str | None] = {}
|
||||||
for entry in ordered([e for e in entries if e["status"] == "waiting"], ranks, agents, now):
|
for entry in ordered([e for e in entries if e["status"] == "waiting"], ranks, agents, now):
|
||||||
kind, need, workdir = entry["kind"], entry["need_gb"], entry["workdir"]
|
kind, need, workdir, share = entry["kind"], entry["need_gb"], entry["workdir"], footprints[entry["id"]]
|
||||||
reason = None
|
reason = host_reason(stage, memory, silenced, need) or slot_reason(kind, workdir, slot_limits, counts, busy)
|
||||||
if stage in ("stop", "critical"):
|
if reason is None:
|
||||||
reason = f"disk stage {stage}"
|
reason, used, available = resources(need, share, used, available, floor, budget, taken)
|
||||||
elif stage == "warn" and need > 0:
|
if reason is None:
|
||||||
reason = "disk stage warn admits only jobs that need no disk"
|
taken += 1
|
||||||
elif silenced:
|
counts[kind] += 1
|
||||||
reason = "radio silence"
|
busy.add(workdir)
|
||||||
elif kind not in slot_limits:
|
|
||||||
reason = f"slot_limits has no {kind}"
|
|
||||||
elif counts[kind] >= slot_limits[kind]:
|
|
||||||
reason = f"all {slot_limits[kind]} {kind} slots taken"
|
|
||||||
elif workdir in busy:
|
|
||||||
reason = f"{workdir} runs another job"
|
|
||||||
elif available - need < floor:
|
|
||||||
reason = f"{need:g} GB would cut into the {floor:g} GB floor; reserved"
|
|
||||||
available -= need
|
|
||||||
else:
|
|
||||||
counts[kind] += 1
|
|
||||||
busy.add(workdir)
|
|
||||||
available -= need
|
|
||||||
verdicts[entry["id"]] = reason
|
verdicts[entry["id"]] = reason
|
||||||
return verdicts
|
return verdicts
|
||||||
|
|
||||||
|
|
||||||
|
def plus(used: dict[str, float], share: dict[str, float]) -> dict[str, float]:
|
||||||
|
return {resource: used[resource] + share[resource] for resource in used}
|
||||||
|
|
||||||
|
|
||||||
|
def resources(need: float, share: dict[str, float], used: dict[str, float], available: float, floor: float,
|
||||||
|
budget: dict[str, float], taken: int) -> tuple[str | None, dict[str, float], float]:
|
||||||
|
"""Returns:
|
||||||
|
(reason or None, RAM and CPU used or reserved, disk available) after
|
||||||
|
the job: one passing the budget beside other jobs reserves its share,
|
||||||
|
one cutting into the floor its disk, a granted one takes both.
|
||||||
|
"""
|
||||||
|
if taken and (reason := over_budget(used, share, budget)) is not None:
|
||||||
|
return reason, plus(used, share), available
|
||||||
|
if available - need < floor:
|
||||||
|
return f"{need:g} GB would cut into the {floor:g} GB floor; reserved", used, available - need
|
||||||
|
return None, plus(used, share), available - need
|
||||||
|
|
||||||
|
|
||||||
|
def host_reason(stage: str, memory: str, silenced: bool, need: float) -> str | None:
|
||||||
|
if stage in ("stop", "critical"):
|
||||||
|
return f"disk stage {stage}"
|
||||||
|
if stage == "warn" and need > 0:
|
||||||
|
return "disk stage warn admits only jobs that need no disk"
|
||||||
|
if memory == "stop":
|
||||||
|
return "memory stage stop"
|
||||||
|
if silenced:
|
||||||
|
return "radio silence"
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def slot_reason(kind: str, workdir: str, slot_limits: dict[str, int], counts: Counter, busy: set[str]) -> str | None:
|
||||||
|
if kind not in slot_limits:
|
||||||
|
return f"slot_limits has no {kind}"
|
||||||
|
if counts[kind] >= slot_limits[kind]:
|
||||||
|
return f"all {slot_limits[kind]} {kind} slots taken"
|
||||||
|
if workdir in busy:
|
||||||
|
return f"{workdir} runs another job"
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
def register_ranks() -> dict[str, int]:
|
def register_ranks() -> dict[str, int]:
|
||||||
rasci = sibling("rasci")
|
rasci = sibling("rasci")
|
||||||
return {task: index for index, task in enumerate(rasci.ranked(rasci.merged(False)))}
|
return {task: index for index, task in enumerate(rasci.ranked(rasci.merged(False)))}
|
||||||
@@ -344,22 +453,62 @@ def silenced(now: float) -> bool:
|
|||||||
return "until" in state and state["until"] > now
|
return "until" in state and state["until"] > now
|
||||||
|
|
||||||
|
|
||||||
def decide(cfg: dict, entries: list[dict], free_gb: float, now: float, kinds: dict[str, dict]) -> dict[str, str | None]:
|
def footprint(entry: dict, signatures: dict[str, dict], kinds: dict[str, dict]) -> dict[str, float] | None:
|
||||||
|
return history.footprint(entry["kind"], signature_of(entry), signatures, kinds)
|
||||||
|
|
||||||
|
|
||||||
|
def decide(cfg: dict, entries: list[dict], free_gb: float, now: float, signatures: dict[str, dict],
|
||||||
|
kinds: dict[str, dict]) -> dict[str, str | None]:
|
||||||
slots = {kind: limit for kind, (limit, _) in limits(cfg, kinds, entries).items()}
|
slots = {kind: limit for kind, (limit, _) in limits(cfg, kinds, entries).items()}
|
||||||
|
budget = cfg["budget_percent"]
|
||||||
|
footprints = {}
|
||||||
|
for entry in entries:
|
||||||
|
share = footprint(entry, signatures, kinds)
|
||||||
|
footprints[entry["id"]] = dict(budget) if share is None else share
|
||||||
return evaluate(
|
return evaluate(
|
||||||
entries, free_gb, cfg["thresholds_gb"]["stop"], stage(cfg, free_gb, now), silenced(now),
|
entries, free_gb, cfg["thresholds_gb"]["stop"], stage(cfg, free_gb, now),
|
||||||
slots, register_ranks(), agent_ranks(), now,
|
watch.memory_stage(watch.memory(), cfg), silenced(now), slots, footprints, budget,
|
||||||
|
register_ranks(), agent_ranks(), now,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
def load_reason(cfg: dict, kind: str) -> str | None:
|
def load_reason(limits: dict, cpu: float, mem: float, ram_share: float) -> str | None:
|
||||||
limits = cfg["load_limits"]
|
"""Args:
|
||||||
if kind not in limits["kinds"]:
|
limits: load_limits of the config.
|
||||||
return None
|
cpu: CPU busy now, in percent.
|
||||||
cpu, mem = watch.load(cfg)
|
mem: RAM used now, in percent; stacks and jobs outside the queue
|
||||||
if cpu < limits["cpu_percent"] and mem < limits["mem_percent"]:
|
included.
|
||||||
return None
|
ram_share: what the job and the active ones may still take, in
|
||||||
return f"cpu or ram at the limit ({limits['cpu_percent']} %, {limits['mem_percent']} %)"
|
percentage points (pending_ram()).
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
None when CPU stays below cpu_percent and RAM used plus the share
|
||||||
|
below mem_percent, else why not.
|
||||||
|
"""
|
||||||
|
if cpu >= limits["cpu_percent"]:
|
||||||
|
return f"CPU {cpu:.0f} % busy reaches the limit {limits['cpu_percent']} %"
|
||||||
|
if mem + ram_share >= limits["mem_percent"]:
|
||||||
|
return f"RAM {mem:.0f} % used + {ram_share:.0f} % still to come reach the limit {limits['mem_percent']} %"
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def ram_share(entry: dict, signatures: dict[str, dict], kinds: dict[str, dict]) -> float:
|
||||||
|
share = footprint(entry, signatures, kinds)
|
||||||
|
return history.seed_ram(kinds) if share is None else share["ram"]
|
||||||
|
|
||||||
|
|
||||||
|
def pending_ram(entries: list[dict], entry_id: str, now: float, signatures: dict[str, dict],
|
||||||
|
kinds: dict[str, dict]) -> float:
|
||||||
|
"""Returns:
|
||||||
|
The RAM share of entry_id plus the whole share of every active job
|
||||||
|
granted less than RAMP_SECONDS ago (or without a grant time): what
|
||||||
|
may still come once they ramp up. An older job counts as ramped, its
|
||||||
|
RAM inside the RAM used; one that peaks later is bounded by the
|
||||||
|
memory stage only.
|
||||||
|
"""
|
||||||
|
mine = [e for e in entries if e["id"] == entry_id]
|
||||||
|
young = [e for e in entries if e["status"] == "active" and now - e.get("granted", now) < RAMP_SECONDS]
|
||||||
|
return ram_share(mine[0], signatures, kinds) + sum(ram_share(e, signatures, kinds) for e in young)
|
||||||
|
|
||||||
|
|
||||||
def workdir() -> str:
|
def workdir() -> str:
|
||||||
@@ -399,13 +548,12 @@ def pause(caught: list[int]) -> None:
|
|||||||
time.sleep(TICK_SECONDS)
|
time.sleep(TICK_SECONDS)
|
||||||
|
|
||||||
|
|
||||||
def wait(cfg: dict, entry_id: str, kind: str, caught: list[int]) -> tuple[int | None, int]:
|
def wait(cfg: dict, entry_id: str, caught: list[int]) -> tuple[int | None, int]:
|
||||||
"""Poll until the grant.
|
"""Poll until the grant.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
cfg: the config.
|
cfg: the config.
|
||||||
entry_id: this hold's entry.
|
entry_id: this hold's entry.
|
||||||
kind: its kind, for the load sample.
|
|
||||||
caught: signals the caller caught; the first ends the wait.
|
caught: signals the caller caught; the first ends the wait.
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
@@ -414,18 +562,19 @@ def wait(cfg: dict, entry_id: str, kind: str, caught: list[int]) -> tuple[int |
|
|||||||
"""
|
"""
|
||||||
shown = None
|
shown = None
|
||||||
while not caught:
|
while not caught:
|
||||||
busy_load = load_reason(cfg, kind)
|
cpu, mem = watch.load(cfg)
|
||||||
free, _ = watch.disk(cfg)
|
free, _ = watch.disk(cfg)
|
||||||
now = time.time()
|
now = time.time()
|
||||||
_, kinds = learned(cfg, now)
|
signatures, kinds = learned(cfg, now)
|
||||||
with store() as data:
|
with store() as data:
|
||||||
data["entries"] = prune(data["entries"])
|
data["entries"] = prune(data["entries"])
|
||||||
mine = [e for e in data["entries"] if e["id"] == entry_id]
|
mine = [e for e in data["entries"] if e["id"] == entry_id]
|
||||||
if not mine:
|
if not mine:
|
||||||
print(f"queue: {entry_id} was cancelled", file=sys.stderr)
|
print(f"queue: {entry_id} was cancelled", file=sys.stderr)
|
||||||
return CANCELLED, 0
|
return CANCELLED, 0
|
||||||
verdicts = decide(cfg, data["entries"], free, now, kinds)
|
verdicts = decide(cfg, data["entries"], free, now, signatures, kinds)
|
||||||
reason = verdicts[entry_id] or busy_load
|
pending = pending_ram(data["entries"], entry_id, now, signatures, kinds)
|
||||||
|
reason = verdicts[entry_id] or load_reason(cfg["load_limits"], cpu, mem, pending)
|
||||||
if reason is None:
|
if reason is None:
|
||||||
concurrent = sum(e["status"] == "active" for e in data["entries"])
|
concurrent = sum(e["status"] == "active" for e in data["entries"])
|
||||||
mine[0].update(status="active", granted=now)
|
mine[0].update(status="active", granted=now)
|
||||||
@@ -477,6 +626,11 @@ def keep(row: dict) -> None:
|
|||||||
print(f"queue: no history row for {row['id']}: {error.strerror}", file=sys.stderr)
|
print(f"queue: no history row for {row['id']}: {error.strerror}", file=sys.stderr)
|
||||||
|
|
||||||
|
|
||||||
|
def record(job: dict, start: float, code: int, samples: list[dict[str, float]], concurrent: int) -> None:
|
||||||
|
if code != NOT_RUN:
|
||||||
|
keep(history.row(job, start, time.time(), code, samples, concurrent))
|
||||||
|
|
||||||
|
|
||||||
def cmd_hold(args: argparse.Namespace) -> int:
|
def cmd_hold(args: argparse.Namespace) -> int:
|
||||||
cfg = config()
|
cfg = config()
|
||||||
if args.kind not in cfg["slot_need_gb"]:
|
if args.kind not in cfg["slot_need_gb"]:
|
||||||
@@ -486,7 +640,8 @@ def cmd_hold(args: argparse.Namespace) -> int:
|
|||||||
fail(f"{args.task} is no open task of the kvb-org-rasci register")
|
fail(f"{args.task} is no open task of the kvb-org-rasci register")
|
||||||
if args.hotfix is not None and HOTFIX + args.hotfix not in rasci.load():
|
if args.hotfix is not None and HOTFIX + args.hotfix not in rasci.load():
|
||||||
fail(f"the register has no row {HOTFIX}{args.hotfix}")
|
fail(f"the register has no row {HOTFIX}{args.hotfix}")
|
||||||
signature = history.signature(args.command)
|
here = workdir()
|
||||||
|
signature = qualified(here, args.command)
|
||||||
signatures, _ = learned(cfg, time.time())
|
signatures, _ = learned(cfg, time.time())
|
||||||
need = args.need_gb
|
need = args.need_gb
|
||||||
if need is None:
|
if need is None:
|
||||||
@@ -511,13 +666,13 @@ def cmd_hold(args: argparse.Namespace) -> int:
|
|||||||
with holding() as (tag, lock_fd):
|
with holding() as (tag, lock_fd):
|
||||||
entry = enqueue({
|
entry = enqueue({
|
||||||
"kind": args.kind, "task": args.task, "need_gb": need, "hotfix": args.hotfix,
|
"kind": args.kind, "task": args.task, "need_gb": need, "hotfix": args.hotfix,
|
||||||
"workdir": workdir(), "session": session, "name": council.live_sessions().get(session) or session,
|
"workdir": here, "session": session, "name": council.live_sessions().get(session) or session,
|
||||||
"lock": tag, "pid": os.getpid(), "pid_start": identity(os.getpid()), "child": None,
|
"lock": tag, "pid": os.getpid(), "pid_start": identity(os.getpid()), "child": None,
|
||||||
"child_start": None, "status": "waiting", "enqueued": time.time(), "granted": None,
|
"child_start": None, "status": "waiting", "enqueued": time.time(), "granted": None,
|
||||||
"cmd": shlex.join(args.command),
|
"cmd": shlex.join(args.command),
|
||||||
})
|
})
|
||||||
try:
|
try:
|
||||||
code, concurrent = wait(cfg, entry["id"], args.kind, caught)
|
code, concurrent = wait(cfg, entry["id"], caught)
|
||||||
if code is not None:
|
if code is not None:
|
||||||
return code
|
return code
|
||||||
if caught:
|
if caught:
|
||||||
@@ -536,7 +691,7 @@ def cmd_hold(args: argparse.Namespace) -> int:
|
|||||||
code = code if code >= 0 else 128 - code
|
code = code if code >= 0 else 128 - code
|
||||||
job = {"id": f"{entry['id']}-{time.time_ns()}", "kind": args.kind, "signature": signature,
|
job = {"id": f"{entry['id']}-{time.time_ns()}", "kind": args.kind, "signature": signature,
|
||||||
"workdir": entry["workdir"]}
|
"workdir": entry["workdir"]}
|
||||||
keep(history.row(job, start, time.time(), code, samples, concurrent))
|
record(job, start, code, samples, concurrent)
|
||||||
return code
|
return code
|
||||||
finally:
|
finally:
|
||||||
leave(entry["id"])
|
leave(entry["id"])
|
||||||
@@ -550,7 +705,7 @@ def cmd_list(_: argparse.Namespace) -> int:
|
|||||||
with store() as data:
|
with store() as data:
|
||||||
data["entries"] = prune(data["entries"])
|
data["entries"] = prune(data["entries"])
|
||||||
entries = list(data["entries"])
|
entries = list(data["entries"])
|
||||||
verdicts = decide(cfg, entries, free, now, kinds)
|
verdicts = decide(cfg, entries, free, now, signatures, kinds)
|
||||||
by_id = {e["id"]: e for e in entries}
|
by_id = {e["id"]: e for e in entries}
|
||||||
running = [(signature_of(e), e["granted"]) for e in entries if e["status"] == "active"]
|
running = [(signature_of(e), e["granted"]) for e in entries if e["status"] == "active"]
|
||||||
rows = [
|
rows = [
|
||||||
@@ -565,13 +720,17 @@ def cmd_list(_: argparse.Namespace) -> int:
|
|||||||
print("position\tid\tstatus\tkind\ttask\tneed_gb\twaited_min\teta_min\tworkdir\treason")
|
print("position\tid\tstatus\tkind\ttask\tneed_gb\twaited_min\teta_min\tworkdir\treason")
|
||||||
for position, e, eta, reason in rows:
|
for position, e, eta, reason in rows:
|
||||||
waited = ((e["granted"] or now) - e["enqueued"]) / 60
|
waited = ((e["granted"] or now) - e["enqueued"]) / 60
|
||||||
print("\t".join(
|
print(tab_line(
|
||||||
[position, e["id"], e["status"], e["kind"], e["task"], f"{e['need_gb']:g}", f"{waited:.0f}",
|
[position, e["id"], e["status"], e["kind"], e["task"], f"{e['need_gb']:g}", f"{waited:.0f}",
|
||||||
"-" if eta is None else f"{eta:.0f}", e["workdir"], reason]
|
"-" if eta is None else f"{eta:.0f}", e["workdir"], reason]
|
||||||
))
|
))
|
||||||
return 0
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
def tab_line(fields: list[object]) -> str:
|
||||||
|
return "\t".join("".join(c if c.isprintable() else c.encode("unicode_escape").decode() for c in str(f)) for f in fields)
|
||||||
|
|
||||||
|
|
||||||
def cmd_history(_: argparse.Namespace) -> int:
|
def cmd_history(_: argparse.Namespace) -> int:
|
||||||
cfg = config()
|
cfg = config()
|
||||||
signatures, kinds = learned(cfg, time.time())
|
signatures, kinds = learned(cfg, time.time())
|
||||||
@@ -633,50 +792,10 @@ def permission(commands: list[str], project: Path) -> str | None:
|
|||||||
return None
|
return None
|
||||||
|
|
||||||
|
|
||||||
def uncommented(command: str) -> str:
|
|
||||||
kept: list[str] = []
|
|
||||||
quote = ""
|
|
||||||
escaped = skipping = False
|
|
||||||
word_start = True
|
|
||||||
for char in command:
|
|
||||||
if skipping and char != "\n":
|
|
||||||
continue
|
|
||||||
skipping = False
|
|
||||||
if escaped:
|
|
||||||
escaped = False
|
|
||||||
elif quote == "'":
|
|
||||||
quote = "" if char == "'" else quote
|
|
||||||
elif char == "\\":
|
|
||||||
escaped = True
|
|
||||||
elif quote == '"':
|
|
||||||
quote = "" if char == '"' else quote
|
|
||||||
elif char in "'\"":
|
|
||||||
quote = char
|
|
||||||
elif char == "#" and word_start:
|
|
||||||
skipping = True
|
|
||||||
continue
|
|
||||||
kept.append(char)
|
|
||||||
word_start = not quote and not escaped and char in WORD_BREAK
|
|
||||||
return "".join(kept)
|
|
||||||
|
|
||||||
|
|
||||||
def segments(command: str) -> list[list[str]]:
|
|
||||||
lexer = shlex.shlex(uncommented(command.replace("\\\n", "")), posix=True, punctuation_chars=True)
|
|
||||||
lexer.whitespace_split = True
|
|
||||||
lexer.commenters = ""
|
|
||||||
parts: list[list[str]] = [[]]
|
|
||||||
for token in lexer:
|
|
||||||
if SEPARATOR.fullmatch(token):
|
|
||||||
parts.append([])
|
|
||||||
else:
|
|
||||||
parts[-1].append(token)
|
|
||||||
return parts
|
|
||||||
|
|
||||||
|
|
||||||
def held_commands(command: str) -> list[list[str]]:
|
def held_commands(command: str) -> list[list[str]]:
|
||||||
return [
|
return [
|
||||||
cli().parse_args(part[at + 1 :]).command
|
cli().parse_args(part[at + 1 :]).command
|
||||||
for part in segments(command)
|
for part in watch.segments(command)
|
||||||
for at, token in enumerate(part)
|
for at, token in enumerate(part)
|
||||||
if token.endswith("queue.py") and part[at + 1 : at + 2] == ["hold"]
|
if token.endswith("queue.py") and part[at + 1 : at + 2] == ["hold"]
|
||||||
]
|
]
|
||||||
@@ -715,10 +834,8 @@ def cmd_block() -> int:
|
|||||||
cfg = watch.load_config()
|
cfg = watch.load_config()
|
||||||
if cfg is None or cfg.get("queue_enforce") is not True:
|
if cfg is None or cfg.get("queue_enforce") is not True:
|
||||||
return 0
|
return 0
|
||||||
command = watch.executable_text((payload.get("tool_input") or {}).get("command") or "")
|
gated = {**cfg, "heavy_commands": [*cfg["heavy_commands"], GATE.pattern]}
|
||||||
if THROUGH_QUEUE.search(command) or any(re.search(p, command) for p in cfg["never_block"]):
|
if watch.heavy(raw, gated, (THROUGH_QUEUE.pattern, *cfg["never_block"])):
|
||||||
return 0
|
|
||||||
if GATE.search(command) or any(re.search(p, command) for p in cfg["heavy_commands"]):
|
|
||||||
print(
|
print(
|
||||||
f"kvb-org-queue: heavy jobs run through the queue: start `python3 {Path(__file__).resolve()} hold "
|
f"kvb-org-queue: heavy jobs run through the queue: start `python3 {Path(__file__).resolve()} hold "
|
||||||
"--kind <kind> --task <register task> -- <command>` in the background; it runs the command on its grant.",
|
"--kind <kind> --task <register task> -- <command>` in the background; it runs the command on its grant.",
|
||||||
|
|||||||
@@ -249,6 +249,90 @@ class TestOperator(unittest.TestCase):
|
|||||||
self.assertNotIn("decide", result.stdout)
|
self.assertNotIn("decide", result.stdout)
|
||||||
self.assertEqual(self._open(), ["operator/ask"])
|
self.assertEqual(self._open(), ["operator/ask"])
|
||||||
|
|
||||||
|
def _queue_config(self, slot_need_gb: dict) -> None:
|
||||||
|
config = self.base / "config.json"
|
||||||
|
open_memory = {"available": -1, "swap_free": -1, "psi_full": 1000, "container_swap_gb": 1e9}
|
||||||
|
config.write_text(json.dumps({
|
||||||
|
"disk": str(self.base), "thresholds_gb": {"warn": 0, "stop": 0, "critical": 0}, "projection_minutes": 30,
|
||||||
|
"slot_need_gb": slot_need_gb, "heavy_commands": [r"\btouch heavy\b"], "never_block": [],
|
||||||
|
"load_limits": {"cpu_percent": 101, "mem_percent": 101, "sample_seconds": 0.1},
|
||||||
|
"budget_percent": {"ram": 75, "cpu": 100}, "memory_limits": {"warn": open_memory, "stop": open_memory},
|
||||||
|
}), encoding="utf-8")
|
||||||
|
self.env["AGENTS_COUNCIL_CONFIG"] = str(config)
|
||||||
|
|
||||||
|
def test_run_offers_the_queue_for_a_heavy_command_and_asks_again_on_the_grant(self):
|
||||||
|
self._queue_config({"operator": 0})
|
||||||
|
work = self.base / "work"
|
||||||
|
work.mkdir()
|
||||||
|
self._add("light", work, "touch light", "plain", "manual")
|
||||||
|
self._add("heavy", work, "touch heavy\x1b[2K", "build", "manual")
|
||||||
|
result = self._feed("y\nh\n")
|
||||||
|
self.assertEqual(result.returncode, 0, result.stderr)
|
||||||
|
light, heavy = result.stdout.split("[2/2]")
|
||||||
|
self.assertIn("[s]kip", light)
|
||||||
|
self.assertNotIn("heavy:", light)
|
||||||
|
self.assertIn("heavy: memory stage ok", heavy)
|
||||||
|
self.assertIn("position\tid\tstatus", heavy)
|
||||||
|
self.assertIn("[h]old in the queue", heavy)
|
||||||
|
self.assertNotIn("\x1b", heavy)
|
||||||
|
self.assertIn("granted operator/heavy: the queue lets it run now", heavy)
|
||||||
|
self.assertIn("not run operator/heavy", heavy)
|
||||||
|
self.assertIn("failed operator/heavy: exit 75; the row stays open", heavy)
|
||||||
|
self.assertTrue((work / "light").exists())
|
||||||
|
self.assertFalse((work / "heavy\x1b[2K").exists())
|
||||||
|
self.assertEqual(self._open(), ["operator/heavy"])
|
||||||
|
self.assertFalse((self.root / "queue-history.jsonl").exists())
|
||||||
|
|
||||||
|
def test_run_offers_no_hold_while_the_config_lacks_the_operator_kind(self):
|
||||||
|
self._queue_config({"test": 0})
|
||||||
|
work = self.base / "work"
|
||||||
|
work.mkdir()
|
||||||
|
self._add("heavy", work, "touch heavy", "build", "manual")
|
||||||
|
result = self._feed("h\ns\n")
|
||||||
|
self.assertEqual(result.returncode, 0, result.stderr)
|
||||||
|
self.assertIn("lacks slot_need_gb.operator;", result.stdout)
|
||||||
|
self.assertIn("no [h]old until it does", result.stdout)
|
||||||
|
self.assertNotIn("[h]old in the queue", result.stdout)
|
||||||
|
self.assertIn("skipped operator/heavy", result.stdout)
|
||||||
|
|
||||||
|
def test_run_offers_no_hold_for_sudo_whose_prompt_needs_the_terminal(self):
|
||||||
|
self._queue_config({"operator": 0})
|
||||||
|
work = self.base / "work"
|
||||||
|
work.mkdir()
|
||||||
|
self._add("heavy", work, "bash -c 'sudo -n touch heavy'", "build", "manual")
|
||||||
|
result = self._feed("h\ns\n")
|
||||||
|
self.assertEqual(result.returncode, 0, result.stderr)
|
||||||
|
self.assertIn("sudo asks on the terminal, which a queued command lacks", result.stdout)
|
||||||
|
self.assertNotIn("[h]old in the queue", result.stdout)
|
||||||
|
|
||||||
|
def test_granted_runs_only_the_approved_row_in_its_directory_after_a_new_yes(self):
|
||||||
|
work = self.base / "work"
|
||||||
|
work.mkdir()
|
||||||
|
self._add("heavy", work, "touch heavy", "build", "manual")
|
||||||
|
env = {**self.env, "SHELL": "/bin/sh"}
|
||||||
|
|
||||||
|
def granted(answers: str, directory: Path, *argv: str) -> subprocess.CompletedProcess:
|
||||||
|
return subprocess.run(
|
||||||
|
[sys.executable, str(OPERATOR), "granted", "operator/heavy", "--dir", str(directory), "--check",
|
||||||
|
"manual", "--", *argv],
|
||||||
|
input=answers, capture_output=True, text=True, env=env, cwd=self.base, check=False,
|
||||||
|
)
|
||||||
|
|
||||||
|
changed = granted("y\n", work, "/bin/sh", "-ic", "touch other")
|
||||||
|
self.assertEqual(changed.returncode, 75)
|
||||||
|
self.assertIn("no longer open with the approved directory, check and command", changed.stderr)
|
||||||
|
moved = granted("y\n", self.base, "/bin/sh", "-ic", "touch heavy")
|
||||||
|
self.assertEqual((moved.returncode, (self.base / "heavy").exists()), (75, False))
|
||||||
|
declined = granted("s\n", work, "/bin/sh", "-ic", "touch heavy")
|
||||||
|
self.assertEqual((declined.returncode, (work / "heavy").exists()), (75, False))
|
||||||
|
silent = granted("", work, "/bin/sh", "-ic", "touch heavy")
|
||||||
|
self.assertEqual((silent.returncode, (work / "heavy").exists()), (75, False))
|
||||||
|
ran = granted("d\ny\n", work, "/bin/sh", "-ic", "touch heavy")
|
||||||
|
self.assertEqual(ran.returncode, 0, ran.stderr)
|
||||||
|
self.assertIn("granted operator/heavy", ran.stdout)
|
||||||
|
self.assertTrue((work / "heavy").exists())
|
||||||
|
self.assertFalse((self.base / "heavy").exists() or (work / "other").exists())
|
||||||
|
|
||||||
def test_run_needs_shell(self):
|
def test_run_needs_shell(self):
|
||||||
self._add("x", self.base, "true", "r", "manual")
|
self._add("x", self.base, "true", "r", "manual")
|
||||||
result = subprocess.run(
|
result = subprocess.run(
|
||||||
|
|||||||
+294
-6
@@ -20,12 +20,15 @@ QUEUE = REPO_ROOT / "skills" / "kvb-org-queue" / "scripts" / "queue.py"
|
|||||||
HISTORY = REPO_ROOT / "skills" / "kvb-org-queue" / "scripts" / "history.py"
|
HISTORY = REPO_ROOT / "skills" / "kvb-org-queue" / "scripts" / "history.py"
|
||||||
NOW = 1_800_000_000.0
|
NOW = 1_800_000_000.0
|
||||||
HOUR = 3600
|
HOUR = 3600
|
||||||
|
OPEN_MEMORY = {"available": -1, "swap_free": -1, "psi_full": 1000, "container_swap_gb": 1e9}
|
||||||
BASE = {
|
BASE = {
|
||||||
"free_gb": 100.0,
|
"free_gb": 100.0,
|
||||||
"floor": 10.0,
|
"floor": 10.0,
|
||||||
"stage": "ok",
|
"stage": "ok",
|
||||||
|
"memory": "ok",
|
||||||
"silenced": False,
|
"silenced": False,
|
||||||
"slot_limits": {"test": 5, "deploy": 1},
|
"slot_limits": {"test": 5, "deploy": 1},
|
||||||
|
"budget": {"ram": 75, "cpu": 100},
|
||||||
"ranks": {"work/a": 0, "work/b": 1, "work/c": 2},
|
"ranks": {"work/a": 0, "work/b": 1, "work/c": 2},
|
||||||
"agents": {},
|
"agents": {},
|
||||||
"now": NOW,
|
"now": NOW,
|
||||||
@@ -83,7 +86,97 @@ class TestEvaluate(unittest.TestCase):
|
|||||||
cls.queue = _load(QUEUE)
|
cls.queue = _load(QUEUE)
|
||||||
|
|
||||||
def decide(self, entries: list[dict], **overrides: object) -> dict:
|
def decide(self, entries: list[dict], **overrides: object) -> dict:
|
||||||
return self.queue.evaluate(entries, **{**BASE, **overrides})
|
footprints = {e["id"]: {"ram": 0, "cpu": 0} for e in entries} | overrides.pop("footprints", {})
|
||||||
|
return self.queue.evaluate(entries, **{**BASE, "footprints": footprints, **overrides})
|
||||||
|
|
||||||
|
def test_the_budget_refuses_a_job_whose_share_would_pass_it_and_reserves_it(self):
|
||||||
|
running = job(1, kind="test", status="active")
|
||||||
|
candidate = job(2, kind="gui-dind", workdir="/x")
|
||||||
|
small = job(3, task="work/b", workdir="/y")
|
||||||
|
shares = {"j1": {"ram": 60, "cpu": 40}, "j2": {"ram": 30, "cpu": 20}, "j3": {"ram": 10, "cpu": 5}}
|
||||||
|
verdicts = self.decide([running, candidate, small], slot_limits={"test": 5, "gui-dind": 1}, footprints=shares)
|
||||||
|
self.assertEqual(verdicts["j2"], "RAM 60 % in use or reserved + 30 % > budget 75 %; reserved")
|
||||||
|
self.assertEqual(verdicts["j3"], "RAM 90 % in use or reserved + 10 % > budget 75 %; reserved")
|
||||||
|
unbounded = {"ram": 1000, "cpu": 1000}
|
||||||
|
self.assertEqual(
|
||||||
|
self.decide([running, candidate, small], slot_limits={"test": 5, "gui-dind": 1}, footprints=shares, budget=unbounded),
|
||||||
|
{"j2": None, "j3": None},
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_a_job_alone_always_fits_and_cpu_counts_like_ram(self):
|
||||||
|
alone = self.decide([job(1)], footprints={"j1": {"ram": 90, "cpu": 150}})
|
||||||
|
self.assertEqual(alone, {"j1": None})
|
||||||
|
busy = self.decide([job(1, status="active"), job(2)], footprints={"j1": {"ram": 1, "cpu": 70}, "j2": {"ram": 1, "cpu": 40}})
|
||||||
|
self.assertEqual(busy, {"j2": "CPU 70 % in use or reserved + 40 % > budget 100 %; reserved"})
|
||||||
|
|
||||||
|
def test_a_share_beyond_the_budget_runs_only_alone_even_beside_a_job_of_no_share(self):
|
||||||
|
whole = {"ram": 80, "cpu": 1}
|
||||||
|
beside = self.decide([job(1, status="active"), job(2)], footprints={"j1": {"ram": 0, "cpu": 0}, "j2": whole})
|
||||||
|
self.assertEqual(beside, {"j2": "RAM 0 % in use or reserved + 80 % > budget 75 %; reserved"})
|
||||||
|
self.assertEqual(self.decide([job(2)], footprints={"j2": whole}), {"j2": None})
|
||||||
|
|
||||||
|
def test_load_counts_the_ram_in_use_plus_what_is_still_to_come(self):
|
||||||
|
limits = {"cpu_percent": 90, "mem_percent": 90}
|
||||||
|
reason = self.queue.load_reason
|
||||||
|
self.assertIsNone(reason(limits, 50, 60, 29))
|
||||||
|
self.assertEqual(reason(limits, 50, 60, 30), "RAM 60 % used + 30 % still to come reach the limit 90 %")
|
||||||
|
self.assertIsNone(reason(limits, 50, 89, 0))
|
||||||
|
self.assertEqual(reason(limits, 90, 10, 0), "CPU 90 % busy reaches the limit 90 %")
|
||||||
|
|
||||||
|
def test_pending_ram_adds_what_active_jobs_have_not_taken_yet_and_seeds_an_unlearned_kind(self):
|
||||||
|
history = _load(HISTORY)
|
||||||
|
rows = [history_row(n, kind=kind, signature=f"w: {kind}", mem_before=20, mem_max=20 + share)
|
||||||
|
for n, (kind, share) in enumerate([("a", 30)] * 5 + [("b", 25)] * 5 + [("c", 10)] * 5)]
|
||||||
|
signatures, kinds = history.learn(rows, NOW, {"load_limits": {"mem_percent": 90, "cpu_percent": 90}})
|
||||||
|
self.assertEqual(history.seed_ram(kinds), 30)
|
||||||
|
self.assertEqual(history.seed_ram({}), 0)
|
||||||
|
cmd = {"workdir": "/w"}
|
||||||
|
fresh = {**job(2, kind="b"), **cmd, "cmd": "b"}
|
||||||
|
young = {**job(1, kind="a", status="active", granted=NOW - 60), **cmd, "cmd": "a"}
|
||||||
|
overshooting = {**job(5, kind="c", status="active", granted=NOW - 120), **cmd, "cmd": "c"}
|
||||||
|
old = {**job(4, kind="c", status="active", granted=NOW - self.queue.RAMP_SECONDS), **cmd, "cmd": "c"}
|
||||||
|
self.assertEqual(self.queue.pending_ram([young, overshooting, old, fresh], "j2", NOW, signatures, kinds), 30 + 10 + 25)
|
||||||
|
undated = {k: v for k, v in old.items() if k != "granted"}
|
||||||
|
self.assertEqual(self.queue.pending_ram([undated, fresh], "j2", NOW, signatures, kinds), 10 + 25)
|
||||||
|
unknown = {**job(3, kind="new"), **cmd, "cmd": "new"}
|
||||||
|
self.assertEqual(self.queue.pending_ram([unknown], "j3", NOW, signatures, kinds), 30)
|
||||||
|
|
||||||
|
def test_a_signature_names_the_repository_every_worktree_shares(self):
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
base = Path(tmp).resolve()
|
||||||
|
main = base / "skills"
|
||||||
|
main.mkdir()
|
||||||
|
identity = {**os.environ, "GIT_AUTHOR_NAME": "T", "GIT_AUTHOR_EMAIL": "t@x", "GIT_COMMITTER_NAME": "T",
|
||||||
|
"GIT_COMMITTER_EMAIL": "t@x", "GIT_CONFIG_NOSYSTEM": "1"}
|
||||||
|
for args in (["init", "-q"], ["commit", "-q", "--allow-empty", "-m", "x"], ["worktree", "add", "-q", str(base / "wt")]):
|
||||||
|
subprocess.run(["git", *args], cwd=main, env=identity, check=True, capture_output=True)
|
||||||
|
self.assertEqual(self.queue.qualified(str(main), ["make", "test"]), "skills: make test")
|
||||||
|
self.assertEqual(self.queue.qualified(str(base / "wt"), ["make", "test"]), "skills: make test")
|
||||||
|
plain = base / "plain"
|
||||||
|
plain.mkdir()
|
||||||
|
self.assertEqual(self.queue.qualified(str(plain), ["true"]), "plain: true")
|
||||||
|
self.assertEqual(self.queue.qualified(str(base / "gone"), ["true"]), "gone: true")
|
||||||
|
for argv, expected in {
|
||||||
|
("env", "TMPDIR=/t", "-i", "make", "test"): "make test",
|
||||||
|
("nice", "-n", "10", "make", "compose-deploy"): "make compose-deploy",
|
||||||
|
("timeout", "3600", "make", "lint"): "make lint",
|
||||||
|
("python3", "/s/gate.py", "hold", "--repo", ".", "--", "make", "verify-test"): "make verify-test",
|
||||||
|
("python3", "/s/operator.py", "granted", "operator/x", "--dir", "/a", "--check", "manual", "--",
|
||||||
|
"zsh", "-ic", "make -C /a/core compose-deploy"): "make compose-deploy",
|
||||||
|
("bash", "-c", "cd core && make test 2>&1 | tee /tmp/log"): "make test",
|
||||||
|
("python3", "/s/x.py", "--"): "x.py",
|
||||||
|
("bash", "-c", "set -e; make verify-test"): "make verify-test",
|
||||||
|
("bash", "-c", "source .venv/bin/activate && make test"): "make test",
|
||||||
|
}.items():
|
||||||
|
with self.subTest(argv=argv):
|
||||||
|
self.assertEqual(self.queue.qualified(str(plain), list(argv)), f"plain: {expected}")
|
||||||
|
module = ["python3", "-m", "pytest", "--", "tests/"]
|
||||||
|
self.assertEqual(self.queue.command_words(module), module)
|
||||||
|
|
||||||
|
def test_memory_stage_stop_blocks_every_job_and_warn_none(self):
|
||||||
|
entries = [job(1, need_gb=0), job(2, task="work/b")]
|
||||||
|
self.assertEqual(set(self.decide(entries, memory="stop").values()), {"memory stage stop"})
|
||||||
|
self.assertEqual(self.decide(entries, memory="warn"), {"j1": None, "j2": None})
|
||||||
|
|
||||||
def test_hotfix_goes_first_then_rank_then_arrival_and_unknown_tasks_last(self):
|
def test_hotfix_goes_first_then_rank_then_arrival_and_unknown_tasks_last(self):
|
||||||
entries = [
|
entries = [
|
||||||
@@ -199,6 +292,23 @@ class TestHistory(unittest.TestCase):
|
|||||||
crowded = [history_row(n, mem_before=90, mem_max=95) for n in range(5)]
|
crowded = [history_row(n, mem_before=90, mem_max=95) for n in range(5)]
|
||||||
self.assertEqual(self.learn(crowded)[1]["test"]["limit"], 1)
|
self.assertEqual(self.learn(crowded)[1]["test"]["limit"], 1)
|
||||||
|
|
||||||
|
def test_footprint_takes_the_larger_of_kind_and_signature_at_least_the_minimum_and_none_for_a_new_kind(self):
|
||||||
|
small = [history_row(n, signature="make test", mem_max=25, cpu_max=20) for n in range(5)]
|
||||||
|
big = [
|
||||||
|
history_row(n + 5, kind="quality-high", signature="make verify-test", mem_max=80, cpu_max=90)
|
||||||
|
for n in range(5)
|
||||||
|
]
|
||||||
|
idle = [history_row(n + 10, kind="idle", signature="true", mem_max=20, cpu_max=5) for n in range(5)]
|
||||||
|
signatures, kinds = self.learn([*small, *big, *idle])
|
||||||
|
footprint = self.history.footprint
|
||||||
|
self.assertEqual(footprint("test", "make test", signatures, kinds), {"ram": 5, "cpu": 10})
|
||||||
|
self.assertEqual(footprint("test", "make verify-test", signatures, kinds), {"ram": 60, "cpu": 80})
|
||||||
|
self.assertEqual(footprint("test", "make unseen", signatures, kinds), {"ram": 5, "cpu": 10})
|
||||||
|
self.assertEqual(footprint("idle", "true", signatures, kinds), {"ram": 1, "cpu": 1})
|
||||||
|
self.assertIsNone(footprint("gui-dind", "make test-up", signatures, kinds))
|
||||||
|
signatures, kinds = self.learn(small[:4])
|
||||||
|
self.assertIsNone(footprint("test", "make test", signatures, kinds))
|
||||||
|
|
||||||
def test_slot_limits_take_the_learned_limit_else_the_seed_and_a_smaller_cap(self):
|
def test_slot_limits_take_the_learned_limit_else_the_seed_and_a_smaller_cap(self):
|
||||||
kinds = {"deploy": {"limit": 3}, "build": {"limit": None}}
|
kinds = {"deploy": {"limit": 3}, "build": {"limit": None}}
|
||||||
self.assertEqual(self.history.slot_limits(kinds, {"deploy": 2, "swarm": 0, "test": 9}, ["test", "gate"]), {
|
self.assertEqual(self.history.slot_limits(kinds, {"deploy": 2, "swarm": 0, "test": 9}, ["test", "gate"]), {
|
||||||
@@ -256,7 +366,9 @@ class TestQueueCommands(unittest.TestCase):
|
|||||||
"projection_minutes": 30,
|
"projection_minutes": 30,
|
||||||
"slot_need_gb": {"test": 0.001, "deploy": 0},
|
"slot_need_gb": {"test": 0.001, "deploy": 0},
|
||||||
"slot_limits": {"test": 1, "deploy": 1},
|
"slot_limits": {"test": 1, "deploy": 1},
|
||||||
"load_limits": {"cpu_percent": 101, "mem_percent": 101, "sample_seconds": 0.1, "kinds": ["test"]},
|
"load_limits": {"cpu_percent": 101, "mem_percent": 101, "sample_seconds": 0.1},
|
||||||
|
"budget_percent": {"ram": 75, "cpu": 100},
|
||||||
|
"memory_limits": {stage: OPEN_MEMORY for stage in ("warn", "stop")},
|
||||||
"heavy_commands": ["^make compose-deploy"],
|
"heavy_commands": ["^make compose-deploy"],
|
||||||
"never_block": ["^make compose-down"],
|
"never_block": ["^make compose-down"],
|
||||||
}
|
}
|
||||||
@@ -338,13 +450,23 @@ class TestQueueCommands(unittest.TestCase):
|
|||||||
[row] = [json.loads(line) for line in self.history_file.read_text(encoding="utf-8").splitlines()]
|
[row] = [json.loads(line) for line in self.history_file.read_text(encoding="utf-8").splitlines()]
|
||||||
self.assertEqual(
|
self.assertEqual(
|
||||||
[row[k] for k in ("kind", "signature", "workdir", "exit", "concurrent")],
|
[row[k] for k in ("kind", "signature", "workdir", "exit", "concurrent")],
|
||||||
["test", f"{Path(sys.executable).name} -c", str(self.base), 7, 0],
|
["test", f"{self.base.name}: {Path(sys.executable).name} -c", str(self.base), 7, 0],
|
||||||
)
|
)
|
||||||
self.assertTrue(row["id"].startswith("test-1-"))
|
self.assertTrue(row["id"].startswith("test-1-"))
|
||||||
self.assertLessEqual(row["start"], row["end"])
|
self.assertLessEqual(row["start"], row["end"])
|
||||||
self.assertEqual(row["disk_gb"], round(max(0.0, row["free_before_gb"] - row["min_free_gb"]), 2))
|
self.assertEqual(row["disk_gb"], round(max(0.0, row["free_before_gb"] - row["min_free_gb"]), 2))
|
||||||
self.assertLessEqual(row["mem_before"], row["mem_max"])
|
self.assertLessEqual(row["mem_before"], row["mem_max"])
|
||||||
|
|
||||||
|
def test_a_command_that_did_no_work_leaves_no_history_row(self):
|
||||||
|
result = self.run_queue(
|
||||||
|
"hold", "--kind", "test", "--task", "work/a", "--session", "s1", "--", sys.executable, "-c",
|
||||||
|
"import sys; sys.exit(75)",
|
||||||
|
)
|
||||||
|
self.assertEqual(result.returncode, 75)
|
||||||
|
self.assertFalse(self.history_file.exists())
|
||||||
|
self.assertEqual(self.run_queue("hold", "--kind", "test", "--task", "work/a", "--session", "s1", "--", "false").returncode, 1)
|
||||||
|
self.assertEqual(len(self.history_file.read_text(encoding="utf-8").splitlines()), 1)
|
||||||
|
|
||||||
def test_hold_runs_without_slot_limits_on_the_seed_limit(self):
|
def test_hold_runs_without_slot_limits_on_the_seed_limit(self):
|
||||||
del self.cfg["slot_limits"]
|
del self.cfg["slot_limits"]
|
||||||
self.write_config()
|
self.write_config()
|
||||||
@@ -464,7 +586,8 @@ class TestQueueCommands(unittest.TestCase):
|
|||||||
self.write_config()
|
self.write_config()
|
||||||
now = time.time()
|
now = time.time()
|
||||||
rows = [
|
rows = [
|
||||||
history_row(n, kind="deploy", signature="true", end=now, disk_gb=gb) for n, gb in enumerate((4, 1, 10, 3, 2))
|
history_row(n, kind="deploy", signature=f"{self.base.name}: true", end=now, disk_gb=gb)
|
||||||
|
for n, gb in enumerate((4, 1, 10, 3, 2))
|
||||||
]
|
]
|
||||||
self.write_history(self.history_file, rows[:3])
|
self.write_history(self.history_file, rows[:3])
|
||||||
self.write_history(self.kept, [{**rows[0], "disk_gb": 100}, *rows[3:]])
|
self.write_history(self.kept, [{**rows[0], "disk_gb": 100}, *rows[3:]])
|
||||||
@@ -480,9 +603,9 @@ class TestQueueCommands(unittest.TestCase):
|
|||||||
me, mine = os.getpid(), queue.identity(os.getpid())
|
me, mine = os.getpid(), queue.identity(os.getpid())
|
||||||
now = time.time()
|
now = time.time()
|
||||||
self.write_history(
|
self.write_history(
|
||||||
self.history_file, [history_row(n, signature="make compose-deploy", end=now) for n in range(5)]
|
self.history_file, [history_row(n, signature="w: make compose-deploy", end=now) for n in range(5)]
|
||||||
)
|
)
|
||||||
deploy = {"cmd": "make compose-deploy apps=x"}
|
deploy = {"cmd": "make compose-deploy apps=x", "workdir": "/w"}
|
||||||
self.write_store([
|
self.write_store([
|
||||||
{**stored(1, "active", me, mine, me, mine), **deploy, "granted": now - 240},
|
{**stored(1, "active", me, mine, me, mine), **deploy, "granted": now - 240},
|
||||||
{**stored(2, "waiting", me, mine, None, None), **deploy},
|
{**stored(2, "waiting", me, mine, None, None), **deploy},
|
||||||
@@ -545,6 +668,9 @@ class TestQueueCommands(unittest.TestCase):
|
|||||||
self.assertEqual(refused.returncode, 2)
|
self.assertEqual(refused.returncode, 2)
|
||||||
self.assertIn("queue.py hold --kind <kind> --task <register task> -- <command>", refused.stderr)
|
self.assertIn("queue.py hold --kind <kind> --task <register task> -- <command>", refused.stderr)
|
||||||
self.assertEqual(self.bash("python3 skills/kvb-check-gate/scripts/gate.py run --repo . --target test"), 2)
|
self.assertEqual(self.bash("python3 skills/kvb-check-gate/scripts/gate.py run --repo . --target test"), 2)
|
||||||
|
for command in ("make -j4 compose-deploy apps=x", "make 'compose-deploy'", "cd x && make compose-deploy"):
|
||||||
|
with self.subTest(command=command):
|
||||||
|
self.assertEqual(self.bash(command), 2)
|
||||||
for command in (
|
for command in (
|
||||||
"python3 /x/queue.py hold --kind deploy --task work/a -- make compose-deploy",
|
"python3 /x/queue.py hold --kind deploy --task work/a -- make compose-deploy",
|
||||||
"make compose-down",
|
"make compose-down",
|
||||||
@@ -557,6 +683,168 @@ class TestQueueCommands(unittest.TestCase):
|
|||||||
self.config.unlink()
|
self.config.unlink()
|
||||||
self.assertEqual(self.bash("make compose-deploy"), 0)
|
self.assertEqual(self.bash("make compose-deploy"), 0)
|
||||||
|
|
||||||
|
def test_block_with_queue_enforce_refuses_builds_installs_and_full_suites_by_itself(self):
|
||||||
|
self.cfg["queue_enforce"] = True
|
||||||
|
self.cfg["heavy_commands"] = []
|
||||||
|
self.write_config()
|
||||||
|
for command in (
|
||||||
|
"docker compose up -d --build",
|
||||||
|
"docker compose -f x.yml build web",
|
||||||
|
"docker-compose up --build",
|
||||||
|
"make install",
|
||||||
|
"make -C core verify-test",
|
||||||
|
"pytest tests/",
|
||||||
|
"python3 -m pytest -q",
|
||||||
|
".venv/bin/pytest tests/lint tests/unit",
|
||||||
|
"pytest tests/a.py tests/b.py",
|
||||||
|
"python3 -m unittest discover -s tests",
|
||||||
|
"python3 -m unittest discover -s tests -p 'test_*.py'",
|
||||||
|
"pytest tests/x.py tests/lint",
|
||||||
|
"pytest tests/lint --ignore=tests/lint/y.py",
|
||||||
|
"python3 -m unittest",
|
||||||
|
"make -j4 install",
|
||||||
|
"make VAR=x install",
|
||||||
|
"cd core && env A=1 timeout 600 python3 -m pytest -q 2>&1 | tee /tmp/log",
|
||||||
|
"bash -c 'pytest tests/'",
|
||||||
|
"pytest tests/ # queue.py hold",
|
||||||
|
"python3 /x/queue.py hold --kind test --task work/a -- true & pytest tests/",
|
||||||
|
"make compose-down; make install",
|
||||||
|
"uv run pytest tests/",
|
||||||
|
"coverage run -m pytest",
|
||||||
|
"python3 -mpytest tests/",
|
||||||
|
"python3 -X dev -m pytest",
|
||||||
|
"timeout -s KILL 900 pytest tests/",
|
||||||
|
"sudo -u nobody pytest tests/",
|
||||||
|
"env -u FOO pytest",
|
||||||
|
"{ pytest tests/; }",
|
||||||
|
"! pytest tests/",
|
||||||
|
"if pytest tests/; then echo ok; fi",
|
||||||
|
"echo `pytest tests/`",
|
||||||
|
"bash -c -- 'pytest tests/'",
|
||||||
|
"bash <<'EOF'\npytest tests/\nEOF",
|
||||||
|
'OUT="$(pytest tests/ 2>&1)"',
|
||||||
|
'echo "r: $(make install)"',
|
||||||
|
"cat <<EOF\n`make install`\nEOF",
|
||||||
|
"uv run --extra dev pytest tests/",
|
||||||
|
"uv run --with pytest-xdist pytest -n 8 tests/",
|
||||||
|
"coverage run --source pkg -m pytest tests/",
|
||||||
|
"poetry -C core run pytest tests/",
|
||||||
|
"python3 -m coverage run -m pytest tests/",
|
||||||
|
"uvx pytest tests/",
|
||||||
|
"uv tool run pytest tests/",
|
||||||
|
"time -p pytest tests/",
|
||||||
|
"/usr/bin/time -v pytest tests/",
|
||||||
|
"/usr/bin/time -f %M make install",
|
||||||
|
"make 2>&1 install",
|
||||||
|
">/tmp/log pytest tests/",
|
||||||
|
"2>/dev/null make install",
|
||||||
|
"pytest tests/test_watch.py 2>/dev/null tests/",
|
||||||
|
"make compose-down install",
|
||||||
|
"make -C core compose-down verify-test",
|
||||||
|
"make --directory core install",
|
||||||
|
"command pytest tests/",
|
||||||
|
"command -p pytest tests/",
|
||||||
|
"command make install",
|
||||||
|
"env -S 'pytest tests/'",
|
||||||
|
"bash -Ec 'pytest tests/'",
|
||||||
|
"exec -a x pytest tests/",
|
||||||
|
"sudo --user root pytest tests/",
|
||||||
|
"env $(printenv | grep -o -E '^INFINITO_[A-Z0-9_]+' | sed 's/^/-u /') -C /x python3 -m pytest tests/",
|
||||||
|
"env $(cat .env) -u X make install",
|
||||||
|
"sudo -u $(id -un) -- pytest tests/",
|
||||||
|
"timeout --signal=$(echo KILL) 900 pytest tests/",
|
||||||
|
"sudo --chdir /x pytest tests/",
|
||||||
|
"sudo bash <<'EOF'\npytest tests/\nEOF",
|
||||||
|
"bash -s <<'END-MSG'\nmake install\nEND-MSG",
|
||||||
|
"out=$(bash <<'EOF'\nmake install\nEOF\n)",
|
||||||
|
"bash -c 'bash <<EOF\nmake install\nEOF\n'",
|
||||||
|
"bash <<'A'\nbash <<'B'\nmake install\nB\nA",
|
||||||
|
"bash -euo pipefail <<'EOF'\nmake test\nmake install\nEOF",
|
||||||
|
"bash -o pipefail <<'EOF'\npytest tests/\nEOF",
|
||||||
|
"bash -s -- one <<'EOF'\nmake install\nEOF",
|
||||||
|
"bash --rcfile x <<'EOF'\nmake install\nEOF",
|
||||||
|
"make -Cinfinito install",
|
||||||
|
"make -Oline install",
|
||||||
|
"bash <<'EOF'\nmake install \ud800\nEOF",
|
||||||
|
):
|
||||||
|
with self.subTest(command=command):
|
||||||
|
self.assertEqual(self.bash(command), 2)
|
||||||
|
for command in (
|
||||||
|
"docker compose up -d",
|
||||||
|
"docker compose ps",
|
||||||
|
"make install-skills",
|
||||||
|
"pytest tests/unit/test_x.py",
|
||||||
|
"python3 -m pytest tests/unit/test_x.py::TestA -q",
|
||||||
|
"pip install pytest",
|
||||||
|
"python3 -m unittest discover -s tests -p test_queue.py",
|
||||||
|
"python3 -m unittest tests.test_queue",
|
||||||
|
"python3 /x/queue.py hold --kind test --task work/a -- pytest tests/",
|
||||||
|
'pytest "tests/unit/test_x.py"',
|
||||||
|
"python3 -m unittest discover -s tests -p 'test_queue.py'",
|
||||||
|
"grep -rn pytest tests/",
|
||||||
|
"which pytest",
|
||||||
|
"pytest --version",
|
||||||
|
"pytest tests/unit/test_x.py -p no:cacheprovider 2>&1 | tee /tmp/log",
|
||||||
|
"command -v pytest",
|
||||||
|
"make -n install",
|
||||||
|
"cat <<'EOF'\npytest tests/\nEOF",
|
||||||
|
"pytest >/tmp/out.log tests/test_watch.py",
|
||||||
|
"git commit -m \"$(cat <<'EOF'\nfix: run `make install` once\nEOF\n)\"",
|
||||||
|
"gh pr create --body-file - <<'EOF'\nruns `make verify-test`\nEOF",
|
||||||
|
"echo '$(make install)'",
|
||||||
|
"uv pip install pytest",
|
||||||
|
"command -V pytest",
|
||||||
|
"time pytest tests/unit/test_x.py",
|
||||||
|
"docker compose -f compose.yml -f build/compose/runner-mirror.override.yml config --services",
|
||||||
|
"docker compose --project-directory build up -d",
|
||||||
|
'echo "see \\`make install\\` and \\$(make install) docs"',
|
||||||
|
"pytest -n auto --dist loadscope tests/x.py",
|
||||||
|
"pytest --cov src --timeout 30 tests/x.py",
|
||||||
|
"cat > /tmp/run.sh <<'EOF'\npytest tests/\nEOF",
|
||||||
|
"gh pr create --title \"fix(bash): quoting\" --body \"$(cat <<'EOF'\nruns `make install`\nEOF\n)\"",
|
||||||
|
"ls # it's $(make install)",
|
||||||
|
"make -nC core install",
|
||||||
|
"make -kn install",
|
||||||
|
"python3 /x/queue.py hold --kind test --task work/a -- bash <<'EOF'\nmake install\nEOF",
|
||||||
|
"python3 /x/queue.py hold --kind test --task work/a -- bash -c \"$(cat <<'EOF'\nmake install\nEOF\n)\"",
|
||||||
|
"cat <<'END-MSG'\nmake install\nEND-MSG",
|
||||||
|
"cat <<\\EOF\n$(make install)\nEOF",
|
||||||
|
"pytest --numprocesses 4 tests/x.py",
|
||||||
|
"pytest tests/lint --collect-only",
|
||||||
|
"bash script.sh <<'EOF'\nmake install\nEOF",
|
||||||
|
"out=$(cat <<'EOF'\nmake install\nEOF\n)",
|
||||||
|
"echo kvbheredoc1 kvbheredoczz",
|
||||||
|
):
|
||||||
|
with self.subTest(command=command):
|
||||||
|
self.assertEqual(self.bash(command), 0)
|
||||||
|
|
||||||
|
def test_block_matches_a_config_regex_on_the_wrapped_program_too(self):
|
||||||
|
self.cfg["queue_enforce"] = True
|
||||||
|
self.cfg["heavy_commands"] = [r"^docker compose up"]
|
||||||
|
self.write_config()
|
||||||
|
for command in ("timeout 900 docker compose up -d", "sudo docker compose up -d", "docker compose up -d"):
|
||||||
|
with self.subTest(command=command):
|
||||||
|
self.assertEqual(self.bash(command), 2)
|
||||||
|
self.assertEqual(self.bash("docker compose ps"), 0)
|
||||||
|
|
||||||
|
def test_hold_refuses_a_config_without_the_budget_or_the_memory_limits(self):
|
||||||
|
for key, cfg in (
|
||||||
|
("budget_percent", {k: v for k, v in self.cfg.items() if k != "budget_percent"}),
|
||||||
|
("memory_limits", {k: v for k, v in self.cfg.items() if k != "memory_limits"}),
|
||||||
|
("budget_percent.cpu", {**self.cfg, "budget_percent": {"ram": 75}}),
|
||||||
|
("budget_percent.ram", {**self.cfg, "budget_percent": {"ram": "75", "cpu": 100}}),
|
||||||
|
("budget_percent.cpu", {**self.cfg, "budget_percent": {"ram": 75, "cpu": float("nan")}}),
|
||||||
|
("memory_limits.warn.container_swap_gb", {**self.cfg, "memory_limits": {
|
||||||
|
"warn": {k: v for k, v in OPEN_MEMORY.items() if k != "container_swap_gb"}, "stop": OPEN_MEMORY,
|
||||||
|
}}),
|
||||||
|
):
|
||||||
|
with self.subTest(key=key):
|
||||||
|
self.config.write_text(json.dumps(cfg), encoding="utf-8")
|
||||||
|
result = self.run_queue("hold", "--kind", "test", "--task", "work/a", "--session", "s1", "--", "true")
|
||||||
|
self.assertEqual(result.returncode, 2)
|
||||||
|
self.assertIn(f"lacks {key};", result.stderr)
|
||||||
|
self.assertFalse((self.root / "queue.json").exists())
|
||||||
|
|
||||||
def test_block_holds_a_held_command_to_the_bash_rules_of_the_settings(self):
|
def test_block_holds_a_held_command_to_the_bash_rules_of_the_settings(self):
|
||||||
project = self.base / "project"
|
project = self.base / "project"
|
||||||
(project / ".claude").mkdir(parents=True)
|
(project / ".claude").mkdir(parents=True)
|
||||||
|
|||||||
+146
-8
@@ -17,6 +17,10 @@ from pathlib import Path
|
|||||||
REPO_ROOT = Path(__file__).resolve().parents[1]
|
REPO_ROOT = Path(__file__).resolve().parents[1]
|
||||||
WATCH = REPO_ROOT / "skills" / "kvb-org-organize" / "scripts" / "watch.py"
|
WATCH = REPO_ROOT / "skills" / "kvb-org-organize" / "scripts" / "watch.py"
|
||||||
LIMITS = {"warn": 40, "stop": 15, "critical": 12}
|
LIMITS = {"warn": 40, "stop": 15, "critical": 12}
|
||||||
|
OPEN_MEMORY = {"available": -1, "swap_free": -1, "psi_full": 1000, "container_swap_gb": 1e9}
|
||||||
|
SHUT_MEMORY = {"available": 101, "swap_free": 101, "psi_full": 0, "container_swap_gb": 0}
|
||||||
|
MEMINFO = "MemTotal: 1000 kB\nMemAvailable: 120 kB\nSwapTotal: 2000 kB\nSwapFree: 300 kB\n"
|
||||||
|
PSI = "some avg10=30.00 avg60=1.00 avg300=0.50 total=1\nfull avg10=12.50 avg60=0.80 avg300=0.10 total=1\n"
|
||||||
|
|
||||||
|
|
||||||
def _load():
|
def _load():
|
||||||
@@ -41,6 +45,66 @@ class TestStage(unittest.TestCase):
|
|||||||
meminfo = "MemTotal: 1000 kB\nMemFree: 100 kB\nMemAvailable: 250 kB\n"
|
meminfo = "MemTotal: 1000 kB\nMemFree: 100 kB\nMemAvailable: 250 kB\n"
|
||||||
self.assertEqual(self.watch.mem_used_percent(meminfo), 75.0)
|
self.assertEqual(self.watch.mem_used_percent(meminfo), 75.0)
|
||||||
|
|
||||||
|
def test_memory_sample_reads_available_swap_and_psi_and_tolerates_their_absence(self):
|
||||||
|
self.assertEqual(
|
||||||
|
self.watch.memory_sample(MEMINFO, PSI, 2.5),
|
||||||
|
{"available": 12.0, "swap_free": 15.0, "psi_full": 12.5, "container_swap_gb": 2.5},
|
||||||
|
)
|
||||||
|
bare = "MemTotal: 1000 kB\nMemAvailable: 500 kB\nSwapTotal: 0 kB\nSwapFree: 0 kB\n"
|
||||||
|
self.assertEqual(
|
||||||
|
self.watch.memory_sample(bare, None, 0.0),
|
||||||
|
{"available": 50.0, "swap_free": None, "psi_full": None, "container_swap_gb": 0.0},
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_memory_takes_the_swap_of_the_container_holding_the_most_nested_groups_included(self):
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
base = Path(tmp)
|
||||||
|
for name, swap in ((f"docker-{'a' * 64}.scope", 3_000_000_000), (f"docker-{'b' * 64}.scope", 18_000_000_000),
|
||||||
|
("other.scope", 90_000_000_000)):
|
||||||
|
(base / name).mkdir()
|
||||||
|
(base / name / "memory.swap.current").write_text(str(swap), encoding="utf-8")
|
||||||
|
self.watch.CGROUPS = base
|
||||||
|
self.assertEqual(self.watch.memory()["container_swap_gb"], 18.0)
|
||||||
|
self.watch.CGROUPS = base / "absent"
|
||||||
|
self.assertEqual(self.watch.memory()["container_swap_gb"], 0.0)
|
||||||
|
|
||||||
|
def test_memory_stage_takes_the_first_bound_crossed_stop_before_warn(self):
|
||||||
|
cfg = {"memory_limits": {
|
||||||
|
"warn": {"available": 15, "swap_free": 25, "psi_full": 5, "container_swap_gb": 4},
|
||||||
|
"stop": {"available": 8, "swap_free": 10, "psi_full": 10, "container_swap_gb": 16},
|
||||||
|
}}
|
||||||
|
stage = self.watch.memory_stage
|
||||||
|
healthy = {"available": 40.0, "swap_free": 80.0, "psi_full": 0.0, "container_swap_gb": 3.9}
|
||||||
|
self.assertEqual(stage(healthy, cfg), "ok")
|
||||||
|
for change, expected in (
|
||||||
|
({"available": 14.0}, "warn"), ({"swap_free": 24.0}, "warn"), ({"psi_full": 5.0}, "warn"),
|
||||||
|
({"container_swap_gb": 4.0}, "warn"),
|
||||||
|
({"available": 7.0}, "stop"), ({"swap_free": 9.0}, "stop"), ({"psi_full": 10.0}, "stop"),
|
||||||
|
({"container_swap_gb": 16.0}, "stop"),
|
||||||
|
({"swap_free": None, "psi_full": None}, "ok"),
|
||||||
|
):
|
||||||
|
with self.subTest(change=change):
|
||||||
|
self.assertEqual(stage(healthy | change, cfg), expected)
|
||||||
|
|
||||||
|
def test_swap_holders_follow_the_nested_group_holding_most_of_the_swap(self):
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
base = Path(tmp)
|
||||||
|
outer = "a" * 64
|
||||||
|
|
||||||
|
def group(path: Path, swap: int) -> None:
|
||||||
|
path.mkdir(parents=True)
|
||||||
|
(path / "memory.swap.current").write_text(str(swap), encoding="utf-8")
|
||||||
|
|
||||||
|
group(base / f"docker-{outer}.scope", 18_000_000_000)
|
||||||
|
group(base / f"docker-{outer}.scope" / "c977e579", 17_500_000_000)
|
||||||
|
group(base / f"docker-{outer}.scope" / "c977e579" / f"docker-{'b' * 64}.scope", 17_200_000_000)
|
||||||
|
group(base / f"docker-{outer}.scope" / "c977e579" / "small", 100)
|
||||||
|
group(base / f"docker-{'c' * 64}.scope", 1_000_000_000)
|
||||||
|
group(base / f"docker-{'d' * 64}.scope", 0)
|
||||||
|
(base / "other.scope").mkdir()
|
||||||
|
holders = self.watch.swap_holders(base, {outer: "iam"})
|
||||||
|
self.assertEqual(holders, [f"iam > c977e579 > {'b' * 12} 18.0 GB", f"{'c' * 12} 1.0 GB"])
|
||||||
|
|
||||||
def test_fast_fall_escalates_one_stage(self):
|
def test_fast_fall_escalates_one_stage(self):
|
||||||
now = time.time()
|
now = time.time()
|
||||||
self.assertEqual(self.watch.stage_for(32, [[now - 600, 48]], self.cfg, now), "stop")
|
self.assertEqual(self.watch.stage_for(32, [[now - 600, 48]], self.cfg, now), "stop")
|
||||||
@@ -73,7 +137,8 @@ class TestWatchCommands(unittest.TestCase):
|
|||||||
"cache_budget_percent": 3,
|
"cache_budget_percent": 3,
|
||||||
"slot_need_gb": {"swarm": 30, "test": 0.001},
|
"slot_need_gb": {"swarm": 30, "test": 0.001},
|
||||||
"notify": ["true"],
|
"notify": ["true"],
|
||||||
"load_limits": {"cpu_percent": 101, "mem_percent": 101, "sample_seconds": 0.1, "kinds": ["test"]},
|
"load_limits": {"cpu_percent": 101, "mem_percent": 101, "sample_seconds": 0.1},
|
||||||
|
"memory_limits": {stage: dict(OPEN_MEMORY) for stage in ("warn", "stop")},
|
||||||
}
|
}
|
||||||
self._write_config()
|
self._write_config()
|
||||||
|
|
||||||
@@ -126,17 +191,21 @@ class TestWatchCommands(unittest.TestCase):
|
|||||||
self.cfg["heavy_commands"] = [r"\bmake\s+(-C\s+\S+\s+)?(compose-deploy|test|swarm-\S+)(\s|$)"]
|
self.cfg["heavy_commands"] = [r"\bmake\s+(-C\s+\S+\s+)?(compose-deploy|test|swarm-\S+)(\s|$)"]
|
||||||
self._write_config()
|
self._write_config()
|
||||||
self._stage("stop")
|
self._stage("stop")
|
||||||
message = "$(cat <<'EOF'\nfix: run \"make test\" and make swarm-zombie\nEOF\n)"
|
message = "$(cat <<'EOF'\nfix: run \"make test\" and make swarm-zombie, `make test` too\nEOF\n)"
|
||||||
for command in (
|
for command in (
|
||||||
'git commit --no-verify -m "fix: passes the vars to make swarm-zombie app=x"',
|
'git commit --no-verify -m "fix: passes the vars to make swarm-zombie app=x"',
|
||||||
"grep -n 'make test' Makefile",
|
"grep -n 'make test' Makefile",
|
||||||
f'git commit --no-verify -m "{message}"',
|
f'git commit --no-verify -m "{message}"',
|
||||||
|
"gh pr create --body-file - <<'EOF'\nruns `make test`\nEOF",
|
||||||
|
"echo 'not run: $(make test)'",
|
||||||
):
|
):
|
||||||
self.assertEqual(self._run("block", stdin=self._bash(command)).returncode, 0, command)
|
self.assertEqual(self._run("block", stdin=self._bash(command)).returncode, 0, command)
|
||||||
for command in (
|
for command in (
|
||||||
"make -C x swarm-zombie app=y",
|
"make -C x swarm-zombie app=y",
|
||||||
"env FOO=1 make swarm-zombie",
|
"env FOO=1 make swarm-zombie",
|
||||||
'make compose-deploy apps="web-app-x"',
|
'make compose-deploy apps="web-app-x"',
|
||||||
|
'OUT="$(make test 2>&1)"',
|
||||||
|
"cat <<EOF\n$(make test)\nEOF",
|
||||||
):
|
):
|
||||||
self.assertEqual(self._run("block", stdin=self._bash(command)).returncode, 2, command)
|
self.assertEqual(self._run("block", stdin=self._bash(command)).returncode, 2, command)
|
||||||
|
|
||||||
@@ -146,16 +215,85 @@ class TestWatchCommands(unittest.TestCase):
|
|||||||
self.assertEqual(self._run("admit", "--need-gb", "0.001").returncode, 0)
|
self.assertEqual(self._run("admit", "--need-gb", "0.001").returncode, 0)
|
||||||
self.assertEqual(self._run("admit", "--need-gb", "1e9").returncode, 3)
|
self.assertEqual(self._run("admit", "--need-gb", "1e9").returncode, 3)
|
||||||
|
|
||||||
def test_admit_for_a_load_kind_also_needs_cpu_and_ram_below_the_limits(self):
|
def test_every_admit_also_needs_cpu_and_ram_below_the_limits_and_memory_short_of_stop(self):
|
||||||
self.cfg["thresholds_gb"] = {"warn": 0, "stop": 0, "critical": 0}
|
self.cfg["thresholds_gb"] = {"warn": 0, "stop": 0, "critical": 0}
|
||||||
self._write_config()
|
self._write_config()
|
||||||
self.assertEqual(self._run("admit", "--kind", "test").returncode, 0)
|
granted = self._run("admit", "--kind", "test")
|
||||||
|
self.assertEqual(granted.returncode, 0)
|
||||||
|
self.assertIn("memory stage ok", granted.stdout)
|
||||||
self.cfg["load_limits"]["cpu_percent"] = 0
|
self.cfg["load_limits"]["cpu_percent"] = 0
|
||||||
self._write_config()
|
self._write_config()
|
||||||
denied = self._run("admit", "--kind", "test")
|
for args in (("--kind", "test"), ("--kind", "swarm"), ("--need-gb", "0.001")):
|
||||||
self.assertEqual(denied.returncode, 3)
|
with self.subTest(args=args):
|
||||||
self.assertIn("cpu", denied.stdout)
|
denied = self._run("admit", *args)
|
||||||
self.assertEqual(self._run("admit", "--need-gb", "0.001").returncode, 0)
|
self.assertEqual(denied.returncode, 3)
|
||||||
|
self.assertIn("cpu", denied.stdout)
|
||||||
|
self.cfg["load_limits"]["cpu_percent"] = 101
|
||||||
|
self.cfg["memory_limits"]["stop"] = dict(SHUT_MEMORY)
|
||||||
|
self._write_config()
|
||||||
|
short = self._run("admit", "--need-gb", "0.001")
|
||||||
|
self.assertEqual(short.returncode, 3)
|
||||||
|
self.assertIn("memory stage stop", short.stdout)
|
||||||
|
|
||||||
|
def test_block_refuses_heavy_jobs_at_memory_stop_only(self):
|
||||||
|
self._stage("ok")
|
||||||
|
self.cfg["memory_limits"]["warn"] = dict(SHUT_MEMORY)
|
||||||
|
self._write_config()
|
||||||
|
self.assertEqual(self._run("block", stdin=self._bash("make compose-deploy")).returncode, 0)
|
||||||
|
self.cfg["memory_limits"]["stop"] = dict(SHUT_MEMORY)
|
||||||
|
self._write_config()
|
||||||
|
blocked = self._run("block", stdin=self._bash("make compose-deploy"))
|
||||||
|
self.assertEqual(blocked.returncode, 2)
|
||||||
|
self.assertIn("memory stage is stop", blocked.stderr)
|
||||||
|
self.assertEqual(self._run("block", stdin=self._bash("docker compose up -d --build")).returncode, 2)
|
||||||
|
self.assertEqual(self._run("block", stdin=self._bash("make swarm-down")).returncode, 0)
|
||||||
|
self.assertEqual(self._run("block", stdin=self._bash("ls")).returncode, 0)
|
||||||
|
|
||||||
|
def test_memory_warn_alert_names_the_stage_once_per_session(self):
|
||||||
|
self.cfg["thresholds_gb"] = {"warn": 0, "stop": 0, "critical": 0}
|
||||||
|
self.cfg["memory_limits"]["warn"] = dict(SHUT_MEMORY)
|
||||||
|
self._write_config()
|
||||||
|
payload = json.dumps({"session_id": "s1", "hook_event_name": "UserPromptSubmit"})
|
||||||
|
first = self._run("watch", stdin=payload).stdout
|
||||||
|
self.assertIn("RESOURCE MEMORY WARN", first)
|
||||||
|
self.assertIn("Top swap holders", first)
|
||||||
|
self.assertEqual(self._run("watch", stdin=payload).stdout, "")
|
||||||
|
|
||||||
|
def test_a_config_without_memory_limits_blocks_heavy_jobs_keeps_the_disk_watch_and_stops_commands(self):
|
||||||
|
del self.cfg["memory_limits"]
|
||||||
|
self.cfg["thresholds_gb"] = {"warn": 1e9, "stop": -1, "critical": -2}
|
||||||
|
self._write_config()
|
||||||
|
blocked = self._run("block", stdin=self._bash("make compose-deploy"))
|
||||||
|
self.assertEqual(blocked.returncode, 2)
|
||||||
|
self.assertIn("lacks memory_limits", blocked.stderr)
|
||||||
|
self.assertEqual(self._run("block", stdin=self._bash("ls")).returncode, 0)
|
||||||
|
payload = json.dumps({"session_id": "s1", "hook_event_name": "UserPromptSubmit"})
|
||||||
|
watched = self._run("watch", stdin=payload)
|
||||||
|
self.assertEqual(watched.returncode, 0)
|
||||||
|
self.assertIn("RESOURCE WARN", watched.stdout)
|
||||||
|
self.assertIn("lacks memory_limits", watched.stdout)
|
||||||
|
command = self._run("admit", "--need-gb", "0")
|
||||||
|
self.assertEqual(command.returncode, 2)
|
||||||
|
self.assertIn("lacks memory_limits", command.stderr)
|
||||||
|
|
||||||
|
def test_a_memory_limit_missing_one_bound_names_it_and_blocks_heavy_jobs(self):
|
||||||
|
del self.cfg["memory_limits"]["stop"]["psi_full"]
|
||||||
|
self.cfg["thresholds_gb"] = {"warn": 0, "stop": 0, "critical": 0}
|
||||||
|
self._write_config()
|
||||||
|
blocked = self._run("block", stdin=self._bash("make compose-deploy"))
|
||||||
|
self.assertEqual(blocked.returncode, 2)
|
||||||
|
self.assertIn("lacks memory_limits.stop.psi_full;", blocked.stderr)
|
||||||
|
self.assertEqual(self._run("block", stdin=self._bash("ls")).returncode, 0)
|
||||||
|
payload = json.dumps({"session_id": "s1", "hook_event_name": "UserPromptSubmit"})
|
||||||
|
watched = self._run("watch", stdin=payload)
|
||||||
|
self.assertEqual(watched.returncode, 0, watched.stderr)
|
||||||
|
self.assertIn("lacks memory_limits.stop.psi_full", watched.stdout)
|
||||||
|
|
||||||
|
def test_segments_split_at_newlines_and_drop_a_redirection_with_its_target(self):
|
||||||
|
self.assertEqual(
|
||||||
|
_load().segments("pytest a.py 2>&1 | tee log\ncd x; make 'a b' # pytest\n>/tmp/o make 2>/dev/null install &>x\ndiff <(pytest t) y"),
|
||||||
|
[["pytest", "a.py"], ["tee", "log"], ["cd", "x"], ["make", "a b"], ["make", "install"], ["diff"], ["pytest", "t"], ["y"]],
|
||||||
|
)
|
||||||
|
|
||||||
def test_warn_alert_reaches_each_session_once(self):
|
def test_warn_alert_reaches_each_session_once(self):
|
||||||
self.cfg["thresholds_gb"] = {"warn": 1e9, "stop": -1, "critical": -2}
|
self.cfg["thresholds_gb"] = {"warn": 1e9, "stop": -1, "critical": -2}
|
||||||
|
|||||||
Reference in new issue
Block a user