fix(kvb-org-queue): global budget so no queue mix can OOM the host
Root cause of the 2026-11-07 20:14 OOM: the queue granted jobs per kind only, so several heavy kinds and direct heavy commands outside the queue ran side by side until the kernel killed the editor. - queue.py: every grant checks the sum of the learned RAM and CPU footprints of active and granted jobs against budget_percent; an unlearned kind takes the whole budget and runs alone. RAM used plus what young jobs (RAMP_SECONDS) may still take must stay below load_limits. Memory stage stop blocks every grant. - watch.py: memory stage from MemAvailable, swap, PSI full and the swap of the container holding most of it; the block hook classifies builds, make install/verify-*, and pytest or unittest beyond one file as heavy through wrappers, sh -c, command substitutions and heredocs a shell reads. - history.py: footprints per kind and signature, seed RAM for pending shares. - operator.py: [h]old queues a heavy row through queue.py hold and asks again on its grant, with a flushed tty and a timeout; sudo rows, which need the terminal, stay direct. Residuals: a shell reading code from a pipe or here-string, its own script, a program named by a substitution, quotes inside an expanding heredoc, an escaped backslash before a substitution; a lone job waits while the host is over load_limits. 🎯 Fix: 92 % 💥 Break: 4 % 💸 Cost: 100 % ⏱️ Time: – 🛡️ Security: 100 % 🏆 Score: 88 🚦 Gate: ✅ 90/5, cost and time <= 100, security 100 % Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
09f68dd75b
commit
4321b97529
8 files changed
+1665
-218
No files matched your search
@@ -9,11 +9,14 @@ Usage:
|
||||
(kvb-check-security) whose id it lacks to the same name
|
||||
under ~/.local/state/agents-council/
|
||||
watch.py block hook (PreToolUse on Bash): exit 2 for a heavy job while the
|
||||
stage is stop or critical, or while a kvb-org-radio mayday
|
||||
holds radio silence (<root>/radio-silence.json, until > now)
|
||||
disk stage is stop or critical, the memory stage is stop
|
||||
or cannot be judged without memory_limits, or a
|
||||
kvb-org-radio mayday holds radio silence
|
||||
(<root>/radio-silence.json, until > now)
|
||||
watch.py admit (--need-gb N | --kind KIND)
|
||||
exit 0 when the grant leaves at least the stop threshold
|
||||
free, 3 otherwise
|
||||
free, CPU busy and RAM used stay below load_limits and
|
||||
the memory stage is not stop, 3 otherwise
|
||||
watch.py cleanup (--caches | --full)
|
||||
trim caches to their budget; --full also runs the
|
||||
configured low-risk cleanup commands
|
||||
@@ -45,8 +48,15 @@ compose project name is skipped), worktree_marker (path relative
|
||||
to a worktree that must exist), busy_processes (process names; a worktree
|
||||
where one of them runs is skipped), cache_dirs, cache_budget_percent,
|
||||
slot_need_gb {kind: GB}, notify (argv the alert text is appended to),
|
||||
load_limits {cpu_percent, mem_percent, sample_seconds, kinds}: admit for a
|
||||
kind listed in kinds also needs CPU busy and RAM used below the limits.
|
||||
load_limits {cpu_percent, mem_percent, sample_seconds}: every admit and every
|
||||
queue grant needs CPU busy and RAM used below them, memory_limits {warn, stop},
|
||||
each {available, swap_free, psi_full, container_swap_gb}: the memory stage is
|
||||
the first of stop and warn whose MemAvailable or SwapFree percentage lies below
|
||||
its bound, whose /proc/pressure/memory full avg10 reaches its bound, or where
|
||||
one container (a docker-*.scope cgroup under system.slice, nested groups
|
||||
counted) holds at least container_swap_gb GB of swap, else ok; a host without
|
||||
swap or PSI skips that bound. heavy_commands extend the built-in rules of
|
||||
heavy().
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -56,6 +66,7 @@ import contextlib
|
||||
import fcntl
|
||||
import importlib.util
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
import re
|
||||
import shlex
|
||||
@@ -99,6 +110,86 @@ CLIP = 1000
|
||||
CLEANUP_LOCK = "cleanup.lock"
|
||||
LOCK_TTL = 1800
|
||||
GB = 1e9
|
||||
MEMINFO = Path("/proc/meminfo")
|
||||
PRESSURE = Path("/proc/pressure/memory")
|
||||
CGROUPS = Path("/sys/fs/cgroup/system.slice")
|
||||
SWAP_HOLDERS = 3
|
||||
NEST_DEPTH = 8
|
||||
MEMORY_STAGES = ("ok", "warn", "stop")
|
||||
MEMORY_BOUNDS = ("available", "swap_free", "psi_full", "container_swap_gb")
|
||||
MEMORY_KEYS = tuple(f"memory_limits.{stage}.{bound}" for stage in MEMORY_STAGES[1:] for bound in MEMORY_BOUNDS)
|
||||
CONTAINER_ID = re.compile(r"[0-9a-f]{64}")
|
||||
COMPOSE_VALUES = frozenset({
|
||||
"-f", "--file", "-p", "--project-name", "--project-directory", "--env-file", "--profile", "--ansi", "--parallel",
|
||||
"--progress",
|
||||
})
|
||||
BUILDING = frozenset({"up", "run", "create"})
|
||||
SEPARATOR = re.compile(r"[();<>|&\n]+")
|
||||
PUNCTUATION = "();<>|&\n"
|
||||
WORD_BREAK = " \t\n;&|()<>"
|
||||
ASSIGNMENT = re.compile(r"[A-Za-z_]\w*=.*", re.DOTALL)
|
||||
WRAPPERS = frozenset({
|
||||
"env", "sudo", "nice", "nohup", "timeout", "exec", "xargs", "stdbuf", "ionice", "uv", "uvx", "poetry", "pdm",
|
||||
"pipenv", "coverage", "time", "command",
|
||||
})
|
||||
UV_VALUES = frozenset({
|
||||
"--with", "--extra", "--group", "--python", "-p", "--project", "--directory", "--package", "--env-file", "--from",
|
||||
})
|
||||
WRAPPER_VALUES = {
|
||||
"sudo": frozenset({
|
||||
"-u", "-g", "-h", "-p", "-C", "-D", "-r", "-t", "-U", "-T", "--user", "--group", "--host", "--chdir", "--role",
|
||||
"--type", "--prompt", "--other-user", "--command-timeout", "--close-from",
|
||||
}),
|
||||
"timeout": frozenset({"-s", "-k", "--signal", "--kill-after"}),
|
||||
"env": frozenset({"-u", "-C", "--unset", "--chdir"}),
|
||||
"nice": frozenset({"-n", "--adjustment"}),
|
||||
"ionice": frozenset({"-c", "-n", "-p", "--class", "--classdata"}),
|
||||
"xargs": frozenset({"-a", "-d", "-E", "-I", "-L", "-n", "-P", "-s"}),
|
||||
"stdbuf": frozenset({"-i", "-o", "-e"}),
|
||||
"time": frozenset({"-f", "-o", "--format", "--output"}),
|
||||
"exec": frozenset({"-a"}),
|
||||
"uv": UV_VALUES,
|
||||
"uvx": UV_VALUES,
|
||||
"poetry": frozenset({"-C", "-P", "--directory", "--project"}),
|
||||
"pdm": frozenset({"-p", "--project"}),
|
||||
"coverage": frozenset({"--source", "--include", "--omit", "--rcfile", "--context", "--data-file"}),
|
||||
}
|
||||
WRAPPER_WORDS = frozenset({"run", "tool"})
|
||||
RESERVED = frozenset({"!", "{", "if", "then", "else", "elif", "do", "while", "until"})
|
||||
WRAPPER_ARGUMENT = re.compile(r"-.*|\d+(\.\d+)?[smhd]?", re.DOTALL)
|
||||
SHELLS = frozenset({"bash", "sh", "zsh", "dash"})
|
||||
INLINE = re.compile(r"-[A-Za-z]*c[A-Za-z]*")
|
||||
REDIRECT = re.compile(r"&?[<>]+&?")
|
||||
TRAILING_REDIRECT = re.compile(r"[<>]+&?$")
|
||||
PAREN = {"(": 1, ")": -1}
|
||||
PYTHON = re.compile(r"python[\d.]*")
|
||||
PYTHON_VALUE_OPTIONS = frozenset({"-X", "-W", "-Q"})
|
||||
DRY_RUN = frozenset({"-n", "--dry-run", "--just-print", "--recon", "-q", "--question"})
|
||||
SHORT_DRY = re.compile(r"-(?:(?![CfIoOWjl])[A-Za-z])*[nq][A-Za-z]*")
|
||||
SHELL_VALUE = re.compile(r"[-+][A-Za-z]*[oO]|--rcfile|--init-file")
|
||||
STDIN_FLAG = re.compile(r"-[A-Za-z]*s[A-Za-z]*")
|
||||
SPACE = re.compile(r"\s")
|
||||
ESCAPED = re.compile(r"\\[`$]")
|
||||
OPENER = re.compile(r"`|\$\(")
|
||||
PLACEHOLDER = "-_"
|
||||
HEREDOC_MARK = "kvbheredoc"
|
||||
MARK_WORD = re.compile(rf"{HEREDOC_MARK}((?:[0-9a-f]{{2}})*)")
|
||||
HELP = frozenset({"-h", "--help", "--version"})
|
||||
PYTEST_LIGHT = HELP | {"--collect-only", "--co"}
|
||||
TEST_FILE = re.compile(r"[^*?\[\]]+\.py(::.*)?", re.DOTALL)
|
||||
PYTEST_VALUE_OPTIONS = frozenset({
|
||||
"-p", "-k", "-m", "-c", "-o", "-W", "-n", "--numprocesses", "-r", "--maxfail", "--tb", "--rootdir", "--basetemp", "--deselect",
|
||||
"--ignore", "--ignore-glob", "--confcutdir", "--durations", "--junitxml", "--log-level", "--dist", "--cov",
|
||||
"--cov-report", "--cov-config", "--timeout", "--import-mode", "--capture", "--color", "--log-cli-level",
|
||||
"--log-file", "--reruns", "--tx", "--max-worker-restart",
|
||||
})
|
||||
UNITTEST_VALUE_OPTIONS = frozenset({"-k"})
|
||||
DISCOVER_VALUE_OPTIONS = frozenset({"-s", "-p", "-t", "-k", "--start-directory", "--pattern", "--top-level-directory"})
|
||||
PATTERN_OPTIONS = frozenset({"-p", "--pattern"})
|
||||
MAKE_VALUE_OPTIONS = frozenset({
|
||||
"-C", "-f", "-I", "-o", "-W", "--directory", "--file", "--makefile", "--include-dir", "--old-file", "--assume-old",
|
||||
"--what-if", "--new-file", "--assume-new",
|
||||
})
|
||||
|
||||
|
||||
def guard_root(hook: bool) -> int | None:
|
||||
@@ -123,6 +214,35 @@ def load_config() -> dict | None:
|
||||
return json.loads(CONFIG.read_text(encoding="utf-8"))
|
||||
|
||||
|
||||
def absent_part(cfg: dict, key: str) -> str | None:
|
||||
node: object = cfg
|
||||
parts = key.split(".")
|
||||
for depth, part in enumerate(parts, 1):
|
||||
if not isinstance(node, dict) or part not in node:
|
||||
return ".".join(parts[:depth])
|
||||
node = node[part]
|
||||
return None if isinstance(node, (int, float)) and not isinstance(node, bool) and math.isfinite(node) else key
|
||||
|
||||
|
||||
def missing(cfg: dict, keys: tuple[str, ...]) -> str | None:
|
||||
"""Args:
|
||||
cfg: the config.
|
||||
keys: required number keys, nested ones dotted
|
||||
(`memory_limits.stop.psi_full`).
|
||||
|
||||
Returns:
|
||||
None when every key holds a number, else the error naming the
|
||||
shortest missing part of each, or the key whose value is no number.
|
||||
"""
|
||||
absent = list(dict.fromkeys(part for key in keys if (part := absent_part(cfg, key)) is not None))
|
||||
if not absent:
|
||||
return None
|
||||
return (
|
||||
f"agents council: {CONFIG} lacks {', '.join(absent)}; add them as numbers as the watch.py and queue.py "
|
||||
"docstrings document"
|
||||
)
|
||||
|
||||
|
||||
def disk(cfg: dict) -> tuple[float, float]:
|
||||
st = os.statvfs(cfg["disk"])
|
||||
return st.f_bavail * st.f_frsize / GB, st.f_blocks * st.f_frsize / GB
|
||||
@@ -237,6 +357,19 @@ def act(cfg: dict, stage: str, previous: str, free: float) -> None:
|
||||
notify(cfg, f"Agents council: {head}. Heavy jobs blocked; a running stack needs the operator's OK to go.")
|
||||
|
||||
|
||||
def act_memory(cfg: dict, stage: str, previous: str, sample: dict[str, float | None]) -> None:
|
||||
head = memory_text(sample)
|
||||
if MEMORY_STAGES.index(stage) < MEMORY_STAGES.index(previous):
|
||||
alert(stage, f"RESOURCE MEMORY {stage.upper()}: recovered, {head}.")
|
||||
return
|
||||
holders = "; ".join(swap_holders(CGROUPS, container_names())) or "no container holds swap"
|
||||
if stage == "warn":
|
||||
alert(stage, f"RESOURCE MEMORY WARN: {head}. Top swap holders: {holders}. Chair: find the holder; peers: start nothing heavy.")
|
||||
else:
|
||||
alert(stage, f"RESOURCE MEMORY STOP: {head}. New heavy jobs are blocked. Top swap holders: {holders}.")
|
||||
notify(cfg, f"Agents council: memory nearly exhausted ({head}). Heavy jobs blocked; top swap holders: {holders}.")
|
||||
|
||||
|
||||
def history_rows(text: str) -> list[tuple[str, str]]:
|
||||
rows = []
|
||||
for line in text.splitlines():
|
||||
@@ -280,27 +413,42 @@ def stale(state: dict, cfg: dict, now: float) -> bool:
|
||||
return now - state.get("checked", 0) >= cfg["interval_seconds"]
|
||||
|
||||
|
||||
def cmd_watch(cfg: dict) -> int:
|
||||
def measure_and_act(cfg: dict, state: dict, now: float, problem: str | None) -> None:
|
||||
free, _ = disk(cfg)
|
||||
samples = [s for s in state.get("samples", []) if now - s[0] <= cfg["projection_minutes"] * 60]
|
||||
stage = stage_for(free, samples, cfg, now)
|
||||
previous = state.get("stage", "ok")
|
||||
samples.append([now, free])
|
||||
state.update(checked=now, free=free, stage=stage, samples=samples)
|
||||
sample = None if problem else memory()
|
||||
was_short = state.get("memory_stage", "ok")
|
||||
if sample is not None:
|
||||
state["memory_stage"] = memory_stage(sample, cfg)
|
||||
due = now - state.get("caches_checked", 0) >= cfg["cache_interval_seconds"]
|
||||
if due:
|
||||
state["caches_checked"] = now
|
||||
write_json(STATE, state)
|
||||
if stage != previous:
|
||||
act(cfg, stage, previous, free)
|
||||
elif due:
|
||||
spawn_cleanup("--caches")
|
||||
if sample is not None and state["memory_stage"] != was_short:
|
||||
act_memory(cfg, state["memory_stage"], was_short, sample)
|
||||
|
||||
|
||||
def cmd_watch(cfg: dict, problem: str | None) -> int:
|
||||
"""Args:
|
||||
cfg: the config.
|
||||
problem: why the memory stage cannot be judged, None when it can; the
|
||||
disk part still runs and every session sees the problem.
|
||||
"""
|
||||
payload = json.load(sys.stdin)
|
||||
now = time.time()
|
||||
state = read_json(STATE)
|
||||
if stale(state, cfg, now):
|
||||
free, _ = disk(cfg)
|
||||
samples = [s for s in state.get("samples", []) if now - s[0] <= cfg["projection_minutes"] * 60]
|
||||
stage = stage_for(free, samples, cfg, now)
|
||||
previous = state.get("stage", "ok")
|
||||
samples.append([now, free])
|
||||
state.update(checked=now, free=free, stage=stage, samples=samples)
|
||||
due = now - state.get("caches_checked", 0) >= cfg["cache_interval_seconds"]
|
||||
if due:
|
||||
state["caches_checked"] = now
|
||||
write_json(STATE, state)
|
||||
if stage != previous:
|
||||
act(cfg, stage, previous, free)
|
||||
elif due:
|
||||
spawn_cleanup("--caches")
|
||||
measure_and_act(cfg, state, now, problem)
|
||||
keep_history()
|
||||
texts = unseen_alerts(payload["session_id"])
|
||||
texts = unseen_alerts(payload["session_id"]) + ([] if problem is None else [problem])
|
||||
if texts:
|
||||
print(json.dumps({"hookSpecificOutput": {
|
||||
"hookEventName": payload["hook_event_name"],
|
||||
@@ -309,7 +457,10 @@ def cmd_watch(cfg: dict) -> int:
|
||||
return 0
|
||||
|
||||
|
||||
HEREDOC = re.compile(r"<<-?\s*(['\"]?)(\w+)\1[^\n]*\n.*?\n\s*\2(?=\s|\)|$)", re.DOTALL)
|
||||
HEREDOC = re.compile(
|
||||
r"<<-?\s*(?P<quote>['\"]?)(?P<escape>\\?)(?P<tag>[\w.-]+)(?P=quote)(?P<rest>[^\n]*)\n(?P<body>.*?)\n\s*(?P=tag)(?=\s|\)|$)",
|
||||
re.DOTALL,
|
||||
)
|
||||
QUOTED = re.compile(r"'[^']*'|\"(?:\\.|[^\"\\])*\"")
|
||||
|
||||
|
||||
@@ -322,32 +473,369 @@ def silence() -> dict | None:
|
||||
return state if state.get("until", 0) > time.time() else None
|
||||
|
||||
|
||||
def cmd_block(cfg: dict) -> int:
|
||||
def uncommented(command: str) -> str:
|
||||
kept: list[str] = []
|
||||
quote = ""
|
||||
escaped = skipping = False
|
||||
word_start = True
|
||||
for char in command:
|
||||
if skipping and char != "\n":
|
||||
continue
|
||||
skipping = False
|
||||
if escaped:
|
||||
escaped = False
|
||||
elif quote == "'":
|
||||
quote = "" if char == "'" else quote
|
||||
elif char == "\\":
|
||||
escaped = True
|
||||
elif quote == '"':
|
||||
quote = "" if char == '"' else quote
|
||||
elif char in "'\"":
|
||||
quote = char
|
||||
elif char == "#" and word_start:
|
||||
skipping = True
|
||||
continue
|
||||
kept.append(char)
|
||||
word_start = not quote and not escaped and char in WORD_BREAK
|
||||
return "".join(kept)
|
||||
|
||||
|
||||
def segments(command: str) -> list[list[str]]:
|
||||
"""Args:
|
||||
command: a shell command line.
|
||||
|
||||
Returns:
|
||||
The words of each simple command, split at newlines, `;`, `&`, `|`
|
||||
and parentheses; a redirection, its target and a file descriptor
|
||||
number in front of it are dropped and the command goes on.
|
||||
|
||||
Raises:
|
||||
ValueError: on an unbalanced quote.
|
||||
"""
|
||||
lexer = shlex.shlex(uncommented(command.replace("\\\n", "")), posix=True, punctuation_chars=PUNCTUATION)
|
||||
lexer.whitespace = " \t\r"
|
||||
lexer.whitespace_split = True
|
||||
lexer.commenters = ""
|
||||
parts: list[list[str]] = [[]]
|
||||
target = False
|
||||
for token in lexer:
|
||||
if REDIRECT.fullmatch(token):
|
||||
if parts[-1] and parts[-1][-1].isdigit():
|
||||
parts[-1].pop()
|
||||
target = True
|
||||
elif SEPARATOR.fullmatch(token):
|
||||
parts.append([])
|
||||
target = TRAILING_REDIRECT.search(token) is not None
|
||||
elif target:
|
||||
target = False
|
||||
else:
|
||||
parts[-1].append(token)
|
||||
return parts
|
||||
|
||||
|
||||
def program(words: list[str]) -> list[str]:
|
||||
"""Returns:
|
||||
The words from the program on: shell reserved words, VAR=value,
|
||||
wrappers such as env, sudo, time, timeout or `uv run`, their options,
|
||||
the values of their options and their numbers dropped; the split
|
||||
string of `env -S`, nothing for the lookup `command -v`.
|
||||
"""
|
||||
at, wrapper = 0, ""
|
||||
while at < len(words):
|
||||
word = words[at]
|
||||
if os.path.basename(word) in WRAPPERS:
|
||||
wrapper = os.path.basename(word)
|
||||
elif word in WRAPPER_VALUES.get(wrapper, ()):
|
||||
at += 1
|
||||
elif word in WRAPPER_TURNS.get(wrapper, {}):
|
||||
return WRAPPER_TURNS[wrapper][word](words[at + 1 :])
|
||||
elif not (word in RESERVED or ASSIGNMENT.fullmatch(word) or (wrapper and wrapped_word(word))):
|
||||
break
|
||||
at += 1
|
||||
return words[at:]
|
||||
|
||||
|
||||
def lookup(rest: list[str]) -> list[str]:
|
||||
return []
|
||||
|
||||
|
||||
def split_string(rest: list[str]) -> list[str]:
|
||||
return program(" ".join(rest[:1]).split() + rest[1:])
|
||||
|
||||
|
||||
WRAPPER_TURNS = {
|
||||
"command": {"-v": lookup, "-V": lookup},
|
||||
"env": {"-S": split_string, "--split-string": split_string},
|
||||
}
|
||||
|
||||
|
||||
def wrapped_word(word: str) -> bool:
|
||||
return word in WRAPPER_WORDS or WRAPPER_ARGUMENT.fullmatch(word) is not None
|
||||
|
||||
|
||||
def inline_code(words: list[str]) -> str | None:
|
||||
"""Args:
|
||||
words: a simple command from its program on.
|
||||
|
||||
Returns:
|
||||
The code a shell runs with -c (also clustered, such as -lc or -ic,
|
||||
and after a `--`), else None.
|
||||
"""
|
||||
if not words or os.path.basename(words[0]) not in SHELLS:
|
||||
return None
|
||||
rest = words[1:]
|
||||
for at, word in enumerate(rest):
|
||||
if INLINE.fullmatch(word):
|
||||
code = [later for later in rest[at + 1 :] if later != "--"][:1]
|
||||
return code[0] if code else None
|
||||
return None
|
||||
|
||||
|
||||
def operands(args: list[str], value_options: frozenset[str]) -> list[str]:
|
||||
found, skip = [], False
|
||||
for word in args:
|
||||
if skip:
|
||||
skip = False
|
||||
elif word in value_options:
|
||||
skip = True
|
||||
elif not word.startswith("-"):
|
||||
found.append(word)
|
||||
return found
|
||||
|
||||
|
||||
def python_module(args: list[str]) -> tuple[str, list[str]]:
|
||||
skip = False
|
||||
for at, word in enumerate(args):
|
||||
if skip:
|
||||
skip = False
|
||||
elif word == "-m":
|
||||
return (args[at + 1], args[at + 2 :]) if at + 1 < len(args) else ("", [])
|
||||
elif word.startswith("-m"):
|
||||
return word[2:], args[at + 1 :]
|
||||
elif word in PYTHON_VALUE_OPTIONS:
|
||||
skip = True
|
||||
elif not word.startswith("-"):
|
||||
return os.path.basename(word), args[at + 1 :]
|
||||
return "", []
|
||||
|
||||
|
||||
def light_unittest(args: list[str]) -> bool:
|
||||
if args[:1] != ["discover"]:
|
||||
return len(operands(args, UNITTEST_VALUE_OPTIONS)) == 1
|
||||
rest = args[1:]
|
||||
patterns = [rest[at + 1] for at, word in enumerate(rest[:-1]) if word in PATTERN_OPTIONS]
|
||||
patterns = patterns or operands(rest, DISCOVER_VALUE_OPTIONS)[1:2]
|
||||
return bool(patterns) and TEST_FILE.fullmatch(patterns[-1]) is not None
|
||||
|
||||
|
||||
def heavy_pytest(args: list[str]) -> bool:
|
||||
files = operands(args, PYTEST_VALUE_OPTIONS)
|
||||
return not (PYTEST_LIGHT.intersection(args) or (len(files) == 1 and TEST_FILE.fullmatch(files[0])))
|
||||
|
||||
|
||||
def heavy_unittest(args: list[str]) -> bool:
|
||||
return not (HELP.intersection(args) or light_unittest(args))
|
||||
|
||||
|
||||
def make_targets(args: list[str]) -> list[str]:
|
||||
return [word for word in operands(args, MAKE_VALUE_OPTIONS) if "=" not in word and not word.isdigit()]
|
||||
|
||||
|
||||
def dry_run(args: list[str]) -> bool:
|
||||
return bool(DRY_RUN.intersection(args)) or any(SHORT_DRY.fullmatch(word) for word in args)
|
||||
|
||||
|
||||
def heavy_make(args: list[str]) -> bool:
|
||||
return not dry_run(args) and any(target == "install" or target.startswith("verify-") for target in make_targets(args))
|
||||
|
||||
|
||||
def heavy_compose(args: list[str]) -> bool:
|
||||
words = operands(args, COMPOSE_VALUES)
|
||||
command = words[0] if words else ""
|
||||
return command == "build" or (command in BUILDING and "--build" in args)
|
||||
|
||||
|
||||
def heavy_docker(args: list[str]) -> bool:
|
||||
return args[:1] == ["compose"] and heavy_compose(args[1:])
|
||||
|
||||
|
||||
HEAVY_TOOLS = {
|
||||
"pytest": heavy_pytest, "py.test": heavy_pytest, "unittest": heavy_unittest, "make": heavy_make,
|
||||
"docker": heavy_docker, "docker-compose": heavy_compose,
|
||||
}
|
||||
|
||||
|
||||
def heavy_words(run: list[str], cfg: dict, exempt: tuple[str, ...]) -> bool:
|
||||
if not run:
|
||||
return False
|
||||
code = inline_code(run)
|
||||
if code is not None:
|
||||
return heavy(code, cfg, exempt)
|
||||
name, args = os.path.basename(run[0]), run[1:]
|
||||
if PYTHON.fullmatch(name):
|
||||
name, args = python_module(args)
|
||||
if name in WRAPPERS:
|
||||
return heavy_words(program([name, *args]), cfg, exempt)
|
||||
return name in HEAVY_TOOLS and HEAVY_TOOLS[name](args)
|
||||
|
||||
|
||||
def segment_texts(words: list[str], run: list[str]) -> list[str]:
|
||||
return list(dict.fromkeys(" ".join("''" if SPACE.search(word) else word for word in part) for part in (words, run)))
|
||||
|
||||
|
||||
def judged(texts: list[str], run: list[str], cfg: dict, exempt: tuple[str, ...]) -> bool:
|
||||
if any(re.search(pattern, text) for pattern in exempt for text in texts):
|
||||
return False
|
||||
if any(re.search(pattern, text) for pattern in cfg["heavy_commands"] for text in texts):
|
||||
return True
|
||||
return heavy_words(run, cfg, exempt)
|
||||
|
||||
|
||||
def heavy_part(words: list[str], cfg: dict, exempt: tuple[str, ...]) -> bool:
|
||||
"""Returns:
|
||||
Whether one simple command is heavy; a make call is judged per target
|
||||
as `make <target>`, so an exempt target never covers the next one.
|
||||
"""
|
||||
run = program(words)
|
||||
if run and os.path.basename(run[0]) == "make" and not dry_run(run[1:]):
|
||||
return any(judged([f"make {target}"], ["make", target], cfg, exempt) for target in make_targets(run[1:]))
|
||||
return judged(segment_texts(words, run), run, cfg, exempt)
|
||||
|
||||
|
||||
def marked_heredocs(command: str) -> tuple[str, list[re.Match[str]]]:
|
||||
"""Returns:
|
||||
The command with each heredoc replaced by the word HEREDOC_MARK
|
||||
followed by its body in hex, so a nested call still reads it, the
|
||||
rest of its first line kept, and the heredoc matches.
|
||||
"""
|
||||
found: list[re.Match[str]] = []
|
||||
|
||||
def mark(match: re.Match[str]) -> str:
|
||||
found.append(match)
|
||||
return f" {HEREDOC_MARK}{match['body'].encode(errors='surrogatepass').hex()} {match['rest']}"
|
||||
|
||||
return HEREDOC.sub(mark, command), found
|
||||
|
||||
|
||||
def shell_operands(args: list[str]) -> list[str]:
|
||||
found, skip = [], False
|
||||
for word in args:
|
||||
if skip:
|
||||
skip = False
|
||||
elif word.startswith(("-", "+")):
|
||||
skip = SHELL_VALUE.fullmatch(word) is not None
|
||||
else:
|
||||
found.append(word)
|
||||
return found
|
||||
|
||||
|
||||
def reads_stdin(run: list[str]) -> bool:
|
||||
"""Returns:
|
||||
Whether a simple command from its program on is a shell that runs the
|
||||
code on its stdin: no -c code, and -s or no operand but heredoc marks.
|
||||
"""
|
||||
if not run or os.path.basename(run[0]) not in SHELLS or inline_code(run) is not None:
|
||||
return False
|
||||
args = run[1:]
|
||||
return any(STDIN_FLAG.fullmatch(word) for word in args) or all(MARK_WORD.fullmatch(word) for word in shell_operands(args))
|
||||
|
||||
|
||||
def expanded_codes(heredocs: list[re.Match[str]]) -> list[str]:
|
||||
"""Returns:
|
||||
The command substitutions in the bodies of heredocs whose delimiter
|
||||
is unquoted, which the shell expands.
|
||||
"""
|
||||
return [code for match in heredocs if not (match["quote"] or match["escape"]) for code in substituted(match["body"])[1]]
|
||||
|
||||
|
||||
def fed_bodies(parts: list[list[str]]) -> list[str]:
|
||||
marks = [MARK_WORD.fullmatch(word) for words in parts if reads_stdin(program(words)) for word in words]
|
||||
return [bytes.fromhex(mark[1]).decode(errors="ignore") for mark in marks if mark]
|
||||
|
||||
|
||||
def closing(text: str, start: int) -> int:
|
||||
depth = 1
|
||||
for at in range(start, len(text)):
|
||||
depth += PAREN.get(text[at], 0)
|
||||
if depth == 0:
|
||||
return at
|
||||
return len(text)
|
||||
|
||||
|
||||
def blanked_single(quoted: re.Match[str]) -> str:
|
||||
text = quoted[0]
|
||||
return text if text.startswith('"') else "'" + " " * (len(text) - 2) + "'"
|
||||
|
||||
|
||||
def substitution_end(shown: str, found: re.Match[str]) -> int:
|
||||
if found[0] == "`":
|
||||
end = shown.find("`", found.end())
|
||||
return len(shown) if end < 0 else end
|
||||
return closing(shown, found.end())
|
||||
|
||||
|
||||
def substituted(text: str) -> tuple[str, list[str]]:
|
||||
"""Returns:
|
||||
The text with every `...` and $(...) outside single quotes and
|
||||
escapes replaced by the option-like word PLACEHOLDER, so a wrapper
|
||||
around it still parses, and the code of each, which the shell runs
|
||||
first; nested ones stay inside their code.
|
||||
"""
|
||||
shown = QUOTED.sub(blanked_single, ESCAPED.sub(" ", text))
|
||||
kept: list[str] = []
|
||||
codes: list[str] = []
|
||||
at = 0
|
||||
found = OPENER.search(shown)
|
||||
while found:
|
||||
end = substitution_end(shown, found)
|
||||
kept.append(text[at : found.start()] + PLACEHOLDER)
|
||||
codes.append(text[found.end() : end])
|
||||
at = end + 1
|
||||
found = OPENER.search(shown, at)
|
||||
return "".join(kept) + text[at:], codes
|
||||
|
||||
|
||||
def cmd_block(cfg: dict, problem: str | None) -> int:
|
||||
"""Args:
|
||||
cfg: the config.
|
||||
problem: why the memory stage cannot be judged, None when it can; a
|
||||
heavy job is then refused with it.
|
||||
"""
|
||||
payload = json.load(sys.stdin)
|
||||
command = executable_text((payload.get("tool_input") or {}).get("command", ""))
|
||||
raw = (payload.get("tool_input") or {}).get("command", "")
|
||||
if not heavy(raw, cfg, tuple(cfg["never_block"])):
|
||||
return 0
|
||||
reason = block_reason(cfg, problem)
|
||||
if reason is None:
|
||||
return 0
|
||||
print(f"kvb-org-organize: {reason}. Teardown commands still run. Ask the resource chair.", file=sys.stderr)
|
||||
return 2
|
||||
|
||||
|
||||
def disk_stage(cfg: dict) -> str:
|
||||
state = read_json(STATE)
|
||||
now = time.time()
|
||||
stage = state.get("stage", "ok")
|
||||
if stale(state, cfg, now):
|
||||
stage = stage_for(disk(cfg)[0], state.get("samples", []), cfg, now)
|
||||
return stage_for(disk(cfg)[0], state.get("samples", []), cfg, now)
|
||||
return state.get("stage", "ok")
|
||||
|
||||
|
||||
def block_reason(cfg: dict, problem: str | None) -> str | None:
|
||||
mayday = silence()
|
||||
if stage not in ("stop", "critical") and not mayday:
|
||||
return 0
|
||||
if any(re.search(p, command) for p in cfg["never_block"]):
|
||||
return 0
|
||||
if any(re.search(p, command) for p in cfg["heavy_commands"]):
|
||||
reason = (
|
||||
if mayday:
|
||||
return (
|
||||
f"a mayday by {clip(mayday.get('by', '?'))} holds radio silence: {clip(mayday.get('text', ''))}; "
|
||||
"it ends with radio.py clear"
|
||||
if mayday
|
||||
else f"resource stage is {stage}; new heavy jobs are blocked until the disk recovers"
|
||||
)
|
||||
print(
|
||||
f"kvb-org-organize: {reason}. Teardown commands still run. Ask the resource chair.",
|
||||
file=sys.stderr,
|
||||
)
|
||||
return 2
|
||||
return 0
|
||||
stage = disk_stage(cfg)
|
||||
if stage in ("stop", "critical"):
|
||||
return f"resource stage is {stage}; new heavy jobs are blocked until the disk recovers"
|
||||
if problem is not None:
|
||||
return f"{problem}; new heavy jobs are blocked until it does"
|
||||
sample = memory()
|
||||
if memory_stage(sample, cfg) == "stop":
|
||||
return f"memory stage is stop ({memory_text(sample)}); new heavy jobs are blocked until memory recovers"
|
||||
return None
|
||||
|
||||
|
||||
def cpu_times() -> tuple[int, int]:
|
||||
@@ -361,12 +849,17 @@ def cpu_busy_percent(before: tuple[int, int], after: tuple[int, int]) -> float:
|
||||
return 100.0 * (total - idle) / total
|
||||
|
||||
|
||||
def mem_used_percent(meminfo: str) -> float:
|
||||
def meminfo_kb(meminfo: str) -> dict[str, int]:
|
||||
values = {}
|
||||
for line in meminfo.splitlines():
|
||||
key, _, rest = line.partition(":")
|
||||
if rest:
|
||||
values[key] = int(rest.split()[0])
|
||||
return values
|
||||
|
||||
|
||||
def mem_used_percent(meminfo: str) -> float:
|
||||
values = meminfo_kb(meminfo)
|
||||
return 100.0 * (1 - values["MemAvailable"] / values["MemTotal"])
|
||||
|
||||
|
||||
@@ -374,7 +867,161 @@ def load(cfg: dict) -> tuple[float, float]:
|
||||
before = cpu_times()
|
||||
time.sleep(cfg["load_limits"]["sample_seconds"])
|
||||
cpu = cpu_busy_percent(before, cpu_times())
|
||||
return cpu, mem_used_percent(Path("/proc/meminfo").read_text(encoding="utf-8"))
|
||||
return cpu, mem_used_percent(MEMINFO.read_text(encoding="utf-8"))
|
||||
|
||||
|
||||
def psi_full(pressure: str) -> float | None:
|
||||
for line in pressure.splitlines():
|
||||
if line.startswith("full "):
|
||||
return float(dict(field.split("=", 1) for field in line.split()[1:])["avg10"])
|
||||
return None
|
||||
|
||||
|
||||
def memory_sample(meminfo: str, pressure: str | None, container_swap_gb: float) -> dict[str, float | None]:
|
||||
"""Args:
|
||||
meminfo: the text of /proc/meminfo.
|
||||
pressure: the text of /proc/pressure/memory, None without PSI.
|
||||
container_swap_gb: the swap of the container holding the most, in GB.
|
||||
|
||||
Returns:
|
||||
{available, swap_free, psi_full, container_swap_gb}: MemAvailable and
|
||||
SwapFree in percent of their totals, the full avg10 stall percentage
|
||||
and container_swap_gb unchanged; swap_free is None without swap,
|
||||
psi_full None without PSI.
|
||||
"""
|
||||
values = meminfo_kb(meminfo)
|
||||
swap = values["SwapTotal"]
|
||||
return {
|
||||
"available": 100.0 * values["MemAvailable"] / values["MemTotal"],
|
||||
"swap_free": 100.0 * values["SwapFree"] / swap if swap else None,
|
||||
"psi_full": None if pressure is None else psi_full(pressure),
|
||||
"container_swap_gb": container_swap_gb,
|
||||
}
|
||||
|
||||
|
||||
def memory() -> dict[str, float | None]:
|
||||
try:
|
||||
pressure = PRESSURE.read_text(encoding="utf-8")
|
||||
except OSError:
|
||||
pressure = None
|
||||
largest = max((size for size, _ in container_swaps(CGROUPS)), default=0)
|
||||
return memory_sample(MEMINFO.read_text(encoding="utf-8"), pressure, largest / GB)
|
||||
|
||||
|
||||
def memory_stage(sample: dict[str, float | None], cfg: dict) -> str:
|
||||
for name in ("stop", "warn"):
|
||||
bound = cfg["memory_limits"][name]
|
||||
if (
|
||||
sample["available"] < bound["available"]
|
||||
or (sample["swap_free"] is not None and sample["swap_free"] < bound["swap_free"])
|
||||
or (sample["psi_full"] is not None and sample["psi_full"] >= bound["psi_full"])
|
||||
or sample["container_swap_gb"] >= bound["container_swap_gb"]
|
||||
):
|
||||
return name
|
||||
return "ok"
|
||||
|
||||
|
||||
def memory_text(sample: dict[str, float | None]) -> str:
|
||||
swap = "no swap" if sample["swap_free"] is None else f"swap free {sample['swap_free']:.0f} %"
|
||||
psi = "no PSI" if sample["psi_full"] is None else f"PSI full avg10 {sample['psi_full']:.1f}"
|
||||
return f"MemAvailable {sample['available']:.0f} %, {swap}, {psi}, largest container swap {sample['container_swap_gb']:.1f} GB"
|
||||
|
||||
|
||||
def swap_bytes(cgroup: Path) -> int:
|
||||
try:
|
||||
return int((cgroup / "memory.swap.current").read_text(encoding="utf-8"))
|
||||
except (OSError, ValueError):
|
||||
return 0
|
||||
|
||||
|
||||
def container_swaps(base: Path) -> list[tuple[int, Path]]:
|
||||
try:
|
||||
return [(swap_bytes(scope), scope) for scope in base.glob("docker-*.scope")]
|
||||
except OSError:
|
||||
return []
|
||||
|
||||
|
||||
def cgroup_label(name: str, names: dict[str, str]) -> str:
|
||||
bare = name.removeprefix("docker-").removesuffix(".scope")
|
||||
if CONTAINER_ID.fullmatch(bare):
|
||||
return names[bare] if bare in names else bare[:12]
|
||||
return bare
|
||||
|
||||
|
||||
def container_names() -> dict[str, str]:
|
||||
try:
|
||||
out = subprocess.run(
|
||||
["docker", "ps", "--no-trunc", "--format", "{{.ID}}\t{{.Names}}"],
|
||||
capture_output=True, text=True, timeout=30, check=False,
|
||||
).stdout
|
||||
except (OSError, subprocess.TimeoutExpired):
|
||||
return {}
|
||||
return dict(line.split("\t", 1) for line in out.splitlines() if "\t" in line)
|
||||
|
||||
|
||||
def swap_chain(cgroup: Path, held: int, names: dict[str, str]) -> list[str]:
|
||||
chain = [cgroup_label(cgroup.name, names)]
|
||||
for _ in range(NEST_DEPTH):
|
||||
try:
|
||||
children = [(swap_bytes(child), child) for child in cgroup.iterdir() if child.is_dir()]
|
||||
except OSError:
|
||||
break
|
||||
if not children:
|
||||
break
|
||||
size, child = max(children)
|
||||
if size * 2 < held:
|
||||
break
|
||||
chain.append(cgroup_label(child.name, names))
|
||||
cgroup, held = child, size
|
||||
return chain
|
||||
|
||||
|
||||
def swap_holders(base: Path, names: dict[str, str]) -> list[str]:
|
||||
"""Args:
|
||||
base: the cgroup directory holding the docker-<id>.scope groups.
|
||||
names: {full container id: name} for the labels.
|
||||
|
||||
Returns:
|
||||
Up to SWAP_HOLDERS lines, the largest swap first: the container, then
|
||||
each nested group holding at least half of its parent's swap, and the
|
||||
container's memory.swap.current in GB.
|
||||
"""
|
||||
ranked = sorted((pair for pair in container_swaps(base) if pair[0] > 0), reverse=True)[:SWAP_HOLDERS]
|
||||
return [f"{' > '.join(swap_chain(scope, size, names))} {size / GB:.1f} GB" for size, scope in ranked]
|
||||
|
||||
|
||||
def heavy(command: str, cfg: dict, exempt: tuple[str, ...]) -> bool:
|
||||
"""Args:
|
||||
command: the shell command line as typed, quotes included.
|
||||
cfg: the config; its heavy_commands search each simple command with
|
||||
quoted text blanked, and a make call also as `make <targets>`.
|
||||
exempt: regexes; a simple command one of them finds is never heavy
|
||||
(never_block, the queue's own hold), the others still count.
|
||||
|
||||
Returns:
|
||||
Whether one simple command is a heavy job: an image build through
|
||||
docker compose (build, or up, run or create with --build), a
|
||||
heavy_commands match, make install or verify-* (not as a dry run), or
|
||||
pytest or unittest on anything but exactly one test file or module,
|
||||
help and version aside. Wrappers (program()), the code of `sh -c`,
|
||||
command substitutions (substituted(), also in heredocs the shell
|
||||
expands) and heredocs a shell reads as its code count the same. A
|
||||
command with an unbalanced quote, which no shell runs, counts by the
|
||||
regexes on its whole text alone. A guard against accidental heavy
|
||||
runs, not a sandbox: a shell reading code from a pipe or a
|
||||
here-string, a script of its own, a program named by a substitution,
|
||||
quotes inside an expanding heredoc and an escaped backslash in front
|
||||
of a substitution are not looked into.
|
||||
"""
|
||||
marked, heredocs = marked_heredocs(command)
|
||||
plain, codes = substituted(uncommented(marked))
|
||||
try:
|
||||
parts = segments(plain)
|
||||
except ValueError:
|
||||
text = executable_text(command)
|
||||
return not any(re.search(p, text) for p in exempt) and any(re.search(p, text) for p in cfg["heavy_commands"])
|
||||
nested = codes + expanded_codes(heredocs) + fed_bodies(parts)
|
||||
return any(heavy_part(words, cfg, exempt) for words in parts) or any(heavy(code, cfg, exempt) for code in nested)
|
||||
|
||||
|
||||
def cmd_admit(cfg: dict, args: argparse.Namespace) -> int:
|
||||
@@ -385,12 +1032,12 @@ def cmd_admit(cfg: dict, args: argparse.Namespace) -> int:
|
||||
passed = [left >= floor]
|
||||
report = [f"free {free:.1f} GB, need {need} GB, leaves {left:.1f} GB (floor {floor} GB)"]
|
||||
limits = cfg["load_limits"]
|
||||
if args.kind in limits["kinds"]:
|
||||
cpu, mem = load(cfg)
|
||||
passed += [cpu < limits["cpu_percent"], mem < limits["mem_percent"]]
|
||||
report.append(
|
||||
f"cpu {cpu:.0f} % (limit {limits['cpu_percent']}), ram {mem:.0f} % (limit {limits['mem_percent']})"
|
||||
)
|
||||
cpu, mem = load(cfg)
|
||||
sample = memory()
|
||||
stage = memory_stage(sample, cfg)
|
||||
passed += [cpu < limits["cpu_percent"], mem < limits["mem_percent"], stage != "stop"]
|
||||
report.append(f"cpu {cpu:.0f} % (limit {limits['cpu_percent']}), ram {mem:.0f} % (limit {limits['mem_percent']})")
|
||||
report.append(f"memory stage {stage} ({memory_text(sample)})")
|
||||
verdict = "grant" if all(passed) else "deny"
|
||||
print(f"{verdict}: {'; '.join(report)}")
|
||||
return 0 if verdict == "grant" else 3
|
||||
@@ -511,7 +1158,11 @@ def cmd_status(cfg: dict) -> int:
|
||||
now = time.time()
|
||||
samples = state.get("samples", [])
|
||||
age = f"{now - state['checked']:.0f}s" if "checked" in state else "unmeasured"
|
||||
print(f"stage={stage_for(free, samples, cfg, now)} free={free:.1f}GB total={total:.1f}GB samples={len(samples)} age={age}")
|
||||
sample = memory()
|
||||
print(
|
||||
f"stage={stage_for(free, samples, cfg, now)} free={free:.1f}GB total={total:.1f}GB samples={len(samples)} age={age}"
|
||||
f" memory={memory_stage(sample, cfg)} ({memory_text(sample)})"
|
||||
)
|
||||
return 0
|
||||
|
||||
|
||||
@@ -537,10 +1188,14 @@ def main() -> int:
|
||||
cfg = load_config()
|
||||
if cfg is None:
|
||||
return 0
|
||||
problem = missing(cfg, MEMORY_KEYS)
|
||||
if args.cmd == "watch":
|
||||
return cmd_watch(cfg)
|
||||
return cmd_watch(cfg, problem)
|
||||
if args.cmd == "block":
|
||||
return cmd_block(cfg)
|
||||
return cmd_block(cfg, problem)
|
||||
if problem is not None:
|
||||
print(problem, file=sys.stderr)
|
||||
return 2
|
||||
if args.cmd == "admit":
|
||||
return cmd_admit(cfg, args)
|
||||
if args.cmd == "cleanup":
|
||||
|
||||
Reference in new issue
Block a user