fix(kvb-org-queue): global budget so no queue mix can OOM the host

Root cause of the 2026-11-07 20:14 OOM: the queue granted jobs per kind
only, so several heavy kinds and direct heavy commands outside the queue
ran side by side until the kernel killed the editor.

- queue.py: every grant checks the sum of the learned RAM and CPU
  footprints of active and granted jobs against budget_percent; an
  unlearned kind takes the whole budget and runs alone. RAM used plus
  what young jobs (RAMP_SECONDS) may still take must stay below
  load_limits. Memory stage stop blocks every grant.
- watch.py: memory stage from MemAvailable, swap, PSI full and the swap
  of the container holding most of it; the block hook classifies
  builds, make install/verify-*, and pytest or unittest beyond one file
  as heavy through wrappers, sh -c, command substitutions and heredocs
  a shell reads.
- history.py: footprints per kind and signature, seed RAM for pending
  shares.
- operator.py: [h]old queues a heavy row through queue.py hold and asks
  again on its grant, with a flushed tty and a timeout; sudo rows, which
  need the terminal, stay direct.

Residuals: a shell reading code from a pipe or here-string, its own
script, a program named by a substitution, quotes inside an expanding
heredoc, an escaped backslash before a substitution; a lone job waits
while the host is over load_limits.

🎯 Fix: 92 %
💥 Break: 4 %
💸 Cost: 100 %
⏱️ Time: –
🛡️ Security: 100 %
🏆 Score: 88
🚦 Gate: ✅ 90/5, cost and time <= 100, security 100 %

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Kevin Veen-BirkenbachandClaude Opus 5.5 committed 2026-11-08 06:54:09 +00:00
1 parent 09f68dd75b
commit 4321b97529
8 files changed
+1665 -218

No files matched your search

+84
View File
@@ -249,6 +249,90 @@ class TestOperator(unittest.TestCase):
self.assertNotIn("decide", result.stdout)
self.assertEqual(self._open(), ["operator/ask"])
def _queue_config(self, slot_need_gb: dict) -> None:
config = self.base / "config.json"
open_memory = {"available": -1, "swap_free": -1, "psi_full": 1000, "container_swap_gb": 1e9}
config.write_text(json.dumps({
"disk": str(self.base), "thresholds_gb": {"warn": 0, "stop": 0, "critical": 0}, "projection_minutes": 30,
"slot_need_gb": slot_need_gb, "heavy_commands": [r"\btouch heavy\b"], "never_block": [],
"load_limits": {"cpu_percent": 101, "mem_percent": 101, "sample_seconds": 0.1},
"budget_percent": {"ram": 75, "cpu": 100}, "memory_limits": {"warn": open_memory, "stop": open_memory},
}), encoding="utf-8")
self.env["AGENTS_COUNCIL_CONFIG"] = str(config)
def test_run_offers_the_queue_for_a_heavy_command_and_asks_again_on_the_grant(self):
self._queue_config({"operator": 0})
work = self.base / "work"
work.mkdir()
self._add("light", work, "touch light", "plain", "manual")
self._add("heavy", work, "touch heavy\x1b[2K", "build", "manual")
result = self._feed("y\nh\n")
self.assertEqual(result.returncode, 0, result.stderr)
light, heavy = result.stdout.split("[2/2]")
self.assertIn("[s]kip", light)
self.assertNotIn("heavy:", light)
self.assertIn("heavy: memory stage ok", heavy)
self.assertIn("position\tid\tstatus", heavy)
self.assertIn("[h]old in the queue", heavy)
self.assertNotIn("\x1b", heavy)
self.assertIn("granted operator/heavy: the queue lets it run now", heavy)
self.assertIn("not run operator/heavy", heavy)
self.assertIn("failed operator/heavy: exit 75; the row stays open", heavy)
self.assertTrue((work / "light").exists())
self.assertFalse((work / "heavy\x1b[2K").exists())
self.assertEqual(self._open(), ["operator/heavy"])
self.assertFalse((self.root / "queue-history.jsonl").exists())
def test_run_offers_no_hold_while_the_config_lacks_the_operator_kind(self):
self._queue_config({"test": 0})
work = self.base / "work"
work.mkdir()
self._add("heavy", work, "touch heavy", "build", "manual")
result = self._feed("h\ns\n")
self.assertEqual(result.returncode, 0, result.stderr)
self.assertIn("lacks slot_need_gb.operator;", result.stdout)
self.assertIn("no [h]old until it does", result.stdout)
self.assertNotIn("[h]old in the queue", result.stdout)
self.assertIn("skipped operator/heavy", result.stdout)
def test_run_offers_no_hold_for_sudo_whose_prompt_needs_the_terminal(self):
self._queue_config({"operator": 0})
work = self.base / "work"
work.mkdir()
self._add("heavy", work, "bash -c 'sudo -n touch heavy'", "build", "manual")
result = self._feed("h\ns\n")
self.assertEqual(result.returncode, 0, result.stderr)
self.assertIn("sudo asks on the terminal, which a queued command lacks", result.stdout)
self.assertNotIn("[h]old in the queue", result.stdout)
def test_granted_runs_only_the_approved_row_in_its_directory_after_a_new_yes(self):
work = self.base / "work"
work.mkdir()
self._add("heavy", work, "touch heavy", "build", "manual")
env = {**self.env, "SHELL": "/bin/sh"}
def granted(answers: str, directory: Path, *argv: str) -> subprocess.CompletedProcess:
return subprocess.run(
[sys.executable, str(OPERATOR), "granted", "operator/heavy", "--dir", str(directory), "--check",
"manual", "--", *argv],
input=answers, capture_output=True, text=True, env=env, cwd=self.base, check=False,
)
changed = granted("y\n", work, "/bin/sh", "-ic", "touch other")
self.assertEqual(changed.returncode, 75)
self.assertIn("no longer open with the approved directory, check and command", changed.stderr)
moved = granted("y\n", self.base, "/bin/sh", "-ic", "touch heavy")
self.assertEqual((moved.returncode, (self.base / "heavy").exists()), (75, False))
declined = granted("s\n", work, "/bin/sh", "-ic", "touch heavy")
self.assertEqual((declined.returncode, (work / "heavy").exists()), (75, False))
silent = granted("", work, "/bin/sh", "-ic", "touch heavy")
self.assertEqual((silent.returncode, (work / "heavy").exists()), (75, False))
ran = granted("d\ny\n", work, "/bin/sh", "-ic", "touch heavy")
self.assertEqual(ran.returncode, 0, ran.stderr)
self.assertIn("granted operator/heavy", ran.stdout)
self.assertTrue((work / "heavy").exists())
self.assertFalse((self.base / "heavy").exists() or (work / "other").exists())
def test_run_needs_shell(self):
self._add("x", self.base, "true", "r", "manual")
result = subprocess.run(