feat(kvb-org-organize): hold heavy jobs back on memory pressure with one shipped default config

watch.py admits heavy work only while PSI, swap-out rate and RAM forecast stay under the configured limits; config.default.json is the single default the install writes once (mode 600, no clobber) and every hook, admit and queue command falls back to. Browser runners and package-manager script forms count as heavy, preflight derives its probes from the light and heavy tool lists, and the shipped default carries no prune command, buildx path or host-specific key. Tests run once on unify/skills-20261114 after the merge (operator order 2026-11-14).

🎯 Fix: 85 %
💥 Break: 8 %
💸 Tokens: 99 %
⏱️ Velocity: –
🛡️ Security: 100 %
🏆 Score: 79
🚦 Gate: ⛔ fix 85 % < 90, break 8 % > 5

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Kevin Veen-BirkenbachandClaude Opus 5.5 committed 2026-11-14 21:17:28 +00:00
1 parent a2a03995cb
commit 8e2db852e5
27 files changed
+2795 -207

No files matched your search

+121 -6
View File
@@ -23,7 +23,9 @@ HISTORY = REPO_ROOT / "skills" / "kvb-org-queue" / "scripts" / "history.py"
GATES = REPO_ROOT / "skills" / "kvb-check-gate" / "scripts" / "gates.py"
NOW = 1_800_000_000.0
HOUR = 3600
OPEN_MEMORY = {"available": -1, "swap_free": -1, "psi_full": 1000, "container_swap_gb": 1e9}
OPEN_MEMORY = {
"available": -1, "swap_free": -1, "psi_some": 1000, "psi_full": 1000, "swap_out_mb_min": 1e9, "container_swap_gb": 1e9,
}
BASE = {
"free_gb": 100.0,
"floor": 10.0,
@@ -141,11 +143,38 @@ class TestEvaluate(unittest.TestCase):
young = {**job(1, kind="a", status="active", granted=NOW - 60), **cmd, "cmd": "a"}
overshooting = {**job(5, kind="c", status="active", granted=NOW - 120), **cmd, "cmd": "c"}
old = {**job(4, kind="c", status="active", granted=NOW - self.queue.RAMP_SECONDS), **cmd, "cmd": "c"}
self.assertEqual(self.queue.pending_ram([young, overshooting, old, fresh], "j2", NOW, signatures, kinds), 30 + 10 + 25)
pending = self.queue.pending_ram
self.assertEqual(pending([young, overshooting, old, fresh], "j2", NOW, signatures, kinds, 50, 0), 30 + 10 + 25)
undated = {k: v for k, v in old.items() if k != "granted"}
self.assertEqual(self.queue.pending_ram([undated, fresh], "j2", NOW, signatures, kinds), 10 + 25)
self.assertEqual(pending([undated, fresh], "j2", NOW, signatures, kinds, 50, 0), 10 + 25)
unknown = {**job(3, kind="new"), **cmd, "cmd": "new"}
self.assertEqual(self.queue.pending_ram([unknown], "j3", NOW, signatures, kinds), 30)
self.assertEqual(pending([unknown], "j3", NOW, signatures, kinds, 50, 0), 30)
self.assertEqual(pending([fresh], "j2", NOW, signatures, kinds, 50, 12), 25 + 12)
def test_a_young_job_counts_only_what_it_has_not_added_since_its_grant(self):
history = _load(HISTORY)
rows = [history_row(n, kind="b", signature="w: b", mem_before=20, mem_max=45) for n in range(5)]
signatures, kinds = history.learn(rows, NOW, {"load_limits": {"mem_percent": 90, "cpu_percent": 90}})
fresh = {**job(2, kind="b"), "workdir": "/w", "cmd": "b"}
def gate(seq: int, ram: float, granted_mem: float) -> dict:
return {**job(seq, kind="g", status="active", granted=NOW - 60), "workdir": "/w", "cmd": "g",
"footprint": {"ram": ram, "cpu": 1}, "granted_mem": granted_mem}
pending = self.queue.pending_ram
self.assertEqual(pending([gate(1, 40, 10), fresh], "j2", NOW, signatures, kinds, 10, 0), 25 + 40)
self.assertEqual(pending([gate(1, 40, 10), fresh], "j2", NOW, signatures, kinds, 45, 0), 25 + 5)
self.assertEqual(pending([gate(1, 40, 10), fresh], "j2", NOW, signatures, kinds, 60, 0), 25)
self.assertEqual(pending([gate(1, 40, 10), gate(3, 20, 30), fresh], "j2", NOW, signatures, kinds, 45, 0), 25 + 25)
undated = {k: v for k, v in gate(1, 40, 10).items() if k != "granted_mem"}
self.assertEqual(pending([undated, fresh], "j2", NOW, signatures, kinds, 60, 0), 25 + 40)
ramp = self.queue.ramping_ram
for mem in (70, 40, 20):
with self.subTest(mem=mem):
self.assertEqual(ramp([gate(1, 40, 70)], NOW, signatures, kinds, mem), 40)
self.assertEqual(ramp([gate(1, 40, 70), {**undated, "id": "j3"}], NOW, signatures, kinds, 20), 80)
self.assertEqual(self.queue.admitted_ram({"admits": [{"ram": 7, "expires": NOW + 1}, {"ram": 9, "expires": NOW}]}, NOW), 7)
self.assertEqual(self.queue.admitted_ram({"entries": []}, NOW), 0)
def test_a_signature_names_the_repository_every_worktree_shares(self):
with tempfile.TemporaryDirectory() as tmp:
@@ -251,7 +280,7 @@ class TestEvaluate(unittest.TestCase):
self.assertEqual(self.queue.footprint(waiting, signatures, kinds), {"ram": 50, "cpu": 80})
older = {k: v for k, v in running.items() if k != "footprint"}
self.assertEqual(self.queue.footprint(older, signatures, kinds), {"ram": 50, "cpu": 80})
self.assertEqual(self.queue.pending_ram([running], "j1", NOW, signatures, kinds), 12 + 12)
self.assertEqual(self.queue.pending_ram([running], "j1", NOW, signatures, kinds, 50, 0), 12 + 12)
def test_an_active_entry_an_older_queue_granted_never_books_less_than_that_queue_did(self):
history = _load(HISTORY)
@@ -754,6 +783,7 @@ class TestQueueCommands(unittest.TestCase):
)
signature = f"{self.base.name}: {Path(sys.executable).name} -c"
self.assertEqual((entry["aliases"], entry["footprint"]), ([signature], None))
self.assertTrue(0 < entry["granted_mem"] < 100, entry)
self.assertEqual(self.entries(), [])
[row] = [json.loads(line) for line in self.history_file.read_text(encoding="utf-8").splitlines()]
self.assertEqual(
@@ -1111,6 +1141,52 @@ class TestQueueCommands(unittest.TestCase):
self.write_config()
self.assertEqual(self.bash("make compose-deploy"), 0)
def test_list_and_hold_on_a_broken_config_warn_and_use_the_shipped_default(self):
self.config.write_text(json.dumps(self.cfg)[:-1] + ",}", encoding="utf-8")
listing = self.run_queue("list")
self.assertEqual(listing.returncode, 0, listing.stderr)
self.assertIn("built-in defaults", listing.stderr)
held = self.run_queue("hold", "--kind", "nosuch", "--task", "work/a", "--session", "s1", "--", "true")
self.assertEqual(held.returncode, 2, held.stderr)
self.assertIn("built-in defaults", held.stderr)
self.assertIn("'nosuch' is not in slot_need_gb", held.stderr)
for done in (listing, held):
self.assertNotIn("Traceback", done.stderr)
def test_a_broken_config_falls_back_to_defaults_so_light_commands_pass_and_heavy_ones_stay_refused(self):
self.cfg["queue_enforce"] = True
good = json.dumps(self.cfg)
def text(content: str) -> Callable[[], None]:
return lambda: self.config.write_text(content, encoding="utf-8")
shapes = {
"comma": text(good[:-1] + ",}"), "empty": text(""), "list": text("[]"),
"string enforce": text(json.dumps({**self.cfg, "queue_enforce": "yes"})),
"empty-matching heavy": text(json.dumps({**self.cfg, "heavy_commands": [""]})),
"dot heavy": text(json.dumps({**self.cfg, "heavy_commands": ["."]})),
"word-boundary heavy": text(json.dumps({**self.cfg, "heavy_commands": [r"\b"]})),
"dot never_block": text(json.dumps({**self.cfg, "never_block": ["."]})),
"dangling link": lambda: self.config.symlink_to(self.base / "absent.json"),
"link loop": lambda: self.config.symlink_to(self.config),
"directory": lambda: self.config.mkdir(),
}
for name, make in shapes.items():
with self.subTest(config=name):
if self.config.is_dir() and not self.config.is_symlink():
self.config.rmdir()
self.config.unlink(missing_ok=True)
make()
passed = self.hook(json.dumps({"tool_input": {"command": "ls -la"}}))
self.assertEqual(passed.returncode, 0, passed.stderr)
self.assertIn(str(self.config), passed.stderr)
self.assertIn("built-in defaults", passed.stderr)
self.assertEqual(self.bash("make test"), 2)
live = ("make compose-deploy ROLE=x", "docker build .", "docker pull nginx", "make swarm-deploy", "make roundtrip",
"make worktree-up")
for command in live if name in ("comma", "directory") else live[:1]:
self.assertEqual(self.bash(command), 2, command)
def test_block_with_queue_enforce_refuses_heavy_commands_outside_the_queue(self):
self.cfg["queue_enforce"] = True
self.write_config()
@@ -1129,7 +1205,12 @@ class TestQueueCommands(unittest.TestCase):
):
with self.subTest(command=command):
self.assertEqual(self.bash(command), 0)
self.assertEqual(self.hook("not json").returncode, 0)
for payload in ("not json", json.dumps({"tool_input": ["make test"]})):
with self.subTest(payload=payload):
crashed = self.hook(payload)
self.assertEqual(crashed.returncode, 2, crashed.stderr)
self.assertIn("counts as heavy", crashed.stderr)
self.assertEqual(self.bash("CARGO_BUILD_JOBS=² cargo build ; make test"), 2)
self.config.unlink()
self.assertEqual(self.bash("make compose-deploy"), 0)
@@ -1290,6 +1371,40 @@ class TestQueueCommands(unittest.TestCase):
self.assertEqual(self.bash(command), 2)
self.assertEqual(self.bash("docker compose ps"), 0)
def test_a_queue_grant_counts_a_live_admit_record_and_ignores_an_expired_one(self):
def admitted(expires: float) -> None:
(self.root / "queue.json").write_text(
json.dumps({"seq": 0, "entries": [], "admits": [{"ram": 300, "expires": expires}]}), encoding="utf-8"
)
admitted(time.time() + 300)
_, err = self.blocked_hold("true")
self.assertRegex(err.read_text(encoding="utf-8"), r"RAM \d+ % used \+ 300 % still to come reach the limit 101 %")
admitted(time.time() - 1)
result = self.run_queue("hold", "--kind", "deploy", "--task", "work/a", "--session", "s2", "--", "true")
self.assertEqual((result.returncode, result.stderr), (0, ""))
def test_a_light_kind_refuses_a_heavy_command_names_the_test_kind_and_enqueues_nothing(self):
self.cfg["slot_need_gb"] |= {"test-nostack": 0, "quick": 0}
self.cfg["light_kinds"] = ["quick"]
self.write_config()
(self.base / "tests" / "e2e").mkdir(parents=True)
(self.base / "tests" / "e2e" / "test_orchestrator_e2e.py").write_text("", encoding="utf-8")
for kind, command, shown in (
("test-nostack", ("python", "-m", "unittest"), "python -m unittest"),
("test-nostack", ("pytest", "tests/e2e/test_orchestrator_e2e.py"), "pytest tests/e2e/test_orchestrator_e2e.py"),
("quick", ("bash", "-c", "make test"), "bash -c 'make test'"),
("quick", ("python3", "gate.py", "run", "--target", "test-unit"), "python3 gate.py run --target test-unit"),
):
with self.subTest(kind=kind, command=command):
result = self.run_queue("hold", "--kind", kind, "--task", "work/a", "--session", "s1", "--", *command)
self.assertEqual(result.returncode, 2)
self.assertIn(f"the light kind {kind} skips every reservation", result.stderr)
self.assertIn(f"run `queue.py hold --kind test --task <register task> -- {shown}`", result.stderr)
self.assertFalse((self.root / "queue.json").exists())
light = self.run_queue("hold", "--kind", "test-nostack", "--task", "work/a", "--session", "s1", "--", "true")
self.assertEqual((light.returncode, light.stderr), (0, ""))
def test_hold_refuses_a_config_without_the_budget_or_the_memory_limits(self):
for key, cfg in (
("budget_percent", {k: v for k, v in self.cfg.items() if k != "budget_percent"}),