feat(kvb-org-queue): cap the booked RAM share per kind with ram_cap_percent

Operator order relayed by Main on 2026-11-14 22:25: book at most 30 % RAM for test jobs.
learn() folds ram_cap_percent {kind: percent} into each kind the way it folds light_kinds;
footprint() and ceiling() book min(share, cap), while the learned shares and the history
rows stay raw. A cap map that is not {kind: percent above 0 up to 100} makes hold exit 2,
keeps fresh() on the last good config and makes admit refuse. learning.md documents the
key and the re-hold after an install.

Security acceptances (kvb-think-criticize with an independent skeptic; false positives or
by design, all in unchanged lines of the touched files): MEDUSA-MM-SCAN-058 queue.py 375
377 621 623 624 629 630 633 634; MEDUSA-ACP-SCAN-028 queue.py 297; MEDUSA-RT-SCAN-054
queue.py 731; MEDUSA-AI-BACKDOOR-006 queue.py 746 806 1140 1166 1189 1190 1243 and
history.py 353; MEDUSA-WM-SCAN-780 queue.py 1215.

🎯 Fix: 88 %
💥 Break: 3 %
💸 Tokens: –
⏱️ Velocity: –
🛡️ Security: ~100 %
🏆 Score: 85
🚦 Gate: ⛔ fix 88 % < 90

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Kevin Veen-BirkenbachandClaude Opus 5.5 committed 2026-11-14 22:58:07 +00:00
1 parent b95e305db0
commit e1445cb88b
5 files changed
+96 -17

No files matched your search

+5
View File
@@ -29,3 +29,8 @@ was granted on.
Until then the seeds hold: `slot_need_gb[kind]` for disk, 1 slot per kind,
the whole budget as footprint.
The optional `slot_limits {kind: N}` only caps a limit from above.
The optional `ram_cap_percent {kind: percent}` caps the RAM share a job of
that kind books, its footprint and its ceiling, while the learned shares and
the history rows stay raw; keep it truthful by limiting the job's own memory
to that share, such as a container memory limit. A waiting hold keeps the
code it loaded: hold it again after an install so it books with the cap.
+26 -11
View File
@@ -32,6 +32,7 @@ SIGNATURE_KEY = operator.itemgetter("kind", "signature")
NAME_KEY = operator.itemgetter("signature")
KIND_KEY = operator.itemgetter("kind")
ANY_KIND = "*"
RAM_CAP = "ram_cap_percent"
SIGNATURE_HEADER = ("Signature", "Kind", "Runs", "Median min", "p90 disk GB", "p90 RAM %", "Need GB")
KIND_HEADER = ("Kind", "Runs", "p90 RAM share %", "Median CPU share %", "Slot limit", "Source")
@@ -189,15 +190,17 @@ def learn(rows: list[dict], now: float, cfg: dict) -> tuple[dict[tuple[str, str]
now: epoch seconds; rows that ended more than WINDOW_SECONDS earlier are
ignored.
cfg: the config; load_limits.mem_percent and cpu_percent bound a limit,
light_kinds marks a kind light.
light_kinds marks a kind light, RAM_CAP {kind: percent} sets its
ram_cap.
Returns:
(signatures, kinds). signatures: {(kind, signature): {runs, minutes,
disk_gb, ram_share, cpu_share, need_gb}}, and {(ANY_KIND, signature):
...} over the rows of every kind with that signature, which is what a
queue before the per-kind learning booked; kinds: {kind: {runs,
ram_share, cpu_share, limit, light}}. With fewer than MIN_RUNS rows
every value but runs and light is None.
ram_share, cpu_share, limit, light, ram_cap}}, ram_cap None without a
cap. With fewer than MIN_RUNS rows every value but runs, light and
ram_cap is None.
minutes is the median, disk_gb and ram_share the p90 of the disk taken
and of max minus before in percentage points, cpu_share the median of
max minus before; need_gb is ceil(disk_gb + DISK_MARGIN_GB); limit is
@@ -206,10 +209,11 @@ def learn(rows: list[dict], now: float, cfg: dict) -> tuple[dict[tuple[str, str]
"""
recent = [r for r in rows if now - r["end"] <= WINDOW_SECONDS]
light = set(cfg.get("light_kinds", []))
caps = cfg.get(RAM_CAP, {})
signatures = {(ANY_KIND, name): command_stats(group) for name, group in grouped(recent, NAME_KEY).items()}
signatures.update((key, command_stats(group)) for key, group in grouped(recent, SIGNATURE_KEY).items())
kinds = {
name: {**kind_stats(group, cfg["load_limits"]), "light": name in light}
name: {**kind_stats(group, cfg["load_limits"]), "light": name in light, "ram_cap": caps.get(name)}
for name, group in grouped(recent, KIND_KEY).items()
}
return signatures, kinds
@@ -241,15 +245,17 @@ def footprint(kind: str, aliases: tuple[str, ...], signatures: dict[tuple[str, s
there, for a kind not light at least booked_before(); else the
largest of the kind's share and each alias's share over every kind,
so a shape without runs of its own never books less than its kind or
the runs it was learned from; at least MIN_SHARE_PERCENT. None while
the kind is not learned yet.
the runs it was learned from; at least MIN_SHARE_PERCENT; RAM at most
the kind's ram_cap (capped()). None while the kind is not learned yet.
"""
if kind not in kinds or kinds[kind]["ram_share"] is None:
return None
mine = own(signatures, kind, aliases[0])
if mine is None:
return largest([kinds[kind], *wider(signatures, aliases)])
return largest([mine] if kinds[kind]["light"] else [mine, booked_before(kind, aliases, signatures, kinds)])
sources = [kinds[kind], *wider(signatures, aliases)]
else:
sources = [mine] if kinds[kind]["light"] else [mine, booked_before(kind, aliases, signatures, kinds)]
return capped(kinds[kind], largest(sources))
def booked_before(kind: str, aliases: tuple[str, ...], signatures: dict[tuple[str, str], dict],
@@ -267,19 +273,28 @@ def ceiling(kind: str, aliases: tuple[str, ...], signatures: dict[tuple[str, str
"""Returns:
The largest of every share footprint() can pick for the job: its
signature under kind once learned, its kind's and each alias's over
every kind, which holds booked_before(); None while the kind is not
learned yet.
every kind, which holds booked_before(); RAM at most the kind's
ram_cap (capped()); None while the kind is not learned yet.
"""
if kind not in kinds or kinds[kind]["ram_share"] is None:
return None
mine = own(signatures, kind, aliases[0])
return largest([kinds[kind], *wider(signatures, aliases), *([mine] if mine is not None else [])])
return capped(kinds[kind], largest([kinds[kind], *wider(signatures, aliases), *([mine] if mine is not None else [])]))
def largest(sources: list[dict]) -> dict[str, float]:
return {resource: max(MIN_SHARE_PERCENT, *(s[f"{resource}_share"] for s in sources)) for resource in RESOURCES}
def capped(stats: dict, shares: dict[str, float]) -> dict[str, float]:
"""Returns:
shares with RAM lowered to the ram_cap of the kind stats, learn()'s
RAM_CAP value; unchanged without one.
"""
cap = stats.get("ram_cap")
return shares if cap is None else {**shares, "ram": min(shares["ram"], cap)}
def learned_value(signatures: dict[tuple[str, str], dict], kind: str, aliases: tuple[str, ...],
field: str) -> float | None:
"""Returns:
+24 -6
View File
@@ -95,7 +95,9 @@ Config keys: budget_percent {ram, cpu}, required: the active and granted jobs'
learned footprints (history.footprint, the whole budget for a kind not learned
yet) may sum to at most these percentage points, a job alone always fits;
slot_limits {kind: N}, optional, caps the learned slot limit of a kind from
above (history.slot_limits); exclusive_kinds [kind] and light_kinds [kind],
above (history.slot_limits); ram_cap_percent {kind: percent above 0 up to
100}, optional, caps the RAM share history.footprint and history.ceiling
book for a job of that kind, the learned shares stay raw; exclusive_kinds [kind] and light_kinds [kind],
optional: a job of an exclusive kind runs only beside jobs of light kinds, and
no job of another kind starts beside it; queue_enforce (bool), optional, read by block only. The watch.py keys
disk, thresholds_gb (stop is the floor every grant leaves free), slot_need_gb (the disk seed per kind), load_limits (every grant
@@ -266,12 +268,25 @@ def config() -> dict:
cfg, _ = watch.announced_config()
if cfg is None:
fail(f"no config at {watch.CONFIG}; the queue reads the kvb-org-organize watch config")
problem = watch.missing(cfg, (*BUDGET_KEYS, *watch.MEMORY_KEYS))
problem = watch.missing(cfg, (*BUDGET_KEYS, *watch.MEMORY_KEYS)) or cap_problem(cfg)
if problem is not None:
fail(problem)
return cfg
def cap_problem(cfg: dict) -> str | None:
"""Returns:
Why history.RAM_CAP is no {kind: percent above 0 up to 100} map, None
when it is one or unset.
"""
caps = cfg.get(history.RAM_CAP, {})
if isinstance(caps, dict) and all(
isinstance(cap, (int, float)) and not isinstance(cap, bool) and 0 < cap <= 100 for cap in caps.values()
):
return None
return f"{history.RAM_CAP} is no {{kind: percent above 0 up to 100}} map"
def fresh(cfg: dict) -> dict:
"""Returns:
The config as it stands now, else cfg while the file is missing,
@@ -282,7 +297,7 @@ def fresh(cfg: dict) -> dict:
now = watch.checked_config(watch.load_config())
except watch.UNUSABLE:
return cfg
if watch.missing(now, (*BUDGET_KEYS, *watch.MEMORY_KEYS)) is not None:
if (watch.missing(now, (*BUDGET_KEYS, *watch.MEMORY_KEYS)) or cap_problem(now)) is not None:
return cfg
return now
@@ -1746,10 +1761,13 @@ def admit(cfg: dict, kind: str, argv: list[str], cpu: float, mem: float, calm: b
(ramping_ram()) counts only when it is not.
Returns:
load_reason() over the job's RAM share plus the live admit records
(admitted_ram()) and the ramp; under the store lock, a grant records
the share for ADMIT_SECONDS and expired records leave.
cap_problem() of cfg first; else load_reason() over the job's RAM
share plus the live admit records (admitted_ram()) and the ramp;
under the store lock, a grant records the share for ADMIT_SECONDS
and expired records leave.
"""
if (problem := cap_problem(cfg)) is not None:
return problem
now = time.time()
signatures, kinds = learned(cfg, now)
mine = {"id": ADMIT, "status": "waiting", "kind": kind, ALIASES: list(aliases(workdir(), argv, os.getcwd()))}
+30
View File
@@ -1339,6 +1339,21 @@ class TestHistory(unittest.TestCase):
signatures, kinds = self.learn(small[:4])
self.assertIsNone(footprint("test", ("make test",), signatures, kinds))
def test_the_ram_cap_lowers_what_footprint_and_ceiling_book_and_leaves_the_learned_shares_raw(self):
small = [history_row(n, signature="make test", mem_max=25, cpu_max=20) for n in range(5)]
big = [history_row(n + 5, kind="quality-high", signature="make verify-test", mem_max=80, cpu_max=90) for n in range(5)]
cfg = {"load_limits": {"mem_percent": 80, "cpu_percent": 90}, "ram_cap_percent": {"quality-high": 30, "test": 90}}
signatures, kinds = self.history.learn([*small, *big], NOW, cfg)
for booked in (self.history.footprint, self.history.ceiling):
with self.subTest(booked=booked.__name__):
self.assertEqual(booked("quality-high", ("make verify-test",), signatures, kinds), {"ram": 30, "cpu": 80})
self.assertEqual(booked("test", ("make test",), signatures, kinds), {"ram": 5, "cpu": 10})
self.assertEqual((kinds["quality-high"]["ram_share"], signatures["quality-high", "make verify-test"]["ram_share"]), (60, 60))
self.assertEqual((kinds["quality-high"]["ram_cap"], kinds["test"]["ram_cap"]), (30, 90))
signatures, kinds = self.learn([*small, *big])
self.assertIsNone(kinds["quality-high"]["ram_cap"])
self.assertEqual(self.history.footprint("quality-high", ("make verify-test",), signatures, kinds), {"ram": 60, "cpu": 80})
def test_slot_limits_take_the_learned_limit_else_the_seed_and_a_smaller_cap(self):
kinds = {"deploy": {"limit": 3}, "build": {"limit": None}}
self.assertEqual(self.history.slot_limits(kinds, {"deploy": 2, "swarm": 0, "test": 9}, ["test", "gate"]), {
@@ -1806,6 +1821,21 @@ class TestQueueCommands(unittest.TestCase):
with self.subTest(error=error), mock.patch.object(queue.watch, "load_config", side_effect=error):
self.assertEqual(queue.fresh(good), good)
def test_a_broken_ram_cap_keeps_the_last_good_config_and_a_hold_refuses_it(self):
queue = _load(QUEUE)
good = {**self.cfg, "ram_cap_percent": {"test": 30}}
capped = {**self.cfg, "ram_cap_percent": {"test": 30, "quality-high": 100}}
with mock.patch.object(queue.watch, "load_config", return_value=capped):
self.assertEqual(queue.fresh(good), capped)
for caps in ({"test": 0}, {"test": 101}, {"test": "30"}, {"test": True}, {"test": float("nan")}, [30]):
with self.subTest(caps=caps), mock.patch.object(queue.watch, "load_config", return_value={**self.cfg, "ram_cap_percent": caps}):
self.assertEqual(queue.fresh(good), good)
self.config.write_text(json.dumps({**self.cfg, "ram_cap_percent": {"test": 101}}), encoding="utf-8")
result = self.run_queue("hold", "--kind", "test", "--task", "work/a", "--session", "s1", "--", "true")
self.assertEqual(result.returncode, 2)
self.assertIn("ram_cap_percent is no {kind: percent above 0 up to 100} map", result.stderr)
self.assertFalse((self.root / "queue.json").exists())
def test_hold_takes_the_need_learned_for_its_command_from_both_history_files(self):
self.cfg["slot_limits"]["deploy"] = 0
self.write_config()
+11
View File
@@ -1148,6 +1148,17 @@ class TestWatchCommands(unittest.TestCase):
self.assertEqual(short.returncode, 3)
self.assertIn("memory stage stop", short.stdout)
def test_admit_refuses_while_the_ram_cap_is_broken_and_grants_under_a_valid_one(self):
self.cfg["thresholds_gb"] = {"warn": 0, "stop": 0, "critical": 0}
self.cfg["ram_cap_percent"] = {"test": "30"}
self._write_config()
denied = self._run("admit", "--kind", "test", "--", "true")
self.assertEqual(denied.returncode, 3)
self.assertIn("ram_cap_percent is no {kind: percent above 0 up to 100} map", denied.stdout)
self.cfg["ram_cap_percent"] = {"test": 30}
self._write_config()
self.assertEqual(self._run("admit", "--kind", "test", "--", "true").returncode, 0)
def test_admit_refuses_a_heavy_command_and_names_the_queue_hold_of_kind_test(self):
self.cfg["thresholds_gb"] = {"warn": 0, "stop": 0, "critical": 0}
self.cfg["slot_need_gb"]["test-nostack"] = 0