From e1445cb88b86c16f71a3ff44be1ce168b31c2fc9 Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Sat, 14 Nov 2026 22:58:07 +0000 Subject: [PATCH] feat(kvb-org-queue): cap the booked RAM share per kind with ram_cap_percent MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Operator order relayed by Main on 2026-11-14 22:25: book at most 30 % RAM for test jobs. learn() folds ram_cap_percent {kind: percent} into each kind the way it folds light_kinds; footprint() and ceiling() book min(share, cap), while the learned shares and the history rows stay raw. A cap map that is not {kind: percent above 0 up to 100} makes hold exit 2, keeps fresh() on the last good config and makes admit refuse. learning.md documents the key and the re-hold after an install. Security acceptances (kvb-think-criticize with an independent skeptic; false positives or by design, all in unchanged lines of the touched files): MEDUSA-MM-SCAN-058 queue.py 375 377 621 623 624 629 630 633 634; MEDUSA-ACP-SCAN-028 queue.py 297; MEDUSA-RT-SCAN-054 queue.py 731; MEDUSA-AI-BACKDOOR-006 queue.py 746 806 1140 1166 1189 1190 1243 and history.py 353; MEDUSA-WM-SCAN-780 queue.py 1215. 🎯 Fix: 88 % 💥 Break: 3 % 💸 Tokens: – ⏱️ Velocity: – 🛡️ Security: ~100 % 🏆 Score: 85 🚦 Gate: ⛔ fix 88 % < 90 Co-Authored-By: Claude Opus 5.5 --- skills/kvb-org-queue/learning.md | 5 ++++ skills/kvb-org-queue/scripts/history.py | 37 +++++++++++++++++-------- skills/kvb-org-queue/scripts/queue.py | 30 ++++++++++++++++---- tests/test_queue.py | 30 ++++++++++++++++++++ tests/test_watch.py | 11 ++++++++ 5 files changed, 96 insertions(+), 17 deletions(-) diff --git a/skills/kvb-org-queue/learning.md b/skills/kvb-org-queue/learning.md index fd7bfeb..327ec59 100644 --- a/skills/kvb-org-queue/learning.md +++ b/skills/kvb-org-queue/learning.md @@ -29,3 +29,8 @@ was granted on. Until then the seeds hold: `slot_need_gb[kind]` for disk, 1 slot per kind, the whole budget as footprint. The optional `slot_limits {kind: N}` only caps a limit from above. +The optional `ram_cap_percent {kind: percent}` caps the RAM share a job of +that kind books, its footprint and its ceiling, while the learned shares and +the history rows stay raw; keep it truthful by limiting the job's own memory +to that share, such as a container memory limit. A waiting hold keeps the +code it loaded: hold it again after an install so it books with the cap. diff --git a/skills/kvb-org-queue/scripts/history.py b/skills/kvb-org-queue/scripts/history.py index 3008fb4..3276c2e 100644 --- a/skills/kvb-org-queue/scripts/history.py +++ b/skills/kvb-org-queue/scripts/history.py @@ -32,6 +32,7 @@ SIGNATURE_KEY = operator.itemgetter("kind", "signature") NAME_KEY = operator.itemgetter("signature") KIND_KEY = operator.itemgetter("kind") ANY_KIND = "*" +RAM_CAP = "ram_cap_percent" SIGNATURE_HEADER = ("Signature", "Kind", "Runs", "Median min", "p90 disk GB", "p90 RAM %", "Need GB") KIND_HEADER = ("Kind", "Runs", "p90 RAM share %", "Median CPU share %", "Slot limit", "Source") @@ -189,15 +190,17 @@ def learn(rows: list[dict], now: float, cfg: dict) -> tuple[dict[tuple[str, str] now: epoch seconds; rows that ended more than WINDOW_SECONDS earlier are ignored. cfg: the config; load_limits.mem_percent and cpu_percent bound a limit, - light_kinds marks a kind light. + light_kinds marks a kind light, RAM_CAP {kind: percent} sets its + ram_cap. Returns: (signatures, kinds). signatures: {(kind, signature): {runs, minutes, disk_gb, ram_share, cpu_share, need_gb}}, and {(ANY_KIND, signature): ...} over the rows of every kind with that signature, which is what a queue before the per-kind learning booked; kinds: {kind: {runs, - ram_share, cpu_share, limit, light}}. With fewer than MIN_RUNS rows - every value but runs and light is None. + ram_share, cpu_share, limit, light, ram_cap}}, ram_cap None without a + cap. With fewer than MIN_RUNS rows every value but runs, light and + ram_cap is None. minutes is the median, disk_gb and ram_share the p90 of the disk taken and of max minus before in percentage points, cpu_share the median of max minus before; need_gb is ceil(disk_gb + DISK_MARGIN_GB); limit is @@ -206,10 +209,11 @@ def learn(rows: list[dict], now: float, cfg: dict) -> tuple[dict[tuple[str, str] """ recent = [r for r in rows if now - r["end"] <= WINDOW_SECONDS] light = set(cfg.get("light_kinds", [])) + caps = cfg.get(RAM_CAP, {}) signatures = {(ANY_KIND, name): command_stats(group) for name, group in grouped(recent, NAME_KEY).items()} signatures.update((key, command_stats(group)) for key, group in grouped(recent, SIGNATURE_KEY).items()) kinds = { - name: {**kind_stats(group, cfg["load_limits"]), "light": name in light} + name: {**kind_stats(group, cfg["load_limits"]), "light": name in light, "ram_cap": caps.get(name)} for name, group in grouped(recent, KIND_KEY).items() } return signatures, kinds @@ -241,15 +245,17 @@ def footprint(kind: str, aliases: tuple[str, ...], signatures: dict[tuple[str, s there, for a kind not light at least booked_before(); else the largest of the kind's share and each alias's share over every kind, so a shape without runs of its own never books less than its kind or - the runs it was learned from; at least MIN_SHARE_PERCENT. None while - the kind is not learned yet. + the runs it was learned from; at least MIN_SHARE_PERCENT; RAM at most + the kind's ram_cap (capped()). None while the kind is not learned yet. """ if kind not in kinds or kinds[kind]["ram_share"] is None: return None mine = own(signatures, kind, aliases[0]) if mine is None: - return largest([kinds[kind], *wider(signatures, aliases)]) - return largest([mine] if kinds[kind]["light"] else [mine, booked_before(kind, aliases, signatures, kinds)]) + sources = [kinds[kind], *wider(signatures, aliases)] + else: + sources = [mine] if kinds[kind]["light"] else [mine, booked_before(kind, aliases, signatures, kinds)] + return capped(kinds[kind], largest(sources)) def booked_before(kind: str, aliases: tuple[str, ...], signatures: dict[tuple[str, str], dict], @@ -267,19 +273,28 @@ def ceiling(kind: str, aliases: tuple[str, ...], signatures: dict[tuple[str, str """Returns: The largest of every share footprint() can pick for the job: its signature under kind once learned, its kind's and each alias's over - every kind, which holds booked_before(); None while the kind is not - learned yet. + every kind, which holds booked_before(); RAM at most the kind's + ram_cap (capped()); None while the kind is not learned yet. """ if kind not in kinds or kinds[kind]["ram_share"] is None: return None mine = own(signatures, kind, aliases[0]) - return largest([kinds[kind], *wider(signatures, aliases), *([mine] if mine is not None else [])]) + return capped(kinds[kind], largest([kinds[kind], *wider(signatures, aliases), *([mine] if mine is not None else [])])) def largest(sources: list[dict]) -> dict[str, float]: return {resource: max(MIN_SHARE_PERCENT, *(s[f"{resource}_share"] for s in sources)) for resource in RESOURCES} +def capped(stats: dict, shares: dict[str, float]) -> dict[str, float]: + """Returns: + shares with RAM lowered to the ram_cap of the kind stats, learn()'s + RAM_CAP value; unchanged without one. + """ + cap = stats.get("ram_cap") + return shares if cap is None else {**shares, "ram": min(shares["ram"], cap)} + + def learned_value(signatures: dict[tuple[str, str], dict], kind: str, aliases: tuple[str, ...], field: str) -> float | None: """Returns: diff --git a/skills/kvb-org-queue/scripts/queue.py b/skills/kvb-org-queue/scripts/queue.py index 1654279..140f922 100755 --- a/skills/kvb-org-queue/scripts/queue.py +++ b/skills/kvb-org-queue/scripts/queue.py @@ -95,7 +95,9 @@ Config keys: budget_percent {ram, cpu}, required: the active and granted jobs' learned footprints (history.footprint, the whole budget for a kind not learned yet) may sum to at most these percentage points, a job alone always fits; slot_limits {kind: N}, optional, caps the learned slot limit of a kind from -above (history.slot_limits); exclusive_kinds [kind] and light_kinds [kind], +above (history.slot_limits); ram_cap_percent {kind: percent above 0 up to +100}, optional, caps the RAM share history.footprint and history.ceiling +book for a job of that kind, the learned shares stay raw; exclusive_kinds [kind] and light_kinds [kind], optional: a job of an exclusive kind runs only beside jobs of light kinds, and no job of another kind starts beside it; queue_enforce (bool), optional, read by block only. The watch.py keys disk, thresholds_gb (stop is the floor every grant leaves free), slot_need_gb (the disk seed per kind), load_limits (every grant @@ -266,12 +268,25 @@ def config() -> dict: cfg, _ = watch.announced_config() if cfg is None: fail(f"no config at {watch.CONFIG}; the queue reads the kvb-org-organize watch config") - problem = watch.missing(cfg, (*BUDGET_KEYS, *watch.MEMORY_KEYS)) + problem = watch.missing(cfg, (*BUDGET_KEYS, *watch.MEMORY_KEYS)) or cap_problem(cfg) if problem is not None: fail(problem) return cfg +def cap_problem(cfg: dict) -> str | None: + """Returns: + Why history.RAM_CAP is no {kind: percent above 0 up to 100} map, None + when it is one or unset. + """ + caps = cfg.get(history.RAM_CAP, {}) + if isinstance(caps, dict) and all( + isinstance(cap, (int, float)) and not isinstance(cap, bool) and 0 < cap <= 100 for cap in caps.values() + ): + return None + return f"{history.RAM_CAP} is no {{kind: percent above 0 up to 100}} map" + + def fresh(cfg: dict) -> dict: """Returns: The config as it stands now, else cfg while the file is missing, @@ -282,7 +297,7 @@ def fresh(cfg: dict) -> dict: now = watch.checked_config(watch.load_config()) except watch.UNUSABLE: return cfg - if watch.missing(now, (*BUDGET_KEYS, *watch.MEMORY_KEYS)) is not None: + if (watch.missing(now, (*BUDGET_KEYS, *watch.MEMORY_KEYS)) or cap_problem(now)) is not None: return cfg return now @@ -1746,10 +1761,13 @@ def admit(cfg: dict, kind: str, argv: list[str], cpu: float, mem: float, calm: b (ramping_ram()) counts only when it is not. Returns: - load_reason() over the job's RAM share plus the live admit records - (admitted_ram()) and the ramp; under the store lock, a grant records - the share for ADMIT_SECONDS and expired records leave. + cap_problem() of cfg first; else load_reason() over the job's RAM + share plus the live admit records (admitted_ram()) and the ramp; + under the store lock, a grant records the share for ADMIT_SECONDS + and expired records leave. """ + if (problem := cap_problem(cfg)) is not None: + return problem now = time.time() signatures, kinds = learned(cfg, now) mine = {"id": ADMIT, "status": "waiting", "kind": kind, ALIASES: list(aliases(workdir(), argv, os.getcwd()))} diff --git a/tests/test_queue.py b/tests/test_queue.py index 393310b..45c2fcc 100644 --- a/tests/test_queue.py +++ b/tests/test_queue.py @@ -1339,6 +1339,21 @@ class TestHistory(unittest.TestCase): signatures, kinds = self.learn(small[:4]) self.assertIsNone(footprint("test", ("make test",), signatures, kinds)) + def test_the_ram_cap_lowers_what_footprint_and_ceiling_book_and_leaves_the_learned_shares_raw(self): + small = [history_row(n, signature="make test", mem_max=25, cpu_max=20) for n in range(5)] + big = [history_row(n + 5, kind="quality-high", signature="make verify-test", mem_max=80, cpu_max=90) for n in range(5)] + cfg = {"load_limits": {"mem_percent": 80, "cpu_percent": 90}, "ram_cap_percent": {"quality-high": 30, "test": 90}} + signatures, kinds = self.history.learn([*small, *big], NOW, cfg) + for booked in (self.history.footprint, self.history.ceiling): + with self.subTest(booked=booked.__name__): + self.assertEqual(booked("quality-high", ("make verify-test",), signatures, kinds), {"ram": 30, "cpu": 80}) + self.assertEqual(booked("test", ("make test",), signatures, kinds), {"ram": 5, "cpu": 10}) + self.assertEqual((kinds["quality-high"]["ram_share"], signatures["quality-high", "make verify-test"]["ram_share"]), (60, 60)) + self.assertEqual((kinds["quality-high"]["ram_cap"], kinds["test"]["ram_cap"]), (30, 90)) + signatures, kinds = self.learn([*small, *big]) + self.assertIsNone(kinds["quality-high"]["ram_cap"]) + self.assertEqual(self.history.footprint("quality-high", ("make verify-test",), signatures, kinds), {"ram": 60, "cpu": 80}) + def test_slot_limits_take_the_learned_limit_else_the_seed_and_a_smaller_cap(self): kinds = {"deploy": {"limit": 3}, "build": {"limit": None}} self.assertEqual(self.history.slot_limits(kinds, {"deploy": 2, "swarm": 0, "test": 9}, ["test", "gate"]), { @@ -1806,6 +1821,21 @@ class TestQueueCommands(unittest.TestCase): with self.subTest(error=error), mock.patch.object(queue.watch, "load_config", side_effect=error): self.assertEqual(queue.fresh(good), good) + def test_a_broken_ram_cap_keeps_the_last_good_config_and_a_hold_refuses_it(self): + queue = _load(QUEUE) + good = {**self.cfg, "ram_cap_percent": {"test": 30}} + capped = {**self.cfg, "ram_cap_percent": {"test": 30, "quality-high": 100}} + with mock.patch.object(queue.watch, "load_config", return_value=capped): + self.assertEqual(queue.fresh(good), capped) + for caps in ({"test": 0}, {"test": 101}, {"test": "30"}, {"test": True}, {"test": float("nan")}, [30]): + with self.subTest(caps=caps), mock.patch.object(queue.watch, "load_config", return_value={**self.cfg, "ram_cap_percent": caps}): + self.assertEqual(queue.fresh(good), good) + self.config.write_text(json.dumps({**self.cfg, "ram_cap_percent": {"test": 101}}), encoding="utf-8") + result = self.run_queue("hold", "--kind", "test", "--task", "work/a", "--session", "s1", "--", "true") + self.assertEqual(result.returncode, 2) + self.assertIn("ram_cap_percent is no {kind: percent above 0 up to 100} map", result.stderr) + self.assertFalse((self.root / "queue.json").exists()) + def test_hold_takes_the_need_learned_for_its_command_from_both_history_files(self): self.cfg["slot_limits"]["deploy"] = 0 self.write_config() diff --git a/tests/test_watch.py b/tests/test_watch.py index a018750..e35ec4a 100644 --- a/tests/test_watch.py +++ b/tests/test_watch.py @@ -1148,6 +1148,17 @@ class TestWatchCommands(unittest.TestCase): self.assertEqual(short.returncode, 3) self.assertIn("memory stage stop", short.stdout) + def test_admit_refuses_while_the_ram_cap_is_broken_and_grants_under_a_valid_one(self): + self.cfg["thresholds_gb"] = {"warn": 0, "stop": 0, "critical": 0} + self.cfg["ram_cap_percent"] = {"test": "30"} + self._write_config() + denied = self._run("admit", "--kind", "test", "--", "true") + self.assertEqual(denied.returncode, 3) + self.assertIn("ram_cap_percent is no {kind: percent above 0 up to 100} map", denied.stdout) + self.cfg["ram_cap_percent"] = {"test": 30} + self._write_config() + self.assertEqual(self._run("admit", "--kind", "test", "--", "true").returncode, 0) + def test_admit_refuses_a_heavy_command_and_names_the_queue_hold_of_kind_test(self): self.cfg["thresholds_gb"] = {"warn": 0, "stop": 0, "critical": 0} self.cfg["slot_need_gb"]["test-nostack"] = 0