From 151031ab4a8189c1c80bfcc018b71873b30f36a3 Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Thu, 12 Nov 2026 08:47:26 +0000 Subject: [PATCH] fix(kvb-org-queue): light kinds meet only running and granted jobs, never a reservation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every entry blocked by budget or disk added its share to the tally of all entries behind it, and a head the host load keeps out counted as granted while a job ran, so test-nostack jobs needing 0 GB and 11 % CPU waited behind 'CPU 170-242 % in use or reserved' (live queue 2026-11-12). Operator pick on the chair's ask: a light_kinds entry is now checked only against running and granted entries; reservations, a load-held head included, hold back heavy kinds alone. resources() returns what a job takes or reserves, so evaluate() keeps both tallies. Heavy-only queues give master's verdicts (skeptic probe: 0 of 40000 random cases differ); a stream of light jobs can delay a heavy head, as the docstring and SKILL.md now state. Tests kill the reviewer's three mutants and fail on master. 🎯 Fix: 92 % 💥 Break: 4 % 💸 Cost: 100 % ⏱️ Time: 100 % 🛡️ Security: 100 % 🏆 Score: 88 🚦 Gate: ✅ 90/5, cost and time <= 100, security 100 % Co-Authored-By: Claude Opus 5.5 --- skills/kvb-org-queue/SKILL.md | 4 ++- skills/kvb-org-queue/scripts/queue.py | 42 +++++++++++++++----------- tests/test_queue.py | 43 ++++++++++++++++++++++++--- 3 files changed, 67 insertions(+), 22 deletions(-) diff --git a/skills/kvb-org-queue/SKILL.md b/skills/kvb-org-queue/SKILL.md index 6de86a8..2827d7e 100644 --- a/skills/kvb-org-queue/SKILL.md +++ b/skills/kvb-org-queue/SKILL.md @@ -42,7 +42,9 @@ Every 20 s each waiting hold evaluates the queue under one lock: `exclusive_kinds` run only beside `light_kinds`. 3. A job whose disk need would cut below the stop threshold, or whose footprint would push the running ones past `budget_percent`, reserves it - against the jobs behind it; a job alone always fits. + against the jobs behind it but `light_kinds`, which meet only running and + granted jobs, so a stream of light jobs can delay a heavy head; a job + alone always fits. 4. An unblocked job counts as granted for the jobs behind it. 5. Every job, alone too, needs CPU, and RAM used plus what it and the running jobs may still take, below `load_limits`. diff --git a/skills/kvb-org-queue/scripts/queue.py b/skills/kvb-org-queue/scripts/queue.py index ee61430..bcbb80d 100755 --- a/skills/kvb-org-queue/scripts/queue.py +++ b/skills/kvb-org-queue/scripts/queue.py @@ -72,7 +72,8 @@ yet) may sum to at most these percentage points, a job alone always fits; slot_limits {kind: N}, optional, caps the learned slot limit of a kind from above (history.slot_limits); exclusive_kinds [kind] and light_kinds [kind], optional: a job of an exclusive kind runs only beside jobs of light kinds, and -no job of another kind starts beside it; queue_enforce (bool), optional, read by block +no job of another kind starts beside it; a light job meets only the budget and +disk of running and granted jobs, never a reservation; queue_enforce (bool), optional, read by block only. The watch.py keys disk, thresholds_gb (stop is the floor every grant leaves free), slot_need_gb (the disk seed per kind), load_limits (every grant also needs CPU busy below cpu_percent and RAM used plus the job's learned RAM @@ -405,20 +406,24 @@ def evaluate(entries: list[dict], free_gb: float, floor: float, stage: str, memo An unblocked entry counts as granted for the entries after it, whoever holds it, so a lower entry never takes a higher entry's slot; one blocked by disk or budget reserves its need against the entries after it. - A hotfix entry that passes every check but the host load holds back - every entry after it, so lighter jobs cannot keep the load up. While no - queue job is active, any other entry the host load alone keeps out - reserves nothing and holds nothing back: that load lies outside the - queue, so holding the queue frees none of it. An - exclusive entry that only running jobs of other kinds keep out holds - back every entry after it but light ones, so none overtakes it. A waiting - entry whose holder has not polled for CLAIM_SECONDS reserves nothing and - comes last. + A light entry meets only the disk, RAM and CPU of running and granted + entries: reservations, a head the host load keeps out included, hold + back heavy kinds alone, so light jobs that keep coming can delay a heavy + head without bound. A hotfix entry that passes every check but the host + load holds back every entry after it, so lighter jobs cannot keep the + load up. While no queue job is active, any other entry the host load + alone keeps out reserves nothing and holds nothing back: that load lies + outside the queue, so holding the queue frees none of it. An exclusive + entry that only running jobs of other kinds keep out holds back every + entry after it but light ones, so none overtakes it. A waiting entry + whose holder has not polled for CLAIM_SECONDS reserves nothing and comes + last. """ active = [e for e in entries if e["status"] == "active"] available = free_gb - sum(e["need_gb"] for e in active) used = {resource: sum(footprints[e["id"]][resource] for e in active) for resource in budget} taken = len(active) + granted = used, available, taken counts = Counter(e["kind"] for e in active) busy = {e["workdir"] for e in active} verdicts: dict[str, str | None] = {} @@ -438,17 +443,20 @@ def evaluate(entries: list[dict], free_gb: float, floor: float, stage: str, memo if reason is not None and kind in exclusive: drain = entry["id"] if reason is None: - reason, after, left = resources(need, share, used, available, floor, budget, taken) + pool_used, pool_available, pool_taken = granted if kind in light else (used, available, taken) + reason, took, disk = resources(need, share, pool_used, pool_available, floor, budget, pool_taken) if reason is None and entry["id"] in loaded and (entry["hotfix"] or not active): reason = loaded[entry["id"]] if entry["hotfix"]: held = entry["id"] else: - used, available = after, left + used, available = plus(used, took), available - disk if reason is None: taken += 1 counts[kind] += 1 busy.add(workdir) + if entry["id"] not in loaded: + granted = plus(granted[0], took), granted[1] - disk, granted[2] + 1 verdicts[entry["id"]] = reason for entry in waiting: if silent(entry, now): @@ -474,15 +482,15 @@ def plus(used: dict[str, float], share: dict[str, float]) -> dict[str, float]: def resources(need: float, share: dict[str, float], used: dict[str, float], available: float, floor: float, budget: dict[str, float], taken: int) -> tuple[str | None, dict[str, float], float]: """Returns: - (reason or None, RAM and CPU used or reserved, disk available) after - the job: one passing the budget beside other jobs reserves its share, + (reason or None, RAM and CPU share, disk GB) the job takes or + reserves: one passing the budget beside other jobs reserves its share, one cutting into the floor its disk, a granted one takes both. """ if taken and (reason := over_budget(used, share, budget)) is not None: - return reason, plus(used, share), available + return reason, share, 0 if available - need < floor: - return f"{need:g} GB would cut into the {floor:g} GB floor; reserved", used, available - need - return None, plus(used, share), available - need + return f"{need:g} GB would cut into the {floor:g} GB floor; reserved", dict.fromkeys(share, 0), need + return None, share, need def host_reason(stage: str, memory: str, silenced: bool, need: float) -> str | None: diff --git a/tests/test_queue.py b/tests/test_queue.py index 79d005e..75d1dd5 100644 --- a/tests/test_queue.py +++ b/tests/test_queue.py @@ -313,12 +313,47 @@ class TestEvaluate(unittest.TestCase): running = job(5, kind="test-nostack", task="work/c", status="active") self.assertEqual( self.decide([running, head, twin, gate, light], loaded=loaded, footprints=shares | {"j5": {"ram": 6, "cpu": 11}}, **rules), - { - "j1": None, "j2": "all 1 quality-high slots taken", "j3": "test runs only beside test-nostack, but quality-high runs", - "j4": "CPU 73 % in use or reserved + 40 % > budget 100 %; reserved", - }, + {"j1": None, "j2": "all 1 quality-high slots taken", "j3": "test runs only beside test-nostack, but quality-high runs", "j4": None}, ) + def test_a_light_entry_meets_only_running_and_granted_jobs_while_heavy_ones_keep_every_reservation(self): + rules = {"slot_limits": {"test": 5, "test-nostack": 5, "deploy": 1}, "light": {"test-nostack"}} + with self.subTest("budget reservations hold back heavy kinds only"): + running, head, twin = job(1, status="active"), job(2), job(3, task="work/b") + light, heavy = job(4, kind="test-nostack", task="work/c", need_gb=0), job(5, task="work/c") + shares = { + "j1": {"ram": 30, "cpu": 50}, "j2": {"ram": 50, "cpu": 60}, "j3": {"ram": 50, "cpu": 60}, + "j4": {"ram": 6, "cpu": 11}, "j5": {"ram": 5, "cpu": 5}, + } + self.assertEqual(self.decide([running, head, twin, light, heavy], footprints=shares, **rules), { + "j2": "RAM 30 % in use or reserved + 50 % > budget 75 %, CPU 50 % in use or reserved + 60 % > budget 100 %; reserved", + "j3": "RAM 80 % in use or reserved + 50 % > budget 75 %, CPU 110 % in use or reserved + 60 % > budget 100 %; reserved", + "j4": None, + "j5": "RAM 136 % in use or reserved + 5 % > budget 75 %, CPU 181 % in use or reserved + 5 % > budget 100 %; reserved", + }) + with self.subTest("disk reservations hold back heavy kinds only"): + big, small, late = job(1, need_gb=50), job(2, kind="test-nostack", task="work/b", need_gb=0), job(3, task="work/c", need_gb=5) + self.assertEqual(self.decide([big, small, late], free_gb=40.0, **rules), { + "j1": "50 GB would cut into the 10 GB floor; reserved", "j2": None, "j3": "5 GB would cut into the 10 GB floor; reserved", + }) + with self.subTest("a granted light counts for later lights, a blocked one reserves against heavy kinds"): + running = job(1, status="active") + first, second = job(2, kind="test-nostack", need_gb=0), job(3, kind="test-nostack", task="work/b", need_gb=0) + heavy = job(4, task="work/c") + shares = {"j1": {"ram": 10, "cpu": 60}, "j2": {"ram": 0, "cpu": 30}, "j3": {"ram": 0, "cpu": 30}, "j4": {"ram": 5, "cpu": 5}} + self.assertEqual(self.decide([running, first, second, heavy], footprints=shares, **rules), { + "j2": None, "j3": "CPU 90 % in use or reserved + 30 % > budget 100 %; reserved", + "j4": "CPU 120 % in use or reserved + 5 % > budget 100 %; reserved", + }) + with self.subTest("a head the host load keeps out while a job runs reserves against heavy kinds only"): + load = "RAM 55 % used + 41 % still to come reach the limit 90 %" + running, head = job(1, kind="test-nostack", status="active", need_gb=0), job(2, kind="deploy") + heavy, light = job(3, task="work/b"), job(4, kind="test-nostack", task="work/c", need_gb=0) + shares = {"j1": {"ram": 0, "cpu": 10}, "j2": {"ram": 0, "cpu": 60}, "j3": {"ram": 0, "cpu": 40}, "j4": {"ram": 0, "cpu": 40}} + self.assertEqual(self.decide([running, head, heavy, light], loaded={"j2": load}, footprints=shares, **rules), { + "j2": None, "j3": "CPU 70 % in use or reserved + 40 % > budget 100 %; reserved", "j4": None, + }) + class TestHistory(unittest.TestCase): @classmethod