Merge fix/kvb-org-queue-light-kinds-skip-reservations into unify/skills-20261111

skills/kvb-org-queue/SKILL.md

@@ -42,7 +42,9 @@ Every 20 s each waiting hold evaluates the queue under one lock:
    `exclusive_kinds` run only beside `light_kinds`.
 3. A job whose disk need would cut below the stop threshold, or whose
    footprint would push the running ones past `budget_percent`, reserves it
-   against the jobs behind it; a job alone always fits.
+   against the jobs behind it but `light_kinds`, which meet only running and
+   granted jobs, so a stream of light jobs can delay a heavy head; a job
+   alone always fits.
 4. An unblocked job counts as granted for the jobs behind it.
 5. Every job, alone too, needs CPU, and RAM used plus what it and the
    running jobs may still take, below `load_limits`.

skills/kvb-org-queue/scripts/queue.py

@@ -72,7 +72,8 @@ yet) may sum to at most these percentage points, a job alone always fits;
 slot_limits {kind: N}, optional, caps the learned slot limit of a kind from
 above (history.slot_limits); exclusive_kinds [kind] and light_kinds [kind],
 optional: a job of an exclusive kind runs only beside jobs of light kinds, and
-no job of another kind starts beside it; queue_enforce (bool), optional, read by block
+no job of another kind starts beside it; a light job meets only the budget and
+disk of running and granted jobs, never a reservation; queue_enforce (bool), optional, read by block
 only. The watch.py keys disk, thresholds_gb (stop is the floor every grant
 leaves free), slot_need_gb (the disk seed per kind), load_limits (every grant
 also needs CPU busy below cpu_percent and RAM used plus the job's learned RAM
@@ -405,20 +406,24 @@ def evaluate(entries: list[dict], free_gb: float, floor: float, stage: str, memo
       An unblocked entry counts as granted for the entries after it, whoever
       holds it, so a lower entry never takes a higher entry's slot; one
       blocked by disk or budget reserves its need against the entries after it.
-      A hotfix entry that passes every check but the host load holds back
-      every entry after it, so lighter jobs cannot keep the load up. While no
-      queue job is active, any other entry the host load alone keeps out
-      reserves nothing and holds nothing back: that load lies outside the
-      queue, so holding the queue frees none of it. An
-      exclusive entry that only running jobs of other kinds keep out holds
-      back every entry after it but light ones, so none overtakes it. A waiting
-      entry whose holder has not polled for CLAIM_SECONDS reserves nothing and
-      comes last.
+      A light entry meets only the disk, RAM and CPU of running and granted
+      entries: reservations, a head the host load keeps out included, hold
+      back heavy kinds alone, so light jobs that keep coming can delay a heavy
+      head without bound. A hotfix entry that passes every check but the host
+      load holds back every entry after it, so lighter jobs cannot keep the
+      load up. While no queue job is active, any other entry the host load
+      alone keeps out reserves nothing and holds nothing back: that load lies
+      outside the queue, so holding the queue frees none of it. An exclusive
+      entry that only running jobs of other kinds keep out holds back every
+      entry after it but light ones, so none overtakes it. A waiting entry
+      whose holder has not polled for CLAIM_SECONDS reserves nothing and comes
+      last.
     """
     active = [e for e in entries if e["status"] == "active"]
     available = free_gb - sum(e["need_gb"] for e in active)
     used = {resource: sum(footprints[e["id"]][resource] for e in active) for resource in budget}
     taken = len(active)
+    granted = used, available, taken
     counts = Counter(e["kind"] for e in active)
     busy = {e["workdir"] for e in active}
     verdicts: dict[str, str | None] = {}
@@ -438,17 +443,20 @@ def evaluate(entries: list[dict], free_gb: float, floor: float, stage: str, memo
             if reason is not None and kind in exclusive:
                 drain = entry["id"]
         if reason is None:
-            reason, after, left = resources(need, share, used, available, floor, budget, taken)
+            pool_used, pool_available, pool_taken = granted if kind in light else (used, available, taken)
+            reason, took, disk = resources(need, share, pool_used, pool_available, floor, budget, pool_taken)
             if reason is None and entry["id"] in loaded and (entry["hotfix"] or not active):
                 reason = loaded[entry["id"]]
                 if entry["hotfix"]:
                     held = entry["id"]
             else:
-                used, available = after, left
+                used, available = plus(used, took), available - disk
                 if reason is None:
                     taken += 1
                     counts[kind] += 1
                     busy.add(workdir)
+                    if entry["id"] not in loaded:
+                        granted = plus(granted[0], took), granted[1] - disk, granted[2] + 1
         verdicts[entry["id"]] = reason
     for entry in waiting:
         if silent(entry, now):
@@ -474,15 +482,15 @@ def plus(used: dict[str, float], share: dict[str, float]) -> dict[str, float]:
 def resources(need: float, share: dict[str, float], used: dict[str, float], available: float, floor: float,
               budget: dict[str, float], taken: int) -> tuple[str | None, dict[str, float], float]:
     """Returns:
-        (reason or None, RAM and CPU used or reserved, disk available) after
-        the job: one passing the budget beside other jobs reserves its share,
+        (reason or None, RAM and CPU share, disk GB) the job takes or
+        reserves: one passing the budget beside other jobs reserves its share,
         one cutting into the floor its disk, a granted one takes both.
     """
     if taken and (reason := over_budget(used, share, budget)) is not None:
-        return reason, plus(used, share), available
+        return reason, share, 0
     if available - need < floor:
-        return f"{need:g} GB would cut into the {floor:g} GB floor; reserved", used, available - need
-    return None, plus(used, share), available - need
+        return f"{need:g} GB would cut into the {floor:g} GB floor; reserved", dict.fromkeys(share, 0), need
+    return None, share, need
 
 
 def host_reason(stage: str, memory: str, silenced: bool, need: float) -> str | None:

tests/test_queue.py

@@ -313,12 +313,47 @@ class TestEvaluate(unittest.TestCase):
         running = job(5, kind="test-nostack", task="work/c", status="active")
         self.assertEqual(
             self.decide([running, head, twin, gate, light], loaded=loaded, footprints=shares | {"j5": {"ram": 6, "cpu": 11}}, **rules),
-            {
-                "j1": None, "j2": "all 1 quality-high slots taken", "j3": "test runs only beside test-nostack, but quality-high runs",
-                "j4": "CPU 73 % in use or reserved + 40 % > budget 100 %; reserved",
-            },
+            {"j1": None, "j2": "all 1 quality-high slots taken", "j3": "test runs only beside test-nostack, but quality-high runs", "j4": None},
         )
 
+    def test_a_light_entry_meets_only_running_and_granted_jobs_while_heavy_ones_keep_every_reservation(self):
+        rules = {"slot_limits": {"test": 5, "test-nostack": 5, "deploy": 1}, "light": {"test-nostack"}}
+        with self.subTest("budget reservations hold back heavy kinds only"):
+            running, head, twin = job(1, status="active"), job(2), job(3, task="work/b")
+            light, heavy = job(4, kind="test-nostack", task="work/c", need_gb=0), job(5, task="work/c")
+            shares = {
+                "j1": {"ram": 30, "cpu": 50}, "j2": {"ram": 50, "cpu": 60}, "j3": {"ram": 50, "cpu": 60},
+                "j4": {"ram": 6, "cpu": 11}, "j5": {"ram": 5, "cpu": 5},
+            }
+            self.assertEqual(self.decide([running, head, twin, light, heavy], footprints=shares, **rules), {
+                "j2": "RAM 30 % in use or reserved + 50 % > budget 75 %, CPU 50 % in use or reserved + 60 % > budget 100 %; reserved",
+                "j3": "RAM 80 % in use or reserved + 50 % > budget 75 %, CPU 110 % in use or reserved + 60 % > budget 100 %; reserved",
+                "j4": None,
+                "j5": "RAM 136 % in use or reserved + 5 % > budget 75 %, CPU 181 % in use or reserved + 5 % > budget 100 %; reserved",
+            })
+        with self.subTest("disk reservations hold back heavy kinds only"):
+            big, small, late = job(1, need_gb=50), job(2, kind="test-nostack", task="work/b", need_gb=0), job(3, task="work/c", need_gb=5)
+            self.assertEqual(self.decide([big, small, late], free_gb=40.0, **rules), {
+                "j1": "50 GB would cut into the 10 GB floor; reserved", "j2": None, "j3": "5 GB would cut into the 10 GB floor; reserved",
+            })
+        with self.subTest("a granted light counts for later lights, a blocked one reserves against heavy kinds"):
+            running = job(1, status="active")
+            first, second = job(2, kind="test-nostack", need_gb=0), job(3, kind="test-nostack", task="work/b", need_gb=0)
+            heavy = job(4, task="work/c")
+            shares = {"j1": {"ram": 10, "cpu": 60}, "j2": {"ram": 0, "cpu": 30}, "j3": {"ram": 0, "cpu": 30}, "j4": {"ram": 5, "cpu": 5}}
+            self.assertEqual(self.decide([running, first, second, heavy], footprints=shares, **rules), {
+                "j2": None, "j3": "CPU 90 % in use or reserved + 30 % > budget 100 %; reserved",
+                "j4": "CPU 120 % in use or reserved + 5 % > budget 100 %; reserved",
+            })
+        with self.subTest("a head the host load keeps out while a job runs reserves against heavy kinds only"):
+            load = "RAM 55 % used + 41 % still to come reach the limit 90 %"
+            running, head = job(1, kind="test-nostack", status="active", need_gb=0), job(2, kind="deploy")
+            heavy, light = job(3, task="work/b"), job(4, kind="test-nostack", task="work/c", need_gb=0)
+            shares = {"j1": {"ram": 0, "cpu": 10}, "j2": {"ram": 0, "cpu": 60}, "j3": {"ram": 0, "cpu": 40}, "j4": {"ram": 0, "cpu": 40}}
+            self.assertEqual(self.decide([running, head, heavy, light], loaded={"j2": load}, footprints=shares, **rules), {
+                "j2": None, "j3": "CPU 70 % in use or reserved + 40 % > budget 100 %; reserved", "j4": None,
+            })
+
 
 class TestHistory(unittest.TestCase):
     @classmethod