feat(kvb-org-queue): let jobs pass holders that can never start and help a disk-blocked head

A waiting job passes a same-line holder that cannot start under the current caps and takes its place; the decision is recomputed every pass, so a holder that recovers regains its place. A hotfix holds others back and heads the line only while it could start. A holder blocked by disk alone gets the cleanup and the Main notice even while smaller jobs backfill. Main-first reservations stay; the replay pins the lines bounds and caps its mean. Tests run once on unify/skills-20261114 after the merge (operator order 2026-11-14).

🎯 Fix: 80 %
💥 Break: 12 %
💸 Tokens: 100 %
⏱️ Velocity: –
🛡️ Security: 100 %
🏆 Score: 70
🚦 Gate: ⛔ fix 80 % < 90, break 12 % > 5

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Kevin Veen-BirkenbachandClaude Opus 5.5 committed 2026-11-14 21:17:30 +00:00
1 parent a2a03995cb
commit 9fc1383e97
9 files changed
+5020 -200

No files matched your search

+296
View File
@@ -0,0 +1,296 @@
from __future__ import annotations
import importlib.util
import json
import os
import statistics
import tempfile
import unittest
from collections import Counter
from pathlib import Path
from unittest import mock
REPO_ROOT = Path(__file__).resolve().parents[1]
QUEUE = REPO_ROOT / "skills" / "kvb-org-queue" / "scripts" / "queue.py"
FIXTURE = Path(__file__).resolve().parent / "queue-replay-20261114.json"
FP = {
"quality-high": {"ram": 40.6, "cpu": 59.8}, "test": {"ram": 34.5, "cpu": 24.5},
"test-nostack": {"ram": 7.2, "cpu": 9.8}, "iteration": {"ram": 10.4, "cpu": 7.9},
"gui-dind": {"ram": 20.7, "cpu": 63.6}, "swarm": {"ram": 40.6, "cpu": 59.8}, "operator": {"ram": 10.0, "cpu": 10.0},
}
SLOTS = {"gui-dind": 1, "iteration": 3, "operator": 1, "quality-high": 1, "swarm": 1, "test": 1, "test-nostack": 4}
BUDGET = {"ram": 75, "cpu": 100}
LIMITS = {"cpu_percent": 101, "mem_percent": 85}
EXCLUSIVE = {"quality-high", "test"}
LIGHT = {"test-nostack"}
FLOOR = 15.0
STEP = 20
RAMP = 900
HOUR = 3600
HORIZON = 24 * HOUR
LINES_MEAN = 13.53
SCENARIOS = {
"plain": {}, "ram45": {"baseline": 45.0}, "hotfix": {"hotfix": True}, "lines": {"hotfix": True, "lines": True, "agents": True},
"real": {"free": 110.0, "ranked": True, "agents": True},
}
def load(path: Path, name: str):
spec = importlib.util.spec_from_file_location(name, path)
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
return module
def scenario(jobs: list[dict], name: str) -> list[dict]:
"""Returns:
jobs as the scenario name replays them: `hotfix` marks every Main
quality-high job a hotfix; `lines` adds before each such hotfix four
light jobs that fill the test-nostack slots, then in the hotfix's
claims an older light job and an older quality-high of Main, the
shapes of the skeptic's probes P3 and P3b, each with a seq below
every job enqueued after it (placed()); the older quality-high is
held to bound() and rivals() instead of master's wait.
"""
spec = SCENARIOS[name]
if not spec.get("hotfix"):
return jobs
marked = [{**j, "hotfix": "main" if j["main"] and j["kind"] == "quality-high" else None} for j in jobs]
if not spec.get("lines"):
return marked
added = []
for h in (j for j in marked if j["hotfix"]):
base = {"task": h["task"], "need_gb": 0, "hotfix": None, "enqueued": h["enqueued"] - 60}
added += [{**base, "id": f"{h['id']}-fill{k}", "seq": placed(marked, h["enqueued"] - 90, 0.1 + 0.1 * k), "kind": "test-nostack",
"name": "other", "main": False, "workdir": f"/replay/{h['id']}/fill{k}", "claims": [f"/replay/{h['id']}/fill{k}"],
"enqueued": h["enqueued"] - 90, "dur": 900.0} for k in range(4)]
added += [{**base, "id": f"{h['id']}-older-light", "seq": placed(marked, base["enqueued"], 0.5), "kind": "test-nostack",
"name": "Main", "main": True, "workdir": h["workdir"], "claims": h["claims"], "dur": 300.0},
{**base, "id": f"{h['id']}-older-gate", "seq": placed(marked, h["enqueued"] - 30, 0.6), "kind": "quality-high",
"name": "Main", "main": True, "workdir": h["workdir"], "claims": h["claims"], "need_gb": 15,
"enqueued": h["enqueued"] - 30, "dur": h["dur"]}]
return sorted(marked + added, key=lambda j: j["seq"])
def placed(jobs: list[dict], enqueued: float, fraction: float) -> float:
"""Returns:
A seq fraction (0 to 1) below the lowest seq of the jobs enqueued
after enqueued, so arrival order agrees with the enqueue stamps.
"""
return min(j["seq"] for j in jobs if j["enqueued"] > enqueued) - 1 + fraction
def world(name: str, jobs: list[dict]) -> dict:
"""Returns:
The host of scenario name: baseline (RAM % used outside the queue,
None for none), free (t -> free disk in GB; `real` drops it below
FLOOR for the hour after the middle job's enqueue), ranks (`real`:
every task by name) and agents (`real`, `lines`: Main first, then by
name).
"""
spec = SCENARIOS[name]
dip = sorted(j["enqueued"] for j in jobs)[len(jobs) // 2]
return {
"baseline": spec.get("baseline"),
"free": (lambda t: 12.0 if dip <= t < dip + HOUR else spec["free"]) if "free" in spec else (lambda t: 1000.0),
"ranks": {task: n for n, task in enumerate(sorted({j["task"] for j in jobs}))} if spec.get("ranked") else {},
"agents": {agent: n for n, agent in enumerate(sorted({j["name"] for j in jobs}, key=lambda a: (a != "Main", a)))}
if spec.get("agents") else {},
}
def loaded(queue, active: list[dict], waiting: list[dict], baseline: float | None, t: float) -> dict[str, str]:
"""Returns:
{waiting job id: load_reason()} with RAM used = baseline + the ramped
active jobs' RAM, the young ones' RAM still to come; queue.Outside
when the baseline alone keeps the job out (held_back()).
"""
if baseline is None:
return {}
ramped = sum(FP[a["kind"]]["ram"] for a in active if t - a["granted"] >= RAMP)
young = sum(FP[a["kind"]]["ram"] for a in active if t - a["granted"] < RAMP)
out = {}
for j in waiting:
reason = queue.load_reason(LIMITS, 0.0, baseline + ramped, young + FP[j["kind"]]["ram"])
if reason is not None:
alone = queue.load_reason(LIMITS, 0.0, baseline, FP[j["kind"]]["ram"])
out[j["id"]] = reason if alone is None else queue.Outside(reason)
return out
def simulate(queue, jobs: list[dict], name: str, branch: bool, events: list | None = None) -> dict[str, float | None]:
"""Args:
queue: the queue.py module to replay.
jobs: the scenario's jobs, each with its claims, enqueued and dur.
name: the scenario, whose world() the replay runs on.
branch: whether queue has this branch's evaluate (drain, latched(),
stalled()) or master's.
events: when given, the branch's stall events are appended as (t,
label, head id): each ESCALATION send, `cleanup` for each
cleanup started, `notice` for each notice.
Returns:
{job id: minutes from enqueue to grant, None when it never started
within HORIZON after the last enqueue}; every STEP seconds each
waiting job polls and every job granted starts at once.
"""
host = world(name, jobs)
pending = sorted(({**j, "status": "waiting", "granted": None} for j in jobs), key=lambda j: (j["enqueued"], j["seq"]))
t, end = pending[0]["enqueued"], max(j["enqueued"] for j in pending) + HORIZON
waiting: list[dict] = []
active: list[dict] = []
data: dict = {"drain": None}
grants: dict[str, float] = {}
while (pending or waiting) and t <= end:
active = [a for a in active if a["granted"] + a["dur"] > t]
while pending and pending[0]["enqueued"] <= t:
waiting.append(pending.pop(0))
if not waiting:
t = pending[0]["enqueued"]
continue
for j in waiting:
j["polled"] = t
entries = active + waiting
load, free = loaded(queue, active, waiting, host["baseline"], t), host["free"](t)
args = (entries, free, FLOOR, "stop" if free < FLOOR else "ok", "ok", False, SLOTS, {j["id"]: FP[j["kind"]] for j in entries},
BUDGET, host["ranks"], host["agents"], load, EXCLUSIVE, LIGHT)
if branch:
verdicts, drain, order = queue.evaluate(*args, data["drain"], t)
data["drain"] = queue.latched(data, drain, t)
go = [j for j in waiting if verdicts[j["id"]] is None]
if events is not None:
data["entries"] = entries
sends, clean, _ = queue.stalled(data, order, verdicts, FLOOR, free, lambda: (False, []), t)
events += [(t, "notice" if label.startswith("noticed") else label, order[0]["id"]) for label, _ in sends]
events += [(t, "cleanup", order[0]["id"])] if clean else []
else:
verdicts = queue.evaluate(*args, t)
go = [j for j in waiting if verdicts[j["id"]] is None and j["id"] not in load]
for j in go:
j.update(status="active", granted=t)
grants[j["id"]] = t
waiting.remove(j)
active.append(j)
if go and branch:
data[queue.GRANTED] = t
data.pop(queue.STALL, None)
t += STEP
return {j["id"]: (grants[j["id"]] - j["enqueued"]) / 60 if j["id"] in grants else None for j in jobs}
def mean(waits: dict[str, float | None], jobs: list[dict]) -> float:
cutoff = max(j["enqueued"] for j in jobs) + HORIZON
return statistics.mean(w if w is not None else (cutoff - j["enqueued"]) / 60 for j, w in ((j, waits[j["id"]]) for j in jobs))
def overlaps(waits: dict[str, float | None], jobs: list[dict]) -> list[tuple[str, str]]:
runs = [(j, j["enqueued"] + waits[j["id"]] * 60) for j in jobs if waits[j["id"]] is not None]
return [(a["id"], b["id"]) for n, (a, ga) in enumerate(runs) for b, gb in runs[n + 1:]
if set(a["claims"]) & set(b["claims"]) and ga < gb + b["dur"] and gb < ga + a["dur"]]
def started(waits: dict[str, float | None], job: dict) -> float:
return job["enqueued"] + waits[job["id"]] * 60
def took(e: dict, g: float, x: dict, runs: list[tuple[dict, float]]) -> bool:
"""Returns:
Whether e, starting at g beside runs ((job, start)), took what x
needs: its slot (e's was the last free one), its exclusive room (one
of the two exclusive, the other not light) or its budget (only the
light jobs count beside an exclusive x).
"""
running = [j for j, s in runs if j is not e and s <= g < s + j["dur"]]
beside = [j for j in running if j["kind"] in LIGHT or x["kind"] not in EXCLUSIVE]
slot = e["kind"] == x["kind"] and sum(j["kind"] == e["kind"] for j in running) + 1 >= SLOTS[e["kind"]]
room = x["kind"] in EXCLUSIVE and e["kind"] not in LIGHT or e["kind"] in EXCLUSIVE and x["kind"] not in LIGHT
return slot or room or any(sum(FP[j["kind"]][r] for j in beside) + FP[e["kind"]][r] + FP[x["kind"]][r] > BUDGET[r] for r in BUDGET)
def bound(waits: dict[str, float | None], jobs: list[dict], x: dict) -> float:
"""Returns:
The latest grant the rule allows x: one STEP after the later of its
enqueue and the end of every job started before it that is older and
shares one of its claims or is a hotfix.
"""
ends = [started(waits, j) + j["dur"] for j in jobs if j is not x and waits[j["id"]] is not None and started(waits, j) < started(waits, x)
and (j["enqueued"] < x["enqueued"] and set(j["claims"]) & set(x["claims"]) or j["hotfix"])]
return max([x["enqueued"], *ends]) + STEP
def rivals(waits: dict[str, float | None], jobs: list[dict], x: dict) -> list[str]:
"""Returns:
The jobs after x in arrival (seq), of other claims and no hotfix,
that started while x waited and took what it needs (took()).
"""
runs = [(j, started(waits, j)) for j in jobs if waits[j["id"]] is not None]
return [e["id"] for e, g in runs if not e["hotfix"] and e["seq"] > x["seq"] and not set(e["claims"]) & set(x["claims"])
and x["enqueued"] <= g < started(waits, x) and took(e, g, x, runs)]
class TestReplay(unittest.TestCase):
@classmethod
def setUpClass(cls):
tmp = tempfile.TemporaryDirectory()
cls.addClassCleanup(tmp.cleanup)
with mock.patch.dict(os.environ, {"AGENTS_COUNCIL_ROOT": str(Path(tmp.name).resolve() / "council")}):
cls.queue = load(QUEUE, "queue_replay_under_test")
cls.fixture = json.loads(FIXTURE.read_text(encoding="utf-8"))
cls.jobs = cls.fixture["jobs"]
cls.scenes = {name: scenario(cls.jobs, name) for name in SCENARIOS}
cls.events: list = []
cls.waits = {name: simulate(cls.queue, cls.scenes[name], name, True, cls.events if name == "real" else None) for name in SCENARIOS}
def test_the_fixture_is_todays_140_finished_jobs_with_frozen_claims_and_masters_waits(self):
self.assertEqual(len(self.jobs), 140)
self.assertTrue(all("claims" in j and j["dur"] > 0 for j in self.jobs))
self.assertEqual(set(self.fixture["master"]), set(SCENARIOS))
self.assertEqual(Counter(j["id"].rsplit("-", 1)[-1] for j in self.scenes["lines"] if j["id"].startswith("quality-high-1930-")),
Counter({"fill0": 1, "fill1": 1, "fill2": 1, "fill3": 1, "light": 1, "gate": 1}))
def test_the_mean_wait_stays_within_a_minute_of_master_and_lines_within_a_minute_of_its_main_first_mean(self):
for name in ("plain", "ram45", "hotfix", "real"):
jobs = self.scenes[name]
with self.subTest(scenario=name):
self.assertLessEqual(mean(self.waits[name], jobs), mean(self.fixture["master"][name], jobs) + 1)
with self.subTest(scenario="lines"):
self.assertLessEqual(mean(self.waits["lines"], self.scenes["lines"]), LINES_MEAN + 1,
"lines trades mean wait for Main's bound, capped a minute above the accepted cost")
def test_no_scenario_starves_a_job_master_runs(self):
for name in SCENARIOS:
with self.subTest(scenario=name):
starved = {i for i, w in self.waits[name].items() if w is None}
self.assertEqual(starved - {i for i, w in self.fixture["master"][name].items() if w is None}, set())
def test_no_hotfix_and_no_main_job_waits_longer_than_on_master(self):
for name in ("plain", "hotfix", "lines", "real"):
master, branch = self.fixture["master"][name], self.waits[name]
jobs = [j for j in self.scenes[name] if (j["main"] or j["hotfix"]) and not j["id"].endswith("-older-gate")]
with self.subTest(scenario=name):
self.assertEqual([(j["id"], master[j["id"]], branch[j["id"]]) for j in jobs if branch[j["id"]] is None or (
master[j["id"]] is not None and branch[j["id"]] > master[j["id"]] + STEP / 60)], [])
def test_an_older_main_gate_starts_by_its_bound_and_no_younger_rival_takes_its_room(self):
waits, jobs = self.waits["lines"], self.scenes["lines"]
gates = [j for j in jobs if j["id"].endswith("-older-gate")]
self.assertEqual(len(gates), 2)
self.assertEqual({x["id"]: round((bound(waits, jobs, x) - x["enqueued"]) / 60, 1) for x in gates},
{"quality-high-16-older-gate": 19.5, "quality-high-1930-older-gate": 33.5})
self.assertEqual([(x["id"], waits[x["id"]]) for x in gates if waits[x["id"]] is None or started(waits, x) > bound(waits, jobs, x)], [])
self.assertEqual({x["id"]: rivals(waits, jobs, x) for x in gates if rivals(waits, jobs, x)}, {})
def test_no_two_jobs_of_one_claim_run_at_once(self):
for name, jobs in self.scenes.items():
with self.subTest(scenario=name):
self.assertEqual(overlaps(self.waits[name], jobs), [])
def test_the_disk_dip_starts_one_cleanup_and_one_notice_per_head_and_escalates(self):
per_head = Counter((label, head) for _, label, head in self.events if label in ("cleanup", "notice"))
self.assertTrue(per_head)
self.assertEqual({count for count in per_head.values()}, {1})
self.assertIn("escalated post", {label for _, label, _ in self.events})
if __name__ == "__main__":
unittest.main()