feat(kvb-org-queue): let jobs pass holders that can never start and help a disk-blocked head
A waiting job passes a same-line holder that cannot start under the current caps and takes its place; the decision is recomputed every pass, so a holder that recovers regains its place. A hotfix holds others back and heads the line only while it could start. A holder blocked by disk alone gets the cleanup and the Main notice even while smaller jobs backfill. Main-first reservations stay; the replay pins the lines bounds and caps its mean. Tests run once on unify/skills-20261114 after the merge (operator order 2026-11-14). 🎯 Fix: 80 % 💥 Break: 12 % 💸 Tokens: 100 % ⏱️ Velocity: – 🛡️ Security: 100 % 🏆 Score: 70 🚦 Gate: ⛔ fix 80 % < 90, break 12 % > 5 Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
a2a03995cb
commit
9fc1383e97
9 files changed
+5020
-200
No files matched your search
File diff suppressed because it is too large.
Load diff
+864
-40
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,296 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib.util
|
||||
import json
|
||||
import os
|
||||
import statistics
|
||||
import tempfile
|
||||
import unittest
|
||||
from collections import Counter
|
||||
from pathlib import Path
|
||||
from unittest import mock
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parents[1]
|
||||
QUEUE = REPO_ROOT / "skills" / "kvb-org-queue" / "scripts" / "queue.py"
|
||||
FIXTURE = Path(__file__).resolve().parent / "queue-replay-20261114.json"
|
||||
FP = {
|
||||
"quality-high": {"ram": 40.6, "cpu": 59.8}, "test": {"ram": 34.5, "cpu": 24.5},
|
||||
"test-nostack": {"ram": 7.2, "cpu": 9.8}, "iteration": {"ram": 10.4, "cpu": 7.9},
|
||||
"gui-dind": {"ram": 20.7, "cpu": 63.6}, "swarm": {"ram": 40.6, "cpu": 59.8}, "operator": {"ram": 10.0, "cpu": 10.0},
|
||||
}
|
||||
SLOTS = {"gui-dind": 1, "iteration": 3, "operator": 1, "quality-high": 1, "swarm": 1, "test": 1, "test-nostack": 4}
|
||||
BUDGET = {"ram": 75, "cpu": 100}
|
||||
LIMITS = {"cpu_percent": 101, "mem_percent": 85}
|
||||
EXCLUSIVE = {"quality-high", "test"}
|
||||
LIGHT = {"test-nostack"}
|
||||
FLOOR = 15.0
|
||||
STEP = 20
|
||||
RAMP = 900
|
||||
HOUR = 3600
|
||||
HORIZON = 24 * HOUR
|
||||
LINES_MEAN = 13.53
|
||||
SCENARIOS = {
|
||||
"plain": {}, "ram45": {"baseline": 45.0}, "hotfix": {"hotfix": True}, "lines": {"hotfix": True, "lines": True, "agents": True},
|
||||
"real": {"free": 110.0, "ranked": True, "agents": True},
|
||||
}
|
||||
|
||||
|
||||
def load(path: Path, name: str):
|
||||
spec = importlib.util.spec_from_file_location(name, path)
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(module)
|
||||
return module
|
||||
|
||||
|
||||
def scenario(jobs: list[dict], name: str) -> list[dict]:
|
||||
"""Returns:
|
||||
jobs as the scenario name replays them: `hotfix` marks every Main
|
||||
quality-high job a hotfix; `lines` adds before each such hotfix four
|
||||
light jobs that fill the test-nostack slots, then in the hotfix's
|
||||
claims an older light job and an older quality-high of Main, the
|
||||
shapes of the skeptic's probes P3 and P3b, each with a seq below
|
||||
every job enqueued after it (placed()); the older quality-high is
|
||||
held to bound() and rivals() instead of master's wait.
|
||||
"""
|
||||
spec = SCENARIOS[name]
|
||||
if not spec.get("hotfix"):
|
||||
return jobs
|
||||
marked = [{**j, "hotfix": "main" if j["main"] and j["kind"] == "quality-high" else None} for j in jobs]
|
||||
if not spec.get("lines"):
|
||||
return marked
|
||||
added = []
|
||||
for h in (j for j in marked if j["hotfix"]):
|
||||
base = {"task": h["task"], "need_gb": 0, "hotfix": None, "enqueued": h["enqueued"] - 60}
|
||||
added += [{**base, "id": f"{h['id']}-fill{k}", "seq": placed(marked, h["enqueued"] - 90, 0.1 + 0.1 * k), "kind": "test-nostack",
|
||||
"name": "other", "main": False, "workdir": f"/replay/{h['id']}/fill{k}", "claims": [f"/replay/{h['id']}/fill{k}"],
|
||||
"enqueued": h["enqueued"] - 90, "dur": 900.0} for k in range(4)]
|
||||
added += [{**base, "id": f"{h['id']}-older-light", "seq": placed(marked, base["enqueued"], 0.5), "kind": "test-nostack",
|
||||
"name": "Main", "main": True, "workdir": h["workdir"], "claims": h["claims"], "dur": 300.0},
|
||||
{**base, "id": f"{h['id']}-older-gate", "seq": placed(marked, h["enqueued"] - 30, 0.6), "kind": "quality-high",
|
||||
"name": "Main", "main": True, "workdir": h["workdir"], "claims": h["claims"], "need_gb": 15,
|
||||
"enqueued": h["enqueued"] - 30, "dur": h["dur"]}]
|
||||
return sorted(marked + added, key=lambda j: j["seq"])
|
||||
|
||||
|
||||
def placed(jobs: list[dict], enqueued: float, fraction: float) -> float:
|
||||
"""Returns:
|
||||
A seq fraction (0 to 1) below the lowest seq of the jobs enqueued
|
||||
after enqueued, so arrival order agrees with the enqueue stamps.
|
||||
"""
|
||||
return min(j["seq"] for j in jobs if j["enqueued"] > enqueued) - 1 + fraction
|
||||
|
||||
|
||||
def world(name: str, jobs: list[dict]) -> dict:
|
||||
"""Returns:
|
||||
The host of scenario name: baseline (RAM % used outside the queue,
|
||||
None for none), free (t -> free disk in GB; `real` drops it below
|
||||
FLOOR for the hour after the middle job's enqueue), ranks (`real`:
|
||||
every task by name) and agents (`real`, `lines`: Main first, then by
|
||||
name).
|
||||
"""
|
||||
spec = SCENARIOS[name]
|
||||
dip = sorted(j["enqueued"] for j in jobs)[len(jobs) // 2]
|
||||
return {
|
||||
"baseline": spec.get("baseline"),
|
||||
"free": (lambda t: 12.0 if dip <= t < dip + HOUR else spec["free"]) if "free" in spec else (lambda t: 1000.0),
|
||||
"ranks": {task: n for n, task in enumerate(sorted({j["task"] for j in jobs}))} if spec.get("ranked") else {},
|
||||
"agents": {agent: n for n, agent in enumerate(sorted({j["name"] for j in jobs}, key=lambda a: (a != "Main", a)))}
|
||||
if spec.get("agents") else {},
|
||||
}
|
||||
|
||||
|
||||
def loaded(queue, active: list[dict], waiting: list[dict], baseline: float | None, t: float) -> dict[str, str]:
|
||||
"""Returns:
|
||||
{waiting job id: load_reason()} with RAM used = baseline + the ramped
|
||||
active jobs' RAM, the young ones' RAM still to come; queue.Outside
|
||||
when the baseline alone keeps the job out (held_back()).
|
||||
"""
|
||||
if baseline is None:
|
||||
return {}
|
||||
ramped = sum(FP[a["kind"]]["ram"] for a in active if t - a["granted"] >= RAMP)
|
||||
young = sum(FP[a["kind"]]["ram"] for a in active if t - a["granted"] < RAMP)
|
||||
out = {}
|
||||
for j in waiting:
|
||||
reason = queue.load_reason(LIMITS, 0.0, baseline + ramped, young + FP[j["kind"]]["ram"])
|
||||
if reason is not None:
|
||||
alone = queue.load_reason(LIMITS, 0.0, baseline, FP[j["kind"]]["ram"])
|
||||
out[j["id"]] = reason if alone is None else queue.Outside(reason)
|
||||
return out
|
||||
|
||||
|
||||
def simulate(queue, jobs: list[dict], name: str, branch: bool, events: list | None = None) -> dict[str, float | None]:
|
||||
"""Args:
|
||||
queue: the queue.py module to replay.
|
||||
jobs: the scenario's jobs, each with its claims, enqueued and dur.
|
||||
name: the scenario, whose world() the replay runs on.
|
||||
branch: whether queue has this branch's evaluate (drain, latched(),
|
||||
stalled()) or master's.
|
||||
events: when given, the branch's stall events are appended as (t,
|
||||
label, head id): each ESCALATION send, `cleanup` for each
|
||||
cleanup started, `notice` for each notice.
|
||||
|
||||
Returns:
|
||||
{job id: minutes from enqueue to grant, None when it never started
|
||||
within HORIZON after the last enqueue}; every STEP seconds each
|
||||
waiting job polls and every job granted starts at once.
|
||||
"""
|
||||
host = world(name, jobs)
|
||||
pending = sorted(({**j, "status": "waiting", "granted": None} for j in jobs), key=lambda j: (j["enqueued"], j["seq"]))
|
||||
t, end = pending[0]["enqueued"], max(j["enqueued"] for j in pending) + HORIZON
|
||||
waiting: list[dict] = []
|
||||
active: list[dict] = []
|
||||
data: dict = {"drain": None}
|
||||
grants: dict[str, float] = {}
|
||||
while (pending or waiting) and t <= end:
|
||||
active = [a for a in active if a["granted"] + a["dur"] > t]
|
||||
while pending and pending[0]["enqueued"] <= t:
|
||||
waiting.append(pending.pop(0))
|
||||
if not waiting:
|
||||
t = pending[0]["enqueued"]
|
||||
continue
|
||||
for j in waiting:
|
||||
j["polled"] = t
|
||||
entries = active + waiting
|
||||
load, free = loaded(queue, active, waiting, host["baseline"], t), host["free"](t)
|
||||
args = (entries, free, FLOOR, "stop" if free < FLOOR else "ok", "ok", False, SLOTS, {j["id"]: FP[j["kind"]] for j in entries},
|
||||
BUDGET, host["ranks"], host["agents"], load, EXCLUSIVE, LIGHT)
|
||||
if branch:
|
||||
verdicts, drain, order = queue.evaluate(*args, data["drain"], t)
|
||||
data["drain"] = queue.latched(data, drain, t)
|
||||
go = [j for j in waiting if verdicts[j["id"]] is None]
|
||||
if events is not None:
|
||||
data["entries"] = entries
|
||||
sends, clean, _ = queue.stalled(data, order, verdicts, FLOOR, free, lambda: (False, []), t)
|
||||
events += [(t, "notice" if label.startswith("noticed") else label, order[0]["id"]) for label, _ in sends]
|
||||
events += [(t, "cleanup", order[0]["id"])] if clean else []
|
||||
else:
|
||||
verdicts = queue.evaluate(*args, t)
|
||||
go = [j for j in waiting if verdicts[j["id"]] is None and j["id"] not in load]
|
||||
for j in go:
|
||||
j.update(status="active", granted=t)
|
||||
grants[j["id"]] = t
|
||||
waiting.remove(j)
|
||||
active.append(j)
|
||||
if go and branch:
|
||||
data[queue.GRANTED] = t
|
||||
data.pop(queue.STALL, None)
|
||||
t += STEP
|
||||
return {j["id"]: (grants[j["id"]] - j["enqueued"]) / 60 if j["id"] in grants else None for j in jobs}
|
||||
|
||||
|
||||
def mean(waits: dict[str, float | None], jobs: list[dict]) -> float:
|
||||
cutoff = max(j["enqueued"] for j in jobs) + HORIZON
|
||||
return statistics.mean(w if w is not None else (cutoff - j["enqueued"]) / 60 for j, w in ((j, waits[j["id"]]) for j in jobs))
|
||||
|
||||
|
||||
def overlaps(waits: dict[str, float | None], jobs: list[dict]) -> list[tuple[str, str]]:
|
||||
runs = [(j, j["enqueued"] + waits[j["id"]] * 60) for j in jobs if waits[j["id"]] is not None]
|
||||
return [(a["id"], b["id"]) for n, (a, ga) in enumerate(runs) for b, gb in runs[n + 1:]
|
||||
if set(a["claims"]) & set(b["claims"]) and ga < gb + b["dur"] and gb < ga + a["dur"]]
|
||||
|
||||
|
||||
def started(waits: dict[str, float | None], job: dict) -> float:
|
||||
return job["enqueued"] + waits[job["id"]] * 60
|
||||
|
||||
|
||||
def took(e: dict, g: float, x: dict, runs: list[tuple[dict, float]]) -> bool:
|
||||
"""Returns:
|
||||
Whether e, starting at g beside runs ((job, start)), took what x
|
||||
needs: its slot (e's was the last free one), its exclusive room (one
|
||||
of the two exclusive, the other not light) or its budget (only the
|
||||
light jobs count beside an exclusive x).
|
||||
"""
|
||||
running = [j for j, s in runs if j is not e and s <= g < s + j["dur"]]
|
||||
beside = [j for j in running if j["kind"] in LIGHT or x["kind"] not in EXCLUSIVE]
|
||||
slot = e["kind"] == x["kind"] and sum(j["kind"] == e["kind"] for j in running) + 1 >= SLOTS[e["kind"]]
|
||||
room = x["kind"] in EXCLUSIVE and e["kind"] not in LIGHT or e["kind"] in EXCLUSIVE and x["kind"] not in LIGHT
|
||||
return slot or room or any(sum(FP[j["kind"]][r] for j in beside) + FP[e["kind"]][r] + FP[x["kind"]][r] > BUDGET[r] for r in BUDGET)
|
||||
|
||||
|
||||
def bound(waits: dict[str, float | None], jobs: list[dict], x: dict) -> float:
|
||||
"""Returns:
|
||||
The latest grant the rule allows x: one STEP after the later of its
|
||||
enqueue and the end of every job started before it that is older and
|
||||
shares one of its claims or is a hotfix.
|
||||
"""
|
||||
ends = [started(waits, j) + j["dur"] for j in jobs if j is not x and waits[j["id"]] is not None and started(waits, j) < started(waits, x)
|
||||
and (j["enqueued"] < x["enqueued"] and set(j["claims"]) & set(x["claims"]) or j["hotfix"])]
|
||||
return max([x["enqueued"], *ends]) + STEP
|
||||
|
||||
|
||||
def rivals(waits: dict[str, float | None], jobs: list[dict], x: dict) -> list[str]:
|
||||
"""Returns:
|
||||
The jobs after x in arrival (seq), of other claims and no hotfix,
|
||||
that started while x waited and took what it needs (took()).
|
||||
"""
|
||||
runs = [(j, started(waits, j)) for j in jobs if waits[j["id"]] is not None]
|
||||
return [e["id"] for e, g in runs if not e["hotfix"] and e["seq"] > x["seq"] and not set(e["claims"]) & set(x["claims"])
|
||||
and x["enqueued"] <= g < started(waits, x) and took(e, g, x, runs)]
|
||||
|
||||
|
||||
class TestReplay(unittest.TestCase):
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
tmp = tempfile.TemporaryDirectory()
|
||||
cls.addClassCleanup(tmp.cleanup)
|
||||
with mock.patch.dict(os.environ, {"AGENTS_COUNCIL_ROOT": str(Path(tmp.name).resolve() / "council")}):
|
||||
cls.queue = load(QUEUE, "queue_replay_under_test")
|
||||
cls.fixture = json.loads(FIXTURE.read_text(encoding="utf-8"))
|
||||
cls.jobs = cls.fixture["jobs"]
|
||||
cls.scenes = {name: scenario(cls.jobs, name) for name in SCENARIOS}
|
||||
cls.events: list = []
|
||||
cls.waits = {name: simulate(cls.queue, cls.scenes[name], name, True, cls.events if name == "real" else None) for name in SCENARIOS}
|
||||
|
||||
def test_the_fixture_is_todays_140_finished_jobs_with_frozen_claims_and_masters_waits(self):
|
||||
self.assertEqual(len(self.jobs), 140)
|
||||
self.assertTrue(all("claims" in j and j["dur"] > 0 for j in self.jobs))
|
||||
self.assertEqual(set(self.fixture["master"]), set(SCENARIOS))
|
||||
self.assertEqual(Counter(j["id"].rsplit("-", 1)[-1] for j in self.scenes["lines"] if j["id"].startswith("quality-high-1930-")),
|
||||
Counter({"fill0": 1, "fill1": 1, "fill2": 1, "fill3": 1, "light": 1, "gate": 1}))
|
||||
|
||||
def test_the_mean_wait_stays_within_a_minute_of_master_and_lines_within_a_minute_of_its_main_first_mean(self):
|
||||
for name in ("plain", "ram45", "hotfix", "real"):
|
||||
jobs = self.scenes[name]
|
||||
with self.subTest(scenario=name):
|
||||
self.assertLessEqual(mean(self.waits[name], jobs), mean(self.fixture["master"][name], jobs) + 1)
|
||||
with self.subTest(scenario="lines"):
|
||||
self.assertLessEqual(mean(self.waits["lines"], self.scenes["lines"]), LINES_MEAN + 1,
|
||||
"lines trades mean wait for Main's bound, capped a minute above the accepted cost")
|
||||
|
||||
def test_no_scenario_starves_a_job_master_runs(self):
|
||||
for name in SCENARIOS:
|
||||
with self.subTest(scenario=name):
|
||||
starved = {i for i, w in self.waits[name].items() if w is None}
|
||||
self.assertEqual(starved - {i for i, w in self.fixture["master"][name].items() if w is None}, set())
|
||||
|
||||
def test_no_hotfix_and_no_main_job_waits_longer_than_on_master(self):
|
||||
for name in ("plain", "hotfix", "lines", "real"):
|
||||
master, branch = self.fixture["master"][name], self.waits[name]
|
||||
jobs = [j for j in self.scenes[name] if (j["main"] or j["hotfix"]) and not j["id"].endswith("-older-gate")]
|
||||
with self.subTest(scenario=name):
|
||||
self.assertEqual([(j["id"], master[j["id"]], branch[j["id"]]) for j in jobs if branch[j["id"]] is None or (
|
||||
master[j["id"]] is not None and branch[j["id"]] > master[j["id"]] + STEP / 60)], [])
|
||||
|
||||
def test_an_older_main_gate_starts_by_its_bound_and_no_younger_rival_takes_its_room(self):
|
||||
waits, jobs = self.waits["lines"], self.scenes["lines"]
|
||||
gates = [j for j in jobs if j["id"].endswith("-older-gate")]
|
||||
self.assertEqual(len(gates), 2)
|
||||
self.assertEqual({x["id"]: round((bound(waits, jobs, x) - x["enqueued"]) / 60, 1) for x in gates},
|
||||
{"quality-high-16-older-gate": 19.5, "quality-high-1930-older-gate": 33.5})
|
||||
self.assertEqual([(x["id"], waits[x["id"]]) for x in gates if waits[x["id"]] is None or started(waits, x) > bound(waits, jobs, x)], [])
|
||||
self.assertEqual({x["id"]: rivals(waits, jobs, x) for x in gates if rivals(waits, jobs, x)}, {})
|
||||
|
||||
def test_no_two_jobs_of_one_claim_run_at_once(self):
|
||||
for name, jobs in self.scenes.items():
|
||||
with self.subTest(scenario=name):
|
||||
self.assertEqual(overlaps(self.waits[name], jobs), [])
|
||||
|
||||
def test_the_disk_dip_starts_one_cleanup_and_one_notice_per_head_and_escalates(self):
|
||||
per_head = Counter((label, head) for _, label, head in self.events if label in ("cleanup", "notice"))
|
||||
self.assertTrue(per_head)
|
||||
self.assertEqual({count for count in per_head.values()}, {1})
|
||||
self.assertIn("escalated post", {label for _, label, _ in self.events})
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -394,6 +394,22 @@ class TestRadio(unittest.TestCase):
|
||||
self._prompt("s3")
|
||||
self.assertEqual(log.read_text(encoding="utf-8"), "Agents radio MAYDAY by One: smoke\n")
|
||||
|
||||
def test_a_mayday_that_sets_no_silence_notifies_at_once_and_none_replaces_another_senders_silence(self):
|
||||
log = self.root.parent / "notified.log"
|
||||
script = "import sys; open(sys.argv[1], 'a').write(sys.argv[2] + '\\n')"
|
||||
self.config.write_text(json.dumps({"notify": [sys.executable, "-c", script, str(log)]}), encoding="utf-8")
|
||||
self._send("s1", "send", "mayday", "smoke")
|
||||
self._send("s2", "send", "mayday", "queue stall", "--no-silence")
|
||||
self._send("s3", "send", "mayday", "leaked credential")
|
||||
silence = json.loads((self.root / "radio-silence.json").read_text(encoding="utf-8"))
|
||||
self._prompt("s2")
|
||||
self._prompt("s3")
|
||||
self.assertEqual((silence["by"], silence["text"]), ("One", "smoke"))
|
||||
self.assertEqual(log.read_text(encoding="utf-8").splitlines(), [
|
||||
"Agents radio MAYDAY by Two: queue stall", "Agents radio MAYDAY by Three: leaked credential",
|
||||
"Agents radio MAYDAY by One: smoke",
|
||||
])
|
||||
|
||||
def test_listener_wakes_on_mayday(self):
|
||||
proc = self._listen("s2")
|
||||
self.assertEqual(self._send("s1", "send", "mayday", "host down"), ["sent mayday to all", "wake: SendMessage to Three"])
|
||||
|
||||
Reference in new issue
Block a user