From 7c0872ea68646878868da8f0abd0b89388135b9f Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Thu, 12 Nov 2026 11:25:24 +0000 Subject: [PATCH] fix(kvb-org-queue): reread the config on every poll of a waiting hold MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A waiting hold kept the config.json it started with, so a raised RAM or slot limit only reached it after cancel and re-hold. wait() now calls fresh() each poll; a missing, unreadable or incomplete file keeps the last complete config, so a half-saved edit never ends a wait. The History and learning details move to learning.md, bringing SKILL.md from 605 to 540 of 600 words and clearing master's red test_lock. 🎯 Fix: 92 % 💥 Break: 3 % 💸 Cost: 100 % ⏱️ Time: 100 % 🛡️ Security: 100 % 🏆 Score: 89 🚦 Gate: ✅ 90/5, cost and time <= 100, security 100 % Co-Authored-By: Claude Opus 5.5 --- skills/kvb-org-queue/SKILL.md | 20 +++++--------------- skills/kvb-org-queue/learning.md | 16 ++++++++++++++++ skills/kvb-org-queue/scripts/queue.py | 17 ++++++++++++++++- tests/test_queue.py | 22 ++++++++++++++++++++++ 4 files changed, 59 insertions(+), 16 deletions(-) create mode 100644 skills/kvb-org-queue/learning.md diff --git a/skills/kvb-org-queue/SKILL.md b/skills/kvb-org-queue/SKILL.md index 2827d7e..296f851 100644 --- a/skills/kvb-org-queue/SKILL.md +++ b/skills/kvb-org-queue/SKILL.md @@ -51,20 +51,9 @@ Every 20 s each waiting hold evaluates the queue under one lock: ## History and learning -Every finished job appends a row (signature such as `: make `, -kind, minutes, disk, RAM and CPU) to `queue-history.jsonl`; the -`kvb-org-organize` watch hook keeps it across reboots. - -From the last 30 days, once 5 runs exist: - -- per signature, the disk need is p90 disk taken plus 1 GB; -- per kind, the slot limit is how many p90 RAM and median CPU shares fit - between the median load at the grant and `load_limits`, at least 1; -- per job, the footprint is its signature's share, else its kind's. - -Until then the seeds hold: `slot_need_gb[kind]` for disk, 1 slot per kind, -the whole budget as footprint. -The optional `slot_limits {kind: N}` only caps a limit from above. +Every finished job appends a row to `queue-history.jsonl`, from which disk +need, slot limits and footprints are learned; read `learning.md` next to this +file before changing a seed or a limit. ## Inspect and cancel @@ -88,4 +77,5 @@ learned values, `cancel` removes your own waiting job (its hold exits 3). `heavy_commands`, image builds, `make install|verify-*`, tests beyond one file and `gate.py run|hold` outside `queue.py hold`. - Without the config, hold exits 2: tell the operator and ask the chair - (`kvb-org-organize`) until it exists. + (`kvb-org-organize`) until it exists. A waiting hold rereads it every poll + (a broken file keeps the last good one), so a limit change needs no re-hold. diff --git a/skills/kvb-org-queue/learning.md b/skills/kvb-org-queue/learning.md new file mode 100644 index 0000000..32bb723 --- /dev/null +++ b/skills/kvb-org-queue/learning.md @@ -0,0 +1,16 @@ +# Queue history and learning + +Every finished job appends a row (signature such as `: make `, +kind, minutes, disk, RAM and CPU) to `queue-history.jsonl`; the +`kvb-org-organize` watch hook keeps it across reboots. + +From the last 30 days, once 5 runs exist: + +- per signature, the disk need is p90 disk taken plus 1 GB; +- per kind, the slot limit is how many p90 RAM and median CPU shares fit + between the median load at the grant and `load_limits`, at least 1; +- per job, the footprint is its signature's share, else its kind's. + +Until then the seeds hold: `slot_need_gb[kind]` for disk, 1 slot per kind, +the whole budget as footprint. +The optional `slot_limits {kind: N}` only caps a limit from above. diff --git a/skills/kvb-org-queue/scripts/queue.py b/skills/kvb-org-queue/scripts/queue.py index bcbb80d..b0ecb2c 100755 --- a/skills/kvb-org-queue/scripts/queue.py +++ b/skills/kvb-org-queue/scripts/queue.py @@ -198,6 +198,20 @@ def config() -> dict: return cfg +def fresh(cfg: dict) -> dict: + """Returns: + The config as it stands now, else cfg while the file is missing, + unreadable or incomplete, so a half-saved edit never ends a wait. + """ + try: + now = watch.load_config() + except (OSError, ValueError): + return cfg + if not isinstance(now, dict) or watch.missing(now, (*BUDGET_KEYS, *watch.MEMORY_KEYS)) is not None: + return cfg + return now + + def caps(cfg: dict) -> dict[str, int]: return cfg.get("slot_limits", {}) @@ -657,7 +671,7 @@ def wait(cfg: dict, entry_id: str, caught: list[int]) -> tuple[int | None, int]: """Poll until the grant. Args: - cfg: the config. + cfg: the config at the hold's start; every poll reads it anew. entry_id: this hold's entry. caught: signals the caller caught; the first ends the wait. @@ -670,6 +684,7 @@ def wait(cfg: dict, entry_id: str, caught: list[int]) -> tuple[int | None, int]: host = code_of(HOST_SKILLS) if host not in (None, CODE): raise Stale(host, caught) + cfg = fresh(cfg) cpu, mem = watch.load(cfg) free, _ = watch.disk(cfg) now = time.time() diff --git a/tests/test_queue.py b/tests/test_queue.py index 75d1dd5..8d6c6cf 100644 --- a/tests/test_queue.py +++ b/tests/test_queue.py @@ -15,6 +15,7 @@ import time import unittest from collections.abc import Callable from pathlib import Path +from unittest import mock REPO_ROOT = Path(__file__).resolve().parents[1] QUEUE = REPO_ROOT / "skills" / "kvb-org-queue" / "scripts" / "queue.py" @@ -757,6 +758,27 @@ class TestQueueCommands(unittest.TestCase): self.assertFalse(lock.exists()) self.assertEqual(self.entries(), []) + def test_a_waiting_hold_is_granted_once_the_config_it_waits_on_changes(self): + self.cfg["slot_limits"]["deploy"] = 0 + self.write_config() + hold, _ = self.blocked_hold("true") + self.cfg["slot_limits"]["deploy"] = 1 + self.write_config() + self.assertEqual(hold.wait(timeout=60), 0) + self.assertEqual(self.entries(), []) + + def test_fresh_takes_the_new_config_and_keeps_the_last_good_one_on_a_broken_or_partial_file(self): + queue = _load(QUEUE) + good = dict(self.cfg) + new = {**self.cfg, "slot_limits": {"test": 2, "deploy": 2}} + partial = {key: value for key, value in self.cfg.items() if key != "budget_percent"} + for loaded, expected in ((new, new), (None, good), (partial, good), ([], good)): + with self.subTest(loaded=loaded), mock.patch.object(queue.watch, "load_config", return_value=loaded): + self.assertEqual(queue.fresh(good), expected) + for error in (ValueError("half-saved"), OSError("gone")): + with self.subTest(error=error), mock.patch.object(queue.watch, "load_config", side_effect=error): + self.assertEqual(queue.fresh(good), good) + def test_hold_takes_the_need_learned_for_its_command_from_both_history_files(self): self.cfg["slot_limits"]["deploy"] = 0 self.write_config()