fix(kvb-org-queue): reread the config on every poll of a waiting hold
A waiting hold kept the config.json it started with, so a raised RAM or slot limit only reached it after cancel and re-hold. wait() now calls fresh() each poll; a missing, unreadable or incomplete file keeps the last complete config, so a half-saved edit never ends a wait. The History and learning details move to learning.md, bringing SKILL.md from 605 to 540 of 600 words and clearing master's red test_lock. 🎯 Fix: 92 % 💥 Break: 3 % 💸 Cost: 100 % ⏱️ Time: 100 % 🛡️ Security: 100 % 🏆 Score: 89 🚦 Gate: ✅ 90/5, cost and time <= 100, security 100 % Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
fd8fc08603
commit
7c0872ea68
4 files changed
+59
-16
No files matched your search
@@ -51,20 +51,9 @@ Every 20 s each waiting hold evaluates the queue under one lock:
|
|||||||
|
|
||||||
## History and learning
|
## History and learning
|
||||||
|
|
||||||
Every finished job appends a row (signature such as `<repo>: make <target>`,
|
Every finished job appends a row to `queue-history.jsonl`, from which disk
|
||||||
kind, minutes, disk, RAM and CPU) to `queue-history.jsonl`; the
|
need, slot limits and footprints are learned; read `learning.md` next to this
|
||||||
`kvb-org-organize` watch hook keeps it across reboots.
|
file before changing a seed or a limit.
|
||||||
|
|
||||||
From the last 30 days, once 5 runs exist:
|
|
||||||
|
|
||||||
- per signature, the disk need is p90 disk taken plus 1 GB;
|
|
||||||
- per kind, the slot limit is how many p90 RAM and median CPU shares fit
|
|
||||||
between the median load at the grant and `load_limits`, at least 1;
|
|
||||||
- per job, the footprint is its signature's share, else its kind's.
|
|
||||||
|
|
||||||
Until then the seeds hold: `slot_need_gb[kind]` for disk, 1 slot per kind,
|
|
||||||
the whole budget as footprint.
|
|
||||||
The optional `slot_limits {kind: N}` only caps a limit from above.
|
|
||||||
|
|
||||||
## Inspect and cancel
|
## Inspect and cancel
|
||||||
|
|
||||||
@@ -88,4 +77,5 @@ learned values, `cancel` removes your own waiting job (its hold exits 3).
|
|||||||
`heavy_commands`, image builds, `make install|verify-*`, tests beyond one
|
`heavy_commands`, image builds, `make install|verify-*`, tests beyond one
|
||||||
file and `gate.py run|hold` outside `queue.py hold`.
|
file and `gate.py run|hold` outside `queue.py hold`.
|
||||||
- Without the config, hold exits 2: tell the operator and ask the chair
|
- Without the config, hold exits 2: tell the operator and ask the chair
|
||||||
(`kvb-org-organize`) until it exists.
|
(`kvb-org-organize`) until it exists. A waiting hold rereads it every poll
|
||||||
|
(a broken file keeps the last good one), so a limit change needs no re-hold.
|
||||||
@@ -0,0 +1,16 @@
|
|||||||
|
# Queue history and learning
|
||||||
|
|
||||||
|
Every finished job appends a row (signature such as `<repo>: make <target>`,
|
||||||
|
kind, minutes, disk, RAM and CPU) to `queue-history.jsonl`; the
|
||||||
|
`kvb-org-organize` watch hook keeps it across reboots.
|
||||||
|
|
||||||
|
From the last 30 days, once 5 runs exist:
|
||||||
|
|
||||||
|
- per signature, the disk need is p90 disk taken plus 1 GB;
|
||||||
|
- per kind, the slot limit is how many p90 RAM and median CPU shares fit
|
||||||
|
between the median load at the grant and `load_limits`, at least 1;
|
||||||
|
- per job, the footprint is its signature's share, else its kind's.
|
||||||
|
|
||||||
|
Until then the seeds hold: `slot_need_gb[kind]` for disk, 1 slot per kind,
|
||||||
|
the whole budget as footprint.
|
||||||
|
The optional `slot_limits {kind: N}` only caps a limit from above.
|
||||||
@@ -198,6 +198,20 @@ def config() -> dict:
|
|||||||
return cfg
|
return cfg
|
||||||
|
|
||||||
|
|
||||||
|
def fresh(cfg: dict) -> dict:
|
||||||
|
"""Returns:
|
||||||
|
The config as it stands now, else cfg while the file is missing,
|
||||||
|
unreadable or incomplete, so a half-saved edit never ends a wait.
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
now = watch.load_config()
|
||||||
|
except (OSError, ValueError):
|
||||||
|
return cfg
|
||||||
|
if not isinstance(now, dict) or watch.missing(now, (*BUDGET_KEYS, *watch.MEMORY_KEYS)) is not None:
|
||||||
|
return cfg
|
||||||
|
return now
|
||||||
|
|
||||||
|
|
||||||
def caps(cfg: dict) -> dict[str, int]:
|
def caps(cfg: dict) -> dict[str, int]:
|
||||||
return cfg.get("slot_limits", {})
|
return cfg.get("slot_limits", {})
|
||||||
|
|
||||||
@@ -657,7 +671,7 @@ def wait(cfg: dict, entry_id: str, caught: list[int]) -> tuple[int | None, int]:
|
|||||||
"""Poll until the grant.
|
"""Poll until the grant.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
cfg: the config.
|
cfg: the config at the hold's start; every poll reads it anew.
|
||||||
entry_id: this hold's entry.
|
entry_id: this hold's entry.
|
||||||
caught: signals the caller caught; the first ends the wait.
|
caught: signals the caller caught; the first ends the wait.
|
||||||
|
|
||||||
@@ -670,6 +684,7 @@ def wait(cfg: dict, entry_id: str, caught: list[int]) -> tuple[int | None, int]:
|
|||||||
host = code_of(HOST_SKILLS)
|
host = code_of(HOST_SKILLS)
|
||||||
if host not in (None, CODE):
|
if host not in (None, CODE):
|
||||||
raise Stale(host, caught)
|
raise Stale(host, caught)
|
||||||
|
cfg = fresh(cfg)
|
||||||
cpu, mem = watch.load(cfg)
|
cpu, mem = watch.load(cfg)
|
||||||
free, _ = watch.disk(cfg)
|
free, _ = watch.disk(cfg)
|
||||||
now = time.time()
|
now = time.time()
|
||||||
|
|||||||
@@ -15,6 +15,7 @@ import time
|
|||||||
import unittest
|
import unittest
|
||||||
from collections.abc import Callable
|
from collections.abc import Callable
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
from unittest import mock
|
||||||
|
|
||||||
REPO_ROOT = Path(__file__).resolve().parents[1]
|
REPO_ROOT = Path(__file__).resolve().parents[1]
|
||||||
QUEUE = REPO_ROOT / "skills" / "kvb-org-queue" / "scripts" / "queue.py"
|
QUEUE = REPO_ROOT / "skills" / "kvb-org-queue" / "scripts" / "queue.py"
|
||||||
@@ -757,6 +758,27 @@ class TestQueueCommands(unittest.TestCase):
|
|||||||
self.assertFalse(lock.exists())
|
self.assertFalse(lock.exists())
|
||||||
self.assertEqual(self.entries(), [])
|
self.assertEqual(self.entries(), [])
|
||||||
|
|
||||||
|
def test_a_waiting_hold_is_granted_once_the_config_it_waits_on_changes(self):
|
||||||
|
self.cfg["slot_limits"]["deploy"] = 0
|
||||||
|
self.write_config()
|
||||||
|
hold, _ = self.blocked_hold("true")
|
||||||
|
self.cfg["slot_limits"]["deploy"] = 1
|
||||||
|
self.write_config()
|
||||||
|
self.assertEqual(hold.wait(timeout=60), 0)
|
||||||
|
self.assertEqual(self.entries(), [])
|
||||||
|
|
||||||
|
def test_fresh_takes_the_new_config_and_keeps_the_last_good_one_on_a_broken_or_partial_file(self):
|
||||||
|
queue = _load(QUEUE)
|
||||||
|
good = dict(self.cfg)
|
||||||
|
new = {**self.cfg, "slot_limits": {"test": 2, "deploy": 2}}
|
||||||
|
partial = {key: value for key, value in self.cfg.items() if key != "budget_percent"}
|
||||||
|
for loaded, expected in ((new, new), (None, good), (partial, good), ([], good)):
|
||||||
|
with self.subTest(loaded=loaded), mock.patch.object(queue.watch, "load_config", return_value=loaded):
|
||||||
|
self.assertEqual(queue.fresh(good), expected)
|
||||||
|
for error in (ValueError("half-saved"), OSError("gone")):
|
||||||
|
with self.subTest(error=error), mock.patch.object(queue.watch, "load_config", side_effect=error):
|
||||||
|
self.assertEqual(queue.fresh(good), good)
|
||||||
|
|
||||||
def test_hold_takes_the_need_learned_for_its_command_from_both_history_files(self):
|
def test_hold_takes_the_need_learned_for_its_command_from_both_history_files(self):
|
||||||
self.cfg["slot_limits"]["deploy"] = 0
|
self.cfg["slot_limits"]["deploy"] = 0
|
||||||
self.write_config()
|
self.write_config()
|
||||||
|
|||||||
Reference in new issue
Block a user