feat(kvb-org-organize): hold heavy jobs back on memory pressure with one shipped default config

watch.py admits heavy work only while PSI, swap-out rate and RAM forecast stay under the configured limits; config.default.json is the single default the install writes once (mode 600, no clobber) and every hook, admit and queue command falls back to. Browser runners and package-manager script forms count as heavy, preflight derives its probes from the light and heavy tool lists, and the shipped default carries no prune command, buildx path or host-specific key. Tests run once on unify/skills-20261114 after the merge (operator order 2026-11-14).

🎯 Fix: 85 %
💥 Break: 8 %
💸 Tokens: 99 %
⏱️ Velocity: –
🛡️ Security: 100 %
🏆 Score: 79
🚦 Gate: ⛔ fix 85 % < 90, break 8 % > 5

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Kevin Veen-BirkenbachandClaude Opus 5.5 committed 2026-11-14 21:17:28 +00:00
1 parent a2a03995cb
commit 8e2db852e5
27 files changed
+2795 -207

No files matched your search

+52 -7
View File
@@ -2,12 +2,17 @@
"""Args:
sample: append one row to <state>/samples.jsonl: {t, disks: [{mount,
used, total}], cpu, ram, swap, load, gpus: [{name, util, vram_used,
vram_total} | {name, asleep: true}], queue: {waiting, active}}; t in
epoch seconds, used, total and vram in bytes, cpu, ram, swap and util
in percent, load the 1-minute load average, a GPU named by its PCI
address. cpu is the busy share of the /proc/stat ticks since those
the run before kept in <state>/cpu.json as [t, idle, total], None on
the first run and after a gap of more than 15 minutes. An NVIDIA GPU
vram_total} | {name, asleep: true}], queue: {waiting, active},
psi_some, psi_full, swap_out_mb_min}; t in epoch seconds, used, total
and vram in bytes, cpu, ram, swap and util in percent, load the
1-minute load average, a GPU named by its PCI address. cpu is the
busy share of the /proc/stat ticks since those the run before kept in
<state>/cpu.json as [t, idle, total], None on the first run and after
a gap of more than 15 minutes. psi_some and psi_full are the avg10 of
/proc/pressure/memory, None without PSI; swap_out_mb_min the MB per
minute swapped out (pswpout of /proc/vmstat) since the count the run
before kept in <state>/swap.json as [t, pswpout], None like cpu and
after a counter reset. An NVIDIA GPU
whose PCI runtime_status is not active is recorded asleep, and
nvidia-smi is not run for it; an awake one nvidia-smi gives no
reading for (no driver, a driver mismatch) keeps its name with util
@@ -75,12 +80,15 @@ NVIDIA_QUERY = "pci.bus_id,utilization.gpu,memory.used,memory.total"
UNREAD = {"util": None, "vram_used": None, "vram_total": None}
COUNTS = (("input", "input_tokens"), ("output", "output_tokens"), ("cache_read", "cache_read_input_tokens"), ("cache_write", "cache_creation_input_tokens"))
NAMES = tuple(name for name, _ in COUNTS)
HOST_DIGITS = (("cpu", 1), ("ram", 1), ("swap", 1), ("load", 2))
HOST_DIGITS = (("cpu", 1), ("ram", 1), ("swap", 1), ("load", 2), ("psi_some", 2), ("psi_full", 2), ("swap_out_mb_min", 1))
CLAIMS = ("*/claim.json", "*/*/claim.json")
SAMPLES = "samples.jsonl"
TOKENS = "tokens.jsonl"
OFFSETS = "offsets.json"
TICKS = "cpu.json"
SWAPPED = "swap.json"
PAGE_BYTES = os.sysconf("SC_PAGE_SIZE")
MB = 1_000_000
def read_text(path: Path) -> str:
@@ -263,9 +271,43 @@ def cpu_since(proc: Path, state: Path, t: int) -> float | None:
return cpu_percent(before, ticks)
def psi_avg10(text: str, kind: str) -> float | None:
for line in text.splitlines():
fields = line.split()
if fields[:1] == [kind]:
values = dict(field.split("=", 1) for field in fields[1:] if "=" in field)
return number(values.get("avg10", ""))
return None
def swapped_pages(text: str) -> int | None:
for line in text.splitlines():
fields = line.split()
if len(fields) == 2 and fields[0] == "pswpout" and fields[1].isdigit():
return int(fields[1])
return None
def swap_out_since(proc: Path, state: Path, t: int) -> float | None:
"""Returns:
The MB per minute swapped out since the pswpout count the run before
kept in <state>/swap.json at most BUCKET seconds before t, else None,
also after a counter reset; this run's count replaces it.
"""
pages = swapped_pages(read_text(proc / "vmstat"))
if pages is None:
return None
stored = read_json(state / SWAPPED)
replace(state / SWAPPED, json.dumps([t, pages]))
if not integers(stored, 2) or not t - BUCKET <= stored[0] < t or stored[1] > pages:
return None
return round((pages - stored[1]) * PAGE_BYTES / MB * 60 / (t - stored[0]), 1)
def sample(proc: Path, sys_root: Path, council: Path, run: Callable[[list[str]], str | None], state: Path) -> dict:
t = int(time.time())
ram, swap = memory(parse_meminfo(read_text(proc / "meminfo")))
pressure = read_text(proc / "pressure" / "memory")
return {
"t": t,
"disks": parse_df(run(["df", "-P", "-k", "-l"]) or ""),
@@ -275,6 +317,9 @@ def sample(proc: Path, sys_root: Path, council: Path, run: Callable[[list[str]],
"load": parse_loadavg(read_text(proc / "loadavg")),
"gpus": gpus(sys_root, run),
"queue": queue(council),
"psi_some": psi_avg10(pressure, "some"),
"psi_full": psi_avg10(pressure, "full"),
"swap_out_mb_min": swap_out_since(proc, state, t),
}
+17
View File
@@ -23,6 +23,12 @@ if [[ ! -d "${TARGET}" ]]; then
exit 2
fi
watch="${REPO_ROOT}/skills/kvb-org-organize/scripts/watch.py"
if [[ -f "${watch}" ]] && ! python3 -I "${watch}" preflight; then
warn "skills: add the keys above to the agents council config first; the new hooks and queue.py refuse to run without them"
exit 2
fi
cd "${REPO_ROOT}"
src="${REPO_ROOT}/.agents/skills"
rm -rf "${src}"
@@ -58,6 +64,17 @@ for root in .agents/skills .claude/skills; do
fi
done
config="${AGENTS_COUNCIL_CONFIG:-${HOME}/.claude/agents-council/config.json}"
shipped="${REPO_ROOT}/skills/kvb-org-organize/config.default.json"
if [[ -f "${shipped}" && ! -e "${config}" && ! -L "${config}" ]]; then
[[ -d "$(dirname -- "${config}")" ]] || install -d -m 0700 "$(dirname -- "${config}")"
if (umask 077 && set -o noclobber && cat -- "${shipped}" >"${config}") 2>/dev/null; then
log "skills: wrote the shipped agents council config to ${config}"
else
warn "skills: kept ${config}, which appeared while the install wrote the shipped default"
fi
fi
settings="${TARGET}/.claude/settings.local.json"
if command -v node &>/dev/null; then
log "skills: auto-enabling caveman + ponytail plugins in ${settings}"
+2 -3
View File
@@ -7,9 +7,8 @@ description: >
Where an Ansible run spends its time: its log, a controller profile, a bench.
Report the script's lines with their log and range, never a guess. The
script and the bench are light runs (`watch.py admit --kind test-nostack` of
`kvb-org-organize`); a deploy to profile is heavy and goes through the
`kvb-org-queue` skill.
script, the bench and a deploy to profile run through the `kvb-org-queue`
skill.
## 1. Pick the log
+2 -3
View File
@@ -60,12 +60,11 @@ It also measures wall time for `kvb-check-speed`.
- **Command**: the targeted tests of the change or a CLI call of the
changed code, runnable on a clean checkout: the base worktree holds no
untracked build output, virtualenv or dependency directory.
- **Heavy commands** (Docker, deploys, whole suites) run only inside
- **Every run** goes inside
`${CLAUDE_SKILL_DIR}/../kvb-org-queue/scripts/queue.py hold --kind <kind>
--task <register task> -- python3 <measure.py> ...`, in the background,
by the `kvb-org-queue` skill; a cancelled wait there also exits 3, but
prints no 💸 line. A light one self-admits with
`watch.py admit --kind test-nostack` of `kvb-org-organize`.
prints no 💸 line.
- Report the run you made; never rerun to pick the best result.
## Exit codes
+3 -4
View File
@@ -32,10 +32,9 @@ commit starts from zero.
`missing hotfix` reason means back to step 1.
4. **Admit**: every gate target is heavy and runs only inside
`${CLAUDE_SKILL_DIR}/../kvb-org-queue/scripts/queue.py hold`.
Only a single test module
self-admits (`${CLAUDE_SKILL_DIR}/../kvb-org-organize/scripts/watch.py
admit --kind test-nostack`), and it never turns a gate target green. A
hotfix gate adds `--hotfix <slug>` and takes the next free gate slot.
Only a single test module self-admits (`kvb-org-organize` Capacity), and
it never turns a gate target green. A hotfix gate adds `--hotfix <slug>`
and takes the next free gate slot.
5. **Run** the missing and red targets, tracked tree clean, in the
background; give the operator `tail -f` on the log:
+2 -5
View File
@@ -15,11 +15,8 @@ skill's Rules apply here too.
first error.
3. **Fix at the root**: apply the real fix for each proven root cause in the
sidetree, with a test and a negative control as the `kvb-org-swarm` skill
demands. Light runs (no Docker, about a minute: one test file or
module, ruff) self-admit with
`${CLAUDE_SKILL_DIR}/../kvb-org-organize/scripts/watch.py admit --kind
test-nostack`; `make test`, gates, deploys and anything with Docker run in
the background through
demands. Light runs self-admit (`kvb-org-organize` Capacity); `make test`,
gates, deploys and anything with Docker run in the background through
`${CLAUDE_SKILL_DIR}/../kvb-org-queue/scripts/queue.py hold --kind <kind>
--task <register task> -- <command>`, which waits for the rule-based
grant and releases on exit. The fix enters the main line only
+2 -3
View File
@@ -26,9 +26,8 @@ cover the change set run now and give the green baseline; a suite already red
is reported, not refactored around. No coverage: read `net.md` next to this
file before editing.
Light runs (one test file or module, ruff) start once `watch.py
admit --kind test-nostack` of the `kvb-org-organize` skill prints `grant`; no
chair request.
Light runs start once `watch.py admit` of the `kvb-org-organize` skill prints
`grant`; no chair request.
## Step 3: refactor each file
+2 -5
View File
@@ -38,11 +38,8 @@ any listed name. Never silently truncate.
## Step 3: run
Before each run, admit it under the `kvb-org-organize` Capacity rules: a
light run (no Docker, about a minute: one test file or module, ruff)
self-admits with
`${CLAUDE_SKILL_DIR}/../kvb-org-organize/scripts/watch.py admit --kind
test-nostack`; `make test`, quality gates and anything with Docker run in the
background through
light run self-admits; `make test`, quality gates and anything with Docker
run in the background through
`${CLAUDE_SKILL_DIR}/../kvb-org-queue/scripts/queue.py hold --kind <kind>
--task <register task> -- make <target>`, which waits for the rule-based
grant and releases on exit.
+2 -3
View File
@@ -49,9 +49,8 @@ single-select `Write it` / `Revise it` / `Discard` with the drafted diff.
owner through the `kvb-org-coordinate` skill.
- Every carried test fails without the code it guards: show the negative
control once.
- Run the carried tests as a light run (no Docker, about a minute) after
`${CLAUDE_SKILL_DIR}/../kvb-org-organize/scripts/watch.py admit --kind
test-nostack`; `make test`, gates and anything with Docker only through
- Run the carried tests as a light run (`kvb-org-organize` Capacity);
`make test`, gates and anything with Docker only through
`queue.py hold` (the `kvb-org-queue` skill). Leave the result
uncommitted for the `kvb-git-commit-next` skill.
- `transcend.py mark` every offered candidate with its verdict.
+2 -4
View File
@@ -71,10 +71,8 @@ order:
Show the full draft, then `AskUserQuestion`: `Write it` / `Revise it` /
`Discard`. Revise loops back to the step the correction touches. Write only
on `Write it`; lint the file with the project's markdown linter, a light run
that self-admits with
`${CLAUDE_SKILL_DIR}/../kvb-org-organize/scripts/watch.py admit --kind
test-nostack`; a lint through `make` or Docker runs through `queue.py hold`
(the `kvb-org-queue` skill).
(`kvb-org-organize` Capacity); a lint through `make` or Docker runs through
`queue.py hold` (the `kvb-org-queue` skill).
## Step 6: hand over
+2 -2
View File
@@ -35,8 +35,8 @@ request, so a red one blocks nothing else.
(`search_graph`, `trace_path`, `get_architecture`, `detect_changes`; the
`kvb-agent-optimize` skill sets it up), grep and read after. Parallel
work, gates, deploys and anything with Docker run through `queue.py hold`
(`kvb-org-queue`); light runs (no Docker, about a minute) self-admit with
`watch.py admit --kind test-nostack`. With the first change, swap the label: `gh issue
(`kvb-org-queue`); light runs self-admit (`kvb-org-organize` Capacity).
With the first change, swap the label: `gh issue
edit <n> --remove-label "in analysis" --add-label "in progress"`.
5. **Open the pull request** with the `kvb-git-pr-create` skill, closing
exactly its issue; one for a failing default branch is posted with
+7 -2
View File
@@ -37,8 +37,13 @@ ends or goes stale, or after a restart.
## Capacity
- Light runs (no Docker, about a minute: one test file or module, ruff)
self-admit: `watch.py admit --kind test-nostack`, start on `grant`.
- Light runs self-admit: `watch.py admit --kind test-nostack -- <command>`,
start on `grant`. Light is only ruff, shellcheck, markdownlint, lizard, jq,
py_compile, cargo fmt, `env CARGO_BUILD_JOBS=4 cargo build|check|clippy|test`,
or one test file or module outside `e2e/` that reaches no Playwright;
anything else: `queue.py hold --kind test`.
- Memory stop (also PSI or swap-out) blocks every admit and grant; running
jobs go on.
- Other test runs, `make test`, `make test-lint`, quality gates, deploys,
iteration loops, swarm, image builds, cloud, shared-data cleanup, anything
with Docker or over a few minutes: run through `queue.py hold`
+13
View File
@@ -63,3 +63,16 @@ and stays silent below the thresholds.
`watch.py status` shows the stage computed from the live free space and the
stored samples, and the age of the stored measurement.
An unusable config (broken JSON, wrong types, a regex refusing a light
probe such as `grep` or freeing a heavy one such as `make swarm-deploy`, a
dangling link, a directory) makes the hooks, admit, status and the queue
warn and use `config.default.json`, which the install writes when no config
exists; without that file every command, teardown included, is refused.
The probes are samples, so a pattern hitting only other commands passes. A
package script is read from the nearest package.json, a named workspace
only from a folder of that name. The light-run check misses
`sys.path.insert` in a conftest, editable installs without pytest
`pythonpath`, browser libraries outside its list and generic third-party
fixtures such as pytest-selenium's `driver`, browsers started by a
subprocess, and `git bisect run` or a suite script behind `bash`.
@@ -0,0 +1,37 @@
{
"disk": "/",
"thresholds_gb": {"warn": 40, "stop": 15, "critical": 12},
"projection_minutes": 30,
"interval_seconds": 120,
"cache_interval_seconds": 1800,
"heavy_commands": [
"\\bmake\\s+(-C\\s+\\S+\\s+)?(compose-deploy|compose-up|compose-playwright|roundtrip|quality|quality-high|test|test-integration|test-unit\\S*|swarm-\\S+|act-workflow|act-runner-image|build\\S*|worktree-up|store-\\S+)(\\s|$)",
"\\bdocker\\s+(build|pull|load)\\b",
"\\bdocker\\s+buildx\\s+build\\b",
"\\brun\\.sh\\s+local\\b"
],
"never_block": [
"\\bmake\\s+(-C\\s+\\S+\\s+)?(swarm-down|swarm-clean|swarm-clean-stale-nfs|swarm-diagnostic|compose-down|compose-entity-purge|worktree-down|store-host-down)(\\s|$)",
"^\\s*git\\s+commit\\b.*\\s--no-verify\\b",
"\\bmake\\s+swarm-exec(\\s|$)"
],
"queue_enforce": true,
"cleanup": [],
"worktree_cleanup": [],
"worktrees_command": "true",
"worktree_marker": ".git",
"busy_processes": ["make"],
"cache_dirs": ["~/.cache/pip", "~/.npm", "~/.cache/pre-commit", "~/.cache/act", "~/.cache/ms-playwright"],
"cache_budget_percent": 3,
"slot_need_gb": {"quality-high": 15, "test": 15, "test-nostack": 0, "iteration": 20, "swarm": 30, "gui-dind": 65, "operator": 10},
"notify": ["notify-send", "-u", "critical", "Agents council"],
"load_limits": {"cpu_percent": 90, "mem_percent": 85, "sample_seconds": 5, "kinds": ["test", "test-nostack", "quality-high"]},
"memory_limits": {
"warn": {"available": 15, "swap_free": 40, "psi_some": 10, "psi_full": 5, "swap_out_mb_min": 64, "container_swap_gb": 4},
"stop": {"available": 8, "swap_free": 10, "psi_full": 5, "swap_out_mb_min": 256, "container_swap_gb": 16}
},
"budget_percent": {"ram": 75, "cpu": 100},
"light_kinds": ["test-nostack"],
"exclusive_kinds": ["quality-high", "test"],
"gate_excluded_targets": ["test-oracle"]
}
File diff suppressed because it is too large. Load diff
+4 -5
View File
@@ -27,9 +27,8 @@ turn: hold runs the command on its grant and exits with its code.
- In a worktree the command stays `gate.py run|hold` (`kvb-check-gate`):
`queue.py hold ... -- python3 <gate.py> run ...`.
One test file or module, or ruff, without Docker self-admits with
`watch.py admit --kind test-nostack` (`kvb-org-organize`); any other test
run goes through hold with kind `test`.
Light runs (`kvb-org-organize` Capacity) self-admit; anything else goes
through hold with kind `test`, and a light kind refuses it.
## Order and grant
@@ -77,8 +76,8 @@ learned values, `cancel` removes your own waiting job.
one down, and `kvb-git-hotfix` may void a gate. A `kvb-org-queue-cleanup`
round deletes only entries whose lock no living process holds.
- With `queue_enforce: true`, the `block` hook (PreToolUse on Bash) refuses
`heavy_commands`, image builds, `make install|verify-*`, tests beyond one
file and `gate.py run|hold` outside `queue.py hold`.
`heavy_commands`, image builds, `make install|test*|quality*|verify-*`,
tests beyond one unit file and `gate.py run|hold` outside `queue.py hold`.
- Without the config, hold exits 2: tell the operator and ask the chair
(`kvb-org-organize`) until it exists. A waiting hold rereads it every poll
(a broken file keeps the last good one), so a limit change needs no re-hold.
+98 -27
View File
@@ -38,7 +38,9 @@ Args:
SIBLINGS); a poll that finds other code installed in
~/.agents/skills leaves the queue ungranted and restarts the same hold
with the installed queue.py.
--kind: a key of slot_need_gb in the config.
--kind: a key of slot_need_gb in the config; a light one
(light_kinds or watch.LIGHT_KIND) refuses a command watch.light()
does not recognise, naming watch.queue_hold().
--task: an open task of the kvb-org-rasci register; its rank orders
the queue, and the operator's agent order (`rasci.py agents
order`) of the holding session breaks ties. While one of its
@@ -87,7 +89,8 @@ disk of running and granted jobs, never a reservation; queue_enforce (bool), opt
only. The watch.py keys disk, thresholds_gb (stop is the floor every grant
leaves free), slot_need_gb (the disk seed per kind), load_limits (every grant
also needs CPU busy below cpu_percent and RAM used plus the job's learned RAM
share below mem_percent; both bound the learned slot limits), memory_limits (no
share, the live admit records (admit()) and the young jobs' ramp
(ramping_ram()) below mem_percent; both bound the learned slot limits), memory_limits (no
grant at memory stage stop), heavy_commands and never_block are used as
watch.py documents them. A signature is `<repository>: <command signature>`
(aliases()).
@@ -165,6 +168,10 @@ XDIST_LONG = "--numprocesses="
ADDOPTS = "PYTEST_ADDOPTS="
FROZEN = "footprint"
ALIASES = "aliases"
ADMIT = "admit"
ADMITS = "admits"
ADMIT_SECONDS = 120
GRANTED_MEM = "granted_mem"
@functools.cache
@@ -206,7 +213,7 @@ def fail(message: str) -> NoReturn:
def config() -> dict:
cfg = watch.load_config()
cfg, _ = watch.announced_config()
if cfg is None:
fail(f"no config at {watch.CONFIG}; the queue reads the kvb-org-organize watch config")
problem = watch.missing(cfg, (*BUDGET_KEYS, *watch.MEMORY_KEYS))
@@ -218,13 +225,14 @@ def config() -> dict:
def fresh(cfg: dict) -> dict:
"""Returns:
The config as it stands now, else cfg while the file is missing,
unreadable or incomplete, so a half-saved edit never ends a wait.
unusable (watch.checked_config()) or incomplete, so a half-saved
edit never ends a wait.
"""
try:
now = watch.load_config()
except (OSError, ValueError):
now = watch.checked_config(watch.load_config())
except watch.UNUSABLE:
return cfg
if not isinstance(now, dict) or watch.missing(now, (*BUDGET_KEYS, *watch.MEMORY_KEYS)) is not None:
if watch.missing(now, (*BUDGET_KEYS, *watch.MEMORY_KEYS)) is not None:
return cfg
return now
@@ -690,11 +698,11 @@ def footprint(entry: dict, signatures: dict[tuple[str, str], dict], kinds: dict[
def held_back(cfg: dict, entries: list[dict], now: float, signatures: dict[str, dict], kinds: dict[str, dict],
cpu: float, mem: float) -> dict[str, str]:
cpu: float, mem: float, admitted: float) -> dict[str, str]:
loaded = {}
for entry in entries:
if entry["status"] == "waiting":
pending = pending_ram(entries, entry["id"], now, signatures, kinds)
pending = pending_ram(entries, entry["id"], now, signatures, kinds, mem, admitted)
if (reason := load_reason(cfg["load_limits"], cpu, mem, pending)) is not None:
loaded[entry["id"]] = reason
return loaded
@@ -742,18 +750,36 @@ def ram_share(entry: dict, signatures: dict[str, dict], kinds: dict[str, dict])
return history.seed_ram(kinds) if share is None else share["ram"]
def pending_ram(entries: list[dict], entry_id: str, now: float, signatures: dict[str, dict],
kinds: dict[str, dict]) -> float:
def ramping_ram(entries: list[dict], now: float, signatures: dict[str, dict], kinds: dict[str, dict],
mem: float) -> float:
"""Returns:
The RAM share of entry_id plus the whole share of every active job
granted less than RAMP_SECONDS ago (or without a grant time): what
may still come once they ramp up. An older job counts as ramped, its
RAM inside the RAM used; one that peaks later is bounded by the
memory stage only.
What the active jobs granted less than RAMP_SECONDS ago (or without a
grant time) may still add: the shares of those whose grant recorded
the RAM used (GRANTED_MEM) minus what RAM used gained since the
earliest of these grants, between 0 and those shares (a RAM used
that fell since counts as no gain), plus the whole share of the
others. An older job counts as ramped, its RAM inside the RAM used;
one that peaks later is bounded by the memory stage only.
"""
young = [e for e in entries if e["status"] == "active" and now - e.get("granted", now) < RAMP_SECONDS]
dated = [e for e in young if GRANTED_MEM in e]
gained = max(0.0, mem - min((e[GRANTED_MEM] for e in dated), default=mem))
ramp = max(0.0, sum(ram_share(e, signatures, kinds) for e in dated) - gained)
return ramp + sum(ram_share(e, signatures, kinds) for e in young if GRANTED_MEM not in e)
def pending_ram(entries: list[dict], entry_id: str, now: float, signatures: dict[str, dict],
kinds: dict[str, dict], mem: float, admitted: float) -> float:
"""Returns:
The RAM share of entry_id plus admitted (admitted_ram()) plus
ramping_ram() at RAM used mem.
"""
mine = [e for e in entries if e["id"] == entry_id]
young = [e for e in entries if e["status"] == "active" and now - e.get("granted", now) < RAMP_SECONDS]
return ram_share(mine[0], signatures, kinds) + sum(ram_share(e, signatures, kinds) for e in young)
return ram_share(mine[0], signatures, kinds) + admitted + ramping_ram(entries, now, signatures, kinds, mem)
def admitted_ram(data: dict, now: float) -> float:
return sum(record["ram"] for record in data.get(ADMITS, []) if record["expires"] > now)
def workdir() -> str:
@@ -822,13 +848,13 @@ def wait(cfg: dict, entry_id: str, caught: list[int]) -> tuple[int | None, int]:
print(f"queue: {entry_id} was cancelled", file=sys.stderr)
return CANCELLED, 0
rejoin(data, mine[0], now)
loaded = held_back(cfg, data["entries"], now, signatures, kinds, cpu, mem)
loaded = held_back(cfg, data["entries"], now, signatures, kinds, cpu, mem, admitted_ram(data, now))
verdicts = decide(cfg, data["entries"], free, now, signatures, kinds, loaded)
reason = verdicts[entry_id] or loaded.get(entry_id)
mine[0]["reason"] = reason
if reason is None:
concurrent = sum(e["status"] == "active" for e in data["entries"])
mine[0].update(status="active", granted=now, **{FROZEN: footprint(mine[0], signatures, kinds)})
mine[0].update(status="active", granted=now, **{GRANTED_MEM: mem, FROZEN: footprint(mine[0], signatures, kinds)})
return None, concurrent
position = list(verdicts).index(entry_id) + 1
if (position, reason) != shown:
@@ -912,15 +938,27 @@ def keep_receipt(gates: ModuleType | None, args: argparse.Namespace, here: str,
print(f"queue: receipt {args.receipt} for tree {before}", file=sys.stderr)
def cmd_hold(args: argparse.Namespace) -> int:
cfg = config()
def check_hold(args: argparse.Namespace, cfg: dict) -> None:
"""Args:
args: the hold arguments; fail() exits on an unknown kind, a command
that is no light run under a light kind, a task outside the
register or a missing hotfix row.
cfg: the watch config.
"""
if args.kind not in cfg["slot_need_gb"]:
fail(f"kind {args.kind!r} is not in slot_need_gb of {watch.CONFIG}")
shown = shlex.join(args.command)
if args.kind in {watch.LIGHT_KIND, *cfg.get("light_kinds", [])} and not watch.light(shown, os.getcwd()):
fail(f"{shown} is no light run and the light kind {args.kind} skips every reservation; run `{watch.queue_hold(shown)}`")
rasci = sibling("rasci")
if args.task not in rasci.merged(False):
fail(f"{args.task} is no open task of the kvb-org-rasci register")
if args.hotfix is not None and HOTFIX + args.hotfix not in rasci.load():
fail(f"the register has no row {HOTFIX}{args.hotfix}")
def cmd_hold(args: argparse.Namespace) -> int:
cfg = config()
check_hold(args, cfg)
here = workdir()
gates, out = gated(args, here)
names = aliases(here, args.command, os.getcwd())
@@ -1091,6 +1129,38 @@ def held_commands(command: str) -> list[list[str]]:
]
def enforced(cfg: dict) -> dict:
return {**cfg, "heavy_commands": [*cfg["heavy_commands"], GATE.pattern]}
def admit(cfg: dict, kind: str, argv: list[str], cpu: float, mem: float, calm: bool) -> str | None:
"""Args:
cfg: the config.
kind: the kind of the light job that asks to start outside the queue.
argv: its command, run in the current directory.
cpu: CPU busy now, in percent.
mem: RAM used now, in percent.
calm: whether the memory stage is ok; the live active jobs' ramp
(ramping_ram()) counts only when it is not.
Returns:
load_reason() over the job's RAM share plus the live admit records
(admitted_ram()) and the ramp; under the store lock, a grant records
the share for ADMIT_SECONDS and expired records leave.
"""
now = time.time()
signatures, kinds = learned(cfg, now)
mine = {"id": ADMIT, "status": "waiting", "kind": kind, ALIASES: list(aliases(workdir(), argv, os.getcwd()))}
share = ram_share(mine, signatures, kinds)
with store() as data:
active = [e for e in data["entries"] if e["status"] == "active" and living(e)]
ramp = 0.0 if calm else ramping_ram(active, now, signatures, kinds, mem)
reason = load_reason(cfg["load_limits"], cpu, mem, share + admitted_ram(data, now) + ramp)
granted = [{"ram": share, "expires": now + ADMIT_SECONDS}] if reason is None else []
data[ADMITS] = [record for record in data.get(ADMITS, []) if record["expires"] > now] + granted
return reason
def tails(argv: list[str]) -> list[str]:
return [shlex.join(argv[at:]) for at, word in enumerate(argv) if not word.startswith("-")]
@@ -1121,11 +1191,10 @@ def cmd_block() -> int:
"hookEventName": "PreToolUse", "permissionDecision": "ask", "permissionDecisionReason": reason,
}}))
return 0
cfg = watch.load_config()
cfg, _ = watch.announced_config()
if cfg is None or cfg.get("queue_enforce") is not True:
return 0
gated = {**cfg, "heavy_commands": [*cfg["heavy_commands"], GATE.pattern]}
if watch.heavy(raw, gated, (THROUGH_QUEUE.pattern, *cfg["never_block"]), watch.payload_cwd(payload)):
if watch.heavy(raw, enforced(cfg), (THROUGH_QUEUE.pattern, *cfg["never_block"]), watch.payload_cwd(payload)):
print(
f"kvb-org-queue: heavy jobs run through the queue: start `python3 {Path(__file__).resolve()} hold "
"--kind <kind> --task <register task> -- <command>` in the background; it runs the command on its grant.",
@@ -1164,8 +1233,10 @@ def main() -> int:
if args.cmd == "block":
try:
return cmd_block()
except (OSError, ValueError, KeyError, TypeError, AttributeError, re.error):
return 0
except Exception as error: # noqa: BLE001
print(f"kvb-org-queue: the block hook failed ({error!r}), so the command counts as heavy and is refused.",
file=sys.stderr)
return 2
commands = {"hold": cmd_hold, "list": cmd_list, "history": cmd_history, "cancel": cmd_cancel}
try:
return commands[args.cmd](args)
+2 -3
View File
@@ -31,9 +31,8 @@ picks do; every other session keeps its own.
## Night rules for the active session
- **Grants**: request none; admit your own runs with `watch.py admit --kind
<slot>` (`${CLAUDE_SKILL_DIR}/../kvb-org-organize/scripts/watch.py`), whose
disk, CPU and RAM limits still apply.
- **Grants**: request none; light runs self-admit and heavy ones queue
(`kvb-org-organize` Capacity), whose limits still apply.
- **Offices**: the suspended agents' claims stay theirs. Take only free work;
a piece a suspended agent holds waits for the resume or the operator.
- **Messages**: no wakes and no peer questions; the Peer protocol's wait is
+3 -4
View File
@@ -30,10 +30,9 @@ sidetree or leaves as a proposal.
- code explored through the codebase-memory-mcp graph first
(`search_graph`, `trace_path`, `get_architecture`, `detect_changes`; the
`kvb-agent-optimize` skill sets it up), grep and read after;
- light runs (no Docker, about a minute) self-admit with
`${CLAUDE_SKILL_DIR}/../kvb-org-organize/scripts/watch.py admit --kind
test-nostack`; `make test`, gates, deploys and anything with Docker only
through `queue.py hold` (`kvb-org-queue`);
- light runs self-admit (`kvb-org-organize` Capacity); `make test`, gates,
deploys and anything with Docker only through `queue.py hold`
(`kvb-org-queue`);
- status that asks nothing goes to
`${CLAUDE_SKILL_DIR}/../kvb-org-radio/scripts/radio.py post "<text>"`,
never into a wake.
+3 -3
View File
@@ -58,9 +58,9 @@ and the prose diff.
- Write the script, then replace the compiled prose with its call
(`${CLAUDE_SKILL_DIR}/scripts/<verb>.py ...`) and one line on reading its
output.
- Every script gets a unit test with a negative control shown once, run once
`watch.py admit --kind test-nostack` (the `kvb-org-organize` skill) prints
`grant`; `make test` runs through `queue.py hold` (`kvb-org-queue`).
- Every script gets a unit test with a negative control shown once, run as a
light run (`kvb-org-organize` Capacity); `make test` runs through
`queue.py hold` (`kvb-org-queue`).
- In another repository follow its agent rules (claims, worktree, branch); a
file another line owns goes to that line through the `kvb-org-coordinate`
skill.
+2 -3
View File
@@ -52,9 +52,8 @@ or diff.
- Read every script whole before carrying it; clean it the way the
`kvb-code-clean` skill does.
- A carried script gets a unit test in the repository's `tests/` with a
negative control, run once `watch.py admit --kind test-nostack` (the
`kvb-org-organize` skill) prints `grant`; `make test` runs through
`queue.py hold` (`kvb-org-queue`).
negative control, run as a light run (`kvb-org-organize` Capacity);
`make test` runs through `queue.py hold` (`kvb-org-queue`).
- `transcend.py mark` every offered candidate with its verdict.
- Leave the result uncommitted; committing is the `kvb-git-commit-next`
skill's job.
+3 -1
View File
@@ -11,7 +11,9 @@ from pathlib import Path
REPO_ROOT = Path(__file__).resolve().parents[1]
GATES = REPO_ROOT / "skills" / "kvb-check-gate" / "scripts" / "gates.py"
QUEUE = REPO_ROOT / "skills" / "kvb-org-queue" / "scripts" / "queue.py"
OPEN_MEMORY = {"available": -1, "swap_free": -1, "psi_full": 1000, "container_swap_gb": 1e9}
OPEN_MEMORY = {
"available": -1, "swap_free": -1, "psi_some": 1000, "psi_full": 1000, "swap_out_mb_min": 1e9, "container_swap_gb": 1e9,
}
class Scratch(unittest.TestCase):
+66 -4
View File
@@ -2,6 +2,7 @@
from __future__ import annotations
import json
import os
import shutil
import subprocess
@@ -10,15 +11,26 @@ import unittest
from pathlib import Path
INSTALL = Path(__file__).resolve().parents[1] / "scripts" / "install.sh"
ORGANIZE = Path(__file__).resolve().parents[1] / "skills" / "kvb-org-organize"
NPX = "#!/bin/sh\nmkdir -p .agents/skills/locked\necho locked > .agents/skills/locked/SKILL.md\n"
BYTECODE = ("__pycache__/radio.cpython-312.pyc", "json.pyc", "legacy.pyo")
LIVE_MEMORY = {
"warn": {"available": 15, "swap_free": 40, "psi_full": 5, "container_swap_gb": 4},
"stop": {"available": 8, "swap_free": 10, "psi_full": 10, "container_swap_gb": 16},
}
SHIPPED = ORGANIZE / "config.default.json"
DEFAULT = json.loads(SHIPPED.read_text(encoding="utf-8"))
ROW_MEMORY = DEFAULT["memory_limits"]
class TestInstall(unittest.TestCase):
def install(self, root: Path, target: Path, npx: str = NPX, planted: tuple[str, ...] = ()) -> None:
def install(self, root: Path, target: Path, npx: str = NPX, planted: tuple[str, ...] = (), config: Path | None = None,
code: int = 0, stubs_extra: tuple[tuple[str, str], ...] = ()) -> subprocess.CompletedProcess:
repo, stubs = root / "repo", root / "bin"
(repo / "scripts").mkdir(parents=True)
shutil.copy(INSTALL, repo / "scripts" / "install.sh")
if config is not None:
shutil.copytree(ORGANIZE, repo / "skills" / ORGANIZE.name, ignore=shutil.ignore_patterns("__pycache__"))
(repo / "scripts" / "enable-plugins.js").write_text("", encoding="utf-8")
(repo / "skills" / "own").mkdir(parents=True)
(repo / "skills" / "own" / "SKILL.md").write_text("own", encoding="utf-8")
@@ -26,11 +38,61 @@ class TestInstall(unittest.TestCase):
(repo / name).parent.mkdir(parents=True, exist_ok=True)
(repo / name).write_bytes(b"planted")
stubs.mkdir()
(stubs / "npx").write_text(npx, encoding="utf-8")
(stubs / "npx").chmod(0o755)
for name, text in (("npx", npx), *stubs_extra):
(stubs / name).write_text(text, encoding="utf-8")
(stubs / name).chmod(0o755)
env = {**os.environ, "PATH": f"{stubs}:/usr/bin:/bin", "TARGET": str(target), "HOME": str(root / "home")}
if config is not None:
env |= {"AGENTS_COUNCIL_CONFIG": str(config), "AGENTS_COUNCIL_ROOT": str(root / "council")}
done = subprocess.run(["bash", str(repo / "scripts" / "install.sh")], env=env, capture_output=True, text=True, check=False)
self.assertEqual(done.returncode, 0, done.stderr)
self.assertEqual(done.returncode, code, done.stderr)
return done
def test_the_install_stops_before_any_copy_while_the_live_config_lacks_a_memory_bound_the_new_code_needs(self):
for limits, code in ((LIVE_MEMORY, 2), (ROW_MEMORY, 0)):
with self.subTest(code=code), tempfile.TemporaryDirectory() as tmp:
root = Path(tmp)
target = root / "target"
target.mkdir()
config = root / "config.json"
config.write_text(json.dumps({**DEFAULT, "memory_limits": limits}), encoding="utf-8")
before = config.stat()
done = self.install(root, target, config=config, code=code)
after = config.stat()
self.assertEqual((after.st_ino, after.st_mtime_ns, after.st_mode), (before.st_ino, before.st_mtime_ns, before.st_mode))
installed = sorted(path.name for path in target.iterdir())
if code:
self.assertIn(
"lacks memory_limits.warn.psi_some, memory_limits.warn.swap_out_mb_min, memory_limits.stop.swap_out_mb_min;",
done.stderr,
)
self.assertEqual(installed, [])
self.assertFalse((root / "repo" / ".agents").exists())
else:
self.assertEqual(installed, [".agents", ".claude"])
self.assertEqual(json.loads(config.read_text(encoding="utf-8")), {**DEFAULT, "memory_limits": limits})
def test_an_install_without_a_config_writes_the_shipped_default_as_the_initial_one(self):
with tempfile.TemporaryDirectory() as tmp:
root = Path(tmp)
target = root / "target"
target.mkdir()
config = root / "council-config" / "config.json"
self.install(root, target, config=config)
self.assertEqual(config.read_bytes(), SHIPPED.read_bytes())
self.assertEqual(config.stat().st_mode & 0o777, 0o600)
self.assertEqual(config.parent.stat().st_mode & 0o777, 0o700)
def test_a_config_written_while_the_install_creates_its_folder_survives(self):
with tempfile.TemporaryDirectory() as tmp:
root = Path(tmp)
target = root / "target"
target.mkdir()
config = root / "council-config" / "config.json"
racer = f"#!/bin/sh\nmkdir -p {config.parent}\necho '{{\"mine\": true}}' > {config}\nexec /usr/bin/install \"$@\"\n"
done = self.install(root, target, config=config, stubs_extra=(("install", racer),))
self.assertEqual(json.loads(config.read_text(encoding="utf-8")), {"mine": True})
self.assertIn(str(config), done.stderr)
def test_reinstall_empties_the_skill_directories_in_place(self):
with tempfile.TemporaryDirectory() as tmp:
+4 -1
View File
@@ -529,7 +529,10 @@ class TestOperator(unittest.TestCase):
def _queue_config(self, slot_need_gb: dict) -> None:
config = self.base / "config.json"
open_memory = {"available": -1, "swap_free": -1, "psi_full": 1000, "container_swap_gb": 1e9}
open_memory = {
"available": -1, "swap_free": -1, "psi_some": 1000, "psi_full": 1000, "swap_out_mb_min": 1e9,
"container_swap_gb": 1e9,
}
config.write_text(json.dumps({
"disk": str(self.base), "thresholds_gb": {"warn": 0, "stop": 0, "critical": 0}, "projection_minutes": 30,
"slot_need_gb": slot_need_gb, "heavy_commands": [r"\btouch heavy\b"], "never_block": [],
+121 -6
View File
@@ -23,7 +23,9 @@ HISTORY = REPO_ROOT / "skills" / "kvb-org-queue" / "scripts" / "history.py"
GATES = REPO_ROOT / "skills" / "kvb-check-gate" / "scripts" / "gates.py"
NOW = 1_800_000_000.0
HOUR = 3600
OPEN_MEMORY = {"available": -1, "swap_free": -1, "psi_full": 1000, "container_swap_gb": 1e9}
OPEN_MEMORY = {
"available": -1, "swap_free": -1, "psi_some": 1000, "psi_full": 1000, "swap_out_mb_min": 1e9, "container_swap_gb": 1e9,
}
BASE = {
"free_gb": 100.0,
"floor": 10.0,
@@ -141,11 +143,38 @@ class TestEvaluate(unittest.TestCase):
young = {**job(1, kind="a", status="active", granted=NOW - 60), **cmd, "cmd": "a"}
overshooting = {**job(5, kind="c", status="active", granted=NOW - 120), **cmd, "cmd": "c"}
old = {**job(4, kind="c", status="active", granted=NOW - self.queue.RAMP_SECONDS), **cmd, "cmd": "c"}
self.assertEqual(self.queue.pending_ram([young, overshooting, old, fresh], "j2", NOW, signatures, kinds), 30 + 10 + 25)
pending = self.queue.pending_ram
self.assertEqual(pending([young, overshooting, old, fresh], "j2", NOW, signatures, kinds, 50, 0), 30 + 10 + 25)
undated = {k: v for k, v in old.items() if k != "granted"}
self.assertEqual(self.queue.pending_ram([undated, fresh], "j2", NOW, signatures, kinds), 10 + 25)
self.assertEqual(pending([undated, fresh], "j2", NOW, signatures, kinds, 50, 0), 10 + 25)
unknown = {**job(3, kind="new"), **cmd, "cmd": "new"}
self.assertEqual(self.queue.pending_ram([unknown], "j3", NOW, signatures, kinds), 30)
self.assertEqual(pending([unknown], "j3", NOW, signatures, kinds, 50, 0), 30)
self.assertEqual(pending([fresh], "j2", NOW, signatures, kinds, 50, 12), 25 + 12)
def test_a_young_job_counts_only_what_it_has_not_added_since_its_grant(self):
history = _load(HISTORY)
rows = [history_row(n, kind="b", signature="w: b", mem_before=20, mem_max=45) for n in range(5)]
signatures, kinds = history.learn(rows, NOW, {"load_limits": {"mem_percent": 90, "cpu_percent": 90}})
fresh = {**job(2, kind="b"), "workdir": "/w", "cmd": "b"}
def gate(seq: int, ram: float, granted_mem: float) -> dict:
return {**job(seq, kind="g", status="active", granted=NOW - 60), "workdir": "/w", "cmd": "g",
"footprint": {"ram": ram, "cpu": 1}, "granted_mem": granted_mem}
pending = self.queue.pending_ram
self.assertEqual(pending([gate(1, 40, 10), fresh], "j2", NOW, signatures, kinds, 10, 0), 25 + 40)
self.assertEqual(pending([gate(1, 40, 10), fresh], "j2", NOW, signatures, kinds, 45, 0), 25 + 5)
self.assertEqual(pending([gate(1, 40, 10), fresh], "j2", NOW, signatures, kinds, 60, 0), 25)
self.assertEqual(pending([gate(1, 40, 10), gate(3, 20, 30), fresh], "j2", NOW, signatures, kinds, 45, 0), 25 + 25)
undated = {k: v for k, v in gate(1, 40, 10).items() if k != "granted_mem"}
self.assertEqual(pending([undated, fresh], "j2", NOW, signatures, kinds, 60, 0), 25 + 40)
ramp = self.queue.ramping_ram
for mem in (70, 40, 20):
with self.subTest(mem=mem):
self.assertEqual(ramp([gate(1, 40, 70)], NOW, signatures, kinds, mem), 40)
self.assertEqual(ramp([gate(1, 40, 70), {**undated, "id": "j3"}], NOW, signatures, kinds, 20), 80)
self.assertEqual(self.queue.admitted_ram({"admits": [{"ram": 7, "expires": NOW + 1}, {"ram": 9, "expires": NOW}]}, NOW), 7)
self.assertEqual(self.queue.admitted_ram({"entries": []}, NOW), 0)
def test_a_signature_names_the_repository_every_worktree_shares(self):
with tempfile.TemporaryDirectory() as tmp:
@@ -251,7 +280,7 @@ class TestEvaluate(unittest.TestCase):
self.assertEqual(self.queue.footprint(waiting, signatures, kinds), {"ram": 50, "cpu": 80})
older = {k: v for k, v in running.items() if k != "footprint"}
self.assertEqual(self.queue.footprint(older, signatures, kinds), {"ram": 50, "cpu": 80})
self.assertEqual(self.queue.pending_ram([running], "j1", NOW, signatures, kinds), 12 + 12)
self.assertEqual(self.queue.pending_ram([running], "j1", NOW, signatures, kinds, 50, 0), 12 + 12)
def test_an_active_entry_an_older_queue_granted_never_books_less_than_that_queue_did(self):
history = _load(HISTORY)
@@ -754,6 +783,7 @@ class TestQueueCommands(unittest.TestCase):
)
signature = f"{self.base.name}: {Path(sys.executable).name} -c"
self.assertEqual((entry["aliases"], entry["footprint"]), ([signature], None))
self.assertTrue(0 < entry["granted_mem"] < 100, entry)
self.assertEqual(self.entries(), [])
[row] = [json.loads(line) for line in self.history_file.read_text(encoding="utf-8").splitlines()]
self.assertEqual(
@@ -1111,6 +1141,52 @@ class TestQueueCommands(unittest.TestCase):
self.write_config()
self.assertEqual(self.bash("make compose-deploy"), 0)
def test_list_and_hold_on_a_broken_config_warn_and_use_the_shipped_default(self):
self.config.write_text(json.dumps(self.cfg)[:-1] + ",}", encoding="utf-8")
listing = self.run_queue("list")
self.assertEqual(listing.returncode, 0, listing.stderr)
self.assertIn("built-in defaults", listing.stderr)
held = self.run_queue("hold", "--kind", "nosuch", "--task", "work/a", "--session", "s1", "--", "true")
self.assertEqual(held.returncode, 2, held.stderr)
self.assertIn("built-in defaults", held.stderr)
self.assertIn("'nosuch' is not in slot_need_gb", held.stderr)
for done in (listing, held):
self.assertNotIn("Traceback", done.stderr)
def test_a_broken_config_falls_back_to_defaults_so_light_commands_pass_and_heavy_ones_stay_refused(self):
self.cfg["queue_enforce"] = True
good = json.dumps(self.cfg)
def text(content: str) -> Callable[[], None]:
return lambda: self.config.write_text(content, encoding="utf-8")
shapes = {
"comma": text(good[:-1] + ",}"), "empty": text(""), "list": text("[]"),
"string enforce": text(json.dumps({**self.cfg, "queue_enforce": "yes"})),
"empty-matching heavy": text(json.dumps({**self.cfg, "heavy_commands": [""]})),
"dot heavy": text(json.dumps({**self.cfg, "heavy_commands": ["."]})),
"word-boundary heavy": text(json.dumps({**self.cfg, "heavy_commands": [r"\b"]})),
"dot never_block": text(json.dumps({**self.cfg, "never_block": ["."]})),
"dangling link": lambda: self.config.symlink_to(self.base / "absent.json"),
"link loop": lambda: self.config.symlink_to(self.config),
"directory": lambda: self.config.mkdir(),
}
for name, make in shapes.items():
with self.subTest(config=name):
if self.config.is_dir() and not self.config.is_symlink():
self.config.rmdir()
self.config.unlink(missing_ok=True)
make()
passed = self.hook(json.dumps({"tool_input": {"command": "ls -la"}}))
self.assertEqual(passed.returncode, 0, passed.stderr)
self.assertIn(str(self.config), passed.stderr)
self.assertIn("built-in defaults", passed.stderr)
self.assertEqual(self.bash("make test"), 2)
live = ("make compose-deploy ROLE=x", "docker build .", "docker pull nginx", "make swarm-deploy", "make roundtrip",
"make worktree-up")
for command in live if name in ("comma", "directory") else live[:1]:
self.assertEqual(self.bash(command), 2, command)
def test_block_with_queue_enforce_refuses_heavy_commands_outside_the_queue(self):
self.cfg["queue_enforce"] = True
self.write_config()
@@ -1129,7 +1205,12 @@ class TestQueueCommands(unittest.TestCase):
):
with self.subTest(command=command):
self.assertEqual(self.bash(command), 0)
self.assertEqual(self.hook("not json").returncode, 0)
for payload in ("not json", json.dumps({"tool_input": ["make test"]})):
with self.subTest(payload=payload):
crashed = self.hook(payload)
self.assertEqual(crashed.returncode, 2, crashed.stderr)
self.assertIn("counts as heavy", crashed.stderr)
self.assertEqual(self.bash("CARGO_BUILD_JOBS=² cargo build ; make test"), 2)
self.config.unlink()
self.assertEqual(self.bash("make compose-deploy"), 0)
@@ -1290,6 +1371,40 @@ class TestQueueCommands(unittest.TestCase):
self.assertEqual(self.bash(command), 2)
self.assertEqual(self.bash("docker compose ps"), 0)
def test_a_queue_grant_counts_a_live_admit_record_and_ignores_an_expired_one(self):
def admitted(expires: float) -> None:
(self.root / "queue.json").write_text(
json.dumps({"seq": 0, "entries": [], "admits": [{"ram": 300, "expires": expires}]}), encoding="utf-8"
)
admitted(time.time() + 300)
_, err = self.blocked_hold("true")
self.assertRegex(err.read_text(encoding="utf-8"), r"RAM \d+ % used \+ 300 % still to come reach the limit 101 %")
admitted(time.time() - 1)
result = self.run_queue("hold", "--kind", "deploy", "--task", "work/a", "--session", "s2", "--", "true")
self.assertEqual((result.returncode, result.stderr), (0, ""))
def test_a_light_kind_refuses_a_heavy_command_names_the_test_kind_and_enqueues_nothing(self):
self.cfg["slot_need_gb"] |= {"test-nostack": 0, "quick": 0}
self.cfg["light_kinds"] = ["quick"]
self.write_config()
(self.base / "tests" / "e2e").mkdir(parents=True)
(self.base / "tests" / "e2e" / "test_orchestrator_e2e.py").write_text("", encoding="utf-8")
for kind, command, shown in (
("test-nostack", ("python", "-m", "unittest"), "python -m unittest"),
("test-nostack", ("pytest", "tests/e2e/test_orchestrator_e2e.py"), "pytest tests/e2e/test_orchestrator_e2e.py"),
("quick", ("bash", "-c", "make test"), "bash -c 'make test'"),
("quick", ("python3", "gate.py", "run", "--target", "test-unit"), "python3 gate.py run --target test-unit"),
):
with self.subTest(kind=kind, command=command):
result = self.run_queue("hold", "--kind", kind, "--task", "work/a", "--session", "s1", "--", *command)
self.assertEqual(result.returncode, 2)
self.assertIn(f"the light kind {kind} skips every reservation", result.stderr)
self.assertIn(f"run `queue.py hold --kind test --task <register task> -- {shown}`", result.stderr)
self.assertFalse((self.root / "queue.json").exists())
light = self.run_queue("hold", "--kind", "test-nostack", "--task", "work/a", "--session", "s1", "--", "true")
self.assertEqual((light.returncode, light.stderr), (0, ""))
def test_hold_refuses_a_config_without_the_budget_or_the_memory_limits(self):
for key, cfg in (
("budget_percent", {k: v for k, v in self.cfg.items() if k != "budget_percent"}),
+49 -1
View File
@@ -149,6 +149,53 @@ class TestCpu(unittest.TestCase):
self.assertEqual(json.loads((state / "cpu.json").read_text(encoding="utf-8")), [NOW + sampler.BUCKET + 1, 9300, 12200])
class TestPressure(unittest.TestCase):
def setUp(self):
tmp = tempfile.TemporaryDirectory()
self.addCleanup(tmp.cleanup)
self.proc, self.state = Path(tmp.name) / "proc", Path(tmp.name) / "state"
(self.proc / "pressure").mkdir(parents=True)
self.state.mkdir()
def vmstat(self, pages: int) -> None:
(self.proc / "vmstat").write_text(f"pgpgin 5\npswpin 3\npswpout {pages}\n", encoding="utf-8")
def test_psi_gives_some_and_full_avg10_and_none_without_pressure(self):
text = "some avg10=12.50 avg60=1.00 avg300=0.50 total=1\nfull avg10=6.25 avg60=0.80 avg300=0.10 total=1\n"
self.assertEqual((sampler.psi_avg10(text, "some"), sampler.psi_avg10(text, "full")), (12.5, 6.25))
self.assertEqual((sampler.psi_avg10("", "some"), sampler.psi_avg10(text.splitlines()[0], "full")), (None, None))
self.assertIsNone(sampler.swapped_pages("pgpgin 5\n"))
def test_swap_out_is_none_first_then_mb_per_minute_and_none_after_a_reset_or_a_gap(self):
with mock.patch.object(sampler, "PAGE_BYTES", 4096):
self.assertIsNone(sampler.swap_out_since(self.proc, self.state, NOW))
self.assertFalse((self.state / "swap.json").exists())
self.vmstat(1000)
self.assertIsNone(sampler.swap_out_since(self.proc, self.state, NOW))
self.vmstat(1000 + 1_400_000_000 // 4096)
self.assertEqual(sampler.swap_out_since(self.proc, self.state, NOW + 60), 1400.0)
self.vmstat(10)
self.assertIsNone(sampler.swap_out_since(self.proc, self.state, NOW + 120))
self.assertEqual(json.loads((self.state / "swap.json").read_text(encoding="utf-8")), [NOW + 120, 10])
self.assertIsNone(sampler.swap_out_since(self.proc, self.state, NOW + 120))
self.assertIsNone(sampler.swap_out_since(self.proc, self.state, NOW + 120 + sampler.BUCKET + 1))
def test_a_sample_row_adds_the_pressure_keys_and_keeps_every_other_key(self):
(self.proc / "pressure" / "memory").write_text("some avg10=3.00 avg60=0 avg300=0 total=1\n", encoding="utf-8")
found = sampler.sample(self.proc, self.proc / "sys", self.proc, lambda argv: None, self.state)
self.assertEqual(sorted(found), ["cpu", "disks", "gpus", "load", "psi_full", "psi_some", "queue", "ram", "swap", "swap_out_mb_min", "t"])
self.assertEqual((found["psi_some"], found["psi_full"], found["swap_out_mb_min"]), (3.0, None, None))
def test_the_fifteen_minute_mean_keeps_the_pressure_keys(self):
asleep = {"name": ASLEEP, "asleep": True}
rows = [
row(NOW + 60, 1.0, 1, 0, asleep) | {"psi_some": 2.0, "psi_full": 1.004, "swap_out_mb_min": 100.0},
row(NOW + 120, 1.0, 1, 0, asleep) | {"psi_some": 4.5, "psi_full": None, "swap_out_mb_min": 301.0},
]
mean = sampler.mean_row(NOW, rows)
self.assertEqual((mean["psi_some"], mean["psi_full"], mean["swap_out_mb_min"]), (3.25, 1.0, 200.5))
class TestGpus(unittest.TestCase):
def setUp(self):
tmp = tempfile.TemporaryDirectory()
@@ -206,6 +253,7 @@ class TestRetention(unittest.TestCase):
self.assertIs(kept[0], mean)
self.assertEqual(kept[1], {
"t": bucket, "disks": [{"mount": "/", "used": 200, "total": 1000}, {"mount": "/mnt", "used": 10, "total": 20}], "cpu": 30.0, "ram": 15.0, "swap": 0.0, "load": 3.0,
"psi_some": None, "psi_full": None, "swap_out_mb_min": None,
"gpus": [{"name": AWAKE, "util": 50.0, "vram_used": 300, "vram_total": 1000}], "queue": {"waiting": 1.0, "active": 1.0},
})
self.assertEqual((kept[2]["cpu"], kept[2]["disks"]), (40.0, [{"mount": "/", "used": 400, "total": 1000}]))
@@ -274,7 +322,7 @@ class TestSamplerInScratchHome(unittest.TestCase):
self.assertEqual(len(rows), 2)
for found in rows:
with self.subTest(t=found["t"]):
self.assertEqual(sorted(found), ["cpu", "disks", "gpus", "load", "queue", "ram", "swap", "t"])
self.assertEqual(sorted(found), ["cpu", "disks", "gpus", "load", "psi_full", "psi_some", "queue", "ram", "swap", "swap_out_mb_min", "t"])
self.assertEqual(found["disks"], DISKS)
self.assertEqual(found["gpus"], [{"name": AWAKE, "util": 37.0, "vram_used": 2048 * MIB, "vram_total": 8188 * MIB}, {"name": ASLEEP, "asleep": True}])
self.assertEqual(found["queue"], {"waiting": 1, "active": 2})
+943 -19
View File
File diff suppressed because it is too large. Load diff