feat(kvb-org-organize): hold heavy jobs back on memory pressure with one shipped default config

watch.py admits heavy work only while PSI, swap-out rate and RAM forecast stay under the configured limits; config.default.json is the single default the install writes once (mode 600, no clobber) and every hook, admit and queue command falls back to. Browser runners and package-manager script forms count as heavy, preflight derives its probes from the light and heavy tool lists, and the shipped default carries no prune command, buildx path or host-specific key. Tests run once on unify/skills-20261114 after the merge (operator order 2026-11-14).

🎯 Fix: 85 %
💥 Break: 8 %
💸 Tokens: 99 %
⏱️ Velocity: –
🛡️ Security: 100 %
🏆 Score: 79
🚦 Gate: ⛔ fix 85 % < 90, break 8 % > 5

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Kevin Veen-BirkenbachandClaude Opus 5.5 committed 2026-11-14 21:17:28 +00:00
1 parent a2a03995cb
commit 8e2db852e5
27 files changed
+2795 -207

No files matched your search

+52 -7
View File
@@ -2,12 +2,17 @@
"""Args: """Args:
sample: append one row to <state>/samples.jsonl: {t, disks: [{mount, sample: append one row to <state>/samples.jsonl: {t, disks: [{mount,
used, total}], cpu, ram, swap, load, gpus: [{name, util, vram_used, used, total}], cpu, ram, swap, load, gpus: [{name, util, vram_used,
vram_total} | {name, asleep: true}], queue: {waiting, active}}; t in vram_total} | {name, asleep: true}], queue: {waiting, active},
epoch seconds, used, total and vram in bytes, cpu, ram, swap and util psi_some, psi_full, swap_out_mb_min}; t in epoch seconds, used, total
in percent, load the 1-minute load average, a GPU named by its PCI and vram in bytes, cpu, ram, swap and util in percent, load the
address. cpu is the busy share of the /proc/stat ticks since those 1-minute load average, a GPU named by its PCI address. cpu is the
the run before kept in <state>/cpu.json as [t, idle, total], None on busy share of the /proc/stat ticks since those the run before kept in
the first run and after a gap of more than 15 minutes. An NVIDIA GPU <state>/cpu.json as [t, idle, total], None on the first run and after
a gap of more than 15 minutes. psi_some and psi_full are the avg10 of
/proc/pressure/memory, None without PSI; swap_out_mb_min the MB per
minute swapped out (pswpout of /proc/vmstat) since the count the run
before kept in <state>/swap.json as [t, pswpout], None like cpu and
after a counter reset. An NVIDIA GPU
whose PCI runtime_status is not active is recorded asleep, and whose PCI runtime_status is not active is recorded asleep, and
nvidia-smi is not run for it; an awake one nvidia-smi gives no nvidia-smi is not run for it; an awake one nvidia-smi gives no
reading for (no driver, a driver mismatch) keeps its name with util reading for (no driver, a driver mismatch) keeps its name with util
@@ -75,12 +80,15 @@ NVIDIA_QUERY = "pci.bus_id,utilization.gpu,memory.used,memory.total"
UNREAD = {"util": None, "vram_used": None, "vram_total": None} UNREAD = {"util": None, "vram_used": None, "vram_total": None}
COUNTS = (("input", "input_tokens"), ("output", "output_tokens"), ("cache_read", "cache_read_input_tokens"), ("cache_write", "cache_creation_input_tokens")) COUNTS = (("input", "input_tokens"), ("output", "output_tokens"), ("cache_read", "cache_read_input_tokens"), ("cache_write", "cache_creation_input_tokens"))
NAMES = tuple(name for name, _ in COUNTS) NAMES = tuple(name for name, _ in COUNTS)
HOST_DIGITS = (("cpu", 1), ("ram", 1), ("swap", 1), ("load", 2)) HOST_DIGITS = (("cpu", 1), ("ram", 1), ("swap", 1), ("load", 2), ("psi_some", 2), ("psi_full", 2), ("swap_out_mb_min", 1))
CLAIMS = ("*/claim.json", "*/*/claim.json") CLAIMS = ("*/claim.json", "*/*/claim.json")
SAMPLES = "samples.jsonl" SAMPLES = "samples.jsonl"
TOKENS = "tokens.jsonl" TOKENS = "tokens.jsonl"
OFFSETS = "offsets.json" OFFSETS = "offsets.json"
TICKS = "cpu.json" TICKS = "cpu.json"
SWAPPED = "swap.json"
PAGE_BYTES = os.sysconf("SC_PAGE_SIZE")
MB = 1_000_000
def read_text(path: Path) -> str: def read_text(path: Path) -> str:
@@ -263,9 +271,43 @@ def cpu_since(proc: Path, state: Path, t: int) -> float | None:
return cpu_percent(before, ticks) return cpu_percent(before, ticks)
def psi_avg10(text: str, kind: str) -> float | None:
for line in text.splitlines():
fields = line.split()
if fields[:1] == [kind]:
values = dict(field.split("=", 1) for field in fields[1:] if "=" in field)
return number(values.get("avg10", ""))
return None
def swapped_pages(text: str) -> int | None:
for line in text.splitlines():
fields = line.split()
if len(fields) == 2 and fields[0] == "pswpout" and fields[1].isdigit():
return int(fields[1])
return None
def swap_out_since(proc: Path, state: Path, t: int) -> float | None:
"""Returns:
The MB per minute swapped out since the pswpout count the run before
kept in <state>/swap.json at most BUCKET seconds before t, else None,
also after a counter reset; this run's count replaces it.
"""
pages = swapped_pages(read_text(proc / "vmstat"))
if pages is None:
return None
stored = read_json(state / SWAPPED)
replace(state / SWAPPED, json.dumps([t, pages]))
if not integers(stored, 2) or not t - BUCKET <= stored[0] < t or stored[1] > pages:
return None
return round((pages - stored[1]) * PAGE_BYTES / MB * 60 / (t - stored[0]), 1)
def sample(proc: Path, sys_root: Path, council: Path, run: Callable[[list[str]], str | None], state: Path) -> dict: def sample(proc: Path, sys_root: Path, council: Path, run: Callable[[list[str]], str | None], state: Path) -> dict:
t = int(time.time()) t = int(time.time())
ram, swap = memory(parse_meminfo(read_text(proc / "meminfo"))) ram, swap = memory(parse_meminfo(read_text(proc / "meminfo")))
pressure = read_text(proc / "pressure" / "memory")
return { return {
"t": t, "t": t,
"disks": parse_df(run(["df", "-P", "-k", "-l"]) or ""), "disks": parse_df(run(["df", "-P", "-k", "-l"]) or ""),
@@ -275,6 +317,9 @@ def sample(proc: Path, sys_root: Path, council: Path, run: Callable[[list[str]],
"load": parse_loadavg(read_text(proc / "loadavg")), "load": parse_loadavg(read_text(proc / "loadavg")),
"gpus": gpus(sys_root, run), "gpus": gpus(sys_root, run),
"queue": queue(council), "queue": queue(council),
"psi_some": psi_avg10(pressure, "some"),
"psi_full": psi_avg10(pressure, "full"),
"swap_out_mb_min": swap_out_since(proc, state, t),
} }
+17
View File
@@ -23,6 +23,12 @@ if [[ ! -d "${TARGET}" ]]; then
exit 2 exit 2
fi fi
watch="${REPO_ROOT}/skills/kvb-org-organize/scripts/watch.py"
if [[ -f "${watch}" ]] && ! python3 -I "${watch}" preflight; then
warn "skills: add the keys above to the agents council config first; the new hooks and queue.py refuse to run without them"
exit 2
fi
cd "${REPO_ROOT}" cd "${REPO_ROOT}"
src="${REPO_ROOT}/.agents/skills" src="${REPO_ROOT}/.agents/skills"
rm -rf "${src}" rm -rf "${src}"
@@ -58,6 +64,17 @@ for root in .agents/skills .claude/skills; do
fi fi
done done
config="${AGENTS_COUNCIL_CONFIG:-${HOME}/.claude/agents-council/config.json}"
shipped="${REPO_ROOT}/skills/kvb-org-organize/config.default.json"
if [[ -f "${shipped}" && ! -e "${config}" && ! -L "${config}" ]]; then
[[ -d "$(dirname -- "${config}")" ]] || install -d -m 0700 "$(dirname -- "${config}")"
if (umask 077 && set -o noclobber && cat -- "${shipped}" >"${config}") 2>/dev/null; then
log "skills: wrote the shipped agents council config to ${config}"
else
warn "skills: kept ${config}, which appeared while the install wrote the shipped default"
fi
fi
settings="${TARGET}/.claude/settings.local.json" settings="${TARGET}/.claude/settings.local.json"
if command -v node &>/dev/null; then if command -v node &>/dev/null; then
log "skills: auto-enabling caveman + ponytail plugins in ${settings}" log "skills: auto-enabling caveman + ponytail plugins in ${settings}"
+2 -3
View File
@@ -7,9 +7,8 @@ description: >
Where an Ansible run spends its time: its log, a controller profile, a bench. Where an Ansible run spends its time: its log, a controller profile, a bench.
Report the script's lines with their log and range, never a guess. The Report the script's lines with their log and range, never a guess. The
script and the bench are light runs (`watch.py admit --kind test-nostack` of script, the bench and a deploy to profile run through the `kvb-org-queue`
`kvb-org-organize`); a deploy to profile is heavy and goes through the skill.
`kvb-org-queue` skill.
## 1. Pick the log ## 1. Pick the log
+2 -3
View File
@@ -60,12 +60,11 @@ It also measures wall time for `kvb-check-speed`.
- **Command**: the targeted tests of the change or a CLI call of the - **Command**: the targeted tests of the change or a CLI call of the
changed code, runnable on a clean checkout: the base worktree holds no changed code, runnable on a clean checkout: the base worktree holds no
untracked build output, virtualenv or dependency directory. untracked build output, virtualenv or dependency directory.
- **Heavy commands** (Docker, deploys, whole suites) run only inside - **Every run** goes inside
`${CLAUDE_SKILL_DIR}/../kvb-org-queue/scripts/queue.py hold --kind <kind> `${CLAUDE_SKILL_DIR}/../kvb-org-queue/scripts/queue.py hold --kind <kind>
--task <register task> -- python3 <measure.py> ...`, in the background, --task <register task> -- python3 <measure.py> ...`, in the background,
by the `kvb-org-queue` skill; a cancelled wait there also exits 3, but by the `kvb-org-queue` skill; a cancelled wait there also exits 3, but
prints no 💸 line. A light one self-admits with prints no 💸 line.
`watch.py admit --kind test-nostack` of `kvb-org-organize`.
- Report the run you made; never rerun to pick the best result. - Report the run you made; never rerun to pick the best result.
## Exit codes ## Exit codes
+3 -4
View File
@@ -32,10 +32,9 @@ commit starts from zero.
`missing hotfix` reason means back to step 1. `missing hotfix` reason means back to step 1.
4. **Admit**: every gate target is heavy and runs only inside 4. **Admit**: every gate target is heavy and runs only inside
`${CLAUDE_SKILL_DIR}/../kvb-org-queue/scripts/queue.py hold`. `${CLAUDE_SKILL_DIR}/../kvb-org-queue/scripts/queue.py hold`.
Only a single test module Only a single test module self-admits (`kvb-org-organize` Capacity), and
self-admits (`${CLAUDE_SKILL_DIR}/../kvb-org-organize/scripts/watch.py it never turns a gate target green. A hotfix gate adds `--hotfix <slug>`
admit --kind test-nostack`), and it never turns a gate target green. A and takes the next free gate slot.
hotfix gate adds `--hotfix <slug>` and takes the next free gate slot.
5. **Run** the missing and red targets, tracked tree clean, in the 5. **Run** the missing and red targets, tracked tree clean, in the
background; give the operator `tail -f` on the log: background; give the operator `tail -f` on the log:
+2 -5
View File
@@ -15,11 +15,8 @@ skill's Rules apply here too.
first error. first error.
3. **Fix at the root**: apply the real fix for each proven root cause in the 3. **Fix at the root**: apply the real fix for each proven root cause in the
sidetree, with a test and a negative control as the `kvb-org-swarm` skill sidetree, with a test and a negative control as the `kvb-org-swarm` skill
demands. Light runs (no Docker, about a minute: one test file or demands. Light runs self-admit (`kvb-org-organize` Capacity); `make test`,
module, ruff) self-admit with gates, deploys and anything with Docker run in the background through
`${CLAUDE_SKILL_DIR}/../kvb-org-organize/scripts/watch.py admit --kind
test-nostack`; `make test`, gates, deploys and anything with Docker run in
the background through
`${CLAUDE_SKILL_DIR}/../kvb-org-queue/scripts/queue.py hold --kind <kind> `${CLAUDE_SKILL_DIR}/../kvb-org-queue/scripts/queue.py hold --kind <kind>
--task <register task> -- <command>`, which waits for the rule-based --task <register task> -- <command>`, which waits for the rule-based
grant and releases on exit. The fix enters the main line only grant and releases on exit. The fix enters the main line only
+2 -3
View File
@@ -26,9 +26,8 @@ cover the change set run now and give the green baseline; a suite already red
is reported, not refactored around. No coverage: read `net.md` next to this is reported, not refactored around. No coverage: read `net.md` next to this
file before editing. file before editing.
Light runs (one test file or module, ruff) start once `watch.py Light runs start once `watch.py admit` of the `kvb-org-organize` skill prints
admit --kind test-nostack` of the `kvb-org-organize` skill prints `grant`; no `grant`; no chair request.
chair request.
## Step 3: refactor each file ## Step 3: refactor each file
+2 -5
View File
@@ -38,11 +38,8 @@ any listed name. Never silently truncate.
## Step 3: run ## Step 3: run
Before each run, admit it under the `kvb-org-organize` Capacity rules: a Before each run, admit it under the `kvb-org-organize` Capacity rules: a
light run (no Docker, about a minute: one test file or module, ruff) light run self-admits; `make test`, quality gates and anything with Docker
self-admits with run in the background through
`${CLAUDE_SKILL_DIR}/../kvb-org-organize/scripts/watch.py admit --kind
test-nostack`; `make test`, quality gates and anything with Docker run in the
background through
`${CLAUDE_SKILL_DIR}/../kvb-org-queue/scripts/queue.py hold --kind <kind> `${CLAUDE_SKILL_DIR}/../kvb-org-queue/scripts/queue.py hold --kind <kind>
--task <register task> -- make <target>`, which waits for the rule-based --task <register task> -- make <target>`, which waits for the rule-based
grant and releases on exit. grant and releases on exit.
+2 -3
View File
@@ -49,9 +49,8 @@ single-select `Write it` / `Revise it` / `Discard` with the drafted diff.
owner through the `kvb-org-coordinate` skill. owner through the `kvb-org-coordinate` skill.
- Every carried test fails without the code it guards: show the negative - Every carried test fails without the code it guards: show the negative
control once. control once.
- Run the carried tests as a light run (no Docker, about a minute) after - Run the carried tests as a light run (`kvb-org-organize` Capacity);
`${CLAUDE_SKILL_DIR}/../kvb-org-organize/scripts/watch.py admit --kind `make test`, gates and anything with Docker only through
test-nostack`; `make test`, gates and anything with Docker only through
`queue.py hold` (the `kvb-org-queue` skill). Leave the result `queue.py hold` (the `kvb-org-queue` skill). Leave the result
uncommitted for the `kvb-git-commit-next` skill. uncommitted for the `kvb-git-commit-next` skill.
- `transcend.py mark` every offered candidate with its verdict. - `transcend.py mark` every offered candidate with its verdict.
+2 -4
View File
@@ -71,10 +71,8 @@ order:
Show the full draft, then `AskUserQuestion`: `Write it` / `Revise it` / Show the full draft, then `AskUserQuestion`: `Write it` / `Revise it` /
`Discard`. Revise loops back to the step the correction touches. Write only `Discard`. Revise loops back to the step the correction touches. Write only
on `Write it`; lint the file with the project's markdown linter, a light run on `Write it`; lint the file with the project's markdown linter, a light run
that self-admits with (`kvb-org-organize` Capacity); a lint through `make` or Docker runs through
`${CLAUDE_SKILL_DIR}/../kvb-org-organize/scripts/watch.py admit --kind `queue.py hold` (the `kvb-org-queue` skill).
test-nostack`; a lint through `make` or Docker runs through `queue.py hold`
(the `kvb-org-queue` skill).
## Step 6: hand over ## Step 6: hand over
+2 -2
View File
@@ -35,8 +35,8 @@ request, so a red one blocks nothing else.
(`search_graph`, `trace_path`, `get_architecture`, `detect_changes`; the (`search_graph`, `trace_path`, `get_architecture`, `detect_changes`; the
`kvb-agent-optimize` skill sets it up), grep and read after. Parallel `kvb-agent-optimize` skill sets it up), grep and read after. Parallel
work, gates, deploys and anything with Docker run through `queue.py hold` work, gates, deploys and anything with Docker run through `queue.py hold`
(`kvb-org-queue`); light runs (no Docker, about a minute) self-admit with (`kvb-org-queue`); light runs self-admit (`kvb-org-organize` Capacity).
`watch.py admit --kind test-nostack`. With the first change, swap the label: `gh issue With the first change, swap the label: `gh issue
edit <n> --remove-label "in analysis" --add-label "in progress"`. edit <n> --remove-label "in analysis" --add-label "in progress"`.
5. **Open the pull request** with the `kvb-git-pr-create` skill, closing 5. **Open the pull request** with the `kvb-git-pr-create` skill, closing
exactly its issue; one for a failing default branch is posted with exactly its issue; one for a failing default branch is posted with
+7 -2
View File
@@ -37,8 +37,13 @@ ends or goes stale, or after a restart.
## Capacity ## Capacity
- Light runs (no Docker, about a minute: one test file or module, ruff) - Light runs self-admit: `watch.py admit --kind test-nostack -- <command>`,
self-admit: `watch.py admit --kind test-nostack`, start on `grant`. start on `grant`. Light is only ruff, shellcheck, markdownlint, lizard, jq,
py_compile, cargo fmt, `env CARGO_BUILD_JOBS=4 cargo build|check|clippy|test`,
or one test file or module outside `e2e/` that reaches no Playwright;
anything else: `queue.py hold --kind test`.
- Memory stop (also PSI or swap-out) blocks every admit and grant; running
jobs go on.
- Other test runs, `make test`, `make test-lint`, quality gates, deploys, - Other test runs, `make test`, `make test-lint`, quality gates, deploys,
iteration loops, swarm, image builds, cloud, shared-data cleanup, anything iteration loops, swarm, image builds, cloud, shared-data cleanup, anything
with Docker or over a few minutes: run through `queue.py hold` with Docker or over a few minutes: run through `queue.py hold`
+13
View File
@@ -63,3 +63,16 @@ and stays silent below the thresholds.
`watch.py status` shows the stage computed from the live free space and the `watch.py status` shows the stage computed from the live free space and the
stored samples, and the age of the stored measurement. stored samples, and the age of the stored measurement.
An unusable config (broken JSON, wrong types, a regex refusing a light
probe such as `grep` or freeing a heavy one such as `make swarm-deploy`, a
dangling link, a directory) makes the hooks, admit, status and the queue
warn and use `config.default.json`, which the install writes when no config
exists; without that file every command, teardown included, is refused.
The probes are samples, so a pattern hitting only other commands passes. A
package script is read from the nearest package.json, a named workspace
only from a folder of that name. The light-run check misses
`sys.path.insert` in a conftest, editable installs without pytest
`pythonpath`, browser libraries outside its list and generic third-party
fixtures such as pytest-selenium's `driver`, browsers started by a
subprocess, and `git bisect run` or a suite script behind `bash`.
@@ -0,0 +1,37 @@
{
"disk": "/",
"thresholds_gb": {"warn": 40, "stop": 15, "critical": 12},
"projection_minutes": 30,
"interval_seconds": 120,
"cache_interval_seconds": 1800,
"heavy_commands": [
"\\bmake\\s+(-C\\s+\\S+\\s+)?(compose-deploy|compose-up|compose-playwright|roundtrip|quality|quality-high|test|test-integration|test-unit\\S*|swarm-\\S+|act-workflow|act-runner-image|build\\S*|worktree-up|store-\\S+)(\\s|$)",
"\\bdocker\\s+(build|pull|load)\\b",
"\\bdocker\\s+buildx\\s+build\\b",
"\\brun\\.sh\\s+local\\b"
],
"never_block": [
"\\bmake\\s+(-C\\s+\\S+\\s+)?(swarm-down|swarm-clean|swarm-clean-stale-nfs|swarm-diagnostic|compose-down|compose-entity-purge|worktree-down|store-host-down)(\\s|$)",
"^\\s*git\\s+commit\\b.*\\s--no-verify\\b",
"\\bmake\\s+swarm-exec(\\s|$)"
],
"queue_enforce": true,
"cleanup": [],
"worktree_cleanup": [],
"worktrees_command": "true",
"worktree_marker": ".git",
"busy_processes": ["make"],
"cache_dirs": ["~/.cache/pip", "~/.npm", "~/.cache/pre-commit", "~/.cache/act", "~/.cache/ms-playwright"],
"cache_budget_percent": 3,
"slot_need_gb": {"quality-high": 15, "test": 15, "test-nostack": 0, "iteration": 20, "swarm": 30, "gui-dind": 65, "operator": 10},
"notify": ["notify-send", "-u", "critical", "Agents council"],
"load_limits": {"cpu_percent": 90, "mem_percent": 85, "sample_seconds": 5, "kinds": ["test", "test-nostack", "quality-high"]},
"memory_limits": {
"warn": {"available": 15, "swap_free": 40, "psi_some": 10, "psi_full": 5, "swap_out_mb_min": 64, "container_swap_gb": 4},
"stop": {"available": 8, "swap_free": 10, "psi_full": 5, "swap_out_mb_min": 256, "container_swap_gb": 16}
},
"budget_percent": {"ram": 75, "cpu": 100},
"light_kinds": ["test-nostack"],
"exclusive_kinds": ["quality-high", "test"],
"gate_excluded_targets": ["test-oracle"]
}
File diff suppressed because it is too large. Load diff
+4 -5
View File
@@ -27,9 +27,8 @@ turn: hold runs the command on its grant and exits with its code.
- In a worktree the command stays `gate.py run|hold` (`kvb-check-gate`): - In a worktree the command stays `gate.py run|hold` (`kvb-check-gate`):
`queue.py hold ... -- python3 <gate.py> run ...`. `queue.py hold ... -- python3 <gate.py> run ...`.
One test file or module, or ruff, without Docker self-admits with Light runs (`kvb-org-organize` Capacity) self-admit; anything else goes
`watch.py admit --kind test-nostack` (`kvb-org-organize`); any other test through hold with kind `test`, and a light kind refuses it.
run goes through hold with kind `test`.
## Order and grant ## Order and grant
@@ -77,8 +76,8 @@ learned values, `cancel` removes your own waiting job.
one down, and `kvb-git-hotfix` may void a gate. A `kvb-org-queue-cleanup` one down, and `kvb-git-hotfix` may void a gate. A `kvb-org-queue-cleanup`
round deletes only entries whose lock no living process holds. round deletes only entries whose lock no living process holds.
- With `queue_enforce: true`, the `block` hook (PreToolUse on Bash) refuses - With `queue_enforce: true`, the `block` hook (PreToolUse on Bash) refuses
`heavy_commands`, image builds, `make install|verify-*`, tests beyond one `heavy_commands`, image builds, `make install|test*|quality*|verify-*`,
file and `gate.py run|hold` outside `queue.py hold`. tests beyond one unit file and `gate.py run|hold` outside `queue.py hold`.
- Without the config, hold exits 2: tell the operator and ask the chair - Without the config, hold exits 2: tell the operator and ask the chair
(`kvb-org-organize`) until it exists. A waiting hold rereads it every poll (`kvb-org-organize`) until it exists. A waiting hold rereads it every poll
(a broken file keeps the last good one), so a limit change needs no re-hold. (a broken file keeps the last good one), so a limit change needs no re-hold.
+98 -27
View File
@@ -38,7 +38,9 @@ Args:
SIBLINGS); a poll that finds other code installed in SIBLINGS); a poll that finds other code installed in
~/.agents/skills leaves the queue ungranted and restarts the same hold ~/.agents/skills leaves the queue ungranted and restarts the same hold
with the installed queue.py. with the installed queue.py.
--kind: a key of slot_need_gb in the config. --kind: a key of slot_need_gb in the config; a light one
(light_kinds or watch.LIGHT_KIND) refuses a command watch.light()
does not recognise, naming watch.queue_hold().
--task: an open task of the kvb-org-rasci register; its rank orders --task: an open task of the kvb-org-rasci register; its rank orders
the queue, and the operator's agent order (`rasci.py agents the queue, and the operator's agent order (`rasci.py agents
order`) of the holding session breaks ties. While one of its order`) of the holding session breaks ties. While one of its
@@ -87,7 +89,8 @@ disk of running and granted jobs, never a reservation; queue_enforce (bool), opt
only. The watch.py keys disk, thresholds_gb (stop is the floor every grant only. The watch.py keys disk, thresholds_gb (stop is the floor every grant
leaves free), slot_need_gb (the disk seed per kind), load_limits (every grant leaves free), slot_need_gb (the disk seed per kind), load_limits (every grant
also needs CPU busy below cpu_percent and RAM used plus the job's learned RAM also needs CPU busy below cpu_percent and RAM used plus the job's learned RAM
share below mem_percent; both bound the learned slot limits), memory_limits (no share, the live admit records (admit()) and the young jobs' ramp
(ramping_ram()) below mem_percent; both bound the learned slot limits), memory_limits (no
grant at memory stage stop), heavy_commands and never_block are used as grant at memory stage stop), heavy_commands and never_block are used as
watch.py documents them. A signature is `<repository>: <command signature>` watch.py documents them. A signature is `<repository>: <command signature>`
(aliases()). (aliases()).
@@ -165,6 +168,10 @@ XDIST_LONG = "--numprocesses="
ADDOPTS = "PYTEST_ADDOPTS=" ADDOPTS = "PYTEST_ADDOPTS="
FROZEN = "footprint" FROZEN = "footprint"
ALIASES = "aliases" ALIASES = "aliases"
ADMIT = "admit"
ADMITS = "admits"
ADMIT_SECONDS = 120
GRANTED_MEM = "granted_mem"
@functools.cache @functools.cache
@@ -206,7 +213,7 @@ def fail(message: str) -> NoReturn:
def config() -> dict: def config() -> dict:
cfg = watch.load_config() cfg, _ = watch.announced_config()
if cfg is None: if cfg is None:
fail(f"no config at {watch.CONFIG}; the queue reads the kvb-org-organize watch config") fail(f"no config at {watch.CONFIG}; the queue reads the kvb-org-organize watch config")
problem = watch.missing(cfg, (*BUDGET_KEYS, *watch.MEMORY_KEYS)) problem = watch.missing(cfg, (*BUDGET_KEYS, *watch.MEMORY_KEYS))
@@ -218,13 +225,14 @@ def config() -> dict:
def fresh(cfg: dict) -> dict: def fresh(cfg: dict) -> dict:
"""Returns: """Returns:
The config as it stands now, else cfg while the file is missing, The config as it stands now, else cfg while the file is missing,
unreadable or incomplete, so a half-saved edit never ends a wait. unusable (watch.checked_config()) or incomplete, so a half-saved
edit never ends a wait.
""" """
try: try:
now = watch.load_config() now = watch.checked_config(watch.load_config())
except (OSError, ValueError): except watch.UNUSABLE:
return cfg return cfg
if not isinstance(now, dict) or watch.missing(now, (*BUDGET_KEYS, *watch.MEMORY_KEYS)) is not None: if watch.missing(now, (*BUDGET_KEYS, *watch.MEMORY_KEYS)) is not None:
return cfg return cfg
return now return now
@@ -690,11 +698,11 @@ def footprint(entry: dict, signatures: dict[tuple[str, str], dict], kinds: dict[
def held_back(cfg: dict, entries: list[dict], now: float, signatures: dict[str, dict], kinds: dict[str, dict], def held_back(cfg: dict, entries: list[dict], now: float, signatures: dict[str, dict], kinds: dict[str, dict],
cpu: float, mem: float) -> dict[str, str]: cpu: float, mem: float, admitted: float) -> dict[str, str]:
loaded = {} loaded = {}
for entry in entries: for entry in entries:
if entry["status"] == "waiting": if entry["status"] == "waiting":
pending = pending_ram(entries, entry["id"], now, signatures, kinds) pending = pending_ram(entries, entry["id"], now, signatures, kinds, mem, admitted)
if (reason := load_reason(cfg["load_limits"], cpu, mem, pending)) is not None: if (reason := load_reason(cfg["load_limits"], cpu, mem, pending)) is not None:
loaded[entry["id"]] = reason loaded[entry["id"]] = reason
return loaded return loaded
@@ -742,18 +750,36 @@ def ram_share(entry: dict, signatures: dict[str, dict], kinds: dict[str, dict])
return history.seed_ram(kinds) if share is None else share["ram"] return history.seed_ram(kinds) if share is None else share["ram"]
def pending_ram(entries: list[dict], entry_id: str, now: float, signatures: dict[str, dict], def ramping_ram(entries: list[dict], now: float, signatures: dict[str, dict], kinds: dict[str, dict],
kinds: dict[str, dict]) -> float: mem: float) -> float:
"""Returns: """Returns:
The RAM share of entry_id plus the whole share of every active job What the active jobs granted less than RAMP_SECONDS ago (or without a
granted less than RAMP_SECONDS ago (or without a grant time): what grant time) may still add: the shares of those whose grant recorded
may still come once they ramp up. An older job counts as ramped, its the RAM used (GRANTED_MEM) minus what RAM used gained since the
RAM inside the RAM used; one that peaks later is bounded by the earliest of these grants, between 0 and those shares (a RAM used
memory stage only. that fell since counts as no gain), plus the whole share of the
others. An older job counts as ramped, its RAM inside the RAM used;
one that peaks later is bounded by the memory stage only.
"""
young = [e for e in entries if e["status"] == "active" and now - e.get("granted", now) < RAMP_SECONDS]
dated = [e for e in young if GRANTED_MEM in e]
gained = max(0.0, mem - min((e[GRANTED_MEM] for e in dated), default=mem))
ramp = max(0.0, sum(ram_share(e, signatures, kinds) for e in dated) - gained)
return ramp + sum(ram_share(e, signatures, kinds) for e in young if GRANTED_MEM not in e)
def pending_ram(entries: list[dict], entry_id: str, now: float, signatures: dict[str, dict],
kinds: dict[str, dict], mem: float, admitted: float) -> float:
"""Returns:
The RAM share of entry_id plus admitted (admitted_ram()) plus
ramping_ram() at RAM used mem.
""" """
mine = [e for e in entries if e["id"] == entry_id] mine = [e for e in entries if e["id"] == entry_id]
young = [e for e in entries if e["status"] == "active" and now - e.get("granted", now) < RAMP_SECONDS] return ram_share(mine[0], signatures, kinds) + admitted + ramping_ram(entries, now, signatures, kinds, mem)
return ram_share(mine[0], signatures, kinds) + sum(ram_share(e, signatures, kinds) for e in young)
def admitted_ram(data: dict, now: float) -> float:
return sum(record["ram"] for record in data.get(ADMITS, []) if record["expires"] > now)
def workdir() -> str: def workdir() -> str:
@@ -822,13 +848,13 @@ def wait(cfg: dict, entry_id: str, caught: list[int]) -> tuple[int | None, int]:
print(f"queue: {entry_id} was cancelled", file=sys.stderr) print(f"queue: {entry_id} was cancelled", file=sys.stderr)
return CANCELLED, 0 return CANCELLED, 0
rejoin(data, mine[0], now) rejoin(data, mine[0], now)
loaded = held_back(cfg, data["entries"], now, signatures, kinds, cpu, mem) loaded = held_back(cfg, data["entries"], now, signatures, kinds, cpu, mem, admitted_ram(data, now))
verdicts = decide(cfg, data["entries"], free, now, signatures, kinds, loaded) verdicts = decide(cfg, data["entries"], free, now, signatures, kinds, loaded)
reason = verdicts[entry_id] or loaded.get(entry_id) reason = verdicts[entry_id] or loaded.get(entry_id)
mine[0]["reason"] = reason mine[0]["reason"] = reason
if reason is None: if reason is None:
concurrent = sum(e["status"] == "active" for e in data["entries"]) concurrent = sum(e["status"] == "active" for e in data["entries"])
mine[0].update(status="active", granted=now, **{FROZEN: footprint(mine[0], signatures, kinds)}) mine[0].update(status="active", granted=now, **{GRANTED_MEM: mem, FROZEN: footprint(mine[0], signatures, kinds)})
return None, concurrent return None, concurrent
position = list(verdicts).index(entry_id) + 1 position = list(verdicts).index(entry_id) + 1
if (position, reason) != shown: if (position, reason) != shown:
@@ -912,15 +938,27 @@ def keep_receipt(gates: ModuleType | None, args: argparse.Namespace, here: str,
print(f"queue: receipt {args.receipt} for tree {before}", file=sys.stderr) print(f"queue: receipt {args.receipt} for tree {before}", file=sys.stderr)
def cmd_hold(args: argparse.Namespace) -> int: def check_hold(args: argparse.Namespace, cfg: dict) -> None:
cfg = config() """Args:
args: the hold arguments; fail() exits on an unknown kind, a command
that is no light run under a light kind, a task outside the
register or a missing hotfix row.
cfg: the watch config.
"""
if args.kind not in cfg["slot_need_gb"]: if args.kind not in cfg["slot_need_gb"]:
fail(f"kind {args.kind!r} is not in slot_need_gb of {watch.CONFIG}") fail(f"kind {args.kind!r} is not in slot_need_gb of {watch.CONFIG}")
shown = shlex.join(args.command)
if args.kind in {watch.LIGHT_KIND, *cfg.get("light_kinds", [])} and not watch.light(shown, os.getcwd()):
fail(f"{shown} is no light run and the light kind {args.kind} skips every reservation; run `{watch.queue_hold(shown)}`")
rasci = sibling("rasci") rasci = sibling("rasci")
if args.task not in rasci.merged(False): if args.task not in rasci.merged(False):
fail(f"{args.task} is no open task of the kvb-org-rasci register") fail(f"{args.task} is no open task of the kvb-org-rasci register")
if args.hotfix is not None and HOTFIX + args.hotfix not in rasci.load(): if args.hotfix is not None and HOTFIX + args.hotfix not in rasci.load():
fail(f"the register has no row {HOTFIX}{args.hotfix}") fail(f"the register has no row {HOTFIX}{args.hotfix}")
def cmd_hold(args: argparse.Namespace) -> int:
cfg = config()
check_hold(args, cfg)
here = workdir() here = workdir()
gates, out = gated(args, here) gates, out = gated(args, here)
names = aliases(here, args.command, os.getcwd()) names = aliases(here, args.command, os.getcwd())
@@ -1091,6 +1129,38 @@ def held_commands(command: str) -> list[list[str]]:
] ]
def enforced(cfg: dict) -> dict:
return {**cfg, "heavy_commands": [*cfg["heavy_commands"], GATE.pattern]}
def admit(cfg: dict, kind: str, argv: list[str], cpu: float, mem: float, calm: bool) -> str | None:
"""Args:
cfg: the config.
kind: the kind of the light job that asks to start outside the queue.
argv: its command, run in the current directory.
cpu: CPU busy now, in percent.
mem: RAM used now, in percent.
calm: whether the memory stage is ok; the live active jobs' ramp
(ramping_ram()) counts only when it is not.
Returns:
load_reason() over the job's RAM share plus the live admit records
(admitted_ram()) and the ramp; under the store lock, a grant records
the share for ADMIT_SECONDS and expired records leave.
"""
now = time.time()
signatures, kinds = learned(cfg, now)
mine = {"id": ADMIT, "status": "waiting", "kind": kind, ALIASES: list(aliases(workdir(), argv, os.getcwd()))}
share = ram_share(mine, signatures, kinds)
with store() as data:
active = [e for e in data["entries"] if e["status"] == "active" and living(e)]
ramp = 0.0 if calm else ramping_ram(active, now, signatures, kinds, mem)
reason = load_reason(cfg["load_limits"], cpu, mem, share + admitted_ram(data, now) + ramp)
granted = [{"ram": share, "expires": now + ADMIT_SECONDS}] if reason is None else []
data[ADMITS] = [record for record in data.get(ADMITS, []) if record["expires"] > now] + granted
return reason
def tails(argv: list[str]) -> list[str]: def tails(argv: list[str]) -> list[str]:
return [shlex.join(argv[at:]) for at, word in enumerate(argv) if not word.startswith("-")] return [shlex.join(argv[at:]) for at, word in enumerate(argv) if not word.startswith("-")]
@@ -1121,11 +1191,10 @@ def cmd_block() -> int:
"hookEventName": "PreToolUse", "permissionDecision": "ask", "permissionDecisionReason": reason, "hookEventName": "PreToolUse", "permissionDecision": "ask", "permissionDecisionReason": reason,
}})) }}))
return 0 return 0
cfg = watch.load_config() cfg, _ = watch.announced_config()
if cfg is None or cfg.get("queue_enforce") is not True: if cfg is None or cfg.get("queue_enforce") is not True:
return 0 return 0
gated = {**cfg, "heavy_commands": [*cfg["heavy_commands"], GATE.pattern]} if watch.heavy(raw, enforced(cfg), (THROUGH_QUEUE.pattern, *cfg["never_block"]), watch.payload_cwd(payload)):
if watch.heavy(raw, gated, (THROUGH_QUEUE.pattern, *cfg["never_block"]), watch.payload_cwd(payload)):
print( print(
f"kvb-org-queue: heavy jobs run through the queue: start `python3 {Path(__file__).resolve()} hold " f"kvb-org-queue: heavy jobs run through the queue: start `python3 {Path(__file__).resolve()} hold "
"--kind <kind> --task <register task> -- <command>` in the background; it runs the command on its grant.", "--kind <kind> --task <register task> -- <command>` in the background; it runs the command on its grant.",
@@ -1164,8 +1233,10 @@ def main() -> int:
if args.cmd == "block": if args.cmd == "block":
try: try:
return cmd_block() return cmd_block()
except (OSError, ValueError, KeyError, TypeError, AttributeError, re.error): except Exception as error: # noqa: BLE001
return 0 print(f"kvb-org-queue: the block hook failed ({error!r}), so the command counts as heavy and is refused.",
file=sys.stderr)
return 2
commands = {"hold": cmd_hold, "list": cmd_list, "history": cmd_history, "cancel": cmd_cancel} commands = {"hold": cmd_hold, "list": cmd_list, "history": cmd_history, "cancel": cmd_cancel}
try: try:
return commands[args.cmd](args) return commands[args.cmd](args)
+2 -3
View File
@@ -31,9 +31,8 @@ picks do; every other session keeps its own.
## Night rules for the active session ## Night rules for the active session
- **Grants**: request none; admit your own runs with `watch.py admit --kind - **Grants**: request none; light runs self-admit and heavy ones queue
<slot>` (`${CLAUDE_SKILL_DIR}/../kvb-org-organize/scripts/watch.py`), whose (`kvb-org-organize` Capacity), whose limits still apply.
disk, CPU and RAM limits still apply.
- **Offices**: the suspended agents' claims stay theirs. Take only free work; - **Offices**: the suspended agents' claims stay theirs. Take only free work;
a piece a suspended agent holds waits for the resume or the operator. a piece a suspended agent holds waits for the resume or the operator.
- **Messages**: no wakes and no peer questions; the Peer protocol's wait is - **Messages**: no wakes and no peer questions; the Peer protocol's wait is
+3 -4
View File
@@ -30,10 +30,9 @@ sidetree or leaves as a proposal.
- code explored through the codebase-memory-mcp graph first - code explored through the codebase-memory-mcp graph first
(`search_graph`, `trace_path`, `get_architecture`, `detect_changes`; the (`search_graph`, `trace_path`, `get_architecture`, `detect_changes`; the
`kvb-agent-optimize` skill sets it up), grep and read after; `kvb-agent-optimize` skill sets it up), grep and read after;
- light runs (no Docker, about a minute) self-admit with - light runs self-admit (`kvb-org-organize` Capacity); `make test`, gates,
`${CLAUDE_SKILL_DIR}/../kvb-org-organize/scripts/watch.py admit --kind deploys and anything with Docker only through `queue.py hold`
test-nostack`; `make test`, gates, deploys and anything with Docker only (`kvb-org-queue`);
through `queue.py hold` (`kvb-org-queue`);
- status that asks nothing goes to - status that asks nothing goes to
`${CLAUDE_SKILL_DIR}/../kvb-org-radio/scripts/radio.py post "<text>"`, `${CLAUDE_SKILL_DIR}/../kvb-org-radio/scripts/radio.py post "<text>"`,
never into a wake. never into a wake.
+3 -3
View File
@@ -58,9 +58,9 @@ and the prose diff.
- Write the script, then replace the compiled prose with its call - Write the script, then replace the compiled prose with its call
(`${CLAUDE_SKILL_DIR}/scripts/<verb>.py ...`) and one line on reading its (`${CLAUDE_SKILL_DIR}/scripts/<verb>.py ...`) and one line on reading its
output. output.
- Every script gets a unit test with a negative control shown once, run once - Every script gets a unit test with a negative control shown once, run as a
`watch.py admit --kind test-nostack` (the `kvb-org-organize` skill) prints light run (`kvb-org-organize` Capacity); `make test` runs through
`grant`; `make test` runs through `queue.py hold` (`kvb-org-queue`). `queue.py hold` (`kvb-org-queue`).
- In another repository follow its agent rules (claims, worktree, branch); a - In another repository follow its agent rules (claims, worktree, branch); a
file another line owns goes to that line through the `kvb-org-coordinate` file another line owns goes to that line through the `kvb-org-coordinate`
skill. skill.
+2 -3
View File
@@ -52,9 +52,8 @@ or diff.
- Read every script whole before carrying it; clean it the way the - Read every script whole before carrying it; clean it the way the
`kvb-code-clean` skill does. `kvb-code-clean` skill does.
- A carried script gets a unit test in the repository's `tests/` with a - A carried script gets a unit test in the repository's `tests/` with a
negative control, run once `watch.py admit --kind test-nostack` (the negative control, run as a light run (`kvb-org-organize` Capacity);
`kvb-org-organize` skill) prints `grant`; `make test` runs through `make test` runs through `queue.py hold` (`kvb-org-queue`).
`queue.py hold` (`kvb-org-queue`).
- `transcend.py mark` every offered candidate with its verdict. - `transcend.py mark` every offered candidate with its verdict.
- Leave the result uncommitted; committing is the `kvb-git-commit-next` - Leave the result uncommitted; committing is the `kvb-git-commit-next`
skill's job. skill's job.
+3 -1
View File
@@ -11,7 +11,9 @@ from pathlib import Path
REPO_ROOT = Path(__file__).resolve().parents[1] REPO_ROOT = Path(__file__).resolve().parents[1]
GATES = REPO_ROOT / "skills" / "kvb-check-gate" / "scripts" / "gates.py" GATES = REPO_ROOT / "skills" / "kvb-check-gate" / "scripts" / "gates.py"
QUEUE = REPO_ROOT / "skills" / "kvb-org-queue" / "scripts" / "queue.py" QUEUE = REPO_ROOT / "skills" / "kvb-org-queue" / "scripts" / "queue.py"
OPEN_MEMORY = {"available": -1, "swap_free": -1, "psi_full": 1000, "container_swap_gb": 1e9} OPEN_MEMORY = {
"available": -1, "swap_free": -1, "psi_some": 1000, "psi_full": 1000, "swap_out_mb_min": 1e9, "container_swap_gb": 1e9,
}
class Scratch(unittest.TestCase): class Scratch(unittest.TestCase):
+66 -4
View File
@@ -2,6 +2,7 @@
from __future__ import annotations from __future__ import annotations
import json
import os import os
import shutil import shutil
import subprocess import subprocess
@@ -10,15 +11,26 @@ import unittest
from pathlib import Path from pathlib import Path
INSTALL = Path(__file__).resolve().parents[1] / "scripts" / "install.sh" INSTALL = Path(__file__).resolve().parents[1] / "scripts" / "install.sh"
ORGANIZE = Path(__file__).resolve().parents[1] / "skills" / "kvb-org-organize"
NPX = "#!/bin/sh\nmkdir -p .agents/skills/locked\necho locked > .agents/skills/locked/SKILL.md\n" NPX = "#!/bin/sh\nmkdir -p .agents/skills/locked\necho locked > .agents/skills/locked/SKILL.md\n"
BYTECODE = ("__pycache__/radio.cpython-312.pyc", "json.pyc", "legacy.pyo") BYTECODE = ("__pycache__/radio.cpython-312.pyc", "json.pyc", "legacy.pyo")
LIVE_MEMORY = {
"warn": {"available": 15, "swap_free": 40, "psi_full": 5, "container_swap_gb": 4},
"stop": {"available": 8, "swap_free": 10, "psi_full": 10, "container_swap_gb": 16},
}
SHIPPED = ORGANIZE / "config.default.json"
DEFAULT = json.loads(SHIPPED.read_text(encoding="utf-8"))
ROW_MEMORY = DEFAULT["memory_limits"]
class TestInstall(unittest.TestCase): class TestInstall(unittest.TestCase):
def install(self, root: Path, target: Path, npx: str = NPX, planted: tuple[str, ...] = ()) -> None: def install(self, root: Path, target: Path, npx: str = NPX, planted: tuple[str, ...] = (), config: Path | None = None,
code: int = 0, stubs_extra: tuple[tuple[str, str], ...] = ()) -> subprocess.CompletedProcess:
repo, stubs = root / "repo", root / "bin" repo, stubs = root / "repo", root / "bin"
(repo / "scripts").mkdir(parents=True) (repo / "scripts").mkdir(parents=True)
shutil.copy(INSTALL, repo / "scripts" / "install.sh") shutil.copy(INSTALL, repo / "scripts" / "install.sh")
if config is not None:
shutil.copytree(ORGANIZE, repo / "skills" / ORGANIZE.name, ignore=shutil.ignore_patterns("__pycache__"))
(repo / "scripts" / "enable-plugins.js").write_text("", encoding="utf-8") (repo / "scripts" / "enable-plugins.js").write_text("", encoding="utf-8")
(repo / "skills" / "own").mkdir(parents=True) (repo / "skills" / "own").mkdir(parents=True)
(repo / "skills" / "own" / "SKILL.md").write_text("own", encoding="utf-8") (repo / "skills" / "own" / "SKILL.md").write_text("own", encoding="utf-8")
@@ -26,11 +38,61 @@ class TestInstall(unittest.TestCase):
(repo / name).parent.mkdir(parents=True, exist_ok=True) (repo / name).parent.mkdir(parents=True, exist_ok=True)
(repo / name).write_bytes(b"planted") (repo / name).write_bytes(b"planted")
stubs.mkdir() stubs.mkdir()
(stubs / "npx").write_text(npx, encoding="utf-8") for name, text in (("npx", npx), *stubs_extra):
(stubs / "npx").chmod(0o755) (stubs / name).write_text(text, encoding="utf-8")
(stubs / name).chmod(0o755)
env = {**os.environ, "PATH": f"{stubs}:/usr/bin:/bin", "TARGET": str(target), "HOME": str(root / "home")} env = {**os.environ, "PATH": f"{stubs}:/usr/bin:/bin", "TARGET": str(target), "HOME": str(root / "home")}
if config is not None:
env |= {"AGENTS_COUNCIL_CONFIG": str(config), "AGENTS_COUNCIL_ROOT": str(root / "council")}
done = subprocess.run(["bash", str(repo / "scripts" / "install.sh")], env=env, capture_output=True, text=True, check=False) done = subprocess.run(["bash", str(repo / "scripts" / "install.sh")], env=env, capture_output=True, text=True, check=False)
self.assertEqual(done.returncode, 0, done.stderr) self.assertEqual(done.returncode, code, done.stderr)
return done
def test_the_install_stops_before_any_copy_while_the_live_config_lacks_a_memory_bound_the_new_code_needs(self):
for limits, code in ((LIVE_MEMORY, 2), (ROW_MEMORY, 0)):
with self.subTest(code=code), tempfile.TemporaryDirectory() as tmp:
root = Path(tmp)
target = root / "target"
target.mkdir()
config = root / "config.json"
config.write_text(json.dumps({**DEFAULT, "memory_limits": limits}), encoding="utf-8")
before = config.stat()
done = self.install(root, target, config=config, code=code)
after = config.stat()
self.assertEqual((after.st_ino, after.st_mtime_ns, after.st_mode), (before.st_ino, before.st_mtime_ns, before.st_mode))
installed = sorted(path.name for path in target.iterdir())
if code:
self.assertIn(
"lacks memory_limits.warn.psi_some, memory_limits.warn.swap_out_mb_min, memory_limits.stop.swap_out_mb_min;",
done.stderr,
)
self.assertEqual(installed, [])
self.assertFalse((root / "repo" / ".agents").exists())
else:
self.assertEqual(installed, [".agents", ".claude"])
self.assertEqual(json.loads(config.read_text(encoding="utf-8")), {**DEFAULT, "memory_limits": limits})
def test_an_install_without_a_config_writes_the_shipped_default_as_the_initial_one(self):
with tempfile.TemporaryDirectory() as tmp:
root = Path(tmp)
target = root / "target"
target.mkdir()
config = root / "council-config" / "config.json"
self.install(root, target, config=config)
self.assertEqual(config.read_bytes(), SHIPPED.read_bytes())
self.assertEqual(config.stat().st_mode & 0o777, 0o600)
self.assertEqual(config.parent.stat().st_mode & 0o777, 0o700)
def test_a_config_written_while_the_install_creates_its_folder_survives(self):
with tempfile.TemporaryDirectory() as tmp:
root = Path(tmp)
target = root / "target"
target.mkdir()
config = root / "council-config" / "config.json"
racer = f"#!/bin/sh\nmkdir -p {config.parent}\necho '{{\"mine\": true}}' > {config}\nexec /usr/bin/install \"$@\"\n"
done = self.install(root, target, config=config, stubs_extra=(("install", racer),))
self.assertEqual(json.loads(config.read_text(encoding="utf-8")), {"mine": True})
self.assertIn(str(config), done.stderr)
def test_reinstall_empties_the_skill_directories_in_place(self): def test_reinstall_empties_the_skill_directories_in_place(self):
with tempfile.TemporaryDirectory() as tmp: with tempfile.TemporaryDirectory() as tmp:
+4 -1
View File
@@ -529,7 +529,10 @@ class TestOperator(unittest.TestCase):
def _queue_config(self, slot_need_gb: dict) -> None: def _queue_config(self, slot_need_gb: dict) -> None:
config = self.base / "config.json" config = self.base / "config.json"
open_memory = {"available": -1, "swap_free": -1, "psi_full": 1000, "container_swap_gb": 1e9} open_memory = {
"available": -1, "swap_free": -1, "psi_some": 1000, "psi_full": 1000, "swap_out_mb_min": 1e9,
"container_swap_gb": 1e9,
}
config.write_text(json.dumps({ config.write_text(json.dumps({
"disk": str(self.base), "thresholds_gb": {"warn": 0, "stop": 0, "critical": 0}, "projection_minutes": 30, "disk": str(self.base), "thresholds_gb": {"warn": 0, "stop": 0, "critical": 0}, "projection_minutes": 30,
"slot_need_gb": slot_need_gb, "heavy_commands": [r"\btouch heavy\b"], "never_block": [], "slot_need_gb": slot_need_gb, "heavy_commands": [r"\btouch heavy\b"], "never_block": [],
+121 -6
View File
@@ -23,7 +23,9 @@ HISTORY = REPO_ROOT / "skills" / "kvb-org-queue" / "scripts" / "history.py"
GATES = REPO_ROOT / "skills" / "kvb-check-gate" / "scripts" / "gates.py" GATES = REPO_ROOT / "skills" / "kvb-check-gate" / "scripts" / "gates.py"
NOW = 1_800_000_000.0 NOW = 1_800_000_000.0
HOUR = 3600 HOUR = 3600
OPEN_MEMORY = {"available": -1, "swap_free": -1, "psi_full": 1000, "container_swap_gb": 1e9} OPEN_MEMORY = {
"available": -1, "swap_free": -1, "psi_some": 1000, "psi_full": 1000, "swap_out_mb_min": 1e9, "container_swap_gb": 1e9,
}
BASE = { BASE = {
"free_gb": 100.0, "free_gb": 100.0,
"floor": 10.0, "floor": 10.0,
@@ -141,11 +143,38 @@ class TestEvaluate(unittest.TestCase):
young = {**job(1, kind="a", status="active", granted=NOW - 60), **cmd, "cmd": "a"} young = {**job(1, kind="a", status="active", granted=NOW - 60), **cmd, "cmd": "a"}
overshooting = {**job(5, kind="c", status="active", granted=NOW - 120), **cmd, "cmd": "c"} overshooting = {**job(5, kind="c", status="active", granted=NOW - 120), **cmd, "cmd": "c"}
old = {**job(4, kind="c", status="active", granted=NOW - self.queue.RAMP_SECONDS), **cmd, "cmd": "c"} old = {**job(4, kind="c", status="active", granted=NOW - self.queue.RAMP_SECONDS), **cmd, "cmd": "c"}
self.assertEqual(self.queue.pending_ram([young, overshooting, old, fresh], "j2", NOW, signatures, kinds), 30 + 10 + 25) pending = self.queue.pending_ram
self.assertEqual(pending([young, overshooting, old, fresh], "j2", NOW, signatures, kinds, 50, 0), 30 + 10 + 25)
undated = {k: v for k, v in old.items() if k != "granted"} undated = {k: v for k, v in old.items() if k != "granted"}
self.assertEqual(self.queue.pending_ram([undated, fresh], "j2", NOW, signatures, kinds), 10 + 25) self.assertEqual(pending([undated, fresh], "j2", NOW, signatures, kinds, 50, 0), 10 + 25)
unknown = {**job(3, kind="new"), **cmd, "cmd": "new"} unknown = {**job(3, kind="new"), **cmd, "cmd": "new"}
self.assertEqual(self.queue.pending_ram([unknown], "j3", NOW, signatures, kinds), 30) self.assertEqual(pending([unknown], "j3", NOW, signatures, kinds, 50, 0), 30)
self.assertEqual(pending([fresh], "j2", NOW, signatures, kinds, 50, 12), 25 + 12)
def test_a_young_job_counts_only_what_it_has_not_added_since_its_grant(self):
history = _load(HISTORY)
rows = [history_row(n, kind="b", signature="w: b", mem_before=20, mem_max=45) for n in range(5)]
signatures, kinds = history.learn(rows, NOW, {"load_limits": {"mem_percent": 90, "cpu_percent": 90}})
fresh = {**job(2, kind="b"), "workdir": "/w", "cmd": "b"}
def gate(seq: int, ram: float, granted_mem: float) -> dict:
return {**job(seq, kind="g", status="active", granted=NOW - 60), "workdir": "/w", "cmd": "g",
"footprint": {"ram": ram, "cpu": 1}, "granted_mem": granted_mem}
pending = self.queue.pending_ram
self.assertEqual(pending([gate(1, 40, 10), fresh], "j2", NOW, signatures, kinds, 10, 0), 25 + 40)
self.assertEqual(pending([gate(1, 40, 10), fresh], "j2", NOW, signatures, kinds, 45, 0), 25 + 5)
self.assertEqual(pending([gate(1, 40, 10), fresh], "j2", NOW, signatures, kinds, 60, 0), 25)
self.assertEqual(pending([gate(1, 40, 10), gate(3, 20, 30), fresh], "j2", NOW, signatures, kinds, 45, 0), 25 + 25)
undated = {k: v for k, v in gate(1, 40, 10).items() if k != "granted_mem"}
self.assertEqual(pending([undated, fresh], "j2", NOW, signatures, kinds, 60, 0), 25 + 40)
ramp = self.queue.ramping_ram
for mem in (70, 40, 20):
with self.subTest(mem=mem):
self.assertEqual(ramp([gate(1, 40, 70)], NOW, signatures, kinds, mem), 40)
self.assertEqual(ramp([gate(1, 40, 70), {**undated, "id": "j3"}], NOW, signatures, kinds, 20), 80)
self.assertEqual(self.queue.admitted_ram({"admits": [{"ram": 7, "expires": NOW + 1}, {"ram": 9, "expires": NOW}]}, NOW), 7)
self.assertEqual(self.queue.admitted_ram({"entries": []}, NOW), 0)
def test_a_signature_names_the_repository_every_worktree_shares(self): def test_a_signature_names_the_repository_every_worktree_shares(self):
with tempfile.TemporaryDirectory() as tmp: with tempfile.TemporaryDirectory() as tmp:
@@ -251,7 +280,7 @@ class TestEvaluate(unittest.TestCase):
self.assertEqual(self.queue.footprint(waiting, signatures, kinds), {"ram": 50, "cpu": 80}) self.assertEqual(self.queue.footprint(waiting, signatures, kinds), {"ram": 50, "cpu": 80})
older = {k: v for k, v in running.items() if k != "footprint"} older = {k: v for k, v in running.items() if k != "footprint"}
self.assertEqual(self.queue.footprint(older, signatures, kinds), {"ram": 50, "cpu": 80}) self.assertEqual(self.queue.footprint(older, signatures, kinds), {"ram": 50, "cpu": 80})
self.assertEqual(self.queue.pending_ram([running], "j1", NOW, signatures, kinds), 12 + 12) self.assertEqual(self.queue.pending_ram([running], "j1", NOW, signatures, kinds, 50, 0), 12 + 12)
def test_an_active_entry_an_older_queue_granted_never_books_less_than_that_queue_did(self): def test_an_active_entry_an_older_queue_granted_never_books_less_than_that_queue_did(self):
history = _load(HISTORY) history = _load(HISTORY)
@@ -754,6 +783,7 @@ class TestQueueCommands(unittest.TestCase):
) )
signature = f"{self.base.name}: {Path(sys.executable).name} -c" signature = f"{self.base.name}: {Path(sys.executable).name} -c"
self.assertEqual((entry["aliases"], entry["footprint"]), ([signature], None)) self.assertEqual((entry["aliases"], entry["footprint"]), ([signature], None))
self.assertTrue(0 < entry["granted_mem"] < 100, entry)
self.assertEqual(self.entries(), []) self.assertEqual(self.entries(), [])
[row] = [json.loads(line) for line in self.history_file.read_text(encoding="utf-8").splitlines()] [row] = [json.loads(line) for line in self.history_file.read_text(encoding="utf-8").splitlines()]
self.assertEqual( self.assertEqual(
@@ -1111,6 +1141,52 @@ class TestQueueCommands(unittest.TestCase):
self.write_config() self.write_config()
self.assertEqual(self.bash("make compose-deploy"), 0) self.assertEqual(self.bash("make compose-deploy"), 0)
def test_list_and_hold_on_a_broken_config_warn_and_use_the_shipped_default(self):
self.config.write_text(json.dumps(self.cfg)[:-1] + ",}", encoding="utf-8")
listing = self.run_queue("list")
self.assertEqual(listing.returncode, 0, listing.stderr)
self.assertIn("built-in defaults", listing.stderr)
held = self.run_queue("hold", "--kind", "nosuch", "--task", "work/a", "--session", "s1", "--", "true")
self.assertEqual(held.returncode, 2, held.stderr)
self.assertIn("built-in defaults", held.stderr)
self.assertIn("'nosuch' is not in slot_need_gb", held.stderr)
for done in (listing, held):
self.assertNotIn("Traceback", done.stderr)
def test_a_broken_config_falls_back_to_defaults_so_light_commands_pass_and_heavy_ones_stay_refused(self):
self.cfg["queue_enforce"] = True
good = json.dumps(self.cfg)
def text(content: str) -> Callable[[], None]:
return lambda: self.config.write_text(content, encoding="utf-8")
shapes = {
"comma": text(good[:-1] + ",}"), "empty": text(""), "list": text("[]"),
"string enforce": text(json.dumps({**self.cfg, "queue_enforce": "yes"})),
"empty-matching heavy": text(json.dumps({**self.cfg, "heavy_commands": [""]})),
"dot heavy": text(json.dumps({**self.cfg, "heavy_commands": ["."]})),
"word-boundary heavy": text(json.dumps({**self.cfg, "heavy_commands": [r"\b"]})),
"dot never_block": text(json.dumps({**self.cfg, "never_block": ["."]})),
"dangling link": lambda: self.config.symlink_to(self.base / "absent.json"),
"link loop": lambda: self.config.symlink_to(self.config),
"directory": lambda: self.config.mkdir(),
}
for name, make in shapes.items():
with self.subTest(config=name):
if self.config.is_dir() and not self.config.is_symlink():
self.config.rmdir()
self.config.unlink(missing_ok=True)
make()
passed = self.hook(json.dumps({"tool_input": {"command": "ls -la"}}))
self.assertEqual(passed.returncode, 0, passed.stderr)
self.assertIn(str(self.config), passed.stderr)
self.assertIn("built-in defaults", passed.stderr)
self.assertEqual(self.bash("make test"), 2)
live = ("make compose-deploy ROLE=x", "docker build .", "docker pull nginx", "make swarm-deploy", "make roundtrip",
"make worktree-up")
for command in live if name in ("comma", "directory") else live[:1]:
self.assertEqual(self.bash(command), 2, command)
def test_block_with_queue_enforce_refuses_heavy_commands_outside_the_queue(self): def test_block_with_queue_enforce_refuses_heavy_commands_outside_the_queue(self):
self.cfg["queue_enforce"] = True self.cfg["queue_enforce"] = True
self.write_config() self.write_config()
@@ -1129,7 +1205,12 @@ class TestQueueCommands(unittest.TestCase):
): ):
with self.subTest(command=command): with self.subTest(command=command):
self.assertEqual(self.bash(command), 0) self.assertEqual(self.bash(command), 0)
self.assertEqual(self.hook("not json").returncode, 0) for payload in ("not json", json.dumps({"tool_input": ["make test"]})):
with self.subTest(payload=payload):
crashed = self.hook(payload)
self.assertEqual(crashed.returncode, 2, crashed.stderr)
self.assertIn("counts as heavy", crashed.stderr)
self.assertEqual(self.bash("CARGO_BUILD_JOBS=² cargo build ; make test"), 2)
self.config.unlink() self.config.unlink()
self.assertEqual(self.bash("make compose-deploy"), 0) self.assertEqual(self.bash("make compose-deploy"), 0)
@@ -1290,6 +1371,40 @@ class TestQueueCommands(unittest.TestCase):
self.assertEqual(self.bash(command), 2) self.assertEqual(self.bash(command), 2)
self.assertEqual(self.bash("docker compose ps"), 0) self.assertEqual(self.bash("docker compose ps"), 0)
def test_a_queue_grant_counts_a_live_admit_record_and_ignores_an_expired_one(self):
def admitted(expires: float) -> None:
(self.root / "queue.json").write_text(
json.dumps({"seq": 0, "entries": [], "admits": [{"ram": 300, "expires": expires}]}), encoding="utf-8"
)
admitted(time.time() + 300)
_, err = self.blocked_hold("true")
self.assertRegex(err.read_text(encoding="utf-8"), r"RAM \d+ % used \+ 300 % still to come reach the limit 101 %")
admitted(time.time() - 1)
result = self.run_queue("hold", "--kind", "deploy", "--task", "work/a", "--session", "s2", "--", "true")
self.assertEqual((result.returncode, result.stderr), (0, ""))
def test_a_light_kind_refuses_a_heavy_command_names_the_test_kind_and_enqueues_nothing(self):
self.cfg["slot_need_gb"] |= {"test-nostack": 0, "quick": 0}
self.cfg["light_kinds"] = ["quick"]
self.write_config()
(self.base / "tests" / "e2e").mkdir(parents=True)
(self.base / "tests" / "e2e" / "test_orchestrator_e2e.py").write_text("", encoding="utf-8")
for kind, command, shown in (
("test-nostack", ("python", "-m", "unittest"), "python -m unittest"),
("test-nostack", ("pytest", "tests/e2e/test_orchestrator_e2e.py"), "pytest tests/e2e/test_orchestrator_e2e.py"),
("quick", ("bash", "-c", "make test"), "bash -c 'make test'"),
("quick", ("python3", "gate.py", "run", "--target", "test-unit"), "python3 gate.py run --target test-unit"),
):
with self.subTest(kind=kind, command=command):
result = self.run_queue("hold", "--kind", kind, "--task", "work/a", "--session", "s1", "--", *command)
self.assertEqual(result.returncode, 2)
self.assertIn(f"the light kind {kind} skips every reservation", result.stderr)
self.assertIn(f"run `queue.py hold --kind test --task <register task> -- {shown}`", result.stderr)
self.assertFalse((self.root / "queue.json").exists())
light = self.run_queue("hold", "--kind", "test-nostack", "--task", "work/a", "--session", "s1", "--", "true")
self.assertEqual((light.returncode, light.stderr), (0, ""))
def test_hold_refuses_a_config_without_the_budget_or_the_memory_limits(self): def test_hold_refuses_a_config_without_the_budget_or_the_memory_limits(self):
for key, cfg in ( for key, cfg in (
("budget_percent", {k: v for k, v in self.cfg.items() if k != "budget_percent"}), ("budget_percent", {k: v for k, v in self.cfg.items() if k != "budget_percent"}),
+49 -1
View File
@@ -149,6 +149,53 @@ class TestCpu(unittest.TestCase):
self.assertEqual(json.loads((state / "cpu.json").read_text(encoding="utf-8")), [NOW + sampler.BUCKET + 1, 9300, 12200]) self.assertEqual(json.loads((state / "cpu.json").read_text(encoding="utf-8")), [NOW + sampler.BUCKET + 1, 9300, 12200])
class TestPressure(unittest.TestCase):
def setUp(self):
tmp = tempfile.TemporaryDirectory()
self.addCleanup(tmp.cleanup)
self.proc, self.state = Path(tmp.name) / "proc", Path(tmp.name) / "state"
(self.proc / "pressure").mkdir(parents=True)
self.state.mkdir()
def vmstat(self, pages: int) -> None:
(self.proc / "vmstat").write_text(f"pgpgin 5\npswpin 3\npswpout {pages}\n", encoding="utf-8")
def test_psi_gives_some_and_full_avg10_and_none_without_pressure(self):
text = "some avg10=12.50 avg60=1.00 avg300=0.50 total=1\nfull avg10=6.25 avg60=0.80 avg300=0.10 total=1\n"
self.assertEqual((sampler.psi_avg10(text, "some"), sampler.psi_avg10(text, "full")), (12.5, 6.25))
self.assertEqual((sampler.psi_avg10("", "some"), sampler.psi_avg10(text.splitlines()[0], "full")), (None, None))
self.assertIsNone(sampler.swapped_pages("pgpgin 5\n"))
def test_swap_out_is_none_first_then_mb_per_minute_and_none_after_a_reset_or_a_gap(self):
with mock.patch.object(sampler, "PAGE_BYTES", 4096):
self.assertIsNone(sampler.swap_out_since(self.proc, self.state, NOW))
self.assertFalse((self.state / "swap.json").exists())
self.vmstat(1000)
self.assertIsNone(sampler.swap_out_since(self.proc, self.state, NOW))
self.vmstat(1000 + 1_400_000_000 // 4096)
self.assertEqual(sampler.swap_out_since(self.proc, self.state, NOW + 60), 1400.0)
self.vmstat(10)
self.assertIsNone(sampler.swap_out_since(self.proc, self.state, NOW + 120))
self.assertEqual(json.loads((self.state / "swap.json").read_text(encoding="utf-8")), [NOW + 120, 10])
self.assertIsNone(sampler.swap_out_since(self.proc, self.state, NOW + 120))
self.assertIsNone(sampler.swap_out_since(self.proc, self.state, NOW + 120 + sampler.BUCKET + 1))
def test_a_sample_row_adds_the_pressure_keys_and_keeps_every_other_key(self):
(self.proc / "pressure" / "memory").write_text("some avg10=3.00 avg60=0 avg300=0 total=1\n", encoding="utf-8")
found = sampler.sample(self.proc, self.proc / "sys", self.proc, lambda argv: None, self.state)
self.assertEqual(sorted(found), ["cpu", "disks", "gpus", "load", "psi_full", "psi_some", "queue", "ram", "swap", "swap_out_mb_min", "t"])
self.assertEqual((found["psi_some"], found["psi_full"], found["swap_out_mb_min"]), (3.0, None, None))
def test_the_fifteen_minute_mean_keeps_the_pressure_keys(self):
asleep = {"name": ASLEEP, "asleep": True}
rows = [
row(NOW + 60, 1.0, 1, 0, asleep) | {"psi_some": 2.0, "psi_full": 1.004, "swap_out_mb_min": 100.0},
row(NOW + 120, 1.0, 1, 0, asleep) | {"psi_some": 4.5, "psi_full": None, "swap_out_mb_min": 301.0},
]
mean = sampler.mean_row(NOW, rows)
self.assertEqual((mean["psi_some"], mean["psi_full"], mean["swap_out_mb_min"]), (3.25, 1.0, 200.5))
class TestGpus(unittest.TestCase): class TestGpus(unittest.TestCase):
def setUp(self): def setUp(self):
tmp = tempfile.TemporaryDirectory() tmp = tempfile.TemporaryDirectory()
@@ -206,6 +253,7 @@ class TestRetention(unittest.TestCase):
self.assertIs(kept[0], mean) self.assertIs(kept[0], mean)
self.assertEqual(kept[1], { self.assertEqual(kept[1], {
"t": bucket, "disks": [{"mount": "/", "used": 200, "total": 1000}, {"mount": "/mnt", "used": 10, "total": 20}], "cpu": 30.0, "ram": 15.0, "swap": 0.0, "load": 3.0, "t": bucket, "disks": [{"mount": "/", "used": 200, "total": 1000}, {"mount": "/mnt", "used": 10, "total": 20}], "cpu": 30.0, "ram": 15.0, "swap": 0.0, "load": 3.0,
"psi_some": None, "psi_full": None, "swap_out_mb_min": None,
"gpus": [{"name": AWAKE, "util": 50.0, "vram_used": 300, "vram_total": 1000}], "queue": {"waiting": 1.0, "active": 1.0}, "gpus": [{"name": AWAKE, "util": 50.0, "vram_used": 300, "vram_total": 1000}], "queue": {"waiting": 1.0, "active": 1.0},
}) })
self.assertEqual((kept[2]["cpu"], kept[2]["disks"]), (40.0, [{"mount": "/", "used": 400, "total": 1000}])) self.assertEqual((kept[2]["cpu"], kept[2]["disks"]), (40.0, [{"mount": "/", "used": 400, "total": 1000}]))
@@ -274,7 +322,7 @@ class TestSamplerInScratchHome(unittest.TestCase):
self.assertEqual(len(rows), 2) self.assertEqual(len(rows), 2)
for found in rows: for found in rows:
with self.subTest(t=found["t"]): with self.subTest(t=found["t"]):
self.assertEqual(sorted(found), ["cpu", "disks", "gpus", "load", "queue", "ram", "swap", "t"]) self.assertEqual(sorted(found), ["cpu", "disks", "gpus", "load", "psi_full", "psi_some", "queue", "ram", "swap", "swap_out_mb_min", "t"])
self.assertEqual(found["disks"], DISKS) self.assertEqual(found["disks"], DISKS)
self.assertEqual(found["gpus"], [{"name": AWAKE, "util": 37.0, "vram_used": 2048 * MIB, "vram_total": 8188 * MIB}, {"name": ASLEEP, "asleep": True}]) self.assertEqual(found["gpus"], [{"name": AWAKE, "util": 37.0, "vram_used": 2048 * MIB, "vram_total": 8188 * MIB}, {"name": ASLEEP, "asleep": True}])
self.assertEqual(found["queue"], {"waiting": 1, "active": 2}) self.assertEqual(found["queue"], {"waiting": 1, "active": 2})
+943 -19
View File
File diff suppressed because it is too large. Load diff