feat(kvb-org-organize): hold heavy jobs back on memory pressure with one shipped default config
watch.py admits heavy work only while PSI, swap-out rate and RAM forecast stay under the configured limits; config.default.json is the single default the install writes once (mode 600, no clobber) and every hook, admit and queue command falls back to. Browser runners and package-manager script forms count as heavy, preflight derives its probes from the light and heavy tool lists, and the shipped default carries no prune command, buildx path or host-specific key. Tests run once on unify/skills-20261114 after the merge (operator order 2026-11-14). 🎯 Fix: 85 % 💥 Break: 8 % 💸 Tokens: 99 % ⏱️ Velocity: – 🛡️ Security: 100 % 🏆 Score: 79 🚦 Gate: ⛔ fix 85 % < 90, break 8 % > 5 Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
a2a03995cb
commit
8e2db852e5
27 files changed
+2795
-207
No files matched your search
@@ -2,12 +2,17 @@
|
|||||||
"""Args:
|
"""Args:
|
||||||
sample: append one row to <state>/samples.jsonl: {t, disks: [{mount,
|
sample: append one row to <state>/samples.jsonl: {t, disks: [{mount,
|
||||||
used, total}], cpu, ram, swap, load, gpus: [{name, util, vram_used,
|
used, total}], cpu, ram, swap, load, gpus: [{name, util, vram_used,
|
||||||
vram_total} | {name, asleep: true}], queue: {waiting, active}}; t in
|
vram_total} | {name, asleep: true}], queue: {waiting, active},
|
||||||
epoch seconds, used, total and vram in bytes, cpu, ram, swap and util
|
psi_some, psi_full, swap_out_mb_min}; t in epoch seconds, used, total
|
||||||
in percent, load the 1-minute load average, a GPU named by its PCI
|
and vram in bytes, cpu, ram, swap and util in percent, load the
|
||||||
address. cpu is the busy share of the /proc/stat ticks since those
|
1-minute load average, a GPU named by its PCI address. cpu is the
|
||||||
the run before kept in <state>/cpu.json as [t, idle, total], None on
|
busy share of the /proc/stat ticks since those the run before kept in
|
||||||
the first run and after a gap of more than 15 minutes. An NVIDIA GPU
|
<state>/cpu.json as [t, idle, total], None on the first run and after
|
||||||
|
a gap of more than 15 minutes. psi_some and psi_full are the avg10 of
|
||||||
|
/proc/pressure/memory, None without PSI; swap_out_mb_min the MB per
|
||||||
|
minute swapped out (pswpout of /proc/vmstat) since the count the run
|
||||||
|
before kept in <state>/swap.json as [t, pswpout], None like cpu and
|
||||||
|
after a counter reset. An NVIDIA GPU
|
||||||
whose PCI runtime_status is not active is recorded asleep, and
|
whose PCI runtime_status is not active is recorded asleep, and
|
||||||
nvidia-smi is not run for it; an awake one nvidia-smi gives no
|
nvidia-smi is not run for it; an awake one nvidia-smi gives no
|
||||||
reading for (no driver, a driver mismatch) keeps its name with util
|
reading for (no driver, a driver mismatch) keeps its name with util
|
||||||
@@ -75,12 +80,15 @@ NVIDIA_QUERY = "pci.bus_id,utilization.gpu,memory.used,memory.total"
|
|||||||
UNREAD = {"util": None, "vram_used": None, "vram_total": None}
|
UNREAD = {"util": None, "vram_used": None, "vram_total": None}
|
||||||
COUNTS = (("input", "input_tokens"), ("output", "output_tokens"), ("cache_read", "cache_read_input_tokens"), ("cache_write", "cache_creation_input_tokens"))
|
COUNTS = (("input", "input_tokens"), ("output", "output_tokens"), ("cache_read", "cache_read_input_tokens"), ("cache_write", "cache_creation_input_tokens"))
|
||||||
NAMES = tuple(name for name, _ in COUNTS)
|
NAMES = tuple(name for name, _ in COUNTS)
|
||||||
HOST_DIGITS = (("cpu", 1), ("ram", 1), ("swap", 1), ("load", 2))
|
HOST_DIGITS = (("cpu", 1), ("ram", 1), ("swap", 1), ("load", 2), ("psi_some", 2), ("psi_full", 2), ("swap_out_mb_min", 1))
|
||||||
CLAIMS = ("*/claim.json", "*/*/claim.json")
|
CLAIMS = ("*/claim.json", "*/*/claim.json")
|
||||||
SAMPLES = "samples.jsonl"
|
SAMPLES = "samples.jsonl"
|
||||||
TOKENS = "tokens.jsonl"
|
TOKENS = "tokens.jsonl"
|
||||||
OFFSETS = "offsets.json"
|
OFFSETS = "offsets.json"
|
||||||
TICKS = "cpu.json"
|
TICKS = "cpu.json"
|
||||||
|
SWAPPED = "swap.json"
|
||||||
|
PAGE_BYTES = os.sysconf("SC_PAGE_SIZE")
|
||||||
|
MB = 1_000_000
|
||||||
|
|
||||||
|
|
||||||
def read_text(path: Path) -> str:
|
def read_text(path: Path) -> str:
|
||||||
@@ -263,9 +271,43 @@ def cpu_since(proc: Path, state: Path, t: int) -> float | None:
|
|||||||
return cpu_percent(before, ticks)
|
return cpu_percent(before, ticks)
|
||||||
|
|
||||||
|
|
||||||
|
def psi_avg10(text: str, kind: str) -> float | None:
|
||||||
|
for line in text.splitlines():
|
||||||
|
fields = line.split()
|
||||||
|
if fields[:1] == [kind]:
|
||||||
|
values = dict(field.split("=", 1) for field in fields[1:] if "=" in field)
|
||||||
|
return number(values.get("avg10", ""))
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def swapped_pages(text: str) -> int | None:
|
||||||
|
for line in text.splitlines():
|
||||||
|
fields = line.split()
|
||||||
|
if len(fields) == 2 and fields[0] == "pswpout" and fields[1].isdigit():
|
||||||
|
return int(fields[1])
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def swap_out_since(proc: Path, state: Path, t: int) -> float | None:
|
||||||
|
"""Returns:
|
||||||
|
The MB per minute swapped out since the pswpout count the run before
|
||||||
|
kept in <state>/swap.json at most BUCKET seconds before t, else None,
|
||||||
|
also after a counter reset; this run's count replaces it.
|
||||||
|
"""
|
||||||
|
pages = swapped_pages(read_text(proc / "vmstat"))
|
||||||
|
if pages is None:
|
||||||
|
return None
|
||||||
|
stored = read_json(state / SWAPPED)
|
||||||
|
replace(state / SWAPPED, json.dumps([t, pages]))
|
||||||
|
if not integers(stored, 2) or not t - BUCKET <= stored[0] < t or stored[1] > pages:
|
||||||
|
return None
|
||||||
|
return round((pages - stored[1]) * PAGE_BYTES / MB * 60 / (t - stored[0]), 1)
|
||||||
|
|
||||||
|
|
||||||
def sample(proc: Path, sys_root: Path, council: Path, run: Callable[[list[str]], str | None], state: Path) -> dict:
|
def sample(proc: Path, sys_root: Path, council: Path, run: Callable[[list[str]], str | None], state: Path) -> dict:
|
||||||
t = int(time.time())
|
t = int(time.time())
|
||||||
ram, swap = memory(parse_meminfo(read_text(proc / "meminfo")))
|
ram, swap = memory(parse_meminfo(read_text(proc / "meminfo")))
|
||||||
|
pressure = read_text(proc / "pressure" / "memory")
|
||||||
return {
|
return {
|
||||||
"t": t,
|
"t": t,
|
||||||
"disks": parse_df(run(["df", "-P", "-k", "-l"]) or ""),
|
"disks": parse_df(run(["df", "-P", "-k", "-l"]) or ""),
|
||||||
@@ -275,6 +317,9 @@ def sample(proc: Path, sys_root: Path, council: Path, run: Callable[[list[str]],
|
|||||||
"load": parse_loadavg(read_text(proc / "loadavg")),
|
"load": parse_loadavg(read_text(proc / "loadavg")),
|
||||||
"gpus": gpus(sys_root, run),
|
"gpus": gpus(sys_root, run),
|
||||||
"queue": queue(council),
|
"queue": queue(council),
|
||||||
|
"psi_some": psi_avg10(pressure, "some"),
|
||||||
|
"psi_full": psi_avg10(pressure, "full"),
|
||||||
|
"swap_out_mb_min": swap_out_since(proc, state, t),
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -23,6 +23,12 @@ if [[ ! -d "${TARGET}" ]]; then
|
|||||||
exit 2
|
exit 2
|
||||||
fi
|
fi
|
||||||
|
|
||||||
|
watch="${REPO_ROOT}/skills/kvb-org-organize/scripts/watch.py"
|
||||||
|
if [[ -f "${watch}" ]] && ! python3 -I "${watch}" preflight; then
|
||||||
|
warn "skills: add the keys above to the agents council config first; the new hooks and queue.py refuse to run without them"
|
||||||
|
exit 2
|
||||||
|
fi
|
||||||
|
|
||||||
cd "${REPO_ROOT}"
|
cd "${REPO_ROOT}"
|
||||||
src="${REPO_ROOT}/.agents/skills"
|
src="${REPO_ROOT}/.agents/skills"
|
||||||
rm -rf "${src}"
|
rm -rf "${src}"
|
||||||
@@ -58,6 +64,17 @@ for root in .agents/skills .claude/skills; do
|
|||||||
fi
|
fi
|
||||||
done
|
done
|
||||||
|
|
||||||
|
config="${AGENTS_COUNCIL_CONFIG:-${HOME}/.claude/agents-council/config.json}"
|
||||||
|
shipped="${REPO_ROOT}/skills/kvb-org-organize/config.default.json"
|
||||||
|
if [[ -f "${shipped}" && ! -e "${config}" && ! -L "${config}" ]]; then
|
||||||
|
[[ -d "$(dirname -- "${config}")" ]] || install -d -m 0700 "$(dirname -- "${config}")"
|
||||||
|
if (umask 077 && set -o noclobber && cat -- "${shipped}" >"${config}") 2>/dev/null; then
|
||||||
|
log "skills: wrote the shipped agents council config to ${config}"
|
||||||
|
else
|
||||||
|
warn "skills: kept ${config}, which appeared while the install wrote the shipped default"
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
|
||||||
settings="${TARGET}/.claude/settings.local.json"
|
settings="${TARGET}/.claude/settings.local.json"
|
||||||
if command -v node &>/dev/null; then
|
if command -v node &>/dev/null; then
|
||||||
log "skills: auto-enabling caveman + ponytail plugins in ${settings}"
|
log "skills: auto-enabling caveman + ponytail plugins in ${settings}"
|
||||||
|
|||||||
@@ -7,9 +7,8 @@ description: >
|
|||||||
|
|
||||||
Where an Ansible run spends its time: its log, a controller profile, a bench.
|
Where an Ansible run spends its time: its log, a controller profile, a bench.
|
||||||
Report the script's lines with their log and range, never a guess. The
|
Report the script's lines with their log and range, never a guess. The
|
||||||
script and the bench are light runs (`watch.py admit --kind test-nostack` of
|
script, the bench and a deploy to profile run through the `kvb-org-queue`
|
||||||
`kvb-org-organize`); a deploy to profile is heavy and goes through the
|
skill.
|
||||||
`kvb-org-queue` skill.
|
|
||||||
|
|
||||||
## 1. Pick the log
|
## 1. Pick the log
|
||||||
|
|
||||||
|
|||||||
@@ -60,12 +60,11 @@ It also measures wall time for `kvb-check-speed`.
|
|||||||
- **Command**: the targeted tests of the change or a CLI call of the
|
- **Command**: the targeted tests of the change or a CLI call of the
|
||||||
changed code, runnable on a clean checkout: the base worktree holds no
|
changed code, runnable on a clean checkout: the base worktree holds no
|
||||||
untracked build output, virtualenv or dependency directory.
|
untracked build output, virtualenv or dependency directory.
|
||||||
- **Heavy commands** (Docker, deploys, whole suites) run only inside
|
- **Every run** goes inside
|
||||||
`${CLAUDE_SKILL_DIR}/../kvb-org-queue/scripts/queue.py hold --kind <kind>
|
`${CLAUDE_SKILL_DIR}/../kvb-org-queue/scripts/queue.py hold --kind <kind>
|
||||||
--task <register task> -- python3 <measure.py> ...`, in the background,
|
--task <register task> -- python3 <measure.py> ...`, in the background,
|
||||||
by the `kvb-org-queue` skill; a cancelled wait there also exits 3, but
|
by the `kvb-org-queue` skill; a cancelled wait there also exits 3, but
|
||||||
prints no 💸 line. A light one self-admits with
|
prints no 💸 line.
|
||||||
`watch.py admit --kind test-nostack` of `kvb-org-organize`.
|
|
||||||
- Report the run you made; never rerun to pick the best result.
|
- Report the run you made; never rerun to pick the best result.
|
||||||
|
|
||||||
## Exit codes
|
## Exit codes
|
||||||
|
|||||||
@@ -32,10 +32,9 @@ commit starts from zero.
|
|||||||
`missing hotfix` reason means back to step 1.
|
`missing hotfix` reason means back to step 1.
|
||||||
4. **Admit**: every gate target is heavy and runs only inside
|
4. **Admit**: every gate target is heavy and runs only inside
|
||||||
`${CLAUDE_SKILL_DIR}/../kvb-org-queue/scripts/queue.py hold`.
|
`${CLAUDE_SKILL_DIR}/../kvb-org-queue/scripts/queue.py hold`.
|
||||||
Only a single test module
|
Only a single test module self-admits (`kvb-org-organize` Capacity), and
|
||||||
self-admits (`${CLAUDE_SKILL_DIR}/../kvb-org-organize/scripts/watch.py
|
it never turns a gate target green. A hotfix gate adds `--hotfix <slug>`
|
||||||
admit --kind test-nostack`), and it never turns a gate target green. A
|
and takes the next free gate slot.
|
||||||
hotfix gate adds `--hotfix <slug>` and takes the next free gate slot.
|
|
||||||
5. **Run** the missing and red targets, tracked tree clean, in the
|
5. **Run** the missing and red targets, tracked tree clean, in the
|
||||||
background; give the operator `tail -f` on the log:
|
background; give the operator `tail -f` on the log:
|
||||||
|
|
||||||
|
|||||||
@@ -15,11 +15,8 @@ skill's Rules apply here too.
|
|||||||
first error.
|
first error.
|
||||||
3. **Fix at the root**: apply the real fix for each proven root cause in the
|
3. **Fix at the root**: apply the real fix for each proven root cause in the
|
||||||
sidetree, with a test and a negative control as the `kvb-org-swarm` skill
|
sidetree, with a test and a negative control as the `kvb-org-swarm` skill
|
||||||
demands. Light runs (no Docker, about a minute: one test file or
|
demands. Light runs self-admit (`kvb-org-organize` Capacity); `make test`,
|
||||||
module, ruff) self-admit with
|
gates, deploys and anything with Docker run in the background through
|
||||||
`${CLAUDE_SKILL_DIR}/../kvb-org-organize/scripts/watch.py admit --kind
|
|
||||||
test-nostack`; `make test`, gates, deploys and anything with Docker run in
|
|
||||||
the background through
|
|
||||||
`${CLAUDE_SKILL_DIR}/../kvb-org-queue/scripts/queue.py hold --kind <kind>
|
`${CLAUDE_SKILL_DIR}/../kvb-org-queue/scripts/queue.py hold --kind <kind>
|
||||||
--task <register task> -- <command>`, which waits for the rule-based
|
--task <register task> -- <command>`, which waits for the rule-based
|
||||||
grant and releases on exit. The fix enters the main line only
|
grant and releases on exit. The fix enters the main line only
|
||||||
|
|||||||
@@ -26,9 +26,8 @@ cover the change set run now and give the green baseline; a suite already red
|
|||||||
is reported, not refactored around. No coverage: read `net.md` next to this
|
is reported, not refactored around. No coverage: read `net.md` next to this
|
||||||
file before editing.
|
file before editing.
|
||||||
|
|
||||||
Light runs (one test file or module, ruff) start once `watch.py
|
Light runs start once `watch.py admit` of the `kvb-org-organize` skill prints
|
||||||
admit --kind test-nostack` of the `kvb-org-organize` skill prints `grant`; no
|
`grant`; no chair request.
|
||||||
chair request.
|
|
||||||
|
|
||||||
## Step 3: refactor each file
|
## Step 3: refactor each file
|
||||||
|
|
||||||
|
|||||||
@@ -38,11 +38,8 @@ any listed name. Never silently truncate.
|
|||||||
## Step 3: run
|
## Step 3: run
|
||||||
|
|
||||||
Before each run, admit it under the `kvb-org-organize` Capacity rules: a
|
Before each run, admit it under the `kvb-org-organize` Capacity rules: a
|
||||||
light run (no Docker, about a minute: one test file or module, ruff)
|
light run self-admits; `make test`, quality gates and anything with Docker
|
||||||
self-admits with
|
run in the background through
|
||||||
`${CLAUDE_SKILL_DIR}/../kvb-org-organize/scripts/watch.py admit --kind
|
|
||||||
test-nostack`; `make test`, quality gates and anything with Docker run in the
|
|
||||||
background through
|
|
||||||
`${CLAUDE_SKILL_DIR}/../kvb-org-queue/scripts/queue.py hold --kind <kind>
|
`${CLAUDE_SKILL_DIR}/../kvb-org-queue/scripts/queue.py hold --kind <kind>
|
||||||
--task <register task> -- make <target>`, which waits for the rule-based
|
--task <register task> -- make <target>`, which waits for the rule-based
|
||||||
grant and releases on exit.
|
grant and releases on exit.
|
||||||
|
|||||||
@@ -49,9 +49,8 @@ single-select `Write it` / `Revise it` / `Discard` with the drafted diff.
|
|||||||
owner through the `kvb-org-coordinate` skill.
|
owner through the `kvb-org-coordinate` skill.
|
||||||
- Every carried test fails without the code it guards: show the negative
|
- Every carried test fails without the code it guards: show the negative
|
||||||
control once.
|
control once.
|
||||||
- Run the carried tests as a light run (no Docker, about a minute) after
|
- Run the carried tests as a light run (`kvb-org-organize` Capacity);
|
||||||
`${CLAUDE_SKILL_DIR}/../kvb-org-organize/scripts/watch.py admit --kind
|
`make test`, gates and anything with Docker only through
|
||||||
test-nostack`; `make test`, gates and anything with Docker only through
|
|
||||||
`queue.py hold` (the `kvb-org-queue` skill). Leave the result
|
`queue.py hold` (the `kvb-org-queue` skill). Leave the result
|
||||||
uncommitted for the `kvb-git-commit-next` skill.
|
uncommitted for the `kvb-git-commit-next` skill.
|
||||||
- `transcend.py mark` every offered candidate with its verdict.
|
- `transcend.py mark` every offered candidate with its verdict.
|
||||||
|
|||||||
@@ -71,10 +71,8 @@ order:
|
|||||||
Show the full draft, then `AskUserQuestion`: `Write it` / `Revise it` /
|
Show the full draft, then `AskUserQuestion`: `Write it` / `Revise it` /
|
||||||
`Discard`. Revise loops back to the step the correction touches. Write only
|
`Discard`. Revise loops back to the step the correction touches. Write only
|
||||||
on `Write it`; lint the file with the project's markdown linter, a light run
|
on `Write it`; lint the file with the project's markdown linter, a light run
|
||||||
that self-admits with
|
(`kvb-org-organize` Capacity); a lint through `make` or Docker runs through
|
||||||
`${CLAUDE_SKILL_DIR}/../kvb-org-organize/scripts/watch.py admit --kind
|
`queue.py hold` (the `kvb-org-queue` skill).
|
||||||
test-nostack`; a lint through `make` or Docker runs through `queue.py hold`
|
|
||||||
(the `kvb-org-queue` skill).
|
|
||||||
|
|
||||||
## Step 6: hand over
|
## Step 6: hand over
|
||||||
|
|
||||||
|
|||||||
@@ -35,8 +35,8 @@ request, so a red one blocks nothing else.
|
|||||||
(`search_graph`, `trace_path`, `get_architecture`, `detect_changes`; the
|
(`search_graph`, `trace_path`, `get_architecture`, `detect_changes`; the
|
||||||
`kvb-agent-optimize` skill sets it up), grep and read after. Parallel
|
`kvb-agent-optimize` skill sets it up), grep and read after. Parallel
|
||||||
work, gates, deploys and anything with Docker run through `queue.py hold`
|
work, gates, deploys and anything with Docker run through `queue.py hold`
|
||||||
(`kvb-org-queue`); light runs (no Docker, about a minute) self-admit with
|
(`kvb-org-queue`); light runs self-admit (`kvb-org-organize` Capacity).
|
||||||
`watch.py admit --kind test-nostack`. With the first change, swap the label: `gh issue
|
With the first change, swap the label: `gh issue
|
||||||
edit <n> --remove-label "in analysis" --add-label "in progress"`.
|
edit <n> --remove-label "in analysis" --add-label "in progress"`.
|
||||||
5. **Open the pull request** with the `kvb-git-pr-create` skill, closing
|
5. **Open the pull request** with the `kvb-git-pr-create` skill, closing
|
||||||
exactly its issue; one for a failing default branch is posted with
|
exactly its issue; one for a failing default branch is posted with
|
||||||
|
|||||||
@@ -37,8 +37,13 @@ ends or goes stale, or after a restart.
|
|||||||
|
|
||||||
## Capacity
|
## Capacity
|
||||||
|
|
||||||
- Light runs (no Docker, about a minute: one test file or module, ruff)
|
- Light runs self-admit: `watch.py admit --kind test-nostack -- <command>`,
|
||||||
self-admit: `watch.py admit --kind test-nostack`, start on `grant`.
|
start on `grant`. Light is only ruff, shellcheck, markdownlint, lizard, jq,
|
||||||
|
py_compile, cargo fmt, `env CARGO_BUILD_JOBS=4 cargo build|check|clippy|test`,
|
||||||
|
or one test file or module outside `e2e/` that reaches no Playwright;
|
||||||
|
anything else: `queue.py hold --kind test`.
|
||||||
|
- Memory stop (also PSI or swap-out) blocks every admit and grant; running
|
||||||
|
jobs go on.
|
||||||
- Other test runs, `make test`, `make test-lint`, quality gates, deploys,
|
- Other test runs, `make test`, `make test-lint`, quality gates, deploys,
|
||||||
iteration loops, swarm, image builds, cloud, shared-data cleanup, anything
|
iteration loops, swarm, image builds, cloud, shared-data cleanup, anything
|
||||||
with Docker or over a few minutes: run through `queue.py hold`
|
with Docker or over a few minutes: run through `queue.py hold`
|
||||||
|
|||||||
@@ -63,3 +63,16 @@ and stays silent below the thresholds.
|
|||||||
|
|
||||||
`watch.py status` shows the stage computed from the live free space and the
|
`watch.py status` shows the stage computed from the live free space and the
|
||||||
stored samples, and the age of the stored measurement.
|
stored samples, and the age of the stored measurement.
|
||||||
|
|
||||||
|
An unusable config (broken JSON, wrong types, a regex refusing a light
|
||||||
|
probe such as `grep` or freeing a heavy one such as `make swarm-deploy`, a
|
||||||
|
dangling link, a directory) makes the hooks, admit, status and the queue
|
||||||
|
warn and use `config.default.json`, which the install writes when no config
|
||||||
|
exists; without that file every command, teardown included, is refused.
|
||||||
|
The probes are samples, so a pattern hitting only other commands passes. A
|
||||||
|
package script is read from the nearest package.json, a named workspace
|
||||||
|
only from a folder of that name. The light-run check misses
|
||||||
|
`sys.path.insert` in a conftest, editable installs without pytest
|
||||||
|
`pythonpath`, browser libraries outside its list and generic third-party
|
||||||
|
fixtures such as pytest-selenium's `driver`, browsers started by a
|
||||||
|
subprocess, and `git bisect run` or a suite script behind `bash`.
|
||||||
@@ -0,0 +1,37 @@
|
|||||||
|
{
|
||||||
|
"disk": "/",
|
||||||
|
"thresholds_gb": {"warn": 40, "stop": 15, "critical": 12},
|
||||||
|
"projection_minutes": 30,
|
||||||
|
"interval_seconds": 120,
|
||||||
|
"cache_interval_seconds": 1800,
|
||||||
|
"heavy_commands": [
|
||||||
|
"\\bmake\\s+(-C\\s+\\S+\\s+)?(compose-deploy|compose-up|compose-playwright|roundtrip|quality|quality-high|test|test-integration|test-unit\\S*|swarm-\\S+|act-workflow|act-runner-image|build\\S*|worktree-up|store-\\S+)(\\s|$)",
|
||||||
|
"\\bdocker\\s+(build|pull|load)\\b",
|
||||||
|
"\\bdocker\\s+buildx\\s+build\\b",
|
||||||
|
"\\brun\\.sh\\s+local\\b"
|
||||||
|
],
|
||||||
|
"never_block": [
|
||||||
|
"\\bmake\\s+(-C\\s+\\S+\\s+)?(swarm-down|swarm-clean|swarm-clean-stale-nfs|swarm-diagnostic|compose-down|compose-entity-purge|worktree-down|store-host-down)(\\s|$)",
|
||||||
|
"^\\s*git\\s+commit\\b.*\\s--no-verify\\b",
|
||||||
|
"\\bmake\\s+swarm-exec(\\s|$)"
|
||||||
|
],
|
||||||
|
"queue_enforce": true,
|
||||||
|
"cleanup": [],
|
||||||
|
"worktree_cleanup": [],
|
||||||
|
"worktrees_command": "true",
|
||||||
|
"worktree_marker": ".git",
|
||||||
|
"busy_processes": ["make"],
|
||||||
|
"cache_dirs": ["~/.cache/pip", "~/.npm", "~/.cache/pre-commit", "~/.cache/act", "~/.cache/ms-playwright"],
|
||||||
|
"cache_budget_percent": 3,
|
||||||
|
"slot_need_gb": {"quality-high": 15, "test": 15, "test-nostack": 0, "iteration": 20, "swarm": 30, "gui-dind": 65, "operator": 10},
|
||||||
|
"notify": ["notify-send", "-u", "critical", "Agents council"],
|
||||||
|
"load_limits": {"cpu_percent": 90, "mem_percent": 85, "sample_seconds": 5, "kinds": ["test", "test-nostack", "quality-high"]},
|
||||||
|
"memory_limits": {
|
||||||
|
"warn": {"available": 15, "swap_free": 40, "psi_some": 10, "psi_full": 5, "swap_out_mb_min": 64, "container_swap_gb": 4},
|
||||||
|
"stop": {"available": 8, "swap_free": 10, "psi_full": 5, "swap_out_mb_min": 256, "container_swap_gb": 16}
|
||||||
|
},
|
||||||
|
"budget_percent": {"ram": 75, "cpu": 100},
|
||||||
|
"light_kinds": ["test-nostack"],
|
||||||
|
"exclusive_kinds": ["quality-high", "test"],
|
||||||
|
"gate_excluded_targets": ["test-oracle"]
|
||||||
|
}
|
||||||
File diff suppressed because it is too large.
Load diff
@@ -27,9 +27,8 @@ turn: hold runs the command on its grant and exits with its code.
|
|||||||
- In a worktree the command stays `gate.py run|hold` (`kvb-check-gate`):
|
- In a worktree the command stays `gate.py run|hold` (`kvb-check-gate`):
|
||||||
`queue.py hold ... -- python3 <gate.py> run ...`.
|
`queue.py hold ... -- python3 <gate.py> run ...`.
|
||||||
|
|
||||||
One test file or module, or ruff, without Docker self-admits with
|
Light runs (`kvb-org-organize` Capacity) self-admit; anything else goes
|
||||||
`watch.py admit --kind test-nostack` (`kvb-org-organize`); any other test
|
through hold with kind `test`, and a light kind refuses it.
|
||||||
run goes through hold with kind `test`.
|
|
||||||
|
|
||||||
## Order and grant
|
## Order and grant
|
||||||
|
|
||||||
@@ -77,8 +76,8 @@ learned values, `cancel` removes your own waiting job.
|
|||||||
one down, and `kvb-git-hotfix` may void a gate. A `kvb-org-queue-cleanup`
|
one down, and `kvb-git-hotfix` may void a gate. A `kvb-org-queue-cleanup`
|
||||||
round deletes only entries whose lock no living process holds.
|
round deletes only entries whose lock no living process holds.
|
||||||
- With `queue_enforce: true`, the `block` hook (PreToolUse on Bash) refuses
|
- With `queue_enforce: true`, the `block` hook (PreToolUse on Bash) refuses
|
||||||
`heavy_commands`, image builds, `make install|verify-*`, tests beyond one
|
`heavy_commands`, image builds, `make install|test*|quality*|verify-*`,
|
||||||
file and `gate.py run|hold` outside `queue.py hold`.
|
tests beyond one unit file and `gate.py run|hold` outside `queue.py hold`.
|
||||||
- Without the config, hold exits 2: tell the operator and ask the chair
|
- Without the config, hold exits 2: tell the operator and ask the chair
|
||||||
(`kvb-org-organize`) until it exists. A waiting hold rereads it every poll
|
(`kvb-org-organize`) until it exists. A waiting hold rereads it every poll
|
||||||
(a broken file keeps the last good one), so a limit change needs no re-hold.
|
(a broken file keeps the last good one), so a limit change needs no re-hold.
|
||||||
@@ -38,7 +38,9 @@ Args:
|
|||||||
SIBLINGS); a poll that finds other code installed in
|
SIBLINGS); a poll that finds other code installed in
|
||||||
~/.agents/skills leaves the queue ungranted and restarts the same hold
|
~/.agents/skills leaves the queue ungranted and restarts the same hold
|
||||||
with the installed queue.py.
|
with the installed queue.py.
|
||||||
--kind: a key of slot_need_gb in the config.
|
--kind: a key of slot_need_gb in the config; a light one
|
||||||
|
(light_kinds or watch.LIGHT_KIND) refuses a command watch.light()
|
||||||
|
does not recognise, naming watch.queue_hold().
|
||||||
--task: an open task of the kvb-org-rasci register; its rank orders
|
--task: an open task of the kvb-org-rasci register; its rank orders
|
||||||
the queue, and the operator's agent order (`rasci.py agents
|
the queue, and the operator's agent order (`rasci.py agents
|
||||||
order`) of the holding session breaks ties. While one of its
|
order`) of the holding session breaks ties. While one of its
|
||||||
@@ -87,7 +89,8 @@ disk of running and granted jobs, never a reservation; queue_enforce (bool), opt
|
|||||||
only. The watch.py keys disk, thresholds_gb (stop is the floor every grant
|
only. The watch.py keys disk, thresholds_gb (stop is the floor every grant
|
||||||
leaves free), slot_need_gb (the disk seed per kind), load_limits (every grant
|
leaves free), slot_need_gb (the disk seed per kind), load_limits (every grant
|
||||||
also needs CPU busy below cpu_percent and RAM used plus the job's learned RAM
|
also needs CPU busy below cpu_percent and RAM used plus the job's learned RAM
|
||||||
share below mem_percent; both bound the learned slot limits), memory_limits (no
|
share, the live admit records (admit()) and the young jobs' ramp
|
||||||
|
(ramping_ram()) below mem_percent; both bound the learned slot limits), memory_limits (no
|
||||||
grant at memory stage stop), heavy_commands and never_block are used as
|
grant at memory stage stop), heavy_commands and never_block are used as
|
||||||
watch.py documents them. A signature is `<repository>: <command signature>`
|
watch.py documents them. A signature is `<repository>: <command signature>`
|
||||||
(aliases()).
|
(aliases()).
|
||||||
@@ -165,6 +168,10 @@ XDIST_LONG = "--numprocesses="
|
|||||||
ADDOPTS = "PYTEST_ADDOPTS="
|
ADDOPTS = "PYTEST_ADDOPTS="
|
||||||
FROZEN = "footprint"
|
FROZEN = "footprint"
|
||||||
ALIASES = "aliases"
|
ALIASES = "aliases"
|
||||||
|
ADMIT = "admit"
|
||||||
|
ADMITS = "admits"
|
||||||
|
ADMIT_SECONDS = 120
|
||||||
|
GRANTED_MEM = "granted_mem"
|
||||||
|
|
||||||
|
|
||||||
@functools.cache
|
@functools.cache
|
||||||
@@ -206,7 +213,7 @@ def fail(message: str) -> NoReturn:
|
|||||||
|
|
||||||
|
|
||||||
def config() -> dict:
|
def config() -> dict:
|
||||||
cfg = watch.load_config()
|
cfg, _ = watch.announced_config()
|
||||||
if cfg is None:
|
if cfg is None:
|
||||||
fail(f"no config at {watch.CONFIG}; the queue reads the kvb-org-organize watch config")
|
fail(f"no config at {watch.CONFIG}; the queue reads the kvb-org-organize watch config")
|
||||||
problem = watch.missing(cfg, (*BUDGET_KEYS, *watch.MEMORY_KEYS))
|
problem = watch.missing(cfg, (*BUDGET_KEYS, *watch.MEMORY_KEYS))
|
||||||
@@ -218,13 +225,14 @@ def config() -> dict:
|
|||||||
def fresh(cfg: dict) -> dict:
|
def fresh(cfg: dict) -> dict:
|
||||||
"""Returns:
|
"""Returns:
|
||||||
The config as it stands now, else cfg while the file is missing,
|
The config as it stands now, else cfg while the file is missing,
|
||||||
unreadable or incomplete, so a half-saved edit never ends a wait.
|
unusable (watch.checked_config()) or incomplete, so a half-saved
|
||||||
|
edit never ends a wait.
|
||||||
"""
|
"""
|
||||||
try:
|
try:
|
||||||
now = watch.load_config()
|
now = watch.checked_config(watch.load_config())
|
||||||
except (OSError, ValueError):
|
except watch.UNUSABLE:
|
||||||
return cfg
|
return cfg
|
||||||
if not isinstance(now, dict) or watch.missing(now, (*BUDGET_KEYS, *watch.MEMORY_KEYS)) is not None:
|
if watch.missing(now, (*BUDGET_KEYS, *watch.MEMORY_KEYS)) is not None:
|
||||||
return cfg
|
return cfg
|
||||||
return now
|
return now
|
||||||
|
|
||||||
@@ -690,11 +698,11 @@ def footprint(entry: dict, signatures: dict[tuple[str, str], dict], kinds: dict[
|
|||||||
|
|
||||||
|
|
||||||
def held_back(cfg: dict, entries: list[dict], now: float, signatures: dict[str, dict], kinds: dict[str, dict],
|
def held_back(cfg: dict, entries: list[dict], now: float, signatures: dict[str, dict], kinds: dict[str, dict],
|
||||||
cpu: float, mem: float) -> dict[str, str]:
|
cpu: float, mem: float, admitted: float) -> dict[str, str]:
|
||||||
loaded = {}
|
loaded = {}
|
||||||
for entry in entries:
|
for entry in entries:
|
||||||
if entry["status"] == "waiting":
|
if entry["status"] == "waiting":
|
||||||
pending = pending_ram(entries, entry["id"], now, signatures, kinds)
|
pending = pending_ram(entries, entry["id"], now, signatures, kinds, mem, admitted)
|
||||||
if (reason := load_reason(cfg["load_limits"], cpu, mem, pending)) is not None:
|
if (reason := load_reason(cfg["load_limits"], cpu, mem, pending)) is not None:
|
||||||
loaded[entry["id"]] = reason
|
loaded[entry["id"]] = reason
|
||||||
return loaded
|
return loaded
|
||||||
@@ -742,18 +750,36 @@ def ram_share(entry: dict, signatures: dict[str, dict], kinds: dict[str, dict])
|
|||||||
return history.seed_ram(kinds) if share is None else share["ram"]
|
return history.seed_ram(kinds) if share is None else share["ram"]
|
||||||
|
|
||||||
|
|
||||||
def pending_ram(entries: list[dict], entry_id: str, now: float, signatures: dict[str, dict],
|
def ramping_ram(entries: list[dict], now: float, signatures: dict[str, dict], kinds: dict[str, dict],
|
||||||
kinds: dict[str, dict]) -> float:
|
mem: float) -> float:
|
||||||
"""Returns:
|
"""Returns:
|
||||||
The RAM share of entry_id plus the whole share of every active job
|
What the active jobs granted less than RAMP_SECONDS ago (or without a
|
||||||
granted less than RAMP_SECONDS ago (or without a grant time): what
|
grant time) may still add: the shares of those whose grant recorded
|
||||||
may still come once they ramp up. An older job counts as ramped, its
|
the RAM used (GRANTED_MEM) minus what RAM used gained since the
|
||||||
RAM inside the RAM used; one that peaks later is bounded by the
|
earliest of these grants, between 0 and those shares (a RAM used
|
||||||
memory stage only.
|
that fell since counts as no gain), plus the whole share of the
|
||||||
|
others. An older job counts as ramped, its RAM inside the RAM used;
|
||||||
|
one that peaks later is bounded by the memory stage only.
|
||||||
|
"""
|
||||||
|
young = [e for e in entries if e["status"] == "active" and now - e.get("granted", now) < RAMP_SECONDS]
|
||||||
|
dated = [e for e in young if GRANTED_MEM in e]
|
||||||
|
gained = max(0.0, mem - min((e[GRANTED_MEM] for e in dated), default=mem))
|
||||||
|
ramp = max(0.0, sum(ram_share(e, signatures, kinds) for e in dated) - gained)
|
||||||
|
return ramp + sum(ram_share(e, signatures, kinds) for e in young if GRANTED_MEM not in e)
|
||||||
|
|
||||||
|
|
||||||
|
def pending_ram(entries: list[dict], entry_id: str, now: float, signatures: dict[str, dict],
|
||||||
|
kinds: dict[str, dict], mem: float, admitted: float) -> float:
|
||||||
|
"""Returns:
|
||||||
|
The RAM share of entry_id plus admitted (admitted_ram()) plus
|
||||||
|
ramping_ram() at RAM used mem.
|
||||||
"""
|
"""
|
||||||
mine = [e for e in entries if e["id"] == entry_id]
|
mine = [e for e in entries if e["id"] == entry_id]
|
||||||
young = [e for e in entries if e["status"] == "active" and now - e.get("granted", now) < RAMP_SECONDS]
|
return ram_share(mine[0], signatures, kinds) + admitted + ramping_ram(entries, now, signatures, kinds, mem)
|
||||||
return ram_share(mine[0], signatures, kinds) + sum(ram_share(e, signatures, kinds) for e in young)
|
|
||||||
|
|
||||||
|
def admitted_ram(data: dict, now: float) -> float:
|
||||||
|
return sum(record["ram"] for record in data.get(ADMITS, []) if record["expires"] > now)
|
||||||
|
|
||||||
|
|
||||||
def workdir() -> str:
|
def workdir() -> str:
|
||||||
@@ -822,13 +848,13 @@ def wait(cfg: dict, entry_id: str, caught: list[int]) -> tuple[int | None, int]:
|
|||||||
print(f"queue: {entry_id} was cancelled", file=sys.stderr)
|
print(f"queue: {entry_id} was cancelled", file=sys.stderr)
|
||||||
return CANCELLED, 0
|
return CANCELLED, 0
|
||||||
rejoin(data, mine[0], now)
|
rejoin(data, mine[0], now)
|
||||||
loaded = held_back(cfg, data["entries"], now, signatures, kinds, cpu, mem)
|
loaded = held_back(cfg, data["entries"], now, signatures, kinds, cpu, mem, admitted_ram(data, now))
|
||||||
verdicts = decide(cfg, data["entries"], free, now, signatures, kinds, loaded)
|
verdicts = decide(cfg, data["entries"], free, now, signatures, kinds, loaded)
|
||||||
reason = verdicts[entry_id] or loaded.get(entry_id)
|
reason = verdicts[entry_id] or loaded.get(entry_id)
|
||||||
mine[0]["reason"] = reason
|
mine[0]["reason"] = reason
|
||||||
if reason is None:
|
if reason is None:
|
||||||
concurrent = sum(e["status"] == "active" for e in data["entries"])
|
concurrent = sum(e["status"] == "active" for e in data["entries"])
|
||||||
mine[0].update(status="active", granted=now, **{FROZEN: footprint(mine[0], signatures, kinds)})
|
mine[0].update(status="active", granted=now, **{GRANTED_MEM: mem, FROZEN: footprint(mine[0], signatures, kinds)})
|
||||||
return None, concurrent
|
return None, concurrent
|
||||||
position = list(verdicts).index(entry_id) + 1
|
position = list(verdicts).index(entry_id) + 1
|
||||||
if (position, reason) != shown:
|
if (position, reason) != shown:
|
||||||
@@ -912,15 +938,27 @@ def keep_receipt(gates: ModuleType | None, args: argparse.Namespace, here: str,
|
|||||||
print(f"queue: receipt {args.receipt} for tree {before}", file=sys.stderr)
|
print(f"queue: receipt {args.receipt} for tree {before}", file=sys.stderr)
|
||||||
|
|
||||||
|
|
||||||
def cmd_hold(args: argparse.Namespace) -> int:
|
def check_hold(args: argparse.Namespace, cfg: dict) -> None:
|
||||||
cfg = config()
|
"""Args:
|
||||||
|
args: the hold arguments; fail() exits on an unknown kind, a command
|
||||||
|
that is no light run under a light kind, a task outside the
|
||||||
|
register or a missing hotfix row.
|
||||||
|
cfg: the watch config.
|
||||||
|
"""
|
||||||
if args.kind not in cfg["slot_need_gb"]:
|
if args.kind not in cfg["slot_need_gb"]:
|
||||||
fail(f"kind {args.kind!r} is not in slot_need_gb of {watch.CONFIG}")
|
fail(f"kind {args.kind!r} is not in slot_need_gb of {watch.CONFIG}")
|
||||||
|
shown = shlex.join(args.command)
|
||||||
|
if args.kind in {watch.LIGHT_KIND, *cfg.get("light_kinds", [])} and not watch.light(shown, os.getcwd()):
|
||||||
|
fail(f"{shown} is no light run and the light kind {args.kind} skips every reservation; run `{watch.queue_hold(shown)}`")
|
||||||
rasci = sibling("rasci")
|
rasci = sibling("rasci")
|
||||||
if args.task not in rasci.merged(False):
|
if args.task not in rasci.merged(False):
|
||||||
fail(f"{args.task} is no open task of the kvb-org-rasci register")
|
fail(f"{args.task} is no open task of the kvb-org-rasci register")
|
||||||
if args.hotfix is not None and HOTFIX + args.hotfix not in rasci.load():
|
if args.hotfix is not None and HOTFIX + args.hotfix not in rasci.load():
|
||||||
fail(f"the register has no row {HOTFIX}{args.hotfix}")
|
fail(f"the register has no row {HOTFIX}{args.hotfix}")
|
||||||
|
|
||||||
|
def cmd_hold(args: argparse.Namespace) -> int:
|
||||||
|
cfg = config()
|
||||||
|
check_hold(args, cfg)
|
||||||
here = workdir()
|
here = workdir()
|
||||||
gates, out = gated(args, here)
|
gates, out = gated(args, here)
|
||||||
names = aliases(here, args.command, os.getcwd())
|
names = aliases(here, args.command, os.getcwd())
|
||||||
@@ -1091,6 +1129,38 @@ def held_commands(command: str) -> list[list[str]]:
|
|||||||
]
|
]
|
||||||
|
|
||||||
|
|
||||||
|
def enforced(cfg: dict) -> dict:
|
||||||
|
return {**cfg, "heavy_commands": [*cfg["heavy_commands"], GATE.pattern]}
|
||||||
|
|
||||||
|
|
||||||
|
def admit(cfg: dict, kind: str, argv: list[str], cpu: float, mem: float, calm: bool) -> str | None:
|
||||||
|
"""Args:
|
||||||
|
cfg: the config.
|
||||||
|
kind: the kind of the light job that asks to start outside the queue.
|
||||||
|
argv: its command, run in the current directory.
|
||||||
|
cpu: CPU busy now, in percent.
|
||||||
|
mem: RAM used now, in percent.
|
||||||
|
calm: whether the memory stage is ok; the live active jobs' ramp
|
||||||
|
(ramping_ram()) counts only when it is not.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
load_reason() over the job's RAM share plus the live admit records
|
||||||
|
(admitted_ram()) and the ramp; under the store lock, a grant records
|
||||||
|
the share for ADMIT_SECONDS and expired records leave.
|
||||||
|
"""
|
||||||
|
now = time.time()
|
||||||
|
signatures, kinds = learned(cfg, now)
|
||||||
|
mine = {"id": ADMIT, "status": "waiting", "kind": kind, ALIASES: list(aliases(workdir(), argv, os.getcwd()))}
|
||||||
|
share = ram_share(mine, signatures, kinds)
|
||||||
|
with store() as data:
|
||||||
|
active = [e for e in data["entries"] if e["status"] == "active" and living(e)]
|
||||||
|
ramp = 0.0 if calm else ramping_ram(active, now, signatures, kinds, mem)
|
||||||
|
reason = load_reason(cfg["load_limits"], cpu, mem, share + admitted_ram(data, now) + ramp)
|
||||||
|
granted = [{"ram": share, "expires": now + ADMIT_SECONDS}] if reason is None else []
|
||||||
|
data[ADMITS] = [record for record in data.get(ADMITS, []) if record["expires"] > now] + granted
|
||||||
|
return reason
|
||||||
|
|
||||||
|
|
||||||
def tails(argv: list[str]) -> list[str]:
|
def tails(argv: list[str]) -> list[str]:
|
||||||
return [shlex.join(argv[at:]) for at, word in enumerate(argv) if not word.startswith("-")]
|
return [shlex.join(argv[at:]) for at, word in enumerate(argv) if not word.startswith("-")]
|
||||||
|
|
||||||
@@ -1121,11 +1191,10 @@ def cmd_block() -> int:
|
|||||||
"hookEventName": "PreToolUse", "permissionDecision": "ask", "permissionDecisionReason": reason,
|
"hookEventName": "PreToolUse", "permissionDecision": "ask", "permissionDecisionReason": reason,
|
||||||
}}))
|
}}))
|
||||||
return 0
|
return 0
|
||||||
cfg = watch.load_config()
|
cfg, _ = watch.announced_config()
|
||||||
if cfg is None or cfg.get("queue_enforce") is not True:
|
if cfg is None or cfg.get("queue_enforce") is not True:
|
||||||
return 0
|
return 0
|
||||||
gated = {**cfg, "heavy_commands": [*cfg["heavy_commands"], GATE.pattern]}
|
if watch.heavy(raw, enforced(cfg), (THROUGH_QUEUE.pattern, *cfg["never_block"]), watch.payload_cwd(payload)):
|
||||||
if watch.heavy(raw, gated, (THROUGH_QUEUE.pattern, *cfg["never_block"]), watch.payload_cwd(payload)):
|
|
||||||
print(
|
print(
|
||||||
f"kvb-org-queue: heavy jobs run through the queue: start `python3 {Path(__file__).resolve()} hold "
|
f"kvb-org-queue: heavy jobs run through the queue: start `python3 {Path(__file__).resolve()} hold "
|
||||||
"--kind <kind> --task <register task> -- <command>` in the background; it runs the command on its grant.",
|
"--kind <kind> --task <register task> -- <command>` in the background; it runs the command on its grant.",
|
||||||
@@ -1164,8 +1233,10 @@ def main() -> int:
|
|||||||
if args.cmd == "block":
|
if args.cmd == "block":
|
||||||
try:
|
try:
|
||||||
return cmd_block()
|
return cmd_block()
|
||||||
except (OSError, ValueError, KeyError, TypeError, AttributeError, re.error):
|
except Exception as error: # noqa: BLE001
|
||||||
return 0
|
print(f"kvb-org-queue: the block hook failed ({error!r}), so the command counts as heavy and is refused.",
|
||||||
|
file=sys.stderr)
|
||||||
|
return 2
|
||||||
commands = {"hold": cmd_hold, "list": cmd_list, "history": cmd_history, "cancel": cmd_cancel}
|
commands = {"hold": cmd_hold, "list": cmd_list, "history": cmd_history, "cancel": cmd_cancel}
|
||||||
try:
|
try:
|
||||||
return commands[args.cmd](args)
|
return commands[args.cmd](args)
|
||||||
|
|||||||
@@ -31,9 +31,8 @@ picks do; every other session keeps its own.
|
|||||||
|
|
||||||
## Night rules for the active session
|
## Night rules for the active session
|
||||||
|
|
||||||
- **Grants**: request none; admit your own runs with `watch.py admit --kind
|
- **Grants**: request none; light runs self-admit and heavy ones queue
|
||||||
<slot>` (`${CLAUDE_SKILL_DIR}/../kvb-org-organize/scripts/watch.py`), whose
|
(`kvb-org-organize` Capacity), whose limits still apply.
|
||||||
disk, CPU and RAM limits still apply.
|
|
||||||
- **Offices**: the suspended agents' claims stay theirs. Take only free work;
|
- **Offices**: the suspended agents' claims stay theirs. Take only free work;
|
||||||
a piece a suspended agent holds waits for the resume or the operator.
|
a piece a suspended agent holds waits for the resume or the operator.
|
||||||
- **Messages**: no wakes and no peer questions; the Peer protocol's wait is
|
- **Messages**: no wakes and no peer questions; the Peer protocol's wait is
|
||||||
|
|||||||
@@ -30,10 +30,9 @@ sidetree or leaves as a proposal.
|
|||||||
- code explored through the codebase-memory-mcp graph first
|
- code explored through the codebase-memory-mcp graph first
|
||||||
(`search_graph`, `trace_path`, `get_architecture`, `detect_changes`; the
|
(`search_graph`, `trace_path`, `get_architecture`, `detect_changes`; the
|
||||||
`kvb-agent-optimize` skill sets it up), grep and read after;
|
`kvb-agent-optimize` skill sets it up), grep and read after;
|
||||||
- light runs (no Docker, about a minute) self-admit with
|
- light runs self-admit (`kvb-org-organize` Capacity); `make test`, gates,
|
||||||
`${CLAUDE_SKILL_DIR}/../kvb-org-organize/scripts/watch.py admit --kind
|
deploys and anything with Docker only through `queue.py hold`
|
||||||
test-nostack`; `make test`, gates, deploys and anything with Docker only
|
(`kvb-org-queue`);
|
||||||
through `queue.py hold` (`kvb-org-queue`);
|
|
||||||
- status that asks nothing goes to
|
- status that asks nothing goes to
|
||||||
`${CLAUDE_SKILL_DIR}/../kvb-org-radio/scripts/radio.py post "<text>"`,
|
`${CLAUDE_SKILL_DIR}/../kvb-org-radio/scripts/radio.py post "<text>"`,
|
||||||
never into a wake.
|
never into a wake.
|
||||||
|
|||||||
@@ -58,9 +58,9 @@ and the prose diff.
|
|||||||
- Write the script, then replace the compiled prose with its call
|
- Write the script, then replace the compiled prose with its call
|
||||||
(`${CLAUDE_SKILL_DIR}/scripts/<verb>.py ...`) and one line on reading its
|
(`${CLAUDE_SKILL_DIR}/scripts/<verb>.py ...`) and one line on reading its
|
||||||
output.
|
output.
|
||||||
- Every script gets a unit test with a negative control shown once, run once
|
- Every script gets a unit test with a negative control shown once, run as a
|
||||||
`watch.py admit --kind test-nostack` (the `kvb-org-organize` skill) prints
|
light run (`kvb-org-organize` Capacity); `make test` runs through
|
||||||
`grant`; `make test` runs through `queue.py hold` (`kvb-org-queue`).
|
`queue.py hold` (`kvb-org-queue`).
|
||||||
- In another repository follow its agent rules (claims, worktree, branch); a
|
- In another repository follow its agent rules (claims, worktree, branch); a
|
||||||
file another line owns goes to that line through the `kvb-org-coordinate`
|
file another line owns goes to that line through the `kvb-org-coordinate`
|
||||||
skill.
|
skill.
|
||||||
|
|||||||
@@ -52,9 +52,8 @@ or diff.
|
|||||||
- Read every script whole before carrying it; clean it the way the
|
- Read every script whole before carrying it; clean it the way the
|
||||||
`kvb-code-clean` skill does.
|
`kvb-code-clean` skill does.
|
||||||
- A carried script gets a unit test in the repository's `tests/` with a
|
- A carried script gets a unit test in the repository's `tests/` with a
|
||||||
negative control, run once `watch.py admit --kind test-nostack` (the
|
negative control, run as a light run (`kvb-org-organize` Capacity);
|
||||||
`kvb-org-organize` skill) prints `grant`; `make test` runs through
|
`make test` runs through `queue.py hold` (`kvb-org-queue`).
|
||||||
`queue.py hold` (`kvb-org-queue`).
|
|
||||||
- `transcend.py mark` every offered candidate with its verdict.
|
- `transcend.py mark` every offered candidate with its verdict.
|
||||||
- Leave the result uncommitted; committing is the `kvb-git-commit-next`
|
- Leave the result uncommitted; committing is the `kvb-git-commit-next`
|
||||||
skill's job.
|
skill's job.
|
||||||
|
|||||||
+3
-1
@@ -11,7 +11,9 @@ from pathlib import Path
|
|||||||
REPO_ROOT = Path(__file__).resolve().parents[1]
|
REPO_ROOT = Path(__file__).resolve().parents[1]
|
||||||
GATES = REPO_ROOT / "skills" / "kvb-check-gate" / "scripts" / "gates.py"
|
GATES = REPO_ROOT / "skills" / "kvb-check-gate" / "scripts" / "gates.py"
|
||||||
QUEUE = REPO_ROOT / "skills" / "kvb-org-queue" / "scripts" / "queue.py"
|
QUEUE = REPO_ROOT / "skills" / "kvb-org-queue" / "scripts" / "queue.py"
|
||||||
OPEN_MEMORY = {"available": -1, "swap_free": -1, "psi_full": 1000, "container_swap_gb": 1e9}
|
OPEN_MEMORY = {
|
||||||
|
"available": -1, "swap_free": -1, "psi_some": 1000, "psi_full": 1000, "swap_out_mb_min": 1e9, "container_swap_gb": 1e9,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
class Scratch(unittest.TestCase):
|
class Scratch(unittest.TestCase):
|
||||||
|
|||||||
+66
-4
@@ -2,6 +2,7 @@
|
|||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
import os
|
import os
|
||||||
import shutil
|
import shutil
|
||||||
import subprocess
|
import subprocess
|
||||||
@@ -10,15 +11,26 @@ import unittest
|
|||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
INSTALL = Path(__file__).resolve().parents[1] / "scripts" / "install.sh"
|
INSTALL = Path(__file__).resolve().parents[1] / "scripts" / "install.sh"
|
||||||
|
ORGANIZE = Path(__file__).resolve().parents[1] / "skills" / "kvb-org-organize"
|
||||||
NPX = "#!/bin/sh\nmkdir -p .agents/skills/locked\necho locked > .agents/skills/locked/SKILL.md\n"
|
NPX = "#!/bin/sh\nmkdir -p .agents/skills/locked\necho locked > .agents/skills/locked/SKILL.md\n"
|
||||||
BYTECODE = ("__pycache__/radio.cpython-312.pyc", "json.pyc", "legacy.pyo")
|
BYTECODE = ("__pycache__/radio.cpython-312.pyc", "json.pyc", "legacy.pyo")
|
||||||
|
LIVE_MEMORY = {
|
||||||
|
"warn": {"available": 15, "swap_free": 40, "psi_full": 5, "container_swap_gb": 4},
|
||||||
|
"stop": {"available": 8, "swap_free": 10, "psi_full": 10, "container_swap_gb": 16},
|
||||||
|
}
|
||||||
|
SHIPPED = ORGANIZE / "config.default.json"
|
||||||
|
DEFAULT = json.loads(SHIPPED.read_text(encoding="utf-8"))
|
||||||
|
ROW_MEMORY = DEFAULT["memory_limits"]
|
||||||
|
|
||||||
|
|
||||||
class TestInstall(unittest.TestCase):
|
class TestInstall(unittest.TestCase):
|
||||||
def install(self, root: Path, target: Path, npx: str = NPX, planted: tuple[str, ...] = ()) -> None:
|
def install(self, root: Path, target: Path, npx: str = NPX, planted: tuple[str, ...] = (), config: Path | None = None,
|
||||||
|
code: int = 0, stubs_extra: tuple[tuple[str, str], ...] = ()) -> subprocess.CompletedProcess:
|
||||||
repo, stubs = root / "repo", root / "bin"
|
repo, stubs = root / "repo", root / "bin"
|
||||||
(repo / "scripts").mkdir(parents=True)
|
(repo / "scripts").mkdir(parents=True)
|
||||||
shutil.copy(INSTALL, repo / "scripts" / "install.sh")
|
shutil.copy(INSTALL, repo / "scripts" / "install.sh")
|
||||||
|
if config is not None:
|
||||||
|
shutil.copytree(ORGANIZE, repo / "skills" / ORGANIZE.name, ignore=shutil.ignore_patterns("__pycache__"))
|
||||||
(repo / "scripts" / "enable-plugins.js").write_text("", encoding="utf-8")
|
(repo / "scripts" / "enable-plugins.js").write_text("", encoding="utf-8")
|
||||||
(repo / "skills" / "own").mkdir(parents=True)
|
(repo / "skills" / "own").mkdir(parents=True)
|
||||||
(repo / "skills" / "own" / "SKILL.md").write_text("own", encoding="utf-8")
|
(repo / "skills" / "own" / "SKILL.md").write_text("own", encoding="utf-8")
|
||||||
@@ -26,11 +38,61 @@ class TestInstall(unittest.TestCase):
|
|||||||
(repo / name).parent.mkdir(parents=True, exist_ok=True)
|
(repo / name).parent.mkdir(parents=True, exist_ok=True)
|
||||||
(repo / name).write_bytes(b"planted")
|
(repo / name).write_bytes(b"planted")
|
||||||
stubs.mkdir()
|
stubs.mkdir()
|
||||||
(stubs / "npx").write_text(npx, encoding="utf-8")
|
for name, text in (("npx", npx), *stubs_extra):
|
||||||
(stubs / "npx").chmod(0o755)
|
(stubs / name).write_text(text, encoding="utf-8")
|
||||||
|
(stubs / name).chmod(0o755)
|
||||||
env = {**os.environ, "PATH": f"{stubs}:/usr/bin:/bin", "TARGET": str(target), "HOME": str(root / "home")}
|
env = {**os.environ, "PATH": f"{stubs}:/usr/bin:/bin", "TARGET": str(target), "HOME": str(root / "home")}
|
||||||
|
if config is not None:
|
||||||
|
env |= {"AGENTS_COUNCIL_CONFIG": str(config), "AGENTS_COUNCIL_ROOT": str(root / "council")}
|
||||||
done = subprocess.run(["bash", str(repo / "scripts" / "install.sh")], env=env, capture_output=True, text=True, check=False)
|
done = subprocess.run(["bash", str(repo / "scripts" / "install.sh")], env=env, capture_output=True, text=True, check=False)
|
||||||
self.assertEqual(done.returncode, 0, done.stderr)
|
self.assertEqual(done.returncode, code, done.stderr)
|
||||||
|
return done
|
||||||
|
|
||||||
|
def test_the_install_stops_before_any_copy_while_the_live_config_lacks_a_memory_bound_the_new_code_needs(self):
|
||||||
|
for limits, code in ((LIVE_MEMORY, 2), (ROW_MEMORY, 0)):
|
||||||
|
with self.subTest(code=code), tempfile.TemporaryDirectory() as tmp:
|
||||||
|
root = Path(tmp)
|
||||||
|
target = root / "target"
|
||||||
|
target.mkdir()
|
||||||
|
config = root / "config.json"
|
||||||
|
config.write_text(json.dumps({**DEFAULT, "memory_limits": limits}), encoding="utf-8")
|
||||||
|
before = config.stat()
|
||||||
|
done = self.install(root, target, config=config, code=code)
|
||||||
|
after = config.stat()
|
||||||
|
self.assertEqual((after.st_ino, after.st_mtime_ns, after.st_mode), (before.st_ino, before.st_mtime_ns, before.st_mode))
|
||||||
|
installed = sorted(path.name for path in target.iterdir())
|
||||||
|
if code:
|
||||||
|
self.assertIn(
|
||||||
|
"lacks memory_limits.warn.psi_some, memory_limits.warn.swap_out_mb_min, memory_limits.stop.swap_out_mb_min;",
|
||||||
|
done.stderr,
|
||||||
|
)
|
||||||
|
self.assertEqual(installed, [])
|
||||||
|
self.assertFalse((root / "repo" / ".agents").exists())
|
||||||
|
else:
|
||||||
|
self.assertEqual(installed, [".agents", ".claude"])
|
||||||
|
self.assertEqual(json.loads(config.read_text(encoding="utf-8")), {**DEFAULT, "memory_limits": limits})
|
||||||
|
|
||||||
|
def test_an_install_without_a_config_writes_the_shipped_default_as_the_initial_one(self):
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
root = Path(tmp)
|
||||||
|
target = root / "target"
|
||||||
|
target.mkdir()
|
||||||
|
config = root / "council-config" / "config.json"
|
||||||
|
self.install(root, target, config=config)
|
||||||
|
self.assertEqual(config.read_bytes(), SHIPPED.read_bytes())
|
||||||
|
self.assertEqual(config.stat().st_mode & 0o777, 0o600)
|
||||||
|
self.assertEqual(config.parent.stat().st_mode & 0o777, 0o700)
|
||||||
|
|
||||||
|
def test_a_config_written_while_the_install_creates_its_folder_survives(self):
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
root = Path(tmp)
|
||||||
|
target = root / "target"
|
||||||
|
target.mkdir()
|
||||||
|
config = root / "council-config" / "config.json"
|
||||||
|
racer = f"#!/bin/sh\nmkdir -p {config.parent}\necho '{{\"mine\": true}}' > {config}\nexec /usr/bin/install \"$@\"\n"
|
||||||
|
done = self.install(root, target, config=config, stubs_extra=(("install", racer),))
|
||||||
|
self.assertEqual(json.loads(config.read_text(encoding="utf-8")), {"mine": True})
|
||||||
|
self.assertIn(str(config), done.stderr)
|
||||||
|
|
||||||
def test_reinstall_empties_the_skill_directories_in_place(self):
|
def test_reinstall_empties_the_skill_directories_in_place(self):
|
||||||
with tempfile.TemporaryDirectory() as tmp:
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
|||||||
@@ -529,7 +529,10 @@ class TestOperator(unittest.TestCase):
|
|||||||
|
|
||||||
def _queue_config(self, slot_need_gb: dict) -> None:
|
def _queue_config(self, slot_need_gb: dict) -> None:
|
||||||
config = self.base / "config.json"
|
config = self.base / "config.json"
|
||||||
open_memory = {"available": -1, "swap_free": -1, "psi_full": 1000, "container_swap_gb": 1e9}
|
open_memory = {
|
||||||
|
"available": -1, "swap_free": -1, "psi_some": 1000, "psi_full": 1000, "swap_out_mb_min": 1e9,
|
||||||
|
"container_swap_gb": 1e9,
|
||||||
|
}
|
||||||
config.write_text(json.dumps({
|
config.write_text(json.dumps({
|
||||||
"disk": str(self.base), "thresholds_gb": {"warn": 0, "stop": 0, "critical": 0}, "projection_minutes": 30,
|
"disk": str(self.base), "thresholds_gb": {"warn": 0, "stop": 0, "critical": 0}, "projection_minutes": 30,
|
||||||
"slot_need_gb": slot_need_gb, "heavy_commands": [r"\btouch heavy\b"], "never_block": [],
|
"slot_need_gb": slot_need_gb, "heavy_commands": [r"\btouch heavy\b"], "never_block": [],
|
||||||
|
|||||||
+121
-6
@@ -23,7 +23,9 @@ HISTORY = REPO_ROOT / "skills" / "kvb-org-queue" / "scripts" / "history.py"
|
|||||||
GATES = REPO_ROOT / "skills" / "kvb-check-gate" / "scripts" / "gates.py"
|
GATES = REPO_ROOT / "skills" / "kvb-check-gate" / "scripts" / "gates.py"
|
||||||
NOW = 1_800_000_000.0
|
NOW = 1_800_000_000.0
|
||||||
HOUR = 3600
|
HOUR = 3600
|
||||||
OPEN_MEMORY = {"available": -1, "swap_free": -1, "psi_full": 1000, "container_swap_gb": 1e9}
|
OPEN_MEMORY = {
|
||||||
|
"available": -1, "swap_free": -1, "psi_some": 1000, "psi_full": 1000, "swap_out_mb_min": 1e9, "container_swap_gb": 1e9,
|
||||||
|
}
|
||||||
BASE = {
|
BASE = {
|
||||||
"free_gb": 100.0,
|
"free_gb": 100.0,
|
||||||
"floor": 10.0,
|
"floor": 10.0,
|
||||||
@@ -141,11 +143,38 @@ class TestEvaluate(unittest.TestCase):
|
|||||||
young = {**job(1, kind="a", status="active", granted=NOW - 60), **cmd, "cmd": "a"}
|
young = {**job(1, kind="a", status="active", granted=NOW - 60), **cmd, "cmd": "a"}
|
||||||
overshooting = {**job(5, kind="c", status="active", granted=NOW - 120), **cmd, "cmd": "c"}
|
overshooting = {**job(5, kind="c", status="active", granted=NOW - 120), **cmd, "cmd": "c"}
|
||||||
old = {**job(4, kind="c", status="active", granted=NOW - self.queue.RAMP_SECONDS), **cmd, "cmd": "c"}
|
old = {**job(4, kind="c", status="active", granted=NOW - self.queue.RAMP_SECONDS), **cmd, "cmd": "c"}
|
||||||
self.assertEqual(self.queue.pending_ram([young, overshooting, old, fresh], "j2", NOW, signatures, kinds), 30 + 10 + 25)
|
pending = self.queue.pending_ram
|
||||||
|
self.assertEqual(pending([young, overshooting, old, fresh], "j2", NOW, signatures, kinds, 50, 0), 30 + 10 + 25)
|
||||||
undated = {k: v for k, v in old.items() if k != "granted"}
|
undated = {k: v for k, v in old.items() if k != "granted"}
|
||||||
self.assertEqual(self.queue.pending_ram([undated, fresh], "j2", NOW, signatures, kinds), 10 + 25)
|
self.assertEqual(pending([undated, fresh], "j2", NOW, signatures, kinds, 50, 0), 10 + 25)
|
||||||
unknown = {**job(3, kind="new"), **cmd, "cmd": "new"}
|
unknown = {**job(3, kind="new"), **cmd, "cmd": "new"}
|
||||||
self.assertEqual(self.queue.pending_ram([unknown], "j3", NOW, signatures, kinds), 30)
|
self.assertEqual(pending([unknown], "j3", NOW, signatures, kinds, 50, 0), 30)
|
||||||
|
self.assertEqual(pending([fresh], "j2", NOW, signatures, kinds, 50, 12), 25 + 12)
|
||||||
|
|
||||||
|
def test_a_young_job_counts_only_what_it_has_not_added_since_its_grant(self):
|
||||||
|
history = _load(HISTORY)
|
||||||
|
rows = [history_row(n, kind="b", signature="w: b", mem_before=20, mem_max=45) for n in range(5)]
|
||||||
|
signatures, kinds = history.learn(rows, NOW, {"load_limits": {"mem_percent": 90, "cpu_percent": 90}})
|
||||||
|
fresh = {**job(2, kind="b"), "workdir": "/w", "cmd": "b"}
|
||||||
|
|
||||||
|
def gate(seq: int, ram: float, granted_mem: float) -> dict:
|
||||||
|
return {**job(seq, kind="g", status="active", granted=NOW - 60), "workdir": "/w", "cmd": "g",
|
||||||
|
"footprint": {"ram": ram, "cpu": 1}, "granted_mem": granted_mem}
|
||||||
|
|
||||||
|
pending = self.queue.pending_ram
|
||||||
|
self.assertEqual(pending([gate(1, 40, 10), fresh], "j2", NOW, signatures, kinds, 10, 0), 25 + 40)
|
||||||
|
self.assertEqual(pending([gate(1, 40, 10), fresh], "j2", NOW, signatures, kinds, 45, 0), 25 + 5)
|
||||||
|
self.assertEqual(pending([gate(1, 40, 10), fresh], "j2", NOW, signatures, kinds, 60, 0), 25)
|
||||||
|
self.assertEqual(pending([gate(1, 40, 10), gate(3, 20, 30), fresh], "j2", NOW, signatures, kinds, 45, 0), 25 + 25)
|
||||||
|
undated = {k: v for k, v in gate(1, 40, 10).items() if k != "granted_mem"}
|
||||||
|
self.assertEqual(pending([undated, fresh], "j2", NOW, signatures, kinds, 60, 0), 25 + 40)
|
||||||
|
ramp = self.queue.ramping_ram
|
||||||
|
for mem in (70, 40, 20):
|
||||||
|
with self.subTest(mem=mem):
|
||||||
|
self.assertEqual(ramp([gate(1, 40, 70)], NOW, signatures, kinds, mem), 40)
|
||||||
|
self.assertEqual(ramp([gate(1, 40, 70), {**undated, "id": "j3"}], NOW, signatures, kinds, 20), 80)
|
||||||
|
self.assertEqual(self.queue.admitted_ram({"admits": [{"ram": 7, "expires": NOW + 1}, {"ram": 9, "expires": NOW}]}, NOW), 7)
|
||||||
|
self.assertEqual(self.queue.admitted_ram({"entries": []}, NOW), 0)
|
||||||
|
|
||||||
def test_a_signature_names_the_repository_every_worktree_shares(self):
|
def test_a_signature_names_the_repository_every_worktree_shares(self):
|
||||||
with tempfile.TemporaryDirectory() as tmp:
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
@@ -251,7 +280,7 @@ class TestEvaluate(unittest.TestCase):
|
|||||||
self.assertEqual(self.queue.footprint(waiting, signatures, kinds), {"ram": 50, "cpu": 80})
|
self.assertEqual(self.queue.footprint(waiting, signatures, kinds), {"ram": 50, "cpu": 80})
|
||||||
older = {k: v for k, v in running.items() if k != "footprint"}
|
older = {k: v for k, v in running.items() if k != "footprint"}
|
||||||
self.assertEqual(self.queue.footprint(older, signatures, kinds), {"ram": 50, "cpu": 80})
|
self.assertEqual(self.queue.footprint(older, signatures, kinds), {"ram": 50, "cpu": 80})
|
||||||
self.assertEqual(self.queue.pending_ram([running], "j1", NOW, signatures, kinds), 12 + 12)
|
self.assertEqual(self.queue.pending_ram([running], "j1", NOW, signatures, kinds, 50, 0), 12 + 12)
|
||||||
|
|
||||||
def test_an_active_entry_an_older_queue_granted_never_books_less_than_that_queue_did(self):
|
def test_an_active_entry_an_older_queue_granted_never_books_less_than_that_queue_did(self):
|
||||||
history = _load(HISTORY)
|
history = _load(HISTORY)
|
||||||
@@ -754,6 +783,7 @@ class TestQueueCommands(unittest.TestCase):
|
|||||||
)
|
)
|
||||||
signature = f"{self.base.name}: {Path(sys.executable).name} -c"
|
signature = f"{self.base.name}: {Path(sys.executable).name} -c"
|
||||||
self.assertEqual((entry["aliases"], entry["footprint"]), ([signature], None))
|
self.assertEqual((entry["aliases"], entry["footprint"]), ([signature], None))
|
||||||
|
self.assertTrue(0 < entry["granted_mem"] < 100, entry)
|
||||||
self.assertEqual(self.entries(), [])
|
self.assertEqual(self.entries(), [])
|
||||||
[row] = [json.loads(line) for line in self.history_file.read_text(encoding="utf-8").splitlines()]
|
[row] = [json.loads(line) for line in self.history_file.read_text(encoding="utf-8").splitlines()]
|
||||||
self.assertEqual(
|
self.assertEqual(
|
||||||
@@ -1111,6 +1141,52 @@ class TestQueueCommands(unittest.TestCase):
|
|||||||
self.write_config()
|
self.write_config()
|
||||||
self.assertEqual(self.bash("make compose-deploy"), 0)
|
self.assertEqual(self.bash("make compose-deploy"), 0)
|
||||||
|
|
||||||
|
def test_list_and_hold_on_a_broken_config_warn_and_use_the_shipped_default(self):
|
||||||
|
self.config.write_text(json.dumps(self.cfg)[:-1] + ",}", encoding="utf-8")
|
||||||
|
listing = self.run_queue("list")
|
||||||
|
self.assertEqual(listing.returncode, 0, listing.stderr)
|
||||||
|
self.assertIn("built-in defaults", listing.stderr)
|
||||||
|
held = self.run_queue("hold", "--kind", "nosuch", "--task", "work/a", "--session", "s1", "--", "true")
|
||||||
|
self.assertEqual(held.returncode, 2, held.stderr)
|
||||||
|
self.assertIn("built-in defaults", held.stderr)
|
||||||
|
self.assertIn("'nosuch' is not in slot_need_gb", held.stderr)
|
||||||
|
for done in (listing, held):
|
||||||
|
self.assertNotIn("Traceback", done.stderr)
|
||||||
|
|
||||||
|
def test_a_broken_config_falls_back_to_defaults_so_light_commands_pass_and_heavy_ones_stay_refused(self):
|
||||||
|
self.cfg["queue_enforce"] = True
|
||||||
|
good = json.dumps(self.cfg)
|
||||||
|
|
||||||
|
def text(content: str) -> Callable[[], None]:
|
||||||
|
return lambda: self.config.write_text(content, encoding="utf-8")
|
||||||
|
|
||||||
|
shapes = {
|
||||||
|
"comma": text(good[:-1] + ",}"), "empty": text(""), "list": text("[]"),
|
||||||
|
"string enforce": text(json.dumps({**self.cfg, "queue_enforce": "yes"})),
|
||||||
|
"empty-matching heavy": text(json.dumps({**self.cfg, "heavy_commands": [""]})),
|
||||||
|
"dot heavy": text(json.dumps({**self.cfg, "heavy_commands": ["."]})),
|
||||||
|
"word-boundary heavy": text(json.dumps({**self.cfg, "heavy_commands": [r"\b"]})),
|
||||||
|
"dot never_block": text(json.dumps({**self.cfg, "never_block": ["."]})),
|
||||||
|
"dangling link": lambda: self.config.symlink_to(self.base / "absent.json"),
|
||||||
|
"link loop": lambda: self.config.symlink_to(self.config),
|
||||||
|
"directory": lambda: self.config.mkdir(),
|
||||||
|
}
|
||||||
|
for name, make in shapes.items():
|
||||||
|
with self.subTest(config=name):
|
||||||
|
if self.config.is_dir() and not self.config.is_symlink():
|
||||||
|
self.config.rmdir()
|
||||||
|
self.config.unlink(missing_ok=True)
|
||||||
|
make()
|
||||||
|
passed = self.hook(json.dumps({"tool_input": {"command": "ls -la"}}))
|
||||||
|
self.assertEqual(passed.returncode, 0, passed.stderr)
|
||||||
|
self.assertIn(str(self.config), passed.stderr)
|
||||||
|
self.assertIn("built-in defaults", passed.stderr)
|
||||||
|
self.assertEqual(self.bash("make test"), 2)
|
||||||
|
live = ("make compose-deploy ROLE=x", "docker build .", "docker pull nginx", "make swarm-deploy", "make roundtrip",
|
||||||
|
"make worktree-up")
|
||||||
|
for command in live if name in ("comma", "directory") else live[:1]:
|
||||||
|
self.assertEqual(self.bash(command), 2, command)
|
||||||
|
|
||||||
def test_block_with_queue_enforce_refuses_heavy_commands_outside_the_queue(self):
|
def test_block_with_queue_enforce_refuses_heavy_commands_outside_the_queue(self):
|
||||||
self.cfg["queue_enforce"] = True
|
self.cfg["queue_enforce"] = True
|
||||||
self.write_config()
|
self.write_config()
|
||||||
@@ -1129,7 +1205,12 @@ class TestQueueCommands(unittest.TestCase):
|
|||||||
):
|
):
|
||||||
with self.subTest(command=command):
|
with self.subTest(command=command):
|
||||||
self.assertEqual(self.bash(command), 0)
|
self.assertEqual(self.bash(command), 0)
|
||||||
self.assertEqual(self.hook("not json").returncode, 0)
|
for payload in ("not json", json.dumps({"tool_input": ["make test"]})):
|
||||||
|
with self.subTest(payload=payload):
|
||||||
|
crashed = self.hook(payload)
|
||||||
|
self.assertEqual(crashed.returncode, 2, crashed.stderr)
|
||||||
|
self.assertIn("counts as heavy", crashed.stderr)
|
||||||
|
self.assertEqual(self.bash("CARGO_BUILD_JOBS=² cargo build ; make test"), 2)
|
||||||
self.config.unlink()
|
self.config.unlink()
|
||||||
self.assertEqual(self.bash("make compose-deploy"), 0)
|
self.assertEqual(self.bash("make compose-deploy"), 0)
|
||||||
|
|
||||||
@@ -1290,6 +1371,40 @@ class TestQueueCommands(unittest.TestCase):
|
|||||||
self.assertEqual(self.bash(command), 2)
|
self.assertEqual(self.bash(command), 2)
|
||||||
self.assertEqual(self.bash("docker compose ps"), 0)
|
self.assertEqual(self.bash("docker compose ps"), 0)
|
||||||
|
|
||||||
|
def test_a_queue_grant_counts_a_live_admit_record_and_ignores_an_expired_one(self):
|
||||||
|
def admitted(expires: float) -> None:
|
||||||
|
(self.root / "queue.json").write_text(
|
||||||
|
json.dumps({"seq": 0, "entries": [], "admits": [{"ram": 300, "expires": expires}]}), encoding="utf-8"
|
||||||
|
)
|
||||||
|
|
||||||
|
admitted(time.time() + 300)
|
||||||
|
_, err = self.blocked_hold("true")
|
||||||
|
self.assertRegex(err.read_text(encoding="utf-8"), r"RAM \d+ % used \+ 300 % still to come reach the limit 101 %")
|
||||||
|
admitted(time.time() - 1)
|
||||||
|
result = self.run_queue("hold", "--kind", "deploy", "--task", "work/a", "--session", "s2", "--", "true")
|
||||||
|
self.assertEqual((result.returncode, result.stderr), (0, ""))
|
||||||
|
|
||||||
|
def test_a_light_kind_refuses_a_heavy_command_names_the_test_kind_and_enqueues_nothing(self):
|
||||||
|
self.cfg["slot_need_gb"] |= {"test-nostack": 0, "quick": 0}
|
||||||
|
self.cfg["light_kinds"] = ["quick"]
|
||||||
|
self.write_config()
|
||||||
|
(self.base / "tests" / "e2e").mkdir(parents=True)
|
||||||
|
(self.base / "tests" / "e2e" / "test_orchestrator_e2e.py").write_text("", encoding="utf-8")
|
||||||
|
for kind, command, shown in (
|
||||||
|
("test-nostack", ("python", "-m", "unittest"), "python -m unittest"),
|
||||||
|
("test-nostack", ("pytest", "tests/e2e/test_orchestrator_e2e.py"), "pytest tests/e2e/test_orchestrator_e2e.py"),
|
||||||
|
("quick", ("bash", "-c", "make test"), "bash -c 'make test'"),
|
||||||
|
("quick", ("python3", "gate.py", "run", "--target", "test-unit"), "python3 gate.py run --target test-unit"),
|
||||||
|
):
|
||||||
|
with self.subTest(kind=kind, command=command):
|
||||||
|
result = self.run_queue("hold", "--kind", kind, "--task", "work/a", "--session", "s1", "--", *command)
|
||||||
|
self.assertEqual(result.returncode, 2)
|
||||||
|
self.assertIn(f"the light kind {kind} skips every reservation", result.stderr)
|
||||||
|
self.assertIn(f"run `queue.py hold --kind test --task <register task> -- {shown}`", result.stderr)
|
||||||
|
self.assertFalse((self.root / "queue.json").exists())
|
||||||
|
light = self.run_queue("hold", "--kind", "test-nostack", "--task", "work/a", "--session", "s1", "--", "true")
|
||||||
|
self.assertEqual((light.returncode, light.stderr), (0, ""))
|
||||||
|
|
||||||
def test_hold_refuses_a_config_without_the_budget_or_the_memory_limits(self):
|
def test_hold_refuses_a_config_without_the_budget_or_the_memory_limits(self):
|
||||||
for key, cfg in (
|
for key, cfg in (
|
||||||
("budget_percent", {k: v for k, v in self.cfg.items() if k != "budget_percent"}),
|
("budget_percent", {k: v for k, v in self.cfg.items() if k != "budget_percent"}),
|
||||||
|
|||||||
@@ -149,6 +149,53 @@ class TestCpu(unittest.TestCase):
|
|||||||
self.assertEqual(json.loads((state / "cpu.json").read_text(encoding="utf-8")), [NOW + sampler.BUCKET + 1, 9300, 12200])
|
self.assertEqual(json.loads((state / "cpu.json").read_text(encoding="utf-8")), [NOW + sampler.BUCKET + 1, 9300, 12200])
|
||||||
|
|
||||||
|
|
||||||
|
class TestPressure(unittest.TestCase):
|
||||||
|
def setUp(self):
|
||||||
|
tmp = tempfile.TemporaryDirectory()
|
||||||
|
self.addCleanup(tmp.cleanup)
|
||||||
|
self.proc, self.state = Path(tmp.name) / "proc", Path(tmp.name) / "state"
|
||||||
|
(self.proc / "pressure").mkdir(parents=True)
|
||||||
|
self.state.mkdir()
|
||||||
|
|
||||||
|
def vmstat(self, pages: int) -> None:
|
||||||
|
(self.proc / "vmstat").write_text(f"pgpgin 5\npswpin 3\npswpout {pages}\n", encoding="utf-8")
|
||||||
|
|
||||||
|
def test_psi_gives_some_and_full_avg10_and_none_without_pressure(self):
|
||||||
|
text = "some avg10=12.50 avg60=1.00 avg300=0.50 total=1\nfull avg10=6.25 avg60=0.80 avg300=0.10 total=1\n"
|
||||||
|
self.assertEqual((sampler.psi_avg10(text, "some"), sampler.psi_avg10(text, "full")), (12.5, 6.25))
|
||||||
|
self.assertEqual((sampler.psi_avg10("", "some"), sampler.psi_avg10(text.splitlines()[0], "full")), (None, None))
|
||||||
|
self.assertIsNone(sampler.swapped_pages("pgpgin 5\n"))
|
||||||
|
|
||||||
|
def test_swap_out_is_none_first_then_mb_per_minute_and_none_after_a_reset_or_a_gap(self):
|
||||||
|
with mock.patch.object(sampler, "PAGE_BYTES", 4096):
|
||||||
|
self.assertIsNone(sampler.swap_out_since(self.proc, self.state, NOW))
|
||||||
|
self.assertFalse((self.state / "swap.json").exists())
|
||||||
|
self.vmstat(1000)
|
||||||
|
self.assertIsNone(sampler.swap_out_since(self.proc, self.state, NOW))
|
||||||
|
self.vmstat(1000 + 1_400_000_000 // 4096)
|
||||||
|
self.assertEqual(sampler.swap_out_since(self.proc, self.state, NOW + 60), 1400.0)
|
||||||
|
self.vmstat(10)
|
||||||
|
self.assertIsNone(sampler.swap_out_since(self.proc, self.state, NOW + 120))
|
||||||
|
self.assertEqual(json.loads((self.state / "swap.json").read_text(encoding="utf-8")), [NOW + 120, 10])
|
||||||
|
self.assertIsNone(sampler.swap_out_since(self.proc, self.state, NOW + 120))
|
||||||
|
self.assertIsNone(sampler.swap_out_since(self.proc, self.state, NOW + 120 + sampler.BUCKET + 1))
|
||||||
|
|
||||||
|
def test_a_sample_row_adds_the_pressure_keys_and_keeps_every_other_key(self):
|
||||||
|
(self.proc / "pressure" / "memory").write_text("some avg10=3.00 avg60=0 avg300=0 total=1\n", encoding="utf-8")
|
||||||
|
found = sampler.sample(self.proc, self.proc / "sys", self.proc, lambda argv: None, self.state)
|
||||||
|
self.assertEqual(sorted(found), ["cpu", "disks", "gpus", "load", "psi_full", "psi_some", "queue", "ram", "swap", "swap_out_mb_min", "t"])
|
||||||
|
self.assertEqual((found["psi_some"], found["psi_full"], found["swap_out_mb_min"]), (3.0, None, None))
|
||||||
|
|
||||||
|
def test_the_fifteen_minute_mean_keeps_the_pressure_keys(self):
|
||||||
|
asleep = {"name": ASLEEP, "asleep": True}
|
||||||
|
rows = [
|
||||||
|
row(NOW + 60, 1.0, 1, 0, asleep) | {"psi_some": 2.0, "psi_full": 1.004, "swap_out_mb_min": 100.0},
|
||||||
|
row(NOW + 120, 1.0, 1, 0, asleep) | {"psi_some": 4.5, "psi_full": None, "swap_out_mb_min": 301.0},
|
||||||
|
]
|
||||||
|
mean = sampler.mean_row(NOW, rows)
|
||||||
|
self.assertEqual((mean["psi_some"], mean["psi_full"], mean["swap_out_mb_min"]), (3.25, 1.0, 200.5))
|
||||||
|
|
||||||
|
|
||||||
class TestGpus(unittest.TestCase):
|
class TestGpus(unittest.TestCase):
|
||||||
def setUp(self):
|
def setUp(self):
|
||||||
tmp = tempfile.TemporaryDirectory()
|
tmp = tempfile.TemporaryDirectory()
|
||||||
@@ -206,6 +253,7 @@ class TestRetention(unittest.TestCase):
|
|||||||
self.assertIs(kept[0], mean)
|
self.assertIs(kept[0], mean)
|
||||||
self.assertEqual(kept[1], {
|
self.assertEqual(kept[1], {
|
||||||
"t": bucket, "disks": [{"mount": "/", "used": 200, "total": 1000}, {"mount": "/mnt", "used": 10, "total": 20}], "cpu": 30.0, "ram": 15.0, "swap": 0.0, "load": 3.0,
|
"t": bucket, "disks": [{"mount": "/", "used": 200, "total": 1000}, {"mount": "/mnt", "used": 10, "total": 20}], "cpu": 30.0, "ram": 15.0, "swap": 0.0, "load": 3.0,
|
||||||
|
"psi_some": None, "psi_full": None, "swap_out_mb_min": None,
|
||||||
"gpus": [{"name": AWAKE, "util": 50.0, "vram_used": 300, "vram_total": 1000}], "queue": {"waiting": 1.0, "active": 1.0},
|
"gpus": [{"name": AWAKE, "util": 50.0, "vram_used": 300, "vram_total": 1000}], "queue": {"waiting": 1.0, "active": 1.0},
|
||||||
})
|
})
|
||||||
self.assertEqual((kept[2]["cpu"], kept[2]["disks"]), (40.0, [{"mount": "/", "used": 400, "total": 1000}]))
|
self.assertEqual((kept[2]["cpu"], kept[2]["disks"]), (40.0, [{"mount": "/", "used": 400, "total": 1000}]))
|
||||||
@@ -274,7 +322,7 @@ class TestSamplerInScratchHome(unittest.TestCase):
|
|||||||
self.assertEqual(len(rows), 2)
|
self.assertEqual(len(rows), 2)
|
||||||
for found in rows:
|
for found in rows:
|
||||||
with self.subTest(t=found["t"]):
|
with self.subTest(t=found["t"]):
|
||||||
self.assertEqual(sorted(found), ["cpu", "disks", "gpus", "load", "queue", "ram", "swap", "t"])
|
self.assertEqual(sorted(found), ["cpu", "disks", "gpus", "load", "psi_full", "psi_some", "queue", "ram", "swap", "swap_out_mb_min", "t"])
|
||||||
self.assertEqual(found["disks"], DISKS)
|
self.assertEqual(found["disks"], DISKS)
|
||||||
self.assertEqual(found["gpus"], [{"name": AWAKE, "util": 37.0, "vram_used": 2048 * MIB, "vram_total": 8188 * MIB}, {"name": ASLEEP, "asleep": True}])
|
self.assertEqual(found["gpus"], [{"name": AWAKE, "util": 37.0, "vram_used": 2048 * MIB, "vram_total": 8188 * MIB}, {"name": ASLEEP, "asleep": True}])
|
||||||
self.assertEqual(found["queue"], {"waiting": 1, "active": 2})
|
self.assertEqual(found["queue"], {"waiting": 1, "active": 2})
|
||||||
|
|||||||
+943
-19
File diff suppressed because it is too large.
Load diff
Reference in new issue
Block a user