Merge branch 'fix/queue-pressure-gate' into unify/skills-20261114
# Conflicts: # scripts/install.sh # skills/kvb-org-organize/scripts/watch.py
This commit is contained in:
commit
0046ab3a0a
27 files changed
+2794
-207
No files matched your search
@@ -2,12 +2,17 @@
|
||||
"""Args:
|
||||
sample: append one row to <state>/samples.jsonl: {t, disks: [{mount,
|
||||
used, total}], cpu, ram, swap, load, gpus: [{name, util, vram_used,
|
||||
vram_total} | {name, asleep: true}], queue: {waiting, active}}; t in
|
||||
epoch seconds, used, total and vram in bytes, cpu, ram, swap and util
|
||||
in percent, load the 1-minute load average, a GPU named by its PCI
|
||||
address. cpu is the busy share of the /proc/stat ticks since those
|
||||
the run before kept in <state>/cpu.json as [t, idle, total], None on
|
||||
the first run and after a gap of more than 15 minutes. An NVIDIA GPU
|
||||
vram_total} | {name, asleep: true}], queue: {waiting, active},
|
||||
psi_some, psi_full, swap_out_mb_min}; t in epoch seconds, used, total
|
||||
and vram in bytes, cpu, ram, swap and util in percent, load the
|
||||
1-minute load average, a GPU named by its PCI address. cpu is the
|
||||
busy share of the /proc/stat ticks since those the run before kept in
|
||||
<state>/cpu.json as [t, idle, total], None on the first run and after
|
||||
a gap of more than 15 minutes. psi_some and psi_full are the avg10 of
|
||||
/proc/pressure/memory, None without PSI; swap_out_mb_min the MB per
|
||||
minute swapped out (pswpout of /proc/vmstat) since the count the run
|
||||
before kept in <state>/swap.json as [t, pswpout], None like cpu and
|
||||
after a counter reset. An NVIDIA GPU
|
||||
whose PCI runtime_status is not active is recorded asleep, and
|
||||
nvidia-smi is not run for it; an awake one nvidia-smi gives no
|
||||
reading for (no driver, a driver mismatch) keeps its name with util
|
||||
@@ -76,12 +81,15 @@ NVIDIA_QUERY = "pci.bus_id,utilization.gpu,memory.used,memory.total"
|
||||
UNREAD = {"util": None, "vram_used": None, "vram_total": None}
|
||||
COUNTS = (("input", "input_tokens"), ("output", "output_tokens"), ("cache_read", "cache_read_input_tokens"), ("cache_write", "cache_creation_input_tokens"))
|
||||
NAMES = tuple(name for name, _ in COUNTS)
|
||||
HOST_DIGITS = (("cpu", 1), ("ram", 1), ("swap", 1), ("load", 2))
|
||||
HOST_DIGITS = (("cpu", 1), ("ram", 1), ("swap", 1), ("load", 2), ("psi_some", 2), ("psi_full", 2), ("swap_out_mb_min", 1))
|
||||
CLAIMS = ("*/claim.json", "*/*/claim.json")
|
||||
SAMPLES = "samples.jsonl"
|
||||
TOKENS = "tokens.jsonl"
|
||||
OFFSETS = "offsets.json"
|
||||
TICKS = "cpu.json"
|
||||
SWAPPED = "swap.json"
|
||||
PAGE_BYTES = os.sysconf("SC_PAGE_SIZE")
|
||||
MB = 1_000_000
|
||||
|
||||
|
||||
def read_text(path: Path) -> str:
|
||||
@@ -264,9 +272,43 @@ def cpu_since(proc: Path, state: Path, t: int) -> float | None:
|
||||
return cpu_percent(before, ticks)
|
||||
|
||||
|
||||
def psi_avg10(text: str, kind: str) -> float | None:
|
||||
for line in text.splitlines():
|
||||
fields = line.split()
|
||||
if fields[:1] == [kind]:
|
||||
values = dict(field.split("=", 1) for field in fields[1:] if "=" in field)
|
||||
return number(values.get("avg10", ""))
|
||||
return None
|
||||
|
||||
|
||||
def swapped_pages(text: str) -> int | None:
|
||||
for line in text.splitlines():
|
||||
fields = line.split()
|
||||
if len(fields) == 2 and fields[0] == "pswpout" and fields[1].isdigit():
|
||||
return int(fields[1])
|
||||
return None
|
||||
|
||||
|
||||
def swap_out_since(proc: Path, state: Path, t: int) -> float | None:
|
||||
"""Returns:
|
||||
The MB per minute swapped out since the pswpout count the run before
|
||||
kept in <state>/swap.json at most BUCKET seconds before t, else None,
|
||||
also after a counter reset; this run's count replaces it.
|
||||
"""
|
||||
pages = swapped_pages(read_text(proc / "vmstat"))
|
||||
if pages is None:
|
||||
return None
|
||||
stored = read_json(state / SWAPPED)
|
||||
replace(state / SWAPPED, json.dumps([t, pages]))
|
||||
if not integers(stored, 2) or not t - BUCKET <= stored[0] < t or stored[1] > pages:
|
||||
return None
|
||||
return round((pages - stored[1]) * PAGE_BYTES / MB * 60 / (t - stored[0]), 1)
|
||||
|
||||
|
||||
def sample(proc: Path, sys_root: Path, council: Path, run: Callable[[list[str]], str | None], state: Path) -> dict:
|
||||
t = int(time.time())
|
||||
ram, swap = memory(parse_meminfo(read_text(proc / "meminfo")))
|
||||
pressure = read_text(proc / "pressure" / "memory")
|
||||
return {
|
||||
"t": t,
|
||||
"disks": parse_df(run(["df", "-P", "-k", "-l"]) or ""),
|
||||
@@ -276,6 +318,9 @@ def sample(proc: Path, sys_root: Path, council: Path, run: Callable[[list[str]],
|
||||
"load": parse_loadavg(read_text(proc / "loadavg")),
|
||||
"gpus": gpus(sys_root, run),
|
||||
"queue": queue(council),
|
||||
"psi_some": psi_avg10(pressure, "some"),
|
||||
"psi_full": psi_avg10(pressure, "full"),
|
||||
"swap_out_mb_min": swap_out_since(proc, state, t),
|
||||
}
|
||||
|
||||
|
||||
|
||||
@@ -48,6 +48,11 @@ if [[ "${TARGET}" == "${HOME}" ]]; then
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
watch="${REPO_ROOT}/skills/kvb-org-organize/scripts/watch.py"
|
||||
if [[ -f "${watch}" ]] && ! python3 -I "${watch}" preflight; then
|
||||
warn "skills: add the keys above to the agents council config first; the new hooks and queue.py refuse to run without them"
|
||||
exit 2
|
||||
fi
|
||||
|
||||
cd "${REPO_ROOT}"
|
||||
src="${REPO_ROOT}/.agents/skills"
|
||||
@@ -120,6 +125,17 @@ if [[ "${TARGET}" == "${HOME}" ]]; then
|
||||
fi
|
||||
fi
|
||||
|
||||
config="${AGENTS_COUNCIL_CONFIG:-${HOME}/.claude/agents-council/config.json}"
|
||||
shipped="${REPO_ROOT}/skills/kvb-org-organize/config.default.json"
|
||||
if [[ -f "${shipped}" && ! -e "${config}" && ! -L "${config}" ]]; then
|
||||
[[ -d "$(dirname -- "${config}")" ]] || install -d -m 0700 "$(dirname -- "${config}")"
|
||||
if (umask 077 && set -o noclobber && cat -- "${shipped}" >"${config}") 2>/dev/null; then
|
||||
log "skills: wrote the shipped agents council config to ${config}"
|
||||
else
|
||||
warn "skills: kept ${config}, which appeared while the install wrote the shipped default"
|
||||
fi
|
||||
fi
|
||||
|
||||
settings="${TARGET}/.claude/settings.local.json"
|
||||
if command -v node &>/dev/null; then
|
||||
log "skills: auto-enabling caveman + ponytail plugins in ${settings}"
|
||||
|
||||
@@ -7,9 +7,8 @@ description: >
|
||||
|
||||
Where an Ansible run spends its time: its log, a controller profile, a bench.
|
||||
Report the script's lines with their log and range, never a guess. The
|
||||
script and the bench are light runs (`watch.py admit --kind test-nostack` of
|
||||
`kvb-org-organize`); a deploy to profile is heavy and goes through the
|
||||
`kvb-org-queue` skill.
|
||||
script, the bench and a deploy to profile run through the `kvb-org-queue`
|
||||
skill.
|
||||
|
||||
## 1. Pick the log
|
||||
|
||||
|
||||
@@ -60,12 +60,11 @@ It also measures wall time for `kvb-check-speed`.
|
||||
- **Command**: the targeted tests of the change or a CLI call of the
|
||||
changed code, runnable on a clean checkout: the base worktree holds no
|
||||
untracked build output, virtualenv or dependency directory.
|
||||
- **Heavy commands** (Docker, deploys, whole suites) run only inside
|
||||
- **Every run** goes inside
|
||||
`${CLAUDE_SKILL_DIR}/../kvb-org-queue/scripts/queue.py hold --kind <kind>
|
||||
--task <register task> -- python3 <measure.py> ...`, in the background,
|
||||
by the `kvb-org-queue` skill; a cancelled wait there also exits 3, but
|
||||
prints no 💸 line. A light one self-admits with
|
||||
`watch.py admit --kind test-nostack` of `kvb-org-organize`.
|
||||
prints no 💸 line.
|
||||
- Report the run you made; never rerun to pick the best result.
|
||||
|
||||
## Exit codes
|
||||
|
||||
@@ -32,10 +32,9 @@ commit starts from zero.
|
||||
`missing hotfix` means step 1.
|
||||
4. **Admit**: every gate target is heavy and runs only inside
|
||||
`${CLAUDE_SKILL_DIR}/../kvb-org-queue/scripts/queue.py hold`.
|
||||
Only a single test module
|
||||
self-admits (`${CLAUDE_SKILL_DIR}/../kvb-org-organize/scripts/watch.py
|
||||
admit --kind test-nostack`), and it never turns a gate target green. A
|
||||
hotfix gate adds `--hotfix <slug>` and takes the next free gate slot.
|
||||
Only a single test module self-admits (`kvb-org-organize` Capacity), and
|
||||
it never turns a gate target green. A hotfix gate adds `--hotfix <slug>`
|
||||
and takes the next free gate slot.
|
||||
5. **Run** the missing and red targets, tracked tree clean, in the
|
||||
background; give the operator `tail -f` on the log:
|
||||
|
||||
|
||||
@@ -15,11 +15,8 @@ skill's Rules apply here too.
|
||||
first error.
|
||||
3. **Fix at the root**: apply the real fix for each proven root cause in the
|
||||
sidetree, with a test and a negative control as the `kvb-org-swarm` skill
|
||||
demands. Light runs (no Docker, about a minute: one test file or
|
||||
module, ruff) self-admit with
|
||||
`${CLAUDE_SKILL_DIR}/../kvb-org-organize/scripts/watch.py admit --kind
|
||||
test-nostack`; `make test`, gates, deploys and anything with Docker run in
|
||||
the background through
|
||||
demands. Light runs self-admit (`kvb-org-organize` Capacity); `make test`,
|
||||
gates, deploys and anything with Docker run in the background through
|
||||
`${CLAUDE_SKILL_DIR}/../kvb-org-queue/scripts/queue.py hold --kind <kind>
|
||||
--task <register task> -- <command>`, which waits for the rule-based
|
||||
grant and releases on exit. The fix enters the main line only
|
||||
|
||||
@@ -26,9 +26,8 @@ cover the change set run now and give the green baseline; a suite already red
|
||||
is reported, not refactored around. No coverage: read `net.md` next to this
|
||||
file before editing.
|
||||
|
||||
Light runs (one test file or module, ruff) start once `watch.py
|
||||
admit --kind test-nostack` of the `kvb-org-organize` skill prints `grant`; no
|
||||
chair request.
|
||||
Light runs start once `watch.py admit` of the `kvb-org-organize` skill prints
|
||||
`grant`; no chair request.
|
||||
|
||||
## Step 3: refactor each file
|
||||
|
||||
|
||||
@@ -38,11 +38,8 @@ any listed name. Never silently truncate.
|
||||
## Step 3: run
|
||||
|
||||
Before each run, admit it under the `kvb-org-organize` Capacity rules: a
|
||||
light run (no Docker, about a minute: one test file or module, ruff)
|
||||
self-admits with
|
||||
`${CLAUDE_SKILL_DIR}/../kvb-org-organize/scripts/watch.py admit --kind
|
||||
test-nostack`; `make test`, quality gates and anything with Docker run in the
|
||||
background through
|
||||
light run self-admits; `make test`, quality gates and anything with Docker
|
||||
run in the background through
|
||||
`${CLAUDE_SKILL_DIR}/../kvb-org-queue/scripts/queue.py hold --kind <kind>
|
||||
--task <register task> -- make <target>`, which waits for the rule-based
|
||||
grant and releases on exit.
|
||||
|
||||
@@ -49,9 +49,8 @@ single-select `Write it` / `Revise it` / `Discard` with the drafted diff.
|
||||
owner through the `kvb-org-coordinate` skill.
|
||||
- Every carried test fails without the code it guards: show the negative
|
||||
control once.
|
||||
- Run the carried tests as a light run (no Docker, about a minute) after
|
||||
`${CLAUDE_SKILL_DIR}/../kvb-org-organize/scripts/watch.py admit --kind
|
||||
test-nostack`; `make test`, gates and anything with Docker only through
|
||||
- Run the carried tests as a light run (`kvb-org-organize` Capacity);
|
||||
`make test`, gates and anything with Docker only through
|
||||
`queue.py hold` (the `kvb-org-queue` skill). Leave the result
|
||||
uncommitted for the `kvb-git-commit-next` skill.
|
||||
- `transcend.py mark` every offered candidate with its verdict.
|
||||
|
||||
@@ -71,10 +71,8 @@ order:
|
||||
Show the full draft, then `AskUserQuestion`: `Write it` / `Revise it` /
|
||||
`Discard`. Revise loops back to the step the correction touches. Write only
|
||||
on `Write it`; lint the file with the project's markdown linter, a light run
|
||||
that self-admits with
|
||||
`${CLAUDE_SKILL_DIR}/../kvb-org-organize/scripts/watch.py admit --kind
|
||||
test-nostack`; a lint through `make` or Docker runs through `queue.py hold`
|
||||
(the `kvb-org-queue` skill).
|
||||
(`kvb-org-organize` Capacity); a lint through `make` or Docker runs through
|
||||
`queue.py hold` (the `kvb-org-queue` skill).
|
||||
|
||||
## Step 6: hand over
|
||||
|
||||
|
||||
@@ -35,8 +35,8 @@ request, so a red one blocks nothing else.
|
||||
(`search_graph`, `trace_path`, `get_architecture`, `detect_changes`; the
|
||||
`kvb-agent-optimize` skill sets it up), grep and read after. Parallel
|
||||
work, gates, deploys and anything with Docker run through `queue.py hold`
|
||||
(`kvb-org-queue`); light runs (no Docker, about a minute) self-admit with
|
||||
`watch.py admit --kind test-nostack`. With the first change, swap the label: `gh issue
|
||||
(`kvb-org-queue`); light runs self-admit (`kvb-org-organize` Capacity).
|
||||
With the first change, swap the label: `gh issue
|
||||
edit <n> --remove-label "in analysis" --add-label "in progress"`.
|
||||
5. **Open the pull request** with the `kvb-git-pr-create` skill, closing
|
||||
exactly its issue; one for a failing default branch is posted with
|
||||
|
||||
@@ -37,8 +37,13 @@ ends or goes stale, or after a restart.
|
||||
|
||||
## Capacity
|
||||
|
||||
- Light runs (no Docker, about a minute: one test file or module, ruff)
|
||||
self-admit: `watch.py admit --kind test-nostack`, start on `grant`.
|
||||
- Light runs self-admit: `watch.py admit --kind test-nostack -- <command>`,
|
||||
start on `grant`. Light is only ruff, shellcheck, markdownlint, lizard, jq,
|
||||
py_compile, cargo fmt, `env CARGO_BUILD_JOBS=4 cargo build|check|clippy|test`,
|
||||
or one test file or module outside `e2e/` that reaches no Playwright;
|
||||
anything else: `queue.py hold --kind test`.
|
||||
- Memory stop (also PSI or swap-out) blocks every admit and grant; running
|
||||
jobs go on.
|
||||
- Other test runs, `make test`, `make test-lint`, quality gates, deploys,
|
||||
iteration loops, swarm, image builds, cloud, shared-data cleanup, anything
|
||||
with Docker or over a few minutes: run through `queue.py hold`
|
||||
|
||||
@@ -63,3 +63,16 @@ and stays silent below the thresholds.
|
||||
|
||||
`watch.py status` shows the stage computed from the live free space and the
|
||||
stored samples, and the age of the stored measurement.
|
||||
|
||||
An unusable config (broken JSON, wrong types, a regex refusing a light
|
||||
probe such as `grep` or freeing a heavy one such as `make swarm-deploy`, a
|
||||
dangling link, a directory) makes the hooks, admit, status and the queue
|
||||
warn and use `config.default.json`, which the install writes when no config
|
||||
exists; without that file every command, teardown included, is refused.
|
||||
The probes are samples, so a pattern hitting only other commands passes. A
|
||||
package script is read from the nearest package.json, a named workspace
|
||||
only from a folder of that name. The light-run check misses
|
||||
`sys.path.insert` in a conftest, editable installs without pytest
|
||||
`pythonpath`, browser libraries outside its list and generic third-party
|
||||
fixtures such as pytest-selenium's `driver`, browsers started by a
|
||||
subprocess, and `git bisect run` or a suite script behind `bash`.
|
||||
@@ -0,0 +1,37 @@
|
||||
{
|
||||
"disk": "/",
|
||||
"thresholds_gb": {"warn": 40, "stop": 15, "critical": 12},
|
||||
"projection_minutes": 30,
|
||||
"interval_seconds": 120,
|
||||
"cache_interval_seconds": 1800,
|
||||
"heavy_commands": [
|
||||
"\\bmake\\s+(-C\\s+\\S+\\s+)?(compose-deploy|compose-up|compose-playwright|roundtrip|quality|quality-high|test|test-integration|test-unit\\S*|swarm-\\S+|act-workflow|act-runner-image|build\\S*|worktree-up|store-\\S+)(\\s|$)",
|
||||
"\\bdocker\\s+(build|pull|load)\\b",
|
||||
"\\bdocker\\s+buildx\\s+build\\b",
|
||||
"\\brun\\.sh\\s+local\\b"
|
||||
],
|
||||
"never_block": [
|
||||
"\\bmake\\s+(-C\\s+\\S+\\s+)?(swarm-down|swarm-clean|swarm-clean-stale-nfs|swarm-diagnostic|compose-down|compose-entity-purge|worktree-down|store-host-down)(\\s|$)",
|
||||
"^\\s*git\\s+commit\\b.*\\s--no-verify\\b",
|
||||
"\\bmake\\s+swarm-exec(\\s|$)"
|
||||
],
|
||||
"queue_enforce": true,
|
||||
"cleanup": [],
|
||||
"worktree_cleanup": [],
|
||||
"worktrees_command": "true",
|
||||
"worktree_marker": ".git",
|
||||
"busy_processes": ["make"],
|
||||
"cache_dirs": ["~/.cache/pip", "~/.npm", "~/.cache/pre-commit", "~/.cache/act", "~/.cache/ms-playwright"],
|
||||
"cache_budget_percent": 3,
|
||||
"slot_need_gb": {"quality-high": 15, "test": 15, "test-nostack": 0, "iteration": 20, "swarm": 30, "gui-dind": 65, "operator": 10},
|
||||
"notify": ["notify-send", "-u", "critical", "Agents council"],
|
||||
"load_limits": {"cpu_percent": 90, "mem_percent": 85, "sample_seconds": 5, "kinds": ["test", "test-nostack", "quality-high"]},
|
||||
"memory_limits": {
|
||||
"warn": {"available": 15, "swap_free": 40, "psi_some": 10, "psi_full": 5, "swap_out_mb_min": 64, "container_swap_gb": 4},
|
||||
"stop": {"available": 8, "swap_free": 10, "psi_full": 5, "swap_out_mb_min": 256, "container_swap_gb": 16}
|
||||
},
|
||||
"budget_percent": {"ram": 75, "cpu": 100},
|
||||
"light_kinds": ["test-nostack"],
|
||||
"exclusive_kinds": ["quality-high", "test"],
|
||||
"gate_excluded_targets": ["test-oracle"]
|
||||
}
|
||||
File diff suppressed because it is too large.
Load diff
@@ -27,9 +27,8 @@ turn: hold runs the command on its grant and exits with its code.
|
||||
- In a worktree the command stays `gate.py run|hold` (`kvb-check-gate`):
|
||||
`queue.py hold ... -- python3 <gate.py> run ...`.
|
||||
|
||||
One test file or module, or ruff, without Docker self-admits with
|
||||
`watch.py admit --kind test-nostack` (`kvb-org-organize`); any other test
|
||||
run goes through hold with kind `test`.
|
||||
Light runs (`kvb-org-organize` Capacity) self-admit; anything else goes
|
||||
through hold with kind `test`, and a light kind refuses it.
|
||||
|
||||
## Order and grant
|
||||
|
||||
@@ -77,8 +76,8 @@ learned values, `cancel` removes your own waiting job.
|
||||
one down, and `kvb-git-hotfix` may void a gate. A `kvb-org-queue-cleanup`
|
||||
round deletes only entries whose lock no living process holds.
|
||||
- With `queue_enforce: true`, the `block` hook (PreToolUse on Bash) refuses
|
||||
`heavy_commands`, image builds, `make install|verify-*`, tests beyond one
|
||||
file and `gate.py run|hold` outside `queue.py hold`.
|
||||
`heavy_commands`, image builds, `make install|test*|quality*|verify-*`,
|
||||
tests beyond one unit file and `gate.py run|hold` outside `queue.py hold`.
|
||||
- Without the config, hold exits 2: tell the operator and ask the chair
|
||||
(`kvb-org-organize`) until it exists. A waiting hold rereads it every poll
|
||||
(a broken file keeps the last good one), so a limit change needs no re-hold.
|
||||
@@ -39,7 +39,9 @@ Args:
|
||||
SIBLINGS); a poll that finds other code installed in
|
||||
~/.agents/skills leaves the queue ungranted and restarts the same hold
|
||||
with the installed queue.py.
|
||||
--kind: a key of slot_need_gb in the config.
|
||||
--kind: a key of slot_need_gb in the config; a light one
|
||||
(light_kinds or watch.LIGHT_KIND) refuses a command watch.light()
|
||||
does not recognise, naming watch.queue_hold().
|
||||
--task: an open task of the kvb-org-rasci register; its rank orders
|
||||
the queue, and the operator's agent order (`rasci.py agents
|
||||
order`) of the holding session breaks ties. While one of its
|
||||
@@ -88,7 +90,8 @@ disk of running and granted jobs, never a reservation; queue_enforce (bool), opt
|
||||
only. The watch.py keys disk, thresholds_gb (stop is the floor every grant
|
||||
leaves free), slot_need_gb (the disk seed per kind), load_limits (every grant
|
||||
also needs CPU busy below cpu_percent and RAM used plus the job's learned RAM
|
||||
share below mem_percent; both bound the learned slot limits), memory_limits (no
|
||||
share, the live admit records (admit()) and the young jobs' ramp
|
||||
(ramping_ram()) below mem_percent; both bound the learned slot limits), memory_limits (no
|
||||
grant at memory stage stop), heavy_commands and never_block are used as
|
||||
watch.py documents them. A signature is `<repository>: <command signature>`
|
||||
(aliases()).
|
||||
@@ -166,6 +169,10 @@ XDIST_LONG = "--numprocesses="
|
||||
ADDOPTS = "PYTEST_ADDOPTS="
|
||||
FROZEN = "footprint"
|
||||
ALIASES = "aliases"
|
||||
ADMIT = "admit"
|
||||
ADMITS = "admits"
|
||||
ADMIT_SECONDS = 120
|
||||
GRANTED_MEM = "granted_mem"
|
||||
|
||||
|
||||
@functools.cache
|
||||
@@ -207,7 +214,7 @@ def fail(message: str) -> NoReturn:
|
||||
|
||||
|
||||
def config() -> dict:
|
||||
cfg = watch.load_config()
|
||||
cfg, _ = watch.announced_config()
|
||||
if cfg is None:
|
||||
fail(f"no config at {watch.CONFIG}; the queue reads the kvb-org-organize watch config")
|
||||
problem = watch.missing(cfg, (*BUDGET_KEYS, *watch.MEMORY_KEYS))
|
||||
@@ -219,13 +226,14 @@ def config() -> dict:
|
||||
def fresh(cfg: dict) -> dict:
|
||||
"""Returns:
|
||||
The config as it stands now, else cfg while the file is missing,
|
||||
unreadable or incomplete, so a half-saved edit never ends a wait.
|
||||
unusable (watch.checked_config()) or incomplete, so a half-saved
|
||||
edit never ends a wait.
|
||||
"""
|
||||
try:
|
||||
now = watch.load_config()
|
||||
except (OSError, ValueError):
|
||||
now = watch.checked_config(watch.load_config())
|
||||
except watch.UNUSABLE:
|
||||
return cfg
|
||||
if not isinstance(now, dict) or watch.missing(now, (*BUDGET_KEYS, *watch.MEMORY_KEYS)) is not None:
|
||||
if watch.missing(now, (*BUDGET_KEYS, *watch.MEMORY_KEYS)) is not None:
|
||||
return cfg
|
||||
return now
|
||||
|
||||
@@ -691,11 +699,11 @@ def footprint(entry: dict, signatures: dict[tuple[str, str], dict], kinds: dict[
|
||||
|
||||
|
||||
def held_back(cfg: dict, entries: list[dict], now: float, signatures: dict[str, dict], kinds: dict[str, dict],
|
||||
cpu: float, mem: float) -> dict[str, str]:
|
||||
cpu: float, mem: float, admitted: float) -> dict[str, str]:
|
||||
loaded = {}
|
||||
for entry in entries:
|
||||
if entry["status"] == "waiting":
|
||||
pending = pending_ram(entries, entry["id"], now, signatures, kinds)
|
||||
pending = pending_ram(entries, entry["id"], now, signatures, kinds, mem, admitted)
|
||||
if (reason := load_reason(cfg["load_limits"], cpu, mem, pending)) is not None:
|
||||
loaded[entry["id"]] = reason
|
||||
return loaded
|
||||
@@ -743,18 +751,36 @@ def ram_share(entry: dict, signatures: dict[str, dict], kinds: dict[str, dict])
|
||||
return history.seed_ram(kinds) if share is None else share["ram"]
|
||||
|
||||
|
||||
def pending_ram(entries: list[dict], entry_id: str, now: float, signatures: dict[str, dict],
|
||||
kinds: dict[str, dict]) -> float:
|
||||
def ramping_ram(entries: list[dict], now: float, signatures: dict[str, dict], kinds: dict[str, dict],
|
||||
mem: float) -> float:
|
||||
"""Returns:
|
||||
The RAM share of entry_id plus the whole share of every active job
|
||||
granted less than RAMP_SECONDS ago (or without a grant time): what
|
||||
may still come once they ramp up. An older job counts as ramped, its
|
||||
RAM inside the RAM used; one that peaks later is bounded by the
|
||||
memory stage only.
|
||||
What the active jobs granted less than RAMP_SECONDS ago (or without a
|
||||
grant time) may still add: the shares of those whose grant recorded
|
||||
the RAM used (GRANTED_MEM) minus what RAM used gained since the
|
||||
earliest of these grants, between 0 and those shares (a RAM used
|
||||
that fell since counts as no gain), plus the whole share of the
|
||||
others. An older job counts as ramped, its RAM inside the RAM used;
|
||||
one that peaks later is bounded by the memory stage only.
|
||||
"""
|
||||
young = [e for e in entries if e["status"] == "active" and now - e.get("granted", now) < RAMP_SECONDS]
|
||||
dated = [e for e in young if GRANTED_MEM in e]
|
||||
gained = max(0.0, mem - min((e[GRANTED_MEM] for e in dated), default=mem))
|
||||
ramp = max(0.0, sum(ram_share(e, signatures, kinds) for e in dated) - gained)
|
||||
return ramp + sum(ram_share(e, signatures, kinds) for e in young if GRANTED_MEM not in e)
|
||||
|
||||
|
||||
def pending_ram(entries: list[dict], entry_id: str, now: float, signatures: dict[str, dict],
|
||||
kinds: dict[str, dict], mem: float, admitted: float) -> float:
|
||||
"""Returns:
|
||||
The RAM share of entry_id plus admitted (admitted_ram()) plus
|
||||
ramping_ram() at RAM used mem.
|
||||
"""
|
||||
mine = [e for e in entries if e["id"] == entry_id]
|
||||
young = [e for e in entries if e["status"] == "active" and now - e.get("granted", now) < RAMP_SECONDS]
|
||||
return ram_share(mine[0], signatures, kinds) + sum(ram_share(e, signatures, kinds) for e in young)
|
||||
return ram_share(mine[0], signatures, kinds) + admitted + ramping_ram(entries, now, signatures, kinds, mem)
|
||||
|
||||
|
||||
def admitted_ram(data: dict, now: float) -> float:
|
||||
return sum(record["ram"] for record in data.get(ADMITS, []) if record["expires"] > now)
|
||||
|
||||
|
||||
def workdir() -> str:
|
||||
@@ -823,13 +849,13 @@ def wait(cfg: dict, entry_id: str, caught: list[int]) -> tuple[int | None, int]:
|
||||
print(f"queue: {entry_id} was cancelled", file=sys.stderr)
|
||||
return CANCELLED, 0
|
||||
rejoin(data, mine[0], now)
|
||||
loaded = held_back(cfg, data["entries"], now, signatures, kinds, cpu, mem)
|
||||
loaded = held_back(cfg, data["entries"], now, signatures, kinds, cpu, mem, admitted_ram(data, now))
|
||||
verdicts = decide(cfg, data["entries"], free, now, signatures, kinds, loaded)
|
||||
reason = verdicts[entry_id] or loaded.get(entry_id)
|
||||
mine[0]["reason"] = reason
|
||||
if reason is None:
|
||||
concurrent = sum(e["status"] == "active" for e in data["entries"])
|
||||
mine[0].update(status="active", granted=now, **{FROZEN: footprint(mine[0], signatures, kinds)})
|
||||
mine[0].update(status="active", granted=now, **{GRANTED_MEM: mem, FROZEN: footprint(mine[0], signatures, kinds)})
|
||||
return None, concurrent
|
||||
position = list(verdicts).index(entry_id) + 1
|
||||
if (position, reason) != shown:
|
||||
@@ -913,15 +939,27 @@ def keep_receipt(gates: ModuleType | None, args: argparse.Namespace, here: str,
|
||||
print(f"queue: receipt {args.receipt} for tree {before}", file=sys.stderr)
|
||||
|
||||
|
||||
def cmd_hold(args: argparse.Namespace) -> int:
|
||||
cfg = config()
|
||||
def check_hold(args: argparse.Namespace, cfg: dict) -> None:
|
||||
"""Args:
|
||||
args: the hold arguments; fail() exits on an unknown kind, a command
|
||||
that is no light run under a light kind, a task outside the
|
||||
register or a missing hotfix row.
|
||||
cfg: the watch config.
|
||||
"""
|
||||
if args.kind not in cfg["slot_need_gb"]:
|
||||
fail(f"kind {args.kind!r} is not in slot_need_gb of {watch.CONFIG}")
|
||||
shown = shlex.join(args.command)
|
||||
if args.kind in {watch.LIGHT_KIND, *cfg.get("light_kinds", [])} and not watch.light(shown, os.getcwd()):
|
||||
fail(f"{shown} is no light run and the light kind {args.kind} skips every reservation; run `{watch.queue_hold(shown)}`")
|
||||
rasci = sibling("rasci")
|
||||
if args.task not in rasci.merged(False):
|
||||
fail(f"{args.task} is no open task of the kvb-org-rasci register")
|
||||
if args.hotfix is not None and HOTFIX + args.hotfix not in rasci.load():
|
||||
fail(f"the register has no row {HOTFIX}{args.hotfix}")
|
||||
|
||||
def cmd_hold(args: argparse.Namespace) -> int:
|
||||
cfg = config()
|
||||
check_hold(args, cfg)
|
||||
here = workdir()
|
||||
gates, out = gated(args, here)
|
||||
names = aliases(here, args.command, os.getcwd())
|
||||
@@ -1084,6 +1122,38 @@ def held_commands(command: str) -> list[list[str]]:
|
||||
]
|
||||
|
||||
|
||||
def enforced(cfg: dict) -> dict:
|
||||
return {**cfg, "heavy_commands": [*cfg["heavy_commands"], GATE.pattern]}
|
||||
|
||||
|
||||
def admit(cfg: dict, kind: str, argv: list[str], cpu: float, mem: float, calm: bool) -> str | None:
|
||||
"""Args:
|
||||
cfg: the config.
|
||||
kind: the kind of the light job that asks to start outside the queue.
|
||||
argv: its command, run in the current directory.
|
||||
cpu: CPU busy now, in percent.
|
||||
mem: RAM used now, in percent.
|
||||
calm: whether the memory stage is ok; the live active jobs' ramp
|
||||
(ramping_ram()) counts only when it is not.
|
||||
|
||||
Returns:
|
||||
load_reason() over the job's RAM share plus the live admit records
|
||||
(admitted_ram()) and the ramp; under the store lock, a grant records
|
||||
the share for ADMIT_SECONDS and expired records leave.
|
||||
"""
|
||||
now = time.time()
|
||||
signatures, kinds = learned(cfg, now)
|
||||
mine = {"id": ADMIT, "status": "waiting", "kind": kind, ALIASES: list(aliases(workdir(), argv, os.getcwd()))}
|
||||
share = ram_share(mine, signatures, kinds)
|
||||
with store() as data:
|
||||
active = [e for e in data["entries"] if e["status"] == "active" and living(e)]
|
||||
ramp = 0.0 if calm else ramping_ram(active, now, signatures, kinds, mem)
|
||||
reason = load_reason(cfg["load_limits"], cpu, mem, share + admitted_ram(data, now) + ramp)
|
||||
granted = [{"ram": share, "expires": now + ADMIT_SECONDS}] if reason is None else []
|
||||
data[ADMITS] = [record for record in data.get(ADMITS, []) if record["expires"] > now] + granted
|
||||
return reason
|
||||
|
||||
|
||||
def tails(argv: list[str]) -> list[str]:
|
||||
return [shlex.join(argv[at:]) for at, word in enumerate(argv) if not word.startswith("-")]
|
||||
|
||||
@@ -1114,11 +1184,10 @@ def cmd_block() -> int:
|
||||
"hookEventName": "PreToolUse", "permissionDecision": "ask", "permissionDecisionReason": reason,
|
||||
}}))
|
||||
return 0
|
||||
cfg = watch.load_config()
|
||||
cfg, _ = watch.announced_config()
|
||||
if cfg is None or cfg.get("queue_enforce") is not True:
|
||||
return 0
|
||||
gated = {**cfg, "heavy_commands": [*cfg["heavy_commands"], GATE.pattern]}
|
||||
if watch.heavy(raw, gated, (THROUGH_QUEUE.pattern, *cfg["never_block"]), watch.payload_cwd(payload)):
|
||||
if watch.heavy(raw, enforced(cfg), (THROUGH_QUEUE.pattern, *cfg["never_block"]), watch.payload_cwd(payload)):
|
||||
print(
|
||||
f"kvb-org-queue: heavy jobs run through the queue: start `python3 {Path(__file__).resolve()} hold "
|
||||
"--kind <kind> --task <register task> -- <command>` in the background; it runs the command on its grant.",
|
||||
@@ -1157,8 +1226,10 @@ def main() -> int:
|
||||
if args.cmd == "block":
|
||||
try:
|
||||
return cmd_block()
|
||||
except (OSError, ValueError, KeyError, TypeError, AttributeError, re.error):
|
||||
return 0
|
||||
except Exception as error: # noqa: BLE001
|
||||
print(f"kvb-org-queue: the block hook failed ({error!r}), so the command counts as heavy and is refused.",
|
||||
file=sys.stderr)
|
||||
return 2
|
||||
commands = {"hold": cmd_hold, "list": cmd_list, "history": cmd_history, "cancel": cmd_cancel}
|
||||
try:
|
||||
return commands[args.cmd](args)
|
||||
|
||||
@@ -31,9 +31,8 @@ picks do; every other session keeps its own.
|
||||
|
||||
## Night rules for the active session
|
||||
|
||||
- **Grants**: request none; admit your own runs with `watch.py admit --kind
|
||||
<slot>` (`${CLAUDE_SKILL_DIR}/../kvb-org-organize/scripts/watch.py`), whose
|
||||
disk, CPU and RAM limits still apply.
|
||||
- **Grants**: request none; light runs self-admit and heavy ones queue
|
||||
(`kvb-org-organize` Capacity), whose limits still apply.
|
||||
- **Offices**: the suspended agents' claims stay theirs. Take only free work;
|
||||
a piece a suspended agent holds waits for the resume or the operator.
|
||||
- **Messages**: no wakes and no peer questions; the Peer protocol's wait is
|
||||
|
||||
@@ -30,10 +30,9 @@ sidetree or leaves as a proposal.
|
||||
- code explored through the codebase-memory-mcp graph first
|
||||
(`search_graph`, `trace_path`, `get_architecture`, `detect_changes`; the
|
||||
`kvb-agent-optimize` skill sets it up), grep and read after;
|
||||
- light runs (no Docker, about a minute) self-admit with
|
||||
`${CLAUDE_SKILL_DIR}/../kvb-org-organize/scripts/watch.py admit --kind
|
||||
test-nostack`; `make test`, gates, deploys and anything with Docker only
|
||||
through `queue.py hold` (`kvb-org-queue`);
|
||||
- light runs self-admit (`kvb-org-organize` Capacity); `make test`, gates,
|
||||
deploys and anything with Docker only through `queue.py hold`
|
||||
(`kvb-org-queue`);
|
||||
- status that asks nothing goes to
|
||||
`${CLAUDE_SKILL_DIR}/../kvb-org-radio/scripts/radio.py post "<text>"`,
|
||||
never into a wake.
|
||||
|
||||
@@ -58,9 +58,9 @@ and the prose diff.
|
||||
- Write the script, then replace the compiled prose with its call
|
||||
(`${CLAUDE_SKILL_DIR}/scripts/<verb>.py ...`) and one line on reading its
|
||||
output.
|
||||
- Every script gets a unit test with a negative control shown once, run once
|
||||
`watch.py admit --kind test-nostack` (the `kvb-org-organize` skill) prints
|
||||
`grant`; `make test` runs through `queue.py hold` (`kvb-org-queue`).
|
||||
- Every script gets a unit test with a negative control shown once, run as a
|
||||
light run (`kvb-org-organize` Capacity); `make test` runs through
|
||||
`queue.py hold` (`kvb-org-queue`).
|
||||
- In another repository follow its agent rules (claims, worktree, branch); a
|
||||
file another line owns goes to that line through the `kvb-org-coordinate`
|
||||
skill.
|
||||
|
||||
@@ -52,9 +52,8 @@ or diff.
|
||||
- Read every script whole before carrying it; clean it the way the
|
||||
`kvb-code-clean` skill does.
|
||||
- A carried script gets a unit test in the repository's `tests/` with a
|
||||
negative control, run once `watch.py admit --kind test-nostack` (the
|
||||
`kvb-org-organize` skill) prints `grant`; `make test` runs through
|
||||
`queue.py hold` (`kvb-org-queue`).
|
||||
negative control, run as a light run (`kvb-org-organize` Capacity);
|
||||
`make test` runs through `queue.py hold` (`kvb-org-queue`).
|
||||
- `transcend.py mark` every offered candidate with its verdict.
|
||||
- Leave the result uncommitted; committing is the `kvb-git-commit-next`
|
||||
skill's job.
|
||||
|
||||
+3
-1
@@ -11,7 +11,9 @@ from pathlib import Path
|
||||
REPO_ROOT = Path(__file__).resolve().parents[1]
|
||||
GATES = REPO_ROOT / "skills" / "kvb-check-gate" / "scripts" / "gates.py"
|
||||
QUEUE = REPO_ROOT / "skills" / "kvb-org-queue" / "scripts" / "queue.py"
|
||||
OPEN_MEMORY = {"available": -1, "swap_free": -1, "psi_full": 1000, "container_swap_gb": 1e9}
|
||||
OPEN_MEMORY = {
|
||||
"available": -1, "swap_free": -1, "psi_some": 1000, "psi_full": 1000, "swap_out_mb_min": 1e9, "container_swap_gb": 1e9,
|
||||
}
|
||||
|
||||
|
||||
class Scratch(unittest.TestCase):
|
||||
|
||||
+66
-4
@@ -2,6 +2,7 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
@@ -10,15 +11,26 @@ import unittest
|
||||
from pathlib import Path
|
||||
|
||||
INSTALL = Path(__file__).resolve().parents[1] / "scripts" / "install.sh"
|
||||
ORGANIZE = Path(__file__).resolve().parents[1] / "skills" / "kvb-org-organize"
|
||||
NPX = "#!/bin/sh\nmkdir -p .agents/skills/locked\necho locked > .agents/skills/locked/SKILL.md\n"
|
||||
BYTECODE = ("__pycache__/radio.cpython-312.pyc", "json.pyc", "legacy.pyo")
|
||||
LIVE_MEMORY = {
|
||||
"warn": {"available": 15, "swap_free": 40, "psi_full": 5, "container_swap_gb": 4},
|
||||
"stop": {"available": 8, "swap_free": 10, "psi_full": 10, "container_swap_gb": 16},
|
||||
}
|
||||
SHIPPED = ORGANIZE / "config.default.json"
|
||||
DEFAULT = json.loads(SHIPPED.read_text(encoding="utf-8"))
|
||||
ROW_MEMORY = DEFAULT["memory_limits"]
|
||||
|
||||
|
||||
class TestInstall(unittest.TestCase):
|
||||
def install(self, root: Path, target: Path, npx: str = NPX, planted: tuple[str, ...] = ()) -> None:
|
||||
def install(self, root: Path, target: Path, npx: str = NPX, planted: tuple[str, ...] = (), config: Path | None = None,
|
||||
code: int = 0, stubs_extra: tuple[tuple[str, str], ...] = ()) -> subprocess.CompletedProcess:
|
||||
repo, stubs = root / "repo", root / "bin"
|
||||
(repo / "scripts").mkdir(parents=True)
|
||||
shutil.copy(INSTALL, repo / "scripts" / "install.sh")
|
||||
if config is not None:
|
||||
shutil.copytree(ORGANIZE, repo / "skills" / ORGANIZE.name, ignore=shutil.ignore_patterns("__pycache__"))
|
||||
(repo / "scripts" / "enable-plugins.js").write_text("", encoding="utf-8")
|
||||
(repo / "skills" / "own").mkdir(parents=True)
|
||||
(repo / "skills" / "own" / "SKILL.md").write_text("own", encoding="utf-8")
|
||||
@@ -26,11 +38,61 @@ class TestInstall(unittest.TestCase):
|
||||
(repo / name).parent.mkdir(parents=True, exist_ok=True)
|
||||
(repo / name).write_bytes(b"planted")
|
||||
stubs.mkdir()
|
||||
(stubs / "npx").write_text(npx, encoding="utf-8")
|
||||
(stubs / "npx").chmod(0o755)
|
||||
for name, text in (("npx", npx), *stubs_extra):
|
||||
(stubs / name).write_text(text, encoding="utf-8")
|
||||
(stubs / name).chmod(0o755)
|
||||
env = {**os.environ, "PATH": f"{stubs}:/usr/bin:/bin", "TARGET": str(target), "HOME": str(root / "home")}
|
||||
if config is not None:
|
||||
env |= {"AGENTS_COUNCIL_CONFIG": str(config), "AGENTS_COUNCIL_ROOT": str(root / "council")}
|
||||
done = subprocess.run(["bash", str(repo / "scripts" / "install.sh")], env=env, capture_output=True, text=True, check=False)
|
||||
self.assertEqual(done.returncode, 0, done.stderr)
|
||||
self.assertEqual(done.returncode, code, done.stderr)
|
||||
return done
|
||||
|
||||
def test_the_install_stops_before_any_copy_while_the_live_config_lacks_a_memory_bound_the_new_code_needs(self):
|
||||
for limits, code in ((LIVE_MEMORY, 2), (ROW_MEMORY, 0)):
|
||||
with self.subTest(code=code), tempfile.TemporaryDirectory() as tmp:
|
||||
root = Path(tmp)
|
||||
target = root / "target"
|
||||
target.mkdir()
|
||||
config = root / "config.json"
|
||||
config.write_text(json.dumps({**DEFAULT, "memory_limits": limits}), encoding="utf-8")
|
||||
before = config.stat()
|
||||
done = self.install(root, target, config=config, code=code)
|
||||
after = config.stat()
|
||||
self.assertEqual((after.st_ino, after.st_mtime_ns, after.st_mode), (before.st_ino, before.st_mtime_ns, before.st_mode))
|
||||
installed = sorted(path.name for path in target.iterdir())
|
||||
if code:
|
||||
self.assertIn(
|
||||
"lacks memory_limits.warn.psi_some, memory_limits.warn.swap_out_mb_min, memory_limits.stop.swap_out_mb_min;",
|
||||
done.stderr,
|
||||
)
|
||||
self.assertEqual(installed, [])
|
||||
self.assertFalse((root / "repo" / ".agents").exists())
|
||||
else:
|
||||
self.assertEqual(installed, [".agents", ".claude"])
|
||||
self.assertEqual(json.loads(config.read_text(encoding="utf-8")), {**DEFAULT, "memory_limits": limits})
|
||||
|
||||
def test_an_install_without_a_config_writes_the_shipped_default_as_the_initial_one(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = Path(tmp)
|
||||
target = root / "target"
|
||||
target.mkdir()
|
||||
config = root / "council-config" / "config.json"
|
||||
self.install(root, target, config=config)
|
||||
self.assertEqual(config.read_bytes(), SHIPPED.read_bytes())
|
||||
self.assertEqual(config.stat().st_mode & 0o777, 0o600)
|
||||
self.assertEqual(config.parent.stat().st_mode & 0o777, 0o700)
|
||||
|
||||
def test_a_config_written_while_the_install_creates_its_folder_survives(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = Path(tmp)
|
||||
target = root / "target"
|
||||
target.mkdir()
|
||||
config = root / "council-config" / "config.json"
|
||||
racer = f"#!/bin/sh\nmkdir -p {config.parent}\necho '{{\"mine\": true}}' > {config}\nexec /usr/bin/install \"$@\"\n"
|
||||
done = self.install(root, target, config=config, stubs_extra=(("install", racer),))
|
||||
self.assertEqual(json.loads(config.read_text(encoding="utf-8")), {"mine": True})
|
||||
self.assertIn(str(config), done.stderr)
|
||||
|
||||
def test_reinstall_empties_the_skill_directories_in_place(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
|
||||
@@ -529,7 +529,10 @@ class TestOperator(unittest.TestCase):
|
||||
|
||||
def _queue_config(self, slot_need_gb: dict) -> None:
|
||||
config = self.base / "config.json"
|
||||
open_memory = {"available": -1, "swap_free": -1, "psi_full": 1000, "container_swap_gb": 1e9}
|
||||
open_memory = {
|
||||
"available": -1, "swap_free": -1, "psi_some": 1000, "psi_full": 1000, "swap_out_mb_min": 1e9,
|
||||
"container_swap_gb": 1e9,
|
||||
}
|
||||
config.write_text(json.dumps({
|
||||
"disk": str(self.base), "thresholds_gb": {"warn": 0, "stop": 0, "critical": 0}, "projection_minutes": 30,
|
||||
"slot_need_gb": slot_need_gb, "heavy_commands": [r"\btouch heavy\b"], "never_block": [],
|
||||
|
||||
+121
-6
@@ -23,7 +23,9 @@ HISTORY = REPO_ROOT / "skills" / "kvb-org-queue" / "scripts" / "history.py"
|
||||
GATES = REPO_ROOT / "skills" / "kvb-check-gate" / "scripts" / "gates.py"
|
||||
NOW = 1_800_000_000.0
|
||||
HOUR = 3600
|
||||
OPEN_MEMORY = {"available": -1, "swap_free": -1, "psi_full": 1000, "container_swap_gb": 1e9}
|
||||
OPEN_MEMORY = {
|
||||
"available": -1, "swap_free": -1, "psi_some": 1000, "psi_full": 1000, "swap_out_mb_min": 1e9, "container_swap_gb": 1e9,
|
||||
}
|
||||
BASE = {
|
||||
"free_gb": 100.0,
|
||||
"floor": 10.0,
|
||||
@@ -141,11 +143,38 @@ class TestEvaluate(unittest.TestCase):
|
||||
young = {**job(1, kind="a", status="active", granted=NOW - 60), **cmd, "cmd": "a"}
|
||||
overshooting = {**job(5, kind="c", status="active", granted=NOW - 120), **cmd, "cmd": "c"}
|
||||
old = {**job(4, kind="c", status="active", granted=NOW - self.queue.RAMP_SECONDS), **cmd, "cmd": "c"}
|
||||
self.assertEqual(self.queue.pending_ram([young, overshooting, old, fresh], "j2", NOW, signatures, kinds), 30 + 10 + 25)
|
||||
pending = self.queue.pending_ram
|
||||
self.assertEqual(pending([young, overshooting, old, fresh], "j2", NOW, signatures, kinds, 50, 0), 30 + 10 + 25)
|
||||
undated = {k: v for k, v in old.items() if k != "granted"}
|
||||
self.assertEqual(self.queue.pending_ram([undated, fresh], "j2", NOW, signatures, kinds), 10 + 25)
|
||||
self.assertEqual(pending([undated, fresh], "j2", NOW, signatures, kinds, 50, 0), 10 + 25)
|
||||
unknown = {**job(3, kind="new"), **cmd, "cmd": "new"}
|
||||
self.assertEqual(self.queue.pending_ram([unknown], "j3", NOW, signatures, kinds), 30)
|
||||
self.assertEqual(pending([unknown], "j3", NOW, signatures, kinds, 50, 0), 30)
|
||||
self.assertEqual(pending([fresh], "j2", NOW, signatures, kinds, 50, 12), 25 + 12)
|
||||
|
||||
def test_a_young_job_counts_only_what_it_has_not_added_since_its_grant(self):
|
||||
history = _load(HISTORY)
|
||||
rows = [history_row(n, kind="b", signature="w: b", mem_before=20, mem_max=45) for n in range(5)]
|
||||
signatures, kinds = history.learn(rows, NOW, {"load_limits": {"mem_percent": 90, "cpu_percent": 90}})
|
||||
fresh = {**job(2, kind="b"), "workdir": "/w", "cmd": "b"}
|
||||
|
||||
def gate(seq: int, ram: float, granted_mem: float) -> dict:
|
||||
return {**job(seq, kind="g", status="active", granted=NOW - 60), "workdir": "/w", "cmd": "g",
|
||||
"footprint": {"ram": ram, "cpu": 1}, "granted_mem": granted_mem}
|
||||
|
||||
pending = self.queue.pending_ram
|
||||
self.assertEqual(pending([gate(1, 40, 10), fresh], "j2", NOW, signatures, kinds, 10, 0), 25 + 40)
|
||||
self.assertEqual(pending([gate(1, 40, 10), fresh], "j2", NOW, signatures, kinds, 45, 0), 25 + 5)
|
||||
self.assertEqual(pending([gate(1, 40, 10), fresh], "j2", NOW, signatures, kinds, 60, 0), 25)
|
||||
self.assertEqual(pending([gate(1, 40, 10), gate(3, 20, 30), fresh], "j2", NOW, signatures, kinds, 45, 0), 25 + 25)
|
||||
undated = {k: v for k, v in gate(1, 40, 10).items() if k != "granted_mem"}
|
||||
self.assertEqual(pending([undated, fresh], "j2", NOW, signatures, kinds, 60, 0), 25 + 40)
|
||||
ramp = self.queue.ramping_ram
|
||||
for mem in (70, 40, 20):
|
||||
with self.subTest(mem=mem):
|
||||
self.assertEqual(ramp([gate(1, 40, 70)], NOW, signatures, kinds, mem), 40)
|
||||
self.assertEqual(ramp([gate(1, 40, 70), {**undated, "id": "j3"}], NOW, signatures, kinds, 20), 80)
|
||||
self.assertEqual(self.queue.admitted_ram({"admits": [{"ram": 7, "expires": NOW + 1}, {"ram": 9, "expires": NOW}]}, NOW), 7)
|
||||
self.assertEqual(self.queue.admitted_ram({"entries": []}, NOW), 0)
|
||||
|
||||
def test_a_signature_names_the_repository_every_worktree_shares(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
@@ -251,7 +280,7 @@ class TestEvaluate(unittest.TestCase):
|
||||
self.assertEqual(self.queue.footprint(waiting, signatures, kinds), {"ram": 50, "cpu": 80})
|
||||
older = {k: v for k, v in running.items() if k != "footprint"}
|
||||
self.assertEqual(self.queue.footprint(older, signatures, kinds), {"ram": 50, "cpu": 80})
|
||||
self.assertEqual(self.queue.pending_ram([running], "j1", NOW, signatures, kinds), 12 + 12)
|
||||
self.assertEqual(self.queue.pending_ram([running], "j1", NOW, signatures, kinds, 50, 0), 12 + 12)
|
||||
|
||||
def test_an_active_entry_an_older_queue_granted_never_books_less_than_that_queue_did(self):
|
||||
history = _load(HISTORY)
|
||||
@@ -754,6 +783,7 @@ class TestQueueCommands(unittest.TestCase):
|
||||
)
|
||||
signature = f"{self.base.name}: {Path(sys.executable).name} -c"
|
||||
self.assertEqual((entry["aliases"], entry["footprint"]), ([signature], None))
|
||||
self.assertTrue(0 < entry["granted_mem"] < 100, entry)
|
||||
self.assertEqual(self.entries(), [])
|
||||
[row] = [json.loads(line) for line in self.history_file.read_text(encoding="utf-8").splitlines()]
|
||||
self.assertEqual(
|
||||
@@ -1111,6 +1141,52 @@ class TestQueueCommands(unittest.TestCase):
|
||||
self.write_config()
|
||||
self.assertEqual(self.bash("make compose-deploy"), 0)
|
||||
|
||||
def test_list_and_hold_on_a_broken_config_warn_and_use_the_shipped_default(self):
|
||||
self.config.write_text(json.dumps(self.cfg)[:-1] + ",}", encoding="utf-8")
|
||||
listing = self.run_queue("list")
|
||||
self.assertEqual(listing.returncode, 0, listing.stderr)
|
||||
self.assertIn("built-in defaults", listing.stderr)
|
||||
held = self.run_queue("hold", "--kind", "nosuch", "--task", "work/a", "--session", "s1", "--", "true")
|
||||
self.assertEqual(held.returncode, 2, held.stderr)
|
||||
self.assertIn("built-in defaults", held.stderr)
|
||||
self.assertIn("'nosuch' is not in slot_need_gb", held.stderr)
|
||||
for done in (listing, held):
|
||||
self.assertNotIn("Traceback", done.stderr)
|
||||
|
||||
def test_a_broken_config_falls_back_to_defaults_so_light_commands_pass_and_heavy_ones_stay_refused(self):
|
||||
self.cfg["queue_enforce"] = True
|
||||
good = json.dumps(self.cfg)
|
||||
|
||||
def text(content: str) -> Callable[[], None]:
|
||||
return lambda: self.config.write_text(content, encoding="utf-8")
|
||||
|
||||
shapes = {
|
||||
"comma": text(good[:-1] + ",}"), "empty": text(""), "list": text("[]"),
|
||||
"string enforce": text(json.dumps({**self.cfg, "queue_enforce": "yes"})),
|
||||
"empty-matching heavy": text(json.dumps({**self.cfg, "heavy_commands": [""]})),
|
||||
"dot heavy": text(json.dumps({**self.cfg, "heavy_commands": ["."]})),
|
||||
"word-boundary heavy": text(json.dumps({**self.cfg, "heavy_commands": [r"\b"]})),
|
||||
"dot never_block": text(json.dumps({**self.cfg, "never_block": ["."]})),
|
||||
"dangling link": lambda: self.config.symlink_to(self.base / "absent.json"),
|
||||
"link loop": lambda: self.config.symlink_to(self.config),
|
||||
"directory": lambda: self.config.mkdir(),
|
||||
}
|
||||
for name, make in shapes.items():
|
||||
with self.subTest(config=name):
|
||||
if self.config.is_dir() and not self.config.is_symlink():
|
||||
self.config.rmdir()
|
||||
self.config.unlink(missing_ok=True)
|
||||
make()
|
||||
passed = self.hook(json.dumps({"tool_input": {"command": "ls -la"}}))
|
||||
self.assertEqual(passed.returncode, 0, passed.stderr)
|
||||
self.assertIn(str(self.config), passed.stderr)
|
||||
self.assertIn("built-in defaults", passed.stderr)
|
||||
self.assertEqual(self.bash("make test"), 2)
|
||||
live = ("make compose-deploy ROLE=x", "docker build .", "docker pull nginx", "make swarm-deploy", "make roundtrip",
|
||||
"make worktree-up")
|
||||
for command in live if name in ("comma", "directory") else live[:1]:
|
||||
self.assertEqual(self.bash(command), 2, command)
|
||||
|
||||
def test_block_with_queue_enforce_refuses_heavy_commands_outside_the_queue(self):
|
||||
self.cfg["queue_enforce"] = True
|
||||
self.write_config()
|
||||
@@ -1129,7 +1205,12 @@ class TestQueueCommands(unittest.TestCase):
|
||||
):
|
||||
with self.subTest(command=command):
|
||||
self.assertEqual(self.bash(command), 0)
|
||||
self.assertEqual(self.hook("not json").returncode, 0)
|
||||
for payload in ("not json", json.dumps({"tool_input": ["make test"]})):
|
||||
with self.subTest(payload=payload):
|
||||
crashed = self.hook(payload)
|
||||
self.assertEqual(crashed.returncode, 2, crashed.stderr)
|
||||
self.assertIn("counts as heavy", crashed.stderr)
|
||||
self.assertEqual(self.bash("CARGO_BUILD_JOBS=² cargo build ; make test"), 2)
|
||||
self.config.unlink()
|
||||
self.assertEqual(self.bash("make compose-deploy"), 0)
|
||||
|
||||
@@ -1290,6 +1371,40 @@ class TestQueueCommands(unittest.TestCase):
|
||||
self.assertEqual(self.bash(command), 2)
|
||||
self.assertEqual(self.bash("docker compose ps"), 0)
|
||||
|
||||
def test_a_queue_grant_counts_a_live_admit_record_and_ignores_an_expired_one(self):
|
||||
def admitted(expires: float) -> None:
|
||||
(self.root / "queue.json").write_text(
|
||||
json.dumps({"seq": 0, "entries": [], "admits": [{"ram": 300, "expires": expires}]}), encoding="utf-8"
|
||||
)
|
||||
|
||||
admitted(time.time() + 300)
|
||||
_, err = self.blocked_hold("true")
|
||||
self.assertRegex(err.read_text(encoding="utf-8"), r"RAM \d+ % used \+ 300 % still to come reach the limit 101 %")
|
||||
admitted(time.time() - 1)
|
||||
result = self.run_queue("hold", "--kind", "deploy", "--task", "work/a", "--session", "s2", "--", "true")
|
||||
self.assertEqual((result.returncode, result.stderr), (0, ""))
|
||||
|
||||
def test_a_light_kind_refuses_a_heavy_command_names_the_test_kind_and_enqueues_nothing(self):
|
||||
self.cfg["slot_need_gb"] |= {"test-nostack": 0, "quick": 0}
|
||||
self.cfg["light_kinds"] = ["quick"]
|
||||
self.write_config()
|
||||
(self.base / "tests" / "e2e").mkdir(parents=True)
|
||||
(self.base / "tests" / "e2e" / "test_orchestrator_e2e.py").write_text("", encoding="utf-8")
|
||||
for kind, command, shown in (
|
||||
("test-nostack", ("python", "-m", "unittest"), "python -m unittest"),
|
||||
("test-nostack", ("pytest", "tests/e2e/test_orchestrator_e2e.py"), "pytest tests/e2e/test_orchestrator_e2e.py"),
|
||||
("quick", ("bash", "-c", "make test"), "bash -c 'make test'"),
|
||||
("quick", ("python3", "gate.py", "run", "--target", "test-unit"), "python3 gate.py run --target test-unit"),
|
||||
):
|
||||
with self.subTest(kind=kind, command=command):
|
||||
result = self.run_queue("hold", "--kind", kind, "--task", "work/a", "--session", "s1", "--", *command)
|
||||
self.assertEqual(result.returncode, 2)
|
||||
self.assertIn(f"the light kind {kind} skips every reservation", result.stderr)
|
||||
self.assertIn(f"run `queue.py hold --kind test --task <register task> -- {shown}`", result.stderr)
|
||||
self.assertFalse((self.root / "queue.json").exists())
|
||||
light = self.run_queue("hold", "--kind", "test-nostack", "--task", "work/a", "--session", "s1", "--", "true")
|
||||
self.assertEqual((light.returncode, light.stderr), (0, ""))
|
||||
|
||||
def test_hold_refuses_a_config_without_the_budget_or_the_memory_limits(self):
|
||||
for key, cfg in (
|
||||
("budget_percent", {k: v for k, v in self.cfg.items() if k != "budget_percent"}),
|
||||
|
||||
@@ -149,6 +149,53 @@ class TestCpu(unittest.TestCase):
|
||||
self.assertEqual(json.loads((state / "cpu.json").read_text(encoding="utf-8")), [NOW + sampler.BUCKET + 1, 9300, 12200])
|
||||
|
||||
|
||||
class TestPressure(unittest.TestCase):
|
||||
def setUp(self):
|
||||
tmp = tempfile.TemporaryDirectory()
|
||||
self.addCleanup(tmp.cleanup)
|
||||
self.proc, self.state = Path(tmp.name) / "proc", Path(tmp.name) / "state"
|
||||
(self.proc / "pressure").mkdir(parents=True)
|
||||
self.state.mkdir()
|
||||
|
||||
def vmstat(self, pages: int) -> None:
|
||||
(self.proc / "vmstat").write_text(f"pgpgin 5\npswpin 3\npswpout {pages}\n", encoding="utf-8")
|
||||
|
||||
def test_psi_gives_some_and_full_avg10_and_none_without_pressure(self):
|
||||
text = "some avg10=12.50 avg60=1.00 avg300=0.50 total=1\nfull avg10=6.25 avg60=0.80 avg300=0.10 total=1\n"
|
||||
self.assertEqual((sampler.psi_avg10(text, "some"), sampler.psi_avg10(text, "full")), (12.5, 6.25))
|
||||
self.assertEqual((sampler.psi_avg10("", "some"), sampler.psi_avg10(text.splitlines()[0], "full")), (None, None))
|
||||
self.assertIsNone(sampler.swapped_pages("pgpgin 5\n"))
|
||||
|
||||
def test_swap_out_is_none_first_then_mb_per_minute_and_none_after_a_reset_or_a_gap(self):
|
||||
with mock.patch.object(sampler, "PAGE_BYTES", 4096):
|
||||
self.assertIsNone(sampler.swap_out_since(self.proc, self.state, NOW))
|
||||
self.assertFalse((self.state / "swap.json").exists())
|
||||
self.vmstat(1000)
|
||||
self.assertIsNone(sampler.swap_out_since(self.proc, self.state, NOW))
|
||||
self.vmstat(1000 + 1_400_000_000 // 4096)
|
||||
self.assertEqual(sampler.swap_out_since(self.proc, self.state, NOW + 60), 1400.0)
|
||||
self.vmstat(10)
|
||||
self.assertIsNone(sampler.swap_out_since(self.proc, self.state, NOW + 120))
|
||||
self.assertEqual(json.loads((self.state / "swap.json").read_text(encoding="utf-8")), [NOW + 120, 10])
|
||||
self.assertIsNone(sampler.swap_out_since(self.proc, self.state, NOW + 120))
|
||||
self.assertIsNone(sampler.swap_out_since(self.proc, self.state, NOW + 120 + sampler.BUCKET + 1))
|
||||
|
||||
def test_a_sample_row_adds_the_pressure_keys_and_keeps_every_other_key(self):
|
||||
(self.proc / "pressure" / "memory").write_text("some avg10=3.00 avg60=0 avg300=0 total=1\n", encoding="utf-8")
|
||||
found = sampler.sample(self.proc, self.proc / "sys", self.proc, lambda argv: None, self.state)
|
||||
self.assertEqual(sorted(found), ["cpu", "disks", "gpus", "load", "psi_full", "psi_some", "queue", "ram", "swap", "swap_out_mb_min", "t"])
|
||||
self.assertEqual((found["psi_some"], found["psi_full"], found["swap_out_mb_min"]), (3.0, None, None))
|
||||
|
||||
def test_the_fifteen_minute_mean_keeps_the_pressure_keys(self):
|
||||
asleep = {"name": ASLEEP, "asleep": True}
|
||||
rows = [
|
||||
row(NOW + 60, 1.0, 1, 0, asleep) | {"psi_some": 2.0, "psi_full": 1.004, "swap_out_mb_min": 100.0},
|
||||
row(NOW + 120, 1.0, 1, 0, asleep) | {"psi_some": 4.5, "psi_full": None, "swap_out_mb_min": 301.0},
|
||||
]
|
||||
mean = sampler.mean_row(NOW, rows)
|
||||
self.assertEqual((mean["psi_some"], mean["psi_full"], mean["swap_out_mb_min"]), (3.25, 1.0, 200.5))
|
||||
|
||||
|
||||
class TestGpus(unittest.TestCase):
|
||||
def setUp(self):
|
||||
tmp = tempfile.TemporaryDirectory()
|
||||
@@ -206,6 +253,7 @@ class TestRetention(unittest.TestCase):
|
||||
self.assertIs(kept[0], mean)
|
||||
self.assertEqual(kept[1], {
|
||||
"t": bucket, "disks": [{"mount": "/", "used": 200, "total": 1000}, {"mount": "/mnt", "used": 10, "total": 20}], "cpu": 30.0, "ram": 15.0, "swap": 0.0, "load": 3.0,
|
||||
"psi_some": None, "psi_full": None, "swap_out_mb_min": None,
|
||||
"gpus": [{"name": AWAKE, "util": 50.0, "vram_used": 300, "vram_total": 1000}], "queue": {"waiting": 1.0, "active": 1.0},
|
||||
})
|
||||
self.assertEqual((kept[2]["cpu"], kept[2]["disks"]), (40.0, [{"mount": "/", "used": 400, "total": 1000}]))
|
||||
@@ -274,7 +322,7 @@ class TestSamplerInScratchHome(unittest.TestCase):
|
||||
self.assertEqual(len(rows), 2)
|
||||
for found in rows:
|
||||
with self.subTest(t=found["t"]):
|
||||
self.assertEqual(sorted(found), ["cpu", "disks", "gpus", "load", "queue", "ram", "swap", "t"])
|
||||
self.assertEqual(sorted(found), ["cpu", "disks", "gpus", "load", "psi_full", "psi_some", "queue", "ram", "swap", "swap_out_mb_min", "t"])
|
||||
self.assertEqual(found["disks"], DISKS)
|
||||
self.assertEqual(found["gpus"], [{"name": AWAKE, "util": 37.0, "vram_used": 2048 * MIB, "vram_total": 8188 * MIB}, {"name": ASLEEP, "asleep": True}])
|
||||
self.assertEqual(found["queue"], {"waiting": 1, "active": 2})
|
||||
|
||||
+943
-19
File diff suppressed because it is too large.
Load diff
Reference in new issue
Block a user