diff --git a/orchestrator/client/resources.py b/orchestrator/client/resources.py index c9f3fa1..4d3d11b 100755 --- a/orchestrator/client/resources.py +++ b/orchestrator/client/resources.py @@ -2,12 +2,17 @@ """Args: sample: append one row to /samples.jsonl: {t, disks: [{mount, used, total}], cpu, ram, swap, load, gpus: [{name, util, vram_used, - vram_total} | {name, asleep: true}], queue: {waiting, active}}; t in - epoch seconds, used, total and vram in bytes, cpu, ram, swap and util - in percent, load the 1-minute load average, a GPU named by its PCI - address. cpu is the busy share of the /proc/stat ticks since those - the run before kept in /cpu.json as [t, idle, total], None on - the first run and after a gap of more than 15 minutes. An NVIDIA GPU + vram_total} | {name, asleep: true}], queue: {waiting, active}, + psi_some, psi_full, swap_out_mb_min}; t in epoch seconds, used, total + and vram in bytes, cpu, ram, swap and util in percent, load the + 1-minute load average, a GPU named by its PCI address. cpu is the + busy share of the /proc/stat ticks since those the run before kept in + /cpu.json as [t, idle, total], None on the first run and after + a gap of more than 15 minutes. psi_some and psi_full are the avg10 of + /proc/pressure/memory, None without PSI; swap_out_mb_min the MB per + minute swapped out (pswpout of /proc/vmstat) since the count the run + before kept in /swap.json as [t, pswpout], None like cpu and + after a counter reset. An NVIDIA GPU whose PCI runtime_status is not active is recorded asleep, and nvidia-smi is not run for it; an awake one nvidia-smi gives no reading for (no driver, a driver mismatch) keeps its name with util @@ -76,12 +81,15 @@ NVIDIA_QUERY = "pci.bus_id,utilization.gpu,memory.used,memory.total" UNREAD = {"util": None, "vram_used": None, "vram_total": None} COUNTS = (("input", "input_tokens"), ("output", "output_tokens"), ("cache_read", "cache_read_input_tokens"), ("cache_write", "cache_creation_input_tokens")) NAMES = tuple(name for name, _ in COUNTS) -HOST_DIGITS = (("cpu", 1), ("ram", 1), ("swap", 1), ("load", 2)) +HOST_DIGITS = (("cpu", 1), ("ram", 1), ("swap", 1), ("load", 2), ("psi_some", 2), ("psi_full", 2), ("swap_out_mb_min", 1)) CLAIMS = ("*/claim.json", "*/*/claim.json") SAMPLES = "samples.jsonl" TOKENS = "tokens.jsonl" OFFSETS = "offsets.json" TICKS = "cpu.json" +SWAPPED = "swap.json" +PAGE_BYTES = os.sysconf("SC_PAGE_SIZE") +MB = 1_000_000 def read_text(path: Path) -> str: @@ -264,9 +272,43 @@ def cpu_since(proc: Path, state: Path, t: int) -> float | None: return cpu_percent(before, ticks) +def psi_avg10(text: str, kind: str) -> float | None: + for line in text.splitlines(): + fields = line.split() + if fields[:1] == [kind]: + values = dict(field.split("=", 1) for field in fields[1:] if "=" in field) + return number(values.get("avg10", "")) + return None + + +def swapped_pages(text: str) -> int | None: + for line in text.splitlines(): + fields = line.split() + if len(fields) == 2 and fields[0] == "pswpout" and fields[1].isdigit(): + return int(fields[1]) + return None + + +def swap_out_since(proc: Path, state: Path, t: int) -> float | None: + """Returns: + The MB per minute swapped out since the pswpout count the run before + kept in /swap.json at most BUCKET seconds before t, else None, + also after a counter reset; this run's count replaces it. + """ + pages = swapped_pages(read_text(proc / "vmstat")) + if pages is None: + return None + stored = read_json(state / SWAPPED) + replace(state / SWAPPED, json.dumps([t, pages])) + if not integers(stored, 2) or not t - BUCKET <= stored[0] < t or stored[1] > pages: + return None + return round((pages - stored[1]) * PAGE_BYTES / MB * 60 / (t - stored[0]), 1) + + def sample(proc: Path, sys_root: Path, council: Path, run: Callable[[list[str]], str | None], state: Path) -> dict: t = int(time.time()) ram, swap = memory(parse_meminfo(read_text(proc / "meminfo"))) + pressure = read_text(proc / "pressure" / "memory") return { "t": t, "disks": parse_df(run(["df", "-P", "-k", "-l"]) or ""), @@ -276,6 +318,9 @@ def sample(proc: Path, sys_root: Path, council: Path, run: Callable[[list[str]], "load": parse_loadavg(read_text(proc / "loadavg")), "gpus": gpus(sys_root, run), "queue": queue(council), + "psi_some": psi_avg10(pressure, "some"), + "psi_full": psi_avg10(pressure, "full"), + "swap_out_mb_min": swap_out_since(proc, state, t), } diff --git a/scripts/install.sh b/scripts/install.sh index 1c09f98..71b7e85 100644 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -48,6 +48,11 @@ if [[ "${TARGET}" == "${HOME}" ]]; then exit 1 fi fi +watch="${REPO_ROOT}/skills/kvb-org-organize/scripts/watch.py" +if [[ -f "${watch}" ]] && ! python3 -I "${watch}" preflight; then + warn "skills: add the keys above to the agents council config first; the new hooks and queue.py refuse to run without them" + exit 2 +fi cd "${REPO_ROOT}" src="${REPO_ROOT}/.agents/skills" @@ -120,6 +125,17 @@ if [[ "${TARGET}" == "${HOME}" ]]; then fi fi +config="${AGENTS_COUNCIL_CONFIG:-${HOME}/.claude/agents-council/config.json}" +shipped="${REPO_ROOT}/skills/kvb-org-organize/config.default.json" +if [[ -f "${shipped}" && ! -e "${config}" && ! -L "${config}" ]]; then + [[ -d "$(dirname -- "${config}")" ]] || install -d -m 0700 "$(dirname -- "${config}")" + if (umask 077 && set -o noclobber && cat -- "${shipped}" >"${config}") 2>/dev/null; then + log "skills: wrote the shipped agents council config to ${config}" + else + warn "skills: kept ${config}, which appeared while the install wrote the shipped default" + fi +fi + settings="${TARGET}/.claude/settings.local.json" if command -v node &>/dev/null; then log "skills: auto-enabling caveman + ponytail plugins in ${settings}" diff --git a/skills/kvb-check-ansible/SKILL.md b/skills/kvb-check-ansible/SKILL.md index 0544457..f7cc796 100644 --- a/skills/kvb-check-ansible/SKILL.md +++ b/skills/kvb-check-ansible/SKILL.md @@ -7,9 +7,8 @@ description: > Where an Ansible run spends its time: its log, a controller profile, a bench. Report the script's lines with their log and range, never a guess. The -script and the bench are light runs (`watch.py admit --kind test-nostack` of -`kvb-org-organize`); a deploy to profile is heavy and goes through the -`kvb-org-queue` skill. +script, the bench and a deploy to profile run through the `kvb-org-queue` +skill. ## 1. Pick the log diff --git a/skills/kvb-check-efficiency/SKILL.md b/skills/kvb-check-efficiency/SKILL.md index cbe6e77..6fdcd4d 100644 --- a/skills/kvb-check-efficiency/SKILL.md +++ b/skills/kvb-check-efficiency/SKILL.md @@ -60,12 +60,11 @@ It also measures wall time for `kvb-check-speed`. - **Command**: the targeted tests of the change or a CLI call of the changed code, runnable on a clean checkout: the base worktree holds no untracked build output, virtualenv or dependency directory. -- **Heavy commands** (Docker, deploys, whole suites) run only inside +- **Every run** goes inside `${CLAUDE_SKILL_DIR}/../kvb-org-queue/scripts/queue.py hold --kind --task -- python3 ...`, in the background, by the `kvb-org-queue` skill; a cancelled wait there also exits 3, but - prints no 💸 line. A light one self-admits with - `watch.py admit --kind test-nostack` of `kvb-org-organize`. + prints no 💸 line. - Report the run you made; never rerun to pick the best result. ## Exit codes diff --git a/skills/kvb-check-gate/SKILL.md b/skills/kvb-check-gate/SKILL.md index 87b07c2..cb6a082 100644 --- a/skills/kvb-check-gate/SKILL.md +++ b/skills/kvb-check-gate/SKILL.md @@ -32,10 +32,9 @@ commit starts from zero. `missing hotfix` means step 1. 4. **Admit**: every gate target is heavy and runs only inside `${CLAUDE_SKILL_DIR}/../kvb-org-queue/scripts/queue.py hold`. - Only a single test module - self-admits (`${CLAUDE_SKILL_DIR}/../kvb-org-organize/scripts/watch.py - admit --kind test-nostack`), and it never turns a gate target green. A - hotfix gate adds `--hotfix ` and takes the next free gate slot. + Only a single test module self-admits (`kvb-org-organize` Capacity), and + it never turns a gate target green. A hotfix gate adds `--hotfix ` + and takes the next free gate slot. 5. **Run** the missing and red targets, tracked tree clean, in the background; give the operator `tail -f` on the log: diff --git a/skills/kvb-ci-triage/funnel.md b/skills/kvb-ci-triage/funnel.md index 371ddbb..99254ef 100644 --- a/skills/kvb-ci-triage/funnel.md +++ b/skills/kvb-ci-triage/funnel.md @@ -15,11 +15,8 @@ skill's Rules apply here too. first error. 3. **Fix at the root**: apply the real fix for each proven root cause in the sidetree, with a test and a negative control as the `kvb-org-swarm` skill - demands. Light runs (no Docker, about a minute: one test file or - module, ruff) self-admit with - `${CLAUDE_SKILL_DIR}/../kvb-org-organize/scripts/watch.py admit --kind - test-nostack`; `make test`, gates, deploys and anything with Docker run in - the background through + demands. Light runs self-admit (`kvb-org-organize` Capacity); `make test`, + gates, deploys and anything with Docker run in the background through `${CLAUDE_SKILL_DIR}/../kvb-org-queue/scripts/queue.py hold --kind --task -- `, which waits for the rule-based grant and releases on exit. The fix enters the main line only diff --git a/skills/kvb-code-refactor/SKILL.md b/skills/kvb-code-refactor/SKILL.md index a2b688b..4c80b0e 100644 --- a/skills/kvb-code-refactor/SKILL.md +++ b/skills/kvb-code-refactor/SKILL.md @@ -26,9 +26,8 @@ cover the change set run now and give the green baseline; a suite already red is reported, not refactored around. No coverage: read `net.md` next to this file before editing. -Light runs (one test file or module, ruff) start once `watch.py -admit --kind test-nostack` of the `kvb-org-organize` skill prints `grant`; no -chair request. +Light runs start once `watch.py admit` of the `kvb-org-organize` skill prints +`grant`; no chair request. ## Step 3: refactor each file diff --git a/skills/kvb-code-test/SKILL.md b/skills/kvb-code-test/SKILL.md index 3c1f89f..b361956 100644 --- a/skills/kvb-code-test/SKILL.md +++ b/skills/kvb-code-test/SKILL.md @@ -38,11 +38,8 @@ any listed name. Never silently truncate. ## Step 3: run Before each run, admit it under the `kvb-org-organize` Capacity rules: a -light run (no Docker, about a minute: one test file or module, ruff) -self-admits with -`${CLAUDE_SKILL_DIR}/../kvb-org-organize/scripts/watch.py admit --kind -test-nostack`; `make test`, quality gates and anything with Docker run in the -background through +light run self-admits; `make test`, quality gates and anything with Docker +run in the background through `${CLAUDE_SKILL_DIR}/../kvb-org-queue/scripts/queue.py hold --kind --task -- make `, which waits for the rule-based grant and releases on exit. diff --git a/skills/kvb-code-transcend/SKILL.md b/skills/kvb-code-transcend/SKILL.md index 6086136..36ec36f 100644 --- a/skills/kvb-code-transcend/SKILL.md +++ b/skills/kvb-code-transcend/SKILL.md @@ -49,9 +49,8 @@ single-select `Write it` / `Revise it` / `Discard` with the drafted diff. owner through the `kvb-org-coordinate` skill. - Every carried test fails without the code it guards: show the negative control once. -- Run the carried tests as a light run (no Docker, about a minute) after - `${CLAUDE_SKILL_DIR}/../kvb-org-organize/scripts/watch.py admit --kind - test-nostack`; `make test`, gates and anything with Docker only through +- Run the carried tests as a light run (`kvb-org-organize` Capacity); + `make test`, gates and anything with Docker only through `queue.py hold` (the `kvb-org-queue` skill). Leave the result uncommitted for the `kvb-git-commit-next` skill. - `transcend.py mark` every offered candidate with its verdict. diff --git a/skills/kvb-create-requirement/SKILL.md b/skills/kvb-create-requirement/SKILL.md index 8a2f658..f33441e 100644 --- a/skills/kvb-create-requirement/SKILL.md +++ b/skills/kvb-create-requirement/SKILL.md @@ -71,10 +71,8 @@ order: Show the full draft, then `AskUserQuestion`: `Write it` / `Revise it` / `Discard`. Revise loops back to the step the correction touches. Write only on `Write it`; lint the file with the project's markdown linter, a light run -that self-admits with -`${CLAUDE_SKILL_DIR}/../kvb-org-organize/scripts/watch.py admit --kind -test-nostack`; a lint through `make` or Docker runs through `queue.py hold` -(the `kvb-org-queue` skill). +(`kvb-org-organize` Capacity); a lint through `make` or Docker runs through +`queue.py hold` (the `kvb-org-queue` skill). ## Step 6: hand over diff --git a/skills/kvb-git-issue-solve/SKILL.md b/skills/kvb-git-issue-solve/SKILL.md index 500bc4c..2985987 100644 --- a/skills/kvb-git-issue-solve/SKILL.md +++ b/skills/kvb-git-issue-solve/SKILL.md @@ -35,8 +35,8 @@ request, so a red one blocks nothing else. (`search_graph`, `trace_path`, `get_architecture`, `detect_changes`; the `kvb-agent-optimize` skill sets it up), grep and read after. Parallel work, gates, deploys and anything with Docker run through `queue.py hold` - (`kvb-org-queue`); light runs (no Docker, about a minute) self-admit with - `watch.py admit --kind test-nostack`. With the first change, swap the label: `gh issue + (`kvb-org-queue`); light runs self-admit (`kvb-org-organize` Capacity). + With the first change, swap the label: `gh issue edit --remove-label "in analysis" --add-label "in progress"`. 5. **Open the pull request** with the `kvb-git-pr-create` skill, closing exactly its issue; one for a failing default branch is posted with diff --git a/skills/kvb-org-organize/SKILL.md b/skills/kvb-org-organize/SKILL.md index ccebd5e..4189066 100644 --- a/skills/kvb-org-organize/SKILL.md +++ b/skills/kvb-org-organize/SKILL.md @@ -37,8 +37,13 @@ ends or goes stale, or after a restart. ## Capacity -- Light runs (no Docker, about a minute: one test file or module, ruff) - self-admit: `watch.py admit --kind test-nostack`, start on `grant`. +- Light runs self-admit: `watch.py admit --kind test-nostack -- `, + start on `grant`. Light is only ruff, shellcheck, markdownlint, lizard, jq, + py_compile, cargo fmt, `env CARGO_BUILD_JOBS=4 cargo build|check|clippy|test`, + or one test file or module outside `e2e/` that reaches no Playwright; + anything else: `queue.py hold --kind test`. +- Memory stop (also PSI or swap-out) blocks every admit and grant; running + jobs go on. - Other test runs, `make test`, `make test-lint`, quality gates, deploys, iteration loops, swarm, image builds, cloud, shared-data cleanup, anything with Docker or over a few minutes: run through `queue.py hold` diff --git a/skills/kvb-org-organize/capacity.md b/skills/kvb-org-organize/capacity.md index be5c2db..076932f 100644 --- a/skills/kvb-org-organize/capacity.md +++ b/skills/kvb-org-organize/capacity.md @@ -63,3 +63,16 @@ and stays silent below the thresholds. `watch.py status` shows the stage computed from the live free space and the stored samples, and the age of the stored measurement. + +An unusable config (broken JSON, wrong types, a regex refusing a light +probe such as `grep` or freeing a heavy one such as `make swarm-deploy`, a +dangling link, a directory) makes the hooks, admit, status and the queue +warn and use `config.default.json`, which the install writes when no config +exists; without that file every command, teardown included, is refused. +The probes are samples, so a pattern hitting only other commands passes. A +package script is read from the nearest package.json, a named workspace +only from a folder of that name. The light-run check misses +`sys.path.insert` in a conftest, editable installs without pytest +`pythonpath`, browser libraries outside its list and generic third-party +fixtures such as pytest-selenium's `driver`, browsers started by a +subprocess, and `git bisect run` or a suite script behind `bash`. diff --git a/skills/kvb-org-organize/config.default.json b/skills/kvb-org-organize/config.default.json new file mode 100644 index 0000000..5adda74 --- /dev/null +++ b/skills/kvb-org-organize/config.default.json @@ -0,0 +1,37 @@ +{ + "disk": "/", + "thresholds_gb": {"warn": 40, "stop": 15, "critical": 12}, + "projection_minutes": 30, + "interval_seconds": 120, + "cache_interval_seconds": 1800, + "heavy_commands": [ + "\\bmake\\s+(-C\\s+\\S+\\s+)?(compose-deploy|compose-up|compose-playwright|roundtrip|quality|quality-high|test|test-integration|test-unit\\S*|swarm-\\S+|act-workflow|act-runner-image|build\\S*|worktree-up|store-\\S+)(\\s|$)", + "\\bdocker\\s+(build|pull|load)\\b", + "\\bdocker\\s+buildx\\s+build\\b", + "\\brun\\.sh\\s+local\\b" + ], + "never_block": [ + "\\bmake\\s+(-C\\s+\\S+\\s+)?(swarm-down|swarm-clean|swarm-clean-stale-nfs|swarm-diagnostic|compose-down|compose-entity-purge|worktree-down|store-host-down)(\\s|$)", + "^\\s*git\\s+commit\\b.*\\s--no-verify\\b", + "\\bmake\\s+swarm-exec(\\s|$)" + ], + "queue_enforce": true, + "cleanup": [], + "worktree_cleanup": [], + "worktrees_command": "true", + "worktree_marker": ".git", + "busy_processes": ["make"], + "cache_dirs": ["~/.cache/pip", "~/.npm", "~/.cache/pre-commit", "~/.cache/act", "~/.cache/ms-playwright"], + "cache_budget_percent": 3, + "slot_need_gb": {"quality-high": 15, "test": 15, "test-nostack": 0, "iteration": 20, "swarm": 30, "gui-dind": 65, "operator": 10}, + "notify": ["notify-send", "-u", "critical", "Agents council"], + "load_limits": {"cpu_percent": 90, "mem_percent": 85, "sample_seconds": 5, "kinds": ["test", "test-nostack", "quality-high"]}, + "memory_limits": { + "warn": {"available": 15, "swap_free": 40, "psi_some": 10, "psi_full": 5, "swap_out_mb_min": 64, "container_swap_gb": 4}, + "stop": {"available": 8, "swap_free": 10, "psi_full": 5, "swap_out_mb_min": 256, "container_swap_gb": 16} + }, + "budget_percent": {"ram": 75, "cpu": 100}, + "light_kinds": ["test-nostack"], + "exclusive_kinds": ["quality-high", "test"], + "gate_excluded_targets": ["test-oracle"] +} diff --git a/skills/kvb-org-organize/scripts/watch.py b/skills/kvb-org-organize/scripts/watch.py index 2295e99..de829d7 100755 --- a/skills/kvb-org-organize/scripts/watch.py +++ b/skills/kvb-org-organize/scripts/watch.py @@ -15,10 +15,15 @@ Usage: or cannot be judged without memory_limits, or a kvb-org-radio mayday holds radio silence (/radio-silence.json, until > now) - watch.py admit (--need-gb N | --kind KIND) + watch.py admit (--need-gb N | --kind KIND) -- CMD... exit 0 when the grant leaves at least the stop threshold - free, CPU busy and RAM used stay below load_limits and - the memory stage is not stop, 3 otherwise + free, the memory stage is not stop and queue.py admit() + keeps CPU and RAM below load_limits, 3 otherwise; 3 at + once for a CMD light() does not recognise, which + queue_hold() runs + watch.py preflight exit 2 naming every missing memory_limits key while the + config lacks one this code needs, else 0, touching no + state watch.py cleanup (--caches | --full) trim caches to their budget; --full also runs the configured low-risk cleanup commands @@ -44,7 +49,8 @@ Environment: the sandbox's file rules do not let it write, because the hooks run its commands outside the sandbox; a Docker daemon the sandbox can reach still writes it. Without a config file every command exits 0 and does - nothing. + nothing; an unusable one makes every command warn and use + ../config.default.json, cleanup and preflight exit 2. PATH: filtered to the directories hookpath.py trusts before any command runs, the config's commands included. @@ -73,22 +79,28 @@ to a worktree that must exist), busy_processes (process names; a worktree where one of them runs is skipped), cache_dirs, cache_budget_percent, slot_need_gb {kind: GB}, notify (argv the alert text is appended to), load_limits {cpu_percent, mem_percent, sample_seconds}: every admit and every -queue grant needs CPU busy and RAM used below them, memory_limits {warn, stop}, -each {available, swap_free, psi_full, container_swap_gb}: the memory stage is -the first of stop and warn whose MemAvailable or SwapFree percentage lies below -its bound, whose /proc/pressure/memory full avg10 reaches its bound, or where -one container (a docker-*.scope cgroup under system.slice, nested groups -counted) holds at least container_swap_gb GB of swap, else ok; a host without -swap or PSI skips that bound. heavy_commands extend the built-in rules of -heavy(). +queue grant needs CPU busy and RAM used below them, memory_limits {warn: +{available, swap_free, psi_some, psi_full, swap_out_mb_min, container_swap_gb}, +stop: the same without psi_some}: the memory stage is the first of stop and warn whose +MemAvailable or SwapFree percentage lies below its bound, whose +/proc/pressure/memory some or full avg10 reaches its bound, whose swap-out +rate (pswpout of /proc/vmstat since the newest reading in +/swap-out.json SWAP_SPAN seconds old, in MB per minute) reaches its +bound, or where one container (a docker-*.scope cgroup under system.slice, +nested groups counted) holds at least container_swap_gb GB of swap, else ok; +a host without swap, PSI or such a reading skips that bound. heavy_commands +extend the built-in rules of heavy(). """ from __future__ import annotations import argparse +import ast +import configparser import contextlib import fcntl import fnmatch +import functools import importlib.util import itertools import json @@ -102,20 +114,27 @@ import subprocess import sys import tempfile import time +import tomllib +from collections.abc import Iterable from pathlib import Path from types import ModuleType -def sibling(name: str) -> ModuleType: - spec = importlib.util.spec_from_file_location(f"kvb_watch_{name}", Path(__file__).resolve().parent / f"{name}.py") +def loaded(name: str, path: Path) -> ModuleType: + spec = importlib.util.spec_from_file_location(f"kvb_watch_{name}", path) module = importlib.util.module_from_spec(spec) spec.loader.exec_module(module) return module +def sibling(name: str) -> ModuleType: + return loaded(name, Path(__file__).resolve().parent / f"{name}.py") + + nofollow = sibling("nofollow") hookpath = sibling("hookpath") councilroot = sibling("councilroot") +QUEUE = Path(__file__).resolve().parents[2] / "kvb-org-queue" / "scripts" / "queue.py" ROOT = councilroot.root() CONFIG = Path( os.environ.get( @@ -147,12 +166,20 @@ LOCK_TTL = 1800 GB = 1e9 MEMINFO = Path("/proc/meminfo") PRESSURE = Path("/proc/pressure/memory") +VMSTAT = Path("/proc/vmstat") +SWAP_OUT = ROOT / "swap-out.json" +SWAP_SPAN = (5, 900) +PAGE_BYTES = os.sysconf("SC_PAGE_SIZE") +MB = 1e6 CGROUPS = Path("/sys/fs/cgroup/system.slice") SWAP_HOLDERS = 3 NEST_DEPTH = 8 MEMORY_STAGES = ("ok", "warn", "stop") -MEMORY_BOUNDS = ("available", "swap_free", "psi_full", "container_swap_gb") -MEMORY_KEYS = tuple(f"memory_limits.{stage}.{bound}" for stage in MEMORY_STAGES[1:] for bound in MEMORY_BOUNDS) +PRESSURES = {"warn": ("psi_some", "psi_full", "swap_out_mb_min"), "stop": ("psi_full", "swap_out_mb_min")} +MEMORY_KEYS = tuple( + f"memory_limits.{stage}.{bound}" + for stage in MEMORY_STAGES[1:] for bound in ("available", "swap_free", *PRESSURES[stage], "container_swap_gb") +) CONTAINER_ID = re.compile(r"[0-9a-f]{64}") COMPOSE_VALUES = frozenset({ "-f", "--file", "-p", "--project-name", "--project-directory", "--env-file", "--profile", "--ansi", "--parallel", @@ -205,7 +232,7 @@ WORD_BREAK = " \t\n;&|()<>" ASSIGNMENT = re.compile(r"[A-Za-z_]\w*=.*", re.DOTALL) WRAPPERS = frozenset({ "env", "sudo", "nice", "nohup", "timeout", "exec", "xargs", "stdbuf", "ionice", "uv", "uvx", "poetry", "pdm", - "pipenv", "coverage", "time", "command", + "pipenv", "coverage", "time", "command", "npx", "pnpx", "bunx", }) UV_VALUES = frozenset({ "--with", "--extra", "--group", "--python", "-p", "--project", "--directory", "--package", "--env-file", "--from", @@ -228,6 +255,7 @@ WRAPPER_VALUES = { "poetry": frozenset({"-C", "-P", "--directory", "--project"}), "pdm": frozenset({"-p", "--project"}), "coverage": frozenset({"--source", "--include", "--omit", "--rcfile", "--context", "--data-file"}), + "npx": frozenset({"-p", "--package", "-c", "--call"}), } WRAPPER_WORDS = frozenset({"run", "tool"}) RESERVED = frozenset({"!", "{", "if", "then", "else", "elif", "do", "while", "until"}) @@ -251,7 +279,94 @@ HEREDOC_MARK = "kvbheredoc" MARK_WORD = re.compile(rf"{HEREDOC_MARK}((?:[0-9a-f]{{2}})*)") HELP = frozenset({"-h", "--help", "--version"}) PYTEST_LIGHT = HELP | {"--collect-only", "--co"} +PYTESTS = frozenset({"pytest", "py.test"}) TEST_FILE = re.compile(r"[^*?\[\]]+\.py(::.*)?", re.DOTALL) +E2E = ("tests", "e2e") +PLAYWRIGHT = "playwright" +BROWSER_MODULES = frozenset({ + "playwright", "pytest_playwright", "selenium", "webdriver", "@playwright", "playwright-core", "playwright-chromium", + "playwright-firefox", "playwright-webkit", "puppeteer", "puppeteer-core", "selenium-webdriver", "webdriverio", + "pyppeteer", "undetected_chromedriver", "puppeteer-extra", "cypress", "seleniumbase", "nodriver", "DrissionPage", + "splinter", "helium", "camoufox", "nightwatch", "testcafe", "@wdio", +}) +BROWSER_CLIS = frozenset({"playwright", "cypress", "nightwatch", "testcafe", "wdio"}) +DYNAMIC_IMPORTS = frozenset({"import_module", "__import__"}) +PLUGIN_LIST = "pytest_plugins" +REQUESTS = frozenset({"getfixturevalue", "usefixtures"}) +INI_SECTIONS = {"pytest.ini": "pytest", "tox.ini": "pytest", "setup.cfg": "tool:pytest"} +LOAD_TESTS = frozenset({"load_tests", "*"}) +FIXTURES = frozenset({ + "playwright", "browser_type", "browser_type_launch_args", "launch_browser", "browser", "browser_context_args", + "context", "new_context", "page", "browser_name", "browser_channel", "is_chromium", "is_firefox", "is_webkit", + "device", "connect_options", "output_path", "browser_context", "selenium", +}) +FUNCTIONS = (ast.FunctionDef, ast.AsyncFunctionDef) +CONFIGS = ("pytest.ini", "pyproject.toml", "setup.cfg", "tox.ini") +ADDOPTS = re.compile(r"addopts.*(?:-n\b|--numprocesses|--dist|-p\b|playwright)", re.IGNORECASE) +REACH_FILES = 2000 +REACH_BYTES = 4 * 2**20 +SHIPPED = Path(__file__).resolve().parents[1] / "config.default.json" +REFUSE_ALL = {"heavy_commands": [r"[\s\S]"], "never_block": [], "queue_enforce": True} +UNUSABLE = (OSError, ValueError, TypeError, RecursionError) +SCRIPT = re.compile(r"\.(?:py|[cm]?[jt]sx?)$") +JS_SCRIPT = re.compile(r"\.[cm]?[jt]sx?$") +JS_TOKEN = re.compile(r"//|/\*|['\"`]") +JS_IMPORTER = re.compile(r"(?:\bfrom|\bimport|\brequire\s*\(|\bimport\s*\()\s*$") +JS_TAIL = 64 +SOURCE_BYTES = 1 << 20 +JS_RUNNERS = frozenset({"node", "deno", "bun"}) +JS_VALUE_OPTIONS = frozenset({"-r", "--require", "--import", "--loader", "-e", "--eval", "-p", "--print", "-c", "--config"}) +PACKAGE_RUNNERS = frozenset({"npm", "pnpm", "yarn", "bun"}) +FOLDER_OPTIONS = frozenset({"--cwd", "-C", "--dir", "--prefix"}) +PACKAGE_OPTIONS = { + runner: frozenset({"--filter", "-F", "--workspace", *(["-w"] if runner == "npm" else [])}) for runner in PACKAGE_RUNNERS +} +ALL_PACKAGES = frozenset({"-r", "--recursive", "--workspaces", "-ws"}) +PACKAGE_COMMANDS = frozenset({ + "add", "install", "i", "ci", "remove", "rm", "uninstall", "upgrade", "update", "up", "init", "info", "view", "why", + "list", "ls", "outdated", "audit", "config", "cache", "version", "help", "link", "unlink", "pack", "dedupe", +}) +SCRIPT_PREFIX = re.compile(r"^(?:npm|yarn|pnpm|bun):") +RESOLVING: set[tuple[str, str]] = set() +RUN_WORDS = frozenset({"run", "run-script", "test", "t", "tst", "exec", "x", "dlx", "start"}) | BROWSER_CLIS +MAKES = frozenset({"make", "gmake"}) +BACKGROUND = re.compile(r"(?|])&(?![&>])") +PYTEST_FLAGS = frozenset({ + "-q", "-qq", "-v", "-vv", "-vvv", "-x", "-s", "-l", "-ra", "-rA", "--quiet", "--verbose", "--exitfirst", "--no-header", + "--showlocals", "--strict-markers", +}) +PYTEST_VALUES = frozenset({"-k", "-m", "--tb", "-r", "--maxfail", "--durations", "--color"}) +UNITTEST_FLAGS = frozenset({"-v", "-q", "-f", "-b", "-c", "--verbose", "--quiet", "--failfast", "--buffer", "--catch", "--locals"}) +UNITTEST_VALUES = frozenset({"-k", "--durations"}) +INERT_ENV = frozenset({ + "LANG", "LANGUAGE", "TZ", "NO_COLOR", "FORCE_COLOR", "TERM", "COLUMNS", "CARGO_BUILD_JOBS", "PYTHONDONTWRITEBYTECODE", + "PYTHONUNBUFFERED", +}) +LOCALE = "LC_" +EXPORTS = frozenset({"export", "declare", "typeset", "readonly", "local"}) +CARGO_LIGHT = frozenset({"fmt", "metadata"}) +CARGO_CAPPED = frozenset({"build", "check", "clippy", "test"}) +CARGO_JOBS = 4 +CARGO_JOB_FLAGS = re.compile(r"-j.*|--jobs.*", re.DOTALL) +FIND_EXEC = frozenset({"-exec", "-execdir", "-ok", "-okdir"}) +LINTERS = frozenset({ + "ruff", "shellcheck", "markdownlint", "markdownlint-cli2", "lizard", "jq", "yamllint", "actionlint", "codespell", +}) +INERT = frozenset({ + "grep", "egrep", "fgrep", "rg", "echo", "printf", "cat", "head", "tail", "wc", "ls", "sort", "uniq", "tee", "true", + "which", "type", "kill", "pkill", "killall", "pgrep", "pidof", "find", "man", "readlink", "whereis", "ps", "stat", + "file", +}) +LIGHT_TOOLS = LINTERS | INERT +LIGHT_MODULES = frozenset({"py_compile", "json.tool"}) +SAFE_PREFIXES = frozenset({"env", "nice", "ionice", "timeout", "time"}) +DIRECTORY_CHANGES = frozenset({"cd", "pushd", "popd"}) +KNOWN = INERT | DIRECTORY_CHANGES | frozenset({ + "git", "gh", "pip", "pip3", "pipx", "apt", "apt-get", "pacman", "brew", "dnf", "bash", "sh", "zsh", "dash", +}) +HEAVY_MAKE = ("test", "quality", "verify-") +LIGHT_KIND = "test-nostack" +HEAVY_KIND = "test" PYTEST_VALUE_OPTIONS = frozenset({ "-p", "-k", "-m", "-c", "-o", "-W", "-n", "--numprocesses", "-r", "--maxfail", "--tb", "--rootdir", "--basetemp", "--deselect", "--ignore", "--ignore-glob", "--confcutdir", "--durations", "--junitxml", "--log-level", "--dist", "--cov", @@ -294,9 +409,75 @@ def guard_root(hook: bool) -> int | None: def load_config() -> dict | None: - if not CONFIG.is_file(): - return None - return json.loads(CONFIG.read_text(encoding="utf-8")) + """Returns: + The parsed config; None only when its path does not exist in any + form. + + Raises: + OSError: when the path exists but is no readable regular file (a + dangling or looping link, a directory). + ValueError: when it holds no JSON. + """ + return read_regular(CONFIG) if os.path.lexists(CONFIG) else None + + +def read_regular(path: Path) -> object: + if not path.is_file(): + raise OSError(f"{path} is no readable regular file") + return json.loads(path.read_text(encoding="utf-8")) + + +def checked_config(cfg: object) -> dict: + """Returns: + cfg when it is a JSON object whose PATTERN_PROBES keys are lists of + valid regexes none of which matches one of that key's probe commands, + and whose queue_enforce, when set, is a bool. + + Raises: + TypeError: naming the first part of the wrong type. + ValueError: naming the first broken regex or one matching a probe. + """ + if not isinstance(cfg, dict): + raise TypeError("the config is no JSON object") + for key, probes in PATTERN_PROBES.items(): + patterns = cfg.get(key) + if not isinstance(patterns, list) or not all(isinstance(pattern, str) for pattern in patterns): + raise TypeError(f"{key} is no list of regexes") + for pattern in patterns: + checked_pattern(key, pattern, probes) + if not isinstance(cfg.get("queue_enforce", False), bool): + raise TypeError("queue_enforce is no true or false") + return cfg + + +def checked_pattern(key: str, pattern: str, probes: tuple[str, ...]) -> None: + try: + found = [probe for probe in probes if re.search(pattern, probe)] + except re.error as error: + raise ValueError(f"{key} holds the broken regex {pattern!r}: {error}") from error + if found: + raise ValueError(f"{key} holds {pattern!r}, which matches the probe command {found[0]!r}") + + +def settled_config() -> tuple[dict | None, str | None]: + """Returns: + The config and None; None and None without a config file (every + command stays off); the SHIPPED config and one warning naming the + file when it cannot be read or checked (checked_config()); + REFUSE_ALL, which counts every command heavy, when SHIPPED cannot be + read or checked either. + """ + try: + cfg = load_config() + return (None if cfg is None else checked_config(cfg)), None + except UNUSABLE as error: + problem = f"agents council: {CONFIG} is unusable ({error})" + try: + return checked_config(read_regular(SHIPPED)), ( + f"{problem}; the built-in defaults of {SHIPPED} apply until it is fixed: light commands pass, heavy ones are refused" + ) + except UNUSABLE as error: + return dict(REFUSE_ALL), f"{problem} and so is {SHIPPED} ({error}); every command is refused until one is fixed" def absent_part(cfg: dict, key: str) -> str | None: @@ -520,16 +701,19 @@ def measure_and_act(cfg: dict, state: dict, now: float, problem: str | None) -> act_memory(cfg, state["memory_stage"], was_short, sample) -def cmd_watch(cfg: dict, problem: str | None) -> int: +def cmd_watch(cfg: dict, problem: str | None, unsettled: bool) -> int: """Args: cfg: the config. problem: why the memory stage cannot be judged, None when it can; the disk part still runs and every session sees the problem. + unsettled: whether cfg stands in for an unusable config file + (settled_config()); nothing is measured then, the alerts still reach the + session and problem names the file. """ payload = json.load(sys.stdin) now = time.time() state = read_json(STATE) - if stale(state, cfg, now): + if not unsettled and stale(state, cfg, now): measure_and_act(cfg, state, now, problem) keep_history() texts = unseen_alerts(payload["session_id"]) + ([] if problem is None else [problem]) @@ -687,7 +871,11 @@ def operands(args: list[str], value_options: frozenset[str]) -> list[str]: return found -def python_module(args: list[str]) -> tuple[str, list[str]]: +def python_target(args: list[str]) -> tuple[str, list[str]]: + """Returns: + The module of `python -m` or the script path as typed, and its + arguments; ("", []) for code given with -c or for neither. + """ skip = False for at, word in enumerate(args): if skip: @@ -696,20 +884,48 @@ def python_module(args: list[str]) -> tuple[str, list[str]]: return (args[at + 1], args[at + 2 :]) if at + 1 < len(args) else ("", []) elif word.startswith("-m"): return word[2:], args[at + 1 :] + elif INLINE.fullmatch(word): + return "", [] elif word in PYTHON_VALUE_OPTIONS: skip = True elif not word.startswith("-"): - return os.path.basename(word), args[at + 1 :] + return word, args[at + 1 :] return "", [] +def bare(word: str) -> str: + """Returns: + The program name of word: its basename, a scoped npm package as + typed, an `@version` suffix dropped. + """ + name = word if word.startswith("@") else os.path.basename(word) + return name[:1] + name[1:].split("@", 1)[0] + + def tool(run: list[str]) -> tuple[str, list[str]]: """Returns: - The program's name and arguments, the module and its arguments for - `python -m` or a Python script. + The program's name (bare()) and arguments, the module and its + arguments for `python -m` or a Python script. """ - name, args = os.path.basename(run[0]), run[1:] - return python_module(args) if PYTHON.fullmatch(name) else (name, args) + name, args = bare(run[0]), run[1:] + if not PYTHON.fullmatch(name): + return name, args + target, rest = python_target(args) + return os.path.basename(target), rest + + +def launched(run: list[str]) -> str: + """Returns: + The script or `-m` module a python call starts, or the script of a + node, deno or bun call (after a `run`); "" for none. + """ + name = os.path.basename(run[0]) + if PYTHON.fullmatch(name): + return python_target(run[1:])[0] + found = operands(run[1:], JS_VALUE_OPTIONS) if name in JS_RUNNERS else [] + if found[:1] == ["run"]: + found = found[1:] + return found[0] if found else "" def discover_patterns(rest: list[str]) -> list[str]: @@ -741,38 +957,540 @@ def started_in(prefix: list[str], cwd: str | None) -> str | None: def one_discovered(rest: list[str], base: str | None) -> bool: + return discovered_file(rest, base) is not None + + +def discover_start(rest: list[str]) -> str: + """Returns: + The start directory of `unittest discover` with these arguments: the + last -s/--start-directory value, else the first operand, else `.`. + """ + starts = [rest[at + 1] for at, word in enumerate(rest[:-1]) if word in START_OPTIONS] + return (starts or operands(rest, DISCOVER_VALUE_OPTIONS)[:1] or ["."])[-1] + +def discovered_file(rest: list[str], base: str | None) -> str | None: """Args: rest: the arguments after `unittest discover`. base: the directory the command starts in (started_in()), None when unknown. Returns: - Whether exactly one file below the start directory matches the - pattern: a plain name can match a file in every package of a tree. - Without base or start directory, whether the pattern is one file + The one file below the start directory that matches the pattern, a + plain name matching a file in every package of a tree counting as + several; None for none or several. Without base or start directory + the start directory joined with the pattern when that is one file name (TEST_FILE), so an unknown tree never blocks a single file. """ patterns = discover_patterns(rest) if not patterns: - return False - starts = [rest[at + 1] for at, word in enumerate(rest[:-1]) if word in START_OPTIONS] - start = (starts or operands(rest, DISCOVER_VALUE_OPTIONS)[:1] or ["."])[-1] + return None + start = discover_start(rest) if base is None or not os.path.isdir(os.path.join(base, start)): - return TEST_FILE.fullmatch(patterns[-1]) is not None - found = (name for _, _, files in os.walk(os.path.join(base, start)) for name in files + return os.path.join(start, patterns[-1]) if TEST_FILE.fullmatch(patterns[-1]) else None + found = (os.path.join(top, name) for top, _, files in os.walk(os.path.join(base, start)) for name in files if fnmatch.fnmatchcase(name, patterns[-1])) - return len([*itertools.islice(found, 2)]) == 1 + first = [*itertools.islice(found, 2)] + return first[0] if len(first) == 1 else None + + +def source(path: str) -> str | None: + try: + with open(os.open(path, os.O_RDONLY | os.O_NONBLOCK | os.O_CLOEXEC), "rb") as handle: + if not stat.S_ISREG(os.fstat(handle.fileno()).st_mode): + return None + data = handle.read(SOURCE_BYTES + 1) + except OSError: + return None + return None if len(data) > SOURCE_BYTES else data.decode("utf-8", "replace") + + +def parsed(path: str) -> ast.Module | None: + text = source(path) + try: + return None if text is None else ast.parse(text) + except (SyntaxError, ValueError, RecursionError): + return None + + +def dynamic_import(node: ast.AST) -> str | None: + """Returns: + The literal module name an importlib.import_module or __import__ + call imports, else None. + """ + if not isinstance(node, ast.Call) or not node.args: + return None + first = node.args[0] + literal = isinstance(first, ast.Constant) and isinstance(first.value, str) + return first.value if call_name(node) in DYNAMIC_IMPORTS and literal else None + + +def call_name(node: ast.Call) -> str: + return node.func.id if isinstance(node.func, ast.Name) else getattr(node.func, "attr", "") + + +def strings(node: ast.AST) -> list[str]: + return [found.value for found in ast.walk(node) if isinstance(found, ast.Constant) and isinstance(found.value, str)] + + +def plugin_names(tree: ast.Module) -> list[str]: + """Returns: + The module names a top-level pytest_plugins assignment of tree + lists. + """ + return [name for node in tree.body if isinstance(node, ast.Assign) + and any(isinstance(target, ast.Name) and target.id == PLUGIN_LIST for target in node.targets) + for name in strings(node.value)] + + +def import_refs(tree: ast.Module) -> list[tuple[str, int]]: + """Returns: + (module, level) per module tree imports: Import nodes, ImportFrom + nodes with each imported name also as a submodule, + dynamic_import() calls and pytest_plugins entries (plugin_names()). + """ + refs: list[tuple[str, int]] = [(name, 0) for name in plugin_names(tree)] + for node in ast.walk(tree): + if isinstance(node, ast.Import): + refs += [(alias.name, 0) for alias in node.names] + elif isinstance(node, ast.ImportFrom): + refs += [(".".join(filter(None, (node.module, alias.name))), node.level) for alias in node.names] + refs.append((node.module or "", node.level)) + elif (name := dynamic_import(node)) is not None: + refs.append((name, 0)) + return refs + + +def ancestors(path: str) -> list[Path]: + """Returns: + The directories from the one holding path up to the repository + root: the first holding .git or a pytest config (CONFIGS), else the + filesystem root. + """ + found = [] + for folder in Path(os.path.abspath(path)).parents: + found.append(folder) + if any((folder / name).exists() for name in (".git", *CONFIGS)): + break + return found + + +def browser_module(name: str) -> bool: + """Returns: + Whether an imported module or package specifier is a browser driver + (BROWSER_MODULES) by its top-level name or npm scope: + `playwright.sync_api` and `@playwright/experimental-ct-react` count, + `playwright_report` does not. + """ + return re.split(r"[./]", name)[0] in BROWSER_MODULES + + +def file_bytes(path: str) -> int: + try: + return os.stat(path).st_size + except OSError: + return REACH_BYTES + 1 + + +@functools.cache +def module_refs(path: str) -> tuple[tuple[str, int], ...] | None: + tree = parsed(path) + return None if tree is None else tuple(import_refs(tree)) + + +@functools.cache +def rooted_files(module: str, roots: tuple[str, ...]) -> frozenset[str]: + """Returns: + The module files and package __init__.py files below roots that + importing module runs; below a root the look stops at the first + part that is no folder. + """ + parts = module.split(".") if module else [] + found: set[str] = set() + for folder in roots: + prefix = folder + for part in parts: + prefix = os.path.join(prefix, part) + files, deeper = module_files(prefix) + found |= files + if not deeper: + break + return frozenset(found) + + +@functools.cache +def module_files(prefix: str) -> tuple[frozenset[str], bool]: + """Returns: + prefix.py and prefix/__init__.py where they are files, and whether + prefix is a folder a deeper module part can lie in. + """ + files = frozenset(path for path in (prefix + ".py", os.path.join(prefix, "__init__.py")) if os.path.isfile(path)) + return files, os.path.isdir(prefix) + + +def local_files(ref: tuple[str, int], here: str, roots: tuple[str, ...]) -> frozenset[str]: + """Returns: + rooted_files() of the module of ref below roots, for a relative + import below here moved up level - 1 directories. + """ + module, level = ref + return rooted_files(module, (os.path.normpath(os.path.join(here, *[".."] * (level - 1))),) if level else roots) + + +def imports_reach(seeds: list[str], roots: list[str]) -> bool: + """Returns: + Whether a seed file or a local module they import (local_files(), + recursively) imports a browser driver (browser_module()); a file + that cannot be read or parsed (over SOURCE_BYTES too) or a walk past + REACH_FILES files or REACH_BYTES bytes counts as one. + """ + seen, todo, bases, spent = set(seeds), list(seeds), tuple(roots), 0 + while todo: + current = todo.pop() + spent += file_bytes(current) + if len(seen) > REACH_FILES or spent > REACH_BYTES: + return True + refs = module_refs(current) + if refs is None or any(browser_module(module) for module, _ in refs): + return True + found = {path for ref in refs for path in local_files(ref, os.path.dirname(current), bases)} - seen + seen |= found + todo += sorted(found) + return False + + +def top_names(tree: ast.Module) -> set[str]: + """Returns: + The names a module binds at its top: functions, assigned names and + names from ImportFrom (`*` for a star import). + """ + names = {node.name for node in tree.body if isinstance(node, FUNCTIONS)} + names |= {target.id for node in tree.body if isinstance(node, ast.Assign) for target in node.targets + if isinstance(target, ast.Name)} + return names | imported_names(tree) + + +def imported_names(tree: ast.Module) -> set[str]: + return {alias.asname or alias.name for node in tree.body if isinstance(node, ast.ImportFrom) for alias in node.names} + + +def parameters(node: ast.FunctionDef | ast.AsyncFunctionDef) -> set[str]: + return {arg.arg for arg in [*node.args.args, *node.args.kwonlyargs]} + + +def asked_fixtures(tree: ast.Module) -> set[str]: + """Returns: + The fixtures a test module asks for: the parameters of its test + functions and the strings its getfixturevalue and usefixtures calls + name. + """ + asked = {name for node in ast.walk(tree) if isinstance(node, FUNCTIONS) and node.name.startswith("test") + for name in parameters(node)} + return asked | requested_fixtures(tree) + + +def requested_fixtures(tree: ast.Module) -> set[str]: + """Returns: + The fixture names the getfixturevalue and usefixtures calls of tree + request; all FIXTURES when an argument is no string literal (a + built name such as `'pa' + 'ge'`). + """ + arguments = [argument for node in ast.walk(tree) if isinstance(node, ast.Call) and call_name(node) in REQUESTS + for argument in node.args] + literal = {argument.value for argument in arguments + if isinstance(argument, ast.Constant) and isinstance(argument.value, str)} + return literal | (FIXTURES if any(not isinstance(argument, ast.Constant) for argument in arguments) else set()) + + +def builds_on_browser(tree: ast.Module) -> bool: + return any(parameters(node) & FIXTURES for node in ast.walk(tree) if isinstance(node, FUNCTIONS)) + + +def browser_fixtures(trees: list[ast.Module]) -> set[str]: + """Returns: + FIXTURES and every function of trees whose parameters reach one of + them, transitively: a fixture built on page, browser or context. + """ + params = [(node.name, parameters(node)) for tree in trees for node in ast.walk(tree) if isinstance(node, FUNCTIONS)] + found, size = set(FIXTURES), 0 + while size != len(found): + size = len(found) + found |= {name for name, args in params if args & found} + return found + + +def own_risk(tree: ast.Module, conftests: list[ast.Module]) -> bool: + """Returns: + Whether a test module defines or imports load_tests (a star import + too), or a test function asks for a browser fixture + (browser_fixtures() of the module and its conftest files). + """ + return bool(LOAD_TESTS & top_names(tree) or browser_fixtures([tree, *conftests]) & asked_fixtures(tree)) + + +def context_risk(folders: list[Path], conftests: list[str]) -> bool: + """Returns: + Whether a conftest.py names playwright or cannot be read, or a + pytest config in folders adds workers, plugins or playwright to + addopts (ADDOPTS). + """ + configs = [source(str(folder / name)) or "" for folder in folders for name in CONFIGS] + named = any(PLAYWRIGHT in (source(conftest) or PLAYWRIGHT).lower() for conftest in conftests) + return named or any(ADDOPTS.search(text) for text in configs) + + +def pythonpath_entries(config: Path) -> list[str]: + """Returns: + The directories the pytest `pythonpath` setting of one config file + (CONFIGS) lists, as written; none when it has none or does not + parse. + """ + text = source(str(config)) + if text is None: + return [] + try: + if config.name == "pyproject.toml": + value = tomllib.loads(text).get("tool", {}).get("pytest", {}).get("ini_options", {}).get("pythonpath", []) + return [value] if isinstance(value, str) else [entry for entry in value if isinstance(entry, str)] + parser = configparser.ConfigParser(interpolation=None) + parser.read_string(text) + return parser.get(INI_SECTIONS[config.name], "pythonpath", fallback="").split() + except (tomllib.TOMLDecodeError, configparser.Error, AttributeError, TypeError): + return [] + + +def search_roots(folders: list[Path]) -> list[str]: + """Returns: + Where a test's imports resolve: the directories from the test up to + the repository root and every pytest `pythonpath` entry of a config + there, relative to that config. + """ + added = [os.path.normpath(folder / entry) for folder in folders for name in CONFIGS + for entry in pythonpath_entries(folder / name)] + return [str(folder) for folder in folders] + added + + +def context_files(tree: ast.Module, conftests: list[str], roots: list[str]) -> list[str]: + """Returns: + The conftest files and the local modules the pytest_plugins of the + test module or of those conftest files name (plugin_names()). + """ + trees = [tree, *(found for found in map(parsed, conftests) if found is not None)] + named = {path for found in trees for name in plugin_names(found) for path in rooted_files(name, tuple(roots))} + return [*conftests, *sorted(named - set(conftests))] + + +def runner_reach(path: str) -> bool: + """Returns: + Whether a test runner running the Python file at path runs more + than one light test file. The context of a test is every conftest.py + from its directory up to the repository root (ancestors()) and every + module their pytest_plugins name (context_files()); the file is heavy + when it does not parse, own_risk(), context_risk(), a context file + does not parse or builds a fixture on page, browser or context + (builds_on_browser(), autouse ones too), or imports_reach() from the + file and its context through search_roots(). + """ + tree = parsed(path) + if tree is None: + return True + folders = ancestors(path) + roots = search_roots(folders) + conftests = [str(folder / "conftest.py") for folder in folders if (folder / "conftest.py").exists()] + context = context_files(tree, conftests, roots) + if context_heavy(tree, context) or context_risk(folders, conftests): + return True + return imports_reach([path, *context], roots) + + +def context_heavy(tree: ast.Module, context: list[str]) -> bool: + """Returns: + Whether a context file (context_files()) does not parse, builds a + fixture on a browser fixture (builds_on_browser()) or requests one + by name (requested_fixtures()), autouse ones too, or the test module + is own_risk() with those files. + """ + trees = [parsed(found) for found in context] + if any(found is None or builds_on_browser(found) for found in trees): + return True + browser = browser_fixtures([tree, *trees]) + return own_risk(tree, trees) or any(requested_fixtures(found) & browser for found in trees) + + +def reached(path: str, runner: bool) -> bool: + """Returns: + For a Python file runner_reach() when a test runner runs it, else + imports_reach() from it alone; playwright_file() for other files, + where an unreadable one counts as reached. + """ + if not path.endswith(".py"): + return playwright_file(path) is not False + return runner_reach(path) if runner else imports_reach([path], [str(folder) for folder in ancestors(path)]) + + +def string_end(text: str, start: int) -> int: + quote, at = text[start], start + 1 + while at < len(text) and text[at] != quote: + at += 2 if text[at] == "\\" else 1 + return at + + +def js_imports(text: str) -> list[str]: + """Returns: + The string literals of text that follow `from`, `import`, `require(` + or `import(` in code; comments and every other string are skipped. + """ + found: list[str] = [] + tail, at = "", 0 + while (match := JS_TOKEN.search(text, at)) is not None: + tail = (tail + text[at : match.start()])[-JS_TAIL:] + if match.group() == "//": + end = text.find("\n", match.end()) + at = len(text) if end < 0 else end + elif match.group() == "/*": + end = text.find("*/", match.end()) + at = len(text) if end < 0 else end + 2 + else: + end = string_end(text, match.start()) + if JS_IMPORTER.search(tail): + found.append(text[match.start() + 1 : end]) + tail, at = tail + " ", end + 1 + return found + + +def playwright_file(path: str) -> bool | None: + """Returns: + Whether the Python (module_refs()) or JavaScript/TypeScript + (js_imports()) file at path itself imports a browser driver + (browser_module()); None when it cannot be read as a regular file + or, for Python, parsed. + """ + if JS_SCRIPT.search(path): + text = source(path) + names = None if text is None else js_imports(text) + else: + refs = module_refs(path) + names = None if refs is None else [module for module, _ in refs] + return None if names is None else any(browser_module(name) for name in names) + + +def named_parts(test: str) -> tuple[str, list[str] | None]: + """Returns: + The file or module part of a test name (a pytest node id loses + `::...`) and its dotted parts when it is a module, else None. + """ + name = test.split("::", 1)[0] + return name, name.split(".") if not SCRIPT.search(name) and os.sep not in name else None + + +def test_file(test: str, base: str | None) -> str | None: + """Returns: + The regular file a test name runs, relative to base: the path, or + the first dotted prefix that is a .py file; None when there is none + or neither base nor an absolute path places it. + """ + name, module = named_parts(os.path.expanduser(test)) + paths = [os.path.join(*module[:end]) + ".py" for end in range(1, len(module) + 1)] if module else [name] + found = [ + os.path.join(base or "", path) for path in paths + if (base is not None or os.path.isabs(path)) and os.path.isfile(os.path.join(base or "", path)) + ] + return found[0] if found else None + + +def e2e_path(test: str, base: str | None) -> bool: + name, module = named_parts(test) + named = Path(*(module or [name])).parts + parts = Path(os.path.join(base or "", *named)).parts + return E2E[1] in named[:-1] or any(parts[at : at + 2] == E2E for at in range(len(parts))) + + +def browser_test(test: str, base: str | None, runner: bool) -> bool: + """Args: + test: a test file or script, a pytest node id (`file::name`) or a + dotted module name. + base: the directory the command starts in, None when unknown. + runner: whether a test runner runs it (reached()). + + Returns: + Whether it lies under a tests/e2e/ directory, passes an e2e + directory itself, or its file (test_file()) reaches playwright or + more tests (reached()). + """ + found = test_file(test, base) + return e2e_path(test, base) or (found is not None and reached(found, runner)) + + +def clean_test(test: str, base: str | None) -> bool: + found = test_file(test, base) + return found is not None and not e2e_path(test, base) and not reached(found, True) def light_unittest(args: list[str], base: str | None) -> bool: - if args[:1] != ["discover"]: - return len(operands(args, UNITTEST_VALUE_OPTIONS)) == 1 - return one_discovered(args[1:], base) + if args[:1] == ["discover"]: + found = discovered_file(args[1:], base) + else: + named = operands(args, UNITTEST_VALUE_OPTIONS) + found = named[0] if len(named) == 1 else None + return found is not None and not browser_test(found, base, True) -def heavy_pytest(args: list[str]) -> bool: +def heavy_pytest(args: list[str], base: str | None) -> bool: files = operands(args, PYTEST_VALUE_OPTIONS) - return not (PYTEST_LIGHT.intersection(args) or (len(files) == 1 and TEST_FILE.fullmatch(files[0]))) + if PYTEST_LIGHT.intersection(args): + return False + return not (len(files) == 1 and TEST_FILE.fullmatch(files[0])) or browser_test(files[0], base, True) + + +def plain_operands(args: list[str], flags: frozenset[str], values: frozenset[str]) -> list[str] | None: + """Returns: + The operands of args when each option is one of flags or values (a + value option with its value, also as --option=value), else None. + """ + found, skip = [], False + for word in args: + if skip: + skip = False + elif not word.startswith("-"): + found.append(word) + elif word.split("=", 1)[0] in values: + skip = "=" not in word + elif word not in flags: + return None + return found + + +def single_pytest(args: list[str], base: str | None) -> bool: + """Returns: + Whether pytest runs help, collects with plain options only, or runs + exactly one clean test file (clean_test()) with plain options only + (PYTEST_FLAGS, PYTEST_VALUES). + """ + files = plain_operands(args, PYTEST_FLAGS | PYTEST_LIGHT, PYTEST_VALUES) + if HELP.intersection(args) or (files is not None and PYTEST_LIGHT.intersection(args)): + return True + return files is not None and len(files) == 1 and bool(TEST_FILE.fullmatch(files[0])) and clean_test(files[0], base) + + +def single_unittest(args: list[str], base: str | None) -> bool: + """Returns: + Whether unittest runs help, or exactly one clean test file or module + (clean_test()) with plain options only (UNITTEST_FLAGS, + UNITTEST_VALUES) and without discovery. + """ + named = plain_operands(args, UNITTEST_FLAGS, UNITTEST_VALUES) + if HELP.intersection(args): + return True + return named is not None and args[:1] != ["discover"] and len(named) == 1 and clean_test(named[0], base) + + +def light_test(name: str, args: list[str], base: str | None) -> bool: + """Returns: + Whether pytest or unittest runs a light form (single_pytest(), + single_unittest()). + """ + if name in PYTESTS: + return single_pytest(args, base) + return name == "unittest" and single_unittest(args, base) def heavy_unittest(args: list[str], base: str | None) -> bool: @@ -788,7 +1506,8 @@ def dry_run(args: list[str]) -> bool: def heavy_make(args: list[str]) -> bool: - return not dry_run(args) and any(target == "install" or target.startswith("verify-") for target in make_targets(args)) + targets = make_targets(args) + return not dry_run(args) and (not targets or any(target == "install" or target.startswith(HEAVY_MAKE) for target in targets)) def heavy_compose(args: list[str]) -> bool: @@ -801,10 +1520,167 @@ def heavy_docker(args: list[str]) -> bool: return args[:1] == ["compose"] and heavy_compose(args[1:]) +def heavy_runner(args: list[str]) -> bool: + return not HELP.intersection(args) + + +def heavy_test_word(args: list[str]) -> bool: + return operands(args, frozenset())[:1] == ["test"] + + +def package_call(runner: str, args: list[str]) -> tuple[list[str], list[str], bool, list[str]]: + """Args: + runner: npm, pnpm, yarn or bun. + args: the words after it. + + Returns: + The folders FOLDER_OPTIONS name, the packages PACKAGE_OPTIONS or + `yarn workspace NAME` name, whether ALL_PACKAGES or `yarn + workspaces` asks for every package, and the words from the first + operand on; options before it and their values are skipped. + """ + folders: list[str] = [] + packages: list[str] = [] + every, rest, words = False, [], iter(args) + for word in words: + option, equals, value = word.partition("=") + if rest or not word.startswith("-"): + rest.append(word) + elif option in FOLDER_OPTIONS or option in PACKAGE_OPTIONS[runner]: + (folders if option in FOLDER_OPTIONS else packages).append(value if equals else next(words, "")) + else: + every = every or word in ALL_PACKAGES + return workspace_call(folders, packages, every, rest) + + +def workspace_call(folders: list[str], packages: list[str], every: bool, + rest: list[str]) -> tuple[list[str], list[str], bool, list[str]]: + if rest[:1] == ["workspace"]: + return folders, [*packages, *rest[1:2]], every, rest[2:] + return folders, packages, every or rest[:1] == ["workspaces"], rest + + +def heavy_packages(runner: str, args: list[str]) -> bool: + found = package_call(runner, args)[3] + return bool(found) and found[0] in RUN_WORDS + + +def heavy_node(args: list[str]) -> bool: + return "--test" in args + + HEAVY_TOOLS = { - "pytest": heavy_pytest, "py.test": heavy_pytest, "make": heavy_make, - "docker": heavy_docker, "docker-compose": heavy_compose, + "make": heavy_make, "gmake": heavy_make, "docker": heavy_docker, "docker-compose": heavy_compose, "tox": heavy_runner, + "nox": heavy_runner, "hatch": heavy_test_word, "deno": heavy_test_word, "@playwright/test": heavy_runner, + "cargo": heavy_runner, "node": heavy_node, + **dict.fromkeys(BROWSER_CLIS, heavy_runner), + **{runner: functools.partial(heavy_packages, runner) for runner in PACKAGE_RUNNERS}, } +OWN_RULE = KNOWN | PYTESTS | JS_RUNNERS | (HEAVY_TOOLS.keys() - {"docker", "docker-compose"}) +TRIGGERS = PYTESTS | JS_RUNNERS | HEAVY_TOOLS.keys() | WRAPPERS | SHELLS +HEAVY_TARGETS = ( + "test", "test-unit", "test-integration", "quality", "quality-high", "verify-x", "compose-deploy", "compose-up", + "compose-playwright", "roundtrip", "swarm-deploy", "act-workflow", "act-runner-image", "build", "worktree-up", "store-up", +) +PATTERN_PROBES = { + "heavy_commands": ( + "", "true", "git status", "ruff check .", "python3 -m pytest tests/test_x.py", "cat a/b.md", f"cat {'a/' * 150}b.md", + *sorted(LIGHT_TOOLS), *(f"{tool} -n x a/b.md" for tool in sorted(LIGHT_TOOLS)), + ), + "never_block": ( + "", "pytest tests/", "pytest a.py b.py", "python3 -m pytest", "python3 -m unittest", "npx playwright test", + "docker build .", "docker buildx build .", "docker compose build", "docker pull nginx", "docker load -i x.tar", + "bash scripts/run.sh local", "npm run e2e", "pnpm exec cypress run", + *(f"{tool} run" for tool in sorted(HEAVY_TOOLS)), *(f"make {target}" for target in HEAVY_TARGETS), + ), +} + + +def package_scripts(base: str | None) -> tuple[str | None, dict]: + """Returns: + The folder of the package.json nearest above base (ancestors()) and + its scripts, {} when they are no object; base and {} without one. + """ + for folder in ancestors(os.path.join(base, "package.json")) if base else []: + text = source(str(folder / "package.json")) + if text is None: + continue + try: + scripts = json.loads(text).get("scripts") + except (ValueError, AttributeError): + scripts = None + return str(folder), scripts if isinstance(scripts, dict) else {} + return base, {} + + +def heavy_tool(name: str, args: list[str], cfg: dict, exempt: tuple[str, ...], base: str | None) -> bool: + """Returns: + HEAVY_TOOLS' rule for name, for a package runner also + heavy_script_call(). + """ + if name in HEAVY_TOOLS and HEAVY_TOOLS[name](args): + return True + return name in PACKAGE_RUNNERS and heavy_script_call(name, args, cfg, exempt, base) + + +def heavy_script_call(runner: str, args: list[str], cfg: dict, exempt: tuple[str, ...], base: str | None) -> bool: + """Returns: + Whether the package script a package-runner call names is heavy + (scripts_heavy()) in the folder its options lead to from base; a + PACKAGE_COMMANDS word such as add or install is no script. Without + base, for every package or for a named package without a folder of + that name it counts heavy. + """ + folders, packages, every, rest = package_call(runner, args) + if not rest or rest[0] in PACKAGE_COMMANDS: + return False + if base is None or every: + return True + folder = os.path.join(base, *folders, *packages[-1:]) + if packages and not os.path.isfile(os.path.join(folder, "package.json")): + return True + root, scripts = package_scripts(folder) + return scripts_heavy(scripts, rest[0], root or folder, cfg, exempt) + + +def scripts_heavy(scripts: dict, name: str, folder: str, cfg: dict, exempt: tuple[str, ...]) -> bool: + """Returns: + Whether script name or its pre or post script is heavy + (script_heavy()). + """ + return any(script_heavy(scripts, found, folder, cfg, exempt) for found in (f"pre{name}", name, f"post{name}") + if isinstance(scripts.get(found), str)) + + +def script_heavy(scripts: dict, name: str, folder: str, cfg: dict, exempt: tuple[str, ...]) -> bool: + """Returns: + Whether the body of script name is heavy() in folder, or a quoted + command in it (`concurrently "cypress run"`) or a script it names + (`start-server-and-test dev URL cy:run`, `npm:dev`) is; a script + reached again through a runner call while it is judged, a loop, + counts heavy. + """ + if (folder, name) in RESOLVING: + return True + RESOLVING.add((folder, name)) + try: + return body_heavy(scripts, scripts[name], folder, cfg, exempt) + finally: + RESOLVING.discard((folder, name)) + + +def body_heavy(scripts: dict, body: str, folder: str, cfg: dict, exempt: tuple[str, ...]) -> bool: + words = body_words(body) + named = {SCRIPT_PREFIX.sub("", word) for word in words} & scripts.keys() + return (heavy(body, cfg, exempt, folder) or any(heavy(word, cfg, exempt, folder) for word in words if SPACE.search(word)) + or any(scripts_heavy(scripts, found, folder, cfg, exempt) for found in named if (folder, found) not in RESOLVING)) + + +def body_words(body: str) -> list[str]: + try: + return [word for part in segments(body) for word in part] + except ValueError: + return [] def heavy_words(run: list[str], cfg: dict, exempt: tuple[str, ...], base: str | None) -> bool: @@ -822,7 +1698,12 @@ def heavy_words(run: list[str], cfg: dict, exempt: tuple[str, ...], base: str | return heavy_words(program([name, *args]), cfg, exempt, base) if name == "unittest": return heavy_unittest(args, base) - return name in HEAVY_TOOLS and HEAVY_TOOLS[name](args) + if name in PYTESTS: + return heavy_pytest(args, base) + script = launched(run) + if script and browser_test(script, base, False): + return True + return heavy_tool(name, args, cfg, exempt, base) def segment_texts(words: list[str], run: list[str]) -> list[str]: @@ -837,16 +1718,225 @@ def judged(texts: list[str], run: list[str], cfg: dict, exempt: tuple[str, ...], return heavy_words(run, cfg, exempt, base) +def inner_start(run: list[str]) -> int | None: + """Returns: + For a program without a rule of its own (not OWN_RULE, docker + included), the first later word naming a runner, wrapper or shell + (TRIGGERS, python): what an unknown wrapper such as systemd-run, + taskset or xvfb-run may start, or the command after a find -exec, + -execdir, -ok or -okdir; None otherwise. + """ + if run and bare(run[0]) == "find": + return find_exec(run) + if not run or bare(run[0]) in OWN_RULE or PYTHON.fullmatch(bare(run[0])): + return None + return next((at for at in range(1, len(run)) if bare(run[at]) in TRIGGERS or PYTHON.fullmatch(bare(run[at]))), None) + + +def find_exec(run: list[str]) -> int | None: + return next((at + 1 for at, word in enumerate(run[:-1]) if word in FIND_EXEC), None) + + +def heavy_targets(args: list[str], cfg: dict, exempt: tuple[str, ...], base: str | None) -> bool: + """Returns: + Whether a make call with these arguments is heavy, judged per target + as `make ` and without a target as `make`. + """ + calls = [["make", target] for target in make_targets(args)] or [["make"]] + return any(judged([" ".join(call)], call, cfg, exempt, base) for call in calls) + + +def exempted(words: list[str], run: list[str], exempt: tuple[str, ...]) -> bool: + return any(re.search(pattern, text) for pattern in exempt for text in segment_texts(words, run)) + + +def assignments(words: list[str]) -> dict[str, str]: + """Returns: + The variables a simple command sets: VAR=value words in front of + its program (program()), after export or declare, or anywhere when + the program is no tail of the words (env -S). + """ + run = program(words) + tail = bool(run) and words[len(words) - len(run) :] == run + prefix = words[1:] if words[:1] and words[0] in EXPORTS else words[: len(words) - len(run)] if tail else words + return dict(word.split("=", 1) for word in prefix if ASSIGNMENT.fullmatch(word)) + + +def stirring(names: Iterable[str]) -> bool: + """Returns: + Whether one of the variables can change what a command runs: any + outside INERT_ENV and the LC_* locale. + """ + return any(name not in INERT_ENV and not name.startswith(LOCALE) for name in names) + + +def cargo_allowed(run: list[str], names: dict[str, str]) -> bool: + """Returns: + Whether a cargo call needs no queue: help, version, fmt or metadata, + or build, check, clippy or test with CARGO_BUILD_JOBS between 1 and + CARGO_JOBS set in front and no -j or --jobs. + """ + args = run[1:] + found = [word for word in operands(args, frozenset()) if not word.startswith("+")] + command = found[0] if found else "" + jobs = names.get("CARGO_BUILD_JOBS", "") + capped = jobs.isdecimal() and 1 <= int(jobs) <= CARGO_JOBS and not any(CARGO_JOB_FLAGS.fullmatch(word) for word in args) + return bool(HELP.intersection(args)) or command in CARGO_LIGHT or (command in CARGO_CAPPED and capped) + + +def runs_tests(run: list[str]) -> bool: + """Returns: + Whether a simple command starts pytest, unittest or cargo, also + behind a program without a rule of its own (inner_start()). + """ + while run: + if tool(run)[0] in PYTESTS | {"unittest"} or bare(run[0]) == "cargo": + return True + at = inner_start(run) + run = run[at:] if at is not None else [] + return False + + +def test_runs(command: str, exempt: tuple[str, ...]) -> int: + """Returns: + How many simple commands of command, the code of its `sh -c` calls + and substitutions included, start tests (runs_tests()), exempt ones + aside; 2 for an unbalanced quote. + """ + plain, codes = substituted(uncommented(marked_heredocs(command)[0])) + try: + parts = [words for words in segments(plain) if words] + except ValueError: + return 2 + count = sum(test_runs(code, exempt) for code in codes) + for words in parts: + run = program(words) + if run and not exempted(words, run, exempt): + code = inline_code(run) + count += test_runs(code, exempt) if code is not None else int(runs_tests(run)) + return count + + +def unparsed_heavy(command: str, cfg: dict, exempt: tuple[str, ...]) -> bool: + text = executable_text(command) + return not any(re.search(p, text) for p in exempt) and any(re.search(p, text) for p in cfg["heavy_commands"]) + + +def crowded(command: str, parts: list[list[str]], exempt: tuple[str, ...]) -> bool: + """Returns: + Whether command starts two test runs (test_runs()), or one beside + an assignment of its simple commands parts that can change it + (stirring()). + """ + runs = test_runs(command, exempt) + return runs > 1 or (runs == 1 and stirring(name for words in parts for name in assignments(words))) + + def heavy_part(words: list[str], cfg: dict, exempt: tuple[str, ...], cwd: str | None) -> bool: """Returns: - Whether one simple command is heavy; a make call is judged per target - as `make `, so an exempt target never covers the next one. + Whether one simple command is heavy; a make or gmake call is judged + per target as `make `, so an exempt target never covers the + next one, and without a target as `make`; a cargo_allowed() call is + light; behind a program without a rule of its own the command from + inner_start() on counts too. """ run = program(words) base = started_in(words[: len(words) - len(run)], cwd) - if run and os.path.basename(run[0]) == "make" and not dry_run(run[1:]): - return any(judged([f"make {target}"], ["make", target], cfg, exempt, base) for target in make_targets(run[1:])) - return judged(segment_texts(words, run), run, cfg, exempt, base) + if run and bare(run[0]) in MAKES and not dry_run(run[1:]): + return heavy_targets(run[1:], cfg, exempt, base) + texts = segment_texts(words, run) + if exempted(words, run, exempt) or (run and bare(run[0]) == "cargo" and cargo_allowed(run, assignments(words))): + return False + if judged(texts, run, cfg, exempt, base): + return True + at = inner_start(run) + return at is not None and heavy_part(run[at:], cfg, exempt, base) + + +def segment_bases(parts: list[list[str]], cwd: str | None) -> list[str | None]: + """Returns: + The directory each simple command of parts starts in: cwd, moved by + every cd or pushd before it to its target (None for a target with a + substitution, a variable or none) and unknown after popd. + """ + bases = [] + for words in parts: + bases.append(cwd) + if words[:1] == ["popd"]: + cwd = None + elif words[:1] in (["cd"], ["pushd"]): + target = [word for word in words[1:] if not word.startswith("-")][:1] + known = cwd is not None and target and "$" not in target[0] + cwd = os.path.join(cwd, os.path.expanduser(target[0])) if known else None + return bases + + +def safe_run(words: list[str]) -> list[str] | None: + """Returns: + The program part of words (program()) when nothing but + SAFE_PREFIXES, their options and VAR=value words precede it, else + None. + """ + run = program(words) + prefix = words[: len(words) - len(run)] + if not run or words[len(prefix) :] != run or any(os.path.basename(word) in WRAPPERS - SAFE_PREFIXES for word in prefix): + return None + return run + + +def light_program(run: list[str], words: list[str], base: str | None) -> bool: + name, args = bare(run[0]), run[1:] + if name == "cargo": + return cargo_allowed(run, assignments(words)) + if name == "find": + return not FIND_EXEC.intersection(args) + return name in LIGHT_TOOLS or light_test(name, args, base) + + +def light_python(run: list[str], base: str | None) -> bool: + name, args = tool(run) + target = python_target(run[1:])[0] + module = target == name and os.sep not in target and not SCRIPT.search(target) + return module and (name in LIGHT_MODULES or light_test(name, args, base)) + + +def light_part(words: list[str], base: str | None) -> bool: + if bare(words[0]) in DIRECTORY_CHANGES: + return True + run = safe_run(words) + if run is None or stirring(assignments(words)): + return False + base = started_in(words[: len(words) - len(run)], base) + code = inline_code(run) + if code is not None: + return light(code, base) + return light_python(run, base) if PYTHON.fullmatch(bare(run[0])) else light_program(run, words, base) + + +def light(command: str, cwd: str | None) -> bool: + """Args: + command: the shell command line as typed. + cwd: the directory it starts in, None when unknown. + + Returns: + Whether every simple command of it is a recognised small form: cd, + pushd or popd, or behind nothing but SAFE_PREFIXES and INERT_ENV or + LC_* assignments a LIGHT_TOOLS program (find without -exec), a + cargo_allowed() call, `python -m` of LIGHT_MODULES, `sh -c` of a + light command, or light_test(); test files resolve relative to cwd + moved by every cd and env -C. A substitution, heredoc, background + job, unbalanced quote or second test run (test_runs()) makes it not + light. + """ + marked, heredocs = marked_heredocs(command) + plain, codes = substituted(uncommented(marked)) + if heredocs or codes or BACKGROUND.search(plain) or test_runs(command, ()) > 1: + return False + try: + parts = [words for words in segments(plain) if words] + except ValueError: + return False + return bool(parts) and all(light_part(words, base) for words, base in zip(parts, segment_bases(parts, cwd))) def marked_heredocs(command: str) -> tuple[str, list[re.Match[str]]]: @@ -951,20 +2041,23 @@ def payload_cwd(payload: dict) -> str | None: return cwd if isinstance(cwd, str) and os.path.isabs(cwd) else None -def cmd_block(cfg: dict, problem: str | None) -> int: +def cmd_block(cfg: dict, problem: str | None, unsettled: bool) -> int: """Args: cfg: the config. problem: why the memory stage cannot be judged, None when it can; a heavy job is then refused with it. + unsettled: whether cfg stands in for an unusable config file + (settled_config()); a heavy job is then refused with problem alone. """ payload = json.load(sys.stdin) raw = (payload.get("tool_input") or {}).get("command", "") if not heavy(raw, cfg, tuple(cfg["never_block"]), payload_cwd(payload)): return 0 - reason = block_reason(cfg, problem) + reason = problem if unsettled else block_reason(cfg, problem) if reason is None: return 0 - print(f"kvb-org-organize: {reason}. Teardown commands still run. Ask the resource chair.", file=sys.stderr) + still = " Teardown commands still run." if cfg["never_block"] else "" + print(f"kvb-org-organize: {reason}.{still} Ask the resource chair.", file=sys.stderr) return 2 @@ -1026,31 +2119,109 @@ def load(cfg: dict) -> tuple[float, float]: return cpu, mem_used_percent(MEMINFO.read_text(encoding="utf-8")) -def psi_full(pressure: str) -> float | None: +def psi_avg10(pressure: str, kind: str) -> float | None: for line in pressure.splitlines(): - if line.startswith("full "): + if line.startswith(f"{kind} "): return float(dict(field.split("=", 1) for field in line.split()[1:])["avg10"]) return None -def memory_sample(meminfo: str, pressure: str | None, container_swap_gb: float) -> dict[str, float | None]: +def swap_pages(vmstat: str) -> int | None: + for line in vmstat.splitlines(): + key, _, value = line.partition(" ") + if key == "pswpout" and value.strip().isdigit(): + return int(value) + return None + + +def swap_out_rate(before: list[float] | None, after: list[float], page_bytes: int) -> float | None: + """Args: + before: [epoch seconds, pswpout] of the earlier reading, None without + one. + after: [epoch seconds, pswpout] of this reading. + page_bytes: the size of one swapped page. + + Returns: + The MB swapped out per minute between both; None without an earlier + reading, after a counter reset and when they lie outside SWAP_SPAN + seconds apart. + """ + if before is None: + return None + span, pages = after[0] - before[0], after[1] - before[1] + if pages < 0 or not SWAP_SPAN[0] <= span <= SWAP_SPAN[1]: + return None + return pages * page_bytes / MB * 60 / span + + +def swap_readings(stored: list[list[float]], reading: list[float]) -> tuple[list[float] | None, list[list[float]]]: + """Args: + stored: the kept [epoch seconds, pswpout] readings, oldest first. + reading: this one. + + Returns: + The newest stored reading at least SWAP_SPAN[0] seconds older than + reading, None without one, and the readings to keep: the newest + stored one and reading once that is SWAP_SPAN[0] seconds old, else + the last two stored. A reading from the future or above the count of + reading (a counter reset) is dropped. + """ + valid = [kept for kept in stored if kept[0] < reading[0] and kept[1] <= reading[1]] + older = [kept for kept in valid if reading[0] - kept[0] >= SWAP_SPAN[0]] + baseline = older[-1] if older else None + if valid and reading[0] - valid[-1][0] < SWAP_SPAN[0]: + return baseline, valid[-2:] + return baseline, [*valid[-1:], reading] + + +def stored_readings() -> list[list[float]]: + readings = read_json(SWAP_OUT).get("readings") + return [ + reading for reading in (readings if isinstance(readings, list) else []) + if isinstance(reading, list) and len(reading) == 2 + and all(isinstance(value, (int, float)) and not isinstance(value, bool) for value in reading) + ] + + +def swap_out() -> float | None: + try: + pages = swap_pages(VMSTAT.read_text(encoding="utf-8")) + except OSError: + return None + if pages is None: + return None + reading = [time.time(), pages] + stored = stored_readings() + baseline, kept = swap_readings(stored, reading) + if kept != stored: + write_json(SWAP_OUT, {"readings": kept}) + return swap_out_rate(baseline, reading, PAGE_BYTES) + + +def memory_sample(meminfo: str, pressure: str | None, container_swap_gb: float, + swap_out_mb_min: float | None) -> dict[str, float | None]: """Args: meminfo: the text of /proc/meminfo. pressure: the text of /proc/pressure/memory, None without PSI. container_swap_gb: the swap of the container holding the most, in GB. + swap_out_mb_min: the swap-out rate in MB per minute (swap_out()), + None without one. Returns: - {available, swap_free, psi_full, container_swap_gb}: MemAvailable and - SwapFree in percent of their totals, the full avg10 stall percentage - and container_swap_gb unchanged; swap_free is None without swap, - psi_full None without PSI. + {available, swap_free, psi_some, psi_full, swap_out_mb_min, + container_swap_gb}: MemAvailable and SwapFree in percent of their + totals, the some and full avg10 stall percentages, the rate and + container_swap_gb unchanged; swap_free is None without swap, psi_some + and psi_full None without PSI. """ values = meminfo_kb(meminfo) swap = values["SwapTotal"] return { "available": 100.0 * values["MemAvailable"] / values["MemTotal"], "swap_free": 100.0 * values["SwapFree"] / swap if swap else None, - "psi_full": None if pressure is None else psi_full(pressure), + "psi_some": None if pressure is None else psi_avg10(pressure, "some"), + "psi_full": None if pressure is None else psi_avg10(pressure, "full"), + "swap_out_mb_min": swap_out_mb_min, "container_swap_gb": container_swap_gb, } @@ -1061,7 +2232,7 @@ def memory() -> dict[str, float | None]: except OSError: pressure = None largest = max((size for size, _ in container_swaps(CGROUPS)), default=0) - return memory_sample(MEMINFO.read_text(encoding="utf-8"), pressure, largest / GB) + return memory_sample(MEMINFO.read_text(encoding="utf-8"), pressure, largest / GB, swap_out()) def memory_stage(sample: dict[str, float | None], cfg: dict) -> str: @@ -1070,17 +2241,24 @@ def memory_stage(sample: dict[str, float | None], cfg: dict) -> str: if ( sample["available"] < bound["available"] or (sample["swap_free"] is not None and sample["swap_free"] < bound["swap_free"]) - or (sample["psi_full"] is not None and sample["psi_full"] >= bound["psi_full"]) + or any(sample[key] is not None and sample[key] >= bound[key] for key in PRESSURES[name]) or sample["container_swap_gb"] >= bound["container_swap_gb"] ): return name return "ok" +def measured(value: float | None, unit: str) -> str: + return "unmeasured" if value is None else f"{value:.1f}{unit}" + + def memory_text(sample: dict[str, float | None]) -> str: swap = "no swap" if sample["swap_free"] is None else f"swap free {sample['swap_free']:.0f} %" - psi = "no PSI" if sample["psi_full"] is None else f"PSI full avg10 {sample['psi_full']:.1f}" - return f"MemAvailable {sample['available']:.0f} %, {swap}, {psi}, largest container swap {sample['container_swap_gb']:.1f} GB" + psi = f"PSI avg10 some {measured(sample['psi_some'], '')} full {measured(sample['psi_full'], '')}" + return ( + f"MemAvailable {sample['available']:.0f} %, {swap}, {psi}, swap-out {measured(sample['swap_out_mb_min'], ' MB/min')}, " + f"largest container swap {sample['container_swap_gb']:.1f} GB" + ) def swap_bytes(cgroup: Path) -> int: @@ -1154,15 +2332,24 @@ def heavy(command: str, cfg: dict, exempt: tuple[str, ...], cwd: str | None) -> exempt: regexes; a simple command one of them finds is never heavy (never_block, the queue's own hold), the others still count. cwd: the directory the shell runs it in (the hook payload's cwd), - None when unknown; a cd, pushd or popd in it makes it unknown. + None when unknown; each simple command starts where the cd and + pushd before it lead (segment_bases()), nested code where none + leads. Returns: Whether one simple command is a heavy job: an image build through docker compose (build, or up, run or create with --build), a - heavy_commands match, make install or verify-* (not as a dry run), or - pytest or unittest on anything but exactly one test file or module - (a discover pattern only when one_discovered()), help and version - aside. Wrappers (program()), the code of `sh -c`, + heavy_commands match, make or gmake without a target or with + install, test*, quality* or verify-* (not as a dry run), tox, nox, + hatch or deno test, playwright, cargo, node --test, npm, pnpm, yarn + or bun with a RUN_WORDS command, pytest or unittest on anything but + exactly one test file or module (a discover pattern only when + one_discovered()), or a test, python script or module, or node, deno + or bun script that is a browser_test(), help and version aside; also + two test runs in one command line (test_runs()), or one beside an + assignment that can change it (stirring()); a cargo_allowed() call + is light. Wrappers (program(), npx included), whatever follows a + program without a rule of its own (inner_start()), the code of `sh -c`, command substitutions (substituted(), also in heredocs the shell expands) and heredocs a shell reads as its code count the same. A command with an unbalanced quote, which no shell runs, counts by the @@ -1177,30 +2364,57 @@ def heavy(command: str, cfg: dict, exempt: tuple[str, ...], cwd: str | None) -> try: parts = segments(plain) except ValueError: - text = executable_text(command) - return not any(re.search(p, text) for p in exempt) and any(re.search(p, text) for p in cfg["heavy_commands"]) + return unparsed_heavy(command, cfg, exempt) + if crowded(command, parts, exempt): + return True nested = codes + expanded_codes(heredocs) + fed_bodies(parts) + bases = segment_bases(parts, cwd) if any(words[:1] in (["cd"], ["pushd"], ["popd"]) for words in parts): cwd = None - return (any(heavy_part(words, cfg, exempt, cwd) for words in parts) + return (any(heavy_part(words, cfg, exempt, base) for words, base in zip(parts, bases)) or any(heavy(code, cfg, exempt, cwd) for code in nested)) +def queue_hold(command: str) -> str: + return f"queue.py hold --kind {HEAVY_KIND} --task -- {command}" + + +def admit_sample(cfg: dict) -> tuple[float, float, dict[str, float | None]]: + """Returns: + CPU busy and RAM used over the load window (load()) and the memory + sample after it. + """ + swap_out() + cpu, mem = load(cfg) + return cpu, mem, memory() + + +def admit_need(cfg: dict, args: argparse.Namespace) -> float | None: + return args.need_gb if args.need_gb is not None else cfg["slot_need_gb"].get(args.kind) + + def cmd_admit(cfg: dict, args: argparse.Namespace) -> int: - need = args.need_gb if args.need_gb is not None else cfg["slot_need_gb"][args.kind] + command = shlex.join(args.command) + if not light(command, os.getcwd()): + print(f"deny: {command} is no light run and never self-admits; run it as `{queue_hold(command)}`") + return 3 + need = admit_need(cfg, args) + if need is None: + print(f"deny: kind {args.kind!r} is not in slot_need_gb of {CONFIG}") + return 3 free, _ = disk(cfg) left = free - need floor = cfg["thresholds_gb"]["stop"] - passed = [left >= floor] report = [f"free {free:.1f} GB, need {need} GB, leaves {left:.1f} GB (floor {floor} GB)"] limits = cfg["load_limits"] - cpu, mem = load(cfg) - sample = memory() + cpu, mem, sample = admit_sample(cfg) stage = memory_stage(sample, cfg) - passed += [cpu < limits["cpu_percent"], mem < limits["mem_percent"], stage != "stop"] + passed = left >= floor and stage != "stop" + reason = loaded("queue", QUEUE).admit(cfg, args.kind or LIGHT_KIND, args.command, cpu, mem, stage == "ok") if passed else None report.append(f"cpu {cpu:.0f} % (limit {limits['cpu_percent']}), ram {mem:.0f} % (limit {limits['mem_percent']})") report.append(f"memory stage {stage} ({memory_text(sample)})") - verdict = "grant" if all(passed) else "deny" + report.append(reason or "RAM used plus the job and the queue's still to come fits") + verdict = "grant" if passed and reason is None else "deny" print(f"{verdict}: {'; '.join(report)}") return 0 if verdict == "grant" else 3 @@ -1562,9 +2776,53 @@ def cmd_status(cfg: dict) -> int: return 0 +def cmd_preflight() -> int: + cfg, warning = settled_config() + problem = warning or (None if cfg is None else missing(cfg, MEMORY_KEYS)) + if problem is None: + return 0 + print(problem, file=sys.stderr) + return 2 + + def main() -> int: if sys.argv[1:2] == ["guard"] and len(sys.argv) > 3: return cmd_guard(sys.argv[2], sys.argv[3], sys.argv[4:]) + if sys.argv[1:2] != ["block"]: + return run() + try: + return run() + except Exception as error: # noqa: BLE001 + print(f"kvb-org-organize: the block hook failed ({error!r}), so the command counts as heavy and is refused.", + file=sys.stderr) + return 2 + + +def announced_config() -> tuple[dict | None, str | None]: + """Returns: + settled_config(), its warning printed to stderr. + """ + cfg, warning = settled_config() + if warning is not None: + print(warning, file=sys.stderr) + return cfg, warning + + +def watch_hook() -> int: + cfg, warning = settled_config() + if cfg is None: + return 0 + return cmd_watch(cfg, warning or missing(cfg, MEMORY_KEYS), warning is not None) + + +def block_hook() -> int: + cfg, warning = announced_config() + if cfg is None: + return 0 + return cmd_block(cfg, warning or missing(cfg, MEMORY_KEYS), warning is not None) + + +def run() -> int: parser = argparse.ArgumentParser(prog="watch") sub = parser.add_subparsers(dest="cmd", required=True) sub.add_parser("watch") @@ -1573,31 +2831,36 @@ def main() -> int: need = admit.add_mutually_exclusive_group(required=True) need.add_argument("--need-gb", type=float) need.add_argument("--kind") + admit.add_argument("command", nargs="+") cleanup = sub.add_parser("cleanup") mode = cleanup.add_mutually_exclusive_group(required=True) mode.add_argument("--caches", action="store_true") mode.add_argument("--full", action="store_true") sub.add_parser("status") + sub.add_parser("preflight") args = parser.parse_args() os.environ["PATH"] = hookpath.trusted(os.environ.get("PATH", "")) + if args.cmd == "preflight": + return cmd_preflight() refused = guard_root(hook=args.cmd in ("watch", "block")) if refused is not None: return refused - cfg = load_config() + if args.cmd == "block": + return block_hook() + if args.cmd == "watch": + return watch_hook() + cfg, warning = announced_config() if cfg is None: return 0 + if warning is not None and args.cmd == "cleanup": + return 2 problem = missing(cfg, MEMORY_KEYS) - if args.cmd == "watch": - return cmd_watch(cfg, problem) - if args.cmd == "block": - return cmd_block(cfg, problem) if problem is not None: print(problem, file=sys.stderr) return 2 - if args.cmd == "admit": - return cmd_admit(cfg, args) - if args.cmd == "cleanup": - return cmd_cleanup(cfg, args) + handlers = {"admit": cmd_admit, "cleanup": cmd_cleanup} + if args.cmd in handlers: + return handlers[args.cmd](cfg, args) return cmd_status(cfg) diff --git a/skills/kvb-org-queue/SKILL.md b/skills/kvb-org-queue/SKILL.md index 4798809..ccd5416 100644 --- a/skills/kvb-org-queue/SKILL.md +++ b/skills/kvb-org-queue/SKILL.md @@ -27,9 +27,8 @@ turn: hold runs the command on its grant and exits with its code. - In a worktree the command stays `gate.py run|hold` (`kvb-check-gate`): `queue.py hold ... -- python3 run ...`. -One test file or module, or ruff, without Docker self-admits with -`watch.py admit --kind test-nostack` (`kvb-org-organize`); any other test -run goes through hold with kind `test`. +Light runs (`kvb-org-organize` Capacity) self-admit; anything else goes +through hold with kind `test`, and a light kind refuses it. ## Order and grant @@ -77,8 +76,8 @@ learned values, `cancel` removes your own waiting job. one down, and `kvb-git-hotfix` may void a gate. A `kvb-org-queue-cleanup` round deletes only entries whose lock no living process holds. - With `queue_enforce: true`, the `block` hook (PreToolUse on Bash) refuses - `heavy_commands`, image builds, `make install|verify-*`, tests beyond one - file and `gate.py run|hold` outside `queue.py hold`. + `heavy_commands`, image builds, `make install|test*|quality*|verify-*`, + tests beyond one unit file and `gate.py run|hold` outside `queue.py hold`. - Without the config, hold exits 2: tell the operator and ask the chair (`kvb-org-organize`) until it exists. A waiting hold rereads it every poll (a broken file keeps the last good one), so a limit change needs no re-hold. diff --git a/skills/kvb-org-queue/scripts/queue.py b/skills/kvb-org-queue/scripts/queue.py index d4ffb59..7378a2a 100755 --- a/skills/kvb-org-queue/scripts/queue.py +++ b/skills/kvb-org-queue/scripts/queue.py @@ -39,7 +39,9 @@ Args: SIBLINGS); a poll that finds other code installed in ~/.agents/skills leaves the queue ungranted and restarts the same hold with the installed queue.py. - --kind: a key of slot_need_gb in the config. + --kind: a key of slot_need_gb in the config; a light one + (light_kinds or watch.LIGHT_KIND) refuses a command watch.light() + does not recognise, naming watch.queue_hold(). --task: an open task of the kvb-org-rasci register; its rank orders the queue, and the operator's agent order (`rasci.py agents order`) of the holding session breaks ties. While one of its @@ -88,7 +90,8 @@ disk of running and granted jobs, never a reservation; queue_enforce (bool), opt only. The watch.py keys disk, thresholds_gb (stop is the floor every grant leaves free), slot_need_gb (the disk seed per kind), load_limits (every grant also needs CPU busy below cpu_percent and RAM used plus the job's learned RAM -share below mem_percent; both bound the learned slot limits), memory_limits (no +share, the live admit records (admit()) and the young jobs' ramp +(ramping_ram()) below mem_percent; both bound the learned slot limits), memory_limits (no grant at memory stage stop), heavy_commands and never_block are used as watch.py documents them. A signature is `: ` (aliases()). @@ -166,6 +169,10 @@ XDIST_LONG = "--numprocesses=" ADDOPTS = "PYTEST_ADDOPTS=" FROZEN = "footprint" ALIASES = "aliases" +ADMIT = "admit" +ADMITS = "admits" +ADMIT_SECONDS = 120 +GRANTED_MEM = "granted_mem" @functools.cache @@ -207,7 +214,7 @@ def fail(message: str) -> NoReturn: def config() -> dict: - cfg = watch.load_config() + cfg, _ = watch.announced_config() if cfg is None: fail(f"no config at {watch.CONFIG}; the queue reads the kvb-org-organize watch config") problem = watch.missing(cfg, (*BUDGET_KEYS, *watch.MEMORY_KEYS)) @@ -219,13 +226,14 @@ def config() -> dict: def fresh(cfg: dict) -> dict: """Returns: The config as it stands now, else cfg while the file is missing, - unreadable or incomplete, so a half-saved edit never ends a wait. + unusable (watch.checked_config()) or incomplete, so a half-saved + edit never ends a wait. """ try: - now = watch.load_config() - except (OSError, ValueError): + now = watch.checked_config(watch.load_config()) + except watch.UNUSABLE: return cfg - if not isinstance(now, dict) or watch.missing(now, (*BUDGET_KEYS, *watch.MEMORY_KEYS)) is not None: + if watch.missing(now, (*BUDGET_KEYS, *watch.MEMORY_KEYS)) is not None: return cfg return now @@ -691,11 +699,11 @@ def footprint(entry: dict, signatures: dict[tuple[str, str], dict], kinds: dict[ def held_back(cfg: dict, entries: list[dict], now: float, signatures: dict[str, dict], kinds: dict[str, dict], - cpu: float, mem: float) -> dict[str, str]: + cpu: float, mem: float, admitted: float) -> dict[str, str]: loaded = {} for entry in entries: if entry["status"] == "waiting": - pending = pending_ram(entries, entry["id"], now, signatures, kinds) + pending = pending_ram(entries, entry["id"], now, signatures, kinds, mem, admitted) if (reason := load_reason(cfg["load_limits"], cpu, mem, pending)) is not None: loaded[entry["id"]] = reason return loaded @@ -743,18 +751,36 @@ def ram_share(entry: dict, signatures: dict[str, dict], kinds: dict[str, dict]) return history.seed_ram(kinds) if share is None else share["ram"] -def pending_ram(entries: list[dict], entry_id: str, now: float, signatures: dict[str, dict], - kinds: dict[str, dict]) -> float: +def ramping_ram(entries: list[dict], now: float, signatures: dict[str, dict], kinds: dict[str, dict], + mem: float) -> float: """Returns: - The RAM share of entry_id plus the whole share of every active job - granted less than RAMP_SECONDS ago (or without a grant time): what - may still come once they ramp up. An older job counts as ramped, its - RAM inside the RAM used; one that peaks later is bounded by the - memory stage only. + What the active jobs granted less than RAMP_SECONDS ago (or without a + grant time) may still add: the shares of those whose grant recorded + the RAM used (GRANTED_MEM) minus what RAM used gained since the + earliest of these grants, between 0 and those shares (a RAM used + that fell since counts as no gain), plus the whole share of the + others. An older job counts as ramped, its RAM inside the RAM used; + one that peaks later is bounded by the memory stage only. + """ + young = [e for e in entries if e["status"] == "active" and now - e.get("granted", now) < RAMP_SECONDS] + dated = [e for e in young if GRANTED_MEM in e] + gained = max(0.0, mem - min((e[GRANTED_MEM] for e in dated), default=mem)) + ramp = max(0.0, sum(ram_share(e, signatures, kinds) for e in dated) - gained) + return ramp + sum(ram_share(e, signatures, kinds) for e in young if GRANTED_MEM not in e) + + +def pending_ram(entries: list[dict], entry_id: str, now: float, signatures: dict[str, dict], + kinds: dict[str, dict], mem: float, admitted: float) -> float: + """Returns: + The RAM share of entry_id plus admitted (admitted_ram()) plus + ramping_ram() at RAM used mem. """ mine = [e for e in entries if e["id"] == entry_id] - young = [e for e in entries if e["status"] == "active" and now - e.get("granted", now) < RAMP_SECONDS] - return ram_share(mine[0], signatures, kinds) + sum(ram_share(e, signatures, kinds) for e in young) + return ram_share(mine[0], signatures, kinds) + admitted + ramping_ram(entries, now, signatures, kinds, mem) + + +def admitted_ram(data: dict, now: float) -> float: + return sum(record["ram"] for record in data.get(ADMITS, []) if record["expires"] > now) def workdir() -> str: @@ -823,13 +849,13 @@ def wait(cfg: dict, entry_id: str, caught: list[int]) -> tuple[int | None, int]: print(f"queue: {entry_id} was cancelled", file=sys.stderr) return CANCELLED, 0 rejoin(data, mine[0], now) - loaded = held_back(cfg, data["entries"], now, signatures, kinds, cpu, mem) + loaded = held_back(cfg, data["entries"], now, signatures, kinds, cpu, mem, admitted_ram(data, now)) verdicts = decide(cfg, data["entries"], free, now, signatures, kinds, loaded) reason = verdicts[entry_id] or loaded.get(entry_id) mine[0]["reason"] = reason if reason is None: concurrent = sum(e["status"] == "active" for e in data["entries"]) - mine[0].update(status="active", granted=now, **{FROZEN: footprint(mine[0], signatures, kinds)}) + mine[0].update(status="active", granted=now, **{GRANTED_MEM: mem, FROZEN: footprint(mine[0], signatures, kinds)}) return None, concurrent position = list(verdicts).index(entry_id) + 1 if (position, reason) != shown: @@ -913,15 +939,27 @@ def keep_receipt(gates: ModuleType | None, args: argparse.Namespace, here: str, print(f"queue: receipt {args.receipt} for tree {before}", file=sys.stderr) -def cmd_hold(args: argparse.Namespace) -> int: - cfg = config() +def check_hold(args: argparse.Namespace, cfg: dict) -> None: + """Args: + args: the hold arguments; fail() exits on an unknown kind, a command + that is no light run under a light kind, a task outside the + register or a missing hotfix row. + cfg: the watch config. + """ if args.kind not in cfg["slot_need_gb"]: fail(f"kind {args.kind!r} is not in slot_need_gb of {watch.CONFIG}") + shown = shlex.join(args.command) + if args.kind in {watch.LIGHT_KIND, *cfg.get("light_kinds", [])} and not watch.light(shown, os.getcwd()): + fail(f"{shown} is no light run and the light kind {args.kind} skips every reservation; run `{watch.queue_hold(shown)}`") rasci = sibling("rasci") if args.task not in rasci.merged(False): fail(f"{args.task} is no open task of the kvb-org-rasci register") if args.hotfix is not None and HOTFIX + args.hotfix not in rasci.load(): fail(f"the register has no row {HOTFIX}{args.hotfix}") + +def cmd_hold(args: argparse.Namespace) -> int: + cfg = config() + check_hold(args, cfg) here = workdir() gates, out = gated(args, here) names = aliases(here, args.command, os.getcwd()) @@ -1084,6 +1122,38 @@ def held_commands(command: str) -> list[list[str]]: ] +def enforced(cfg: dict) -> dict: + return {**cfg, "heavy_commands": [*cfg["heavy_commands"], GATE.pattern]} + + +def admit(cfg: dict, kind: str, argv: list[str], cpu: float, mem: float, calm: bool) -> str | None: + """Args: + cfg: the config. + kind: the kind of the light job that asks to start outside the queue. + argv: its command, run in the current directory. + cpu: CPU busy now, in percent. + mem: RAM used now, in percent. + calm: whether the memory stage is ok; the live active jobs' ramp + (ramping_ram()) counts only when it is not. + + Returns: + load_reason() over the job's RAM share plus the live admit records + (admitted_ram()) and the ramp; under the store lock, a grant records + the share for ADMIT_SECONDS and expired records leave. + """ + now = time.time() + signatures, kinds = learned(cfg, now) + mine = {"id": ADMIT, "status": "waiting", "kind": kind, ALIASES: list(aliases(workdir(), argv, os.getcwd()))} + share = ram_share(mine, signatures, kinds) + with store() as data: + active = [e for e in data["entries"] if e["status"] == "active" and living(e)] + ramp = 0.0 if calm else ramping_ram(active, now, signatures, kinds, mem) + reason = load_reason(cfg["load_limits"], cpu, mem, share + admitted_ram(data, now) + ramp) + granted = [{"ram": share, "expires": now + ADMIT_SECONDS}] if reason is None else [] + data[ADMITS] = [record for record in data.get(ADMITS, []) if record["expires"] > now] + granted + return reason + + def tails(argv: list[str]) -> list[str]: return [shlex.join(argv[at:]) for at, word in enumerate(argv) if not word.startswith("-")] @@ -1114,11 +1184,10 @@ def cmd_block() -> int: "hookEventName": "PreToolUse", "permissionDecision": "ask", "permissionDecisionReason": reason, }})) return 0 - cfg = watch.load_config() + cfg, _ = watch.announced_config() if cfg is None or cfg.get("queue_enforce") is not True: return 0 - gated = {**cfg, "heavy_commands": [*cfg["heavy_commands"], GATE.pattern]} - if watch.heavy(raw, gated, (THROUGH_QUEUE.pattern, *cfg["never_block"]), watch.payload_cwd(payload)): + if watch.heavy(raw, enforced(cfg), (THROUGH_QUEUE.pattern, *cfg["never_block"]), watch.payload_cwd(payload)): print( f"kvb-org-queue: heavy jobs run through the queue: start `python3 {Path(__file__).resolve()} hold " "--kind --task -- ` in the background; it runs the command on its grant.", @@ -1157,8 +1226,10 @@ def main() -> int: if args.cmd == "block": try: return cmd_block() - except (OSError, ValueError, KeyError, TypeError, AttributeError, re.error): - return 0 + except Exception as error: # noqa: BLE001 + print(f"kvb-org-queue: the block hook failed ({error!r}), so the command counts as heavy and is refused.", + file=sys.stderr) + return 2 commands = {"hold": cmd_hold, "list": cmd_list, "history": cmd_history, "cancel": cmd_cancel} try: return commands[args.cmd](args) diff --git a/skills/kvb-org-suspend/SKILL.md b/skills/kvb-org-suspend/SKILL.md index af38e0c..ea1f82a 100644 --- a/skills/kvb-org-suspend/SKILL.md +++ b/skills/kvb-org-suspend/SKILL.md @@ -31,9 +31,8 @@ picks do; every other session keeps its own. ## Night rules for the active session -- **Grants**: request none; admit your own runs with `watch.py admit --kind - ` (`${CLAUDE_SKILL_DIR}/../kvb-org-organize/scripts/watch.py`), whose - disk, CPU and RAM limits still apply. +- **Grants**: request none; light runs self-admit and heavy ones queue + (`kvb-org-organize` Capacity), whose limits still apply. - **Offices**: the suspended agents' claims stay theirs. Take only free work; a piece a suspended agent holds waits for the resume or the operator. - **Messages**: no wakes and no peer questions; the Peer protocol's wait is diff --git a/skills/kvb-org-swarm/SKILL.md b/skills/kvb-org-swarm/SKILL.md index c030d43..a051e36 100644 --- a/skills/kvb-org-swarm/SKILL.md +++ b/skills/kvb-org-swarm/SKILL.md @@ -30,10 +30,9 @@ sidetree or leaves as a proposal. - code explored through the codebase-memory-mcp graph first (`search_graph`, `trace_path`, `get_architecture`, `detect_changes`; the `kvb-agent-optimize` skill sets it up), grep and read after; - - light runs (no Docker, about a minute) self-admit with - `${CLAUDE_SKILL_DIR}/../kvb-org-organize/scripts/watch.py admit --kind - test-nostack`; `make test`, gates, deploys and anything with Docker only - through `queue.py hold` (`kvb-org-queue`); + - light runs self-admit (`kvb-org-organize` Capacity); `make test`, gates, + deploys and anything with Docker only through `queue.py hold` + (`kvb-org-queue`); - status that asks nothing goes to `${CLAUDE_SKILL_DIR}/../kvb-org-radio/scripts/radio.py post ""`, never into a wake. diff --git a/skills/kvb-skill-compile/SKILL.md b/skills/kvb-skill-compile/SKILL.md index 740e3c2..6eaeda1 100644 --- a/skills/kvb-skill-compile/SKILL.md +++ b/skills/kvb-skill-compile/SKILL.md @@ -58,9 +58,9 @@ and the prose diff. - Write the script, then replace the compiled prose with its call (`${CLAUDE_SKILL_DIR}/scripts/.py ...`) and one line on reading its output. -- Every script gets a unit test with a negative control shown once, run once - `watch.py admit --kind test-nostack` (the `kvb-org-organize` skill) prints - `grant`; `make test` runs through `queue.py hold` (`kvb-org-queue`). +- Every script gets a unit test with a negative control shown once, run as a + light run (`kvb-org-organize` Capacity); `make test` runs through + `queue.py hold` (`kvb-org-queue`). - In another repository follow its agent rules (claims, worktree, branch); a file another line owns goes to that line through the `kvb-org-coordinate` skill. diff --git a/skills/kvb-skill-transcend/SKILL.md b/skills/kvb-skill-transcend/SKILL.md index 8522336..07d9d3e 100644 --- a/skills/kvb-skill-transcend/SKILL.md +++ b/skills/kvb-skill-transcend/SKILL.md @@ -52,9 +52,8 @@ or diff. - Read every script whole before carrying it; clean it the way the `kvb-code-clean` skill does. - A carried script gets a unit test in the repository's `tests/` with a - negative control, run once `watch.py admit --kind test-nostack` (the - `kvb-org-organize` skill) prints `grant`; `make test` runs through - `queue.py hold` (`kvb-org-queue`). + negative control, run as a light run (`kvb-org-organize` Capacity); + `make test` runs through `queue.py hold` (`kvb-org-queue`). - `transcend.py mark` every offered candidate with its verdict. - Leave the result uncommitted; committing is the `kvb-git-commit-next` skill's job. diff --git a/tests/test_gates.py b/tests/test_gates.py index 4ab3433..c18b0ee 100644 --- a/tests/test_gates.py +++ b/tests/test_gates.py @@ -11,7 +11,9 @@ from pathlib import Path REPO_ROOT = Path(__file__).resolve().parents[1] GATES = REPO_ROOT / "skills" / "kvb-check-gate" / "scripts" / "gates.py" QUEUE = REPO_ROOT / "skills" / "kvb-org-queue" / "scripts" / "queue.py" -OPEN_MEMORY = {"available": -1, "swap_free": -1, "psi_full": 1000, "container_swap_gb": 1e9} +OPEN_MEMORY = { + "available": -1, "swap_free": -1, "psi_some": 1000, "psi_full": 1000, "swap_out_mb_min": 1e9, "container_swap_gb": 1e9, +} class Scratch(unittest.TestCase): diff --git a/tests/test_install.py b/tests/test_install.py index a26f16e..64de01f 100644 --- a/tests/test_install.py +++ b/tests/test_install.py @@ -2,6 +2,7 @@ from __future__ import annotations +import json import os import shutil import subprocess @@ -10,15 +11,26 @@ import unittest from pathlib import Path INSTALL = Path(__file__).resolve().parents[1] / "scripts" / "install.sh" +ORGANIZE = Path(__file__).resolve().parents[1] / "skills" / "kvb-org-organize" NPX = "#!/bin/sh\nmkdir -p .agents/skills/locked\necho locked > .agents/skills/locked/SKILL.md\n" BYTECODE = ("__pycache__/radio.cpython-312.pyc", "json.pyc", "legacy.pyo") +LIVE_MEMORY = { + "warn": {"available": 15, "swap_free": 40, "psi_full": 5, "container_swap_gb": 4}, + "stop": {"available": 8, "swap_free": 10, "psi_full": 10, "container_swap_gb": 16}, +} +SHIPPED = ORGANIZE / "config.default.json" +DEFAULT = json.loads(SHIPPED.read_text(encoding="utf-8")) +ROW_MEMORY = DEFAULT["memory_limits"] class TestInstall(unittest.TestCase): - def install(self, root: Path, target: Path, npx: str = NPX, planted: tuple[str, ...] = ()) -> None: + def install(self, root: Path, target: Path, npx: str = NPX, planted: tuple[str, ...] = (), config: Path | None = None, + code: int = 0, stubs_extra: tuple[tuple[str, str], ...] = ()) -> subprocess.CompletedProcess: repo, stubs = root / "repo", root / "bin" (repo / "scripts").mkdir(parents=True) shutil.copy(INSTALL, repo / "scripts" / "install.sh") + if config is not None: + shutil.copytree(ORGANIZE, repo / "skills" / ORGANIZE.name, ignore=shutil.ignore_patterns("__pycache__")) (repo / "scripts" / "enable-plugins.js").write_text("", encoding="utf-8") (repo / "skills" / "own").mkdir(parents=True) (repo / "skills" / "own" / "SKILL.md").write_text("own", encoding="utf-8") @@ -26,11 +38,61 @@ class TestInstall(unittest.TestCase): (repo / name).parent.mkdir(parents=True, exist_ok=True) (repo / name).write_bytes(b"planted") stubs.mkdir() - (stubs / "npx").write_text(npx, encoding="utf-8") - (stubs / "npx").chmod(0o755) + for name, text in (("npx", npx), *stubs_extra): + (stubs / name).write_text(text, encoding="utf-8") + (stubs / name).chmod(0o755) env = {**os.environ, "PATH": f"{stubs}:/usr/bin:/bin", "TARGET": str(target), "HOME": str(root / "home")} + if config is not None: + env |= {"AGENTS_COUNCIL_CONFIG": str(config), "AGENTS_COUNCIL_ROOT": str(root / "council")} done = subprocess.run(["bash", str(repo / "scripts" / "install.sh")], env=env, capture_output=True, text=True, check=False) - self.assertEqual(done.returncode, 0, done.stderr) + self.assertEqual(done.returncode, code, done.stderr) + return done + + def test_the_install_stops_before_any_copy_while_the_live_config_lacks_a_memory_bound_the_new_code_needs(self): + for limits, code in ((LIVE_MEMORY, 2), (ROW_MEMORY, 0)): + with self.subTest(code=code), tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + target = root / "target" + target.mkdir() + config = root / "config.json" + config.write_text(json.dumps({**DEFAULT, "memory_limits": limits}), encoding="utf-8") + before = config.stat() + done = self.install(root, target, config=config, code=code) + after = config.stat() + self.assertEqual((after.st_ino, after.st_mtime_ns, after.st_mode), (before.st_ino, before.st_mtime_ns, before.st_mode)) + installed = sorted(path.name for path in target.iterdir()) + if code: + self.assertIn( + "lacks memory_limits.warn.psi_some, memory_limits.warn.swap_out_mb_min, memory_limits.stop.swap_out_mb_min;", + done.stderr, + ) + self.assertEqual(installed, []) + self.assertFalse((root / "repo" / ".agents").exists()) + else: + self.assertEqual(installed, [".agents", ".claude"]) + self.assertEqual(json.loads(config.read_text(encoding="utf-8")), {**DEFAULT, "memory_limits": limits}) + + def test_an_install_without_a_config_writes_the_shipped_default_as_the_initial_one(self): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + target = root / "target" + target.mkdir() + config = root / "council-config" / "config.json" + self.install(root, target, config=config) + self.assertEqual(config.read_bytes(), SHIPPED.read_bytes()) + self.assertEqual(config.stat().st_mode & 0o777, 0o600) + self.assertEqual(config.parent.stat().st_mode & 0o777, 0o700) + + def test_a_config_written_while_the_install_creates_its_folder_survives(self): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + target = root / "target" + target.mkdir() + config = root / "council-config" / "config.json" + racer = f"#!/bin/sh\nmkdir -p {config.parent}\necho '{{\"mine\": true}}' > {config}\nexec /usr/bin/install \"$@\"\n" + done = self.install(root, target, config=config, stubs_extra=(("install", racer),)) + self.assertEqual(json.loads(config.read_text(encoding="utf-8")), {"mine": True}) + self.assertIn(str(config), done.stderr) def test_reinstall_empties_the_skill_directories_in_place(self): with tempfile.TemporaryDirectory() as tmp: diff --git a/tests/test_operator.py b/tests/test_operator.py index 1424e9e..be4e998 100644 --- a/tests/test_operator.py +++ b/tests/test_operator.py @@ -529,7 +529,10 @@ class TestOperator(unittest.TestCase): def _queue_config(self, slot_need_gb: dict) -> None: config = self.base / "config.json" - open_memory = {"available": -1, "swap_free": -1, "psi_full": 1000, "container_swap_gb": 1e9} + open_memory = { + "available": -1, "swap_free": -1, "psi_some": 1000, "psi_full": 1000, "swap_out_mb_min": 1e9, + "container_swap_gb": 1e9, + } config.write_text(json.dumps({ "disk": str(self.base), "thresholds_gb": {"warn": 0, "stop": 0, "critical": 0}, "projection_minutes": 30, "slot_need_gb": slot_need_gb, "heavy_commands": [r"\btouch heavy\b"], "never_block": [], diff --git a/tests/test_queue.py b/tests/test_queue.py index 9d9cec3..b27ed2d 100644 --- a/tests/test_queue.py +++ b/tests/test_queue.py @@ -23,7 +23,9 @@ HISTORY = REPO_ROOT / "skills" / "kvb-org-queue" / "scripts" / "history.py" GATES = REPO_ROOT / "skills" / "kvb-check-gate" / "scripts" / "gates.py" NOW = 1_800_000_000.0 HOUR = 3600 -OPEN_MEMORY = {"available": -1, "swap_free": -1, "psi_full": 1000, "container_swap_gb": 1e9} +OPEN_MEMORY = { + "available": -1, "swap_free": -1, "psi_some": 1000, "psi_full": 1000, "swap_out_mb_min": 1e9, "container_swap_gb": 1e9, +} BASE = { "free_gb": 100.0, "floor": 10.0, @@ -141,11 +143,38 @@ class TestEvaluate(unittest.TestCase): young = {**job(1, kind="a", status="active", granted=NOW - 60), **cmd, "cmd": "a"} overshooting = {**job(5, kind="c", status="active", granted=NOW - 120), **cmd, "cmd": "c"} old = {**job(4, kind="c", status="active", granted=NOW - self.queue.RAMP_SECONDS), **cmd, "cmd": "c"} - self.assertEqual(self.queue.pending_ram([young, overshooting, old, fresh], "j2", NOW, signatures, kinds), 30 + 10 + 25) + pending = self.queue.pending_ram + self.assertEqual(pending([young, overshooting, old, fresh], "j2", NOW, signatures, kinds, 50, 0), 30 + 10 + 25) undated = {k: v for k, v in old.items() if k != "granted"} - self.assertEqual(self.queue.pending_ram([undated, fresh], "j2", NOW, signatures, kinds), 10 + 25) + self.assertEqual(pending([undated, fresh], "j2", NOW, signatures, kinds, 50, 0), 10 + 25) unknown = {**job(3, kind="new"), **cmd, "cmd": "new"} - self.assertEqual(self.queue.pending_ram([unknown], "j3", NOW, signatures, kinds), 30) + self.assertEqual(pending([unknown], "j3", NOW, signatures, kinds, 50, 0), 30) + self.assertEqual(pending([fresh], "j2", NOW, signatures, kinds, 50, 12), 25 + 12) + + def test_a_young_job_counts_only_what_it_has_not_added_since_its_grant(self): + history = _load(HISTORY) + rows = [history_row(n, kind="b", signature="w: b", mem_before=20, mem_max=45) for n in range(5)] + signatures, kinds = history.learn(rows, NOW, {"load_limits": {"mem_percent": 90, "cpu_percent": 90}}) + fresh = {**job(2, kind="b"), "workdir": "/w", "cmd": "b"} + + def gate(seq: int, ram: float, granted_mem: float) -> dict: + return {**job(seq, kind="g", status="active", granted=NOW - 60), "workdir": "/w", "cmd": "g", + "footprint": {"ram": ram, "cpu": 1}, "granted_mem": granted_mem} + + pending = self.queue.pending_ram + self.assertEqual(pending([gate(1, 40, 10), fresh], "j2", NOW, signatures, kinds, 10, 0), 25 + 40) + self.assertEqual(pending([gate(1, 40, 10), fresh], "j2", NOW, signatures, kinds, 45, 0), 25 + 5) + self.assertEqual(pending([gate(1, 40, 10), fresh], "j2", NOW, signatures, kinds, 60, 0), 25) + self.assertEqual(pending([gate(1, 40, 10), gate(3, 20, 30), fresh], "j2", NOW, signatures, kinds, 45, 0), 25 + 25) + undated = {k: v for k, v in gate(1, 40, 10).items() if k != "granted_mem"} + self.assertEqual(pending([undated, fresh], "j2", NOW, signatures, kinds, 60, 0), 25 + 40) + ramp = self.queue.ramping_ram + for mem in (70, 40, 20): + with self.subTest(mem=mem): + self.assertEqual(ramp([gate(1, 40, 70)], NOW, signatures, kinds, mem), 40) + self.assertEqual(ramp([gate(1, 40, 70), {**undated, "id": "j3"}], NOW, signatures, kinds, 20), 80) + self.assertEqual(self.queue.admitted_ram({"admits": [{"ram": 7, "expires": NOW + 1}, {"ram": 9, "expires": NOW}]}, NOW), 7) + self.assertEqual(self.queue.admitted_ram({"entries": []}, NOW), 0) def test_a_signature_names_the_repository_every_worktree_shares(self): with tempfile.TemporaryDirectory() as tmp: @@ -251,7 +280,7 @@ class TestEvaluate(unittest.TestCase): self.assertEqual(self.queue.footprint(waiting, signatures, kinds), {"ram": 50, "cpu": 80}) older = {k: v for k, v in running.items() if k != "footprint"} self.assertEqual(self.queue.footprint(older, signatures, kinds), {"ram": 50, "cpu": 80}) - self.assertEqual(self.queue.pending_ram([running], "j1", NOW, signatures, kinds), 12 + 12) + self.assertEqual(self.queue.pending_ram([running], "j1", NOW, signatures, kinds, 50, 0), 12 + 12) def test_an_active_entry_an_older_queue_granted_never_books_less_than_that_queue_did(self): history = _load(HISTORY) @@ -754,6 +783,7 @@ class TestQueueCommands(unittest.TestCase): ) signature = f"{self.base.name}: {Path(sys.executable).name} -c" self.assertEqual((entry["aliases"], entry["footprint"]), ([signature], None)) + self.assertTrue(0 < entry["granted_mem"] < 100, entry) self.assertEqual(self.entries(), []) [row] = [json.loads(line) for line in self.history_file.read_text(encoding="utf-8").splitlines()] self.assertEqual( @@ -1111,6 +1141,52 @@ class TestQueueCommands(unittest.TestCase): self.write_config() self.assertEqual(self.bash("make compose-deploy"), 0) + def test_list_and_hold_on_a_broken_config_warn_and_use_the_shipped_default(self): + self.config.write_text(json.dumps(self.cfg)[:-1] + ",}", encoding="utf-8") + listing = self.run_queue("list") + self.assertEqual(listing.returncode, 0, listing.stderr) + self.assertIn("built-in defaults", listing.stderr) + held = self.run_queue("hold", "--kind", "nosuch", "--task", "work/a", "--session", "s1", "--", "true") + self.assertEqual(held.returncode, 2, held.stderr) + self.assertIn("built-in defaults", held.stderr) + self.assertIn("'nosuch' is not in slot_need_gb", held.stderr) + for done in (listing, held): + self.assertNotIn("Traceback", done.stderr) + + def test_a_broken_config_falls_back_to_defaults_so_light_commands_pass_and_heavy_ones_stay_refused(self): + self.cfg["queue_enforce"] = True + good = json.dumps(self.cfg) + + def text(content: str) -> Callable[[], None]: + return lambda: self.config.write_text(content, encoding="utf-8") + + shapes = { + "comma": text(good[:-1] + ",}"), "empty": text(""), "list": text("[]"), + "string enforce": text(json.dumps({**self.cfg, "queue_enforce": "yes"})), + "empty-matching heavy": text(json.dumps({**self.cfg, "heavy_commands": [""]})), + "dot heavy": text(json.dumps({**self.cfg, "heavy_commands": ["."]})), + "word-boundary heavy": text(json.dumps({**self.cfg, "heavy_commands": [r"\b"]})), + "dot never_block": text(json.dumps({**self.cfg, "never_block": ["."]})), + "dangling link": lambda: self.config.symlink_to(self.base / "absent.json"), + "link loop": lambda: self.config.symlink_to(self.config), + "directory": lambda: self.config.mkdir(), + } + for name, make in shapes.items(): + with self.subTest(config=name): + if self.config.is_dir() and not self.config.is_symlink(): + self.config.rmdir() + self.config.unlink(missing_ok=True) + make() + passed = self.hook(json.dumps({"tool_input": {"command": "ls -la"}})) + self.assertEqual(passed.returncode, 0, passed.stderr) + self.assertIn(str(self.config), passed.stderr) + self.assertIn("built-in defaults", passed.stderr) + self.assertEqual(self.bash("make test"), 2) + live = ("make compose-deploy ROLE=x", "docker build .", "docker pull nginx", "make swarm-deploy", "make roundtrip", + "make worktree-up") + for command in live if name in ("comma", "directory") else live[:1]: + self.assertEqual(self.bash(command), 2, command) + def test_block_with_queue_enforce_refuses_heavy_commands_outside_the_queue(self): self.cfg["queue_enforce"] = True self.write_config() @@ -1129,7 +1205,12 @@ class TestQueueCommands(unittest.TestCase): ): with self.subTest(command=command): self.assertEqual(self.bash(command), 0) - self.assertEqual(self.hook("not json").returncode, 0) + for payload in ("not json", json.dumps({"tool_input": ["make test"]})): + with self.subTest(payload=payload): + crashed = self.hook(payload) + self.assertEqual(crashed.returncode, 2, crashed.stderr) + self.assertIn("counts as heavy", crashed.stderr) + self.assertEqual(self.bash("CARGO_BUILD_JOBS=² cargo build ; make test"), 2) self.config.unlink() self.assertEqual(self.bash("make compose-deploy"), 0) @@ -1290,6 +1371,40 @@ class TestQueueCommands(unittest.TestCase): self.assertEqual(self.bash(command), 2) self.assertEqual(self.bash("docker compose ps"), 0) + def test_a_queue_grant_counts_a_live_admit_record_and_ignores_an_expired_one(self): + def admitted(expires: float) -> None: + (self.root / "queue.json").write_text( + json.dumps({"seq": 0, "entries": [], "admits": [{"ram": 300, "expires": expires}]}), encoding="utf-8" + ) + + admitted(time.time() + 300) + _, err = self.blocked_hold("true") + self.assertRegex(err.read_text(encoding="utf-8"), r"RAM \d+ % used \+ 300 % still to come reach the limit 101 %") + admitted(time.time() - 1) + result = self.run_queue("hold", "--kind", "deploy", "--task", "work/a", "--session", "s2", "--", "true") + self.assertEqual((result.returncode, result.stderr), (0, "")) + + def test_a_light_kind_refuses_a_heavy_command_names_the_test_kind_and_enqueues_nothing(self): + self.cfg["slot_need_gb"] |= {"test-nostack": 0, "quick": 0} + self.cfg["light_kinds"] = ["quick"] + self.write_config() + (self.base / "tests" / "e2e").mkdir(parents=True) + (self.base / "tests" / "e2e" / "test_orchestrator_e2e.py").write_text("", encoding="utf-8") + for kind, command, shown in ( + ("test-nostack", ("python", "-m", "unittest"), "python -m unittest"), + ("test-nostack", ("pytest", "tests/e2e/test_orchestrator_e2e.py"), "pytest tests/e2e/test_orchestrator_e2e.py"), + ("quick", ("bash", "-c", "make test"), "bash -c 'make test'"), + ("quick", ("python3", "gate.py", "run", "--target", "test-unit"), "python3 gate.py run --target test-unit"), + ): + with self.subTest(kind=kind, command=command): + result = self.run_queue("hold", "--kind", kind, "--task", "work/a", "--session", "s1", "--", *command) + self.assertEqual(result.returncode, 2) + self.assertIn(f"the light kind {kind} skips every reservation", result.stderr) + self.assertIn(f"run `queue.py hold --kind test --task -- {shown}`", result.stderr) + self.assertFalse((self.root / "queue.json").exists()) + light = self.run_queue("hold", "--kind", "test-nostack", "--task", "work/a", "--session", "s1", "--", "true") + self.assertEqual((light.returncode, light.stderr), (0, "")) + def test_hold_refuses_a_config_without_the_budget_or_the_memory_limits(self): for key, cfg in ( ("budget_percent", {k: v for k, v in self.cfg.items() if k != "budget_percent"}), diff --git a/tests/test_resources_sample.py b/tests/test_resources_sample.py index 01f08af..08d16be 100644 --- a/tests/test_resources_sample.py +++ b/tests/test_resources_sample.py @@ -149,6 +149,53 @@ class TestCpu(unittest.TestCase): self.assertEqual(json.loads((state / "cpu.json").read_text(encoding="utf-8")), [NOW + sampler.BUCKET + 1, 9300, 12200]) +class TestPressure(unittest.TestCase): + def setUp(self): + tmp = tempfile.TemporaryDirectory() + self.addCleanup(tmp.cleanup) + self.proc, self.state = Path(tmp.name) / "proc", Path(tmp.name) / "state" + (self.proc / "pressure").mkdir(parents=True) + self.state.mkdir() + + def vmstat(self, pages: int) -> None: + (self.proc / "vmstat").write_text(f"pgpgin 5\npswpin 3\npswpout {pages}\n", encoding="utf-8") + + def test_psi_gives_some_and_full_avg10_and_none_without_pressure(self): + text = "some avg10=12.50 avg60=1.00 avg300=0.50 total=1\nfull avg10=6.25 avg60=0.80 avg300=0.10 total=1\n" + self.assertEqual((sampler.psi_avg10(text, "some"), sampler.psi_avg10(text, "full")), (12.5, 6.25)) + self.assertEqual((sampler.psi_avg10("", "some"), sampler.psi_avg10(text.splitlines()[0], "full")), (None, None)) + self.assertIsNone(sampler.swapped_pages("pgpgin 5\n")) + + def test_swap_out_is_none_first_then_mb_per_minute_and_none_after_a_reset_or_a_gap(self): + with mock.patch.object(sampler, "PAGE_BYTES", 4096): + self.assertIsNone(sampler.swap_out_since(self.proc, self.state, NOW)) + self.assertFalse((self.state / "swap.json").exists()) + self.vmstat(1000) + self.assertIsNone(sampler.swap_out_since(self.proc, self.state, NOW)) + self.vmstat(1000 + 1_400_000_000 // 4096) + self.assertEqual(sampler.swap_out_since(self.proc, self.state, NOW + 60), 1400.0) + self.vmstat(10) + self.assertIsNone(sampler.swap_out_since(self.proc, self.state, NOW + 120)) + self.assertEqual(json.loads((self.state / "swap.json").read_text(encoding="utf-8")), [NOW + 120, 10]) + self.assertIsNone(sampler.swap_out_since(self.proc, self.state, NOW + 120)) + self.assertIsNone(sampler.swap_out_since(self.proc, self.state, NOW + 120 + sampler.BUCKET + 1)) + + def test_a_sample_row_adds_the_pressure_keys_and_keeps_every_other_key(self): + (self.proc / "pressure" / "memory").write_text("some avg10=3.00 avg60=0 avg300=0 total=1\n", encoding="utf-8") + found = sampler.sample(self.proc, self.proc / "sys", self.proc, lambda argv: None, self.state) + self.assertEqual(sorted(found), ["cpu", "disks", "gpus", "load", "psi_full", "psi_some", "queue", "ram", "swap", "swap_out_mb_min", "t"]) + self.assertEqual((found["psi_some"], found["psi_full"], found["swap_out_mb_min"]), (3.0, None, None)) + + def test_the_fifteen_minute_mean_keeps_the_pressure_keys(self): + asleep = {"name": ASLEEP, "asleep": True} + rows = [ + row(NOW + 60, 1.0, 1, 0, asleep) | {"psi_some": 2.0, "psi_full": 1.004, "swap_out_mb_min": 100.0}, + row(NOW + 120, 1.0, 1, 0, asleep) | {"psi_some": 4.5, "psi_full": None, "swap_out_mb_min": 301.0}, + ] + mean = sampler.mean_row(NOW, rows) + self.assertEqual((mean["psi_some"], mean["psi_full"], mean["swap_out_mb_min"]), (3.25, 1.0, 200.5)) + + class TestGpus(unittest.TestCase): def setUp(self): tmp = tempfile.TemporaryDirectory() @@ -206,6 +253,7 @@ class TestRetention(unittest.TestCase): self.assertIs(kept[0], mean) self.assertEqual(kept[1], { "t": bucket, "disks": [{"mount": "/", "used": 200, "total": 1000}, {"mount": "/mnt", "used": 10, "total": 20}], "cpu": 30.0, "ram": 15.0, "swap": 0.0, "load": 3.0, + "psi_some": None, "psi_full": None, "swap_out_mb_min": None, "gpus": [{"name": AWAKE, "util": 50.0, "vram_used": 300, "vram_total": 1000}], "queue": {"waiting": 1.0, "active": 1.0}, }) self.assertEqual((kept[2]["cpu"], kept[2]["disks"]), (40.0, [{"mount": "/", "used": 400, "total": 1000}])) @@ -274,7 +322,7 @@ class TestSamplerInScratchHome(unittest.TestCase): self.assertEqual(len(rows), 2) for found in rows: with self.subTest(t=found["t"]): - self.assertEqual(sorted(found), ["cpu", "disks", "gpus", "load", "queue", "ram", "swap", "t"]) + self.assertEqual(sorted(found), ["cpu", "disks", "gpus", "load", "psi_full", "psi_some", "queue", "ram", "swap", "swap_out_mb_min", "t"]) self.assertEqual(found["disks"], DISKS) self.assertEqual(found["gpus"], [{"name": AWAKE, "util": 37.0, "vram_used": 2048 * MIB, "vram_total": 8188 * MIB}, {"name": ASLEEP, "asleep": True}]) self.assertEqual(found["queue"], {"waiting": 1, "active": 2}) diff --git a/tests/test_watch.py b/tests/test_watch.py index bffe47e..a018750 100644 --- a/tests/test_watch.py +++ b/tests/test_watch.py @@ -2,8 +2,11 @@ from __future__ import annotations +import contextlib import fcntl import importlib.util +import io +import itertools import json import os import re @@ -14,14 +17,40 @@ import sys import tempfile import time import unittest +from collections.abc import Callable from pathlib import Path from unittest import mock REPO_ROOT = Path(__file__).resolve().parents[1] WATCH = REPO_ROOT / "skills" / "kvb-org-organize" / "scripts" / "watch.py" -LIMITS = {"warn": 40, "stop": 15, "critical": 12} -OPEN_MEMORY = {"available": -1, "swap_free": -1, "psi_full": 1000, "container_swap_gb": 1e9} -SHUT_MEMORY = {"available": 101, "swap_free": 101, "psi_full": 0, "container_swap_gb": 0} +SHIPPED = REPO_ROOT / "skills" / "kvb-org-organize" / "config.default.json" +DEFAULT = json.loads(SHIPPED.read_text(encoding="utf-8")) +LIMITS = DEFAULT["thresholds_gb"] +OPEN_MEMORY = { + "available": -1, "swap_free": -1, "psi_some": 1000, "psi_full": 1000, "swap_out_mb_min": 1e9, "container_swap_gb": 1e9, +} +SHUT_MEMORY = {"available": 101, "swap_free": 101, "psi_some": 0, "psi_full": 0, "swap_out_mb_min": 0, "container_swap_gb": 0} +LIVE_MEMORY = DEFAULT["memory_limits"] +BROWSER_IMPORTS = { + "seleniumbase": "from seleniumbase import BaseCase", "nodriver": "import nodriver", + "DrissionPage": "from DrissionPage import ChromiumPage", "splinter": "from splinter import Browser", + "helium": "from helium import start_chrome", "camoufox": "from camoufox.sync_api import Camoufox", +} +PACKAGE_SCRIPTS = { + "e2e": "playwright test", "test:e2e": "cypress run", "lint": "eslint .", "build": "vite build", "dev": "vite", + "cy:run": "cypress run", "cy": "start-server-and-test dev http://localhost:3000 cy:run", + "both": 'concurrently "npm:dev" "cypress run"', "pree2e2": "cypress run", "e2e2": "echo ok", "loop": "yarn loop", +} +HEAVY_LIVE = ("make compose-deploy ROLE=x", "docker build .", "docker pull nginx", "make swarm-deploy", "make roundtrip", "make worktree-up") +OLD_MEMORY = { + "warn": {"available": 15, "swap_free": 40, "psi_full": 5, "container_swap_gb": 4}, + "stop": {"available": 8, "swap_free": 10, "psi_full": 10, "container_swap_gb": 16}, +} +CALM = { + "available": 50.0, "swap_free": 80.0, "psi_some": 1.0, "psi_full": 0.0, "swap_out_mb_min": None, "container_swap_gb": 0.0, +} +PAGE = 4096 +GB_PER_MINUTE_PAGES = 1_400_000_000 // PAGE MEMINFO = "MemTotal: 1000 kB\nMemAvailable: 120 kB\nSwapTotal: 2000 kB\nSwapFree: 300 kB\n" PSI = "some avg10=30.00 avg60=1.00 avg300=0.50 total=1\nfull avg10=12.50 avg60=0.80 avg300=0.10 total=1\n" LISTED = ["images", "--filter", "dangling=true", "--format", "{{.ID}} {{.Repository}} {{.Digest}}"] @@ -120,18 +149,21 @@ class TestStage(unittest.TestCase): def test_memory_sample_reads_available_swap_and_psi_and_tolerates_their_absence(self): self.assertEqual( - self.watch.memory_sample(MEMINFO, PSI, 2.5), - {"available": 12.0, "swap_free": 15.0, "psi_full": 12.5, "container_swap_gb": 2.5}, + self.watch.memory_sample(MEMINFO, PSI, 2.5, 70.0), + {"available": 12.0, "swap_free": 15.0, "psi_some": 30.0, "psi_full": 12.5, "swap_out_mb_min": 70.0, + "container_swap_gb": 2.5}, ) bare = "MemTotal: 1000 kB\nMemAvailable: 500 kB\nSwapTotal: 0 kB\nSwapFree: 0 kB\n" self.assertEqual( - self.watch.memory_sample(bare, None, 0.0), - {"available": 50.0, "swap_free": None, "psi_full": None, "container_swap_gb": 0.0}, + self.watch.memory_sample(bare, None, 0.0, None), + {"available": 50.0, "swap_free": None, "psi_some": None, "psi_full": None, "swap_out_mb_min": None, + "container_swap_gb": 0.0}, ) def test_memory_takes_the_swap_of_the_container_holding_the_most_nested_groups_included(self): with tempfile.TemporaryDirectory() as tmp: base = Path(tmp) + self.watch.ROOT, self.watch.SWAP_OUT = base, base / "swap-out.json" for name, swap in ((f"docker-{'a' * 64}.scope", 3_000_000_000), (f"docker-{'b' * 64}.scope", 18_000_000_000), ("other.scope", 90_000_000_000)): (base / name).mkdir() @@ -143,18 +175,20 @@ class TestStage(unittest.TestCase): def test_memory_stage_takes_the_first_bound_crossed_stop_before_warn(self): cfg = {"memory_limits": { - "warn": {"available": 15, "swap_free": 25, "psi_full": 5, "container_swap_gb": 4}, - "stop": {"available": 8, "swap_free": 10, "psi_full": 10, "container_swap_gb": 16}, + "warn": {"available": 15, "swap_free": 25, "psi_some": 10, "psi_full": 5, "swap_out_mb_min": 64, + "container_swap_gb": 4}, + "stop": {"available": 8, "swap_free": 10, "psi_full": 10, "swap_out_mb_min": 256, "container_swap_gb": 16}, }} stage = self.watch.memory_stage - healthy = {"available": 40.0, "swap_free": 80.0, "psi_full": 0.0, "container_swap_gb": 3.9} + healthy = {"available": 40.0, "swap_free": 80.0, "psi_some": 9.9, "psi_full": 0.0, "swap_out_mb_min": 63.9, + "container_swap_gb": 3.9} self.assertEqual(stage(healthy, cfg), "ok") for change, expected in ( ({"available": 14.0}, "warn"), ({"swap_free": 24.0}, "warn"), ({"psi_full": 5.0}, "warn"), - ({"container_swap_gb": 4.0}, "warn"), + ({"container_swap_gb": 4.0}, "warn"), ({"psi_some": 10.0}, "warn"), ({"swap_out_mb_min": 64.0}, "warn"), ({"available": 7.0}, "stop"), ({"swap_free": 9.0}, "stop"), ({"psi_full": 10.0}, "stop"), - ({"container_swap_gb": 16.0}, "stop"), - ({"swap_free": None, "psi_full": None}, "ok"), + ({"container_swap_gb": 16.0}, "stop"), ({"psi_some": 90.0}, "warn"), ({"swap_out_mb_min": 256.0}, "stop"), + ({"swap_free": None, "psi_some": None, "psi_full": None, "swap_out_mb_min": None}, "ok"), ): with self.subTest(change=change): self.assertEqual(stage(healthy | change, cfg), expected) @@ -184,6 +218,619 @@ class TestStage(unittest.TestCase): self.assertEqual(self.watch.stage_for(32, [[now - 600, 32.1]], self.cfg, now), "warn") +class TestPressure(unittest.TestCase): + def setUp(self): + self.watch = _load() + self.cfg = {"memory_limits": LIVE_MEMORY} + self.now = 1_800_000_000.0 + + def test_the_hang_inputs_stop_and_a_light_swap_out_with_low_psi_stays_ok(self): + rate = self.watch.swap_out_rate([self.now, 1000], [self.now + 60, 1000 + GB_PER_MINUTE_PAGES], PAGE) + self.assertAlmostEqual(rate, 1400.0, delta=0.01) + self.assertEqual(self.watch.memory_stage(CALM | {"swap_out_mb_min": rate}, self.cfg), "stop") + self.assertEqual(self.watch.memory_stage(CALM | {"available": 21.0, "psi_full": 6.0}, self.cfg), "stop") + self.assertEqual(self.watch.memory_stage(CALM | {"swap_out_mb_min": 40.0}, self.cfg), "ok") + self.assertEqual(self.watch.memory_stage(CALM | {"swap_out_mb_min": 64.0}, self.cfg), "warn") + self.assertEqual(self.watch.memory_stage(CALM | {"psi_some": 10.0}, self.cfg), "warn") + + def test_a_rate_needs_an_earlier_reading_inside_the_span_and_a_counter_reset_gives_none(self): + rate, now = self.watch.swap_out_rate, self.now + self.assertIsNone(rate(None, [now, 5], PAGE)) + self.assertIsNone(rate([now, 9000], [now + 60, 10], PAGE)) + self.assertIsNone(rate([now, 0], [now + 4, 10_000], PAGE)) + self.assertIsNone(rate([now - 901, 0], [now, 10], PAGE)) + self.assertEqual(rate([now, 7], [now + 60, 7], PAGE), 0.0) + + def test_the_readings_keep_a_baseline_at_least_the_minimum_span_old_and_drop_a_reset(self): + readings, t = self.watch.swap_readings, self.now + self.assertEqual(readings([], [t, 10]), (None, [[t, 10]])) + self.assertEqual(readings([[t, 10]], [t + 2, 12]), (None, [[t, 10]])) + self.assertEqual(readings([[t, 10]], [t + 6, 16]), ([t, 10], [[t, 10], [t + 6, 16]])) + self.assertEqual(readings([[t, 10], [t + 6, 16]], [t + 8, 18]), ([t, 10], [[t, 10], [t + 6, 16]])) + self.assertEqual(readings([[t, 10], [t + 6, 16]], [t + 12, 30]), ([t + 6, 16], [[t + 6, 16], [t + 12, 30]])) + self.assertEqual(readings([[t, 10], [t + 6, 16]], [t + 20, 3]), (None, [[t + 20, 3]])) + self.assertEqual(readings([[t + 99, 1]], [t, 3]), (None, [[t, 3]])) + + def test_the_sampler_reads_pswpout_and_both_psi_lines_and_none_without_them(self): + self.assertEqual(self.watch.swap_pages("pgpgin 1\npswpin 2\npswpout 345\n"), 345) + self.assertIsNone(self.watch.swap_pages("pgpgin 1\n")) + self.assertEqual((self.watch.psi_avg10(PSI, "some"), self.watch.psi_avg10(PSI, "full")), (30.0, 12.5)) + self.assertIsNone(self.watch.psi_avg10(PSI.splitlines()[0], "full")) + + def test_swap_out_keeps_its_readings_below_the_council_root_and_rates_from_the_second(self): + with tempfile.TemporaryDirectory() as tmp: + base = Path(tmp) + watch = self.watch + watch.ROOT, watch.SWAP_OUT, watch.VMSTAT, watch.PAGE_BYTES = base, base / "swap-out.json", base / "vmstat", PAGE + self.assertIsNone(watch.swap_out()) + self.assertFalse(watch.SWAP_OUT.exists()) + watch.VMSTAT.write_text("pswpout 1000\n", encoding="utf-8") + with mock.patch.object(watch.time, "time", return_value=self.now): + self.assertIsNone(watch.swap_out()) + watch.VMSTAT.write_text(f"pswpout {1000 + GB_PER_MINUTE_PAGES}\n", encoding="utf-8") + with mock.patch.object(watch.time, "time", return_value=self.now + 60): + self.assertAlmostEqual(watch.swap_out(), 1400.0, delta=0.01) + kept = json.loads(watch.SWAP_OUT.read_text(encoding="utf-8"))["readings"] + self.assertEqual(kept, [[self.now, 1000], [self.now + 60, 1000 + GB_PER_MINUTE_PAGES]]) + watch.SWAP_OUT.write_text('{"readings": [["x", 1], [1, 2, 3], true]}', encoding="utf-8") + with mock.patch.object(watch.time, "time", return_value=self.now + 120): + self.assertIsNone(watch.swap_out()) + + def test_admit_reads_swap_out_before_its_load_window_so_the_first_check_after_idle_has_a_rate(self): + with tempfile.TemporaryDirectory() as tmp: + base = Path(tmp) + watch = self.watch + watch.ROOT, watch.SWAP_OUT, watch.VMSTAT, watch.PAGE_BYTES = base, base / "swap-out.json", base / "vmstat", PAGE + watch.SWAP_OUT.write_text(json.dumps({"readings": [[self.now - 1000, 1000]]}), encoding="utf-8") + watch.VMSTAT.write_text(f"pswpout {1000 + 7_000_000_000 // PAGE}\n", encoding="utf-8") + clock = [self.now] + + def window(cfg: dict) -> tuple[float, float]: + clock[0] += 5 + watch.VMSTAT.write_text(f"pswpout {1000 + 7_300_000_000 // PAGE}\n", encoding="utf-8") + return 10.0, 50.0 + + with mock.patch.object(watch.time, "time", side_effect=lambda: clock[0]), mock.patch.object(watch, "load", side_effect=window): + cpu, mem, sample = watch.admit_sample({}) + self.assertEqual((cpu, mem), (10.0, 50.0)) + self.assertAlmostEqual(sample["swap_out_mb_min"], 3600.0, delta=0.5) + self.assertEqual(watch.memory_stage(sample | {"available": 50.0, "swap_free": 80.0}, self.cfg), "stop") + + def test_only_the_warn_stage_has_a_psi_some_bound_and_a_stop_one_is_ignored(self): + self.assertIn("memory_limits.warn.psi_some", self.watch.MEMORY_KEYS) + self.assertNotIn("memory_limits.stop.psi_some", self.watch.MEMORY_KEYS) + stray = {"memory_limits": {**LIVE_MEMORY, "stop": {**LIVE_MEMORY["stop"], "psi_some": 50}}} + self.assertEqual(self.watch.memory_stage(CALM | {"psi_some": 60.0}, stray), "warn") + + def test_the_pressure_bounds_are_required_config_and_the_text_names_them(self): + self.assertIsNone(self.watch.missing(self.cfg, self.watch.MEMORY_KEYS)) + for stage, bound in (("warn", "psi_some"), ("warn", "swap_out_mb_min"), ("stop", "swap_out_mb_min")): + with self.subTest(stage=stage, bound=bound): + limits = {**LIVE_MEMORY, stage: {k: v for k, v in LIVE_MEMORY[stage].items() if k != bound}} + problem = self.watch.missing({"memory_limits": limits}, self.watch.MEMORY_KEYS) + self.assertIn(f"lacks memory_limits.{stage}.{bound};", problem) + problem = self.watch.missing({"memory_limits": OLD_MEMORY}, self.watch.MEMORY_KEYS) + self.assertIn("lacks memory_limits.warn.psi_some, memory_limits.warn.swap_out_mb_min, memory_limits.stop.swap_out_mb_min;", problem) + text = self.watch.memory_text(CALM | {"swap_out_mb_min": 1400.0}) + self.assertIn("PSI avg10 some 1.0 full 0.0, swap-out 1400.0 MB/min", text) + self.assertIn("swap-out unmeasured", self.watch.memory_text(CALM)) + + +class TestHeavy(unittest.TestCase): + def setUp(self): + self.watch = _load() + tmp = tempfile.TemporaryDirectory() + self.addCleanup(tmp.cleanup) + self.base = Path(tmp.name) + for name, text in ( + ("tests/e2e/test_orchestrator_e2e.py", "import unittest\n"), + ("tests/test_watch.py", "import unittest\n"), + ("tests/test_page.py", "import json\nfrom playwright.sync_api import sync_playwright\n"), + ("tests/e2e/cdp.mjs", "export {};\n"), + ("tools/shot.py", "from playwright.sync_api import sync_playwright\n"), + ("tools/shot.mjs", "import { chromium } from 'playwright';\n"), + ("tools/plain.py", "print(1)\n"), + ("tools/plain.mjs", "console.log(1);\n"), + ): + (self.base / name).parent.mkdir(parents=True, exist_ok=True) + (self.base / name).write_text(text, encoding="utf-8") + (self.base / "tests" / "fifo.py").parent.mkdir(exist_ok=True) + os.mkfifo(self.base / "tests" / "fifo.py") + + def heavy(self, command: str, cwd: Path | None) -> bool: + return self.watch.heavy(command, {"heavy_commands": []}, (), None if cwd is None else str(cwd)) + + def test_the_hang_inputs_are_heavy_and_a_single_unit_file_stays_light(self): + for command in ( + "python -m unittest", + "python3 -m unittest -v", + "python3 -m unittest discover -s tests", + "pytest tests/e2e/test_orchestrator_e2e.py", + "python3 -m pytest tests/e2e/test_orchestrator_e2e.py::TestHub::test_home -q", + "python3 -m unittest tests.e2e.test_orchestrator_e2e", + "python3 -m unittest tests/e2e/test_orchestrator_e2e.py", + "python3 -m unittest discover -s tests/e2e -p test_orchestrator_e2e.py", + "pytest tests/test_page.py", + "python3 -m unittest tests.test_page", + "make test", + "make -C core test-lint", + "make quality-high", + "env -C tests pytest e2e/test_orchestrator_e2e.py", + ): + with self.subTest(command=command): + self.assertTrue(self.heavy(command, self.base)) + for command in ( + "pytest tests/test_watch.py", + "python3 -m unittest tests.test_watch", + "python3 -m unittest tests/test_watch.py", + "python3 -m unittest discover -s tests -p test_watch.py", + "pytest tests/test_page.py --collect-only", + "pytest tests/fifo.py", + "make install-skills", + "make -n test", + "ruff check skills tests", + ): + with self.subTest(command=command): + self.assertFalse(self.heavy(command, self.base)) + + def test_scripts_modules_browser_runners_and_suite_runners_are_heavy_beside_light_plain_ones(self): + for command in ( + "python3 tests/e2e/test_orchestrator_e2e.py", + "python3 -m tests.e2e.test_orchestrator_e2e", + "sh -c 'exec python3 tests/e2e/test_orchestrator_e2e.py'", + "python3 tools/shot.py", + "python3 -m playwright install chromium", + "node tests/e2e/cdp.mjs", + "node tools/shot.mjs", + "deno run -A tests/e2e/cdp.mjs", + "npx playwright test", + "npx --yes @playwright/test test", + "playwright test", + "cd tests ; python3 -m pytest e2e/test_orchestrator_e2e.py", + "bash -c 'cd tests && python3 -m unittest e2e/test_orchestrator_e2e.py'", + "tox", + "nox -s tests", + "hatch test", + "make", + "make VAR=1 -j4", + "gmake test", + ): + with self.subTest(command=command): + self.assertTrue(self.heavy(command, self.base)) + for command in ( + "python3 tests/test_watch.py", + "python3 -m tests.test_watch", + "python3 tools/plain.py", + "python3 -c 'import json'", + "node tools/plain.mjs", + "npx prettier --check .", + "playwright --version", + "tox --version", + "hatch fmt", + "make -n", + "gmake install-skills", + ): + with self.subTest(command=command): + self.assertFalse(self.heavy(command, self.base)) + + def light(self, command: str, cwd: Path | None) -> bool: + return self.watch.light(command, None if cwd is None else str(cwd)) + + def test_only_recognised_small_forms_are_light_and_everything_else_queues(self): + for command in ( + "ruff check skills tests", + "shellcheck scripts/install.sh", + "markdownlint-cli2 '**/*.md'", + "lizard -w -C 10 tools/plain.py", + "jq . config.json", + "python3 -m py_compile tools/plain.py", + "python3 -m pytest tests/test_watch.py", + "pytest tests/test_watch.py::TestA -q", + "python3 -m unittest tests.test_watch", + "python3 -m unittest tests.test_watch.TestA.test_a", + "python3 -m unittest tests/test_watch.py", + "env -C tests python3 -m pytest test_watch.py", + "cd tests && python3 -m pytest test_watch.py", + "timeout 900 nice -n 5 env LANG=C python3 -m unittest tests.test_watch", + "python3 -m pytest tests/test_watch.py 2>&1 | tee /tmp/log | tail -n 5", + "bash -c 'python3 -m pytest tests/test_watch.py'", + "true", + ): + with self.subTest(command=command): + self.assertTrue(self.light(command, self.base)) + for command in ( + "python3 -m unittest discover -s tests -p test_watch.py", + "python3 -m unittest tests", + "python3 -m unittest tests.test_watch tests.test_page", + "pytest tests/", + "pytest tests/test_missing.py", + "pytest tests/test_page.py", + "pytest -n 4 tests/test_watch.py", + "python3 -m unittest tests.e2e.test_orchestrator_e2e", + "systemd-run --user --scope python3 -m pytest tests/test_watch.py", + "taskset -c 0 ruff check .", + "xvfb-run -a python3 -m pytest tests/test_watch.py", + "uv run pytest tests/test_watch.py", + "sudo ruff check .", + "npx -y playwright@1.48.0 test", + "npm run test:e2e", + "cargo test", + "docker run --rm img ruff check .", + "make", + "make lint", + "python3 tools/plain.py", + "python3 -c 'import json'", + "ruff check $(cat list)", + "cd $(mktemp -d) && python3 -m pytest test_watch.py", + "cd tests ; python3 -m pytest test_page.py", + ): + with self.subTest(command=command): + self.assertFalse(self.light(command, self.base)) + self.assertFalse(self.light("python3 -m pytest tests/test_watch.py", None)) + + def test_a_playwright_import_counts_only_as_an_import_statement_never_inside_a_string_or_comment(self): + files = { + "string.py": "FIXTURE = \"from playwright.sync_api import sync_playwright\"\n", + "string.mjs": ( + "const s = \"import { chromium } from 'playwright'\";\n// import x from 'playwright'\n" + "/* require('playwright') */\nconst t = `\nimport { chromium } from 'playwright'\n`;\n" + ), + "nested.py": "def f():\n from playwright.sync_api import sync_playwright\n", + "aliased.py": "import os, playwright as pw\n", + "required.cjs": "const { test } = require(\"@playwright/test\");\n", + "dynamic.mjs": "const core = await import('playwright-core');\n", + "side.ts": "import 'playwright';\n", + "broken.py": "def (:\n", + } + folder = self.base / "imports" + folder.mkdir() + for name, text in files.items(): + (folder / name).write_text(text, encoding="utf-8") + found = {name: self.watch.playwright_file(str(folder / name)) for name in files} + self.assertEqual(found, { + "string.py": False, "string.mjs": False, "nested.py": True, "aliased.py": True, "required.cjs": True, + "dynamic.mjs": True, "side.ts": True, "broken.py": None, + }) + self.assertIsNone(self.watch.playwright_file(str(folder / "missing.py"))) + self.assertTrue(self.light("python3 -m unittest tests.test_watch", REPO_ROOT)) + self.assertFalse(self.heavy("python3 -m unittest tests.test_watch", REPO_ROOT)) + + def test_an_unknown_program_in_front_of_a_runner_never_hides_it_from_the_hook(self): + for command in ( + "systemd-run --user --scope -p MemoryMax=8G python3 -m unittest discover -s tests", + "taskset -c 0-3 python3 -m unittest discover -s tests", + "flock /tmp/lane.lock python3 -m unittest discover -s tests", + "xvfb-run -a python3 -m unittest tests.e2e.test_orchestrator_e2e", + "chrt -i 0 python3 -m unittest tests.e2e.test_orchestrator_e2e", + "npx -y playwright@1.48.0 test", + "npx @playwright/test@1.48.0 test", + "pnpm exec playwright test", + "pnpm dlx playwright test", + "yarn playwright test", + "npm exec -- playwright test", + "npm run test:e2e", + "docker run --rm -v /w:/w img python3 -m unittest discover -s /w/tests", + "cargo test", + "node --test", + "deno test", + "watch -n 5 pytest", + "cd tests && python3 -m pytest test_page.py", + ): + with self.subTest(command=command): + self.assertTrue(self.heavy(command, self.base)) + for command in ( + "grep -rn pytest tests/", + "which pytest", + "pip install pytest playwright", + "uv pip install pytest", + "npm install playwright", + "pkill pytest", + "git commit -m x", + "make lint", + "make -C . help", + "docker ps", + "ruff check skills tests", + "python3 -m unittest tests.test_watch", + "python3 tools/plain.py", + "xvfb-run python3 tools/plain.py", + "cd tests && python3 -m pytest test_watch.py", + ): + with self.subTest(command=command): + self.assertFalse(self.heavy(command, self.base)) + + def _repo(self) -> Path: + repo = self.base / "repo" + (repo / ".git").mkdir(parents=True) + for name, text in ( + ("tests/__init__.py", ""), + ("tests/test_unit.py", "import unittest\n"), + ("tests/helper.py", "from playwright.sync_api import sync_playwright\n"), + ("tests/test_indirect.py", "from tests import helper\n"), + ("tests/test_relative.py", "from .helper import sync_playwright\n"), + ("tests/test_beside.py", "import helper\n"), + ("tests/test_fixture.py", "def test_home(page):\n page.goto('/')\n"), + ("tests/test_dynamic.py", "import importlib\nimportlib.import_module('playwright.sync_api')\n"), + ("tests/test_dunder.py", "__import__('playwright')\n"), + ("tests/test_suite_runner.py", "def load_tests(loader, tests, pattern):\n return loader.discover('.')\n"), + ("tests/test_chain.py", "import deep.a\n"), + ("tests/test_cyclic.py", "import loop.a\n"), + ("deep/__init__.py", ""), + ("deep/a.py", "from deep import b\n"), + ("deep/b.py", "import playwright\n"), + ("loop/__init__.py", ""), + ("loop/a.py", "from loop import b\n"), + ("loop/b.py", "from loop import a\n"), + ("tests/fx/conftest.py", ( + "import pytest\n\n\n@pytest.fixture\ndef logged_in(page):\n return page\n\n\n" + "@pytest.fixture\ndef admin(logged_in):\n return logged_in\n" + )), + ("tests/fx/test_admin.py", "def test_panel(admin):\n assert admin\n"), + ("tests/local_fixture.py", ( + "import pytest\n\n\n@pytest.fixture\ndef shop(page):\n return page\n\n\ndef test_cart(shop):\n assert shop\n" + )), + ("tests/test_selenium.py", "from selenium import webdriver\n"), + ("tests/test_named.py", "import playwright_report\n"), + ("playwright_report.py", "VALUE = 1\n"), + ("web/conftest.py", "pytest_plugins = ['pytest_playwright']\n"), + ("web/test_plain.py", "import unittest\n"), + ("workers/pytest.ini", "[pytest]\naddopts = -n 4\n"), + ("workers/test_x.py", "import unittest\n"), + ("pyproject.toml", "[tool.pytest.ini_options]\npythonpath = [\n 'src',\n]\n"), + ("src/app/__init__.py", ""), + ("src/app/browser.py", "from playwright.sync_api import sync_playwright\n"), + ("tests/test_src_layout.py", "from app.browser import sync_playwright\n"), + ("tests/ui/conftest.py", ( + "import pytest\n\n\n@pytest.fixture(autouse=True)\ndef _login(page):\n page.goto('/login')\n" + )), + ("tests/ui/test_autouse.py", "def test_home():\n assert True\n"), + ("tests/pl/conftest.py", "pytest_plugins = ['fixtures_browser']\n"), + ("tests/pl/fixtures_browser.py", ( + "import pytest\nfrom playwright.sync_api import sync_playwright\n\n\n@pytest.fixture\ndef shop():\n" + " return sync_playwright().start()\n" + )), + ("tests/pl/test_plugin.py", "def test_x(shop):\n assert shop\n"), + ("tests/uf/test_usefixtures.py", "import pytest\n\n\n@pytest.mark.usefixtures('page')\ndef test_x():\n assert True\n"), + ("tests/rq/test_request.py", "def test_x(request):\n request.getfixturevalue('page').goto('/')\n"), + ("tests/bm/test_bigmod.py", "import bigmod\n"), + ("tests/bm/bigmod.py", "x = 1\n" * 200_000 + "from playwright.sync_api import sync_playwright\n"), + ("tests/ct/test_ct.spec.ts", "import { test } from '@playwright/experimental-ct-react';\n"), + ("tests/rc/conftest.py", ( + "import pytest\n\n\n@pytest.fixture(autouse=True)\ndef _login(request):\n" + " request.getfixturevalue('page').goto('/')\n" + )), + ("tests/dy/conftest.py", ( + "import pytest\n\n\n@pytest.fixture(autouse=True)\ndef _login(request):\n" + " request.getfixturevalue('pa' + 'ge').goto('/')\n" + )), + ("tests/lb/conftest.py", ( + "import pytest\n\n\n@pytest.fixture(autouse=True)\ndef _go(launch_browser):\n launch_browser().new_page()\n" + )), + ("tests/pr/conftest.py", "pytest_plugins = ['req_plugin']\n"), + ("tests/pr/req_plugin.py", ( + "import pytest\n\n\n@pytest.fixture(autouse=True)\ndef shop(request):\n" + " return request.getfixturevalue('page')\n" + )), + *((f"tests/{folder}/test_x.py", "def test_x():\n assert True\n") for folder in ("rc", "dy", "lb", "pr")), + ("tools/chromium.mjs", "const pw = require('playwright-chromium');\n"), + ("tools/pup.mjs", "import puppeteer from 'puppeteer-core';\n"), + ("tools/extra.mjs", "import puppeteer from 'puppeteer-extra';\n"), + ("tools/cy.mjs", "const cypress = require('cypress');\n"), + ("tests/test_pyppeteer.py", "import pyppeteer\n"), + ("tests/test_undetected.py", "import undetected_chromedriver as uc\n"), + *((f"tests/test_{name.lower()}.py", f"{line}\n") for name, line in BROWSER_IMPORTS.items()), + *((f"tools/{name}.js", f"const x = require('{name.replace('wdio', '@wdio/globals')}');\n") + for name in ("nightwatch", "testcafe", "wdio")), + ("package.json", json.dumps({"scripts": PACKAGE_SCRIPTS})), + ("web/package.json", json.dumps({"scripts": {"e2e": "cypress run", "lint": "eslint ."}})), + ): + (repo / name).parent.mkdir(parents=True, exist_ok=True) + (repo / name).write_text(text, encoding="utf-8") + return repo + + def test_playwright_reached_through_imports_conftest_fixtures_or_load_tests_is_never_light(self): + repo = self._repo() + transitive = [ + "python3 -m pytest tests/test_indirect.py", + "python3 -m pytest tests/test_relative.py", + "python3 -m pytest tests/test_beside.py", + "python3 -m pytest tests/test_fixture.py", + "python3 -m pytest tests/test_dynamic.py", + "python3 -m pytest tests/test_dunder.py", + "python3 -m unittest tests.test_suite_runner", + "python3 -m pytest tests/test_chain.py", + "python3 -m pytest web/test_plain.py", + "python3 -m pytest workers/test_x.py", + "python3 -m pytest tests/fx/test_admin.py", + "python3 -m pytest tests/local_fixture.py", + "python3 -m pytest tests/test_selenium.py", + "python3 -m pytest tests/test_src_layout.py", + "python3 -m pytest tests/ui/test_autouse.py", + "python3 -m pytest tests/pl/test_plugin.py", + "python3 -m pytest tests/uf/test_usefixtures.py", + "python3 -m pytest tests/rq/test_request.py", + "python3 -m pytest tests/bm/test_bigmod.py", + "python3 -m pytest tests/rc/test_x.py", + "python3 -m pytest tests/dy/test_x.py", + "python3 -m pytest tests/lb/test_x.py", + "python3 -m pytest tests/pr/test_x.py", + "python3 -m pytest tests/test_pyppeteer.py", + "python3 -m pytest tests/test_undetected.py", + *(f"python3 -m pytest tests/test_{name.lower()}.py" for name in BROWSER_IMPORTS), + ] + for command in transitive: + with self.subTest(command=command): + self.assertFalse(self.light(command, repo)) + self.assertTrue(self.heavy(command, repo)) + for command in ( + "node tests/ct/test_ct.spec.ts", "node tools/chromium.mjs", "node tools/pup.mjs", "node tools/extra.mjs", + "node tools/cy.mjs", "npx cypress run", "yarn cypress run", "pnpm cypress run", "yarn e2e", "pnpm test:e2e", + "npx nightwatch", "npx testcafe chrome tests/", "npx wdio run wdio.conf.js", "node tools/nightwatch.js", + "node tools/testcafe.js", "node tools/wdio.js", + ): + with self.subTest(command=command): + self.assertFalse(self.light(command, repo)) + self.assertTrue(self.heavy(command, repo)) + for command in ( + "yarn --cwd web e2e", "yarn --cwd=web e2e", "pnpm -C web e2e", "pnpm --filter web e2e", "yarn workspace web e2e", + "npm --prefix web run e2e", "npm -w web run e2e", 'cd "$W" && yarn e2e', "yarn cy", "yarn both", "yarn e2e2", + "pnpm -r e2e", "yarn loop", + ): + with self.subTest(command=command): + self.assertTrue(self.heavy(command, repo)) + for command in ( + "yarn lint", "yarn build", "yarn add -D cypress", "yarn install", "yarn", "pnpm install", "yarn --cwd web lint", + ): + with self.subTest(command=command): + self.assertFalse(self.heavy(command, repo)) + for name in ( + "@playwright/experimental-ct-react", "playwright-chromium", "playwright-firefox", "playwright-webkit", + "playwright-core", "puppeteer", "puppeteer-core", "selenium-webdriver", "webdriverio", "pyppeteer", + "undetected_chromedriver", "puppeteer-extra", "cypress", *BROWSER_IMPORTS, "nightwatch", "testcafe", "@wdio/globals", + ): + with self.subTest(name=name): + self.assertTrue(self.watch.browser_module(name)) + plugin = { + "playwright", "browser_type", "browser_type_launch_args", "launch_browser", "browser", "browser_context_args", + "context", "new_context", "page", "browser_name", "browser_channel", "is_chromium", "is_firefox", + "is_webkit", "device", "connect_options", "output_path", + } + self.assertLessEqual(plugin, self.watch.FIXTURES) + for command in ( + "python3 -m pytest tests/test_unit.py", "python3 -m unittest tests.test_cyclic", "python3 -m pytest tests/test_named.py", + ): + with self.subTest(command=command): + self.assertTrue(self.light(command, repo)) + self.assertFalse(self.heavy(command, repo)) + + def _closure(self, modules: int) -> Path: + repo = self.base / f"closure{modules}" + (repo / ".git").mkdir(parents=True) + (repo / "pkg").mkdir() + (repo / "pkg" / "__init__.py").write_text("", encoding="utf-8") + for n in range(modules): + (repo / "pkg" / f"m{n}.py").write_text(f"from pkg import m{n + 1}\n" if n + 1 < modules else "", encoding="utf-8") + (repo / "test_wide.py").write_text("import pkg.m0\n", encoding="utf-8") + return repo + + def test_a_large_import_closure_is_walked_and_one_past_the_bound_is_heavy_in_admit_and_hook(self): + command = "python3 -m pytest test_wide.py" + small = self._closure(1500) + large = self._closure(2500) + jumps = itertools.count(0, 3600) + with mock.patch.object(time, "monotonic", side_effect=lambda: next(jumps)): + self.assertTrue(self.light(command, small)) + self.assertFalse(self.heavy(command, small)) + self.assertFalse(self.light(command, large)) + self.assertTrue(self.heavy(command, large)) + + def test_a_walk_past_the_byte_budget_is_heavy_in_admit_and_hook(self): + command = "python3 -m pytest test_wide.py" + self.assertEqual(self.watch.REACH_BYTES, 4 * 2**20) + within, past = self._closure(4), self._closure(5) + for repo, modules in ((within, 4), (past, 5)): + for n in range(modules): + with (repo / "pkg" / f"m{n}.py").open("a", encoding="utf-8") as module: + module.write("# " + "x" * 900_000 + "\n") + self.assertTrue(self.light(command, within)) + self.assertFalse(self.heavy(command, within)) + self.assertFalse(self.light(command, past)) + self.assertTrue(self.heavy(command, past)) + + def test_a_deep_dotted_import_stops_looking_at_the_first_missing_folder(self): + (self.base / "ns" / "deep").mkdir(parents=True) + (self.base / "ns" / "deep" / "mod.py").write_text("", encoding="utf-8") + looked: list[str] = [] + isfile, isdir = os.path.isfile, os.path.isdir + + def counted(check: Callable[[str], bool]) -> Callable[[str], bool]: + return lambda path: looked.append(path) or check(path) + + with mock.patch.object(self.watch.os.path, "isfile", counted(isfile)), mock.patch.object(self.watch.os.path, "isdir", counted(isdir)): + self.assertEqual(self.watch.rooted_files(".".join(["absent"] + ["b"] * 97), (str(self.base),)), frozenset()) + self.assertLessEqual(len(looked), 3) + self.assertEqual(self.watch.rooted_files("ns.deep.mod", (str(self.base),)), frozenset({str(self.base / "ns" / "deep" / "mod.py")})) + + def test_deep_imports_through_existing_folders_look_at_each_folder_once(self): + repo = self.base / "tower" + chain = ["b"] * 98 + os.makedirs(repo.joinpath(*chain)) + (repo / ".git").mkdir() + imports = "".join(f"import {'.'.join(chain)}.x{k}\n" for k in range(2000)) + (repo / "test_tower.py").write_text(imports + "def test_x():\n assert True\n", encoding="utf-8") + looked: list[str] = [] + isfile, isdir = os.path.isfile, os.path.isdir + + def counted(check: Callable[[str], bool]) -> Callable[[str], bool]: + return lambda path: looked.append(path) or check(path) + + start = time.perf_counter() + with mock.patch.object(self.watch.os.path, "isfile", counted(isfile)), mock.patch.object(self.watch.os.path, "isdir", counted(isdir)): + self.assertFalse(self.heavy("python3 -m pytest test_tower.py", repo)) + self.assertLess(time.perf_counter() - start, 10) + self.assertLessEqual(len([path for path in looked if path.startswith(str(repo / "b"))]), 3 * (98 + 2000)) + + def test_env_that_changes_the_run_or_a_second_runner_makes_a_test_command_heavy(self): + repo = self._repo() + for command in ( + "env LANG=C.UTF-8 TZ=UTC python3 -m pytest tests/test_unit.py", + "LC_ALL=C NO_COLOR=1 python3 -m unittest tests.test_unit", + "python3 -m pytest -q -x --tb=short -k unit tests/test_unit.py", + "python3 -m unittest -v tests.test_unit", + ): + with self.subTest(command=command): + self.assertTrue(self.light(command, repo)) + self.assertFalse(self.heavy(command, repo)) + for command in ( + "env PYTEST_ADDOPTS='-n 14 tests' python3 -m pytest tests/test_unit.py", + "PYTEST_PLUGINS=xdist python3 -m pytest tests/test_unit.py", + "PYTHONPATH=evil python3 -m unittest tests.test_unit", + "export PYTEST_ADDOPTS=-n14; python3 -m pytest tests/test_unit.py", + "python3 -m pytest tests/test_unit.py | python3 -m pytest tests/test_unit.py", + "bash -c 'python3 -m pytest tests/test_unit.py & python3 -m pytest tests/test_unit.py'", + "python3 -m pytest tests/test_unit.py ; python3 -m unittest tests.test_unit", + ): + with self.subTest(command=command): + self.assertFalse(self.light(command, repo)) + self.assertTrue(self.heavy(command, repo)) + for command in ( + "python3 -m pytest -p xdist tests/test_unit.py", + "python3 -m pytest -o addopts=-n4 tests/test_unit.py", + "python3 -m pytest -c other.ini tests/test_unit.py", + "env -S 'PYTEST_ADDOPTS=-n4 python3 -m pytest' tests/test_unit.py", + ): + with self.subTest(command=command): + self.assertFalse(self.light(command, repo)) + + def test_read_only_programs_never_hide_a_runner_and_capped_cargo_stays_light(self): + for command in ( + "find / -name pytest", "man make", "readlink -f /usr/bin/make", "whereis cargo", "ps -C make", + "stat /usr/bin/pytest", "file /usr/bin/make", "cargo fmt --check", "cargo metadata --format-version 1", + "env CARGO_BUILD_JOBS=4 cargo build", "CARGO_BUILD_JOBS=2 cargo clippy", "FOO=1 git status", + ): + with self.subTest(command=command): + self.assertFalse(self.heavy(command, self.base)) + for command in ( + "find . -name '*.py' -exec pytest {} +", "cargo build", "env CARGO_BUILD_JOBS=8 cargo test", + "env CARGO_BUILD_JOBS=4 cargo build -j 14", "RUSTFLAGS=-g CARGO_BUILD_JOBS=4 cargo build", "npm run lint", + "CARGO_BUILD_JOBS=² cargo build", "CARGO_BUILD_JOBS=0 cargo check", + ): + with self.subTest(command=command): + self.assertTrue(self.heavy(command, self.base)) + self.assertFalse(self.light(command, self.base)) + for command in ( + "find . -name '*.py'", "cargo fmt --check", "cargo metadata", "env CARGO_BUILD_JOBS=4 cargo build", + "CARGO_BUILD_JOBS=1 cargo test", + ): + with self.subTest(command=command): + self.assertTrue(self.light(command, self.base)) + + def test_without_a_start_directory_only_the_e2e_path_counts(self): + self.assertTrue(self.heavy("pytest tests/e2e/test_orchestrator_e2e.py", None)) + self.assertTrue(self.heavy("python3 -m unittest tests.e2e.test_orchestrator_e2e", None)) + self.assertFalse(self.heavy("pytest tests/test_page.py", None)) + self.assertTrue(self.heavy(f"pytest {self.base / 'tests' / 'test_page.py'}", None)) + + class TestWatchCommands(unittest.TestCase): def setUp(self): self.tmp = tempfile.TemporaryDirectory() @@ -239,6 +886,187 @@ class TestWatchCommands(unittest.TestCase): def _bash(self, command: str) -> str: return json.dumps({"tool_input": {"command": command}}) + def _broken_configs(self) -> list[tuple[str, Callable[[], None]]]: + good = json.dumps(self.cfg) + + def text(content: str) -> Callable[[], None]: + return lambda: self.config.write_text(content, encoding="utf-8") + + return [ + ("comma", text(good[:-1] + ",}")), ("empty", text("")), ("list", text("[]")), + ("string never_block", text(json.dumps({**self.cfg, "never_block": "."}))), + ("broken regex", text(json.dumps({**self.cfg, "heavy_commands": ["("]}))), + ("empty-matching heavy", text(json.dumps({**self.cfg, "heavy_commands": [""]}))), + ("empty-matching never_block", text(json.dumps({**self.cfg, "never_block": ["x*"]}))), + ("dot heavy", text(json.dumps({**self.cfg, "heavy_commands": ["."]}))), + ("word-boundary heavy", text(json.dumps({**self.cfg, "heavy_commands": [r"\b"]}))), + ("dot never_block", text(json.dumps({**self.cfg, "never_block": ["."]}))), + ("dangling link", lambda: self.config.symlink_to(self.base / "absent.json")), + ("link loop", lambda: self.config.symlink_to(self.config)), + ("directory", lambda: self.config.mkdir()), + ] + + def _reset_config(self) -> None: + if self.config.is_dir() and not self.config.is_symlink(): + self.config.rmdir() + else: + self.config.unlink(missing_ok=True) + + def test_a_broken_config_falls_back_to_defaults_so_light_commands_pass_and_heavy_ones_stay_refused(self): + self._stage("ok") + for name, make in self._broken_configs(): + with self.subTest(config=name): + self._reset_config() + make() + passed = self._run("block", stdin=self._bash("ls -la")) + self.assertEqual(passed.returncode, 0, passed.stderr) + self.assertIn(str(self.config), passed.stderr) + self.assertIn("built-in defaults", passed.stderr) + self.assertEqual(self._run("block", stdin=self._bash("make test")).returncode, 2) + self.assertEqual(self._run("block", stdin=self._bash(HEAVY_LIVE[0])).returncode, 2) + watched = self._run("watch", stdin=json.dumps({"session_id": "s1", "hook_event_name": "UserPromptSubmit"})) + self.assertEqual(watched.returncode, 0, watched.stderr) + self.assertIn(str(self.config), watched.stdout) + self._reset_config() + self.assertEqual(self._run("block", stdin=self._bash("make test")).returncode, 0) + + def test_a_broken_config_keeps_the_shipped_patterns_so_every_live_heavy_command_stays_refused(self): + self._stage("ok") + self._broken_configs()[0][1]() + for command in HEAVY_LIVE: + with self.subTest(command=command): + refused = self._run("block", stdin=self._bash(command)) + self.assertEqual(refused.returncode, 2, refused.stderr) + self.assertIn(str(self.config), refused.stderr) + + def test_a_pattern_matching_a_probe_command_is_refused(self): + watch = _load() + reproducers = { + "heavy_commands": (r"\.", "/", ".{11}", ".{34}", r"\bgrep\b", r"\becho\b", r"\bhead\b"), + "never_block": (r"\.", "compose", "quality", ".{20}", "swarm", "pull", "roundtrip", "act-"), + } + derived = { + "heavy_commands": [rf"\b{re.escape(tool)}\b" for tool in sorted(watch.LIGHT_TOOLS)], + "never_block": [rf"^{re.escape(tool)}\b" for tool in sorted(watch.HEAVY_TOOLS)], + } + for key, probes in watch.PATTERN_PROBES.items(): + for pattern in (*map(re.escape, probes), *reproducers[key], *derived[key]): + with self.subTest(key=key, pattern=pattern), self.assertRaises(ValueError): + watch.checked_config({"heavy_commands": [], "never_block": [], key: [pattern]}) + self.assertEqual(watch.checked_config(self.cfg), self.cfg) + self.assertEqual(watch.checked_config(DEFAULT), DEFAULT) + for key, pattern in (("heavy_commands", r"\bhead\b"), ("never_block", "swarm")): + with self.subTest(preflight=pattern): + self.cfg[key] = [*self.cfg[key], pattern] + self._write_config() + refused = self._run("preflight") + self.assertEqual(refused.returncode, 2, refused.stderr) + self.assertIn(repr(pattern), refused.stderr) + self.cfg[key] = self.cfg[key][:-1] + self._write_config() + self.assertEqual(self._run("preflight").returncode, 0) + + def test_the_refuse_everything_mode_never_promises_that_teardown_still_runs(self): + scripts = self.base / "skills" / "kvb-org-organize" / "scripts" + shutil.copytree(WATCH.parent, scripts, ignore=shutil.ignore_patterns("__pycache__")) + self._stage("ok") + self._broken_configs()[0][1]() + for command in ("make compose-down", "ls -la"): + with self.subTest(command=command): + refused = subprocess.run( + [sys.executable, str(scripts / "watch.py"), "block"], capture_output=True, text=True, env=self._env(), + input=self._bash(command), check=False, + ) + self.assertEqual(refused.returncode, 2, refused.stderr) + self.assertIn("every command is refused", refused.stderr) + self.assertNotIn("Teardown commands still run", refused.stderr) + + def test_the_shipped_default_is_safe_on_any_host(self): + text = SHIPPED.read_text(encoding="utf-8") + self.assertEqual((DEFAULT["cleanup"], DEFAULT["worktree_cleanup"]), ([], [])) + self.assertNotIn("prune", text) + self.assertNotIn("infinito", text.lower()) + self.assertFalse([folder for folder in DEFAULT["cache_dirs"] if "buildx" in folder]) + self.assertTrue(DEFAULT["queue_enforce"]) + + def test_the_shipped_default_is_complete_and_samples_long_enough_for_a_swap_out_rate(self): + watch = _load() + self.assertIsNone(watch.missing(DEFAULT, watch.MEMORY_KEYS)) + self.assertGreaterEqual(DEFAULT["load_limits"]["sample_seconds"], watch.SWAP_SPAN[0]) + self.assertEqual(DEFAULT["memory_limits"]["stop"]["psi_full"], 5) + self.assertLessEqual(set(self.cfg), set(DEFAULT)) + for command in HEAVY_LIVE: + with self.subTest(command=command): + self.assertTrue(watch.heavy(command, DEFAULT, tuple(DEFAULT["never_block"]), None)) + + def test_a_broken_config_uses_the_shipped_file_and_an_unreadable_one_refuses_every_command(self): + watch = _load() + watch.CONFIG = self.config + self.config.write_text("{", encoding="utf-8") + cfg, warning = watch.settled_config() + self.assertEqual(cfg, DEFAULT) + self.assertIn(str(SHIPPED), warning) + watch.SHIPPED = self.base / "absent.json" + cfg, warning = watch.settled_config() + self.assertIn(str(watch.SHIPPED), warning) + for command in ("ls -la", *HEAVY_LIVE): + with self.subTest(command=command): + self.assertTrue(watch.heavy(command, cfg, tuple(cfg["never_block"]), None)) + + def test_admit_under_the_shipped_default_denies_at_psi_full_5_and_rates_swap_out_after_idle(self): + watch = _load() + watch.ROOT, watch.SWAP_OUT, watch.VMSTAT, watch.PAGE_BYTES = self.base, self.base / "swap-out.json", self.base / "vmstat", PAGE + watch.CONFIG = self.config + self.config.write_text("{", encoding="utf-8") + cfg, _ = watch.settled_config() + args = mock.Mock(kind="test-nostack", need_gb=None, command=["true"]) + out = io.StringIO() + with mock.patch.object(watch, "admit_sample", return_value=(1.0, 1.0, CALM | {"psi_full": 5.0})), \ + mock.patch.object(watch, "disk", return_value=(1e6, 2e6)), mock.patch.object(watch, "loaded", return_value=mock.Mock(admit=mock.Mock(return_value=None))), \ + contextlib.redirect_stdout(out): + self.assertEqual(watch.cmd_admit(cfg, args), 3) + self.assertIn("memory stage stop", out.getvalue()) + now = 1_800_000_000.0 + clock = [now] + watch.SWAP_OUT.write_text(json.dumps({"readings": [[now - 1000, 1000]]}), encoding="utf-8") + watch.VMSTAT.write_text("pswpout 1000\n", encoding="utf-8") + + def window(limits: dict) -> tuple[float, float]: + clock[0] += limits["load_limits"]["sample_seconds"] + return 10.0, 50.0 + + with mock.patch.object(watch.time, "time", side_effect=lambda: clock[0]), \ + mock.patch.object(watch, "load", side_effect=window): + _, _, sample = watch.admit_sample(cfg) + self.assertEqual(sample["swap_out_mb_min"], 0.0) + + def test_admit_and_status_on_a_broken_config_warn_and_use_the_built_in_limits(self): + for name, make in self._broken_configs()[:1] + self._broken_configs()[-1:]: + with self.subTest(config=name): + self._reset_config() + make() + runs = [self._run("status"), self._run("admit", "--kind", "nosuch", "--", "true")] + self.assertEqual(runs[0].returncode, 0, runs[0].stderr) + self.assertTrue(runs[0].stdout.startswith("stage="), runs[0].stdout) + self.assertEqual(runs[1].returncode, 3, runs[1].stderr) + if name == "comma": + runs.append(self._run("admit", "--kind", "test-nostack", "--", "true")) + self.assertIn(runs[-1].returncode, (0, 3), runs[-1].stderr) + self.assertRegex(runs[-1].stdout, r"^(grant|deny): free .* memory stage ") + for done in runs: + self.assertEqual(len(done.stderr.splitlines()), 1, done.stderr) + self.assertIn(str(self.config), done.stderr) + self.assertIn("built-in defaults", done.stderr) + self._reset_config() + + def test_a_block_hook_that_cannot_read_its_input_refuses_the_command(self): + self._stage("ok") + for payload in ("not json", json.dumps({"tool_input": ["make test"]})): + with self.subTest(payload=payload): + refused = self._run("block", stdin=payload) + self.assertEqual(refused.returncode, 2, refused.stderr) + self.assertIn("counts as heavy", refused.stderr) + def test_block_only_heavy_jobs_at_stop_and_never_teardown(self): self._stage("stop") self.assertEqual(self._run("block", stdin=self._bash("make compose-deploy apps=x")).returncode, 2) @@ -297,29 +1125,125 @@ class TestWatchCommands(unittest.TestCase): def test_admit_keeps_the_stop_floor(self): self.cfg["thresholds_gb"] = {"warn": 0, "stop": 0, "critical": 0} self._write_config() - self.assertEqual(self._run("admit", "--need-gb", "0.001").returncode, 0) - self.assertEqual(self._run("admit", "--need-gb", "1e9").returncode, 3) + self.assertEqual(self._run("admit", "--need-gb", "0.001", "--", "true").returncode, 0) + self.assertEqual(self._run("admit", "--need-gb", "1e9", "--", "true").returncode, 3) def test_every_admit_also_needs_cpu_and_ram_below_the_limits_and_memory_short_of_stop(self): self.cfg["thresholds_gb"] = {"warn": 0, "stop": 0, "critical": 0} self._write_config() - granted = self._run("admit", "--kind", "test") + granted = self._run("admit", "--kind", "test", "--", "true") self.assertEqual(granted.returncode, 0) self.assertIn("memory stage ok", granted.stdout) self.cfg["load_limits"]["cpu_percent"] = 0 self._write_config() for args in (("--kind", "test"), ("--kind", "swarm"), ("--need-gb", "0.001")): with self.subTest(args=args): - denied = self._run("admit", *args) + denied = self._run("admit", *args, "--", "true") self.assertEqual(denied.returncode, 3) self.assertIn("cpu", denied.stdout) self.cfg["load_limits"]["cpu_percent"] = 101 self.cfg["memory_limits"]["stop"] = dict(SHUT_MEMORY) self._write_config() - short = self._run("admit", "--need-gb", "0.001") + short = self._run("admit", "--need-gb", "0.001", "--", "true") self.assertEqual(short.returncode, 3) self.assertIn("memory stage stop", short.stdout) + def test_admit_refuses_a_heavy_command_and_names_the_queue_hold_of_kind_test(self): + self.cfg["thresholds_gb"] = {"warn": 0, "stop": 0, "critical": 0} + self.cfg["slot_need_gb"]["test-nostack"] = 0 + self._write_config() + for command in ( + ("python", "-m", "unittest"), ("pytest", "tests/e2e/test_orchestrator_e2e.py"), ("make", "quality-high"), + ): + with self.subTest(command=command): + denied = self._run("admit", "--kind", "test-nostack", "--", *command) + self.assertEqual(denied.returncode, 3) + self.assertIn(f"run it as `queue.py hold --kind test --task -- {shlex.join(command)}`", denied.stdout) + light = self.base / "tests" / "test_x.py" + light.parent.mkdir() + light.write_text("import unittest\n", encoding="utf-8") + granted = self._run("admit", "--kind", "test-nostack", "--", "pytest", str(light)) + self.assertEqual(granted.returncode, 0, granted.stdout) + self.assertIn("swap-out", granted.stdout) + + def _admit_beside(self, entries: list[dict], ram_share: float) -> subprocess.CompletedProcess: + self.root.mkdir(parents=True, exist_ok=True) + (self.root / "queue.json").write_text(json.dumps({"seq": 2, "entries": entries}), encoding="utf-8") + rows = [ + {"id": f"r{n}", "kind": "test-nostack", "signature": "s", "end": time.time(), "minutes": 1, "disk_gb": 0, + "mem_before": 0, "mem_max": ram_share, "cpu_before": 0, "cpu_max": 1} + for n in range(5) + ] + (self.root / "queue-history.jsonl").write_text("".join(json.dumps(row) + "\n" for row in rows), encoding="utf-8") + return self._run("admit", "--kind", "test-nostack", "--", "true") + + def _light_config(self) -> None: + self.cfg["thresholds_gb"] = {"warn": 0, "stop": 0, "critical": 0} + self.cfg["slot_need_gb"]["test-nostack"] = 0 + self.cfg["load_limits"]["mem_percent"] = 200 + self.cfg |= {"exclusive_kinds": ["quality-high"], "light_kinds": ["test-nostack"]} + self._write_config() + + def test_a_light_run_beside_a_young_gate_is_admitted_while_memory_stays_under_warn(self): + self._light_config() + (self.root / "holds").mkdir(parents=True) + lock = (self.root / "holds" / "7-1.lock").open("w", encoding="utf-8") + self.addCleanup(lock.close) + fcntl.flock(lock, fcntl.LOCK_EX) + gate = {"id": "quality-high-1", "kind": "quality-high", "status": "active", "lock": "7-1", "granted": time.time() - 60, + "footprint": {"ram": 150.0, "cpu": 1.0}, "aliases": ["g"], "workdir": "/w", "need_gb": 0} + calm = self._admit_beside([gate], 50) + self.assertEqual(calm.returncode, 0, calm.stdout) + self.cfg["memory_limits"]["warn"] = dict(SHUT_MEMORY) + self._write_config() + warned = self._admit_beside([gate], 50) + self.assertEqual(warned.returncode, 3, warned.stdout) + self.assertRegex(warned.stdout, r"RAM \d+ % used \+ 200 % still to come reach the limit 200 %") + self.assertEqual(self._admit_beside([{**gate, "lock": "7-2"}], 50).returncode, 0) + + def test_two_admits_count_each_other_until_the_record_expires(self): + self._light_config() + first = self._admit_beside([], 100) + self.assertEqual(first.returncode, 0, first.stdout) + [record] = json.loads((self.root / "queue.json").read_text(encoding="utf-8"))["admits"] + self.assertEqual(record["ram"], 100) + self.assertGreater(record["expires"], time.time()) + second = self._run("admit", "--kind", "test-nostack", "--", "true") + self.assertEqual(second.returncode, 3, second.stdout) + self.assertRegex(second.stdout, r"RAM \d+ % used \+ 200 % still to come reach the limit 200 %") + stored = json.loads((self.root / "queue.json").read_text(encoding="utf-8")) + stored["admits"] = [{**record, "expires": time.time() - 1}] + (self.root / "queue.json").write_text(json.dumps(stored), encoding="utf-8") + third = self._run("admit", "--kind", "test-nostack", "--", "true") + self.assertEqual(third.returncode, 0, third.stdout) + + def test_admit_needs_the_command_and_refuses_a_gate_run_like_the_block_hook(self): + self.cfg["thresholds_gb"] = {"warn": 0, "stop": 0, "critical": 0} + self.cfg["slot_need_gb"]["test-nostack"] = 0 + self._write_config() + bare = self._run("admit", "--kind", "test-nostack") + self.assertEqual(bare.returncode, 2) + self.assertIn("command", bare.stderr) + gated = self._run("admit", "--kind", "test-nostack", "--", "python3", "gate.py", "run", "--target", "lint") + self.assertEqual(gated.returncode, 3, gated.stdout) + self.assertIn("queue.py hold --kind test --task -- python3 gate.py run --target lint", gated.stdout) + + def test_preflight_names_every_missing_memory_bound_and_passes_a_complete_or_absent_config(self): + self.cfg["memory_limits"] = OLD_MEMORY + self._write_config() + refused = self._run("preflight") + self.assertEqual(refused.returncode, 2) + self.assertIn( + "lacks memory_limits.warn.psi_some, memory_limits.warn.swap_out_mb_min, memory_limits.stop.swap_out_mb_min;", + refused.stderr, + ) + self.assertFalse(self.root.exists()) + self.cfg["memory_limits"] = LIVE_MEMORY + self._write_config() + self.assertEqual(self._run("preflight").returncode, 0) + self.config.unlink() + self.assertEqual(self._run("preflight").returncode, 0) + def test_block_refuses_heavy_jobs_at_memory_stop_only(self): self._stage("ok") self.cfg["memory_limits"]["warn"] = dict(SHUT_MEMORY) @@ -357,7 +1281,7 @@ class TestWatchCommands(unittest.TestCase): self.assertEqual(watched.returncode, 0) self.assertIn("RESOURCE WARN", watched.stdout) self.assertIn("lacks memory_limits", watched.stdout) - command = self._run("admit", "--need-gb", "0") + command = self._run("admit", "--need-gb", "0", "--", "true") self.assertEqual(command.returncode, 2) self.assertIn("lacks memory_limits", command.stderr)