diff --git a/README.md b/README.md index 9d07d85..5c6ce3d 100644 --- a/README.md +++ b/README.md @@ -464,6 +464,8 @@ make project TARGET=/path/to/repo Both commands also patch the target's `.claude/settings.local.json` — the machine-local, untracked file, so a project's reviewed `.claude/settings.json` is never rewritten by an install: the caveman and ponytail plugins are enabled so their modes auto-activate on session start, and two reminder hooks are registered — `kvb-dream` points each agent at `/kvb-dream` at most once every twelve hours (stamped per session in `~/.claude/.dream-stamps/`), and `kvb-coach` runs its short form at the first prompt of every full hour, once across all agents (stamped in `~/.claude/.coach-hours/`); the global install also enables the `kvb-dream-notify.timer` systemd user timer, which recommends `/kvb-dream` in a GNOME notification daily at 17:00 local time and catches up once after a missed run, and the `kvb-org-queue-cleanup.timer` and `kvb-org-queue-cleanup.path` user units, which start a `kvb-org-queue-cleanup` round at 00:00 and 12:00 local time without catch-up and whenever an agent runs `cleanup.py request`. A PreToolUse gate from the `kvb-org-organize` skill blocks edits to files in a git worktree until the session holds a claim (`skills/kvb-org-organize/scripts/council.py`), and its resource watch (`skills/kvb-org-organize/scripts/watch.py`) measures disk, Docker and caches, cleans up low-risk data and blocks new heavy jobs before the disk runs out; it stays inactive until `~/.claude/agents-council/config.json` exists, and because its hooks run that config's commands outside the sandbox, the install adds `Edit(~/.claude/agents-council/**)` to `permissions.deny`, which also covers Write in every permission mode; `Edit(~/.local/state/agents-council/**)` joins it, so the state the skills' scripts keep there changes only through those scripts. The global install writes both rules into `~/.claude/settings.json` too (`scripts/council-deny.js`), the only user-wide settings file, so they bind sessions in every repository. The `kvb-org-radio` skill registers five hooks: its prompt hook and a PostToolBatch hook deliver the immediate radio (`call`, `broadcast`, `security`, `pan-pan`, `mayday`) mid-turn, a Stop hook delivers the post (`post`, `newsletter` and call receipts) when the turn ends, an `asyncRewake` Stop hook wakes an idle session for the radio, and a PreToolUse hook refuses every writing action while a mayday holds, except those of its sender and of the agents that join it with `radio.py assist`. The PreToolUse hook of the `kvb-org-queue` skill sends heavy commands through `queue.py hold` once the config sets `queue_enforce`, and holds a held command to the settings' Bash deny and ask rules, which see only `queue.py hold`. The global install also installs the MEDUSA security scanner with pipx for `kvb-check-security`, unless `medusa` is already on the PATH. Two hooks of the `kvb-org-perform` skill (`skills/kvb-org-perform/scripts/log.py`) log every Agent spawn with its task class, model and effort, and store the quality feedback a sandboxed session prints, in `~/.local/state/agents-council/subagent-model.jsonl`. Every hook runs `/usr/bin/python3 -I`, so neither the `python3` of a sandbox-writable virtualenv on the `PATH` nor `PYTHONPATH` reaches the hooks, which run outside the sandbox; older bare `python3` hook entries are rewritten. The reminder hooks run `/usr/bin/bash` and pin `PATH=/usr/bin:/bin`, and the council scripts keep only the `PATH` directories this uid cannot write (`skills/kvb-org-organize/scripts/hookpath.py`), so no binary an agent plants on the inherited `PATH` runs outside the sandbox. Existing settings are preserved. +The council root is the one directory every `kvb-org` script, hook and the hub share: the register, radio, queue, claims, operator rows, score journal, gate receipts, security cache and night state, plus `pages/` (the `kvb-git-diff` review pages the hub reads), `approvals/` (the replay stores of `approval.py verify`) and `gates/` (gate logs). `skills/kvb-org-organize/scripts/councilroot.py` alone resolves it: `$AGENTS_COUNCIL_ROOT`, else `$XDG_STATE_HOME/agents-shared`, else `~/.local/state/agents-shared`, so it survives a reboot; `councilroot.py root` prints it and `councilroot.py dir ` creates and prints a part. It and `agents-release` below are the only state paths a sandbox's `allowWrite` may name. Its sibling `~/.local/state/agents-council` (`councilroot.py durable`) is host-only and never belongs in `allowWrite`: only hooks, the runner and the systemd units write it (`gates.db`, the watch hook's copies of the receipt, parent, acceptance and queue-history journals, `queue-cleanup.json`, `resources/`, `subagent-model.jsonl`), and the hub mounts it read-only; no directory is both agent-writable and host-written except the council root, where every host writer opens files through `nofollow.py`. The `kvb-release` drafts lie in `agents-release/` beside both (`councilroot.py release `). The global install first runs `councilroot.py preflight` and stops before it copies anything unless `~/.claude/settings.json` lists both `agents-shared` and `agents-release` in `sandbox.filesystem.allowWrite`, no `allowWrite` entry reaches `agents-council` in the managed file, its `managed-settings.d` drop-ins, the user or user-local file (the list `councilroot.py` shares with the queue's permission check), in a project `.claude/settings.json` or `.claude/settings.local.json` below the pkgmgr repositories and worktrees directories (directory and file links followed), or in the start directory of any session listed in `~/.claude/projects`, and the move below would succeed; it warns while the agents are not suspended. Fixing those entries is the operator's step. It then builds the skills, stops the hub, the runner, the offices, resources and queue-cleanup units, runs `councilroot.py migrate`, which moves an earlier council root in `/tmp/agents-council` into the council root, copies the skills, runs `migrate --catch-up` for what the old skills wrote meanwhile, and starts the units again; the hub image is built before the container is replaced, and a failed build leaves the hub down and fails the install until `bash scripts/orchestrator.sh install` succeeds. Migrate refuses while a lock there is held or a queue job waits, and, checked under those locks, moves nothing (exit 4) while a file cannot be moved or a whole file both roots hold differs; the install then stops with the earlier root intact. Each file is renamed to a `.migrating` name before it is copied, so a writer that opens the old name meanwhile makes a new file the catch-up moves. An identical file leaves one copy, a `.jsonl` journal gets the rows it lacks (by id, else each exact line as often as the earlier root holds it more often, in `ts` order), links are skipped, and a second run moves nothing and duplicates no row. The catch-up keeps a differing whole file beside its namesake as `.legacy-` (exit 3); a file that fails after the check stays behind (exit 5). After exit 3 or 5 the install goes on so the new skills read the new root, names every such file and exits 1 at the end. `~/.local/state/agents-council/migrate.started` exists from the first move to the end of a run; when a crash leaves it, the install copies the new skills anyway, names it and exits 1, and the next completed migrate removes it. Because the council root survives reboots, each `kvb-org-queue-cleanup` round first runs `councilroot.py retire`, which moves gate logs and review pages older than 30 days into `archive//`, and the radio appends the rows it drops past its last 500 to `archive/news-.jsonl`; no reader opens `archive/`, which `kvb-cleanup` offers for deletion. Run the install with every agent suspended (`kvb-org-suspend`) and the queue empty, so nothing writes either root during the move. + The global install also runs the Agent Orchestrator, a local review page in which the operator approves or rejects the agents' staged commits (the `kvb-git-diff` skill): it copies `orchestrator/` to `~/.local/share/kvb-orchestrator`, starts it there with `docker compose up -d --build` (restart `unless-stopped`, bound to `127.0.0.1` only, state in `~/.local/state/kvb-orchestrator`) and registers its MCP server for Claude Code at user scope. Its port is `KVB_ORCHESTRATOR_PORT` in `orchestrator/.env`; the hub page is `http://localhost:/`, the MCP endpoint `/mcp`. Without docker the hub is skipped and the commit skills ask in chat. The hub reads new radio traffic aloud in maritime phraseology and takes push-to-talk input; its image builds both engines from pinned, sha256-verified sources and runs each as a separate process: Piper (`piper-tts`, GPL-3.0-or-later) with the `en_US-libritts-high` voice (904 speakers, trained on LibriTTS, CC BY 4.0, from `rhasspy/piper-voices`) and whisper.cpp (MIT) with the multilingual `ggml-base-q8_0` model (OpenAI Whisper weights, MIT, 82 MB). The operator's own voice is `KVB_ORCHESTRATOR_OPERATOR_VOICE` in `orchestrator/.env`. Its Tree reads only the file `pkgmgr branch tree` writes: the install clones `PKGMGR_REPO` (default the GitHub repository, never the sandbox-writable local checkout), builds the newest `v*` tag a fully trusted GPG key signed into `~/.local/share/kvb-orchestrator/pkgmgr`, and enables the `kvb-orchestrator-tree.timer` user timer, which writes `~/.local/state/pkgmgr/tree/branch-tree.json` a minute after its previous run while the hub page polls, on at most half a core; the hub mounts that directory read-only and starts no git. A signed tag without `branch tree` leaves the timer off. Stream → Radio (`#stream/radio`) is the hub's radio interface. Its station list shows one line per station, the full name in the tooltip, with its state and a play or stop button; when the browser blocks autoplay, the stream fails or the CSP refuses its origin, the row and a toast say so, and one click starts the station. New stations play daily around the clock. While the hub reads a message aloud, the speaking agent pulses there and in the Orbit and Chord views, under the caption " speaking" with the message's first line; under `prefers-reduced-motion` the ring stays still. The speech bubble icon in the header (a hub setting, kept across reloads) puts a bubble at each speaking agent in the Agents graph, 3D, Hierarchy, Radio and Stream views, or in a caption bar at the bottom when the view does not show that agent: the agent being read gets the largest bubble and its words appear in time with the audio, waiting messages show a smaller `…` bubble per agent, and without speech output the same rows run at 180 words a minute, at least 1.5 s each. The composer sends on all seven channels: post and call to one agent, newsletter, broadcast, pan-pan, security and mayday to all. "Record" records with MediaRecorder; the hub transcribes the recording with whisper.cpp off the request thread (at most 120 s of audio, a 120 s timeout, four at a time), keeps it for replay in `~/.local/state/kvb-orchestrator/voice` (the newest 50) and puts the transcript into the field to edit; the message goes out as text, with the recording's host path and sha256 only when "Attach recording" is ticked. The hub transcribes at most six recordings a minute and 600 s of audio an hour. Every operator message is passkey-signed, and the hub verifies the signature against the pinned keys, each once, before it relays; `operator_message(id, text, to)` answers `match` only for the signed text and recipient; for a denial the signature covers the recipient the inbox resolved at signing, and the answer's text is only the signed reason, never the agent's own words the radio row quotes. "Cancel mayday", Evening, Sunset, Sunrise and a settings save that adds a stream URL take the passkey the same way. @@ -476,7 +478,7 @@ The Tasks tab (key 4, `#tasks/list`, `#tasks/board`, `#tasks/rasci`) shows the ` Without reinstalling the skills, `make orchestrator-install` installs, builds and starts only the hub, `make orchestrator-check` checks the installed hub (image, container, logs, HTTP, pinned keys, hook, runner, tree timer, disk; exit 1 on a failure), `make orchestrator-pin CREDENTIAL=` pins the passkey that "Register passkey" shows, `make orchestrator-runner` enables the runner service, and `make orchestrator-hook` registers the PermissionRequest hook in `~/.claude/settings.json`, replacing an earlier Agent Orchestrator entry and keeping the file mode, after a copy to `settings.json.before-kvb-orchestrator-hook` (Claude Code reads `~/.claude/settings.local.json` only in sessions started in the home directory). Agents that reach the Docker socket are root-equivalent: until the agent VMs of core requirement 093 isolate them, the signatures do not hold against such an agent. `make watch-check` shows whether the resource watch hook is stuck (running hooks, holders of the kept queue history, age of the stored stage). -For approvals the hub is only a relay: every approval of an ask, a staged commit or an operator command is signed end to end with a WebAuthn passkey held in KeePassXC (through KeePassXC-Browser, relying party `localhost`, origin `http://localhost:`; ES256, EdDSA and RS256, so a hardware key can be pinned beside it). To register, open the hub at `http://localhost:/` (WebAuthn needs the domain; on `127.0.0.1` the page links there), choose "Register passkey" in the settings and run the host command the page shows, `python3 ~/.local/share/kvb-orchestrator/client/approval.py pin --port --approvers ~/.config/kvb-orchestrator/approvers.json --credential ''`, in a terminal outside every sandbox: it pins the key in `~/.config/kvb-orchestrator/approvers.json` with mode 0600, a file no sandboxed agent can write and the hub mounts read-only; the hub never pins. Each approval, or one bulk selection, is one signature over the hashes of every item, a nonce and an expiry; each auto-approval rule is signed once. In the inbox, an operator command run with "Run" shows the runner's output in a console under its row until "Close". "Deny" on an ask, an operator row or a rule request signs the denial with its channel and optional reason (a hash no approval shares) and, after the 3 s undo, tells the requesting agent by post or by the radio class picked, never mayday; the hub verifies that signature against the pinned keys before it stores or sends anything, the hook passes the reason on only when it verifies the signature too, and a denied operator row ends as `stopped`. Agents ask for an auto-approval rule with the MCP tool `rule_request` (the rules form's limits; a request stays open for the ask lifetime of one hour, at most 20 in all, and 3 per agent name, which only spreads the queue since MCP names no caller); its row shows a dry run over the last 7 days, and only "Accept" with the passkey stores it. The hub stores no rule, of any kind, whose passkey assertion it cannot verify. The executors verify it themselves against the pinned keys and fail closed: the runner (`client/runner.py`, the `kvb-orchestrator-runner` user service) runs only signed commands, each signature once, recorded in `~/.local/state/kvb-orchestrator-approvals/runner.json`; the PermissionRequest hook (`client/permission_hook.py --port --timeout --approvers ~/.config/kvb-orchestrator/approvers.json --replay ~/.local/state/kvb-orchestrator-approvals/hook.json`) allows only a signed item that matches its own tool, command, cwd and session, else it leaves the decision to Claude Code's own prompt; the commit skills run `approval.py verify` on the staged tree before committing, else they ask in the terminal. With no key pinned, every executor refuses; review rejections and a decision a rule takes stay unsigned. A signed rule never covers a command holding `;`, `&`, `|`, a backtick, `$`, `<`, `>`, parentheses or a line break: such a command always asks. The hub page is trusted while it asks for a signature or shows the blob to pin: whoever controls the hub container can show one item and have another signed, or hand over a key of its own. Any local process, sandboxed agents included, can also send the unsigned operator actions (the bare decision choice `/api/decisions//choose`, which sends no answer, the priority order, settings that add no stream origin, review rejections, the bare ask denial `/api/asks//deny`, console dismissals) with the hub's guard headers, and the container reads the host `/tmp` for the review pages in `/tmp/kvb-git-diff`. All three close with the agent VMs of core requirement 093, like the Docker-socket gap. +For approvals the hub is only a relay: every approval of an ask, a staged commit or an operator command is signed end to end with a WebAuthn passkey held in KeePassXC (through KeePassXC-Browser, relying party `localhost`, origin `http://localhost:`; ES256, EdDSA and RS256, so a hardware key can be pinned beside it). To register, open the hub at `http://localhost:/` (WebAuthn needs the domain; on `127.0.0.1` the page links there), choose "Register passkey" in the settings and run the host command the page shows, `python3 ~/.local/share/kvb-orchestrator/client/approval.py pin --port --approvers ~/.config/kvb-orchestrator/approvers.json --credential ''`, in a terminal outside every sandbox: it pins the key in `~/.config/kvb-orchestrator/approvers.json` with mode 0600, a file no sandboxed agent can write and the hub mounts read-only; the hub never pins. Each approval, or one bulk selection, is one signature over the hashes of every item, a nonce and an expiry; each auto-approval rule is signed once. In the inbox, an operator command run with "Run" shows the runner's output in a console under its row until "Close". "Deny" on an ask, an operator row or a rule request signs the denial with its channel and optional reason (a hash no approval shares) and, after the 3 s undo, tells the requesting agent by post or by the radio class picked, never mayday; the hub verifies that signature against the pinned keys before it stores or sends anything, the hook passes the reason on only when it verifies the signature too, and a denied operator row ends as `stopped`. Agents ask for an auto-approval rule with the MCP tool `rule_request` (the rules form's limits; a request stays open for the ask lifetime of one hour, at most 20 in all, and 3 per agent name, which only spreads the queue since MCP names no caller); its row shows a dry run over the last 7 days, and only "Accept" with the passkey stores it. The hub stores no rule, of any kind, whose passkey assertion it cannot verify. The executors verify it themselves against the pinned keys and fail closed: the runner (`client/runner.py`, the `kvb-orchestrator-runner` user service) runs only signed commands, each signature once, recorded in `~/.local/state/kvb-orchestrator-approvals/runner.json`; the PermissionRequest hook (`client/permission_hook.py --port --timeout --approvers ~/.config/kvb-orchestrator/approvers.json --replay ~/.local/state/kvb-orchestrator-approvals/hook.json`) allows only a signed item that matches its own tool, command, cwd and session, else it leaves the decision to Claude Code's own prompt; the commit skills run `approval.py verify` on the staged tree before committing, else they ask in the terminal. With no key pinned, every executor refuses; review rejections and a decision a rule takes stay unsigned. A signed rule never covers a command holding `;`, `&`, `|`, a backtick, `$`, `<`, `>`, parentheses or a line break: such a command always asks. The hub page is trusted while it asks for a signature or shows the blob to pin: whoever controls the hub container can show one item and have another signed, or hand over a key of its own. Any local process, sandboxed agents included, can also send the unsigned operator actions (the bare decision choice `/api/decisions//choose`, which sends no answer, the priority order, settings that add no stream origin, review rejections, the bare ask denial `/api/asks//deny`, console dismissals) with the hub's guard headers, and the container reads the review pages from `pages/` of the council root, which every sandboxed agent can write; it mounts the council root at its host path and no longer reads the host `/tmp`. All three close with the agent VMs of core requirement 093, like the Docker-socket gap. Restart your agent afterwards so it loads the new skills. diff --git a/docs/requirements/003-operator-workbench.md b/docs/requirements/003-operator-workbench.md index 94e3d80..63cdda7 100644 --- a/docs/requirements/003-operator-workbench.md +++ b/docs/requirements/003-operator-workbench.md @@ -61,8 +61,8 @@ changed and what it costs, and a gate means something again. 12. **Gates**: a level (none < quick < test < quality-high < e2e) per branch and action (commit, merge, push, pr), inherited from all ancestors, the strictest wins, an own rule wins over all; kept in the local database - `~/.local/state/agents-council/gates.db` (the council root lives in the - tmpfs `/tmp` and loses it at reboot) that agents read through `gates.py`; + `~/.local/state/agents-council/gates.db` (host-only; agents share the + council root `~/.local/state/agents-shared`) that agents read through `gates.py`; set in every Artifacts view. Rejected: a committed `.gates.yml`, repo plus local. 13. **Hooks with receipts**: pre-commit runs only quick checks of the staged @@ -243,6 +243,9 @@ flowchart TD `gate-parents.jsonl` in the council root, which the watch hook of `kvb-org-organize` keeps in `~/.local/state/agents-council/` beside `gates.db`; reads merge both, and a database row for an edge wins. + Amended 2026-11-14: the council root is the durable + `~/.local/state/agents-shared` (`councilroot.py root`); the copy stays in + the host-only `~/.local/state/agents-council` (`councilroot.py durable`). - [x] `gates.py resolve --repo R --branch B --action A` prints the level and its source: the branch's own rule, else the strictest resolved level of all ancestors, else none. @@ -293,6 +296,10 @@ flowchart TD sandboxed gates write receipts to `/tmp/agents-council` (the council root) and the watch hook of `kvb-org-organize` mirrors them to `~/.local/state/agents-council`, as it does `security-accepted.jsonl`. + Amended 2026-11-14 after a host crash wiped `/tmp`: the council root + moved to the durable `~/.local/state/agents-shared`, the one state path + the sandbox may write; `~/.local/state/agents-council` stays host-only, + and the watch hook still mirrors receipts and acceptances into it. (4) Agents commit and push with `--no-verify` per operator rule, so the hooks bind humans and CI; the push hand-over keeps its own green gate on the exact commit. @@ -347,7 +354,8 @@ The work packages meet only through these interfaces: cache_write}`, t the hour), `offsets.json` and `cpu.json` under `~/.local/state/agents-council/resources/`. - **Mounts**: the hub mounts `~/.local/state/agents-council` read-only and - opens `gates.db` read-only. + opens `gates.db` read-only; the council root `~/.local/state/agents-shared` + is mounted read-write at its host path. ### W6 Speech bubbles diff --git a/docs/requirements/006-score-metric-overlays.md b/docs/requirements/006-score-metric-overlays.md index d87b2e3..c4956a5 100644 --- a/docs/requirements/006-score-metric-overlays.md +++ b/docs/requirements/006-score-metric-overlays.md @@ -25,7 +25,8 @@ the burndown, so that I see how quality and cost move alongside the work. [--labels ]` parses score blocks from commit messages through `git log` with the label file, keyed by (remote, commit sha), idempotent (a second run adds nothing), runnable on any host, after a reboot (the journal - lives in the RAM-backed `/tmp`) or on another PC; no model calls. + lived in the RAM-backed `/tmp` until 2026-11-14 and is durable since) or + on another PC; no model calls. Rejected: an agent reading the history (tokens); parsing at view time in the hub (the hub would need every repository). 5. **Overlays**: the burndown gets one switch per metric (six), all off by diff --git a/orchestrator/client/offices.py b/orchestrator/client/offices.py index b71b642..7c2ebb7 100755 --- a/orchestrator/client/offices.py +++ b/orchestrator/client/offices.py @@ -9,7 +9,7 @@ --replay: this runner's replay store, outside the hub's reach, e.g. ~/.local/state/kvb-orchestrator-approvals/offices.json. --council: the council directory every agent shares, e.g. - /tmp/agents-council. + ~/.local/state/agents-shared. --script: the installed council.py of the kvb-org-organize skill. Returns: diff --git a/orchestrator/client/resources.py b/orchestrator/client/resources.py index a8abc74..c9f3fa1 100755 --- a/orchestrator/client/resources.py +++ b/orchestrator/client/resources.py @@ -43,7 +43,9 @@ from __future__ import annotations import argparse import base64 +import contextlib import datetime +import errno import functools import hashlib import json @@ -51,7 +53,6 @@ import os import shutil import stat import subprocess -import tempfile import time from collections.abc import Callable from pathlib import Path @@ -367,20 +368,47 @@ def encoded(rows: list[dict]) -> str: return "".join(json.dumps(row, separators=(",", ":")) + "\n" for row in rows) +def folder(path: Path) -> int: + return os.open(path.parent, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW | os.O_CLOEXEC) + + def append_rows(path: Path, rows: list[dict]) -> None: + """Args: + path: the file; a link at it or its directory, or a file that is not + regular with a single link, raises OSError. + rows: appended as JSON lines. + """ if rows: - with path.open("a", encoding="utf-8") as handle: + parent = folder(path) + try: + fd = os.open(path.name, os.O_WRONLY | os.O_CREAT | os.O_APPEND | os.O_NOFOLLOW | os.O_NONBLOCK | os.O_CLOEXEC, 0o600, dir_fd=parent) + finally: + os.close(parent) + with open(fd, "a", encoding="utf-8") as handle: + info = os.fstat(fd) + if not stat.S_ISREG(info.st_mode) or info.st_nlink != 1: + raise OSError(errno.EINVAL, "not a regular file with a single link", str(path)) handle.write(encoded(rows)) def replace(path: Path, text: str) -> None: - with tempfile.NamedTemporaryFile("w", encoding="utf-8", dir=path.parent, prefix=f".{path.name}.", delete=False) as handle: - try: + """Args: + path: the file, replaced through a fresh temp file beside it; a link + at its directory raises OSError, one at path is replaced. + text: the new content. + """ + parent = folder(path) + temp = f".{path.name}.{os.urandom(8).hex()}" + try: + with open(os.open(temp, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW | os.O_CLOEXEC, 0o600, dir_fd=parent), "w", encoding="utf-8") as handle: handle.write(text) - except BaseException: - os.unlink(handle.name) - raise - os.replace(handle.name, path) + os.replace(temp, path.name, src_dir_fd=parent, dst_dir_fd=parent) + except BaseException: + with contextlib.suppress(OSError): + os.unlink(temp, dir_fd=parent) + raise + finally: + os.close(parent) def rewrite(path: Path, keep: Callable[[list[dict], float], list[dict]], now: float) -> None: diff --git a/orchestrator/compose.yml b/orchestrator/compose.yml index 261afa7..70b52e1 100644 --- a/orchestrator/compose.yml +++ b/orchestrator/compose.yml @@ -13,6 +13,7 @@ services: read_only: true environment: TMPDIR: /scratch + AGENTS_COUNCIL_ROOT: "${KVB_ORCHESTRATOR_COUNCIL:?written into .env by scripts/install.sh}" cap_drop: - ALL security_opt: @@ -36,9 +37,9 @@ services: - --public-port - "${KVB_ORCHESTRATOR_PORT:?set in .env}" - --council - - /tmp/agents-council + - "${KVB_ORCHESTRATOR_COUNCIL:?written into .env by scripts/install.sh}" - --pages - - /tmp/kvb-git-diff + - "${KVB_ORCHESTRATOR_COUNCIL:?written into .env by scripts/install.sh}/pages" - --state - /state - --host-state @@ -88,13 +89,9 @@ services: tmpfs: - /scratch:size=64m,mode=1777,noexec,nosuid,nodev volumes: - - type: bind - source: /tmp - target: /tmp - read_only: true - type: bind source: "${KVB_ORCHESTRATOR_COUNCIL:?written into .env by scripts/install.sh}" - target: /tmp/agents-council + target: "${KVB_ORCHESTRATOR_COUNCIL:?written into .env by scripts/install.sh}" bind: create_host_path: false - type: bind diff --git a/orchestrator/kvb_orchestrator/gating.py b/orchestrator/kvb_orchestrator/gating.py index 19385ec..0f443c1 100644 --- a/orchestrator/kvb_orchestrator/gating.py +++ b/orchestrator/kvb_orchestrator/gating.py @@ -48,7 +48,7 @@ def branch_gates(gates: ModuleType, db: sqlite3.Connection, repo: str, branch: s def read_gates(gates: ModuleType | None, db: Path, scan: dict) -> dict: """Args: gates: the gates.py module, or None. - db: gates.db under the read-only mount of ~/.local/state/agents-council. + db: gates.db under the read-only mount of the durable council root. scan: the tree; its repositories and branches are the ones shown. Returns: diff --git a/orchestrator/server.py b/orchestrator/server.py index 637e0e2..7a1f21e 100755 --- a/orchestrator/server.py +++ b/orchestrator/server.py @@ -25,8 +25,8 @@ --tree-file: the JSON file `pkgmgr branch tree --output` writes, mounted read-only; read again whenever it changes. --timers: directory holding the systemd user units (*.timer). - --kept: ~/.local/state/agents-council, mounted read-only: gates.db and - the resources/ files of the host sampler. + --kept: the durable council root (councilroot.py durable), mounted + read-only: gates.db and the resources/ files of the host sampler. --vendor: directory holding the vendored Bootstrap Icons sprite, 3d-force-graph and three with their licence texts. --cleanup: host-mounted directory the hub writes signed branch deletions diff --git a/scripts/install.sh b/scripts/install.sh index 0a9aa9d..1c09f98 100644 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -23,6 +23,32 @@ if [[ ! -d "${TARGET}" ]]; then exit 2 fi +council_root="${REPO_ROOT}/skills/kvb-org-organize/scripts/councilroot.py" +has_user_systemd() { command -v systemctl &>/dev/null && systemctl --user show-environment &>/dev/null; } +resume_units() { + bash "${REPO_ROOT}/scripts/orchestrator.sh" start || true + if has_user_systemd; then systemctl --user start kvb-org-queue-cleanup.timer kvb-org-queue-cleanup.path || true; fi +} +# Args: extra councilroot.py migrate arguments. Returns its exit code; on 3 +# or 5 every file it left behind or kept beside its namesake is named on stderr. +migrate_council() { + local report code=0 + report="$(python3 "${council_root}" migrate "$@")" || code=$? + log "${report}" + if ((code == 3 || code == 5)); then + warn "skills: the council root migration left these files in the earlier root or beside their namesakes; merge them by hand, the install exits 1 at the end:" + grep -E '^(unmoved|conflict|kept) ' <<<"${report}" | sed 's/^/skills: /' >&2 || true + fi + return "${code}" +} +status=0 +if [[ "${TARGET}" == "${HOME}" ]]; then + if ! python3 "${council_root}" preflight; then + warn "skills: install refused before copying anything; the installed skills keep the earlier council root until the reasons above are gone." + exit 1 + fi +fi + cd "${REPO_ROOT}" src="${REPO_ROOT}/.agents/skills" rm -rf "${src}" @@ -39,6 +65,31 @@ if [[ -d "${REPO_ROOT}/skills" ]]; then cp -a "${REPO_ROOT}/skills/." "${src}/" fi +if [[ "${TARGET}" == "${HOME}" ]]; then + install -d -m 0700 "$(python3 "${council_root}" durable)" + if has_user_systemd; then + systemctl --user stop kvb-org-queue-cleanup.path kvb-org-queue-cleanup.timer kvb-org-queue-cleanup.service &>/dev/null || true + fi + if ! bash "${REPO_ROOT}/scripts/orchestrator.sh" stop; then + resume_units + warn "skills: install refused before copying anything: the Agent Orchestrator did not stop." + exit 1 + fi + moved=0 + migrate_council || moved=$? + marker="$(python3 "${council_root}" durable)/migrate.started" + if [[ -e "${marker}" ]]; then + warn "skills: the council root migration stopped halfway (exit ${moved}): ${marker} exists; the install copies the new skills so they read the new root and exits 1 at the end; 'python3 ${council_root} migrate --catch-up' finishes the move and removes ${marker}." + status=1 + elif ((moved == 3 || moved == 5)); then + status=1 + elif ((moved != 0)); then + resume_units + warn "skills: install refused before copying anything: the council root migration failed (exit ${moved}); run the install again once the reasons above are gone." + exit 1 + fi +fi + for root in .agents/skills .claude/skills; do dst="${TARGET}/${root}" log "skills: copying -> ${dst}" @@ -58,6 +109,17 @@ for root in .agents/skills .claude/skills; do fi done +if [[ "${TARGET}" == "${HOME}" ]]; then + caught=0 + migrate_council --catch-up || caught=$? + if ((caught != 0)); then + status=1 + fi + if ((caught != 0 && caught != 3 && caught != 5)); then + warn "skills: the catch-up council root migration failed (exit ${caught}); what the old skills wrote during the install stays in the earlier root until 'python3 ${council_root} migrate --catch-up' succeeds." + fi +fi + settings="${TARGET}/.claude/settings.local.json" if command -v node &>/dev/null; then log "skills: auto-enabling caveman + ponytail plugins in ${settings}" @@ -70,8 +132,7 @@ else fi if [[ "${TARGET}" == "${HOME}" ]]; then - install -d -m 0700 "${HOME}/.local/state/agents-council" - if command -v systemctl &>/dev/null && systemctl --user show-environment &>/dev/null; then + if has_user_systemd; then log "skills: enabling the daily kvb-dream notification timer and the kvb-org-queue-cleanup rounds at 00:00, 12:00 and on request" units="${HOME}/.config/systemd/user" mkdir -p "${units}" @@ -83,7 +144,10 @@ if [[ "${TARGET}" == "${HOME}" ]]; then warn "skills: no systemd user session; skipped the kvb-dream notification timer and the kvb-org-queue-cleanup rounds." fi if command -v docker &>/dev/null; then - bash "${REPO_ROOT}/scripts/orchestrator.sh" install + if ! bash "${REPO_ROOT}/scripts/orchestrator.sh" install; then + warn "skills: the Agent Orchestrator is down or partly installed; run 'bash scripts/orchestrator.sh install' once the reasons above are gone." + status=1 + fi bash "${REPO_ROOT}/scripts/orchestrator.sh" runner || true else warn "skills: docker not found; skipped the Agent Orchestrator." @@ -129,3 +193,4 @@ if [[ "${TARGET}" == "${HOME}" ]]; then fi log "skills: done. Restart your agent to load the skills." +exit "${status}" diff --git a/scripts/orchestrator.sh b/scripts/orchestrator.sh index ad1afcf..03f9888 100644 --- a/scripts/orchestrator.sh +++ b/scripts/orchestrator.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Usage: orchestrator.sh install | migrate | check | pin | runner | hook +# Usage: orchestrator.sh install | migrate | stop | start | check | pin | runner | hook # install migrate, then copy orchestrator/ to ~/.local/share/kvb-orchestrator, # write its .env, build and start the container, register the MCP # server for Claude Code, build pkgmgr from the newest signed tag of @@ -12,12 +12,16 @@ # once: container, image, code, state, approvals, pinned keys, MCP # server, PermissionRequest hook, runner unit; a second run changes # nothing +# stop stop the container, the runner, the offices path unit and the +# resources timer, which write the council roots, keeping them +# enabled; exit 1 when the container does not stop +# start start them again, the units only while enabled # check image imports, container, logs, HTTP, pinned keys, hook, runner, # disk; exit 1 when a check fails # pin pin the credential blob the "Passkey registrieren" settings show # runner install, enable and restart the runner user service; disable it -# and exit 1 while the installed council.py, nofollow.py or -# rasci.py differ from this tree +# and exit 1 while the installed council.py, nofollow.py, +# councilroot.py or rasci.py differ from this tree # hook register the PermissionRequest hook in ~/.claude/settings.json, # replacing an earlier entry, keeping everything else and the file # mode, after a copy to settings.json.before-kvb-orchestrator-hook @@ -31,8 +35,9 @@ STATE="${HOME}/.local/state/${NAME}" APPROVALS="${HOME}/.local/state/${NAME}-approvals" APPROVERS="${HOME}/.config/${NAME}" UNITS="${HOME}/.config/systemd/user" -# council.py ROOT: every agent must agree on this directory -COUNCIL="${AGENTS_COUNCIL_ROOT:-/tmp/agents-council}" +COUNCIL_ROOT="${REPO_ROOT}/skills/kvb-org-organize/scripts/councilroot.py" +COUNCIL="$(python3 "${COUNCIL_ROOT}" root)" +KEPT="$(python3 "${COUNCIL_ROOT}" durable)" SETTINGS="${HOME}/.claude/settings.json" LOCAL_SETTINGS="${HOME}/.claude/settings.local.json" HOOK_TIMEOUT=110 @@ -44,7 +49,7 @@ SESSIONS_UNIT="${NAME}-sessions" RESOURCES_UNIT="${NAME}-resources" FIREWALL_UNIT="${NAME}-firewall" INSTALLED_SKILLS="${HOME}/.claude/skills" -HOST_SCRIPTS=(kvb-org-organize/scripts/council.py kvb-org-organize/scripts/nofollow.py kvb-org-rasci/scripts/rasci.py kvb-org-operator/scripts/operator.py kvb-org-radio/scripts/radio.py) +HOST_SCRIPTS=(kvb-org-organize/scripts/council.py kvb-org-organize/scripts/nofollow.py kvb-org-organize/scripts/councilroot.py kvb-org-rasci/scripts/rasci.py kvb-org-operator/scripts/operator.py kvb-org-radio/scripts/radio.py) TREE_DIR="${HOME}/.local/state/pkgmgr/tree" PKGMGR_REPO="${PKGMGR_REPO:-https://github.com/kevinveenbirkenbach/package-manager.git}" IMAGE="${NAME}:local" @@ -153,7 +158,7 @@ install_hub() { port="$(port_of "${REPO_ROOT}/orchestrator/.env")" log "skills: starting the Agent Orchestrator on http://localhost:${port}/" rm -rf "${HUB}" - mkdir -p "${HUB}" "${STATE}" "${APPROVALS}" "${COUNCIL}" "${HOME}/.claude/projects" "${TREE_DIR}" "${UNITS}" "${APPROVERS}" "${HOME}/.local/state/agents-council/resources" + mkdir -p "${HUB}" "${STATE}" "${APPROVALS}" "${COUNCIL}" "${HOME}/.claude/projects" "${TREE_DIR}" "${UNITS}" "${APPROVERS}" "${KEPT}/resources" chmod 700 "${APPROVERS}" cp -a "${REPO_ROOT}/orchestrator/." "${HUB}/" # Python loads a cached .pyc whose header matches the source mtime and size, and cp -a keeps the mtime: one planted in the sandbox-writable tree would run on the host @@ -162,8 +167,12 @@ install_hub() { log "skills: the PermissionRequest hook client is ${HUB}/client/permission_hook.py" log "skills: approvals need a pinned passkey: settings, 'Passkey registrieren', then 'make orchestrator-pin CREDENTIAL='; until then every executor refuses" printf 'KVB_ORCHESTRATOR_USER=%s:%s\nKVB_ORCHESTRATOR_STATE=%s\nKVB_ORCHESTRATOR_COUNCIL=%s\nKVB_ORCHESTRATOR_SKILLS=%s\nKVB_ORCHESTRATOR_PROJECTS=%s\nKVB_ORCHESTRATOR_TREE=%s\nKVB_ORCHESTRATOR_TIMERS=%s\nKVB_ORCHESTRATOR_APPROVERS=%s\nKVB_ORCHESTRATOR_KEPT=%s\n' \ - "$(id -u)" "$(id -g)" "${STATE}" "${COUNCIL}" "${HOME}/.claude/skills" "${HOME}/.claude/projects" "${TREE_DIR}" "${UNITS}" "${APPROVERS}" "${HOME}/.local/state/agents-council" >>"${HUB}/.env" - if ! docker compose --file "${HUB}/compose.yml" up -d --build; then + "$(id -u)" "$(id -g)" "${STATE}" "${COUNCIL}" "${HOME}/.claude/skills" "${HOME}/.claude/projects" "${TREE_DIR}" "${UNITS}" "${APPROVERS}" "${KEPT}" >>"${HUB}/.env" + if ! docker compose --file "${HUB}/compose.yml" build; then + docker compose --file "${HUB}/compose.yml" stop || true + warn "skills: building the Agent Orchestrator failed; it stays down until 'bash scripts/orchestrator.sh install' succeeds." + return 1 + elif ! docker compose --file "${HUB}/compose.yml" up -d; then warn "skills: docker compose failed; the Agent Orchestrator stays down until 'make orchestrator-install' succeeds." fi if ! command -v claude &>/dev/null; then @@ -255,7 +264,7 @@ resources_timer() { systemctl --user daemon-reload systemctl --user enable --now "${RESOURCES_UNIT}.timer" systemctl --user restart "${RESOURCES_UNIT}.timer" - log "skills: ${RESOURCES_UNIT}.timer samples the host and counts the transcripts' tokens into ${HOME}/.local/state/agents-council/resources every minute" + log "skills: ${RESOURCES_UNIT}.timer samples the host and counts the transcripts' tokens into ${KEPT}/resources every minute" } firewall_unit() { @@ -285,7 +294,7 @@ offices_unit() { fi if ! skills_current; then systemctl --user disable --now "${OFFICES_UNIT}.path" >/dev/null 2>&1 || true - warn "skills: ${INSTALLED_SKILLS} holds other council.py, nofollow.py, rasci.py, operator.py or radio.py than this tree; run 'make install' to enable ${OFFICES_UNIT}.path." + warn "skills: ${INSTALLED_SKILLS} holds other council.py, nofollow.py, councilroot.py, rasci.py, operator.py or radio.py than this tree; run 'make install' to enable ${OFFICES_UNIT}.path." return 0 fi cp -f "${HUB}/systemd/${OFFICES_UNIT}.service" "${HUB}/systemd/${OFFICES_UNIT}.path" "${UNITS}/" @@ -296,7 +305,8 @@ offices_unit() { install_all() { migrate || return 0 - install_hub + local hub=0 + install_hub || hub=1 tree_timer sessions_timer resources_timer @@ -306,6 +316,27 @@ install_all() { if ((TOOK_HOOK)); then if [[ -f "${HUB}/client/permission_hook.py" ]]; then hook; else warn "skills: run 'make orchestrator-hook' once the Agent Orchestrator is installed"; fi fi + return "${hub}" +} + +stop_all() { + if [[ -f "${HUB}/compose.yml" ]] && command -v docker &>/dev/null; then + docker compose --file "${HUB}/compose.yml" stop || return 1 + fi + if has_systemd; then + systemctl --user stop "${RUNNER}" "${OFFICES_UNIT}.path" "${OFFICES_UNIT}.service" "${RESOURCES_UNIT}.timer" "${RESOURCES_UNIT}.service" &>/dev/null || true + fi +} + +start_all() { + if [[ -f "${HUB}/compose.yml" ]] && command -v docker &>/dev/null; then + docker compose --file "${HUB}/compose.yml" start || warn "skills: starting the Agent Orchestrator failed" + fi + if has_systemd; then + for unit in "${OFFICES_UNIT}.path" "${RESOURCES_UNIT}.timer" "${RUNNER}"; do + if [[ "$(systemctl --user is-enabled "${unit}" 2>/dev/null || true)" == "enabled" ]]; then systemctl --user start "${unit}" || true; fi + done + fi } runner_enabled() { @@ -319,7 +350,7 @@ runner() { fi if ! skills_current; then systemctl --user disable --now "${RUNNER}" >/dev/null 2>&1 || true - warn "skills: ${INSTALLED_SKILLS} holds other council.py, nofollow.py, rasci.py, operator.py or radio.py than this tree; run 'make install' to enable ${RUNNER}." + warn "skills: ${INSTALLED_SKILLS} holds other council.py, nofollow.py, councilroot.py, rasci.py, operator.py or radio.py than this tree; run 'make install' to enable ${RUNNER}." return 1 fi cp -f "${HUB}/systemd/${RUNNER}" "${UNITS}/" @@ -425,12 +456,14 @@ hook() { case "${1:-}" in install) install_all ;; migrate) migrate ;; +stop) stop_all ;; +start) start_all ;; check) check ;; pin) pin "${2:-}" ;; runner) runner ;; hook) hook ;; *) - warn "usage: orchestrator.sh install | migrate | check | pin | runner | hook" + warn "usage: orchestrator.sh install | migrate | stop | start | check | pin | runner | hook" exit 2 ;; esac diff --git a/scripts/watch-check.sh b/scripts/watch-check.sh index 4d64322..247c158 100644 --- a/scripts/watch-check.sh +++ b/scripts/watch-check.sh @@ -5,9 +5,9 @@ # resource stage. Exit 1 when the stored stage is older than STALE_SECONDS. set -euo pipefail -# council.py ROOT: every agent must agree on this directory -STATE="${AGENTS_COUNCIL_ROOT:-/tmp/agents-council}/watch.json" -KEPT="${HOME}/.local/state/agents-council/queue-history.jsonl" +COUNCIL_ROOT="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd)/skills/kvb-org-organize/scripts/councilroot.py" +STATE="$(python3 "${COUNCIL_ROOT}" root)/watch.json" +KEPT="$(python3 "${COUNCIL_ROOT}" durable)/queue-history.jsonl" STALE_SECONDS=600 printf 'watch hooks running: %s\n' "$(pgrep -fc 'watch.py watch' || true)" diff --git a/skills/kvb-check-gate/SKILL.md b/skills/kvb-check-gate/SKILL.md index 2f7a526..87b07c2 100644 --- a/skills/kvb-check-gate/SKILL.md +++ b/skills/kvb-check-gate/SKILL.md @@ -28,8 +28,8 @@ commit starts from zero. include or target expression means the list may be short: read that line and name the gap. A project without any target keeps the gate open; ask the operator. -3. **Status**: `gate.py status --repo `. `gate: green` ends here; a - `missing hotfix` reason means back to step 1. +3. **Status**: `gate.py status --repo `. `gate: green` ends here; + `missing hotfix` means step 1. 4. **Admit**: every gate target is heavy and runs only inside `${CLAUDE_SKILL_DIR}/../kvb-org-queue/scripts/queue.py hold`. Only a single test module @@ -40,17 +40,18 @@ commit starts from zero. background; give the operator `tail -f` on the log: ```bash - queue.py hold --kind --task -- gate.py run --repo --target [--target ...] --log-dir /tmp/ [--fail-regex ...] + queue.py hold --kind --task -- gate.py run --repo --target [--target ...] --log-dir [--fail-regex ...] ``` + ``: what `${CLAUDE_SKILL_DIR}/../kvb-org-organize/scripts/councilroot.py dir gates ` prints; stop when it fails. + Pass one `--fail-regex` per failure line the project prints while exiting 0; infinito-nexus needs `FAILED TARGETS: \S`. A non-zero exit, a `make: ***` line, a matching regex or modified tracked files make a target red; the last stops the run. Before each target `run` asks `hotfix.py void` for it and every later one. Exit 3 (`void: merge main for hotfix , then rerun`): back to step 1, then through the queue - again. Exit 75: the worktree is - busy, the message names the holder; wait, then rerun. + again. Exit 75: busy worktree, holder named; wait, rerun. 6. **Judge** every red target by its `status` line and the log around it: caused by the change, already red on the default branch, or the environment. Fix the cause and rerun; looking unrelated never makes it diff --git a/skills/kvb-check-gate/scripts/gates.py b/skills/kvb-check-gate/scripts/gates.py index c8d4041..54fdc19 100755 --- a/skills/kvb-check-gate/scripts/gates.py +++ b/skills/kvb-check-gate/scripts/gates.py @@ -2,11 +2,11 @@ """Gate rules, branch parents and gate receipts of every repository on this host. Args: - --db: the SQLite database of rules and parents, default - ~/.local/state/agents-council/gates.db (GATES_DB overrides it); it - survives reboots, the council root under /tmp does not. Only rule and - parents add|drop|derive write it; every other command opens it - read-only and an absent one as empty. + --db: the SQLite database of rules and parents, default gates.db in the + durable council root (councilroot.py durable of kvb-org-organize; + GATES_DB overrides it). Only rule and parents add|drop|derive write + it; every other command opens it read-only and an absent one as + empty. rule set --repo R --branch B --action A --level L: the branch's own level for one action. rule drop --repo R --branch B --action A removes it. @@ -48,7 +48,8 @@ Args: branch's merge level before it leaves. Each passes when a receipt for the tree meets the resolved level, else prints the command that produces it and exits 1: `gates.py quick` for quick; above it `queue.py hold --receipt ` - of `make ` with INFINITO_TEST_GPU=false, held as kind + of `make ` with INFINITO_TEST_GPU=false and its log in the + council root's gates/, which it creates, held as kind quality-high for e2e, else as the level, with the open register task whose branch is B. post-checkout: parents record on a branch checkout; it exits 0 even when recording fails. @@ -63,9 +64,9 @@ Args: Journals: Receipts {id, repo, tree, level, gpu, log, at} and recorded parents {id, repo, branch, parent} are rows of gate-receipts.jsonl and - gate-parents.jsonl in the council root (AGENTS_COUNCIL_ROOT, else - /tmp/agents-council), which sandboxed agents can write; the watch hook of - kvb-org-organize copies them to ~/.local/state/agents-council, and reading + gate-parents.jsonl in the council root (councilroot.py root of + kvb-org-organize), which sandboxed agents can write; the watch hook of + kvb-org-organize copies them to the host-only durable root, and reading merges both by id; a row whose id is no string is skipped. A receipt carries INFINITO_TEST_GPU of the environment that wrote it as gpu; it meets its own level and every lower one, above quick only with gpu @@ -99,7 +100,6 @@ LEVELS = ("none", "quick", "test", "quality-high", "e2e") ACTIONS = ("commit", "merge", "push", "pr") SOURCES = ("recorded", "derived", "operator") DROPPED = "dropped" -DEFAULT_DB = Path.home() / ".local" / "state" / "agents-council" / "gates.db" SKILLS = Path(__file__).resolve().parents[2] QUEUE = SKILLS / "kvb-org-queue" / "scripts" / "queue.py" RASCI = SKILLS / "kvb-org-rasci" / "scripts" / "rasci.py" @@ -137,6 +137,8 @@ def module(name: str, path: Path) -> ModuleType: watch = module("kvb_gates_watch", SKILLS / "kvb-org-organize" / "scripts" / "watch.py") +DEFAULT_DB = watch.KEPT / "gates.db" +LOGS = "gates" def fail(message: str) -> None: @@ -250,7 +252,8 @@ def receipted(repo: str, tree: str, level: str) -> bool: def opened(path: Path, write: bool, kept_dir: Path) -> sqlite3.Connection: """Args: path: gates.db; read-only unless write, in memory when it is missing. - write: open it for rule and parent changes. + write: open it for rule and parent changes; OSError when path or its + directory is a link or path no regular file with a single link. kept_dir: the directory holding the kept gate-parents.jsonl: watch.KEPT on the host, the read-only mount of it in the hub. @@ -260,6 +263,7 @@ def opened(path: Path, write: bool, kept_dir: Path) -> sqlite3.Connection: """ if write: path.parent.mkdir(parents=True, exist_ok=True) + os.close(watch.nofollow.open_below(path.parent, path, os.O_RDWR | os.O_CREAT, FILE_MODE)) db = sqlite3.connect(path) elif path.exists(): db = sqlite3.connect(f"{path.resolve().as_uri()}?mode=ro", uri=True) @@ -466,7 +470,10 @@ def task_of(branch: str) -> str: def producer(worktree: str, branch: str, level: str, tree: str) -> str: if level == "quick": return shlex.join(["python3", str(Path(__file__).resolve()), "quick", "--worktree", worktree]) - log = Path(tempfile.gettempdir()) / f"gate-{tree[:12]}-{level}.log" + try: + log = watch.councilroot.directory(watch.ROOT, LOGS) / f"gate-{tree[:12]}-{level}.log" + except (OSError, ValueError) as error: + return f"nothing until the council root is fixed: {error}" return shlex.join([ "env", "-C", worktree, f"INFINITO_TEST_GPU={GPU_OFF}", "python3", str(QUEUE), "hold", "--kind", KINDS[level], "--task", task_of(branch), "--receipt", level, "--log", str(log), "--", "make", level, diff --git a/skills/kvb-check-score/journal.md b/skills/kvb-check-score/journal.md index 0a5ded5..a721352 100644 --- a/skills/kvb-check-score/journal.md +++ b/skills/kvb-check-score/journal.md @@ -1,16 +1,16 @@ # Journal Each `--message-block` render appends one row to -`${AGENTS_COUNCIL_ROOT:-/tmp/agents-council}/scores.jsonl`: time, origin +`scores.jsonl` in the council root that `councilroot.py root` prints: time, origin remote, top level path, branch, `sha` null, the six values and the block's digest. The hub's burndown draws it as overlays; no gate reads it. A failed append prints one stderr line and keeps the exit code; a linked council root or a link, FIFO or other non-regular file at the journal fails it without blocking, through `nofollow.py` of `kvb-org-organize`; when that -helper is missing or unusable, the line names it. A new council root gets mode +helper or `councilroot.py` is missing or unusable, the line names it. A new council root gets mode 0700, a new journal 0600. -The journal lives in `/tmp`, so a reboot or another PC starts empty. Refill +The journal survives a reboot, but another PC starts empty. Refill it from the blocks in git history, without a model: ```bash diff --git a/skills/kvb-check-score/scripts/backfill.py b/skills/kvb-check-score/scripts/backfill.py index 4bdacd8..8e1fe33 100755 --- a/skills/kvb-check-score/scripts/backfill.py +++ b/skills/kvb-check-score/scripts/backfill.py @@ -217,7 +217,11 @@ def main() -> int: except LookupError as error: print(f"backfill: {error}", file=sys.stderr) return 2 - path = journal_path() + try: + path = journal_path() + except OSError as error: + print(f"backfill: journal: {error}", file=sys.stderr) + return 2 try: keys = known(path) except (OSError, ValueError) as error: diff --git a/skills/kvb-check-score/scripts/journal.py b/skills/kvb-check-score/scripts/journal.py index cb45268..9035a1d 100644 --- a/skills/kvb-check-score/scripts/journal.py +++ b/skills/kvb-check-score/scripts/journal.py @@ -1,9 +1,10 @@ """Labels and score journal shared by score.py and backfill.py of kvb-check-score. Environment: - AGENTS_COUNCIL_ROOT: council directory shared by every agent on the host; - /tmp/agents-council when unset or empty. The journal is its scores.jsonl, - display data for the hub's burndown only, never a gate input. + AGENTS_COUNCIL_ROOT, XDG_STATE_HOME, HOME: the council directory the + kvb-org-organize councilroot.py resolves, shared by every agent on the + host. The journal is its scores.jsonl, display data for the hub's + burndown only, never a gate input. """ from __future__ import annotations @@ -11,7 +12,6 @@ from __future__ import annotations import hashlib import importlib.util import json -import os import re import subprocess from collections.abc import Callable @@ -20,9 +20,10 @@ from types import ModuleType from urllib.parse import urlsplit LABELS = Path(__file__).resolve().parents[1] / "labels.json" -NOFOLLOW = Path(__file__).resolve().parents[2] / "kvb-org-organize" / "scripts" / "nofollow.py" +ORGANIZE = Path(__file__).resolve().parents[2] / "kvb-org-organize" / "scripts" +NOFOLLOW = ORGANIZE / "nofollow.py" +COUNCIL_ROOT = ORGANIZE / "councilroot.py" JOURNAL = "scores.jsonl" -COUNCIL = "/tmp/agents-council" SCP = re.compile(r"(?:[^@/:]+@)?([^/:]+):(?!//)(.+)") FILE_MODE = 0o600 @@ -39,7 +40,11 @@ def load_labels(path: Path) -> dict: def journal_path() -> Path: - return Path(os.environ.get("AGENTS_COUNCIL_ROOT") or COUNCIL) / JOURNAL + """Returns: + scores.jsonl in the council root; OSError naming councilroot.py of + kvb-org-organize when it is missing or unusable. + """ + return helper(COUNCIL_ROOT, "kvb_score_councilroot", lambda loaded: loaded.root()) / JOURNAL def git(repo: Path | str, *args: str) -> str: @@ -130,13 +135,15 @@ def line(row: dict) -> str: return json.dumps(row, ensure_ascii=False) + "\n" -def unusable(error: Exception) -> OSError: - return OSError(f"helper {NOFOLLOW} unusable ({type(error).__name__}): install or update kvb-org-organize") +def unusable(path: Path, error: Exception) -> OSError: + return OSError(f"helper {path} unusable ({type(error).__name__}): install or update kvb-org-organize") -def nofollow[T](use: Callable[[ModuleType], T]) -> T: +def helper[T](path: Path, name: str, use: Callable[[ModuleType], T]) -> T: """Args: - use: what to do with nofollow.py of kvb-org-organize. + path: a script of kvb-org-organize. + name: the module name it loads under. + use: what to do with it. Returns: The result of use; its OSError unchanged, any other failure loading @@ -144,17 +151,27 @@ def nofollow[T](use: Callable[[ModuleType], T]) -> T: FileNotFoundError. """ try: - spec = importlib.util.spec_from_file_location("kvb_score_nofollow", NOFOLLOW) + spec = importlib.util.spec_from_file_location(name, path) loaded = importlib.util.module_from_spec(spec) spec.loader.exec_module(loaded) except Exception as error: # noqa: BLE001 - raise unusable(error) from None + raise unusable(path, error) from None try: return use(loaded) except OSError: raise except Exception as error: # noqa: BLE001 - raise unusable(error) from None + raise unusable(path, error) from None + + +def nofollow[T](use: Callable[[ModuleType], T]) -> T: + """Args: + use: what to do with nofollow.py of kvb-org-organize. + + Returns: + helper() of it. + """ + return helper(NOFOLLOW, "kvb_score_nofollow", use) def append(path: Path, rows: list[dict]) -> None: diff --git a/skills/kvb-check-score/scripts/score.py b/skills/kvb-check-score/scripts/score.py index af40373..e936d56 100755 --- a/skills/kvb-check-score/scripts/score.py +++ b/skills/kvb-check-score/scripts/score.py @@ -294,7 +294,11 @@ def journal(args: argparse.Namespace, total: int, block: str) -> None: "fix": plain(args.fix), "break": plain(args.risk), "tokens": plain(args.efficiency), "velocity": plain(args.speed), "security": plain(args.security[0]), "score": total, } - path = journal_path() + try: + path = journal_path() + except OSError as error: + print(f"score: journal: {error}", file=sys.stderr) + return try: here = Path.cwd() append(path, [entry(int(time.time()), origin(here), git_read(here, "branch", "--show-current") or None, None, values, block)]) diff --git a/skills/kvb-cleanup/SKILL.md b/skills/kvb-cleanup/SKILL.md index 9051c98..ca754cd 100644 --- a/skills/kvb-cleanup/SKILL.md +++ b/skills/kvb-cleanup/SKILL.md @@ -32,6 +32,7 @@ rules forbidding raw commands. joins Step 4) and `worktrees.md` next to this file when present. - **Agent dirs**: pip, npm, pre-commit, act, gh, buildx caches; Ansible and build temp dirs. +- **Council root** (`~/.local/state/agents-shared`): `archive/` only. - **Register**: `council.py list --long`; `rasci.py artifacts` (`kvb-org-artefacts`, `--ended` for candidates). - **Queue**: `${CLAUDE_SKILL_DIR}/../kvb-org-queue-cleanup/scripts/cleanup.py diff --git a/skills/kvb-git-diff/SKILL.md b/skills/kvb-git-diff/SKILL.md index d48eb4d..c99e4df 100644 --- a/skills/kvb-git-diff/SKILL.md +++ b/skills/kvb-git-diff/SKILL.md @@ -9,11 +9,14 @@ approval is the operator's word for one commit, the exact tree it names, only once the operator's pinned passkey signature over it verifies here. 1. Render the page. `` is the repository's name, `` the - branch with every `/` as `-`, `` the branch's base and `` a - file under `/tmp` holding the kvb-check-score commit table: + branch with every `/` as `-`, `` the branch's base, `
` a + file under `/tmp` holding the kvb-check-score commit table, `` + and `` the directories + `${CLAUDE_SKILL_DIR}/../kvb-org-organize/scripts/councilroot.py dir pages` + and `... dir approvals` print, in the council root the hub reads: ```bash - pkgmgr review --out /tmp/kvb-git-diff/-.html --max-file-lines 400 --base --note
+ pkgmgr review --out /-.html --max-file-lines 400 --base --note
``` 2. Request the review in the worktree; it ships the staged diff, every file @@ -23,7 +26,7 @@ once the operator's pinned passkey signature over it verifies here. `` your agent name, which inline feedback is posted to: ```bash - python3 ~/.local/share/kvb-orchestrator/client/approval.py request --port 8737 --title "" --page /tmp/kvb-git-diff/-.html --notes
--gate --score "" --agent "" + python3 ~/.local/share/kvb-orchestrator/client/approval.py request --port 8737 --title "" --page /-.html --notes
--gate --score "" --agent "" ``` Only a `pass` review can be approved in bulk or by a rule. Tell the @@ -37,7 +40,7 @@ once the operator's pinned passkey signature over it verifies here. first, `` being the `gate` from step 2: ```bash - python3 ~/.local/share/kvb-orchestrator/client/approval.py verify --port 8737 --approvers ~/.config/kvb-orchestrator/approvers.json --replay /tmp/kvb-orchestrator-approvals/commit.json --review --gate + python3 ~/.local/share/kvb-orchestrator/client/approval.py verify --port 8737 --approvers ~/.config/kvb-orchestrator/approvers.json --replay /commit.json --review --gate ``` Exit 0 only: commit as the calling skill says. Any other exit, diff --git a/skills/kvb-git-push/SKILL.md b/skills/kvb-git-push/SKILL.md index 77ecfe2..91bcddf 100644 --- a/skills/kvb-git-push/SKILL.md +++ b/skills/kvb-git-push/SKILL.md @@ -26,11 +26,14 @@ hands over, `kvb-git-pr-create`, `kvb-git-update-branches` and output streamed to a log; give the operator `tail -f `: ```bash - queue.py hold --kind --task -- python3 hold --repo -- make 2>&1 | tee /tmp/kvb-git-push--.log + queue.py hold --kind --task -- python3 hold --repo -- make 2>&1 | tee /kvb-git-push--.log ``` `` is `${CLAUDE_SKILL_DIR}/../kvb-check-gate/scripts/gate.py`; - its worktree lock keeps a sync out while the gate runs. + its worktree lock keeps a sync out while the gate runs. `` is the + directory + `${CLAUDE_SKILL_DIR}/../kvb-org-organize/scripts/councilroot.py dir gates` + prints, so the evidence survives a reboot. 5. **Judge** by the gate's own result tables and its failure line (infinito-nexus: the `📊 per-target wall-clock` table and `FAILED TARGETS:`), never by the exit code of a pipe. Green also needs HEAD and diff --git a/skills/kvb-org-organize/SKILL.md b/skills/kvb-org-organize/SKILL.md index f4cf759..ccebd5e 100644 --- a/skills/kvb-org-organize/SKILL.md +++ b/skills/kvb-org-organize/SKILL.md @@ -14,7 +14,7 @@ outranks them and appoints or recalls at will. ## Election `${CLAUDE_SKILL_DIR}/scripts/council.py` keeps atomic claims in -`/tmp/agents-council`; other runtimes add `--session `. +`$(${CLAUDE_SKILL_DIR}/scripts/councilroot.py root)`; other runtimes add `--session `. 1. Name the office: `work/` (a piece of work), `cluster/` (a strand across lines), `/` (a line) or `chair`, the slug diff --git a/skills/kvb-org-organize/scripts/council.py b/skills/kvb-org-organize/scripts/council.py index 84956cb..1e248e8 100755 --- a/skills/kvb-org-organize/scripts/council.py +++ b/skills/kvb-org-organize/scripts/council.py @@ -2,8 +2,8 @@ """Claim, release and list kvb-org-organize offices in the shared council directory. Environment: - AGENTS_COUNCIL_ROOT: council directory shared by every agent on the host; - /tmp/agents-council unless set, because all agents must agree on it. + AGENTS_COUNCIL_ROOT, XDG_STATE_HOME, HOME: the council directory + councilroot.py resolves, shared by every agent on the host. CLAUDE_CODE_SESSION_ID: the caller's session, taken over --session; other runtimes pass --session. PATH: filtered to the directories hookpath.py trusts before git runs. @@ -44,7 +44,7 @@ def sibling(name: str) -> ModuleType: nofollow = sibling("nofollow") -ROOT = Path(os.environ.get("AGENTS_COUNCIL_ROOT", "/tmp/agents-council")) +ROOT = sibling("councilroot").root() HEARTBEAT_LIMIT = 3600 OFFICE = re.compile(r"^[a-z0-9][a-z0-9._-]*(/[a-z0-9][a-z0-9._-]*)*$") CLAIM = "claim.json" diff --git a/skills/kvb-org-organize/scripts/councilroot.py b/skills/kvb-org-organize/scripts/councilroot.py new file mode 100755 index 0000000..e3625be --- /dev/null +++ b/skills/kvb-org-organize/scripts/councilroot.py @@ -0,0 +1,851 @@ +#!/usr/bin/env python3 +"""Usage: + councilroot.py root + councilroot.py durable + councilroot.py dir PART [PART ...] + councilroot.py release VERSION + councilroot.py migrate [--legacy DIR] [--catch-up] + councilroot.py preflight [--legacy DIR] [--settings FILE] [--managed DIR] + councilroot.py retire [--days N] + +Environment: + AGENTS_COUNCIL_ROOT: the council root when it is an absolute path. + XDG_STATE_HOME: else both roots lie below it, when it is an absolute path. + HOME: else both lie below ~/.local/state. + AGENTS_COUNCIL_LEGACY: the default DIR of migrate and preflight, LEGACY + unless set. + +Args: + root: print the council root. + durable: print the host-only root; AGENTS_COUNCIL_ROOT never moves it. + dir: create /PART/... with mode 700 where missing, refusing a + level that is a link, not the current user's or writable by group or + others, and print it: + pages holds the kvb-git-diff review pages the hub reads, approvals + the replay stores of approval.py verify, gates the gate logs. + release: create agents-release/VERSION beside the roots with mode 700 + and print it: the kvb-release drafts, which the release skills write + with the Edit and Write tools. + migrate: move every regular file below DIR, LEGACY unless given, to the + same place below the root and remove the emptied directories, through + nofollow.py, so no link on either side is followed; each copy is + written to a temp file and linked into place, so it never replaces a + file. Refuses, moving nothing, while a *.lock below DIR is held or + its queue.json holds a job, and, under those locks and unless + --catch-up is given, while a regular file cannot be opened or its + place below the root is reached through a link (`unmoved : + `) or a whole file the root holds differs (`conflict + differs from `). A file the root already holds is never + overwritten: an identical one leaves only the root's copy; a .jsonl + journal gets the legacy rows it lacks, by id when every row on both + sides carries one, else each exact line as often as the legacy side + holds it more often, appended or, when every row carries a ts, in ts + order; with --catch-up a differing whole file is copied beside its + namesake as .legacy-<12 hex digits of its sha256> (`kept + beside as `). Each file is first renamed + to .<16 hex digits>.migrating and copied from there, so a + writer that opens the old name meanwhile makes a new file for the + catch-up; a later run moves a staged file to its old name. A link or + another non-regular entry stays (`skipped : `); a file + that fails after the check stays (`unmoved : `). + /migrate.started exists from the first move to the end of + the run, so after a crash it tells the install the move stopped + halfway; a completed run removes it. A second run moves nothing new + and duplicates no journal row. Run it while no agent, unit or queue + job writes either root; the install runs it again with --catch-up + after the copy for what the old skills wrote meanwhile. + preflight: print to stderr every reason migrate would refuse, and every + missing or wrong sandbox.filesystem.allowWrite entry, all files only + read: the root and agents-release must be in the user file + (--settings, ~/.claude/settings.json unless given); no entry of it, + of the managed and user files settings_files() lists (--managed DIR, + /etc/claude-code unless given, with its managed-settings.d drop-ins; + the queue's permission check reads the same list), of a + .claude/settings.json or .claude/settings.local.json up to 8 levels + below the pkgmgr repositories and worktrees directories, directory + links followed, or in the start directory of a session listed in + ~/.claude/projects may be the host-only root, inside it or above it. + It warns, without failing, while the agents are not suspended. The + global install runs it before it copies anything. + retire: move every regular file below /gates and /pages + changed more than N days ago (30 unless given) to the same place + below /archive/, which no reader opens, never following + a link, and print the count. + +Exit codes: + 0 done, 2 usage error, a refused preflight, a busy legacy root, a root + that cannot be created, or a legacy or council root that is a link, not + a directory of the current user, writable by group or others, or inside + the other, 3 a whole file migrate kept in both roots or beside its + namesake, 4 nothing moved for the unmoved and conflict lines printed, 5 + a partial move: a file that failed after the check stays in DIR; any + other code is a crash. +""" + +from __future__ import annotations + +import argparse +import contextlib +import fcntl +import hashlib +import importlib.util +import json +import os +import re +import shutil +import stat +import sys +import time +from collections import Counter +from collections.abc import Sequence +from pathlib import Path +from types import ModuleType + +NAME = "agents-council" +SHARED = "agents-shared" +RELEASES = "agents-release" +QUEUE = "queue.json" +LOCK_SUFFIX = ".lock" +ARCHIVE = "archive" +RETIRED = ("gates", "pages") +RETIRE_DAYS = 30 +DAY_SECONDS = 86400 +LEGACY = Path("/tmp/agents-council") +DIRECTORY_MODE = 0o700 +PART = re.compile(r"[A-Za-z0-9][A-Za-z0-9._+-]*") +JOURNAL_SUFFIX = ".jsonl" +FILE_BITS = 0o755 +MANAGED = Path("/etc/claude-code") +MARKER = "migrate.started" +STAGED = re.compile(r"\.[0-9a-f]{16}\.migrating\Z") +PROJECT_FILES = ("settings.json", "settings.local.json") +PROJECT_DEPTH = 8 +PRUNED = frozenset({".git", "node_modules", ".venv", "venv", "__pycache__", ".tox", ".cache", ".mypy_cache", ".pytest_cache", ".ruff_cache", "site-packages"}) +PKGMGR_DIRECTORIES = {"repositories": "~/Repositories/", "worktrees": "~/.local/share/worktrees/"} +NIGHT = "night.json" +SUSPENDED = ("suspend", "sunset") +LOCK_WAIT = 30 +ENCODED = re.compile(r"[^A-Za-z0-9]") + + +def state_home() -> Path: + state = os.environ.get("XDG_STATE_HOME", "") + return Path(state) if os.path.isabs(state) else Path.home() / ".local" / "state" + + +def durable() -> Path: + return state_home() / NAME + + +def root() -> Path: + explicit = os.environ.get("AGENTS_COUNCIL_ROOT", "") + return Path(explicit) if os.path.isabs(explicit) else state_home() / SHARED + + +def directory(base: Path, *parts: str) -> Path: + """Args: + base: the directory the parts lie below; its missing parents are + created with the default mode. + parts: path segments, each matching PART. + + Returns: + base/parts, each level from base down created with mode 700 where + missing; ValueError for a part that does not match PART or a level + that refusal() names: a link, not a directory of the current user, + or writable by group or others. + """ + bad = [part for part in parts if not PART.fullmatch(part)] + if bad: + raise ValueError(f"not a directory name: {bad[0]!r}") + base.parent.mkdir(parents=True, exist_ok=True) + levels = [base] + for part in parts: + levels.append(levels[-1] / part) + for level in levels: + with contextlib.suppress(FileExistsError): + level.mkdir(mode=DIRECTORY_MODE) + problem = refusal(level) + if problem is not None: + raise ValueError(problem) + return levels[-1] + + +def nofollow() -> ModuleType: + spec = importlib.util.spec_from_file_location("kvb_councilroot_nofollow", Path(__file__).resolve().parent / "nofollow.py") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def refusal(path: Path) -> str | None: + info = path.lstat() + if stat.S_ISDIR(info.st_mode) and info.st_uid == os.getuid() and not info.st_mode & 0o022: + return None + return f"{path} must be a directory of uid {os.getuid()} that group and others cannot write" + + +def row_ids(text: str) -> list[tuple[str, str]] | None: + """Returns: + (id key, line) per non-empty line, the id serialised as JSON; None + when a line is no JSON object with an id. + """ + rows = [] + for line in text.splitlines(): + if not line.strip(): + continue + try: + row = json.loads(line) + except (ValueError, RecursionError): + return None + if not isinstance(row, dict) or "id" not in row: + return None + rows.append((json.dumps(row["id"], sort_keys=True), line)) + return rows + + +def copied(helper: ModuleType, legacy: Path, source: Path, target_root: Path, target: Path) -> bool: + """Returns: + True once source is copied, with its mode and times, to a temp file + that is then linked to the name target; False when target exists, + which stays untouched. + """ + *dirs, name = helper.parts_below(target_root, target) + with open(helper.open_below(legacy, source, os.O_RDONLY, 0), "rb") as handle: + info = os.fstat(handle.fileno()) + folder = helper.directory(target_root, tuple(dirs), True) + temp = f".{name}.{os.urandom(8).hex()}" + try: + with open(os.open(temp, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW | os.O_CLOEXEC, 0o600, dir_fd=folder), "wb") as out: + shutil.copyfileobj(handle, out) + out.flush() + os.fchmod(out.fileno(), info.st_mode & FILE_BITS) + os.utime(out.fileno(), ns=(info.st_atime_ns, info.st_mtime_ns)) + os.fsync(out.fileno()) + try: + os.link(temp, name, src_dir_fd=folder, dst_dir_fd=folder, follow_symlinks=False) + except FileExistsError: + return False + return True + finally: + with contextlib.suppress(OSError): + os.unlink(temp, dir_fd=folder) + os.close(folder) + + +def content(helper: ModuleType, base: Path, path: Path) -> bytes: + with open(helper.open_below(base, path, os.O_RDONLY, 0), "rb") as handle: + return handle.read() + + +def novel(text: str, legacy_text: str) -> tuple[list[str], bool]: + """Returns: + The legacy lines text lacks and True when both are id journals: each + id once; else each exact line as often as legacy_text holds it more + often than text, and False. + """ + ours, theirs = row_ids(text), row_ids(legacy_text) + if ours is not None and theirs is not None: + known, fresh = {key for key, _ in ours}, [] + for key, line in theirs: + if key not in known: + known.add(key) + fresh.append(line) + return fresh, True + have, fresh = Counter(text.splitlines()), [] + for line in legacy_text.splitlines(): + if not line.strip(): + continue + if have[line]: + have[line] -= 1 + else: + fresh.append(line) + return fresh, False + + +def in_ts_order(lines: list[str]) -> list[str] | None: + """Returns: + lines stably sorted by their numeric ts; None when a line is no JSON + object with one. + """ + stamps = [] + for line in lines: + try: + row = json.loads(line) + except (ValueError, RecursionError): + return None + stamp = row.get("ts") if isinstance(row, dict) else None + if isinstance(stamp, bool) or not isinstance(stamp, int | float): + return None + stamps.append(stamp) + return [line for _, line in sorted(zip(stamps, lines, strict=True), key=lambda pair: pair[0])] + + +def merged(helper: ModuleType, legacy: Path, source: Path, target_root: Path, target: Path) -> int: + """Returns: + The number of novel() legacy rows target gained, under the flock of + target and of its .lock when one exists (the radio's lock): + appended, or for a journal compared by line whose rows all carry a + ts and would fall out of order, the whole file replaced in ts order. + """ + legacy_text = helper.read_text(legacy, source) + lock = target.with_suffix(LOCK_SUFFIX) + with contextlib.ExitStack() as stack: + if lock.is_file() and not lock.is_symlink(): + stack.enter_context(helper.locked(target_root, lock, 0o600, LOCK_WAIT)) + handle = stack.enter_context(open(helper.open_below(target_root, target, os.O_RDWR | os.O_APPEND, 0), "a+", encoding="utf-8", errors="replace")) + handle.seek(0) + fcntl.flock(handle, fcntl.LOCK_EX) + text = handle.read() + fresh, by_id = novel(text, legacy_text) + lines = [line for line in text.splitlines() if line.strip()] + fresh + order = None if by_id or not fresh else in_ts_order(lines) + if order is None or order == lines: + handle.write(("" if not fresh or not text or text.endswith("\n") else "\n") + "".join(f"{line}\n" for line in fresh)) + else: + helper.replace(target_root, target, "".join(f"{line}\n" for line in order), os.fstat(handle.fileno()).st_mode & FILE_BITS) + return len(fresh) + + +def beside(helper: ModuleType, legacy: Path, source: Path, target_root: Path, target: Path) -> Path: + """Returns: + target.legacy-, a copy + of source beside target unless that name already holds it. + """ + kept = target.with_name(f"{target.name}.legacy-{hashlib.sha256(content(helper, legacy, source)).hexdigest()[:12]}") + copied(helper, legacy, source, target_root, kept) + return kept + + +def destination(legacy: Path, source: Path, target_root: Path) -> Path: + """Returns: + The place of source below target_root, a staged() suffix dropped. + """ + relative = source.relative_to(legacy) + return target_root / relative.with_name(STAGED.sub("", relative.name)) + + +def staged(helper: ModuleType, legacy: Path, source: Path) -> Path: + """Returns: + source renamed in its directory to .<16 hex digits>.migrating, + so a writer that opens the old name afterwards creates a new file the + catch-up moves; source itself when it already carries such a name. + """ + if STAGED.search(source.name): + return source + *dirs, name = helper.parts_below(legacy, source) + stage = f"{name}.{os.urandom(8).hex()}.migrating" + fd = helper.directory(legacy, tuple(dirs), False) + try: + os.rename(name, stage, src_dir_fd=fd, dst_dir_fd=fd) + finally: + os.close(fd) + return source.with_name(stage) + + +def settle(helper: ModuleType, legacy: Path, source: Path, target_root: Path, catch_up: bool) -> tuple[str, str | None]: + """Args: + helper: nofollow.py. + legacy: the earlier council root. + source: a regular file below it. + target_root: the council root. + catch_up: keep a differing whole file beside its namesake instead of + leaving it in legacy. + + Returns: + (moved, merged, kept, conflict or unmoved, report line), the line + None for a plain move or an identical copy; every copy is made from + the staged() file, which a line naming a file left in legacy names. + """ + target, stage = destination(legacy, source, target_root), source + try: + stage = staged(helper, legacy, source) + if copied(helper, legacy, stage, target_root, target) or content(helper, legacy, stage) == content(helper, target_root, target): + helper.unlink(legacy, stage) + return "moved", None + if target.suffix == JOURNAL_SUFFIX: + count = merged(helper, legacy, stage, target_root, target) + helper.unlink(legacy, stage) + return "merged", f"merged {count} rows of {source} into {target}" + if catch_up: + kept = beside(helper, legacy, stage, target_root, target) + helper.unlink(legacy, stage) + return "kept", f"kept {source} beside {target} as {kept}" + return "conflict", f"conflict {target} differs from {stage}; both kept" + except OSError as error: + return "unmoved", f"unmoved {stage}: {error.strerror or error}" + + +def held(helper: ModuleType, legacy: Path, stack: contextlib.ExitStack) -> list[str]: + """Args: + helper: nofollow.py. + legacy: the earlier council root. + stack: keeps every lock this takes until the move ends. + + Returns: + Why the move must wait: each *.lock below legacy another process + holds, and a queue.json with jobs or that cannot be read. + """ + problems = [] + for folder, _, files in os.walk(legacy): + for name in files: + path = Path(folder) / name + if not name.endswith(LOCK_SUFFIX) or not stat.S_ISREG(path.lstat().st_mode): + continue + try: + lock = helper.open_below(legacy, path, os.O_RDONLY, 0) + stack.callback(os.close, lock) + fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) + except BlockingIOError: + problems.append(f"{path} is held") + except OSError as error: + problems.append(f"{path} cannot be locked: {error.strerror or error}") + queue = legacy / QUEUE + if queue.exists() or queue.is_symlink(): + try: + entries = json.loads(helper.read_text(legacy, queue)).get("entries") + except (OSError, ValueError, AttributeError): + entries = None + if not isinstance(entries, list): + problems.append(f"{queue} is unreadable") + elif entries: + problems.append(f"{queue} holds {len(entries)} queue jobs") + return problems + + +def prune(helper: ModuleType, legacy: Path) -> None: + for folder, _, _ in os.walk(legacy, topdown=False): + here = Path(folder) + if here == legacy: + continue + *dirs, name = helper.parts_below(legacy, here) + with contextlib.suppress(OSError): + fd = helper.directory(legacy, tuple(dirs), False) + try: + os.rmdir(name, dir_fd=fd) + finally: + os.close(fd) + with contextlib.suppress(OSError): + os.rmdir(legacy) + + +class Blocked(Exception): + """Args: + lines: the conflicts() lines that kept migrate from moving anything. + """ + + def __init__(self, lines: list[str]) -> None: + super().__init__("; ".join(lines)) + self.lines = lines + + +def migrate(legacy: Path, target: Path, catch_up: bool = False) -> list[str]: + """Args: + legacy: the earlier council root. + target: the council root. + catch_up: move what conflicts() names too, keeping a differing whole + file beside its namesake, instead of refusing. + + Returns: + The report lines, the moved count first, the rest sorted; ValueError + naming the refused root when either is a link, not a directory of the + current user, writable by group or others, or lies inside the other, + and naming every reason held() finds, moving nothing; without + catch_up, Blocked naming every conflicts() line found under the + held locks, moving nothing. + """ + helper = nofollow() + if not legacy.exists() and not legacy.is_symlink(): + unmark(helper) + return [] + problem = refusal(legacy) + if problem is not None: + raise ValueError(problem) + if legacy.resolve() == target.resolve(): + return [] + with contextlib.ExitStack() as stack: + problems = held(helper, legacy, stack) + if problems: + raise ValueError("; ".join(problems) + "; nothing moved") + blocked = [] if catch_up else conflicts(helper, legacy, target) + if blocked: + raise Blocked(blocked) + return move(legacy, target, catch_up) + + +def move(legacy: Path, target: Path, catch_up: bool) -> list[str]: + target.mkdir(mode=DIRECTORY_MODE, parents=True, exist_ok=True) + problem = refusal(target) + if problem is not None: + raise ValueError(problem) + old, new = legacy.resolve(), target.resolve() + if old.is_relative_to(new) or new.is_relative_to(old): + raise ValueError(f"{legacy} and {target} must not lie inside each other") + helper = nofollow() + host = directory(durable()) + helper.replace(host, host / MARKER, f"{legacy} -> {target}\n", 0o600) + moved, lines = 0, [] + for folder, folders, files in os.walk(legacy): + here = Path(folder) + lines += [f"skipped {here / name}: a link" for name in folders if (here / name).is_symlink()] + for name in sorted(files): + source = here / name + try: + regular = stat.S_ISREG(source.lstat().st_mode) + except OSError: + continue + if not regular: + lines.append(f"skipped {source}: not a regular file") + continue + outcome, line = settle(helper, legacy, source, target, catch_up) + moved += outcome == "moved" + if line is not None: + lines.append(line) + prune(helper, legacy) + unmark(helper) + head = [f"moved {moved} files from {legacy} to {target}"] if moved else [] + return head + sorted(lines) + + +def unmark(helper: ModuleType) -> None: + with contextlib.suppress(FileNotFoundError): + helper.unlink(durable(), durable() / MARKER) + + +def unmovable(helper: ModuleType, legacy: Path, source: Path, target: Path, twin: Path) -> str | None: + """Returns: + The `unmoved` line migrate would print because source cannot be + opened as it opens it, or a directory on the way to twin is a link + or no directory; None when nothing stops the copy. + """ + try: + os.close(helper.open_below(legacy, source, os.O_RDONLY, 0)) + *dirs, _ = helper.parts_below(target, twin) + with contextlib.suppress(FileNotFoundError): + os.close(helper.directory(target, tuple(dirs), False)) + except OSError as error: + return f"unmoved {source}: {error.strerror or error}" + return None + + +def conflicts(helper: ModuleType, legacy: Path, target: Path) -> list[str]: + """Returns: + A line per regular file below legacy that migrate could not move, or + that is no .jsonl journal and whose namesake below target differs. + """ + lines = [] + for folder, _, files in os.walk(legacy): + for name in files: + source = Path(folder) / name + twin = destination(legacy, source, target) + if not stat.S_ISREG(source.lstat().st_mode): + continue + stuck = unmovable(helper, legacy, source, target, twin) + if stuck is not None: + lines.append(stuck) + continue + if not (twin.exists() or twin.is_symlink()): + continue + try: + ours, theirs = content(helper, legacy, source), content(helper, target, twin) + except OSError as error: + lines.append(f"unmoved {source}: {error.strerror or error}") + continue + if ours != theirs and source.suffix != JOURNAL_SUFFIX: + lines.append(f"conflict {twin} differs from {source}") + return lines + + +def written(entry: object, base: Path) -> Path | None: + """Args: + entry: one allowWrite entry. + base: the directory a relative entry lies below. + + Returns: + The normalised path the entry names, a trailing /** dropped; None for + anything but a string. + """ + if not isinstance(entry, str): + return None + text = entry.removesuffix("/**").rstrip("/") or "/" + path = Path(os.path.expanduser(text[1:] if text.startswith("//") else text)) + return Path(os.path.normpath(base / path)) + + +def allowed(settings: Path, base: Path) -> list[tuple[str, Path]] | None: + """Returns: + (entry, path) per sandbox.filesystem.allowWrite entry of settings, + relative ones below base; None when it holds no such list. + """ + try: + entries = json.loads(settings.read_text(encoding="utf-8"))["sandbox"]["filesystem"]["allowWrite"] + except (OSError, ValueError, KeyError, TypeError): + return None + if not isinstance(entries, list): + return None + return [(entry, path) for entry in entries if (path := written(entry, base)) is not None] + + +def pkgmgr_roots() -> list[Path]: + """Returns: + The repositories and worktrees directories of the directories block + of ~/.config/pkgmgr/config.yaml, pkgmgr's defaults where it names + none. + """ + found = dict(PKGMGR_DIRECTORIES) + inside = False + with contextlib.suppress(OSError): + for line in (Path.home() / ".config" / "pkgmgr" / "config.yaml").read_text(encoding="utf-8").splitlines(): + text = line.split("#", 1)[0] + if text[:1].strip(): + inside = text.strip() == "directories:" + continue + key, _, value = text.strip().partition(":") + if inside and key in found and value.strip(): + found[key] = value.strip().strip("'\"") + return [Path(os.path.expanduser(value)) for value in found.values()] + + +def settings_files(project: Path | None, managed: Path | None = None) -> list[Path]: + """Args: + project: the start directory of a session; None for only the managed + and user files. + managed: the managed settings directory, MANAGED unless given. + + Returns: + Every Claude Code settings file that can apply there: + managed-settings.json and managed-settings.d/*.json in name order, + ~/.claude/settings.json and settings.local.json, then the project's + .claude/settings.json and settings.local.json. + """ + managed = managed or MANAGED + files = [managed / "managed-settings.json", *sorted((managed / "managed-settings.d").glob("*.json"))] + files += [Path.home() / ".claude" / name for name in PROJECT_FILES] + return files + ([] if project is None else [project / ".claude" / name for name in PROJECT_FILES]) + + +def settings_in(folder: Path) -> list[Path]: + """Returns: + folder/.claude/settings.json and settings.local.json where they are + files, read through a link like Claude Code does. + """ + return [path for name in PROJECT_FILES if (path := folder / ".claude" / name).is_file()] + + +def project_settings(top: Path) -> list[Path]: + """Returns: + The settings_in() of every directory at most PROJECT_DEPTH levels + below top, in walk order, following directory links but entering each + real directory once and never one of PRUNED. + """ + found, seen = [], set() + for folder, folders, _ in os.walk(top, followlinks=True): + here = Path(folder) + real = os.path.realpath(here) + if real in seen: + folders[:] = [] + continue + seen.add(real) + depth = len(here.relative_to(top).parts) + folders[:] = sorted(name for name in folders if name not in PRUNED) if depth < PROJECT_DEPTH else [] + found += settings_in(here) + return found + + +def decoded(folder: Path, rest: str) -> list[Path]: + """Returns: + Every directory below folder whose path after folder encodes to rest + the way Claude Code names a project: each character but an ASCII + letter or digit turned into '-'. + """ + if not rest: + return [folder] + try: + with os.scandir(folder) as entries: + children = sorted((entry.name, entry.path) for entry in entries) + except OSError: + return [] + found = [] + for name, path in children: + mark = "-" + ENCODED.sub("-", name) + if rest.startswith(mark) and rest[len(mark):][:1] in ("", "-") and os.path.isdir(path): + found += decoded(Path(path), rest[len(mark):]) + return found + + +def started(projects: Path) -> list[Path]: + """Returns: + The decoded() start directory of every Claude Code session folder + below projects that still exists. + """ + try: + with os.scandir(projects) as entries: + names = sorted(entry.name for entry in entries if entry.is_dir(follow_symlinks=False)) + except OSError: + return [] + return [folder for name in names for folder in decoded(Path("/"), name)] + + +def sandbox_problems(settings: Path, managed: Path) -> list[str]: + """Args: + settings: the user settings file. + managed: the managed settings directory. + + Returns: + An add line per agent path the user file's allowWrite lacks, and a + drop line per allowWrite entry that is the host-only root, inside it + or above it, of settings, of a settings_files() file, of a + project_settings() file below pkgmgr_roots(), or of a settings_in() + file of a started() session directory; each real file once. + """ + release, host = state_home() / RELEASES, durable() + user = allowed(settings, settings.parent) + if user is None: + problems = [f"{settings} holds no sandbox.filesystem.allowWrite list; the operator adds {root()} and {release} there"] + else: + paths = {path for _, path in user} + problems = [f"add {path} to sandbox.filesystem.allowWrite in {settings} (operator step)" for path in (root(), release) if path not in paths] + files = [(path, path.parent) for path in (*settings_files(None, managed), settings)] + files += [(path, path.parent.parent) for top in pkgmgr_roots() for path in project_settings(top)] + files += [(path, path.parent.parent) for folder in started(Path.home() / ".claude" / "projects") for path in settings_in(folder)] + unique: dict[tuple[str, Path], tuple[Path, Path]] = {} + for file, base in files: + unique.setdefault((os.path.realpath(file), base.resolve()), (file, base)) + for file, base in unique.values(): + problems += [ + f"{file}: drop allowWrite entry {entry!r} ({path}): the sandbox must never write the host-only {host}" + for entry, path in allowed(file, base) or [] if path == host or path in host.parents or host in path.parents + ] + return problems + + +def suspended(legacy: Path) -> bool: + """Returns: + True when night.json exists in legacy or the council root and every + one there is in a suspend mode. + """ + helper, modes = nofollow(), [] + for base in (legacy, root()): + with contextlib.suppress(OSError, ValueError, AttributeError): + modes.append(json.loads(helper.read_text(base, base / NIGHT)).get("mode")) + return bool(modes) and all(mode in SUSPENDED for mode in modes) + + +def preflight(settings: Path, managed: Path, legacy: Path) -> list[str]: + """Args: + settings: the user settings file, only read. + managed: the managed settings directory, only read. + legacy: the earlier council root. + + Returns: + Every reason the install must stop before it copies anything: the + sandbox_problems(), a refused council root, and for an existing + legacy root its refusal, held() and conflicts(). + """ + problems = sandbox_problems(settings, managed) + target = root() + if target.exists() or target.is_symlink(): + problems += [problem for problem in (refusal(target),) if problem is not None] + if not legacy.exists() and not legacy.is_symlink(): + return problems + problem = refusal(legacy) + if problem is not None: + return [*problems, problem] + helper = nofollow() + with contextlib.ExitStack() as stack: + problems += held(helper, legacy, stack) + return problems + conflicts(helper, legacy, target) + + +def retire(base: Path, days: int, now: float) -> tuple[Path, int]: + """Args: + base: the council root. + days: files changed longer ago than this move. + now: the current epoch time. + + Returns: + The dated archive directory and the number of files moved there. + """ + helper = nofollow() + archive = base / ARCHIVE / time.strftime("%Y-%m-%d", time.localtime(now)) + cutoff, count = now - days * DAY_SECONDS, 0 + for part in RETIRED: + top = base / part + if top.is_symlink() or not top.is_dir(): + continue + for folder, _, files in os.walk(top): + for name in files: + path = Path(folder) / name + info = path.lstat() + if not stat.S_ISREG(info.st_mode) or info.st_mtime > cutoff: + continue + *dirs, _ = helper.parts_below(base, path) + source = helper.directory(base, tuple(dirs), False) + try: + target = helper.directory(base, (*helper.parts_below(base, archive), *dirs), True) + try: + os.rename(name, name, src_dir_fd=source, dst_dir_fd=target) + finally: + os.close(target) + finally: + os.close(source) + count += 1 + return archive, count + + +def legacy_default() -> Path: + return Path(os.environ.get("AGENTS_COUNCIL_LEGACY") or LEGACY) + + +def main(argv: Sequence[str] | None = None) -> int: + parser = argparse.ArgumentParser(prog="councilroot.py", description=__doc__.splitlines()[0]) + commands = parser.add_subparsers(dest="command", required=True) + commands.add_parser("root") + commands.add_parser("durable") + commands.add_parser("dir").add_argument("parts", nargs="+") + commands.add_parser("release").add_argument("version") + moving = commands.add_parser("migrate") + moving.add_argument("--legacy", type=Path, default=legacy_default()) + moving.add_argument("--catch-up", action="store_true") + check = commands.add_parser("preflight") + check.add_argument("--legacy", type=Path, default=legacy_default()) + check.add_argument("--settings", type=Path, default=Path.home() / ".claude" / "settings.json") + check.add_argument("--managed", type=Path, default=MANAGED) + commands.add_parser("retire").add_argument("--days", type=int, default=RETIRE_DAYS) + args = parser.parse_args(argv) + try: + if args.command == "root": + print(root()) + elif args.command == "durable": + print(durable()) + elif args.command == "dir": + print(directory(root(), *args.parts)) + elif args.command == "release": + print(directory(state_home() / RELEASES, args.version)) + elif args.command == "preflight": + problems = preflight(args.settings, args.managed, args.legacy) + for problem in problems: + print(f"councilroot: preflight: {problem}", file=sys.stderr) + if not suspended(args.legacy): + print("councilroot: preflight: warning: the agents are not suspended (kvb-org-suspend); the install moves what they write meanwhile in a second migrate after the copy", file=sys.stderr) + return 2 if problems else 0 + elif args.command == "retire": + archive, count = retire(root(), args.days, time.time()) + print(f"archived {count} files into {archive}") + else: + try: + report = migrate(args.legacy, root(), args.catch_up) + except Blocked as blocked: + print("\n".join([*blocked.lines, f"nothing moved from {args.legacy}"])) + return 4 + print("\n".join(report) or f"nothing to move from {args.legacy}") + for code, kinds in ((5, ("unmoved ",)), (3, ("conflict ", "kept "))): + if any(line.startswith(kinds) for line in report): + return code + except (OSError, ValueError) as error: + print(f"councilroot: {error}", file=sys.stderr) + return 2 + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/skills/kvb-org-organize/scripts/nofollow.py b/skills/kvb-org-organize/scripts/nofollow.py index 848a309..0789955 100644 --- a/skills/kvb-org-organize/scripts/nofollow.py +++ b/skills/kvb-org-organize/scripts/nofollow.py @@ -86,6 +86,25 @@ def read_text(root: Path, path: Path) -> str: return data.decode("utf-8", errors="replace") +def read_tail(root: Path, path: Path, limit: int) -> str: + """Args: + root: the trusted directory. + path: the file, below root. + limit: bytes to read at most; a longer file yields its last limit + bytes after the first line break in them, so only whole lines. + + Returns: + That text, UTF-8 decoded, undecodable bytes replaced. + """ + with open(open_below(root, path, os.O_RDONLY, 0), "rb") as handle: + size = os.fstat(handle.fileno()).st_size + if size <= limit: + return handle.read(limit).decode("utf-8", errors="replace") + handle.seek(size - limit) + data = handle.read(limit) + return data[data.find(b"\n") + 1:].decode("utf-8", errors="replace") + + @contextlib.contextmanager def locked(root: Path, path: Path, mode: int, wait: float | None) -> Iterator[TextIO]: """Hold an exclusive flock on a lock file below root, created without truncation. diff --git a/skills/kvb-org-organize/scripts/watch.py b/skills/kvb-org-organize/scripts/watch.py index bb88fdf..2295e99 100755 --- a/skills/kvb-org-organize/scripts/watch.py +++ b/skills/kvb-org-organize/scripts/watch.py @@ -8,8 +8,8 @@ Usage: (kvb-org-queue), /security-accepted.jsonl (kvb-check-security), /gate-receipts.jsonl and /gate-parents.jsonl (kvb-check-gate) whose id it - lacks to the same name under - ~/.local/state/agents-council/ + lacks to the same name in the host-only root + (councilroot.py durable), never through a link watch.py block hook (PreToolUse on Bash): exit 2 for a heavy job while the disk stage is stop or critical, the memory stage is stop or cannot be judged without memory_limits, or a @@ -33,8 +33,9 @@ Usage: REPORT Environment: - AGENTS_COUNCIL_ROOT: shared state directory; /tmp/agents-council unless set, - because every agent on the host must agree on it. Every command creates + AGENTS_COUNCIL_ROOT, XDG_STATE_HOME, HOME: the shared state directory + councilroot.py resolves, which survives a reboot and on which every + agent on the host agrees. Every command creates it with mode 700; while it is a symlink, another uid's, or group or other writable, a hook mode prints why and exits 0 doing nothing and any other mode exits 2. queue.py and radio.py check it the same way. @@ -114,7 +115,8 @@ def sibling(name: str) -> ModuleType: nofollow = sibling("nofollow") hookpath = sibling("hookpath") -ROOT = Path(os.environ.get("AGENTS_COUNCIL_ROOT", "/tmp/agents-council")) +councilroot = sibling("councilroot") +ROOT = councilroot.root() CONFIG = Path( os.environ.get( "AGENTS_COUNCIL_CONFIG", @@ -126,7 +128,7 @@ STAGES = ("ok", "warn", "stop", "critical") STATE = ROOT / "watch.json" SILENCE = ROOT / "radio-silence.json" ALERTS = ROOT / "alerts.jsonl" -KEPT = Path.home() / ".local" / "state" / "agents-council" +KEPT = councilroot.durable() HISTORY = ROOT / "queue-history.jsonl" KEPT_HISTORY = KEPT / "queue-history.jsonl" ACCEPTED = ROOT / "security-accepted.jsonl" @@ -273,6 +275,13 @@ def guard_root(hook: bool) -> int | None: ROOT.mkdir(mode=0o700, parents=True, exist_ok=True) except FileExistsError: pass + except OSError as error: + print( + f"agents council: cannot create {ROOT} ({error.strerror or error}); a sandbox needs it in allowWrite; " + "the council hooks stay off until it exists", + file=sys.stdout if hook else sys.stderr, + ) + return 0 if hook else 2 st = ROOT.lstat() if stat.S_ISDIR(st.st_mode) and st.st_uid == os.getuid() and not st.st_mode & 0o022: return None @@ -459,14 +468,13 @@ def history_rows(text: str) -> list[tuple[str, str]]: def keep_rows(origin: Path, target: Path) -> None: - # The agents' sandbox cannot write ~/.local/state and /tmp is wiped on reboot, so this hook keeps the rows. # The kept file takes the source's mtime: an unchanged source is skipped without parsing. try: source = origin.stat().st_mtime_ns if target.exists() and target.stat().st_mtime_ns == source: return - target.parent.mkdir(parents=True, exist_ok=True) - with target.open("a+", encoding="utf-8") as kept: + KEPT.mkdir(mode=0o700, parents=True, exist_ok=True) + with open(nofollow.open_below(KEPT, target, os.O_RDWR | os.O_CREAT | os.O_APPEND, 0o600), "a+", encoding="utf-8") as kept: fcntl.flock(kept, fcntl.LOCK_EX | fcntl.LOCK_NB) kept.seek(0) known = {row_id for row_id, _ in history_rows(kept.read())} @@ -475,7 +483,7 @@ def keep_rows(origin: Path, target: Path) -> None: known.add(row_id) kept.write(line + "\n") kept.flush() - os.utime(target, ns=(source, source)) + os.utime(kept.fileno(), ns=(source, source)) except (OSError, ValueError, TypeError): pass diff --git a/skills/kvb-org-organize/succession.md b/skills/kvb-org-organize/succession.md index 62296b7..891143e 100644 --- a/skills/kvb-org-organize/succession.md +++ b/skills/kvb-org-organize/succession.md @@ -21,6 +21,7 @@ `council.py takeover --work "..."` and posts the succession to the radio; everyone else leaves it alone. - After a restart or reboot, run `council.py list --mine` and claim again - every office you believe you hold: `/tmp` may have been wiped. + every office you believe you hold: claims survive a reboot, but a new + session holds none of them. - Only the operator, through the Orchestrator's passkey-signed Hierarchy view, removes a live holder; agents have no override. diff --git a/skills/kvb-org-perform/scripts/log.py b/skills/kvb-org-perform/scripts/log.py index f84d702..013bf38 100755 --- a/skills/kvb-org-perform/scripts/log.py +++ b/skills/kvb-org-perform/scripts/log.py @@ -16,7 +16,9 @@ Usage: Environment: SUBAGENT_MODEL_LOG: the log shared by every session; - ~/.local/state/agents-council/subagent-model.jsonl unless set. + subagent-model.jsonl in the host-only root (councilroot.py durable of + kvb-org-organize) unless set. Writes go through its nofollow.py, so + a link at the log or its directory fails them. CLAUDE_CODE_SESSION_ID: the session that rates its spawns. """ @@ -24,19 +26,27 @@ from __future__ import annotations import argparse import datetime +import importlib.util import json import os import re import shlex import sys from pathlib import Path +from types import ModuleType -LOG = Path( - os.environ.get( - "SUBAGENT_MODEL_LOG", - str(Path.home() / ".local" / "state" / "agents-council" / "subagent-model.jsonl"), - ) -) +ORGANIZE = Path(__file__).resolve().parents[2] / "kvb-org-organize" / "scripts" + + +def organize(name: str) -> ModuleType: + spec = importlib.util.spec_from_file_location(f"kvb_perform_{name}", ORGANIZE / f"{name}.py") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +nofollow = organize("nofollow") +LOG = Path(os.environ.get("SUBAGENT_MODEL_LOG") or organize("councilroot").durable() / "subagent-model.jsonl") TASK_CLASS = re.compile(r"^task-class:\s*(\S+)", re.MULTILINE) PENDING = "pending: " SCRIPT = "kvb-org-perform/scripts/log.py" @@ -54,9 +64,8 @@ def now() -> str: def append(line: dict) -> None: - LOG.parent.mkdir(parents=True, exist_ok=True) - with LOG.open("a", encoding="utf-8") as handle: - handle.write(json.dumps(line, sort_keys=True) + "\n") + LOG.parent.mkdir(mode=0o700, parents=True, exist_ok=True) + nofollow.append(LOG.parent, LOG, json.dumps(line, sort_keys=True) + "\n", 0o600) def record(line: dict) -> int: diff --git a/skills/kvb-org-queue-cleanup/scripts/cleanup.py b/skills/kvb-org-queue-cleanup/scripts/cleanup.py index f8a9465..d92464d 100755 --- a/skills/kvb-org-queue-cleanup/scripts/cleanup.py +++ b/skills/kvb-org-queue-cleanup/scripts/cleanup.py @@ -7,10 +7,11 @@ Usage: cleanup.py request Environment: - AGENTS_COUNCIL_ROOT: shared state directory; /tmp/agents-council unless - set. Agents append their requests to /queue-cleanup.request, - which kvb-org-queue-cleanup.path watches below /tmp/agents-council - only. The round lives in ~/.local/state/agents-council, which only + AGENTS_COUNCIL_ROOT, XDG_STATE_HOME, HOME: the council root the + kvb-org-organize councilroot.py resolves. Agents append their + requests to /queue-cleanup.request, which + kvb-org-queue-cleanup.path watches below %S/agents-shared only. The + round lives in the host-only root (councilroot.py durable), which only the host can write, in queue-cleanup.json under queue-cleanup.lock, written by run alone. CLAUDE_CODE_SESSION_ID: run refuses to start while it is set. diff --git a/skills/kvb-org-queue-cleanup/systemd/kvb-org-queue-cleanup.path b/skills/kvb-org-queue-cleanup/systemd/kvb-org-queue-cleanup.path index a80c42a..647fb9b 100644 --- a/skills/kvb-org-queue-cleanup/systemd/kvb-org-queue-cleanup.path +++ b/skills/kvb-org-queue-cleanup/systemd/kvb-org-queue-cleanup.path @@ -2,7 +2,7 @@ Description=kvb-org-queue-cleanup round on cleanup.py request [Path] -PathModified=/tmp/agents-council/queue-cleanup.request +PathModified=%S/agents-shared/queue-cleanup.request [Install] WantedBy=paths.target diff --git a/skills/kvb-org-queue-cleanup/systemd/kvb-org-queue-cleanup.service b/skills/kvb-org-queue-cleanup/systemd/kvb-org-queue-cleanup.service index 04a4688..6eb325d 100644 --- a/skills/kvb-org-queue-cleanup/systemd/kvb-org-queue-cleanup.service +++ b/skills/kvb-org-queue-cleanup/systemd/kvb-org-queue-cleanup.service @@ -3,5 +3,6 @@ Description=One kvb-org-queue-cleanup round: pan-pan, security, delete the jobs [Service] Type=oneshot +ExecStartPre=-/usr/bin/python3 -I %h/.claude/skills/kvb-org-organize/scripts/councilroot.py retire ExecStart=/usr/bin/python3 -I %h/.claude/skills/kvb-org-queue-cleanup/scripts/cleanup.py run --window 900 TimeoutStartSec=2400 diff --git a/skills/kvb-org-queue-estimate/scripts/estimate.py b/skills/kvb-org-queue-estimate/scripts/estimate.py index 3632772..5daddda 100755 --- a/skills/kvb-org-queue-estimate/scripts/estimate.py +++ b/skills/kvb-org-queue-estimate/scripts/estimate.py @@ -2,8 +2,9 @@ """Ask every agent once, by one newsletter, to estimate the open register tasks that carry no complexity. Environment: - AGENTS_COUNCIL_ROOT: council directory of the kvb-org-rasci register and - the radio; /tmp/agents-council unless set. + AGENTS_COUNCIL_ROOT, XDG_STATE_HOME, HOME: the council directory of the + kvb-org-rasci register and the radio, as the kvb-org-organize + councilroot.py resolves it. Args: show: print the newsletter's text, send nothing. diff --git a/skills/kvb-org-queue/scripts/queue.py b/skills/kvb-org-queue/scripts/queue.py index 693c195..d4ffb59 100755 --- a/skills/kvb-org-queue/scripts/queue.py +++ b/skills/kvb-org-queue/scripts/queue.py @@ -10,8 +10,9 @@ Usage: queue.py block hook (PreToolUse on Bash) Environment: - AGENTS_COUNCIL_ROOT: shared state directory; /tmp/agents-council unless set, - because every agent on the host must agree on it. The queue is + AGENTS_COUNCIL_ROOT, XDG_STATE_HOME, HOME: the shared state directory the + kvb-org-organize councilroot.py resolves, on which every agent on the + host agrees. The queue is /queue.json, every read-modify-write holds /queue.lock. Each hold keeps an exclusive flock on /holds/.lock, shared with its command, from before its entry exists until it leaves; an @@ -138,6 +139,7 @@ SIBLINGS = { "rasci": SKILLS / "kvb-org-rasci" / "scripts" / "rasci.py", "history": Path(__file__).resolve().parent / "history.py", "nofollow": SKILLS / "kvb-org-organize" / "scripts" / "nofollow.py", + "councilroot": SKILLS / "kvb-org-organize" / "scripts" / "councilroot.py", } TOOLS = {"gates": SKILLS / "kvb-check-gate" / "scripts" / "gates.py"} HOST_SKILLS = Path.home() / ".agents" / "skills" @@ -156,7 +158,6 @@ GATE = re.compile(r"\bgate\.py\s+(run|hold)\b") THROUGH_QUEUE = re.compile(r"\bqueue\.py\s+hold\b") BASH_RULE = re.compile(r"Bash(?:\((.*)\))?", re.DOTALL) RULE_PART = re.compile(r"( \*|\*)") -MANAGED = Path("/etc/claude-code") BUDGET_KEYS = ("budget_percent.ram", "budget_percent.cpu") HOTFIX = "hotfix/" PYTEST = frozenset({"pytest", "py.test"}) @@ -1043,15 +1044,7 @@ def cmd_cancel(args: argparse.Namespace) -> int: def settings_files(project: Path) -> list[Path]: - user = Path.home() / ".claude" - return [ - MANAGED / "managed-settings.json", - *sorted((MANAGED / "managed-settings.d").glob("*.json")), - user / "settings.json", - user / "settings.local.json", - project / ".claude" / "settings.json", - project / ".claude" / "settings.local.json", - ] + return sibling("councilroot").settings_files(project) def bash_rule(rule: str) -> re.Pattern | None: diff --git a/skills/kvb-org-radio/SKILL.md b/skills/kvb-org-radio/SKILL.md index d9acbfb..ab60bfd 100644 --- a/skills/kvb-org-radio/SKILL.md +++ b/skills/kvb-org-radio/SKILL.md @@ -7,7 +7,8 @@ description: > `${CLAUDE_SKILL_DIR}/scripts/radio.py` is the channel every skill uses to reach other agents, on every runtime. Both channels share one store, -`news.jsonl` in the council directory (`/tmp/agents-council`). +`news.jsonl` in the council directory, which +`${CLAUDE_SKILL_DIR}/../kvb-org-organize/scripts/councilroot.py root` prints. ```bash radio.py post "" [--to ...] [--session ] diff --git a/skills/kvb-org-radio/scripts/radio.py b/skills/kvb-org-radio/scripts/radio.py index c288ddc..49a3118 100755 --- a/skills/kvb-org-radio/scripts/radio.py +++ b/skills/kvb-org-radio/scripts/radio.py @@ -2,8 +2,9 @@ """Reach other agents: the radio is immediate, the post waits until a turn ends; one shared store. Environment: - AGENTS_COUNCIL_ROOT: council directory shared by every agent on the host; - /tmp/agents-council unless set. Rows live in /news.jsonl under + AGENTS_COUNCIL_ROOT, XDG_STATE_HOME, HOME: the council directory the + kvb-org-organize councilroot.py resolves, shared by every agent on the + host. Rows live in /news.jsonl under /news.lock, the store the old news.py reads and writes too. AGENTS_COUNCIL_CONFIG: config file; its "notify" argv tells the operator of a mayday, its "never_block" and "mayday_allow" regexes name the @@ -89,18 +90,9 @@ from pathlib import Path from types import ModuleType # Every agent on the host must agree on one council directory and one config without being told. -ROOT = Path(os.environ.get("AGENTS_COUNCIL_ROOT", "/tmp/agents-council")) CONFIG = Path( os.environ.get("AGENTS_COUNCIL_CONFIG", str(Path.home() / ".claude" / "agents-council" / "config.json")) ) -NEWS = ROOT / "news.jsonl" -LOCK = ROOT / "news.lock" -SEEN = ROOT / "news-seen" -STATE = ROOT / "radio-seen" -LISTEN = ROOT / "radio-listen" -SILENCE = ROOT / "radio-silence.json" -NOTIFIED = ROOT / "radio-notified" -FEEDBACK = ROOT / "feedback.json" ORGANIZE = Path(__file__).resolve().parents[2] / "kvb-org-organize" / "scripts" SELF = Path(__file__).resolve() RADIO = ("mayday", "pan-pan", "security", "call", "broadcast") @@ -140,6 +132,17 @@ def organize(name: str) -> ModuleType: return module +ROOT = organize("councilroot").root() +NEWS = ROOT / "news.jsonl" +LOCK = ROOT / "news.lock" +SEEN = ROOT / "news-seen" +STATE = ROOT / "radio-seen" +LISTEN = ROOT / "radio-listen" +SILENCE = ROOT / "radio-silence.json" +NOTIFIED = ROOT / "radio-notified" +FEEDBACK = ROOT / "feedback.json" + + def council() -> ModuleType: return organize("council") @@ -149,8 +152,9 @@ def name_of(sid: str) -> str: def items() -> list[dict]: + nofollow = organize("nofollow") try: - lines = organize("nofollow").read_text(ROOT, NEWS).splitlines() + lines = nofollow.read_tail(ROOT, NEWS, nofollow.READ_LIMIT).splitlines() except OSError: return [] rows = [] @@ -361,7 +365,11 @@ def store(entries: list[dict]) -> float: for entry in entries: entry["ts"] = ts ts = math.nextafter(ts, math.inf) - write_atomic(NEWS, "".join(json.dumps(row) + "\n" for row in [*rows[-(KEPT - len(entries)):], *entries])) + cut = max(0, len(rows) - max(0, KEPT - len(entries))) + if cut: + archive = ROOT / "archive" / f"news-{time.strftime('%Y-%m-%d')}.jsonl" + organize("nofollow").append(ROOT, archive, "".join(json.dumps(row) + "\n" for row in rows[:cut]), 0o600) + write_atomic(NEWS, "".join(json.dumps(row) + "\n" for row in [*rows[cut:], *entries])) return entries[-1]["ts"] diff --git a/skills/kvb-org-rasci/scripts/rasci.py b/skills/kvb-org-rasci/scripts/rasci.py index 7376f0c..c36cedf 100755 --- a/skills/kvb-org-rasci/scripts/rasci.py +++ b/skills/kvb-org-rasci/scripts/rasci.py @@ -2,12 +2,12 @@ """Keep the RASCI register of the organisation's tasks in a local JSON file. Environment: - AGENTS_COUNCIL_ROOT: council directory shared by every agent on the host; - the register is /rasci.json, and `show` also lists every work/ - and cluster/ claim of the council there. The operator's agent order - (`agents order`) lies beside it in /agent-order.json, apart from - every task row: it breaks ties of equal task priority only. - /tmp/agents-council unless set. + AGENTS_COUNCIL_ROOT, XDG_STATE_HOME, HOME: the council directory the + kvb-org-organize councilroot.py resolves, shared by every agent on the + host; the register is /rasci.json, and `show` also lists every + work/ and cluster/ claim of the council there. The operator's agent + order (`agents order`) lies beside it in /agent-order.json, apart + from every task row: it breaks ties of equal task priority only. Exit codes: 0 done, 2 usage error. @@ -20,7 +20,6 @@ import contextlib import datetime import importlib.util import json -import os import re import statistics import sys @@ -28,10 +27,6 @@ from collections.abc import Iterator from pathlib import Path from types import ModuleType -ROOT = Path(os.environ.get("AGENTS_COUNCIL_ROOT", "/tmp/agents-council")) -DB = ROOT / "rasci.json" -LOCK = ROOT / "rasci.lock" -AGENT_ORDER = ROOT / "agent-order.json" FILE_MODE = 0o600 ORGANIZE = Path(__file__).resolve().parents[2] / "kvb-org-organize" / "scripts" COUNCIL = ORGANIZE / "council.py" @@ -88,6 +83,10 @@ def script(name: str, path: Path) -> ModuleType: nofollow = script("kvb_rasci_nofollow", ORGANIZE / "nofollow.py") +ROOT = script("kvb_rasci_councilroot", ORGANIZE / "councilroot.py").root() +DB = ROOT / "rasci.json" +LOCK = ROOT / "rasci.lock" +AGENT_ORDER = ROOT / "agent-order.json" def load() -> dict[str, dict]: diff --git a/skills/kvb-org-suspend/scripts/night.py b/skills/kvb-org-suspend/scripts/night.py index d27a7a5..c499793 100755 --- a/skills/kvb-org-suspend/scripts/night.py +++ b/skills/kvb-org-suspend/scripts/night.py @@ -2,8 +2,9 @@ """Keep the organisation's night mode: one active session, the others drained or suspended. Environment: - AGENTS_COUNCIL_ROOT: council directory shared by every agent on the host; - the state is /night.json. /tmp/agents-council unless set. + AGENTS_COUNCIL_ROOT, XDG_STATE_HOME, HOME: the council directory the + kvb-org-organize councilroot.py resolves, shared by every agent on the + host; the state is /night.json. CLAUDE_CODE_SESSION_ID: the calling session. Args: @@ -43,20 +44,27 @@ import tempfile from pathlib import Path from types import ModuleType -ROOT = Path(os.environ.get("AGENTS_COUNCIL_ROOT", "/tmp/agents-council")) -STATE = ROOT / "night.json" -COUNCIL = Path(__file__).resolve().parents[2] / "kvb-org-organize" / "scripts" / "council.py" +ORGANIZE = Path(__file__).resolve().parents[2] / "kvb-org-organize" / "scripts" +COUNCIL = ORGANIZE / "council.py" PARTS = {"drain": "drain (kvb-agent-drain)", "suspend": "suspend (kvb-agent-suspend)"} RENAMED = {"evening": "drain", "sunset": "suspend"} -def council() -> ModuleType: - spec = importlib.util.spec_from_file_location("council", COUNCIL) +def loaded(name: str, path: Path) -> ModuleType: + spec = importlib.util.spec_from_file_location(name, path) module = importlib.util.module_from_spec(spec) spec.loader.exec_module(module) return module +def council() -> ModuleType: + return loaded("council", COUNCIL) + + +ROOT = loaded("kvb_night_councilroot", ORGANIZE / "councilroot.py").root() +STATE = ROOT / "night.json" + + def migrated(state: object) -> object: if not isinstance(state, dict): return state diff --git a/skills/kvb-org-unify/SKILL.md b/skills/kvb-org-unify/SKILL.md index efde7f5..1e384ca 100644 --- a/skills/kvb-org-unify/SKILL.md +++ b/skills/kvb-org-unify/SKILL.md @@ -19,10 +19,11 @@ bar on ``", key results gate green, landed, cleaned up, overview posted. ## Step 1: collect The register is the swarm's record (stage 2 of `kvb-org-swarm` sets `--branch` -and `--base`): +and `--base`); `` is what +`${CLAUDE_SKILL_DIR}/../kvb-org-organize/scripts/councilroot.py root` prints: ```bash -jq -r --arg base "" 'to_entries[] | select(.value.base == $base and .value.branch) | [.key, .value.branch, .value.workdir // "-", .value.status] | @tsv' "${AGENTS_COUNCIL_ROOT:-/tmp/agents-council}/rasci.json" +jq -r --arg base "" 'to_entries[] | select(.value.base == $base and .value.branch) | [.key, .value.branch, .value.workdir // "-", .value.status] | @tsv' "/rasci.json" ``` Per branch: `git log --oneline ..` (empty: already merged, go @@ -44,7 +45,8 @@ the branch becomes a proposal naming the conflicting files. The project's full gate on `unify/` through `queue.py hold` (`kvb-check-gate`), and `security.py` over `..unify/` -(`kvb-check-security`), logs under `/tmp`. Red: find the merge that broke it +(`kvb-check-security`), logs under the directory `councilroot.py dir gates` +prints. Red: find the merge that broke it (`git bisect start --first-parent`), rebuild the branch without it, which becomes a proposal naming the failing test, and gate again. diff --git a/skills/kvb-org-update/scripts/update.py b/skills/kvb-org-update/scripts/update.py index 0a22d94..a9c3300 100755 --- a/skills/kvb-org-update/scripts/update.py +++ b/skills/kvb-org-update/scripts/update.py @@ -2,8 +2,9 @@ """Tell every agent which skills changed and how to use them: one newsletter per update. Environment: - AGENTS_COUNCIL_ROOT: council directory shared by every agent on the host; - /tmp/agents-council unless set. The last announced revision per + AGENTS_COUNCIL_ROOT, XDG_STATE_HOME, HOME: the council directory the + kvb-org-organize councilroot.py resolves, shared by every agent on the + host. The last announced revision per repository lives in /skills-announced.json. CLAUDE_CODE_SESSION_ID: the sending session, passed on to radio.py. @@ -25,14 +26,11 @@ import argparse import functools import importlib.util import json -import os import subprocess import sys from pathlib import Path from types import ModuleType -ROOT = Path(os.environ.get("AGENTS_COUNCIL_ROOT", "/tmp/agents-council")) -ANNOUNCED = ROOT / "skills-announced.json" SKILLS = Path(__file__).resolve().parents[2] RADIO = SKILLS / "kvb-org-radio" / "scripts" / "radio.py" ORGANIZE = SKILLS / "kvb-org-organize" / "scripts" @@ -48,6 +46,10 @@ def organize(name: str) -> ModuleType: return module +ROOT = organize("councilroot").root() +ANNOUNCED = ROOT / "skills-announced.json" + + def git(repo: Path, *args: str) -> str: return subprocess.run([*GIT, *args], cwd=repo, capture_output=True, text=True, check=True).stdout diff --git a/skills/kvb-release-article/SKILL.md b/skills/kvb-release-article/SKILL.md index c2af68c..5d1ac94 100644 --- a/skills/kvb-release-article/SKILL.md +++ b/skills/kvb-release-article/SKILL.md @@ -41,5 +41,8 @@ mechanism and the upgrade path, investors read only what the brief proves. - Stay inside the channel's length bounds. Without project bounds: a blog article 600 to 1200 words, a LinkedIn article 400 to 900, a newsletter 250 to 500. -- Write `/tmp/release-//.md`, or the format the - project's template names, and report each file's word count. +- Write `//.md`, or the format the project's + template names, and report each file's word count. `` is the + directory `kvb-release` handed over, else the one + `${CLAUDE_SKILL_DIR}/../kvb-org-organize/scripts/councilroot.py release ` + prints. diff --git a/skills/kvb-release-post/SKILL.md b/skills/kvb-release-post/SKILL.md index 9c1fce4..e85f866 100644 --- a/skills/kvb-release-post/SKILL.md +++ b/skills/kvb-release-post/SKILL.md @@ -43,6 +43,9 @@ replaces this one: - Every number, name and date comes from `brief.yml`. - No em dashes, no placeholder, no claim the brief does not prove. -- Write into `/tmp/release-/` under the file name the project skill - sets, else `/.txt`, and report each file's character - count against its limit. +- Write into `/` under the file name the project skill sets, else + `/.txt`, and report each file's character count + against its limit. `` is the directory `kvb-release` handed over, + else the one + `${CLAUDE_SKILL_DIR}/../kvb-org-organize/scripts/councilroot.py release ` + prints. diff --git a/skills/kvb-release/SKILL.md b/skills/kvb-release/SKILL.md index c1008cb..79131a2 100644 --- a/skills/kvb-release/SKILL.md +++ b/skills/kvb-release/SKILL.md @@ -14,8 +14,10 @@ the project's own `-release-` skills. Run the `kvb-think-listen` skill once for what only the operator knows: the tag, the languages (default English), the audiences, the highlights, what to -leave out and the channels wanted. Write the answers to -`/tmp/release-/listen.yml` (`tag`, `languages`, `audience`, +leave out and the channels wanted. `` is the directory +`${CLAUDE_SKILL_DIR}/../kvb-org-organize/scripts/councilroot.py release ` +prints, which survives a reboot. Write the answers to +`/listen.yml` (`tag`, `languages`, `audience`, `highlights`, `leave_out`, `channels`) and hand that file to every channel, so the operator answers each question once. @@ -31,7 +33,7 @@ so the operator answers each question once. ## 3. Brief Run the `kvb-think-brainstorm` skill on the headline benefit, then write -`/tmp/release-/brief.yml`: version, date, languages, headline +`/brief.yml`: version, date, languages, headline benefit, three to five benefits each with the changelog lines that prove it, upgrade notes and links. `languages` is the only answer the brief copies from `listen.yml`: an answer steers what a channel stresses, it never backs a @@ -51,12 +53,12 @@ the general method: | entry | changelog | `kvb-release-changelog` | Each gets `brief.yml`, `listen.yml` and its channel, runs its own listen and -brainstorm, and writes into `/tmp/release-/` under the file name +brainstorm, and writes into `/` under the file name the project skill sets, else `/.`. ## 5. Check -Fix in every file under `/tmp/release-/`, subfolders included, +Fix in every file under `/`, subfolders included, before the hand-over: a leftover placeholder (`TODO`, `XXX`, `<...>`), an em dash, a length outside the channel's bounds, a number absent from `brief.yml`. diff --git a/tests/test_council_root.py b/tests/test_council_root.py new file mode 100644 index 0000000..9d61dc7 --- /dev/null +++ b/tests/test_council_root.py @@ -0,0 +1,872 @@ +from __future__ import annotations + +import contextlib +import errno +import fcntl +import hashlib +import importlib.util +import io +import json +import os +import re +import shutil +import subprocess +import sys +import tempfile +import time +import unittest +from pathlib import Path +from types import ModuleType +from unittest import mock + +REPO_ROOT = Path(__file__).resolve().parents[1] +ORGANIZE = REPO_ROOT / "skills" / "kvb-org-organize" / "scripts" +COUNCIL_ROOT = ORGANIZE / "councilroot.py" +WATCH = ORGANIZE / "watch.py" +QUEUE = REPO_ROOT / "skills" / "kvb-org-queue" / "scripts" / "queue.py" +SECURITY = REPO_ROOT / "skills" / "kvb-check-security" / "scripts" / "security.py" +GATES = REPO_ROOT / "skills" / "kvb-check-gate" / "scripts" / "gates.py" +LOG = REPO_ROOT / "skills" / "kvb-org-perform" / "scripts" / "log.py" +CLIENT = REPO_ROOT / "orchestrator" / "client" +INSTALL = REPO_ROOT / "scripts" / "install.sh" +GATE_SKILL = REPO_ROOT / "skills" / "kvb-check-gate" +SCANNED = ("skills", "orchestrator", "scripts") +FORBIDDEN = re.compile(r"/tmp/(?:agents-council|kvb-git-diff|kvb-orchestrator-approvals|release-)") +LEGACY_LINE = 'LEGACY = Path("/tmp/agents-council")' +DIR_SLASH = re.compile(r"councilroot\.py dir [^)\n]*\)/") +RADIO = REPO_ROOT / "skills" / "kvb-org-radio" / "scripts" / "radio.py" +CLEANUP_SERVICE = REPO_ROOT / "skills" / "kvb-org-queue-cleanup" / "systemd" / "kvb-org-queue-cleanup.service" + + +def load(path: Path, name: str) -> ModuleType: + spec = importlib.util.spec_from_file_location(name, path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def rows(path: Path) -> list[str]: + return [json.loads(line)["id"] for line in path.read_text(encoding="utf-8").splitlines()] + + +class Home(unittest.TestCase): + def setUp(self): + tmp = tempfile.TemporaryDirectory() + self.addCleanup(tmp.cleanup) + self.base = Path(tmp.name).resolve() + self.home = self.base / "home" + self.home.mkdir() + patch = mock.patch.dict(os.environ, {"HOME": str(self.home)}) + patch.start() + self.addCleanup(patch.stop) + for name in ("AGENTS_COUNCIL_ROOT", "XDG_STATE_HOME"): + os.environ.pop(name, None) + self.durable = self.home / ".local" / "state" / "agents-council" + self.shared = self.home / ".local" / "state" / "agents-shared" + self.councilroot = load(COUNCIL_ROOT, f"test_councilroot_{id(self)}") + + def cli(self, *args: str, **env: str) -> subprocess.CompletedProcess: + return subprocess.run( + [sys.executable, str(COUNCIL_ROOT), *args], env={"PATH": os.environ["PATH"], "HOME": str(self.home), **env}, + capture_output=True, text=True, check=False, + ) + + +class TestResolver(Home): + def test_agents_share_one_root_and_the_host_keeps_another_by_default(self): + self.assertEqual((self.councilroot.root(), self.councilroot.durable()), (self.shared, self.durable)) + self.assertNotEqual(self.councilroot.root(), self.councilroot.durable()) + + def test_xdg_state_home_moves_both_roots_only_when_absolute(self): + for value, state in (("/srv/state", Path("/srv/state")), ("relative", self.durable.parent), ("", self.durable.parent)): + with self.subTest(value=value), mock.patch.dict(os.environ, {"XDG_STATE_HOME": value}): + self.assertEqual((self.councilroot.root(), self.councilroot.durable()), + (state / "agents-shared", state / "agents-council")) + + def test_agents_council_root_wins_and_an_empty_or_relative_one_means_unset(self): + with mock.patch.dict(os.environ, {"AGENTS_COUNCIL_ROOT": "/srv/council", "XDG_STATE_HOME": "/srv/state"}): + self.assertEqual((self.councilroot.root(), self.councilroot.durable()), (Path("/srv/council"), Path("/srv/state/agents-council"))) + for value in ("", "relative-council"): + with self.subTest(value=value), mock.patch.dict(os.environ, {"AGENTS_COUNCIL_ROOT": value}): + self.assertEqual(self.councilroot.root(), self.shared) + + def test_the_cli_prints_the_roots_and_creates_private_parts(self): + self.assertEqual(self.cli("root").stdout, f"{self.shared}\n") + self.assertEqual(self.cli("root", AGENTS_COUNCIL_ROOT=str(self.base / "c")).stdout, f"{self.base / 'c'}\n") + self.assertEqual(self.cli("durable", AGENTS_COUNCIL_ROOT=str(self.base / "c")).stdout, f"{self.durable}\n") + for args, made in ((("dir", "gates"), self.shared / "gates"), (("release", "1.2.3"), self.durable.parent / "agents-release" / "1.2.3")): + with self.subTest(args=args): + done = self.cli(*args) + self.assertEqual((done.returncode, done.stdout), (0, f"{made}\n"), done.stderr) + self.assertEqual(made.stat().st_mode & 0o777, 0o700) + self.assertEqual(self.shared.stat().st_mode & 0o777, 0o700) + self.assertFalse(self.durable.exists()) + refused = self.cli("dir", "../escape") + self.assertEqual((refused.returncode, refused.stderr), (2, "councilroot: not a directory name: '../escape'\n")) + self.assertFalse((self.shared.parent / "escape").exists()) + + def test_a_root_the_sandbox_cannot_create_turns_the_hooks_off_with_the_reason(self): + state = self.base / "frozen" + state.mkdir(mode=0o500) + self.addCleanup(state.chmod, 0o700) + with mock.patch.dict(os.environ, {"XDG_STATE_HOME": str(state)}): + watch = load(WATCH, f"test_council_root_frozen_{id(self)}") + out = io.StringIO() + with contextlib.redirect_stdout(out): + self.assertEqual(watch.guard_root(hook=True), 0) + self.assertIn(f"cannot create {state / 'agents-shared'}", out.getvalue()) + + def test_a_part_the_cli_cannot_create_exits_2_naming_why_and_prints_nothing(self): + state = self.base / "frozen" + state.mkdir(mode=0o500) + self.addCleanup(state.chmod, 0o700) + done = self.cli("dir", "gates", "slug", XDG_STATE_HOME=str(state)) + self.assertEqual((done.returncode, done.stdout), (2, "")) + self.assertTrue(done.stderr.startswith("councilroot: ") and "Permission denied" in done.stderr and "Traceback" not in done.stderr, done.stderr) + + +class TestMigrate(Home): + def setUp(self): + super().setUp() + self.legacy = self.base / "legacy" + self.legacy.mkdir(mode=0o700) + self.target = self.base / "state" / "agents-shared" + + def write(self, path: Path, text: str, mtime: int | None = None) -> Path: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(text, encoding="utf-8") + path.chmod(0o600) + if mtime is not None: + os.utime(path, (mtime, mtime)) + return path + + def migrate(self) -> list[str]: + return self.councilroot.migrate(self.legacy, self.target) + + def test_every_file_moves_and_a_second_run_moves_nothing(self): + self.write(self.legacy / "rasci.json", "{}\n", 1000) + self.write(self.legacy / "work" / "a" / "claim.json", '{"session": "s1"}\n') + self.write(self.legacy / "holds" / "1-2.lock", "") + self.assertEqual(self.migrate(), [f"moved 3 files from {self.legacy} to {self.target}"]) + self.assertEqual((self.target / "rasci.json").read_text(encoding="utf-8"), "{}\n") + self.assertEqual((self.target / "rasci.json").stat().st_mtime, 1000) + self.assertEqual((self.target / "rasci.json").stat().st_mode & 0o777, 0o600) + self.assertEqual((self.target / "work" / "a" / "claim.json").read_text(encoding="utf-8"), '{"session": "s1"}\n') + self.assertTrue((self.target / "holds" / "1-2.lock").is_file()) + self.assertFalse(self.legacy.exists()) + self.assertEqual(sorted(path.name for path in self.target.iterdir()), ["holds", "rasci.json", "work"]) + self.assertEqual(self.migrate(), []) + + def test_a_differing_whole_file_refuses_the_move_and_a_catch_up_keeps_it_beside(self): + newer = self.write(self.target / "rasci.json", "durable\n", 2000) + self.write(self.legacy / "rasci.json", "legacy\n", 1000) + self.write(self.legacy / "queue.json", '{"seq": 1, "entries": []}\n') + line = f"conflict {newer} differs from {self.legacy / 'rasci.json'}" + with self.assertRaises(self.councilroot.Blocked) as blocked: + self.migrate() + self.assertEqual(blocked.exception.lines, [line]) + done = self.cli("migrate", "--legacy", str(self.legacy), AGENTS_COUNCIL_ROOT=str(self.target)) + self.assertEqual((done.returncode, done.stdout), (4, f"{line}\nnothing moved from {self.legacy}\n"), done.stderr) + self.assertEqual(sorted(path.name for path in self.legacy.iterdir()), ["queue.json", "rasci.json"]) + self.assertFalse((self.target / "queue.json").exists()) + kept = self.target / f"rasci.json.legacy-{hashlib.sha256(b"legacy\n").hexdigest()[:12]}" + for run in (1, 2): + with self.subTest(run=run): + if run == 2: + self.write(self.legacy / "rasci.json", "legacy\n") + caught = self.cli("migrate", "--legacy", str(self.legacy), "--catch-up", AGENTS_COUNCIL_ROOT=str(self.target)) + self.assertEqual(caught.returncode, 3, caught.stderr) + self.assertIn(f"kept {self.legacy / 'rasci.json'} beside {newer} as {kept}", caught.stdout) + self.assertEqual((newer.read_text(encoding="utf-8"), kept.read_text(encoding="utf-8")), ("durable\n", "legacy\n")) + self.assertFalse(self.legacy.exists()) + self.assertEqual(sorted(path.name for path in self.target.iterdir()), ["queue.json", "rasci.json", kept.name]) + + def test_a_file_it_cannot_move_refuses_the_whole_move_and_preflight_predicts_it(self): + self.write(self.legacy / "rasci.json", '{"work/x": {}}\n') + self.write(self.legacy / "queue.json", '{"seq": 9, "entries": []}\n') + source = self.write(self.legacy / "late.json", "{}\n") + os.link(source, self.base / "second-link") + done = self.cli("migrate", "--legacy", str(self.legacy), AGENTS_COUNCIL_ROOT=str(self.target)) + line = f"unmoved {source}: not a regular file with a single link" + self.assertEqual((done.returncode, done.stdout), (4, f"{line}\nnothing moved from {self.legacy}\n"), done.stderr) + self.assertEqual(sorted(path.name for path in self.legacy.iterdir()), ["late.json", "queue.json", "rasci.json"]) + self.assertFalse(self.target.exists() and any(self.target.iterdir())) + checked = self.cli("preflight", "--legacy", str(self.legacy), "--managed", str(self.base / "no-managed"), AGENTS_COUNCIL_ROOT=str(self.target)) + self.assertEqual(checked.returncode, 2) + self.assertIn(line, checked.stderr) + + def test_a_move_failing_after_the_check_leaves_a_partial_state_with_exit_5(self): + self.write(self.legacy / "rasci.json", "{}\n") + late = self.write(self.legacy / "late.json", "{}\n") + real = self.councilroot.copied + + def racing(helper, legacy, source, *rest): + if source.name.startswith(late.name): + raise OSError(errno.EBUSY, "raced", str(source)) + return real(helper, legacy, source, *rest) + + out = io.StringIO() + with mock.patch.object(self.councilroot, "copied", racing), mock.patch.dict(os.environ, {"AGENTS_COUNCIL_ROOT": str(self.target)}), contextlib.redirect_stdout(out): + code = self.councilroot.main(["migrate", "--legacy", str(self.legacy)]) + self.assertEqual(code, 5) + self.assertTrue(any(line.startswith(f"unmoved {late}") and line.endswith(": raced") for line in out.getvalue().splitlines()), out.getvalue()) + self.assertTrue((self.target / "rasci.json").is_file()) + self.assertEqual([path.name.startswith("late.json") for path in self.legacy.iterdir()], [True]) + + def test_a_row_appended_while_its_journal_moves_reaches_the_root_through_the_catch_up(self): + journal = self.write(self.legacy / "scores.jsonl", '{"time": 1, "digest": "a"}\n') + late = '{"time": 2, "digest": "written between the copy and the unlink"}\n' + real = self.councilroot.copied + + def racing(helper, legacy, source, *rest): + done = real(helper, legacy, source, *rest) + with open(journal, "a", encoding="utf-8") as writer: + writer.write(late) + return done + + with mock.patch.object(self.councilroot, "copied", racing): + self.migrate() + self.councilroot.migrate(self.legacy, self.target, True) + self.assertEqual((self.target / "scores.jsonl").read_text(encoding="utf-8"), '{"time": 1, "digest": "a"}\n' + late) + self.assertFalse(self.legacy.exists()) + + def test_a_crash_mid_move_leaves_the_started_marker_until_a_run_completes_the_move(self): + self.write(self.legacy / "a.json", "{}\n") + self.write(self.legacy / "b.json", '{"b": 1}\n') + real = self.councilroot.copied + + def crashing(helper, legacy, source, *rest): + if source.name.startswith("b.json"): + raise RuntimeError("boom") + return real(helper, legacy, source, *rest) + + marker = self.durable / "migrate.started" + with mock.patch.object(self.councilroot, "copied", crashing), self.assertRaises(RuntimeError): + self.migrate() + self.assertTrue(marker.is_file() and (self.target / "a.json").is_file()) + self.assertEqual(self.migrate(), [f"moved 1 files from {self.legacy} to {self.target}"]) + self.assertEqual((self.target / "b.json").read_text(encoding="utf-8"), '{"b": 1}\n') + self.assertFalse(marker.exists() or self.legacy.exists()) + + def test_identical_copies_are_one_and_leave_the_legacy_root(self): + same = self.write(self.target / "rasci.json", "{}\n", 2000) + self.write(self.legacy / "rasci.json", "{}\n", 1000) + self.assertEqual(self.migrate(), [f"moved 1 files from {self.legacy} to {self.target}"]) + self.assertEqual((same.read_text(encoding="utf-8"), same.stat().st_mtime), ("{}\n", 2000)) + self.assertFalse(self.legacy.exists()) + + def test_a_held_lock_or_a_queued_job_refuses_the_whole_move(self): + hold = self.write(self.legacy / "holds" / "1-2.lock", "") + self.write(self.legacy / "rasci.json", "{}\n") + with open(hold, encoding="utf-8") as held: + fcntl.flock(held, fcntl.LOCK_EX) + with self.assertRaises(ValueError) as raised: + self.migrate() + self.assertIn(f"{hold} is held", str(raised.exception)) + self.write(self.legacy / "queue.json", json.dumps({"seq": 3, "entries": [{"id": "test-3"}]})) + with self.assertRaises(ValueError) as raised: + self.migrate() + self.assertIn(f"{self.legacy / 'queue.json'} holds 1 queue jobs", str(raised.exception)) + self.assertFalse(self.target.exists()) + self.assertEqual(sorted(path.name for path in self.legacy.iterdir()), ["holds", "queue.json", "rasci.json"]) + self.write(self.legacy / "queue.json", json.dumps({"seq": 3, "entries": []})) + self.assertEqual(self.migrate(), [f"moved 3 files from {self.legacy} to {self.target}"]) + + def test_an_id_journal_gets_only_the_rows_it_lacks(self): + kept = self.write(self.target / "queue-history.jsonl", '{"id": "a"}\n{"id": "b"}\n') + self.write(self.legacy / "queue-history.jsonl", '{"id": "b"}\n{"id": "c"}\n{"id": "c"}\n') + self.assertEqual(self.migrate(), [f"merged 1 rows of {self.legacy / 'queue-history.jsonl'} into {kept}"]) + self.assertEqual(rows(kept), ["a", "b", "c"]) + self.assertFalse(self.legacy.exists()) + self.assertEqual(self.migrate(), []) + self.assertEqual(rows(kept), ["a", "b", "c"]) + + def test_a_merge_appends_after_a_row_another_writer_adds_meanwhile(self): + self.write(self.legacy / "news.jsonl", '{"id": "L1"}\n') + kept = self.write(self.target / "news.jsonl", '{"id": "T1"}\n') + real, calls = self.councilroot.row_ids, [] + + def racing(text: str): + calls.append(text) + if len(calls) == 2: + with open(kept, "a", encoding="utf-8") as other: + other.write('{"id": "CONCURRENT"}\n') + return real(text) + + with mock.patch.object(self.councilroot, "row_ids", racing): + self.assertEqual(self.migrate(), [f"merged 1 rows of {self.legacy / 'news.jsonl'} into {kept}"]) + self.assertEqual(rows(kept), ["T1", "CONCURRENT", "L1"]) + + def test_a_journal_without_ids_merges_each_line_as_often_as_the_legacy_side_has_it_more_in_ts_order(self): + kept = self.write(self.target / "news.jsonl", '{"ts": 1.0, "text": "a"}\n{"ts": 3.0, "text": "c"}\n') + self.write(self.legacy / "news.jsonl", '{"ts": 3.0, "text": "c"}\n{"ts": 2.0, "text": "b"}\n{"ts": 2.0, "text": "b"}\n') + untimed = self.write(self.target / "scores.jsonl", '{"time": 1}\n') + self.write(self.legacy / "scores.jsonl", '{"time": 2}\n{"time": 1}\n{"time": 1}\n{"time": 1}\n') + self.assertEqual(self.migrate(), [ + f"merged 2 rows of {self.legacy / 'news.jsonl'} into {kept}", + f"merged 3 rows of {self.legacy / 'scores.jsonl'} into {untimed}", + ]) + self.assertEqual([json.loads(line)["text"] for line in kept.read_text(encoding="utf-8").splitlines()], ["a", "b", "b", "c"]) + self.assertEqual(untimed.read_text(encoding="utf-8"), '{"time": 1}\n{"time": 2}\n{"time": 1}\n{"time": 1}\n') + self.assertEqual(kept.stat().st_mode & 0o777, 0o600) + self.assertFalse(self.legacy.exists()) + + def test_no_link_on_either_side_is_followed(self): + secret = self.write(self.base / "secret.txt", "secret\n") + outside = self.base / "outside" + self.write(outside / "inner.txt", "inner\n") + elsewhere = self.base / "elsewhere" + elsewhere.mkdir() + (self.legacy / "evil").symlink_to(secret) + (self.legacy / "linked").symlink_to(outside) + self.write(self.legacy / "work" / "a" / "claim.json", "{}\n") + self.target.mkdir(parents=True, mode=0o700) + (self.target / "work").symlink_to(elsewhere) + with self.assertRaises(self.councilroot.Blocked) as blocked: + self.migrate() + self.assertTrue(all(line.startswith(f"unmoved {self.legacy / 'work' / 'a' / 'claim.json'}: ") for line in blocked.exception.lines), blocked.exception.lines) + report = self.councilroot.migrate(self.legacy, self.target, True) + self.assertIn(f"skipped {self.legacy / 'evil'}: not a regular file", report) + self.assertIn(f"skipped {self.legacy / 'linked'}: a link", report) + self.assertTrue(any(re.match(rf"unmoved {re.escape(str(self.legacy / 'work' / 'a' / 'claim.json'))}\.[0-9a-f]{{16}}\.migrating: ", line) for line in report), report) + self.assertEqual(list(elsewhere.iterdir()), []) + self.assertFalse((self.target / "evil").exists() or (self.target / "linked").exists()) + self.assertEqual((secret.read_text(encoding="utf-8"), (outside / "inner.txt").read_text(encoding="utf-8")), ("secret\n", "inner\n")) + + def test_a_linked_or_shared_legacy_root_is_refused(self): + linked = self.base / "linked-legacy" + linked.symlink_to(self.legacy) + with self.assertRaises(ValueError): + self.councilroot.migrate(linked, self.target) + self.legacy.chmod(0o777) + self.addCleanup(self.legacy.chmod, 0o700) + done = self.cli("migrate", "--legacy", str(self.legacy), AGENTS_COUNCIL_ROOT=str(self.target)) + self.assertEqual(done.returncode, 2) + self.assertIn("group and others cannot write", done.stderr) + + def test_a_missing_legacy_root_moves_nothing(self): + done = self.cli("migrate", "--legacy", str(self.base / "gone"), AGENTS_COUNCIL_ROOT=str(self.target)) + self.assertEqual((done.returncode, done.stdout), (0, f"nothing to move from {self.base / 'gone'}\n")) + + def test_orchestrator_stop_covers_every_unit_that_writes_a_council_root(self): + units = (REPO_ROOT / "scripts" / "orchestrator.sh").read_text(encoding="utf-8") + body = units[units.index("stop_all() {"):] + body = body[:body.index("\n}\n")] + for unit in ("${RUNNER}", "${OFFICES_UNIT}.path", "${RESOURCES_UNIT}.timer", "${RESOURCES_UNIT}.service", "compose"): + with self.subTest(unit=unit): + self.assertIn(unit, body) + self.assertIn("stop) stop_all ;;", units) + + +NPX_STUB = """#!/usr/bin/env python3 +import json, os +from pathlib import Path +with open(os.environ["STUB_RECORD"], "a") as record: + record.write(f"npx {Path(os.environ['STUB_CHECK']).is_file()}\\n") +if os.environ.get("STUB_LINK"): + os.link(*json.loads(os.environ["STUB_LINK"])) +skill = Path(".agents/skills/kvb-new/SKILL.md") +skill.parent.mkdir(parents=True) +skill.write_text("new\\n") +""" +CP_STUB = """#!/usr/bin/env python3 +import json, os, sys +from pathlib import Path +with open(os.environ["STUB_RECORD"], "a") as record: + record.write(f"cp {Path(os.environ['STUB_CHECK']).is_file()} {sys.argv[-1]}\\n") +if sys.argv[-1].startswith(os.environ["HOME"]): + for late, text in json.loads(os.environ.get("STUB_LATE", "{}")).items(): + if not Path(late).exists(): + Path(late).parent.mkdir(parents=True, exist_ok=True) + Path(late).write_text(text) +os.execv(os.environ["REAL_CP"], ["cp", *sys.argv[1:]]) +""" +TOOLS = ("bash", "python3", "dirname", "rm", "find", "mkdir", "install", "grep", "sed", "ln", "cat", "id") + + +class Settings(Home): + def setUp(self): + super().setUp() + self.legacy = self.base / "legacy" + self.legacy.mkdir(mode=0o700) + self.settings = self.home / ".claude" / "settings.json" + self.release = self.home / ".local" / "state" / "agents-release" + self.allow(["~/.local/state/agents-shared", f"{self.release}/**", "/var/cache/unrelated", "."]) + + def allow(self, entries: list[str]) -> None: + self.settings.parent.mkdir(parents=True, exist_ok=True) + self.settings.write_text(json.dumps({"sandbox": {"enabled": True, "filesystem": {"allowWrite": entries}}}), encoding="utf-8") + + def preflight(self) -> subprocess.CompletedProcess: + return self.cli("preflight", "--legacy", str(self.legacy), "--managed", str(self.base / "managed")) + + def settings_file(self, path: Path, entries: list[str]) -> Path: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps({"sandbox": {"filesystem": {"allowWrite": entries}}}), encoding="utf-8") + return path + + +class TestPreflight(Settings): + def test_both_agent_paths_must_be_writable_for_the_sandbox_and_the_host_root_never(self): + self.assertEqual(self.preflight().returncode, 0, self.preflight().stderr) + for entries, wanted in ( + (["~/.local/state/agents-shared"], f"add {self.release} to sandbox.filesystem.allowWrite in {self.settings}"), + ([f"{self.release}"], f"add {self.shared} to sandbox.filesystem.allowWrite in {self.settings}"), + (["~/.local/state/agents-shared", str(self.release), "~/.local/state/agents-council/**"], f"{self.settings}: drop allowWrite entry '~/.local/state/agents-council/**' ({self.durable})"), + (["~/.local/state/agents-shared", str(self.release), "~/.local/state"], f"{self.settings}: drop allowWrite entry '~/.local/state' ({self.durable.parent})"), + ): + with self.subTest(entries=entries): + self.allow(entries) + done = self.preflight() + self.assertEqual(done.returncode, 2) + self.assertIn(wanted, done.stderr) + self.settings.unlink() + self.assertIn(f"{self.settings} holds no sandbox.filesystem.allowWrite list", self.preflight().stderr) + + def test_every_settings_file_that_can_apply_is_checked_and_named_with_its_entry(self): + repo = self.home / "Repositories" / "github.com" / "o" / "r" + worktree = self.home / ".local" / "share" / "worktrees" / "github.com" / "o" / "r" / "fix" / "x" + widening = ( + (self.base / "managed" / "managed-settings.json", "~"), + (self.base / "managed" / "managed-settings.d" / "50-widen.json", "~/.local"), + (self.home / ".claude" / "settings.local.json", "~/.local/state/agents-council"), + (repo / ".claude" / "settings.json", "~/.local/state"), + (repo / ".claude" / "settings.local.json", "../../../../.local"), + (worktree / ".claude" / "settings.json", "~/.local/state/agents-council/resources/**"), + ) + self.settings_file(repo / "nested" / ".claude" / "settings.json", ["/var/cache/fine"]) + self.assertEqual(self.preflight().returncode, 0, self.preflight().stderr) + for path, entry in widening: + with self.subTest(path=path): + self.settings_file(path, [entry, "/var/cache/fine"]) + done = self.preflight() + self.assertEqual(done.returncode, 2) + self.assertIn(f"{path}: drop allowWrite entry {entry!r}", done.stderr) + self.assertNotIn("add ", done.stderr) + path.unlink() + moved = self.home / "Code" / "o" / "r" / ".claude" / "settings.json" + self.settings_file(moved, ["~/.local/state"]) + self.assertEqual(self.preflight().returncode, 0) + config = self.home / ".config" / "pkgmgr" / "config.yaml" + config.parent.mkdir(parents=True) + config.write_text("directories:\n repositories: ~/Code/\n binaries: ~/.local/bin/\n", encoding="utf-8") + self.assertIn(f"{moved}: drop allowWrite entry '~/.local/state'", self.preflight().stderr) + + def test_the_queue_and_the_preflight_share_one_list_of_settings_files(self): + self.assertNotIn("managed-settings", QUEUE.read_text(encoding="utf-8")) + managed = self.base / "managed" + dropin = managed / "managed-settings.d" / "50-deny.json" + dropin.parent.mkdir(parents=True) + dropin.write_text(json.dumps({"permissions": {"deny": ["Bash(rm *)"]}}), encoding="utf-8") + queue = load(QUEUE, f"test_council_root_queue_settings_{id(self)}") + project = self.base / "project" + with mock.patch.object(queue.sibling("councilroot"), "MANAGED", managed): + self.assertIn(dropin, queue.settings_files(project)) + self.assertEqual(queue.permission(["rm -rf x"], project), "deny") + + def test_linked_repositories_linked_files_and_every_session_start_directory_are_checked(self): + repositories = self.home / "Repositories" + data = self.base / "data" / "r" + linked = repositories / "github.com" / "o" / "r" + linked.parent.mkdir(parents=True) + linked.symlink_to(data) + (repositories / "loop").symlink_to(repositories) + project = self.home / "work" / "proj" + session = self.home / ".claude" / "projects" / re.sub(r"[^A-Za-z0-9]", "-", str(project)) + session.mkdir(parents=True) + (self.home / ".claude" / "projects" / "-gone-project").mkdir() + target = self.settings_file(self.base / "shared-settings.json", ["~/.local/state"]) + cases = ( + (linked / ".claude" / "settings.json", lambda path: self.settings_file(data / ".claude" / "settings.json", ["~/.local/state"])), + (project / ".claude" / "settings.local.json", lambda path: self.settings_file(path, ["~/.local/state"])), + (repositories / "github.com" / "o" / "s" / ".claude" / "settings.json", lambda path: (path.parent.mkdir(parents=True), path.symlink_to(target))), + ) + self.settings_file(project / "deeper" / ".claude" / "settings.json", ["~/.local/state"]) + self.assertEqual(self.preflight().returncode, 0, self.preflight().stderr) + for path, make in cases: + with self.subTest(path=path): + make(path) + done = self.preflight() + self.assertEqual(done.returncode, 2) + self.assertEqual(done.stderr.count(f"{path}: drop allowWrite entry '~/.local/state'"), 1, done.stderr) + path.unlink() + + def test_a_council_that_is_not_suspended_is_only_warned_about(self): + done = self.preflight() + self.assertEqual(done.returncode, 0, done.stderr) + self.assertIn("warning: the agents are not suspended", done.stderr) + (self.legacy / "night.json").write_text(json.dumps({"mode": "suspend"}), encoding="utf-8") + done = self.preflight() + self.assertEqual((done.returncode, done.stderr), (0, "")) + + def test_a_busy_or_conflicting_legacy_root_is_refused_before_anything_moves(self): + hold = self.legacy / "holds" / "1-2.lock" + hold.parent.mkdir() + hold.write_text("", encoding="utf-8") + with open(hold, encoding="utf-8") as held: + fcntl.flock(held, fcntl.LOCK_EX) + self.assertIn(f"{hold} is held", self.preflight().stderr) + (self.legacy / "queue.json").write_text(json.dumps({"seq": 3, "entries": [{"id": "test-3"}]}), encoding="utf-8") + self.assertIn("holds 1 queue jobs", self.preflight().stderr) + (self.legacy / "queue.json").write_text(json.dumps({"seq": 3, "entries": []}), encoding="utf-8") + for name, legacy, shared, ok in (("rasci.json", "a", "b", False), ("night.json", "same", "same", True), ("news.jsonl", '{"id": 1}\n', '{"id": 2}\n', True)): + (self.legacy / name).write_text(legacy, encoding="utf-8") + self.shared.mkdir(parents=True, mode=0o700, exist_ok=True) + (self.shared / name).write_text(shared, encoding="utf-8") + with self.subTest(name=name): + self.assertEqual(f"conflict {self.shared / name} differs" in self.preflight().stderr, not ok) + (self.legacy / "rasci.json").unlink() + self.assertEqual(self.preflight().returncode, 0, self.preflight().stderr) + self.assertEqual(sorted(path.name for path in self.legacy.iterdir()), ["holds", "news.jsonl", "night.json", "queue.json"]) + + +class TestInstall(Settings): + def setUp(self): + super().setUp() + self.repo = self.base / "repo" + for part in ("scripts/install.sh", "scripts/orchestrator.sh", "skills/kvb-org-organize/scripts/councilroot.py", "skills/kvb-org-organize/scripts/nofollow.py"): + (self.repo / part).parent.mkdir(parents=True, exist_ok=True) + (self.repo / part).write_bytes((REPO_ROOT / part).read_bytes()) + stubs = self.base / "bin" + stubs.mkdir() + (stubs / "npx").write_text(NPX_STUB, encoding="utf-8") + (stubs / "cp").write_text(CP_STUB, encoding="utf-8") + (stubs / "systemctl").write_text("#!/bin/sh\nexit 1\n", encoding="utf-8") + for stub in stubs.iterdir(): + stub.chmod(0o755) + tools = self.base / "tools" + tools.mkdir() + for name in TOOLS: + (tools / name).symlink_to(shutil.which(name)) + self.record = self.base / "stubs.log" + self.installed = [self.home / root / "skills" / "kvb-old" / "SKILL.md" for root in (".agents", ".claude")] + for marker in self.installed: + marker.parent.mkdir(parents=True) + marker.write_text("old\n", encoding="utf-8") + self.env = { + "PATH": f"{stubs}:{tools}", "HOME": str(self.home), "AGENTS_COUNCIL_LEGACY": str(self.legacy), + "STUB_RECORD": str(self.record), "STUB_CHECK": str(self.shared / "rasci.json"), "REAL_CP": shutil.which("cp"), + } + + def install(self) -> subprocess.CompletedProcess: + return subprocess.run(["bash", str(self.repo / "scripts" / "install.sh")], env=self.env, capture_output=True, text=True, timeout=120, check=False) + + def test_a_refused_preflight_aborts_before_any_copy_and_the_old_skills_keep_the_legacy_state(self): + (self.legacy / "queue.json").write_text(json.dumps({"seq": 3, "entries": [{"id": "test-3"}]}), encoding="utf-8") + busy = self.install() + self.allow(["~/.local/state/agents-shared"]) + (self.legacy / "queue.json").write_text(json.dumps({"seq": 3, "entries": []}), encoding="utf-8") + unset = self.install() + for done, reason in ((busy, "holds 1 queue jobs"), (unset, f"add {self.release} to sandbox.filesystem.allowWrite")): + with self.subTest(reason=reason): + self.assertEqual(done.returncode, 1, done.stderr) + self.assertIn(reason, done.stderr) + self.assertIn("skills: install refused before copying anything", done.stderr) + self.assertFalse(self.record.exists() or self.shared.exists() or (self.repo / ".agents").exists()) + self.assertEqual([marker.read_text(encoding="utf-8") for marker in self.installed], ["old\n", "old\n"]) + self.assertTrue((self.legacy / "queue.json").is_file()) + + def test_a_migrate_that_fails_in_any_other_way_aborts_before_the_copy(self): + script = self.repo / "skills" / "kvb-org-organize" / "scripts" / "councilroot.py" + script.write_text(script.read_text(encoding="utf-8").replace("report = migrate(args.legacy, root(), args.catch_up)", 'raise RuntimeError("boom")'), encoding="utf-8") + (self.legacy / "rasci.json").write_text("{}\n", encoding="utf-8") + done = self.install() + self.assertEqual(done.returncode, 1, done.stderr) + self.assertIn("RuntimeError: boom", done.stderr) + self.assertIn("skills: install refused before copying anything: the council root migration failed", done.stderr) + self.assertEqual([marker.read_text(encoding="utf-8") for marker in self.installed], ["old\n", "old\n"]) + self.assertTrue((self.legacy / "rasci.json").is_file()) + + def test_a_file_that_turns_unmovable_after_the_preflight_aborts_with_the_old_root_intact(self): + (self.legacy / "rasci.json").write_text('{"work/x": {}}\n', encoding="utf-8") + (self.legacy / "queue.json").write_text('{"seq": 9, "entries": []}\n', encoding="utf-8") + self.env["STUB_LINK"] = json.dumps([str(self.legacy / "rasci.json"), str(self.base / "alias")]) + done = self.install() + self.assertEqual(done.returncode, 1, done.stderr) + self.assertIn(f"unmoved {self.legacy / 'rasci.json'}: not a regular file with a single link", done.stdout) + self.assertIn("skills: install refused before copying anything: the council root migration failed (exit 4)", done.stderr) + self.assertEqual(sorted(path.name for path in self.legacy.iterdir()), ["queue.json", "rasci.json"]) + self.assertFalse((self.shared / "rasci.json").exists() or (self.shared / "queue.json").exists()) + self.assertEqual([marker.read_text(encoding="utf-8") for marker in self.installed], ["old\n", "old\n"]) + + def test_a_partial_move_still_installs_the_new_skills_names_every_file_left_and_fails_at_the_end(self): + script = self.repo / "skills" / "kvb-org-organize" / "scripts" / "councilroot.py" + script.write_text(script.read_text(encoding="utf-8").replace( + "outcome, line = settle(helper, legacy, source, target, catch_up)", + 'outcome, line = ("unmoved", f"unmoved {source}: raced") if source.name == "late.json" else settle(helper, legacy, source, target, catch_up)', + ), encoding="utf-8") + (self.legacy / "rasci.json").write_text("{}\n", encoding="utf-8") + (self.legacy / "late.json").write_text("{}\n", encoding="utf-8") + done = self.install() + self.assertEqual(done.returncode, 1, done.stderr) + self.assertGreaterEqual(done.stderr.count(f"unmoved {self.legacy / 'late.json'}: raced"), 2, done.stderr) + self.assertTrue((self.shared / "rasci.json").is_file() and (self.legacy / "late.json").is_file()) + self.assertEqual([(marker.parent.parent / "kvb-new" / "SKILL.md").read_text(encoding="utf-8") for marker in self.installed], ["new\n"] * 2) + + def test_a_crash_after_a_partial_move_still_installs_the_new_skills_and_names_the_marker(self): + script = self.repo / "skills" / "kvb-org-organize" / "scripts" / "councilroot.py" + settle = "outcome, line = settle(helper, legacy, source, target, catch_up)" + script.write_text(script.read_text(encoding="utf-8").replace(settle, f'if source.name == "zz.json":\n raise RuntimeError("boom")\n {settle}'), encoding="utf-8") + (self.legacy / "rasci.json").write_text("{}\n", encoding="utf-8") + (self.legacy / "zz.json").write_text("{}\n", encoding="utf-8") + done = self.install() + marker = self.durable / "migrate.started" + self.assertEqual(done.returncode, 1, done.stderr) + self.assertIn("RuntimeError: boom", done.stderr) + self.assertIn(f"skills: the council root migration stopped halfway (exit 1): {marker} exists", done.stderr) + self.assertTrue((self.shared / "rasci.json").is_file() and marker.is_file() and (self.legacy / "zz.json").is_file()) + self.assertEqual([(old.parent.parent / "kvb-new" / "SKILL.md").read_text(encoding="utf-8") for old in self.installed], ["new\n"] * 2) + + def test_rows_and_register_changes_written_meanwhile_arrive_or_are_reported_never_lost(self): + (self.legacy / "queue-history.jsonl").write_text('{"id": "early"}\n', encoding="utf-8") + (self.legacy / "news.jsonl").write_text(json.dumps({"from": "A", "text": "before", "ts": 1.0}) + "\n", encoding="utf-8") + (self.legacy / "rasci.json").write_text("{}\n", encoding="utf-8") + window = '{"work/x": {"status": "window"}}\n' + self.env["STUB_LATE"] = json.dumps({ + str(self.legacy / "queue-history.jsonl"): '{"id": "late"}\n', + str(self.legacy / "news.jsonl"): json.dumps({"from": "B", "text": "posted in the window", "ts": 2.0}) + "\n", + str(self.legacy / "rasci.json"): window, + }) + done = self.install() + self.assertEqual(done.returncode, 1, done.stderr) + self.assertEqual(rows(self.shared / "queue-history.jsonl"), ["early", "late"]) + self.assertEqual([json.loads(line)["text"] for line in (self.shared / "news.jsonl").read_text(encoding="utf-8").splitlines()], ["before", "posted in the window"]) + kept = [path for path in self.shared.iterdir() if path.name.startswith("rasci.json.legacy-")] + self.assertEqual([path.read_text(encoding="utf-8") for path in kept], [window]) + self.assertEqual((self.shared / "rasci.json").read_text(encoding="utf-8"), "{}\n") + self.assertIn(f"kept {self.legacy / 'rasci.json'} beside {self.shared / 'rasci.json'} as {kept[0]}", done.stderr) + self.assertFalse(self.legacy.exists()) + + def test_the_skills_are_built_first_and_reach_the_target_only_after_the_legacy_state_moved(self): + (self.legacy / "rasci.json").write_text("{}\n", encoding="utf-8") + done = self.install() + self.assertEqual(done.returncode, 0, done.stderr) + calls = self.record.read_text(encoding="utf-8").splitlines() + self.assertEqual(calls[0], "npx False", done.stderr) + into_target = [call for call in calls if call.startswith("cp ") and call.split(" ", 2)[2].startswith(str(self.home))] + self.assertEqual(into_target, [f"cp True {self.home / root / 'skills'}/" for root in (".agents", ".claude")], done.stderr) + self.assertFalse(self.legacy.exists()) + self.assertEqual([(marker.exists(), (marker.parent.parent / "kvb-new" / "SKILL.md").read_text(encoding="utf-8")) for marker in self.installed], [(False, "new\n")] * 2) + + +class TestKeptCopy(Home): + def setUp(self): + super().setUp() + self.watch = load(WATCH, f"test_council_root_watch_{id(self)}") + + def write_rows(self, path: Path) -> None: + path.write_text('{"id": "r1", "rule": "R", "path": "p", "sha1": "s"}\n{"id": "r2", "rule": "R", "path": "q", "sha1": "s"}\n', encoding="utf-8") + + def test_agents_write_their_root_and_the_host_keeps_the_copies_apart(self): + watch = self.watch + self.assertEqual((watch.ROOT, watch.KEPT), (self.shared, self.durable)) + for origin, kept in watch.MIRRORED: + with self.subTest(name=origin.name): + self.assertEqual((origin.parent, kept.parent, origin.name), (self.shared, self.durable, kept.name)) + queue = load(QUEUE, f"test_council_root_queue_{id(self)}") + self.assertEqual(queue.HISTORIES, (self.shared / "queue-history.jsonl", self.durable / "queue-history.jsonl")) + + def test_the_hook_mirrors_acceptances_and_receipts_into_the_host_root_and_readers_merge_both(self): + watch = self.watch + self.shared.mkdir(parents=True, mode=0o700) + for origin, _ in watch.MIRRORED: + self.write_rows(origin) + watch.keep_history() + for _, kept in watch.MIRRORED: + with self.subTest(name=kept.name): + self.assertEqual((rows(kept), kept.stat().st_mode & 0o777), (["r1", "r2"], 0o600)) + self.assertEqual(self.durable.stat().st_mode & 0o777, 0o700) + with watch.KEPT_ACCEPTED.open("a", encoding="utf-8") as kept: + kept.write('{"id": "r0", "rule": "R", "path": "o", "sha1": "s"}\n') + security = load(SECURITY, f"test_council_root_security_{id(self)}") + self.assertEqual(security.accepted((watch.ACCEPTED, watch.KEPT_ACCEPTED)), {("R", name, "s", None) for name in "opq"}) + + def test_the_mirror_never_follows_a_link_in_the_host_root(self): + watch = self.watch + self.shared.mkdir(parents=True, mode=0o700) + self.durable.mkdir(parents=True, mode=0o700) + self.write_rows(watch.ACCEPTED) + outside = self.base / "outside.jsonl" + outside.write_text("", encoding="utf-8") + watch.KEPT_ACCEPTED.symlink_to(outside) + watch.keep_rows(watch.ACCEPTED, watch.KEPT_ACCEPTED) + self.assertEqual(outside.read_text(encoding="utf-8"), "") + watch.KEPT_ACCEPTED.unlink() + os.link(outside, watch.KEPT_ACCEPTED) + watch.keep_rows(watch.ACCEPTED, watch.KEPT_ACCEPTED) + self.assertEqual(outside.read_text(encoding="utf-8"), "") + watch.KEPT_ACCEPTED.unlink() + elsewhere = self.base / "elsewhere" + elsewhere.mkdir() + self.durable.rmdir() + self.durable.symlink_to(elsewhere) + watch.keep_rows(watch.ACCEPTED, watch.KEPT_ACCEPTED) + self.assertEqual(list(elsewhere.iterdir()), []) + + def test_the_watch_hook_never_copies_a_journal_onto_itself(self): + with mock.patch.dict(os.environ, {"AGENTS_COUNCIL_ROOT": str(self.durable)}): + watch = load(WATCH, f"test_council_root_same_{id(self)}") + self.durable.mkdir(parents=True, mode=0o700) + for origin, _ in watch.MIRRORED: + self.write_rows(origin) + before = {origin: origin.read_bytes() for origin, _ in watch.MIRRORED} + watch.keep_history() + self.assertEqual({origin: origin.read_bytes() for origin, _ in watch.MIRRORED}, before) + + def test_an_explicit_root_still_keeps_its_rows_in_the_durable_one(self): + with mock.patch.dict(os.environ, {"AGENTS_COUNCIL_ROOT": str(self.base / "council")}): + watch = load(WATCH, f"test_council_root_explicit_{id(self)}") + self.assertEqual((watch.ROOT, watch.KEPT), (self.base / "council", self.durable)) + watch.ROOT.mkdir(mode=0o700) + self.write_rows(watch.HISTORY) + watch.keep_history() + self.assertEqual(rows(watch.KEPT_HISTORY), ["r1", "r2"]) + + +class TestHostWriters(Home): + def outside(self) -> Path: + path = self.base / "outside" + path.write_text("", encoding="utf-8") + return path + + def test_the_subagent_log_lies_in_the_host_root_and_never_follows_a_link(self): + payload = json.dumps({"session_id": "s1", "tool_input": {"description": "d", "prompt": "task-class: judgement"}}) + state = self.base / "xdg" + env = {"PATH": os.environ["PATH"], "HOME": str(self.home), "XDG_STATE_HOME": str(state)} + spawn = subprocess.run([sys.executable, str(LOG), "spawn"], input=payload, env=env, capture_output=True, text=True, check=False) + self.assertEqual(spawn.returncode, 0, spawn.stderr) + log = state / "agents-council" / "subagent-model.jsonl" + self.assertEqual(json.loads(log.read_text(encoding="utf-8"))["task_class"], "judgement") + outside = self.outside() + log.unlink() + log.symlink_to(outside) + subprocess.run([sys.executable, str(LOG), "spawn"], input=payload, env=env, capture_output=True, text=True, check=False) + self.assertEqual(outside.read_text(encoding="utf-8"), "") + + def resources(self) -> ModuleType: + with mock.patch.object(sys, "path", [str(CLIENT), *sys.path]): + return load(CLIENT / "resources.py", f"test_council_root_resources_{id(self)}") + + def test_the_resources_sampler_never_follows_a_link(self): + resources = self.resources() + state = self.durable / "resources" + state.mkdir(parents=True, mode=0o700) + outside = self.outside() + (state / "samples.jsonl").symlink_to(outside) + with self.assertRaises(OSError): + resources.append_rows(state / "samples.jsonl", [{"t": 1}]) + self.assertEqual(outside.read_text(encoding="utf-8"), "") + elsewhere = self.base / "elsewhere" + elsewhere.mkdir() + (state / "samples.jsonl").unlink() + state.rmdir() + state.symlink_to(elsewhere) + for write in (lambda: resources.append_rows(state / "tokens.jsonl", [{"t": 1}]), lambda: resources.replace(state / "offsets.json", "{}")): + with self.assertRaises(OSError): + write() + self.assertEqual(list(elsewhere.iterdir()), []) + + def gates(self) -> ModuleType: + return load(GATES, f"test_council_root_gates_{id(self)}") + + def test_gates_db_is_never_written_through_a_link(self): + gates = self.gates() + self.durable.mkdir(parents=True, mode=0o700) + outside = self.outside() + (self.durable / "gates.db").symlink_to(outside) + with self.assertRaises(OSError): + gates.opened(self.durable / "gates.db", True, self.durable) + self.assertEqual(outside.read_bytes(), b"") + + def test_an_unusable_council_root_is_named_in_the_gate_hint_instead_of_a_traceback(self): + target = self.base / "target" + target.mkdir() + linked = self.base / "linked" + linked.symlink_to(target) + with mock.patch.dict(os.environ, {"AGENTS_COUNCIL_ROOT": str(linked)}): + gates = self.gates() + hint = gates.producer(str(self.base), "main", "test", "0" * 40) + self.assertIn(f"{linked} must be a directory of uid {os.getuid()}", hint) + self.assertEqual(list(target.iterdir()), []) + + +class TestRetention(Home): + def setUp(self): + super().setUp() + self.shared.mkdir(parents=True, mode=0o700) + self.radio = load(RADIO, f"test_council_root_radio_{id(self)}") + + def test_an_oversized_radio_log_still_yields_its_newest_rows(self): + row = json.dumps({"from": "A", "to": ["B"], "class": "mayday", "text": "mayday", "ts": 2.0}) + "\n" + filler = json.dumps({"from": "A", "text": "x" * 200, "ts": 1.0}) + "\n" + self.radio.NEWS.write_text(filler * 20 + row, encoding="utf-8") + with mock.patch.object(self.radio.organize("nofollow"), "READ_LIMIT", 1024): + found = self.radio.items() + self.assertEqual(found[-1]["text"], "mayday") + self.assertLess(len(found), 6) + + def test_rows_the_radio_drops_go_to_a_dated_archive_the_readers_skip(self): + with mock.patch.object(self.radio, "KEPT", 3): + for number in range(5): + self.radio.store([{"from": "A", "text": f"m{number}"}]) + self.assertEqual([row["text"] for row in self.radio.items()], ["m2", "m3", "m4"]) + archive = self.shared / "archive" / f"news-{time.strftime('%Y-%m-%d')}.jsonl" + self.assertEqual([json.loads(line)["text"] for line in archive.read_text(encoding="utf-8").splitlines()], ["m0", "m1"]) + self.assertEqual(archive.stat().st_mode & 0o777, 0o600) + + def test_old_gate_logs_and_pages_move_to_a_dated_archive_and_links_stay(self): + old = time.time() - 31 * 86400 + stale = {self.shared / "gates" / "slug" / "gate-a.log", self.shared / "pages" / "branch" / "index.html"} + fresh = self.shared / "gates" / "slug" / "gate-b.log" + for path in (*stale, fresh): + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text("x\n", encoding="utf-8") + for path in stale: + os.utime(path, (old, old)) + outside = self.base / "outside.log" + outside.write_text("keep\n", encoding="utf-8") + os.utime(outside, (old, old)) + (self.shared / "gates" / "linked.log").symlink_to(outside) + done = self.cli("retire") + archive = self.shared / "archive" / time.strftime("%Y-%m-%d") + self.assertEqual((done.returncode, done.stdout), (0, f"archived 2 files into {archive}\n"), done.stderr) + self.assertEqual(sorted(str(path.relative_to(archive)) for path in archive.rglob("*") if path.is_file()), ["gates/slug/gate-a.log", "pages/branch/index.html"]) + self.assertTrue(fresh.is_file() and (self.shared / "gates" / "linked.log").is_symlink()) + self.assertEqual(outside.read_text(encoding="utf-8"), "keep\n") + self.assertIn("ExecStartPre=-/usr/bin/python3 -I %h/.claude/skills/kvb-org-organize/scripts/councilroot.py retire\n", CLEANUP_SERVICE.read_text(encoding="utf-8")) + + +class TestGuard(unittest.TestCase): + def test_no_script_or_skill_text_keeps_persistent_data_in_tmp(self): + hits = [] + for top in SCANNED: + for path in sorted((REPO_ROOT / top).rglob("*")): + if not path.is_file() or "__pycache__" in path.parts: + continue + for number, line in enumerate(path.read_text(encoding="utf-8", errors="replace").splitlines(), 1): + if FORBIDDEN.search(line) and not (path == COUNCIL_ROOT and line == LEGACY_LINE): + hits.append(f"{path.relative_to(REPO_ROOT)}:{number}") + self.assertEqual(hits, []) + + def test_the_legacy_root_is_named_once_as_the_migration_source(self): + found = [line for line in COUNCIL_ROOT.read_text(encoding="utf-8").splitlines() if "/tmp/agents-council" in line] + self.assertEqual(found, [LEGACY_LINE]) + + def test_no_skill_text_turns_a_failed_dir_call_into_a_root_path(self): + hits = [str(path.relative_to(REPO_ROOT)) for path in sorted((REPO_ROOT / "skills").rglob("*.md")) if DIR_SLASH.search(path.read_text(encoding="utf-8"))] + self.assertEqual(hits, []) + + def test_the_agent_root_is_named_where_it_is_inventoried_or_mounted(self): + self.assertIn("~/.local/state/agents-shared", (REPO_ROOT / "skills" / "kvb-cleanup" / "SKILL.md").read_text(encoding="utf-8")) + self.assertIn("~/.local/state/agents-shared.", (CLIENT / "offices.py").read_text(encoding="utf-8")) + mounts = (REPO_ROOT / "docs" / "requirements" / "003-operator-workbench.md").read_text(encoding="utf-8").split("- **Mounts**:", 1)[1].split("\n\n", 1)[0] + self.assertIn("`~/.local/state/agents-shared`", mounts) + self.assertNotIn("the same directory is also", mounts) + + def test_gate_logs_never_go_to_the_temp_directory(self): + found = [str(path.relative_to(REPO_ROOT)) for path in sorted(GATE_SKILL.rglob("*.py")) if "gettempdir" in path.read_text(encoding="utf-8")] + self.assertEqual(found, []) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_gates.py b/tests/test_gates.py index 7188007..4ab3433 100644 --- a/tests/test_gates.py +++ b/tests/test_gates.py @@ -276,7 +276,7 @@ class TestInstalledHooks(Scratch): tree = self.out("write-tree") self.assertIn(f"gates: commit on main needs test (rule of main); no receipt for tree {tree} meets it.", refused.stderr) command = self.produce_command(refused.stderr) - log = self.base / f"gate-{tree[:12]}-test.log" + log = self.council / "gates" / f"gate-{tree[:12]}-test.log" self.assertEqual(command[:6], ["env", "-C", str(self.repo), "INFINITO_TEST_GPU=false", "python3", str(QUEUE)]) self.assertEqual( command[6:], ["hold", "--kind", "test", "--task", "work/gate", "--receipt", "test", "--log", str(log), "--", "make", "test"] diff --git a/tests/test_orchestrator_migration.py b/tests/test_orchestrator_migration.py index f724a72..02da632 100644 --- a/tests/test_orchestrator_migration.py +++ b/tests/test_orchestrator_migration.py @@ -14,8 +14,8 @@ from pathlib import Path REPO_ROOT = Path(__file__).resolve().parents[1] SCRIPT = REPO_ROOT / "scripts" / "orchestrator.sh" HOST_SCRIPTS = ( - "kvb-org-organize/scripts/council.py", "kvb-org-organize/scripts/nofollow.py", "kvb-org-rasci/scripts/rasci.py", - "kvb-org-operator/scripts/operator.py", "kvb-org-radio/scripts/radio.py", + "kvb-org-organize/scripts/council.py", "kvb-org-organize/scripts/nofollow.py", "kvb-org-organize/scripts/councilroot.py", + "kvb-org-rasci/scripts/rasci.py", "kvb-org-operator/scripts/operator.py", "kvb-org-radio/scripts/radio.py", ) OLD = "kvb-hub" NEW = "kvb-orchestrator" @@ -33,6 +33,8 @@ if tool == "docker" and args[:2] == ["image", "inspect"]: elif tool == "docker" and args[:2] == ["image", "rm"]: code = 0 if args[2] in state["images"] else 1 state["images"] = [image for image in state["images"] if image != args[2]] +elif tool == "docker" and args[0] == "compose" and "build" in args: + code = 1 if state.get("fail_build") else 0 elif tool == "docker" and args[0] == "compose" and "up" in args: state["images"] = sorted({*state["images"], "kvb-orchestrator:local"}) elif tool == "claude" and args[:2] == ["mcp", "get"]: @@ -88,6 +90,7 @@ class TestOrchestratorMigration(unittest.TestCase): self.state = self.root / "stub.json" self.state.write_text(json.dumps({"calls": [], "images": [f"{OLD}:local"], "mcp": {OLD: "http://127.0.0.1:8737/mcp"}, "enabled": [f"{OLD}-runner.service"]}), encoding="utf-8") self.env = {**os.environ, "HOME": str(self.home), "PATH": f"{stubs}:{os.environ['PATH']}", "STUB_STATE": str(self.state), "AGENTS_COUNCIL_ROOT": str(self.root / "council"), "PKGMGR_REPO": str(self.root / "no-pkgmgr")} + self.env.pop("XDG_STATE_HOME", None) share, local, config = self.home / ".local" / "share", self.home / ".local" / "state", self.home / ".config" self.write(share / OLD / "compose.yml", f"name: {OLD}\n") self.write(share / OLD / ".env", "KVB_HUB_PORT=8737\n") @@ -146,6 +149,29 @@ class TestOrchestratorMigration(unittest.TestCase): legacy_calls = [call for call in second["calls"][len(first["calls"]) :] if any(OLD in arg for arg in call)] self.assertEqual(legacy_calls, [["docker", "image", "inspect", f"{OLD}:local"], ["claude", "mcp", "get", OLD]]) + def test_install_hands_the_hub_the_agents_root_and_the_host_only_root_apart(self): + del self.env["AGENTS_COUNCIL_ROOT"] + self.install() + state = self.home / ".local" / "state" + env = (self.home / ".local" / "share" / NEW / ".env").read_text(encoding="utf-8").splitlines() + self.assertLessEqual({f"KVB_ORCHESTRATOR_COUNCIL={state / 'agents-shared'}", f"KVB_ORCHESTRATOR_KEPT={state / 'agents-council'}"}, set(env)) + self.assertTrue((state / "agents-council" / "resources").is_dir()) + + def test_the_hub_is_built_before_it_is_replaced_and_a_failed_build_leaves_it_down_and_fails(self): + self.install() + hub = str(self.home / ".local" / "share" / NEW / "compose.yml") + first = json.loads(self.state.read_text(encoding="utf-8")) + self.assertEqual([call[4:] for call in first["calls"] if call[:4] == ["docker", "compose", "--file", hub]], [["build"], ["up", "-d"]]) + self.state.write_text(json.dumps({**first, "calls": [], "fail_build": True}), encoding="utf-8") + done = subprocess.run(["bash", str(SCRIPT), "install"], env=self.env, capture_output=True, text=True, timeout=120, check=False) + self.assertEqual(done.returncode, 1, done.stderr) + self.assertIn("building the Agent Orchestrator failed; it stays down until 'bash scripts/orchestrator.sh install' succeeds", done.stderr) + after = json.loads(self.state.read_text(encoding="utf-8")) + self.assertEqual([call[4:] for call in after["calls"] if call[:4] == ["docker", "compose", "--file", hub]], [["build"], ["stop"]]) + self.assertIn(f"{NEW}-resources.timer", after["enabled"]) + installer = (REPO_ROOT / "scripts" / "install.sh").read_text(encoding="utf-8") + self.assertIn('if ! bash "${REPO_ROOT}/scripts/orchestrator.sh" install; then', installer) + def test_check_reports_the_sessions_timer_the_install_enabled_as_active(self): for tool in ("curl", "journalctl"): (self.root / "bin" / tool).symlink_to("stub") diff --git a/tests/test_queue_cleanup.py b/tests/test_queue_cleanup.py index f7983c8..0cb0d71 100644 --- a/tests/test_queue_cleanup.py +++ b/tests/test_queue_cleanup.py @@ -693,7 +693,13 @@ class TestQueueCleanup(unittest.TestCase): inode = request.stat().st_ino self.cli("request") self.assertEqual((request.stat().st_ino, len(request.read_text(encoding="utf-8").splitlines())), (inode, 2)) - self.assertIn("PathModified=/tmp/agents-council/queue-cleanup.request\n", unit.read_text(encoding="utf-8")) + self.assertIn("PathModified=%S/agents-shared/queue-cleanup.request\n", unit.read_text(encoding="utf-8")) + + def test_agents_request_in_their_root_and_the_round_state_stays_host_only(self): + with mock.patch.dict(os.environ, {"AGENTS_COUNCIL_ROOT": ""}): + cleanup = _load(CLEANUP) + state = self.home / ".local" / "state" + self.assertEqual((cleanup.REQUEST, cleanup.STATE), (state / "agents-shared" / "queue-cleanup.request", self.kept / "queue-cleanup.json")) def test_request_refuses_during_a_round_and_for_30_minutes_after_one(self): self.install_unit() diff --git a/tests/test_score_backfill.py b/tests/test_score_backfill.py index 1bcca93..62ec53e 100644 --- a/tests/test_score_backfill.py +++ b/tests/test_score_backfill.py @@ -139,6 +139,8 @@ class TestBackfill(unittest.TestCase): shutil.copy(BACKFILL.parent / name, scripts / name) shutil.copy(LABELS, scripts.parent / "labels.json") helper = scripts.parents[1] / "kvb-org-organize" / "scripts" / "nofollow.py" + helper.parent.mkdir(parents=True) + shutil.copy(REPO_ROOT / "skills" / "kvb-org-organize" / "scripts" / "councilroot.py", helper.parent / "councilroot.py") council = self.council("missing") env = {"PATH": os.environ["PATH"], "AGENTS_COUNCIL_ROOT": str(council)} done = subprocess.run([sys.executable, "-B", str(scripts / "backfill.py"), "--repo", str(self.fixture.a)], env=env, capture_output=True, text=True, check=False, timeout=HANG) diff --git a/tests/test_score_journal.py b/tests/test_score_journal.py index 5eb5cb3..848ec29 100644 --- a/tests/test_score_journal.py +++ b/tests/test_score_journal.py @@ -26,6 +26,7 @@ REQUIREMENTS = REPO_ROOT / "docs" / "requirements" SCORE = SKILL / "scripts" / "score.py" EFFICIENCY = REPO_ROOT / "skills" / "kvb-check-efficiency" / "scripts" NOFOLLOW = REPO_ROOT / "skills" / "kvb-org-organize" / "scripts" / "nofollow.py" +COUNCIL_ROOT = NOFOLLOW.with_name("councilroot.py") HANG = 20 FORMER = re.compile(r"💸 Cost|⏱️ Time|(?i:💸 cost|⏱️ time|\bcost (?:and|or) time\b|\bcost, time\b|\b(?:cost|time) from the `kvb-check-(?:efficiency|speed)`)") REMOTES = ( @@ -97,6 +98,7 @@ class ScoreCase(unittest.TestCase): organize = self.tmp / "skills" / "kvb-org-organize" / "scripts" organize.mkdir(parents=True) shutil.copy(NOFOLLOW, organize / "nofollow.py") + shutil.copy(COUNCIL_ROOT, organize / "councilroot.py") (skill / "labels.json").write_text(labels, encoding="utf-8") return skill / "scripts" / "score.py" @@ -171,12 +173,14 @@ class TestJournalPlace(unittest.TestCase): def setUp(self): self.journal = module(SKILL / "scripts" / "journal.py") - def test_an_empty_council_root_means_the_default(self): - for value in ("", None): - with self.subTest(value=value), mock.patch.dict(os.environ, {} if value is None else {"AGENTS_COUNCIL_ROOT": value}): - if value is None: - os.environ.pop("AGENTS_COUNCIL_ROOT", None) - self.assertEqual(self.journal.journal_path(), Path("/tmp/agents-council/scores.jsonl")) + def test_an_empty_council_root_means_the_durable_agents_default(self): + with tempfile.TemporaryDirectory() as home: + for value in ("", None): + with self.subTest(value=value), mock.patch.dict(os.environ, {"HOME": home} if value is None else {"HOME": home, "AGENTS_COUNCIL_ROOT": value}): + os.environ.pop("XDG_STATE_HOME", None) + if value is None: + os.environ.pop("AGENTS_COUNCIL_ROOT", None) + self.assertEqual(self.journal.journal_path(), Path(home) / ".local" / "state" / "agents-shared" / "scores.jsonl") def test_ssh_scp_and_https_remotes_share_one_key(self): self.assertEqual({self.journal.identity(url) for url in REMOTES}, {"github.com/org/repo"}) @@ -291,6 +295,18 @@ class TestScoreJournal(ScoreCase): self.assertEqual(len(done.stdout.splitlines()), 7) self.assertFalse(self.council.exists()) + def test_a_missing_council_root_helper_is_named_in_the_one_stderr_line(self): + script = self.copy_skills((SKILL / "labels.json").read_text(encoding="utf-8")) + helper = self.tmp.resolve() / "skills" / "kvb-org-organize" / "scripts" / "councilroot.py" + helper.unlink() + for values, code in (({}, 0), ({"critical": "1"}, 4)): + with self.subTest(values=values): + done = self.score(script, *flags(values), "--message-block", cwd=self.tmp) + self.assertEqual(done.returncode, code) + self.assertEqual(done.stderr, f"score: journal: helper {helper} unusable (FileNotFoundError): install or update kvb-org-organize\n") + self.assertEqual(len(done.stdout.splitlines()), 7) + self.assertFalse(self.council.exists()) + def test_a_branch_or_commit_outside_utf8_keeps_the_score_exit(self): repo = self.tmp / "latin" env = git_env(self.tmp)