Merge branch 'fix/council-root-durable' into unify/skills-20261114
This commit is contained in:
commit
cdd4df5234
51 files changed
+2871
-263
No files matched your search
@@ -464,6 +464,8 @@ make project TARGET=/path/to/repo
|
||||
|
||||
Both commands also patch the target's `.claude/settings.local.json` — the machine-local, untracked file, so a project's reviewed `.claude/settings.json` is never rewritten by an install: the caveman and ponytail plugins are enabled so their modes auto-activate on session start, and two reminder hooks are registered — `kvb-dream` points each agent at `/kvb-dream` at most once every twelve hours (stamped per session in `~/.claude/.dream-stamps/`), and `kvb-coach` runs its short form at the first prompt of every full hour, once across all agents (stamped in `~/.claude/.coach-hours/`); the global install also enables the `kvb-dream-notify.timer` systemd user timer, which recommends `/kvb-dream` in a GNOME notification daily at 17:00 local time and catches up once after a missed run, and the `kvb-org-queue-cleanup.timer` and `kvb-org-queue-cleanup.path` user units, which start a `kvb-org-queue-cleanup` round at 00:00 and 12:00 local time without catch-up and whenever an agent runs `cleanup.py request`. A PreToolUse gate from the `kvb-org-organize` skill blocks edits to files in a git worktree until the session holds a claim (`skills/kvb-org-organize/scripts/council.py`), and its resource watch (`skills/kvb-org-organize/scripts/watch.py`) measures disk, Docker and caches, cleans up low-risk data and blocks new heavy jobs before the disk runs out; it stays inactive until `~/.claude/agents-council/config.json` exists, and because its hooks run that config's commands outside the sandbox, the install adds `Edit(~/.claude/agents-council/**)` to `permissions.deny`, which also covers Write in every permission mode; `Edit(~/.local/state/agents-council/**)` joins it, so the state the skills' scripts keep there changes only through those scripts. The global install writes both rules into `~/.claude/settings.json` too (`scripts/council-deny.js`), the only user-wide settings file, so they bind sessions in every repository. The `kvb-org-radio` skill registers five hooks: its prompt hook and a PostToolBatch hook deliver the immediate radio (`call`, `broadcast`, `security`, `pan-pan`, `mayday`) mid-turn, a Stop hook delivers the post (`post`, `newsletter` and call receipts) when the turn ends, an `asyncRewake` Stop hook wakes an idle session for the radio, and a PreToolUse hook refuses every writing action while a mayday holds, except those of its sender and of the agents that join it with `radio.py assist`. The PreToolUse hook of the `kvb-org-queue` skill sends heavy commands through `queue.py hold` once the config sets `queue_enforce`, and holds a held command to the settings' Bash deny and ask rules, which see only `queue.py hold`. The global install also installs the MEDUSA security scanner with pipx for `kvb-check-security`, unless `medusa` is already on the PATH. Two hooks of the `kvb-org-perform` skill (`skills/kvb-org-perform/scripts/log.py`) log every Agent spawn with its task class, model and effort, and store the quality feedback a sandboxed session prints, in `~/.local/state/agents-council/subagent-model.jsonl`. Every hook runs `/usr/bin/python3 -I`, so neither the `python3` of a sandbox-writable virtualenv on the `PATH` nor `PYTHONPATH` reaches the hooks, which run outside the sandbox; older bare `python3` hook entries are rewritten. The reminder hooks run `/usr/bin/bash` and pin `PATH=/usr/bin:/bin`, and the council scripts keep only the `PATH` directories this uid cannot write (`skills/kvb-org-organize/scripts/hookpath.py`), so no binary an agent plants on the inherited `PATH` runs outside the sandbox. Existing settings are preserved.
|
||||
|
||||
The council root is the one directory every `kvb-org` script, hook and the hub share: the register, radio, queue, claims, operator rows, score journal, gate receipts, security cache and night state, plus `pages/` (the `kvb-git-diff` review pages the hub reads), `approvals/` (the replay stores of `approval.py verify`) and `gates/` (gate logs). `skills/kvb-org-organize/scripts/councilroot.py` alone resolves it: `$AGENTS_COUNCIL_ROOT`, else `$XDG_STATE_HOME/agents-shared`, else `~/.local/state/agents-shared`, so it survives a reboot; `councilroot.py root` prints it and `councilroot.py dir <part>` creates and prints a part. It and `agents-release` below are the only state paths a sandbox's `allowWrite` may name. Its sibling `~/.local/state/agents-council` (`councilroot.py durable`) is host-only and never belongs in `allowWrite`: only hooks, the runner and the systemd units write it (`gates.db`, the watch hook's copies of the receipt, parent, acceptance and queue-history journals, `queue-cleanup.json`, `resources/`, `subagent-model.jsonl`), and the hub mounts it read-only; no directory is both agent-writable and host-written except the council root, where every host writer opens files through `nofollow.py`. The `kvb-release` drafts lie in `agents-release/<version>` beside both (`councilroot.py release <version>`). The global install first runs `councilroot.py preflight` and stops before it copies anything unless `~/.claude/settings.json` lists both `agents-shared` and `agents-release` in `sandbox.filesystem.allowWrite`, no `allowWrite` entry reaches `agents-council` in the managed file, its `managed-settings.d` drop-ins, the user or user-local file (the list `councilroot.py` shares with the queue's permission check), in a project `.claude/settings.json` or `.claude/settings.local.json` below the pkgmgr repositories and worktrees directories (directory and file links followed), or in the start directory of any session listed in `~/.claude/projects`, and the move below would succeed; it warns while the agents are not suspended. Fixing those entries is the operator's step. It then builds the skills, stops the hub, the runner, the offices, resources and queue-cleanup units, runs `councilroot.py migrate`, which moves an earlier council root in `/tmp/agents-council` into the council root, copies the skills, runs `migrate --catch-up` for what the old skills wrote meanwhile, and starts the units again; the hub image is built before the container is replaced, and a failed build leaves the hub down and fails the install until `bash scripts/orchestrator.sh install` succeeds. Migrate refuses while a lock there is held or a queue job waits, and, checked under those locks, moves nothing (exit 4) while a file cannot be moved or a whole file both roots hold differs; the install then stops with the earlier root intact. Each file is renamed to a `.migrating` name before it is copied, so a writer that opens the old name meanwhile makes a new file the catch-up moves. An identical file leaves one copy, a `.jsonl` journal gets the rows it lacks (by id, else each exact line as often as the earlier root holds it more often, in `ts` order), links are skipped, and a second run moves nothing and duplicates no row. The catch-up keeps a differing whole file beside its namesake as `<name>.legacy-<hash>` (exit 3); a file that fails after the check stays behind (exit 5). After exit 3 or 5 the install goes on so the new skills read the new root, names every such file and exits 1 at the end. `~/.local/state/agents-council/migrate.started` exists from the first move to the end of a run; when a crash leaves it, the install copies the new skills anyway, names it and exits 1, and the next completed migrate removes it. Because the council root survives reboots, each `kvb-org-queue-cleanup` round first runs `councilroot.py retire`, which moves gate logs and review pages older than 30 days into `archive/<date>/`, and the radio appends the rows it drops past its last 500 to `archive/news-<date>.jsonl`; no reader opens `archive/`, which `kvb-cleanup` offers for deletion. Run the install with every agent suspended (`kvb-org-suspend`) and the queue empty, so nothing writes either root during the move.
|
||||
|
||||
The global install also runs the Agent Orchestrator, a local review page in which the operator approves or rejects the agents' staged commits (the `kvb-git-diff` skill): it copies `orchestrator/` to `~/.local/share/kvb-orchestrator`, starts it there with `docker compose up -d --build` (restart `unless-stopped`, bound to `127.0.0.1` only, state in `~/.local/state/kvb-orchestrator`) and registers its MCP server for Claude Code at user scope. Its port is `KVB_ORCHESTRATOR_PORT` in `orchestrator/.env`; the hub page is `http://localhost:<port>/`, the MCP endpoint `/mcp`. Without docker the hub is skipped and the commit skills ask in chat. The hub reads new radio traffic aloud in maritime phraseology and takes push-to-talk input; its image builds both engines from pinned, sha256-verified sources and runs each as a separate process: Piper (`piper-tts`, GPL-3.0-or-later) with the `en_US-libritts-high` voice (904 speakers, trained on LibriTTS, CC BY 4.0, from `rhasspy/piper-voices`) and whisper.cpp (MIT) with the multilingual `ggml-base-q8_0` model (OpenAI Whisper weights, MIT, 82 MB). The operator's own voice is `KVB_ORCHESTRATOR_OPERATOR_VOICE` in `orchestrator/.env`. Its Tree reads only the file `pkgmgr branch tree` writes: the install clones `PKGMGR_REPO` (default the GitHub repository, never the sandbox-writable local checkout), builds the newest `v*` tag a fully trusted GPG key signed into `~/.local/share/kvb-orchestrator/pkgmgr`, and enables the `kvb-orchestrator-tree.timer` user timer, which writes `~/.local/state/pkgmgr/tree/branch-tree.json` a minute after its previous run while the hub page polls, on at most half a core; the hub mounts that directory read-only and starts no git. A signed tag without `branch tree` leaves the timer off.
|
||||
|
||||
Stream → Radio (`#stream/radio`) is the hub's radio interface. Its station list shows one line per station, the full name in the tooltip, with its state and a play or stop button; when the browser blocks autoplay, the stream fails or the CSP refuses its origin, the row and a toast say so, and one click starts the station. New stations play daily around the clock. While the hub reads a message aloud, the speaking agent pulses there and in the Orbit and Chord views, under the caption "<Agent> speaking" with the message's first line; under `prefers-reduced-motion` the ring stays still. The speech bubble icon in the header (a hub setting, kept across reloads) puts a bubble at each speaking agent in the Agents graph, 3D, Hierarchy, Radio and Stream views, or in a caption bar at the bottom when the view does not show that agent: the agent being read gets the largest bubble and its words appear in time with the audio, waiting messages show a smaller `…` bubble per agent, and without speech output the same rows run at 180 words a minute, at least 1.5 s each. The composer sends on all seven channels: post and call to one agent, newsletter, broadcast, pan-pan, security and mayday to all. "Record" records with MediaRecorder; the hub transcribes the recording with whisper.cpp off the request thread (at most 120 s of audio, a 120 s timeout, four at a time), keeps it for replay in `~/.local/state/kvb-orchestrator/voice` (the newest 50) and puts the transcript into the field to edit; the message goes out as text, with the recording's host path and sha256 only when "Attach recording" is ticked. The hub transcribes at most six recordings a minute and 600 s of audio an hour. Every operator message is passkey-signed, and the hub verifies the signature against the pinned keys, each once, before it relays; `operator_message(id, text, to)` answers `match` only for the signed text and recipient; for a denial the signature covers the recipient the inbox resolved at signing, and the answer's text is only the signed reason, never the agent's own words the radio row quotes. "Cancel mayday", Evening, Sunset, Sunrise and a settings save that adds a stream URL take the passkey the same way.
|
||||
@@ -476,7 +478,7 @@ The Tasks tab (key 4, `#tasks/list`, `#tasks/board`, `#tasks/rasci`) shows the `
|
||||
|
||||
Without reinstalling the skills, `make orchestrator-install` installs, builds and starts only the hub, `make orchestrator-check` checks the installed hub (image, container, logs, HTTP, pinned keys, hook, runner, tree timer, disk; exit 1 on a failure), `make orchestrator-pin CREDENTIAL=<blob>` pins the passkey that "Register passkey" shows, `make orchestrator-runner` enables the runner service, and `make orchestrator-hook` registers the PermissionRequest hook in `~/.claude/settings.json`, replacing an earlier Agent Orchestrator entry and keeping the file mode, after a copy to `settings.json.before-kvb-orchestrator-hook` (Claude Code reads `~/.claude/settings.local.json` only in sessions started in the home directory). Agents that reach the Docker socket are root-equivalent: until the agent VMs of core requirement 093 isolate them, the signatures do not hold against such an agent. `make watch-check` shows whether the resource watch hook is stuck (running hooks, holders of the kept queue history, age of the stored stage).
|
||||
|
||||
For approvals the hub is only a relay: every approval of an ask, a staged commit or an operator command is signed end to end with a WebAuthn passkey held in KeePassXC (through KeePassXC-Browser, relying party `localhost`, origin `http://localhost:<port>`; ES256, EdDSA and RS256, so a hardware key can be pinned beside it). To register, open the hub at `http://localhost:<port>/` (WebAuthn needs the domain; on `127.0.0.1` the page links there), choose "Register passkey" in the settings and run the host command the page shows, `python3 ~/.local/share/kvb-orchestrator/client/approval.py pin --port <port> --approvers ~/.config/kvb-orchestrator/approvers.json --credential '<blob>'`, in a terminal outside every sandbox: it pins the key in `~/.config/kvb-orchestrator/approvers.json` with mode 0600, a file no sandboxed agent can write and the hub mounts read-only; the hub never pins. Each approval, or one bulk selection, is one signature over the hashes of every item, a nonce and an expiry; each auto-approval rule is signed once. In the inbox, an operator command run with "Run" shows the runner's output in a console under its row until "Close". "Deny" on an ask, an operator row or a rule request signs the denial with its channel and optional reason (a hash no approval shares) and, after the 3 s undo, tells the requesting agent by post or by the radio class picked, never mayday; the hub verifies that signature against the pinned keys before it stores or sends anything, the hook passes the reason on only when it verifies the signature too, and a denied operator row ends as `stopped`. Agents ask for an auto-approval rule with the MCP tool `rule_request` (the rules form's limits; a request stays open for the ask lifetime of one hour, at most 20 in all, and 3 per agent name, which only spreads the queue since MCP names no caller); its row shows a dry run over the last 7 days, and only "Accept" with the passkey stores it. The hub stores no rule, of any kind, whose passkey assertion it cannot verify. The executors verify it themselves against the pinned keys and fail closed: the runner (`client/runner.py`, the `kvb-orchestrator-runner` user service) runs only signed commands, each signature once, recorded in `~/.local/state/kvb-orchestrator-approvals/runner.json`; the PermissionRequest hook (`client/permission_hook.py --port <port> --timeout <seconds> --approvers ~/.config/kvb-orchestrator/approvers.json --replay ~/.local/state/kvb-orchestrator-approvals/hook.json`) allows only a signed item that matches its own tool, command, cwd and session, else it leaves the decision to Claude Code's own prompt; the commit skills run `approval.py verify` on the staged tree before committing, else they ask in the terminal. With no key pinned, every executor refuses; review rejections and a decision a rule takes stay unsigned. A signed rule never covers a command holding `;`, `&`, `|`, a backtick, `$`, `<`, `>`, parentheses or a line break: such a command always asks. The hub page is trusted while it asks for a signature or shows the blob to pin: whoever controls the hub container can show one item and have another signed, or hand over a key of its own. Any local process, sandboxed agents included, can also send the unsigned operator actions (the bare decision choice `/api/decisions/<id>/choose`, which sends no answer, the priority order, settings that add no stream origin, review rejections, the bare ask denial `/api/asks/<id>/deny`, console dismissals) with the hub's guard headers, and the container reads the host `/tmp` for the review pages in `/tmp/kvb-git-diff`. All three close with the agent VMs of core requirement 093, like the Docker-socket gap.
|
||||
For approvals the hub is only a relay: every approval of an ask, a staged commit or an operator command is signed end to end with a WebAuthn passkey held in KeePassXC (through KeePassXC-Browser, relying party `localhost`, origin `http://localhost:<port>`; ES256, EdDSA and RS256, so a hardware key can be pinned beside it). To register, open the hub at `http://localhost:<port>/` (WebAuthn needs the domain; on `127.0.0.1` the page links there), choose "Register passkey" in the settings and run the host command the page shows, `python3 ~/.local/share/kvb-orchestrator/client/approval.py pin --port <port> --approvers ~/.config/kvb-orchestrator/approvers.json --credential '<blob>'`, in a terminal outside every sandbox: it pins the key in `~/.config/kvb-orchestrator/approvers.json` with mode 0600, a file no sandboxed agent can write and the hub mounts read-only; the hub never pins. Each approval, or one bulk selection, is one signature over the hashes of every item, a nonce and an expiry; each auto-approval rule is signed once. In the inbox, an operator command run with "Run" shows the runner's output in a console under its row until "Close". "Deny" on an ask, an operator row or a rule request signs the denial with its channel and optional reason (a hash no approval shares) and, after the 3 s undo, tells the requesting agent by post or by the radio class picked, never mayday; the hub verifies that signature against the pinned keys before it stores or sends anything, the hook passes the reason on only when it verifies the signature too, and a denied operator row ends as `stopped`. Agents ask for an auto-approval rule with the MCP tool `rule_request` (the rules form's limits; a request stays open for the ask lifetime of one hour, at most 20 in all, and 3 per agent name, which only spreads the queue since MCP names no caller); its row shows a dry run over the last 7 days, and only "Accept" with the passkey stores it. The hub stores no rule, of any kind, whose passkey assertion it cannot verify. The executors verify it themselves against the pinned keys and fail closed: the runner (`client/runner.py`, the `kvb-orchestrator-runner` user service) runs only signed commands, each signature once, recorded in `~/.local/state/kvb-orchestrator-approvals/runner.json`; the PermissionRequest hook (`client/permission_hook.py --port <port> --timeout <seconds> --approvers ~/.config/kvb-orchestrator/approvers.json --replay ~/.local/state/kvb-orchestrator-approvals/hook.json`) allows only a signed item that matches its own tool, command, cwd and session, else it leaves the decision to Claude Code's own prompt; the commit skills run `approval.py verify` on the staged tree before committing, else they ask in the terminal. With no key pinned, every executor refuses; review rejections and a decision a rule takes stay unsigned. A signed rule never covers a command holding `;`, `&`, `|`, a backtick, `$`, `<`, `>`, parentheses or a line break: such a command always asks. The hub page is trusted while it asks for a signature or shows the blob to pin: whoever controls the hub container can show one item and have another signed, or hand over a key of its own. Any local process, sandboxed agents included, can also send the unsigned operator actions (the bare decision choice `/api/decisions/<id>/choose`, which sends no answer, the priority order, settings that add no stream origin, review rejections, the bare ask denial `/api/asks/<id>/deny`, console dismissals) with the hub's guard headers, and the container reads the review pages from `pages/` of the council root, which every sandboxed agent can write; it mounts the council root at its host path and no longer reads the host `/tmp`. All three close with the agent VMs of core requirement 093, like the Docker-socket gap.
|
||||
|
||||
Restart your agent afterwards so it loads the new skills.
|
||||
|
||||
|
||||
@@ -61,8 +61,8 @@ changed and what it costs, and a gate means something again.
|
||||
12. **Gates**: a level (none < quick < test < quality-high < e2e) per branch
|
||||
and action (commit, merge, push, pr), inherited from all ancestors, the
|
||||
strictest wins, an own rule wins over all; kept in the local database
|
||||
`~/.local/state/agents-council/gates.db` (the council root lives in the
|
||||
tmpfs `/tmp` and loses it at reboot) that agents read through `gates.py`;
|
||||
`~/.local/state/agents-council/gates.db` (host-only; agents share the
|
||||
council root `~/.local/state/agents-shared`) that agents read through `gates.py`;
|
||||
set in every Artifacts view. Rejected: a committed `.gates.yml`, repo plus
|
||||
local.
|
||||
13. **Hooks with receipts**: pre-commit runs only quick checks of the staged
|
||||
@@ -243,6 +243,9 @@ flowchart TD
|
||||
`gate-parents.jsonl` in the council root, which the watch hook of
|
||||
`kvb-org-organize` keeps in `~/.local/state/agents-council/` beside
|
||||
`gates.db`; reads merge both, and a database row for an edge wins.
|
||||
Amended 2026-11-14: the council root is the durable
|
||||
`~/.local/state/agents-shared` (`councilroot.py root`); the copy stays in
|
||||
the host-only `~/.local/state/agents-council` (`councilroot.py durable`).
|
||||
- [x] `gates.py resolve --repo R --branch B --action A` prints the level and its
|
||||
source: the branch's own rule, else the strictest resolved level of all
|
||||
ancestors, else none.
|
||||
@@ -293,6 +296,10 @@ flowchart TD
|
||||
sandboxed gates write receipts to `/tmp/agents-council` (the council
|
||||
root) and the watch hook of `kvb-org-organize` mirrors them to
|
||||
`~/.local/state/agents-council`, as it does `security-accepted.jsonl`.
|
||||
Amended 2026-11-14 after a host crash wiped `/tmp`: the council root
|
||||
moved to the durable `~/.local/state/agents-shared`, the one state path
|
||||
the sandbox may write; `~/.local/state/agents-council` stays host-only,
|
||||
and the watch hook still mirrors receipts and acceptances into it.
|
||||
(4) Agents commit and push with `--no-verify` per operator rule, so the
|
||||
hooks bind humans and CI; the push hand-over keeps its own green gate on
|
||||
the exact commit.
|
||||
@@ -347,7 +354,8 @@ The work packages meet only through these interfaces:
|
||||
cache_write}`, t the hour), `offsets.json` and `cpu.json` under
|
||||
`~/.local/state/agents-council/resources/`.
|
||||
- **Mounts**: the hub mounts `~/.local/state/agents-council` read-only and
|
||||
opens `gates.db` read-only.
|
||||
opens `gates.db` read-only; the council root `~/.local/state/agents-shared`
|
||||
is mounted read-write at its host path.
|
||||
|
||||
### W6 Speech bubbles
|
||||
|
||||
|
||||
@@ -25,7 +25,8 @@ the burndown, so that I see how quality and cost move alongside the work.
|
||||
[--labels <file>]` parses score blocks from commit messages through
|
||||
`git log` with the label file, keyed by (remote, commit sha), idempotent (a
|
||||
second run adds nothing), runnable on any host, after a reboot (the journal
|
||||
lives in the RAM-backed `/tmp`) or on another PC; no model calls.
|
||||
lived in the RAM-backed `/tmp` until 2026-11-14 and is durable since) or
|
||||
on another PC; no model calls.
|
||||
Rejected: an agent reading the history (tokens); parsing at view time in
|
||||
the hub (the hub would need every repository).
|
||||
5. **Overlays**: the burndown gets one switch per metric (six), all off by
|
||||
|
||||
@@ -9,7 +9,7 @@
|
||||
--replay: this runner's replay store, outside the hub's reach, e.g.
|
||||
~/.local/state/kvb-orchestrator-approvals/offices.json.
|
||||
--council: the council directory every agent shares, e.g.
|
||||
/tmp/agents-council.
|
||||
~/.local/state/agents-shared.
|
||||
--script: the installed council.py of the kvb-org-organize skill.
|
||||
|
||||
Returns:
|
||||
|
||||
@@ -43,7 +43,9 @@ from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import base64
|
||||
import contextlib
|
||||
import datetime
|
||||
import errno
|
||||
import functools
|
||||
import hashlib
|
||||
import json
|
||||
@@ -51,7 +53,6 @@ import os
|
||||
import shutil
|
||||
import stat
|
||||
import subprocess
|
||||
import tempfile
|
||||
import time
|
||||
from collections.abc import Callable
|
||||
from pathlib import Path
|
||||
@@ -367,20 +368,47 @@ def encoded(rows: list[dict]) -> str:
|
||||
return "".join(json.dumps(row, separators=(",", ":")) + "\n" for row in rows)
|
||||
|
||||
|
||||
def folder(path: Path) -> int:
|
||||
return os.open(path.parent, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW | os.O_CLOEXEC)
|
||||
|
||||
|
||||
def append_rows(path: Path, rows: list[dict]) -> None:
|
||||
"""Args:
|
||||
path: the file; a link at it or its directory, or a file that is not
|
||||
regular with a single link, raises OSError.
|
||||
rows: appended as JSON lines.
|
||||
"""
|
||||
if rows:
|
||||
with path.open("a", encoding="utf-8") as handle:
|
||||
parent = folder(path)
|
||||
try:
|
||||
fd = os.open(path.name, os.O_WRONLY | os.O_CREAT | os.O_APPEND | os.O_NOFOLLOW | os.O_NONBLOCK | os.O_CLOEXEC, 0o600, dir_fd=parent)
|
||||
finally:
|
||||
os.close(parent)
|
||||
with open(fd, "a", encoding="utf-8") as handle:
|
||||
info = os.fstat(fd)
|
||||
if not stat.S_ISREG(info.st_mode) or info.st_nlink != 1:
|
||||
raise OSError(errno.EINVAL, "not a regular file with a single link", str(path))
|
||||
handle.write(encoded(rows))
|
||||
|
||||
|
||||
def replace(path: Path, text: str) -> None:
|
||||
with tempfile.NamedTemporaryFile("w", encoding="utf-8", dir=path.parent, prefix=f".{path.name}.", delete=False) as handle:
|
||||
try:
|
||||
"""Args:
|
||||
path: the file, replaced through a fresh temp file beside it; a link
|
||||
at its directory raises OSError, one at path is replaced.
|
||||
text: the new content.
|
||||
"""
|
||||
parent = folder(path)
|
||||
temp = f".{path.name}.{os.urandom(8).hex()}"
|
||||
try:
|
||||
with open(os.open(temp, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW | os.O_CLOEXEC, 0o600, dir_fd=parent), "w", encoding="utf-8") as handle:
|
||||
handle.write(text)
|
||||
except BaseException:
|
||||
os.unlink(handle.name)
|
||||
raise
|
||||
os.replace(handle.name, path)
|
||||
os.replace(temp, path.name, src_dir_fd=parent, dst_dir_fd=parent)
|
||||
except BaseException:
|
||||
with contextlib.suppress(OSError):
|
||||
os.unlink(temp, dir_fd=parent)
|
||||
raise
|
||||
finally:
|
||||
os.close(parent)
|
||||
|
||||
|
||||
def rewrite(path: Path, keep: Callable[[list[dict], float], list[dict]], now: float) -> None:
|
||||
|
||||
@@ -13,6 +13,7 @@ services:
|
||||
read_only: true
|
||||
environment:
|
||||
TMPDIR: /scratch
|
||||
AGENTS_COUNCIL_ROOT: "${KVB_ORCHESTRATOR_COUNCIL:?written into .env by scripts/install.sh}"
|
||||
cap_drop:
|
||||
- ALL
|
||||
security_opt:
|
||||
@@ -36,9 +37,9 @@ services:
|
||||
- --public-port
|
||||
- "${KVB_ORCHESTRATOR_PORT:?set in .env}"
|
||||
- --council
|
||||
- /tmp/agents-council
|
||||
- "${KVB_ORCHESTRATOR_COUNCIL:?written into .env by scripts/install.sh}"
|
||||
- --pages
|
||||
- /tmp/kvb-git-diff
|
||||
- "${KVB_ORCHESTRATOR_COUNCIL:?written into .env by scripts/install.sh}/pages"
|
||||
- --state
|
||||
- /state
|
||||
- --host-state
|
||||
@@ -88,13 +89,9 @@ services:
|
||||
tmpfs:
|
||||
- /scratch:size=64m,mode=1777,noexec,nosuid,nodev
|
||||
volumes:
|
||||
- type: bind
|
||||
source: /tmp
|
||||
target: /tmp
|
||||
read_only: true
|
||||
- type: bind
|
||||
source: "${KVB_ORCHESTRATOR_COUNCIL:?written into .env by scripts/install.sh}"
|
||||
target: /tmp/agents-council
|
||||
target: "${KVB_ORCHESTRATOR_COUNCIL:?written into .env by scripts/install.sh}"
|
||||
bind:
|
||||
create_host_path: false
|
||||
- type: bind
|
||||
|
||||
@@ -48,7 +48,7 @@ def branch_gates(gates: ModuleType, db: sqlite3.Connection, repo: str, branch: s
|
||||
def read_gates(gates: ModuleType | None, db: Path, scan: dict) -> dict:
|
||||
"""Args:
|
||||
gates: the gates.py module, or None.
|
||||
db: gates.db under the read-only mount of ~/.local/state/agents-council.
|
||||
db: gates.db under the read-only mount of the durable council root.
|
||||
scan: the tree; its repositories and branches are the ones shown.
|
||||
|
||||
Returns:
|
||||
|
||||
@@ -25,8 +25,8 @@
|
||||
--tree-file: the JSON file `pkgmgr branch tree --output` writes, mounted
|
||||
read-only; read again whenever it changes.
|
||||
--timers: directory holding the systemd user units (*.timer).
|
||||
--kept: ~/.local/state/agents-council, mounted read-only: gates.db and
|
||||
the resources/ files of the host sampler.
|
||||
--kept: the durable council root (councilroot.py durable), mounted
|
||||
read-only: gates.db and the resources/ files of the host sampler.
|
||||
--vendor: directory holding the vendored Bootstrap Icons sprite,
|
||||
3d-force-graph and three with their licence texts.
|
||||
--cleanup: host-mounted directory the hub writes signed branch deletions
|
||||
|
||||
+68
-3
@@ -23,6 +23,32 @@ if [[ ! -d "${TARGET}" ]]; then
|
||||
exit 2
|
||||
fi
|
||||
|
||||
council_root="${REPO_ROOT}/skills/kvb-org-organize/scripts/councilroot.py"
|
||||
has_user_systemd() { command -v systemctl &>/dev/null && systemctl --user show-environment &>/dev/null; }
|
||||
resume_units() {
|
||||
bash "${REPO_ROOT}/scripts/orchestrator.sh" start || true
|
||||
if has_user_systemd; then systemctl --user start kvb-org-queue-cleanup.timer kvb-org-queue-cleanup.path || true; fi
|
||||
}
|
||||
# Args: extra councilroot.py migrate arguments. Returns its exit code; on 3
|
||||
# or 5 every file it left behind or kept beside its namesake is named on stderr.
|
||||
migrate_council() {
|
||||
local report code=0
|
||||
report="$(python3 "${council_root}" migrate "$@")" || code=$?
|
||||
log "${report}"
|
||||
if ((code == 3 || code == 5)); then
|
||||
warn "skills: the council root migration left these files in the earlier root or beside their namesakes; merge them by hand, the install exits 1 at the end:"
|
||||
grep -E '^(unmoved|conflict|kept) ' <<<"${report}" | sed 's/^/skills: /' >&2 || true
|
||||
fi
|
||||
return "${code}"
|
||||
}
|
||||
status=0
|
||||
if [[ "${TARGET}" == "${HOME}" ]]; then
|
||||
if ! python3 "${council_root}" preflight; then
|
||||
warn "skills: install refused before copying anything; the installed skills keep the earlier council root until the reasons above are gone."
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
cd "${REPO_ROOT}"
|
||||
src="${REPO_ROOT}/.agents/skills"
|
||||
rm -rf "${src}"
|
||||
@@ -39,6 +65,31 @@ if [[ -d "${REPO_ROOT}/skills" ]]; then
|
||||
cp -a "${REPO_ROOT}/skills/." "${src}/"
|
||||
fi
|
||||
|
||||
if [[ "${TARGET}" == "${HOME}" ]]; then
|
||||
install -d -m 0700 "$(python3 "${council_root}" durable)"
|
||||
if has_user_systemd; then
|
||||
systemctl --user stop kvb-org-queue-cleanup.path kvb-org-queue-cleanup.timer kvb-org-queue-cleanup.service &>/dev/null || true
|
||||
fi
|
||||
if ! bash "${REPO_ROOT}/scripts/orchestrator.sh" stop; then
|
||||
resume_units
|
||||
warn "skills: install refused before copying anything: the Agent Orchestrator did not stop."
|
||||
exit 1
|
||||
fi
|
||||
moved=0
|
||||
migrate_council || moved=$?
|
||||
marker="$(python3 "${council_root}" durable)/migrate.started"
|
||||
if [[ -e "${marker}" ]]; then
|
||||
warn "skills: the council root migration stopped halfway (exit ${moved}): ${marker} exists; the install copies the new skills so they read the new root and exits 1 at the end; 'python3 ${council_root} migrate --catch-up' finishes the move and removes ${marker}."
|
||||
status=1
|
||||
elif ((moved == 3 || moved == 5)); then
|
||||
status=1
|
||||
elif ((moved != 0)); then
|
||||
resume_units
|
||||
warn "skills: install refused before copying anything: the council root migration failed (exit ${moved}); run the install again once the reasons above are gone."
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
for root in .agents/skills .claude/skills; do
|
||||
dst="${TARGET}/${root}"
|
||||
log "skills: copying -> ${dst}"
|
||||
@@ -58,6 +109,17 @@ for root in .agents/skills .claude/skills; do
|
||||
fi
|
||||
done
|
||||
|
||||
if [[ "${TARGET}" == "${HOME}" ]]; then
|
||||
caught=0
|
||||
migrate_council --catch-up || caught=$?
|
||||
if ((caught != 0)); then
|
||||
status=1
|
||||
fi
|
||||
if ((caught != 0 && caught != 3 && caught != 5)); then
|
||||
warn "skills: the catch-up council root migration failed (exit ${caught}); what the old skills wrote during the install stays in the earlier root until 'python3 ${council_root} migrate --catch-up' succeeds."
|
||||
fi
|
||||
fi
|
||||
|
||||
settings="${TARGET}/.claude/settings.local.json"
|
||||
if command -v node &>/dev/null; then
|
||||
log "skills: auto-enabling caveman + ponytail plugins in ${settings}"
|
||||
@@ -70,8 +132,7 @@ else
|
||||
fi
|
||||
|
||||
if [[ "${TARGET}" == "${HOME}" ]]; then
|
||||
install -d -m 0700 "${HOME}/.local/state/agents-council"
|
||||
if command -v systemctl &>/dev/null && systemctl --user show-environment &>/dev/null; then
|
||||
if has_user_systemd; then
|
||||
log "skills: enabling the daily kvb-dream notification timer and the kvb-org-queue-cleanup rounds at 00:00, 12:00 and on request"
|
||||
units="${HOME}/.config/systemd/user"
|
||||
mkdir -p "${units}"
|
||||
@@ -83,7 +144,10 @@ if [[ "${TARGET}" == "${HOME}" ]]; then
|
||||
warn "skills: no systemd user session; skipped the kvb-dream notification timer and the kvb-org-queue-cleanup rounds."
|
||||
fi
|
||||
if command -v docker &>/dev/null; then
|
||||
bash "${REPO_ROOT}/scripts/orchestrator.sh" install
|
||||
if ! bash "${REPO_ROOT}/scripts/orchestrator.sh" install; then
|
||||
warn "skills: the Agent Orchestrator is down or partly installed; run 'bash scripts/orchestrator.sh install' once the reasons above are gone."
|
||||
status=1
|
||||
fi
|
||||
bash "${REPO_ROOT}/scripts/orchestrator.sh" runner || true
|
||||
else
|
||||
warn "skills: docker not found; skipped the Agent Orchestrator."
|
||||
@@ -129,3 +193,4 @@ if [[ "${TARGET}" == "${HOME}" ]]; then
|
||||
fi
|
||||
|
||||
log "skills: done. Restart your agent to load the skills."
|
||||
exit "${status}"
|
||||
+47
-14
@@ -1,5 +1,5 @@
|
||||
#!/usr/bin/env bash
|
||||
# Usage: orchestrator.sh install | migrate | check | pin <blob> | runner | hook
|
||||
# Usage: orchestrator.sh install | migrate | stop | start | check | pin <blob> | runner | hook
|
||||
# install migrate, then copy orchestrator/ to ~/.local/share/kvb-orchestrator,
|
||||
# write its .env, build and start the container, register the MCP
|
||||
# server for Claude Code, build pkgmgr from the newest signed tag of
|
||||
@@ -12,12 +12,16 @@
|
||||
# once: container, image, code, state, approvals, pinned keys, MCP
|
||||
# server, PermissionRequest hook, runner unit; a second run changes
|
||||
# nothing
|
||||
# stop stop the container, the runner, the offices path unit and the
|
||||
# resources timer, which write the council roots, keeping them
|
||||
# enabled; exit 1 when the container does not stop
|
||||
# start start them again, the units only while enabled
|
||||
# check image imports, container, logs, HTTP, pinned keys, hook, runner,
|
||||
# disk; exit 1 when a check fails
|
||||
# pin pin the credential blob the "Passkey registrieren" settings show
|
||||
# runner install, enable and restart the runner user service; disable it
|
||||
# and exit 1 while the installed council.py, nofollow.py or
|
||||
# rasci.py differ from this tree
|
||||
# and exit 1 while the installed council.py, nofollow.py,
|
||||
# councilroot.py or rasci.py differ from this tree
|
||||
# hook register the PermissionRequest hook in ~/.claude/settings.json,
|
||||
# replacing an earlier entry, keeping everything else and the file
|
||||
# mode, after a copy to settings.json.before-kvb-orchestrator-hook
|
||||
@@ -31,8 +35,9 @@ STATE="${HOME}/.local/state/${NAME}"
|
||||
APPROVALS="${HOME}/.local/state/${NAME}-approvals"
|
||||
APPROVERS="${HOME}/.config/${NAME}"
|
||||
UNITS="${HOME}/.config/systemd/user"
|
||||
# council.py ROOT: every agent must agree on this directory
|
||||
COUNCIL="${AGENTS_COUNCIL_ROOT:-/tmp/agents-council}"
|
||||
COUNCIL_ROOT="${REPO_ROOT}/skills/kvb-org-organize/scripts/councilroot.py"
|
||||
COUNCIL="$(python3 "${COUNCIL_ROOT}" root)"
|
||||
KEPT="$(python3 "${COUNCIL_ROOT}" durable)"
|
||||
SETTINGS="${HOME}/.claude/settings.json"
|
||||
LOCAL_SETTINGS="${HOME}/.claude/settings.local.json"
|
||||
HOOK_TIMEOUT=110
|
||||
@@ -44,7 +49,7 @@ SESSIONS_UNIT="${NAME}-sessions"
|
||||
RESOURCES_UNIT="${NAME}-resources"
|
||||
FIREWALL_UNIT="${NAME}-firewall"
|
||||
INSTALLED_SKILLS="${HOME}/.claude/skills"
|
||||
HOST_SCRIPTS=(kvb-org-organize/scripts/council.py kvb-org-organize/scripts/nofollow.py kvb-org-rasci/scripts/rasci.py kvb-org-operator/scripts/operator.py kvb-org-radio/scripts/radio.py)
|
||||
HOST_SCRIPTS=(kvb-org-organize/scripts/council.py kvb-org-organize/scripts/nofollow.py kvb-org-organize/scripts/councilroot.py kvb-org-rasci/scripts/rasci.py kvb-org-operator/scripts/operator.py kvb-org-radio/scripts/radio.py)
|
||||
TREE_DIR="${HOME}/.local/state/pkgmgr/tree"
|
||||
PKGMGR_REPO="${PKGMGR_REPO:-https://github.com/kevinveenbirkenbach/package-manager.git}"
|
||||
IMAGE="${NAME}:local"
|
||||
@@ -153,7 +158,7 @@ install_hub() {
|
||||
port="$(port_of "${REPO_ROOT}/orchestrator/.env")"
|
||||
log "skills: starting the Agent Orchestrator on http://localhost:${port}/"
|
||||
rm -rf "${HUB}"
|
||||
mkdir -p "${HUB}" "${STATE}" "${APPROVALS}" "${COUNCIL}" "${HOME}/.claude/projects" "${TREE_DIR}" "${UNITS}" "${APPROVERS}" "${HOME}/.local/state/agents-council/resources"
|
||||
mkdir -p "${HUB}" "${STATE}" "${APPROVALS}" "${COUNCIL}" "${HOME}/.claude/projects" "${TREE_DIR}" "${UNITS}" "${APPROVERS}" "${KEPT}/resources"
|
||||
chmod 700 "${APPROVERS}"
|
||||
cp -a "${REPO_ROOT}/orchestrator/." "${HUB}/"
|
||||
# Python loads a cached .pyc whose header matches the source mtime and size, and cp -a keeps the mtime: one planted in the sandbox-writable tree would run on the host
|
||||
@@ -162,8 +167,12 @@ install_hub() {
|
||||
log "skills: the PermissionRequest hook client is ${HUB}/client/permission_hook.py"
|
||||
log "skills: approvals need a pinned passkey: settings, 'Passkey registrieren', then 'make orchestrator-pin CREDENTIAL=<blob>'; until then every executor refuses"
|
||||
printf 'KVB_ORCHESTRATOR_USER=%s:%s\nKVB_ORCHESTRATOR_STATE=%s\nKVB_ORCHESTRATOR_COUNCIL=%s\nKVB_ORCHESTRATOR_SKILLS=%s\nKVB_ORCHESTRATOR_PROJECTS=%s\nKVB_ORCHESTRATOR_TREE=%s\nKVB_ORCHESTRATOR_TIMERS=%s\nKVB_ORCHESTRATOR_APPROVERS=%s\nKVB_ORCHESTRATOR_KEPT=%s\n' \
|
||||
"$(id -u)" "$(id -g)" "${STATE}" "${COUNCIL}" "${HOME}/.claude/skills" "${HOME}/.claude/projects" "${TREE_DIR}" "${UNITS}" "${APPROVERS}" "${HOME}/.local/state/agents-council" >>"${HUB}/.env"
|
||||
if ! docker compose --file "${HUB}/compose.yml" up -d --build; then
|
||||
"$(id -u)" "$(id -g)" "${STATE}" "${COUNCIL}" "${HOME}/.claude/skills" "${HOME}/.claude/projects" "${TREE_DIR}" "${UNITS}" "${APPROVERS}" "${KEPT}" >>"${HUB}/.env"
|
||||
if ! docker compose --file "${HUB}/compose.yml" build; then
|
||||
docker compose --file "${HUB}/compose.yml" stop || true
|
||||
warn "skills: building the Agent Orchestrator failed; it stays down until 'bash scripts/orchestrator.sh install' succeeds."
|
||||
return 1
|
||||
elif ! docker compose --file "${HUB}/compose.yml" up -d; then
|
||||
warn "skills: docker compose failed; the Agent Orchestrator stays down until 'make orchestrator-install' succeeds."
|
||||
fi
|
||||
if ! command -v claude &>/dev/null; then
|
||||
@@ -255,7 +264,7 @@ resources_timer() {
|
||||
systemctl --user daemon-reload
|
||||
systemctl --user enable --now "${RESOURCES_UNIT}.timer"
|
||||
systemctl --user restart "${RESOURCES_UNIT}.timer"
|
||||
log "skills: ${RESOURCES_UNIT}.timer samples the host and counts the transcripts' tokens into ${HOME}/.local/state/agents-council/resources every minute"
|
||||
log "skills: ${RESOURCES_UNIT}.timer samples the host and counts the transcripts' tokens into ${KEPT}/resources every minute"
|
||||
}
|
||||
|
||||
firewall_unit() {
|
||||
@@ -285,7 +294,7 @@ offices_unit() {
|
||||
fi
|
||||
if ! skills_current; then
|
||||
systemctl --user disable --now "${OFFICES_UNIT}.path" >/dev/null 2>&1 || true
|
||||
warn "skills: ${INSTALLED_SKILLS} holds other council.py, nofollow.py, rasci.py, operator.py or radio.py than this tree; run 'make install' to enable ${OFFICES_UNIT}.path."
|
||||
warn "skills: ${INSTALLED_SKILLS} holds other council.py, nofollow.py, councilroot.py, rasci.py, operator.py or radio.py than this tree; run 'make install' to enable ${OFFICES_UNIT}.path."
|
||||
return 0
|
||||
fi
|
||||
cp -f "${HUB}/systemd/${OFFICES_UNIT}.service" "${HUB}/systemd/${OFFICES_UNIT}.path" "${UNITS}/"
|
||||
@@ -296,7 +305,8 @@ offices_unit() {
|
||||
|
||||
install_all() {
|
||||
migrate || return 0
|
||||
install_hub
|
||||
local hub=0
|
||||
install_hub || hub=1
|
||||
tree_timer
|
||||
sessions_timer
|
||||
resources_timer
|
||||
@@ -306,6 +316,27 @@ install_all() {
|
||||
if ((TOOK_HOOK)); then
|
||||
if [[ -f "${HUB}/client/permission_hook.py" ]]; then hook; else warn "skills: run 'make orchestrator-hook' once the Agent Orchestrator is installed"; fi
|
||||
fi
|
||||
return "${hub}"
|
||||
}
|
||||
|
||||
stop_all() {
|
||||
if [[ -f "${HUB}/compose.yml" ]] && command -v docker &>/dev/null; then
|
||||
docker compose --file "${HUB}/compose.yml" stop || return 1
|
||||
fi
|
||||
if has_systemd; then
|
||||
systemctl --user stop "${RUNNER}" "${OFFICES_UNIT}.path" "${OFFICES_UNIT}.service" "${RESOURCES_UNIT}.timer" "${RESOURCES_UNIT}.service" &>/dev/null || true
|
||||
fi
|
||||
}
|
||||
|
||||
start_all() {
|
||||
if [[ -f "${HUB}/compose.yml" ]] && command -v docker &>/dev/null; then
|
||||
docker compose --file "${HUB}/compose.yml" start || warn "skills: starting the Agent Orchestrator failed"
|
||||
fi
|
||||
if has_systemd; then
|
||||
for unit in "${OFFICES_UNIT}.path" "${RESOURCES_UNIT}.timer" "${RUNNER}"; do
|
||||
if [[ "$(systemctl --user is-enabled "${unit}" 2>/dev/null || true)" == "enabled" ]]; then systemctl --user start "${unit}" || true; fi
|
||||
done
|
||||
fi
|
||||
}
|
||||
|
||||
runner_enabled() {
|
||||
@@ -319,7 +350,7 @@ runner() {
|
||||
fi
|
||||
if ! skills_current; then
|
||||
systemctl --user disable --now "${RUNNER}" >/dev/null 2>&1 || true
|
||||
warn "skills: ${INSTALLED_SKILLS} holds other council.py, nofollow.py, rasci.py, operator.py or radio.py than this tree; run 'make install' to enable ${RUNNER}."
|
||||
warn "skills: ${INSTALLED_SKILLS} holds other council.py, nofollow.py, councilroot.py, rasci.py, operator.py or radio.py than this tree; run 'make install' to enable ${RUNNER}."
|
||||
return 1
|
||||
fi
|
||||
cp -f "${HUB}/systemd/${RUNNER}" "${UNITS}/"
|
||||
@@ -425,12 +456,14 @@ hook() {
|
||||
case "${1:-}" in
|
||||
install) install_all ;;
|
||||
migrate) migrate ;;
|
||||
stop) stop_all ;;
|
||||
start) start_all ;;
|
||||
check) check ;;
|
||||
pin) pin "${2:-}" ;;
|
||||
runner) runner ;;
|
||||
hook) hook ;;
|
||||
*)
|
||||
warn "usage: orchestrator.sh install | migrate | check | pin <blob> | runner | hook"
|
||||
warn "usage: orchestrator.sh install | migrate | stop | start | check | pin <blob> | runner | hook"
|
||||
exit 2
|
||||
;;
|
||||
esac
|
||||
@@ -5,9 +5,9 @@
|
||||
# resource stage. Exit 1 when the stored stage is older than STALE_SECONDS.
|
||||
set -euo pipefail
|
||||
|
||||
# council.py ROOT: every agent must agree on this directory
|
||||
STATE="${AGENTS_COUNCIL_ROOT:-/tmp/agents-council}/watch.json"
|
||||
KEPT="${HOME}/.local/state/agents-council/queue-history.jsonl"
|
||||
COUNCIL_ROOT="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd)/skills/kvb-org-organize/scripts/councilroot.py"
|
||||
STATE="$(python3 "${COUNCIL_ROOT}" root)/watch.json"
|
||||
KEPT="$(python3 "${COUNCIL_ROOT}" durable)/queue-history.jsonl"
|
||||
STALE_SECONDS=600
|
||||
|
||||
printf 'watch hooks running: %s\n' "$(pgrep -fc 'watch.py watch' || true)"
|
||||
|
||||
@@ -28,8 +28,8 @@ commit starts from zero.
|
||||
include or target expression means the list may be short: read that line
|
||||
and name the gap. A project without any target keeps the gate open; ask
|
||||
the operator.
|
||||
3. **Status**: `gate.py status --repo <root>`. `gate: green` ends here; a
|
||||
`missing hotfix` reason means back to step 1.
|
||||
3. **Status**: `gate.py status --repo <root>`. `gate: green` ends here;
|
||||
`missing hotfix` means step 1.
|
||||
4. **Admit**: every gate target is heavy and runs only inside
|
||||
`${CLAUDE_SKILL_DIR}/../kvb-org-queue/scripts/queue.py hold`.
|
||||
Only a single test module
|
||||
@@ -40,17 +40,18 @@ commit starts from zero.
|
||||
background; give the operator `tail -f` on the log:
|
||||
|
||||
```bash
|
||||
queue.py hold --kind <kind> --task <register task> -- gate.py run --repo <root> --target <t> [--target ...] --log-dir /tmp/<slug> [--fail-regex <re> ...]
|
||||
queue.py hold --kind <kind> --task <register task> -- gate.py run --repo <root> --target <t> [--target ...] --log-dir <logs> [--fail-regex <re> ...]
|
||||
```
|
||||
|
||||
`<logs>`: what `${CLAUDE_SKILL_DIR}/../kvb-org-organize/scripts/councilroot.py dir gates <slug>` prints; stop when it fails.
|
||||
|
||||
Pass one `--fail-regex` per failure line the project prints while exiting
|
||||
0; infinito-nexus needs `FAILED TARGETS: \S`. A non-zero exit, a
|
||||
`make: ***` line, a matching regex or modified tracked files make a
|
||||
target red; the last stops the run. Before each target `run` asks
|
||||
`hotfix.py void` for it and every later one. Exit 3 (`void: merge main
|
||||
for hotfix <slug>, then rerun`): back to step 1, then through the queue
|
||||
again. Exit 75: the worktree is
|
||||
busy, the message names the holder; wait, then rerun.
|
||||
again. Exit 75: busy worktree, holder named; wait, rerun.
|
||||
6. **Judge** every red target by its `status` line and the log around it:
|
||||
caused by the change, already red on the default branch, or the
|
||||
environment. Fix the cause and rerun; looking unrelated never makes it
|
||||
|
||||
@@ -28,15 +28,42 @@ Args:
|
||||
command's process group while hold keeps waiting for it. For deploys
|
||||
and every other heavy command that is no `make test*` target.
|
||||
|
||||
The worktree lock is `kvb-gate-run.lock` in the worktree's git dir,
|
||||
taken without waiting and passed to make or the command, so it stays
|
||||
held while they or a process they left running on this host live; a
|
||||
process inside a container (`docker compose exec`) runs on there after
|
||||
they die, and the lock no longer covers it. run and hold write their PID
|
||||
and command into it, and a busy message names them. `hotfix.py sync`
|
||||
takes `kvb-gate-sync.lock` exclusively, recording its PID and command
|
||||
there, then the worktree lock, and releases them in reverse order; run
|
||||
and hold take the worktree lock while holding `kvb-gate-sync.lock`
|
||||
The worktree lock is `kvb-gate-run.lock` in `/tmp/agents-locks/<key>`,
|
||||
key the first 16 hex digits of the SHA-256 of the worktree's real
|
||||
top-level path: a fixed path no environment variable moves, so every
|
||||
caller sharing this /tmp meets the same lock whether the git dir is
|
||||
writable or not; a reboot ends every holder, so it needs no durability.
|
||||
Both directories are created with mode 700 where missing; each level and
|
||||
lock file is opened relative to its parent without following a link and
|
||||
refused unless it is the current user's and not writable by group or
|
||||
others, a lock file a regular file with one link. /tmp belongs to one
|
||||
mount namespace, so run, hold and sync also hold the git dir lock,
|
||||
`kvb-gate-run.lock` in the worktree's git dir: the one inode that a
|
||||
caller with another /tmp (an agentbox box with the project bind-mounted,
|
||||
the host) and a gate.py from before /tmp/agents-locks share. It is
|
||||
created with mode 600 where missing, opened read-only where the git dir
|
||||
or the file refuses writing, opened without following a link and refused
|
||||
unless it is a regular file of the current user with one link; while it
|
||||
is missing and cannot be created, only the /tmp lock guards the worktree
|
||||
and a caller with another /tmp is not kept out. `kvb-gate.json` and
|
||||
`kvb-gate.json.lock` in the git common dir pass the same checks, and
|
||||
`kvb-gate.json` is replaced by a new file, never written in place; run
|
||||
opens the lock, creates and removes a temporary file beside
|
||||
`kvb-gate.json` and reads it before it takes the locks, so a read-only
|
||||
git common dir or a refused `kvb-gate.json` ends it before any target
|
||||
runs. Both
|
||||
locks are taken without waiting and passed to make or the command, so
|
||||
they stay held while they or a process they left running on this host
|
||||
live; a process inside a container (`docker compose exec`) runs on there
|
||||
after they die, and the locks no longer cover it. run and hold write
|
||||
their PID and command into both, the git dir lock only when it opened
|
||||
for writing, and a busy message names them; one on the git dir lock
|
||||
shows its last recorded line as possibly stale, since a holder that
|
||||
opened it read-only records nothing there. `hotfix.py sync` takes
|
||||
`kvb-gate-sync.lock` beside the worktree lock exclusively, recording its
|
||||
PID and command there, then the worktree lock and the git dir lock,
|
||||
recording them in both too, and releases them in reverse
|
||||
order; run and hold take the locks while holding `kvb-gate-sync.lock`
|
||||
shared, so they wait for a running sync, naming it, for at most
|
||||
SYNC_WAIT_SECONDS, and a busy worktree lock then means another run or
|
||||
hold.
|
||||
@@ -58,28 +85,38 @@ Output:
|
||||
|
||||
Exit codes:
|
||||
list: 0 done, 2 usage error.
|
||||
status: 0 gate green, 1 gate open, 2 usage error.
|
||||
status: 0 gate green, 1 gate open, 2 usage error or a `kvb-gate.json`
|
||||
that cannot be opened or trusted.
|
||||
run: 0 every target green, 1 a target red, 2 usage error, an excluded
|
||||
target, dirty tracked files or HEAD moved during a run, 3 void: the remaining targets are
|
||||
neither run nor recorded, 75 (EX_TEMPFAIL) busy: another run or hold
|
||||
holds the worktree, or a sync still holds it after SYNC_WAIT_SECONDS.
|
||||
target, a worktree lock, git dir lock, `kvb-gate.json` or its lock
|
||||
that cannot be created, opened or trusted, a git common dir it cannot
|
||||
write (before any target runs), dirty tracked files or
|
||||
HEAD moved during a run, 3 void: the remaining targets are neither
|
||||
run nor recorded, 75 (EX_TEMPFAIL) busy: another run, hold or sync
|
||||
holds the worktree or its git dir lock, or a sync still holds it
|
||||
after SYNC_WAIT_SECONDS.
|
||||
hold: the command's exit code unchanged, 128+N when signal N ended it;
|
||||
before the command started 2 on a usage error, 75 busy as for run,
|
||||
128+N when signal N reached hold.
|
||||
before the command started 2 on a usage error, a worktree lock or
|
||||
git dir lock that cannot be created, opened or trusted or a command
|
||||
that cannot start, 75 busy as for run, 128+N when signal N reached
|
||||
hold.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import contextlib
|
||||
import errno
|
||||
import fcntl
|
||||
import glob
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import pwd
|
||||
import re
|
||||
import shlex
|
||||
import signal
|
||||
import stat
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
@@ -109,6 +146,12 @@ POLL_SECONDS = 0.1
|
||||
BUSY = os.EX_TEMPFAIL
|
||||
FORWARDED = (signal.SIGINT, signal.SIGTERM, signal.SIGHUP)
|
||||
HOTFIX = Path(__file__).resolve().parents[2] / "kvb-git-hotfix" / "scripts" / "hotfix.py"
|
||||
LOCK_BASE = Path("/tmp/agents-locks")
|
||||
KEY_HEX = 16
|
||||
DIRECTORY_MODE = 0o700
|
||||
LOCK_MODE = 0o600
|
||||
READ_ONLY = (errno.EROFS, errno.EACCES, errno.EPERM)
|
||||
HOLDER_BYTES = 4096
|
||||
CONFIG = Path(pwd.getpwuid(os.getuid()).pw_dir) / ".claude" / "agents-council" / "config.json"
|
||||
EXCLUDED = "gate_excluded_targets"
|
||||
|
||||
@@ -241,21 +284,61 @@ def state_path(repo: Path) -> Path:
|
||||
|
||||
|
||||
def load(path: Path) -> dict:
|
||||
if not path.exists():
|
||||
return {}
|
||||
try:
|
||||
data = json.loads(path.read_text(encoding="utf-8"))
|
||||
except ValueError:
|
||||
fail(f"{path} is not valid JSON; move it aside to start over")
|
||||
handle = os.fdopen(git_file(path, os.O_RDONLY), encoding="utf-8")
|
||||
except FileNotFoundError:
|
||||
return {}
|
||||
except OSError as error:
|
||||
fail(f"cannot read {path}: {error.strerror or error}")
|
||||
with handle:
|
||||
try:
|
||||
data = json.load(handle)
|
||||
except ValueError:
|
||||
fail(f"{path} is not valid JSON; move it aside to start over")
|
||||
if not isinstance(data, dict):
|
||||
fail(f"{path} holds no object; move it aside to start over")
|
||||
return data
|
||||
|
||||
|
||||
def record(path: Path, sha: str, target: str, entry: dict) -> None:
|
||||
def unwritable(path: Path, error: OSError) -> NoReturn:
|
||||
fail(f"cannot write {path}: {error.strerror or error}; nothing ran. Run `gate.py run` as an operator row on the "
|
||||
"host, where the git common dir is writable; `make <target>` alone runs but records nothing")
|
||||
|
||||
|
||||
def state_lock(path: Path) -> TextIO:
|
||||
"""Args:
|
||||
path: STATE in the git common dir.
|
||||
|
||||
Returns:
|
||||
Its lock file, opened by git_file() for reading and writing, once a
|
||||
temporary file could be created and removed beside path and load()
|
||||
read path; exit 2 naming the host-side route to a writable git common
|
||||
dir, or as load() does, otherwise.
|
||||
"""
|
||||
lock_path = path.with_name(f"{STATE}.lock")
|
||||
try:
|
||||
lock = os.fdopen(git_file(lock_path, os.O_RDWR | os.O_CREAT), "r+", encoding="utf-8")
|
||||
except OSError as error:
|
||||
unwritable(lock_path, error)
|
||||
try:
|
||||
handle, temp = tempfile.mkstemp(dir=path.parent, prefix=f".{STATE}.")
|
||||
except OSError as error:
|
||||
lock.close()
|
||||
unwritable(path.parent, error)
|
||||
os.close(handle)
|
||||
os.unlink(temp)
|
||||
try:
|
||||
load(path)
|
||||
except SystemExit:
|
||||
lock.close()
|
||||
raise
|
||||
return lock
|
||||
|
||||
|
||||
def record(lock: TextIO, path: Path, sha: str, target: str, entry: dict) -> None:
|
||||
# Linked worktrees share the git common dir, so parallel runs write this file.
|
||||
with open(path.with_name(f"{STATE}.lock"), "w", encoding="utf-8") as lock:
|
||||
fcntl.flock(lock, fcntl.LOCK_EX)
|
||||
fcntl.flock(lock, fcntl.LOCK_EX)
|
||||
try:
|
||||
data = load(path)
|
||||
data.setdefault(sha, {})[target] = entry
|
||||
handle, temp = tempfile.mkstemp(dir=path.parent, prefix=f".{STATE}.")
|
||||
@@ -263,6 +346,8 @@ def record(path: Path, sha: str, target: str, entry: dict) -> None:
|
||||
json.dump(data, out, indent=1, sort_keys=True)
|
||||
out.write("\n")
|
||||
os.replace(temp, path)
|
||||
finally:
|
||||
fcntl.flock(lock, fcntl.LOCK_UN)
|
||||
|
||||
|
||||
def first_failure(log: Path, checks: list[re.Pattern]) -> str | None:
|
||||
@@ -314,22 +399,26 @@ def claim(lock: TextIO, command: list[str]) -> None:
|
||||
lock: the open, exclusively held lock file.
|
||||
command: the holder's command line.
|
||||
"""
|
||||
lock.seek(0)
|
||||
lock.truncate(0)
|
||||
lock.write(f"{os.getpid()}\t{shlex.join(command)}\n")
|
||||
lock.flush()
|
||||
|
||||
|
||||
def holder(path: Path) -> str:
|
||||
def holder(fd: int) -> str:
|
||||
"""Name the holder that `claim` recorded in a lock file.
|
||||
|
||||
Args:
|
||||
path: the lock file.
|
||||
fd: an open descriptor of the lock file.
|
||||
"""
|
||||
try:
|
||||
pid, _, command = path.read_text(encoding="utf-8").strip().partition("\t")
|
||||
pid, _, command = os.pread(fd, HOLDER_BYTES, 0).decode("utf-8", "replace").strip().partition("\t")
|
||||
except OSError:
|
||||
pid, command = "", ""
|
||||
return f"pid {pid}: {command}" if pid and command else "holder not recorded"
|
||||
if not (pid.isdecimal() and command):
|
||||
return "holder not recorded"
|
||||
shown = "".join(char if char.isprintable() else repr(char)[1:-1] for char in command)
|
||||
return f"pid {pid}: {shown}"
|
||||
|
||||
|
||||
def shared(sync: TextIO, repo: Path, caught: list[int]) -> None:
|
||||
@@ -351,34 +440,169 @@ def shared(sync: TextIO, repo: Path, caught: list[int]) -> None:
|
||||
if caught:
|
||||
sys.exit(128 + caught[0])
|
||||
if time.monotonic() >= deadline:
|
||||
busy(f"a hotfix.py sync still holds {repo} after {SYNC_WAIT_SECONDS} s ({holder(Path(sync.name))}); "
|
||||
busy(f"a hotfix.py sync still holds {repo} after {SYNC_WAIT_SECONDS} s ({holder(sync.fileno())}); "
|
||||
"retry once it ends")
|
||||
if not waiting:
|
||||
print(f"gate: a hotfix.py sync merges in {repo} ({holder(Path(sync.name))}); waiting for it at most "
|
||||
print(f"gate: a hotfix.py sync merges in {repo} ({holder(sync.fileno())}); waiting for it at most "
|
||||
f"{SYNC_WAIT_SECONDS} s", file=sys.stderr, flush=True)
|
||||
waiting = True
|
||||
time.sleep(POLL_SECONDS)
|
||||
|
||||
|
||||
class Busy(Exception):
|
||||
pass
|
||||
|
||||
|
||||
def owned(fd: int, path: Path, regular: bool, private: bool) -> int:
|
||||
"""Args:
|
||||
fd: an open descriptor of path, closed on a refusal.
|
||||
path: named in the refusal.
|
||||
regular: whether path is a lock or state file rather than a directory.
|
||||
private: whether group and others must not be able to write path.
|
||||
|
||||
Returns:
|
||||
fd when it is a directory, or with regular a regular file with one
|
||||
link, of the current user that, with private, group and others cannot
|
||||
write; exit 2 naming path otherwise.
|
||||
"""
|
||||
info = os.fstat(fd)
|
||||
if regular:
|
||||
kind, fits = f"regular file of uid {os.getuid()} with one link", stat.S_ISREG(info.st_mode) and info.st_nlink == 1
|
||||
else:
|
||||
kind, fits = f"directory of uid {os.getuid()}", stat.S_ISDIR(info.st_mode)
|
||||
if fits and info.st_uid == os.getuid() and not (private and info.st_mode & 0o022):
|
||||
return fd
|
||||
os.close(fd)
|
||||
fail(f"{path} must be a {kind}" + (" that group and others cannot write" if private else ""))
|
||||
|
||||
|
||||
def git_file(path: Path, flags: int) -> int:
|
||||
"""Args:
|
||||
path: a lock or state file in a git dir, where a sandboxed caller may
|
||||
have put a link, FIFO or second name.
|
||||
flags: os.open flags; O_NOFOLLOW, O_NONBLOCK and O_CLOEXEC are added,
|
||||
and a created file gets LOCK_MODE.
|
||||
|
||||
Returns:
|
||||
Its descriptor once owned() passes it as a regular file, without the
|
||||
private check; OSError of the open propagates.
|
||||
"""
|
||||
return owned(os.open(path, flags | os.O_NOFOLLOW | os.O_NONBLOCK | os.O_CLOEXEC, LOCK_MODE), path, True, False)
|
||||
|
||||
|
||||
def lock_files(top: Path, base: Path) -> tuple[TextIO, TextIO]:
|
||||
"""Args:
|
||||
top: the worktree's top-level directory, as `git rev-parse
|
||||
--show-toplevel` prints it.
|
||||
base: the directory holding one lock directory per worktree, LOCK_BASE
|
||||
outside tests.
|
||||
|
||||
Returns:
|
||||
RUN_LOCK and SYNC_LOCK of base/<key>, open for reading and writing,
|
||||
key the first KEY_HEX hex digits of the SHA-256 of the real path of
|
||||
top. base and base/<key> are created with DIRECTORY_MODE where
|
||||
missing; each level and lock file is opened relative to its parent's
|
||||
descriptor without following a link and passes owned() before
|
||||
anything is written to it. Exit 2 naming the level otherwise.
|
||||
"""
|
||||
key = hashlib.sha256(os.fsencode(os.path.realpath(top))).hexdigest()[:KEY_HEX]
|
||||
where = base.parent
|
||||
try:
|
||||
fd = os.open(where, os.O_RDONLY | os.O_DIRECTORY | os.O_CLOEXEC)
|
||||
for name in (base.name, key):
|
||||
where = where / name
|
||||
with contextlib.suppress(FileExistsError):
|
||||
os.mkdir(name, DIRECTORY_MODE, dir_fd=fd)
|
||||
parent, fd = fd, os.open(name, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW | os.O_CLOEXEC, dir_fd=fd)
|
||||
os.close(parent)
|
||||
owned(fd, where, False, True)
|
||||
files = []
|
||||
for name in (RUN_LOCK, SYNC_LOCK):
|
||||
where = base / key / name
|
||||
flags = os.O_RDWR | os.O_CREAT | os.O_NOFOLLOW | os.O_NONBLOCK | os.O_CLOEXEC
|
||||
opened = owned(os.open(name, flags, LOCK_MODE, dir_fd=fd), where, True, True)
|
||||
files.append(os.fdopen(opened, "r+", encoding="utf-8"))
|
||||
os.close(fd)
|
||||
except OSError as error:
|
||||
fail(f"cannot open the worktree lock {where}: {error.strerror or error}")
|
||||
return files[0], files[1]
|
||||
|
||||
|
||||
def git_dir_file(path: Path) -> TextIO | None:
|
||||
"""Args:
|
||||
path: RUN_LOCK in a worktree's git dir.
|
||||
|
||||
Returns:
|
||||
path opened by git_file() for reading and writing, created where
|
||||
missing, or read-only where the git dir or the file refuses writing;
|
||||
None when it is missing and cannot be created; exit 2 when it cannot
|
||||
be opened otherwise.
|
||||
"""
|
||||
try:
|
||||
return os.fdopen(git_file(path, os.O_RDWR | os.O_CREAT), "r+", encoding="utf-8")
|
||||
except OSError as error:
|
||||
if error.errno not in READ_ONLY:
|
||||
fail(f"cannot open the git dir lock {path}: {error.strerror or error}")
|
||||
try:
|
||||
return os.fdopen(git_file(path, os.O_RDONLY), "r", encoding="utf-8")
|
||||
except FileNotFoundError:
|
||||
return None
|
||||
except OSError as error:
|
||||
fail(f"cannot open the git dir lock {path}: {error.strerror or error}")
|
||||
|
||||
|
||||
@contextlib.contextmanager
|
||||
def worktree_lock(repo: Path, command: list[str], caught: list[int]) -> Iterator[int]:
|
||||
"""Hold the worktree lock; yields its file descriptor for pass_fds.
|
||||
def git_dir_lock(git_dir: Path, command: list[str]) -> Iterator[int | None]:
|
||||
"""Args:
|
||||
git_dir: the worktree's git dir, one inode for every mount namespace
|
||||
that sees the worktree and for gate.py from before LOCK_BASE.
|
||||
command: the holder's command line, recorded in it when git_dir_file()
|
||||
opened it for writing.
|
||||
|
||||
Returns:
|
||||
A context holding RUN_LOCK of git_dir exclusively and yielding its
|
||||
descriptor, or holding nothing and yielding None when git_dir_file()
|
||||
finds it missing and not creatable; Busy with the line last recorded
|
||||
there, labelled possibly stale, while another run, hold or sync holds it.
|
||||
"""
|
||||
lock = git_dir_file(git_dir / RUN_LOCK)
|
||||
if lock is None:
|
||||
yield None
|
||||
return
|
||||
with lock:
|
||||
try:
|
||||
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
||||
except BlockingIOError:
|
||||
raise Busy(f"last recorded {holder(lock.fileno())}, possibly stale") from None
|
||||
if lock.writable():
|
||||
claim(lock, command)
|
||||
yield lock.fileno()
|
||||
|
||||
|
||||
@contextlib.contextmanager
|
||||
def worktree_lock(repo: Path, command: list[str], caught: list[int]) -> Iterator[tuple[int, ...]]:
|
||||
"""Hold the worktree lock and the git dir lock; yields their file descriptors for pass_fds.
|
||||
|
||||
Args:
|
||||
repo: the worktree whose git dir holds RUN_LOCK and SYNC_LOCK.
|
||||
command: the holder's command line, recorded in RUN_LOCK.
|
||||
repo: the worktree whose lock_files and git_dir_lock are taken.
|
||||
command: the holder's command line, recorded in both RUN_LOCKs.
|
||||
caught: signals the caller caught; the first ends a wait for a sync with 128+N.
|
||||
"""
|
||||
git_dir = Path(git(repo, "rev-parse", "--absolute-git-dir").strip())
|
||||
with open(git_dir / RUN_LOCK, "a", encoding="utf-8") as lock:
|
||||
with open(git_dir / SYNC_LOCK, "a", encoding="utf-8") as sync:
|
||||
run_lock, sync_lock = lock_files(Path(git(repo, "rev-parse", "--show-toplevel").strip()), LOCK_BASE)
|
||||
with run_lock as lock, contextlib.ExitStack() as held:
|
||||
with sync_lock as sync:
|
||||
shared(sync, repo, caught)
|
||||
try:
|
||||
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
||||
except BlockingIOError:
|
||||
busy(f"another gate.py run or hold holds {repo} ({holder(git_dir / RUN_LOCK)})")
|
||||
busy(f"another gate.py run or hold holds {repo} ({holder(lock.fileno())})")
|
||||
try:
|
||||
git_fd = held.enter_context(git_dir_lock(git_dir, command))
|
||||
except Busy as other:
|
||||
busy(f"another gate.py run or hold or a hotfix.py sync holds the git dir lock of {repo} ({other})")
|
||||
claim(lock, command)
|
||||
yield lock.fileno()
|
||||
yield (lock.fileno(),) if git_fd is None else (lock.fileno(), git_fd)
|
||||
|
||||
|
||||
def repository(value: str) -> Path:
|
||||
@@ -483,7 +707,11 @@ def cmd_run(args: argparse.Namespace) -> int:
|
||||
barred = sorted(set(args.target) & excluded())
|
||||
if barred:
|
||||
fail(f"excluded from every gate by {EXCLUDED} in {CONFIG}: {', '.join(barred)}")
|
||||
with worktree_lock(repo, ["gate.py", "run", "--repo", str(repo), *target_flags(args.target)], []) as lock:
|
||||
state = state_path(repo)
|
||||
with (
|
||||
state_lock(state) as state_file,
|
||||
worktree_lock(repo, ["gate.py", "run", "--repo", str(repo), *target_flags(args.target)], []) as locks,
|
||||
):
|
||||
sha = head(repo)
|
||||
dirty, untracked = tree(repo)
|
||||
if dirty:
|
||||
@@ -492,7 +720,6 @@ def cmd_run(args: argparse.Namespace) -> int:
|
||||
print(f"untracked\t{len(untracked)} file(s), not part of the gate")
|
||||
log_dir = Path(args.log_dir).resolve()
|
||||
log_dir.mkdir(parents=True, exist_ok=True)
|
||||
state = state_path(repo)
|
||||
red = False
|
||||
order = list(dict.fromkeys(args.target))
|
||||
checking = hotfix_checks()
|
||||
@@ -505,7 +732,7 @@ def cmd_run(args: argparse.Namespace) -> int:
|
||||
with open(log, "w", encoding="utf-8") as out:
|
||||
code = subprocess.run(
|
||||
["make", "-C", str(repo), target],
|
||||
stdin=subprocess.DEVNULL, stdout=out, stderr=subprocess.STDOUT, pass_fds=(lock,), check=False,
|
||||
stdin=subprocess.DEVNULL, stdout=out, stderr=subprocess.STDOUT, pass_fds=locks, check=False,
|
||||
).returncode
|
||||
if head(repo) != sha:
|
||||
fail(f"HEAD moved during {target}; nothing recorded for it")
|
||||
@@ -525,7 +752,7 @@ def cmd_run(args: argparse.Namespace) -> int:
|
||||
"fail_regex": [c.pattern for c in checks]}
|
||||
if reasons:
|
||||
entry["reason"] = "; ".join(reasons)
|
||||
record(state, sha, target, entry)
|
||||
record(state_file, state, sha, target, entry)
|
||||
print("\t".join([status, target, str(log), cell(entry.get("reason"))]))
|
||||
if changed:
|
||||
fail(f"{target} left tracked files dirty; remaining targets not run")
|
||||
@@ -551,11 +778,11 @@ def cmd_hold(args: argparse.Namespace) -> int:
|
||||
|
||||
for signum in FORWARDED:
|
||||
signal.signal(signum, forward)
|
||||
with worktree_lock(repo, ["gate.py", "hold", "--repo", str(repo), "--", *args.command], caught) as lock:
|
||||
with worktree_lock(repo, ["gate.py", "hold", "--repo", str(repo), "--", *args.command], caught) as locks:
|
||||
if caught:
|
||||
return 128 + caught[0]
|
||||
try:
|
||||
child = subprocess.Popen(args.command, cwd=repo, pass_fds=(lock,), start_new_session=True)
|
||||
child = subprocess.Popen(args.command, cwd=repo, pass_fds=locks, start_new_session=True)
|
||||
except OSError as error:
|
||||
fail(f"cannot start {args.command[0]}: {error.strerror}")
|
||||
for signum in caught:
|
||||
|
||||
@@ -2,11 +2,11 @@
|
||||
"""Gate rules, branch parents and gate receipts of every repository on this host.
|
||||
|
||||
Args:
|
||||
--db: the SQLite database of rules and parents, default
|
||||
~/.local/state/agents-council/gates.db (GATES_DB overrides it); it
|
||||
survives reboots, the council root under /tmp does not. Only rule and
|
||||
parents add|drop|derive write it; every other command opens it
|
||||
read-only and an absent one as empty.
|
||||
--db: the SQLite database of rules and parents, default gates.db in the
|
||||
durable council root (councilroot.py durable of kvb-org-organize;
|
||||
GATES_DB overrides it). Only rule and parents add|drop|derive write
|
||||
it; every other command opens it read-only and an absent one as
|
||||
empty.
|
||||
|
||||
rule set --repo R --branch B --action A --level L: the branch's own level
|
||||
for one action. rule drop --repo R --branch B --action A removes it.
|
||||
@@ -48,7 +48,8 @@ Args:
|
||||
branch's merge level before it leaves. Each passes when a receipt for the tree meets the resolved
|
||||
level, else prints the command that produces it and exits 1:
|
||||
`gates.py quick` for quick; above it `queue.py hold --receipt <level>`
|
||||
of `make <level>` with INFINITO_TEST_GPU=false, held as kind
|
||||
of `make <level>` with INFINITO_TEST_GPU=false and its log in the
|
||||
council root's gates/, which it creates, held as kind
|
||||
quality-high for e2e, else as the level, with the open register task
|
||||
whose branch is B. post-checkout: parents record on a branch
|
||||
checkout; it exits 0 even when recording fails.
|
||||
@@ -63,9 +64,9 @@ Args:
|
||||
Journals:
|
||||
Receipts {id, repo, tree, level, gpu, log, at} and recorded parents {id,
|
||||
repo, branch, parent} are rows of gate-receipts.jsonl and
|
||||
gate-parents.jsonl in the council root (AGENTS_COUNCIL_ROOT, else
|
||||
/tmp/agents-council), which sandboxed agents can write; the watch hook of
|
||||
kvb-org-organize copies them to ~/.local/state/agents-council, and reading
|
||||
gate-parents.jsonl in the council root (councilroot.py root of
|
||||
kvb-org-organize), which sandboxed agents can write; the watch hook of
|
||||
kvb-org-organize copies them to the host-only durable root, and reading
|
||||
merges both by id; a row whose id is no string is skipped. A receipt
|
||||
carries INFINITO_TEST_GPU of the environment that wrote it as gpu; it
|
||||
meets its own level and every lower one, above quick only with gpu
|
||||
@@ -99,7 +100,6 @@ LEVELS = ("none", "quick", "test", "quality-high", "e2e")
|
||||
ACTIONS = ("commit", "merge", "push", "pr")
|
||||
SOURCES = ("recorded", "derived", "operator")
|
||||
DROPPED = "dropped"
|
||||
DEFAULT_DB = Path.home() / ".local" / "state" / "agents-council" / "gates.db"
|
||||
SKILLS = Path(__file__).resolve().parents[2]
|
||||
QUEUE = SKILLS / "kvb-org-queue" / "scripts" / "queue.py"
|
||||
RASCI = SKILLS / "kvb-org-rasci" / "scripts" / "rasci.py"
|
||||
@@ -137,6 +137,8 @@ def module(name: str, path: Path) -> ModuleType:
|
||||
|
||||
|
||||
watch = module("kvb_gates_watch", SKILLS / "kvb-org-organize" / "scripts" / "watch.py")
|
||||
DEFAULT_DB = watch.KEPT / "gates.db"
|
||||
LOGS = "gates"
|
||||
|
||||
|
||||
def fail(message: str) -> None:
|
||||
@@ -250,7 +252,8 @@ def receipted(repo: str, tree: str, level: str) -> bool:
|
||||
def opened(path: Path, write: bool, kept_dir: Path) -> sqlite3.Connection:
|
||||
"""Args:
|
||||
path: gates.db; read-only unless write, in memory when it is missing.
|
||||
write: open it for rule and parent changes.
|
||||
write: open it for rule and parent changes; OSError when path or its
|
||||
directory is a link or path no regular file with a single link.
|
||||
kept_dir: the directory holding the kept gate-parents.jsonl: watch.KEPT
|
||||
on the host, the read-only mount of it in the hub.
|
||||
|
||||
@@ -260,6 +263,7 @@ def opened(path: Path, write: bool, kept_dir: Path) -> sqlite3.Connection:
|
||||
"""
|
||||
if write:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
os.close(watch.nofollow.open_below(path.parent, path, os.O_RDWR | os.O_CREAT, FILE_MODE))
|
||||
db = sqlite3.connect(path)
|
||||
elif path.exists():
|
||||
db = sqlite3.connect(f"{path.resolve().as_uri()}?mode=ro", uri=True)
|
||||
@@ -466,7 +470,10 @@ def task_of(branch: str) -> str:
|
||||
def producer(worktree: str, branch: str, level: str, tree: str) -> str:
|
||||
if level == "quick":
|
||||
return shlex.join(["python3", str(Path(__file__).resolve()), "quick", "--worktree", worktree])
|
||||
log = Path(tempfile.gettempdir()) / f"gate-{tree[:12]}-{level}.log"
|
||||
try:
|
||||
log = watch.councilroot.directory(watch.ROOT, LOGS) / f"gate-{tree[:12]}-{level}.log"
|
||||
except (OSError, ValueError) as error:
|
||||
return f"nothing until the council root is fixed: {error}"
|
||||
return shlex.join([
|
||||
"env", "-C", worktree, f"INFINITO_TEST_GPU={GPU_OFF}", "python3", str(QUEUE), "hold", "--kind", KINDS[level],
|
||||
"--task", task_of(branch), "--receipt", level, "--log", str(log), "--", "make", level,
|
||||
|
||||
@@ -1,16 +1,16 @@
|
||||
# Journal
|
||||
|
||||
Each `--message-block` render appends one row to
|
||||
`${AGENTS_COUNCIL_ROOT:-/tmp/agents-council}/scores.jsonl`: time, origin
|
||||
`scores.jsonl` in the council root that `councilroot.py root` prints: time, origin
|
||||
remote, top level path, branch, `sha` null, the six values and the block's
|
||||
digest. The hub's burndown draws it as overlays; no gate reads it. A failed
|
||||
append prints one stderr line and keeps the exit code; a linked council
|
||||
root or a link, FIFO or other non-regular file at the journal fails it
|
||||
without blocking, through `nofollow.py` of `kvb-org-organize`; when that
|
||||
helper is missing or unusable, the line names it. A new council root gets mode
|
||||
helper or `councilroot.py` is missing or unusable, the line names it. A new council root gets mode
|
||||
0700, a new journal 0600.
|
||||
|
||||
The journal lives in `/tmp`, so a reboot or another PC starts empty. Refill
|
||||
The journal survives a reboot, but another PC starts empty. Refill
|
||||
it from the blocks in git history, without a model:
|
||||
|
||||
```bash
|
||||
|
||||
@@ -217,7 +217,11 @@ def main() -> int:
|
||||
except LookupError as error:
|
||||
print(f"backfill: {error}", file=sys.stderr)
|
||||
return 2
|
||||
path = journal_path()
|
||||
try:
|
||||
path = journal_path()
|
||||
except OSError as error:
|
||||
print(f"backfill: journal: {error}", file=sys.stderr)
|
||||
return 2
|
||||
try:
|
||||
keys = known(path)
|
||||
except (OSError, ValueError) as error:
|
||||
|
||||
@@ -1,9 +1,10 @@
|
||||
"""Labels and score journal shared by score.py and backfill.py of kvb-check-score.
|
||||
|
||||
Environment:
|
||||
AGENTS_COUNCIL_ROOT: council directory shared by every agent on the host;
|
||||
/tmp/agents-council when unset or empty. The journal is its scores.jsonl,
|
||||
display data for the hub's burndown only, never a gate input.
|
||||
AGENTS_COUNCIL_ROOT, XDG_STATE_HOME, HOME: the council directory the
|
||||
kvb-org-organize councilroot.py resolves, shared by every agent on the
|
||||
host. The journal is its scores.jsonl, display data for the hub's
|
||||
burndown only, never a gate input.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -11,7 +12,6 @@ from __future__ import annotations
|
||||
import hashlib
|
||||
import importlib.util
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
from collections.abc import Callable
|
||||
@@ -20,9 +20,10 @@ from types import ModuleType
|
||||
from urllib.parse import urlsplit
|
||||
|
||||
LABELS = Path(__file__).resolve().parents[1] / "labels.json"
|
||||
NOFOLLOW = Path(__file__).resolve().parents[2] / "kvb-org-organize" / "scripts" / "nofollow.py"
|
||||
ORGANIZE = Path(__file__).resolve().parents[2] / "kvb-org-organize" / "scripts"
|
||||
NOFOLLOW = ORGANIZE / "nofollow.py"
|
||||
COUNCIL_ROOT = ORGANIZE / "councilroot.py"
|
||||
JOURNAL = "scores.jsonl"
|
||||
COUNCIL = "/tmp/agents-council"
|
||||
SCP = re.compile(r"(?:[^@/:]+@)?([^/:]+):(?!//)(.+)")
|
||||
FILE_MODE = 0o600
|
||||
|
||||
@@ -39,7 +40,11 @@ def load_labels(path: Path) -> dict:
|
||||
|
||||
|
||||
def journal_path() -> Path:
|
||||
return Path(os.environ.get("AGENTS_COUNCIL_ROOT") or COUNCIL) / JOURNAL
|
||||
"""Returns:
|
||||
scores.jsonl in the council root; OSError naming councilroot.py of
|
||||
kvb-org-organize when it is missing or unusable.
|
||||
"""
|
||||
return helper(COUNCIL_ROOT, "kvb_score_councilroot", lambda loaded: loaded.root()) / JOURNAL
|
||||
|
||||
|
||||
def git(repo: Path | str, *args: str) -> str:
|
||||
@@ -130,13 +135,15 @@ def line(row: dict) -> str:
|
||||
return json.dumps(row, ensure_ascii=False) + "\n"
|
||||
|
||||
|
||||
def unusable(error: Exception) -> OSError:
|
||||
return OSError(f"helper {NOFOLLOW} unusable ({type(error).__name__}): install or update kvb-org-organize")
|
||||
def unusable(path: Path, error: Exception) -> OSError:
|
||||
return OSError(f"helper {path} unusable ({type(error).__name__}): install or update kvb-org-organize")
|
||||
|
||||
|
||||
def nofollow[T](use: Callable[[ModuleType], T]) -> T:
|
||||
def helper[T](path: Path, name: str, use: Callable[[ModuleType], T]) -> T:
|
||||
"""Args:
|
||||
use: what to do with nofollow.py of kvb-org-organize.
|
||||
path: a script of kvb-org-organize.
|
||||
name: the module name it loads under.
|
||||
use: what to do with it.
|
||||
|
||||
Returns:
|
||||
The result of use; its OSError unchanged, any other failure loading
|
||||
@@ -144,17 +151,27 @@ def nofollow[T](use: Callable[[ModuleType], T]) -> T:
|
||||
FileNotFoundError.
|
||||
"""
|
||||
try:
|
||||
spec = importlib.util.spec_from_file_location("kvb_score_nofollow", NOFOLLOW)
|
||||
spec = importlib.util.spec_from_file_location(name, path)
|
||||
loaded = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(loaded)
|
||||
except Exception as error: # noqa: BLE001
|
||||
raise unusable(error) from None
|
||||
raise unusable(path, error) from None
|
||||
try:
|
||||
return use(loaded)
|
||||
except OSError:
|
||||
raise
|
||||
except Exception as error: # noqa: BLE001
|
||||
raise unusable(error) from None
|
||||
raise unusable(path, error) from None
|
||||
|
||||
|
||||
def nofollow[T](use: Callable[[ModuleType], T]) -> T:
|
||||
"""Args:
|
||||
use: what to do with nofollow.py of kvb-org-organize.
|
||||
|
||||
Returns:
|
||||
helper() of it.
|
||||
"""
|
||||
return helper(NOFOLLOW, "kvb_score_nofollow", use)
|
||||
|
||||
|
||||
def append(path: Path, rows: list[dict]) -> None:
|
||||
|
||||
@@ -294,7 +294,11 @@ def journal(args: argparse.Namespace, total: int, block: str) -> None:
|
||||
"fix": plain(args.fix), "break": plain(args.risk), "tokens": plain(args.efficiency),
|
||||
"velocity": plain(args.speed), "security": plain(args.security[0]), "score": total,
|
||||
}
|
||||
path = journal_path()
|
||||
try:
|
||||
path = journal_path()
|
||||
except OSError as error:
|
||||
print(f"score: journal: {error}", file=sys.stderr)
|
||||
return
|
||||
try:
|
||||
here = Path.cwd()
|
||||
append(path, [entry(int(time.time()), origin(here), git_read(here, "branch", "--show-current") or None, None, values, block)])
|
||||
|
||||
@@ -32,6 +32,7 @@ rules forbidding raw commands.
|
||||
joins Step 4) and `worktrees.md` next to this file when present.
|
||||
- **Agent dirs**: pip, npm, pre-commit, act, gh, buildx caches; Ansible and
|
||||
build temp dirs.
|
||||
- **Council root** (`~/.local/state/agents-shared`): `archive/` only.
|
||||
- **Register**: `council.py list --long`; `rasci.py artifacts` (`kvb-org-artefacts`,
|
||||
`--ended` for candidates).
|
||||
- **Queue**: `${CLAUDE_SKILL_DIR}/../kvb-org-queue-cleanup/scripts/cleanup.py
|
||||
|
||||
@@ -9,11 +9,14 @@ approval is the operator's word for one commit, the exact tree it names, only
|
||||
once the operator's pinned passkey signature over it verifies here.
|
||||
|
||||
1. Render the page. `<repo>` is the repository's name, `<branch>` the
|
||||
branch with every `/` as `-`, `<base>` the branch's base and `<table>` a
|
||||
file under `/tmp` holding the kvb-check-score commit table:
|
||||
branch with every `/` as `-`, `<base>` the branch's base, `<table>` a
|
||||
file under `/tmp` holding the kvb-check-score commit table, `<pages>`
|
||||
and `<approvals>` the directories
|
||||
`${CLAUDE_SKILL_DIR}/../kvb-org-organize/scripts/councilroot.py dir pages`
|
||||
and `... dir approvals` print, in the council root the hub reads:
|
||||
|
||||
```bash
|
||||
pkgmgr review --out /tmp/kvb-git-diff/<repo>-<branch>.html --max-file-lines 400 --base <base> --note <table>
|
||||
pkgmgr review --out <pages>/<repo>-<branch>.html --max-file-lines 400 --base <base> --note <table>
|
||||
```
|
||||
|
||||
2. Request the review in the worktree; it ships the staged diff, every file
|
||||
@@ -23,7 +26,7 @@ once the operator's pinned passkey signature over it verifies here.
|
||||
`<agent>` your agent name, which inline feedback is posted to:
|
||||
|
||||
```bash
|
||||
python3 ~/.local/share/kvb-orchestrator/client/approval.py request --port 8737 --title "<subject>" --page /tmp/kvb-git-diff/<repo>-<branch>.html --notes <table> --gate <gate> --score "<score line>" --agent "<agent>"
|
||||
python3 ~/.local/share/kvb-orchestrator/client/approval.py request --port 8737 --title "<subject>" --page <pages>/<repo>-<branch>.html --notes <table> --gate <gate> --score "<score line>" --agent "<agent>"
|
||||
```
|
||||
|
||||
Only a `pass` review can be approved in bulk or by a rule. Tell the
|
||||
@@ -37,7 +40,7 @@ once the operator's pinned passkey signature over it verifies here.
|
||||
first, `<gate>` being the `gate` from step 2:
|
||||
|
||||
```bash
|
||||
python3 ~/.local/share/kvb-orchestrator/client/approval.py verify --port 8737 --approvers ~/.config/kvb-orchestrator/approvers.json --replay /tmp/kvb-orchestrator-approvals/commit.json --review <id> --gate <gate>
|
||||
python3 ~/.local/share/kvb-orchestrator/client/approval.py verify --port 8737 --approvers ~/.config/kvb-orchestrator/approvers.json --replay <approvals>/commit.json --review <id> --gate <gate>
|
||||
```
|
||||
|
||||
Exit 0 only: commit as the calling skill says. Any other exit,
|
||||
|
||||
@@ -46,9 +46,10 @@ Args:
|
||||
tracked tree, merge origin/<default> and each base up the chain, each
|
||||
with `git merge --no-verify --no-edit`. From the tree check to the
|
||||
last merge or abort it holds `kvb-gate-sync.lock`, with its PID and
|
||||
command recorded there, and the worktree lock of the kvb-check-gate
|
||||
skill's gate.py, and refuses while a `gate.py run` or `gate.py hold`
|
||||
holds it, naming the holder. A watch.py busy process it sees in <dir>
|
||||
command recorded there, and the worktree lock and git dir lock of the
|
||||
kvb-check-gate skill's gate.py, and refuses while a `gate.py run` or
|
||||
`gate.py hold` or, by the git dir lock, a sync with another /tmp
|
||||
holds one, naming the holder. A watch.py busy process it sees in <dir>
|
||||
also refuses, but processes in another sandbox's PID namespace stay
|
||||
unseen, so the lock is the guard. The default branch is origin/HEAD; --default is read
|
||||
only when that ref is missing and must agree with it otherwise.
|
||||
@@ -742,27 +743,30 @@ def cmd_void(rasci: ModuleType, args: argparse.Namespace) -> int:
|
||||
|
||||
@contextlib.contextmanager
|
||||
def locked(repo: str, command: list[str]) -> Iterator[None]:
|
||||
"""Hold gate.py's sync lock and worktree lock of the worktree.
|
||||
"""Hold gate.py's sync lock, worktree lock and git dir lock of the worktree.
|
||||
|
||||
Args:
|
||||
repo: the worktree.
|
||||
command: sync's command line, recorded in the sync lock.
|
||||
command: sync's command line, recorded in every lock it holds that it can write.
|
||||
"""
|
||||
if not GATE.is_file():
|
||||
yield
|
||||
return
|
||||
gate = module("gate", GATE)
|
||||
git_dir = Path(git_out(repo, "rev-parse", "--absolute-git-dir").strip())
|
||||
with (
|
||||
open(git_dir / gate.SYNC_LOCK, "a", encoding="utf-8") as sync,
|
||||
open(git_dir / gate.RUN_LOCK, "a", encoding="utf-8") as run,
|
||||
):
|
||||
run_lock, sync_lock = gate.lock_files(Path(git_out(repo, "rev-parse", "--show-toplevel").strip()), gate.LOCK_BASE)
|
||||
with sync_lock as sync, run_lock as run, contextlib.ExitStack() as held:
|
||||
fcntl.flock(sync, fcntl.LOCK_EX)
|
||||
gate.claim(sync, command)
|
||||
try:
|
||||
fcntl.flock(run, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
||||
except BlockingIOError:
|
||||
fail(f"a gate.py run or hold holds {repo} ({gate.holder(git_dir / gate.RUN_LOCK)}); sync once it ends")
|
||||
fail(f"a gate.py run or hold holds {repo} ({gate.holder(run.fileno())}); sync once it ends")
|
||||
try:
|
||||
held.enter_context(gate.git_dir_lock(git_dir, command))
|
||||
except gate.Busy as other:
|
||||
fail(f"a gate.py run or hold or another sync holds the git dir lock of {repo} ({other}); sync once it ends")
|
||||
gate.claim(run, command)
|
||||
yield
|
||||
|
||||
|
||||
|
||||
@@ -26,11 +26,14 @@ hands over, `kvb-git-pr-create`, `kvb-git-update-branches` and
|
||||
output streamed to a log; give the operator `tail -f <log>`:
|
||||
|
||||
```bash
|
||||
queue.py hold --kind <kind> --task <register task> -- python3 <gate.py> hold --repo <root> -- make <target> 2>&1 | tee /tmp/kvb-git-push-<slug>-<N>.log
|
||||
queue.py hold --kind <kind> --task <register task> -- python3 <gate.py> hold --repo <root> -- make <target> 2>&1 | tee <gates>/kvb-git-push-<slug>-<N>.log
|
||||
```
|
||||
|
||||
`<gate.py>` is `${CLAUDE_SKILL_DIR}/../kvb-check-gate/scripts/gate.py`;
|
||||
its worktree lock keeps a sync out while the gate runs.
|
||||
its worktree lock keeps a sync out while the gate runs. `<gates>` is the
|
||||
directory
|
||||
`${CLAUDE_SKILL_DIR}/../kvb-org-organize/scripts/councilroot.py dir gates`
|
||||
prints, so the evidence survives a reboot.
|
||||
5. **Judge** by the gate's own result tables and its failure line
|
||||
(infinito-nexus: the `📊 per-target wall-clock` table and `FAILED
|
||||
TARGETS:`), never by the exit code of a pipe. Green also needs HEAD and
|
||||
|
||||
@@ -14,7 +14,7 @@ outranks them and appoints or recalls at will.
|
||||
## Election
|
||||
|
||||
`${CLAUDE_SKILL_DIR}/scripts/council.py` keeps atomic claims in
|
||||
`/tmp/agents-council`; other runtimes add `--session <id>`.
|
||||
`$(${CLAUDE_SKILL_DIR}/scripts/councilroot.py root)`; other runtimes add `--session <id>`.
|
||||
|
||||
1. Name the office: `work/<slug>` (a piece of work), `cluster/<slug>` (a
|
||||
strand across lines), `<dimension>/<line>` (a line) or `chair`, the slug
|
||||
|
||||
@@ -2,8 +2,8 @@
|
||||
"""Claim, release and list kvb-org-organize offices in the shared council directory.
|
||||
|
||||
Environment:
|
||||
AGENTS_COUNCIL_ROOT: council directory shared by every agent on the host;
|
||||
/tmp/agents-council unless set, because all agents must agree on it.
|
||||
AGENTS_COUNCIL_ROOT, XDG_STATE_HOME, HOME: the council directory
|
||||
councilroot.py resolves, shared by every agent on the host.
|
||||
CLAUDE_CODE_SESSION_ID: the caller's session, taken over --session; other
|
||||
runtimes pass --session.
|
||||
PATH: filtered to the directories hookpath.py trusts before git runs.
|
||||
@@ -44,7 +44,7 @@ def sibling(name: str) -> ModuleType:
|
||||
|
||||
nofollow = sibling("nofollow")
|
||||
|
||||
ROOT = Path(os.environ.get("AGENTS_COUNCIL_ROOT", "/tmp/agents-council"))
|
||||
ROOT = sibling("councilroot").root()
|
||||
HEARTBEAT_LIMIT = 3600
|
||||
OFFICE = re.compile(r"^[a-z0-9][a-z0-9._-]*(/[a-z0-9][a-z0-9._-]*)*$")
|
||||
CLAIM = "claim.json"
|
||||
|
||||
+851
@@ -0,0 +1,851 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Usage:
|
||||
councilroot.py root
|
||||
councilroot.py durable
|
||||
councilroot.py dir PART [PART ...]
|
||||
councilroot.py release VERSION
|
||||
councilroot.py migrate [--legacy DIR] [--catch-up]
|
||||
councilroot.py preflight [--legacy DIR] [--settings FILE] [--managed DIR]
|
||||
councilroot.py retire [--days N]
|
||||
|
||||
Environment:
|
||||
AGENTS_COUNCIL_ROOT: the council root when it is an absolute path.
|
||||
XDG_STATE_HOME: else both roots lie below it, when it is an absolute path.
|
||||
HOME: else both lie below ~/.local/state.
|
||||
AGENTS_COUNCIL_LEGACY: the default DIR of migrate and preflight, LEGACY
|
||||
unless set.
|
||||
|
||||
Args:
|
||||
root: print the council root.
|
||||
durable: print the host-only root; AGENTS_COUNCIL_ROOT never moves it.
|
||||
dir: create <root>/PART/... with mode 700 where missing, refusing a
|
||||
level that is a link, not the current user's or writable by group or
|
||||
others, and print it:
|
||||
pages holds the kvb-git-diff review pages the hub reads, approvals
|
||||
the replay stores of approval.py verify, gates the gate logs.
|
||||
release: create agents-release/VERSION beside the roots with mode 700
|
||||
and print it: the kvb-release drafts, which the release skills write
|
||||
with the Edit and Write tools.
|
||||
migrate: move every regular file below DIR, LEGACY unless given, to the
|
||||
same place below the root and remove the emptied directories, through
|
||||
nofollow.py, so no link on either side is followed; each copy is
|
||||
written to a temp file and linked into place, so it never replaces a
|
||||
file. Refuses, moving nothing, while a *.lock below DIR is held or
|
||||
its queue.json holds a job, and, under those locks and unless
|
||||
--catch-up is given, while a regular file cannot be opened or its
|
||||
place below the root is reached through a link (`unmoved <path>:
|
||||
<reason>`) or a whole file the root holds differs (`conflict <file>
|
||||
differs from <legacy file>`). A file the root already holds is never
|
||||
overwritten: an identical one leaves only the root's copy; a .jsonl
|
||||
journal gets the legacy rows it lacks, by id when every row on both
|
||||
sides carries one, else each exact line as often as the legacy side
|
||||
holds it more often, appended or, when every row carries a ts, in ts
|
||||
order; with --catch-up a differing whole file is copied beside its
|
||||
namesake as <name>.legacy-<12 hex digits of its sha256> (`kept
|
||||
<legacy file> beside <file> as <copy>`). Each file is first renamed
|
||||
to <name>.<16 hex digits>.migrating and copied from there, so a
|
||||
writer that opens the old name meanwhile makes a new file for the
|
||||
catch-up; a later run moves a staged file to its old name. A link or
|
||||
another non-regular entry stays (`skipped <path>: <reason>`); a file
|
||||
that fails after the check stays (`unmoved <path>: <reason>`).
|
||||
<durable>/migrate.started exists from the first move to the end of
|
||||
the run, so after a crash it tells the install the move stopped
|
||||
halfway; a completed run removes it. A second run moves nothing new
|
||||
and duplicates no journal row. Run it while no agent, unit or queue
|
||||
job writes either root; the install runs it again with --catch-up
|
||||
after the copy for what the old skills wrote meanwhile.
|
||||
preflight: print to stderr every reason migrate would refuse, and every
|
||||
missing or wrong sandbox.filesystem.allowWrite entry, all files only
|
||||
read: the root and agents-release must be in the user file
|
||||
(--settings, ~/.claude/settings.json unless given); no entry of it,
|
||||
of the managed and user files settings_files() lists (--managed DIR,
|
||||
/etc/claude-code unless given, with its managed-settings.d drop-ins;
|
||||
the queue's permission check reads the same list), of a
|
||||
.claude/settings.json or .claude/settings.local.json up to 8 levels
|
||||
below the pkgmgr repositories and worktrees directories, directory
|
||||
links followed, or in the start directory of a session listed in
|
||||
~/.claude/projects may be the host-only root, inside it or above it.
|
||||
It warns, without failing, while the agents are not suspended. The
|
||||
global install runs it before it copies anything.
|
||||
retire: move every regular file below <root>/gates and <root>/pages
|
||||
changed more than N days ago (30 unless given) to the same place
|
||||
below <root>/archive/<date>, which no reader opens, never following
|
||||
a link, and print the count.
|
||||
|
||||
Exit codes:
|
||||
0 done, 2 usage error, a refused preflight, a busy legacy root, a root
|
||||
that cannot be created, or a legacy or council root that is a link, not
|
||||
a directory of the current user, writable by group or others, or inside
|
||||
the other, 3 a whole file migrate kept in both roots or beside its
|
||||
namesake, 4 nothing moved for the unmoved and conflict lines printed, 5
|
||||
a partial move: a file that failed after the check stays in DIR; any
|
||||
other code is a crash.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import contextlib
|
||||
import fcntl
|
||||
import hashlib
|
||||
import importlib.util
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import stat
|
||||
import sys
|
||||
import time
|
||||
from collections import Counter
|
||||
from collections.abc import Sequence
|
||||
from pathlib import Path
|
||||
from types import ModuleType
|
||||
|
||||
NAME = "agents-council"
|
||||
SHARED = "agents-shared"
|
||||
RELEASES = "agents-release"
|
||||
QUEUE = "queue.json"
|
||||
LOCK_SUFFIX = ".lock"
|
||||
ARCHIVE = "archive"
|
||||
RETIRED = ("gates", "pages")
|
||||
RETIRE_DAYS = 30
|
||||
DAY_SECONDS = 86400
|
||||
LEGACY = Path("/tmp/agents-council")
|
||||
DIRECTORY_MODE = 0o700
|
||||
PART = re.compile(r"[A-Za-z0-9][A-Za-z0-9._+-]*")
|
||||
JOURNAL_SUFFIX = ".jsonl"
|
||||
FILE_BITS = 0o755
|
||||
MANAGED = Path("/etc/claude-code")
|
||||
MARKER = "migrate.started"
|
||||
STAGED = re.compile(r"\.[0-9a-f]{16}\.migrating\Z")
|
||||
PROJECT_FILES = ("settings.json", "settings.local.json")
|
||||
PROJECT_DEPTH = 8
|
||||
PRUNED = frozenset({".git", "node_modules", ".venv", "venv", "__pycache__", ".tox", ".cache", ".mypy_cache", ".pytest_cache", ".ruff_cache", "site-packages"})
|
||||
PKGMGR_DIRECTORIES = {"repositories": "~/Repositories/", "worktrees": "~/.local/share/worktrees/"}
|
||||
NIGHT = "night.json"
|
||||
SUSPENDED = ("suspend", "sunset")
|
||||
LOCK_WAIT = 30
|
||||
ENCODED = re.compile(r"[^A-Za-z0-9]")
|
||||
|
||||
|
||||
def state_home() -> Path:
|
||||
state = os.environ.get("XDG_STATE_HOME", "")
|
||||
return Path(state) if os.path.isabs(state) else Path.home() / ".local" / "state"
|
||||
|
||||
|
||||
def durable() -> Path:
|
||||
return state_home() / NAME
|
||||
|
||||
|
||||
def root() -> Path:
|
||||
explicit = os.environ.get("AGENTS_COUNCIL_ROOT", "")
|
||||
return Path(explicit) if os.path.isabs(explicit) else state_home() / SHARED
|
||||
|
||||
|
||||
def directory(base: Path, *parts: str) -> Path:
|
||||
"""Args:
|
||||
base: the directory the parts lie below; its missing parents are
|
||||
created with the default mode.
|
||||
parts: path segments, each matching PART.
|
||||
|
||||
Returns:
|
||||
base/parts, each level from base down created with mode 700 where
|
||||
missing; ValueError for a part that does not match PART or a level
|
||||
that refusal() names: a link, not a directory of the current user,
|
||||
or writable by group or others.
|
||||
"""
|
||||
bad = [part for part in parts if not PART.fullmatch(part)]
|
||||
if bad:
|
||||
raise ValueError(f"not a directory name: {bad[0]!r}")
|
||||
base.parent.mkdir(parents=True, exist_ok=True)
|
||||
levels = [base]
|
||||
for part in parts:
|
||||
levels.append(levels[-1] / part)
|
||||
for level in levels:
|
||||
with contextlib.suppress(FileExistsError):
|
||||
level.mkdir(mode=DIRECTORY_MODE)
|
||||
problem = refusal(level)
|
||||
if problem is not None:
|
||||
raise ValueError(problem)
|
||||
return levels[-1]
|
||||
|
||||
|
||||
def nofollow() -> ModuleType:
|
||||
spec = importlib.util.spec_from_file_location("kvb_councilroot_nofollow", Path(__file__).resolve().parent / "nofollow.py")
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(module)
|
||||
return module
|
||||
|
||||
|
||||
def refusal(path: Path) -> str | None:
|
||||
info = path.lstat()
|
||||
if stat.S_ISDIR(info.st_mode) and info.st_uid == os.getuid() and not info.st_mode & 0o022:
|
||||
return None
|
||||
return f"{path} must be a directory of uid {os.getuid()} that group and others cannot write"
|
||||
|
||||
|
||||
def row_ids(text: str) -> list[tuple[str, str]] | None:
|
||||
"""Returns:
|
||||
(id key, line) per non-empty line, the id serialised as JSON; None
|
||||
when a line is no JSON object with an id.
|
||||
"""
|
||||
rows = []
|
||||
for line in text.splitlines():
|
||||
if not line.strip():
|
||||
continue
|
||||
try:
|
||||
row = json.loads(line)
|
||||
except (ValueError, RecursionError):
|
||||
return None
|
||||
if not isinstance(row, dict) or "id" not in row:
|
||||
return None
|
||||
rows.append((json.dumps(row["id"], sort_keys=True), line))
|
||||
return rows
|
||||
|
||||
|
||||
def copied(helper: ModuleType, legacy: Path, source: Path, target_root: Path, target: Path) -> bool:
|
||||
"""Returns:
|
||||
True once source is copied, with its mode and times, to a temp file
|
||||
that is then linked to the name target; False when target exists,
|
||||
which stays untouched.
|
||||
"""
|
||||
*dirs, name = helper.parts_below(target_root, target)
|
||||
with open(helper.open_below(legacy, source, os.O_RDONLY, 0), "rb") as handle:
|
||||
info = os.fstat(handle.fileno())
|
||||
folder = helper.directory(target_root, tuple(dirs), True)
|
||||
temp = f".{name}.{os.urandom(8).hex()}"
|
||||
try:
|
||||
with open(os.open(temp, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW | os.O_CLOEXEC, 0o600, dir_fd=folder), "wb") as out:
|
||||
shutil.copyfileobj(handle, out)
|
||||
out.flush()
|
||||
os.fchmod(out.fileno(), info.st_mode & FILE_BITS)
|
||||
os.utime(out.fileno(), ns=(info.st_atime_ns, info.st_mtime_ns))
|
||||
os.fsync(out.fileno())
|
||||
try:
|
||||
os.link(temp, name, src_dir_fd=folder, dst_dir_fd=folder, follow_symlinks=False)
|
||||
except FileExistsError:
|
||||
return False
|
||||
return True
|
||||
finally:
|
||||
with contextlib.suppress(OSError):
|
||||
os.unlink(temp, dir_fd=folder)
|
||||
os.close(folder)
|
||||
|
||||
|
||||
def content(helper: ModuleType, base: Path, path: Path) -> bytes:
|
||||
with open(helper.open_below(base, path, os.O_RDONLY, 0), "rb") as handle:
|
||||
return handle.read()
|
||||
|
||||
|
||||
def novel(text: str, legacy_text: str) -> tuple[list[str], bool]:
|
||||
"""Returns:
|
||||
The legacy lines text lacks and True when both are id journals: each
|
||||
id once; else each exact line as often as legacy_text holds it more
|
||||
often than text, and False.
|
||||
"""
|
||||
ours, theirs = row_ids(text), row_ids(legacy_text)
|
||||
if ours is not None and theirs is not None:
|
||||
known, fresh = {key for key, _ in ours}, []
|
||||
for key, line in theirs:
|
||||
if key not in known:
|
||||
known.add(key)
|
||||
fresh.append(line)
|
||||
return fresh, True
|
||||
have, fresh = Counter(text.splitlines()), []
|
||||
for line in legacy_text.splitlines():
|
||||
if not line.strip():
|
||||
continue
|
||||
if have[line]:
|
||||
have[line] -= 1
|
||||
else:
|
||||
fresh.append(line)
|
||||
return fresh, False
|
||||
|
||||
|
||||
def in_ts_order(lines: list[str]) -> list[str] | None:
|
||||
"""Returns:
|
||||
lines stably sorted by their numeric ts; None when a line is no JSON
|
||||
object with one.
|
||||
"""
|
||||
stamps = []
|
||||
for line in lines:
|
||||
try:
|
||||
row = json.loads(line)
|
||||
except (ValueError, RecursionError):
|
||||
return None
|
||||
stamp = row.get("ts") if isinstance(row, dict) else None
|
||||
if isinstance(stamp, bool) or not isinstance(stamp, int | float):
|
||||
return None
|
||||
stamps.append(stamp)
|
||||
return [line for _, line in sorted(zip(stamps, lines, strict=True), key=lambda pair: pair[0])]
|
||||
|
||||
|
||||
def merged(helper: ModuleType, legacy: Path, source: Path, target_root: Path, target: Path) -> int:
|
||||
"""Returns:
|
||||
The number of novel() legacy rows target gained, under the flock of
|
||||
target and of its <stem>.lock when one exists (the radio's lock):
|
||||
appended, or for a journal compared by line whose rows all carry a
|
||||
ts and would fall out of order, the whole file replaced in ts order.
|
||||
"""
|
||||
legacy_text = helper.read_text(legacy, source)
|
||||
lock = target.with_suffix(LOCK_SUFFIX)
|
||||
with contextlib.ExitStack() as stack:
|
||||
if lock.is_file() and not lock.is_symlink():
|
||||
stack.enter_context(helper.locked(target_root, lock, 0o600, LOCK_WAIT))
|
||||
handle = stack.enter_context(open(helper.open_below(target_root, target, os.O_RDWR | os.O_APPEND, 0), "a+", encoding="utf-8", errors="replace"))
|
||||
handle.seek(0)
|
||||
fcntl.flock(handle, fcntl.LOCK_EX)
|
||||
text = handle.read()
|
||||
fresh, by_id = novel(text, legacy_text)
|
||||
lines = [line for line in text.splitlines() if line.strip()] + fresh
|
||||
order = None if by_id or not fresh else in_ts_order(lines)
|
||||
if order is None or order == lines:
|
||||
handle.write(("" if not fresh or not text or text.endswith("\n") else "\n") + "".join(f"{line}\n" for line in fresh))
|
||||
else:
|
||||
helper.replace(target_root, target, "".join(f"{line}\n" for line in order), os.fstat(handle.fileno()).st_mode & FILE_BITS)
|
||||
return len(fresh)
|
||||
|
||||
|
||||
def beside(helper: ModuleType, legacy: Path, source: Path, target_root: Path, target: Path) -> Path:
|
||||
"""Returns:
|
||||
target.legacy-<first 12 hex digits of the sha256 of source>, a copy
|
||||
of source beside target unless that name already holds it.
|
||||
"""
|
||||
kept = target.with_name(f"{target.name}.legacy-{hashlib.sha256(content(helper, legacy, source)).hexdigest()[:12]}")
|
||||
copied(helper, legacy, source, target_root, kept)
|
||||
return kept
|
||||
|
||||
|
||||
def destination(legacy: Path, source: Path, target_root: Path) -> Path:
|
||||
"""Returns:
|
||||
The place of source below target_root, a staged() suffix dropped.
|
||||
"""
|
||||
relative = source.relative_to(legacy)
|
||||
return target_root / relative.with_name(STAGED.sub("", relative.name))
|
||||
|
||||
|
||||
def staged(helper: ModuleType, legacy: Path, source: Path) -> Path:
|
||||
"""Returns:
|
||||
source renamed in its directory to <name>.<16 hex digits>.migrating,
|
||||
so a writer that opens the old name afterwards creates a new file the
|
||||
catch-up moves; source itself when it already carries such a name.
|
||||
"""
|
||||
if STAGED.search(source.name):
|
||||
return source
|
||||
*dirs, name = helper.parts_below(legacy, source)
|
||||
stage = f"{name}.{os.urandom(8).hex()}.migrating"
|
||||
fd = helper.directory(legacy, tuple(dirs), False)
|
||||
try:
|
||||
os.rename(name, stage, src_dir_fd=fd, dst_dir_fd=fd)
|
||||
finally:
|
||||
os.close(fd)
|
||||
return source.with_name(stage)
|
||||
|
||||
|
||||
def settle(helper: ModuleType, legacy: Path, source: Path, target_root: Path, catch_up: bool) -> tuple[str, str | None]:
|
||||
"""Args:
|
||||
helper: nofollow.py.
|
||||
legacy: the earlier council root.
|
||||
source: a regular file below it.
|
||||
target_root: the council root.
|
||||
catch_up: keep a differing whole file beside its namesake instead of
|
||||
leaving it in legacy.
|
||||
|
||||
Returns:
|
||||
(moved, merged, kept, conflict or unmoved, report line), the line
|
||||
None for a plain move or an identical copy; every copy is made from
|
||||
the staged() file, which a line naming a file left in legacy names.
|
||||
"""
|
||||
target, stage = destination(legacy, source, target_root), source
|
||||
try:
|
||||
stage = staged(helper, legacy, source)
|
||||
if copied(helper, legacy, stage, target_root, target) or content(helper, legacy, stage) == content(helper, target_root, target):
|
||||
helper.unlink(legacy, stage)
|
||||
return "moved", None
|
||||
if target.suffix == JOURNAL_SUFFIX:
|
||||
count = merged(helper, legacy, stage, target_root, target)
|
||||
helper.unlink(legacy, stage)
|
||||
return "merged", f"merged {count} rows of {source} into {target}"
|
||||
if catch_up:
|
||||
kept = beside(helper, legacy, stage, target_root, target)
|
||||
helper.unlink(legacy, stage)
|
||||
return "kept", f"kept {source} beside {target} as {kept}"
|
||||
return "conflict", f"conflict {target} differs from {stage}; both kept"
|
||||
except OSError as error:
|
||||
return "unmoved", f"unmoved {stage}: {error.strerror or error}"
|
||||
|
||||
|
||||
def held(helper: ModuleType, legacy: Path, stack: contextlib.ExitStack) -> list[str]:
|
||||
"""Args:
|
||||
helper: nofollow.py.
|
||||
legacy: the earlier council root.
|
||||
stack: keeps every lock this takes until the move ends.
|
||||
|
||||
Returns:
|
||||
Why the move must wait: each *.lock below legacy another process
|
||||
holds, and a queue.json with jobs or that cannot be read.
|
||||
"""
|
||||
problems = []
|
||||
for folder, _, files in os.walk(legacy):
|
||||
for name in files:
|
||||
path = Path(folder) / name
|
||||
if not name.endswith(LOCK_SUFFIX) or not stat.S_ISREG(path.lstat().st_mode):
|
||||
continue
|
||||
try:
|
||||
lock = helper.open_below(legacy, path, os.O_RDONLY, 0)
|
||||
stack.callback(os.close, lock)
|
||||
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
||||
except BlockingIOError:
|
||||
problems.append(f"{path} is held")
|
||||
except OSError as error:
|
||||
problems.append(f"{path} cannot be locked: {error.strerror or error}")
|
||||
queue = legacy / QUEUE
|
||||
if queue.exists() or queue.is_symlink():
|
||||
try:
|
||||
entries = json.loads(helper.read_text(legacy, queue)).get("entries")
|
||||
except (OSError, ValueError, AttributeError):
|
||||
entries = None
|
||||
if not isinstance(entries, list):
|
||||
problems.append(f"{queue} is unreadable")
|
||||
elif entries:
|
||||
problems.append(f"{queue} holds {len(entries)} queue jobs")
|
||||
return problems
|
||||
|
||||
|
||||
def prune(helper: ModuleType, legacy: Path) -> None:
|
||||
for folder, _, _ in os.walk(legacy, topdown=False):
|
||||
here = Path(folder)
|
||||
if here == legacy:
|
||||
continue
|
||||
*dirs, name = helper.parts_below(legacy, here)
|
||||
with contextlib.suppress(OSError):
|
||||
fd = helper.directory(legacy, tuple(dirs), False)
|
||||
try:
|
||||
os.rmdir(name, dir_fd=fd)
|
||||
finally:
|
||||
os.close(fd)
|
||||
with contextlib.suppress(OSError):
|
||||
os.rmdir(legacy)
|
||||
|
||||
|
||||
class Blocked(Exception):
|
||||
"""Args:
|
||||
lines: the conflicts() lines that kept migrate from moving anything.
|
||||
"""
|
||||
|
||||
def __init__(self, lines: list[str]) -> None:
|
||||
super().__init__("; ".join(lines))
|
||||
self.lines = lines
|
||||
|
||||
|
||||
def migrate(legacy: Path, target: Path, catch_up: bool = False) -> list[str]:
|
||||
"""Args:
|
||||
legacy: the earlier council root.
|
||||
target: the council root.
|
||||
catch_up: move what conflicts() names too, keeping a differing whole
|
||||
file beside its namesake, instead of refusing.
|
||||
|
||||
Returns:
|
||||
The report lines, the moved count first, the rest sorted; ValueError
|
||||
naming the refused root when either is a link, not a directory of the
|
||||
current user, writable by group or others, or lies inside the other,
|
||||
and naming every reason held() finds, moving nothing; without
|
||||
catch_up, Blocked naming every conflicts() line found under the
|
||||
held locks, moving nothing.
|
||||
"""
|
||||
helper = nofollow()
|
||||
if not legacy.exists() and not legacy.is_symlink():
|
||||
unmark(helper)
|
||||
return []
|
||||
problem = refusal(legacy)
|
||||
if problem is not None:
|
||||
raise ValueError(problem)
|
||||
if legacy.resolve() == target.resolve():
|
||||
return []
|
||||
with contextlib.ExitStack() as stack:
|
||||
problems = held(helper, legacy, stack)
|
||||
if problems:
|
||||
raise ValueError("; ".join(problems) + "; nothing moved")
|
||||
blocked = [] if catch_up else conflicts(helper, legacy, target)
|
||||
if blocked:
|
||||
raise Blocked(blocked)
|
||||
return move(legacy, target, catch_up)
|
||||
|
||||
|
||||
def move(legacy: Path, target: Path, catch_up: bool) -> list[str]:
|
||||
target.mkdir(mode=DIRECTORY_MODE, parents=True, exist_ok=True)
|
||||
problem = refusal(target)
|
||||
if problem is not None:
|
||||
raise ValueError(problem)
|
||||
old, new = legacy.resolve(), target.resolve()
|
||||
if old.is_relative_to(new) or new.is_relative_to(old):
|
||||
raise ValueError(f"{legacy} and {target} must not lie inside each other")
|
||||
helper = nofollow()
|
||||
host = directory(durable())
|
||||
helper.replace(host, host / MARKER, f"{legacy} -> {target}\n", 0o600)
|
||||
moved, lines = 0, []
|
||||
for folder, folders, files in os.walk(legacy):
|
||||
here = Path(folder)
|
||||
lines += [f"skipped {here / name}: a link" for name in folders if (here / name).is_symlink()]
|
||||
for name in sorted(files):
|
||||
source = here / name
|
||||
try:
|
||||
regular = stat.S_ISREG(source.lstat().st_mode)
|
||||
except OSError:
|
||||
continue
|
||||
if not regular:
|
||||
lines.append(f"skipped {source}: not a regular file")
|
||||
continue
|
||||
outcome, line = settle(helper, legacy, source, target, catch_up)
|
||||
moved += outcome == "moved"
|
||||
if line is not None:
|
||||
lines.append(line)
|
||||
prune(helper, legacy)
|
||||
unmark(helper)
|
||||
head = [f"moved {moved} files from {legacy} to {target}"] if moved else []
|
||||
return head + sorted(lines)
|
||||
|
||||
|
||||
def unmark(helper: ModuleType) -> None:
|
||||
with contextlib.suppress(FileNotFoundError):
|
||||
helper.unlink(durable(), durable() / MARKER)
|
||||
|
||||
|
||||
def unmovable(helper: ModuleType, legacy: Path, source: Path, target: Path, twin: Path) -> str | None:
|
||||
"""Returns:
|
||||
The `unmoved` line migrate would print because source cannot be
|
||||
opened as it opens it, or a directory on the way to twin is a link
|
||||
or no directory; None when nothing stops the copy.
|
||||
"""
|
||||
try:
|
||||
os.close(helper.open_below(legacy, source, os.O_RDONLY, 0))
|
||||
*dirs, _ = helper.parts_below(target, twin)
|
||||
with contextlib.suppress(FileNotFoundError):
|
||||
os.close(helper.directory(target, tuple(dirs), False))
|
||||
except OSError as error:
|
||||
return f"unmoved {source}: {error.strerror or error}"
|
||||
return None
|
||||
|
||||
|
||||
def conflicts(helper: ModuleType, legacy: Path, target: Path) -> list[str]:
|
||||
"""Returns:
|
||||
A line per regular file below legacy that migrate could not move, or
|
||||
that is no .jsonl journal and whose namesake below target differs.
|
||||
"""
|
||||
lines = []
|
||||
for folder, _, files in os.walk(legacy):
|
||||
for name in files:
|
||||
source = Path(folder) / name
|
||||
twin = destination(legacy, source, target)
|
||||
if not stat.S_ISREG(source.lstat().st_mode):
|
||||
continue
|
||||
stuck = unmovable(helper, legacy, source, target, twin)
|
||||
if stuck is not None:
|
||||
lines.append(stuck)
|
||||
continue
|
||||
if not (twin.exists() or twin.is_symlink()):
|
||||
continue
|
||||
try:
|
||||
ours, theirs = content(helper, legacy, source), content(helper, target, twin)
|
||||
except OSError as error:
|
||||
lines.append(f"unmoved {source}: {error.strerror or error}")
|
||||
continue
|
||||
if ours != theirs and source.suffix != JOURNAL_SUFFIX:
|
||||
lines.append(f"conflict {twin} differs from {source}")
|
||||
return lines
|
||||
|
||||
|
||||
def written(entry: object, base: Path) -> Path | None:
|
||||
"""Args:
|
||||
entry: one allowWrite entry.
|
||||
base: the directory a relative entry lies below.
|
||||
|
||||
Returns:
|
||||
The normalised path the entry names, a trailing /** dropped; None for
|
||||
anything but a string.
|
||||
"""
|
||||
if not isinstance(entry, str):
|
||||
return None
|
||||
text = entry.removesuffix("/**").rstrip("/") or "/"
|
||||
path = Path(os.path.expanduser(text[1:] if text.startswith("//") else text))
|
||||
return Path(os.path.normpath(base / path))
|
||||
|
||||
|
||||
def allowed(settings: Path, base: Path) -> list[tuple[str, Path]] | None:
|
||||
"""Returns:
|
||||
(entry, path) per sandbox.filesystem.allowWrite entry of settings,
|
||||
relative ones below base; None when it holds no such list.
|
||||
"""
|
||||
try:
|
||||
entries = json.loads(settings.read_text(encoding="utf-8"))["sandbox"]["filesystem"]["allowWrite"]
|
||||
except (OSError, ValueError, KeyError, TypeError):
|
||||
return None
|
||||
if not isinstance(entries, list):
|
||||
return None
|
||||
return [(entry, path) for entry in entries if (path := written(entry, base)) is not None]
|
||||
|
||||
|
||||
def pkgmgr_roots() -> list[Path]:
|
||||
"""Returns:
|
||||
The repositories and worktrees directories of the directories block
|
||||
of ~/.config/pkgmgr/config.yaml, pkgmgr's defaults where it names
|
||||
none.
|
||||
"""
|
||||
found = dict(PKGMGR_DIRECTORIES)
|
||||
inside = False
|
||||
with contextlib.suppress(OSError):
|
||||
for line in (Path.home() / ".config" / "pkgmgr" / "config.yaml").read_text(encoding="utf-8").splitlines():
|
||||
text = line.split("#", 1)[0]
|
||||
if text[:1].strip():
|
||||
inside = text.strip() == "directories:"
|
||||
continue
|
||||
key, _, value = text.strip().partition(":")
|
||||
if inside and key in found and value.strip():
|
||||
found[key] = value.strip().strip("'\"")
|
||||
return [Path(os.path.expanduser(value)) for value in found.values()]
|
||||
|
||||
|
||||
def settings_files(project: Path | None, managed: Path | None = None) -> list[Path]:
|
||||
"""Args:
|
||||
project: the start directory of a session; None for only the managed
|
||||
and user files.
|
||||
managed: the managed settings directory, MANAGED unless given.
|
||||
|
||||
Returns:
|
||||
Every Claude Code settings file that can apply there:
|
||||
managed-settings.json and managed-settings.d/*.json in name order,
|
||||
~/.claude/settings.json and settings.local.json, then the project's
|
||||
.claude/settings.json and settings.local.json.
|
||||
"""
|
||||
managed = managed or MANAGED
|
||||
files = [managed / "managed-settings.json", *sorted((managed / "managed-settings.d").glob("*.json"))]
|
||||
files += [Path.home() / ".claude" / name for name in PROJECT_FILES]
|
||||
return files + ([] if project is None else [project / ".claude" / name for name in PROJECT_FILES])
|
||||
|
||||
|
||||
def settings_in(folder: Path) -> list[Path]:
|
||||
"""Returns:
|
||||
folder/.claude/settings.json and settings.local.json where they are
|
||||
files, read through a link like Claude Code does.
|
||||
"""
|
||||
return [path for name in PROJECT_FILES if (path := folder / ".claude" / name).is_file()]
|
||||
|
||||
|
||||
def project_settings(top: Path) -> list[Path]:
|
||||
"""Returns:
|
||||
The settings_in() of every directory at most PROJECT_DEPTH levels
|
||||
below top, in walk order, following directory links but entering each
|
||||
real directory once and never one of PRUNED.
|
||||
"""
|
||||
found, seen = [], set()
|
||||
for folder, folders, _ in os.walk(top, followlinks=True):
|
||||
here = Path(folder)
|
||||
real = os.path.realpath(here)
|
||||
if real in seen:
|
||||
folders[:] = []
|
||||
continue
|
||||
seen.add(real)
|
||||
depth = len(here.relative_to(top).parts)
|
||||
folders[:] = sorted(name for name in folders if name not in PRUNED) if depth < PROJECT_DEPTH else []
|
||||
found += settings_in(here)
|
||||
return found
|
||||
|
||||
|
||||
def decoded(folder: Path, rest: str) -> list[Path]:
|
||||
"""Returns:
|
||||
Every directory below folder whose path after folder encodes to rest
|
||||
the way Claude Code names a project: each character but an ASCII
|
||||
letter or digit turned into '-'.
|
||||
"""
|
||||
if not rest:
|
||||
return [folder]
|
||||
try:
|
||||
with os.scandir(folder) as entries:
|
||||
children = sorted((entry.name, entry.path) for entry in entries)
|
||||
except OSError:
|
||||
return []
|
||||
found = []
|
||||
for name, path in children:
|
||||
mark = "-" + ENCODED.sub("-", name)
|
||||
if rest.startswith(mark) and rest[len(mark):][:1] in ("", "-") and os.path.isdir(path):
|
||||
found += decoded(Path(path), rest[len(mark):])
|
||||
return found
|
||||
|
||||
|
||||
def started(projects: Path) -> list[Path]:
|
||||
"""Returns:
|
||||
The decoded() start directory of every Claude Code session folder
|
||||
below projects that still exists.
|
||||
"""
|
||||
try:
|
||||
with os.scandir(projects) as entries:
|
||||
names = sorted(entry.name for entry in entries if entry.is_dir(follow_symlinks=False))
|
||||
except OSError:
|
||||
return []
|
||||
return [folder for name in names for folder in decoded(Path("/"), name)]
|
||||
|
||||
|
||||
def sandbox_problems(settings: Path, managed: Path) -> list[str]:
|
||||
"""Args:
|
||||
settings: the user settings file.
|
||||
managed: the managed settings directory.
|
||||
|
||||
Returns:
|
||||
An add line per agent path the user file's allowWrite lacks, and a
|
||||
drop line per allowWrite entry that is the host-only root, inside it
|
||||
or above it, of settings, of a settings_files() file, of a
|
||||
project_settings() file below pkgmgr_roots(), or of a settings_in()
|
||||
file of a started() session directory; each real file once.
|
||||
"""
|
||||
release, host = state_home() / RELEASES, durable()
|
||||
user = allowed(settings, settings.parent)
|
||||
if user is None:
|
||||
problems = [f"{settings} holds no sandbox.filesystem.allowWrite list; the operator adds {root()} and {release} there"]
|
||||
else:
|
||||
paths = {path for _, path in user}
|
||||
problems = [f"add {path} to sandbox.filesystem.allowWrite in {settings} (operator step)" for path in (root(), release) if path not in paths]
|
||||
files = [(path, path.parent) for path in (*settings_files(None, managed), settings)]
|
||||
files += [(path, path.parent.parent) for top in pkgmgr_roots() for path in project_settings(top)]
|
||||
files += [(path, path.parent.parent) for folder in started(Path.home() / ".claude" / "projects") for path in settings_in(folder)]
|
||||
unique: dict[tuple[str, Path], tuple[Path, Path]] = {}
|
||||
for file, base in files:
|
||||
unique.setdefault((os.path.realpath(file), base.resolve()), (file, base))
|
||||
for file, base in unique.values():
|
||||
problems += [
|
||||
f"{file}: drop allowWrite entry {entry!r} ({path}): the sandbox must never write the host-only {host}"
|
||||
for entry, path in allowed(file, base) or [] if path == host or path in host.parents or host in path.parents
|
||||
]
|
||||
return problems
|
||||
|
||||
|
||||
def suspended(legacy: Path) -> bool:
|
||||
"""Returns:
|
||||
True when night.json exists in legacy or the council root and every
|
||||
one there is in a suspend mode.
|
||||
"""
|
||||
helper, modes = nofollow(), []
|
||||
for base in (legacy, root()):
|
||||
with contextlib.suppress(OSError, ValueError, AttributeError):
|
||||
modes.append(json.loads(helper.read_text(base, base / NIGHT)).get("mode"))
|
||||
return bool(modes) and all(mode in SUSPENDED for mode in modes)
|
||||
|
||||
|
||||
def preflight(settings: Path, managed: Path, legacy: Path) -> list[str]:
|
||||
"""Args:
|
||||
settings: the user settings file, only read.
|
||||
managed: the managed settings directory, only read.
|
||||
legacy: the earlier council root.
|
||||
|
||||
Returns:
|
||||
Every reason the install must stop before it copies anything: the
|
||||
sandbox_problems(), a refused council root, and for an existing
|
||||
legacy root its refusal, held() and conflicts().
|
||||
"""
|
||||
problems = sandbox_problems(settings, managed)
|
||||
target = root()
|
||||
if target.exists() or target.is_symlink():
|
||||
problems += [problem for problem in (refusal(target),) if problem is not None]
|
||||
if not legacy.exists() and not legacy.is_symlink():
|
||||
return problems
|
||||
problem = refusal(legacy)
|
||||
if problem is not None:
|
||||
return [*problems, problem]
|
||||
helper = nofollow()
|
||||
with contextlib.ExitStack() as stack:
|
||||
problems += held(helper, legacy, stack)
|
||||
return problems + conflicts(helper, legacy, target)
|
||||
|
||||
|
||||
def retire(base: Path, days: int, now: float) -> tuple[Path, int]:
|
||||
"""Args:
|
||||
base: the council root.
|
||||
days: files changed longer ago than this move.
|
||||
now: the current epoch time.
|
||||
|
||||
Returns:
|
||||
The dated archive directory and the number of files moved there.
|
||||
"""
|
||||
helper = nofollow()
|
||||
archive = base / ARCHIVE / time.strftime("%Y-%m-%d", time.localtime(now))
|
||||
cutoff, count = now - days * DAY_SECONDS, 0
|
||||
for part in RETIRED:
|
||||
top = base / part
|
||||
if top.is_symlink() or not top.is_dir():
|
||||
continue
|
||||
for folder, _, files in os.walk(top):
|
||||
for name in files:
|
||||
path = Path(folder) / name
|
||||
info = path.lstat()
|
||||
if not stat.S_ISREG(info.st_mode) or info.st_mtime > cutoff:
|
||||
continue
|
||||
*dirs, _ = helper.parts_below(base, path)
|
||||
source = helper.directory(base, tuple(dirs), False)
|
||||
try:
|
||||
target = helper.directory(base, (*helper.parts_below(base, archive), *dirs), True)
|
||||
try:
|
||||
os.rename(name, name, src_dir_fd=source, dst_dir_fd=target)
|
||||
finally:
|
||||
os.close(target)
|
||||
finally:
|
||||
os.close(source)
|
||||
count += 1
|
||||
return archive, count
|
||||
|
||||
|
||||
def legacy_default() -> Path:
|
||||
return Path(os.environ.get("AGENTS_COUNCIL_LEGACY") or LEGACY)
|
||||
|
||||
|
||||
def main(argv: Sequence[str] | None = None) -> int:
|
||||
parser = argparse.ArgumentParser(prog="councilroot.py", description=__doc__.splitlines()[0])
|
||||
commands = parser.add_subparsers(dest="command", required=True)
|
||||
commands.add_parser("root")
|
||||
commands.add_parser("durable")
|
||||
commands.add_parser("dir").add_argument("parts", nargs="+")
|
||||
commands.add_parser("release").add_argument("version")
|
||||
moving = commands.add_parser("migrate")
|
||||
moving.add_argument("--legacy", type=Path, default=legacy_default())
|
||||
moving.add_argument("--catch-up", action="store_true")
|
||||
check = commands.add_parser("preflight")
|
||||
check.add_argument("--legacy", type=Path, default=legacy_default())
|
||||
check.add_argument("--settings", type=Path, default=Path.home() / ".claude" / "settings.json")
|
||||
check.add_argument("--managed", type=Path, default=MANAGED)
|
||||
commands.add_parser("retire").add_argument("--days", type=int, default=RETIRE_DAYS)
|
||||
args = parser.parse_args(argv)
|
||||
try:
|
||||
if args.command == "root":
|
||||
print(root())
|
||||
elif args.command == "durable":
|
||||
print(durable())
|
||||
elif args.command == "dir":
|
||||
print(directory(root(), *args.parts))
|
||||
elif args.command == "release":
|
||||
print(directory(state_home() / RELEASES, args.version))
|
||||
elif args.command == "preflight":
|
||||
problems = preflight(args.settings, args.managed, args.legacy)
|
||||
for problem in problems:
|
||||
print(f"councilroot: preflight: {problem}", file=sys.stderr)
|
||||
if not suspended(args.legacy):
|
||||
print("councilroot: preflight: warning: the agents are not suspended (kvb-org-suspend); the install moves what they write meanwhile in a second migrate after the copy", file=sys.stderr)
|
||||
return 2 if problems else 0
|
||||
elif args.command == "retire":
|
||||
archive, count = retire(root(), args.days, time.time())
|
||||
print(f"archived {count} files into {archive}")
|
||||
else:
|
||||
try:
|
||||
report = migrate(args.legacy, root(), args.catch_up)
|
||||
except Blocked as blocked:
|
||||
print("\n".join([*blocked.lines, f"nothing moved from {args.legacy}"]))
|
||||
return 4
|
||||
print("\n".join(report) or f"nothing to move from {args.legacy}")
|
||||
for code, kinds in ((5, ("unmoved ",)), (3, ("conflict ", "kept "))):
|
||||
if any(line.startswith(kinds) for line in report):
|
||||
return code
|
||||
except (OSError, ValueError) as error:
|
||||
print(f"councilroot: {error}", file=sys.stderr)
|
||||
return 2
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -86,6 +86,25 @@ def read_text(root: Path, path: Path) -> str:
|
||||
return data.decode("utf-8", errors="replace")
|
||||
|
||||
|
||||
def read_tail(root: Path, path: Path, limit: int) -> str:
|
||||
"""Args:
|
||||
root: the trusted directory.
|
||||
path: the file, below root.
|
||||
limit: bytes to read at most; a longer file yields its last limit
|
||||
bytes after the first line break in them, so only whole lines.
|
||||
|
||||
Returns:
|
||||
That text, UTF-8 decoded, undecodable bytes replaced.
|
||||
"""
|
||||
with open(open_below(root, path, os.O_RDONLY, 0), "rb") as handle:
|
||||
size = os.fstat(handle.fileno()).st_size
|
||||
if size <= limit:
|
||||
return handle.read(limit).decode("utf-8", errors="replace")
|
||||
handle.seek(size - limit)
|
||||
data = handle.read(limit)
|
||||
return data[data.find(b"\n") + 1:].decode("utf-8", errors="replace")
|
||||
|
||||
|
||||
@contextlib.contextmanager
|
||||
def locked(root: Path, path: Path, mode: int, wait: float | None) -> Iterator[TextIO]:
|
||||
"""Hold an exclusive flock on a lock file below root, created without truncation.
|
||||
|
||||
@@ -8,8 +8,8 @@ Usage:
|
||||
(kvb-org-queue), <root>/security-accepted.jsonl
|
||||
(kvb-check-security), <root>/gate-receipts.jsonl and
|
||||
<root>/gate-parents.jsonl (kvb-check-gate) whose id it
|
||||
lacks to the same name under
|
||||
~/.local/state/agents-council/
|
||||
lacks to the same name in the host-only root
|
||||
(councilroot.py durable), never through a link
|
||||
watch.py block hook (PreToolUse on Bash): exit 2 for a heavy job while the
|
||||
disk stage is stop or critical, the memory stage is stop
|
||||
or cannot be judged without memory_limits, or a
|
||||
@@ -33,8 +33,9 @@ Usage:
|
||||
REPORT
|
||||
|
||||
Environment:
|
||||
AGENTS_COUNCIL_ROOT: shared state directory; /tmp/agents-council unless set,
|
||||
because every agent on the host must agree on it. Every command creates
|
||||
AGENTS_COUNCIL_ROOT, XDG_STATE_HOME, HOME: the shared state directory
|
||||
councilroot.py resolves, which survives a reboot and on which every
|
||||
agent on the host agrees. Every command creates
|
||||
it with mode 700; while it is a symlink, another uid's, or group or
|
||||
other writable, a hook mode prints why and exits 0 doing nothing and
|
||||
any other mode exits 2. queue.py and radio.py check it the same way.
|
||||
@@ -114,7 +115,8 @@ def sibling(name: str) -> ModuleType:
|
||||
|
||||
nofollow = sibling("nofollow")
|
||||
hookpath = sibling("hookpath")
|
||||
ROOT = Path(os.environ.get("AGENTS_COUNCIL_ROOT", "/tmp/agents-council"))
|
||||
councilroot = sibling("councilroot")
|
||||
ROOT = councilroot.root()
|
||||
CONFIG = Path(
|
||||
os.environ.get(
|
||||
"AGENTS_COUNCIL_CONFIG",
|
||||
@@ -126,7 +128,7 @@ STAGES = ("ok", "warn", "stop", "critical")
|
||||
STATE = ROOT / "watch.json"
|
||||
SILENCE = ROOT / "radio-silence.json"
|
||||
ALERTS = ROOT / "alerts.jsonl"
|
||||
KEPT = Path.home() / ".local" / "state" / "agents-council"
|
||||
KEPT = councilroot.durable()
|
||||
HISTORY = ROOT / "queue-history.jsonl"
|
||||
KEPT_HISTORY = KEPT / "queue-history.jsonl"
|
||||
ACCEPTED = ROOT / "security-accepted.jsonl"
|
||||
@@ -273,6 +275,13 @@ def guard_root(hook: bool) -> int | None:
|
||||
ROOT.mkdir(mode=0o700, parents=True, exist_ok=True)
|
||||
except FileExistsError:
|
||||
pass
|
||||
except OSError as error:
|
||||
print(
|
||||
f"agents council: cannot create {ROOT} ({error.strerror or error}); a sandbox needs it in allowWrite; "
|
||||
"the council hooks stay off until it exists",
|
||||
file=sys.stdout if hook else sys.stderr,
|
||||
)
|
||||
return 0 if hook else 2
|
||||
st = ROOT.lstat()
|
||||
if stat.S_ISDIR(st.st_mode) and st.st_uid == os.getuid() and not st.st_mode & 0o022:
|
||||
return None
|
||||
@@ -459,14 +468,13 @@ def history_rows(text: str) -> list[tuple[str, str]]:
|
||||
|
||||
|
||||
def keep_rows(origin: Path, target: Path) -> None:
|
||||
# The agents' sandbox cannot write ~/.local/state and /tmp is wiped on reboot, so this hook keeps the rows.
|
||||
# The kept file takes the source's mtime: an unchanged source is skipped without parsing.
|
||||
try:
|
||||
source = origin.stat().st_mtime_ns
|
||||
if target.exists() and target.stat().st_mtime_ns == source:
|
||||
return
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
with target.open("a+", encoding="utf-8") as kept:
|
||||
KEPT.mkdir(mode=0o700, parents=True, exist_ok=True)
|
||||
with open(nofollow.open_below(KEPT, target, os.O_RDWR | os.O_CREAT | os.O_APPEND, 0o600), "a+", encoding="utf-8") as kept:
|
||||
fcntl.flock(kept, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
||||
kept.seek(0)
|
||||
known = {row_id for row_id, _ in history_rows(kept.read())}
|
||||
@@ -475,7 +483,7 @@ def keep_rows(origin: Path, target: Path) -> None:
|
||||
known.add(row_id)
|
||||
kept.write(line + "\n")
|
||||
kept.flush()
|
||||
os.utime(target, ns=(source, source))
|
||||
os.utime(kept.fileno(), ns=(source, source))
|
||||
except (OSError, ValueError, TypeError):
|
||||
pass
|
||||
|
||||
|
||||
@@ -21,6 +21,7 @@
|
||||
`council.py takeover <office> --work "..."` and posts the succession to the
|
||||
radio; everyone else leaves it alone.
|
||||
- After a restart or reboot, run `council.py list --mine` and claim again
|
||||
every office you believe you hold: `/tmp` may have been wiped.
|
||||
every office you believe you hold: claims survive a reboot, but a new
|
||||
session holds none of them.
|
||||
- Only the operator, through the Orchestrator's passkey-signed Hierarchy
|
||||
view, removes a live holder; agents have no override.
|
||||
@@ -16,7 +16,9 @@ Usage:
|
||||
|
||||
Environment:
|
||||
SUBAGENT_MODEL_LOG: the log shared by every session;
|
||||
~/.local/state/agents-council/subagent-model.jsonl unless set.
|
||||
subagent-model.jsonl in the host-only root (councilroot.py durable of
|
||||
kvb-org-organize) unless set. Writes go through its nofollow.py, so
|
||||
a link at the log or its directory fails them.
|
||||
CLAUDE_CODE_SESSION_ID: the session that rates its spawns.
|
||||
"""
|
||||
|
||||
@@ -24,19 +26,27 @@ from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import datetime
|
||||
import importlib.util
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import shlex
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from types import ModuleType
|
||||
|
||||
LOG = Path(
|
||||
os.environ.get(
|
||||
"SUBAGENT_MODEL_LOG",
|
||||
str(Path.home() / ".local" / "state" / "agents-council" / "subagent-model.jsonl"),
|
||||
)
|
||||
)
|
||||
ORGANIZE = Path(__file__).resolve().parents[2] / "kvb-org-organize" / "scripts"
|
||||
|
||||
|
||||
def organize(name: str) -> ModuleType:
|
||||
spec = importlib.util.spec_from_file_location(f"kvb_perform_{name}", ORGANIZE / f"{name}.py")
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(module)
|
||||
return module
|
||||
|
||||
|
||||
nofollow = organize("nofollow")
|
||||
LOG = Path(os.environ.get("SUBAGENT_MODEL_LOG") or organize("councilroot").durable() / "subagent-model.jsonl")
|
||||
TASK_CLASS = re.compile(r"^task-class:\s*(\S+)", re.MULTILINE)
|
||||
PENDING = "pending: "
|
||||
SCRIPT = "kvb-org-perform/scripts/log.py"
|
||||
@@ -54,9 +64,8 @@ def now() -> str:
|
||||
|
||||
|
||||
def append(line: dict) -> None:
|
||||
LOG.parent.mkdir(parents=True, exist_ok=True)
|
||||
with LOG.open("a", encoding="utf-8") as handle:
|
||||
handle.write(json.dumps(line, sort_keys=True) + "\n")
|
||||
LOG.parent.mkdir(mode=0o700, parents=True, exist_ok=True)
|
||||
nofollow.append(LOG.parent, LOG, json.dumps(line, sort_keys=True) + "\n", 0o600)
|
||||
|
||||
|
||||
def record(line: dict) -> int:
|
||||
|
||||
@@ -7,10 +7,11 @@ Usage:
|
||||
cleanup.py request
|
||||
|
||||
Environment:
|
||||
AGENTS_COUNCIL_ROOT: shared state directory; /tmp/agents-council unless
|
||||
set. Agents append their requests to <root>/queue-cleanup.request,
|
||||
which kvb-org-queue-cleanup.path watches below /tmp/agents-council
|
||||
only. The round lives in ~/.local/state/agents-council, which only
|
||||
AGENTS_COUNCIL_ROOT, XDG_STATE_HOME, HOME: the council root the
|
||||
kvb-org-organize councilroot.py resolves. Agents append their
|
||||
requests to <root>/queue-cleanup.request, which
|
||||
kvb-org-queue-cleanup.path watches below %S/agents-shared only. The
|
||||
round lives in the host-only root (councilroot.py durable), which only
|
||||
the host can write, in queue-cleanup.json under queue-cleanup.lock,
|
||||
written by run alone.
|
||||
CLAUDE_CODE_SESSION_ID: run refuses to start while it is set.
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
Description=kvb-org-queue-cleanup round on cleanup.py request
|
||||
|
||||
[Path]
|
||||
PathModified=/tmp/agents-council/queue-cleanup.request
|
||||
PathModified=%S/agents-shared/queue-cleanup.request
|
||||
|
||||
[Install]
|
||||
WantedBy=paths.target
|
||||
@@ -3,5 +3,6 @@ Description=One kvb-org-queue-cleanup round: pan-pan, security, delete the jobs
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
ExecStartPre=-/usr/bin/python3 -I %h/.claude/skills/kvb-org-organize/scripts/councilroot.py retire
|
||||
ExecStart=/usr/bin/python3 -I %h/.claude/skills/kvb-org-queue-cleanup/scripts/cleanup.py run --window 900
|
||||
TimeoutStartSec=2400
|
||||
@@ -2,8 +2,9 @@
|
||||
"""Ask every agent once, by one newsletter, to estimate the open register tasks that carry no complexity.
|
||||
|
||||
Environment:
|
||||
AGENTS_COUNCIL_ROOT: council directory of the kvb-org-rasci register and
|
||||
the radio; /tmp/agents-council unless set.
|
||||
AGENTS_COUNCIL_ROOT, XDG_STATE_HOME, HOME: the council directory of the
|
||||
kvb-org-rasci register and the radio, as the kvb-org-organize
|
||||
councilroot.py resolves it.
|
||||
|
||||
Args:
|
||||
show: print the newsletter's text, send nothing.
|
||||
|
||||
@@ -10,8 +10,9 @@ Usage:
|
||||
queue.py block hook (PreToolUse on Bash)
|
||||
|
||||
Environment:
|
||||
AGENTS_COUNCIL_ROOT: shared state directory; /tmp/agents-council unless set,
|
||||
because every agent on the host must agree on it. The queue is
|
||||
AGENTS_COUNCIL_ROOT, XDG_STATE_HOME, HOME: the shared state directory the
|
||||
kvb-org-organize councilroot.py resolves, on which every agent on the
|
||||
host agrees. The queue is
|
||||
<root>/queue.json, every read-modify-write holds <root>/queue.lock.
|
||||
Each hold keeps an exclusive flock on <root>/holds/<tag>.lock, shared
|
||||
with its command, from before its entry exists until it leaves; an
|
||||
@@ -138,6 +139,7 @@ SIBLINGS = {
|
||||
"rasci": SKILLS / "kvb-org-rasci" / "scripts" / "rasci.py",
|
||||
"history": Path(__file__).resolve().parent / "history.py",
|
||||
"nofollow": SKILLS / "kvb-org-organize" / "scripts" / "nofollow.py",
|
||||
"councilroot": SKILLS / "kvb-org-organize" / "scripts" / "councilroot.py",
|
||||
}
|
||||
TOOLS = {"gates": SKILLS / "kvb-check-gate" / "scripts" / "gates.py"}
|
||||
HOST_SKILLS = Path.home() / ".agents" / "skills"
|
||||
@@ -156,7 +158,6 @@ GATE = re.compile(r"\bgate\.py\s+(run|hold)\b")
|
||||
THROUGH_QUEUE = re.compile(r"\bqueue\.py\s+hold\b")
|
||||
BASH_RULE = re.compile(r"Bash(?:\((.*)\))?", re.DOTALL)
|
||||
RULE_PART = re.compile(r"( \*|\*)")
|
||||
MANAGED = Path("/etc/claude-code")
|
||||
BUDGET_KEYS = ("budget_percent.ram", "budget_percent.cpu")
|
||||
HOTFIX = "hotfix/"
|
||||
PYTEST = frozenset({"pytest", "py.test"})
|
||||
@@ -1043,15 +1044,7 @@ def cmd_cancel(args: argparse.Namespace) -> int:
|
||||
|
||||
|
||||
def settings_files(project: Path) -> list[Path]:
|
||||
user = Path.home() / ".claude"
|
||||
return [
|
||||
MANAGED / "managed-settings.json",
|
||||
*sorted((MANAGED / "managed-settings.d").glob("*.json")),
|
||||
user / "settings.json",
|
||||
user / "settings.local.json",
|
||||
project / ".claude" / "settings.json",
|
||||
project / ".claude" / "settings.local.json",
|
||||
]
|
||||
return sibling("councilroot").settings_files(project)
|
||||
|
||||
|
||||
def bash_rule(rule: str) -> re.Pattern | None:
|
||||
|
||||
@@ -7,7 +7,8 @@ description: >
|
||||
|
||||
`${CLAUDE_SKILL_DIR}/scripts/radio.py` is the channel every skill uses to
|
||||
reach other agents, on every runtime. Both channels share one store,
|
||||
`news.jsonl` in the council directory (`/tmp/agents-council`).
|
||||
`news.jsonl` in the council directory, which
|
||||
`${CLAUDE_SKILL_DIR}/../kvb-org-organize/scripts/councilroot.py root` prints.
|
||||
|
||||
```bash
|
||||
radio.py post "<text>" [--to <name> ...] [--session <id>]
|
||||
|
||||
@@ -2,8 +2,9 @@
|
||||
"""Reach other agents: the radio is immediate, the post waits until a turn ends; one shared store.
|
||||
|
||||
Environment:
|
||||
AGENTS_COUNCIL_ROOT: council directory shared by every agent on the host;
|
||||
/tmp/agents-council unless set. Rows live in <root>/news.jsonl under
|
||||
AGENTS_COUNCIL_ROOT, XDG_STATE_HOME, HOME: the council directory the
|
||||
kvb-org-organize councilroot.py resolves, shared by every agent on the
|
||||
host. Rows live in <root>/news.jsonl under
|
||||
<root>/news.lock, the store the old news.py reads and writes too.
|
||||
AGENTS_COUNCIL_CONFIG: config file; its "notify" argv tells the operator
|
||||
of a mayday, its "never_block" and "mayday_allow" regexes name the
|
||||
@@ -89,18 +90,9 @@ from pathlib import Path
|
||||
from types import ModuleType
|
||||
|
||||
# Every agent on the host must agree on one council directory and one config without being told.
|
||||
ROOT = Path(os.environ.get("AGENTS_COUNCIL_ROOT", "/tmp/agents-council"))
|
||||
CONFIG = Path(
|
||||
os.environ.get("AGENTS_COUNCIL_CONFIG", str(Path.home() / ".claude" / "agents-council" / "config.json"))
|
||||
)
|
||||
NEWS = ROOT / "news.jsonl"
|
||||
LOCK = ROOT / "news.lock"
|
||||
SEEN = ROOT / "news-seen"
|
||||
STATE = ROOT / "radio-seen"
|
||||
LISTEN = ROOT / "radio-listen"
|
||||
SILENCE = ROOT / "radio-silence.json"
|
||||
NOTIFIED = ROOT / "radio-notified"
|
||||
FEEDBACK = ROOT / "feedback.json"
|
||||
ORGANIZE = Path(__file__).resolve().parents[2] / "kvb-org-organize" / "scripts"
|
||||
SELF = Path(__file__).resolve()
|
||||
RADIO = ("mayday", "pan-pan", "security", "call", "broadcast")
|
||||
@@ -140,6 +132,17 @@ def organize(name: str) -> ModuleType:
|
||||
return module
|
||||
|
||||
|
||||
ROOT = organize("councilroot").root()
|
||||
NEWS = ROOT / "news.jsonl"
|
||||
LOCK = ROOT / "news.lock"
|
||||
SEEN = ROOT / "news-seen"
|
||||
STATE = ROOT / "radio-seen"
|
||||
LISTEN = ROOT / "radio-listen"
|
||||
SILENCE = ROOT / "radio-silence.json"
|
||||
NOTIFIED = ROOT / "radio-notified"
|
||||
FEEDBACK = ROOT / "feedback.json"
|
||||
|
||||
|
||||
def council() -> ModuleType:
|
||||
return organize("council")
|
||||
|
||||
@@ -149,8 +152,9 @@ def name_of(sid: str) -> str:
|
||||
|
||||
|
||||
def items() -> list[dict]:
|
||||
nofollow = organize("nofollow")
|
||||
try:
|
||||
lines = organize("nofollow").read_text(ROOT, NEWS).splitlines()
|
||||
lines = nofollow.read_tail(ROOT, NEWS, nofollow.READ_LIMIT).splitlines()
|
||||
except OSError:
|
||||
return []
|
||||
rows = []
|
||||
@@ -361,7 +365,11 @@ def store(entries: list[dict]) -> float:
|
||||
for entry in entries:
|
||||
entry["ts"] = ts
|
||||
ts = math.nextafter(ts, math.inf)
|
||||
write_atomic(NEWS, "".join(json.dumps(row) + "\n" for row in [*rows[-(KEPT - len(entries)):], *entries]))
|
||||
cut = max(0, len(rows) - max(0, KEPT - len(entries)))
|
||||
if cut:
|
||||
archive = ROOT / "archive" / f"news-{time.strftime('%Y-%m-%d')}.jsonl"
|
||||
organize("nofollow").append(ROOT, archive, "".join(json.dumps(row) + "\n" for row in rows[:cut]), 0o600)
|
||||
write_atomic(NEWS, "".join(json.dumps(row) + "\n" for row in [*rows[cut:], *entries]))
|
||||
return entries[-1]["ts"]
|
||||
|
||||
|
||||
|
||||
@@ -2,12 +2,12 @@
|
||||
"""Keep the RASCI register of the organisation's tasks in a local JSON file.
|
||||
|
||||
Environment:
|
||||
AGENTS_COUNCIL_ROOT: council directory shared by every agent on the host;
|
||||
the register is <root>/rasci.json, and `show` also lists every work/
|
||||
and cluster/ claim of the council there. The operator's agent order
|
||||
(`agents order`) lies beside it in <root>/agent-order.json, apart from
|
||||
every task row: it breaks ties of equal task priority only.
|
||||
/tmp/agents-council unless set.
|
||||
AGENTS_COUNCIL_ROOT, XDG_STATE_HOME, HOME: the council directory the
|
||||
kvb-org-organize councilroot.py resolves, shared by every agent on the
|
||||
host; the register is <root>/rasci.json, and `show` also lists every
|
||||
work/ and cluster/ claim of the council there. The operator's agent
|
||||
order (`agents order`) lies beside it in <root>/agent-order.json, apart
|
||||
from every task row: it breaks ties of equal task priority only.
|
||||
|
||||
Exit codes:
|
||||
0 done, 2 usage error.
|
||||
@@ -20,7 +20,6 @@ import contextlib
|
||||
import datetime
|
||||
import importlib.util
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import statistics
|
||||
import sys
|
||||
@@ -28,10 +27,6 @@ from collections.abc import Iterator
|
||||
from pathlib import Path
|
||||
from types import ModuleType
|
||||
|
||||
ROOT = Path(os.environ.get("AGENTS_COUNCIL_ROOT", "/tmp/agents-council"))
|
||||
DB = ROOT / "rasci.json"
|
||||
LOCK = ROOT / "rasci.lock"
|
||||
AGENT_ORDER = ROOT / "agent-order.json"
|
||||
FILE_MODE = 0o600
|
||||
ORGANIZE = Path(__file__).resolve().parents[2] / "kvb-org-organize" / "scripts"
|
||||
COUNCIL = ORGANIZE / "council.py"
|
||||
@@ -88,6 +83,10 @@ def script(name: str, path: Path) -> ModuleType:
|
||||
|
||||
|
||||
nofollow = script("kvb_rasci_nofollow", ORGANIZE / "nofollow.py")
|
||||
ROOT = script("kvb_rasci_councilroot", ORGANIZE / "councilroot.py").root()
|
||||
DB = ROOT / "rasci.json"
|
||||
LOCK = ROOT / "rasci.lock"
|
||||
AGENT_ORDER = ROOT / "agent-order.json"
|
||||
|
||||
|
||||
def load() -> dict[str, dict]:
|
||||
|
||||
@@ -2,8 +2,9 @@
|
||||
"""Keep the organisation's night mode: one active session, the others drained or suspended.
|
||||
|
||||
Environment:
|
||||
AGENTS_COUNCIL_ROOT: council directory shared by every agent on the host;
|
||||
the state is <root>/night.json. /tmp/agents-council unless set.
|
||||
AGENTS_COUNCIL_ROOT, XDG_STATE_HOME, HOME: the council directory the
|
||||
kvb-org-organize councilroot.py resolves, shared by every agent on the
|
||||
host; the state is <root>/night.json.
|
||||
CLAUDE_CODE_SESSION_ID: the calling session.
|
||||
|
||||
Args:
|
||||
@@ -43,20 +44,27 @@ import tempfile
|
||||
from pathlib import Path
|
||||
from types import ModuleType
|
||||
|
||||
ROOT = Path(os.environ.get("AGENTS_COUNCIL_ROOT", "/tmp/agents-council"))
|
||||
STATE = ROOT / "night.json"
|
||||
COUNCIL = Path(__file__).resolve().parents[2] / "kvb-org-organize" / "scripts" / "council.py"
|
||||
ORGANIZE = Path(__file__).resolve().parents[2] / "kvb-org-organize" / "scripts"
|
||||
COUNCIL = ORGANIZE / "council.py"
|
||||
PARTS = {"drain": "drain (kvb-agent-drain)", "suspend": "suspend (kvb-agent-suspend)"}
|
||||
RENAMED = {"evening": "drain", "sunset": "suspend"}
|
||||
|
||||
|
||||
def council() -> ModuleType:
|
||||
spec = importlib.util.spec_from_file_location("council", COUNCIL)
|
||||
def loaded(name: str, path: Path) -> ModuleType:
|
||||
spec = importlib.util.spec_from_file_location(name, path)
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(module)
|
||||
return module
|
||||
|
||||
|
||||
def council() -> ModuleType:
|
||||
return loaded("council", COUNCIL)
|
||||
|
||||
|
||||
ROOT = loaded("kvb_night_councilroot", ORGANIZE / "councilroot.py").root()
|
||||
STATE = ROOT / "night.json"
|
||||
|
||||
|
||||
def migrated(state: object) -> object:
|
||||
if not isinstance(state, dict):
|
||||
return state
|
||||
|
||||
@@ -19,10 +19,11 @@ bar on `<line>`", key results gate green, landed, cleaned up, overview posted.
|
||||
## Step 1: collect
|
||||
|
||||
The register is the swarm's record (stage 2 of `kvb-org-swarm` sets `--branch`
|
||||
and `--base`):
|
||||
and `--base`); `<council>` is what
|
||||
`${CLAUDE_SKILL_DIR}/../kvb-org-organize/scripts/councilroot.py root` prints:
|
||||
|
||||
```bash
|
||||
jq -r --arg base "<line>" 'to_entries[] | select(.value.base == $base and .value.branch) | [.key, .value.branch, .value.workdir // "-", .value.status] | @tsv' "${AGENTS_COUNCIL_ROOT:-/tmp/agents-council}/rasci.json"
|
||||
jq -r --arg base "<line>" 'to_entries[] | select(.value.base == $base and .value.branch) | [.key, .value.branch, .value.workdir // "-", .value.status] | @tsv' "<council>/rasci.json"
|
||||
```
|
||||
|
||||
Per branch: `git log --oneline <line>..<branch>` (empty: already merged, go
|
||||
@@ -44,7 +45,8 @@ the branch becomes a proposal naming the conflicting files.
|
||||
|
||||
The project's full gate on `unify/<goal>` through `queue.py hold`
|
||||
(`kvb-check-gate`), and `security.py` over `<line>..unify/<goal>`
|
||||
(`kvb-check-security`), logs under `/tmp`. Red: find the merge that broke it
|
||||
(`kvb-check-security`), logs under the directory `councilroot.py dir gates`
|
||||
prints. Red: find the merge that broke it
|
||||
(`git bisect start --first-parent`), rebuild the branch without it, which
|
||||
becomes a proposal naming the failing test, and gate again.
|
||||
|
||||
|
||||
@@ -2,8 +2,9 @@
|
||||
"""Tell every agent which skills changed and how to use them: one newsletter per update.
|
||||
|
||||
Environment:
|
||||
AGENTS_COUNCIL_ROOT: council directory shared by every agent on the host;
|
||||
/tmp/agents-council unless set. The last announced revision per
|
||||
AGENTS_COUNCIL_ROOT, XDG_STATE_HOME, HOME: the council directory the
|
||||
kvb-org-organize councilroot.py resolves, shared by every agent on the
|
||||
host. The last announced revision per
|
||||
repository lives in <root>/skills-announced.json.
|
||||
CLAUDE_CODE_SESSION_ID: the sending session, passed on to radio.py.
|
||||
|
||||
@@ -25,14 +26,11 @@ import argparse
|
||||
import functools
|
||||
import importlib.util
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from types import ModuleType
|
||||
|
||||
ROOT = Path(os.environ.get("AGENTS_COUNCIL_ROOT", "/tmp/agents-council"))
|
||||
ANNOUNCED = ROOT / "skills-announced.json"
|
||||
SKILLS = Path(__file__).resolve().parents[2]
|
||||
RADIO = SKILLS / "kvb-org-radio" / "scripts" / "radio.py"
|
||||
ORGANIZE = SKILLS / "kvb-org-organize" / "scripts"
|
||||
@@ -48,6 +46,10 @@ def organize(name: str) -> ModuleType:
|
||||
return module
|
||||
|
||||
|
||||
ROOT = organize("councilroot").root()
|
||||
ANNOUNCED = ROOT / "skills-announced.json"
|
||||
|
||||
|
||||
def git(repo: Path, *args: str) -> str:
|
||||
return subprocess.run([*GIT, *args], cwd=repo, capture_output=True, text=True, check=True).stdout
|
||||
|
||||
|
||||
@@ -41,5 +41,8 @@ mechanism and the upgrade path, investors read only what the brief proves.
|
||||
- Stay inside the channel's length bounds. Without project bounds: a blog
|
||||
article 600 to 1200 words, a LinkedIn article 400 to 900, a newsletter
|
||||
250 to 500.
|
||||
- Write `/tmp/release-<version>/<channel>/<language>.md`, or the format the
|
||||
project's template names, and report each file's word count.
|
||||
- Write `<drafts>/<channel>/<language>.md`, or the format the project's
|
||||
template names, and report each file's word count. `<drafts>` is the
|
||||
directory `kvb-release` handed over, else the one
|
||||
`${CLAUDE_SKILL_DIR}/../kvb-org-organize/scripts/councilroot.py release <version>`
|
||||
prints.
|
||||
@@ -43,6 +43,9 @@ replaces this one:
|
||||
|
||||
- Every number, name and date comes from `brief.yml`.
|
||||
- No em dashes, no placeholder, no claim the brief does not prove.
|
||||
- Write into `/tmp/release-<version>/` under the file name the project skill
|
||||
sets, else `<channel>/<language>.txt`, and report each file's character
|
||||
count against its limit.
|
||||
- Write into `<drafts>/` under the file name the project skill sets, else
|
||||
`<channel>/<language>.txt`, and report each file's character count
|
||||
against its limit. `<drafts>` is the directory `kvb-release` handed over,
|
||||
else the one
|
||||
`${CLAUDE_SKILL_DIR}/../kvb-org-organize/scripts/councilroot.py release <version>`
|
||||
prints.
|
||||
@@ -14,8 +14,10 @@ the project's own `<prefix>-release-<channel>` skills.
|
||||
|
||||
Run the `kvb-think-listen` skill once for what only the operator knows: the
|
||||
tag, the languages (default English), the audiences, the highlights, what to
|
||||
leave out and the channels wanted. Write the answers to
|
||||
`/tmp/release-<version>/listen.yml` (`tag`, `languages`, `audience`,
|
||||
leave out and the channels wanted. `<drafts>` is the directory
|
||||
`${CLAUDE_SKILL_DIR}/../kvb-org-organize/scripts/councilroot.py release <version>`
|
||||
prints, which survives a reboot. Write the answers to
|
||||
`<drafts>/listen.yml` (`tag`, `languages`, `audience`,
|
||||
`highlights`, `leave_out`, `channels`) and hand that file to every channel,
|
||||
so the operator answers each question once.
|
||||
|
||||
@@ -31,7 +33,7 @@ so the operator answers each question once.
|
||||
## 3. Brief
|
||||
|
||||
Run the `kvb-think-brainstorm` skill on the headline benefit, then write
|
||||
`/tmp/release-<version>/brief.yml`: version, date, languages, headline
|
||||
`<drafts>/brief.yml`: version, date, languages, headline
|
||||
benefit, three to five benefits each with the changelog lines that prove it,
|
||||
upgrade notes and links. `languages` is the only answer the brief copies from
|
||||
`listen.yml`: an answer steers what a channel stresses, it never backs a
|
||||
@@ -51,12 +53,12 @@ the general method:
|
||||
| entry | changelog | `kvb-release-changelog` |
|
||||
|
||||
Each gets `brief.yml`, `listen.yml` and its channel, runs its own listen and
|
||||
brainstorm, and writes into `/tmp/release-<version>/` under the file name
|
||||
brainstorm, and writes into `<drafts>/` under the file name
|
||||
the project skill sets, else `<channel>/<language>.<ext>`.
|
||||
|
||||
## 5. Check
|
||||
|
||||
Fix in every file under `/tmp/release-<version>/`, subfolders included,
|
||||
Fix in every file under `<drafts>/`, subfolders included,
|
||||
before the hand-over: a leftover placeholder (`TODO`, `XXX`, `<...>`), an
|
||||
em dash, a length outside the channel's bounds, a number absent from
|
||||
`brief.yml`.
|
||||
|
||||
@@ -0,0 +1,872 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import contextlib
|
||||
import errno
|
||||
import fcntl
|
||||
import hashlib
|
||||
import importlib.util
|
||||
import io
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import time
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from types import ModuleType
|
||||
from unittest import mock
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parents[1]
|
||||
ORGANIZE = REPO_ROOT / "skills" / "kvb-org-organize" / "scripts"
|
||||
COUNCIL_ROOT = ORGANIZE / "councilroot.py"
|
||||
WATCH = ORGANIZE / "watch.py"
|
||||
QUEUE = REPO_ROOT / "skills" / "kvb-org-queue" / "scripts" / "queue.py"
|
||||
SECURITY = REPO_ROOT / "skills" / "kvb-check-security" / "scripts" / "security.py"
|
||||
GATES = REPO_ROOT / "skills" / "kvb-check-gate" / "scripts" / "gates.py"
|
||||
LOG = REPO_ROOT / "skills" / "kvb-org-perform" / "scripts" / "log.py"
|
||||
CLIENT = REPO_ROOT / "orchestrator" / "client"
|
||||
INSTALL = REPO_ROOT / "scripts" / "install.sh"
|
||||
GATE_SKILL = REPO_ROOT / "skills" / "kvb-check-gate"
|
||||
SCANNED = ("skills", "orchestrator", "scripts")
|
||||
FORBIDDEN = re.compile(r"/tmp/(?:agents-council|kvb-git-diff|kvb-orchestrator-approvals|release-)")
|
||||
LEGACY_LINE = 'LEGACY = Path("/tmp/agents-council")'
|
||||
DIR_SLASH = re.compile(r"councilroot\.py dir [^)\n]*\)/")
|
||||
RADIO = REPO_ROOT / "skills" / "kvb-org-radio" / "scripts" / "radio.py"
|
||||
CLEANUP_SERVICE = REPO_ROOT / "skills" / "kvb-org-queue-cleanup" / "systemd" / "kvb-org-queue-cleanup.service"
|
||||
|
||||
|
||||
def load(path: Path, name: str) -> ModuleType:
|
||||
spec = importlib.util.spec_from_file_location(name, path)
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(module)
|
||||
return module
|
||||
|
||||
|
||||
def rows(path: Path) -> list[str]:
|
||||
return [json.loads(line)["id"] for line in path.read_text(encoding="utf-8").splitlines()]
|
||||
|
||||
|
||||
class Home(unittest.TestCase):
|
||||
def setUp(self):
|
||||
tmp = tempfile.TemporaryDirectory()
|
||||
self.addCleanup(tmp.cleanup)
|
||||
self.base = Path(tmp.name).resolve()
|
||||
self.home = self.base / "home"
|
||||
self.home.mkdir()
|
||||
patch = mock.patch.dict(os.environ, {"HOME": str(self.home)})
|
||||
patch.start()
|
||||
self.addCleanup(patch.stop)
|
||||
for name in ("AGENTS_COUNCIL_ROOT", "XDG_STATE_HOME"):
|
||||
os.environ.pop(name, None)
|
||||
self.durable = self.home / ".local" / "state" / "agents-council"
|
||||
self.shared = self.home / ".local" / "state" / "agents-shared"
|
||||
self.councilroot = load(COUNCIL_ROOT, f"test_councilroot_{id(self)}")
|
||||
|
||||
def cli(self, *args: str, **env: str) -> subprocess.CompletedProcess:
|
||||
return subprocess.run(
|
||||
[sys.executable, str(COUNCIL_ROOT), *args], env={"PATH": os.environ["PATH"], "HOME": str(self.home), **env},
|
||||
capture_output=True, text=True, check=False,
|
||||
)
|
||||
|
||||
|
||||
class TestResolver(Home):
|
||||
def test_agents_share_one_root_and_the_host_keeps_another_by_default(self):
|
||||
self.assertEqual((self.councilroot.root(), self.councilroot.durable()), (self.shared, self.durable))
|
||||
self.assertNotEqual(self.councilroot.root(), self.councilroot.durable())
|
||||
|
||||
def test_xdg_state_home_moves_both_roots_only_when_absolute(self):
|
||||
for value, state in (("/srv/state", Path("/srv/state")), ("relative", self.durable.parent), ("", self.durable.parent)):
|
||||
with self.subTest(value=value), mock.patch.dict(os.environ, {"XDG_STATE_HOME": value}):
|
||||
self.assertEqual((self.councilroot.root(), self.councilroot.durable()),
|
||||
(state / "agents-shared", state / "agents-council"))
|
||||
|
||||
def test_agents_council_root_wins_and_an_empty_or_relative_one_means_unset(self):
|
||||
with mock.patch.dict(os.environ, {"AGENTS_COUNCIL_ROOT": "/srv/council", "XDG_STATE_HOME": "/srv/state"}):
|
||||
self.assertEqual((self.councilroot.root(), self.councilroot.durable()), (Path("/srv/council"), Path("/srv/state/agents-council")))
|
||||
for value in ("", "relative-council"):
|
||||
with self.subTest(value=value), mock.patch.dict(os.environ, {"AGENTS_COUNCIL_ROOT": value}):
|
||||
self.assertEqual(self.councilroot.root(), self.shared)
|
||||
|
||||
def test_the_cli_prints_the_roots_and_creates_private_parts(self):
|
||||
self.assertEqual(self.cli("root").stdout, f"{self.shared}\n")
|
||||
self.assertEqual(self.cli("root", AGENTS_COUNCIL_ROOT=str(self.base / "c")).stdout, f"{self.base / 'c'}\n")
|
||||
self.assertEqual(self.cli("durable", AGENTS_COUNCIL_ROOT=str(self.base / "c")).stdout, f"{self.durable}\n")
|
||||
for args, made in ((("dir", "gates"), self.shared / "gates"), (("release", "1.2.3"), self.durable.parent / "agents-release" / "1.2.3")):
|
||||
with self.subTest(args=args):
|
||||
done = self.cli(*args)
|
||||
self.assertEqual((done.returncode, done.stdout), (0, f"{made}\n"), done.stderr)
|
||||
self.assertEqual(made.stat().st_mode & 0o777, 0o700)
|
||||
self.assertEqual(self.shared.stat().st_mode & 0o777, 0o700)
|
||||
self.assertFalse(self.durable.exists())
|
||||
refused = self.cli("dir", "../escape")
|
||||
self.assertEqual((refused.returncode, refused.stderr), (2, "councilroot: not a directory name: '../escape'\n"))
|
||||
self.assertFalse((self.shared.parent / "escape").exists())
|
||||
|
||||
def test_a_root_the_sandbox_cannot_create_turns_the_hooks_off_with_the_reason(self):
|
||||
state = self.base / "frozen"
|
||||
state.mkdir(mode=0o500)
|
||||
self.addCleanup(state.chmod, 0o700)
|
||||
with mock.patch.dict(os.environ, {"XDG_STATE_HOME": str(state)}):
|
||||
watch = load(WATCH, f"test_council_root_frozen_{id(self)}")
|
||||
out = io.StringIO()
|
||||
with contextlib.redirect_stdout(out):
|
||||
self.assertEqual(watch.guard_root(hook=True), 0)
|
||||
self.assertIn(f"cannot create {state / 'agents-shared'}", out.getvalue())
|
||||
|
||||
def test_a_part_the_cli_cannot_create_exits_2_naming_why_and_prints_nothing(self):
|
||||
state = self.base / "frozen"
|
||||
state.mkdir(mode=0o500)
|
||||
self.addCleanup(state.chmod, 0o700)
|
||||
done = self.cli("dir", "gates", "slug", XDG_STATE_HOME=str(state))
|
||||
self.assertEqual((done.returncode, done.stdout), (2, ""))
|
||||
self.assertTrue(done.stderr.startswith("councilroot: ") and "Permission denied" in done.stderr and "Traceback" not in done.stderr, done.stderr)
|
||||
|
||||
|
||||
class TestMigrate(Home):
|
||||
def setUp(self):
|
||||
super().setUp()
|
||||
self.legacy = self.base / "legacy"
|
||||
self.legacy.mkdir(mode=0o700)
|
||||
self.target = self.base / "state" / "agents-shared"
|
||||
|
||||
def write(self, path: Path, text: str, mtime: int | None = None) -> Path:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(text, encoding="utf-8")
|
||||
path.chmod(0o600)
|
||||
if mtime is not None:
|
||||
os.utime(path, (mtime, mtime))
|
||||
return path
|
||||
|
||||
def migrate(self) -> list[str]:
|
||||
return self.councilroot.migrate(self.legacy, self.target)
|
||||
|
||||
def test_every_file_moves_and_a_second_run_moves_nothing(self):
|
||||
self.write(self.legacy / "rasci.json", "{}\n", 1000)
|
||||
self.write(self.legacy / "work" / "a" / "claim.json", '{"session": "s1"}\n')
|
||||
self.write(self.legacy / "holds" / "1-2.lock", "")
|
||||
self.assertEqual(self.migrate(), [f"moved 3 files from {self.legacy} to {self.target}"])
|
||||
self.assertEqual((self.target / "rasci.json").read_text(encoding="utf-8"), "{}\n")
|
||||
self.assertEqual((self.target / "rasci.json").stat().st_mtime, 1000)
|
||||
self.assertEqual((self.target / "rasci.json").stat().st_mode & 0o777, 0o600)
|
||||
self.assertEqual((self.target / "work" / "a" / "claim.json").read_text(encoding="utf-8"), '{"session": "s1"}\n')
|
||||
self.assertTrue((self.target / "holds" / "1-2.lock").is_file())
|
||||
self.assertFalse(self.legacy.exists())
|
||||
self.assertEqual(sorted(path.name for path in self.target.iterdir()), ["holds", "rasci.json", "work"])
|
||||
self.assertEqual(self.migrate(), [])
|
||||
|
||||
def test_a_differing_whole_file_refuses_the_move_and_a_catch_up_keeps_it_beside(self):
|
||||
newer = self.write(self.target / "rasci.json", "durable\n", 2000)
|
||||
self.write(self.legacy / "rasci.json", "legacy\n", 1000)
|
||||
self.write(self.legacy / "queue.json", '{"seq": 1, "entries": []}\n')
|
||||
line = f"conflict {newer} differs from {self.legacy / 'rasci.json'}"
|
||||
with self.assertRaises(self.councilroot.Blocked) as blocked:
|
||||
self.migrate()
|
||||
self.assertEqual(blocked.exception.lines, [line])
|
||||
done = self.cli("migrate", "--legacy", str(self.legacy), AGENTS_COUNCIL_ROOT=str(self.target))
|
||||
self.assertEqual((done.returncode, done.stdout), (4, f"{line}\nnothing moved from {self.legacy}\n"), done.stderr)
|
||||
self.assertEqual(sorted(path.name for path in self.legacy.iterdir()), ["queue.json", "rasci.json"])
|
||||
self.assertFalse((self.target / "queue.json").exists())
|
||||
kept = self.target / f"rasci.json.legacy-{hashlib.sha256(b"legacy\n").hexdigest()[:12]}"
|
||||
for run in (1, 2):
|
||||
with self.subTest(run=run):
|
||||
if run == 2:
|
||||
self.write(self.legacy / "rasci.json", "legacy\n")
|
||||
caught = self.cli("migrate", "--legacy", str(self.legacy), "--catch-up", AGENTS_COUNCIL_ROOT=str(self.target))
|
||||
self.assertEqual(caught.returncode, 3, caught.stderr)
|
||||
self.assertIn(f"kept {self.legacy / 'rasci.json'} beside {newer} as {kept}", caught.stdout)
|
||||
self.assertEqual((newer.read_text(encoding="utf-8"), kept.read_text(encoding="utf-8")), ("durable\n", "legacy\n"))
|
||||
self.assertFalse(self.legacy.exists())
|
||||
self.assertEqual(sorted(path.name for path in self.target.iterdir()), ["queue.json", "rasci.json", kept.name])
|
||||
|
||||
def test_a_file_it_cannot_move_refuses_the_whole_move_and_preflight_predicts_it(self):
|
||||
self.write(self.legacy / "rasci.json", '{"work/x": {}}\n')
|
||||
self.write(self.legacy / "queue.json", '{"seq": 9, "entries": []}\n')
|
||||
source = self.write(self.legacy / "late.json", "{}\n")
|
||||
os.link(source, self.base / "second-link")
|
||||
done = self.cli("migrate", "--legacy", str(self.legacy), AGENTS_COUNCIL_ROOT=str(self.target))
|
||||
line = f"unmoved {source}: not a regular file with a single link"
|
||||
self.assertEqual((done.returncode, done.stdout), (4, f"{line}\nnothing moved from {self.legacy}\n"), done.stderr)
|
||||
self.assertEqual(sorted(path.name for path in self.legacy.iterdir()), ["late.json", "queue.json", "rasci.json"])
|
||||
self.assertFalse(self.target.exists() and any(self.target.iterdir()))
|
||||
checked = self.cli("preflight", "--legacy", str(self.legacy), "--managed", str(self.base / "no-managed"), AGENTS_COUNCIL_ROOT=str(self.target))
|
||||
self.assertEqual(checked.returncode, 2)
|
||||
self.assertIn(line, checked.stderr)
|
||||
|
||||
def test_a_move_failing_after_the_check_leaves_a_partial_state_with_exit_5(self):
|
||||
self.write(self.legacy / "rasci.json", "{}\n")
|
||||
late = self.write(self.legacy / "late.json", "{}\n")
|
||||
real = self.councilroot.copied
|
||||
|
||||
def racing(helper, legacy, source, *rest):
|
||||
if source.name.startswith(late.name):
|
||||
raise OSError(errno.EBUSY, "raced", str(source))
|
||||
return real(helper, legacy, source, *rest)
|
||||
|
||||
out = io.StringIO()
|
||||
with mock.patch.object(self.councilroot, "copied", racing), mock.patch.dict(os.environ, {"AGENTS_COUNCIL_ROOT": str(self.target)}), contextlib.redirect_stdout(out):
|
||||
code = self.councilroot.main(["migrate", "--legacy", str(self.legacy)])
|
||||
self.assertEqual(code, 5)
|
||||
self.assertTrue(any(line.startswith(f"unmoved {late}") and line.endswith(": raced") for line in out.getvalue().splitlines()), out.getvalue())
|
||||
self.assertTrue((self.target / "rasci.json").is_file())
|
||||
self.assertEqual([path.name.startswith("late.json") for path in self.legacy.iterdir()], [True])
|
||||
|
||||
def test_a_row_appended_while_its_journal_moves_reaches_the_root_through_the_catch_up(self):
|
||||
journal = self.write(self.legacy / "scores.jsonl", '{"time": 1, "digest": "a"}\n')
|
||||
late = '{"time": 2, "digest": "written between the copy and the unlink"}\n'
|
||||
real = self.councilroot.copied
|
||||
|
||||
def racing(helper, legacy, source, *rest):
|
||||
done = real(helper, legacy, source, *rest)
|
||||
with open(journal, "a", encoding="utf-8") as writer:
|
||||
writer.write(late)
|
||||
return done
|
||||
|
||||
with mock.patch.object(self.councilroot, "copied", racing):
|
||||
self.migrate()
|
||||
self.councilroot.migrate(self.legacy, self.target, True)
|
||||
self.assertEqual((self.target / "scores.jsonl").read_text(encoding="utf-8"), '{"time": 1, "digest": "a"}\n' + late)
|
||||
self.assertFalse(self.legacy.exists())
|
||||
|
||||
def test_a_crash_mid_move_leaves_the_started_marker_until_a_run_completes_the_move(self):
|
||||
self.write(self.legacy / "a.json", "{}\n")
|
||||
self.write(self.legacy / "b.json", '{"b": 1}\n')
|
||||
real = self.councilroot.copied
|
||||
|
||||
def crashing(helper, legacy, source, *rest):
|
||||
if source.name.startswith("b.json"):
|
||||
raise RuntimeError("boom")
|
||||
return real(helper, legacy, source, *rest)
|
||||
|
||||
marker = self.durable / "migrate.started"
|
||||
with mock.patch.object(self.councilroot, "copied", crashing), self.assertRaises(RuntimeError):
|
||||
self.migrate()
|
||||
self.assertTrue(marker.is_file() and (self.target / "a.json").is_file())
|
||||
self.assertEqual(self.migrate(), [f"moved 1 files from {self.legacy} to {self.target}"])
|
||||
self.assertEqual((self.target / "b.json").read_text(encoding="utf-8"), '{"b": 1}\n')
|
||||
self.assertFalse(marker.exists() or self.legacy.exists())
|
||||
|
||||
def test_identical_copies_are_one_and_leave_the_legacy_root(self):
|
||||
same = self.write(self.target / "rasci.json", "{}\n", 2000)
|
||||
self.write(self.legacy / "rasci.json", "{}\n", 1000)
|
||||
self.assertEqual(self.migrate(), [f"moved 1 files from {self.legacy} to {self.target}"])
|
||||
self.assertEqual((same.read_text(encoding="utf-8"), same.stat().st_mtime), ("{}\n", 2000))
|
||||
self.assertFalse(self.legacy.exists())
|
||||
|
||||
def test_a_held_lock_or_a_queued_job_refuses_the_whole_move(self):
|
||||
hold = self.write(self.legacy / "holds" / "1-2.lock", "")
|
||||
self.write(self.legacy / "rasci.json", "{}\n")
|
||||
with open(hold, encoding="utf-8") as held:
|
||||
fcntl.flock(held, fcntl.LOCK_EX)
|
||||
with self.assertRaises(ValueError) as raised:
|
||||
self.migrate()
|
||||
self.assertIn(f"{hold} is held", str(raised.exception))
|
||||
self.write(self.legacy / "queue.json", json.dumps({"seq": 3, "entries": [{"id": "test-3"}]}))
|
||||
with self.assertRaises(ValueError) as raised:
|
||||
self.migrate()
|
||||
self.assertIn(f"{self.legacy / 'queue.json'} holds 1 queue jobs", str(raised.exception))
|
||||
self.assertFalse(self.target.exists())
|
||||
self.assertEqual(sorted(path.name for path in self.legacy.iterdir()), ["holds", "queue.json", "rasci.json"])
|
||||
self.write(self.legacy / "queue.json", json.dumps({"seq": 3, "entries": []}))
|
||||
self.assertEqual(self.migrate(), [f"moved 3 files from {self.legacy} to {self.target}"])
|
||||
|
||||
def test_an_id_journal_gets_only_the_rows_it_lacks(self):
|
||||
kept = self.write(self.target / "queue-history.jsonl", '{"id": "a"}\n{"id": "b"}\n')
|
||||
self.write(self.legacy / "queue-history.jsonl", '{"id": "b"}\n{"id": "c"}\n{"id": "c"}\n')
|
||||
self.assertEqual(self.migrate(), [f"merged 1 rows of {self.legacy / 'queue-history.jsonl'} into {kept}"])
|
||||
self.assertEqual(rows(kept), ["a", "b", "c"])
|
||||
self.assertFalse(self.legacy.exists())
|
||||
self.assertEqual(self.migrate(), [])
|
||||
self.assertEqual(rows(kept), ["a", "b", "c"])
|
||||
|
||||
def test_a_merge_appends_after_a_row_another_writer_adds_meanwhile(self):
|
||||
self.write(self.legacy / "news.jsonl", '{"id": "L1"}\n')
|
||||
kept = self.write(self.target / "news.jsonl", '{"id": "T1"}\n')
|
||||
real, calls = self.councilroot.row_ids, []
|
||||
|
||||
def racing(text: str):
|
||||
calls.append(text)
|
||||
if len(calls) == 2:
|
||||
with open(kept, "a", encoding="utf-8") as other:
|
||||
other.write('{"id": "CONCURRENT"}\n')
|
||||
return real(text)
|
||||
|
||||
with mock.patch.object(self.councilroot, "row_ids", racing):
|
||||
self.assertEqual(self.migrate(), [f"merged 1 rows of {self.legacy / 'news.jsonl'} into {kept}"])
|
||||
self.assertEqual(rows(kept), ["T1", "CONCURRENT", "L1"])
|
||||
|
||||
def test_a_journal_without_ids_merges_each_line_as_often_as_the_legacy_side_has_it_more_in_ts_order(self):
|
||||
kept = self.write(self.target / "news.jsonl", '{"ts": 1.0, "text": "a"}\n{"ts": 3.0, "text": "c"}\n')
|
||||
self.write(self.legacy / "news.jsonl", '{"ts": 3.0, "text": "c"}\n{"ts": 2.0, "text": "b"}\n{"ts": 2.0, "text": "b"}\n')
|
||||
untimed = self.write(self.target / "scores.jsonl", '{"time": 1}\n')
|
||||
self.write(self.legacy / "scores.jsonl", '{"time": 2}\n{"time": 1}\n{"time": 1}\n{"time": 1}\n')
|
||||
self.assertEqual(self.migrate(), [
|
||||
f"merged 2 rows of {self.legacy / 'news.jsonl'} into {kept}",
|
||||
f"merged 3 rows of {self.legacy / 'scores.jsonl'} into {untimed}",
|
||||
])
|
||||
self.assertEqual([json.loads(line)["text"] for line in kept.read_text(encoding="utf-8").splitlines()], ["a", "b", "b", "c"])
|
||||
self.assertEqual(untimed.read_text(encoding="utf-8"), '{"time": 1}\n{"time": 2}\n{"time": 1}\n{"time": 1}\n')
|
||||
self.assertEqual(kept.stat().st_mode & 0o777, 0o600)
|
||||
self.assertFalse(self.legacy.exists())
|
||||
|
||||
def test_no_link_on_either_side_is_followed(self):
|
||||
secret = self.write(self.base / "secret.txt", "secret\n")
|
||||
outside = self.base / "outside"
|
||||
self.write(outside / "inner.txt", "inner\n")
|
||||
elsewhere = self.base / "elsewhere"
|
||||
elsewhere.mkdir()
|
||||
(self.legacy / "evil").symlink_to(secret)
|
||||
(self.legacy / "linked").symlink_to(outside)
|
||||
self.write(self.legacy / "work" / "a" / "claim.json", "{}\n")
|
||||
self.target.mkdir(parents=True, mode=0o700)
|
||||
(self.target / "work").symlink_to(elsewhere)
|
||||
with self.assertRaises(self.councilroot.Blocked) as blocked:
|
||||
self.migrate()
|
||||
self.assertTrue(all(line.startswith(f"unmoved {self.legacy / 'work' / 'a' / 'claim.json'}: ") for line in blocked.exception.lines), blocked.exception.lines)
|
||||
report = self.councilroot.migrate(self.legacy, self.target, True)
|
||||
self.assertIn(f"skipped {self.legacy / 'evil'}: not a regular file", report)
|
||||
self.assertIn(f"skipped {self.legacy / 'linked'}: a link", report)
|
||||
self.assertTrue(any(re.match(rf"unmoved {re.escape(str(self.legacy / 'work' / 'a' / 'claim.json'))}\.[0-9a-f]{{16}}\.migrating: ", line) for line in report), report)
|
||||
self.assertEqual(list(elsewhere.iterdir()), [])
|
||||
self.assertFalse((self.target / "evil").exists() or (self.target / "linked").exists())
|
||||
self.assertEqual((secret.read_text(encoding="utf-8"), (outside / "inner.txt").read_text(encoding="utf-8")), ("secret\n", "inner\n"))
|
||||
|
||||
def test_a_linked_or_shared_legacy_root_is_refused(self):
|
||||
linked = self.base / "linked-legacy"
|
||||
linked.symlink_to(self.legacy)
|
||||
with self.assertRaises(ValueError):
|
||||
self.councilroot.migrate(linked, self.target)
|
||||
self.legacy.chmod(0o777)
|
||||
self.addCleanup(self.legacy.chmod, 0o700)
|
||||
done = self.cli("migrate", "--legacy", str(self.legacy), AGENTS_COUNCIL_ROOT=str(self.target))
|
||||
self.assertEqual(done.returncode, 2)
|
||||
self.assertIn("group and others cannot write", done.stderr)
|
||||
|
||||
def test_a_missing_legacy_root_moves_nothing(self):
|
||||
done = self.cli("migrate", "--legacy", str(self.base / "gone"), AGENTS_COUNCIL_ROOT=str(self.target))
|
||||
self.assertEqual((done.returncode, done.stdout), (0, f"nothing to move from {self.base / 'gone'}\n"))
|
||||
|
||||
def test_orchestrator_stop_covers_every_unit_that_writes_a_council_root(self):
|
||||
units = (REPO_ROOT / "scripts" / "orchestrator.sh").read_text(encoding="utf-8")
|
||||
body = units[units.index("stop_all() {"):]
|
||||
body = body[:body.index("\n}\n")]
|
||||
for unit in ("${RUNNER}", "${OFFICES_UNIT}.path", "${RESOURCES_UNIT}.timer", "${RESOURCES_UNIT}.service", "compose"):
|
||||
with self.subTest(unit=unit):
|
||||
self.assertIn(unit, body)
|
||||
self.assertIn("stop) stop_all ;;", units)
|
||||
|
||||
|
||||
NPX_STUB = """#!/usr/bin/env python3
|
||||
import json, os
|
||||
from pathlib import Path
|
||||
with open(os.environ["STUB_RECORD"], "a") as record:
|
||||
record.write(f"npx {Path(os.environ['STUB_CHECK']).is_file()}\\n")
|
||||
if os.environ.get("STUB_LINK"):
|
||||
os.link(*json.loads(os.environ["STUB_LINK"]))
|
||||
skill = Path(".agents/skills/kvb-new/SKILL.md")
|
||||
skill.parent.mkdir(parents=True)
|
||||
skill.write_text("new\\n")
|
||||
"""
|
||||
CP_STUB = """#!/usr/bin/env python3
|
||||
import json, os, sys
|
||||
from pathlib import Path
|
||||
with open(os.environ["STUB_RECORD"], "a") as record:
|
||||
record.write(f"cp {Path(os.environ['STUB_CHECK']).is_file()} {sys.argv[-1]}\\n")
|
||||
if sys.argv[-1].startswith(os.environ["HOME"]):
|
||||
for late, text in json.loads(os.environ.get("STUB_LATE", "{}")).items():
|
||||
if not Path(late).exists():
|
||||
Path(late).parent.mkdir(parents=True, exist_ok=True)
|
||||
Path(late).write_text(text)
|
||||
os.execv(os.environ["REAL_CP"], ["cp", *sys.argv[1:]])
|
||||
"""
|
||||
TOOLS = ("bash", "python3", "dirname", "rm", "find", "mkdir", "install", "grep", "sed", "ln", "cat", "id")
|
||||
|
||||
|
||||
class Settings(Home):
|
||||
def setUp(self):
|
||||
super().setUp()
|
||||
self.legacy = self.base / "legacy"
|
||||
self.legacy.mkdir(mode=0o700)
|
||||
self.settings = self.home / ".claude" / "settings.json"
|
||||
self.release = self.home / ".local" / "state" / "agents-release"
|
||||
self.allow(["~/.local/state/agents-shared", f"{self.release}/**", "/var/cache/unrelated", "."])
|
||||
|
||||
def allow(self, entries: list[str]) -> None:
|
||||
self.settings.parent.mkdir(parents=True, exist_ok=True)
|
||||
self.settings.write_text(json.dumps({"sandbox": {"enabled": True, "filesystem": {"allowWrite": entries}}}), encoding="utf-8")
|
||||
|
||||
def preflight(self) -> subprocess.CompletedProcess:
|
||||
return self.cli("preflight", "--legacy", str(self.legacy), "--managed", str(self.base / "managed"))
|
||||
|
||||
def settings_file(self, path: Path, entries: list[str]) -> Path:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(json.dumps({"sandbox": {"filesystem": {"allowWrite": entries}}}), encoding="utf-8")
|
||||
return path
|
||||
|
||||
|
||||
class TestPreflight(Settings):
|
||||
def test_both_agent_paths_must_be_writable_for_the_sandbox_and_the_host_root_never(self):
|
||||
self.assertEqual(self.preflight().returncode, 0, self.preflight().stderr)
|
||||
for entries, wanted in (
|
||||
(["~/.local/state/agents-shared"], f"add {self.release} to sandbox.filesystem.allowWrite in {self.settings}"),
|
||||
([f"{self.release}"], f"add {self.shared} to sandbox.filesystem.allowWrite in {self.settings}"),
|
||||
(["~/.local/state/agents-shared", str(self.release), "~/.local/state/agents-council/**"], f"{self.settings}: drop allowWrite entry '~/.local/state/agents-council/**' ({self.durable})"),
|
||||
(["~/.local/state/agents-shared", str(self.release), "~/.local/state"], f"{self.settings}: drop allowWrite entry '~/.local/state' ({self.durable.parent})"),
|
||||
):
|
||||
with self.subTest(entries=entries):
|
||||
self.allow(entries)
|
||||
done = self.preflight()
|
||||
self.assertEqual(done.returncode, 2)
|
||||
self.assertIn(wanted, done.stderr)
|
||||
self.settings.unlink()
|
||||
self.assertIn(f"{self.settings} holds no sandbox.filesystem.allowWrite list", self.preflight().stderr)
|
||||
|
||||
def test_every_settings_file_that_can_apply_is_checked_and_named_with_its_entry(self):
|
||||
repo = self.home / "Repositories" / "github.com" / "o" / "r"
|
||||
worktree = self.home / ".local" / "share" / "worktrees" / "github.com" / "o" / "r" / "fix" / "x"
|
||||
widening = (
|
||||
(self.base / "managed" / "managed-settings.json", "~"),
|
||||
(self.base / "managed" / "managed-settings.d" / "50-widen.json", "~/.local"),
|
||||
(self.home / ".claude" / "settings.local.json", "~/.local/state/agents-council"),
|
||||
(repo / ".claude" / "settings.json", "~/.local/state"),
|
||||
(repo / ".claude" / "settings.local.json", "../../../../.local"),
|
||||
(worktree / ".claude" / "settings.json", "~/.local/state/agents-council/resources/**"),
|
||||
)
|
||||
self.settings_file(repo / "nested" / ".claude" / "settings.json", ["/var/cache/fine"])
|
||||
self.assertEqual(self.preflight().returncode, 0, self.preflight().stderr)
|
||||
for path, entry in widening:
|
||||
with self.subTest(path=path):
|
||||
self.settings_file(path, [entry, "/var/cache/fine"])
|
||||
done = self.preflight()
|
||||
self.assertEqual(done.returncode, 2)
|
||||
self.assertIn(f"{path}: drop allowWrite entry {entry!r}", done.stderr)
|
||||
self.assertNotIn("add ", done.stderr)
|
||||
path.unlink()
|
||||
moved = self.home / "Code" / "o" / "r" / ".claude" / "settings.json"
|
||||
self.settings_file(moved, ["~/.local/state"])
|
||||
self.assertEqual(self.preflight().returncode, 0)
|
||||
config = self.home / ".config" / "pkgmgr" / "config.yaml"
|
||||
config.parent.mkdir(parents=True)
|
||||
config.write_text("directories:\n repositories: ~/Code/\n binaries: ~/.local/bin/\n", encoding="utf-8")
|
||||
self.assertIn(f"{moved}: drop allowWrite entry '~/.local/state'", self.preflight().stderr)
|
||||
|
||||
def test_the_queue_and_the_preflight_share_one_list_of_settings_files(self):
|
||||
self.assertNotIn("managed-settings", QUEUE.read_text(encoding="utf-8"))
|
||||
managed = self.base / "managed"
|
||||
dropin = managed / "managed-settings.d" / "50-deny.json"
|
||||
dropin.parent.mkdir(parents=True)
|
||||
dropin.write_text(json.dumps({"permissions": {"deny": ["Bash(rm *)"]}}), encoding="utf-8")
|
||||
queue = load(QUEUE, f"test_council_root_queue_settings_{id(self)}")
|
||||
project = self.base / "project"
|
||||
with mock.patch.object(queue.sibling("councilroot"), "MANAGED", managed):
|
||||
self.assertIn(dropin, queue.settings_files(project))
|
||||
self.assertEqual(queue.permission(["rm -rf x"], project), "deny")
|
||||
|
||||
def test_linked_repositories_linked_files_and_every_session_start_directory_are_checked(self):
|
||||
repositories = self.home / "Repositories"
|
||||
data = self.base / "data" / "r"
|
||||
linked = repositories / "github.com" / "o" / "r"
|
||||
linked.parent.mkdir(parents=True)
|
||||
linked.symlink_to(data)
|
||||
(repositories / "loop").symlink_to(repositories)
|
||||
project = self.home / "work" / "proj"
|
||||
session = self.home / ".claude" / "projects" / re.sub(r"[^A-Za-z0-9]", "-", str(project))
|
||||
session.mkdir(parents=True)
|
||||
(self.home / ".claude" / "projects" / "-gone-project").mkdir()
|
||||
target = self.settings_file(self.base / "shared-settings.json", ["~/.local/state"])
|
||||
cases = (
|
||||
(linked / ".claude" / "settings.json", lambda path: self.settings_file(data / ".claude" / "settings.json", ["~/.local/state"])),
|
||||
(project / ".claude" / "settings.local.json", lambda path: self.settings_file(path, ["~/.local/state"])),
|
||||
(repositories / "github.com" / "o" / "s" / ".claude" / "settings.json", lambda path: (path.parent.mkdir(parents=True), path.symlink_to(target))),
|
||||
)
|
||||
self.settings_file(project / "deeper" / ".claude" / "settings.json", ["~/.local/state"])
|
||||
self.assertEqual(self.preflight().returncode, 0, self.preflight().stderr)
|
||||
for path, make in cases:
|
||||
with self.subTest(path=path):
|
||||
make(path)
|
||||
done = self.preflight()
|
||||
self.assertEqual(done.returncode, 2)
|
||||
self.assertEqual(done.stderr.count(f"{path}: drop allowWrite entry '~/.local/state'"), 1, done.stderr)
|
||||
path.unlink()
|
||||
|
||||
def test_a_council_that_is_not_suspended_is_only_warned_about(self):
|
||||
done = self.preflight()
|
||||
self.assertEqual(done.returncode, 0, done.stderr)
|
||||
self.assertIn("warning: the agents are not suspended", done.stderr)
|
||||
(self.legacy / "night.json").write_text(json.dumps({"mode": "suspend"}), encoding="utf-8")
|
||||
done = self.preflight()
|
||||
self.assertEqual((done.returncode, done.stderr), (0, ""))
|
||||
|
||||
def test_a_busy_or_conflicting_legacy_root_is_refused_before_anything_moves(self):
|
||||
hold = self.legacy / "holds" / "1-2.lock"
|
||||
hold.parent.mkdir()
|
||||
hold.write_text("", encoding="utf-8")
|
||||
with open(hold, encoding="utf-8") as held:
|
||||
fcntl.flock(held, fcntl.LOCK_EX)
|
||||
self.assertIn(f"{hold} is held", self.preflight().stderr)
|
||||
(self.legacy / "queue.json").write_text(json.dumps({"seq": 3, "entries": [{"id": "test-3"}]}), encoding="utf-8")
|
||||
self.assertIn("holds 1 queue jobs", self.preflight().stderr)
|
||||
(self.legacy / "queue.json").write_text(json.dumps({"seq": 3, "entries": []}), encoding="utf-8")
|
||||
for name, legacy, shared, ok in (("rasci.json", "a", "b", False), ("night.json", "same", "same", True), ("news.jsonl", '{"id": 1}\n', '{"id": 2}\n', True)):
|
||||
(self.legacy / name).write_text(legacy, encoding="utf-8")
|
||||
self.shared.mkdir(parents=True, mode=0o700, exist_ok=True)
|
||||
(self.shared / name).write_text(shared, encoding="utf-8")
|
||||
with self.subTest(name=name):
|
||||
self.assertEqual(f"conflict {self.shared / name} differs" in self.preflight().stderr, not ok)
|
||||
(self.legacy / "rasci.json").unlink()
|
||||
self.assertEqual(self.preflight().returncode, 0, self.preflight().stderr)
|
||||
self.assertEqual(sorted(path.name for path in self.legacy.iterdir()), ["holds", "news.jsonl", "night.json", "queue.json"])
|
||||
|
||||
|
||||
class TestInstall(Settings):
|
||||
def setUp(self):
|
||||
super().setUp()
|
||||
self.repo = self.base / "repo"
|
||||
for part in ("scripts/install.sh", "scripts/orchestrator.sh", "skills/kvb-org-organize/scripts/councilroot.py", "skills/kvb-org-organize/scripts/nofollow.py"):
|
||||
(self.repo / part).parent.mkdir(parents=True, exist_ok=True)
|
||||
(self.repo / part).write_bytes((REPO_ROOT / part).read_bytes())
|
||||
stubs = self.base / "bin"
|
||||
stubs.mkdir()
|
||||
(stubs / "npx").write_text(NPX_STUB, encoding="utf-8")
|
||||
(stubs / "cp").write_text(CP_STUB, encoding="utf-8")
|
||||
(stubs / "systemctl").write_text("#!/bin/sh\nexit 1\n", encoding="utf-8")
|
||||
for stub in stubs.iterdir():
|
||||
stub.chmod(0o755)
|
||||
tools = self.base / "tools"
|
||||
tools.mkdir()
|
||||
for name in TOOLS:
|
||||
(tools / name).symlink_to(shutil.which(name))
|
||||
self.record = self.base / "stubs.log"
|
||||
self.installed = [self.home / root / "skills" / "kvb-old" / "SKILL.md" for root in (".agents", ".claude")]
|
||||
for marker in self.installed:
|
||||
marker.parent.mkdir(parents=True)
|
||||
marker.write_text("old\n", encoding="utf-8")
|
||||
self.env = {
|
||||
"PATH": f"{stubs}:{tools}", "HOME": str(self.home), "AGENTS_COUNCIL_LEGACY": str(self.legacy),
|
||||
"STUB_RECORD": str(self.record), "STUB_CHECK": str(self.shared / "rasci.json"), "REAL_CP": shutil.which("cp"),
|
||||
}
|
||||
|
||||
def install(self) -> subprocess.CompletedProcess:
|
||||
return subprocess.run(["bash", str(self.repo / "scripts" / "install.sh")], env=self.env, capture_output=True, text=True, timeout=120, check=False)
|
||||
|
||||
def test_a_refused_preflight_aborts_before_any_copy_and_the_old_skills_keep_the_legacy_state(self):
|
||||
(self.legacy / "queue.json").write_text(json.dumps({"seq": 3, "entries": [{"id": "test-3"}]}), encoding="utf-8")
|
||||
busy = self.install()
|
||||
self.allow(["~/.local/state/agents-shared"])
|
||||
(self.legacy / "queue.json").write_text(json.dumps({"seq": 3, "entries": []}), encoding="utf-8")
|
||||
unset = self.install()
|
||||
for done, reason in ((busy, "holds 1 queue jobs"), (unset, f"add {self.release} to sandbox.filesystem.allowWrite")):
|
||||
with self.subTest(reason=reason):
|
||||
self.assertEqual(done.returncode, 1, done.stderr)
|
||||
self.assertIn(reason, done.stderr)
|
||||
self.assertIn("skills: install refused before copying anything", done.stderr)
|
||||
self.assertFalse(self.record.exists() or self.shared.exists() or (self.repo / ".agents").exists())
|
||||
self.assertEqual([marker.read_text(encoding="utf-8") for marker in self.installed], ["old\n", "old\n"])
|
||||
self.assertTrue((self.legacy / "queue.json").is_file())
|
||||
|
||||
def test_a_migrate_that_fails_in_any_other_way_aborts_before_the_copy(self):
|
||||
script = self.repo / "skills" / "kvb-org-organize" / "scripts" / "councilroot.py"
|
||||
script.write_text(script.read_text(encoding="utf-8").replace("report = migrate(args.legacy, root(), args.catch_up)", 'raise RuntimeError("boom")'), encoding="utf-8")
|
||||
(self.legacy / "rasci.json").write_text("{}\n", encoding="utf-8")
|
||||
done = self.install()
|
||||
self.assertEqual(done.returncode, 1, done.stderr)
|
||||
self.assertIn("RuntimeError: boom", done.stderr)
|
||||
self.assertIn("skills: install refused before copying anything: the council root migration failed", done.stderr)
|
||||
self.assertEqual([marker.read_text(encoding="utf-8") for marker in self.installed], ["old\n", "old\n"])
|
||||
self.assertTrue((self.legacy / "rasci.json").is_file())
|
||||
|
||||
def test_a_file_that_turns_unmovable_after_the_preflight_aborts_with_the_old_root_intact(self):
|
||||
(self.legacy / "rasci.json").write_text('{"work/x": {}}\n', encoding="utf-8")
|
||||
(self.legacy / "queue.json").write_text('{"seq": 9, "entries": []}\n', encoding="utf-8")
|
||||
self.env["STUB_LINK"] = json.dumps([str(self.legacy / "rasci.json"), str(self.base / "alias")])
|
||||
done = self.install()
|
||||
self.assertEqual(done.returncode, 1, done.stderr)
|
||||
self.assertIn(f"unmoved {self.legacy / 'rasci.json'}: not a regular file with a single link", done.stdout)
|
||||
self.assertIn("skills: install refused before copying anything: the council root migration failed (exit 4)", done.stderr)
|
||||
self.assertEqual(sorted(path.name for path in self.legacy.iterdir()), ["queue.json", "rasci.json"])
|
||||
self.assertFalse((self.shared / "rasci.json").exists() or (self.shared / "queue.json").exists())
|
||||
self.assertEqual([marker.read_text(encoding="utf-8") for marker in self.installed], ["old\n", "old\n"])
|
||||
|
||||
def test_a_partial_move_still_installs_the_new_skills_names_every_file_left_and_fails_at_the_end(self):
|
||||
script = self.repo / "skills" / "kvb-org-organize" / "scripts" / "councilroot.py"
|
||||
script.write_text(script.read_text(encoding="utf-8").replace(
|
||||
"outcome, line = settle(helper, legacy, source, target, catch_up)",
|
||||
'outcome, line = ("unmoved", f"unmoved {source}: raced") if source.name == "late.json" else settle(helper, legacy, source, target, catch_up)',
|
||||
), encoding="utf-8")
|
||||
(self.legacy / "rasci.json").write_text("{}\n", encoding="utf-8")
|
||||
(self.legacy / "late.json").write_text("{}\n", encoding="utf-8")
|
||||
done = self.install()
|
||||
self.assertEqual(done.returncode, 1, done.stderr)
|
||||
self.assertGreaterEqual(done.stderr.count(f"unmoved {self.legacy / 'late.json'}: raced"), 2, done.stderr)
|
||||
self.assertTrue((self.shared / "rasci.json").is_file() and (self.legacy / "late.json").is_file())
|
||||
self.assertEqual([(marker.parent.parent / "kvb-new" / "SKILL.md").read_text(encoding="utf-8") for marker in self.installed], ["new\n"] * 2)
|
||||
|
||||
def test_a_crash_after_a_partial_move_still_installs_the_new_skills_and_names_the_marker(self):
|
||||
script = self.repo / "skills" / "kvb-org-organize" / "scripts" / "councilroot.py"
|
||||
settle = "outcome, line = settle(helper, legacy, source, target, catch_up)"
|
||||
script.write_text(script.read_text(encoding="utf-8").replace(settle, f'if source.name == "zz.json":\n raise RuntimeError("boom")\n {settle}'), encoding="utf-8")
|
||||
(self.legacy / "rasci.json").write_text("{}\n", encoding="utf-8")
|
||||
(self.legacy / "zz.json").write_text("{}\n", encoding="utf-8")
|
||||
done = self.install()
|
||||
marker = self.durable / "migrate.started"
|
||||
self.assertEqual(done.returncode, 1, done.stderr)
|
||||
self.assertIn("RuntimeError: boom", done.stderr)
|
||||
self.assertIn(f"skills: the council root migration stopped halfway (exit 1): {marker} exists", done.stderr)
|
||||
self.assertTrue((self.shared / "rasci.json").is_file() and marker.is_file() and (self.legacy / "zz.json").is_file())
|
||||
self.assertEqual([(old.parent.parent / "kvb-new" / "SKILL.md").read_text(encoding="utf-8") for old in self.installed], ["new\n"] * 2)
|
||||
|
||||
def test_rows_and_register_changes_written_meanwhile_arrive_or_are_reported_never_lost(self):
|
||||
(self.legacy / "queue-history.jsonl").write_text('{"id": "early"}\n', encoding="utf-8")
|
||||
(self.legacy / "news.jsonl").write_text(json.dumps({"from": "A", "text": "before", "ts": 1.0}) + "\n", encoding="utf-8")
|
||||
(self.legacy / "rasci.json").write_text("{}\n", encoding="utf-8")
|
||||
window = '{"work/x": {"status": "window"}}\n'
|
||||
self.env["STUB_LATE"] = json.dumps({
|
||||
str(self.legacy / "queue-history.jsonl"): '{"id": "late"}\n',
|
||||
str(self.legacy / "news.jsonl"): json.dumps({"from": "B", "text": "posted in the window", "ts": 2.0}) + "\n",
|
||||
str(self.legacy / "rasci.json"): window,
|
||||
})
|
||||
done = self.install()
|
||||
self.assertEqual(done.returncode, 1, done.stderr)
|
||||
self.assertEqual(rows(self.shared / "queue-history.jsonl"), ["early", "late"])
|
||||
self.assertEqual([json.loads(line)["text"] for line in (self.shared / "news.jsonl").read_text(encoding="utf-8").splitlines()], ["before", "posted in the window"])
|
||||
kept = [path for path in self.shared.iterdir() if path.name.startswith("rasci.json.legacy-")]
|
||||
self.assertEqual([path.read_text(encoding="utf-8") for path in kept], [window])
|
||||
self.assertEqual((self.shared / "rasci.json").read_text(encoding="utf-8"), "{}\n")
|
||||
self.assertIn(f"kept {self.legacy / 'rasci.json'} beside {self.shared / 'rasci.json'} as {kept[0]}", done.stderr)
|
||||
self.assertFalse(self.legacy.exists())
|
||||
|
||||
def test_the_skills_are_built_first_and_reach_the_target_only_after_the_legacy_state_moved(self):
|
||||
(self.legacy / "rasci.json").write_text("{}\n", encoding="utf-8")
|
||||
done = self.install()
|
||||
self.assertEqual(done.returncode, 0, done.stderr)
|
||||
calls = self.record.read_text(encoding="utf-8").splitlines()
|
||||
self.assertEqual(calls[0], "npx False", done.stderr)
|
||||
into_target = [call for call in calls if call.startswith("cp ") and call.split(" ", 2)[2].startswith(str(self.home))]
|
||||
self.assertEqual(into_target, [f"cp True {self.home / root / 'skills'}/" for root in (".agents", ".claude")], done.stderr)
|
||||
self.assertFalse(self.legacy.exists())
|
||||
self.assertEqual([(marker.exists(), (marker.parent.parent / "kvb-new" / "SKILL.md").read_text(encoding="utf-8")) for marker in self.installed], [(False, "new\n")] * 2)
|
||||
|
||||
|
||||
class TestKeptCopy(Home):
|
||||
def setUp(self):
|
||||
super().setUp()
|
||||
self.watch = load(WATCH, f"test_council_root_watch_{id(self)}")
|
||||
|
||||
def write_rows(self, path: Path) -> None:
|
||||
path.write_text('{"id": "r1", "rule": "R", "path": "p", "sha1": "s"}\n{"id": "r2", "rule": "R", "path": "q", "sha1": "s"}\n', encoding="utf-8")
|
||||
|
||||
def test_agents_write_their_root_and_the_host_keeps_the_copies_apart(self):
|
||||
watch = self.watch
|
||||
self.assertEqual((watch.ROOT, watch.KEPT), (self.shared, self.durable))
|
||||
for origin, kept in watch.MIRRORED:
|
||||
with self.subTest(name=origin.name):
|
||||
self.assertEqual((origin.parent, kept.parent, origin.name), (self.shared, self.durable, kept.name))
|
||||
queue = load(QUEUE, f"test_council_root_queue_{id(self)}")
|
||||
self.assertEqual(queue.HISTORIES, (self.shared / "queue-history.jsonl", self.durable / "queue-history.jsonl"))
|
||||
|
||||
def test_the_hook_mirrors_acceptances_and_receipts_into_the_host_root_and_readers_merge_both(self):
|
||||
watch = self.watch
|
||||
self.shared.mkdir(parents=True, mode=0o700)
|
||||
for origin, _ in watch.MIRRORED:
|
||||
self.write_rows(origin)
|
||||
watch.keep_history()
|
||||
for _, kept in watch.MIRRORED:
|
||||
with self.subTest(name=kept.name):
|
||||
self.assertEqual((rows(kept), kept.stat().st_mode & 0o777), (["r1", "r2"], 0o600))
|
||||
self.assertEqual(self.durable.stat().st_mode & 0o777, 0o700)
|
||||
with watch.KEPT_ACCEPTED.open("a", encoding="utf-8") as kept:
|
||||
kept.write('{"id": "r0", "rule": "R", "path": "o", "sha1": "s"}\n')
|
||||
security = load(SECURITY, f"test_council_root_security_{id(self)}")
|
||||
self.assertEqual(security.accepted((watch.ACCEPTED, watch.KEPT_ACCEPTED)), {("R", name, "s", None) for name in "opq"})
|
||||
|
||||
def test_the_mirror_never_follows_a_link_in_the_host_root(self):
|
||||
watch = self.watch
|
||||
self.shared.mkdir(parents=True, mode=0o700)
|
||||
self.durable.mkdir(parents=True, mode=0o700)
|
||||
self.write_rows(watch.ACCEPTED)
|
||||
outside = self.base / "outside.jsonl"
|
||||
outside.write_text("", encoding="utf-8")
|
||||
watch.KEPT_ACCEPTED.symlink_to(outside)
|
||||
watch.keep_rows(watch.ACCEPTED, watch.KEPT_ACCEPTED)
|
||||
self.assertEqual(outside.read_text(encoding="utf-8"), "")
|
||||
watch.KEPT_ACCEPTED.unlink()
|
||||
os.link(outside, watch.KEPT_ACCEPTED)
|
||||
watch.keep_rows(watch.ACCEPTED, watch.KEPT_ACCEPTED)
|
||||
self.assertEqual(outside.read_text(encoding="utf-8"), "")
|
||||
watch.KEPT_ACCEPTED.unlink()
|
||||
elsewhere = self.base / "elsewhere"
|
||||
elsewhere.mkdir()
|
||||
self.durable.rmdir()
|
||||
self.durable.symlink_to(elsewhere)
|
||||
watch.keep_rows(watch.ACCEPTED, watch.KEPT_ACCEPTED)
|
||||
self.assertEqual(list(elsewhere.iterdir()), [])
|
||||
|
||||
def test_the_watch_hook_never_copies_a_journal_onto_itself(self):
|
||||
with mock.patch.dict(os.environ, {"AGENTS_COUNCIL_ROOT": str(self.durable)}):
|
||||
watch = load(WATCH, f"test_council_root_same_{id(self)}")
|
||||
self.durable.mkdir(parents=True, mode=0o700)
|
||||
for origin, _ in watch.MIRRORED:
|
||||
self.write_rows(origin)
|
||||
before = {origin: origin.read_bytes() for origin, _ in watch.MIRRORED}
|
||||
watch.keep_history()
|
||||
self.assertEqual({origin: origin.read_bytes() for origin, _ in watch.MIRRORED}, before)
|
||||
|
||||
def test_an_explicit_root_still_keeps_its_rows_in_the_durable_one(self):
|
||||
with mock.patch.dict(os.environ, {"AGENTS_COUNCIL_ROOT": str(self.base / "council")}):
|
||||
watch = load(WATCH, f"test_council_root_explicit_{id(self)}")
|
||||
self.assertEqual((watch.ROOT, watch.KEPT), (self.base / "council", self.durable))
|
||||
watch.ROOT.mkdir(mode=0o700)
|
||||
self.write_rows(watch.HISTORY)
|
||||
watch.keep_history()
|
||||
self.assertEqual(rows(watch.KEPT_HISTORY), ["r1", "r2"])
|
||||
|
||||
|
||||
class TestHostWriters(Home):
|
||||
def outside(self) -> Path:
|
||||
path = self.base / "outside"
|
||||
path.write_text("", encoding="utf-8")
|
||||
return path
|
||||
|
||||
def test_the_subagent_log_lies_in_the_host_root_and_never_follows_a_link(self):
|
||||
payload = json.dumps({"session_id": "s1", "tool_input": {"description": "d", "prompt": "task-class: judgement"}})
|
||||
state = self.base / "xdg"
|
||||
env = {"PATH": os.environ["PATH"], "HOME": str(self.home), "XDG_STATE_HOME": str(state)}
|
||||
spawn = subprocess.run([sys.executable, str(LOG), "spawn"], input=payload, env=env, capture_output=True, text=True, check=False)
|
||||
self.assertEqual(spawn.returncode, 0, spawn.stderr)
|
||||
log = state / "agents-council" / "subagent-model.jsonl"
|
||||
self.assertEqual(json.loads(log.read_text(encoding="utf-8"))["task_class"], "judgement")
|
||||
outside = self.outside()
|
||||
log.unlink()
|
||||
log.symlink_to(outside)
|
||||
subprocess.run([sys.executable, str(LOG), "spawn"], input=payload, env=env, capture_output=True, text=True, check=False)
|
||||
self.assertEqual(outside.read_text(encoding="utf-8"), "")
|
||||
|
||||
def resources(self) -> ModuleType:
|
||||
with mock.patch.object(sys, "path", [str(CLIENT), *sys.path]):
|
||||
return load(CLIENT / "resources.py", f"test_council_root_resources_{id(self)}")
|
||||
|
||||
def test_the_resources_sampler_never_follows_a_link(self):
|
||||
resources = self.resources()
|
||||
state = self.durable / "resources"
|
||||
state.mkdir(parents=True, mode=0o700)
|
||||
outside = self.outside()
|
||||
(state / "samples.jsonl").symlink_to(outside)
|
||||
with self.assertRaises(OSError):
|
||||
resources.append_rows(state / "samples.jsonl", [{"t": 1}])
|
||||
self.assertEqual(outside.read_text(encoding="utf-8"), "")
|
||||
elsewhere = self.base / "elsewhere"
|
||||
elsewhere.mkdir()
|
||||
(state / "samples.jsonl").unlink()
|
||||
state.rmdir()
|
||||
state.symlink_to(elsewhere)
|
||||
for write in (lambda: resources.append_rows(state / "tokens.jsonl", [{"t": 1}]), lambda: resources.replace(state / "offsets.json", "{}")):
|
||||
with self.assertRaises(OSError):
|
||||
write()
|
||||
self.assertEqual(list(elsewhere.iterdir()), [])
|
||||
|
||||
def gates(self) -> ModuleType:
|
||||
return load(GATES, f"test_council_root_gates_{id(self)}")
|
||||
|
||||
def test_gates_db_is_never_written_through_a_link(self):
|
||||
gates = self.gates()
|
||||
self.durable.mkdir(parents=True, mode=0o700)
|
||||
outside = self.outside()
|
||||
(self.durable / "gates.db").symlink_to(outside)
|
||||
with self.assertRaises(OSError):
|
||||
gates.opened(self.durable / "gates.db", True, self.durable)
|
||||
self.assertEqual(outside.read_bytes(), b"")
|
||||
|
||||
def test_an_unusable_council_root_is_named_in_the_gate_hint_instead_of_a_traceback(self):
|
||||
target = self.base / "target"
|
||||
target.mkdir()
|
||||
linked = self.base / "linked"
|
||||
linked.symlink_to(target)
|
||||
with mock.patch.dict(os.environ, {"AGENTS_COUNCIL_ROOT": str(linked)}):
|
||||
gates = self.gates()
|
||||
hint = gates.producer(str(self.base), "main", "test", "0" * 40)
|
||||
self.assertIn(f"{linked} must be a directory of uid {os.getuid()}", hint)
|
||||
self.assertEqual(list(target.iterdir()), [])
|
||||
|
||||
|
||||
class TestRetention(Home):
|
||||
def setUp(self):
|
||||
super().setUp()
|
||||
self.shared.mkdir(parents=True, mode=0o700)
|
||||
self.radio = load(RADIO, f"test_council_root_radio_{id(self)}")
|
||||
|
||||
def test_an_oversized_radio_log_still_yields_its_newest_rows(self):
|
||||
row = json.dumps({"from": "A", "to": ["B"], "class": "mayday", "text": "mayday", "ts": 2.0}) + "\n"
|
||||
filler = json.dumps({"from": "A", "text": "x" * 200, "ts": 1.0}) + "\n"
|
||||
self.radio.NEWS.write_text(filler * 20 + row, encoding="utf-8")
|
||||
with mock.patch.object(self.radio.organize("nofollow"), "READ_LIMIT", 1024):
|
||||
found = self.radio.items()
|
||||
self.assertEqual(found[-1]["text"], "mayday")
|
||||
self.assertLess(len(found), 6)
|
||||
|
||||
def test_rows_the_radio_drops_go_to_a_dated_archive_the_readers_skip(self):
|
||||
with mock.patch.object(self.radio, "KEPT", 3):
|
||||
for number in range(5):
|
||||
self.radio.store([{"from": "A", "text": f"m{number}"}])
|
||||
self.assertEqual([row["text"] for row in self.radio.items()], ["m2", "m3", "m4"])
|
||||
archive = self.shared / "archive" / f"news-{time.strftime('%Y-%m-%d')}.jsonl"
|
||||
self.assertEqual([json.loads(line)["text"] for line in archive.read_text(encoding="utf-8").splitlines()], ["m0", "m1"])
|
||||
self.assertEqual(archive.stat().st_mode & 0o777, 0o600)
|
||||
|
||||
def test_old_gate_logs_and_pages_move_to_a_dated_archive_and_links_stay(self):
|
||||
old = time.time() - 31 * 86400
|
||||
stale = {self.shared / "gates" / "slug" / "gate-a.log", self.shared / "pages" / "branch" / "index.html"}
|
||||
fresh = self.shared / "gates" / "slug" / "gate-b.log"
|
||||
for path in (*stale, fresh):
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text("x\n", encoding="utf-8")
|
||||
for path in stale:
|
||||
os.utime(path, (old, old))
|
||||
outside = self.base / "outside.log"
|
||||
outside.write_text("keep\n", encoding="utf-8")
|
||||
os.utime(outside, (old, old))
|
||||
(self.shared / "gates" / "linked.log").symlink_to(outside)
|
||||
done = self.cli("retire")
|
||||
archive = self.shared / "archive" / time.strftime("%Y-%m-%d")
|
||||
self.assertEqual((done.returncode, done.stdout), (0, f"archived 2 files into {archive}\n"), done.stderr)
|
||||
self.assertEqual(sorted(str(path.relative_to(archive)) for path in archive.rglob("*") if path.is_file()), ["gates/slug/gate-a.log", "pages/branch/index.html"])
|
||||
self.assertTrue(fresh.is_file() and (self.shared / "gates" / "linked.log").is_symlink())
|
||||
self.assertEqual(outside.read_text(encoding="utf-8"), "keep\n")
|
||||
self.assertIn("ExecStartPre=-/usr/bin/python3 -I %h/.claude/skills/kvb-org-organize/scripts/councilroot.py retire\n", CLEANUP_SERVICE.read_text(encoding="utf-8"))
|
||||
|
||||
|
||||
class TestGuard(unittest.TestCase):
|
||||
def test_no_script_or_skill_text_keeps_persistent_data_in_tmp(self):
|
||||
hits = []
|
||||
for top in SCANNED:
|
||||
for path in sorted((REPO_ROOT / top).rglob("*")):
|
||||
if not path.is_file() or "__pycache__" in path.parts:
|
||||
continue
|
||||
for number, line in enumerate(path.read_text(encoding="utf-8", errors="replace").splitlines(), 1):
|
||||
if FORBIDDEN.search(line) and not (path == COUNCIL_ROOT and line == LEGACY_LINE):
|
||||
hits.append(f"{path.relative_to(REPO_ROOT)}:{number}")
|
||||
self.assertEqual(hits, [])
|
||||
|
||||
def test_the_legacy_root_is_named_once_as_the_migration_source(self):
|
||||
found = [line for line in COUNCIL_ROOT.read_text(encoding="utf-8").splitlines() if "/tmp/agents-council" in line]
|
||||
self.assertEqual(found, [LEGACY_LINE])
|
||||
|
||||
def test_no_skill_text_turns_a_failed_dir_call_into_a_root_path(self):
|
||||
hits = [str(path.relative_to(REPO_ROOT)) for path in sorted((REPO_ROOT / "skills").rglob("*.md")) if DIR_SLASH.search(path.read_text(encoding="utf-8"))]
|
||||
self.assertEqual(hits, [])
|
||||
|
||||
def test_the_agent_root_is_named_where_it_is_inventoried_or_mounted(self):
|
||||
self.assertIn("~/.local/state/agents-shared", (REPO_ROOT / "skills" / "kvb-cleanup" / "SKILL.md").read_text(encoding="utf-8"))
|
||||
self.assertIn("~/.local/state/agents-shared.", (CLIENT / "offices.py").read_text(encoding="utf-8"))
|
||||
mounts = (REPO_ROOT / "docs" / "requirements" / "003-operator-workbench.md").read_text(encoding="utf-8").split("- **Mounts**:", 1)[1].split("\n\n", 1)[0]
|
||||
self.assertIn("`~/.local/state/agents-shared`", mounts)
|
||||
self.assertNotIn("the same directory is also", mounts)
|
||||
|
||||
def test_gate_logs_never_go_to_the_temp_directory(self):
|
||||
found = [str(path.relative_to(REPO_ROOT)) for path in sorted(GATE_SKILL.rglob("*.py")) if "gettempdir" in path.read_text(encoding="utf-8")]
|
||||
self.assertEqual(found, [])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
+348
-6
@@ -5,6 +5,7 @@ from __future__ import annotations
|
||||
import argparse
|
||||
import contextlib
|
||||
import fcntl
|
||||
import hashlib
|
||||
import importlib.util
|
||||
import io
|
||||
import json
|
||||
@@ -12,6 +13,7 @@ import os
|
||||
import pwd
|
||||
import shutil
|
||||
import signal
|
||||
import stat
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
@@ -26,6 +28,18 @@ SCRIPT = REPO_ROOT / "skills" / "kvb-check-gate" / "scripts" / "gate.py"
|
||||
HOTFIX = REPO_ROOT / "skills" / "kvb-git-hotfix" / "scripts" / "hotfix.py"
|
||||
RUN_LOCK = "kvb-gate-run.lock"
|
||||
SYNC_LOCK = "kvb-gate-sync.lock"
|
||||
STATE = "kvb-gate.json"
|
||||
LOCK_BASE = Path("/tmp/agents-locks")
|
||||
EARLIER_GATE = (
|
||||
"import fcntl, sys\n"
|
||||
"with open(sys.argv[1], 'a', encoding='utf-8') as lock:\n"
|
||||
" try:\n"
|
||||
" fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)\n"
|
||||
" except BlockingIOError:\n"
|
||||
" print('held')\n"
|
||||
" else:\n"
|
||||
" print('free')\n"
|
||||
)
|
||||
GREEN = {
|
||||
"Makefile": (
|
||||
"include mk/*.mk\n"
|
||||
@@ -70,6 +84,7 @@ RED = {
|
||||
"tracked.txt": "one\n",
|
||||
}
|
||||
USAGE = {"Makefile": "test:\n\t@true\nbuild:\n\t@true\n"}
|
||||
MARKING = {"Makefile": ".PHONY: test\ntest:\n\t@touch \"$$MARKER\"\n"}
|
||||
VOID = {"Makefile": ".PHONY: test test-a test-b\ntest:\n\t@echo all\ntest-a:\n\t@echo a\ntest-b:\n\t@echo b\n"}
|
||||
ORACLE = {"Makefile": ".PHONY: test test-oracle\ntest:\n\t@echo all\ntest-oracle:\n\t@echo oracle\n"}
|
||||
FAILED = ("--fail-regex", r"FAILED TARGETS: \S")
|
||||
@@ -95,6 +110,17 @@ def load_gate() -> object:
|
||||
return gate
|
||||
|
||||
|
||||
def lock_dir(worktree: Path) -> Path:
|
||||
return LOCK_BASE / hashlib.sha256(os.fsencode(str(worktree))).hexdigest()[:16]
|
||||
|
||||
|
||||
def drop_lock_dir(path: Path) -> None:
|
||||
if path.is_symlink():
|
||||
path.unlink()
|
||||
else:
|
||||
shutil.rmtree(path, ignore_errors=True)
|
||||
|
||||
|
||||
class GateCase(unittest.TestCase):
|
||||
files: ClassVar[dict[str, str]]
|
||||
|
||||
@@ -105,6 +131,7 @@ class GateCase(unittest.TestCase):
|
||||
self.repo = self.root / "repo"
|
||||
self.logs = self.root / "logs"
|
||||
self.repo.mkdir()
|
||||
self.addCleanup(drop_lock_dir, lock_dir(self.repo))
|
||||
self.env = {
|
||||
"PATH": "/usr/bin:/bin",
|
||||
"HOME": tmp.name,
|
||||
@@ -378,16 +405,280 @@ class TestWorktreeLock(GateCase):
|
||||
files = VOID
|
||||
|
||||
def lock(self, name: str) -> Path:
|
||||
return Path(self._git("rev-parse", "--absolute-git-dir").strip()) / name
|
||||
LOCK_BASE.mkdir(mode=0o700, exist_ok=True)
|
||||
lock_dir(self.repo).mkdir(mode=0o700, exist_ok=True)
|
||||
return lock_dir(self.repo) / name
|
||||
|
||||
def lock_free(self) -> bool:
|
||||
with open(self.lock(RUN_LOCK), "a", encoding="utf-8") as handle:
|
||||
def test_a_read_only_git_dir_holds_no_lock_and_the_hold_still_runs(self):
|
||||
git_dir = self.repo / ".git"
|
||||
git_dir.chmod(0o555)
|
||||
self.addCleanup(git_dir.chmod, 0o755)
|
||||
result = self.gate("hold", "--repo", str(self.repo), "--", "sh", "-c", "exit 7")
|
||||
self.assertEqual(result.returncode, 7, result.stderr)
|
||||
self.assertEqual(list(git_dir.glob("kvb-gate-*.lock")), [])
|
||||
locks = lock_dir(self.repo)
|
||||
self.assertEqual(sorted(path.name for path in locks.iterdir()), [RUN_LOCK, SYNC_LOCK])
|
||||
self.assertEqual(stat.S_IMODE(locks.stat().st_mode), 0o700)
|
||||
self.assertEqual({stat.S_IMODE((locks / name).stat().st_mode) for name in (RUN_LOCK, SYNC_LOCK)}, {0o600})
|
||||
|
||||
def test_the_lock_path_ignores_tmpdir_council_root_state_and_home(self):
|
||||
with open(self.lock(RUN_LOCK), "a", encoding="utf-8") as lock:
|
||||
fcntl.flock(lock, fcntl.LOCK_EX)
|
||||
for name in ("TMPDIR", "AGENTS_COUNCIL_ROOT", "XDG_STATE_HOME", "HOME"):
|
||||
for value in (str(self.root / f"other-{name}"), "relative"):
|
||||
with self.subTest(name=name, value=value):
|
||||
result = subprocess.run(
|
||||
[sys.executable, str(SCRIPT), "hold", "--repo", str(self.repo), "--", "true"],
|
||||
cwd=self.root, env={**self.env, name: value}, capture_output=True, text=True, check=False,
|
||||
)
|
||||
self.assertEqual(result.returncode, 75, result.stderr)
|
||||
self.assertEqual([path.name for path in self.root.iterdir() if path.name.startswith(("other-", "relative"))], [])
|
||||
|
||||
def test_a_link_shared_directory_or_second_name_at_any_level_is_refused(self):
|
||||
gate = load_gate()
|
||||
elsewhere, victim = self.root / "elsewhere", self.root / "victim"
|
||||
elsewhere.mkdir()
|
||||
victim.write_text("keep\n", encoding="utf-8")
|
||||
key = lock_dir(self.repo).name
|
||||
for case, refusal in (
|
||||
("base link", "cannot open the worktree lock {base}:"),
|
||||
("key link", "cannot open the worktree lock {base}/{key}:"),
|
||||
("run link", f"cannot open the worktree lock {{base}}/{{key}}/{RUN_LOCK}:"),
|
||||
("sync link", f"cannot open the worktree lock {{base}}/{{key}}/{SYNC_LOCK}:"),
|
||||
("base shared", f"{{base}} must be a directory of uid {os.getuid()} that group and others cannot write"),
|
||||
("key shared", f"{{base}}/{{key}} must be a directory of uid {os.getuid()} that group and others cannot write"),
|
||||
("run hardlink", f"{{base}}/{{key}}/{RUN_LOCK} must be a regular file of uid {os.getuid()} with one link"),
|
||||
("run fifo", f"{{base}}/{{key}}/{RUN_LOCK} must be a regular file of uid {os.getuid()} with one link"),
|
||||
):
|
||||
with self.subTest(case):
|
||||
base = self.root / case.replace(" ", "-")
|
||||
level, kind = case.split()
|
||||
path = {"base": base, "key": base / key, "run": base / key / RUN_LOCK, "sync": base / key / SYNC_LOCK}[level]
|
||||
for directory in {"base": (), "key": (base,)}.get(level, (base, base / key)):
|
||||
directory.mkdir(mode=0o700)
|
||||
if kind == "link":
|
||||
path.symlink_to(elsewhere if level in ("base", "key") else elsewhere / path.name)
|
||||
elif kind == "shared":
|
||||
path.mkdir()
|
||||
path.chmod(0o777)
|
||||
elif kind == "hardlink":
|
||||
os.link(victim, path)
|
||||
else:
|
||||
os.mkfifo(path, 0o600)
|
||||
with (
|
||||
contextlib.redirect_stderr(io.StringIO()) as err,
|
||||
self.assertRaises(SystemExit) as raised,
|
||||
):
|
||||
gate.lock_files(self.repo, base)
|
||||
self.assertEqual(raised.exception.code, 2)
|
||||
self.assertIn(refusal.format(base=base, key=key), err.getvalue())
|
||||
self.assertEqual(list(elsewhere.iterdir()), [])
|
||||
self.assertEqual(victim.read_text(encoding="utf-8"), "keep\n")
|
||||
marker = self.root / "ran"
|
||||
lock_dir(self.repo).parent.mkdir(mode=0o700, exist_ok=True)
|
||||
lock_dir(self.repo).symlink_to(elsewhere, target_is_directory=True)
|
||||
result = self.gate("hold", "--repo", str(self.repo), "--", "touch", str(marker))
|
||||
self.assertEqual(result.returncode, 2, result.stderr)
|
||||
self.assertIn(f"gate: cannot open the worktree lock {lock_dir(self.repo)}:", result.stderr)
|
||||
self.assertNotIn("Traceback", result.stderr)
|
||||
self.assertEqual((list(elsewhere.iterdir()), marker.exists()), ([], False))
|
||||
|
||||
def test_a_held_git_dir_lock_keeps_run_and_hold_out_and_a_read_only_one_is_still_held(self):
|
||||
git_lock = self.repo / ".git" / RUN_LOCK
|
||||
with open(git_lock, "a", encoding="utf-8") as lock:
|
||||
fcntl.flock(lock, fcntl.LOCK_EX)
|
||||
lock.write("4242\tgate.py hold -- make deploy\n")
|
||||
lock.flush()
|
||||
run = self.run_targets("test-a", extra=())
|
||||
hold = self.gate("hold", "--repo", str(self.repo), "--", "true")
|
||||
for result in (run, hold):
|
||||
with self.subTest(command=result.args[2]):
|
||||
self.assertEqual(result.returncode, 75, result.stderr)
|
||||
self.assertIn(f"another gate.py run or hold or a hotfix.py sync holds the git dir lock of {self.repo} "
|
||||
"(last recorded pid 4242: gate.py hold -- make deploy, possibly stale)", result.stderr)
|
||||
git_lock.chmod(0o400)
|
||||
git_dir = self.repo / ".git"
|
||||
git_dir.chmod(0o555)
|
||||
self.addCleanup(git_dir.chmod, 0o755)
|
||||
reader = EARLIER_GATE.replace("'a'", "'r'")
|
||||
result = self.gate("hold", "--repo", str(self.repo), "--", sys.executable, "-c", reader, str(git_lock))
|
||||
self.assertEqual((result.returncode, result.stdout), (0, "held\n"), result.stderr)
|
||||
self.assertEqual(git_lock.read_text(encoding="utf-8"), "4242\tgate.py hold -- make deploy\n")
|
||||
|
||||
def test_a_missing_git_dir_lock_is_created_and_held_so_an_earlier_gate_py_stays_out(self):
|
||||
git_lock = self.repo / ".git" / RUN_LOCK
|
||||
self.assertFalse(git_lock.exists())
|
||||
result = self.gate("hold", "--repo", str(self.repo), "--", sys.executable, "-c", EARLIER_GATE, str(git_lock))
|
||||
self.assertEqual((result.returncode, result.stdout), (0, "held\n"), result.stderr)
|
||||
self.assertEqual(stat.S_IMODE(git_lock.stat().st_mode), 0o600)
|
||||
self.assertIn(f"\tgate.py hold --repo {self.repo} -- {sys.executable} -c ", git_lock.read_text(encoding="utf-8"))
|
||||
self.assertTrue(self.free(git_lock))
|
||||
|
||||
def test_a_gate_py_with_a_tmp_of_its_own_is_kept_out_by_the_git_dir_lock(self):
|
||||
host, box = load_gate(), load_gate()
|
||||
for gate, name in ((host, "host-tmp"), (box, "box-tmp")):
|
||||
(self.root / name).mkdir(mode=0o700)
|
||||
gate.LOCK_BASE = self.root / name / "agents-locks"
|
||||
with (
|
||||
host.worktree_lock(self.repo, ["gate.py", "hold", "--", "make", "deploy"], []),
|
||||
contextlib.redirect_stderr(io.StringIO()) as err,
|
||||
self.assertRaises(SystemExit) as raised,
|
||||
box.worktree_lock(self.repo, ["gate.py", "hold"], []),
|
||||
):
|
||||
self.fail("a gate.py with a /tmp of its own took the worktree lock")
|
||||
self.assertEqual(raised.exception.code, 75)
|
||||
self.assertIn(f"holds the git dir lock of {self.repo} (last recorded pid {os.getpid()}: gate.py hold -- make "
|
||||
"deploy, possibly stale)", err.getvalue())
|
||||
with box.worktree_lock(self.repo, ["gate.py", "hold"], []) as fds:
|
||||
self.assertEqual(len(fds), 2)
|
||||
|
||||
def test_a_read_only_git_dir_lock_holder_is_named_live_by_its_tmp_lock_and_the_git_dir_line_as_stale(self):
|
||||
git_lock = self.repo / ".git" / RUN_LOCK
|
||||
git_lock.write_text("4242\tgate.py run --repo OLD-FINISHED-RUN\n", encoding="utf-8")
|
||||
git_lock.chmod(0o400)
|
||||
self.addCleanup(git_lock.chmod, 0o600)
|
||||
live, box = load_gate(), load_gate()
|
||||
(self.root / "box-tmp").mkdir(mode=0o700)
|
||||
box.LOCK_BASE = self.root / "box-tmp" / "agents-locks"
|
||||
with live.worktree_lock(self.repo, ["gate.py", "hold", "--", "live"], []):
|
||||
same_tmp = self.gate("hold", "--repo", str(self.repo), "--", "true")
|
||||
with (
|
||||
contextlib.redirect_stderr(io.StringIO()) as err,
|
||||
self.assertRaises(SystemExit) as raised,
|
||||
box.worktree_lock(self.repo, ["gate.py", "hold"], []),
|
||||
):
|
||||
self.fail("a gate.py with a /tmp of its own took the worktree lock")
|
||||
self.assertEqual(same_tmp.returncode, 75, same_tmp.stderr)
|
||||
self.assertIn(f"another gate.py run or hold holds {self.repo} (pid {os.getpid()}: gate.py hold -- live)",
|
||||
same_tmp.stderr)
|
||||
self.assertEqual(raised.exception.code, 75)
|
||||
self.assertIn("(last recorded pid 4242: gate.py run --repo OLD-FINISHED-RUN, possibly stale)", err.getvalue())
|
||||
self.assertEqual(git_lock.read_text(encoding="utf-8"), "4242\tgate.py run --repo OLD-FINISHED-RUN\n")
|
||||
|
||||
def test_a_recorded_holder_reaches_the_message_with_control_characters_escaped(self):
|
||||
gate, lock = load_gate(), self.root / "holder.lock"
|
||||
for recorded, named in (
|
||||
("4242\tgate.py run \x1b[2Jforged\nERROR: done\x07\n", "pid 4242: gate.py run \\x1b[2Jforged\\nERROR: done\\x07"),
|
||||
("4242 \x1b[2J\tgate.py run\n", "holder not recorded"),
|
||||
("4242\tgate.py hold -- make test-é\n", "pid 4242: gate.py hold -- make test-é"),
|
||||
):
|
||||
with self.subTest(recorded=recorded):
|
||||
lock.write_text(recorded, encoding="utf-8")
|
||||
with lock.open("rb") as handle:
|
||||
self.assertEqual(gate.holder(handle.fileno()), named)
|
||||
|
||||
def test_a_fifo_link_or_second_name_as_git_dir_lock_ends_the_hold_with_2_at_once(self):
|
||||
git_lock = self.repo / ".git" / RUN_LOCK
|
||||
victim, marker = self.root / "victim", self.root / "ran"
|
||||
victim.write_text("keep\n", encoding="utf-8")
|
||||
second_name = f"{git_lock} must be a regular file of uid {os.getuid()} with one link"
|
||||
for case, plant, refusal in (
|
||||
("fifo", lambda: os.mkfifo(git_lock, 0o600), second_name),
|
||||
("hardlink", lambda: os.link(victim, git_lock), second_name),
|
||||
("symlink", lambda: git_lock.symlink_to(victim), f"cannot open the git dir lock {git_lock}:"),
|
||||
):
|
||||
with self.subTest(case):
|
||||
plant()
|
||||
try:
|
||||
result = subprocess.run(
|
||||
[sys.executable, str(SCRIPT), "hold", "--repo", str(self.repo), "--", "touch", str(marker)],
|
||||
env=self.env, capture_output=True, text=True, timeout=30, check=False,
|
||||
)
|
||||
finally:
|
||||
git_lock.unlink()
|
||||
self.assertEqual(result.returncode, 2, result.stderr)
|
||||
self.assertIn(refusal, result.stderr)
|
||||
self.assertNotIn("Traceback", result.stderr)
|
||||
self.assertFalse(marker.exists())
|
||||
self.assertEqual(victim.read_text(encoding="utf-8"), "keep\n")
|
||||
|
||||
def test_record_and_status_never_follow_a_link_or_second_name_in_the_git_common_dir(self):
|
||||
gate = load_gate()
|
||||
victim = self.root / "victim.json"
|
||||
secret = '{"auths": {"registry": "secret"}}\n'
|
||||
for case in ("lock link", "lock hardlink", "state link", "state hardlink"):
|
||||
with self.subTest(case):
|
||||
common = self.root / case.replace(" ", "-")
|
||||
common.mkdir()
|
||||
victim.write_text(secret, encoding="utf-8")
|
||||
planted = common / (f"{STATE}.lock" if case.startswith("lock") else STATE)
|
||||
if case.endswith("hardlink"):
|
||||
os.link(victim, planted)
|
||||
else:
|
||||
planted.symlink_to(victim)
|
||||
with (
|
||||
contextlib.redirect_stderr(io.StringIO()) as err,
|
||||
self.assertRaises(SystemExit) as raised,
|
||||
gate.state_lock(common / STATE) as lock,
|
||||
):
|
||||
gate.record(lock, common / STATE, "0" * 40, "test-a", {"status": "green"})
|
||||
self.assertEqual(raised.exception.code, 2)
|
||||
self.assertIn(str(planted), err.getvalue())
|
||||
self.assertEqual(victim.read_text(encoding="utf-8"), secret)
|
||||
self.assertEqual(planted.is_symlink(), case.endswith(" link"))
|
||||
self.assertEqual([path.name for path in common.iterdir() if path.name.startswith(f".{STATE}.")], [])
|
||||
self.assertEqual((common / STATE).exists() and not (common / STATE).is_symlink(),
|
||||
case == "state hardlink")
|
||||
state = self.repo / ".git" / STATE
|
||||
for case, plant, refusal in (
|
||||
("link", lambda: state.symlink_to(victim), f"gate: cannot read {state}:"),
|
||||
("fifo", lambda: os.mkfifo(state, 0o600), f"gate: {state} must be a regular file of uid {os.getuid()}"),
|
||||
):
|
||||
with self.subTest(f"status through a {case}"):
|
||||
plant()
|
||||
try:
|
||||
result = subprocess.run(
|
||||
[sys.executable, str(SCRIPT), "status", "--repo", str(self.repo), "--json"],
|
||||
env=self.env, capture_output=True, text=True, timeout=30, check=False,
|
||||
)
|
||||
finally:
|
||||
state.unlink()
|
||||
self.assertEqual((result.returncode, result.stdout), (2, ""), result.stderr)
|
||||
self.assertIn(refusal, result.stderr)
|
||||
self.assertNotIn("secret", result.stdout + result.stderr)
|
||||
|
||||
def test_each_worktree_has_its_own_lock_and_a_subdirectory_shares_its_worktrees(self):
|
||||
other = self.root / "other"
|
||||
self._git("worktree", "add", "-q", "-b", "other", str(other))
|
||||
self.addCleanup(drop_lock_dir, lock_dir(other))
|
||||
(self.repo / "sub").mkdir()
|
||||
self.assertNotEqual(lock_dir(self.repo), lock_dir(other))
|
||||
with open(self.lock(RUN_LOCK), "a", encoding="utf-8") as lock:
|
||||
fcntl.flock(lock, fcntl.LOCK_EX)
|
||||
busy = self.gate("hold", "--repo", str(self.repo / "sub"), "--", "true")
|
||||
free = self.gate("hold", "--repo", str(other), "--", "true")
|
||||
self.assertEqual((busy.returncode, free.returncode), (75, 0), busy.stderr + free.stderr)
|
||||
self.assertTrue((lock_dir(other) / RUN_LOCK).is_file())
|
||||
|
||||
def test_a_lock_or_command_that_cannot_start_ends_the_hold_non_zero(self):
|
||||
marker = self.root / "ran"
|
||||
locks = self.lock(RUN_LOCK).parent
|
||||
locks.chmod(0o500)
|
||||
self.addCleanup(locks.chmod, 0o700)
|
||||
unwritable = self.gate("hold", "--repo", str(self.repo), "--", "touch", str(marker))
|
||||
locks.chmod(0o700)
|
||||
missing = self.gate("hold", "--repo", str(self.repo), "--", str(self.root / "no-such-command"))
|
||||
for result, message in (
|
||||
(unwritable, f"gate: cannot open the worktree lock {locks / RUN_LOCK}: Permission denied"),
|
||||
(missing, "gate: cannot start"),
|
||||
):
|
||||
with self.subTest(message):
|
||||
self.assertEqual(result.returncode, 2, result.stderr)
|
||||
self.assertIn(message, result.stderr)
|
||||
self.assertNotIn("Traceback", result.stderr)
|
||||
self.assertFalse(marker.exists())
|
||||
|
||||
def free(self, path: Path) -> bool:
|
||||
with open(path, "a", encoding="utf-8") as handle:
|
||||
try:
|
||||
fcntl.flock(handle, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
||||
except BlockingIOError:
|
||||
return False
|
||||
return True
|
||||
|
||||
def lock_free(self) -> bool:
|
||||
return self.free(self.lock(RUN_LOCK))
|
||||
|
||||
def test_a_busy_worktree_makes_run_and_hold_exit_75_naming_the_holder(self):
|
||||
with open(self.lock(RUN_LOCK), "a", encoding="utf-8") as lock:
|
||||
fcntl.flock(lock, fcntl.LOCK_EX)
|
||||
@@ -459,8 +750,10 @@ class TestWorktreeLock(GateCase):
|
||||
self.assertEqual(result.returncode, 128 + signal.SIGTERM)
|
||||
self.assertEqual(self.gate("hold", "--repo", str(self.repo)).returncode, 2)
|
||||
|
||||
def test_an_orphan_of_a_hold_keeps_the_lock(self):
|
||||
def test_an_orphan_of_a_hold_keeps_both_locks(self):
|
||||
pid_file = self.root / "deploy.pid"
|
||||
git_lock = self.repo / ".git" / RUN_LOCK
|
||||
git_lock.touch(mode=0o600)
|
||||
holder = subprocess.Popen(
|
||||
[sys.executable, str(SCRIPT), "hold", "--repo", str(self.repo), "--",
|
||||
"sh", "-c", f"echo $$ > {pid_file}.tmp; mv {pid_file}.tmp {pid_file}; exec sleep 60"],
|
||||
@@ -477,14 +770,15 @@ class TestWorktreeLock(GateCase):
|
||||
self.addCleanup(self.kill, deploy)
|
||||
holder.kill()
|
||||
holder.wait()
|
||||
self.assertFalse(self.free(git_lock), "the git dir lock died with the hold")
|
||||
result = self.run_targets("test-a", extra=())
|
||||
self.assertEqual(result.returncode, 75)
|
||||
self.assertIn(f"another gate.py run or hold holds {self.repo} (pid {holder.pid}: "
|
||||
f"gate.py hold --repo {self.repo} -- sh -c", result.stderr)
|
||||
self.kill(deploy)
|
||||
deadline = time.monotonic() + 30
|
||||
while not self.lock_free():
|
||||
self.assertLess(time.monotonic(), deadline, "the killed deploy kept the lock")
|
||||
while not (self.lock_free() and self.free(git_lock)):
|
||||
self.assertLess(time.monotonic(), deadline, "the killed deploy kept a lock")
|
||||
time.sleep(0.05)
|
||||
self.assertEqual(self.run_targets("test-a", extra=()).returncode, 0)
|
||||
|
||||
@@ -536,6 +830,54 @@ class TestWorktreeLock(GateCase):
|
||||
self.assertTrue(self.lock_free())
|
||||
|
||||
|
||||
class TestReadOnlyCommonDir(GateCase):
|
||||
files = MARKING
|
||||
|
||||
def test_a_read_only_common_dir_ends_run_with_2_before_any_target_runs(self):
|
||||
marker = self.root / "ran"
|
||||
self.env["MARKER"] = str(marker)
|
||||
git_dir = self.repo / ".git"
|
||||
self.addCleanup(git_dir.chmod, 0o755)
|
||||
for case in ("no state yet", "state of an earlier run"):
|
||||
with self.subTest(case):
|
||||
git_dir.chmod(0o755)
|
||||
if case == "state of an earlier run":
|
||||
self.assertEqual(self.run_targets("test", extra=()).returncode, 0)
|
||||
marker.unlink()
|
||||
before = {path.name: path.read_bytes() for path in git_dir.glob(f"{STATE}*")}
|
||||
git_dir.chmod(0o555)
|
||||
result = self.run_targets("test", extra=())
|
||||
self.assertEqual((result.returncode, marker.exists()), (2, False), result.stdout + result.stderr)
|
||||
self.assertIn("nothing ran", result.stderr)
|
||||
self.assertIn("as an operator row on the host", result.stderr)
|
||||
self.assertIn("`make <target>` alone runs but records nothing", result.stderr)
|
||||
self.assertNotIn("queue.py", result.stderr)
|
||||
self.assertNotIn("Traceback", result.stderr)
|
||||
self.assertEqual({path.name: path.read_bytes() for path in git_dir.glob(f"{STATE}*")}, before)
|
||||
self.assertEqual(list(git_dir.glob(f".{STATE}.*")), [])
|
||||
|
||||
def test_a_linked_or_invalid_state_file_ends_run_with_2_before_any_target_runs(self):
|
||||
marker, victim = self.root / "ran", self.root / "victim.json"
|
||||
self.env["MARKER"] = str(marker)
|
||||
victim.write_text('{"auths": {"registry": "secret"}}\n', encoding="utf-8")
|
||||
state = self.repo / ".git" / STATE
|
||||
for case, plant, refusal in (
|
||||
("link", lambda: state.symlink_to(victim), f"gate: cannot read {state}:"),
|
||||
("invalid JSON", lambda: state.write_text("{broken\n", encoding="utf-8"), f"gate: {state} is not valid JSON"),
|
||||
):
|
||||
with self.subTest(case):
|
||||
plant()
|
||||
try:
|
||||
result = self.run_targets("test", extra=())
|
||||
finally:
|
||||
state.unlink()
|
||||
self.assertEqual((result.returncode, marker.exists()), (2, False), result.stdout + result.stderr)
|
||||
self.assertIn(refusal, result.stderr)
|
||||
self.assertNotIn("secret", result.stdout + result.stderr)
|
||||
self.assertEqual(victim.read_text(encoding="utf-8"), '{"auths": {"registry": "secret"}}\n')
|
||||
self.assertEqual(list((self.repo / ".git").glob(f".{STATE}.*")), [])
|
||||
|
||||
|
||||
class TestUsage(GateCase):
|
||||
files = USAGE
|
||||
|
||||
|
||||
+1
-1
@@ -276,7 +276,7 @@ class TestInstalledHooks(Scratch):
|
||||
tree = self.out("write-tree")
|
||||
self.assertIn(f"gates: commit on main needs test (rule of main); no receipt for tree {tree} meets it.", refused.stderr)
|
||||
command = self.produce_command(refused.stderr)
|
||||
log = self.base / f"gate-{tree[:12]}-test.log"
|
||||
log = self.council / "gates" / f"gate-{tree[:12]}-test.log"
|
||||
self.assertEqual(command[:6], ["env", "-C", str(self.repo), "INFINITO_TEST_GPU=false", "python3", str(QUEUE)])
|
||||
self.assertEqual(
|
||||
command[6:], ["hold", "--kind", "test", "--task", "work/gate", "--receipt", "test", "--log", str(log), "--", "make", "test"]
|
||||
|
||||
+40
-13
@@ -5,6 +5,7 @@ from __future__ import annotations
|
||||
import argparse
|
||||
import contextlib
|
||||
import fcntl
|
||||
import hashlib
|
||||
import importlib.util
|
||||
import io
|
||||
import json
|
||||
@@ -37,6 +38,7 @@ PROBE = (
|
||||
SIGNATURE = r"connection refused .* port 5432"
|
||||
RUN_LOCK = "kvb-gate-run.lock"
|
||||
SYNC_LOCK = "kvb-gate-sync.lock"
|
||||
LOCK_BASE = Path("/tmp/agents-locks")
|
||||
IDENTITY = {
|
||||
"GIT_AUTHOR_NAME": "Test",
|
||||
"GIT_AUTHOR_EMAIL": "test@example.org",
|
||||
@@ -53,6 +55,16 @@ def load_hotfix() -> object:
|
||||
return hotfix
|
||||
|
||||
|
||||
def gate_locks(worktree: Path) -> Path:
|
||||
return LOCK_BASE / hashlib.sha256(os.fsencode(str(worktree))).hexdigest()[:16]
|
||||
|
||||
|
||||
def gate_lock(worktree: Path, name: str) -> Path:
|
||||
LOCK_BASE.mkdir(mode=0o700, exist_ok=True)
|
||||
gate_locks(worktree).mkdir(mode=0o700, exist_ok=True)
|
||||
return gate_locks(worktree) / name
|
||||
|
||||
|
||||
class TestOriginUrls(unittest.TestCase):
|
||||
def test_spellings_of_one_repository_normalize_alike(self):
|
||||
hotfix = load_hotfix()
|
||||
@@ -172,6 +184,8 @@ class TestHotfix(unittest.TestCase):
|
||||
self.seed = self.published(self.origin, self.base / "seed", "app.txt")
|
||||
|
||||
self.work = self.base / "work"
|
||||
for worktree in (self.work, self.base / "feature", self.base / "line", self.base / "lone"):
|
||||
self.addCleanup(shutil.rmtree, gate_locks(worktree), True)
|
||||
self.git(self.base, "clone", "-q", str(self.origin), str(self.work))
|
||||
self.feature = self.base / "feature"
|
||||
self.git(self.work, "worktree", "add", "-q", "-b", "feature", str(self.feature), "origin/main")
|
||||
@@ -709,12 +723,20 @@ class TestHotfix(unittest.TestCase):
|
||||
|
||||
def test_sync_refuses_while_a_gate_or_a_busy_process_holds_the_worktree(self):
|
||||
before = self.head(self.work)
|
||||
git_dir = Path(self.git(self.work, "rev-parse", "--absolute-git-dir").strip())
|
||||
with open(git_dir / RUN_LOCK, "a", encoding="utf-8") as lock:
|
||||
with open(gate_lock(self.work, RUN_LOCK), "a", encoding="utf-8") as lock:
|
||||
fcntl.flock(lock, fcntl.LOCK_EX)
|
||||
result = self.sync(self.work, "work/side")
|
||||
self.assertEqual(result.returncode, 2)
|
||||
self.assertIn(f"a gate.py run or hold holds {self.work}", result.stderr)
|
||||
git_lock = Path(self.git(self.work, "rev-parse", "--absolute-git-dir").strip()) / RUN_LOCK
|
||||
with open(git_lock, "a", encoding="utf-8") as lock:
|
||||
fcntl.flock(lock, fcntl.LOCK_EX)
|
||||
lock.write("4242\tgate.py hold -- make deploy\n")
|
||||
lock.flush()
|
||||
result = self.sync(self.work, "work/side")
|
||||
self.assertEqual(result.returncode, 2)
|
||||
self.assertIn(f"a gate.py run or hold or another sync holds the git dir lock of {self.work} "
|
||||
"(last recorded pid 4242: gate.py hold -- make deploy, possibly stale)", result.stderr)
|
||||
config = self.base / "config.json"
|
||||
config.write_text(json.dumps({"busy_processes": ["sleep"]}), encoding="utf-8")
|
||||
self.env["AGENTS_COUNCIL_CONFIG"] = str(config)
|
||||
@@ -771,7 +793,7 @@ class TestHotfix(unittest.TestCase):
|
||||
self.assertIn(f"a gate.py run or hold holds {self.work} (pid {holder.pid}: "
|
||||
f"gate.py hold --repo {self.work} -- sh", result.stderr)
|
||||
self.assertEqual(self.head(self.work), before)
|
||||
lock = Path(self.git(self.work, "rev-parse", "--absolute-git-dir").strip()) / RUN_LOCK
|
||||
lock = gate_lock(self.work, RUN_LOCK)
|
||||
os.kill(deploy, signal.SIGKILL)
|
||||
deadline = time.monotonic() + 30
|
||||
while not self.lock_free(lock):
|
||||
@@ -783,15 +805,17 @@ class TestHotfix(unittest.TestCase):
|
||||
line = self.base / "line"
|
||||
self.git(self.work, "worktree", "add", "-q", str(line), "line")
|
||||
self.commit(line, "side.txt", "other\n", "clashing line work")
|
||||
lock = Path(self.git(self.work, "rev-parse", "--absolute-git-dir").strip()) / RUN_LOCK
|
||||
probe, calls = self.base / "probe.py", self.base / "lock-states.log"
|
||||
lock = gate_lock(self.work, RUN_LOCK)
|
||||
git_lock = self.work / ".git" / RUN_LOCK
|
||||
probe, calls, git_calls = self.base / "probe.py", self.base / "lock-states.log", self.base / "git-dir-states.log"
|
||||
probe.write_text(PROBE, encoding="utf-8")
|
||||
wrapper = self.base / "bin" / "git"
|
||||
wrapper.parent.mkdir()
|
||||
real = shutil.which("git", path=self.env["PATH"])
|
||||
wrapper.write_text(
|
||||
"#!/bin/sh\n"
|
||||
f"case \" $* \" in *\" fetch \"*|*\" merge \"*) {sys.executable} {probe} {lock} \"$*\" >> {calls} ;; esac\n"
|
||||
f"case \" $* \" in *\" fetch \"*|*\" merge \"*) {sys.executable} {probe} {lock} \"$*\" >> {calls}\n"
|
||||
f" {sys.executable} {probe} {git_lock} \"$*\" >> {git_calls} ;; esac\n"
|
||||
f"exec {real} \"$@\"\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
@@ -799,13 +823,16 @@ class TestHotfix(unittest.TestCase):
|
||||
traced = {**self.env, "PATH": f"{wrapper.parent}:{self.env['PATH']}"}
|
||||
result = self.hotfix("sync", "--repo", str(self.work), "--task", "work/side", env=traced)
|
||||
self.assertEqual(result.returncode, 4, result.stderr)
|
||||
states = [entry.split(" ", 1) for entry in calls.read_text(encoding="utf-8").splitlines()]
|
||||
commands = [command.split(" ")[2] + (" --abort" if command.endswith("--abort") else "") for _, command in states]
|
||||
self.assertEqual(commands, ["fetch", "merge", "merge", "merge --abort"])
|
||||
self.assertEqual([state for state, _ in states], ["free", "held", "held", "held"], states)
|
||||
self.assertTrue(self.lock_free(lock))
|
||||
self.assertIn(f"\thotfix.py sync --repo {self.work} --task work/side\n",
|
||||
lock.with_name(SYNC_LOCK).read_text(encoding="utf-8"))
|
||||
for log, path in ((calls, lock), (git_calls, git_lock)):
|
||||
with self.subTest(lock=str(path)):
|
||||
states = [entry.split(" ", 1) for entry in log.read_text(encoding="utf-8").splitlines()]
|
||||
commands = [command.split(" ")[2] + (" --abort" if command.endswith("--abort") else "")
|
||||
for _, command in states]
|
||||
self.assertEqual(commands, ["fetch", "merge", "merge", "merge --abort"])
|
||||
self.assertEqual([state for state, _ in states], ["free", "held", "held", "held"], states)
|
||||
self.assertTrue(self.lock_free(path))
|
||||
for path in (lock.with_name(SYNC_LOCK), lock, git_lock):
|
||||
self.assertIn(f"\thotfix.py sync --repo {self.work} --task work/side\n", path.read_text(encoding="utf-8"))
|
||||
|
||||
def test_sync_and_update_keep_git_maintenance_off(self):
|
||||
calls = self.base / "git-calls.log"
|
||||
|
||||
@@ -14,8 +14,8 @@ from pathlib import Path
|
||||
REPO_ROOT = Path(__file__).resolve().parents[1]
|
||||
SCRIPT = REPO_ROOT / "scripts" / "orchestrator.sh"
|
||||
HOST_SCRIPTS = (
|
||||
"kvb-org-organize/scripts/council.py", "kvb-org-organize/scripts/nofollow.py", "kvb-org-rasci/scripts/rasci.py",
|
||||
"kvb-org-operator/scripts/operator.py", "kvb-org-radio/scripts/radio.py",
|
||||
"kvb-org-organize/scripts/council.py", "kvb-org-organize/scripts/nofollow.py", "kvb-org-organize/scripts/councilroot.py",
|
||||
"kvb-org-rasci/scripts/rasci.py", "kvb-org-operator/scripts/operator.py", "kvb-org-radio/scripts/radio.py",
|
||||
)
|
||||
OLD = "kvb-hub"
|
||||
NEW = "kvb-orchestrator"
|
||||
@@ -33,6 +33,8 @@ if tool == "docker" and args[:2] == ["image", "inspect"]:
|
||||
elif tool == "docker" and args[:2] == ["image", "rm"]:
|
||||
code = 0 if args[2] in state["images"] else 1
|
||||
state["images"] = [image for image in state["images"] if image != args[2]]
|
||||
elif tool == "docker" and args[0] == "compose" and "build" in args:
|
||||
code = 1 if state.get("fail_build") else 0
|
||||
elif tool == "docker" and args[0] == "compose" and "up" in args:
|
||||
state["images"] = sorted({*state["images"], "kvb-orchestrator:local"})
|
||||
elif tool == "claude" and args[:2] == ["mcp", "get"]:
|
||||
@@ -88,6 +90,7 @@ class TestOrchestratorMigration(unittest.TestCase):
|
||||
self.state = self.root / "stub.json"
|
||||
self.state.write_text(json.dumps({"calls": [], "images": [f"{OLD}:local"], "mcp": {OLD: "http://127.0.0.1:8737/mcp"}, "enabled": [f"{OLD}-runner.service"]}), encoding="utf-8")
|
||||
self.env = {**os.environ, "HOME": str(self.home), "PATH": f"{stubs}:{os.environ['PATH']}", "STUB_STATE": str(self.state), "AGENTS_COUNCIL_ROOT": str(self.root / "council"), "PKGMGR_REPO": str(self.root / "no-pkgmgr")}
|
||||
self.env.pop("XDG_STATE_HOME", None)
|
||||
share, local, config = self.home / ".local" / "share", self.home / ".local" / "state", self.home / ".config"
|
||||
self.write(share / OLD / "compose.yml", f"name: {OLD}\n")
|
||||
self.write(share / OLD / ".env", "KVB_HUB_PORT=8737\n")
|
||||
@@ -146,6 +149,29 @@ class TestOrchestratorMigration(unittest.TestCase):
|
||||
legacy_calls = [call for call in second["calls"][len(first["calls"]) :] if any(OLD in arg for arg in call)]
|
||||
self.assertEqual(legacy_calls, [["docker", "image", "inspect", f"{OLD}:local"], ["claude", "mcp", "get", OLD]])
|
||||
|
||||
def test_install_hands_the_hub_the_agents_root_and_the_host_only_root_apart(self):
|
||||
del self.env["AGENTS_COUNCIL_ROOT"]
|
||||
self.install()
|
||||
state = self.home / ".local" / "state"
|
||||
env = (self.home / ".local" / "share" / NEW / ".env").read_text(encoding="utf-8").splitlines()
|
||||
self.assertLessEqual({f"KVB_ORCHESTRATOR_COUNCIL={state / 'agents-shared'}", f"KVB_ORCHESTRATOR_KEPT={state / 'agents-council'}"}, set(env))
|
||||
self.assertTrue((state / "agents-council" / "resources").is_dir())
|
||||
|
||||
def test_the_hub_is_built_before_it_is_replaced_and_a_failed_build_leaves_it_down_and_fails(self):
|
||||
self.install()
|
||||
hub = str(self.home / ".local" / "share" / NEW / "compose.yml")
|
||||
first = json.loads(self.state.read_text(encoding="utf-8"))
|
||||
self.assertEqual([call[4:] for call in first["calls"] if call[:4] == ["docker", "compose", "--file", hub]], [["build"], ["up", "-d"]])
|
||||
self.state.write_text(json.dumps({**first, "calls": [], "fail_build": True}), encoding="utf-8")
|
||||
done = subprocess.run(["bash", str(SCRIPT), "install"], env=self.env, capture_output=True, text=True, timeout=120, check=False)
|
||||
self.assertEqual(done.returncode, 1, done.stderr)
|
||||
self.assertIn("building the Agent Orchestrator failed; it stays down until 'bash scripts/orchestrator.sh install' succeeds", done.stderr)
|
||||
after = json.loads(self.state.read_text(encoding="utf-8"))
|
||||
self.assertEqual([call[4:] for call in after["calls"] if call[:4] == ["docker", "compose", "--file", hub]], [["build"], ["stop"]])
|
||||
self.assertIn(f"{NEW}-resources.timer", after["enabled"])
|
||||
installer = (REPO_ROOT / "scripts" / "install.sh").read_text(encoding="utf-8")
|
||||
self.assertIn('if ! bash "${REPO_ROOT}/scripts/orchestrator.sh" install; then', installer)
|
||||
|
||||
def test_check_reports_the_sessions_timer_the_install_enabled_as_active(self):
|
||||
for tool in ("curl", "journalctl"):
|
||||
(self.root / "bin" / tool).symlink_to("stub")
|
||||
|
||||
@@ -693,7 +693,13 @@ class TestQueueCleanup(unittest.TestCase):
|
||||
inode = request.stat().st_ino
|
||||
self.cli("request")
|
||||
self.assertEqual((request.stat().st_ino, len(request.read_text(encoding="utf-8").splitlines())), (inode, 2))
|
||||
self.assertIn("PathModified=/tmp/agents-council/queue-cleanup.request\n", unit.read_text(encoding="utf-8"))
|
||||
self.assertIn("PathModified=%S/agents-shared/queue-cleanup.request\n", unit.read_text(encoding="utf-8"))
|
||||
|
||||
def test_agents_request_in_their_root_and_the_round_state_stays_host_only(self):
|
||||
with mock.patch.dict(os.environ, {"AGENTS_COUNCIL_ROOT": ""}):
|
||||
cleanup = _load(CLEANUP)
|
||||
state = self.home / ".local" / "state"
|
||||
self.assertEqual((cleanup.REQUEST, cleanup.STATE), (state / "agents-shared" / "queue-cleanup.request", self.kept / "queue-cleanup.json"))
|
||||
|
||||
def test_request_refuses_during_a_round_and_for_30_minutes_after_one(self):
|
||||
self.install_unit()
|
||||
|
||||
@@ -139,6 +139,8 @@ class TestBackfill(unittest.TestCase):
|
||||
shutil.copy(BACKFILL.parent / name, scripts / name)
|
||||
shutil.copy(LABELS, scripts.parent / "labels.json")
|
||||
helper = scripts.parents[1] / "kvb-org-organize" / "scripts" / "nofollow.py"
|
||||
helper.parent.mkdir(parents=True)
|
||||
shutil.copy(REPO_ROOT / "skills" / "kvb-org-organize" / "scripts" / "councilroot.py", helper.parent / "councilroot.py")
|
||||
council = self.council("missing")
|
||||
env = {"PATH": os.environ["PATH"], "AGENTS_COUNCIL_ROOT": str(council)}
|
||||
done = subprocess.run([sys.executable, "-B", str(scripts / "backfill.py"), "--repo", str(self.fixture.a)], env=env, capture_output=True, text=True, check=False, timeout=HANG)
|
||||
|
||||
@@ -26,6 +26,7 @@ REQUIREMENTS = REPO_ROOT / "docs" / "requirements"
|
||||
SCORE = SKILL / "scripts" / "score.py"
|
||||
EFFICIENCY = REPO_ROOT / "skills" / "kvb-check-efficiency" / "scripts"
|
||||
NOFOLLOW = REPO_ROOT / "skills" / "kvb-org-organize" / "scripts" / "nofollow.py"
|
||||
COUNCIL_ROOT = NOFOLLOW.with_name("councilroot.py")
|
||||
HANG = 20
|
||||
FORMER = re.compile(r"💸 Cost|⏱️ Time|(?i:💸 cost|⏱️ time|\bcost (?:and|or) time\b|\bcost, time\b|\b(?:cost|time) from the `kvb-check-(?:efficiency|speed)`)")
|
||||
REMOTES = (
|
||||
@@ -97,6 +98,7 @@ class ScoreCase(unittest.TestCase):
|
||||
organize = self.tmp / "skills" / "kvb-org-organize" / "scripts"
|
||||
organize.mkdir(parents=True)
|
||||
shutil.copy(NOFOLLOW, organize / "nofollow.py")
|
||||
shutil.copy(COUNCIL_ROOT, organize / "councilroot.py")
|
||||
(skill / "labels.json").write_text(labels, encoding="utf-8")
|
||||
return skill / "scripts" / "score.py"
|
||||
|
||||
@@ -171,12 +173,14 @@ class TestJournalPlace(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.journal = module(SKILL / "scripts" / "journal.py")
|
||||
|
||||
def test_an_empty_council_root_means_the_default(self):
|
||||
for value in ("", None):
|
||||
with self.subTest(value=value), mock.patch.dict(os.environ, {} if value is None else {"AGENTS_COUNCIL_ROOT": value}):
|
||||
if value is None:
|
||||
os.environ.pop("AGENTS_COUNCIL_ROOT", None)
|
||||
self.assertEqual(self.journal.journal_path(), Path("/tmp/agents-council/scores.jsonl"))
|
||||
def test_an_empty_council_root_means_the_durable_agents_default(self):
|
||||
with tempfile.TemporaryDirectory() as home:
|
||||
for value in ("", None):
|
||||
with self.subTest(value=value), mock.patch.dict(os.environ, {"HOME": home} if value is None else {"HOME": home, "AGENTS_COUNCIL_ROOT": value}):
|
||||
os.environ.pop("XDG_STATE_HOME", None)
|
||||
if value is None:
|
||||
os.environ.pop("AGENTS_COUNCIL_ROOT", None)
|
||||
self.assertEqual(self.journal.journal_path(), Path(home) / ".local" / "state" / "agents-shared" / "scores.jsonl")
|
||||
|
||||
def test_ssh_scp_and_https_remotes_share_one_key(self):
|
||||
self.assertEqual({self.journal.identity(url) for url in REMOTES}, {"github.com/org/repo"})
|
||||
@@ -291,6 +295,18 @@ class TestScoreJournal(ScoreCase):
|
||||
self.assertEqual(len(done.stdout.splitlines()), 7)
|
||||
self.assertFalse(self.council.exists())
|
||||
|
||||
def test_a_missing_council_root_helper_is_named_in_the_one_stderr_line(self):
|
||||
script = self.copy_skills((SKILL / "labels.json").read_text(encoding="utf-8"))
|
||||
helper = self.tmp.resolve() / "skills" / "kvb-org-organize" / "scripts" / "councilroot.py"
|
||||
helper.unlink()
|
||||
for values, code in (({}, 0), ({"critical": "1"}, 4)):
|
||||
with self.subTest(values=values):
|
||||
done = self.score(script, *flags(values), "--message-block", cwd=self.tmp)
|
||||
self.assertEqual(done.returncode, code)
|
||||
self.assertEqual(done.stderr, f"score: journal: helper {helper} unusable (FileNotFoundError): install or update kvb-org-organize\n")
|
||||
self.assertEqual(len(done.stdout.splitlines()), 7)
|
||||
self.assertFalse(self.council.exists())
|
||||
|
||||
def test_a_branch_or_commit_outside_utf8_keeps_the_score_exit(self):
|
||||
repo = self.tmp / "latin"
|
||||
env = git_env(self.tmp)
|
||||
|
||||
Reference in new issue
Block a user