diff --git a/README.md b/README.md index 68a2638..9743101 100644 --- a/README.md +++ b/README.md @@ -95,6 +95,7 @@ flowchart LR org --> org_radio[kvb-org-radio] org --> org_queue[kvb-org-queue] org --> org_queue_estimate[kvb-org-queue-estimate] + org --> org_queue_cleanup[kvb-org-queue-cleanup] org --> org_update[kvb-org-update] agent --> agent_optimize[kvb-agent-optimize] agent --> agent_drain[kvb-agent-drain] @@ -128,6 +129,7 @@ flowchart LR stage_req([staging or fix ready]) edit_hook([edit in a git worktree]) reminders([12 h agent hint, 17:00 GNOME popup]) + noon_midnight([00:00 and 12:00 timer]) full_hour([full hour]) code_task([code task]) spawn([subagent spawn]) @@ -212,6 +214,7 @@ flowchart LR radio[kvb-org-radio] queue[kvb-org-queue] queue_estimate[kvb-org-queue-estimate] + queue_cleanup[kvb-org-queue-cleanup] org_update[kvb-org-update] end subgraph cat_agent [agent] @@ -247,6 +250,7 @@ flowchart LR stage_req -.-> confidence edit_hook -. council gate .-> democracy reminders -.-> dream + noon_midnight -.-> queue_cleanup full_hour -.-> coach code_task -.-> clean code_task -.-> scope_guard @@ -318,6 +322,10 @@ flowchart LR unify --> radio status_matrix --> rasci cleanup --> artefacts + cleanup -. request .-> queue_cleanup + queue_cleanup --> queue + queue_cleanup -. pan-pan, security .-> radio + queue_cleanup --> rasci artefacts --> rasci democracy --> wsjf wsjf --> rasci @@ -454,7 +462,7 @@ Copy the skills into a specific project (`/.agents/skills` and `/.cl make project TARGET=/path/to/repo ``` -Both commands also patch the target's `.claude/settings.local.json` — the machine-local, untracked file, so a project's reviewed `.claude/settings.json` is never rewritten by an install: the caveman and ponytail plugins are enabled so their modes auto-activate on session start, and two reminder hooks are registered — `kvb-dream` points each agent at `/kvb-dream` at most once every twelve hours (stamped per session in `~/.claude/.dream-stamps/`), and `kvb-coach` runs its short form at the first prompt of every full hour, once across all agents (stamped in `~/.claude/.coach-hours/`); the global install also enables the `kvb-dream-notify.timer` systemd user timer, which recommends `/kvb-dream` in a GNOME notification daily at 17:00 local time and catches up once after a missed run. A PreToolUse gate from the `kvb-org-organize` skill blocks edits to files in a git worktree until the session holds a claim (`skills/kvb-org-organize/scripts/council.py`), and its resource watch (`skills/kvb-org-organize/scripts/watch.py`) measures disk, Docker and caches, cleans up low-risk data and blocks new heavy jobs before the disk runs out; it stays inactive until `~/.claude/agents-council/config.json` exists, and because its hooks run that config's commands outside the sandbox, the install adds `Edit(~/.claude/agents-council/**)` to `permissions.deny`, which also covers Write in every permission mode. The `kvb-org-radio` skill registers five hooks: its prompt hook and a PostToolBatch hook deliver the immediate radio (`call`, `broadcast`, `security`, `pan-pan`, `mayday`) mid-turn, a Stop hook delivers the post (`post`, `newsletter` and call receipts) when the turn ends, an `asyncRewake` Stop hook wakes an idle session for the radio, and a PreToolUse hook refuses every writing action while a mayday holds, except those of its sender and of the agents that join it with `radio.py assist`. The PreToolUse hook of the `kvb-org-queue` skill sends heavy commands through `queue.py hold` once the config sets `queue_enforce`, and holds a held command to the settings' Bash deny and ask rules, which see only `queue.py hold`. The global install also installs the MEDUSA security scanner with pipx for `kvb-check-security`, unless `medusa` is already on the PATH. Two hooks of the `kvb-org-perform` skill (`skills/kvb-org-perform/scripts/log.py`) log every Agent spawn with its task class, model and effort, and store the quality feedback a sandboxed session prints, in `~/.local/state/agents-council/subagent-model.jsonl`. Every hook runs `/usr/bin/python3 -I`, so neither the `python3` of a sandbox-writable virtualenv on the `PATH` nor `PYTHONPATH` reaches the hooks, which run outside the sandbox; older bare `python3` hook entries are rewritten. The reminder hooks run `/usr/bin/bash` and pin `PATH=/usr/bin:/bin`, and the council scripts keep only the `PATH` directories this uid cannot write (`skills/kvb-org-organize/scripts/hookpath.py`), so no binary an agent plants on the inherited `PATH` runs outside the sandbox. Existing settings are preserved. +Both commands also patch the target's `.claude/settings.local.json` — the machine-local, untracked file, so a project's reviewed `.claude/settings.json` is never rewritten by an install: the caveman and ponytail plugins are enabled so their modes auto-activate on session start, and two reminder hooks are registered — `kvb-dream` points each agent at `/kvb-dream` at most once every twelve hours (stamped per session in `~/.claude/.dream-stamps/`), and `kvb-coach` runs its short form at the first prompt of every full hour, once across all agents (stamped in `~/.claude/.coach-hours/`); the global install also enables the `kvb-dream-notify.timer` systemd user timer, which recommends `/kvb-dream` in a GNOME notification daily at 17:00 local time and catches up once after a missed run, and the `kvb-org-queue-cleanup.timer` and `kvb-org-queue-cleanup.path` user units, which start a `kvb-org-queue-cleanup` round at 00:00 and 12:00 local time without catch-up and whenever an agent runs `cleanup.py request`. A PreToolUse gate from the `kvb-org-organize` skill blocks edits to files in a git worktree until the session holds a claim (`skills/kvb-org-organize/scripts/council.py`), and its resource watch (`skills/kvb-org-organize/scripts/watch.py`) measures disk, Docker and caches, cleans up low-risk data and blocks new heavy jobs before the disk runs out; it stays inactive until `~/.claude/agents-council/config.json` exists, and because its hooks run that config's commands outside the sandbox, the install adds `Edit(~/.claude/agents-council/**)` to `permissions.deny`, which also covers Write in every permission mode. The `kvb-org-radio` skill registers five hooks: its prompt hook and a PostToolBatch hook deliver the immediate radio (`call`, `broadcast`, `security`, `pan-pan`, `mayday`) mid-turn, a Stop hook delivers the post (`post`, `newsletter` and call receipts) when the turn ends, an `asyncRewake` Stop hook wakes an idle session for the radio, and a PreToolUse hook refuses every writing action while a mayday holds, except those of its sender and of the agents that join it with `radio.py assist`. The PreToolUse hook of the `kvb-org-queue` skill sends heavy commands through `queue.py hold` once the config sets `queue_enforce`, and holds a held command to the settings' Bash deny and ask rules, which see only `queue.py hold`. The global install also installs the MEDUSA security scanner with pipx for `kvb-check-security`, unless `medusa` is already on the PATH. Two hooks of the `kvb-org-perform` skill (`skills/kvb-org-perform/scripts/log.py`) log every Agent spawn with its task class, model and effort, and store the quality feedback a sandboxed session prints, in `~/.local/state/agents-council/subagent-model.jsonl`. Every hook runs `/usr/bin/python3 -I`, so neither the `python3` of a sandbox-writable virtualenv on the `PATH` nor `PYTHONPATH` reaches the hooks, which run outside the sandbox; older bare `python3` hook entries are rewritten. The reminder hooks run `/usr/bin/bash` and pin `PATH=/usr/bin:/bin`, and the council scripts keep only the `PATH` directories this uid cannot write (`skills/kvb-org-organize/scripts/hookpath.py`), so no binary an agent plants on the inherited `PATH` runs outside the sandbox. Existing settings are preserved. The global install also runs the Agent Orchestrator, a local review page in which the operator approves or rejects the agents' staged commits (the `kvb-git-diff` skill): it copies `orchestrator/` to `~/.local/share/kvb-orchestrator`, starts it there with `docker compose up -d --build` (restart `unless-stopped`, bound to `127.0.0.1` only, state in `~/.local/state/kvb-orchestrator`) and registers its MCP server for Claude Code at user scope. Its port is `KVB_ORCHESTRATOR_PORT` in `orchestrator/.env`; the hub page is `http://localhost:/`, the MCP endpoint `/mcp`. Without docker the hub is skipped and the commit skills ask in chat. The hub reads new radio traffic aloud in maritime phraseology and takes push-to-talk input; its image builds both engines from pinned, sha256-verified sources and runs each as a separate process: Piper (`piper-tts`, GPL-3.0-or-later) with the `en_US-libritts-high` voice (904 speakers, trained on LibriTTS, CC BY 4.0, from `rhasspy/piper-voices`) and whisper.cpp (MIT) with the multilingual `ggml-base-q8_0` model (OpenAI Whisper weights, MIT, 82 MB). The operator's own voice is `KVB_ORCHESTRATOR_OPERATOR_VOICE` in `orchestrator/.env`. Its Tree reads only the file `pkgmgr branch tree` writes: the install clones `PKGMGR_REPO` (default the GitHub repository, never the sandbox-writable local checkout), builds the newest `v*` tag a fully trusted GPG key signed into `~/.local/share/kvb-orchestrator/pkgmgr`, and enables the `kvb-orchestrator-tree.timer` user timer, which writes `~/.local/state/pkgmgr/tree/branch-tree.json` a minute after its previous run while the hub page polls, on at most half a core; the hub mounts that directory read-only and starts no git. A signed tag without `branch tree` leaves the timer off. diff --git a/scripts/install.sh b/scripts/install.sh index 9e3db73..fad6605 100644 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -66,15 +66,17 @@ else fi if [[ "${TARGET}" == "${HOME}" ]]; then + install -d -m 0700 "${HOME}/.local/state/agents-council" if command -v systemctl &>/dev/null && systemctl --user show-environment &>/dev/null; then - log "skills: enabling the daily kvb-dream notification timer" + log "skills: enabling the daily kvb-dream notification timer and the kvb-org-queue-cleanup rounds at 00:00, 12:00 and on request" units="${HOME}/.config/systemd/user" mkdir -p "${units}" cp -f "${REPO_ROOT}/skills/kvb-dream/systemd/kvb-dream-notify.service" "${REPO_ROOT}/skills/kvb-dream/systemd/kvb-dream-notify.timer" "${units}/" + cp -f "${REPO_ROOT}/skills/kvb-org-queue-cleanup/systemd/kvb-org-queue-cleanup."{service,timer,path} "${units}/" systemctl --user daemon-reload - systemctl --user enable --now kvb-dream-notify.timer + systemctl --user enable --now kvb-dream-notify.timer kvb-org-queue-cleanup.timer kvb-org-queue-cleanup.path else - warn "skills: no systemd user session; skipped the kvb-dream notification timer." + warn "skills: no systemd user session; skipped the kvb-dream notification timer and the kvb-org-queue-cleanup rounds." fi if command -v docker &>/dev/null; then bash "${REPO_ROOT}/scripts/orchestrator.sh" install diff --git a/skills/kvb-cleanup/SKILL.md b/skills/kvb-cleanup/SKILL.md index 219fda5..9051c98 100644 --- a/skills/kvb-cleanup/SKILL.md +++ b/skills/kvb-cleanup/SKILL.md @@ -34,6 +34,8 @@ rules forbidding raw commands. build temp dirs. - **Register**: `council.py list --long`; `rasci.py artifacts` (`kvb-org-artefacts`, `--ended` for candidates). +- **Queue**: `${CLAUDE_SKILL_DIR}/../kvb-org-queue-cleanup/scripts/cleanup.py + request`; go on meanwhile. - **Live use**: `fuser`, `lsof`, `ps -o pid,etime,comm`, never full command lines or environments (credentials). Sandboxed, an empty result proves nothing. diff --git a/skills/kvb-org-queue-cleanup/SKILL.md b/skills/kvb-org-queue-cleanup/SKILL.md new file mode 100644 index 0000000..32a7c49 --- /dev/null +++ b/skills/kvb-org-queue-cleanup/SKILL.md @@ -0,0 +1,61 @@ +--- +name: kvb-org-queue-cleanup +description: > + Twice daily, delete dead queue jobs. +--- + +`${CLAUDE_SKILL_DIR}/scripts/cleanup.py` runs one cleanup round of the +`kvb-org-queue`, accountable to the top chair (council office `chair`); +without a live chair it deletes nothing. + +## Round + +1. A job (waiting or running) is dead only when no line of `/proc/locks`, + of any type, pid or waiting state, names its hold's lock file + `holds/.lock` and a flock on that file would not block, or when the + file is gone. Whoever owns a job, and whatever its queue entry says, a + living holder keeps it. Nothing is dead while `/proc/locks` is + unreadable, unparsable or misses the round's own flock on `holds/`, nor + is a job whose lock file is unreadable, no regular file, or changed + after its hold created it. Malformed entries stay. +2. Without a dead job the round sends nothing. Else a pan-pan names the dead + jobs; a 15 min window runs on both the wall and the monotonic clock. +3. A security names the jobs still dead and every job left alone with "held + by a living process, not deleted"; another 15 min window runs. +4. Each job still dead under the same id and lock tag, judged again under + the queue lock, leaves the queue. Nothing is signalled, no stop is + written. +5. The chair gets the counts by post; the `kvb-org-rasci` register records + the round as task `queue-cleanup`. + +The round keeps its state in +`~/.local/state/agents-council/queue-cleanup.json`, which only the host +writes. Radio silence on any poll, a stall above 3 minutes, a newer round, +or a chair gone stale before the delete ends the round at once, deletes +nothing and records it `stopped`; so does any other error, with one post to +the chair. + +`cleanup.py status` shows the open round's jobs: delete, held or unclear. +There is nothing to answer. + +## Triggers + +- `kvb-org-queue-cleanup.timer`, which `make install` enables, at 00:00 and + 12:00 host local time, without catch-up. +- `kvb-org-queue-cleanup.path` on `cleanup.py request`, which `kvb-cleanup` + runs; `request` exits 2 without the installed path unit, while a round + runs and for 30 minutes after one ended. +- Another interval: `systemctl --user edit kvb-org-queue-cleanup.timer`, a + `[Timer]` drop-in of an empty `OnCalendar=`, then `OnCalendar=`. +- `cleanup.py run --window ` refuses inside an agent session, below + 900 s and without write access to the host state; the service passes 900. + +## Limits + +- A lock file removed while its hold lives reads as gone, so its job is + deleted; `queue.py` itself prunes such an entry on its next poll. +- Started outside the initial PID namespace, the round misses POSIX locks of + processes it cannot see; the service runs in it. +- The alarms name at most 6 ids per list; `status` lists all. +- The 30-minute spacing binds `cleanup.py request` only; a line appended to + the request file otherwise starts a round once none runs. diff --git a/skills/kvb-org-queue-cleanup/scripts/cleanup.py b/skills/kvb-org-queue-cleanup/scripts/cleanup.py new file mode 100755 index 0000000..4035dba --- /dev/null +++ b/skills/kvb-org-queue-cleanup/scripts/cleanup.py @@ -0,0 +1,600 @@ +#!/usr/bin/env python3 +"""Delete the kvb-org-queue jobs whose hold lock no living process holds, announced by a pan-pan and a security. + +Usage: + cleanup.py run --window SECONDS + cleanup.py status + cleanup.py request + +Environment: + AGENTS_COUNCIL_ROOT: shared state directory; /tmp/agents-council unless + set. Agents append their requests to /queue-cleanup.request, + which kvb-org-queue-cleanup.path watches below /tmp/agents-council + only. The round lives in ~/.local/state/agents-council, which only + the host can write, in queue-cleanup.json under queue-cleanup.lock, + written by run alone. + CLAUDE_CODE_SESSION_ID: run refuses to start while it is set. + +Args: + run: one round, accountable to the live holder of the council office + chair; without one it deletes nothing. --window: seconds of each + window, at least WINDOW_FLOOR (the service passes 900). It refuses + unless it can write the host state. A job is a waiting or active + entry with a string id, status, kind and lock; every other entry + stays and counts as malformed. A job is dead when no line of + /proc/locks, whatever its type, pid or blocked state, names the + device and inode of /holds/.lock and a flock on it + would not block, or when that file is gone. Nothing is dead while + /proc/locks cannot be read, holds a line without exactly one device + and inode, or misses the shared flock the round holds on + /holds while it reads them; nor is a job whose lock file + cannot be opened, is no regular file with one link, or changed more + than CREATED_SECONDS away from the time in its tag. + Without a dead job it sends nothing. Else a pan-pan names the dead + jobs and counts the others; the window starts once it is sent and + ends once both the wall clock and the monotonic clock have run + --window seconds. A security then names the round's jobs still dead + and every job held by a living process, and a second window starts + once it is sent. Each job of the round still queued under the same + id and lock tag and still dead at the end, judged under the queue + lock, leaves the queue; nothing is signalled and no stop is written. + Radio silence on any poll, a gap above DEAD_SECONDS between the + round's own last beat and the end of a poll's sleep or the moment + the delete holds the queue lock, a newer round in the host state, or + radio silence or a vacant or stale chair before the delete ends the + round at once without deleting; it is recorded stopped. Any other + error after the claim, a queue store gone invalid included, records + phase stopped, deletes nothing more, posts one line to the chair and + exits FAILED. The result goes by post to the chair and into task + queue-cleanup of the kvb-org-rasci register. Every row it sends + comes from the station queue-cleanup. + status: the open round's phase and deadline, then one line per job it + judged: id, status, kind and delete for a dead one, id and held or + unclear for the others; without an open round the last result. + request: append a line to the request file in place, so + kvb-org-queue-cleanup.path starts a round outside the sandbox; + refused while a round runs and for SPACING_SECONDS after one ended. + +Exit codes: + 0 done, 1 run failed after its claim, 2 usage error, run inside an agent + session, with a window below WINDOW_FLOOR or without write access to the + host state, request without the installed path unit, during a round or + within SPACING_SECONDS after one, a council root that watch.py refuses; + run: 3 while another round runs. +""" + +from __future__ import annotations + +import argparse +import contextlib +import datetime +import fcntl +import importlib.util +import json +import os +import re +import sys +import time +from pathlib import Path +from types import ModuleType +from typing import NoReturn, Protocol + +SKILLS = Path(__file__).resolve().parents[2] +SELF = Path(__file__).resolve() +STATION = "queue-cleanup" +TASK = "queue-cleanup" +WORKER = "systemd kvb-org-queue-cleanup" +OBJECTIVE = "Delete the queue jobs no living process holds" +WINDOW_FLOOR = 900 +POLL_SECONDS = 5 +DEAD_SECONDS = 180 +SPACING_SECONDS = 1800 +CREATED_SECONDS = 10 +NAMES_SHOWN = 6 +NAME_WIDTH = 32 +OPEN = ("need", "warn") +ENDED = ("done", "stopped") +STATUSES = ("waiting", "active") +DEAD, HELD, UNCLEAR = "dead", "held", "unclear" +GONE = "gone" +RUNNING = 3 +FAILED = 1 +PROC = Path("/proc") +DEVICE = re.compile(r"(? float: ... + + def monotonic(self) -> float: ... + + def sleep(self, seconds: float) -> None: ... + + +class Superseded(Exception): + pass + + +def script(name: str, path: Path) -> ModuleType: + spec = importlib.util.spec_from_file_location(f"kvb_queue_cleanup_{name}", path) + loaded = importlib.util.module_from_spec(spec) + spec.loader.exec_module(loaded) + return loaded + + +queue = script("queue", SKILLS / "kvb-org-queue" / "scripts" / "queue.py") +radio = script("radio", SKILLS / "kvb-org-radio" / "scripts" / "radio.py") +rasci = script("rasci", SKILLS / "kvb-org-rasci" / "scripts" / "rasci.py") +watch, nofollow = queue.watch, queue.nofollow +REQUEST = watch.ROOT / "queue-cleanup.request" +STATE = watch.KEPT / "queue-cleanup.json" +STATE_LOCK = watch.KEPT / "queue-cleanup.lock" + + +def fail(message: str) -> NoReturn: + print(f"queue-cleanup: {message}", file=sys.stderr) + sys.exit(2) + + +def council() -> ModuleType: + return radio.council() + + +def host_state() -> dict: + try: + data = json.loads(nofollow.read_text(watch.KEPT, STATE)) + except (OSError, ValueError): + return {} + return data if isinstance(data, dict) else {} + + +def save(state: dict) -> None: + nofollow.replace(watch.KEPT, STATE, json.dumps(state, indent=1), 0o600) + + +def publish(state: dict) -> None: + with nofollow.locked(watch.KEPT, STATE_LOCK, 0o600, None): + if host_state().get("start") != state["start"]: + raise Superseded + save(state) + + +def alive(state: dict, now: float) -> bool: + return state.get("phase") in OPEN and now - state["beat"] <= DEAD_SECONDS + + +def busy(state: dict, now: float) -> bool: + return "beat" in state and state["phase"] not in ENDED and now - state["beat"] <= DEAD_SECONDS + + +def stalled(state: dict, now: float) -> str | None: + gap = now - state["beat"] + return f"the round stalled for {gap:.0f} s" if gap > DEAD_SECONDS else None + + +def top_chair() -> tuple[str | None, str]: + """Returns: + (session, name) of the live holder of the council office chair, else + (None, why no one is accountable). + """ + claim = council().read_claim(council().ROOT / "chair") + if claim is None: + return None, "the council office chair is vacant" + if council().is_stale(claim, council().live_sessions()): + return None, f"the chair {radio.clip(claim.get('name') or claim['session'])} is stale" + return claim["session"], radio.name_of(claim["session"]) + + +def hushed() -> str | None: + active = radio.silence() + return None if active is None else f"radio silence by {radio.clip(active.get('by'))} holds" + + +def tagged(tag: object) -> bool: + return isinstance(tag, str) and bool(queue.TAG.fullmatch(tag)) + + +def key(info: os.stat_result) -> tuple[int, int, int]: + return os.major(info.st_dev), os.minor(info.st_dev), info.st_ino + + +def lock_lines(root: Path) -> set[tuple[int, int, int]] | None: + """Returns: + The (device major, minor, inode) of every line of locks below root, + a /proc; None where it cannot be read or a line names not exactly one. + """ + try: + lines = (root / "locks").read_text(encoding="utf-8").splitlines() + except (OSError, ValueError): + return None + found = [DEVICE.findall(line) for line in lines] + if any(len(names) != 1 for names in found): + return None + return {(int(major, 16), int(minor, 16), int(inode)) for [(major, minor, inode)] in found} + + +def holders() -> set[tuple[int, int, int]] | None: + """Returns: + lock_lines of PROC, read under a shared flock of this process on + /holds; None unless they show that flock. + """ + try: + fd = nofollow.directory(watch.ROOT, nofollow.parts_below(watch.ROOT, queue.HOLDS), False) + except OSError: + return None + try: + fcntl.flock(fd, fcntl.LOCK_SH | fcntl.LOCK_NB) + found = lock_lines(PROC) + return found if found is not None and key(os.fstat(fd)) in found else None + except OSError: + return None + finally: + os.close(fd) + + +def lock_file(tag: object) -> tuple[int, int, int] | str | None: + """Returns: + HELD where a flock on the lock file of tag would block, GONE where it + is missing, else its key; None for no lock tag or a file that cannot + be opened, is no regular file with one link or changed more than + CREATED_SECONDS away from its tag's time. + """ + if not tagged(tag): + return None + try: + fd = nofollow.open_below(watch.ROOT, queue.HOLDS / f"{tag}.lock", os.O_RDONLY, 0) + except FileNotFoundError: + return GONE + except OSError: + return None + try: + info = os.fstat(fd) + fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB) + except BlockingIOError: + return HELD + except OSError: + return None + finally: + os.close(fd) + made = abs(info.st_ctime_ns - int(tag.split("-")[1])) <= CREATED_SECONDS * 1_000_000_000 + return key(info) if made else None + + +def judge(jobs: list[dict]) -> dict[str, list[dict]]: + """Returns: + The jobs by verdict: DEAD, HELD or UNCLEAR. + """ + files = {job["lock"]: lock_file(job["lock"]) for job in jobs} + found = holders() + groups: dict[str, list[dict]] = {DEAD: [], HELD: [], UNCLEAR: []} + for job in jobs: + name = files[job["lock"]] + if name == HELD or (found is not None and name in found): + groups[HELD].append(job) + else: + groups[UNCLEAR if found is None or name is None else DEAD].append(job) + return groups + + +def sound(entry: object) -> bool: + return (isinstance(entry, dict) and all(isinstance(entry.get(field), str) for field in ("id", "status", "kind", "lock")) + and entry["status"] in STATUSES) + + +def snapshot() -> tuple[dict[str, list[dict]], int]: + """Returns: + (the jobs of the queue by verdict, the number of malformed entries). + """ + with queue.store() as data: + jobs = [entry for entry in data["entries"] if sound(entry)] + groups = judge(jobs) + return groups, len(data["entries"]) - len(jobs) + + +def idents(entries: list[dict]) -> list[str]: + return [entry["id"] for entry in entries] + + +def standing(state: dict) -> list[dict]: + """Returns: + The round's jobs still queued under their lock tag and dead; state + takes the ids of the queue's held and unclear jobs. + """ + groups, _ = snapshot() + state.update(held=idents(groups[HELD]), unclear=idents(groups[UNCLEAR])) + dead = pairs(groups[DEAD]) + return [job for job in state["jobs"] if (job["id"], job["lock"]) in dead] + + +def host_writable() -> bool: + try: + watch.KEPT.mkdir(mode=0o700, parents=True, exist_ok=True) + with nofollow.locked(watch.KEPT, STATE_LOCK, 0o600, None): + return True + except OSError: + return False + + +def pairs(jobs: list[dict]) -> set[tuple[str, str]]: + return {(job["id"], job["lock"]) for job in jobs} + + +def cancel(state: dict, doomed: list[dict], clock: Clock) -> dict[str, list[str]] | str: + """Returns: + The stall found once the queue lock is held, and nothing deleted; + else the ids of the round's jobs still queued under the same lock tag + by outcome: deleted (dead and among doomed, out of the queue), else + left alone. + """ + named, chosen = pairs(state["jobs"]), pairs(doomed) + with queue.store() as data: + problem = stalled(state, clock.time()) + if problem is not None: + return problem + publish(state) + mine = [entry for entry in data["entries"] if sound(entry) and (entry["id"], entry["lock"]) in named] + groups = judge(mine) + gone = [entry for entry in groups[DEAD] if (entry["id"], entry["lock"]) in chosen] + data["entries"] = [entry for entry in data["entries"] if entry not in gone] + return outcome(groups, gone) + + +def outcome(groups: dict[str, list[dict]], gone: list[dict]) -> dict[str, list[str]]: + found = {"deleted": idents(gone), "held by a living process, not deleted": idents(groups[HELD]), + "unclear, not deleted": idents(groups[UNCLEAR]), + "not deleted, alive at the security": [entry["id"] for entry in groups[DEAD] if entry not in gone]} + return {word: ids for word, ids in found.items() if ids or word == "deleted"} + + +def send(cls: str, to: list[str], text: str) -> None: + radio.transmit(cls, to, text, STATION) + + +def capped(ids: list[str]) -> str: + hidden = len(ids) - NAMES_SHOWN + shown = ", ".join(council().short(radio.clip(i), NAME_WIDTH) for i in ids[:NAMES_SHOWN]) + return (shown or "none") + (f", +{hidden} more" if hidden > 0 else "") + + +def head(state: dict) -> str: + return f"queue cleanup {state['round']}, chair {council().short(radio.clip(state['chair']), NAME_WIDTH)}" + + +def unclear(state: dict) -> str: + return f"; unclear, not deleted: {capped(state['unclear'])}" if state["unclear"] else "" + + +def ask(state: dict) -> str: + odd = f", {state['malformed']} malformed entries" if state["malformed"] else "" + return ( + f"{head(state)}: no living process holds the lock of {len(state['jobs'])} queue jobs: {capped(idents(state['jobs']))}. " + f"A security at {radio.clock(state['deadline'])} names those still dead, deleted {state['window'] / 60:g} min after it; " + f"nothing is signalled. {len(state['held'])} held and {len(state['unclear'])} unclear jobs{odd} stay; " + f"python3 {SELF} status lists them." + ) + + +def warn(state: dict) -> str: + return ( + f"{head(state)}: still no living lock holder, deleted at {radio.clock(state['deadline'])}: {capped(state['delete'])}; " + f"held by a living process, not deleted: {capped(state['held'])}{unclear(state)}." + ) + + +ALARMS = {"need": ("pan-pan", ask), "warn": ("security", warn)} + + +def alarm(state: dict, phase: str, clock: Clock) -> str | None: + """Args: + state: the round; its deadline moves to the window's end. + phase: need (pan-pan) or warn (security). + clock: time, monotonic and sleep. + + Returns: + Why the round must stop (radio silence or a stall on a poll), else + None once the window has run on both clocks. + """ + cls, text = ALARMS[phase] + window = state["window"] + state.update(phase=phase, deadline=clock.time() + window, beat=clock.time()) + publish(state) + send(cls, [], text(state)) + wall, mono = clock.time(), clock.monotonic() + state.update(deadline=wall + window, beat=wall) + publish(state) + while True: + problem = hushed() + left = max(wall + window - clock.time(), mono + window - clock.monotonic()) + if problem is not None or left <= 0: + return problem + clock.sleep(min(POLL_SECONDS, left)) + problem = stalled(state, clock.time()) + if problem is not None: + return problem + state["beat"] = clock.time() + publish(state) + + +def stamp(ts: float) -> str: + return datetime.datetime.fromtimestamp(ts, datetime.UTC).isoformat(timespec="seconds") + + +def register(state: dict, status: str) -> None: + try: + with rasci.database() as tasks: + entry = tasks.setdefault(TASK, {"roles": {}}) + entry.update( + objective=OBJECTIVE, description=state["result"], worker=WORKER, status=status, + roles={state["chair"]: "A", WORKER: "R"}, start=stamp(state["start"]), end=stamp(state["end"]), + ) + except (OSError, SystemExit) as error: + print(f"queue-cleanup: the round is missing from the rasci register ({error})", file=sys.stderr) + + +def listed(ids: list[str]) -> str: + return ", ".join(ids) if ids else "none" + + +def finish(state: dict, named: dict[str, list[str]], status: str) -> int: + handled = {job_id for ids in named.values() for job_id in ids} + outcome = {**named, "left the queue": [job["id"] for job in state["jobs"] if job["id"] not in handled]} + result = f"queue cleanup {state['round']}: " + "; ".join(f"{word} {listed(ids)}" for word, ids in outcome.items()) + state.update(phase="done", result=result, end=time.time()) + publish(state) + counts = "; ".join(f"{word} {len(ids)}" for word, ids in outcome.items()) + send("post", [state["chair"]], f"queue cleanup {state['round']}: {counts}; python3 {SELF} status lists the ids") + register(state, status) + print(result) + return 0 + + +def halt(state: dict, problem: str) -> int: + return finish(state, {f"not deleted, {problem}": idents(state["jobs"])}, "stopped") + + +def abort(state: dict, error: BaseException) -> int: + why = f"queue cleanup {state['round']}: the round failed ({type(error).__name__}); nothing more is deleted" + state.update(phase="stopped", result=why, end=time.time()) + with contextlib.suppress(Exception): + publish(state) + with contextlib.suppress(Exception): + send("post", [state["chair"]], why) + with contextlib.suppress(Exception): + register(state, "stopped") + print(f"queue-cleanup: {why}: {error!r}", file=sys.stderr) + return FAILED + + +def claim(state: dict) -> bool: + with nofollow.locked(watch.KEPT, STATE_LOCK, 0o600, None): + if busy(host_state(), state["start"]): + return False + save(state) + return True + + +def run_round(window: float, clock: Clock) -> int: + """Args: + window: seconds each of the two windows lasts. + clock: time, monotonic and sleep; the time module outside tests. + + Returns: + The exit code. + """ + session, chair = top_chair() + if session is None: + print(f"queue-cleanup: {chair}; no one is accountable, nothing is deleted") + return 0 + groups, malformed = snapshot() + if not groups[DEAD]: + print(f"queue-cleanup: no queue job without a living lock holder ({len(groups[HELD])} held, " + f"{len(groups[UNCLEAR])} unclear, {malformed} malformed); nothing sent") + return 0 + now = clock.time() + state = { + "round": radio.clock(now), "start": now, "beat": now, "phase": "need", "deadline": now + window, + "window": window, "chair": chair, "malformed": malformed, "held": idents(groups[HELD]), + "unclear": idents(groups[UNCLEAR]), + "jobs": [{field: job[field] for field in ("id", "lock", "status", "kind")} for job in groups[DEAD]], + } + if not claim(state): + print("queue-cleanup: another round runs; status shows it", file=sys.stderr) + return RUNNING + try: + return proceed(state, clock) + except Superseded: + print("queue-cleanup: a newer round took over this one; nothing more is deleted", file=sys.stderr) + return 0 + except (Exception, SystemExit) as error: # noqa: BLE001 + return abort(state, error) + + +def proceed(state: dict, clock: Clock) -> int: + problem = hushed() + if problem is None: + problem = alarm(state, "need", clock) + if problem is not None: + return halt(state, problem) + state.update(phase="warn", beat=clock.time()) + publish(state) + doomed = standing(state) + if not doomed: + return close(state, doomed, clock) + state["delete"] = idents(doomed) + problem = alarm(state, "warn", clock) + if problem is not None: + return halt(state, problem) + state.update(phase="closing", beat=clock.time()) + publish(state) + return close(state, doomed, clock) + + +def close(state: dict, doomed: list[dict], clock: Clock) -> int: + session, chair = top_chair() + if session is not None: + state["chair"] = chair + problem = chair if session is None else hushed() or stalled(state, clock.time()) + if problem is not None and doomed: + return halt(state, problem) + found = cancel(state, doomed, clock) + return halt(state, found) if isinstance(found, str) else finish(state, found, "done") + + +def cmd_status(_: argparse.Namespace) -> int: + state = host_state() + if not alive(state, time.time()): + print(f"no open round; last: {state['result']}" if "result" in state else "no open round") + return 0 + print(f"round {state['round']} {state['phase']} until {radio.clock(state['deadline'])}") + for job in state["jobs"]: + print(f"{job['id']} {job['status']} {job['kind']} delete") + for verdict in (HELD, UNCLEAR): + for job_id in state[verdict]: + print(f"{job_id} {verdict}") + return 0 + + +def cmd_request(_: argparse.Namespace) -> int: + if not UNIT.is_file(): + fail(f"{UNIT} is missing, so no round would start; make install enables it") + state, now = host_state(), time.time() + if busy(state, now): + fail(f"round {state['round']} runs; status shows it") + if "end" in state and now - state["end"] < SPACING_SECONDS: + fail(f"the last round ended at {radio.clock(state['end'])}; request again from {radio.clock(state['end'] + SPACING_SECONDS)}") + nofollow.append(watch.ROOT, REQUEST, f"{now}\n", 0o600) + print(f"requested a queue-cleanup round through {REQUEST}") + return 0 + + +def cmd_run(args: argparse.Namespace) -> int: + if os.environ.get("CLAUDE_CODE_SESSION_ID"): + fail("run starts from kvb-org-queue-cleanup.service, never in an agent session; agents use request") + if args.window < WINDOW_FLOOR: + fail(f"--window must be at least {WINDOW_FLOOR} seconds") + if not host_writable(): + fail(f"cannot write {watch.KEPT}; run starts outside the sandbox only") + return run_round(args.window, time) + + +def cli() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + sub = parser.add_subparsers(dest="cmd", required=True) + run = sub.add_parser("run") + run.add_argument("--window", type=float, required=True) + sub.add_parser("status") + sub.add_parser("request") + return parser + + +def main(argv: list[str]) -> int: + args = cli().parse_args(argv) + refused = watch.guard_root(hook=False) + if refused is not None: + return refused + return {"run": cmd_run, "status": cmd_status, "request": cmd_request}[args.cmd](args) + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) diff --git a/skills/kvb-org-queue-cleanup/systemd/kvb-org-queue-cleanup.path b/skills/kvb-org-queue-cleanup/systemd/kvb-org-queue-cleanup.path new file mode 100644 index 0000000..a80c42a --- /dev/null +++ b/skills/kvb-org-queue-cleanup/systemd/kvb-org-queue-cleanup.path @@ -0,0 +1,8 @@ +[Unit] +Description=kvb-org-queue-cleanup round on cleanup.py request + +[Path] +PathModified=/tmp/agents-council/queue-cleanup.request + +[Install] +WantedBy=paths.target diff --git a/skills/kvb-org-queue-cleanup/systemd/kvb-org-queue-cleanup.service b/skills/kvb-org-queue-cleanup/systemd/kvb-org-queue-cleanup.service new file mode 100644 index 0000000..04a4688 --- /dev/null +++ b/skills/kvb-org-queue-cleanup/systemd/kvb-org-queue-cleanup.service @@ -0,0 +1,7 @@ +[Unit] +Description=One kvb-org-queue-cleanup round: pan-pan, security, delete the jobs no living process holds + +[Service] +Type=oneshot +ExecStart=/usr/bin/python3 -I %h/.claude/skills/kvb-org-queue-cleanup/scripts/cleanup.py run --window 900 +TimeoutStartSec=2400 diff --git a/skills/kvb-org-queue-cleanup/systemd/kvb-org-queue-cleanup.timer b/skills/kvb-org-queue-cleanup/systemd/kvb-org-queue-cleanup.timer new file mode 100644 index 0000000..e657663 --- /dev/null +++ b/skills/kvb-org-queue-cleanup/systemd/kvb-org-queue-cleanup.timer @@ -0,0 +1,9 @@ +[Unit] +Description=kvb-org-queue-cleanup round at 00:00 and 12:00 local time + +[Timer] +OnCalendar=*-*-* 00,12:00:00 +Persistent=false + +[Install] +WantedBy=timers.target diff --git a/skills/kvb-org-queue/SKILL.md b/skills/kvb-org-queue/SKILL.md index 061f919..4798809 100644 --- a/skills/kvb-org-queue/SKILL.md +++ b/skills/kvb-org-queue/SKILL.md @@ -1,8 +1,8 @@ --- name: kvb-org-queue description: > - Run every heavy job (gate, deploy, build, swarm) through a deterministic - queue instead of asking the chair: hotfix first, then register rank. + Queue every heavy job (gate, deploy, build, swarm), never ask the chair: + hotfix first, then rank. --- `${CLAUDE_SKILL_DIR}/scripts/queue.py` replaces asking the chair for heavy @@ -74,7 +74,8 @@ learned values, `cancel` removes your own waiting job. only exceptions such as a stuck queue or a hung job (`capacity.md` of `kvb-org-organize`). - A running job is never cut short for the queue; only the operator orders - one down, and `kvb-git-hotfix` may void a gate. + one down, and `kvb-git-hotfix` may void a gate. A `kvb-org-queue-cleanup` + round deletes only entries whose lock no living process holds. - With `queue_enforce: true`, the `block` hook (PreToolUse on Bash) refuses `heavy_commands`, image builds, `make install|verify-*`, tests beyond one file and `gate.py run|hold` outside `queue.py hold`. diff --git a/skills/kvb-org-radio/scripts/radio.py b/skills/kvb-org-radio/scripts/radio.py index d790533..eb776de 100755 --- a/skills/kvb-org-radio/scripts/radio.py +++ b/skills/kvb-org-radio/scripts/radio.py @@ -152,7 +152,7 @@ def items() -> list[dict]: for line in lines: try: row = json.loads(line) - except ValueError: + except (ValueError, RecursionError): continue if isinstance(row, dict) and isinstance(row.get("ts"), int | float): rows.append(row) @@ -225,7 +225,7 @@ def for_me(row: dict, name: str, sid: str) -> bool: def read_json(path: Path) -> dict: try: data = json.loads(organize("nofollow").read_text(ROOT, path)) - except (OSError, ValueError): + except (OSError, ValueError, RecursionError): return {} return data if isinstance(data, dict) else {} diff --git a/tests/test_queue_cleanup.py b/tests/test_queue_cleanup.py new file mode 100644 index 0000000..f7983c8 --- /dev/null +++ b/tests/test_queue_cleanup.py @@ -0,0 +1,724 @@ +"""Exercise kvb-org-queue-cleanup: jobs whose hold lock no living process holds leave the queue after a pan-pan and a security, every other job stays.""" + +from __future__ import annotations + +import contextlib +import importlib.util +import io +import json +import os +import re +import subprocess +import sys +import tempfile +import time +import unittest +from collections.abc import Callable +from pathlib import Path +from unittest import mock + +REPO_ROOT = Path(__file__).resolve().parents[1] +CLEANUP = REPO_ROOT / "skills" / "kvb-org-queue-cleanup" / "scripts" / "cleanup.py" +SYSTEMD = CLEANUP.parents[1] / "systemd" +SESSIONS = {"s0": "Main", "s1": "Skills", "s2": "IAM"} +WINDOW = 900 +WORKER = "systemd kvb-org-queue-cleanup" +HOLD = ( + "import ctypes, fcntl, struct, subprocess, sys\n" + "ctypes.CDLL(None).prctl(1, 9)\n" + "handle = open(sys.argv[1], 'a')\n" + "if sys.argv[2] == 'posix':\n" + " fcntl.lockf(handle, fcntl.LOCK_EX)\n" + "elif sys.argv[2] == 'ofd':\n" + " fcntl.fcntl(handle, fcntl.F_OFD_SETLK, struct.pack('hhqqi4x', fcntl.F_WRLCK, 0, 0, 0, 0))\n" + "else:\n" + " fcntl.flock(handle, fcntl.LOCK_EX)\n" + "if sys.argv[2] == 'child':\n" + " subprocess.Popen([sys.executable, '-c', 'import sys; sys.stdin.read()'], pass_fds=[handle.fileno()])\n" + "print('ready', flush=True)\n" + "sys.stdin.read()\n" +) +FORGED = {"session": "s9", "name": "Main", "pid": 1, "pid_start": "1", "child": 2, "child_start": "2", "beat": 0, + "host": True, "granted": 0, "workdir": "/elsewhere"} + + +def _load(path: Path): + spec = importlib.util.spec_from_file_location(f"{path.stem}_under_test", path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def ids_of(entries: list[dict]) -> list[str]: + return [entry["id"] for entry in entries] + + +class Killed(BaseException): + pass + + +class FakeClock: + def __init__(self, wall: float, hook: Callable[[float], None]): + self.wall, self.mono, self.hook = wall, 0.0, hook + + def time(self) -> float: + return self.wall + + def monotonic(self) -> float: + return self.mono + + def sleep(self, seconds: float) -> None: + self.hook(seconds) + self.wall += seconds + self.mono += seconds + + +class TestQueueCleanup(unittest.TestCase): + def setUp(self): + tmp = tempfile.TemporaryDirectory() + self.addCleanup(tmp.cleanup) + base = Path(tmp.name).resolve() + self.root = base / "council" + self.holds = self.root / "holds" + self.holds.mkdir(parents=True, mode=0o700) + self.home = base / "home" + sessions = self.home / ".claude" / "sessions" + sessions.mkdir(parents=True) + for sid, name in SESSIONS.items(): + (sessions / f"{sid}.json").write_text(json.dumps({"sessionId": sid, "name": name}), encoding="utf-8") + self.kept = self.home / ".local" / "state" / "agents-council" + self.kept.mkdir(parents=True, mode=0o700) + env = mock.patch.dict(os.environ, { + "AGENTS_COUNCIL_ROOT": str(self.root), "HOME": str(self.home), "AGENTS_COUNCIL_CONFIG": str(base / "config.json"), + }) + env.start() + self.addCleanup(env.stop) + os.environ.pop("CLAUDE_CODE_SESSION_ID", None) + self.cleanup = _load(CLEANUP) + self.last = WINDOW // self.cleanup.POLL_SECONDS - 1 + self.clock = FakeClock(float(int(time.time())), self.tick) + for module in (self.cleanup, self.cleanup.radio): + patch = mock.patch.object(module, "time", self.clock) + patch.start() + self.addCleanup(patch.stop) + self.actions: dict[tuple[str, int], Callable[[], object]] = {} + self.jumps: dict[tuple[str, int], tuple[float, float]] = {} + self.slept: list[tuple[str, float]] = [] + self.seq = 0 + self.chair("s0") + + def tick(self, seconds: float) -> None: + phase = self.state()["phase"] + step = (phase, sum(p == phase for p, _ in self.slept)) + self.slept.append((phase, seconds)) + if step in self.actions: + self.actions[step]() + if step in self.jumps: + wall, mono = self.jumps[step] + self.clock.wall += wall + self.clock.mono += mono + + def state(self) -> dict: + return json.loads((self.kept / "queue-cleanup.json").read_text(encoding="utf-8")) + + def write_state(self, state: dict) -> None: + (self.kept / "queue-cleanup.json").write_text(json.dumps(state), encoding="utf-8") + + def chair(self, session: str) -> None: + office = self.root / "chair" + office.mkdir(exist_ok=True) + claim = {"office": "chair", "session": session, "runtime": "claude-code", "name": SESSIONS.get(session), + "beat": time.time(), "work": "chair"} + (office / "claim.json").write_text(json.dumps(claim), encoding="utf-8") + + def entry(self, status: str, born: float = 0) -> dict: + self.seq += 1 + return { + "id": f"test-{self.seq}", "seq": self.seq, "kind": "test", "task": "work/a", "need_gb": 0, "hotfix": None, + "workdir": f"/w{self.seq}", "session": "s1", "name": "Skills", "lock": f"{self.seq}-{time.time_ns() + int(born * 1e9)}", + "host": False, "pid": 1, "pid_start": None, "child": None, "child_start": None, "status": status, + "enqueued": time.time(), "granted": None, "cmd": "true", + } + + def lock_file(self, entry: dict) -> Path: + return self.holds / f"{entry['lock']}.lock" + + def holder(self, path: Path, mode: str = "flock") -> subprocess.Popen: + proc = subprocess.Popen([sys.executable, "-c", HOLD, str(path), mode], stdin=subprocess.PIPE, stdout=subprocess.PIPE, text=True) + self.addCleanup(proc.wait) + self.addCleanup(proc.stdin.close) + self.addCleanup(proc.stdout.close) + self.assertEqual(proc.stdout.readline(), "ready\n") + return proc + + def kill(self, proc: subprocess.Popen) -> None: + proc.kill() + proc.wait() + + def held(self, status: str, mode: str = "flock") -> dict: + entry = self.entry(status) + self.holder(self.lock_file(entry), mode) + return entry + + def dead(self, status: str, born: float = 0) -> dict: + entry = self.entry(status, born) + self.kill(self.holder(self.lock_file(entry))) + return entry + + def verdict(self, job: dict) -> str: + [found] = [verdict for verdict, jobs in self.cleanup.judge([job]).items() if jobs] + return found + + def queue(self, *entries: object) -> None: + (self.root / "queue.json").write_text(json.dumps({"seq": self.seq, "entries": list(entries)}), encoding="utf-8") + + def entries(self) -> list[object]: + return json.loads((self.root / "queue.json").read_text(encoding="utf-8"))["entries"] + + def ids(self) -> list[str]: + return ids_of(self.entries()) + + def rows(self) -> list[dict]: + news = self.root / "news.jsonl" + return [json.loads(line) for line in news.read_text(encoding="utf-8").splitlines()] if news.exists() else [] + + def classes(self) -> list[str]: + return [row["class"] for row in self.rows()] + + def register(self) -> dict: + return json.loads((self.root / "rasci.json").read_text(encoding="utf-8"))["queue-cleanup"] + + def cli(self, *args: str) -> tuple[int, str, str]: + out, err = io.StringIO(), io.StringIO() + with contextlib.redirect_stdout(out), contextlib.redirect_stderr(err): + try: + code = self.cleanup.main(list(args)) + except SystemExit as stop: + code = stop.code + return code, out.getvalue(), err.getvalue() + + def silence(self) -> None: + hush = {"since": self.clock.wall, "until": self.clock.wall + 7200, "by": "IAM", "text": "disk full"} + (self.root / "radio-silence.json").write_text(json.dumps(hush), encoding="utf-8") + + def round(self) -> tuple[int, str]: + out = io.StringIO() + with contextlib.redirect_stdout(out), contextlib.redirect_stderr(io.StringIO()): + code = self.cleanup.run_round(WINDOW, self.clock) + return code, out.getvalue() + + def nothing(self, held: int, unclear: int, malformed: int) -> tuple[int, str]: + return 0, f"queue-cleanup: no queue job without a living lock holder ({held} held, {unclear} unclear, {malformed} malformed); nothing sent\n" + + def waited(self, phase: str) -> float: + return sum(seconds for p, seconds in self.slept if p == phase) + + def ends(self, out: str, tail: str) -> None: + self.assertTrue(out.endswith(tail + "\n"), out) + + def at_the_close(self, action: Callable[[], None]) -> tuple[int, str, str]: + original = self.cleanup.top_chair + + def chair_then_act() -> tuple[str | None, str]: + if (self.kept / "queue-cleanup.json").exists() and self.state()["phase"] == "closing": + action() + return original() + + out, err = io.StringIO(), io.StringIO() + with mock.patch.object(self.cleanup, "top_chair", chair_then_act), contextlib.redirect_stdout(out), contextlib.redirect_stderr(err): + code = self.cleanup.run_round(WINDOW, self.clock) + return code, out.getvalue(), err.getvalue() + + def install_unit(self) -> Path: + unit = self.home / ".config" / "systemd" / "user" / "kvb-org-queue-cleanup.path" + unit.parent.mkdir(parents=True) + unit.write_text((SYSTEMD / "kvb-org-queue-cleanup.path").read_text(encoding="utf-8"), encoding="utf-8") + return unit + + def test_a_vacant_or_stale_chair_deletes_nothing_and_sends_nothing(self): + entries = [self.dead("waiting"), self.dead("active")] + self.queue(*entries) + for name, vacate in (("stale", lambda: self.chair("gone")), ("vacant", (self.root / "chair" / "claim.json").unlink)): + with self.subTest(name): + vacate() + code, out = self.round() + self.assertEqual(code, 0) + self.ends(out, "; no one is accountable, nothing is deleted") + self.assertEqual((self.entries(), self.rows()), (entries, [])) + + def test_jobs_whose_hold_died_or_whose_lock_file_is_gone_leave_the_queue_after_a_pan_pan_and_a_security(self): + waiting, active, gone, alive = self.dead("waiting"), self.dead("active"), self.entry("waiting"), self.held("active") + self.queue(waiting, active, gone, alive) + code, out = self.round() + self.assertEqual((code, self.entries()), (0, [alive])) + self.assertFalse((self.kept / "queue-stops.json").exists()) + self.assertEqual( + [(row["class"], row["from"], row["to"]) for row in self.rows()], + [("pan-pan", "queue-cleanup", []), ("security", "queue-cleanup", []), ("post", "queue-cleanup", ["Main"])], + ) + pan_pan, security, post = (row["text"] for row in self.rows()) + self.assertRegex( + pan_pan, + r"^queue cleanup \d\d:\d\d, chair Main: no living process holds the lock of 3 queue jobs: test-1, test-2, test-3\. " + r"A security at \d\d:\d\d names those still dead, deleted 15 min after it; nothing is signalled\. " + r"1 held and 0 unclear jobs stay; ", + ) + self.assertRegex( + security, + r"^queue cleanup \d\d:\d\d, chair Main: still no living lock holder, deleted at \d\d:\d\d: test-1, test-2, test-3; " + r"held by a living process, not deleted: test-4\.$", + ) + self.assertTrue(post.endswith(f": deleted 3; left the queue 0; python3 {CLEANUP} status lists the ids"), post) + self.ends(out, ": deleted test-1, test-2, test-3; left the queue none") + + def test_a_hold_that_died_while_its_child_keeps_the_inherited_lock_is_not_deleted(self): + inherited = self.entry("active") + self.kill(self.holder(self.lock_file(inherited), "child")) + dead = self.dead("waiting") + self.queue(inherited, dead) + self.assertEqual(self.cleanup.lock_file(inherited["lock"]), self.cleanup.HELD) + code, out = self.round() + self.assertEqual((code, self.entries()), (0, [inherited])) + self.assertTrue(self.rows()[1]["text"].endswith(": test-2; held by a living process, not deleted: test-1."), self.rows()[1]["text"]) + self.ends(out, ": deleted test-2; left the queue none") + + def test_waiting_and_active_jobs_with_a_living_holder_of_any_lock_type_stay_whatever_their_entry_says(self): + living = [{**self.held(status, mode), **FORGED} for status, mode in + (("waiting", "flock"), ("active", "flock"), ("waiting", "posix"), ("active", "ofd"))] + self.queue(*living) + self.assertEqual(self.round(), self.nothing(4, 0, 0)) + self.assertEqual((self.entries(), self.rows()), (living, [])) + dead = self.dead("waiting") + self.queue(*living, dead) + code, out = self.round() + self.assertEqual((code, self.entries()), (0, living)) + self.assertTrue(self.rows()[1]["text"].endswith("; held by a living process, not deleted: test-1, test-2, test-3, test-4.")) + self.ends(out, ": deleted test-5; left the queue none") + + def test_a_lock_file_swapped_linked_redirected_unreadable_or_odd_is_never_dead(self): + swapped = self.entry("waiting", born=-60) + self.holder(self.lock_file(swapped)) + fresh = self.holds / "fresh" + fresh.write_text("", encoding="utf-8") + fresh.rename(self.lock_file(swapped)) + left, right = self.held("waiting"), self.held("active") + self.lock_file(left).rename(self.holds / "spare") + self.lock_file(right).rename(self.lock_file(left)) + (self.holds / "spare").rename(self.lock_file(right)) + redirected = self.dead("waiting") + self.lock_file(redirected).rename(self.root / "target.lock") + self.lock_file(redirected).symlink_to(self.root / "target.lock") + linked = self.dead("waiting") + os.link(self.lock_file(linked), self.root / "second.lock") + unreadable = self.dead("active") + self.lock_file(unreadable).chmod(0) + directory, fifo = self.entry("waiting"), self.entry("active") + self.lock_file(directory).mkdir() + os.mkfifo(self.lock_file(fifo)) + future = self.dead("waiting", born=60) + untagged = {**self.dead("waiting"), "lock": "1-2-3"} + cases = { + "swapped after its hold started": (swapped, "unclear"), "exchanged with another living one": (left, "held"), + "the other one exchanged": (right, "held"), "a symlink": (redirected, "unclear"), "a second link": (linked, "unclear"), + "unreadable": (unreadable, "unclear"), "a directory": (directory, "unclear"), "a fifo": (fifo, "unclear"), + "a tag from the future": (future, "unclear"), "no lock tag": (untagged, "unclear"), + } + for name, (job, expected) in cases.items(): + with self.subTest(name): + self.assertEqual(self.verdict(job), expected) + jobs = [job for job, _ in cases.values()] + self.queue(*jobs) + self.assertEqual(self.round(), self.nothing(2, 8, 0)) + self.assertEqual((self.entries(), self.rows()), (jobs, [])) + + def test_proc_locks_unreadable_unparsable_or_without_the_rounds_own_flock_makes_nothing_dead(self): + job = self.dead("waiting") + self.queue(job) + info = os.stat(self.holds) + own = f"1: FLOCK ADVISORY READ 1 {os.major(info.st_dev):02x}:{os.minor(info.st_dev):02x}:{info.st_ino} 0 EOF\n" + fakes = { + "missing": None, "a directory": "", "empty": "", "without the round's flock": "1: FLOCK ADVISORY WRITE 9 00:01:5 0 EOF\n", + "with a line without an inode": own + "2: FLOCK ADVISORY WRITE 9 :0 0 EOF\n", "with garbage": own + "garbage\n", + } + for index, (name, text) in enumerate(fakes.items()): + with self.subTest(name): + proc = self.home / f"proc-{index}" + proc.mkdir() + if name == "a directory": + (proc / "locks").mkdir() + elif text is not None: + (proc / "locks").write_text(text, encoding="utf-8") + with mock.patch.object(self.cleanup, "PROC", proc): + self.assertEqual(self.round(), self.nothing(0, 1, 0)) + self.assertEqual((self.entries(), self.rows()), ([job], [])) + control = self.home / "control" + control.mkdir() + (control / "locks").write_text(own, encoding="utf-8") + with mock.patch.object(self.cleanup, "PROC", control): + self.assertEqual(self.verdict(job), "dead") + + def test_lock_lines_name_every_line_of_any_type_pid_or_waiting_state_and_refuse_any_other(self): + proc = self.home / "proc" + proc.mkdir() + first = "1: FLOCK ADVISORY WRITE 100 00:2d:555 0 EOF\n" + (proc / "locks").write_text( + first + + "1: -> FLOCK ADVISORY WRITE 101 00:2d:556 0 EOF\n" + "2: POSIX ADVISORY WRITE 102 00:2d:557 0 0\n" + "3: OFDLCK ADVISORY WRITE -1 00:2d:558 10 10\n" + "4: FLOCK ADVISORY READ 0 fd:01:777 0 EOF\n" + "5: LEASE ACTIVE READ 105 103:02:888 0 EOF\n" + "6: POSIX ADVISORY WRITE 106 00:2d:12345678901 0 EOF\n", + encoding="utf-8", + ) + self.assertEqual( + self.cleanup.lock_lines(proc), + {(0, 0x2D, 555), (0, 0x2D, 556), (0, 0x2D, 557), (0, 0x2D, 558), (0xFD, 1, 777), (0x103, 2, 888), (0, 0x2D, 12345678901)}, + ) + odd = {"garbage": "garbage\n", "no inode": "7: FLOCK ADVISORY WRITE 106 :0 0 EOF\n", "blank": "\n", + "two devices": "8: FLOCK ADVISORY WRITE 1 00:2d:1 00:2d:2 EOF\n"} + for name, line in odd.items(): + with self.subTest(name): + (proc / "locks").write_text(first + line, encoding="utf-8") + self.assertIsNone(self.cleanup.lock_lines(proc)) + (proc / "locks").write_text("", encoding="utf-8") + self.assertEqual(self.cleanup.lock_lines(proc), set()) + (proc / "locks").unlink() + self.assertIsNone(self.cleanup.lock_lines(proc)) + + def test_a_job_that_dies_after_the_snapshot_waits_for_the_next_round_and_one_living_again_stays(self): + revived, later = self.dead("waiting"), self.entry("active") + hold = self.holder(self.lock_file(later)) + self.queue(revived, later) + self.actions[("need", 0)] = lambda: (self.holder(self.lock_file(revived)), self.kill(hold)) + code, out = self.round() + self.assertEqual((code, self.entries(), self.classes()), (0, [revived, later], ["pan-pan", "post"])) + self.ends(out, ": deleted none; held by a living process, not deleted test-1; left the queue none") + + def test_a_job_alive_at_the_security_and_dead_again_at_the_end_is_not_deleted(self): + flapping, other = self.dead("waiting"), self.dead("waiting") + self.queue(flapping, other) + holds = {} + self.actions[("need", 0)] = lambda: holds.update(proc=self.holder(self.lock_file(flapping))) + self.actions[("warn", 0)] = lambda: self.kill(holds["proc"]) + code, out = self.round() + self.assertEqual((code, self.entries()), (0, [flapping])) + self.assertRegex(self.rows()[1]["text"], r": still no living lock holder, deleted at \d\d:\d\d: test-2; held by a living process, not deleted: test-1\.$") + self.ends(out, ": deleted test-2; not deleted, alive at the security test-1; left the queue none") + + def test_a_job_living_again_replaced_or_gone_by_the_delete_is_left_alone(self): + revived, replaced, gone = self.dead("waiting"), self.dead("active"), self.dead("waiting") + self.queue(revived, replaced, gone) + swap = {**self.dead("active"), "id": replaced["id"]} + + def act() -> None: + self.holder(self.lock_file(revived)) + self.queue(revived, swap) + + code, out, _ = self.at_the_close(act) + self.assertEqual((code, self.entries()), (0, [revived, swap])) + self.ends(out, ": deleted none; held by a living process, not deleted test-1; left the queue test-2, test-3") + + def test_an_entry_replaced_in_the_first_window_is_no_candidate_for_the_security(self): + old = self.dead("waiting") + self.queue(old) + replaced = {**self.dead("waiting"), "id": old["id"]} + self.actions[("need", 0)] = lambda: self.queue(replaced) + code, out = self.round() + self.assertEqual((code, self.entries(), self.classes()), (0, [replaced], ["pan-pan", "post"])) + self.ends(out, ": deleted none; left the queue test-1") + + def test_a_dead_job_queued_after_the_snapshot_is_neither_named_nor_deleted(self): + old, new = self.dead("waiting"), self.dead("waiting") + self.queue(old) + self.actions[("need", 0)] = lambda: self.queue(old, new) + code, out = self.round() + self.assertEqual((code, self.entries()), (0, [new])) + self.ends(out, ": deleted test-1; left the queue none") + self.assertNotIn("test-2", " ".join(row["text"] for row in self.rows())) + + def test_malformed_entries_stay_counted_and_block_no_round(self): + job = self.dead("waiting") + odd = [1, {"id": "x"}, {"seq": 2, "status": "waiting", "kind": "test", "lock": "2-2"}, + {"id": 5, "status": "waiting", "kind": "test", "lock": "3-3"}, {"id": "y", "status": "waiting", "lock": "4-4"}, + {"id": "z", "status": "waiting", "kind": "test"}, {**self.dead("waiting"), "status": "done"}] + self.queue(job, *odd) + code, out = self.round() + self.assertEqual((code, self.entries()), (0, odd)) + self.ends(out, f": deleted {job['id']}; left the queue none") + self.assertIn(" 0 held and 0 unclear jobs, 7 malformed entries stay; ", self.rows()[0]["text"]) + + def test_the_alarms_stay_bounded_for_50_long_ids(self): + ids = [f"test-nostack-{1000 + n}-" + "x" * 50 for n in range(50)] + state = {"round": "12:00", "chair": "Main", "jobs": [{"id": i, "lock": f"{n}-{n}", "status": "waiting", "kind": "test"} + for n, i in enumerate(ids)], "held": ids, "unclear": ids, "malformed": 3, "deadline": time.time() + WINDOW, + "window": WINDOW, "delete": ids} + for text in (self.cleanup.ask(state), self.cleanup.warn(state)): + with self.subTest(text=text[:40]): + self.assertLess(len(text), self.cleanup.watch.CLIP) + self.assertIn(", +44 more", text) + + def test_each_window_lasts_its_length_on_both_clocks_from_its_own_alarm(self): + cases = ( + ("wall ahead within the stall limit", {("need", 0): (100, 0)}, 900, 900), + ("wall back", {("need", 0): (-600, 0)}, 1500, 900), + ("monotonic ahead", {("need", 0): (0, 2000)}, 900, 900), + ("monotonic ahead in the second window", {("warn", 0): (0, 2000)}, 900, 900), + ) + for name, jumps, need, warn in cases: + with self.subTest(name): + self.slept.clear() + self.jumps = jumps + self.queue(self.dead("waiting")) + self.assertEqual(self.round()[0], 0) + self.assertGreaterEqual(self.waited("need"), need) + self.assertGreaterEqual(self.waited("warn"), warn) + self.assertEqual(self.entries(), []) + + def test_a_killed_round_counts_as_dead_and_blocks_no_next_one(self): + self.queue(self.dead("waiting")) + + def die() -> None: + raise Killed + + self.actions[("need", 0)] = die + with contextlib.suppress(Killed), contextlib.redirect_stdout(io.StringIO()): + self.cleanup.run_round(WINDOW, self.clock) + self.actions = {} + self.assertEqual(self.round()[0], 3) + self.clock.wall += self.cleanup.DEAD_SECONDS + 1 + self.assertEqual(self.cli("status")[1], "no open round\n") + code, out = self.round() + self.assertEqual((code, self.classes().count("pan-pan")), (0, 2)) + self.ends(out, ": deleted test-1; left the queue none") + + def test_a_stall_above_the_dead_limit_ends_the_round_without_deleting(self): + job = self.dead("active") + self.queue(job) + self.jumps[("need", 5)] = (self.cleanup.DEAD_SECONDS + 30, 0) + code, out = self.round() + self.assertEqual((code, self.ids()), (0, [job["id"]])) + self.ends(out, f": not deleted, the round stalled for {self.cleanup.DEAD_SECONDS + 35} s {job['id']}; left the queue none") + self.assertEqual(self.register()["status"], "stopped") + + def test_a_gap_of_exactly_the_dead_limit_is_no_stall(self): + self.queue(self.dead("waiting")) + self.jumps[("need", 5)] = (self.cleanup.DEAD_SECONDS - self.cleanup.POLL_SECONDS, 0) + code, out = self.round() + self.assertEqual((code, self.entries()), (0, [])) + self.ends(out, ": deleted test-1; left the queue none") + + def test_the_round_age_limits_are_inclusive(self): + cleanup, beat = self.cleanup, 1000.0 + dead = cleanup.DEAD_SECONDS + self.assertTrue(cleanup.alive({"phase": "warn", "beat": beat}, beat + dead)) + self.assertFalse(cleanup.alive({"phase": "warn", "beat": beat}, beat + dead + 1)) + self.assertFalse(cleanup.alive({"phase": "closing", "beat": beat}, beat)) + self.assertTrue(cleanup.busy({"phase": "closing", "beat": beat}, beat + dead)) + self.assertFalse(cleanup.busy({"phase": "closing", "beat": beat}, beat + dead + 1)) + self.assertFalse(cleanup.busy({"phase": "done", "beat": beat}, beat)) + self.assertFalse(cleanup.busy({}, beat)) + self.assertIsNone(cleanup.stalled({"beat": beat}, beat + dead)) + self.assertEqual(cleanup.stalled({"beat": beat}, beat + dead + 1), f"the round stalled for {dead + 1} s") + + def test_a_round_taken_over_by_a_newer_one_stops_without_deleting(self): + job = self.dead("waiting") + self.queue(job) + self.actions[("need", 2)] = lambda: self.write_state({**self.state(), "start": self.state()["start"] + 1}) + err = io.StringIO() + with contextlib.redirect_stderr(err), contextlib.redirect_stdout(io.StringIO()): + code = self.cleanup.run_round(WINDOW, self.clock) + self.assertEqual((code, self.entries(), self.classes()), (0, [job], ["pan-pan"])) + self.assertEqual(err.getvalue(), "queue-cleanup: a newer round took over this one; nothing more is deleted\n") + + def test_radio_silence_at_the_start_on_a_poll_or_on_the_last_poll_deletes_nothing(self): + for when in ("start", "poll", "last poll"): + with self.subTest(when): + (self.root / "radio-silence.json").unlink(missing_ok=True) + (self.root / "news.jsonl").unlink(missing_ok=True) + self.slept.clear() + job = self.dead("waiting") + self.queue(job) + if when == "start": + self.silence() + self.actions = {"poll": {("need", 1): self.silence}, "last poll": {("warn", self.last): self.silence}}.get(when, {}) + code, out = self.round() + self.assertEqual((code, self.entries()), (0, [job])) + self.ends(out, f": not deleted, radio silence by IAM holds {job['id']}; left the queue none") + self.assertEqual(self.register()["status"], "stopped") + expected = {"start": ["post"], "poll": ["pan-pan", "post"], "last poll": ["pan-pan", "security", "post"]} + self.assertEqual(self.classes(), expected[when]) + + def test_radio_silence_found_by_the_close_deletes_nothing(self): + job = self.dead("waiting") + self.queue(job) + code, out, _ = self.at_the_close(self.silence) + self.assertEqual((code, self.entries()), (0, [job])) + self.ends(out, f": not deleted, radio silence by IAM holds {job['id']}; left the queue none") + + def test_a_stall_after_the_closing_publish_deletes_nothing(self): + job = self.dead("active") + self.queue(job) + + def stall() -> None: + self.clock.wall += self.cleanup.DEAD_SECONDS + 10 + + code, out, _ = self.at_the_close(stall) + self.assertEqual((code, self.ids()), (0, [job["id"]])) + self.ends(out, f": not deleted, the round stalled for {self.cleanup.DEAD_SECONDS + 10} s {job['id']}; left the queue none") + + def test_a_newer_round_found_right_before_the_delete_deletes_nothing(self): + job = self.dead("waiting") + self.queue(job) + code, _, err = self.at_the_close(lambda: self.write_state({**self.state(), "start": self.state()["start"] + 1})) + self.assertEqual((code, self.entries(), self.classes()), (0, [job], ["pan-pan", "security"])) + self.assertEqual(err, "queue-cleanup: a newer round took over this one; nothing more is deleted\n") + + def test_an_unexpected_error_after_the_claim_stops_the_round_posts_once_and_fails(self): + job = self.dead("waiting") + self.queue(job) + err = io.StringIO() + with mock.patch.object(self.cleanup, "standing", side_effect=RuntimeError("boom")), contextlib.redirect_stderr(err), \ + contextlib.redirect_stdout(io.StringIO()): + code = self.cleanup.run_round(WINDOW, self.clock) + self.assertEqual((code, self.entries()), (1, [job])) + self.assertEqual(self.classes(), ["pan-pan", "post"]) + self.assertRegex(self.rows()[-1]["text"], r"^queue cleanup \d\d:\d\d: the round failed \(RuntimeError\); nothing more is deleted$") + self.assertEqual((self.state()["phase"], self.register()["status"]), ("stopped", "stopped")) + self.assertIn("RuntimeError('boom')", err.getvalue()) + self.assertEqual(self.round()[0], 0) + + def test_a_queue_store_gone_invalid_mid_round_stops_it_posts_once_and_fails(self): + cases = {"invalid JSON": ("{", "SystemExit"), "too deep": ("[" * 100000 + "]" * 100000, "RecursionError"), + "no entries": ("{}", "KeyError"), "a list": ("[]", "TypeError")} + for name, (text, kind) in cases.items(): + with self.subTest(name): + (self.root / "news.jsonl").unlink(missing_ok=True) + self.slept.clear() + self.queue(self.dead("waiting")) + self.actions = {("need", 0): lambda text=text: (self.root / "queue.json").write_text(text, encoding="utf-8")} + with contextlib.redirect_stderr(io.StringIO()), contextlib.redirect_stdout(io.StringIO()): + code = self.cleanup.run_round(WINDOW, self.clock) + self.assertEqual((code, self.classes()), (1, ["pan-pan", "post"])) + self.assertRegex(self.rows()[-1]["text"], rf"^queue cleanup \d\d:\d\d: the round failed \({kind}\); nothing more is deleted$") + self.assertEqual((self.state()["phase"], self.register()["status"]), ("stopped", "stopped")) + self.assertEqual((self.root / "queue.json").read_text(encoding="utf-8"), text) + + def test_a_queue_lock_won_past_the_stall_limit_or_after_a_newer_round_deletes_nothing(self): + original = self.cleanup.queue.store + cases = { + "stall": (lambda: setattr(self.clock, "wall", self.clock.wall + self.cleanup.DEAD_SECONDS + 10), + f": not deleted, the round stalled for {self.cleanup.DEAD_SECONDS + 10} s test-1; left the queue none\n"), + "newer round": (lambda: self.write_state({**self.state(), "start": self.state()["start"] + 1}), ""), + } + for name, (wait, tail) in cases.items(): + with self.subTest(name): + job = self.dead("waiting") + self.queue(job) + + @contextlib.contextmanager + def slow(wait=wait): + if (self.kept / "queue-cleanup.json").exists() and self.state()["phase"] == "closing": + wait() + with original() as data: + yield data + + with mock.patch.object(self.cleanup.queue, "store", slow): + code, out = self.round() + self.assertEqual((code, self.entries()), (0, [job])) + self.assertTrue(out.endswith(tail), out) + + def test_status_shows_the_phase_and_each_judged_job(self): + self.queue(self.dead("waiting"), self.held("active"), {**self.dead("waiting"), "lock": "x"}) + shown = {} + self.actions[("need", 1)] = lambda: shown.update(status=self.cli("status")) + self.round() + lines = shown["status"][1].splitlines() + self.assertRegex(lines[0], r"^round \d\d:\d\d need until \d\d:\d\d$") + self.assertEqual(lines[1:], ["test-1 waiting test delete", "test-2 held", "test-3 unclear"]) + code, out, _ = self.cli("status") + self.assertEqual(code, 0) + self.assertRegex(out, r"^no open round; last: queue cleanup \d\d:\d\d: deleted test-1; left the queue none\n$") + + def test_the_round_lands_in_the_register_with_the_chair_accountable(self): + self.queue(self.dead("waiting")) + self.round() + row = self.register() + self.assertEqual( + (row["status"], row["worker"], row["roles"], row["objective"]), + ("done", WORKER, {"Main": "A", WORKER: "R"}, self.cleanup.OBJECTIVE), + ) + self.assertTrue(row["description"].endswith(": deleted test-1; left the queue none")) + + def test_a_chair_gone_before_the_delete_stops_the_round_and_deletes_nothing(self): + job = self.dead("waiting") + self.queue(job) + self.actions[("warn", 0)] = lambda: self.chair("gone") + code, out = self.round() + self.assertEqual((code, self.entries()), (0, [job])) + self.ends(out, ": not deleted, the chair gone is stale test-1; left the queue none") + self.assertEqual(self.register()["status"], "stopped") + + def test_a_second_round_refuses_while_one_runs(self): + self.queue(self.dead("waiting")) + inner = {} + self.actions[("need", 0)] = lambda: inner.update(need=self.round()[0]) + self.actions[("warn", 0)] = lambda: inner.update(warn=self.round()[0]) + self.round() + self.assertEqual(inner, {"need": 3, "warn": 3}) + self.assertEqual(self.classes(), ["pan-pan", "security", "post"]) + + def test_run_refuses_an_agent_session_a_short_window_and_a_read_only_host_state(self): + self.queue(self.dead("waiting")) + with mock.patch.dict(os.environ, {"CLAUDE_CODE_SESSION_ID": "s1"}): + inside = self.cli("run", "--window", "900") + short = self.cli("run", "--window", "899") + self.kept.chmod(0o500) + self.addCleanup(self.kept.chmod, 0o700) + sandboxed = self.cli("run", "--window", "900") + self.assertEqual(inside, (2, "", "queue-cleanup: run starts from kvb-org-queue-cleanup.service, never in an agent session; agents use request\n")) + self.assertEqual(short, (2, "", "queue-cleanup: --window must be at least 900 seconds\n")) + self.assertEqual(sandboxed, (2, "", f"queue-cleanup: cannot write {self.kept}; run starts outside the sandbox only\n")) + self.assertEqual(self.rows(), []) + self.assertFalse((self.kept / "queue-cleanup.json").exists()) + + def test_request_needs_the_path_unit_and_appends_in_place(self): + request = self.root / "queue-cleanup.request" + unit = self.home / ".config" / "systemd" / "user" / "kvb-org-queue-cleanup.path" + self.assertEqual(self.cli("request"), (2, "", f"queue-cleanup: {unit} is missing, so no round would start; make install enables it\n")) + self.assertFalse(request.exists()) + self.install_unit() + code, out, _ = self.cli("request") + self.assertEqual((code, out), (0, f"requested a queue-cleanup round through {request}\n")) + inode = request.stat().st_ino + self.cli("request") + self.assertEqual((request.stat().st_ino, len(request.read_text(encoding="utf-8").splitlines())), (inode, 2)) + self.assertIn("PathModified=/tmp/agents-council/queue-cleanup.request\n", unit.read_text(encoding="utf-8")) + + def test_request_refuses_during_a_round_and_for_30_minutes_after_one(self): + self.install_unit() + now, clock = self.clock.wall, self.cleanup.radio.clock + self.write_state({"round": "12:00", "start": now - 100, "beat": now, "phase": "closing"}) + self.assertEqual(self.cli("request"), (2, "", "queue-cleanup: round 12:00 runs; status shows it\n")) + ended = now - 600 + self.write_state({"round": "12:00", "start": now - 2000, "beat": ended, "phase": "done", "end": ended}) + self.assertEqual( + self.cli("request"), + (2, "", f"queue-cleanup: the last round ended at {clock(ended)}; request again from {clock(ended + 1800)}\n"), + ) + self.write_state({"round": "12:00", "start": now - 4000, "beat": now - 1800, "phase": "done", "end": now - 1800}) + self.assertEqual(self.cli("request")[0], 0) + + def test_the_units_run_at_midnight_and_noon_without_catch_up_and_outlast_both_windows(self): + timer = (SYSTEMD / "kvb-org-queue-cleanup.timer").read_text(encoding="utf-8") + service = (SYSTEMD / "kvb-org-queue-cleanup.service").read_text(encoding="utf-8") + self.assertIn("OnCalendar=*-*-* 00,12:00:00\nPersistent=false\n", timer) + self.assertIn("Type=oneshot\n", service) + window = int(re.search(r"^ExecStart=/usr/bin/python3 -I \S+/cleanup\.py run --window (\d+)$", service, re.MULTILINE)[1]) + self.assertEqual(window, WINDOW) + budget = 2 * window + 4 * self.cleanup.POLL_SECONDS + self.assertGreater(int(re.search(r"^TimeoutStartSec=(\d+)$", service, re.MULTILINE)[1]), budget) + + +if __name__ == "__main__": + unittest.main()