Merge branch 'feature/kvb-org-queue-cleanup' into unify/skills-20261112-night

This commit is contained in:
Kevin Veen-Birkenbach committed 2026-11-13 10:45:59 +00:00
commit 1fa52eb092
11 files changed
+1431 -9

No files matched your search

+9 -1
View File
@@ -95,6 +95,7 @@ flowchart LR
org --> org_radio[kvb-org-radio]
org --> org_queue[kvb-org-queue]
org --> org_queue_estimate[kvb-org-queue-estimate]
org --> org_queue_cleanup[kvb-org-queue-cleanup]
org --> org_update[kvb-org-update]
agent --> agent_optimize[kvb-agent-optimize]
agent --> agent_drain[kvb-agent-drain]
@@ -128,6 +129,7 @@ flowchart LR
stage_req([staging or fix ready])
edit_hook([edit in a git worktree])
reminders([12 h agent hint, 17:00 GNOME popup])
noon_midnight([00:00 and 12:00 timer])
full_hour([full hour])
code_task([code task])
spawn([subagent spawn])
@@ -212,6 +214,7 @@ flowchart LR
radio[kvb-org-radio]
queue[kvb-org-queue]
queue_estimate[kvb-org-queue-estimate]
queue_cleanup[kvb-org-queue-cleanup]
org_update[kvb-org-update]
end
subgraph cat_agent [agent]
@@ -247,6 +250,7 @@ flowchart LR
stage_req -.-> confidence
edit_hook -. council gate .-> democracy
reminders -.-> dream
noon_midnight -.-> queue_cleanup
full_hour -.-> coach
code_task -.-> clean
code_task -.-> scope_guard
@@ -318,6 +322,10 @@ flowchart LR
unify --> radio
status_matrix --> rasci
cleanup --> artefacts
cleanup -. request .-> queue_cleanup
queue_cleanup --> queue
queue_cleanup -. pan-pan, security .-> radio
queue_cleanup --> rasci
artefacts --> rasci
democracy --> wsjf
wsjf --> rasci
@@ -454,7 +462,7 @@ Copy the skills into a specific project (`<repo>/.agents/skills` and `<repo>/.cl
make project TARGET=/path/to/repo
```
Both commands also patch the target's `.claude/settings.local.json` — the machine-local, untracked file, so a project's reviewed `.claude/settings.json` is never rewritten by an install: the caveman and ponytail plugins are enabled so their modes auto-activate on session start, and two reminder hooks are registered — `kvb-dream` points each agent at `/kvb-dream` at most once every twelve hours (stamped per session in `~/.claude/.dream-stamps/`), and `kvb-coach` runs its short form at the first prompt of every full hour, once across all agents (stamped in `~/.claude/.coach-hours/`); the global install also enables the `kvb-dream-notify.timer` systemd user timer, which recommends `/kvb-dream` in a GNOME notification daily at 17:00 local time and catches up once after a missed run. A PreToolUse gate from the `kvb-org-organize` skill blocks edits to files in a git worktree until the session holds a claim (`skills/kvb-org-organize/scripts/council.py`), and its resource watch (`skills/kvb-org-organize/scripts/watch.py`) measures disk, Docker and caches, cleans up low-risk data and blocks new heavy jobs before the disk runs out; it stays inactive until `~/.claude/agents-council/config.json` exists, and because its hooks run that config's commands outside the sandbox, the install adds `Edit(~/.claude/agents-council/**)` to `permissions.deny`, which also covers Write in every permission mode. The `kvb-org-radio` skill registers five hooks: its prompt hook and a PostToolBatch hook deliver the immediate radio (`call`, `broadcast`, `security`, `pan-pan`, `mayday`) mid-turn, a Stop hook delivers the post (`post`, `newsletter` and call receipts) when the turn ends, an `asyncRewake` Stop hook wakes an idle session for the radio, and a PreToolUse hook refuses every writing action while a mayday holds, except those of its sender and of the agents that join it with `radio.py assist`. The PreToolUse hook of the `kvb-org-queue` skill sends heavy commands through `queue.py hold` once the config sets `queue_enforce`, and holds a held command to the settings' Bash deny and ask rules, which see only `queue.py hold`. The global install also installs the MEDUSA security scanner with pipx for `kvb-check-security`, unless `medusa` is already on the PATH. Two hooks of the `kvb-org-perform` skill (`skills/kvb-org-perform/scripts/log.py`) log every Agent spawn with its task class, model and effort, and store the quality feedback a sandboxed session prints, in `~/.local/state/agents-council/subagent-model.jsonl`. Every hook runs `/usr/bin/python3 -I`, so neither the `python3` of a sandbox-writable virtualenv on the `PATH` nor `PYTHONPATH` reaches the hooks, which run outside the sandbox; older bare `python3` hook entries are rewritten. The reminder hooks run `/usr/bin/bash` and pin `PATH=/usr/bin:/bin`, and the council scripts keep only the `PATH` directories this uid cannot write (`skills/kvb-org-organize/scripts/hookpath.py`), so no binary an agent plants on the inherited `PATH` runs outside the sandbox. Existing settings are preserved.
Both commands also patch the target's `.claude/settings.local.json` — the machine-local, untracked file, so a project's reviewed `.claude/settings.json` is never rewritten by an install: the caveman and ponytail plugins are enabled so their modes auto-activate on session start, and two reminder hooks are registered — `kvb-dream` points each agent at `/kvb-dream` at most once every twelve hours (stamped per session in `~/.claude/.dream-stamps/`), and `kvb-coach` runs its short form at the first prompt of every full hour, once across all agents (stamped in `~/.claude/.coach-hours/`); the global install also enables the `kvb-dream-notify.timer` systemd user timer, which recommends `/kvb-dream` in a GNOME notification daily at 17:00 local time and catches up once after a missed run, and the `kvb-org-queue-cleanup.timer` and `kvb-org-queue-cleanup.path` user units, which start a `kvb-org-queue-cleanup` round at 00:00 and 12:00 local time without catch-up and whenever an agent runs `cleanup.py request`. A PreToolUse gate from the `kvb-org-organize` skill blocks edits to files in a git worktree until the session holds a claim (`skills/kvb-org-organize/scripts/council.py`), and its resource watch (`skills/kvb-org-organize/scripts/watch.py`) measures disk, Docker and caches, cleans up low-risk data and blocks new heavy jobs before the disk runs out; it stays inactive until `~/.claude/agents-council/config.json` exists, and because its hooks run that config's commands outside the sandbox, the install adds `Edit(~/.claude/agents-council/**)` to `permissions.deny`, which also covers Write in every permission mode. The `kvb-org-radio` skill registers five hooks: its prompt hook and a PostToolBatch hook deliver the immediate radio (`call`, `broadcast`, `security`, `pan-pan`, `mayday`) mid-turn, a Stop hook delivers the post (`post`, `newsletter` and call receipts) when the turn ends, an `asyncRewake` Stop hook wakes an idle session for the radio, and a PreToolUse hook refuses every writing action while a mayday holds, except those of its sender and of the agents that join it with `radio.py assist`. The PreToolUse hook of the `kvb-org-queue` skill sends heavy commands through `queue.py hold` once the config sets `queue_enforce`, and holds a held command to the settings' Bash deny and ask rules, which see only `queue.py hold`. The global install also installs the MEDUSA security scanner with pipx for `kvb-check-security`, unless `medusa` is already on the PATH. Two hooks of the `kvb-org-perform` skill (`skills/kvb-org-perform/scripts/log.py`) log every Agent spawn with its task class, model and effort, and store the quality feedback a sandboxed session prints, in `~/.local/state/agents-council/subagent-model.jsonl`. Every hook runs `/usr/bin/python3 -I`, so neither the `python3` of a sandbox-writable virtualenv on the `PATH` nor `PYTHONPATH` reaches the hooks, which run outside the sandbox; older bare `python3` hook entries are rewritten. The reminder hooks run `/usr/bin/bash` and pin `PATH=/usr/bin:/bin`, and the council scripts keep only the `PATH` directories this uid cannot write (`skills/kvb-org-organize/scripts/hookpath.py`), so no binary an agent plants on the inherited `PATH` runs outside the sandbox. Existing settings are preserved.
The global install also runs the Agent Orchestrator, a local review page in which the operator approves or rejects the agents' staged commits (the `kvb-git-diff` skill): it copies `orchestrator/` to `~/.local/share/kvb-orchestrator`, starts it there with `docker compose up -d --build` (restart `unless-stopped`, bound to `127.0.0.1` only, state in `~/.local/state/kvb-orchestrator`) and registers its MCP server for Claude Code at user scope. Its port is `KVB_ORCHESTRATOR_PORT` in `orchestrator/.env`; the hub page is `http://localhost:<port>/`, the MCP endpoint `/mcp`. Without docker the hub is skipped and the commit skills ask in chat. The hub reads new radio traffic aloud in maritime phraseology and takes push-to-talk input; its image builds both engines from pinned, sha256-verified sources and runs each as a separate process: Piper (`piper-tts`, GPL-3.0-or-later) with the `en_US-libritts-high` voice (904 speakers, trained on LibriTTS, CC BY 4.0, from `rhasspy/piper-voices`) and whisper.cpp (MIT) with the multilingual `ggml-base-q8_0` model (OpenAI Whisper weights, MIT, 82 MB). The operator's own voice is `KVB_ORCHESTRATOR_OPERATOR_VOICE` in `orchestrator/.env`. Its Tree reads only the file `pkgmgr branch tree` writes: the install clones `PKGMGR_REPO` (default the GitHub repository, never the sandbox-writable local checkout), builds the newest `v*` tag a fully trusted GPG key signed into `~/.local/share/kvb-orchestrator/pkgmgr`, and enables the `kvb-orchestrator-tree.timer` user timer, which writes `~/.local/state/pkgmgr/tree/branch-tree.json` a minute after its previous run while the hub page polls, on at most half a core; the hub mounts that directory read-only and starts no git. A signed tag without `branch tree` leaves the timer off.
+5 -3
View File
@@ -66,15 +66,17 @@ else
fi
if [[ "${TARGET}" == "${HOME}" ]]; then
install -d -m 0700 "${HOME}/.local/state/agents-council"
if command -v systemctl &>/dev/null && systemctl --user show-environment &>/dev/null; then
log "skills: enabling the daily kvb-dream notification timer"
log "skills: enabling the daily kvb-dream notification timer and the kvb-org-queue-cleanup rounds at 00:00, 12:00 and on request"
units="${HOME}/.config/systemd/user"
mkdir -p "${units}"
cp -f "${REPO_ROOT}/skills/kvb-dream/systemd/kvb-dream-notify.service" "${REPO_ROOT}/skills/kvb-dream/systemd/kvb-dream-notify.timer" "${units}/"
cp -f "${REPO_ROOT}/skills/kvb-org-queue-cleanup/systemd/kvb-org-queue-cleanup."{service,timer,path} "${units}/"
systemctl --user daemon-reload
systemctl --user enable --now kvb-dream-notify.timer
systemctl --user enable --now kvb-dream-notify.timer kvb-org-queue-cleanup.timer kvb-org-queue-cleanup.path
else
warn "skills: no systemd user session; skipped the kvb-dream notification timer."
warn "skills: no systemd user session; skipped the kvb-dream notification timer and the kvb-org-queue-cleanup rounds."
fi
if command -v docker &>/dev/null; then
bash "${REPO_ROOT}/scripts/orchestrator.sh" install
+2
View File
@@ -34,6 +34,8 @@ rules forbidding raw commands.
build temp dirs.
- **Register**: `council.py list --long`; `rasci.py artifacts` (`kvb-org-artefacts`,
`--ended` for candidates).
- **Queue**: `${CLAUDE_SKILL_DIR}/../kvb-org-queue-cleanup/scripts/cleanup.py
request`; go on meanwhile.
- **Live use**: `fuser`, `lsof`, `ps -o pid,etime,comm`, never full command
lines or environments (credentials). Sandboxed, an empty result proves
nothing.
+61
View File
@@ -0,0 +1,61 @@
---
name: kvb-org-queue-cleanup
description: >
Twice daily, delete dead queue jobs.
---
`${CLAUDE_SKILL_DIR}/scripts/cleanup.py` runs one cleanup round of the
`kvb-org-queue`, accountable to the top chair (council office `chair`);
without a live chair it deletes nothing.
## Round
1. A job (waiting or running) is dead only when no line of `/proc/locks`,
of any type, pid or waiting state, names its hold's lock file
`holds/<tag>.lock` and a flock on that file would not block, or when the
file is gone. Whoever owns a job, and whatever its queue entry says, a
living holder keeps it. Nothing is dead while `/proc/locks` is
unreadable, unparsable or misses the round's own flock on `holds/`, nor
is a job whose lock file is unreadable, no regular file, or changed
after its hold created it. Malformed entries stay.
2. Without a dead job the round sends nothing. Else a pan-pan names the dead
jobs; a 15 min window runs on both the wall and the monotonic clock.
3. A security names the jobs still dead and every job left alone with "held
by a living process, not deleted"; another 15 min window runs.
4. Each job still dead under the same id and lock tag, judged again under
the queue lock, leaves the queue. Nothing is signalled, no stop is
written.
5. The chair gets the counts by post; the `kvb-org-rasci` register records
the round as task `queue-cleanup`.
The round keeps its state in
`~/.local/state/agents-council/queue-cleanup.json`, which only the host
writes. Radio silence on any poll, a stall above 3 minutes, a newer round,
or a chair gone stale before the delete ends the round at once, deletes
nothing and records it `stopped`; so does any other error, with one post to
the chair.
`cleanup.py status` shows the open round's jobs: delete, held or unclear.
There is nothing to answer.
## Triggers
- `kvb-org-queue-cleanup.timer`, which `make install` enables, at 00:00 and
12:00 host local time, without catch-up.
- `kvb-org-queue-cleanup.path` on `cleanup.py request`, which `kvb-cleanup`
runs; `request` exits 2 without the installed path unit, while a round
runs and for 30 minutes after one ended.
- Another interval: `systemctl --user edit kvb-org-queue-cleanup.timer`, a
`[Timer]` drop-in of an empty `OnCalendar=`, then `OnCalendar=<spec>`.
- `cleanup.py run --window <seconds>` refuses inside an agent session, below
900 s and without write access to the host state; the service passes 900.
## Limits
- A lock file removed while its hold lives reads as gone, so its job is
deleted; `queue.py` itself prunes such an entry on its next poll.
- Started outside the initial PID namespace, the round misses POSIX locks of
processes it cannot see; the service runs in it.
- The alarms name at most 6 ids per list; `status` lists all.
- The 30-minute spacing binds `cleanup.py request` only; a line appended to
the request file otherwise starts a round once none runs.
+600
View File
@@ -0,0 +1,600 @@
#!/usr/bin/env python3
"""Delete the kvb-org-queue jobs whose hold lock no living process holds, announced by a pan-pan and a security.
Usage:
cleanup.py run --window SECONDS
cleanup.py status
cleanup.py request
Environment:
AGENTS_COUNCIL_ROOT: shared state directory; /tmp/agents-council unless
set. Agents append their requests to <root>/queue-cleanup.request,
which kvb-org-queue-cleanup.path watches below /tmp/agents-council
only. The round lives in ~/.local/state/agents-council, which only
the host can write, in queue-cleanup.json under queue-cleanup.lock,
written by run alone.
CLAUDE_CODE_SESSION_ID: run refuses to start while it is set.
Args:
run: one round, accountable to the live holder of the council office
chair; without one it deletes nothing. --window: seconds of each
window, at least WINDOW_FLOOR (the service passes 900). It refuses
unless it can write the host state. A job is a waiting or active
entry with a string id, status, kind and lock; every other entry
stays and counts as malformed. A job is dead when no line of
/proc/locks, whatever its type, pid or blocked state, names the
device and inode of <root>/holds/<lock>.lock and a flock on it
would not block, or when that file is gone. Nothing is dead while
/proc/locks cannot be read, holds a line without exactly one device
and inode, or misses the shared flock the round holds on
<root>/holds while it reads them; nor is a job whose lock file
cannot be opened, is no regular file with one link, or changed more
than CREATED_SECONDS away from the time in its tag.
Without a dead job it sends nothing. Else a pan-pan names the dead
jobs and counts the others; the window starts once it is sent and
ends once both the wall clock and the monotonic clock have run
--window seconds. A security then names the round's jobs still dead
and every job held by a living process, and a second window starts
once it is sent. Each job of the round still queued under the same
id and lock tag and still dead at the end, judged under the queue
lock, leaves the queue; nothing is signalled and no stop is written.
Radio silence on any poll, a gap above DEAD_SECONDS between the
round's own last beat and the end of a poll's sleep or the moment
the delete holds the queue lock, a newer round in the host state, or
radio silence or a vacant or stale chair before the delete ends the
round at once without deleting; it is recorded stopped. Any other
error after the claim, a queue store gone invalid included, records
phase stopped, deletes nothing more, posts one line to the chair and
exits FAILED. The result goes by post to the chair and into task
queue-cleanup of the kvb-org-rasci register. Every row it sends
comes from the station queue-cleanup.
status: the open round's phase and deadline, then one line per job it
judged: id, status, kind and delete for a dead one, id and held or
unclear for the others; without an open round the last result.
request: append a line to the request file in place, so
kvb-org-queue-cleanup.path starts a round outside the sandbox;
refused while a round runs and for SPACING_SECONDS after one ended.
Exit codes:
0 done, 1 run failed after its claim, 2 usage error, run inside an agent
session, with a window below WINDOW_FLOOR or without write access to the
host state, request without the installed path unit, during a round or
within SPACING_SECONDS after one, a council root that watch.py refuses;
run: 3 while another round runs.
"""
from __future__ import annotations
import argparse
import contextlib
import datetime
import fcntl
import importlib.util
import json
import os
import re
import sys
import time
from pathlib import Path
from types import ModuleType
from typing import NoReturn, Protocol
SKILLS = Path(__file__).resolve().parents[2]
SELF = Path(__file__).resolve()
STATION = "queue-cleanup"
TASK = "queue-cleanup"
WORKER = "systemd kvb-org-queue-cleanup"
OBJECTIVE = "Delete the queue jobs no living process holds"
WINDOW_FLOOR = 900
POLL_SECONDS = 5
DEAD_SECONDS = 180
SPACING_SECONDS = 1800
CREATED_SECONDS = 10
NAMES_SHOWN = 6
NAME_WIDTH = 32
OPEN = ("need", "warn")
ENDED = ("done", "stopped")
STATUSES = ("waiting", "active")
DEAD, HELD, UNCLEAR = "dead", "held", "unclear"
GONE = "gone"
RUNNING = 3
FAILED = 1
PROC = Path("/proc")
DEVICE = re.compile(r"(?<!\S)([0-9a-f]+):([0-9a-f]+):(\d+)(?!\S)")
UNIT = Path.home() / ".config" / "systemd" / "user" / "kvb-org-queue-cleanup.path"
class Clock(Protocol):
def time(self) -> float: ...
def monotonic(self) -> float: ...
def sleep(self, seconds: float) -> None: ...
class Superseded(Exception):
pass
def script(name: str, path: Path) -> ModuleType:
spec = importlib.util.spec_from_file_location(f"kvb_queue_cleanup_{name}", path)
loaded = importlib.util.module_from_spec(spec)
spec.loader.exec_module(loaded)
return loaded
queue = script("queue", SKILLS / "kvb-org-queue" / "scripts" / "queue.py")
radio = script("radio", SKILLS / "kvb-org-radio" / "scripts" / "radio.py")
rasci = script("rasci", SKILLS / "kvb-org-rasci" / "scripts" / "rasci.py")
watch, nofollow = queue.watch, queue.nofollow
REQUEST = watch.ROOT / "queue-cleanup.request"
STATE = watch.KEPT / "queue-cleanup.json"
STATE_LOCK = watch.KEPT / "queue-cleanup.lock"
def fail(message: str) -> NoReturn:
print(f"queue-cleanup: {message}", file=sys.stderr)
sys.exit(2)
def council() -> ModuleType:
return radio.council()
def host_state() -> dict:
try:
data = json.loads(nofollow.read_text(watch.KEPT, STATE))
except (OSError, ValueError):
return {}
return data if isinstance(data, dict) else {}
def save(state: dict) -> None:
nofollow.replace(watch.KEPT, STATE, json.dumps(state, indent=1), 0o600)
def publish(state: dict) -> None:
with nofollow.locked(watch.KEPT, STATE_LOCK, 0o600, None):
if host_state().get("start") != state["start"]:
raise Superseded
save(state)
def alive(state: dict, now: float) -> bool:
return state.get("phase") in OPEN and now - state["beat"] <= DEAD_SECONDS
def busy(state: dict, now: float) -> bool:
return "beat" in state and state["phase"] not in ENDED and now - state["beat"] <= DEAD_SECONDS
def stalled(state: dict, now: float) -> str | None:
gap = now - state["beat"]
return f"the round stalled for {gap:.0f} s" if gap > DEAD_SECONDS else None
def top_chair() -> tuple[str | None, str]:
"""Returns:
(session, name) of the live holder of the council office chair, else
(None, why no one is accountable).
"""
claim = council().read_claim(council().ROOT / "chair")
if claim is None:
return None, "the council office chair is vacant"
if council().is_stale(claim, council().live_sessions()):
return None, f"the chair {radio.clip(claim.get('name') or claim['session'])} is stale"
return claim["session"], radio.name_of(claim["session"])
def hushed() -> str | None:
active = radio.silence()
return None if active is None else f"radio silence by {radio.clip(active.get('by'))} holds"
def tagged(tag: object) -> bool:
return isinstance(tag, str) and bool(queue.TAG.fullmatch(tag))
def key(info: os.stat_result) -> tuple[int, int, int]:
return os.major(info.st_dev), os.minor(info.st_dev), info.st_ino
def lock_lines(root: Path) -> set[tuple[int, int, int]] | None:
"""Returns:
The (device major, minor, inode) of every line of locks below root,
a /proc; None where it cannot be read or a line names not exactly one.
"""
try:
lines = (root / "locks").read_text(encoding="utf-8").splitlines()
except (OSError, ValueError):
return None
found = [DEVICE.findall(line) for line in lines]
if any(len(names) != 1 for names in found):
return None
return {(int(major, 16), int(minor, 16), int(inode)) for [(major, minor, inode)] in found}
def holders() -> set[tuple[int, int, int]] | None:
"""Returns:
lock_lines of PROC, read under a shared flock of this process on
<root>/holds; None unless they show that flock.
"""
try:
fd = nofollow.directory(watch.ROOT, nofollow.parts_below(watch.ROOT, queue.HOLDS), False)
except OSError:
return None
try:
fcntl.flock(fd, fcntl.LOCK_SH | fcntl.LOCK_NB)
found = lock_lines(PROC)
return found if found is not None and key(os.fstat(fd)) in found else None
except OSError:
return None
finally:
os.close(fd)
def lock_file(tag: object) -> tuple[int, int, int] | str | None:
"""Returns:
HELD where a flock on the lock file of tag would block, GONE where it
is missing, else its key; None for no lock tag or a file that cannot
be opened, is no regular file with one link or changed more than
CREATED_SECONDS away from its tag's time.
"""
if not tagged(tag):
return None
try:
fd = nofollow.open_below(watch.ROOT, queue.HOLDS / f"{tag}.lock", os.O_RDONLY, 0)
except FileNotFoundError:
return GONE
except OSError:
return None
try:
info = os.fstat(fd)
fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
except BlockingIOError:
return HELD
except OSError:
return None
finally:
os.close(fd)
made = abs(info.st_ctime_ns - int(tag.split("-")[1])) <= CREATED_SECONDS * 1_000_000_000
return key(info) if made else None
def judge(jobs: list[dict]) -> dict[str, list[dict]]:
"""Returns:
The jobs by verdict: DEAD, HELD or UNCLEAR.
"""
files = {job["lock"]: lock_file(job["lock"]) for job in jobs}
found = holders()
groups: dict[str, list[dict]] = {DEAD: [], HELD: [], UNCLEAR: []}
for job in jobs:
name = files[job["lock"]]
if name == HELD or (found is not None and name in found):
groups[HELD].append(job)
else:
groups[UNCLEAR if found is None or name is None else DEAD].append(job)
return groups
def sound(entry: object) -> bool:
return (isinstance(entry, dict) and all(isinstance(entry.get(field), str) for field in ("id", "status", "kind", "lock"))
and entry["status"] in STATUSES)
def snapshot() -> tuple[dict[str, list[dict]], int]:
"""Returns:
(the jobs of the queue by verdict, the number of malformed entries).
"""
with queue.store() as data:
jobs = [entry for entry in data["entries"] if sound(entry)]
groups = judge(jobs)
return groups, len(data["entries"]) - len(jobs)
def idents(entries: list[dict]) -> list[str]:
return [entry["id"] for entry in entries]
def standing(state: dict) -> list[dict]:
"""Returns:
The round's jobs still queued under their lock tag and dead; state
takes the ids of the queue's held and unclear jobs.
"""
groups, _ = snapshot()
state.update(held=idents(groups[HELD]), unclear=idents(groups[UNCLEAR]))
dead = pairs(groups[DEAD])
return [job for job in state["jobs"] if (job["id"], job["lock"]) in dead]
def host_writable() -> bool:
try:
watch.KEPT.mkdir(mode=0o700, parents=True, exist_ok=True)
with nofollow.locked(watch.KEPT, STATE_LOCK, 0o600, None):
return True
except OSError:
return False
def pairs(jobs: list[dict]) -> set[tuple[str, str]]:
return {(job["id"], job["lock"]) for job in jobs}
def cancel(state: dict, doomed: list[dict], clock: Clock) -> dict[str, list[str]] | str:
"""Returns:
The stall found once the queue lock is held, and nothing deleted;
else the ids of the round's jobs still queued under the same lock tag
by outcome: deleted (dead and among doomed, out of the queue), else
left alone.
"""
named, chosen = pairs(state["jobs"]), pairs(doomed)
with queue.store() as data:
problem = stalled(state, clock.time())
if problem is not None:
return problem
publish(state)
mine = [entry for entry in data["entries"] if sound(entry) and (entry["id"], entry["lock"]) in named]
groups = judge(mine)
gone = [entry for entry in groups[DEAD] if (entry["id"], entry["lock"]) in chosen]
data["entries"] = [entry for entry in data["entries"] if entry not in gone]
return outcome(groups, gone)
def outcome(groups: dict[str, list[dict]], gone: list[dict]) -> dict[str, list[str]]:
found = {"deleted": idents(gone), "held by a living process, not deleted": idents(groups[HELD]),
"unclear, not deleted": idents(groups[UNCLEAR]),
"not deleted, alive at the security": [entry["id"] for entry in groups[DEAD] if entry not in gone]}
return {word: ids for word, ids in found.items() if ids or word == "deleted"}
def send(cls: str, to: list[str], text: str) -> None:
radio.transmit(cls, to, text, STATION)
def capped(ids: list[str]) -> str:
hidden = len(ids) - NAMES_SHOWN
shown = ", ".join(council().short(radio.clip(i), NAME_WIDTH) for i in ids[:NAMES_SHOWN])
return (shown or "none") + (f", +{hidden} more" if hidden > 0 else "")
def head(state: dict) -> str:
return f"queue cleanup {state['round']}, chair {council().short(radio.clip(state['chair']), NAME_WIDTH)}"
def unclear(state: dict) -> str:
return f"; unclear, not deleted: {capped(state['unclear'])}" if state["unclear"] else ""
def ask(state: dict) -> str:
odd = f", {state['malformed']} malformed entries" if state["malformed"] else ""
return (
f"{head(state)}: no living process holds the lock of {len(state['jobs'])} queue jobs: {capped(idents(state['jobs']))}. "
f"A security at {radio.clock(state['deadline'])} names those still dead, deleted {state['window'] / 60:g} min after it; "
f"nothing is signalled. {len(state['held'])} held and {len(state['unclear'])} unclear jobs{odd} stay; "
f"python3 {SELF} status lists them."
)
def warn(state: dict) -> str:
return (
f"{head(state)}: still no living lock holder, deleted at {radio.clock(state['deadline'])}: {capped(state['delete'])}; "
f"held by a living process, not deleted: {capped(state['held'])}{unclear(state)}."
)
ALARMS = {"need": ("pan-pan", ask), "warn": ("security", warn)}
def alarm(state: dict, phase: str, clock: Clock) -> str | None:
"""Args:
state: the round; its deadline moves to the window's end.
phase: need (pan-pan) or warn (security).
clock: time, monotonic and sleep.
Returns:
Why the round must stop (radio silence or a stall on a poll), else
None once the window has run on both clocks.
"""
cls, text = ALARMS[phase]
window = state["window"]
state.update(phase=phase, deadline=clock.time() + window, beat=clock.time())
publish(state)
send(cls, [], text(state))
wall, mono = clock.time(), clock.monotonic()
state.update(deadline=wall + window, beat=wall)
publish(state)
while True:
problem = hushed()
left = max(wall + window - clock.time(), mono + window - clock.monotonic())
if problem is not None or left <= 0:
return problem
clock.sleep(min(POLL_SECONDS, left))
problem = stalled(state, clock.time())
if problem is not None:
return problem
state["beat"] = clock.time()
publish(state)
def stamp(ts: float) -> str:
return datetime.datetime.fromtimestamp(ts, datetime.UTC).isoformat(timespec="seconds")
def register(state: dict, status: str) -> None:
try:
with rasci.database() as tasks:
entry = tasks.setdefault(TASK, {"roles": {}})
entry.update(
objective=OBJECTIVE, description=state["result"], worker=WORKER, status=status,
roles={state["chair"]: "A", WORKER: "R"}, start=stamp(state["start"]), end=stamp(state["end"]),
)
except (OSError, SystemExit) as error:
print(f"queue-cleanup: the round is missing from the rasci register ({error})", file=sys.stderr)
def listed(ids: list[str]) -> str:
return ", ".join(ids) if ids else "none"
def finish(state: dict, named: dict[str, list[str]], status: str) -> int:
handled = {job_id for ids in named.values() for job_id in ids}
outcome = {**named, "left the queue": [job["id"] for job in state["jobs"] if job["id"] not in handled]}
result = f"queue cleanup {state['round']}: " + "; ".join(f"{word} {listed(ids)}" for word, ids in outcome.items())
state.update(phase="done", result=result, end=time.time())
publish(state)
counts = "; ".join(f"{word} {len(ids)}" for word, ids in outcome.items())
send("post", [state["chair"]], f"queue cleanup {state['round']}: {counts}; python3 {SELF} status lists the ids")
register(state, status)
print(result)
return 0
def halt(state: dict, problem: str) -> int:
return finish(state, {f"not deleted, {problem}": idents(state["jobs"])}, "stopped")
def abort(state: dict, error: BaseException) -> int:
why = f"queue cleanup {state['round']}: the round failed ({type(error).__name__}); nothing more is deleted"
state.update(phase="stopped", result=why, end=time.time())
with contextlib.suppress(Exception):
publish(state)
with contextlib.suppress(Exception):
send("post", [state["chair"]], why)
with contextlib.suppress(Exception):
register(state, "stopped")
print(f"queue-cleanup: {why}: {error!r}", file=sys.stderr)
return FAILED
def claim(state: dict) -> bool:
with nofollow.locked(watch.KEPT, STATE_LOCK, 0o600, None):
if busy(host_state(), state["start"]):
return False
save(state)
return True
def run_round(window: float, clock: Clock) -> int:
"""Args:
window: seconds each of the two windows lasts.
clock: time, monotonic and sleep; the time module outside tests.
Returns:
The exit code.
"""
session, chair = top_chair()
if session is None:
print(f"queue-cleanup: {chair}; no one is accountable, nothing is deleted")
return 0
groups, malformed = snapshot()
if not groups[DEAD]:
print(f"queue-cleanup: no queue job without a living lock holder ({len(groups[HELD])} held, "
f"{len(groups[UNCLEAR])} unclear, {malformed} malformed); nothing sent")
return 0
now = clock.time()
state = {
"round": radio.clock(now), "start": now, "beat": now, "phase": "need", "deadline": now + window,
"window": window, "chair": chair, "malformed": malformed, "held": idents(groups[HELD]),
"unclear": idents(groups[UNCLEAR]),
"jobs": [{field: job[field] for field in ("id", "lock", "status", "kind")} for job in groups[DEAD]],
}
if not claim(state):
print("queue-cleanup: another round runs; status shows it", file=sys.stderr)
return RUNNING
try:
return proceed(state, clock)
except Superseded:
print("queue-cleanup: a newer round took over this one; nothing more is deleted", file=sys.stderr)
return 0
except (Exception, SystemExit) as error: # noqa: BLE001
return abort(state, error)
def proceed(state: dict, clock: Clock) -> int:
problem = hushed()
if problem is None:
problem = alarm(state, "need", clock)
if problem is not None:
return halt(state, problem)
state.update(phase="warn", beat=clock.time())
publish(state)
doomed = standing(state)
if not doomed:
return close(state, doomed, clock)
state["delete"] = idents(doomed)
problem = alarm(state, "warn", clock)
if problem is not None:
return halt(state, problem)
state.update(phase="closing", beat=clock.time())
publish(state)
return close(state, doomed, clock)
def close(state: dict, doomed: list[dict], clock: Clock) -> int:
session, chair = top_chair()
if session is not None:
state["chair"] = chair
problem = chair if session is None else hushed() or stalled(state, clock.time())
if problem is not None and doomed:
return halt(state, problem)
found = cancel(state, doomed, clock)
return halt(state, found) if isinstance(found, str) else finish(state, found, "done")
def cmd_status(_: argparse.Namespace) -> int:
state = host_state()
if not alive(state, time.time()):
print(f"no open round; last: {state['result']}" if "result" in state else "no open round")
return 0
print(f"round {state['round']} {state['phase']} until {radio.clock(state['deadline'])}")
for job in state["jobs"]:
print(f"{job['id']} {job['status']} {job['kind']} delete")
for verdict in (HELD, UNCLEAR):
for job_id in state[verdict]:
print(f"{job_id} {verdict}")
return 0
def cmd_request(_: argparse.Namespace) -> int:
if not UNIT.is_file():
fail(f"{UNIT} is missing, so no round would start; make install enables it")
state, now = host_state(), time.time()
if busy(state, now):
fail(f"round {state['round']} runs; status shows it")
if "end" in state and now - state["end"] < SPACING_SECONDS:
fail(f"the last round ended at {radio.clock(state['end'])}; request again from {radio.clock(state['end'] + SPACING_SECONDS)}")
nofollow.append(watch.ROOT, REQUEST, f"{now}\n", 0o600)
print(f"requested a queue-cleanup round through {REQUEST}")
return 0
def cmd_run(args: argparse.Namespace) -> int:
if os.environ.get("CLAUDE_CODE_SESSION_ID"):
fail("run starts from kvb-org-queue-cleanup.service, never in an agent session; agents use request")
if args.window < WINDOW_FLOOR:
fail(f"--window must be at least {WINDOW_FLOOR} seconds")
if not host_writable():
fail(f"cannot write {watch.KEPT}; run starts outside the sandbox only")
return run_round(args.window, time)
def cli() -> argparse.ArgumentParser:
parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
sub = parser.add_subparsers(dest="cmd", required=True)
run = sub.add_parser("run")
run.add_argument("--window", type=float, required=True)
sub.add_parser("status")
sub.add_parser("request")
return parser
def main(argv: list[str]) -> int:
args = cli().parse_args(argv)
refused = watch.guard_root(hook=False)
if refused is not None:
return refused
return {"run": cmd_run, "status": cmd_status, "request": cmd_request}[args.cmd](args)
if __name__ == "__main__":
sys.exit(main(sys.argv[1:]))
@@ -0,0 +1,8 @@
[Unit]
Description=kvb-org-queue-cleanup round on cleanup.py request
[Path]
PathModified=/tmp/agents-council/queue-cleanup.request
[Install]
WantedBy=paths.target
@@ -0,0 +1,7 @@
[Unit]
Description=One kvb-org-queue-cleanup round: pan-pan, security, delete the jobs no living process holds
[Service]
Type=oneshot
ExecStart=/usr/bin/python3 -I %h/.claude/skills/kvb-org-queue-cleanup/scripts/cleanup.py run --window 900
TimeoutStartSec=2400
@@ -0,0 +1,9 @@
[Unit]
Description=kvb-org-queue-cleanup round at 00:00 and 12:00 local time
[Timer]
OnCalendar=*-*-* 00,12:00:00
Persistent=false
[Install]
WantedBy=timers.target
+4 -3
View File
@@ -1,8 +1,8 @@
---
name: kvb-org-queue
description: >
Run every heavy job (gate, deploy, build, swarm) through a deterministic
queue instead of asking the chair: hotfix first, then register rank.
Queue every heavy job (gate, deploy, build, swarm), never ask the chair:
hotfix first, then rank.
---
`${CLAUDE_SKILL_DIR}/scripts/queue.py` replaces asking the chair for heavy
@@ -74,7 +74,8 @@ learned values, `cancel` removes your own waiting job.
only exceptions such as a stuck queue or a hung job (`capacity.md` of
`kvb-org-organize`).
- A running job is never cut short for the queue; only the operator orders
one down, and `kvb-git-hotfix` may void a gate.
one down, and `kvb-git-hotfix` may void a gate. A `kvb-org-queue-cleanup`
round deletes only entries whose lock no living process holds.
- With `queue_enforce: true`, the `block` hook (PreToolUse on Bash) refuses
`heavy_commands`, image builds, `make install|verify-*`, tests beyond one
file and `gate.py run|hold` outside `queue.py hold`.
+2 -2
View File
@@ -152,7 +152,7 @@ def items() -> list[dict]:
for line in lines:
try:
row = json.loads(line)
except ValueError:
except (ValueError, RecursionError):
continue
if isinstance(row, dict) and isinstance(row.get("ts"), int | float):
rows.append(row)
@@ -225,7 +225,7 @@ def for_me(row: dict, name: str, sid: str) -> bool:
def read_json(path: Path) -> dict:
try:
data = json.loads(organize("nofollow").read_text(ROOT, path))
except (OSError, ValueError):
except (OSError, ValueError, RecursionError):
return {}
return data if isinstance(data, dict) else {}
+724
View File
@@ -0,0 +1,724 @@
"""Exercise kvb-org-queue-cleanup: jobs whose hold lock no living process holds leave the queue after a pan-pan and a security, every other job stays."""
from __future__ import annotations
import contextlib
import importlib.util
import io
import json
import os
import re
import subprocess
import sys
import tempfile
import time
import unittest
from collections.abc import Callable
from pathlib import Path
from unittest import mock
REPO_ROOT = Path(__file__).resolve().parents[1]
CLEANUP = REPO_ROOT / "skills" / "kvb-org-queue-cleanup" / "scripts" / "cleanup.py"
SYSTEMD = CLEANUP.parents[1] / "systemd"
SESSIONS = {"s0": "Main", "s1": "Skills", "s2": "IAM"}
WINDOW = 900
WORKER = "systemd kvb-org-queue-cleanup"
HOLD = (
"import ctypes, fcntl, struct, subprocess, sys\n"
"ctypes.CDLL(None).prctl(1, 9)\n"
"handle = open(sys.argv[1], 'a')\n"
"if sys.argv[2] == 'posix':\n"
" fcntl.lockf(handle, fcntl.LOCK_EX)\n"
"elif sys.argv[2] == 'ofd':\n"
" fcntl.fcntl(handle, fcntl.F_OFD_SETLK, struct.pack('hhqqi4x', fcntl.F_WRLCK, 0, 0, 0, 0))\n"
"else:\n"
" fcntl.flock(handle, fcntl.LOCK_EX)\n"
"if sys.argv[2] == 'child':\n"
" subprocess.Popen([sys.executable, '-c', 'import sys; sys.stdin.read()'], pass_fds=[handle.fileno()])\n"
"print('ready', flush=True)\n"
"sys.stdin.read()\n"
)
FORGED = {"session": "s9", "name": "Main", "pid": 1, "pid_start": "1", "child": 2, "child_start": "2", "beat": 0,
"host": True, "granted": 0, "workdir": "/elsewhere"}
def _load(path: Path):
spec = importlib.util.spec_from_file_location(f"{path.stem}_under_test", path)
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
return module
def ids_of(entries: list[dict]) -> list[str]:
return [entry["id"] for entry in entries]
class Killed(BaseException):
pass
class FakeClock:
def __init__(self, wall: float, hook: Callable[[float], None]):
self.wall, self.mono, self.hook = wall, 0.0, hook
def time(self) -> float:
return self.wall
def monotonic(self) -> float:
return self.mono
def sleep(self, seconds: float) -> None:
self.hook(seconds)
self.wall += seconds
self.mono += seconds
class TestQueueCleanup(unittest.TestCase):
def setUp(self):
tmp = tempfile.TemporaryDirectory()
self.addCleanup(tmp.cleanup)
base = Path(tmp.name).resolve()
self.root = base / "council"
self.holds = self.root / "holds"
self.holds.mkdir(parents=True, mode=0o700)
self.home = base / "home"
sessions = self.home / ".claude" / "sessions"
sessions.mkdir(parents=True)
for sid, name in SESSIONS.items():
(sessions / f"{sid}.json").write_text(json.dumps({"sessionId": sid, "name": name}), encoding="utf-8")
self.kept = self.home / ".local" / "state" / "agents-council"
self.kept.mkdir(parents=True, mode=0o700)
env = mock.patch.dict(os.environ, {
"AGENTS_COUNCIL_ROOT": str(self.root), "HOME": str(self.home), "AGENTS_COUNCIL_CONFIG": str(base / "config.json"),
})
env.start()
self.addCleanup(env.stop)
os.environ.pop("CLAUDE_CODE_SESSION_ID", None)
self.cleanup = _load(CLEANUP)
self.last = WINDOW // self.cleanup.POLL_SECONDS - 1
self.clock = FakeClock(float(int(time.time())), self.tick)
for module in (self.cleanup, self.cleanup.radio):
patch = mock.patch.object(module, "time", self.clock)
patch.start()
self.addCleanup(patch.stop)
self.actions: dict[tuple[str, int], Callable[[], object]] = {}
self.jumps: dict[tuple[str, int], tuple[float, float]] = {}
self.slept: list[tuple[str, float]] = []
self.seq = 0
self.chair("s0")
def tick(self, seconds: float) -> None:
phase = self.state()["phase"]
step = (phase, sum(p == phase for p, _ in self.slept))
self.slept.append((phase, seconds))
if step in self.actions:
self.actions[step]()
if step in self.jumps:
wall, mono = self.jumps[step]
self.clock.wall += wall
self.clock.mono += mono
def state(self) -> dict:
return json.loads((self.kept / "queue-cleanup.json").read_text(encoding="utf-8"))
def write_state(self, state: dict) -> None:
(self.kept / "queue-cleanup.json").write_text(json.dumps(state), encoding="utf-8")
def chair(self, session: str) -> None:
office = self.root / "chair"
office.mkdir(exist_ok=True)
claim = {"office": "chair", "session": session, "runtime": "claude-code", "name": SESSIONS.get(session),
"beat": time.time(), "work": "chair"}
(office / "claim.json").write_text(json.dumps(claim), encoding="utf-8")
def entry(self, status: str, born: float = 0) -> dict:
self.seq += 1
return {
"id": f"test-{self.seq}", "seq": self.seq, "kind": "test", "task": "work/a", "need_gb": 0, "hotfix": None,
"workdir": f"/w{self.seq}", "session": "s1", "name": "Skills", "lock": f"{self.seq}-{time.time_ns() + int(born * 1e9)}",
"host": False, "pid": 1, "pid_start": None, "child": None, "child_start": None, "status": status,
"enqueued": time.time(), "granted": None, "cmd": "true",
}
def lock_file(self, entry: dict) -> Path:
return self.holds / f"{entry['lock']}.lock"
def holder(self, path: Path, mode: str = "flock") -> subprocess.Popen:
proc = subprocess.Popen([sys.executable, "-c", HOLD, str(path), mode], stdin=subprocess.PIPE, stdout=subprocess.PIPE, text=True)
self.addCleanup(proc.wait)
self.addCleanup(proc.stdin.close)
self.addCleanup(proc.stdout.close)
self.assertEqual(proc.stdout.readline(), "ready\n")
return proc
def kill(self, proc: subprocess.Popen) -> None:
proc.kill()
proc.wait()
def held(self, status: str, mode: str = "flock") -> dict:
entry = self.entry(status)
self.holder(self.lock_file(entry), mode)
return entry
def dead(self, status: str, born: float = 0) -> dict:
entry = self.entry(status, born)
self.kill(self.holder(self.lock_file(entry)))
return entry
def verdict(self, job: dict) -> str:
[found] = [verdict for verdict, jobs in self.cleanup.judge([job]).items() if jobs]
return found
def queue(self, *entries: object) -> None:
(self.root / "queue.json").write_text(json.dumps({"seq": self.seq, "entries": list(entries)}), encoding="utf-8")
def entries(self) -> list[object]:
return json.loads((self.root / "queue.json").read_text(encoding="utf-8"))["entries"]
def ids(self) -> list[str]:
return ids_of(self.entries())
def rows(self) -> list[dict]:
news = self.root / "news.jsonl"
return [json.loads(line) for line in news.read_text(encoding="utf-8").splitlines()] if news.exists() else []
def classes(self) -> list[str]:
return [row["class"] for row in self.rows()]
def register(self) -> dict:
return json.loads((self.root / "rasci.json").read_text(encoding="utf-8"))["queue-cleanup"]
def cli(self, *args: str) -> tuple[int, str, str]:
out, err = io.StringIO(), io.StringIO()
with contextlib.redirect_stdout(out), contextlib.redirect_stderr(err):
try:
code = self.cleanup.main(list(args))
except SystemExit as stop:
code = stop.code
return code, out.getvalue(), err.getvalue()
def silence(self) -> None:
hush = {"since": self.clock.wall, "until": self.clock.wall + 7200, "by": "IAM", "text": "disk full"}
(self.root / "radio-silence.json").write_text(json.dumps(hush), encoding="utf-8")
def round(self) -> tuple[int, str]:
out = io.StringIO()
with contextlib.redirect_stdout(out), contextlib.redirect_stderr(io.StringIO()):
code = self.cleanup.run_round(WINDOW, self.clock)
return code, out.getvalue()
def nothing(self, held: int, unclear: int, malformed: int) -> tuple[int, str]:
return 0, f"queue-cleanup: no queue job without a living lock holder ({held} held, {unclear} unclear, {malformed} malformed); nothing sent\n"
def waited(self, phase: str) -> float:
return sum(seconds for p, seconds in self.slept if p == phase)
def ends(self, out: str, tail: str) -> None:
self.assertTrue(out.endswith(tail + "\n"), out)
def at_the_close(self, action: Callable[[], None]) -> tuple[int, str, str]:
original = self.cleanup.top_chair
def chair_then_act() -> tuple[str | None, str]:
if (self.kept / "queue-cleanup.json").exists() and self.state()["phase"] == "closing":
action()
return original()
out, err = io.StringIO(), io.StringIO()
with mock.patch.object(self.cleanup, "top_chair", chair_then_act), contextlib.redirect_stdout(out), contextlib.redirect_stderr(err):
code = self.cleanup.run_round(WINDOW, self.clock)
return code, out.getvalue(), err.getvalue()
def install_unit(self) -> Path:
unit = self.home / ".config" / "systemd" / "user" / "kvb-org-queue-cleanup.path"
unit.parent.mkdir(parents=True)
unit.write_text((SYSTEMD / "kvb-org-queue-cleanup.path").read_text(encoding="utf-8"), encoding="utf-8")
return unit
def test_a_vacant_or_stale_chair_deletes_nothing_and_sends_nothing(self):
entries = [self.dead("waiting"), self.dead("active")]
self.queue(*entries)
for name, vacate in (("stale", lambda: self.chair("gone")), ("vacant", (self.root / "chair" / "claim.json").unlink)):
with self.subTest(name):
vacate()
code, out = self.round()
self.assertEqual(code, 0)
self.ends(out, "; no one is accountable, nothing is deleted")
self.assertEqual((self.entries(), self.rows()), (entries, []))
def test_jobs_whose_hold_died_or_whose_lock_file_is_gone_leave_the_queue_after_a_pan_pan_and_a_security(self):
waiting, active, gone, alive = self.dead("waiting"), self.dead("active"), self.entry("waiting"), self.held("active")
self.queue(waiting, active, gone, alive)
code, out = self.round()
self.assertEqual((code, self.entries()), (0, [alive]))
self.assertFalse((self.kept / "queue-stops.json").exists())
self.assertEqual(
[(row["class"], row["from"], row["to"]) for row in self.rows()],
[("pan-pan", "queue-cleanup", []), ("security", "queue-cleanup", []), ("post", "queue-cleanup", ["Main"])],
)
pan_pan, security, post = (row["text"] for row in self.rows())
self.assertRegex(
pan_pan,
r"^queue cleanup \d\d:\d\d, chair Main: no living process holds the lock of 3 queue jobs: test-1, test-2, test-3\. "
r"A security at \d\d:\d\d names those still dead, deleted 15 min after it; nothing is signalled\. "
r"1 held and 0 unclear jobs stay; ",
)
self.assertRegex(
security,
r"^queue cleanup \d\d:\d\d, chair Main: still no living lock holder, deleted at \d\d:\d\d: test-1, test-2, test-3; "
r"held by a living process, not deleted: test-4\.$",
)
self.assertTrue(post.endswith(f": deleted 3; left the queue 0; python3 {CLEANUP} status lists the ids"), post)
self.ends(out, ": deleted test-1, test-2, test-3; left the queue none")
def test_a_hold_that_died_while_its_child_keeps_the_inherited_lock_is_not_deleted(self):
inherited = self.entry("active")
self.kill(self.holder(self.lock_file(inherited), "child"))
dead = self.dead("waiting")
self.queue(inherited, dead)
self.assertEqual(self.cleanup.lock_file(inherited["lock"]), self.cleanup.HELD)
code, out = self.round()
self.assertEqual((code, self.entries()), (0, [inherited]))
self.assertTrue(self.rows()[1]["text"].endswith(": test-2; held by a living process, not deleted: test-1."), self.rows()[1]["text"])
self.ends(out, ": deleted test-2; left the queue none")
def test_waiting_and_active_jobs_with_a_living_holder_of_any_lock_type_stay_whatever_their_entry_says(self):
living = [{**self.held(status, mode), **FORGED} for status, mode in
(("waiting", "flock"), ("active", "flock"), ("waiting", "posix"), ("active", "ofd"))]
self.queue(*living)
self.assertEqual(self.round(), self.nothing(4, 0, 0))
self.assertEqual((self.entries(), self.rows()), (living, []))
dead = self.dead("waiting")
self.queue(*living, dead)
code, out = self.round()
self.assertEqual((code, self.entries()), (0, living))
self.assertTrue(self.rows()[1]["text"].endswith("; held by a living process, not deleted: test-1, test-2, test-3, test-4."))
self.ends(out, ": deleted test-5; left the queue none")
def test_a_lock_file_swapped_linked_redirected_unreadable_or_odd_is_never_dead(self):
swapped = self.entry("waiting", born=-60)
self.holder(self.lock_file(swapped))
fresh = self.holds / "fresh"
fresh.write_text("", encoding="utf-8")
fresh.rename(self.lock_file(swapped))
left, right = self.held("waiting"), self.held("active")
self.lock_file(left).rename(self.holds / "spare")
self.lock_file(right).rename(self.lock_file(left))
(self.holds / "spare").rename(self.lock_file(right))
redirected = self.dead("waiting")
self.lock_file(redirected).rename(self.root / "target.lock")
self.lock_file(redirected).symlink_to(self.root / "target.lock")
linked = self.dead("waiting")
os.link(self.lock_file(linked), self.root / "second.lock")
unreadable = self.dead("active")
self.lock_file(unreadable).chmod(0)
directory, fifo = self.entry("waiting"), self.entry("active")
self.lock_file(directory).mkdir()
os.mkfifo(self.lock_file(fifo))
future = self.dead("waiting", born=60)
untagged = {**self.dead("waiting"), "lock": "1-2-3"}
cases = {
"swapped after its hold started": (swapped, "unclear"), "exchanged with another living one": (left, "held"),
"the other one exchanged": (right, "held"), "a symlink": (redirected, "unclear"), "a second link": (linked, "unclear"),
"unreadable": (unreadable, "unclear"), "a directory": (directory, "unclear"), "a fifo": (fifo, "unclear"),
"a tag from the future": (future, "unclear"), "no lock tag": (untagged, "unclear"),
}
for name, (job, expected) in cases.items():
with self.subTest(name):
self.assertEqual(self.verdict(job), expected)
jobs = [job for job, _ in cases.values()]
self.queue(*jobs)
self.assertEqual(self.round(), self.nothing(2, 8, 0))
self.assertEqual((self.entries(), self.rows()), (jobs, []))
def test_proc_locks_unreadable_unparsable_or_without_the_rounds_own_flock_makes_nothing_dead(self):
job = self.dead("waiting")
self.queue(job)
info = os.stat(self.holds)
own = f"1: FLOCK ADVISORY READ 1 {os.major(info.st_dev):02x}:{os.minor(info.st_dev):02x}:{info.st_ino} 0 EOF\n"
fakes = {
"missing": None, "a directory": "", "empty": "", "without the round's flock": "1: FLOCK ADVISORY WRITE 9 00:01:5 0 EOF\n",
"with a line without an inode": own + "2: FLOCK ADVISORY WRITE 9 <none>:0 0 EOF\n", "with garbage": own + "garbage\n",
}
for index, (name, text) in enumerate(fakes.items()):
with self.subTest(name):
proc = self.home / f"proc-{index}"
proc.mkdir()
if name == "a directory":
(proc / "locks").mkdir()
elif text is not None:
(proc / "locks").write_text(text, encoding="utf-8")
with mock.patch.object(self.cleanup, "PROC", proc):
self.assertEqual(self.round(), self.nothing(0, 1, 0))
self.assertEqual((self.entries(), self.rows()), ([job], []))
control = self.home / "control"
control.mkdir()
(control / "locks").write_text(own, encoding="utf-8")
with mock.patch.object(self.cleanup, "PROC", control):
self.assertEqual(self.verdict(job), "dead")
def test_lock_lines_name_every_line_of_any_type_pid_or_waiting_state_and_refuse_any_other(self):
proc = self.home / "proc"
proc.mkdir()
first = "1: FLOCK ADVISORY WRITE 100 00:2d:555 0 EOF\n"
(proc / "locks").write_text(
first
+ "1: -> FLOCK ADVISORY WRITE 101 00:2d:556 0 EOF\n"
"2: POSIX ADVISORY WRITE 102 00:2d:557 0 0\n"
"3: OFDLCK ADVISORY WRITE -1 00:2d:558 10 10\n"
"4: FLOCK ADVISORY READ 0 fd:01:777 0 EOF\n"
"5: LEASE ACTIVE READ 105 103:02:888 0 EOF\n"
"6: POSIX ADVISORY WRITE 106 00:2d:12345678901 0 EOF\n",
encoding="utf-8",
)
self.assertEqual(
self.cleanup.lock_lines(proc),
{(0, 0x2D, 555), (0, 0x2D, 556), (0, 0x2D, 557), (0, 0x2D, 558), (0xFD, 1, 777), (0x103, 2, 888), (0, 0x2D, 12345678901)},
)
odd = {"garbage": "garbage\n", "no inode": "7: FLOCK ADVISORY WRITE 106 <none>:0 0 EOF\n", "blank": "\n",
"two devices": "8: FLOCK ADVISORY WRITE 1 00:2d:1 00:2d:2 EOF\n"}
for name, line in odd.items():
with self.subTest(name):
(proc / "locks").write_text(first + line, encoding="utf-8")
self.assertIsNone(self.cleanup.lock_lines(proc))
(proc / "locks").write_text("", encoding="utf-8")
self.assertEqual(self.cleanup.lock_lines(proc), set())
(proc / "locks").unlink()
self.assertIsNone(self.cleanup.lock_lines(proc))
def test_a_job_that_dies_after_the_snapshot_waits_for_the_next_round_and_one_living_again_stays(self):
revived, later = self.dead("waiting"), self.entry("active")
hold = self.holder(self.lock_file(later))
self.queue(revived, later)
self.actions[("need", 0)] = lambda: (self.holder(self.lock_file(revived)), self.kill(hold))
code, out = self.round()
self.assertEqual((code, self.entries(), self.classes()), (0, [revived, later], ["pan-pan", "post"]))
self.ends(out, ": deleted none; held by a living process, not deleted test-1; left the queue none")
def test_a_job_alive_at_the_security_and_dead_again_at_the_end_is_not_deleted(self):
flapping, other = self.dead("waiting"), self.dead("waiting")
self.queue(flapping, other)
holds = {}
self.actions[("need", 0)] = lambda: holds.update(proc=self.holder(self.lock_file(flapping)))
self.actions[("warn", 0)] = lambda: self.kill(holds["proc"])
code, out = self.round()
self.assertEqual((code, self.entries()), (0, [flapping]))
self.assertRegex(self.rows()[1]["text"], r": still no living lock holder, deleted at \d\d:\d\d: test-2; held by a living process, not deleted: test-1\.$")
self.ends(out, ": deleted test-2; not deleted, alive at the security test-1; left the queue none")
def test_a_job_living_again_replaced_or_gone_by_the_delete_is_left_alone(self):
revived, replaced, gone = self.dead("waiting"), self.dead("active"), self.dead("waiting")
self.queue(revived, replaced, gone)
swap = {**self.dead("active"), "id": replaced["id"]}
def act() -> None:
self.holder(self.lock_file(revived))
self.queue(revived, swap)
code, out, _ = self.at_the_close(act)
self.assertEqual((code, self.entries()), (0, [revived, swap]))
self.ends(out, ": deleted none; held by a living process, not deleted test-1; left the queue test-2, test-3")
def test_an_entry_replaced_in_the_first_window_is_no_candidate_for_the_security(self):
old = self.dead("waiting")
self.queue(old)
replaced = {**self.dead("waiting"), "id": old["id"]}
self.actions[("need", 0)] = lambda: self.queue(replaced)
code, out = self.round()
self.assertEqual((code, self.entries(), self.classes()), (0, [replaced], ["pan-pan", "post"]))
self.ends(out, ": deleted none; left the queue test-1")
def test_a_dead_job_queued_after_the_snapshot_is_neither_named_nor_deleted(self):
old, new = self.dead("waiting"), self.dead("waiting")
self.queue(old)
self.actions[("need", 0)] = lambda: self.queue(old, new)
code, out = self.round()
self.assertEqual((code, self.entries()), (0, [new]))
self.ends(out, ": deleted test-1; left the queue none")
self.assertNotIn("test-2", " ".join(row["text"] for row in self.rows()))
def test_malformed_entries_stay_counted_and_block_no_round(self):
job = self.dead("waiting")
odd = [1, {"id": "x"}, {"seq": 2, "status": "waiting", "kind": "test", "lock": "2-2"},
{"id": 5, "status": "waiting", "kind": "test", "lock": "3-3"}, {"id": "y", "status": "waiting", "lock": "4-4"},
{"id": "z", "status": "waiting", "kind": "test"}, {**self.dead("waiting"), "status": "done"}]
self.queue(job, *odd)
code, out = self.round()
self.assertEqual((code, self.entries()), (0, odd))
self.ends(out, f": deleted {job['id']}; left the queue none")
self.assertIn(" 0 held and 0 unclear jobs, 7 malformed entries stay; ", self.rows()[0]["text"])
def test_the_alarms_stay_bounded_for_50_long_ids(self):
ids = [f"test-nostack-{1000 + n}-" + "x" * 50 for n in range(50)]
state = {"round": "12:00", "chair": "Main", "jobs": [{"id": i, "lock": f"{n}-{n}", "status": "waiting", "kind": "test"}
for n, i in enumerate(ids)], "held": ids, "unclear": ids, "malformed": 3, "deadline": time.time() + WINDOW,
"window": WINDOW, "delete": ids}
for text in (self.cleanup.ask(state), self.cleanup.warn(state)):
with self.subTest(text=text[:40]):
self.assertLess(len(text), self.cleanup.watch.CLIP)
self.assertIn(", +44 more", text)
def test_each_window_lasts_its_length_on_both_clocks_from_its_own_alarm(self):
cases = (
("wall ahead within the stall limit", {("need", 0): (100, 0)}, 900, 900),
("wall back", {("need", 0): (-600, 0)}, 1500, 900),
("monotonic ahead", {("need", 0): (0, 2000)}, 900, 900),
("monotonic ahead in the second window", {("warn", 0): (0, 2000)}, 900, 900),
)
for name, jumps, need, warn in cases:
with self.subTest(name):
self.slept.clear()
self.jumps = jumps
self.queue(self.dead("waiting"))
self.assertEqual(self.round()[0], 0)
self.assertGreaterEqual(self.waited("need"), need)
self.assertGreaterEqual(self.waited("warn"), warn)
self.assertEqual(self.entries(), [])
def test_a_killed_round_counts_as_dead_and_blocks_no_next_one(self):
self.queue(self.dead("waiting"))
def die() -> None:
raise Killed
self.actions[("need", 0)] = die
with contextlib.suppress(Killed), contextlib.redirect_stdout(io.StringIO()):
self.cleanup.run_round(WINDOW, self.clock)
self.actions = {}
self.assertEqual(self.round()[0], 3)
self.clock.wall += self.cleanup.DEAD_SECONDS + 1
self.assertEqual(self.cli("status")[1], "no open round\n")
code, out = self.round()
self.assertEqual((code, self.classes().count("pan-pan")), (0, 2))
self.ends(out, ": deleted test-1; left the queue none")
def test_a_stall_above_the_dead_limit_ends_the_round_without_deleting(self):
job = self.dead("active")
self.queue(job)
self.jumps[("need", 5)] = (self.cleanup.DEAD_SECONDS + 30, 0)
code, out = self.round()
self.assertEqual((code, self.ids()), (0, [job["id"]]))
self.ends(out, f": not deleted, the round stalled for {self.cleanup.DEAD_SECONDS + 35} s {job['id']}; left the queue none")
self.assertEqual(self.register()["status"], "stopped")
def test_a_gap_of_exactly_the_dead_limit_is_no_stall(self):
self.queue(self.dead("waiting"))
self.jumps[("need", 5)] = (self.cleanup.DEAD_SECONDS - self.cleanup.POLL_SECONDS, 0)
code, out = self.round()
self.assertEqual((code, self.entries()), (0, []))
self.ends(out, ": deleted test-1; left the queue none")
def test_the_round_age_limits_are_inclusive(self):
cleanup, beat = self.cleanup, 1000.0
dead = cleanup.DEAD_SECONDS
self.assertTrue(cleanup.alive({"phase": "warn", "beat": beat}, beat + dead))
self.assertFalse(cleanup.alive({"phase": "warn", "beat": beat}, beat + dead + 1))
self.assertFalse(cleanup.alive({"phase": "closing", "beat": beat}, beat))
self.assertTrue(cleanup.busy({"phase": "closing", "beat": beat}, beat + dead))
self.assertFalse(cleanup.busy({"phase": "closing", "beat": beat}, beat + dead + 1))
self.assertFalse(cleanup.busy({"phase": "done", "beat": beat}, beat))
self.assertFalse(cleanup.busy({}, beat))
self.assertIsNone(cleanup.stalled({"beat": beat}, beat + dead))
self.assertEqual(cleanup.stalled({"beat": beat}, beat + dead + 1), f"the round stalled for {dead + 1} s")
def test_a_round_taken_over_by_a_newer_one_stops_without_deleting(self):
job = self.dead("waiting")
self.queue(job)
self.actions[("need", 2)] = lambda: self.write_state({**self.state(), "start": self.state()["start"] + 1})
err = io.StringIO()
with contextlib.redirect_stderr(err), contextlib.redirect_stdout(io.StringIO()):
code = self.cleanup.run_round(WINDOW, self.clock)
self.assertEqual((code, self.entries(), self.classes()), (0, [job], ["pan-pan"]))
self.assertEqual(err.getvalue(), "queue-cleanup: a newer round took over this one; nothing more is deleted\n")
def test_radio_silence_at_the_start_on_a_poll_or_on_the_last_poll_deletes_nothing(self):
for when in ("start", "poll", "last poll"):
with self.subTest(when):
(self.root / "radio-silence.json").unlink(missing_ok=True)
(self.root / "news.jsonl").unlink(missing_ok=True)
self.slept.clear()
job = self.dead("waiting")
self.queue(job)
if when == "start":
self.silence()
self.actions = {"poll": {("need", 1): self.silence}, "last poll": {("warn", self.last): self.silence}}.get(when, {})
code, out = self.round()
self.assertEqual((code, self.entries()), (0, [job]))
self.ends(out, f": not deleted, radio silence by IAM holds {job['id']}; left the queue none")
self.assertEqual(self.register()["status"], "stopped")
expected = {"start": ["post"], "poll": ["pan-pan", "post"], "last poll": ["pan-pan", "security", "post"]}
self.assertEqual(self.classes(), expected[when])
def test_radio_silence_found_by_the_close_deletes_nothing(self):
job = self.dead("waiting")
self.queue(job)
code, out, _ = self.at_the_close(self.silence)
self.assertEqual((code, self.entries()), (0, [job]))
self.ends(out, f": not deleted, radio silence by IAM holds {job['id']}; left the queue none")
def test_a_stall_after_the_closing_publish_deletes_nothing(self):
job = self.dead("active")
self.queue(job)
def stall() -> None:
self.clock.wall += self.cleanup.DEAD_SECONDS + 10
code, out, _ = self.at_the_close(stall)
self.assertEqual((code, self.ids()), (0, [job["id"]]))
self.ends(out, f": not deleted, the round stalled for {self.cleanup.DEAD_SECONDS + 10} s {job['id']}; left the queue none")
def test_a_newer_round_found_right_before_the_delete_deletes_nothing(self):
job = self.dead("waiting")
self.queue(job)
code, _, err = self.at_the_close(lambda: self.write_state({**self.state(), "start": self.state()["start"] + 1}))
self.assertEqual((code, self.entries(), self.classes()), (0, [job], ["pan-pan", "security"]))
self.assertEqual(err, "queue-cleanup: a newer round took over this one; nothing more is deleted\n")
def test_an_unexpected_error_after_the_claim_stops_the_round_posts_once_and_fails(self):
job = self.dead("waiting")
self.queue(job)
err = io.StringIO()
with mock.patch.object(self.cleanup, "standing", side_effect=RuntimeError("boom")), contextlib.redirect_stderr(err), \
contextlib.redirect_stdout(io.StringIO()):
code = self.cleanup.run_round(WINDOW, self.clock)
self.assertEqual((code, self.entries()), (1, [job]))
self.assertEqual(self.classes(), ["pan-pan", "post"])
self.assertRegex(self.rows()[-1]["text"], r"^queue cleanup \d\d:\d\d: the round failed \(RuntimeError\); nothing more is deleted$")
self.assertEqual((self.state()["phase"], self.register()["status"]), ("stopped", "stopped"))
self.assertIn("RuntimeError('boom')", err.getvalue())
self.assertEqual(self.round()[0], 0)
def test_a_queue_store_gone_invalid_mid_round_stops_it_posts_once_and_fails(self):
cases = {"invalid JSON": ("{", "SystemExit"), "too deep": ("[" * 100000 + "]" * 100000, "RecursionError"),
"no entries": ("{}", "KeyError"), "a list": ("[]", "TypeError")}
for name, (text, kind) in cases.items():
with self.subTest(name):
(self.root / "news.jsonl").unlink(missing_ok=True)
self.slept.clear()
self.queue(self.dead("waiting"))
self.actions = {("need", 0): lambda text=text: (self.root / "queue.json").write_text(text, encoding="utf-8")}
with contextlib.redirect_stderr(io.StringIO()), contextlib.redirect_stdout(io.StringIO()):
code = self.cleanup.run_round(WINDOW, self.clock)
self.assertEqual((code, self.classes()), (1, ["pan-pan", "post"]))
self.assertRegex(self.rows()[-1]["text"], rf"^queue cleanup \d\d:\d\d: the round failed \({kind}\); nothing more is deleted$")
self.assertEqual((self.state()["phase"], self.register()["status"]), ("stopped", "stopped"))
self.assertEqual((self.root / "queue.json").read_text(encoding="utf-8"), text)
def test_a_queue_lock_won_past_the_stall_limit_or_after_a_newer_round_deletes_nothing(self):
original = self.cleanup.queue.store
cases = {
"stall": (lambda: setattr(self.clock, "wall", self.clock.wall + self.cleanup.DEAD_SECONDS + 10),
f": not deleted, the round stalled for {self.cleanup.DEAD_SECONDS + 10} s test-1; left the queue none\n"),
"newer round": (lambda: self.write_state({**self.state(), "start": self.state()["start"] + 1}), ""),
}
for name, (wait, tail) in cases.items():
with self.subTest(name):
job = self.dead("waiting")
self.queue(job)
@contextlib.contextmanager
def slow(wait=wait):
if (self.kept / "queue-cleanup.json").exists() and self.state()["phase"] == "closing":
wait()
with original() as data:
yield data
with mock.patch.object(self.cleanup.queue, "store", slow):
code, out = self.round()
self.assertEqual((code, self.entries()), (0, [job]))
self.assertTrue(out.endswith(tail), out)
def test_status_shows_the_phase_and_each_judged_job(self):
self.queue(self.dead("waiting"), self.held("active"), {**self.dead("waiting"), "lock": "x"})
shown = {}
self.actions[("need", 1)] = lambda: shown.update(status=self.cli("status"))
self.round()
lines = shown["status"][1].splitlines()
self.assertRegex(lines[0], r"^round \d\d:\d\d need until \d\d:\d\d$")
self.assertEqual(lines[1:], ["test-1 waiting test delete", "test-2 held", "test-3 unclear"])
code, out, _ = self.cli("status")
self.assertEqual(code, 0)
self.assertRegex(out, r"^no open round; last: queue cleanup \d\d:\d\d: deleted test-1; left the queue none\n$")
def test_the_round_lands_in_the_register_with_the_chair_accountable(self):
self.queue(self.dead("waiting"))
self.round()
row = self.register()
self.assertEqual(
(row["status"], row["worker"], row["roles"], row["objective"]),
("done", WORKER, {"Main": "A", WORKER: "R"}, self.cleanup.OBJECTIVE),
)
self.assertTrue(row["description"].endswith(": deleted test-1; left the queue none"))
def test_a_chair_gone_before_the_delete_stops_the_round_and_deletes_nothing(self):
job = self.dead("waiting")
self.queue(job)
self.actions[("warn", 0)] = lambda: self.chair("gone")
code, out = self.round()
self.assertEqual((code, self.entries()), (0, [job]))
self.ends(out, ": not deleted, the chair gone is stale test-1; left the queue none")
self.assertEqual(self.register()["status"], "stopped")
def test_a_second_round_refuses_while_one_runs(self):
self.queue(self.dead("waiting"))
inner = {}
self.actions[("need", 0)] = lambda: inner.update(need=self.round()[0])
self.actions[("warn", 0)] = lambda: inner.update(warn=self.round()[0])
self.round()
self.assertEqual(inner, {"need": 3, "warn": 3})
self.assertEqual(self.classes(), ["pan-pan", "security", "post"])
def test_run_refuses_an_agent_session_a_short_window_and_a_read_only_host_state(self):
self.queue(self.dead("waiting"))
with mock.patch.dict(os.environ, {"CLAUDE_CODE_SESSION_ID": "s1"}):
inside = self.cli("run", "--window", "900")
short = self.cli("run", "--window", "899")
self.kept.chmod(0o500)
self.addCleanup(self.kept.chmod, 0o700)
sandboxed = self.cli("run", "--window", "900")
self.assertEqual(inside, (2, "", "queue-cleanup: run starts from kvb-org-queue-cleanup.service, never in an agent session; agents use request\n"))
self.assertEqual(short, (2, "", "queue-cleanup: --window must be at least 900 seconds\n"))
self.assertEqual(sandboxed, (2, "", f"queue-cleanup: cannot write {self.kept}; run starts outside the sandbox only\n"))
self.assertEqual(self.rows(), [])
self.assertFalse((self.kept / "queue-cleanup.json").exists())
def test_request_needs_the_path_unit_and_appends_in_place(self):
request = self.root / "queue-cleanup.request"
unit = self.home / ".config" / "systemd" / "user" / "kvb-org-queue-cleanup.path"
self.assertEqual(self.cli("request"), (2, "", f"queue-cleanup: {unit} is missing, so no round would start; make install enables it\n"))
self.assertFalse(request.exists())
self.install_unit()
code, out, _ = self.cli("request")
self.assertEqual((code, out), (0, f"requested a queue-cleanup round through {request}\n"))
inode = request.stat().st_ino
self.cli("request")
self.assertEqual((request.stat().st_ino, len(request.read_text(encoding="utf-8").splitlines())), (inode, 2))
self.assertIn("PathModified=/tmp/agents-council/queue-cleanup.request\n", unit.read_text(encoding="utf-8"))
def test_request_refuses_during_a_round_and_for_30_minutes_after_one(self):
self.install_unit()
now, clock = self.clock.wall, self.cleanup.radio.clock
self.write_state({"round": "12:00", "start": now - 100, "beat": now, "phase": "closing"})
self.assertEqual(self.cli("request"), (2, "", "queue-cleanup: round 12:00 runs; status shows it\n"))
ended = now - 600
self.write_state({"round": "12:00", "start": now - 2000, "beat": ended, "phase": "done", "end": ended})
self.assertEqual(
self.cli("request"),
(2, "", f"queue-cleanup: the last round ended at {clock(ended)}; request again from {clock(ended + 1800)}\n"),
)
self.write_state({"round": "12:00", "start": now - 4000, "beat": now - 1800, "phase": "done", "end": now - 1800})
self.assertEqual(self.cli("request")[0], 0)
def test_the_units_run_at_midnight_and_noon_without_catch_up_and_outlast_both_windows(self):
timer = (SYSTEMD / "kvb-org-queue-cleanup.timer").read_text(encoding="utf-8")
service = (SYSTEMD / "kvb-org-queue-cleanup.service").read_text(encoding="utf-8")
self.assertIn("OnCalendar=*-*-* 00,12:00:00\nPersistent=false\n", timer)
self.assertIn("Type=oneshot\n", service)
window = int(re.search(r"^ExecStart=/usr/bin/python3 -I \S+/cleanup\.py run --window (\d+)$", service, re.MULTILINE)[1])
self.assertEqual(window, WINDOW)
budget = 2 * window + 4 * self.cleanup.POLL_SECONDS
self.assertGreater(int(re.search(r"^TimeoutStartSec=(\d+)$", service, re.MULTILINE)[1]), budget)
if __name__ == "__main__":
unittest.main()