feat(democracy): watch disk, Docker and caches and act before the disk runs out
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
a2aad89c0e
commit
d6fb75ba22
8 files changed
+542
-9
No files matched your search
@@ -155,7 +155,7 @@ Copy the skills into a specific project (`<repo>/.agents/skills` and `<repo>/.cl
|
||||
make project TARGET=/path/to/repo
|
||||
```
|
||||
|
||||
Both commands also patch the target's `.claude/settings.local.json` — the machine-local, untracked file, so a project's reviewed `.claude/settings.json` is never rewritten by an install: the caveman and ponytail plugins are enabled so their modes auto-activate on session start, and two reminder hooks are registered — `autotune` points you at `/autotune` at most once an hour, `dream` points you at `/dream` at most once every twelve hours per repository (stamped in that repository's git dir). A PreToolUse gate from the `democracy` skill blocks edits to files in a git worktree until the session holds a claim (`skills/democracy/scripts/council.py`). Existing settings are preserved.
|
||||
Both commands also patch the target's `.claude/settings.local.json` — the machine-local, untracked file, so a project's reviewed `.claude/settings.json` is never rewritten by an install: the caveman and ponytail plugins are enabled so their modes auto-activate on session start, and two reminder hooks are registered — `autotune` points you at `/autotune` at most once an hour, `dream` points you at `/dream` at most once every twelve hours per repository (stamped in that repository's git dir). A PreToolUse gate from the `democracy` skill blocks edits to files in a git worktree until the session holds a claim (`skills/democracy/scripts/council.py`), and its resource watch (`skills/democracy/scripts/watch.py`) measures disk, Docker and caches, cleans up low-risk data and blocks new heavy jobs before the disk runs out; it stays inactive until `~/.config/agents-council/config.json` exists. Existing settings are preserved.
|
||||
|
||||
Restart your agent afterwards so it loads the new skills.
|
||||
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
#!/usr/bin/env node
|
||||
// Enable the caveman and ponytail plugins in a Claude Code settings.json so
|
||||
// their SessionStart hooks auto-activate both modes, register the autotune
|
||||
// and dream reminder hooks and the democracy claim gate. Merges non-destructively: existing marketplaces,
|
||||
// and dream reminder hooks, the democracy claim gate and the resource watch. Merges non-destructively: existing marketplaces,
|
||||
// plugins and hooks are preserved, unparseable files are left untouched.
|
||||
"use strict";
|
||||
|
||||
@@ -60,5 +60,21 @@ if (!JSON.stringify(data.hooks.PreToolUse).includes("democracy/scripts/council.p
|
||||
});
|
||||
}
|
||||
|
||||
const watch = path.join(path.dirname(settingsPath), "skills", "democracy", "scripts", "watch.py");
|
||||
const WATCH_HOOKS = [
|
||||
["UserPromptSubmit", null, "watch"],
|
||||
["PostToolUse", "Bash", "watch"],
|
||||
["PreToolUse", "Bash", "block"],
|
||||
];
|
||||
for (const [event, matcher, sub] of WATCH_HOOKS) {
|
||||
data.hooks[event] = data.hooks[event] || [];
|
||||
const registered = data.hooks[event].some((entry) =>
|
||||
(entry.hooks || []).some((hook) =>
|
||||
String(hook.command).includes("democracy/scripts/watch.py") && String(hook.command).endsWith(` ${sub}`)));
|
||||
if (registered) continue;
|
||||
const entry = { hooks: [{ type: "command", command: `python3 ${JSON.stringify(watch)} ${sub}` }] };
|
||||
data.hooks[event].push(matcher ? { matcher, ...entry } : entry);
|
||||
}
|
||||
|
||||
fs.writeFileSync(settingsPath, JSON.stringify(data, null, 2) + "\n");
|
||||
console.log(`skills: enabled caveman + ponytail plugins, the ${REMINDERS.join(" + ")} reminders and the democracy claim gate in ${settingsPath}`);
|
||||
console.log(`skills: enabled caveman + ponytail plugins, the ${REMINDERS.join(" + ")} reminders, the democracy claim gate and resource watch in ${settingsPath}`);
|
||||
@@ -15,7 +15,9 @@ its owner, its size, and its risk.
|
||||
Shared resources belong to the resource chair of the `democracy` skill. If
|
||||
another session holds the chair, send it the cleanup request and stop, unless
|
||||
the operator ordered the cleanup in this session; then tell the chair before
|
||||
step 1, so its grants and queue stay consistent.
|
||||
step 1, so its grants and queue stay consistent. The `democracy` resource
|
||||
watch already runs the configured low-risk cleanup at its stop threshold; this
|
||||
skill covers everything that needs the operator's pick.
|
||||
|
||||
## Step 1: ask the peers
|
||||
|
||||
|
||||
@@ -73,14 +73,38 @@ records the result.
|
||||
the operator has not ended the appointment.
|
||||
- Before starting anything that consumes shared capacity (gates, deploys,
|
||||
iteration loops, swarm or cloud resources, cleanup of shared data), an agent
|
||||
sends the chair what, where and for how long, waits for the grant, and
|
||||
reports the release. The chair keeps one queue, answers every request with
|
||||
its position, enforces the limits the operator set, and never cancels a
|
||||
running grant to free capacity.
|
||||
sends the chair what, where, for how long and its expected disk need, waits
|
||||
for the grant, and reports the release. The chair keeps one queue, answers
|
||||
every request with its position, enforces the limits the operator set,
|
||||
checks every grant with `watch.py admit --need-gb <N>` (or `--kind <slot>`),
|
||||
and never cancels a running grant to free capacity.
|
||||
- Handover is explicit: the outgoing chair sends every peer the new holder,
|
||||
the queue and the running grants, and keeps allocating until the incoming
|
||||
chair confirms. Two allocators never run at once.
|
||||
|
||||
## Resource watch
|
||||
|
||||
The chair keeps the system from running out instead of reacting once it has.
|
||||
`${CLAUDE_SKILL_DIR}/scripts/watch.py` does the measuring as hooks in every
|
||||
session, so no agent probes the disk by hand; it reads its thresholds, job
|
||||
patterns, cleanup commands and cache directories from
|
||||
`~/.config/agents-council/config.json` (keys in the script's docstring) and
|
||||
stays silent below the thresholds.
|
||||
|
||||
- **Warn**: the chair and every session get the biggest consumers; the chair
|
||||
plans a cleanup and grants no large slot.
|
||||
- **Stop**: new heavy jobs are blocked in every session (teardown commands
|
||||
never), and the hook runs the configured low-risk cleanup by itself.
|
||||
- **Critical**: additionally a desktop alarm to the operator. A running stack
|
||||
is torn down only with the operator's OK.
|
||||
- A measured fall that would reach the critical threshold within the
|
||||
projection window raises the stage by one at once.
|
||||
- Caches together stay under their budget; the least recently used go first.
|
||||
|
||||
`watch.py status` shows the last measurement. A `RESOURCE` line in a hook's
|
||||
context is addressed to the chair; any other session only adjusts its own
|
||||
plans to it.
|
||||
|
||||
## Succession
|
||||
|
||||
- A piece of work or a cluster ends with its result, a proposal, or an
|
||||
|
||||
Executable
+349
@@ -0,0 +1,349 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Watch disk, Docker and caches for the democracy resource chair.
|
||||
|
||||
Usage:
|
||||
watch.py watch hook (UserPromptSubmit, PostToolUse): measure at most once
|
||||
per interval machine-wide, act on the stage, show alerts
|
||||
watch.py block hook (PreToolUse on Bash): exit 2 for a heavy job while the
|
||||
stage is stop or critical
|
||||
watch.py admit (--need-gb N | --kind KIND)
|
||||
exit 0 when the grant leaves at least the stop threshold
|
||||
free, 3 otherwise
|
||||
watch.py cleanup (--caches | --full)
|
||||
trim caches to their budget; --full also runs the
|
||||
configured low-risk cleanup commands
|
||||
watch.py status print the last measurement
|
||||
|
||||
Environment:
|
||||
AGENTS_COUNCIL_ROOT: shared state directory; /tmp/agents-council unless set,
|
||||
because every agent on the host must agree on it.
|
||||
AGENTS_COUNCIL_CONFIG: config file; ~/.config/agents-council/config.json
|
||||
unless set. Without a config file every command exits 0 and does nothing.
|
||||
|
||||
Config keys, all required: disk, thresholds_gb {warn, stop, critical},
|
||||
projection_minutes, interval_seconds, cache_interval_seconds, heavy_commands,
|
||||
never_block (regexes), cleanup (shell commands), worktree_cleanup (shell
|
||||
commands with {worktree}), worktrees (globs), cache_dirs, cache_budget_percent,
|
||||
slot_need_gb {kind: GB}, notify (argv the alert text is appended to).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import glob
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import shlex
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(os.environ.get("AGENTS_COUNCIL_ROOT", "/tmp/agents-council"))
|
||||
CONFIG = Path(
|
||||
os.environ.get(
|
||||
"AGENTS_COUNCIL_CONFIG",
|
||||
str(Path.home() / ".config" / "agents-council" / "config.json"),
|
||||
)
|
||||
)
|
||||
STAGES = ("ok", "warn", "stop", "critical")
|
||||
STATE = ROOT / "watch.json"
|
||||
ALERTS = ROOT / "alerts.jsonl"
|
||||
ALERT_TTL = 3600
|
||||
LOCK_TTL = 1800
|
||||
GB = 1e9
|
||||
|
||||
|
||||
def load_config() -> dict | None:
|
||||
if not CONFIG.is_file():
|
||||
return None
|
||||
return json.loads(CONFIG.read_text(encoding="utf-8"))
|
||||
|
||||
|
||||
def disk(cfg: dict) -> tuple[float, float]:
|
||||
st = os.statvfs(cfg["disk"])
|
||||
return st.f_bavail * st.f_frsize / GB, st.f_blocks * st.f_frsize / GB
|
||||
|
||||
|
||||
def read_json(path: Path) -> dict:
|
||||
try:
|
||||
return json.loads(path.read_text(encoding="utf-8"))
|
||||
except (OSError, ValueError):
|
||||
return {}
|
||||
|
||||
|
||||
def write_json(path: Path, data: dict) -> None:
|
||||
# A full disk refuses the write; the stage must still act, so it is dropped.
|
||||
try:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
tmp = path.with_name(f"{path.name}.{os.getpid()}")
|
||||
tmp.write_text(json.dumps(data), encoding="utf-8")
|
||||
tmp.replace(path)
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
|
||||
def stage_for(free: float, samples: list[list[float]], cfg: dict, now: float) -> str:
|
||||
limits = cfg["thresholds_gb"]
|
||||
level = 0
|
||||
for index, name in ((1, "warn"), (2, "stop"), (3, "critical")):
|
||||
if free < limits[name]:
|
||||
level = index
|
||||
window = cfg["projection_minutes"] * 60
|
||||
recent = [s for s in samples if now - s[0] <= window and now > s[0]]
|
||||
if recent and level < 3:
|
||||
then, before = recent[0]
|
||||
rate = (before - free) / (now - then)
|
||||
if rate > 0 and (free - limits["critical"]) / rate < window:
|
||||
level += 1
|
||||
return STAGES[level]
|
||||
|
||||
|
||||
def alert(stage: str, text: str) -> None:
|
||||
try:
|
||||
ROOT.mkdir(parents=True, exist_ok=True)
|
||||
with ALERTS.open("a", encoding="utf-8") as handle:
|
||||
handle.write(json.dumps({"id": time.time_ns(), "stage": stage, "text": text}) + "\n")
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
|
||||
def unseen_alerts(session: str) -> list[str]:
|
||||
seen_path = ROOT / "seen" / session
|
||||
try:
|
||||
seen = int(seen_path.read_text(encoding="utf-8"))
|
||||
except (OSError, ValueError):
|
||||
seen = 0
|
||||
try:
|
||||
lines = ALERTS.read_text(encoding="utf-8").splitlines()
|
||||
except OSError:
|
||||
return []
|
||||
cutoff = time.time_ns() - ALERT_TTL * 10**9
|
||||
fresh = [json.loads(line) for line in lines if line]
|
||||
fresh = [a for a in fresh if a["id"] > max(seen, cutoff)]
|
||||
if fresh:
|
||||
write_text(seen_path, str(fresh[-1]["id"]))
|
||||
return [a["text"] for a in fresh]
|
||||
|
||||
|
||||
def write_text(path: Path, text: str) -> None:
|
||||
try:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(text, encoding="utf-8")
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
|
||||
def consumers() -> str:
|
||||
try:
|
||||
out = subprocess.run(
|
||||
["docker", "system", "df", "--format", "{{.Type}}: {{.Size}} ({{.Reclaimable}} reclaimable)"],
|
||||
capture_output=True, text=True, timeout=30, check=False,
|
||||
).stdout.strip()
|
||||
except (OSError, subprocess.TimeoutExpired):
|
||||
return "docker system df unavailable"
|
||||
return "; ".join(out.splitlines()) or "docker system df printed nothing"
|
||||
|
||||
|
||||
def notify(cfg: dict, text: str) -> None:
|
||||
try:
|
||||
subprocess.run([*cfg["notify"], text], capture_output=True, timeout=10, check=False)
|
||||
except (OSError, subprocess.TimeoutExpired):
|
||||
alert("critical", f"RESOURCE: desktop notification failed: {cfg['notify'][0]}")
|
||||
|
||||
|
||||
def spawn_cleanup(mode: str) -> None:
|
||||
subprocess.Popen(
|
||||
[sys.executable, str(Path(__file__).resolve()), "cleanup", mode],
|
||||
stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL,
|
||||
start_new_session=True,
|
||||
)
|
||||
|
||||
|
||||
def act(cfg: dict, stage: str, previous: str, free: float) -> None:
|
||||
limits = cfg["thresholds_gb"]
|
||||
head = f"/ has {free:.1f} GB free (warn < {limits['warn']}, stop < {limits['stop']}, critical < {limits['critical']} GB)"
|
||||
if STAGES.index(stage) < STAGES.index(previous):
|
||||
alert(stage, f"RESOURCE {stage.upper()}: recovered, {head}.")
|
||||
return
|
||||
if stage == "warn":
|
||||
alert(stage, f"RESOURCE WARN: {head}. Biggest consumers: {consumers()}. Chair: plan a cleanup; peers: request no large slot.")
|
||||
elif stage in ("stop", "critical"):
|
||||
alert(stage, f"RESOURCE {stage.upper()}: {head}. New heavy jobs are blocked; low-risk cleanup is running. {consumers()}")
|
||||
spawn_cleanup("--full")
|
||||
if stage == "critical":
|
||||
notify(cfg, f"Agents council: {head}. Heavy jobs blocked; a running stack needs the operator's OK to go.")
|
||||
|
||||
|
||||
def cmd_watch(cfg: dict) -> int:
|
||||
payload = json.load(sys.stdin)
|
||||
now = time.time()
|
||||
state = read_json(STATE)
|
||||
if now - state.get("checked", 0) >= cfg["interval_seconds"]:
|
||||
free, _ = disk(cfg)
|
||||
samples = [s for s in state.get("samples", []) if now - s[0] <= cfg["projection_minutes"] * 60]
|
||||
stage = stage_for(free, samples, cfg, now)
|
||||
previous = state.get("stage", "ok")
|
||||
samples.append([now, free])
|
||||
state.update(checked=now, free=free, stage=stage, samples=samples)
|
||||
due = now - state.get("caches_checked", 0) >= cfg["cache_interval_seconds"]
|
||||
if due:
|
||||
state["caches_checked"] = now
|
||||
write_json(STATE, state)
|
||||
if stage != previous:
|
||||
act(cfg, stage, previous, free)
|
||||
elif due:
|
||||
spawn_cleanup("--caches")
|
||||
texts = unseen_alerts(payload["session_id"])
|
||||
if texts:
|
||||
print(json.dumps({"hookSpecificOutput": {
|
||||
"hookEventName": payload["hook_event_name"],
|
||||
"additionalContext": "\n".join(texts),
|
||||
}}))
|
||||
return 0
|
||||
|
||||
|
||||
def cmd_block(cfg: dict) -> int:
|
||||
payload = json.load(sys.stdin)
|
||||
command = (payload.get("tool_input") or {}).get("command", "")
|
||||
stage = read_json(STATE).get("stage", "ok")
|
||||
if stage not in ("stop", "critical"):
|
||||
return 0
|
||||
if any(re.search(p, command) for p in cfg["never_block"]):
|
||||
return 0
|
||||
if any(re.search(p, command) for p in cfg["heavy_commands"]):
|
||||
print(
|
||||
f"democracy: resource stage is {stage}; new heavy jobs are blocked until "
|
||||
"the disk recovers. Teardown commands still run. Ask the resource chair.",
|
||||
file=sys.stderr,
|
||||
)
|
||||
return 2
|
||||
return 0
|
||||
|
||||
|
||||
def cmd_admit(cfg: dict, args: argparse.Namespace) -> int:
|
||||
need = args.need_gb if args.need_gb is not None else cfg["slot_need_gb"][args.kind]
|
||||
free, _ = disk(cfg)
|
||||
left = free - need
|
||||
floor = cfg["thresholds_gb"]["stop"]
|
||||
verdict = "grant" if left >= floor else "deny"
|
||||
print(f"{verdict}: free {free:.1f} GB, need {need} GB, leaves {left:.1f} GB (floor {floor} GB)")
|
||||
return 0 if verdict == "grant" else 3
|
||||
|
||||
|
||||
def cache_files(dirs: list[str]) -> list[tuple[float, int, str]]:
|
||||
files = []
|
||||
for root in dirs:
|
||||
for base, _, names in os.walk(os.path.expanduser(root)):
|
||||
for name in names:
|
||||
path = os.path.join(base, name)
|
||||
try:
|
||||
st = os.lstat(path)
|
||||
except OSError:
|
||||
continue
|
||||
files.append((st.st_atime, st.st_size, path))
|
||||
return files
|
||||
|
||||
|
||||
def trim_caches(cfg: dict) -> float:
|
||||
_, total = disk(cfg)
|
||||
budget = total * GB * cfg["cache_budget_percent"] / 100
|
||||
files = sorted(cache_files(cfg["cache_dirs"]))
|
||||
used = sum(size for _, size, _ in files)
|
||||
freed = 0
|
||||
for _, size, path in files:
|
||||
if used - freed <= budget:
|
||||
break
|
||||
try:
|
||||
os.remove(path)
|
||||
freed += size
|
||||
except OSError:
|
||||
continue
|
||||
return freed / GB
|
||||
|
||||
|
||||
def run_cleanup(cfg: dict) -> list[str]:
|
||||
commands = list(cfg["cleanup"])
|
||||
for pattern in cfg["worktrees"]:
|
||||
for tree in sorted(glob.glob(os.path.expanduser(pattern))):
|
||||
if os.path.isdir(tree):
|
||||
commands += [t.format(worktree=shlex.quote(tree)) for t in cfg["worktree_cleanup"]]
|
||||
results = []
|
||||
for command in commands:
|
||||
try:
|
||||
code = subprocess.run(command, shell=True, capture_output=True, timeout=900, check=False).returncode
|
||||
except subprocess.TimeoutExpired:
|
||||
code = "timeout"
|
||||
results.append(f"{'ok' if code == 0 else f'failed ({code})'}: {command}")
|
||||
return results
|
||||
|
||||
|
||||
def cmd_cleanup(cfg: dict, args: argparse.Namespace) -> int:
|
||||
lock = ROOT / "cleanup.lock"
|
||||
ROOT.mkdir(parents=True, exist_ok=True)
|
||||
try:
|
||||
if time.time() - lock.stat().st_mtime > LOCK_TTL:
|
||||
lock.rmdir()
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
try:
|
||||
lock.mkdir()
|
||||
except FileExistsError:
|
||||
return 0
|
||||
try:
|
||||
before, _ = disk(cfg)
|
||||
results = run_cleanup(cfg) if args.full else []
|
||||
cache_gb = trim_caches(cfg)
|
||||
after, _ = disk(cfg)
|
||||
failed = [r for r in results if not r.startswith("ok")]
|
||||
if args.full or cache_gb:
|
||||
alert(
|
||||
"cleanup",
|
||||
f"RESOURCE CLEANUP: freed {after - before:.1f} GB ({cache_gb:.1f} GB caches), "
|
||||
f"/ now {after:.1f} GB free; {len(results) - len(failed)} commands ok"
|
||||
+ (f", failed: {' | '.join(failed)}" if failed else "."),
|
||||
)
|
||||
finally:
|
||||
lock.rmdir()
|
||||
return 0
|
||||
|
||||
|
||||
def cmd_status(cfg: dict) -> int:
|
||||
state = read_json(STATE)
|
||||
free, total = disk(cfg)
|
||||
print(f"stage={state.get('stage', 'unmeasured')} free={free:.1f}GB total={total:.1f}GB samples={len(state.get('samples', []))}")
|
||||
return 0
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(prog="watch")
|
||||
sub = parser.add_subparsers(dest="cmd", required=True)
|
||||
sub.add_parser("watch")
|
||||
sub.add_parser("block")
|
||||
admit = sub.add_parser("admit")
|
||||
need = admit.add_mutually_exclusive_group(required=True)
|
||||
need.add_argument("--need-gb", type=float)
|
||||
need.add_argument("--kind")
|
||||
cleanup = sub.add_parser("cleanup")
|
||||
mode = cleanup.add_mutually_exclusive_group(required=True)
|
||||
mode.add_argument("--caches", action="store_true")
|
||||
mode.add_argument("--full", action="store_true")
|
||||
sub.add_parser("status")
|
||||
args = parser.parse_args()
|
||||
cfg = load_config()
|
||||
if cfg is None:
|
||||
return 0
|
||||
if args.cmd == "watch":
|
||||
return cmd_watch(cfg)
|
||||
if args.cmd == "block":
|
||||
return cmd_block(cfg)
|
||||
if args.cmd == "admit":
|
||||
return cmd_admit(cfg, args)
|
||||
if args.cmd == "cleanup":
|
||||
return cmd_cleanup(cfg, args)
|
||||
return cmd_status(cfg)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -116,7 +116,7 @@ class TestSettingsRegistration(unittest.TestCase):
|
||||
encoding="utf-8",
|
||||
)
|
||||
data = self._patch()
|
||||
self.assertEqual(len(data["hooks"]["UserPromptSubmit"]), 3)
|
||||
self.assertEqual(len(data["hooks"]["UserPromptSubmit"]), 4)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
@@ -128,6 +128,10 @@ class TestSettingsRegistration(unittest.TestCase):
|
||||
self.assertEqual(entries.count("autotune/hooks/reminder.sh"), 1)
|
||||
gate = json.dumps(data["hooks"]["PreToolUse"])
|
||||
self.assertEqual(gate.count("democracy/scripts/council.py"), 1)
|
||||
self.assertEqual(gate.count("democracy/scripts/watch.py"), 1)
|
||||
self.assertEqual(entries.count("democracy/scripts/watch.py"), 1)
|
||||
post = json.dumps(data["hooks"]["PostToolUse"])
|
||||
self.assertEqual(post.count("democracy/scripts/watch.py"), 1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
@@ -0,0 +1,138 @@
|
||||
"""Exercise the democracy resource watch: stages, projection, block, admit, alerts, caches."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib.util
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import time
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parents[1]
|
||||
WATCH = REPO_ROOT / "skills" / "democracy" / "scripts" / "watch.py"
|
||||
LIMITS = {"warn": 40, "stop": 15, "critical": 12}
|
||||
|
||||
|
||||
def _load():
|
||||
spec = importlib.util.spec_from_file_location("watch", WATCH)
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(module)
|
||||
return module
|
||||
|
||||
|
||||
class TestStage(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.watch = _load()
|
||||
self.cfg = {"thresholds_gb": LIMITS, "projection_minutes": 30}
|
||||
|
||||
def test_thresholds(self):
|
||||
now = time.time()
|
||||
for free, stage in ((50, "ok"), (30, "warn"), (14, "stop"), (5, "critical")):
|
||||
self.assertEqual(self.watch.stage_for(free, [], self.cfg, now), stage)
|
||||
|
||||
def test_fast_fall_escalates_one_stage(self):
|
||||
now = time.time()
|
||||
self.assertEqual(self.watch.stage_for(32, [[now - 600, 48]], self.cfg, now), "stop")
|
||||
self.assertEqual(self.watch.stage_for(32, [[now - 600, 32.1]], self.cfg, now), "warn")
|
||||
|
||||
|
||||
class TestWatchCommands(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.tmp = tempfile.TemporaryDirectory()
|
||||
self.addCleanup(self.tmp.cleanup)
|
||||
self.base = Path(self.tmp.name)
|
||||
self.root = self.base / "council"
|
||||
self.config = self.base / "config.json"
|
||||
self.cache = self.base / "cache"
|
||||
self.cache.mkdir()
|
||||
self.cfg = {
|
||||
"disk": str(self.base),
|
||||
"thresholds_gb": LIMITS,
|
||||
"projection_minutes": 30,
|
||||
"interval_seconds": 0,
|
||||
"cache_interval_seconds": 86400,
|
||||
"heavy_commands": ["^make compose-deploy", "^make swarm-"],
|
||||
"never_block": ["^make swarm-down"],
|
||||
"cleanup": [],
|
||||
"worktree_cleanup": [],
|
||||
"worktrees": [],
|
||||
"cache_dirs": [str(self.cache)],
|
||||
"cache_budget_percent": 3,
|
||||
"slot_need_gb": {"swarm": 30},
|
||||
"notify": ["true"],
|
||||
}
|
||||
self._write_config()
|
||||
|
||||
def _write_config(self):
|
||||
self.config.write_text(json.dumps(self.cfg), encoding="utf-8")
|
||||
|
||||
def _run(self, *args: str, stdin: str = "") -> subprocess.CompletedProcess:
|
||||
env = {
|
||||
"PATH": str(self.base),
|
||||
"HOME": str(self.base),
|
||||
"AGENTS_COUNCIL_ROOT": str(self.root),
|
||||
"AGENTS_COUNCIL_CONFIG": str(self.config),
|
||||
}
|
||||
return subprocess.run(
|
||||
[sys.executable, str(WATCH), *args],
|
||||
capture_output=True, text=True, env=env, input=stdin, check=False,
|
||||
)
|
||||
|
||||
def _stage(self, stage: str) -> None:
|
||||
self.root.mkdir(exist_ok=True)
|
||||
(self.root / "watch.json").write_text(json.dumps({"stage": stage}), encoding="utf-8")
|
||||
|
||||
def _bash(self, command: str) -> str:
|
||||
return json.dumps({"tool_input": {"command": command}})
|
||||
|
||||
def test_block_only_heavy_jobs_at_stop_and_never_teardown(self):
|
||||
self._stage("stop")
|
||||
self.assertEqual(self._run("block", stdin=self._bash("make compose-deploy apps=x")).returncode, 2)
|
||||
self.assertEqual(self._run("block", stdin=self._bash("make swarm-down")).returncode, 0)
|
||||
self.assertEqual(self._run("block", stdin=self._bash("ls")).returncode, 0)
|
||||
self._stage("warn")
|
||||
self.assertEqual(self._run("block", stdin=self._bash("make compose-deploy")).returncode, 0)
|
||||
|
||||
def test_admit_keeps_the_stop_floor(self):
|
||||
self.cfg["thresholds_gb"] = {"warn": 0, "stop": 0, "critical": 0}
|
||||
self._write_config()
|
||||
self.assertEqual(self._run("admit", "--need-gb", "0.001").returncode, 0)
|
||||
self.assertEqual(self._run("admit", "--need-gb", "1e9").returncode, 3)
|
||||
|
||||
def test_warn_alert_reaches_each_session_once(self):
|
||||
self.cfg["thresholds_gb"] = {"warn": 1e9, "stop": -1, "critical": -2}
|
||||
self._write_config()
|
||||
|
||||
def hook(session: str) -> str:
|
||||
payload = {"session_id": session, "hook_event_name": "UserPromptSubmit"}
|
||||
return self._run("watch", stdin=json.dumps(payload)).stdout
|
||||
|
||||
self.assertIn("RESOURCE WARN", hook("s1"))
|
||||
self.assertEqual(hook("s1"), "")
|
||||
self.assertIn("RESOURCE WARN", hook("s2"))
|
||||
|
||||
def test_cache_trim_evicts_least_recently_used_first(self):
|
||||
old, new = self.cache / "old", self.cache / "new"
|
||||
old.write_bytes(b"x" * 4096)
|
||||
new.write_bytes(b"y" * 4096)
|
||||
os.utime(old, (1_000_000, 1_000_000))
|
||||
os.utime(new, (2_000_000, 2_000_000))
|
||||
st = os.statvfs(self.base)
|
||||
self.cfg["cache_budget_percent"] = (4096 + 100) / (st.f_blocks * st.f_frsize) * 100
|
||||
self._write_config()
|
||||
self.assertEqual(self._run("cleanup", "--caches").returncode, 0)
|
||||
self.assertFalse(old.exists())
|
||||
self.assertTrue(new.exists())
|
||||
|
||||
def test_without_config_every_command_is_a_no_op(self):
|
||||
self.config.unlink()
|
||||
result = self._run("status")
|
||||
self.assertEqual((result.returncode, result.stdout), (0, ""))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in new issue
Block a user