feat(democracy): watch disk, Docker and caches and act before the disk runs out
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
a2aad89c0e
commit
d6fb75ba22
8 files changed
+542
-9
No files matched your search
@@ -155,7 +155,7 @@ Copy the skills into a specific project (`<repo>/.agents/skills` and `<repo>/.cl
|
|||||||
make project TARGET=/path/to/repo
|
make project TARGET=/path/to/repo
|
||||||
```
|
```
|
||||||
|
|
||||||
Both commands also patch the target's `.claude/settings.local.json` — the machine-local, untracked file, so a project's reviewed `.claude/settings.json` is never rewritten by an install: the caveman and ponytail plugins are enabled so their modes auto-activate on session start, and two reminder hooks are registered — `autotune` points you at `/autotune` at most once an hour, `dream` points you at `/dream` at most once every twelve hours per repository (stamped in that repository's git dir). A PreToolUse gate from the `democracy` skill blocks edits to files in a git worktree until the session holds a claim (`skills/democracy/scripts/council.py`). Existing settings are preserved.
|
Both commands also patch the target's `.claude/settings.local.json` — the machine-local, untracked file, so a project's reviewed `.claude/settings.json` is never rewritten by an install: the caveman and ponytail plugins are enabled so their modes auto-activate on session start, and two reminder hooks are registered — `autotune` points you at `/autotune` at most once an hour, `dream` points you at `/dream` at most once every twelve hours per repository (stamped in that repository's git dir). A PreToolUse gate from the `democracy` skill blocks edits to files in a git worktree until the session holds a claim (`skills/democracy/scripts/council.py`), and its resource watch (`skills/democracy/scripts/watch.py`) measures disk, Docker and caches, cleans up low-risk data and blocks new heavy jobs before the disk runs out; it stays inactive until `~/.config/agents-council/config.json` exists. Existing settings are preserved.
|
||||||
|
|
||||||
Restart your agent afterwards so it loads the new skills.
|
Restart your agent afterwards so it loads the new skills.
|
||||||
|
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
#!/usr/bin/env node
|
#!/usr/bin/env node
|
||||||
// Enable the caveman and ponytail plugins in a Claude Code settings.json so
|
// Enable the caveman and ponytail plugins in a Claude Code settings.json so
|
||||||
// their SessionStart hooks auto-activate both modes, register the autotune
|
// their SessionStart hooks auto-activate both modes, register the autotune
|
||||||
// and dream reminder hooks and the democracy claim gate. Merges non-destructively: existing marketplaces,
|
// and dream reminder hooks, the democracy claim gate and the resource watch. Merges non-destructively: existing marketplaces,
|
||||||
// plugins and hooks are preserved, unparseable files are left untouched.
|
// plugins and hooks are preserved, unparseable files are left untouched.
|
||||||
"use strict";
|
"use strict";
|
||||||
|
|
||||||
@@ -60,5 +60,21 @@ if (!JSON.stringify(data.hooks.PreToolUse).includes("democracy/scripts/council.p
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const watch = path.join(path.dirname(settingsPath), "skills", "democracy", "scripts", "watch.py");
|
||||||
|
const WATCH_HOOKS = [
|
||||||
|
["UserPromptSubmit", null, "watch"],
|
||||||
|
["PostToolUse", "Bash", "watch"],
|
||||||
|
["PreToolUse", "Bash", "block"],
|
||||||
|
];
|
||||||
|
for (const [event, matcher, sub] of WATCH_HOOKS) {
|
||||||
|
data.hooks[event] = data.hooks[event] || [];
|
||||||
|
const registered = data.hooks[event].some((entry) =>
|
||||||
|
(entry.hooks || []).some((hook) =>
|
||||||
|
String(hook.command).includes("democracy/scripts/watch.py") && String(hook.command).endsWith(` ${sub}`)));
|
||||||
|
if (registered) continue;
|
||||||
|
const entry = { hooks: [{ type: "command", command: `python3 ${JSON.stringify(watch)} ${sub}` }] };
|
||||||
|
data.hooks[event].push(matcher ? { matcher, ...entry } : entry);
|
||||||
|
}
|
||||||
|
|
||||||
fs.writeFileSync(settingsPath, JSON.stringify(data, null, 2) + "\n");
|
fs.writeFileSync(settingsPath, JSON.stringify(data, null, 2) + "\n");
|
||||||
console.log(`skills: enabled caveman + ponytail plugins, the ${REMINDERS.join(" + ")} reminders and the democracy claim gate in ${settingsPath}`);
|
console.log(`skills: enabled caveman + ponytail plugins, the ${REMINDERS.join(" + ")} reminders, the democracy claim gate and resource watch in ${settingsPath}`);
|
||||||
@@ -15,7 +15,9 @@ its owner, its size, and its risk.
|
|||||||
Shared resources belong to the resource chair of the `democracy` skill. If
|
Shared resources belong to the resource chair of the `democracy` skill. If
|
||||||
another session holds the chair, send it the cleanup request and stop, unless
|
another session holds the chair, send it the cleanup request and stop, unless
|
||||||
the operator ordered the cleanup in this session; then tell the chair before
|
the operator ordered the cleanup in this session; then tell the chair before
|
||||||
step 1, so its grants and queue stay consistent.
|
step 1, so its grants and queue stay consistent. The `democracy` resource
|
||||||
|
watch already runs the configured low-risk cleanup at its stop threshold; this
|
||||||
|
skill covers everything that needs the operator's pick.
|
||||||
|
|
||||||
## Step 1: ask the peers
|
## Step 1: ask the peers
|
||||||
|
|
||||||
|
|||||||
@@ -73,14 +73,38 @@ records the result.
|
|||||||
the operator has not ended the appointment.
|
the operator has not ended the appointment.
|
||||||
- Before starting anything that consumes shared capacity (gates, deploys,
|
- Before starting anything that consumes shared capacity (gates, deploys,
|
||||||
iteration loops, swarm or cloud resources, cleanup of shared data), an agent
|
iteration loops, swarm or cloud resources, cleanup of shared data), an agent
|
||||||
sends the chair what, where and for how long, waits for the grant, and
|
sends the chair what, where, for how long and its expected disk need, waits
|
||||||
reports the release. The chair keeps one queue, answers every request with
|
for the grant, and reports the release. The chair keeps one queue, answers
|
||||||
its position, enforces the limits the operator set, and never cancels a
|
every request with its position, enforces the limits the operator set,
|
||||||
running grant to free capacity.
|
checks every grant with `watch.py admit --need-gb <N>` (or `--kind <slot>`),
|
||||||
|
and never cancels a running grant to free capacity.
|
||||||
- Handover is explicit: the outgoing chair sends every peer the new holder,
|
- Handover is explicit: the outgoing chair sends every peer the new holder,
|
||||||
the queue and the running grants, and keeps allocating until the incoming
|
the queue and the running grants, and keeps allocating until the incoming
|
||||||
chair confirms. Two allocators never run at once.
|
chair confirms. Two allocators never run at once.
|
||||||
|
|
||||||
|
## Resource watch
|
||||||
|
|
||||||
|
The chair keeps the system from running out instead of reacting once it has.
|
||||||
|
`${CLAUDE_SKILL_DIR}/scripts/watch.py` does the measuring as hooks in every
|
||||||
|
session, so no agent probes the disk by hand; it reads its thresholds, job
|
||||||
|
patterns, cleanup commands and cache directories from
|
||||||
|
`~/.config/agents-council/config.json` (keys in the script's docstring) and
|
||||||
|
stays silent below the thresholds.
|
||||||
|
|
||||||
|
- **Warn**: the chair and every session get the biggest consumers; the chair
|
||||||
|
plans a cleanup and grants no large slot.
|
||||||
|
- **Stop**: new heavy jobs are blocked in every session (teardown commands
|
||||||
|
never), and the hook runs the configured low-risk cleanup by itself.
|
||||||
|
- **Critical**: additionally a desktop alarm to the operator. A running stack
|
||||||
|
is torn down only with the operator's OK.
|
||||||
|
- A measured fall that would reach the critical threshold within the
|
||||||
|
projection window raises the stage by one at once.
|
||||||
|
- Caches together stay under their budget; the least recently used go first.
|
||||||
|
|
||||||
|
`watch.py status` shows the last measurement. A `RESOURCE` line in a hook's
|
||||||
|
context is addressed to the chair; any other session only adjusts its own
|
||||||
|
plans to it.
|
||||||
|
|
||||||
## Succession
|
## Succession
|
||||||
|
|
||||||
- A piece of work or a cluster ends with its result, a proposal, or an
|
- A piece of work or a cluster ends with its result, a proposal, or an
|
||||||
|
|||||||
Executable
+349
@@ -0,0 +1,349 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""Watch disk, Docker and caches for the democracy resource chair.
|
||||||
|
|
||||||
|
Usage:
|
||||||
|
watch.py watch hook (UserPromptSubmit, PostToolUse): measure at most once
|
||||||
|
per interval machine-wide, act on the stage, show alerts
|
||||||
|
watch.py block hook (PreToolUse on Bash): exit 2 for a heavy job while the
|
||||||
|
stage is stop or critical
|
||||||
|
watch.py admit (--need-gb N | --kind KIND)
|
||||||
|
exit 0 when the grant leaves at least the stop threshold
|
||||||
|
free, 3 otherwise
|
||||||
|
watch.py cleanup (--caches | --full)
|
||||||
|
trim caches to their budget; --full also runs the
|
||||||
|
configured low-risk cleanup commands
|
||||||
|
watch.py status print the last measurement
|
||||||
|
|
||||||
|
Environment:
|
||||||
|
AGENTS_COUNCIL_ROOT: shared state directory; /tmp/agents-council unless set,
|
||||||
|
because every agent on the host must agree on it.
|
||||||
|
AGENTS_COUNCIL_CONFIG: config file; ~/.config/agents-council/config.json
|
||||||
|
unless set. Without a config file every command exits 0 and does nothing.
|
||||||
|
|
||||||
|
Config keys, all required: disk, thresholds_gb {warn, stop, critical},
|
||||||
|
projection_minutes, interval_seconds, cache_interval_seconds, heavy_commands,
|
||||||
|
never_block (regexes), cleanup (shell commands), worktree_cleanup (shell
|
||||||
|
commands with {worktree}), worktrees (globs), cache_dirs, cache_budget_percent,
|
||||||
|
slot_need_gb {kind: GB}, notify (argv the alert text is appended to).
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import glob
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import re
|
||||||
|
import shlex
|
||||||
|
import subprocess
|
||||||
|
import sys
|
||||||
|
import time
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
ROOT = Path(os.environ.get("AGENTS_COUNCIL_ROOT", "/tmp/agents-council"))
|
||||||
|
CONFIG = Path(
|
||||||
|
os.environ.get(
|
||||||
|
"AGENTS_COUNCIL_CONFIG",
|
||||||
|
str(Path.home() / ".config" / "agents-council" / "config.json"),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
STAGES = ("ok", "warn", "stop", "critical")
|
||||||
|
STATE = ROOT / "watch.json"
|
||||||
|
ALERTS = ROOT / "alerts.jsonl"
|
||||||
|
ALERT_TTL = 3600
|
||||||
|
LOCK_TTL = 1800
|
||||||
|
GB = 1e9
|
||||||
|
|
||||||
|
|
||||||
|
def load_config() -> dict | None:
|
||||||
|
if not CONFIG.is_file():
|
||||||
|
return None
|
||||||
|
return json.loads(CONFIG.read_text(encoding="utf-8"))
|
||||||
|
|
||||||
|
|
||||||
|
def disk(cfg: dict) -> tuple[float, float]:
|
||||||
|
st = os.statvfs(cfg["disk"])
|
||||||
|
return st.f_bavail * st.f_frsize / GB, st.f_blocks * st.f_frsize / GB
|
||||||
|
|
||||||
|
|
||||||
|
def read_json(path: Path) -> dict:
|
||||||
|
try:
|
||||||
|
return json.loads(path.read_text(encoding="utf-8"))
|
||||||
|
except (OSError, ValueError):
|
||||||
|
return {}
|
||||||
|
|
||||||
|
|
||||||
|
def write_json(path: Path, data: dict) -> None:
|
||||||
|
# A full disk refuses the write; the stage must still act, so it is dropped.
|
||||||
|
try:
|
||||||
|
path.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
tmp = path.with_name(f"{path.name}.{os.getpid()}")
|
||||||
|
tmp.write_text(json.dumps(data), encoding="utf-8")
|
||||||
|
tmp.replace(path)
|
||||||
|
except OSError:
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
def stage_for(free: float, samples: list[list[float]], cfg: dict, now: float) -> str:
|
||||||
|
limits = cfg["thresholds_gb"]
|
||||||
|
level = 0
|
||||||
|
for index, name in ((1, "warn"), (2, "stop"), (3, "critical")):
|
||||||
|
if free < limits[name]:
|
||||||
|
level = index
|
||||||
|
window = cfg["projection_minutes"] * 60
|
||||||
|
recent = [s for s in samples if now - s[0] <= window and now > s[0]]
|
||||||
|
if recent and level < 3:
|
||||||
|
then, before = recent[0]
|
||||||
|
rate = (before - free) / (now - then)
|
||||||
|
if rate > 0 and (free - limits["critical"]) / rate < window:
|
||||||
|
level += 1
|
||||||
|
return STAGES[level]
|
||||||
|
|
||||||
|
|
||||||
|
def alert(stage: str, text: str) -> None:
|
||||||
|
try:
|
||||||
|
ROOT.mkdir(parents=True, exist_ok=True)
|
||||||
|
with ALERTS.open("a", encoding="utf-8") as handle:
|
||||||
|
handle.write(json.dumps({"id": time.time_ns(), "stage": stage, "text": text}) + "\n")
|
||||||
|
except OSError:
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
def unseen_alerts(session: str) -> list[str]:
|
||||||
|
seen_path = ROOT / "seen" / session
|
||||||
|
try:
|
||||||
|
seen = int(seen_path.read_text(encoding="utf-8"))
|
||||||
|
except (OSError, ValueError):
|
||||||
|
seen = 0
|
||||||
|
try:
|
||||||
|
lines = ALERTS.read_text(encoding="utf-8").splitlines()
|
||||||
|
except OSError:
|
||||||
|
return []
|
||||||
|
cutoff = time.time_ns() - ALERT_TTL * 10**9
|
||||||
|
fresh = [json.loads(line) for line in lines if line]
|
||||||
|
fresh = [a for a in fresh if a["id"] > max(seen, cutoff)]
|
||||||
|
if fresh:
|
||||||
|
write_text(seen_path, str(fresh[-1]["id"]))
|
||||||
|
return [a["text"] for a in fresh]
|
||||||
|
|
||||||
|
|
||||||
|
def write_text(path: Path, text: str) -> None:
|
||||||
|
try:
|
||||||
|
path.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
path.write_text(text, encoding="utf-8")
|
||||||
|
except OSError:
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
def consumers() -> str:
|
||||||
|
try:
|
||||||
|
out = subprocess.run(
|
||||||
|
["docker", "system", "df", "--format", "{{.Type}}: {{.Size}} ({{.Reclaimable}} reclaimable)"],
|
||||||
|
capture_output=True, text=True, timeout=30, check=False,
|
||||||
|
).stdout.strip()
|
||||||
|
except (OSError, subprocess.TimeoutExpired):
|
||||||
|
return "docker system df unavailable"
|
||||||
|
return "; ".join(out.splitlines()) or "docker system df printed nothing"
|
||||||
|
|
||||||
|
|
||||||
|
def notify(cfg: dict, text: str) -> None:
|
||||||
|
try:
|
||||||
|
subprocess.run([*cfg["notify"], text], capture_output=True, timeout=10, check=False)
|
||||||
|
except (OSError, subprocess.TimeoutExpired):
|
||||||
|
alert("critical", f"RESOURCE: desktop notification failed: {cfg['notify'][0]}")
|
||||||
|
|
||||||
|
|
||||||
|
def spawn_cleanup(mode: str) -> None:
|
||||||
|
subprocess.Popen(
|
||||||
|
[sys.executable, str(Path(__file__).resolve()), "cleanup", mode],
|
||||||
|
stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL,
|
||||||
|
start_new_session=True,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def act(cfg: dict, stage: str, previous: str, free: float) -> None:
|
||||||
|
limits = cfg["thresholds_gb"]
|
||||||
|
head = f"/ has {free:.1f} GB free (warn < {limits['warn']}, stop < {limits['stop']}, critical < {limits['critical']} GB)"
|
||||||
|
if STAGES.index(stage) < STAGES.index(previous):
|
||||||
|
alert(stage, f"RESOURCE {stage.upper()}: recovered, {head}.")
|
||||||
|
return
|
||||||
|
if stage == "warn":
|
||||||
|
alert(stage, f"RESOURCE WARN: {head}. Biggest consumers: {consumers()}. Chair: plan a cleanup; peers: request no large slot.")
|
||||||
|
elif stage in ("stop", "critical"):
|
||||||
|
alert(stage, f"RESOURCE {stage.upper()}: {head}. New heavy jobs are blocked; low-risk cleanup is running. {consumers()}")
|
||||||
|
spawn_cleanup("--full")
|
||||||
|
if stage == "critical":
|
||||||
|
notify(cfg, f"Agents council: {head}. Heavy jobs blocked; a running stack needs the operator's OK to go.")
|
||||||
|
|
||||||
|
|
||||||
|
def cmd_watch(cfg: dict) -> int:
|
||||||
|
payload = json.load(sys.stdin)
|
||||||
|
now = time.time()
|
||||||
|
state = read_json(STATE)
|
||||||
|
if now - state.get("checked", 0) >= cfg["interval_seconds"]:
|
||||||
|
free, _ = disk(cfg)
|
||||||
|
samples = [s for s in state.get("samples", []) if now - s[0] <= cfg["projection_minutes"] * 60]
|
||||||
|
stage = stage_for(free, samples, cfg, now)
|
||||||
|
previous = state.get("stage", "ok")
|
||||||
|
samples.append([now, free])
|
||||||
|
state.update(checked=now, free=free, stage=stage, samples=samples)
|
||||||
|
due = now - state.get("caches_checked", 0) >= cfg["cache_interval_seconds"]
|
||||||
|
if due:
|
||||||
|
state["caches_checked"] = now
|
||||||
|
write_json(STATE, state)
|
||||||
|
if stage != previous:
|
||||||
|
act(cfg, stage, previous, free)
|
||||||
|
elif due:
|
||||||
|
spawn_cleanup("--caches")
|
||||||
|
texts = unseen_alerts(payload["session_id"])
|
||||||
|
if texts:
|
||||||
|
print(json.dumps({"hookSpecificOutput": {
|
||||||
|
"hookEventName": payload["hook_event_name"],
|
||||||
|
"additionalContext": "\n".join(texts),
|
||||||
|
}}))
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
def cmd_block(cfg: dict) -> int:
|
||||||
|
payload = json.load(sys.stdin)
|
||||||
|
command = (payload.get("tool_input") or {}).get("command", "")
|
||||||
|
stage = read_json(STATE).get("stage", "ok")
|
||||||
|
if stage not in ("stop", "critical"):
|
||||||
|
return 0
|
||||||
|
if any(re.search(p, command) for p in cfg["never_block"]):
|
||||||
|
return 0
|
||||||
|
if any(re.search(p, command) for p in cfg["heavy_commands"]):
|
||||||
|
print(
|
||||||
|
f"democracy: resource stage is {stage}; new heavy jobs are blocked until "
|
||||||
|
"the disk recovers. Teardown commands still run. Ask the resource chair.",
|
||||||
|
file=sys.stderr,
|
||||||
|
)
|
||||||
|
return 2
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
def cmd_admit(cfg: dict, args: argparse.Namespace) -> int:
|
||||||
|
need = args.need_gb if args.need_gb is not None else cfg["slot_need_gb"][args.kind]
|
||||||
|
free, _ = disk(cfg)
|
||||||
|
left = free - need
|
||||||
|
floor = cfg["thresholds_gb"]["stop"]
|
||||||
|
verdict = "grant" if left >= floor else "deny"
|
||||||
|
print(f"{verdict}: free {free:.1f} GB, need {need} GB, leaves {left:.1f} GB (floor {floor} GB)")
|
||||||
|
return 0 if verdict == "grant" else 3
|
||||||
|
|
||||||
|
|
||||||
|
def cache_files(dirs: list[str]) -> list[tuple[float, int, str]]:
|
||||||
|
files = []
|
||||||
|
for root in dirs:
|
||||||
|
for base, _, names in os.walk(os.path.expanduser(root)):
|
||||||
|
for name in names:
|
||||||
|
path = os.path.join(base, name)
|
||||||
|
try:
|
||||||
|
st = os.lstat(path)
|
||||||
|
except OSError:
|
||||||
|
continue
|
||||||
|
files.append((st.st_atime, st.st_size, path))
|
||||||
|
return files
|
||||||
|
|
||||||
|
|
||||||
|
def trim_caches(cfg: dict) -> float:
|
||||||
|
_, total = disk(cfg)
|
||||||
|
budget = total * GB * cfg["cache_budget_percent"] / 100
|
||||||
|
files = sorted(cache_files(cfg["cache_dirs"]))
|
||||||
|
used = sum(size for _, size, _ in files)
|
||||||
|
freed = 0
|
||||||
|
for _, size, path in files:
|
||||||
|
if used - freed <= budget:
|
||||||
|
break
|
||||||
|
try:
|
||||||
|
os.remove(path)
|
||||||
|
freed += size
|
||||||
|
except OSError:
|
||||||
|
continue
|
||||||
|
return freed / GB
|
||||||
|
|
||||||
|
|
||||||
|
def run_cleanup(cfg: dict) -> list[str]:
|
||||||
|
commands = list(cfg["cleanup"])
|
||||||
|
for pattern in cfg["worktrees"]:
|
||||||
|
for tree in sorted(glob.glob(os.path.expanduser(pattern))):
|
||||||
|
if os.path.isdir(tree):
|
||||||
|
commands += [t.format(worktree=shlex.quote(tree)) for t in cfg["worktree_cleanup"]]
|
||||||
|
results = []
|
||||||
|
for command in commands:
|
||||||
|
try:
|
||||||
|
code = subprocess.run(command, shell=True, capture_output=True, timeout=900, check=False).returncode
|
||||||
|
except subprocess.TimeoutExpired:
|
||||||
|
code = "timeout"
|
||||||
|
results.append(f"{'ok' if code == 0 else f'failed ({code})'}: {command}")
|
||||||
|
return results
|
||||||
|
|
||||||
|
|
||||||
|
def cmd_cleanup(cfg: dict, args: argparse.Namespace) -> int:
|
||||||
|
lock = ROOT / "cleanup.lock"
|
||||||
|
ROOT.mkdir(parents=True, exist_ok=True)
|
||||||
|
try:
|
||||||
|
if time.time() - lock.stat().st_mtime > LOCK_TTL:
|
||||||
|
lock.rmdir()
|
||||||
|
except FileNotFoundError:
|
||||||
|
pass
|
||||||
|
try:
|
||||||
|
lock.mkdir()
|
||||||
|
except FileExistsError:
|
||||||
|
return 0
|
||||||
|
try:
|
||||||
|
before, _ = disk(cfg)
|
||||||
|
results = run_cleanup(cfg) if args.full else []
|
||||||
|
cache_gb = trim_caches(cfg)
|
||||||
|
after, _ = disk(cfg)
|
||||||
|
failed = [r for r in results if not r.startswith("ok")]
|
||||||
|
if args.full or cache_gb:
|
||||||
|
alert(
|
||||||
|
"cleanup",
|
||||||
|
f"RESOURCE CLEANUP: freed {after - before:.1f} GB ({cache_gb:.1f} GB caches), "
|
||||||
|
f"/ now {after:.1f} GB free; {len(results) - len(failed)} commands ok"
|
||||||
|
+ (f", failed: {' | '.join(failed)}" if failed else "."),
|
||||||
|
)
|
||||||
|
finally:
|
||||||
|
lock.rmdir()
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
def cmd_status(cfg: dict) -> int:
|
||||||
|
state = read_json(STATE)
|
||||||
|
free, total = disk(cfg)
|
||||||
|
print(f"stage={state.get('stage', 'unmeasured')} free={free:.1f}GB total={total:.1f}GB samples={len(state.get('samples', []))}")
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> int:
|
||||||
|
parser = argparse.ArgumentParser(prog="watch")
|
||||||
|
sub = parser.add_subparsers(dest="cmd", required=True)
|
||||||
|
sub.add_parser("watch")
|
||||||
|
sub.add_parser("block")
|
||||||
|
admit = sub.add_parser("admit")
|
||||||
|
need = admit.add_mutually_exclusive_group(required=True)
|
||||||
|
need.add_argument("--need-gb", type=float)
|
||||||
|
need.add_argument("--kind")
|
||||||
|
cleanup = sub.add_parser("cleanup")
|
||||||
|
mode = cleanup.add_mutually_exclusive_group(required=True)
|
||||||
|
mode.add_argument("--caches", action="store_true")
|
||||||
|
mode.add_argument("--full", action="store_true")
|
||||||
|
sub.add_parser("status")
|
||||||
|
args = parser.parse_args()
|
||||||
|
cfg = load_config()
|
||||||
|
if cfg is None:
|
||||||
|
return 0
|
||||||
|
if args.cmd == "watch":
|
||||||
|
return cmd_watch(cfg)
|
||||||
|
if args.cmd == "block":
|
||||||
|
return cmd_block(cfg)
|
||||||
|
if args.cmd == "admit":
|
||||||
|
return cmd_admit(cfg, args)
|
||||||
|
if args.cmd == "cleanup":
|
||||||
|
return cmd_cleanup(cfg, args)
|
||||||
|
return cmd_status(cfg)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
sys.exit(main())
|
||||||
@@ -116,7 +116,7 @@ class TestSettingsRegistration(unittest.TestCase):
|
|||||||
encoding="utf-8",
|
encoding="utf-8",
|
||||||
)
|
)
|
||||||
data = self._patch()
|
data = self._patch()
|
||||||
self.assertEqual(len(data["hooks"]["UserPromptSubmit"]), 3)
|
self.assertEqual(len(data["hooks"]["UserPromptSubmit"]), 4)
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
|
|||||||
@@ -128,6 +128,10 @@ class TestSettingsRegistration(unittest.TestCase):
|
|||||||
self.assertEqual(entries.count("autotune/hooks/reminder.sh"), 1)
|
self.assertEqual(entries.count("autotune/hooks/reminder.sh"), 1)
|
||||||
gate = json.dumps(data["hooks"]["PreToolUse"])
|
gate = json.dumps(data["hooks"]["PreToolUse"])
|
||||||
self.assertEqual(gate.count("democracy/scripts/council.py"), 1)
|
self.assertEqual(gate.count("democracy/scripts/council.py"), 1)
|
||||||
|
self.assertEqual(gate.count("democracy/scripts/watch.py"), 1)
|
||||||
|
self.assertEqual(entries.count("democracy/scripts/watch.py"), 1)
|
||||||
|
post = json.dumps(data["hooks"]["PostToolUse"])
|
||||||
|
self.assertEqual(post.count("democracy/scripts/watch.py"), 1)
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
|
|||||||
@@ -0,0 +1,138 @@
|
|||||||
|
"""Exercise the democracy resource watch: stages, projection, block, admit, alerts, caches."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import importlib.util
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import subprocess
|
||||||
|
import sys
|
||||||
|
import tempfile
|
||||||
|
import time
|
||||||
|
import unittest
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
REPO_ROOT = Path(__file__).resolve().parents[1]
|
||||||
|
WATCH = REPO_ROOT / "skills" / "democracy" / "scripts" / "watch.py"
|
||||||
|
LIMITS = {"warn": 40, "stop": 15, "critical": 12}
|
||||||
|
|
||||||
|
|
||||||
|
def _load():
|
||||||
|
spec = importlib.util.spec_from_file_location("watch", WATCH)
|
||||||
|
module = importlib.util.module_from_spec(spec)
|
||||||
|
spec.loader.exec_module(module)
|
||||||
|
return module
|
||||||
|
|
||||||
|
|
||||||
|
class TestStage(unittest.TestCase):
|
||||||
|
def setUp(self):
|
||||||
|
self.watch = _load()
|
||||||
|
self.cfg = {"thresholds_gb": LIMITS, "projection_minutes": 30}
|
||||||
|
|
||||||
|
def test_thresholds(self):
|
||||||
|
now = time.time()
|
||||||
|
for free, stage in ((50, "ok"), (30, "warn"), (14, "stop"), (5, "critical")):
|
||||||
|
self.assertEqual(self.watch.stage_for(free, [], self.cfg, now), stage)
|
||||||
|
|
||||||
|
def test_fast_fall_escalates_one_stage(self):
|
||||||
|
now = time.time()
|
||||||
|
self.assertEqual(self.watch.stage_for(32, [[now - 600, 48]], self.cfg, now), "stop")
|
||||||
|
self.assertEqual(self.watch.stage_for(32, [[now - 600, 32.1]], self.cfg, now), "warn")
|
||||||
|
|
||||||
|
|
||||||
|
class TestWatchCommands(unittest.TestCase):
|
||||||
|
def setUp(self):
|
||||||
|
self.tmp = tempfile.TemporaryDirectory()
|
||||||
|
self.addCleanup(self.tmp.cleanup)
|
||||||
|
self.base = Path(self.tmp.name)
|
||||||
|
self.root = self.base / "council"
|
||||||
|
self.config = self.base / "config.json"
|
||||||
|
self.cache = self.base / "cache"
|
||||||
|
self.cache.mkdir()
|
||||||
|
self.cfg = {
|
||||||
|
"disk": str(self.base),
|
||||||
|
"thresholds_gb": LIMITS,
|
||||||
|
"projection_minutes": 30,
|
||||||
|
"interval_seconds": 0,
|
||||||
|
"cache_interval_seconds": 86400,
|
||||||
|
"heavy_commands": ["^make compose-deploy", "^make swarm-"],
|
||||||
|
"never_block": ["^make swarm-down"],
|
||||||
|
"cleanup": [],
|
||||||
|
"worktree_cleanup": [],
|
||||||
|
"worktrees": [],
|
||||||
|
"cache_dirs": [str(self.cache)],
|
||||||
|
"cache_budget_percent": 3,
|
||||||
|
"slot_need_gb": {"swarm": 30},
|
||||||
|
"notify": ["true"],
|
||||||
|
}
|
||||||
|
self._write_config()
|
||||||
|
|
||||||
|
def _write_config(self):
|
||||||
|
self.config.write_text(json.dumps(self.cfg), encoding="utf-8")
|
||||||
|
|
||||||
|
def _run(self, *args: str, stdin: str = "") -> subprocess.CompletedProcess:
|
||||||
|
env = {
|
||||||
|
"PATH": str(self.base),
|
||||||
|
"HOME": str(self.base),
|
||||||
|
"AGENTS_COUNCIL_ROOT": str(self.root),
|
||||||
|
"AGENTS_COUNCIL_CONFIG": str(self.config),
|
||||||
|
}
|
||||||
|
return subprocess.run(
|
||||||
|
[sys.executable, str(WATCH), *args],
|
||||||
|
capture_output=True, text=True, env=env, input=stdin, check=False,
|
||||||
|
)
|
||||||
|
|
||||||
|
def _stage(self, stage: str) -> None:
|
||||||
|
self.root.mkdir(exist_ok=True)
|
||||||
|
(self.root / "watch.json").write_text(json.dumps({"stage": stage}), encoding="utf-8")
|
||||||
|
|
||||||
|
def _bash(self, command: str) -> str:
|
||||||
|
return json.dumps({"tool_input": {"command": command}})
|
||||||
|
|
||||||
|
def test_block_only_heavy_jobs_at_stop_and_never_teardown(self):
|
||||||
|
self._stage("stop")
|
||||||
|
self.assertEqual(self._run("block", stdin=self._bash("make compose-deploy apps=x")).returncode, 2)
|
||||||
|
self.assertEqual(self._run("block", stdin=self._bash("make swarm-down")).returncode, 0)
|
||||||
|
self.assertEqual(self._run("block", stdin=self._bash("ls")).returncode, 0)
|
||||||
|
self._stage("warn")
|
||||||
|
self.assertEqual(self._run("block", stdin=self._bash("make compose-deploy")).returncode, 0)
|
||||||
|
|
||||||
|
def test_admit_keeps_the_stop_floor(self):
|
||||||
|
self.cfg["thresholds_gb"] = {"warn": 0, "stop": 0, "critical": 0}
|
||||||
|
self._write_config()
|
||||||
|
self.assertEqual(self._run("admit", "--need-gb", "0.001").returncode, 0)
|
||||||
|
self.assertEqual(self._run("admit", "--need-gb", "1e9").returncode, 3)
|
||||||
|
|
||||||
|
def test_warn_alert_reaches_each_session_once(self):
|
||||||
|
self.cfg["thresholds_gb"] = {"warn": 1e9, "stop": -1, "critical": -2}
|
||||||
|
self._write_config()
|
||||||
|
|
||||||
|
def hook(session: str) -> str:
|
||||||
|
payload = {"session_id": session, "hook_event_name": "UserPromptSubmit"}
|
||||||
|
return self._run("watch", stdin=json.dumps(payload)).stdout
|
||||||
|
|
||||||
|
self.assertIn("RESOURCE WARN", hook("s1"))
|
||||||
|
self.assertEqual(hook("s1"), "")
|
||||||
|
self.assertIn("RESOURCE WARN", hook("s2"))
|
||||||
|
|
||||||
|
def test_cache_trim_evicts_least_recently_used_first(self):
|
||||||
|
old, new = self.cache / "old", self.cache / "new"
|
||||||
|
old.write_bytes(b"x" * 4096)
|
||||||
|
new.write_bytes(b"y" * 4096)
|
||||||
|
os.utime(old, (1_000_000, 1_000_000))
|
||||||
|
os.utime(new, (2_000_000, 2_000_000))
|
||||||
|
st = os.statvfs(self.base)
|
||||||
|
self.cfg["cache_budget_percent"] = (4096 + 100) / (st.f_blocks * st.f_frsize) * 100
|
||||||
|
self._write_config()
|
||||||
|
self.assertEqual(self._run("cleanup", "--caches").returncode, 0)
|
||||||
|
self.assertFalse(old.exists())
|
||||||
|
self.assertTrue(new.exists())
|
||||||
|
|
||||||
|
def test_without_config_every_command_is_a_no_op(self):
|
||||||
|
self.config.unlink()
|
||||||
|
result = self._run("status")
|
||||||
|
self.assertEqual((result.returncode, result.stdout), (0, ""))
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
Reference in new issue
Block a user