diff --git a/README.md b/README.md index 28bce55..f5c3447 100644 --- a/README.md +++ b/README.md @@ -19,6 +19,7 @@ flowchart LR edit_hook([edit in a git worktree]) reminders([reminder hooks]) code_task([code task]) + spawn([subagent spawn]) end subgraph commit [Commit] commit_next[commit-next] @@ -34,6 +35,7 @@ flowchart LR coordinate[coordinate] funnel[funnel] cleanup[cleanup] + subagent_model[subagent-model] end subgraph ci [CI] observe[observe] @@ -81,6 +83,7 @@ flowchart LR reminders -.-> autotune code_task -.-> no_defaults code_task -.-> scope_guard + spawn -.-> subagent_model commit_next --> merge_fix commit_next --> commit_no_verify @@ -96,6 +99,8 @@ flowchart LR cleanup --> democracy funnel --> democracy funnel --> dialectic + funnel --> subagent_model + dialectic --> subagent_model democracy --> merge_fix observe --> triage @@ -155,7 +160,7 @@ Copy the skills into a specific project (`/.agents/skills` and `/.cl make project TARGET=/path/to/repo ``` -Both commands also patch the target's `.claude/settings.local.json` — the machine-local, untracked file, so a project's reviewed `.claude/settings.json` is never rewritten by an install: the caveman and ponytail plugins are enabled so their modes auto-activate on session start, and two reminder hooks are registered — `autotune` points you at `/autotune` at most once an hour, `dream` points you at `/dream` at most once every twelve hours per repository (stamped in that repository's git dir). A PreToolUse gate from the `democracy` skill blocks edits to files in a git worktree until the session holds a claim (`skills/democracy/scripts/council.py`), and its resource watch (`skills/democracy/scripts/watch.py`) measures disk, Docker and caches, cleans up low-risk data and blocks new heavy jobs before the disk runs out; it stays inactive until `~/.config/agents-council/config.json` exists. Existing settings are preserved. +Both commands also patch the target's `.claude/settings.local.json` — the machine-local, untracked file, so a project's reviewed `.claude/settings.json` is never rewritten by an install: the caveman and ponytail plugins are enabled so their modes auto-activate on session start, and two reminder hooks are registered — `autotune` points you at `/autotune` at most once an hour, `dream` points you at `/dream` at most once every twelve hours per repository (stamped in that repository's git dir). A PreToolUse gate from the `democracy` skill blocks edits to files in a git worktree until the session holds a claim (`skills/democracy/scripts/council.py`), and its resource watch (`skills/democracy/scripts/watch.py`) measures disk, Docker and caches, cleans up low-risk data and blocks new heavy jobs before the disk runs out; it stays inactive until `~/.config/agents-council/config.json` exists. Two hooks of the `subagent-model` skill (`skills/subagent-model/scripts/log.py`) log every Agent spawn with its task class, model and effort, and store the quality feedback a sandboxed session prints, in `~/.local/state/agents-council/subagent-model.jsonl`. Existing settings are preserved. Restart your agent afterwards so it loads the new skills. diff --git a/scripts/enable-plugins.js b/scripts/enable-plugins.js index 8bef697..93e3189 100644 --- a/scripts/enable-plugins.js +++ b/scripts/enable-plugins.js @@ -1,7 +1,7 @@ #!/usr/bin/env node // Enable the caveman and ponytail plugins in a Claude Code settings.json so // their SessionStart hooks auto-activate both modes, register the autotune -// and dream reminder hooks, the democracy claim gate and the resource watch. Merges non-destructively: existing marketplaces, +// and dream reminder hooks, the democracy claim gate, the resource watch and the subagent-model log. Merges non-destructively: existing marketplaces, // plugins and hooks are preserved, unparseable files are left untouched. "use strict"; @@ -60,21 +60,23 @@ if (!JSON.stringify(data.hooks.PreToolUse).includes("democracy/scripts/council.p }); } -const watch = path.join(path.dirname(settingsPath), "skills", "democracy", "scripts", "watch.py"); -const WATCH_HOOKS = [ - ["UserPromptSubmit", null, "watch"], - ["PostToolUse", "Bash", "watch"], - ["PreToolUse", "Bash", "block"], +const SCRIPT_HOOKS = [ + ["democracy/scripts/watch.py", "UserPromptSubmit", null, "watch"], + ["democracy/scripts/watch.py", "PostToolUse", "Bash", "watch"], + ["democracy/scripts/watch.py", "PreToolUse", "Bash", "block"], + ["subagent-model/scripts/log.py", "PreToolUse", "Agent|Task", "spawn"], + ["subagent-model/scripts/log.py", "PostToolUse", "Bash", "persist"], ]; -for (const [event, matcher, sub] of WATCH_HOOKS) { +for (const [script, event, matcher, sub] of SCRIPT_HOOKS) { data.hooks[event] = data.hooks[event] || []; const registered = data.hooks[event].some((entry) => (entry.hooks || []).some((hook) => - String(hook.command).includes("democracy/scripts/watch.py") && String(hook.command).endsWith(` ${sub}`))); + String(hook.command).includes(script) && String(hook.command).endsWith(` ${sub}`))); if (registered) continue; - const entry = { hooks: [{ type: "command", command: `python3 ${JSON.stringify(watch)} ${sub}` }] }; + const file = path.join(path.dirname(settingsPath), "skills", ...script.split("/")); + const entry = { hooks: [{ type: "command", command: `python3 ${JSON.stringify(file)} ${sub}` }] }; data.hooks[event].push(matcher ? { matcher, ...entry } : entry); } fs.writeFileSync(settingsPath, JSON.stringify(data, null, 2) + "\n"); -console.log(`skills: enabled caveman + ponytail plugins, the ${REMINDERS.join(" + ")} reminders, the democracy claim gate and resource watch in ${settingsPath}`); +console.log(`skills: enabled caveman + ponytail plugins, the ${REMINDERS.join(" + ")} reminders, the democracy claim gate, the resource watch and the subagent-model log in ${settingsPath}`); diff --git a/skills/dialectic/SKILL.md b/skills/dialectic/SKILL.md index 66cbd95..7374134 100644 --- a/skills/dialectic/SKILL.md +++ b/skills/dialectic/SKILL.md @@ -30,7 +30,8 @@ asking; ask only when an operator-only fact blocks the conclusion. make or CLI helpers the project exposes); CI logs and artifacts; tests; docs. Cite `file:line` and command output — never a bare assertion. 2. **Antithesis.** Spawn MULTIPLE independent skeptics (parallel subagents or a - workflow) whose only job is to REFUTE the thesis. Each re-inspects the + workflow, on the default model as the `subagent-model` skill requires) + whose only job is to REFUTE the thesis. Each re-inspects the sources itself and defaults to "refuted" on any concrete hole. Give diverse skeptics diverse lenses: correctness, an alternate mechanism, does-it- reproduce, does-the-fix-break-something-else. diff --git a/skills/funnel/SKILL.md b/skills/funnel/SKILL.md index f38891a..279e9e4 100644 --- a/skills/funnel/SKILL.md +++ b/skills/funnel/SKILL.md @@ -20,7 +20,8 @@ in a sidetree until it does, or leaves as a proposal. ## The stages -1. **Brief.** One concern per agent. Name the goal, the exact base commit, the +1. **Brief.** One concern per agent, on the model the `subagent-model` skill + picks. Name the goal, the exact base commit, the scope that may change, the evidence expected back, a time cap (default 60 minutes) at which it reports its interim state, and what the agent must not do: no pushes, no gates or deploys diff --git a/skills/subagent-model/SKILL.md b/skills/subagent-model/SKILL.md new file mode 100644 index 0000000..c18df3e --- /dev/null +++ b/skills/subagent-model/SKILL.md @@ -0,0 +1,72 @@ +--- +name: subagent-model +description: > + Pick `model` and `effort` per subagent from a fixed table: cheaper only on a + clear match at 90 % that the result equals the default's; log spawns and + quality. Apply AUTOMATICALLY before every Agent tool call. +--- + +A subagent runs on the main model and effort unless the spawn names others. +Set `model` and `effort` on an Agent call only when the task clearly matches a +row below and you are at least 90 % sure the cheaper setting delivers the same +result as the default: no lost findings, no rework. Cost counts per finished +task, retries included. Otherwise set neither, and the default stays. + +## Table + +| Task class | Subagent's task | `model` | `effort` | Evidence | +| --- | --- | --- | --- | --- | +| `lookup` | Read-only: find files, symbols or call sites, grep, list, read and quote, pull a log line or field, summarise a given text without judging it | `haiku` | `low` | Haiku 5.5 costs 1/40 of Opus 5.5 per token and is the speed tier; Anthropic names Haiku at low effort for simple, speed-critical and reading-heavy worker tasks; Explore and the cavecrew investigator already run on it | +| `mechanical-edit` | An edit the brief fully specifies (exact old and new text, a rename, deleting listed lines) in one or two files | `haiku` | `low` | nothing is left to decide, and the parent checks the diff | +| `judgement` | Everything else: root cause, skeptic or dialectic antithesis, review, security or legal checks, design, new or multi-file code, writing tests, performance work | omit | omit | no evidence at 90 % that a cheaper setting gives the same result | + +Prices per million tokens in and out (claude-api reference, cached +2026-10-06): Haiku 5.5 $0.10 / $0.50, Sonnet 5.5 $2 / $10, Opus 5.5 $4 / $20, +Fable 5.1 $10 / $50. + +## Spawn + +1. Pick the class. A task that mixes classes, or whose class you hesitate + over, is `judgement`. +2. Start the brief with the line `task-class: `; the spawn hook logs + it together with `model`, `effort` and the Agent `description`. +3. Set `model` and `effort` from the row; for `judgement` set neither. + +- An agent type whose definition sets its own model (Explore, Plan, the + cavecrew agents) keeps it; never pass `model` or `effort` for it. +- Never pass `fable` or `opus`: neither is cheaper than the default. +- A Haiku agent that reports the task needs judgement, or returns an + incomplete result, is not retried on Haiku: the default continues it. +- Never change the default itself (settings, `CLAUDE_CODE_SUBAGENT_MODEL`, + agent definitions); this skill only chooses per call. + +## Feedback + +Once you have judged the subagent's result, rate it under the same Agent +`description`: + +```bash +${CLAUDE_SKILL_DIR}/scripts/log.py feedback --description "" --completed yes|no --tests pass|fail|none --redo yes|no +``` + +`--redo yes` when the default had to redo or finish the work. When the +operator later asks a follow-up question about that result or corrects it: + +```bash +${CLAUDE_SKILL_DIR}/scripts/log.py followup --description "" --text "" +``` + +All sessions share `~/.local/state/agents-council/subagent-model.jsonl`. +Inside a sandbox that cannot write it, the script prints the line as +`pending:` and the PostToolUse hook stores it. + +## Evaluation + +`log.py report` shows per task class, model and effort the runs, the rated +runs and the share with the same result: completed, not redone, no failing +test, no follow-up. A row is added, or a cheaper setting extended to a class, +only when the report shows at least 90 % over at least 20 rated runs, and only +after the operator confirms it (for example through `/autotune`). A row whose +rate falls below 90 % over 20 rated runs goes back to the default. `sonnet` +enters the table only this way, and a trial run on a setting the table does +not hold needs the operator's order. diff --git a/skills/subagent-model/scripts/log.py b/skills/subagent-model/scripts/log.py new file mode 100755 index 0000000..1dbb405 --- /dev/null +++ b/skills/subagent-model/scripts/log.py @@ -0,0 +1,205 @@ +#!/usr/bin/env python3 +"""Log subagent spawns with their quality feedback, and evaluate the table. + +Usage: + log.py spawn hook (PreToolUse on Agent): append the spawn line + log.py feedback --description D --completed yes|no --tests pass|fail|none --redo yes|no + rate the latest spawn with this Agent description + log.py followup --description D --text T + record an operator follow-up question or correction on it + log.py persist hook (PostToolUse on Bash): append the one line a sandboxed + feedback or followup printed as `pending: `, only + when the command was that bare call and the line is + well-formed and from the hook's own session + log.py report runs, rated runs and same-result rate per task class, + model and effort; a row qualifies at >= 90 % over >= 20 + +Environment: + SUBAGENT_MODEL_LOG: the log shared by every session; + ~/.local/state/agents-council/subagent-model.jsonl unless set. + CLAUDE_CODE_SESSION_ID: the session that rates its spawns. +""" + +from __future__ import annotations + +import argparse +import datetime +import json +import os +import re +import shlex +import sys +from pathlib import Path + +LOG = Path( + os.environ.get( + "SUBAGENT_MODEL_LOG", + str(Path.home() / ".local" / "state" / "agents-council" / "subagent-model.jsonl"), + ) +) +TASK_CLASS = re.compile(r"^task-class:\s*(\S+)", re.MULTILINE) +PENDING = "pending: " +SCRIPT = "subagent-model/scripts/log.py" +CONTROL = re.compile(r"^[();<>|&]+$") +KEYS = { + "feedback": {"kind", "time", "session", "description", "completed", "tests", "redo"}, + "followup": {"kind", "time", "session", "description", "text"}, +} +QUALIFY_RUNS = 20 +QUALIFY_RATE = 0.9 + + +def now() -> str: + return datetime.datetime.now(datetime.UTC).isoformat(timespec="seconds") + + +def append(line: dict) -> None: + LOG.parent.mkdir(parents=True, exist_ok=True) + with LOG.open("a", encoding="utf-8") as handle: + handle.write(json.dumps(line, sort_keys=True) + "\n") + + +def record(line: dict) -> int: + try: + append(line) + except OSError: + print(PENDING + json.dumps(line, sort_keys=True)) + return 0 + + +def cmd_spawn() -> int: + payload = json.load(sys.stdin) + tool = payload.get("tool_input") or {} + match = TASK_CLASS.search(tool.get("prompt", "")) + append({ + "kind": "spawn", + "time": now(), + "session": payload.get("session_id", ""), + "description": tool.get("description", ""), + "subagent_type": tool.get("subagent_type") or "general-purpose", + "task_class": match.group(1) if match else "unclassified", + "model": tool.get("model") or "default", + "effort": tool.get("effort") or "default", + }) + return 0 + + +def rated(kind: str, args: argparse.Namespace, **fields: object) -> dict: + session = os.environ.get("CLAUDE_CODE_SESSION_ID") + if not session: + sys.exit("log.py: CLAUDE_CODE_SESSION_ID is not set") + return {"kind": kind, "time": now(), "session": session, "description": args.description, **fields} + + +def invoked_kind(command: str) -> str | None: + if "$(" in command or "`" in command: + return None + lexer = shlex.shlex(command, posix=True, punctuation_chars=True) + lexer.whitespace_split = True + try: + tokens = list(lexer) + except ValueError: + return None + if any(CONTROL.match(token) for token in tokens): + return None + if tokens[:1] in (["python3"], ["python"]): + tokens = tokens[1:] + if len(tokens) < 2 or not tokens[0].endswith(SCRIPT) or tokens[1] not in KEYS: + return None + return tokens[1] + + +def well_formed(line: object, kind: str, session: str) -> bool: + if not isinstance(line, dict) or set(line) != KEYS[kind] or line["kind"] != kind or line["session"] != session: + return False + if not all(isinstance(line[key], str) for key in ("time", "description")): + return False + if kind == "feedback": + return isinstance(line["completed"], bool) and isinstance(line["redo"], bool) and line["tests"] in ("pass", "fail", "none") + return isinstance(line["text"], str) + + +def cmd_persist() -> int: + payload = json.load(sys.stdin) + kind = invoked_kind((payload.get("tool_input") or {}).get("command", "")) + if not kind: + return 0 + response = payload.get("tool_response") + stdout = response.get("stdout", "") if isinstance(response, dict) else str(response or "") + pending = [text[len(PENDING):] for text in stdout.splitlines() if text.startswith(PENDING)] + if len(pending) != 1: + return 0 + try: + line = json.loads(pending[0]) + except json.JSONDecodeError: + return 0 + if well_formed(line, kind, payload.get("session_id", "")): + append(line) + return 0 + + +def runs() -> list[dict]: + current: dict[tuple[str, str], dict] = {} + found: list[dict] = [] + for text in LOG.read_text(encoding="utf-8").splitlines() if LOG.exists() else []: + line = json.loads(text) + key = (line["session"], line["description"]) + if line["kind"] == "spawn": + current[key] = {**line, "feedback": None, "followups": 0} + found.append(current[key]) + elif key in current and line["kind"] == "feedback": + current[key]["feedback"] = line + elif key in current and line["kind"] == "followup": + current[key]["followups"] += 1 + return found + + +def same_result(run: dict) -> bool: + feedback = run["feedback"] + return feedback["completed"] and not feedback["redo"] and feedback["tests"] != "fail" and not run["followups"] + + +def cmd_report() -> int: + rows: dict[tuple[str, str, str], list[dict]] = {} + for run in runs(): + rows.setdefault((run["task_class"], run["model"], run["effort"]), []).append(run) + print("task_class\tmodel\teffort\truns\trated\tsame\trate\tqualifies") + for (task_class, model, effort), group in sorted(rows.items()): + rated_runs = [r for r in group if r["feedback"]] + same = sum(map(same_result, rated_runs)) + rate = same / len(rated_runs) if rated_runs else 0.0 + qualifies = len(rated_runs) >= QUALIFY_RUNS and rate >= QUALIFY_RATE + print(f"{task_class}\t{model}\t{effort}\t{len(group)}\t{len(rated_runs)}\t{same}\t{rate:.0%}\t{'yes' if qualifies else 'no'}") + return 0 + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + sub = parser.add_subparsers(dest="command", required=True) + sub.add_parser("spawn") + feedback = sub.add_parser("feedback") + feedback.add_argument("--description", required=True) + feedback.add_argument("--completed", required=True, choices=("yes", "no")) + feedback.add_argument("--tests", required=True, choices=("pass", "fail", "none")) + feedback.add_argument("--redo", required=True, choices=("yes", "no")) + followup = sub.add_parser("followup") + followup.add_argument("--description", required=True) + followup.add_argument("--text", required=True) + sub.add_parser("persist") + sub.add_parser("report") + args = parser.parse_args() + if args.command == "spawn": + return cmd_spawn() + if args.command == "feedback": + return record(rated( + "feedback", args, completed=args.completed == "yes", tests=args.tests, redo=args.redo == "yes" + )) + if args.command == "followup": + return record(rated("followup", args, text=args.text)) + if args.command == "persist": + return cmd_persist() + return cmd_report() + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/test_dream.py b/tests/test_dream.py index 74c5fc4..87ee609 100644 --- a/tests/test_dream.py +++ b/tests/test_dream.py @@ -132,6 +132,8 @@ class TestSettingsRegistration(unittest.TestCase): self.assertEqual(entries.count("democracy/scripts/watch.py"), 1) post = json.dumps(data["hooks"]["PostToolUse"]) self.assertEqual(post.count("democracy/scripts/watch.py"), 1) + self.assertEqual(gate.count("subagent-model/scripts/log.py"), 1) + self.assertEqual(post.count("subagent-model/scripts/log.py"), 1) if __name__ == "__main__": diff --git a/tests/test_subagent_model.py b/tests/test_subagent_model.py new file mode 100644 index 0000000..bc5e856 --- /dev/null +++ b/tests/test_subagent_model.py @@ -0,0 +1,107 @@ +"""Exercise the subagent-model log: spawn hook, feedback, sandbox persist, report.""" + +from __future__ import annotations + +import json +import subprocess +import sys +import tempfile +import unittest +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parents[1] +LOG_PY = REPO_ROOT / "skills" / "subagent-model" / "scripts" / "log.py" + + +class TestSubagentModelLog(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.TemporaryDirectory() + self.addCleanup(self.tmp.cleanup) + self.base = Path(self.tmp.name) + self.log = self.base / "state" / "subagent-model.jsonl" + + def _run(self, *args: str, stdin: str = "", log: Path | None = None) -> subprocess.CompletedProcess: + env = {"SUBAGENT_MODEL_LOG": str(log or self.log), "CLAUDE_CODE_SESSION_ID": "s1"} + return subprocess.run( + [sys.executable, str(LOG_PY), *args], capture_output=True, text=True, env=env, input=stdin, check=False + ) + + def _spawn(self, description: str, prompt: str, **tool: str) -> None: + payload = {"session_id": "s1", "tool_input": {"description": description, "prompt": prompt, **tool}} + self.assertEqual(self._run("spawn", stdin=json.dumps(payload)).returncode, 0) + + def _feedback(self, description: str, completed: str = "yes", redo: str = "no", tests: str = "none") -> None: + result = self._run( + "feedback", "--description", description, "--completed", completed, "--tests", tests, "--redo", redo + ) + self.assertEqual(result.returncode, 0, result.stderr) + + def _lines(self) -> list[dict]: + return [json.loads(text) for text in self.log.read_text(encoding="utf-8").splitlines()] + + def _report(self) -> dict[str, list[str]]: + rows = self._run("report").stdout.splitlines()[1:] + return {row.split("\t")[0]: row.split("\t") for row in rows} + + def test_spawn_logs_class_model_and_effort_or_the_defaults(self): + self._spawn("find callers", "task-class: lookup\nFind every caller.", model="haiku", effort="low") + self._spawn("fix it", "Find the root cause.") + first, second = self._lines() + self.assertEqual((first["task_class"], first["model"], first["effort"]), ("lookup", "haiku", "low")) + self.assertEqual((second["task_class"], second["model"], second["effort"]), ("unclassified", "default", "default")) + + def _pending_feedback(self) -> str: + blocked = self.base / "file" + blocked.write_text("", encoding="utf-8") + result = self._run( + "feedback", "--description", "d", "--completed", "yes", "--tests", "none", "--redo", "no", + log=blocked / "log.jsonl", + ) + self.assertTrue(result.stdout.startswith("pending: "), result.stdout) + return result.stdout + + def _persist(self, command: str, stdout: str, session: str = "s1") -> None: + payload = {"session_id": session, "tool_input": {"command": command}, "tool_response": {"stdout": stdout}} + self.assertEqual(self._run("persist", stdin=json.dumps(payload)).returncode, 0) + + def test_sandboxed_feedback_is_printed_and_stored_by_the_persist_hook(self): + stdout = self._pending_feedback() + self._persist("python3 /x/skills/subagent-model/scripts/log.py feedback --description 'd'", stdout) + self.assertEqual(self._lines()[0]["kind"], "feedback") + + def test_persist_rejects_pending_lines_it_did_not_get_from_a_bare_log_call(self): + stdout = self._pending_feedback() + line = json.loads(stdout[len("pending: "):]) + call = "/x/skills/subagent-model/scripts/log.py feedback --description d" + for command, output, session in ( + ("cat ci.log", stdout, "s1"), + (f"{call}; cat ci.log", stdout, "s1"), + (f"{call} | tee /tmp/x", stdout, "s1"), + (f"echo $({call})", stdout, "s1"), + (call, stdout, "other-session"), + (call, stdout + stdout, "s1"), + (call, "pending: " + json.dumps({**line, "extra": 1}), "s1"), + (call, "pending: " + json.dumps({**line, "completed": "yes"}), "s1"), + ("/x/skills/subagent-model/scripts/log.py followup --description d", stdout, "s1"), + ): + self._persist(command, output, session) + self.assertFalse(self.log.exists()) + + def test_report_qualifies_a_row_only_at_ninety_percent_over_twenty_rated_runs(self): + for i in range(20): + self._spawn(f"q{i}", "task-class: lookup", model="haiku", effort="low") + self._feedback(f"q{i}", redo="yes" if i < 2 else "no") + for i in range(19): + self._spawn(f"r{i}", "task-class: mechanical-edit", model="haiku", effort="low") + self._feedback(f"r{i}") + self._run("followup", "--description", "q5", "--text", "that list missed a caller") + report = self._report() + self.assertEqual(report["lookup"][3:], ["20", "20", "17", "85%", "no"]) + self.assertEqual(report["mechanical-edit"][3:], ["19", "19", "19", "100%", "no"]) + self._spawn("r19", "task-class: mechanical-edit", model="haiku", effort="low") + self._feedback("r19") + self.assertEqual(self._report()["mechanical-edit"][-1], "yes") + + +if __name__ == "__main__": + unittest.main()