perf(skills): cut skill tokens and load hooks from cached bytecode
Hooks start their script as __main__ from cached bytecode instead of recompiling it on every call, and the installer migrates registered old hook commands to that form. A disabled plugin is written as false instead of deleted, because a deleted key in settings.local.json lets a true in settings.json win; the auto-compact window drops to 300000. The coach hour prints one ready line chosen by the time of day instead of asking the agent to load the skill, kvb-cleanup moves its procedure into plan.md, and the token counter reads subagent transcripts too and counts with Counter and defaultdict instead of .get defaults. 🎯 Fix: 88 % 💥 Break: 8 % 💸 Tokens: – ⏱️ Velocity: – 🛡️ Security: 100 % 🏆 Score: 81 🚦 Gate: ⛔ fix 88 % < 90, break 8 % > 5 Created-with: https://s.veen.world/skills Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
d695bb2b58
commit
6c63c364bd
13 files changed
+250
-130
No files matched your search
@@ -89,6 +89,12 @@ class TestCoachHour(unittest.TestCase):
|
||||
self.assertIn("kvb-coach", self._run("2026-11-03T15"))
|
||||
self.assertEqual(sorted(p.name for p in self.hours.iterdir()), ["2026-11-03T15"])
|
||||
|
||||
def test_the_hour_picks_one_fixed_tip_and_loads_no_skill(self):
|
||||
tips = {hour: self._run(f"2026-11-03T{hour}").splitlines() for hour in ("23", "04", "12", "19", "09", "16")}
|
||||
firsts = {hour: lines[1].split(" ", 1)[0] for hour, lines in tips.items()}
|
||||
self.assertEqual(firsts, {"23": "😴", "04": "😴", "12": "🥗", "19": "🥗", "09": "🚶", "16": "🚶"})
|
||||
self.assertTrue(all(len(lines) == 2 and "load no skill" in lines[0] for lines in tips.values()))
|
||||
|
||||
def test_a_binary_planted_on_the_inherited_path_never_runs(self):
|
||||
planted = Path(self.tmp.name) / "planted"
|
||||
planted.mkdir()
|
||||
|
||||
@@ -36,6 +36,9 @@ class TestContext(unittest.TestCase):
|
||||
]
|
||||
(self.root / "project").mkdir()
|
||||
(self.root / "project" / "session.jsonl").write_text("\n".join(lines) + "\n", encoding="utf-8")
|
||||
subagents = self.root / "project" / "session" / "subagents"
|
||||
subagents.mkdir(parents=True)
|
||||
(subagents / "agent-a1.jsonl").write_text(turn(self.today, "s1", 0) + "\n", encoding="utf-8")
|
||||
|
||||
def _run(self, *args: str) -> subprocess.CompletedProcess:
|
||||
return subprocess.run(
|
||||
@@ -52,7 +55,7 @@ class TestContext(unittest.TestCase):
|
||||
)
|
||||
self.assertEqual(
|
||||
(today["day"], today["turns"], today["tokens"], today["compactions"], today["over"]["300000"]),
|
||||
(self.today, 2, 450_100, 1, {"turns": 50, "tokens": 22}),
|
||||
(self.today, 3, 450_200, 1, {"turns": 33, "tokens": 22}),
|
||||
)
|
||||
|
||||
def test_usage_errors_exit_2(self):
|
||||
|
||||
@@ -858,7 +858,7 @@ class TestGuard(unittest.TestCase):
|
||||
self.assertEqual(hits, [])
|
||||
|
||||
def test_the_agent_root_is_named_where_it_is_inventoried_or_mounted(self):
|
||||
self.assertIn("~/.local/state/agents-shared", (REPO_ROOT / "skills" / "kvb-cleanup" / "SKILL.md").read_text(encoding="utf-8"))
|
||||
self.assertIn("~/.local/state/agents-shared", (REPO_ROOT / "skills" / "kvb-cleanup" / "plan.md").read_text(encoding="utf-8"))
|
||||
self.assertIn("~/.local/state/agents-shared.", (CLIENT / "offices.py").read_text(encoding="utf-8"))
|
||||
mounts = (REPO_ROOT / "docs" / "requirements" / "003-operator-workbench.md").read_text(encoding="utf-8").split("- **Mounts**:", 1)[1].split("\n\n", 1)[0]
|
||||
self.assertIn("`~/.local/state/agents-shared`", mounts)
|
||||
|
||||
+37
-4
@@ -3,6 +3,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
import tempfile
|
||||
@@ -222,6 +223,35 @@ class TestSettingsRegistration(unittest.TestCase):
|
||||
result = subprocess.run(["sh", "-c", gate["command"]], capture_output=True, check=False)
|
||||
self.assertEqual(result.returncode, 1)
|
||||
|
||||
def test_a_hook_runs_its_script_as_main_from_cached_bytecode_and_keeps_its_exit_code(self):
|
||||
(gate,) = self._hooks(self._patch(), "PreToolUse", "kvb-org-queue/scripts/queue.py")
|
||||
script = Path(self.tmp.name) / "skills" / "kvb-org-queue" / "scripts" / "queue.py"
|
||||
script.parent.mkdir(parents=True)
|
||||
script.write_text("import sys\nprint(__name__, sys.argv[1:], sys.argv[0] == __file__)\nsys.exit(3)\n", encoding="utf-8")
|
||||
runs = [subprocess.run(["sh", "-c", gate["command"]], capture_output=True, text=True, check=False) for _ in range(2)]
|
||||
self.assertEqual([(run.returncode, run.stdout) for run in runs], [(3, "__main__ ['block'] True\n")] * 2)
|
||||
self.assertEqual([path.name.split(".")[0] for path in (script.parent / "__pycache__").iterdir()], ["queue"])
|
||||
|
||||
def test_every_bash_gate_answers_under_the_cached_loader_as_it_does_started_directly(self):
|
||||
data = self._patch()
|
||||
(Path(self.tmp.name) / "skills").symlink_to(REPO_ROOT / "skills")
|
||||
|
||||
def answer(command: str, home: Path) -> tuple:
|
||||
home.mkdir()
|
||||
env = {"PATH": "/usr/bin:/bin", "HOME": str(home), "AGENTS_COUNCIL_ROOT": str(home / "council"), "XDG_STATE_HOME": str(home / "state")}
|
||||
payload = json.dumps({"session_id": "s", "hook_event_name": "PreToolUse", "tool_name": "Bash", "tool_input": {"command": "ls"}, "cwd": "/"})
|
||||
done = subprocess.run(["sh", "-c", command], input=payload, cwd="/", env=env, capture_output=True, text=True, timeout=60, check=False)
|
||||
return done.returncode, done.stdout.replace(str(home), "~"), done.stderr.replace(str(home), "~")
|
||||
|
||||
bash = [entry for entry in data["hooks"]["PreToolUse"] if "matcher" not in entry or "Bash" in entry["matcher"].split("|")]
|
||||
gates = [re.search(r' ("[^"]+\.py") (\w+)$', hook["command"]) for entry in bash for hook in entry["hooks"] if "importlib" in hook["command"]]
|
||||
self.assertEqual(len(gates), 4)
|
||||
for n, gate in enumerate(gates):
|
||||
with self.subTest(script=gate[1]):
|
||||
cached = answer(gate.string, Path(self.tmp.name) / f"cached-{n}")
|
||||
direct = answer(f"/usr/bin/python3 -I {gate[1]} {gate[2]}", Path(self.tmp.name) / f"direct-{n}")
|
||||
self.assertEqual(cached, direct)
|
||||
|
||||
def test_news_hook_moves_to_radio(self):
|
||||
base = Path(self.tmp.name) / "skills"
|
||||
self.settings.write_text(
|
||||
@@ -243,14 +273,16 @@ class TestSettingsRegistration(unittest.TestCase):
|
||||
encoding="utf-8",
|
||||
)
|
||||
data = self._patch()
|
||||
self.assertNotIn("codebase-memory@codebase-memory", data["enabledPlugins"])
|
||||
self.assertIs(data["enabledPlugins"]["codebase-memory@codebase-memory"], False)
|
||||
self.assertNotIn("codebase-memory", data["extraKnownMarketplaces"])
|
||||
self.assertTrue(data["enabledPlugins"]["caveman@caveman"])
|
||||
self.settings.write_text("{}", encoding="utf-8")
|
||||
self.assertIs(self._patch()["enabledPlugins"]["codebase-memory@codebase-memory"], False)
|
||||
|
||||
def test_compact_window_is_set_once_and_never_overwritten(self):
|
||||
self.assertEqual(self._patch()["env"]["CLAUDE_CODE_AUTO_COMPACT_WINDOW"], "500000")
|
||||
self.settings.write_text(json.dumps({"env": {"CLAUDE_CODE_AUTO_COMPACT_WINDOW": "300000"}}), encoding="utf-8")
|
||||
self.assertEqual(self._patch()["env"]["CLAUDE_CODE_AUTO_COMPACT_WINDOW"], "300000")
|
||||
self.settings.write_text(json.dumps({"env": {"CLAUDE_CODE_AUTO_COMPACT_WINDOW": "500000"}}), encoding="utf-8")
|
||||
self.assertEqual(self._patch()["env"]["CLAUDE_CODE_AUTO_COMPACT_WINDOW"], "500000")
|
||||
|
||||
def test_hooks_run_the_system_python_isolated_and_old_entries_move_to_it(self):
|
||||
old = f'[ -f "{self.tmp.name}/skills/kvb-org-queue/scripts/queue.py" ] && python3 "{self.tmp.name}/skills/kvb-org-queue/scripts/queue.py" block'
|
||||
@@ -266,7 +298,8 @@ class TestSettingsRegistration(unittest.TestCase):
|
||||
with self.subTest(command=command):
|
||||
self.assertIn("/usr/bin/python3 -I ", command)
|
||||
self.assertNotRegex(command, r"(^|&& )python3 ")
|
||||
self.assertEqual(len(self._hooks(data, "PreToolUse", "kvb-org-queue/scripts/queue.py")), 1)
|
||||
(gate,) = self._hooks(data, "PreToolUse", "kvb-org-queue/scripts/queue.py")
|
||||
self.assertIn("spec_from_file_location", gate["command"])
|
||||
|
||||
def test_reminders_run_the_system_bash_and_old_entries_move_to_it(self):
|
||||
old = f'bash "{self.tmp.name}/skills/kvb-dream/hooks/reminder.sh"'
|
||||
|
||||
@@ -29,7 +29,7 @@ BUDGET = REPO_ROOT / "scripts" / "budget.py"
|
||||
SKILL_A = "---\nname: kvb-a\ndescription: >\n one two three\n---\n"
|
||||
SKILL_B = "---\nname: kvb-b\ndescription: >\n four five\n---\n"
|
||||
NO_VALUE = "💰 value ?:? (cost 0/day; pass --value with evidence)"
|
||||
NEW_B = "💰 value ?:? (cost 0/day; pass --value with evidence; kvb-b new: all skills median 0 invocations/day)"
|
||||
NEW_B = "💰 value ?:? (cost 0/day; pass --value with evidence; kvb-b new: all skills median 0 invocations/day, 0 resident turns/day)"
|
||||
CHEAP = "print('cheap')\n"
|
||||
COSTLY = "data = list(range(2_000_000))\nprint(sum(data))\n"
|
||||
LINE = re.compile(
|
||||
@@ -149,6 +149,7 @@ class TestTokens(GitRepo):
|
||||
"evidence": None,
|
||||
"requests_per_day": 0,
|
||||
"invocations_per_day": {"kvb-b": 0},
|
||||
"resident_turns_per_day": {"kvb-b": 0},
|
||||
"median_for_new": {"kvb-b": "all skills"},
|
||||
},
|
||||
},
|
||||
@@ -194,13 +195,13 @@ class TestTokens(GitRepo):
|
||||
result.stdout.splitlines(),
|
||||
[
|
||||
"💸 tokens 869 % (skills 13→113 tokens; listing 3→5 words)",
|
||||
"💰 value ?:? (cost 203/day; pass --value with evidence)",
|
||||
"💰 value ?:? (cost 333/day; pass --value with evidence)",
|
||||
],
|
||||
)
|
||||
self.assertEqual(result.returncode, 3)
|
||||
for value, line in (
|
||||
("4053", "💰 value 1:20 (cost 203/day, value 4053/day; 4053 tokens/day saved)"),
|
||||
("50", "💰 value 4.1:1 (cost 203/day, value 50/day; 4053 tokens/day saved)"),
|
||||
("4053", "💰 value 1:12 (cost 333/day, value 4053/day; 4053 tokens/day saved)"),
|
||||
("50", "💰 value 6.7:1 (cost 333/day, value 50/day; 4053 tokens/day saved)"),
|
||||
):
|
||||
with self.subTest(value=value):
|
||||
result = self.tokens("--value", value, "--evidence", "4053 tokens/day saved")
|
||||
@@ -209,16 +210,30 @@ class TestTokens(GitRepo):
|
||||
self.assertEqual(
|
||||
found,
|
||||
{
|
||||
"quotient": "1:20",
|
||||
"cost": 202.66,
|
||||
"quotient": "1:12",
|
||||
"cost": 332.66,
|
||||
"value": 4053,
|
||||
"evidence": "e",
|
||||
"requests_per_day": 10,
|
||||
"invocations_per_day": {"kvb-a": 2},
|
||||
"resident_turns_per_day": {"kvb-a": 13},
|
||||
"median_for_new": {},
|
||||
},
|
||||
)
|
||||
|
||||
def test_subagent_transcripts_count_and_an_invoked_skill_stays_resident(self):
|
||||
subagent = self.base / "projects" / "project" / "session" / "subagents" / "agent-a1.jsonl"
|
||||
subagent.parent.mkdir(parents=True)
|
||||
lines = [assistant("s0", self.recent, ("u0", "kvb-a")), *(assistant(f"s{n}", self.recent) for n in range(1, 14))]
|
||||
subagent.write_text("\n".join(lines) + "\n", encoding="utf-8")
|
||||
self.write("skills/kvb-a/SKILL.md", SKILL_A.ljust(452, "y"))
|
||||
found = json.loads(self.tokens("--json").stdout)["value"]
|
||||
self.assertEqual(
|
||||
(found["requests_per_day"], found["invocations_per_day"], found["resident_turns_per_day"]),
|
||||
(2, {"kvb-a": round(1 / 7, 2)}, {"kvb-a": round(13 / 7, 2)}),
|
||||
)
|
||||
self.assertEqual(found["cost"], round(100 * (1 / 7 + 0.1 * 13 / 7), 2))
|
||||
|
||||
def test_new_skill_takes_its_category_median(self):
|
||||
self.write("skills/kvb-c-one/SKILL.md", skill_md("kvb-c-one", "one"))
|
||||
self.write("skills/kvb-c-two/SKILL.md", skill_md("kvb-c-two", "two"))
|
||||
@@ -231,7 +246,7 @@ class TestTokens(GitRepo):
|
||||
result = self.tokens()
|
||||
self.assertEqual(
|
||||
result.stdout.splitlines()[1],
|
||||
"💰 value ?:? (cost 202/day; pass --value with evidence; kvb-c-new new: kvb-c median 2 invocations/day)",
|
||||
"💰 value ?:? (cost 1872/day; pass --value with evidence; kvb-c-new new: kvb-c median 2 invocations/day, 167 resident turns/day)",
|
||||
)
|
||||
self.assertEqual(result.returncode, 3)
|
||||
|
||||
|
||||
Reference in new issue
Block a user