kevinveenbirkenbach/skills
1
0 Fork 0
Code Issues Pull Requests Actions 4 Packages Projects Releases Wiki Activity

feat(orchestrator): W4 resources view and host samples

Browse Source 
A resources timer samples host CPU, RAM, swap and per-session tokens into samples.jsonl; the hub serves them and a Resources view charts them. Committed below the bar on the operator's word 2026-11-12 (no further per-bundle tests, one gate after the merge): the W4 e2e never ran in a real browser.

🎯 Fix: 92 %
💥 Break: 6 %
💸 Cost: 100 %
⏱️ Time: 100 %
🛡️ Security: 100 %
🏆 Score: 86
🚦 Gate: ⛔ break 6 % > 5

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Kevin Veen-BirkenbachandClaude Opus 5.5 committed 2026-11-12 16:15:43 +00:00
1 parent fd8fc08603
commit 35c270e120
21 files changed
+1937 -18
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

tests/test_resources_sample.py
+294
View File
@@ -0,0 +1,294 @@
from __future__ import annotations
import json
import os
import subprocess
import sys
import tempfile
import unittest
from pathlib import Path
from unittest import mock
import hub_support
import resources as sampler
SCRIPT = hub_support.HUB / "client" / "resources.py"
UNITS = hub_support.HUB / "systemd"
NOW = 1_800_000_000
DAY = 86400
MIB = 1 << 20
DF = """Filesystem 1024-blocks Used Available Capacity Mounted on
/dev/nvme0n1p2 976762584 512000000 460000000 53% /
devtmpfs 8000000 0 8000000 0% /dev
tmpfs 8000000 1000 7999000 1% /dev/shm
/dev/nvme0n1p2 976762584 512000000 460000000 53% /home
/dev/nvme0n1p1 523248 12000 511248 3% /boot/efi
/dev/loop3 65536 65536 0 100% /snap/core/1
/dev/sda1 1953514584 100000000 1853514584 6% /mnt/Backup Disk
overlay 976762584 512000000 460000000 53% /var/lib/docker/overlay2/x/merged
"""
DISKS = [
{"mount": "/", "used": 512000000 * 1024, "total": 976762584 * 1024},
{"mount": "/boot/efi", "used": 12000 * 1024, "total": 523248 * 1024},
{"mount": "/mnt/Backup Disk", "used": 100000000 * 1024, "total": 1953514584 * 1024},
]
MEMINFO = """MemTotal: 32000000 kB
MemFree: 2000000 kB
MemAvailable: 24000000 kB
Buffers: 500000 kB
SwapTotal: 8000000 kB
SwapFree: 6000000 kB
HugePages_Total: 0
"""
STAT_BEFORE = "cpu 1000 0 500 8000 500 0 0 0 0 0\ncpu0 500 0 250 4000 250 0 0 0 0 0\n"
STAT_AFTER = "cpu 1600 0 700 8400 600 0 0 0 0 0\ncpu0 800 0 350 4200 300 0 0 0 0 0\n"
STAT_LATER = "cpu 2000 0 900 8600 700 0 0 0 0 0\n"
LOADAVG = "1.25 0.80 0.50 2/1234 56789\n"
NVIDIA_SMI = "00000000:01:00.0, 37, 2048, 8188\n00000000:03:00.0, [N/A], [N/A], 4096\n"
AWAKE, ASLEEP = "0000:01:00.0", "0000:02:00.0"
STUB = """#!/usr/bin/env python3
import json, os, sys
from pathlib import Path
with Path(os.environ["STUB_LOG"]).open("a", encoding="utf-8") as handle:
handle.write(json.dumps([Path(sys.argv[0]).name, *sys.argv[1:]]) + "\\n")
sys.stdout.write(os.environ["STUB_" + Path(sys.argv[0]).name.replace("-", "_").upper()])
"""
def device(root: Path, address: str, vendor: str, kind: str, status: str | None) -> None:
folder = root / "bus" / "pci" / "devices" / address
(folder / "power").mkdir(parents=True)
(folder / "vendor").write_text(f"{vendor}\n", encoding="utf-8")
(folder / "class").write_text(f"{kind}\n", encoding="utf-8")
if status is not None:
(folder / "power" / "runtime_status").write_text(f"{status}\n", encoding="utf-8")
def host_sys(root: Path) -> Path:
device(root, AWAKE, "0x10de", "0x030000", "active")
device(root, "0000:01:00.1", "0x10de", "0x040300", "active")
device(root, ASLEEP, "0x10de", "0x030200", "suspended")
device(root, "0000:00:02.0", "0x8086", "0x030000", "active")
return root
def row(t: int, cpu: float, used: int, waiting: int, gpu: dict) -> dict:
return {"t": t, "disks": [{"mount": "/", "used": used, "total": 1000}], "cpu": cpu, "ram": cpu / 2, "swap": 0.0, "load": cpu / 10, "gpus": [gpu], "queue": {"waiting": waiting, "active": 1}}
class TestParsing(unittest.TestCase):
def test_df_keeps_each_local_block_device_once_with_its_first_mount_in_bytes(self):
self.assertEqual(sampler.parse_df(DF), DISKS)
self.assertEqual(sampler.parse_df(""), [])
def test_meminfo_gives_ram_and_swap_in_percent_and_no_swap_as_zero(self):
self.assertEqual(sampler.memory(sampler.parse_meminfo(MEMINFO)), (25.0, 25.0))
without = MEMINFO.replace("SwapTotal: 8000000", "SwapTotal: 0").replace("SwapFree: 6000000", "SwapFree: 0")
self.assertEqual(sampler.memory(sampler.parse_meminfo(without)), (25.0, 0.0))
self.assertEqual(sampler.memory({}), (None, None))
def test_cpu_percent_is_the_busy_share_of_the_ticks_between_two_reads(self):
before, after = sampler.cpu_ticks(STAT_BEFORE), sampler.cpu_ticks(STAT_AFTER)
self.assertEqual((before, after), ((8500, 10000), (9000, 11300)))
self.assertEqual(sampler.cpu_percent(before, after), 61.5)
self.assertIsNone(sampler.cpu_percent(after, after))
self.assertIsNone(sampler.cpu_ticks("intr 1 2 3\n"))
def test_loadavg_is_the_one_minute_load(self):
self.assertEqual(sampler.parse_loadavg(LOADAVG), 1.25)
self.assertIsNone(sampler.parse_loadavg(""))
def test_nvidia_smi_csv_gives_util_and_vram_in_bytes_per_pci_address(self):
self.assertEqual(sampler.parse_nvidia(NVIDIA_SMI), {
AWAKE: {"util": 37.0, "vram_used": 2048 * MIB, "vram_total": 8188 * MIB},
"0000:03:00.0": {"util": None, "vram_used": None, "vram_total": 4096 * MIB},
})
def test_the_queue_counts_waiting_and_active_entries(self):
with tempfile.TemporaryDirectory() as tmp:
council = Path(tmp)
self.assertEqual(sampler.queue(council), {"waiting": 0, "active": 0})
entries = [{"status": "waiting"}, {"status": "waiting"}, {"status": "active"}, {"status": "done"}, "junk"]
(council / "queue.json").write_text(json.dumps({"entries": entries}), encoding="utf-8")
self.assertEqual(sampler.queue(council), {"waiting": 2, "active": 1})
class TestCpu(unittest.TestCase):
def test_cpu_is_the_busy_share_since_the_ticks_the_run_before_kept_and_the_sampler_never_sleeps(self):
with tempfile.TemporaryDirectory() as tmp:
proc, state = Path(tmp) / "proc", Path(tmp) / "state"
proc.mkdir()
state.mkdir()
(proc / "stat").write_text(STAT_BEFORE, encoding="utf-8")
with mock.patch.object(sampler.time, "sleep", side_effect=AssertionError("slept")), mock.patch.object(sampler.time, "time", return_value=NOW - 60):
self.assertIsNone(sampler.sample(proc, Path(tmp) / "sys", Path(tmp), lambda argv: None, state)["cpu"])
(proc / "stat").write_text(STAT_AFTER, encoding="utf-8")
self.assertEqual(sampler.cpu_since(proc, state, NOW), 61.5)
(proc / "stat").write_text(STAT_LATER, encoding="utf-8")
self.assertIsNone(sampler.cpu_since(proc, state, NOW + sampler.BUCKET + 1))
self.assertEqual(json.loads((state / "cpu.json").read_text(encoding="utf-8")), [NOW + sampler.BUCKET + 1, 9300, 12200])
class TestGpus(unittest.TestCase):
def setUp(self):
tmp = tempfile.TemporaryDirectory()
self.addCleanup(tmp.cleanup)
self.root = Path(tmp.name)
self.calls = []
def run_smi(self, argv: list[str]) -> str:
self.calls.append(argv)
return NVIDIA_SMI
def test_an_asleep_gpu_is_recorded_asleep_and_nvidia_smi_queries_only_the_awake_one(self):
found = sampler.gpus(host_sys(self.root), self.run_smi)
self.assertEqual(found, [{"name": AWAKE, "util": 37.0, "vram_used": 2048 * MIB, "vram_total": 8188 * MIB}, {"name": ASLEEP, "asleep": True}])
self.assertEqual(len(self.calls), 1)
self.assertEqual(self.calls[0][0], "nvidia-smi")
self.assertIn(f"--id={AWAKE}", self.calls[0])
self.assertNotIn(ASLEEP, " ".join(self.calls[0]))
def test_nvidia_smi_never_runs_while_every_nvidia_gpu_sleeps_or_lacks_a_runtime_status(self):
device(self.root, ASLEEP, "0x10de", "0x030000", "suspended")
device(self.root, "0000:04:00.0", "0x10de", "0x030000", None)
self.assertEqual(sampler.gpus(self.root, self.run_smi), [{"name": ASLEEP, "asleep": True}, {"name": "0000:04:00.0", "asleep": True}])
self.assertEqual(self.calls, [])
def test_an_awake_gpu_nvidia_smi_gives_no_reading_for_is_recorded_unknown(self):
unknown = {"name": AWAKE, "util": None, "vram_used": None, "vram_total": None}
self.assertEqual(sampler.gpus(host_sys(self.root), lambda argv: None), [unknown, {"name": ASLEEP, "asleep": True}])
self.assertEqual(sampler.gpus(self.root, lambda argv: NVIDIA_SMI.splitlines()[1]), [unknown, {"name": ASLEEP, "asleep": True}])
self.assertEqual(sampler.mean_row(NOW, [row(NOW + 60, 1.0, 1, 0, unknown), row(NOW + 120, 1.0, 1, 0, unknown)])["gpus"], [unknown])
def test_no_nvidia_gpu_means_no_gpu_row_and_no_call(self):
device(self.root, "0000:00:02.0", "0x8086", "0x030000", "active")
self.assertEqual(sampler.gpus(self.root, self.run_smi), [])
self.assertEqual(sampler.gpus(self.root / "missing", self.run_smi), [])
self.assertEqual(self.calls, [])
class TestRetention(unittest.TestCase):
def test_rows_older_than_seven_days_fold_into_fifteen_minute_means_and_means_past_ninety_days_drop(self):
bucket = NOW - 8 * DAY
asleep, awake = {"name": AWAKE, "asleep": True}, {"name": AWAKE, "util": 50.0, "vram_used": 300, "vram_total": 1000}
mean = {**row(NOW - 30 * DAY, 12.0, 100, 0, asleep)}
rows = [
row(NOW - 91 * DAY, 5.0, 1, 0, asleep),
mean,
row(bucket, 10.0, 100, 0, asleep),
row(bucket + 60, 20.0, 200, 1, asleep),
{**row(bucket + 120, 60.0, 300, 2, awake), "disks": [{"mount": "/", "used": 300, "total": 1000}, {"mount": "/mnt", "used": 10, "total": 20}]},
row(bucket + 1000, 40.0, 400, 0, awake),
row(NOW - 60, 70.0, 700, 3, awake),
]
kept = sampler.retain(rows, NOW)
self.assertEqual([entry["t"] for entry in kept], [NOW - 30 * DAY, bucket, bucket + 900, NOW - 60])
self.assertIs(kept[0], mean)
self.assertEqual(kept[1], {
"t": bucket, "disks": [{"mount": "/", "used": 200, "total": 1000}, {"mount": "/mnt", "used": 10, "total": 20}], "cpu": 30.0, "ram": 15.0, "swap": 0.0, "load": 3.0,
"gpus": [{"name": AWAKE, "util": 50.0, "vram_used": 300, "vram_total": 1000}], "queue": {"waiting": 1.0, "active": 1.0},
})
self.assertEqual((kept[2]["cpu"], kept[2]["disks"]), (40.0, [{"mount": "/", "used": 400, "total": 1000}]))
self.assertEqual(kept[3], rows[-1])
self.assertEqual(sampler.retain(kept, NOW), kept)
def test_a_bucket_still_reaching_into_the_last_seven_days_stays_raw_and_asleep_buckets_stay_asleep(self):
now = NOW + 300
asleep = {"name": ASLEEP, "asleep": True}
rows = [row(NOW - 7 * DAY - 890, 10.0, 1, 0, asleep), row(NOW - 7 * DAY - 830, 30.0, 3, 0, asleep), row(NOW - 7 * DAY + 10, 50.0, 5, 0, asleep)]
kept = sampler.retain(rows, now)
self.assertEqual([entry["t"] for entry in kept], [NOW - 7 * DAY - 900, NOW - 7 * DAY + 10])
self.assertEqual((kept[0]["cpu"], kept[0]["gpus"]), (20.0, [asleep]))
self.assertEqual(kept[1], rows[2])
def test_record_appends_and_folds_only_on_the_first_minute_of_a_bucket(self):
with tempfile.TemporaryDirectory() as tmp:
state = Path(tmp)
asleep = {"name": AWAKE, "asleep": True}
old = [row(NOW - 8 * DAY, 10.0, 1, 0, asleep), row(NOW - 8 * DAY + 60, 30.0, 1, 0, asleep)]
(state / "samples.jsonl").write_text("".join(json.dumps(entry) + "\n" for entry in old), encoding="utf-8")
later = row(NOW + 400, 1.0, 1, 0, asleep)
sampler.record(state, later, NOW + 400)
lines = [json.loads(line) for line in (state / "samples.jsonl").read_text(encoding="utf-8").splitlines()]
self.assertEqual(lines, [*old, later])
first = row(NOW + 930, 2.0, 1, 0, asleep)
sampler.record(state, first, NOW + 930)
lines = [json.loads(line) for line in (state / "samples.jsonl").read_text(encoding="utf-8").splitlines()]
self.assertEqual([entry["t"] for entry in lines], [NOW - 8 * DAY, NOW + 400, NOW + 930])
self.assertEqual(lines[0]["cpu"], 20.0)
class TestSamplerInScratchHome(unittest.TestCase):
def setUp(self):
tmp = tempfile.TemporaryDirectory()
self.addCleanup(tmp.cleanup)
base = Path(tmp.name)
self.home, self.council, stubs, self.proc = base / "home", base / "council", base / "bin", base / "proc"
for folder in (self.home, self.council, stubs, self.proc):
folder.mkdir()
for name, text in (("stat", STAT_BEFORE), ("meminfo", MEMINFO), ("loadavg", LOADAVG)):
(self.proc / name).write_text(text, encoding="utf-8")
for tool in ("df", "nvidia-smi"):
(stubs / tool).write_text(STUB, encoding="utf-8")
(stubs / tool).chmod(0o755)
self.sys = host_sys(base / "sys")
self.log = base / "calls.jsonl"
(self.council / "queue.json").write_text(json.dumps({"entries": [{"status": "waiting"}, {"status": "active"}, {"status": "active"}]}), encoding="utf-8")
self.env = {
**os.environ, "HOME": str(self.home), "PATH": f"{stubs}:/usr/bin:/bin",
"STUB_LOG": str(self.log), "STUB_DF": DF, "STUB_NVIDIA_SMI": NVIDIA_SMI.splitlines()[0] + "\n",
}
self.resources = self.home / ".local" / "state" / "agents-council" / "resources"
def sample(self) -> None:
argv = [sys.executable, str(SCRIPT), "sample", "--state", str(self.resources), "--council", str(self.council), "--proc", str(self.proc), "--sys", str(self.sys)]
done = subprocess.run(argv, env=self.env, capture_output=True, text=True, timeout=60, check=False)
self.assertEqual(done.returncode, 0, done.stderr)
def test_the_sampler_appends_one_row_per_run_under_the_scratch_home(self):
self.sample()
(self.proc / "stat").write_text(STAT_AFTER, encoding="utf-8")
self.sample()
samples = self.resources / "samples.jsonl"
rows = [json.loads(line) for line in samples.read_text(encoding="utf-8").splitlines()]
self.assertEqual(len(rows), 2)
for found in rows:
with self.subTest(t=found["t"]):
self.assertEqual(sorted(found), ["cpu", "disks", "gpus", "load", "queue", "ram", "swap", "t"])
self.assertEqual(found["disks"], DISKS)
self.assertEqual(found["gpus"], [{"name": AWAKE, "util": 37.0, "vram_used": 2048 * MIB, "vram_total": 8188 * MIB}, {"name": ASLEEP, "asleep": True}])
self.assertEqual(found["queue"], {"waiting": 1, "active": 2})
self.assertEqual((found["ram"], found["swap"], found["load"]), (25.0, 25.0, 1.25))
self.assertEqual([found["cpu"] for found in rows], [None, 61.5])
self.assertLessEqual(rows[0]["t"], rows[1]["t"])
calls = [json.loads(line) for line in self.log.read_text(encoding="utf-8").splitlines()]
self.assertEqual([call[0] for call in calls], ["df", "nvidia-smi"] * 2)
self.assertTrue(all(ASLEEP not in " ".join(call) for call in calls), calls)
self.assertEqual(sorted(path.name for path in samples.parent.iterdir()), ["cpu.json", "samples.jsonl"])
class TestTimer(unittest.TestCase):
def test_a_user_timer_runs_sample_and_tokens_every_minute_without_a_page_condition(self):
timer = (UNITS / "kvb-orchestrator-resources.timer").read_text(encoding="utf-8").splitlines()
service = (UNITS / "kvb-orchestrator-resources.service").read_text(encoding="utf-8").splitlines()
self.assertIn("OnCalendar=*-*-* *:*:00", timer)
self.assertIn("WantedBy=timers.target", timer)
script = "/usr/bin/env python3 %h/.local/share/kvb-orchestrator/client/resources.py"
shared = "--state ${KVB_ORCHESTRATOR_KEPT}/resources --council ${KVB_ORCHESTRATOR_COUNCIL}"
self.assertEqual([line for line in service if line.startswith("ExecStart=")], [
f"ExecStart={script} sample {shared} --proc /proc --sys /sys",
f"ExecStart={script} tokens {shared} --transcripts ${{KVB_ORCHESTRATOR_PROJECTS}}",
])
self.assertIn("EnvironmentFile=%h/.local/share/kvb-orchestrator/.env", service)
self.assertIn("Type=oneshot", service)
self.assertFalse(any(line.startswith("ExecCondition=") for line in service))
def test_orchestrator_install_enables_the_resources_timer(self):
script = (hub_support.REPO_ROOT / "scripts" / "orchestrator.sh").read_text(encoding="utf-8")
self.assertIn('RESOURCES_UNIT="${NAME}-resources"', script)
self.assertIn('systemctl --user enable --now "${RESOURCES_UNIT}.timer"', script)
self.assertIn('"${HUB}/client/resources.py"', script)
if __name__ == "__main__":
unittest.main()