Merge feature/workbench-w4-resources into unify/skills-20261112
Conflicts combined W1 (Inbox leaves the Stream for Tasks, Waits on Operator) with W4 (Resources tab): five tabs with Resources, Stream keeps only the Firewall, the inbox hash keeps its redirect; index.html.j2 includes both diffs.js and resources.js. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Kevin Veen-BirkenbachandClaude Opus 5.5 committed
commit
515fe61009
21 files changed
+1937
-18
No files matched your search
@@ -35,9 +35,9 @@ OUT = Path(os.environ.get("E2E_OUT", "/tmp/orchestrator-phase4-shots"))
|
||||
WIDTHS = ((1920, 1080), (1366, 768), (390, 844))
|
||||
VIEWS = ("orbit", "chord", "swimlane", "tree", "3d")
|
||||
TASK_VIEWS = ("list", "board", "rasci", "gantt", "burn", "3d")
|
||||
SCREENS = ["stream", "tasks/operator", "stream/rules", *(f"{tab}/{view}" for tab in ("agents", "artifacts") for view in VIEWS), "agents/hierarchy", *(f"tasks/{view}" for view in TASK_VIEWS)]
|
||||
SCREENS = ["stream", "tasks/operator", "stream/rules", *(f"{tab}/{view}" for tab in ("agents", "artifacts") for view in VIEWS), "agents/hierarchy", *(f"tasks/{view}" for view in TASK_VIEWS), "resources"]
|
||||
SUBS = ["Firewall"]
|
||||
TABS = ["Stream", "Agents", "Artifacts", "Tasks"]
|
||||
TABS = ["Stream", "Agents", "Artifacts", "Tasks", "Resources"]
|
||||
HELPERS = """
|
||||
const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
|
||||
const until = async (check, ms) => {
|
||||
@@ -1785,7 +1785,7 @@ class TestOrchestratorE2E(HubCase):
|
||||
self.assertEqual((probe["title"], probe["h1"]), ("(2) Agent Orchestrator", "Agent Orchestrator"))
|
||||
self.assertEqual(probe["header"], ["radio-open", "bubbles-toggle", "palette-open", "settings-open"])
|
||||
self.assertEqual((probe["tabs"], probe["subs"]), (TABS, SUBS))
|
||||
self.assertEqual(probe["badges"], {"tab-stream": None, "tab-rules": None, "tab-agents": "1", "tab-artifacts": None, "tab-tasks": "2"})
|
||||
self.assertEqual(probe["badges"], {"tab-stream": None, "tab-rules": None, "tab-agents": "1", "tab-artifacts": None, "tab-tasks": "2", "tab-resources": None})
|
||||
self.assertEqual(probe["banner"], "MAYDAY · Docs: disk full")
|
||||
self.assertFalse(probe["rail"])
|
||||
self.assertEqual((probe["time"], probe["replay"]), ("24 h ▾", True))
|
||||
|
||||
@@ -0,0 +1,152 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import sys
|
||||
import time
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
HERE = Path(__file__).resolve().parent
|
||||
sys.path[:0] = [str(HERE.parent), str(HERE)]
|
||||
|
||||
import test_orchestrator_e2e as base
|
||||
from hub_case import HubCase
|
||||
|
||||
OUT = Path(os.environ.get("E2E_OUT", "/tmp/workbench-w4-shots"))
|
||||
GPU = "0000:01:00.0"
|
||||
ROWS = 360
|
||||
WAITED = (120, 180)
|
||||
ASLEEP = 180
|
||||
UNREAD = (300, 320)
|
||||
PANELS = {"disks": "Disks", "host": "CPU, RAM and swap", "gpu": "GPU and VRAM", "tokens": "Tokens per agent"}
|
||||
GUARD = """() => {
|
||||
window.e2eViolations = [];
|
||||
document.addEventListener("securitypolicyviolation", (event) => window.e2eViolations.push(`${event.violatedDirective} ${event.blockedURI}`));
|
||||
}"""
|
||||
CHARTS = """(async () => {@HELPERS@
|
||||
const counted = () => text(document.querySelector('#main [aria-label="Samples"]'));
|
||||
if (!(await until(() => all("#main svg.res-chart").length === 4 && counted() === "@ROWS@", 15000))) {
|
||||
return {error: text(document.getElementById("main")), count: counted()};
|
||||
}
|
||||
await sleep(300);
|
||||
const charts = all("#main svg.res-chart");
|
||||
const number = (node, key) => Number(node.dataset[key]);
|
||||
const attr = (node, key) => Number(node.getAttribute(key));
|
||||
const stacks = {};
|
||||
for (const rect of all("#main rect.res-bar")) {
|
||||
(stacks[rect.dataset.t] = stacks[rect.dataset.t] || []).push({agent: rect.dataset.agent, y: attr(rect, "y"), height: attr(rect, "height")});
|
||||
}
|
||||
const piles = Object.values(stacks).map((parts) => parts.sort((a, b) => b.y - a.y));
|
||||
const labelled = (node) => (node ? [node.getAttribute("title"), node.getAttribute("aria-label")] : null);
|
||||
return {
|
||||
panels: all("#main .res-panel").map((panel) => panel.dataset.panel),
|
||||
axes: [...new Set(charts.map((chart) => `${chart.dataset.from}|${chart.dataset.to}`))],
|
||||
since: number(charts[0], "from"),
|
||||
bands: charts.filter((chart) => chart.dataset.panel !== "tokens").map((chart) => {
|
||||
const [idle, saturated] = [chart.querySelector("rect.band.idle"), chart.querySelector("rect.band.saturated")];
|
||||
return [chart.dataset.panel, attr(idle, "height") / attr(saturated, "height"), attr(idle, "y") > attr(saturated, "y")];
|
||||
}),
|
||||
series: charts.map((chart) => [chart.dataset.panel, [...new Set(all("path.res-line", chart).map((path) => `${path.dataset.series}|${path.dataset.kind}`))]]),
|
||||
waits: all("#main rect.res-mark.wait").map((rect) => [number(rect, "from"), number(rect, "to"), number(rect, "peak")]),
|
||||
asleep: all("#main rect.res-mark.asleep").map((rect) => [number(rect, "from"), number(rect, "to")]),
|
||||
unknown: all("#main rect.res-mark.unknown").map((rect) => [number(rect, "from"), number(rect, "to")]),
|
||||
agents: [...new Set(all("#main rect.res-bar").map((rect) => rect.dataset.agent))].sort(),
|
||||
bottoms: [...new Set(piles.map((parts) => parts[0].agent))],
|
||||
stacked: piles.every((parts) => parts.every((part, index) => !index || Math.abs(part.y + part.height - parts[index - 1].y) < 0.2)),
|
||||
legend: all("#main .res-host .res-key").map(text),
|
||||
labels: {tab: labelled(document.getElementById("tab-resources")), reload: labelled(document.querySelector('#main [aria-label="Reload"]')), titles: all("#main .res-title").map(labelled)},
|
||||
unlabelled: all("#main svg.bi").map((svg) => svg.parentElement).filter((owner) => !text(owner) && !(owner.getAttribute("title") && owner.getAttribute("aria-label"))).length,
|
||||
plots: all("#main .res-plot").map((plot) => plot.getBoundingClientRect().height > 20),
|
||||
tall: document.documentElement.scrollHeight > innerHeight,
|
||||
wide: document.documentElement.scrollWidth > innerWidth + 1,
|
||||
violations: window.e2eViolations || null,
|
||||
};
|
||||
})()"""
|
||||
KEY = """(async () => {@HELPERS@
|
||||
document.body.dispatchEvent(new KeyboardEvent("keydown", {key: "5", bubbles: true}));
|
||||
const opened = Boolean(await until(() => location.hash === "#resources" && document.querySelector("#main svg.res-chart"), 8000));
|
||||
return {opened, current: document.getElementById("tab-resources").getAttribute("aria-current")};
|
||||
})()"""
|
||||
|
||||
|
||||
def probe(template: str) -> str:
|
||||
return template.replace("@HELPERS@", base.HELPERS).replace("@ROWS@", str(ROWS))
|
||||
|
||||
|
||||
def quiet(found: list[str] | None) -> list[str] | None:
|
||||
return None if found is None else [seen for seen in found if not seen.endswith("/favicon.ico")]
|
||||
|
||||
|
||||
@unittest.skipUnless(shutil.which(os.environ.get("CHROMIUM", "chromium")), "no Chromium")
|
||||
class TestWorkbenchResources(HubCase):
|
||||
drive = base.TestOrchestratorE2E.drive
|
||||
|
||||
def setUp(self):
|
||||
super().setUp()
|
||||
OUT.mkdir(parents=True, exist_ok=True)
|
||||
now = int(time.time())
|
||||
self.start = now - now % 60 - ROWS * 60
|
||||
samples = [self.sample(index) for index in range(ROWS)]
|
||||
hour = self.start - self.start % 3600
|
||||
tokens = [{"t": hour + step * 3600, "agent": "Skills", "model": "claude-opus-5-5", "input": 1000 * (step + 1), "output": 500, "cache_read": 20000, "cache_write": 1000} for step in range(7)]
|
||||
tokens += [{"t": hour + step * 3600, "agent": "Stage", "model": "claude-haiku-4-5", "input": 200, "output": 100, "cache_read": 3000, "cache_write": 0} for step in range(0, 7, 2)]
|
||||
folder = self.kept / "resources"
|
||||
folder.mkdir()
|
||||
(folder / "samples.jsonl").write_text(self.jsonl(samples), encoding="utf-8")
|
||||
(folder / "tokens.jsonl").write_text(self.jsonl([row for row in tokens if row["t"] <= now]), encoding="utf-8")
|
||||
self.base = f"http://{self.host}/"
|
||||
|
||||
def sample(self, index: int) -> dict:
|
||||
waited, saturated = WAITED[0] <= index < WAITED[1], WAITED[1] <= index < WAITED[1] + 60
|
||||
cpu, ram, waiting = (25, 35, 2) if waited else (95, 60, 1) if saturated else (10, 20, 0) if index < WAITED[0] else (60, 55, 0)
|
||||
gpu = {"name": GPU, "asleep": True} if index < ASLEEP else {"name": GPU, "util": None, "vram_used": None, "vram_total": None} if UNREAD[0] <= index < UNREAD[1] else {"name": GPU, "util": 50, "vram_used": 2048 << 20, "vram_total": 8192 << 20}
|
||||
root = 950 if index >= ROWS - 60 else 400
|
||||
disks = [{"mount": "/", "used": root << 30, "total": 1000 << 30}, {"mount": "/boot/efi", "used": 3 << 20, "total": 100 << 20}]
|
||||
return {"t": self.start + index * 60, "disks": disks, "cpu": cpu, "ram": ram, "swap": 5, "load": cpu / 25, "gpus": [gpu], "queue": {"waiting": waiting, "active": 1}}
|
||||
|
||||
def steps(self, browser: str) -> list[dict]:
|
||||
steps = [{"width": width, "height": height, "url": f"{self.base}?{width}#resources", "wait": 1200, "probe": probe(CHARTS), "shot": str(OUT / f"{browser}-{width}-resources.png")} for width, height in base.WIDTHS]
|
||||
steps.append({"width": 1366, "height": 768, "url": f"{self.base}?key#stream", "wait": 1200, "probe": probe(KEY), "shot": None})
|
||||
return steps
|
||||
|
||||
def check(self, data: dict, browser: str) -> None:
|
||||
results = data["results"]
|
||||
for (width, _), found in zip(base.WIDTHS, results, strict=False):
|
||||
with self.subTest(browser=browser, width=width):
|
||||
self.assertNotIn("error", found, found)
|
||||
self.assertEqual(found["panels"], list(PANELS))
|
||||
self.assertEqual(len(found["axes"]), 1, found["axes"])
|
||||
self.assertLess(abs(found["since"] - (time.time() - 86400)), 600)
|
||||
self.assertEqual([(panel, lower) for panel, _, lower in found["bands"]], [("disks", True), ("host", True), ("gpu", True)])
|
||||
for panel, ratio, _ in found["bands"]:
|
||||
self.assertAlmostEqual(ratio, 3, delta=0.2, msg=panel)
|
||||
self.assertEqual(found["series"], [["disks", ["/|disk", "/boot/efi|disk"]], ["host", ["CPU|host", "RAM|host", "Swap|host"]], ["gpu", [f"{GPU}|util", f"{GPU}|vram"]], ["tokens", []]])
|
||||
self.assertEqual(found["waits"], [[self.start + WAITED[0] * 60, self.start + WAITED[1] * 60, 2]])
|
||||
self.assertEqual(found["asleep"], [[self.start, self.start + ASLEEP * 60]])
|
||||
self.assertEqual(found["unknown"], [[self.start + UNREAD[0] * 60, self.start + UNREAD[1] * 60]])
|
||||
self.assertEqual((found["agents"], found["bottoms"], found["stacked"]), (["Skills", "Stage"], ["Skills"], True))
|
||||
self.assertEqual(found["legend"], ["CPU 60 %", "RAM 55 %", "Swap 5 %"])
|
||||
self.assertEqual(found["labels"], {"tab": ["Resources (key 5)", "Resources"], "reload": ["Reload: read the sampler files again", "Reload"], "titles": [[name, name] for name in PANELS.values()]})
|
||||
self.assertEqual((found["unlabelled"], found["plots"]), (0, [True] * 4))
|
||||
self.assertEqual((found["tall"], found["wide"]), (False, False), "the page scrolls")
|
||||
self.assertEqual(quiet(found["violations"]), [])
|
||||
self.assertEqual(results[len(base.WIDTHS)], {"opened": True, "current": "page"})
|
||||
|
||||
def test_chromium_draws_the_resources_charts_from_the_sampler_files_at_three_widths(self):
|
||||
data = self.drive(self.steps("chromium"), None, init=GUARD)
|
||||
(OUT / "chromium-resources.json").write_text(json.dumps(data, indent=1), encoding="utf-8")
|
||||
self.assertEqual(data["errors"], [])
|
||||
self.check(data, "chromium")
|
||||
|
||||
@unittest.skipUnless(shutil.which(os.environ.get("FIREFOX", "firefox")), "no Firefox")
|
||||
def test_firefox_draws_the_resources_charts_from_the_sampler_files_at_three_widths(self):
|
||||
data = self.drive(self.steps("firefox"), None, "bidi.mjs", os.environ.get("FIREFOX", "firefox"), init=GUARD)
|
||||
(OUT / "firefox-resources.json").write_text(json.dumps(data, indent=1), encoding="utf-8")
|
||||
self.assertEqual([error for error in data["errors"] if "/favicon.ico" not in error], [])
|
||||
self.check(data, "firefox")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -65,9 +65,9 @@ class TestNavModule(unittest.TestCase):
|
||||
done = subprocess.run(["node", "-e", NAV_PROBE, str(NAV)], capture_output=True, text=True, timeout=60, check=True)
|
||||
cls.data = json.loads(done.stdout)
|
||||
|
||||
def test_the_navigation_has_four_entries_and_no_inbox_and_stream_keeps_only_the_firewall(self):
|
||||
def test_the_navigation_has_five_entries_and_no_inbox_and_stream_keeps_only_the_firewall(self):
|
||||
stream = ["stream", "Stream", [["rules", "Firewall"]]]
|
||||
self.assertEqual(self.data["tabs"], [stream, ["agents", "Agents", []], ["artifacts", "Artifacts", []], ["tasks", "Tasks", []]])
|
||||
self.assertEqual(self.data["tabs"], [stream, ["agents", "Agents", []], ["artifacts", "Artifacts", []], ["tasks", "Tasks", []], ["resources", "Resources", []]])
|
||||
self.assertNotIn("inbox", json.dumps(self.data["tabs"]))
|
||||
|
||||
def test_agents_and_artifacts_offer_five_views_in_order_and_tasks_six(self):
|
||||
|
||||
@@ -0,0 +1,229 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import subprocess
|
||||
import time
|
||||
import unittest
|
||||
|
||||
from hub_case import HubCase
|
||||
from hub_support import HUB, NAV
|
||||
|
||||
MODULE = HUB / "templates" / "resources.js"
|
||||
PAGE = HUB / "templates" / "page-resources.js"
|
||||
STYLE = HUB / "templates" / "style-resources.css"
|
||||
T = 1_760_000_000
|
||||
FIXTURE = """
|
||||
const asleep = {name: "0000:01:00.0", asleep: true};
|
||||
const awake = (util, used) => ({name: "0000:01:00.0", util, vram_used: used, vram_total: 1000});
|
||||
const unread = {name: "0000:01:00.0", util: null, vram_used: null, vram_total: null};
|
||||
const row = (t, cpu, ram, waiting, gpu, disks) => ({t, cpu, ram, swap: 0, load: 1, disks, gpus: [gpu], queue: {waiting, active: 1}});
|
||||
const root = [{mount: "/", used: 250, total: 1000}];
|
||||
const samples = [
|
||||
row(T, 20, 30, 0, asleep, root),
|
||||
row(T + 60, 40, 45, 2, asleep, root),
|
||||
row(T + 120, 45, 40, 3, awake(80, 500), [...root, {mount: "/mnt", used: 1, total: 4}]),
|
||||
row(T + 180, 70, 40, 3, awake(90, 600), root),
|
||||
row(T + 240, 30, 30, 1, awake(10, 100), root),
|
||||
row(T + 300, 35, 30, 0, unread, root),
|
||||
row(T + 5000, 10, 10, 1, asleep, root),
|
||||
row(T + 5060, 10, 10, 0, asleep, root),
|
||||
];
|
||||
const tokens = [
|
||||
{t: T, agent: "Skills", model: "opus", input: 10, output: 20, cache_read: 300, cache_write: 40},
|
||||
{t: T, agent: "Stage", model: "opus", input: 1, output: 2, cache_read: 3, cache_write: 4},
|
||||
{t: T, agent: "Skills", model: "haiku", input: 5, output: 0, cache_read: 0, cache_write: 0},
|
||||
{t: T + 3600, agent: "Stage", model: "opus", input: 500, output: 0, cache_read: 0, cache_write: 0},
|
||||
{t: "x", agent: "Bad", model: "opus", input: 1, output: 0, cache_read: 0, cache_write: 0},
|
||||
];
|
||||
"""
|
||||
PROBE = """
|
||||
const R = require(process.argv[1]);
|
||||
const T = Number(process.argv[2]);
|
||||
@FIXTURE@
|
||||
const lines = R.lines(samples);
|
||||
const pick = (list) => list.map((entry) => [entry.name, entry.kind, entry.points]);
|
||||
console.log(JSON.stringify({
|
||||
zones: R.ZONES,
|
||||
disks: pick(lines.disks),
|
||||
host: pick(lines.host),
|
||||
gpu: pick(lines.gpu),
|
||||
segments: R.segments([[T, 1], [T + 60, null], [T + 120, 3], [T + 180, 4], [T + 4000, 5]]),
|
||||
waits: R.waits(samples),
|
||||
asleep: R.asleep(samples),
|
||||
unknown: R.unknown(samples),
|
||||
stack: R.stack(tokens),
|
||||
}));
|
||||
""".replace("@FIXTURE@", FIXTURE)
|
||||
PAGE_PROBE = """
|
||||
const fs = require("fs");
|
||||
const vm = require("vm");
|
||||
const Resources = require(process.argv[1]);
|
||||
const code = fs.readFileSync(process.argv[2], "utf8");
|
||||
const T = Number(process.argv[3]);
|
||||
@FIXTURE@
|
||||
const make = (tag, attrs, children) => {
|
||||
const node = {tag, attrs: {}, children: [], textContent: "", isConnected: true,
|
||||
append(...items) { this.children.push(...items); },
|
||||
replaceChildren(...items) { this.children = items; },
|
||||
getBoundingClientRect() { return {width: 800, height: 120}; }};
|
||||
for (const [key, value] of Object.entries(attrs)) {
|
||||
if (key !== "on" && value !== null && value !== undefined && value !== false) {
|
||||
node.attrs[key] = String(value);
|
||||
}
|
||||
}
|
||||
node.children = children.flat(Infinity).filter((child) => child !== null && child !== undefined && child !== false);
|
||||
return node;
|
||||
};
|
||||
const h = (tag, attrs, ...children) => make(tag, attrs, children);
|
||||
const icon = (name) => make("svg", {class: "bi", "data-icon": name}, []);
|
||||
const fetched = [];
|
||||
const context = {
|
||||
Resources, h, s: h, icon, DAY: 86400, ui: {preset: 86400, from: null, to: null, asOf: null, offline: false},
|
||||
iconBtn: (name, label, cls, run, hint) => make("button", {title: hint, "aria-label": label}, [icon(name)]),
|
||||
fixed: (value) => value.toFixed(1), dash: (value) => (value === null ? "–" : value), absolute: {format: () => "d"}, clock: {format: () => "c"}, stampOf: (t) => new Date(t * 1000),
|
||||
window: {setTimeout: (run) => run(), setInterval: () => 1, addEventListener: () => null}, document: {hidden: false},
|
||||
windowQuery: () => "from=1", nowSeconds: () => 0,
|
||||
fetch: async (url) => {
|
||||
fetched.push(url);
|
||||
return {ok: true, json: async () => ({since: T - 600, until: T + 6000, samples, tokens})};
|
||||
},
|
||||
};
|
||||
vm.runInNewContext(`${code}\\nscreen = resourcesScreen();`, context);
|
||||
const walk = (node, found) => {
|
||||
if (node && typeof node === "object") {
|
||||
found.push(node);
|
||||
node.children.forEach((child) => walk(child, found));
|
||||
}
|
||||
return found;
|
||||
};
|
||||
const has = (node, name) => (node.attrs.class || "").split(" ").includes(name);
|
||||
(async () => {
|
||||
for (let round = 0; round < 5; round += 1) {
|
||||
await new Promise((resolve) => setImmediate(resolve));
|
||||
}
|
||||
const all = walk(context.screen, []);
|
||||
const charts = all.filter((node) => has(node, "res-chart"));
|
||||
const marks = (kind) => all.filter((node) => has(node, "res-mark") && has(node, kind)).map((node) => [Number(node.attrs["data-from"]), Number(node.attrs["data-to"])]);
|
||||
console.log(JSON.stringify({
|
||||
fetched,
|
||||
count: all.find((node) => node.attrs["aria-label"] === "Samples").textContent,
|
||||
charts: charts.map((chart) => [chart.attrs["data-panel"], chart.attrs["data-from"], chart.attrs["data-to"]]),
|
||||
bands: charts.map((chart) => walk(chart, []).filter((node) => has(node, "band")).map((node) => node.attrs["data-zone"])),
|
||||
lines: [...new Set(all.filter((node) => has(node, "res-line")).map((node) => `${node.attrs["data-series"]}|${node.attrs["data-kind"]}`))],
|
||||
waits: marks("wait"),
|
||||
asleep: marks("asleep"),
|
||||
unknown: marks("unknown"),
|
||||
words: all.filter((node) => has(node, "res-mark") && has(node, "unknown")).map((node) => node.children[0].children.join("")),
|
||||
bars: all.filter((node) => has(node, "res-bar")).map((node) => [node.attrs["data-agent"], Number(node.attrs["data-t"])]),
|
||||
legends: all.filter((node) => has(node, "res-key")).map((node) => node.children.join("")),
|
||||
titles: all.filter((node) => has(node, "res-title")).map((node) => [node.attrs.title, node.attrs["aria-label"], node.children[0].attrs["data-icon"]]),
|
||||
}));
|
||||
})();
|
||||
""".replace("@FIXTURE@", FIXTURE)
|
||||
|
||||
|
||||
class TestResourcesModule(unittest.TestCase):
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
done = subprocess.run(["node", "-e", PROBE, str(MODULE), str(T)], capture_output=True, text=True, timeout=60, check=True)
|
||||
cls.data = json.loads(done.stdout)
|
||||
|
||||
def test_bands_mark_idle_below_thirty_and_saturated_above_ninety_percent(self):
|
||||
self.assertEqual(self.data["zones"], [{"kind": "idle", "from": 0, "to": 30}, {"kind": "saturated", "from": 90, "to": 100}])
|
||||
|
||||
def test_disks_cpu_ram_swap_gpu_and_vram_become_percent_lines_on_one_time_axis(self):
|
||||
times = [T, T + 60, T + 120, T + 180, T + 240, T + 300, T + 5000, T + 5060]
|
||||
self.assertEqual(self.data["disks"], [["/", "disk", [[t, 25] for t in times]], ["/mnt", "disk", [[T + 120, 25]]]])
|
||||
self.assertEqual([name for name, _, _ in self.data["host"]], ["CPU", "RAM", "Swap"])
|
||||
self.assertEqual(self.data["host"][0][2], [[t, cpu] for t, cpu in zip(times, (20, 40, 45, 70, 30, 35, 10, 10), strict=True)])
|
||||
gpu, vram = self.data["gpu"]
|
||||
self.assertEqual((gpu[0], gpu[1], vram[0], vram[1]), ("0000:01:00.0", "util", "0000:01:00.0", "vram"))
|
||||
self.assertEqual(gpu[2], [[T, None], [T + 60, None], [T + 120, 80], [T + 180, 90], [T + 240, 10], [T + 300, None], [T + 5000, None], [T + 5060, None]])
|
||||
self.assertEqual(vram[2][2:5], [[T + 120, 50], [T + 180, 60], [T + 240, 10]])
|
||||
|
||||
def test_a_line_breaks_at_a_missing_value_and_at_a_gap_of_more_than_half_an_hour(self):
|
||||
self.assertEqual(self.data["segments"], [[[T, 1]], [[T + 120, 3], [T + 180, 4]], [[T + 4000, 5]]])
|
||||
|
||||
def test_a_marker_spans_rows_where_jobs_waited_while_cpu_and_ram_stayed_under_half(self):
|
||||
self.assertEqual(self.data["waits"], [{"from": T + 60, "to": T + 180, "peak": 3}, {"from": T + 240, "to": T + 300, "peak": 1}, {"from": T + 5000, "to": T + 5060, "peak": 1}])
|
||||
|
||||
def test_an_asleep_gpu_gets_its_own_spans(self):
|
||||
self.assertEqual(self.data["asleep"], [{"name": "0000:01:00.0", "spans": [{"from": T, "to": T + 120, "peak": 1}, {"from": T + 5000, "to": T + 5120, "peak": 1}]}])
|
||||
|
||||
def test_an_awake_gpu_without_a_reading_gets_unknown_spans_apart_from_asleep_ones(self):
|
||||
self.assertEqual(self.data["unknown"], [{"name": "0000:01:00.0", "spans": [{"from": T + 300, "to": T + 360, "peak": 1}]}])
|
||||
|
||||
def test_tokens_stack_per_hour_with_the_biggest_agent_at_the_bottom(self):
|
||||
stack = self.data["stack"]
|
||||
self.assertEqual((stack["agents"], stack["top"]), (["Stage", "Skills"], 500))
|
||||
first, second = stack["bars"]
|
||||
self.assertEqual((first["t"], first["total"]), (T, 385))
|
||||
self.assertEqual([(part["agent"], part["from"], part["to"], part["models"]) for part in first["parts"]], [("Stage", 0, 10, ["opus"]), ("Skills", 10, 385, ["opus", "haiku"])])
|
||||
self.assertEqual({key: first["parts"][1][key] for key in ("input", "output", "cache_read", "cache_write")}, {"input": 15, "output": 20, "cache_read": 300, "cache_write": 40})
|
||||
self.assertEqual((second["t"], [(part["agent"], part["to"]) for part in second["parts"]]), (T + 3600, [("Stage", 500)]))
|
||||
|
||||
|
||||
class TestResourcesPage(unittest.TestCase):
|
||||
def test_the_view_draws_four_panels_on_one_time_axis_from_the_api_rows_alone(self):
|
||||
done = subprocess.run(["node", "-e", PAGE_PROBE, str(MODULE), str(PAGE), str(T)], capture_output=True, text=True, timeout=60, check=True)
|
||||
found = json.loads(done.stdout)
|
||||
self.assertEqual((found["fetched"], found["count"]), (["/api/resources?from=1"], "8"))
|
||||
axis = [str(T - 600), str(T + 6000)]
|
||||
self.assertEqual(found["charts"], [[panel, *axis] for panel in ("disks", "host", "gpu", "tokens")])
|
||||
self.assertEqual(found["bands"], [["idle", "saturated"]] * 3 + [[]])
|
||||
self.assertEqual(found["lines"], ["/|disk", "/mnt|disk", "CPU|host", "RAM|host", "Swap|host", "0000:01:00.0|util", "0000:01:00.0|vram"])
|
||||
self.assertEqual(found["waits"], [[T + 60, T + 180], [T + 240, T + 300], [T + 5000, T + 5060]])
|
||||
self.assertEqual(found["asleep"], [[T, T + 120], [T + 5000, T + 5120]])
|
||||
self.assertEqual((found["unknown"], found["words"]), ([[T + 300, T + 360]], ["0000:01:00.0 unknown: no reading"]))
|
||||
self.assertEqual(found["bars"], [["Stage", T], ["Skills", T], ["Stage", T + 3600]])
|
||||
self.assertEqual(found["legends"], ["/ 25 %", "/mnt 25 %", "CPU 10 %", "RAM 10 %", "Swap 0 %", "0000:01:00.0 10 %", "0000:01:00.0 VRAM 10 %", "Stage 510", "Skills 375"])
|
||||
names = [("Disks", "bi-hdd-stack"), ("CPU, RAM and swap", "bi-cpu"), ("GPU and VRAM", "bi-gpu-card"), ("Tokens per agent", "bi-coin")]
|
||||
self.assertEqual(found["titles"], [[name, name, icon] for name, icon in names])
|
||||
|
||||
|
||||
class TestHubResources(HubCase):
|
||||
def write(self, name: str, rows: list[dict], tail: str = "") -> None:
|
||||
folder = self.kept / "resources"
|
||||
folder.mkdir(exist_ok=True)
|
||||
(folder / name).write_text(self.jsonl(rows) + tail, encoding="utf-8")
|
||||
|
||||
def fetch(self, query: str) -> tuple[int, bytes]:
|
||||
status, _, body = self.get(f"/api/resources?{query}")
|
||||
return status, body
|
||||
|
||||
def test_the_view_reads_only_the_sampler_files_within_the_window(self):
|
||||
samples = [{"t": T + step, "cpu": step / 60, "ram": 1, "swap": 0, "load": 0, "disks": [], "gpus": [], "queue": {"waiting": 0, "active": 0}} for step in (0, 60, 120, 180)]
|
||||
tokens = [{"t": T - 3600, "agent": "Skills", "model": "opus", "input": 1, "output": 0, "cache_read": 0, "cache_write": 0}, {"t": T + 3600, "agent": "Skills", "model": "opus", "input": 2, "output": 0, "cache_read": 0, "cache_write": 0}]
|
||||
self.write("samples.jsonl", samples, "{broken\n[1]\n")
|
||||
self.write("tokens.jsonl", tokens)
|
||||
status, body = self.fetch(f"from={T + 60}&to={T + 120}")
|
||||
self.assertEqual(status, 200, body)
|
||||
self.assertEqual(json.loads(body), {"since": T + 60, "until": T + 120, "samples": samples[1:3], "tokens": []})
|
||||
status, body = self.fetch(f"from={T - 1800}&to={T + 3600}")
|
||||
self.assertEqual(json.loads(body)["tokens"], tokens)
|
||||
self.assertEqual(json.loads(body)["samples"], samples)
|
||||
|
||||
def test_missing_files_answer_empty_rows_and_a_reversed_window_is_refused(self):
|
||||
status, body = self.fetch("")
|
||||
self.assertEqual(status, 200, body)
|
||||
found = json.loads(body)
|
||||
self.assertEqual((found["samples"], found["tokens"]), ([], []))
|
||||
self.assertAlmostEqual(found["until"] - found["since"], 86400)
|
||||
self.assertLess(abs(found["until"] - time.time()), 30)
|
||||
self.assertEqual(self.fetch(f"from={T + 10}&to={T}")[0], 400)
|
||||
self.assertEqual(self.fetch("from=x")[0], 400)
|
||||
|
||||
def test_the_page_carries_the_resources_tab_module_view_and_icons(self):
|
||||
html = self.index()
|
||||
self.assertIn(MODULE.read_text(encoding="utf-8").strip(), html)
|
||||
self.assertIn(PAGE.read_text(encoding="utf-8").strip(), html)
|
||||
self.assertIn(STYLE.read_text(encoding="utf-8").strip(), html)
|
||||
for name in ("speedometer2", "hdd-stack", "cpu", "gpu-card", "coin", "hourglass-split", "moon", "question-circle"):
|
||||
with self.subTest(icon=name):
|
||||
self.assertIn(f'<symbol id="bi-{name}"', html)
|
||||
done = subprocess.run(["node", "-e", "const N = require(process.argv[1]); console.log(JSON.stringify([N.TABS.at(-1), N.ICONS.resources, N.route('#resources')]));", str(NAV)], capture_output=True, text=True, timeout=60, check=True)
|
||||
self.assertEqual(json.loads(done.stdout), [["resources", "Resources"], "bi-speedometer2", {"screen": "resources", "view": None, "redirect": None}])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -123,6 +123,7 @@ class TestOrchestratorMigration(unittest.TestCase):
|
||||
self.assertEqual(((config / NEW).stat().st_mode & 0o777, (config / NEW / "approvers.json").stat().st_mode & 0o777), (0o700, 0o600))
|
||||
self.assertTrue((units / f"{NEW}-runner.service").is_file())
|
||||
self.assertTrue((units / f"{NEW}-sessions.timer").is_file())
|
||||
self.assertTrue((units / f"{NEW}-resources.timer").is_file())
|
||||
self.assertIn(f"KVB_ORCHESTRATOR_STATE={local / NEW}", (share / NEW / ".env").read_text(encoding="utf-8"))
|
||||
settings = self.home / ".claude" / "settings.json"
|
||||
data = json.loads(settings.read_text(encoding="utf-8"))
|
||||
@@ -131,7 +132,7 @@ class TestOrchestratorMigration(unittest.TestCase):
|
||||
self.assertIn(f"{share / NEW}/client/permission_hook.py", hooks[1]["hooks"][0]["command"])
|
||||
self.assertEqual(settings.stat().st_mode & 0o777, 0o640)
|
||||
self.assertIn(f"{OLD}/client", (self.home / ".claude" / f"settings.json.before-{NEW}-migration").read_text(encoding="utf-8"))
|
||||
self.assertEqual((first["images"], first["mcp"], first["enabled"]), ([f"{NEW}:local"], {NEW: "http://127.0.0.1:8737/mcp"}, [f"{NEW}-offices.path", f"{NEW}-runner.service", f"{NEW}-sessions.timer"]))
|
||||
self.assertEqual((first["images"], first["mcp"], first["enabled"]), ([f"{NEW}:local"], {NEW: "http://127.0.0.1:8737/mcp"}, [f"{NEW}-offices.path", f"{NEW}-resources.timer", f"{NEW}-runner.service", f"{NEW}-sessions.timer"]))
|
||||
self.assertIn(["docker", "compose", "--file", str(share / OLD / "compose.yml"), "down", "--remove-orphans"], first["calls"])
|
||||
|
||||
before = snapshot(self.home)
|
||||
|
||||
@@ -0,0 +1,294 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from unittest import mock
|
||||
|
||||
import hub_support
|
||||
import resources as sampler
|
||||
|
||||
SCRIPT = hub_support.HUB / "client" / "resources.py"
|
||||
UNITS = hub_support.HUB / "systemd"
|
||||
NOW = 1_800_000_000
|
||||
DAY = 86400
|
||||
MIB = 1 << 20
|
||||
DF = """Filesystem 1024-blocks Used Available Capacity Mounted on
|
||||
/dev/nvme0n1p2 976762584 512000000 460000000 53% /
|
||||
devtmpfs 8000000 0 8000000 0% /dev
|
||||
tmpfs 8000000 1000 7999000 1% /dev/shm
|
||||
/dev/nvme0n1p2 976762584 512000000 460000000 53% /home
|
||||
/dev/nvme0n1p1 523248 12000 511248 3% /boot/efi
|
||||
/dev/loop3 65536 65536 0 100% /snap/core/1
|
||||
/dev/sda1 1953514584 100000000 1853514584 6% /mnt/Backup Disk
|
||||
overlay 976762584 512000000 460000000 53% /var/lib/docker/overlay2/x/merged
|
||||
"""
|
||||
DISKS = [
|
||||
{"mount": "/", "used": 512000000 * 1024, "total": 976762584 * 1024},
|
||||
{"mount": "/boot/efi", "used": 12000 * 1024, "total": 523248 * 1024},
|
||||
{"mount": "/mnt/Backup Disk", "used": 100000000 * 1024, "total": 1953514584 * 1024},
|
||||
]
|
||||
MEMINFO = """MemTotal: 32000000 kB
|
||||
MemFree: 2000000 kB
|
||||
MemAvailable: 24000000 kB
|
||||
Buffers: 500000 kB
|
||||
SwapTotal: 8000000 kB
|
||||
SwapFree: 6000000 kB
|
||||
HugePages_Total: 0
|
||||
"""
|
||||
STAT_BEFORE = "cpu 1000 0 500 8000 500 0 0 0 0 0\ncpu0 500 0 250 4000 250 0 0 0 0 0\n"
|
||||
STAT_AFTER = "cpu 1600 0 700 8400 600 0 0 0 0 0\ncpu0 800 0 350 4200 300 0 0 0 0 0\n"
|
||||
STAT_LATER = "cpu 2000 0 900 8600 700 0 0 0 0 0\n"
|
||||
LOADAVG = "1.25 0.80 0.50 2/1234 56789\n"
|
||||
NVIDIA_SMI = "00000000:01:00.0, 37, 2048, 8188\n00000000:03:00.0, [N/A], [N/A], 4096\n"
|
||||
AWAKE, ASLEEP = "0000:01:00.0", "0000:02:00.0"
|
||||
STUB = """#!/usr/bin/env python3
|
||||
import json, os, sys
|
||||
from pathlib import Path
|
||||
with Path(os.environ["STUB_LOG"]).open("a", encoding="utf-8") as handle:
|
||||
handle.write(json.dumps([Path(sys.argv[0]).name, *sys.argv[1:]]) + "\\n")
|
||||
sys.stdout.write(os.environ["STUB_" + Path(sys.argv[0]).name.replace("-", "_").upper()])
|
||||
"""
|
||||
|
||||
|
||||
def device(root: Path, address: str, vendor: str, kind: str, status: str | None) -> None:
|
||||
folder = root / "bus" / "pci" / "devices" / address
|
||||
(folder / "power").mkdir(parents=True)
|
||||
(folder / "vendor").write_text(f"{vendor}\n", encoding="utf-8")
|
||||
(folder / "class").write_text(f"{kind}\n", encoding="utf-8")
|
||||
if status is not None:
|
||||
(folder / "power" / "runtime_status").write_text(f"{status}\n", encoding="utf-8")
|
||||
|
||||
|
||||
def host_sys(root: Path) -> Path:
|
||||
device(root, AWAKE, "0x10de", "0x030000", "active")
|
||||
device(root, "0000:01:00.1", "0x10de", "0x040300", "active")
|
||||
device(root, ASLEEP, "0x10de", "0x030200", "suspended")
|
||||
device(root, "0000:00:02.0", "0x8086", "0x030000", "active")
|
||||
return root
|
||||
|
||||
|
||||
def row(t: int, cpu: float, used: int, waiting: int, gpu: dict) -> dict:
|
||||
return {"t": t, "disks": [{"mount": "/", "used": used, "total": 1000}], "cpu": cpu, "ram": cpu / 2, "swap": 0.0, "load": cpu / 10, "gpus": [gpu], "queue": {"waiting": waiting, "active": 1}}
|
||||
|
||||
|
||||
class TestParsing(unittest.TestCase):
|
||||
def test_df_keeps_each_local_block_device_once_with_its_first_mount_in_bytes(self):
|
||||
self.assertEqual(sampler.parse_df(DF), DISKS)
|
||||
self.assertEqual(sampler.parse_df(""), [])
|
||||
|
||||
def test_meminfo_gives_ram_and_swap_in_percent_and_no_swap_as_zero(self):
|
||||
self.assertEqual(sampler.memory(sampler.parse_meminfo(MEMINFO)), (25.0, 25.0))
|
||||
without = MEMINFO.replace("SwapTotal: 8000000", "SwapTotal: 0").replace("SwapFree: 6000000", "SwapFree: 0")
|
||||
self.assertEqual(sampler.memory(sampler.parse_meminfo(without)), (25.0, 0.0))
|
||||
self.assertEqual(sampler.memory({}), (None, None))
|
||||
|
||||
def test_cpu_percent_is_the_busy_share_of_the_ticks_between_two_reads(self):
|
||||
before, after = sampler.cpu_ticks(STAT_BEFORE), sampler.cpu_ticks(STAT_AFTER)
|
||||
self.assertEqual((before, after), ((8500, 10000), (9000, 11300)))
|
||||
self.assertEqual(sampler.cpu_percent(before, after), 61.5)
|
||||
self.assertIsNone(sampler.cpu_percent(after, after))
|
||||
self.assertIsNone(sampler.cpu_ticks("intr 1 2 3\n"))
|
||||
|
||||
def test_loadavg_is_the_one_minute_load(self):
|
||||
self.assertEqual(sampler.parse_loadavg(LOADAVG), 1.25)
|
||||
self.assertIsNone(sampler.parse_loadavg(""))
|
||||
|
||||
def test_nvidia_smi_csv_gives_util_and_vram_in_bytes_per_pci_address(self):
|
||||
self.assertEqual(sampler.parse_nvidia(NVIDIA_SMI), {
|
||||
AWAKE: {"util": 37.0, "vram_used": 2048 * MIB, "vram_total": 8188 * MIB},
|
||||
"0000:03:00.0": {"util": None, "vram_used": None, "vram_total": 4096 * MIB},
|
||||
})
|
||||
|
||||
def test_the_queue_counts_waiting_and_active_entries(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
council = Path(tmp)
|
||||
self.assertEqual(sampler.queue(council), {"waiting": 0, "active": 0})
|
||||
entries = [{"status": "waiting"}, {"status": "waiting"}, {"status": "active"}, {"status": "done"}, "junk"]
|
||||
(council / "queue.json").write_text(json.dumps({"entries": entries}), encoding="utf-8")
|
||||
self.assertEqual(sampler.queue(council), {"waiting": 2, "active": 1})
|
||||
|
||||
|
||||
class TestCpu(unittest.TestCase):
|
||||
def test_cpu_is_the_busy_share_since_the_ticks_the_run_before_kept_and_the_sampler_never_sleeps(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
proc, state = Path(tmp) / "proc", Path(tmp) / "state"
|
||||
proc.mkdir()
|
||||
state.mkdir()
|
||||
(proc / "stat").write_text(STAT_BEFORE, encoding="utf-8")
|
||||
with mock.patch.object(sampler.time, "sleep", side_effect=AssertionError("slept")), mock.patch.object(sampler.time, "time", return_value=NOW - 60):
|
||||
self.assertIsNone(sampler.sample(proc, Path(tmp) / "sys", Path(tmp), lambda argv: None, state)["cpu"])
|
||||
(proc / "stat").write_text(STAT_AFTER, encoding="utf-8")
|
||||
self.assertEqual(sampler.cpu_since(proc, state, NOW), 61.5)
|
||||
(proc / "stat").write_text(STAT_LATER, encoding="utf-8")
|
||||
self.assertIsNone(sampler.cpu_since(proc, state, NOW + sampler.BUCKET + 1))
|
||||
self.assertEqual(json.loads((state / "cpu.json").read_text(encoding="utf-8")), [NOW + sampler.BUCKET + 1, 9300, 12200])
|
||||
|
||||
|
||||
class TestGpus(unittest.TestCase):
|
||||
def setUp(self):
|
||||
tmp = tempfile.TemporaryDirectory()
|
||||
self.addCleanup(tmp.cleanup)
|
||||
self.root = Path(tmp.name)
|
||||
self.calls = []
|
||||
|
||||
def run_smi(self, argv: list[str]) -> str:
|
||||
self.calls.append(argv)
|
||||
return NVIDIA_SMI
|
||||
|
||||
def test_an_asleep_gpu_is_recorded_asleep_and_nvidia_smi_queries_only_the_awake_one(self):
|
||||
found = sampler.gpus(host_sys(self.root), self.run_smi)
|
||||
self.assertEqual(found, [{"name": AWAKE, "util": 37.0, "vram_used": 2048 * MIB, "vram_total": 8188 * MIB}, {"name": ASLEEP, "asleep": True}])
|
||||
self.assertEqual(len(self.calls), 1)
|
||||
self.assertEqual(self.calls[0][0], "nvidia-smi")
|
||||
self.assertIn(f"--id={AWAKE}", self.calls[0])
|
||||
self.assertNotIn(ASLEEP, " ".join(self.calls[0]))
|
||||
|
||||
def test_nvidia_smi_never_runs_while_every_nvidia_gpu_sleeps_or_lacks_a_runtime_status(self):
|
||||
device(self.root, ASLEEP, "0x10de", "0x030000", "suspended")
|
||||
device(self.root, "0000:04:00.0", "0x10de", "0x030000", None)
|
||||
self.assertEqual(sampler.gpus(self.root, self.run_smi), [{"name": ASLEEP, "asleep": True}, {"name": "0000:04:00.0", "asleep": True}])
|
||||
self.assertEqual(self.calls, [])
|
||||
|
||||
def test_an_awake_gpu_nvidia_smi_gives_no_reading_for_is_recorded_unknown(self):
|
||||
unknown = {"name": AWAKE, "util": None, "vram_used": None, "vram_total": None}
|
||||
self.assertEqual(sampler.gpus(host_sys(self.root), lambda argv: None), [unknown, {"name": ASLEEP, "asleep": True}])
|
||||
self.assertEqual(sampler.gpus(self.root, lambda argv: NVIDIA_SMI.splitlines()[1]), [unknown, {"name": ASLEEP, "asleep": True}])
|
||||
self.assertEqual(sampler.mean_row(NOW, [row(NOW + 60, 1.0, 1, 0, unknown), row(NOW + 120, 1.0, 1, 0, unknown)])["gpus"], [unknown])
|
||||
|
||||
def test_no_nvidia_gpu_means_no_gpu_row_and_no_call(self):
|
||||
device(self.root, "0000:00:02.0", "0x8086", "0x030000", "active")
|
||||
self.assertEqual(sampler.gpus(self.root, self.run_smi), [])
|
||||
self.assertEqual(sampler.gpus(self.root / "missing", self.run_smi), [])
|
||||
self.assertEqual(self.calls, [])
|
||||
|
||||
|
||||
class TestRetention(unittest.TestCase):
|
||||
def test_rows_older_than_seven_days_fold_into_fifteen_minute_means_and_means_past_ninety_days_drop(self):
|
||||
bucket = NOW - 8 * DAY
|
||||
asleep, awake = {"name": AWAKE, "asleep": True}, {"name": AWAKE, "util": 50.0, "vram_used": 300, "vram_total": 1000}
|
||||
mean = {**row(NOW - 30 * DAY, 12.0, 100, 0, asleep)}
|
||||
rows = [
|
||||
row(NOW - 91 * DAY, 5.0, 1, 0, asleep),
|
||||
mean,
|
||||
row(bucket, 10.0, 100, 0, asleep),
|
||||
row(bucket + 60, 20.0, 200, 1, asleep),
|
||||
{**row(bucket + 120, 60.0, 300, 2, awake), "disks": [{"mount": "/", "used": 300, "total": 1000}, {"mount": "/mnt", "used": 10, "total": 20}]},
|
||||
row(bucket + 1000, 40.0, 400, 0, awake),
|
||||
row(NOW - 60, 70.0, 700, 3, awake),
|
||||
]
|
||||
kept = sampler.retain(rows, NOW)
|
||||
self.assertEqual([entry["t"] for entry in kept], [NOW - 30 * DAY, bucket, bucket + 900, NOW - 60])
|
||||
self.assertIs(kept[0], mean)
|
||||
self.assertEqual(kept[1], {
|
||||
"t": bucket, "disks": [{"mount": "/", "used": 200, "total": 1000}, {"mount": "/mnt", "used": 10, "total": 20}], "cpu": 30.0, "ram": 15.0, "swap": 0.0, "load": 3.0,
|
||||
"gpus": [{"name": AWAKE, "util": 50.0, "vram_used": 300, "vram_total": 1000}], "queue": {"waiting": 1.0, "active": 1.0},
|
||||
})
|
||||
self.assertEqual((kept[2]["cpu"], kept[2]["disks"]), (40.0, [{"mount": "/", "used": 400, "total": 1000}]))
|
||||
self.assertEqual(kept[3], rows[-1])
|
||||
self.assertEqual(sampler.retain(kept, NOW), kept)
|
||||
|
||||
def test_a_bucket_still_reaching_into_the_last_seven_days_stays_raw_and_asleep_buckets_stay_asleep(self):
|
||||
now = NOW + 300
|
||||
asleep = {"name": ASLEEP, "asleep": True}
|
||||
rows = [row(NOW - 7 * DAY - 890, 10.0, 1, 0, asleep), row(NOW - 7 * DAY - 830, 30.0, 3, 0, asleep), row(NOW - 7 * DAY + 10, 50.0, 5, 0, asleep)]
|
||||
kept = sampler.retain(rows, now)
|
||||
self.assertEqual([entry["t"] for entry in kept], [NOW - 7 * DAY - 900, NOW - 7 * DAY + 10])
|
||||
self.assertEqual((kept[0]["cpu"], kept[0]["gpus"]), (20.0, [asleep]))
|
||||
self.assertEqual(kept[1], rows[2])
|
||||
|
||||
def test_record_appends_and_folds_only_on_the_first_minute_of_a_bucket(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
state = Path(tmp)
|
||||
asleep = {"name": AWAKE, "asleep": True}
|
||||
old = [row(NOW - 8 * DAY, 10.0, 1, 0, asleep), row(NOW - 8 * DAY + 60, 30.0, 1, 0, asleep)]
|
||||
(state / "samples.jsonl").write_text("".join(json.dumps(entry) + "\n" for entry in old), encoding="utf-8")
|
||||
later = row(NOW + 400, 1.0, 1, 0, asleep)
|
||||
sampler.record(state, later, NOW + 400)
|
||||
lines = [json.loads(line) for line in (state / "samples.jsonl").read_text(encoding="utf-8").splitlines()]
|
||||
self.assertEqual(lines, [*old, later])
|
||||
first = row(NOW + 930, 2.0, 1, 0, asleep)
|
||||
sampler.record(state, first, NOW + 930)
|
||||
lines = [json.loads(line) for line in (state / "samples.jsonl").read_text(encoding="utf-8").splitlines()]
|
||||
self.assertEqual([entry["t"] for entry in lines], [NOW - 8 * DAY, NOW + 400, NOW + 930])
|
||||
self.assertEqual(lines[0]["cpu"], 20.0)
|
||||
|
||||
|
||||
class TestSamplerInScratchHome(unittest.TestCase):
|
||||
def setUp(self):
|
||||
tmp = tempfile.TemporaryDirectory()
|
||||
self.addCleanup(tmp.cleanup)
|
||||
base = Path(tmp.name)
|
||||
self.home, self.council, stubs, self.proc = base / "home", base / "council", base / "bin", base / "proc"
|
||||
for folder in (self.home, self.council, stubs, self.proc):
|
||||
folder.mkdir()
|
||||
for name, text in (("stat", STAT_BEFORE), ("meminfo", MEMINFO), ("loadavg", LOADAVG)):
|
||||
(self.proc / name).write_text(text, encoding="utf-8")
|
||||
for tool in ("df", "nvidia-smi"):
|
||||
(stubs / tool).write_text(STUB, encoding="utf-8")
|
||||
(stubs / tool).chmod(0o755)
|
||||
self.sys = host_sys(base / "sys")
|
||||
self.log = base / "calls.jsonl"
|
||||
(self.council / "queue.json").write_text(json.dumps({"entries": [{"status": "waiting"}, {"status": "active"}, {"status": "active"}]}), encoding="utf-8")
|
||||
self.env = {
|
||||
**os.environ, "HOME": str(self.home), "PATH": f"{stubs}:/usr/bin:/bin",
|
||||
"STUB_LOG": str(self.log), "STUB_DF": DF, "STUB_NVIDIA_SMI": NVIDIA_SMI.splitlines()[0] + "\n",
|
||||
}
|
||||
self.resources = self.home / ".local" / "state" / "agents-council" / "resources"
|
||||
|
||||
def sample(self) -> None:
|
||||
argv = [sys.executable, str(SCRIPT), "sample", "--state", str(self.resources), "--council", str(self.council), "--proc", str(self.proc), "--sys", str(self.sys)]
|
||||
done = subprocess.run(argv, env=self.env, capture_output=True, text=True, timeout=60, check=False)
|
||||
self.assertEqual(done.returncode, 0, done.stderr)
|
||||
|
||||
def test_the_sampler_appends_one_row_per_run_under_the_scratch_home(self):
|
||||
self.sample()
|
||||
(self.proc / "stat").write_text(STAT_AFTER, encoding="utf-8")
|
||||
self.sample()
|
||||
samples = self.resources / "samples.jsonl"
|
||||
rows = [json.loads(line) for line in samples.read_text(encoding="utf-8").splitlines()]
|
||||
self.assertEqual(len(rows), 2)
|
||||
for found in rows:
|
||||
with self.subTest(t=found["t"]):
|
||||
self.assertEqual(sorted(found), ["cpu", "disks", "gpus", "load", "queue", "ram", "swap", "t"])
|
||||
self.assertEqual(found["disks"], DISKS)
|
||||
self.assertEqual(found["gpus"], [{"name": AWAKE, "util": 37.0, "vram_used": 2048 * MIB, "vram_total": 8188 * MIB}, {"name": ASLEEP, "asleep": True}])
|
||||
self.assertEqual(found["queue"], {"waiting": 1, "active": 2})
|
||||
self.assertEqual((found["ram"], found["swap"], found["load"]), (25.0, 25.0, 1.25))
|
||||
self.assertEqual([found["cpu"] for found in rows], [None, 61.5])
|
||||
self.assertLessEqual(rows[0]["t"], rows[1]["t"])
|
||||
calls = [json.loads(line) for line in self.log.read_text(encoding="utf-8").splitlines()]
|
||||
self.assertEqual([call[0] for call in calls], ["df", "nvidia-smi"] * 2)
|
||||
self.assertTrue(all(ASLEEP not in " ".join(call) for call in calls), calls)
|
||||
self.assertEqual(sorted(path.name for path in samples.parent.iterdir()), ["cpu.json", "samples.jsonl"])
|
||||
|
||||
|
||||
class TestTimer(unittest.TestCase):
|
||||
def test_a_user_timer_runs_sample_and_tokens_every_minute_without_a_page_condition(self):
|
||||
timer = (UNITS / "kvb-orchestrator-resources.timer").read_text(encoding="utf-8").splitlines()
|
||||
service = (UNITS / "kvb-orchestrator-resources.service").read_text(encoding="utf-8").splitlines()
|
||||
self.assertIn("OnCalendar=*-*-* *:*:00", timer)
|
||||
self.assertIn("WantedBy=timers.target", timer)
|
||||
script = "/usr/bin/env python3 %h/.local/share/kvb-orchestrator/client/resources.py"
|
||||
shared = "--state ${KVB_ORCHESTRATOR_KEPT}/resources --council ${KVB_ORCHESTRATOR_COUNCIL}"
|
||||
self.assertEqual([line for line in service if line.startswith("ExecStart=")], [
|
||||
f"ExecStart={script} sample {shared} --proc /proc --sys /sys",
|
||||
f"ExecStart={script} tokens {shared} --transcripts ${{KVB_ORCHESTRATOR_PROJECTS}}",
|
||||
])
|
||||
self.assertIn("EnvironmentFile=%h/.local/share/kvb-orchestrator/.env", service)
|
||||
self.assertIn("Type=oneshot", service)
|
||||
self.assertFalse(any(line.startswith("ExecCondition=") for line in service))
|
||||
|
||||
def test_orchestrator_install_enables_the_resources_timer(self):
|
||||
script = (hub_support.REPO_ROOT / "scripts" / "orchestrator.sh").read_text(encoding="utf-8")
|
||||
self.assertIn('RESOURCES_UNIT="${NAME}-resources"', script)
|
||||
self.assertIn('systemctl --user enable --now "${RESOURCES_UNIT}.timer"', script)
|
||||
self.assertIn('"${HUB}/client/resources.py"', script)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,212 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import ast
|
||||
import datetime
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from unittest import mock
|
||||
|
||||
import hub_support
|
||||
import resources as sampler
|
||||
|
||||
SCRIPT = hub_support.HUB / "client" / "resources.py"
|
||||
NOW = 1_800_000_000
|
||||
HOUR = 3600
|
||||
SESSION, OTHER = "5d88e9aa-0000-4000-8000-000000000001", "f45bf5bb-0000-4000-8000-000000000002"
|
||||
OPUS, HAIKU = "claude-opus-5-5", "claude-haiku-4-5"
|
||||
|
||||
|
||||
def iso(ts: float) -> str:
|
||||
return datetime.datetime.fromtimestamp(ts, datetime.UTC).isoformat().replace("+00:00", "Z")
|
||||
|
||||
|
||||
def assistant(ts: float, session: str, model: str, ident: str, counts: tuple[int, int, int, int], request: str = "") -> dict:
|
||||
usage = dict(zip(("input_tokens", "output_tokens", "cache_read_input_tokens", "cache_creation_input_tokens"), counts, strict=True))
|
||||
message = {"id": ident, "type": "message", "role": "assistant", "model": model, "content": [{"type": "text", "text": "ok"}], "usage": {**usage, "service_tier": "standard"}}
|
||||
return {"type": "assistant", "sessionId": session, "timestamp": iso(ts), "message": message, **({"requestId": request} if request else {})}
|
||||
|
||||
|
||||
def lines(rows: list[dict]) -> str:
|
||||
return "".join(json.dumps(entry) + "\n" for entry in rows)
|
||||
|
||||
|
||||
class TokensCase(unittest.TestCase):
|
||||
def setUp(self):
|
||||
tmp = tempfile.TemporaryDirectory()
|
||||
self.addCleanup(tmp.cleanup)
|
||||
base = Path(tmp.name)
|
||||
self.projects, self.council, self.state = base / "projects", base / "council", base / "state"
|
||||
for folder in (self.projects / "-repo", self.council / "work" / "s1", self.state):
|
||||
folder.mkdir(parents=True)
|
||||
(self.council / "work" / "s1" / "claim.json").write_text(json.dumps({"office": "work/s1", "session": SESSION, "name": " Skills ", "beat": NOW}), encoding="utf-8")
|
||||
self.transcript = self.projects / "-repo" / f"{SESSION}.jsonl"
|
||||
|
||||
def count(self, now: float = NOW + 120, budget: int = sampler.BUDGET) -> list[dict]:
|
||||
return sampler.count_tokens(self.projects, self.council, self.state, now, budget)
|
||||
|
||||
def offsets(self) -> dict:
|
||||
return json.loads((self.state / "offsets.json").read_text(encoding="utf-8"))
|
||||
|
||||
def written(self) -> list[dict]:
|
||||
path = self.state / "tokens.jsonl"
|
||||
return [json.loads(line) for line in path.read_text(encoding="utf-8").splitlines()] if path.exists() else []
|
||||
|
||||
def totals(self) -> dict[tuple[int, str, str], tuple[int, ...]]:
|
||||
found: dict[tuple[int, str, str], tuple[int, ...]] = {}
|
||||
for entry in self.written():
|
||||
key = (entry["t"], entry["agent"], entry["model"])
|
||||
before = found.get(key, (0, 0, 0, 0))
|
||||
found[key] = tuple(a + b for a, b in zip(before, (entry["input"], entry["output"], entry["cache_read"], entry["cache_write"]), strict=True))
|
||||
return found
|
||||
|
||||
|
||||
class TestTokens(TokensCase):
|
||||
def test_usage_fields_add_up_per_hour_agent_and_model_and_a_message_split_over_lines_counts_once(self):
|
||||
hour = NOW
|
||||
rows = [
|
||||
{"type": "user", "sessionId": SESSION, "timestamp": iso(NOW + 5), "message": {"role": "user", "content": "x", "usage": {"input_tokens": 999}}},
|
||||
assistant(NOW + 10, SESSION, OPUS, "msg_1", (10, 20, 300, 40)),
|
||||
assistant(NOW + 10, SESSION, OPUS, "msg_1", (10, 20, 300, 40)),
|
||||
assistant(NOW + 20, SESSION, HAIKU, "msg_2", (1, 2, 3, 4)),
|
||||
assistant(NOW + 30, SESSION, OPUS, "msg_3", (5, 6, 7, 8)),
|
||||
assistant(NOW + 40, SESSION, "<synthetic>", "msg_4", (0, 0, 0, 0)),
|
||||
assistant(NOW - HOUR + 10, OTHER, OPUS, "msg_5", (100, 200, 0, 0)),
|
||||
]
|
||||
self.transcript.write_text(lines(rows) + "{not json\n", encoding="utf-8")
|
||||
added = self.count()
|
||||
self.assertEqual(sorted((entry["t"], entry["agent"], entry["model"]) for entry in added), sorted(self.totals()))
|
||||
self.assertEqual(self.totals(), {
|
||||
(hour, "Skills", OPUS): (15, 26, 307, 48),
|
||||
(hour, "Skills", HAIKU): (1, 2, 3, 4),
|
||||
(hour - HOUR, OTHER[:8], OPUS): (100, 200, 0, 0),
|
||||
})
|
||||
self.assertEqual(sorted(self.written()[0]), ["agent", "cache_read", "cache_write", "input", "model", "output", "t"])
|
||||
|
||||
def test_a_second_run_adds_only_the_bytes_after_the_last_offset(self):
|
||||
hour = NOW
|
||||
self.transcript.write_text(lines([assistant(NOW + 10, SESSION, OPUS, "msg_1", (10, 20, 30, 40))]), encoding="utf-8")
|
||||
self.count()
|
||||
self.assertEqual(self.offsets()["files"], {f"-repo/{SESSION}.jsonl": self.transcript.stat().st_size})
|
||||
self.assertEqual(self.count(), [])
|
||||
last = json.dumps(assistant(NOW + 50, SESSION, OPUS, "msg_3", (7, 7, 7, 7)))
|
||||
more = [assistant(NOW + 10, SESSION, OPUS, "msg_1", (10, 20, 30, 40)), assistant(NOW + 40, SESSION, OPUS, "msg_2", (1, 1, 1, 1))]
|
||||
with self.transcript.open("a", encoding="utf-8") as handle:
|
||||
handle.write(lines(more) + last[:40])
|
||||
self.count()
|
||||
self.assertEqual(self.totals(), {(hour, "Skills", OPUS): (11, 21, 31, 41)})
|
||||
with self.transcript.open("a", encoding="utf-8") as handle:
|
||||
handle.write(last[40:] + "\n")
|
||||
self.count()
|
||||
self.assertEqual(self.totals(), {(hour, "Skills", OPUS): (18, 28, 38, 48)})
|
||||
self.assertEqual(self.offsets()["files"][f"-repo/{SESSION}.jsonl"], self.transcript.stat().st_size)
|
||||
|
||||
def test_a_byte_budget_spreads_a_long_backlog_over_runs_without_losing_tokens(self):
|
||||
rows = [assistant(NOW + index, SESSION, OPUS, f"msg_{index}", (1, 0, 0, 0)) for index in range(40)]
|
||||
self.transcript.write_text(lines(rows), encoding="utf-8")
|
||||
line = len(lines(rows[:1]))
|
||||
self.count(budget=10 * line)
|
||||
self.assertEqual(sum(entry["input"] for entry in self.written()), 10)
|
||||
for _ in range(3):
|
||||
self.count(budget=10 * line)
|
||||
self.assertEqual(sum(entry["input"] for entry in self.written()), 40)
|
||||
|
||||
def test_a_shrunk_file_restarts_at_zero_a_vanished_one_leaves_the_offsets_and_an_old_one_starts_at_its_end(self):
|
||||
self.transcript.write_text(lines([assistant(NOW - 40, SESSION, OPUS, "msg_1", (5, 0, 0, 0)), assistant(NOW - 30, SESSION, OPUS, "msg_2", (5, 0, 0, 0))]), encoding="utf-8")
|
||||
old = self.projects / "-repo" / f"{OTHER}.jsonl"
|
||||
old.write_text(lines([assistant(NOW - 100 * 86400, OTHER, OPUS, "msg_9", (9, 0, 0, 0))]), encoding="utf-8")
|
||||
os.utime(old, (NOW - 100 * 86400, NOW - 100 * 86400))
|
||||
self.count()
|
||||
self.assertEqual(sum(entry["input"] for entry in self.written()), 10)
|
||||
self.transcript.write_text(lines([assistant(NOW - 20, SESSION, OPUS, "msg_7", (1, 0, 0, 0))]), encoding="utf-8")
|
||||
old.unlink()
|
||||
self.count()
|
||||
self.assertEqual(sum(entry["input"] for entry in self.written()), 11)
|
||||
self.assertEqual(sorted(self.offsets()["files"]), [f"-repo/{SESSION}.jsonl"])
|
||||
|
||||
def test_a_message_that_fork_transcripts_copy_counts_once_across_files_and_runs(self):
|
||||
shared = assistant(NOW + 10, SESSION, OPUS, "msg_1", (10, 20, 30, 40), "req_1")
|
||||
subagents = self.projects / "-repo" / SESSION / "subagents"
|
||||
subagents.mkdir(parents=True)
|
||||
self.transcript.write_text(lines([shared]), encoding="utf-8")
|
||||
(subagents / "agent-a.jsonl").write_text(lines([{**shared, "timestamp": iso(NOW + 40)}, assistant(NOW + 50, SESSION, OPUS, "msg_2", (1, 1, 1, 1), "req_2")]), encoding="utf-8")
|
||||
self.count()
|
||||
self.assertEqual(self.totals(), {(NOW, "Skills", OPUS): (11, 21, 31, 41)})
|
||||
(subagents / "agent-b.jsonl").write_text(lines([shared, assistant(NOW + 60, SESSION, OPUS, "msg_1", (5, 5, 5, 5), "req_9")]), encoding="utf-8")
|
||||
self.count(now=NOW + 180)
|
||||
self.assertEqual(self.totals(), {(NOW, "Skills", OPUS): (16, 26, 36, 46)})
|
||||
|
||||
def test_the_seen_messages_keep_the_hours_of_this_run_and_the_last_two_hours(self):
|
||||
self.transcript.write_text(lines([assistant(NOW - 5 * HOUR, SESSION, OPUS, "msg_1", (1, 0, 0, 0), "req_1"), assistant(NOW + 10, SESSION, OPUS, "msg_2", (1, 0, 0, 0), "req_2")]), encoding="utf-8")
|
||||
self.count()
|
||||
self.assertEqual({hour: len(keys) for hour, keys in self.offsets()["seen"].items()}, {str(NOW - 5 * HOUR): 1, str(NOW): 1})
|
||||
self.count(now=NOW + HOUR + 120)
|
||||
self.assertEqual(sorted(self.offsets()["seen"]), [str(NOW)])
|
||||
self.count(now=NOW + 2 * HOUR + 120)
|
||||
self.assertEqual(self.offsets()["seen"], {})
|
||||
|
||||
def test_a_transcript_that_fails_to_read_keeps_its_offset_and_counts_nothing_twice(self):
|
||||
old = NOW - 5 * HOUR
|
||||
self.transcript.write_text(lines([assistant(old + 10, SESSION, OPUS, "msg_1", (5, 0, 0, 0), "req_1")]), encoding="utf-8")
|
||||
self.count()
|
||||
end = self.transcript.stat().st_size
|
||||
with self.transcript.open("a", encoding="utf-8") as handle:
|
||||
handle.write(lines([assistant(old + 20, SESSION, OPUS, "msg_2", (1, 0, 0, 0), "req_2")]))
|
||||
with mock.patch.object(sampler, "scan", side_effect=PermissionError("denied")):
|
||||
self.assertEqual(self.count(now=NOW + HOUR + 120), [])
|
||||
self.assertEqual(self.offsets()["files"], {f"-repo/{SESSION}.jsonl": end})
|
||||
self.count(now=NOW + 2 * HOUR + 120)
|
||||
self.assertEqual(self.totals(), {(old, "Skills", OPUS): (6, 0, 0, 0)})
|
||||
|
||||
def test_a_run_without_new_bytes_leaves_offsets_json_unwritten(self):
|
||||
self.transcript.write_text(lines([assistant(NOW + 10, SESSION, OPUS, "msg_1", (1, 0, 0, 0), "req_1")]), encoding="utf-8")
|
||||
self.count()
|
||||
stored = self.offsets()
|
||||
with mock.patch.object(sampler, "replace", side_effect=AssertionError("offsets.json rewritten")):
|
||||
self.assertEqual(self.count(now=NOW + 180), [])
|
||||
self.assertEqual(self.offsets(), stored)
|
||||
|
||||
def test_the_first_minute_of_an_hour_merges_rows_and_drops_those_older_than_ninety_days(self):
|
||||
hour = NOW
|
||||
stale = {"t": hour - 91 * 86400, "agent": "Old", "model": OPUS, "input": 1, "output": 1, "cache_read": 1, "cache_write": 1}
|
||||
parts = [{"t": hour - HOUR, "agent": "Skills", "model": OPUS, "input": value, "output": 0, "cache_read": 0, "cache_write": 0} for value in (1, 2, 3)]
|
||||
(self.state / "tokens.jsonl").write_text(lines([stale, *parts]), encoding="utf-8")
|
||||
self.transcript.write_text(lines([assistant(NOW + 10, SESSION, OPUS, "msg_1", (4, 0, 0, 0))]), encoding="utf-8")
|
||||
self.count(now=hour + 1800)
|
||||
self.assertEqual(len(self.written()), 5)
|
||||
self.count(now=hour + 30)
|
||||
self.assertEqual(self.written(), [{**parts[0], "input": 6}, {"t": hour, "agent": "Skills", "model": OPUS, "input": 4, "output": 0, "cache_read": 0, "cache_write": 0}])
|
||||
|
||||
|
||||
class TestNoLanguageModel(unittest.TestCase):
|
||||
def test_the_sampler_and_the_parser_import_no_network_client(self):
|
||||
tree = ast.parse(SCRIPT.read_text(encoding="utf-8"))
|
||||
imported = {alias.name.split(".")[0] for node in ast.walk(tree) if isinstance(node, ast.Import) for alias in node.names}
|
||||
imported |= {node.module.split(".")[0] for node in ast.walk(tree) if isinstance(node, ast.ImportFrom) and node.module}
|
||||
self.assertEqual(imported & {"urllib", "http", "socket", "ssl", "requests", "hub_api", "anthropic", "openai"}, set())
|
||||
|
||||
|
||||
class TestTokensInScratchHome(unittest.TestCase):
|
||||
def test_the_tokens_command_reads_the_scratch_home_transcripts_and_writes_beside_the_samples(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
home, council = Path(tmp) / "home", Path(tmp) / "council"
|
||||
folder = home / ".claude" / "projects" / "-repo"
|
||||
folder.mkdir(parents=True)
|
||||
council.mkdir()
|
||||
ts = datetime.datetime.now(datetime.UTC).timestamp() - 60
|
||||
(folder / f"{SESSION}.jsonl").write_text(lines([assistant(ts, SESSION, OPUS, "msg_1", (3, 4, 5, 6))]), encoding="utf-8")
|
||||
state = home / ".local" / "state" / "agents-council" / "resources"
|
||||
argv = [sys.executable, str(SCRIPT), "tokens", "--state", str(state), "--council", str(council), "--transcripts", str(home / ".claude" / "projects")]
|
||||
done = subprocess.run(argv, env={**os.environ, "HOME": str(home), "PATH": "/usr/bin:/bin"}, capture_output=True, text=True, timeout=60, check=False)
|
||||
self.assertEqual(done.returncode, 0, done.stderr)
|
||||
self.assertEqual(sorted(path.name for path in state.iterdir()), ["offsets.json", "tokens.jsonl"])
|
||||
written = [json.loads(line) for line in (state / "tokens.jsonl").read_text(encoding="utf-8").splitlines()]
|
||||
self.assertEqual(written, [{"t": int(ts - ts % HOUR), "agent": SESSION[:8], "model": OPUS, "input": 3, "output": 4, "cache_read": 5, "cache_write": 6}])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in new issue
Block a user