Merge branch 'feature/hub-header-rates' into unify/skills-20261114

# Conflicts:
#	docs/requirements/README.md
#	orchestrator/templates/page-resources.js
#	orchestrator/templates/page-settings.js
#	skills/kvb-check-score/scripts/score.py
#	tests/test_resources_sample.py
This commit is contained in:
Kevin Veen-Birkenbach committed 2026-11-14 22:02:20 +00:00
commit f824ea27cf
28 files changed
+1449 -102

No files matched your search

+198
View File
@@ -0,0 +1,198 @@
from __future__ import annotations
import json
import os
import shutil
import sys
import time
import unittest
from pathlib import Path
HERE = Path(__file__).resolve().parent
sys.path[:0] = [str(HERE.parent), str(HERE)]
import test_orchestrator_e2e as base
import test_workbench_w4_e2e as w4
from hub_case import HubCase
from hub_support import iso
OUT = Path(os.environ.get("E2E_OUT", "/tmp/hub-header-rates-shots"))
HOUR = 3600
GIB = 1 << 30
PERIODS = {1920: "day", 1366: "week", 390: "minute"}
BEFORE = {1920: "hour", 1366: "day", 390: "week"}
CHOSEN = {1920: ["tasks", "tokens", "commits"], 1366: ["tasks", "tokens", "cpu"], 390: ["tasks", "tokens", "commits"]}
TASKS = {"minute": ("≈ 0.02/min", "true"), "hour": ("1/h", "false"), "day": ("2/d", "false"), "week": ("2/wk", "false"), "month": ("2/mo", "false"), "year": ("2/yr", "false"), "total": ("2", "false")}
METRICS = ["tasks", "commits", "krs", "tokens", "cpu", "ram", "vram", "swap", "disk"]
HEAD = ["Metric", "Chip", "min", "h", "d", "wk", "mo", "yr", "all"]
READ = """
const chip = () => document.querySelector("#rates summary.rates-chip");
const period = () => document.querySelector('#rates select[aria-label="Rate period"]');
const menu = () => document.querySelector("#rates details.rates-menu");
const box = () => document.querySelector("#rates .rates-box");
const pick = (id) => document.querySelector(`#rates tr[data-metric="${id}"] input[type="checkbox"]`);
const metrics = () => all("#rates summary .rate").map((node) => node.dataset.metric);
const values = () => Object.fromEntries(all("#rates summary .rate").map((node) => [node.dataset.metric, text(node)]));
const loaded = () => Boolean(chip() && metrics().length && !text(chip()).includes("…"));
const stored = async () => (await (await fetch("/api/state", {cache: "no-store"})).json()).settings.rates;
const storedUntil = async (test, ms) => {
const end = Date.now() + ms;
while (Date.now() < end) {
const found = await stored();
if (test(found)) {
return found;
}
await sleep(200);
}
return null;
};
const inside = (node) => {
const rect = node.getBoundingClientRect();
return rect.width > 0 && rect.left >= -1 && rect.right <= innerWidth + 1;
};
const fits = () => {
const bar = document.querySelector(".navbar");
return {wide: document.documentElement.scrollWidth > innerWidth + 1, bar: bar.scrollWidth > bar.clientWidth + 1, chip: inside(chip()), period: inside(period())};
};
"""
CHOOSE = """(async () => {@HELPERS@@READ@
if (!(await until(loaded, 15000))) {
return {error: text(document.getElementById("rates"))};
}
const before = {period: period().value, metrics: metrics()};
period().value = "@PERIOD@";
period().dispatchEvent(new Event("change", {bubbles: true}));
const saved = await storedUntil((found) => found.period === "@PERIOD@", 8000);
await until(() => period().value === "@PERIOD@" && loaded(), 4000);
return {before, saved, after: {period: period().value, metrics: metrics(), values: values(), label: chip().getAttribute("aria-label")}, fits: fits(), open: menu().open, violations: window.e2eViolations || null};
})()"""
OPEN = """(async () => {@HELPERS@@READ@
if (!(await until(loaded, 15000))) {
return {error: text(document.getElementById("rates"))};
}
const kept = {period: period().value, metrics: metrics(), values: values()};
chip().click();
const opened = Boolean(await until(() => menu().open && box().getClientRects().length, 4000));
const head = all("#rates .rates-table thead th").map(text);
const rows = all("#rates .rates-table tbody tr").map((row) => [row.dataset.metric, all("td[data-period]", row).length]);
const current = all('#rates .rates-table td[aria-current="true"]').map((cell) => cell.dataset.period);
const tasks = Object.fromEntries(all('#rates tr[data-metric="tasks"] td[data-period]').map((cell) => [cell.dataset.period, [text(cell), cell.dataset.approx]]));
const hint = document.querySelector('#rates tr[data-metric="tasks"] td[data-period="minute"]').title;
const [drop, add] = metrics().includes("cpu") ? ["cpu", "commits"] : ["commits", "cpu"];
const locked = pick(add).disabled;
pick(drop).click();
await storedUntil((found) => !found.metrics.includes(drop), 8000);
await until(() => !metrics().includes(drop) && pick(add) && !pick(add).disabled, 4000);
const fewest = all("#rates tbody input:checked").map((input) => input.disabled);
pick(add).click();
const swapped = await storedUntil((found) => found.metrics.includes(add), 8000);
await until(() => metrics().includes(add), 4000);
return {kept, opened, head, rows, current, tasks, hint, locked, fewest, swapped, chip: metrics(), box: inside(box()), fits: fits(), open: menu().open, violations: window.e2eViolations || null};
})()"""
CLOSE = """(async () => {@HELPERS@@READ@
const was = menu().open;
document.body.dispatchEvent(new KeyboardEvent("keydown", {key: "Escape", bubbles: true}));
const escaped = Boolean(await until(() => !menu().open, 3000));
chip().click();
const reopened = Boolean(await until(() => menu().open, 3000));
chip().click();
const clicked = Boolean(await until(() => !menu().open, 3000));
chip().click();
await until(() => menu().open, 3000);
document.getElementById("main").click();
const outside = Boolean(await until(() => !menu().open, 3000));
return {was, escaped, reopened, clicked, outside, fits: fits(), violations: window.e2eViolations || null};
})()"""
def probe(template: str, width: int) -> str:
names = {"@HELPERS@": base.HELPERS, "@READ@": READ, "@PERIOD@": PERIODS[width]}
for key, value in names.items():
template = template.replace(key, value)
return template
def sample(t: int) -> dict:
gpus = [{"name": "0000:01:00.0", "util": 30.0, "vram_used": 2 * GIB, "vram_total": 8 * GIB}]
return {"t": t, "disks": [{"mount": "/", "used": 400 << 30, "total": 1000 << 30}], "cpu": 25.0, "ram": 40.0, "swap": 5.0, "load": 1.0, "gpus": gpus, "queue": {"waiting": 0, "active": 0}}
@unittest.skipUnless(shutil.which(os.environ.get("CHROMIUM", "chromium")), "no Chromium")
class TestRatesChip(HubCase):
drive = base.TestOrchestratorE2E.drive
def setUp(self):
super().setUp()
OUT.mkdir(parents=True, exist_ok=True)
now = int(time.time())
hour = now - now % HOUR
folder = self.kept / "resources"
folder.mkdir()
(folder / "samples.jsonl").write_text(self.jsonl([sample(now - 2 * HOUR + step * 60) for step in range(140)]), encoding="utf-8")
tokens = [{"t": t, "agent": "Skills", "model": "claude-opus-5-5", "input": 4000, "output": 1000, "cache_read": 20000, "cache_write": 500} for t in (hour - HOUR, hour)]
(folder / "tokens.jsonl").write_text(self.jsonl(tokens), encoding="utf-8")
self.write_council({"rasci.json": {
"work/first": {"title": "First", "status": "done", "worker": "Skills", "start": iso(now - 5 * HOUR), "end": iso(now - 3 * HOUR), "roles": {"Skills": "RA"}},
"work/second": {"title": "Second", "status": "done", "worker": "Skills", "start": iso(now - 2 * HOUR), "end": iso(now - 1200), "roles": {"Skills": "RA"}, "key_results": [{"text": "shipped", "met": True, "met_at": iso(now - 1300)}]},
"work/third": {"title": "Third", "status": "running", "worker": "Stage", "start": iso(now - HOUR), "roles": {"Stage": "RA"}},
}})
(self.council / "scores.jsonl").write_text(json.dumps({"time": now - 2 * HOUR, "remote": "example.org/skills", "path": "/repo", "branch": "main", "sha": "d" * 40, "values": {"score": 90}, "digest": "0" * 64}) + "\n", encoding="utf-8")
self.base = f"http://{self.host}/"
def steps(self, browser: str) -> list[dict]:
steps = []
for width, height in base.WIDTHS:
shot = OUT / f"{browser}-{width}"
steps += [
{"width": width, "height": height, "url": f"{self.base}?{width}#home", "wait": 1200, "probe": probe(CHOOSE, width), "shot": f"{shot}-chip.png"},
{"width": width, "height": height, "url": f"{self.base}?{width}-reloaded#home", "wait": 1200, "probe": probe(OPEN, width), "shot": f"{shot}-popover.png"},
{"width": width, "height": height, "url": None, "wait": 0, "probe": probe(CLOSE, width), "shot": None},
]
return steps
def check(self, data: dict, browser: str) -> None:
results = iter(data["results"])
fitting = {"wide": False, "bar": False, "chip": True, "period": True}
previous = ["tasks", "tokens", "cpu"]
for width, _ in base.WIDTHS:
chosen, opened, closed = next(results), next(results), next(results)
chip = {"day": "2/d", "week": "2/wk", "minute": "≈ 0.02/min"}[PERIODS[width]]
with self.subTest(browser=browser, width=width):
self.assertNotIn("error", chosen, chosen)
self.assertEqual(chosen["before"], {"period": BEFORE[width], "metrics": previous})
self.assertEqual(chosen["saved"], {"period": PERIODS[width], "metrics": previous})
self.assertEqual((chosen["after"]["period"], chosen["after"]["metrics"], chosen["after"]["values"]["tasks"]), (PERIODS[width], previous, chip))
self.assertTrue(chosen["after"]["label"].startswith("Rates per "), chosen["after"])
self.assertEqual((chosen["fits"], chosen["open"]), (fitting, False))
self.assertEqual(w4.quiet(chosen["violations"]), [])
self.assertNotIn("error", opened, opened)
self.assertEqual((opened["kept"]["period"], opened["kept"]["metrics"], opened["kept"]["values"]["tasks"]), (PERIODS[width], previous, chip))
self.assertTrue(opened["opened"], opened)
self.assertEqual((opened["head"], opened["rows"], opened["current"]), (HEAD, [[metric, 7] for metric in METRICS], [PERIODS[width]] * len(METRICS)))
self.assertEqual({period: tuple(cell) for period, cell in opened["tasks"].items()}, TASKS)
self.assertIn("scaled from the last hour", opened["hint"])
self.assertEqual((opened["locked"], opened["fewest"]), (True, [True, True]))
self.assertEqual((opened["swapped"], opened["chip"]), ({"period": PERIODS[width], "metrics": CHOSEN[width]}, CHOSEN[width]))
self.assertEqual((opened["box"], opened["fits"], opened["open"]), (True, fitting, True))
self.assertEqual(w4.quiet(opened["violations"]), [])
self.assertEqual({key: closed[key] for key in ("was", "escaped", "reopened", "clicked", "outside")}, dict.fromkeys(("was", "escaped", "reopened", "clicked", "outside"), True))
self.assertEqual((closed["fits"], w4.quiet(closed["violations"])), (fitting, []))
previous = CHOSEN[width]
self.assertEqual(self.httpd.hub.settings.get()["rates"], {"period": PERIODS[390], "metrics": CHOSEN[390]})
def test_chromium_shows_the_rates_chip_keeps_its_period_over_a_reload_and_opens_and_closes_the_table_at_three_widths(self):
data = self.drive(self.steps("chromium"), None, init=w4.GUARD)
(OUT / "chromium-rates.json").write_text(json.dumps(data, indent=1), encoding="utf-8")
self.assertEqual(data["errors"], [])
self.check(data, "chromium")
@unittest.skipUnless(shutil.which(os.environ.get("FIREFOX", "firefox")), "no Firefox")
def test_firefox_shows_the_rates_chip_keeps_its_period_over_a_reload_and_opens_and_closes_the_table_at_three_widths(self):
data = self.drive(self.steps("firefox"), None, "bidi.mjs", os.environ.get("FIREFOX", "firefox"), init=w4.GUARD)
(OUT / "firefox-rates.json").write_text(json.dumps(data, indent=1), encoding="utf-8")
self.assertEqual([error for error in data["errors"] if "/favicon.ico" not in error], [])
self.check(data, "firefox")
if __name__ == "__main__":
unittest.main()
+225
View File
@@ -0,0 +1,225 @@
from __future__ import annotations
import json
import subprocess
import tempfile
import unittest
from pathlib import Path
import hub_support
import resources as sampler
from hub_case import HubCase
from kvb_orchestrator import rates
iso = hub_support.iso
HUB = hub_support.HUB
HOUR = 3600
DAY = 86400
GIB = 1 << 30
NOW = 1_800_000_000 + 1800
EXACT_ZERO = {"value": 0.0, "approx": False, "source": None}
def token(t: int, given: int, cached: int) -> dict:
return {"t": t, "agent": "Skills", "model": "claude-opus-5-5", "input": given, "output": 0, "cache_read": cached, "cache_write": 0}
def sample(t: float, cpu: float, span: int | None = None, vram: tuple[int, int] | None = None, boot: int = 50) -> dict:
gpus = [{"name": "0000:01:00.0", "util": 10.0, "vram_used": vram[0] * GIB, "vram_total": vram[1] * GIB}, {"name": "0000:02:00.0", "asleep": True}] if vram else [{"name": "0000:01:00.0", "util": None, "vram_used": None, "vram_total": None}]
row = {"t": t, "cpu": cpu, "ram": cpu + 1, "swap": 2.0, "disks": [{"mount": "/", "used": 50, "total": 100}, {"mount": "/boot", "used": boot, "total": 100}], "gpus": gpus}
return row if span is None else {**row, "span": span}
class TestRates(unittest.TestCase):
def setUp(self):
tmp = tempfile.TemporaryDirectory()
self.addCleanup(tmp.cleanup)
self.council = Path(tmp.name) / "council"
self.folder = Path(tmp.name) / "kept" / "resources"
self.council.mkdir()
self.folder.mkdir(parents=True)
def register(self, tasks: dict) -> None:
(self.council / "rasci.json").write_text(json.dumps(tasks), encoding="utf-8")
def lines(self, path: Path, rows: list[dict]) -> None:
path.write_text("".join(json.dumps(row) + "\n" for row in rows), encoding="utf-8")
def rated(self) -> dict:
return rates.rated(self.council, self.folder.parent, NOW)
def column(self, metric: str) -> dict:
found = self.rated()
return {period: found["rates"][period][metric] for period in rates.PERIOD_NAMES}
def test_done_tasks_count_in_every_window_their_end_lies_in_and_total_counts_each_done_task(self):
ends = [NOW - 30, NOW - 1800, NOW - 2 * DAY, NOW - 20 * DAY, NOW - 200 * DAY, NOW - 800 * DAY]
tasks = {f"work/t{index}": {"status": "done", "end": iso(end)} for index, end in enumerate(ends)}
others = {"status": "done", "end": iso(NOW - 10)}
self.register({**tasks, "work/legacy": {"status": "done"}, "work/open": {**others, "status": "running"}, "work/halted": {**others, "status": "stopped"}, "operator/run": others})
counts = dict(zip(rates.PERIOD_NAMES, (1.0, 2.0, 2.0, 3.0, 4.0, 5.0, 7.0), strict=True))
self.assertEqual(self.column("tasks"), {period: {"value": value, "approx": False, "source": None} for period, value in counts.items()})
def test_an_empty_window_takes_the_shortest_longer_window_with_events_scaled_down_and_marked_approx(self):
results = [{"text": "met", "met": True, "met_at": iso(NOW - 1800)}, {"text": "open", "met": False}]
self.register({"work/a": {"status": "running", "key_results": results}})
commit = {"time": NOW - 5 * HOUR, "remote": "example.org/repo", "sha": "a" * 40, "values": {}}
self.lines(self.council / "scores.jsonl", [commit, commit, {**commit, "sha": None, "time": NOW - 60}, {**commit, "sha": "b" * 40, "values": None}])
krs = self.column("krs")
self.assertEqual(krs["minute"], {"value": 0.01667, "approx": True, "source": "hour"})
self.assertEqual([krs[period] for period in rates.PERIOD_NAMES[1:]], [{"value": 1.0, "approx": False, "source": None}] * 6)
commits = self.column("commits")
self.assertEqual(commits["minute"], {"value": 0.0006944, "approx": True, "source": "day"})
self.assertEqual(commits["hour"], {"value": 0.04167, "approx": True, "source": "day"})
self.assertEqual([commits[period] for period in rates.PERIOD_NAMES[2:]], [{"value": 1.0, "approx": False, "source": None}] * 5)
def test_key_results_met_before_met_at_existed_count_only_in_total_and_scale_from_there(self):
self.register({"work/a": {"status": "running", "key_results": [{"text": "a", "met": True}]}})
self.lines(self.folder / "samples.jsonl", [sample(NOW - 10 * DAY, 5.0)])
krs = self.column("krs")
self.assertEqual(krs["total"], {"value": 1.0, "approx": False, "source": None})
self.assertEqual(krs["year"], {"value": 1.0, "approx": True, "source": "total"})
self.assertEqual(krs["week"], {"value": 0.7, "approx": True, "source": "total"})
self.assertEqual(krs["minute"], {"value": 6.944e-05, "approx": True, "source": "total"})
def test_tokens_share_their_hour_by_overlap_and_the_minute_always_takes_a_longer_window(self):
hour = NOW - NOW % HOUR
self.lines(self.folder / "tokens.jsonl", [token(hour, 1000, 800), token(hour - HOUR, 3000, 600), token(hour - 20 * DAY, 7, 0)])
tokens = self.column("tokens")
self.assertEqual(tokens["minute"], {"value": 60.0, "approx": True, "source": "hour"})
self.assertEqual(tokens["hour"], {"value": 3600.0, "approx": False, "source": None})
self.assertEqual((tokens["day"]["value"], tokens["month"]["value"], tokens["total"]["value"]), (5400.0, 5407.0, 5407.0))
def test_an_empty_hour_takes_the_day_for_tokens_and_so_does_the_minute(self):
hour = NOW - NOW % HOUR
self.lines(self.folder / "tokens.jsonl", [token(hour - 5 * HOUR, 8000, 640)])
tokens = self.column("tokens")
self.assertEqual(tokens["minute"], {"value": 6.0, "approx": True, "source": "day"})
self.assertEqual(tokens["hour"], {"value": 360.0, "approx": True, "source": "day"})
self.assertEqual(tokens["day"], {"value": 8640.0, "approx": False, "source": None})
def test_gauges_average_each_window_weighting_raw_rows_fifteen_minute_means_and_daily_means_by_their_span(self):
self.lines(self.folder / "samples.jsonl", [
sample(NOW - 500 * DAY, 0.0, span=DAY),
sample(NOW - 200 * DAY, 90.0, span=DAY),
sample(NOW - 10 * DAY, 50.0, span=900),
sample(NOW - 90, 30.0, vram=(4, 8), boot=70),
sample(NOW - 30, 10.0, vram=(2, 8), boot=90),
])
found = self.rated()["rates"]
self.assertEqual([found[period]["cpu"]["value"] for period in rates.PERIOD_NAMES], [10.0, 20.0, 20.0, 20.0, 46.5, 89.5, 45.0])
self.assertEqual([found[period]["ram"]["value"] for period in ("minute", "hour")], [11.0, 21.0])
self.assertEqual(found["minute"]["vram"], {"value": 25.0, "used": 2 * GIB, "approx": False, "source": None})
self.assertEqual((found["hour"]["vram"]["value"], found["hour"]["vram"]["used"], found["month"]["vram"]["value"]), (37.5, 3 * GIB, 37.5))
self.assertEqual(found["minute"]["disk"], {"value": 90.0, "mount": "/boot", "used": 90, "total": 100, "approx": False, "source": None})
self.assertEqual((found["hour"]["disk"]["value"], found["month"]["disk"]["value"], found["month"]["disk"]["used"]), (80.0, 53.5, 54))
self.assertEqual(self.rated()["since"], NOW - 500 * DAY)
def test_a_window_without_a_sample_shows_no_gauge_and_kinds_without_any_event_count_an_exact_zero(self):
self.lines(self.folder / "samples.jsonl", [sample(NOW - 2 * HOUR, 40.0)])
found = self.rated()
self.assertEqual([found["rates"][period]["cpu"]["value"] for period in rates.PERIOD_NAMES], [None, None, 40.0, 40.0, 40.0, 40.0, 40.0])
self.assertEqual(found["rates"]["hour"]["vram"], {"value": None, "used": None, "approx": False, "source": None})
self.assertEqual(found["rates"]["minute"]["disk"], {"value": None, "mount": None, "used": None, "total": None, "approx": False, "source": None})
self.assertEqual({found["rates"][period][kind] == EXACT_ZERO for period in rates.PERIOD_NAMES for kind in rates.COUNTS}, {True})
self.assertEqual((found["now"], found["since"], [entry["name"] for entry in found["periods"]]), (NOW, NOW - 2 * HOUR, list(rates.PERIOD_NAMES)))
def test_folded_sampler_rows_feed_the_windows_with_the_seconds_they_sampled(self):
old = NOW - NOW % DAY - 100 * DAY
raw = [sample(old + 3600 + index * 60, 20.0) for index in range(30)] + [sample(NOW - 9 * DAY + index * 60, 60.0) for index in range(30)] + [sample(NOW - 59, 10.0)]
kept = sampler.retain(sorted(raw, key=lambda row: row["t"]), NOW)
self.assertEqual([(row["t"], row["span"], row["width"]) for row in kept[:1]], [(old, 1800, DAY)])
self.assertEqual(sum(row["span"] for row in kept[1:-1]), 1800)
self.assertEqual({row["width"] for row in kept[1:-1]}, {900})
self.assertNotIn("span", kept[-1])
self.lines(self.folder / "samples.jsonl", kept)
found = self.rated()["rates"]
month = round((10 * 60 + 60 * 1800) / (60 + 1800), 1)
year = round((10 * 60 + 60 * 1800 + 20 * 1800) / (60 + 1800 * 2), 1)
self.assertEqual([found[period]["cpu"]["value"] for period in ("minute", "week", "month", "year")], [10.0, 10.0, month, year])
def test_a_lone_fifteen_minute_bucket_weighs_its_sampled_seconds_wherever_it_lies_in_its_day(self):
day = NOW - NOW % DAY - 200 * DAY
raw = [sample(day + 60 * index, 50.0) for index in range(15)] + [sample(day + DAY + 900 + 60 * index, 50.0) for index in range(15)]
kept = sampler.retain(raw, NOW)
self.assertEqual([(row["t"], row["span"], row["width"]) for row in kept], [(day, 900, DAY), (day + DAY, 900, DAY)])
self.assertEqual(sampler.retain(kept, NOW), kept)
def test_a_fifteen_minute_mean_written_before_rows_carried_span_stands_for_its_whole_bucket(self):
start = NOW - NOW % 900 - 20 * DAY
legacy = {**sample(start, 30.0), "queue": {"waiting": 0.0, "active": 1.0}}
kept = sampler.retain([legacy, sample(start + 900 + 60, 90.0)], NOW)
self.assertEqual([(row["t"], row["span"], row["width"]) for row in kept], [(start, 900, 900), (start + 900, 60, 900)])
def test_the_year_weighs_every_tier_by_the_seconds_it_sampled(self):
old, recent = NOW - NOW % DAY - 100 * DAY, NOW - NOW % DAY - 8 * DAY
raw = [sample(old + 60 * index, 90.0) for index in range(60)] + [sample(recent + 60 * index, 10.0) for index in range(60)]
self.lines(self.folder / "samples.jsonl", sampler.retain(raw, NOW))
self.assertEqual(self.rated()["rates"]["year"]["cpu"]["value"], 50.0)
def test_a_folded_row_straddling_a_window_start_counts_by_its_share_inside(self):
week, year = NOW - 7 * DAY, NOW - 365 * DAY
self.lines(self.folder / "samples.jsonl", [
{**sample(year - year % DAY, 40.0), "span": 3600, "width": DAY},
{**sample(week - 300, 70.0, boot=80), "span": 600, "width": 900},
sample(NOW - 30, 10.0, boot=20),
])
found = self.rated()["rates"]
inside = (year - year % DAY + DAY - year) / DAY * 3600
self.assertEqual(found["week"]["cpu"]["value"], round((400 * 70 + 60 * 10) / 460, 1))
self.assertEqual(found["month"]["cpu"]["value"], round((600 * 70 + 60 * 10) / 660, 1))
self.assertEqual(found["year"]["cpu"]["value"], round((inside * 40 + 600 * 70 + 60 * 10) / (inside + 660), 1))
self.assertEqual(found["total"]["cpu"]["value"], round((3600 * 40 + 600 * 70 + 60 * 10) / 4260, 1))
self.assertEqual(found["week"]["disk"]["value"], round((400 * 80 + 60 * 20) / 460, 1))
def test_token_rows_older_than_ninety_days_fold_into_one_row_per_day_agent_and_model_kept_forever(self):
day, hour = NOW - NOW % DAY - 100 * DAY, NOW - NOW % HOUR
rows = [token(day + HOUR, 10, 0), token(day + 5 * HOUR, 20, 5), {**token(day + 2 * HOUR, 1, 0), "model": "other"}, token(hour - HOUR, 7, 0)]
kept = sampler.compact(rows, NOW)
daily = [{**token(day, 30, 5), "width": DAY}, {**token(day, 1, 0), "model": "other", "width": DAY}]
self.assertEqual(kept, [*daily, token(hour - HOUR, 7, 0)])
self.assertEqual(sampler.compact(kept, NOW + 400 * DAY)[:2], daily)
def test_a_daily_token_row_counts_by_the_share_of_its_day_inside_a_window(self):
year = NOW - 365 * DAY
self.lines(self.folder / "tokens.jsonl", [{**token(year - year % DAY, 8640, 0), "width": DAY}])
tokens = self.column("tokens")
self.assertEqual(tokens["year"], {"value": rates.precise((year - year % DAY + DAY - year) / DAY * 8640), "approx": False, "source": None})
self.assertEqual(tokens["total"], {"value": 8640.0, "approx": False, "source": None})
def test_token_windows_reaching_before_the_first_token_row_are_scaled_estimates_from_the_rows_there_are(self):
hour = NOW - NOW % HOUR
begins = hour - 10 * DAY
self.register({"work/old": {"status": "done", "end": iso(NOW - 400 * DAY)}})
self.lines(self.folder / "tokens.jsonl", [token(begins, 1000, 0), token(hour - HOUR, 500, 0)])
tokens = self.column("tokens")
covered = NOW - begins
self.assertEqual(tokens["minute"], {"value": rates.precise(250 / 60), "approx": True, "source": "hour"})
self.assertEqual([tokens[period] for period in ("hour", "day", "week")], [{"value": value, "approx": False, "source": None} for value in (250.0, 500.0, 500.0)])
for period, seconds in (("month", 30 * DAY), ("year", 365 * DAY), ("total", 400 * DAY)):
with self.subTest(period=period):
self.assertEqual(tokens[period], {"value": rates.precise(1500 * seconds / covered), "approx": True, "source": "total"})
def test_counts_shorten_to_thousands_millions_billions_and_trillions(self):
probe = "const R = require(process.argv[1]); console.log(JSON.stringify(JSON.parse(process.argv[2]).map(R.compact)));"
values = [999, 1000, 999_949, 999_950, 14_370_239_246.104, 999_960_000_000, 2.5e12, 3e15]
done = subprocess.run(["node", "-e", probe, str(HUB / "templates" / "resources.js"), json.dumps(values)], capture_output=True, text=True, timeout=60, check=True)
self.assertEqual(json.loads(done.stdout), ["999", "1.0k", "999.9k", "1.0M", "14.4B", "1.0T", "2.5T", "3000.0T"])
class TestHubRates(HubCase):
def test_the_hub_serves_the_rates_of_the_register_the_journal_and_the_sampler(self):
self.write_council({"rasci.json": {"work/a": {"status": "done", "end": iso(1_700_000_000), "key_results": [{"text": "a", "met": True, "met_at": iso(1_700_000_000)}]}}})
(self.council / "scores.jsonl").write_text(json.dumps({"time": 1_700_000_000, "remote": "example.org/repo", "sha": "c" * 40, "values": {}}) + "\n", encoding="utf-8")
status, headers, body = self.get("/api/rates")
self.assertEqual((status, headers["content-type"]), (200, "application/json"), body)
found = json.loads(body)
self.assertEqual([entry["name"] for entry in found["periods"]], list(rates.PERIOD_NAMES))
self.assertEqual({kind: found["rates"]["total"][kind]["value"] for kind in rates.COUNTS}, {"tasks": 1.0, "commits": 1.0, "krs": 1.0, "tokens": 0.0})
self.assertEqual(set(found["rates"]["hour"]), set(rates.METRICS))
self.assertEqual(found["since"], 1_700_000_000)
if __name__ == "__main__":
unittest.main()
+29
View File
@@ -194,6 +194,35 @@ class TestHubSettings(HubCase):
(self.state / "settings.json").write_text(json.dumps(older), encoding="utf-8")
self.assertEqual(settings.Settings(self.state / "settings.json").get()["trend"], {"mode": "off", "days": 7})
def test_the_rates_chip_defaults_to_tasks_tokens_and_cpu_per_hour_is_validated_stored_without_a_passkey_survives_a_reload_and_a_sheet_without_it_keeps_it(self):
default = {"period": "hour", "metrics": ["tasks", "tokens", "cpu"]}
self.assertEqual((settings.SETTINGS["rates"], self.snapshot()["settings"]["rates"]), (default, default))
good = self.guard(self.host)
wrongs = [{"period": period, "metrics": ["tasks", "cpu"]} for period in ("second", "Hour", "", None, 1)]
wrongs += [{"period": "day", "metrics": metrics} for metrics in (["tasks"], [], ["tasks", "cpu", "ram", "swap"], ["tasks", "tasks"], ["tasks", "gpu"], [1, 2], "tasks,cpu", None)]
wrongs += [{"period": "day"}, {"metrics": ["tasks", "cpu"]}, {**default, "extra": 1}, "day", ["day"]]
for wrong in wrongs:
with self.subTest(wrong=wrong):
self.assertEqual(self.post_json("/api/settings", {**settings.SETTINGS, "rates": wrong}, good)[0], 400)
self.assertEqual(self.snapshot()["settings"]["rates"], default)
for chosen in ({"period": "minute", "metrics": ["commits", "krs"]}, {"period": "total", "metrics": ["disk", "vram", "swap"]}, {"period": "week", "metrics": ["ram", "tokens"]}):
with self.subTest(chosen=chosen):
self.assertIsNone(self.prepared({**self.snapshot()["settings"], "rates": chosen})["payload"])
status, _, body = self.post_json("/api/settings", {**self.snapshot()["settings"], "rates": chosen, "proof": None}, good)
self.assertEqual((status, json.loads(body)["settings"]["rates"]), (200, chosen))
last = {"period": "week", "metrics": ["ram", "tokens"]}
self.assertEqual(json.loads((self.state / "settings.json").read_text(encoding="utf-8"))["rates"], last)
self.assertEqual(settings.Settings(self.state / "settings.json").get()["rates"], last)
self.assertEqual(self.snapshot()["settings"]["rates"], last)
sheet = {key: value for key, value in self.snapshot()["settings"].items() if key != "rates"}
self.assertIsNone(self.prepared({**sheet, "estimation": "points"})["payload"])
status, _, body = self.post_json("/api/settings", {**sheet, "estimation": "points", "proof": None}, good)
self.assertEqual((status, json.loads(body)["settings"]["estimation"], json.loads(body)["settings"]["rates"]), (200, "points", last))
self.assertEqual(settings.Settings(self.state / "settings.json").get()["rates"], last)
older = {key: value for key, value in self.settings_payload(STREAMS).items() if key != "rates"}
(self.state / "settings.json").write_text(json.dumps(older), encoding="utf-8")
self.assertEqual(settings.Settings(self.state / "settings.json").get()["rates"], default)
def test_settings_written_before_the_radio_filter_keep_their_values(self):
older = {key: value for key, value in self.settings_payload(STREAMS).items() if key != "speech"}
(self.state / "settings.json").write_text(json.dumps(older), encoding="utf-8")
+24
View File
@@ -142,6 +142,30 @@ class TestRasci(unittest.TestCase):
self.assertEqual(self._run("set", "work/a", "--status", "done").returncode, 0)
self.assertEqual(self._out("show"), ["no tasks"])
def test_a_key_result_set_met_records_met_at_once_and_open_drops_it(self):
self._run("kr", "work/a", "add", "tests green")
self.assertEqual(self._first_kr("work/a"), {"text": "tests green", "met": False})
before = datetime.datetime.now(datetime.UTC).replace(microsecond=0)
self._out("kr", "work/a", "met", "1")
met = self._first_kr("work/a")
moment = datetime.datetime.fromisoformat(met["met_at"])
self.assertEqual((met["met"], moment.utcoffset()), (True, datetime.timedelta(0)))
self.assertLessEqual(before, moment)
self.assertLessEqual(moment, datetime.datetime.now(datetime.UTC))
self._out("kr", "work/a", "met", "1")
self.assertEqual(self._first_kr("work/a"), met)
self._out("kr", "work/a", "open", "1")
self.assertEqual(self._first_kr("work/a"), {"text": "tests green", "met": False})
def test_a_key_result_met_before_met_at_existed_stays_unstamped_when_met_again(self):
self.root.mkdir()
(self.root / "rasci.json").write_text(json.dumps({"work/a": {"roles": {}, "key_results": [{"text": "old", "met": True}]}}), encoding="utf-8")
self._out("kr", "work/a", "met", "1")
self.assertEqual(self._first_kr("work/a"), {"text": "old", "met": True})
def _first_kr(self, task: str) -> dict:
return json.loads((self.root / "rasci.json").read_text(encoding="utf-8"))[task]["key_results"][0]
def test_status_sets_start_and_end(self):
self._run("set", "work/a", "--status", "running")
self._run("set", "work/a", "--status", "done", "--description", "long text")
+31 -9
View File
@@ -225,7 +225,7 @@ class TestGpus(unittest.TestCase):
unknown = {"name": AWAKE, "util": None, "vram_used": None, "vram_total": None}
self.assertEqual(sampler.gpus(host_sys(self.root), lambda argv: None), [unknown, {"name": ASLEEP, "asleep": True}])
self.assertEqual(sampler.gpus(self.root, lambda argv: NVIDIA_SMI.splitlines()[1]), [unknown, {"name": ASLEEP, "asleep": True}])
self.assertEqual(sampler.mean_row(NOW, [row(NOW + 60, 1.0, 1, 0, unknown), row(NOW + 120, 1.0, 1, 0, unknown)])["gpus"], [unknown])
self.assertEqual(sampler.mean_row(NOW, sampler.BUCKET, [row(NOW + 60, 1.0, 1, 0, unknown), row(NOW + 120, 1.0, 1, 0, unknown)])["gpus"], [unknown])
def test_no_nvidia_gpu_means_no_gpu_row_and_no_call(self):
device(self.root, "0000:00:02.0", "0x8086", "0x030000", "active")
@@ -235,12 +235,13 @@ class TestGpus(unittest.TestCase):
class TestRetention(unittest.TestCase):
def test_rows_older_than_seven_days_fold_into_fifteen_minute_means_and_means_past_ninety_days_drop(self):
def test_rows_older_than_seven_days_fold_into_fifteen_minute_means_and_rows_past_ninety_days_into_their_day(self):
bucket = NOW - 8 * DAY
day = NOW - 91 * DAY - NOW % DAY
asleep, awake = {"name": AWAKE, "asleep": True}, {"name": AWAKE, "util": 50.0, "vram_used": 300, "vram_total": 1000}
mean = {**row(NOW - 30 * DAY, 12.0, 100, 0, asleep)}
rows = [
row(NOW - 91 * DAY, 5.0, 1, 0, asleep),
row(NOW - 91 * DAY + 60, 5.0, 1, 0, asleep),
mean,
row(bucket, 10.0, 100, 0, asleep),
row(bucket + 60, 20.0, 200, 1, asleep),
@@ -249,17 +250,38 @@ class TestRetention(unittest.TestCase):
row(NOW - 60, 70.0, 700, 3, awake),
]
kept = sampler.retain(rows, NOW)
self.assertEqual([entry["t"] for entry in kept], [NOW - 30 * DAY, bucket, bucket + 900, NOW - 60])
self.assertIs(kept[0], mean)
self.assertEqual(kept[1], {
self.assertEqual([entry["t"] for entry in kept], [day, NOW - 30 * DAY, bucket, bucket + 900, NOW - 60])
self.assertEqual(kept[0], {**row(day, 5.0, 1, 0, asleep), "queue": {"waiting": 0.0, "active": 1.0}, "span": 60, "width": DAY})
self.assertEqual(kept[1], {**mean, "span": sampler.BUCKET, "width": sampler.BUCKET})
self.assertEqual(kept[2], {
"t": bucket, "disks": [{"mount": "/", "used": 200, "total": 1000}, {"mount": "/mnt", "used": 10, "total": 20}], "cpu": 30.0, "ram": 15.0, "swap": 0.0, "load": 3.0,
"psi_some": None, "psi_full": None, "swap_out_mb_min": None,
"gpus": [{"name": AWAKE, "util": 50.0, "vram_used": 300, "vram_total": 1000}], "queue": {"waiting": 1.0, "active": 1.0},
"gpus": [{"name": AWAKE, "util": 50.0, "vram_used": 300, "vram_total": 1000}], "queue": {"waiting": 1.0, "active": 1.0}, "span": 180, "width": sampler.BUCKET,
})
self.assertEqual((kept[2]["cpu"], kept[2]["disks"]), (40.0, [{"mount": "/", "used": 400, "total": 1000}]))
self.assertEqual(kept[3], rows[-1])
self.assertEqual((kept[3]["cpu"], kept[3]["disks"], kept[3]["span"], kept[3]["width"]), (40.0, [{"mount": "/", "used": 400, "total": 1000}], 60, sampler.BUCKET))
self.assertEqual(kept[4], rows[-1])
self.assertEqual(sampler.retain(kept, NOW), kept)
def test_days_that_ended_more_than_ninety_days_ago_fold_into_one_mean_each_kept_forever(self):
asleep = {"name": AWAKE, "asleep": True}
day = NOW - NOW % DAY - 100 * DAY
edge = NOW - NOW % DAY - 90 * DAY
ancient = {**row(day - 1000 * DAY, 7.0, 7, 0, asleep), "span": 3600, "width": DAY}
means = [{**row(day + index * sampler.BUCKET, cpu, 100, 0, asleep), "span": sampler.BUCKET, "width": sampler.BUCKET} for index, cpu in enumerate((10.0, 20.0, 60.0))]
late = {**row(edge + sampler.BUCKET, 40.0, 400, 0, asleep), "span": sampler.BUCKET, "width": sampler.BUCKET}
recent = row(NOW - 60, 70.0, 700, 3, asleep)
kept = sampler.retain([ancient, *means, late, recent], NOW)
self.assertEqual([entry["t"] for entry in kept], [day - 1000 * DAY, day, edge + sampler.BUCKET, NOW - 60])
self.assertEqual(kept[0], ancient)
self.assertEqual(kept[1], {**row(day, 30.0, 100, 0, asleep), "load": 3.0, "queue": {"waiting": 0.0, "active": 1.0}, "span": 3 * sampler.BUCKET, "width": DAY})
self.assertEqual(kept[2:], [late, recent])
self.assertEqual(sampler.retain(kept, NOW), kept)
later = sampler.retain(kept, NOW + 2 * DAY)
self.assertEqual([entry["t"] for entry in later], [day - 1000 * DAY, day, edge, NOW - 60])
self.assertEqual(later[:2], kept[:2])
self.assertEqual((later[2]["cpu"], later[2]["span"], later[2]["width"]), (40.0, sampler.BUCKET, DAY))
self.assertEqual(sampler.retain(later, NOW + 400 * DAY)[:3], later[:3])
def test_a_bucket_still_reaching_into_the_last_seven_days_stays_raw_and_asleep_buckets_stay_asleep(self):
now = NOW + 300
asleep = {"name": ASLEEP, "asleep": True}
+6 -3
View File
@@ -170,16 +170,19 @@ class TestTokens(TokensCase):
self.assertEqual(self.count(now=NOW + 180), [])
self.assertEqual(self.offsets(), stored)
def test_the_first_minute_of_an_hour_merges_rows_and_drops_those_older_than_ninety_days(self):
def test_the_first_minute_of_an_hour_merges_rows_and_folds_days_past_ninety_days_into_one_row_kept_forever(self):
hour = NOW
stale = {"t": hour - 91 * 86400, "agent": "Old", "model": OPUS, "input": 1, "output": 1, "cache_read": 1, "cache_write": 1}
stale = {"t": hour - 92 * 86400, "agent": "Old", "model": OPUS, "input": 1, "output": 1, "cache_read": 1, "cache_write": 1}
parts = [{"t": hour - HOUR, "agent": "Skills", "model": OPUS, "input": value, "output": 0, "cache_read": 0, "cache_write": 0} for value in (1, 2, 3)]
(self.state / "tokens.jsonl").write_text(lines([stale, *parts]), encoding="utf-8")
self.transcript.write_text(lines([assistant(NOW + 10, SESSION, OPUS, "msg_1", (4, 0, 0, 0))]), encoding="utf-8")
self.count(now=hour + 1800)
self.assertEqual(len(self.written()), 5)
self.count(now=hour + 30)
self.assertEqual(self.written(), [{**parts[0], "input": 6}, {"t": hour, "agent": "Skills", "model": OPUS, "input": 4, "output": 0, "cache_read": 0, "cache_write": 0}])
daily = {**stale, "t": stale["t"] - stale["t"] % 86400, "width": 86400}
self.assertEqual(self.written(), [daily, {**parts[0], "input": 6}, {"t": hour, "agent": "Skills", "model": OPUS, "input": 4, "output": 0, "cache_read": 0, "cache_write": 0}])
self.count(now=hour + 400 * 86400 + 30)
self.assertEqual(self.written()[0], daily)
class TestNoLanguageModel(unittest.TestCase):
+21
View File
@@ -218,6 +218,27 @@ class TestScoreJournal(ScoreCase):
self.score(SCORE, *flags({}), "--table", cwd=repo)
self.assertEqual(len(self.rows()), 2)
def test_the_commit_table_of_a_commit_records_its_sha_and_commit_time_once_per_remote_and_sha(self):
repo = self.repo()
env = git_env(self.tmp)
sha = subprocess.run(["git", "rev-parse", "HEAD"], cwd=repo, env=env, capture_output=True, text=True, check=True).stdout.strip()
stamp = int(subprocess.run(["git", "show", "-s", "--format=%ct", sha], cwd=repo, env=env, capture_output=True, text=True, check=True).stdout)
self.score(SCORE, *flags({}), "--message-block", cwd=repo)
for _ in range(2):
done = self.score(SCORE, *flags({}), "--commit-table", str(repo), "--commit", sha[:7], cwd=self.tmp)
self.assertEqual((done.returncode, done.stderr), (0, ""))
rendered, committed = self.rows()
self.assertEqual(committed, {**rendered, "time": stamp, "sha": sha})
self.score(SCORE, *flags({"fix": "80"}), "--commit-table", str(repo), cwd=self.tmp)
self.assertEqual(len(self.rows()), 2)
self.council.joinpath(JOURNAL_NAME).unlink()
self.council.rmdir()
self.council.write_text("not a directory", encoding="utf-8")
failed = self.score(SCORE, *flags({"critical": "1"}), "--commit-table", str(repo), "--commit", sha, cwd=self.tmp)
self.assertEqual(failed.returncode, 4)
self.assertEqual(len(failed.stderr.splitlines()), 1, failed.stderr)
self.assertTrue(failed.stderr.startswith(f"score: journal {self.council / JOURNAL_NAME}: "), failed.stderr)
def test_a_failed_append_prints_one_stderr_line_and_keeps_the_exit_code(self):
self.council.write_text("not a directory", encoding="utf-8")
for values, code in (({}, 0), ({"efficiency": "120"}, 3), ({"break": "9"}, 1), ({"critical": "1"}, 4)):