Files
skills/tests/test_hub_rates.py
T

206 lines
14 KiB
Python
Raw Normal View History

from __future__ import annotations
import json
import subprocess
import tempfile
import unittest
from pathlib import Path
import hub_support
import resources as sampler
from hub_case import HubCase
from kvb_orchestrator import rates
iso = hub_support.iso
HUB = hub_support.HUB
HOUR = 3600
DAY = 86400
GIB = 1 << 30
NOW = 1_800_000_000 + 1800
LAST_HOUR = NOW - NOW % HOUR
NOTHING = {"value": None, "approx": False, "covered": None}
def token(t: int, given: int, cached: int) -> dict:
return {"t": t, "agent": "Skills", "model": "claude-opus-5-5", "input": given, "output": 0, "cache_read": cached, "cache_write": 0}
def sample(t: float, cpu: float, span: int | None = None, vram: tuple[int, int] | None = None, boot: int = 50) -> dict:
gpus = [{"name": "0000:01:00.0", "util": 10.0, "vram_used": vram[0] * GIB, "vram_total": vram[1] * GIB}, {"name": "0000:02:00.0", "asleep": True}] if vram else [{"name": "0000:01:00.0", "util": None, "vram_used": None, "vram_total": None}]
row = {"t": t, "cpu": cpu, "ram": cpu + 1, "swap": 2.0, "disks": [{"mount": "/", "used": 50, "total": 100}, {"mount": "/boot", "used": boot, "total": 100}], "gpus": gpus}
return row if span is None else {**row, "span": span}
def per_unit(amount: float, covered: float, approx: bool) -> dict[str, dict]:
return {name: {"value": rates.precise(amount if seconds is None else amount * seconds / covered), "approx": approx and seconds is not None, "covered": covered} for name, seconds in rates.PERIODS}
class TestRates(unittest.TestCase):
def setUp(self):
tmp = tempfile.TemporaryDirectory()
self.addCleanup(tmp.cleanup)
self.council = Path(tmp.name) / "council"
self.folder = Path(tmp.name) / "kept" / "resources"
self.council.mkdir()
self.folder.mkdir(parents=True)
def register(self, tasks: dict) -> None:
(self.council / "rasci.json").write_text(json.dumps(tasks), encoding="utf-8")
def lines(self, path: Path, rows: list[dict]) -> None:
path.write_text("".join(json.dumps(row) + "\n" for row in rows), encoding="utf-8")
def rated(self, begin: float, end: float) -> dict:
return rates.rated(self.council, self.folder.parent, NOW, (begin, end))
def column(self, metric: str, begin: float, end: float) -> dict:
found = self.rated(begin, end)
return {period: found["rates"][period][metric] for period in rates.PERIOD_NAMES}
def test_a_week_of_seven_million_tokens_gives_a_million_a_day_seven_a_week_and_seven_in_all(self):
self.lines(self.folder / "tokens.jsonl", [token(LAST_HOUR - 10 * DAY, 5, 0), token(LAST_HOUR - 2 * DAY, 6_000_000, 1_000_000)])
tokens = self.column("tokens", NOW - 7 * DAY, NOW)
self.assertEqual(tokens, per_unit(7_000_000, 7 * DAY, False))
self.assertEqual((tokens["day"]["value"], tokens["week"]["value"], tokens["total"]["value"], tokens["month"]["value"]), (1_000_000.0, 7_000_000.0, 7_000_000.0, 30_000_000.0))
def test_done_tasks_key_results_and_commits_count_inside_the_window_and_scale_per_unit(self):
ends = [NOW - 30, NOW - 1800, NOW - 2 * DAY, NOW - 20 * DAY]
tasks = {f"work/t{index}": {"status": "done", "end": iso(end)} for index, end in enumerate(ends)}
others = {"status": "done", "end": iso(NOW - 10)}
results = [{"text": "met", "met": True, "met_at": iso(NOW - 600)}, {"text": "old", "met": True}, {"text": "open", "met": False}]
self.register({**tasks, "work/legacy": {"status": "done"}, "work/open": {**others, "status": "running", "key_results": results}, "work/halted": {**others, "status": "stopped"}, "operator/run": others})
commit = {"time": NOW - 5 * HOUR, "remote": "example.org/repo", "sha": "a" * 40, "values": {}}
self.lines(self.council / "scores.jsonl", [commit, commit, {**commit, "sha": None, "time": NOW - 60}, {**commit, "sha": "b" * 40, "time": NOW - 3 * DAY}])
day = self.rated(NOW - DAY, NOW)["rates"]
self.assertEqual({period: day[period]["tasks"] for period in rates.PERIOD_NAMES}, per_unit(2, DAY, False))
self.assertEqual({period: day[period]["krs"] for period in rates.PERIOD_NAMES}, per_unit(1, DAY, False))
self.assertEqual({period: day[period]["commits"] for period in rates.PERIOD_NAMES}, per_unit(1, DAY, False))
past = self.rated(NOW - 3 * DAY, NOW - DAY)["rates"]
self.assertEqual((past["total"]["tasks"]["value"], past["total"]["commits"]["value"], past["day"]["tasks"]["value"]), (1.0, 1.0, 0.5))
def test_a_window_reaching_before_the_first_token_row_divides_by_the_covered_part_and_marks_the_rates_approx(self):
self.register({"work/old": {"status": "done", "end": iso(NOW - 30 * DAY)}})
self.lines(self.folder / "tokens.jsonl", [token(LAST_HOUR - 2 * DAY, 4000, 800)])
tokens = self.column("tokens", NOW - 7 * DAY, NOW)
covered = NOW - (LAST_HOUR - 2 * DAY)
self.assertEqual(tokens, per_unit(4800, covered, True))
self.assertEqual(tokens["total"], {"value": 4800.0, "approx": False, "covered": covered})
self.assertEqual(self.column("tasks", NOW - 7 * DAY, NOW)["day"], {"value": 0.0, "approx": False, "covered": 7 * DAY})
def test_a_window_before_every_record_holds_no_value(self):
self.register({"work/a": {"status": "done", "end": iso(NOW - DAY)}})
self.lines(self.folder / "tokens.jsonl", [token(LAST_HOUR - DAY, 100, 0)])
self.lines(self.folder / "samples.jsonl", [sample(NOW - DAY, 40.0)])
found = self.rated(NOW - 9 * DAY, NOW - 8 * DAY)
self.assertEqual({found["rates"][period][kind] == NOTHING for period in rates.PERIOD_NAMES for kind in rates.COUNTS}, {True})
self.assertIsNone(found["rates"]["day"]["cpu"]["value"])
self.assertEqual((found["from"], found["to"], found["since"]), (NOW - 9 * DAY, NOW - 8 * DAY, LAST_HOUR - DAY))
def test_a_window_shorter_than_an_hour_shares_the_token_hour_and_marks_every_token_rate_approx(self):
self.lines(self.folder / "tokens.jsonl", [token(LAST_HOUR, 1000, 800)])
tokens = self.column("tokens", NOW - 600, NOW)
self.assertEqual(tokens["minute"], {"value": 60.0, "approx": True, "covered": 600})
self.assertEqual(tokens["total"], {"value": 600.0, "approx": True, "covered": 600})
def test_gauges_average_the_window_weighting_each_row_by_its_sampled_seconds_inside_on_both_ends(self):
start = NOW - 10 * DAY - (NOW - 10 * DAY) % DAY
self.lines(self.folder / "samples.jsonl", [
{**sample(start, 40.0), "span": 3600, "width": DAY},
{**sample(NOW - 2 * DAY - 300, 70.0, boot=80), "span": 600, "width": 900},
sample(NOW - 30, 10.0, boot=20),
])
found = self.rated(start + DAY / 2, NOW - 2 * DAY)["rates"]
self.assertEqual({found[period]["cpu"]["value"] for period in rates.PERIOD_NAMES}, {round((1800 * 40 + 200 * 70) / 2000, 1)})
self.assertEqual(found["hour"]["disk"], {"value": round((1800 * 50 + 200 * 80) / 2000, 1), "mount": "/boot", "used": 53, "total": 100, "approx": False})
self.assertEqual(found["week"]["vram"], {"value": None, "used": None, "approx": False})
def test_gauges_of_the_last_minutes_read_vram_and_the_fullest_disk(self):
self.lines(self.folder / "samples.jsonl", [sample(NOW - 90, 30.0, vram=(4, 8), boot=70), sample(NOW - 30, 10.0, vram=(2, 8), boot=90)])
last = self.rated(NOW - 60, NOW)["rates"]["minute"]
self.assertEqual((last["cpu"]["value"], last["ram"]["value"]), (10.0, 11.0))
self.assertEqual(last["vram"], {"value": 25.0, "used": 2 * GIB, "approx": False})
self.assertEqual(last["disk"], {"value": 90.0, "mount": "/boot", "used": 90, "total": 100, "approx": False})
both = self.rated(NOW - 120, NOW)["rates"]["hour"]
self.assertEqual((both["cpu"]["value"], both["vram"]["value"], both["vram"]["used"], both["disk"]["value"]), (20.0, 37.5, 3 * GIB, 80.0))
def test_a_window_without_a_sample_shows_no_gauge_and_kinds_with_records_but_none_inside_count_an_exact_zero(self):
self.lines(self.folder / "samples.jsonl", [sample(NOW - 2 * HOUR, 40.0)])
found = self.rated(NOW - HOUR, NOW)["rates"]
self.assertIsNone(found["day"]["cpu"]["value"])
self.assertEqual(found["minute"]["disk"], {"value": None, "mount": None, "used": None, "total": None, "approx": False})
self.assertEqual({found[period][kind]["value"] for period in rates.PERIOD_NAMES for kind in ("tasks", "commits", "krs")}, {0.0})
self.assertEqual(found["day"]["tokens"], NOTHING)
def test_folded_sampler_rows_feed_the_window_with_the_seconds_they_sampled(self):
old = NOW - NOW % DAY - 100 * DAY
raw = [sample(old + 3600 + index * 60, 20.0) for index in range(30)] + [sample(NOW - 9 * DAY + index * 60, 60.0) for index in range(30)] + [sample(NOW - 59, 10.0)]
kept = sampler.retain(sorted(raw, key=lambda row: row["t"]), NOW)
self.assertEqual([(row["t"], row["span"], row["width"]) for row in kept[:1]], [(old, 1800, DAY)])
self.assertEqual(sum(row["span"] for row in kept[1:-1]), 1800)
self.assertEqual({row["width"] for row in kept[1:-1]}, {900})
self.assertNotIn("span", kept[-1])
self.lines(self.folder / "samples.jsonl", kept)
month = round((10 * 60 + 60 * 1800) / (60 + 1800), 1)
year = round((10 * 60 + 60 * 1800 + 20 * 1800) / (60 + 1800 * 2), 1)
cpu = [self.rated(NOW - seconds, NOW)["rates"]["day"]["cpu"]["value"] for seconds in (60, 7 * DAY, 30 * DAY, 365 * DAY)]
self.assertEqual(cpu, [10.0, 10.0, month, year])
def test_a_lone_fifteen_minute_bucket_weighs_its_sampled_seconds_wherever_it_lies_in_its_day(self):
day = NOW - NOW % DAY - 200 * DAY
raw = [sample(day + 60 * index, 50.0) for index in range(15)] + [sample(day + DAY + 900 + 60 * index, 50.0) for index in range(15)]
kept = sampler.retain(raw, NOW)
self.assertEqual([(row["t"], row["span"], row["width"]) for row in kept], [(day, 900, DAY), (day + DAY, 900, DAY)])
self.assertEqual(sampler.retain(kept, NOW), kept)
def test_a_fifteen_minute_mean_written_before_rows_carried_span_stands_for_its_whole_bucket(self):
start = NOW - NOW % 900 - 20 * DAY
legacy = {**sample(start, 30.0), "queue": {"waiting": 0.0, "active": 1.0}}
kept = sampler.retain([legacy, sample(start + 900 + 60, 90.0)], NOW)
self.assertEqual([(row["t"], row["span"], row["width"]) for row in kept], [(start, 900, 900), (start + 900, 60, 900)])
def test_the_year_weighs_every_tier_by_the_seconds_it_sampled(self):
old, recent = NOW - NOW % DAY - 100 * DAY, NOW - NOW % DAY - 8 * DAY
raw = [sample(old + 60 * index, 90.0) for index in range(60)] + [sample(recent + 60 * index, 10.0) for index in range(60)]
self.lines(self.folder / "samples.jsonl", sampler.retain(raw, NOW))
self.assertEqual(self.rated(NOW - 365 * DAY, NOW)["rates"]["year"]["cpu"]["value"], 50.0)
def test_token_rows_older_than_ninety_days_fold_into_one_row_per_day_agent_and_model_kept_forever(self):
day, hour = NOW - NOW % DAY - 100 * DAY, NOW - NOW % HOUR
rows = [token(day + HOUR, 10, 0), token(day + 5 * HOUR, 20, 5), {**token(day + 2 * HOUR, 1, 0), "model": "other"}, token(hour - HOUR, 7, 0)]
kept = sampler.compact(rows, NOW)
daily = [{**token(day, 30, 5), "width": DAY}, {**token(day, 1, 0), "model": "other", "width": DAY}]
self.assertEqual(kept, [*daily, token(hour - HOUR, 7, 0)])
self.assertEqual(sampler.compact(kept, NOW + 400 * DAY)[:2], daily)
def test_a_daily_token_row_counts_by_the_share_of_its_day_inside_the_window(self):
year = NOW - 365 * DAY
self.lines(self.folder / "tokens.jsonl", [{**token(year - year % DAY, 8640, 0), "width": DAY}])
total = self.column("tokens", year, NOW)["total"]
self.assertEqual(total, {"value": rates.precise((year - year % DAY + DAY - year) / DAY * 8640), "approx": False, "covered": NOW - year})
def test_counts_shorten_to_thousands_millions_billions_and_trillions(self):
probe = "const R = require(process.argv[1]); console.log(JSON.stringify(JSON.parse(process.argv[2]).map(R.compact)));"
values = [999, 1000, 999_949, 999_950, 14_370_239_246.104, 999_960_000_000, 2.5e12, 3e15]
done = subprocess.run(["node", "-e", probe, str(HUB / "templates" / "resources.js"), json.dumps(values)], capture_output=True, text=True, timeout=60, check=True)
self.assertEqual(json.loads(done.stdout), ["999", "1.0k", "999.9k", "1.0M", "14.4B", "1.0T", "2.5T", "3000.0T"])
class TestHubRates(HubCase):
def test_the_hub_serves_the_rates_of_the_window_from_the_register_the_journal_and_the_sampler(self):
done = 1_700_000_000
self.write_council({"rasci.json": {"work/a": {"status": "done", "end": iso(done), "key_results": [{"text": "a", "met": True, "met_at": iso(done)}]}}})
(self.council / "scores.jsonl").write_text(json.dumps({"time": done, "remote": "example.org/repo", "sha": "c" * 40, "values": {}}) + "\n", encoding="utf-8")
status, headers, body = self.get(f"/api/rates?from={done - DAY}&to={done + DAY}")
self.assertEqual((status, headers["content-type"]), (200, "application/json"), body)
found = json.loads(body)
self.assertEqual([entry["name"] for entry in found["periods"]], list(rates.PERIOD_NAMES))
self.assertEqual({kind: found["rates"]["total"][kind]["value"] for kind in rates.COUNTS}, {"tasks": 1.0, "commits": 1.0, "krs": 1.0, "tokens": None})
self.assertEqual({kind: found["rates"]["day"][kind]["approx"] for kind in ("tasks", "commits", "krs")}, {"tasks": True, "commits": True, "krs": True})
self.assertEqual(set(found["rates"]["hour"]), set(rates.METRICS))
self.assertEqual((found["since"], found["from"], found["to"]), (done, done - DAY, done + DAY))
self.assertEqual(self.get(f"/api/rates?from={done}&to={done - 1}")[0], 400)
if __name__ == "__main__":
unittest.main()