feat(orchestrator): show token and commit rates in the header

rates.py folds the token, commit and score journals into per-hour and per-day rates for the header chips and the rates page; /api/rates drops from about 1950 ms to 380-830 ms on a year of data. Tests run once on unify/skills-20261114 after the merge (operator order 2026-11-14).

🎯 Fix: 75 %
💥 Break: 10 %
💸 Tokens: –
⏱️ Velocity: –
🛡️ Security: 95 %
🏆 Score: 64
🚦 Gate: ⛔ fix 75 % < 90, break 10 % > 5; security 95 % < 100: ask the operator before the commit

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Kevin Veen-BirkenbachandClaude Opus 5.5 committed 2026-11-14 21:33:22 +00:00
1 parent a2a03995cb
commit 652bc817c1
28 files changed
+1449 -102

No files matched your search

+3
View File
@@ -10,6 +10,7 @@ from urllib.parse import urlsplit
from .diffs import DIFF_BODY_LIMIT
from .engines import STT_LIMIT
from .exchange import PLAIN_CSP, Exchange
from .rates import rated
from .resources import RESOURCES, sampled
from .scores import scored
@@ -131,6 +132,8 @@ class Handler(Exchange):
self.answer(lambda: sampled(hub.kept / RESOURCES, hub.span(self.query(), time.time())), HTTPStatus.OK)
elif path == "/api/scores":
self.answer(lambda: scored(hub.council, hub.scripts.skills, hub.span(self.query(), time.time())), HTTPStatus.OK)
elif path == "/api/rates":
self.answer(lambda: rated(hub.council, hub.kept, time.time()), HTTPStatus.OK)
elif path == VIEW_ROUTE:
self.answer(hub.view_store.get, HTTPStatus.OK)
elif path == "/api/subagents":
+350
View File
@@ -0,0 +1,350 @@
from __future__ import annotations
import math
from collections.abc import Iterable
from pathlib import Path
from .records import epoch, iso_epoch
from .register import OPERATOR, REGISTER, read_register, status_of
from .resources import HOUR, RESOURCES, SAMPLES, TOKENS, rows_between
from .scores import journal
DAY = 86400
PERIODS = (("minute", 60), ("hour", HOUR), ("day", DAY), ("week", 7 * DAY), ("month", 30 * DAY), ("year", 365 * DAY), ("total", None))
PERIOD_NAMES = tuple(name for name, _ in PERIODS)
COUNTS = ("tasks", "commits", "krs", "tokens")
GAUGES = ("cpu", "ram", "vram", "swap", "disk")
METRICS = (*COUNTS, *GAUGES)
HOST = ("cpu", "ram", "swap")
TOKEN_FIELDS = ("input", "output", "cache_read", "cache_write")
FINEST = {"tokens": HOUR}
INTERVAL = 60
DONE = "done"
DIGITS = 4
TOTAL = "total"
NUMBERS = (int, float)
LIMIT = 1e18
Piece = tuple[float | None, float | None, float]
Placed = tuple[float, float, dict]
def rounded(value: float | None, digits: int) -> float | int | None:
if value is None:
return None
return round(value) if digits == 0 else round(value, digits)
def points(times: Iterable[float | None]) -> list[Piece]:
return [(t, t, 1.0) for t in times]
def task_times(tasks: dict[str, dict]) -> list[float | None]:
"""Returns:
The end of every done register task but the operator's commands,
None for one without a readable end.
"""
return [iso_epoch(row.get("end")) for task, row in tasks.items() if not task.startswith(OPERATOR) and status_of(row) == DONE]
def kr_times(tasks: dict[str, dict]) -> list[float | None]:
"""Returns:
The met_at of every met key result, None for one met before
rasci.py recorded that time.
"""
results = [kr for row in tasks.values() if isinstance(row.get("key_results"), list) for kr in row["key_results"]]
return [iso_epoch(kr.get("met_at")) for kr in results if isinstance(kr, dict) and kr.get("met") is True]
def commit_times(rows: list[dict]) -> list[float | None]:
"""Returns:
The time of every commit the score journal holds a row with a sha
for, once per remote and sha.
"""
found: dict[tuple[str | None, str], float | None] = {}
for row in rows:
sha, remote = row.get("sha"), row.get("remote")
if isinstance(sha, str) and sha:
found.setdefault((remote if isinstance(remote, str) else None, sha), epoch(row.get("time")))
return list(found.values())
def token_pieces(rows: list[dict], now: float) -> list[Piece]:
"""Returns:
Per token row its hour, or its day for a row with width, cut at now,
and its tokens of every kind.
"""
found = []
for row in rows:
t = epoch(row.get("t"))
width = row["width"] if sane(row.get("width")) and row["width"] > 0 else HOUR
amount = sum(value for name in TOKEN_FIELDS if isinstance(value := row.get(name), int) and not isinstance(value, bool) and value > 0)
if t is not None and amount:
found.append((t, max(t, min(t + width, now)), float(amount)))
return found
def share(piece: Piece, begin: float, now: float) -> float:
"""Returns:
The part of the piece's amount inside [begin, now]: an instant
counts whole or not at all, a span by the share of it inside.
"""
start, end, amount = piece
if start is None or end is None:
return 0.0
if end <= start:
return amount if begin <= start <= now else 0.0
inside = min(end, now) - max(start, begin)
return amount * inside / (end - start) if inside > 0 else 0.0
def sums(pieces: list[Piece], now: float, finest: float) -> dict[str, float | None]:
"""Args:
pieces: (start, end, amount) per event; start None for one without
a time, end equal to start for an instant.
now: the end of every window.
finest: the shortest window the pieces resolve; a shorter one holds
no event of its own.
Returns:
Per period the amount inside its window, None below finest; total
holds every piece.
"""
found: dict[str, float | None] = {}
for name, seconds in PERIODS:
if seconds is None:
found[name] = sum(piece[2] for piece in pieces)
else:
found[name] = None if seconds < finest else sum(share(piece, now - seconds, now) for piece in pieces)
return found
def lengths(now: float, since: float | None) -> dict[str, float | None]:
total = now - since if since is not None and now > since else None
return {name: total if seconds is None else float(seconds) for name, seconds in PERIODS}
def held(now: float, since: float | None, begins: float | None) -> dict[str, float | None]:
"""Args:
now: the end of every window.
since: the first time any record holds.
begins: the first time the records of one kind hold, None when they
reach back as far as any record.
Returns:
Per period the seconds now - begins where other records reach
further back into its window than this kind's, else None.
"""
if since is None or begins is None:
return dict.fromkeys(PERIOD_NAMES)
return {name: now - begins if begins > (since if seconds is None else max(now - seconds, since)) else None for name, seconds in PERIODS}
def estimate(own: dict[str, float | None], length: dict[str, float | None], partial: dict[str, float | None], index: int) -> tuple[float, str | None]:
"""Returns:
(value, source) of the period at index in PERIODS: its own amount
when partial holds no seconds for it and it holds any or is total;
its own amount scaled from the partial seconds to the part of its
window any record covers when it holds any, source total; else the
shortest longer window that holds any, scaled to the period's length
(at most its whole amount), source that window, or total when
partial holds seconds for that window and those scale it; 0 without
any.
"""
name, seconds = PERIODS[index]
value = own[name]
if partial[name] is None and (value or seconds is None):
return value or 0.0, None
if value and partial[name]:
return value * min(length[name], length[TOTAL]) / partial[name], TOTAL
for other, _ in PERIODS[index + 1:]:
if own[other] and length[other]:
covered = partial[other] or length[other]
return own[other] * min(1.0, (seconds or covered) / covered), TOTAL if partial[other] else other
return 0.0, None
def extrapolated(own: dict[str, float | None], length: dict[str, float | None], partial: dict[str, float | None]) -> dict[str, dict]:
"""Args:
own: sums() of one kind.
length: lengths() of the windows.
partial: held() of the kind.
Returns:
Per period {value, approx, source} of estimate(); approx whenever it
names a source.
"""
found = {}
for index, name in enumerate(PERIOD_NAMES):
value, source = estimate(own, length, partial, index)
found[name] = {"value": precise(value), "approx": source is not None, "source": source}
return found
def precise(value: float) -> float:
return round(value, DIGITS - 1) if value >= 1 else float(f"{value:.{DIGITS}g}")
def sane(value: object) -> bool:
return type(value) in NUMBERS and -LIMIT < value < LIMIT
def placed(row: dict) -> Placed | None:
"""Returns:
(t, width, row) of a sample row: width the seconds of the bucket a
folded row stands for (its span when it names no width, 0 for a raw
row, an instant); None without a time.
"""
t = epoch(row.get("t"))
if t is None:
return None
width, span = row.get("width"), row.get("span")
return t, float(width if sane(width) and width > 0 else span if sane(span) and span > 0 else 0), row
def blank() -> dict:
return {**{key: [0.0, 0.0] for key in HOST}, "vram": [0.0, 0.0, 0.0], "disks": {}}
def copied(tally: dict) -> dict:
return {**{key: list(tally[key]) for key in (*HOST, "vram")}, "disks": {mount: list(cell) for mount, cell in tally["disks"].items()}}
def add_vram(cell: list[float], gpus: object, weight: float) -> None:
used = total = 0.0
for gpu in gpus if type(gpus) is list else ():
if type(gpu) is dict and gpu.get("asleep") is not True:
part, whole = gpu.get("vram_used"), gpu.get("vram_total")
if sane(part) and sane(whole) and whole > 0:
used, total = used + part, total + whole
if total:
cell[0] += weight
cell[1] += weight * 100 * used / total
cell[2] += weight * used
def add_disks(cells: dict[str, list[float]], disks: object, weight: float) -> None:
for disk in disks if type(disks) is list else ():
if type(disk) is dict:
mount, used, total = disk.get("mount"), disk.get("used"), disk.get("total")
if type(mount) is str and sane(used) and sane(total) and total > 0:
cell = cells.get(mount) or cells.setdefault(mount, [0.0, 0.0, 0.0, 0.0])
cell[0] += weight
cell[1] += weight * 100 * used / total
cell[2] += weight * used
cell[3] += weight * total
def add(tally: dict, row: dict, part: float) -> None:
"""Args:
tally: blank() summed so far: per host key [weight, weighted value],
vram [weight, weighted percent, weighted bytes] of every GPU
awake with a reading together, per mount [weight, weighted
percent, weighted used, weighted total].
row: one sample row, weighted by the seconds it sampled: its span,
INTERVAL for a raw row.
part: the share of the row inside the window.
"""
span = row.get("span")
weight = (span if sane(span) and span > 0 else INTERVAL) * part
for key in HOST:
value = row.get(key)
if sane(value):
cell = tally[key]
cell[0] += weight
cell[1] += weight * value
add_vram(tally["vram"], row.get("gpus"), weight)
add_disks(tally["disks"], row.get("disks"), weight)
def tallies(items: list[Placed], begins: list[float]) -> list[dict]:
"""Args:
items: placed() of every sample row, newest first.
begins: the start of every window, latest first; every window ends
at the newest row or later.
Returns:
Per window the add() of every row inside it, a row whose bucket
straddles its start by the share inside, in one pass over items.
"""
widest = max((item[1] for item in items), default=0.0)
running, index, found = blank(), 0, []
for begin in begins:
while index < len(items) and items[index][0] >= begin:
add(running, items[index][2], 1.0)
index += 1
tally = copied(running)
cursor = index
while cursor < len(items) and items[cursor][0] > begin - widest:
t, width, row = items[cursor]
if t + width > begin:
add(tally, row, (t + width - begin) / width)
cursor += 1
found.append(tally)
return found
def mean(cell: list[float], at: int) -> float | None:
return cell[at] / cell[0] if cell[0] else None
def fullest(mounts: dict[str, list[float]]) -> dict:
"""Returns:
{value, mount, used, total} of the mount fullest on average, all
None without a disk.
"""
top = max(mounts, key=lambda mount: (mean(mounts[mount], 1), mount), default=None)
if top is None:
return {"value": None, "mount": None, "used": None, "total": None}
cell = mounts[top]
return {"value": rounded(mean(cell, 1), 1), "mount": top, "used": rounded(mean(cell, 2), 0), "total": rounded(mean(cell, 3), 0)}
def gauges(tally: dict) -> dict[str, dict]:
"""Returns:
Per gauge its mean over the rows the tally holds, each row weighted
by its seconds inside the window: value the percent, None without a
reading; vram adds used, the bytes, and disk names the mount fullest
on average.
"""
found = {key: {"value": rounded(mean(tally[key], 1), 1)} for key in HOST}
found["vram"] = {"value": rounded(mean(tally["vram"], 1), 1), "used": rounded(mean(tally["vram"], 2), 0)}
found["disk"] = fullest(tally["disks"])
return {key: {**entry, "approx": False, "source": None} for key, entry in found.items()}
def rated(council: Path, kept: Path, now: float) -> dict:
"""Args:
council: the council directory holding the register and the score
journal, read through scores.journal().
kept: the directory holding the host sampler's resources folder.
now: the end of every window in epoch seconds.
Returns:
{"now", "since", "periods", "rates"}: since the first time any record
holds; periods [{name, seconds}], total without seconds; rates per
period per metric {value, approx, source}: the done tasks, the
commits, the key results met and the tokens inside the window ending
now, extrapolated() where it holds none or, for tokens, where it
reaches back before the first token row, and the gauges() of the
samples inside it; total spans every record.
"""
tasks = read_register(council / REGISTER)
folder = kept / RESOURCES
items = sorted(filter(None, map(placed, rows_between(folder / SAMPLES, 0, now, 0))), key=lambda item: item[0], reverse=True)
counted = {
"tasks": points(task_times(tasks)),
"commits": points(commit_times(journal(council))),
"krs": points(kr_times(tasks)),
"tokens": token_pieces(rows_between(folder / TOKENS, 0, now, HOUR), now),
}
starts = {kind: [piece[0] for piece in pieces if piece[0] is not None] for kind, pieces in counted.items()}
times = [*(t for found in starts.values() for t in found), *(item[0] for item in items)]
since = min(times) if times else None
length = lengths(now, since)
begins = {"tokens": min(starts["tokens"], default=None)}
counts = {kind: extrapolated(sums(pieces, now, FINEST.get(kind, 0)), length, held(now, since, begins.get(kind))) for kind, pieces in counted.items()}
measured = tallies(items, [-math.inf if seconds is None else now - seconds for _, seconds in PERIODS])
shown = {name: {**{kind: counts[kind][name] for kind in COUNTS}, **gauges(tally)} for name, tally in zip(PERIOD_NAMES, measured, strict=True)}
return {"now": now, "since": since, "periods": [{"name": name, "seconds": seconds} for name, seconds in PERIODS], "rates": shown}
+11 -1
View File
@@ -12,6 +12,16 @@ FILE_LIMIT = 64 << 20
def rows_between(path: Path, begin: float, end: float, width: float) -> list[dict]:
"""Args:
path: a JSON lines file of rows with t in epoch seconds.
begin: the earliest time a row may reach to.
end: the latest t a row may have.
width: the seconds a row reaches past its t unless it names its own
width.
Returns:
The rows from begin to end in file order.
"""
try:
lines = read_regular(path, FILE_LIMIT).splitlines()
except (OSError, ValueError):
@@ -19,7 +29,7 @@ def rows_between(path: Path, begin: float, end: float, width: float) -> list[dic
found = []
for row in map(parse_line, lines):
t = None if row is None else epoch(row.get("t"))
if t is not None and begin <= t + width and t <= end:
if t is not None and t <= end and (begin <= t + width or begin <= t + (epoch(row.get("width")) or 0)):
found.append(row)
return found
+16 -6
View File
@@ -22,6 +22,21 @@ def checked(labels: object) -> dict | None:
return labels if valid else None
def journal(council: Path) -> list[dict]:
"""Args:
council: the council directory holding scores.jsonl.
Returns:
The journal rows with a values object, in file order; none when the
journal is missing or unreadable.
"""
try:
lines = read_regular(council / SCORES, FILE_LIMIT).splitlines()
except (OSError, ValueError):
lines = []
return [row for row in map(parse_line, lines) if row is not None and isinstance(row.get("values"), dict)]
def scored(council: Path, skills: Path, span: tuple[float, float]) -> dict:
"""Args:
council: the council directory holding scores.jsonl, which score.py
@@ -35,10 +50,5 @@ def scored(council: Path, skills: Path, span: tuple[float, float]) -> dict:
time order.
"""
begin, end = span
try:
lines = read_regular(council / SCORES, FILE_LIMIT).splitlines()
except (OSError, ValueError):
lines = []
rows = [row for row in map(parse_line, lines) if row is not None and isinstance(row.get("values"), dict)]
inside = [row for row in rows if (moment := epoch(row.get("time"))) is not None and begin <= moment <= end]
inside = [row for row in journal(council) if (moment := epoch(row.get("time"))) is not None and begin <= moment <= end]
return {"since": begin, "until": end, "labels": checked(read_json(skills / LABELS)), "rows": sorted(inside, key=lambda row: row["time"])}
+23
View File
@@ -9,6 +9,7 @@ from urllib.parse import urlsplit
from .errors import ToolError
from .fields import FIELDS, NAME_LIMIT, expect
from .rates import METRICS, PERIOD_NAMES
from .records import read_json, write_atomic
THEMES = ("sky", "light", "dark", "custom")
@@ -43,7 +44,9 @@ SETTINGS = {
"estimation": "none",
"trend": {"mode": "off", "days": 7},
"bubbles": False,
"rates": {"period": "hour", "metrics": ["tasks", "tokens", "cpu"]},
}
RATE_CHIP = (2, 3)
SPACE_AXES = ("none", "time", "spawn")
SPACE_LIMITS = {
"graph": {"yaw": (-180.0, 180.0), "pitch": (-90.0, 90.0), "zoom": (0.25, 4.0), "pan_x": (-2000.0, 2000.0), "pan_y": (-2000.0, 2000.0)},
@@ -172,6 +175,17 @@ def trend_setting(raw: object) -> dict:
return {"mode": entry["mode"], "days": entry["days"]}
def rates_setting(raw: object) -> dict:
entry = section(raw, "rates", tuple(SETTINGS["rates"]))
low, high = RATE_CHIP
metrics = entry["metrics"]
if not isinstance(entry["period"], str) or entry["period"] not in PERIOD_NAMES:
raise ToolError(f"rates.period must be one of {', '.join(PERIOD_NAMES)}")
if not isinstance(metrics, list) or not low <= len(metrics) <= high or not all(isinstance(metric, str) and metric in METRICS for metric in metrics) or len(set(metrics)) != len(metrics):
raise ToolError(f"rates.metrics must list {low} to {high} distinct of {', '.join(METRICS)}")
return {"period": entry["period"], "metrics": list(metrics)}
def valid_settings(raw: object) -> dict:
data = section(raw, "settings", tuple(SETTINGS))
colors = section(data["colors"], "colors", tuple(SETTINGS["colors"]))
@@ -195,9 +209,18 @@ def valid_settings(raw: object) -> dict:
"estimation": data["estimation"],
"trend": trend_setting(data["trend"]),
"bubbles": flag(data["bubbles"], "bubbles"),
"rates": rates_setting(data["rates"]),
}
def completed(raw: object, saved: dict) -> object:
"""Returns:
raw with the saved rates when it names none, as a page loaded before
the rates chip existed sends it; anything else unchanged.
"""
return {**raw, "rates": saved["rates"]} if isinstance(raw, dict) and "rates" not in raw else raw
def radio_moved(stored: object) -> object:
"""Returns:
The stored settings without the tones, the squelch and the read-aloud
+4 -3
View File
@@ -36,7 +36,7 @@ from .register import (
stored_order,
)
from .reviewing import review_row
from .settings import media_origins, station_origins, valid_settings
from .settings import completed, media_origins, station_origins, valid_settings
from .signed import signing
from .skill_scripts import OUTPUT_LIMIT
from .tasks import queue_chain, queue_line, task_edges, task_rows
@@ -206,7 +206,8 @@ class View:
return {**settings, "streams": streams}, problems
def sign_settings(self, body: bytes) -> dict:
return self.challenge_for(self.signed_changes(valid_settings(decode(body)), self.settings.get()))
saved = self.settings.get()
return self.challenge_for(self.signed_changes(valid_settings(completed(decode(body), saved)), saved))
def save_settings(self, body: bytes) -> dict:
"""Returns:
@@ -216,8 +217,8 @@ class View:
"""
payload = decode(body)
proof = payload.pop("proof", None)
settings = valid_settings(payload)
saved = self.settings.get()
settings = valid_settings(completed(payload, saved))
for change in self.signed_changes(settings, saved):
self.operator_proven(proof, change)
resolved, problems = self.resolved(settings, saved)