feat(orchestrator): show token and commit rates in the header

rates.py folds the token, commit and score journals into per-hour and per-day rates for the header chips and the rates page; /api/rates drops from about 1950 ms to 380-830 ms on a year of data. Tests run once on unify/skills-20261114 after the merge (operator order 2026-11-14).

🎯 Fix: 75 %
💥 Break: 10 %
💸 Tokens: –
⏱️ Velocity: –
🛡️ Security: 95 %
🏆 Score: 64
🚦 Gate: ⛔ fix 75 % < 90, break 10 % > 5; security 95 % < 100: ask the operator before the commit

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Kevin Veen-BirkenbachandClaude Opus 5.5 committed 2026-11-14 21:33:22 +00:00
1 parent a2a03995cb
commit 652bc817c1
28 files changed
+1449 -102

No files matched your search

+119 -48
View File
@@ -13,7 +13,9 @@
reading for (no driver, a driver mismatch) keeps its name with util
and vram None. In the first minute of a 15-minute bucket, rows older
than 7 days fold into 15-minute means and means older than 90 days
are dropped.
into one mean per day, kept forever, each row weighted by the seconds
it sampled; a folded row carries span, the seconds its rows sampled
(60 per raw row), and width, the seconds of its bucket.
tokens: add the input, output, cache-read and cache-write tokens of the
usage fields in the Claude Code transcripts under --transcripts since
each file's byte offset to <state>/tokens.jsonl as {t, agent, model,
@@ -25,7 +27,8 @@
seen: {hour: [key digest]}}, seen keeping the hours this run read and
the last two, and is replaced only when it changes. In the first
minute of an hour, rows of the same hour, agent and model merge and
rows older than 90 days are dropped.
the rows of a day that ended more than 90 days ago fold into one row
per day, agent and model with width: 86400, kept forever.
--state: the directory of these files, the unit passes
~/.local/state/agents-council/resources.
--council: the council directory with queue.json and the claims.
@@ -61,8 +64,9 @@ import queue_live
INTERVAL = 60
BUCKET = 900
HOUR = 3600
FINE = 7 * 86400
COARSE = 90 * 86400
DAY = 86400
FINE = 7 * DAY
COARSE = 90 * DAY
COMMAND_SECONDS = 10
KEY_BYTES = 9
KIB = 1 << 10
@@ -278,77 +282,122 @@ def sample(proc: Path, sys_root: Path, council: Path, run: Callable[[list[str]],
}
def average(values: list[object], digits: int) -> float | None:
found = [value for value in values if isinstance(value, int | float) and not isinstance(value, bool)]
return round(sum(found) / len(found), digits) if found else None
def is_number(value: object) -> bool:
return isinstance(value, int | float) and not isinstance(value, bool)
def whole(values: list[object]) -> int | None:
found = average(values, 0)
def weight(row: dict) -> int | float:
"""Returns:
The seconds row sampled: its span, INTERVAL for a raw row.
"""
span = row.get("span")
return span if is_number(span) and span > 0 else INTERVAL
def average(pairs: list[tuple[int | float, object]], digits: int) -> float | None:
"""Args:
pairs: (weight, value) per reading; a value that is no number is
left out.
"""
found = [(part, value) for part, value in pairs if is_number(value)]
total = sum(part for part, _ in found)
return round(sum(part * value for part, value in found) / total, digits) if total else None
def whole(pairs: list[tuple[int | float, object]]) -> int | None:
found = average(pairs, 0)
return None if found is None else int(found)
def listed(rows: list[dict], key: str) -> list[dict]:
return [item for row in rows if isinstance(row.get(key), list) for item in row[key] if isinstance(item, dict)]
def listed(rows: list[dict], key: str) -> list[tuple[int | float, dict]]:
return [(weight(row), item) for row in rows if isinstance(row.get(key), list) for item in row[key] if isinstance(item, dict)]
def grouped(items: list[dict], key: str) -> dict[object, list[dict]]:
found: dict[object, list[dict]] = {}
for item in items:
found.setdefault(item.get(key), []).append(item)
def grouped(items: list[tuple[int | float, dict]], key: str) -> dict[object, list[tuple[int | float, dict]]]:
found: dict[object, list[tuple[int | float, dict]]] = {}
for part, item in items:
found.setdefault(item.get(key), []).append((part, item))
return found
def gpu_mean(name: object, readings: list[dict]) -> dict:
awake = [reading for reading in readings if reading.get("asleep") is not True]
def means(readings: list[tuple[int | float, dict]], key: str) -> list[tuple[int | float, object]]:
return [(part, reading.get(key)) for part, reading in readings]
def gpu_mean(name: object, readings: list[tuple[int | float, dict]]) -> dict:
awake = [(part, reading) for part, reading in readings if reading.get("asleep") is not True]
if not awake:
return {"name": name, "asleep": True}
return {"name": name, "util": average([reading.get("util") for reading in awake], 1), "vram_used": whole([reading.get("vram_used") for reading in awake]), "vram_total": whole([reading.get("vram_total") for reading in awake])}
return {"name": name, "util": average(means(awake, "util"), 1), "vram_used": whole(means(awake, "vram_used")), "vram_total": whole(means(awake, "vram_total"))}
def disk_means(rows: list[dict]) -> list[dict]:
return [{"mount": mount, "used": whole([disk.get("used") for disk in disks]), "total": whole([disk.get("total") for disk in disks])} for mount, disks in grouped(listed(rows, "disks"), "mount").items()]
return [{"mount": mount, "used": whole(means(disks, "used")), "total": whole(means(disks, "total"))} for mount, disks in grouped(listed(rows, "disks"), "mount").items()]
def queue_mean(rows: list[dict]) -> dict:
queues = [row["queue"] for row in rows if isinstance(row.get("queue"), dict)]
return {key: average([entry.get(key) for entry in queues], 2) for key in ("waiting", "active")}
queues = [(weight(row), row["queue"]) for row in rows if isinstance(row.get("queue"), dict)]
return {key: average(means(queues, key), 2) for key in ("waiting", "active")}
def mean_row(start: int, rows: list[dict]) -> dict:
def mean_row(start: int, width: int, rows: list[dict]) -> dict:
"""Returns:
The 15-minute mean of rows at start: per mount and per GPU the mean
of the rows that list it, a GPU asleep in every row stays asleep;
a single row already at start comes back unchanged.
The mean of rows in the bucket of width seconds at start, each row
weighted by weight(): per mount and per GPU the mean of the rows that
list it, a GPU asleep in every row stays asleep; span the seconds the
rows sampled together, width the bucket's.
"""
host = {key: average([(weight(row), row.get(key)) for row in rows], digits) for key, digits in HOST_DIGITS}
gpu_rows = [gpu_mean(name, readings) for name, readings in grouped(listed(rows, "gpus"), "name").items()]
return {"t": start, "disks": disk_means(rows), **host, "gpus": gpu_rows, "queue": queue_mean(rows), "span": sum(weight(row) for row in rows), "width": width}
def folded(start: int, width: int, rows: list[dict]) -> dict:
"""Returns:
The one row the bucket of width seconds at start keeps: a lone row
at start already folded to width or wider unchanged; a lone row at
start without span, a 15-minute mean written before folded rows
carried span, with span and width the whole bucket; else mean_row().
"""
if len(rows) == 1 and rows[0]["t"] == start:
return rows[0]
host = {key: average([row.get(key) for row in rows], digits) for key, digits in HOST_DIGITS}
gpu_rows = [gpu_mean(name, readings) for name, readings in grouped(listed(rows, "gpus"), "name").items()]
return {"t": start, "disks": disk_means(rows), **host, "gpus": gpu_rows, "queue": queue_mean(rows)}
kept = rows[0].get("width")
if is_number(kept) and kept >= width:
return rows[0]
if "span" not in rows[0]:
return {**rows[0], "span": width, "width": width}
return mean_row(start, width, rows)
def timed(row: object) -> bool:
return isinstance(row, dict) and isinstance(row.get("t"), int | float) and not isinstance(row.get("t"), bool)
return isinstance(row, dict) and is_number(row.get("t"))
def fold(rows: list[dict], width: int, line: float) -> list[dict]:
"""Returns:
rows with every bucket of width seconds that ended by line folded()
into one row at its start, the other rows following unchanged and in
order.
"""
buckets: dict[int, list[dict]] = {}
rest = []
for row in rows:
start = int(row["t"] // width * width)
if start + width <= line:
buckets.setdefault(start, []).append(row)
else:
rest.append(row)
return [folded(start, width, buckets[start]) for start in sorted(buckets)] + rest
def retain(rows: list[dict], now: float) -> list[dict]:
"""Returns:
rows without those older than 90 days, every 15-minute bucket that
ended more than 7 days ago folded into one mean at its start, the
newer rows unchanged and in order.
rows with every 15-minute bucket that ended more than 7 days ago
folded into one mean at its start, every day (UTC) that ended more
than 90 days ago folded into one mean of those, kept forever, and the
newer rows unchanged and in order; a folded row carries span, the
seconds its rows sampled, and width, its bucket's seconds.
"""
buckets: dict[int, list[dict]] = {}
recent = []
for row in rows:
if not timed(row) or row["t"] < now - COARSE:
continue
start = int(row["t"] // BUCKET * BUCKET)
if start + BUCKET <= now - FINE:
buckets.setdefault(start, []).append(row)
else:
recent.append(row)
return [mean_row(start, buckets[start]) for start in sorted(buckets)] + recent
return fold(fold([row for row in rows if timed(row)], BUCKET, now - FINE), DAY, now - COARSE)
def read_rows(path: Path) -> list[dict]:
@@ -498,7 +547,25 @@ def scan(path: Path, start: int, budget: int, seen: dict[int, set[str]]) -> tupl
def token_rows(sums: dict[tuple, list[int]]) -> list[dict]:
return [{"t": t, "agent": agent, "model": model, **dict(zip(NAMES, counts, strict=True))} for (t, agent, model), counts in sorted(sums.items())]
"""Args:
sums: per (t, agent, model, width) the counts in COUNTS order.
Returns:
One token row per key in key order; a row other than hourly names
its width.
"""
return [{"t": t, "agent": agent, "model": model, **dict(zip(NAMES, counts, strict=True)), **({} if width == HOUR else {"width": width})} for (t, agent, model, width), counts in sorted(sums.items())]
def token_key(row: dict, line: float) -> tuple:
"""Returns:
(t, agent, model, width) of the row a token row merges into: its day
(UTC) when the row is daily or the day ended by line, else its hour.
"""
day = int(row["t"] // DAY * DAY)
if row.get("width") == DAY or day + DAY <= line:
return day, row["agent"], row["model"], DAY
return row["t"], row["agent"], row["model"], HOUR
def add(sums: dict[tuple, list[int]], key: tuple, counts: tuple[int, ...]) -> None:
@@ -508,10 +575,14 @@ def add(sums: dict[tuple, list[int]], key: tuple, counts: tuple[int, ...]) -> No
def compact(rows: list[dict], now: float) -> list[dict]:
"""Returns:
The token rows merged per token_key(), every day that ended more than
90 days ago one row per agent and model, kept forever.
"""
sums: dict[tuple, list[int]] = {}
for row in rows:
if timed(row) and row["t"] >= now - COARSE and isinstance(row.get("agent"), str) and isinstance(row.get("model"), str):
add(sums, (row["t"], row["agent"], row["model"]), tuple(count(row.get(name)) for name in NAMES))
if timed(row) and isinstance(row.get("agent"), str) and isinstance(row.get("model"), str):
add(sums, token_key(row, now - COARSE), tuple(count(row.get(name)) for name in NAMES))
return token_rows(sums)
@@ -568,7 +639,7 @@ def count_tokens(transcripts: Path, council: Path, state: Path, now: float, budg
seen.setdefault(hour, set()).update(read)
touched.add(hour)
for ts, session, model, counts in found:
add(sums, (int(ts // HOUR * HOUR), names.get(session) or session[:8], model), counts)
add(sums, (int(ts // HOUR * HOUR), names.get(session) or session[:8], model, HOUR), counts)
rows = token_rows(sums)
append_rows(state / TOKENS, rows)
kept = {"files": files, "seen": kept_seen(seen, touched, now)}
+3
View File
@@ -10,6 +10,7 @@ from urllib.parse import urlsplit
from .diffs import DIFF_BODY_LIMIT
from .engines import STT_LIMIT
from .exchange import PLAIN_CSP, Exchange
from .rates import rated
from .resources import RESOURCES, sampled
from .scores import scored
@@ -131,6 +132,8 @@ class Handler(Exchange):
self.answer(lambda: sampled(hub.kept / RESOURCES, hub.span(self.query(), time.time())), HTTPStatus.OK)
elif path == "/api/scores":
self.answer(lambda: scored(hub.council, hub.scripts.skills, hub.span(self.query(), time.time())), HTTPStatus.OK)
elif path == "/api/rates":
self.answer(lambda: rated(hub.council, hub.kept, time.time()), HTTPStatus.OK)
elif path == VIEW_ROUTE:
self.answer(hub.view_store.get, HTTPStatus.OK)
elif path == "/api/subagents":
+350
View File
@@ -0,0 +1,350 @@
from __future__ import annotations
import math
from collections.abc import Iterable
from pathlib import Path
from .records import epoch, iso_epoch
from .register import OPERATOR, REGISTER, read_register, status_of
from .resources import HOUR, RESOURCES, SAMPLES, TOKENS, rows_between
from .scores import journal
DAY = 86400
PERIODS = (("minute", 60), ("hour", HOUR), ("day", DAY), ("week", 7 * DAY), ("month", 30 * DAY), ("year", 365 * DAY), ("total", None))
PERIOD_NAMES = tuple(name for name, _ in PERIODS)
COUNTS = ("tasks", "commits", "krs", "tokens")
GAUGES = ("cpu", "ram", "vram", "swap", "disk")
METRICS = (*COUNTS, *GAUGES)
HOST = ("cpu", "ram", "swap")
TOKEN_FIELDS = ("input", "output", "cache_read", "cache_write")
FINEST = {"tokens": HOUR}
INTERVAL = 60
DONE = "done"
DIGITS = 4
TOTAL = "total"
NUMBERS = (int, float)
LIMIT = 1e18
Piece = tuple[float | None, float | None, float]
Placed = tuple[float, float, dict]
def rounded(value: float | None, digits: int) -> float | int | None:
if value is None:
return None
return round(value) if digits == 0 else round(value, digits)
def points(times: Iterable[float | None]) -> list[Piece]:
return [(t, t, 1.0) for t in times]
def task_times(tasks: dict[str, dict]) -> list[float | None]:
"""Returns:
The end of every done register task but the operator's commands,
None for one without a readable end.
"""
return [iso_epoch(row.get("end")) for task, row in tasks.items() if not task.startswith(OPERATOR) and status_of(row) == DONE]
def kr_times(tasks: dict[str, dict]) -> list[float | None]:
"""Returns:
The met_at of every met key result, None for one met before
rasci.py recorded that time.
"""
results = [kr for row in tasks.values() if isinstance(row.get("key_results"), list) for kr in row["key_results"]]
return [iso_epoch(kr.get("met_at")) for kr in results if isinstance(kr, dict) and kr.get("met") is True]
def commit_times(rows: list[dict]) -> list[float | None]:
"""Returns:
The time of every commit the score journal holds a row with a sha
for, once per remote and sha.
"""
found: dict[tuple[str | None, str], float | None] = {}
for row in rows:
sha, remote = row.get("sha"), row.get("remote")
if isinstance(sha, str) and sha:
found.setdefault((remote if isinstance(remote, str) else None, sha), epoch(row.get("time")))
return list(found.values())
def token_pieces(rows: list[dict], now: float) -> list[Piece]:
"""Returns:
Per token row its hour, or its day for a row with width, cut at now,
and its tokens of every kind.
"""
found = []
for row in rows:
t = epoch(row.get("t"))
width = row["width"] if sane(row.get("width")) and row["width"] > 0 else HOUR
amount = sum(value for name in TOKEN_FIELDS if isinstance(value := row.get(name), int) and not isinstance(value, bool) and value > 0)
if t is not None and amount:
found.append((t, max(t, min(t + width, now)), float(amount)))
return found
def share(piece: Piece, begin: float, now: float) -> float:
"""Returns:
The part of the piece's amount inside [begin, now]: an instant
counts whole or not at all, a span by the share of it inside.
"""
start, end, amount = piece
if start is None or end is None:
return 0.0
if end <= start:
return amount if begin <= start <= now else 0.0
inside = min(end, now) - max(start, begin)
return amount * inside / (end - start) if inside > 0 else 0.0
def sums(pieces: list[Piece], now: float, finest: float) -> dict[str, float | None]:
"""Args:
pieces: (start, end, amount) per event; start None for one without
a time, end equal to start for an instant.
now: the end of every window.
finest: the shortest window the pieces resolve; a shorter one holds
no event of its own.
Returns:
Per period the amount inside its window, None below finest; total
holds every piece.
"""
found: dict[str, float | None] = {}
for name, seconds in PERIODS:
if seconds is None:
found[name] = sum(piece[2] for piece in pieces)
else:
found[name] = None if seconds < finest else sum(share(piece, now - seconds, now) for piece in pieces)
return found
def lengths(now: float, since: float | None) -> dict[str, float | None]:
total = now - since if since is not None and now > since else None
return {name: total if seconds is None else float(seconds) for name, seconds in PERIODS}
def held(now: float, since: float | None, begins: float | None) -> dict[str, float | None]:
"""Args:
now: the end of every window.
since: the first time any record holds.
begins: the first time the records of one kind hold, None when they
reach back as far as any record.
Returns:
Per period the seconds now - begins where other records reach
further back into its window than this kind's, else None.
"""
if since is None or begins is None:
return dict.fromkeys(PERIOD_NAMES)
return {name: now - begins if begins > (since if seconds is None else max(now - seconds, since)) else None for name, seconds in PERIODS}
def estimate(own: dict[str, float | None], length: dict[str, float | None], partial: dict[str, float | None], index: int) -> tuple[float, str | None]:
"""Returns:
(value, source) of the period at index in PERIODS: its own amount
when partial holds no seconds for it and it holds any or is total;
its own amount scaled from the partial seconds to the part of its
window any record covers when it holds any, source total; else the
shortest longer window that holds any, scaled to the period's length
(at most its whole amount), source that window, or total when
partial holds seconds for that window and those scale it; 0 without
any.
"""
name, seconds = PERIODS[index]
value = own[name]
if partial[name] is None and (value or seconds is None):
return value or 0.0, None
if value and partial[name]:
return value * min(length[name], length[TOTAL]) / partial[name], TOTAL
for other, _ in PERIODS[index + 1:]:
if own[other] and length[other]:
covered = partial[other] or length[other]
return own[other] * min(1.0, (seconds or covered) / covered), TOTAL if partial[other] else other
return 0.0, None
def extrapolated(own: dict[str, float | None], length: dict[str, float | None], partial: dict[str, float | None]) -> dict[str, dict]:
"""Args:
own: sums() of one kind.
length: lengths() of the windows.
partial: held() of the kind.
Returns:
Per period {value, approx, source} of estimate(); approx whenever it
names a source.
"""
found = {}
for index, name in enumerate(PERIOD_NAMES):
value, source = estimate(own, length, partial, index)
found[name] = {"value": precise(value), "approx": source is not None, "source": source}
return found
def precise(value: float) -> float:
return round(value, DIGITS - 1) if value >= 1 else float(f"{value:.{DIGITS}g}")
def sane(value: object) -> bool:
return type(value) in NUMBERS and -LIMIT < value < LIMIT
def placed(row: dict) -> Placed | None:
"""Returns:
(t, width, row) of a sample row: width the seconds of the bucket a
folded row stands for (its span when it names no width, 0 for a raw
row, an instant); None without a time.
"""
t = epoch(row.get("t"))
if t is None:
return None
width, span = row.get("width"), row.get("span")
return t, float(width if sane(width) and width > 0 else span if sane(span) and span > 0 else 0), row
def blank() -> dict:
return {**{key: [0.0, 0.0] for key in HOST}, "vram": [0.0, 0.0, 0.0], "disks": {}}
def copied(tally: dict) -> dict:
return {**{key: list(tally[key]) for key in (*HOST, "vram")}, "disks": {mount: list(cell) for mount, cell in tally["disks"].items()}}
def add_vram(cell: list[float], gpus: object, weight: float) -> None:
used = total = 0.0
for gpu in gpus if type(gpus) is list else ():
if type(gpu) is dict and gpu.get("asleep") is not True:
part, whole = gpu.get("vram_used"), gpu.get("vram_total")
if sane(part) and sane(whole) and whole > 0:
used, total = used + part, total + whole
if total:
cell[0] += weight
cell[1] += weight * 100 * used / total
cell[2] += weight * used
def add_disks(cells: dict[str, list[float]], disks: object, weight: float) -> None:
for disk in disks if type(disks) is list else ():
if type(disk) is dict:
mount, used, total = disk.get("mount"), disk.get("used"), disk.get("total")
if type(mount) is str and sane(used) and sane(total) and total > 0:
cell = cells.get(mount) or cells.setdefault(mount, [0.0, 0.0, 0.0, 0.0])
cell[0] += weight
cell[1] += weight * 100 * used / total
cell[2] += weight * used
cell[3] += weight * total
def add(tally: dict, row: dict, part: float) -> None:
"""Args:
tally: blank() summed so far: per host key [weight, weighted value],
vram [weight, weighted percent, weighted bytes] of every GPU
awake with a reading together, per mount [weight, weighted
percent, weighted used, weighted total].
row: one sample row, weighted by the seconds it sampled: its span,
INTERVAL for a raw row.
part: the share of the row inside the window.
"""
span = row.get("span")
weight = (span if sane(span) and span > 0 else INTERVAL) * part
for key in HOST:
value = row.get(key)
if sane(value):
cell = tally[key]
cell[0] += weight
cell[1] += weight * value
add_vram(tally["vram"], row.get("gpus"), weight)
add_disks(tally["disks"], row.get("disks"), weight)
def tallies(items: list[Placed], begins: list[float]) -> list[dict]:
"""Args:
items: placed() of every sample row, newest first.
begins: the start of every window, latest first; every window ends
at the newest row or later.
Returns:
Per window the add() of every row inside it, a row whose bucket
straddles its start by the share inside, in one pass over items.
"""
widest = max((item[1] for item in items), default=0.0)
running, index, found = blank(), 0, []
for begin in begins:
while index < len(items) and items[index][0] >= begin:
add(running, items[index][2], 1.0)
index += 1
tally = copied(running)
cursor = index
while cursor < len(items) and items[cursor][0] > begin - widest:
t, width, row = items[cursor]
if t + width > begin:
add(tally, row, (t + width - begin) / width)
cursor += 1
found.append(tally)
return found
def mean(cell: list[float], at: int) -> float | None:
return cell[at] / cell[0] if cell[0] else None
def fullest(mounts: dict[str, list[float]]) -> dict:
"""Returns:
{value, mount, used, total} of the mount fullest on average, all
None without a disk.
"""
top = max(mounts, key=lambda mount: (mean(mounts[mount], 1), mount), default=None)
if top is None:
return {"value": None, "mount": None, "used": None, "total": None}
cell = mounts[top]
return {"value": rounded(mean(cell, 1), 1), "mount": top, "used": rounded(mean(cell, 2), 0), "total": rounded(mean(cell, 3), 0)}
def gauges(tally: dict) -> dict[str, dict]:
"""Returns:
Per gauge its mean over the rows the tally holds, each row weighted
by its seconds inside the window: value the percent, None without a
reading; vram adds used, the bytes, and disk names the mount fullest
on average.
"""
found = {key: {"value": rounded(mean(tally[key], 1), 1)} for key in HOST}
found["vram"] = {"value": rounded(mean(tally["vram"], 1), 1), "used": rounded(mean(tally["vram"], 2), 0)}
found["disk"] = fullest(tally["disks"])
return {key: {**entry, "approx": False, "source": None} for key, entry in found.items()}
def rated(council: Path, kept: Path, now: float) -> dict:
"""Args:
council: the council directory holding the register and the score
journal, read through scores.journal().
kept: the directory holding the host sampler's resources folder.
now: the end of every window in epoch seconds.
Returns:
{"now", "since", "periods", "rates"}: since the first time any record
holds; periods [{name, seconds}], total without seconds; rates per
period per metric {value, approx, source}: the done tasks, the
commits, the key results met and the tokens inside the window ending
now, extrapolated() where it holds none or, for tokens, where it
reaches back before the first token row, and the gauges() of the
samples inside it; total spans every record.
"""
tasks = read_register(council / REGISTER)
folder = kept / RESOURCES
items = sorted(filter(None, map(placed, rows_between(folder / SAMPLES, 0, now, 0))), key=lambda item: item[0], reverse=True)
counted = {
"tasks": points(task_times(tasks)),
"commits": points(commit_times(journal(council))),
"krs": points(kr_times(tasks)),
"tokens": token_pieces(rows_between(folder / TOKENS, 0, now, HOUR), now),
}
starts = {kind: [piece[0] for piece in pieces if piece[0] is not None] for kind, pieces in counted.items()}
times = [*(t for found in starts.values() for t in found), *(item[0] for item in items)]
since = min(times) if times else None
length = lengths(now, since)
begins = {"tokens": min(starts["tokens"], default=None)}
counts = {kind: extrapolated(sums(pieces, now, FINEST.get(kind, 0)), length, held(now, since, begins.get(kind))) for kind, pieces in counted.items()}
measured = tallies(items, [-math.inf if seconds is None else now - seconds for _, seconds in PERIODS])
shown = {name: {**{kind: counts[kind][name] for kind in COUNTS}, **gauges(tally)} for name, tally in zip(PERIOD_NAMES, measured, strict=True)}
return {"now": now, "since": since, "periods": [{"name": name, "seconds": seconds} for name, seconds in PERIODS], "rates": shown}
+11 -1
View File
@@ -12,6 +12,16 @@ FILE_LIMIT = 64 << 20
def rows_between(path: Path, begin: float, end: float, width: float) -> list[dict]:
"""Args:
path: a JSON lines file of rows with t in epoch seconds.
begin: the earliest time a row may reach to.
end: the latest t a row may have.
width: the seconds a row reaches past its t unless it names its own
width.
Returns:
The rows from begin to end in file order.
"""
try:
lines = read_regular(path, FILE_LIMIT).splitlines()
except (OSError, ValueError):
@@ -19,7 +29,7 @@ def rows_between(path: Path, begin: float, end: float, width: float) -> list[dic
found = []
for row in map(parse_line, lines):
t = None if row is None else epoch(row.get("t"))
if t is not None and begin <= t + width and t <= end:
if t is not None and t <= end and (begin <= t + width or begin <= t + (epoch(row.get("width")) or 0)):
found.append(row)
return found
+16 -6
View File
@@ -22,6 +22,21 @@ def checked(labels: object) -> dict | None:
return labels if valid else None
def journal(council: Path) -> list[dict]:
"""Args:
council: the council directory holding scores.jsonl.
Returns:
The journal rows with a values object, in file order; none when the
journal is missing or unreadable.
"""
try:
lines = read_regular(council / SCORES, FILE_LIMIT).splitlines()
except (OSError, ValueError):
lines = []
return [row for row in map(parse_line, lines) if row is not None and isinstance(row.get("values"), dict)]
def scored(council: Path, skills: Path, span: tuple[float, float]) -> dict:
"""Args:
council: the council directory holding scores.jsonl, which score.py
@@ -35,10 +50,5 @@ def scored(council: Path, skills: Path, span: tuple[float, float]) -> dict:
time order.
"""
begin, end = span
try:
lines = read_regular(council / SCORES, FILE_LIMIT).splitlines()
except (OSError, ValueError):
lines = []
rows = [row for row in map(parse_line, lines) if row is not None and isinstance(row.get("values"), dict)]
inside = [row for row in rows if (moment := epoch(row.get("time"))) is not None and begin <= moment <= end]
inside = [row for row in journal(council) if (moment := epoch(row.get("time"))) is not None and begin <= moment <= end]
return {"since": begin, "until": end, "labels": checked(read_json(skills / LABELS)), "rows": sorted(inside, key=lambda row: row["time"])}
+23
View File
@@ -9,6 +9,7 @@ from urllib.parse import urlsplit
from .errors import ToolError
from .fields import FIELDS, NAME_LIMIT, expect
from .rates import METRICS, PERIOD_NAMES
from .records import read_json, write_atomic
THEMES = ("sky", "light", "dark", "custom")
@@ -43,7 +44,9 @@ SETTINGS = {
"estimation": "none",
"trend": {"mode": "off", "days": 7},
"bubbles": False,
"rates": {"period": "hour", "metrics": ["tasks", "tokens", "cpu"]},
}
RATE_CHIP = (2, 3)
SPACE_AXES = ("none", "time", "spawn")
SPACE_LIMITS = {
"graph": {"yaw": (-180.0, 180.0), "pitch": (-90.0, 90.0), "zoom": (0.25, 4.0), "pan_x": (-2000.0, 2000.0), "pan_y": (-2000.0, 2000.0)},
@@ -172,6 +175,17 @@ def trend_setting(raw: object) -> dict:
return {"mode": entry["mode"], "days": entry["days"]}
def rates_setting(raw: object) -> dict:
entry = section(raw, "rates", tuple(SETTINGS["rates"]))
low, high = RATE_CHIP
metrics = entry["metrics"]
if not isinstance(entry["period"], str) or entry["period"] not in PERIOD_NAMES:
raise ToolError(f"rates.period must be one of {', '.join(PERIOD_NAMES)}")
if not isinstance(metrics, list) or not low <= len(metrics) <= high or not all(isinstance(metric, str) and metric in METRICS for metric in metrics) or len(set(metrics)) != len(metrics):
raise ToolError(f"rates.metrics must list {low} to {high} distinct of {', '.join(METRICS)}")
return {"period": entry["period"], "metrics": list(metrics)}
def valid_settings(raw: object) -> dict:
data = section(raw, "settings", tuple(SETTINGS))
colors = section(data["colors"], "colors", tuple(SETTINGS["colors"]))
@@ -195,9 +209,18 @@ def valid_settings(raw: object) -> dict:
"estimation": data["estimation"],
"trend": trend_setting(data["trend"]),
"bubbles": flag(data["bubbles"], "bubbles"),
"rates": rates_setting(data["rates"]),
}
def completed(raw: object, saved: dict) -> object:
"""Returns:
raw with the saved rates when it names none, as a page loaded before
the rates chip existed sends it; anything else unchanged.
"""
return {**raw, "rates": saved["rates"]} if isinstance(raw, dict) and "rates" not in raw else raw
def radio_moved(stored: object) -> object:
"""Returns:
The stored settings without the tones, the squelch and the read-aloud
+4 -3
View File
@@ -36,7 +36,7 @@ from .register import (
stored_order,
)
from .reviewing import review_row
from .settings import media_origins, station_origins, valid_settings
from .settings import completed, media_origins, station_origins, valid_settings
from .signed import signing
from .skill_scripts import OUTPUT_LIMIT
from .tasks import queue_chain, queue_line, task_edges, task_rows
@@ -206,7 +206,8 @@ class View:
return {**settings, "streams": streams}, problems
def sign_settings(self, body: bytes) -> dict:
return self.challenge_for(self.signed_changes(valid_settings(decode(body)), self.settings.get()))
saved = self.settings.get()
return self.challenge_for(self.signed_changes(valid_settings(completed(decode(body), saved)), saved))
def save_settings(self, body: bytes) -> dict:
"""Returns:
@@ -216,8 +217,8 @@ class View:
"""
payload = decode(body)
proof = payload.pop("proof", None)
settings = valid_settings(payload)
saved = self.settings.get()
settings = valid_settings(completed(payload, saved))
for change in self.signed_changes(settings, saved):
self.operator_proven(proof, change)
resolved, problems = self.resolved(settings, saved)
+5
View File
@@ -415,6 +415,8 @@ pre.notes { margin: 0; padding: 8px 10px; background: var(--sunken); border-radi
{% include "style-resources.css" %}
{% include "style-rates.css" %}
{% include "style-legend-chips.css" %}
{% include "style-inbox.css" %}
@@ -454,6 +456,7 @@ pre.notes { margin: 0; padding: 8px 10px; background: var(--sunken); border-radi
<span class="spacer"></span>
<details class="menu" id="time"></details>
<span class="updated" id="updated" aria-live="polite"></span>
<div class="rates" id="rates"></div>
</div>
<div class="banner" id="mayday" role="alert" hidden></div>
</div>
@@ -611,6 +614,8 @@ pre.notes { margin: 0; padding: 8px 10px; background: var(--sunken); border-radi
{% include "page-resources.js" %}
{% include "page-rates.js" %}
{% include "page-home.js" %}
{% include "page-rules.js" %}
+1
View File
@@ -66,6 +66,7 @@
const updated = document.getElementById("updated");
updated.classList.toggle("offline", ui.offline);
updated.textContent = `${ui.offline ? "No connection · " : ""}${live() ? "Live" : "As of"} ${absolute.format(stampOf(live() ? state.now : state.until))}`;
pollRates();
}
const faceParts = (key) => Faces.of(key).parts.map(([tag, attrs]) => s(tag, attrs));
+159
View File
@@ -0,0 +1,159 @@
const RATES = {poll: 60, fewest: 2, most: 3};
const RATE_PERIODS = [
["minute", "min", "per minute"],
["hour", "h", "per hour"],
["day", "d", "per day"],
["week", "wk", "per week"],
["month", "mo", "per month of 30 days"],
["year", "yr", "per year of 365 days"],
["total", "all", "in total since records begin"],
];
const RATE_METRICS = [
["tasks", "bi-check2-square", "Tasks done", "count"],
["commits", "bi-git", "Commits", "count"],
["krs", "bi-bullseye", "Key results met", "count"],
["tokens", "bi-coin", "Tokens", "count"],
["cpu", "bi-cpu", "CPU", "percent"],
["ram", "bi-memory", "RAM", "percent"],
["vram", "bi-gpu-card", "VRAM", "vram"],
["swap", "bi-arrow-left-right", "Swap", "percent"],
["disk", "bi-hdd", "Disk", "disk"],
];
const BYTE_UNITS = ["B", "KiB", "MiB", "GiB", "TiB", "PiB"];
const rateView = {data: null, dom: null, busy: false, at: -Infinity};
const periodOf = (id) => RATE_PERIODS.find(([name]) => name === id) || RATE_PERIODS[1];
const metricOf = (id) => RATE_METRICS.find(([name]) => name === id);
function bytesText(value) {
const power = value > 0 ? Math.min(Math.floor(Math.log2(value) / 10), BYTE_UNITS.length - 1) : 0;
return `${power ? fixed(value / 2 ** (10 * power)) : Math.round(value)} ${BYTE_UNITS[power]}`;
}
function countText(value) {
if (value >= 1000) {
return compactCount(value);
}
if (value >= 10 || Number.isInteger(value)) {
return String(Math.round(value));
}
return value < 0.1 ? String(Number(value.toPrecision(1))) : fixed(value);
}
/**
* @param entry - one metric of one period from /api/rates, or null before the first answer.
* @param kind - the metric's kind: count, percent, vram or disk.
* @param short - the period's short name a count is per, "" for none.
* @returns the cell text: ≈ before an extrapolated count, n/a for a gauge without a sample.
*/
function rateText(entry, kind, short) {
if (!entry) {
return "…";
}
if (entry.value === null) {
return "n/a";
}
if (kind === "count") {
return `${entry.approx ? "≈ " : ""}${countText(entry.value)}${short ? `/${short}` : ""}`;
}
return kind === "vram" && entry.used !== null ? `${percentOf(entry.value)} · ${bytesText(entry.used)}` : percentOf(entry.value);
}
function rateHint(entry, metric, period) {
const [id, , label, kind] = metric;
const [, , per] = period;
if (!entry) {
return `${label} ${per}: loading`;
}
if (entry.value === null) {
return `${label} ${per}: n/a, no sample in this window`;
}
const source = entry.source === "total" ? "all records of this kind" : `the last ${entry.source}`;
const scaled = entry.approx ? `; ≈ none in this window or its records begin later, scaled from ${source}` : "";
const where = id === "disk" ? ` on ${entry.mount}, ${bytesText(entry.used)} of ${bytesText(entry.total)}` : "";
const mean = kind === "count" ? "" : " on average";
return `${label} ${per}: ${rateText(entry, kind, "")}${mean}${where}${scaled}`;
}
function ratesBar() {
const host = document.getElementById("rates");
const period = h("select", {class: "field rates-period", ...named("Rate period", "Rate period: the window the chip counts over"), on: {change: (event) => chooseRates({...settings.rates, period: event.target.value})}},
RATE_PERIODS.map(([id, short, per]) => h("option", {value: id, title: per}, short)));
const chip = h("summary", {class: "rates-chip"});
const body = h("tbody", {});
const head = h("tr", {}, h("th", {scope: "col"}, "Metric"), h("th", {scope: "col", title: "Shown in the chip"}, "Chip"), RATE_PERIODS.map(([id, short, per]) => h("th", {scope: "col", "data-period": id, title: per}, short)));
const box = h("div", {class: "menu-box end rates-box", role: "group", "aria-label": "Rates per period"},
h("h3", {}, "Rates per period"),
h("div", {class: "rates-scroll"}, h("table", {class: "rates-table"}, h("thead", {}, head), body)),
h("p", {class: "muted"}, `≈ means an estimate: none in the window, so the shortest longer window that holds any, scaled down, or a window reaching back before the records of its kind begin, scaled from those records. n/a means no sample in the window. Total counts since records begin; the gauges average over the window. The chip shows ${RATES.fewest} to ${RATES.most} metrics.`));
const menu = h("details", {class: "menu rates-menu"}, chip, box);
host.replaceChildren(period, menu);
rateView.dom = {period, chip, body, menu};
}
function rateCell(metric, period, chosen) {
const entry = rateView.data ? rateView.data.rates[period[0]][metric[0]] : null;
const short = period[0] === "total" ? "" : period[1];
return h("td", {"data-period": period[0], "data-approx": String(Boolean(entry && entry.approx)), "aria-current": period[0] === chosen.period ? "true" : null, title: rateHint(entry, metric, period)},
rateText(entry, metric[3], metric[3] === "count" ? short : ""));
}
function rateRow(metric, chosen) {
const [id, name, label] = metric;
const shown = chosen.metrics.includes(id);
const locked = shown ? chosen.metrics.length <= RATES.fewest : chosen.metrics.length >= RATES.most;
const pick = h("input", {type: "checkbox", checked: shown, disabled: locked, "aria-label": `${label} in the chip`, on: {change: () => chooseRates({...chosen, metrics: shown ? chosen.metrics.filter((other) => other !== id) : [...chosen.metrics, id]})}});
return h("tr", {"data-metric": id}, h("th", {scope: "row"}, icon(name), ` ${label}`), h("td", {}, pick), RATE_PERIODS.map((period) => rateCell(metric, period, chosen)));
}
function drawRates() {
const dom = rateView.dom;
if (!dom) {
return;
}
const chosen = settings.rates;
const period = periodOf(chosen.period);
const short = period[0] === "total" ? "" : period[1];
const entries = chosen.metrics.map((id) => [metricOf(id), rateView.data ? rateView.data.rates[period[0]][id] : null]);
dom.period.value = period[0];
dom.chip.replaceChildren(...entries.map(([metric, entry]) => h("span", {class: "rate", "data-metric": metric[0], "data-approx": String(Boolean(entry && entry.approx)), title: rateHint(entry, metric, period)},
icon(metric[1]), h("span", {class: "rate-value"}, rateText(entry, metric[3], metric[3] === "count" ? short : "")))), caret());
dom.chip.setAttribute("aria-label", `Rates ${period[2]}: ${entries.map(([metric, entry]) => `${metric[2]} ${rateText(entry, metric[3], "")}`).join(", ")}`);
dom.chip.title = "Rates: every metric per period";
const focused = document.activeElement instanceof Element ? document.activeElement.closest("#rates tr[data-metric]") : null;
dom.body.replaceChildren(...RATE_METRICS.map((metric) => rateRow(metric, chosen)));
const again = focused ? dom.body.querySelector(`tr[data-metric="${focused.dataset.metric}"] input`) : null;
if (again) {
again.focus();
}
}
async function chooseRates(next) {
if (!(await saveSettings({...settings, rates: next}))) {
drawRates();
}
}
async function fetchRates() {
rateView.busy = true;
try {
const response = await fetch("/api/rates", {cache: "no-store"});
if (response.ok) {
rateView.data = await response.json();
}
} catch {
ui.offline = true;
}
rateView.at = nowSeconds();
rateView.busy = false;
drawRates();
}
function pollRates() {
if (!rateView.dom) {
ratesBar();
drawRates();
}
if (!document.hidden && !rateView.busy && nowSeconds() - rateView.at >= RATES.poll) {
fetchRates();
}
}
+1 -1
View File
@@ -9,7 +9,7 @@
const RESOURCE_BURN = {done: "Done", scope: "Scope"};
const resourceView = {data: null, dom: null, busy: false, key: "", at: 0, timer: 0};
const resourceKey = () => JSON.stringify([ui.preset, ui.from, ui.to, ui.asOf]);
const compactCount = (value) => (value >= 1e6 ? `${fixed(value / 1e6)}M` : value >= 1e3 ? `${fixed(value / 1e3)}k` : String(Math.round(value)));
const compactCount = Resources.compact;
const resourceLabel = (line) => (line.kind === "vram" ? `${line.name} VRAM` : line.name);
const percentOf = (value) => `${value} %`;
+2
View File
@@ -14,6 +14,7 @@
syncRadio();
musicButton();
syncBubbles();
drawRates();
}
async function saveSettings(next) {
@@ -153,6 +154,7 @@
estimation: settings.estimation,
trend: settings.trend,
bubbles: settings.bubbles,
rates: settings.rates,
};
if (await saveSettings(next)) {
closeSheet();
+13 -1
View File
@@ -8,6 +8,8 @@ const Resources = (() => {
const HOUR = 3600;
const COUNTS = ["input", "output", "cache_read", "cache_write"];
const ZONES = [{kind: "idle", from: 0, to: IDLE}, {kind: "saturated", from: SATURATED, to: 100}];
const COUNT_UNITS = [[1e12, "T"], [1e9, "B"], [1e6, "M"], [1e3, "k"]];
const ROUNDS_UP = 0.99995;
const finite = (value) => typeof value === "number" && Number.isFinite(value);
const share = (part, whole) => (finite(part) && finite(whole) && whole > 0 ? Math.round((1000 * part) / whole) / 10 : null);
const listOf = (row, key) => (Array.isArray(row[key]) ? row[key].filter((item) => item && typeof item === "object") : []);
@@ -178,7 +180,17 @@ const Resources = (() => {
return {top, done: points.map((point) => [point.t, percent(point.done)]), scope: points.map((point) => [point.t, percent(point.scope)])};
}
return {IDLE, SATURATED, CALM, GAP, HOUR, COUNTS, ZONES, lines, hardware, segments, spans, waits, asleep, unknown, stack, accumulate, burnup};
/**
* @param value - a count.
* @returns the count with one decimal in the largest of k, M, B and T it reaches once rounded to that decimal,
* else as a whole number.
*/
function compact(value) {
const unit = COUNT_UNITS.find(([size]) => value >= size * ROUNDS_UP);
return unit ? `${(value / unit[0]).toFixed(1)}${unit[1]}` : String(Math.round(value));
}
return {IDLE, SATURATED, CALM, GAP, HOUR, COUNTS, ZONES, lines, hardware, segments, spans, waits, asleep, unknown, stack, accumulate, burnup, compact};
})();
if (typeof module === "object" && module.exports) {
module.exports = Resources;
+20
View File
@@ -0,0 +1,20 @@
.rates { display: inline-flex; align-items: center; gap: 4px; min-width: 0; max-width: 100%; }
.rates-period { padding: 3px 6px; border-radius: 999px; font-size: var(--fs-xs); }
.rates-menu > summary.rates-chip { gap: 10px; padding: 3px 10px; border-radius: 999px; font-size: var(--fs-sm); font-weight: 560; font-variant-numeric: tabular-nums; white-space: nowrap; }
.rates-chip .rate { display: inline-flex; align-items: center; gap: 4px; }
.rates-chip .bi { color: var(--muted); }
.rates-chip .rate[data-approx="true"] .rate-value, .rates-table td[data-approx="true"] { color: var(--muted); }
.rates-box { max-width: min(1100px, calc(100vw - 32px)); }
.rates-scroll { max-width: 100%; overflow-x: auto; }
.rates-box > p { width: 0; min-width: 100%; margin: 0; }
.rates-table { border-collapse: collapse; font-size: var(--fs-sm); font-variant-numeric: tabular-nums; }
.rates-table th, .rates-table td { padding: 3px 8px; border-bottom: 1px solid var(--line); text-align: right; white-space: nowrap; }
.rates-table thead th { color: var(--muted); font-size: var(--fs-xs); font-weight: 650; }
.rates-table th[scope="row"], .rates-table thead th:first-child { position: sticky; left: 0; z-index: 1; background: var(--panel); font-weight: 560; text-align: left; }
.rates-table td[aria-current="true"] { background: var(--accent-soft); color: var(--accent); }
@media (min-width: 641px) {
.rates-box { width: max-content; }
}
@media (max-width: 640px) {
.rates-menu > summary.rates-chip { gap: 6px; padding: 3px 8px; }
}