Files
skills/orchestrator/kvb_orchestrator/rates.py
T

273 lines
11 KiB
Python
Raw Normal View History

from __future__ import annotations
from collections.abc import Iterable
from pathlib import Path
from .records import epoch, iso_epoch
from .register import OPERATOR, REGISTER, read_register, status_of
from .resources import HOUR, RESOURCES, SAMPLES, TOKENS, rows_between
from .scores import journal
DAY = 86400
PERIODS = (("minute", 60), ("hour", HOUR), ("day", DAY), ("week", 7 * DAY), ("month", 30 * DAY), ("year", 365 * DAY), ("total", None))
PERIOD_NAMES = tuple(name for name, _ in PERIODS)
COUNTS = ("tasks", "commits", "krs", "tokens")
GAUGES = ("cpu", "ram", "vram", "swap", "disk")
METRICS = (*COUNTS, *GAUGES)
HOST = ("cpu", "ram", "swap")
TOKEN_FIELDS = ("input", "output", "cache_read", "cache_write")
FINEST = {"tasks": 0, "commits": 0, "krs": 0, "tokens": HOUR}
INTERVAL = 60
DONE = "done"
DIGITS = 4
NUMBERS = (int, float)
LIMIT = 1e18
Piece = tuple[float | None, float | None, float]
Placed = tuple[float, float, dict]
def rounded(value: float | None, digits: int) -> float | int | None:
if value is None:
return None
return round(value) if digits == 0 else round(value, digits)
def points(times: Iterable[float | None]) -> list[Piece]:
return [(t, t, 1.0) for t in times]
def task_times(tasks: dict[str, dict]) -> list[float | None]:
"""Returns:
The end of every done register task but the operator's commands,
None for one without a readable end.
"""
return [iso_epoch(row.get("end")) for task, row in tasks.items() if not task.startswith(OPERATOR) and status_of(row) == DONE]
def kr_times(tasks: dict[str, dict]) -> list[float | None]:
"""Returns:
The met_at of every met key result, None for one met before
rasci.py recorded that time.
"""
results = [kr for row in tasks.values() if isinstance(row.get("key_results"), list) for kr in row["key_results"]]
return [iso_epoch(kr.get("met_at")) for kr in results if isinstance(kr, dict) and kr.get("met") is True]
def commit_times(rows: list[dict]) -> list[float | None]:
"""Returns:
The time of every commit the score journal holds a row with a sha
for, once per remote and sha.
"""
found: dict[tuple[str | None, str], float | None] = {}
for row in rows:
sha, remote = row.get("sha"), row.get("remote")
if isinstance(sha, str) and sha:
found.setdefault((remote if isinstance(remote, str) else None, sha), epoch(row.get("time")))
return list(found.values())
def token_pieces(rows: list[dict], now: float) -> list[Piece]:
"""Returns:
Per token row its hour, or its day for a row with width, cut at now,
and its tokens of every kind.
"""
found = []
for row in rows:
t = epoch(row.get("t"))
width = row["width"] if sane(row.get("width")) and row["width"] > 0 else HOUR
amount = sum(value for name in TOKEN_FIELDS if isinstance(value := row.get(name), int) and not isinstance(value, bool) and value > 0)
if t is not None and amount:
found.append((t, max(t, min(t + width, now)), float(amount)))
return found
def share(piece: Piece, begin: float, now: float) -> float:
"""Returns:
The part of the piece's amount inside [begin, now]: an instant
counts whole or not at all, a span by the share of it inside.
"""
start, end, amount = piece
if start is None or end is None:
return 0.0
if end <= start:
return amount if begin <= start <= now else 0.0
inside = min(end, now) - max(start, begin)
return amount * inside / (end - start) if inside > 0 else 0.0
def counted(pieces: list[Piece], window: tuple[float, float], first: float | None, finest: float) -> dict[str, dict]:
"""Args:
pieces: (start, end, amount) per event of one kind; start None for
one without a time, end equal to start for an instant.
window: (begin, end) the rates cover, in epoch seconds.
first: the first time the records of the kind hold, None without any.
finest: the shortest span the kind's pieces resolve.
Returns:
Per period {value, approx, covered}: covered the seconds of the
window from first on; total the amount inside the window, every
other period that amount per its seconds of covered; value None
where no record covers the window; approx where the window is
shorter than finest, and for every period but total where it starts
before first.
"""
begin, end = window
covered = end - max(begin, first) if first is not None else 0.0
if covered <= 0:
return {name: {"value": None, "approx": False, "covered": None} for name in PERIOD_NAMES}
amount = sum(share(piece, begin, end) for piece in pieces)
short, early = end - begin < finest, begin < first
return {
name: {"value": precise(amount if seconds is None else amount * seconds / covered), "approx": short or (early and seconds is not None), "covered": covered}
for name, seconds in PERIODS
}
def precise(value: float) -> float:
return round(value, DIGITS - 1) if value >= 1 else float(f"{value:.{DIGITS}g}")
def sane(value: object) -> bool:
return type(value) in NUMBERS and -LIMIT < value < LIMIT
def placed(row: dict) -> Placed | None:
"""Returns:
(t, width, row) of a sample row: width the seconds of the bucket a
folded row stands for (its span when it names no width, 0 for a raw
row, an instant); None without a time.
"""
t = epoch(row.get("t"))
if t is None:
return None
width, span = row.get("width"), row.get("span")
return t, float(width if sane(width) and width > 0 else span if sane(span) and span > 0 else 0), row
def blank() -> dict:
return {**{key: [0.0, 0.0] for key in HOST}, "vram": [0.0, 0.0, 0.0], "disks": {}}
def add_vram(cell: list[float], gpus: object, weight: float) -> None:
used = total = 0.0
for gpu in gpus if type(gpus) is list else ():
if type(gpu) is dict and gpu.get("asleep") is not True:
part, whole = gpu.get("vram_used"), gpu.get("vram_total")
if sane(part) and sane(whole) and whole > 0:
used, total = used + part, total + whole
if total:
cell[0] += weight
cell[1] += weight * 100 * used / total
cell[2] += weight * used
def add_disks(cells: dict[str, list[float]], disks: object, weight: float) -> None:
for disk in disks if type(disks) is list else ():
if type(disk) is dict:
mount, used, total = disk.get("mount"), disk.get("used"), disk.get("total")
if type(mount) is str and sane(used) and sane(total) and total > 0:
cell = cells.get(mount) or cells.setdefault(mount, [0.0, 0.0, 0.0, 0.0])
cell[0] += weight
cell[1] += weight * 100 * used / total
cell[2] += weight * used
cell[3] += weight * total
def add(tally: dict, row: dict, part: float) -> None:
"""Args:
tally: blank() summed so far: per host key [weight, weighted value],
vram [weight, weighted percent, weighted bytes] of every GPU
awake with a reading together, per mount [weight, weighted
percent, weighted used, weighted total].
row: one sample row, weighted by the seconds it sampled: its span,
INTERVAL for a raw row.
part: the share of the row inside the window.
"""
span = row.get("span")
weight = (span if sane(span) and span > 0 else INTERVAL) * part
for key in HOST:
value = row.get(key)
if sane(value):
cell = tally[key]
cell[0] += weight
cell[1] += weight * value
add_vram(tally["vram"], row.get("gpus"), weight)
add_disks(tally["disks"], row.get("disks"), weight)
def tally(items: list[Placed], window: tuple[float, float]) -> dict:
"""Returns:
The add() of every placed() row inside the window, a row whose
bucket straddles either end by its share inside.
"""
found = blank()
for t, width, row in items:
part = share((t, t + width, 1.0), *window)
if part:
add(found, row, part)
return found
def mean(cell: list[float], at: int) -> float | None:
return cell[at] / cell[0] if cell[0] else None
def fullest(mounts: dict[str, list[float]]) -> dict:
"""Returns:
{value, mount, used, total} of the mount fullest on average, all
None without a disk.
"""
top = max(mounts, key=lambda mount: (mean(mounts[mount], 1), mount), default=None)
if top is None:
return {"value": None, "mount": None, "used": None, "total": None}
cell = mounts[top]
return {"value": rounded(mean(cell, 1), 1), "mount": top, "used": rounded(mean(cell, 2), 0), "total": rounded(mean(cell, 3), 0)}
def gauges(tally: dict) -> dict[str, dict]:
"""Returns:
Per gauge its mean over the rows the tally holds, each row weighted
by its seconds inside the window: value the percent, None without a
reading; vram adds used, the bytes, and disk names the mount fullest
on average.
"""
found = {key: {"value": rounded(mean(tally[key], 1), 1)} for key in HOST}
found["vram"] = {"value": rounded(mean(tally["vram"], 1), 1), "used": rounded(mean(tally["vram"], 2), 0)}
found["disk"] = fullest(tally["disks"])
return {key: {**entry, "approx": False} for key, entry in found.items()}
def rated(council: Path, kept: Path, now: float, window: tuple[float, float]) -> dict:
"""Args:
council: the council directory holding the register and the score
journal, read through scores.journal().
kept: the directory holding the host sampler's resources folder.
now: the present in epoch seconds; the current token hour ends here.
window: (begin, end) the rates cover, end at most now.
Returns:
{"now", "since", "from", "to", "periods", "rates"}: since the first
time any record holds; periods [{name, seconds}], total without
seconds; rates per period per metric: counted() of the done tasks,
the commits, the key results met and the tokens, each kind covered
from since on, tokens from their first row, and the gauges() of the
samples inside the window, the same in every period.
"""
tasks = read_register(council / REGISTER)
folder = kept / RESOURCES
items = list(filter(None, map(placed, rows_between(folder / SAMPLES, 0, now, 0))))
pieces = {
"tasks": points(task_times(tasks)),
"commits": points(commit_times(journal(council))),
"krs": points(kr_times(tasks)),
"tokens": token_pieces(rows_between(folder / TOKENS, 0, now, HOUR), now),
}
starts = {kind: [piece[0] for piece in found if piece[0] is not None] for kind, found in pieces.items()}
since = min([*(t for found in starts.values() for t in found), *(item[0] for item in items)], default=None)
firsts = {**dict.fromkeys(COUNTS, since), "tokens": min(starts["tokens"], default=None)}
counts = {kind: counted(found, window, firsts[kind], FINEST[kind]) for kind, found in pieces.items()}
measured = gauges(tally(items, window))
shown = {name: {**{kind: counts[kind][name] for kind in COUNTS}, **measured} for name in PERIOD_NAMES}
return {"now": now, "since": since, "from": window[0], "to": window[1], "periods": [{"name": name, "seconds": seconds} for name, seconds in PERIODS], "rates": shown}