perf(hub): cache the names directory, share state builds, retry stations
Concurrent /api/state polls share one build through LiveState for a short TTL, and every POST and PATCH invalidates it, so a write is never followed by a stale view. The names directory is cached, the first transcript scan reads outside the hub lock, the graph timeline starts at the window begin, and unresolved radio stations are retried on a daemon thread every RETRY_SECONDS instead of failing until restart. 🎯 Fix: 85 % 💥 Break: 8 % 💸 Tokens: – ⏱️ Velocity: – 🛡️ Security: 100 % 🏆 Score: 78 🚦 Gate: ⛔ fix 85 % < 90, break 8 % > 5 Created-with: https://s.veen.world/skills Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
6c63c364bd
commit
96d4d2933c
18 files changed
+299
-32
No files matched your search
@@ -44,7 +44,8 @@ def edges(radio: list[dict], grant_rows: list[dict], chair: str) -> list[dict]:
|
||||
return [{"source": s, "target": t, "class": c, "count": n} for (s, t, c), n in sorted(counts.items())]
|
||||
|
||||
|
||||
def graph(radio: list[dict], grant_rows: list[dict], tasks: dict[str, dict], claims: list[dict], end: float, directory: Directory) -> dict:
|
||||
def graph(radio: list[dict], grant_rows: list[dict], tasks: dict[str, dict], claims: list[dict], span: tuple[float, float], directory: Directory) -> dict:
|
||||
begin, end = span
|
||||
current = {task: row for task, row in sorted(tasks.items()) if status_of(row) not in ENDED}
|
||||
chair = chair_of(claims, directory, end)
|
||||
top = chair or HUMAN
|
||||
@@ -82,7 +83,7 @@ def graph(radio: list[dict], grant_rows: list[dict], tasks: dict[str, dict], cla
|
||||
"parent": parent,
|
||||
"label": label,
|
||||
"names": directory.names(key) or [label],
|
||||
"timeline": directory.timeline(key) or [{"name": label, "since": end}],
|
||||
"timeline": directory.timeline(key, begin) or [{"name": label, "since": end}],
|
||||
"tasks": list(holders.get(key, {}).values()),
|
||||
"claims": [claim for claim in claims if claim["session"] == key],
|
||||
}
|
||||
|
||||
@@ -312,7 +312,10 @@ class Handler(Exchange):
|
||||
return
|
||||
receive = self.receiver(path, host)
|
||||
if receive is not None:
|
||||
receive(self.rfile.read(length))
|
||||
try:
|
||||
receive(self.rfile.read(length))
|
||||
finally:
|
||||
self.server.hub.live_state.invalidate()
|
||||
|
||||
def do_PATCH(self) -> None:
|
||||
path = urlsplit(self.path).path
|
||||
@@ -328,6 +331,7 @@ class Handler(Exchange):
|
||||
if length is not None:
|
||||
body = self.rfile.read(length)
|
||||
self.answer(lambda: self.server.hub.view_store.patch(body), HTTPStatus.OK)
|
||||
self.server.hub.live_state.invalidate()
|
||||
|
||||
def index(self) -> None:
|
||||
nonce = secrets.token_urlsafe(18)
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import functools
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
from dataclasses import dataclass
|
||||
@@ -27,6 +28,7 @@ from .inbox import Inbox
|
||||
from .jobs import Jobs
|
||||
from .language import Language, Model
|
||||
from .ledger import Ledger, Marks, Reviews, Sealed
|
||||
from .livestate import LiveState
|
||||
from .matching import Matching
|
||||
from .mcp import Mcp
|
||||
from .messaging import Messaging
|
||||
@@ -43,7 +45,7 @@ from .receipts import (
|
||||
sight,
|
||||
sighting_pass,
|
||||
)
|
||||
from .redirects import Redirects
|
||||
from .redirects import RETRY_SECONDS, Redirects
|
||||
from .requesting import Requesting
|
||||
from .resources import RESOURCES, SAMPLES
|
||||
from .revealing import Revealing
|
||||
@@ -134,6 +136,7 @@ class Hub(Mcp, Reviewing, Revealing, Asking, Messaging, Choosing, Approving, Den
|
||||
queue_order: QueueOrder
|
||||
view_store: ViewStore
|
||||
match_snapshots: Ledger
|
||||
live_state: LiveState
|
||||
|
||||
def hosts(self) -> tuple[str, str]:
|
||||
return f"127.0.0.1:{self.public_port}", f"localhost:{self.public_port}"
|
||||
@@ -158,7 +161,29 @@ def sighting(hub: Hub, stop: threading.Event) -> threading.Thread:
|
||||
return thread
|
||||
|
||||
|
||||
def open_hub(council: Path, pages: Path, public_port: int, state: Path, host_state: Path, transcripts: Path, scripts: Runner, templates: Path, places: Places, model: Model, engines: Engines, approvers: Path, redirects: Redirects) -> Hub:
|
||||
def stations(hub: Hub, stop: threading.Event) -> threading.Thread:
|
||||
"""Args:
|
||||
hub: the hub whose unresolved stations are retried.
|
||||
stop: ends the thread once set.
|
||||
|
||||
Returns:
|
||||
The started daemon thread that runs retry_stations() at the start and
|
||||
every RETRY_SECONDS, printing why a station still does not resolve.
|
||||
"""
|
||||
|
||||
def run() -> None:
|
||||
while True:
|
||||
for problem in hub.retry_stations():
|
||||
print(f"station redirect: {problem}", file=sys.stderr, flush=True)
|
||||
if stop.wait(RETRY_SECONDS):
|
||||
return
|
||||
|
||||
thread = threading.Thread(target=run, name="station redirects", daemon=True)
|
||||
thread.start()
|
||||
return thread
|
||||
|
||||
|
||||
def open_hub(council: Path, pages: Path, public_port: int, state: Path, host_state: Path, transcripts: Path, scripts: Runner, templates: Path, places: Places, model: Model, engines: Engines, approvers: Path, redirects: Redirects, live_ttl: float) -> Hub:
|
||||
uploads = state / "uploads"
|
||||
uploads.mkdir(exist_ok=True)
|
||||
reviews = Reviews(state)
|
||||
@@ -181,6 +206,7 @@ def open_hub(council: Path, pages: Path, public_port: int, state: Path, host_sta
|
||||
replay=state / "replay" / "messages.json",
|
||||
view_store=ViewStore(state / "view.json"),
|
||||
match_snapshots=Ledger(state / "match.json"),
|
||||
live_state=LiveState(live_ttl),
|
||||
rules=Ledger(state / "rules.json"),
|
||||
offers=Offers(),
|
||||
jobs=Ledger(state / "jobs.json"),
|
||||
|
||||
@@ -0,0 +1,43 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import threading
|
||||
import time
|
||||
from collections.abc import Callable
|
||||
|
||||
LIVE_TTL = 2.0
|
||||
|
||||
|
||||
class LiveState:
|
||||
def __init__(self, ttl: float) -> None:
|
||||
"""Args:
|
||||
ttl: seconds a finished build serves later callers of the same
|
||||
key; 0 rebuilds every time but still one build at a time.
|
||||
"""
|
||||
self.ttl = ttl
|
||||
self.guard = threading.Lock()
|
||||
self.locks: dict[object, threading.Lock] = {}
|
||||
self.built: dict[object, tuple[int, float, dict]] = {}
|
||||
self.generation = 0
|
||||
|
||||
def invalidate(self) -> None:
|
||||
with self.guard:
|
||||
self.generation += 1
|
||||
|
||||
def shared(self, key: object, build: Callable[[], dict]) -> dict:
|
||||
"""Returns:
|
||||
build()'s result; callers arriving while it runs wait for it and
|
||||
share it, and so do callers within ttl after it, until
|
||||
invalidate() is called.
|
||||
"""
|
||||
with self.guard:
|
||||
lock = self.locks.setdefault(key, threading.Lock())
|
||||
with lock:
|
||||
with self.guard:
|
||||
generation = self.generation
|
||||
hit = self.built.get(key)
|
||||
if hit is not None and hit[0] == generation and time.monotonic() - hit[1] < self.ttl:
|
||||
return hit[2]
|
||||
value = build()
|
||||
with self.guard:
|
||||
self.built[key] = (generation, time.monotonic(), value)
|
||||
return value
|
||||
@@ -1,5 +1,6 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import bisect
|
||||
import itertools
|
||||
import json
|
||||
import math
|
||||
@@ -27,13 +28,14 @@ class Directory:
|
||||
if entry["session"] in holders:
|
||||
holders.remove(entry["session"])
|
||||
holders.append(entry["session"])
|
||||
self.times = {session: [entry["first_seen"] for entry in own] for session, own in self.by_session.items()}
|
||||
|
||||
def at(self, session: str, ts: float) -> str:
|
||||
own = self.by_session.get(session)
|
||||
if not own:
|
||||
return ""
|
||||
valid = [entry for entry in own if entry["first_seen"] <= ts]
|
||||
return (valid[-1] if valid else own[0])["name"]
|
||||
index = bisect.bisect_right(self.times[session], ts)
|
||||
return own[index - 1 if index else 0]["name"]
|
||||
|
||||
def now(self, session: str) -> str:
|
||||
own = self.by_session.get(session)
|
||||
@@ -57,8 +59,18 @@ class Directory:
|
||||
def names(self, key: str) -> list[str]:
|
||||
return list(dict.fromkeys(entry["name"] for entry in self.by_session.get(key, [])))
|
||||
|
||||
def timeline(self, key: str) -> list[dict]:
|
||||
return [{"name": entry["name"], "since": entry["first_seen"]} for entry in self.by_session.get(key, [])]
|
||||
def timeline(self, key: str, begin: float | None = None) -> list[dict]:
|
||||
"""Args:
|
||||
key: the session.
|
||||
begin: when given, the names between the first one and the last
|
||||
one seen at or before begin are left out; the first stays as
|
||||
the session's birth.
|
||||
"""
|
||||
own = self.by_session.get(key, [])
|
||||
if begin is not None and own:
|
||||
start = bisect.bisect_right(self.times[key], begin) - 1
|
||||
own = own[:1] + own[max(start, 1):]
|
||||
return [{"name": entry["name"], "since": entry["first_seen"]} for entry in own]
|
||||
|
||||
def aliases(self, name: str) -> set[str]:
|
||||
return set(self.by_name.get(name, []))
|
||||
@@ -103,6 +115,8 @@ class Names:
|
||||
entries, observed = self.data.get("entries"), self.data.get("observed")
|
||||
self.entries = [entry for entry in entries if valid_entry(entry)] if isinstance(entries, list) else []
|
||||
self.observed: dict[str, str] = {key: name for key, name in observed.items() if isinstance(name, str)} if isinstance(observed, dict) else {}
|
||||
self.version = 0
|
||||
self.cached: tuple[int, Directory] | None = None
|
||||
|
||||
def payload(self) -> dict:
|
||||
return {"entries": self.entries, "observed": self.observed}
|
||||
@@ -136,6 +150,7 @@ class Names:
|
||||
floor = history[-1]["first_seen"] if history else -math.inf
|
||||
first = seen if seen is not None and seen > floor else max(now, floor)
|
||||
self.entries.append({"session": session, "name": name, "first_seen": first, "source": source})
|
||||
self.version += 1
|
||||
if changed:
|
||||
self.save()
|
||||
|
||||
@@ -143,6 +158,16 @@ class Names:
|
||||
with self.lock:
|
||||
return [dict(entry) for entry in self.entries]
|
||||
|
||||
def directory(self) -> Directory:
|
||||
"""Returns:
|
||||
The Directory of the current entries, rebuilt only after observe()
|
||||
added one; it shares the entry dicts, which nothing mutates.
|
||||
"""
|
||||
with self.lock:
|
||||
if self.cached is None or self.cached[0] != self.version:
|
||||
self.cached = (self.version, Directory(list(self.entries)))
|
||||
return self.cached[1]
|
||||
|
||||
def due(self, now: float) -> bool:
|
||||
with self.lock:
|
||||
if now - self.titled < TITLE_SCAN:
|
||||
|
||||
@@ -20,7 +20,7 @@ def office_request(raw: object) -> bool:
|
||||
|
||||
class Offices:
|
||||
def office_sessions(self, claims: list[dict]) -> set[str]:
|
||||
named = {entry["session"] for entry in self.names.listed()} | {claim["session"] for claim in claims}
|
||||
named = set(self.names.directory().by_session) | {claim["session"] for claim in claims}
|
||||
return {session for session in named if signing.SESSION.fullmatch(session) and session != HUMAN}
|
||||
|
||||
def appointable(self, raw: object) -> list[dict]:
|
||||
|
||||
@@ -7,7 +7,6 @@ import time
|
||||
from .council import read_claims
|
||||
from .errors import Conflict, Missing, ToolError
|
||||
from .fields import COMMAND_LIMIT, PATH_LIMIT, bounded, decode, expect, visible
|
||||
from .names import Directory
|
||||
from .signed import ask_signed, passkey, review_signed, signing, verifier
|
||||
|
||||
RULE_KINDS = ("ask", "review", "decision")
|
||||
@@ -155,7 +154,7 @@ class Rules:
|
||||
"""
|
||||
if kind_name != "decision":
|
||||
return set()
|
||||
directory = Directory(self.names.listed())
|
||||
directory = self.names.directory()
|
||||
holders = {session for session in directory.aliases(record["agent"]) if directory.now(session) == record["agent"]}
|
||||
return holders if len(holders) == 1 else set()
|
||||
|
||||
|
||||
@@ -34,7 +34,7 @@ class Sources:
|
||||
sightings = title_sightings(self.transcripts.root, claimed, now) if self.names.due(now) else []
|
||||
sightings += [(claim["session"], claim["agent"], iso_epoch(claim["since"]), "claim") for claim in claims if claim["agent"] and claim["session"] != "-"]
|
||||
self.names.observe(sorted(sightings, key=lambda sighting: (sighting[2] is None, sighting[2] or 0.0)), now)
|
||||
known = {entry["name"] for entry in self.names.listed()}
|
||||
known = set(self.names.directory().by_name)
|
||||
radio = []
|
||||
for row in news:
|
||||
name, ts = agent(row.get("from")), epoch(row.get("ts"))
|
||||
@@ -42,7 +42,7 @@ class Sources:
|
||||
radio.append((f"{PSEUDO}{name}", name, ts, "radio"))
|
||||
known.add(name)
|
||||
self.names.observe(radio, now)
|
||||
return Directory(self.names.listed())
|
||||
return self.names.directory()
|
||||
|
||||
def repositories(self) -> dict:
|
||||
tasks = read_register(self.council / REGISTER)
|
||||
|
||||
@@ -82,6 +82,7 @@ class Transcripts:
|
||||
def __init__(self, root: Path) -> None:
|
||||
self.root = root
|
||||
self.lock = threading.Lock()
|
||||
self.scanning = threading.Lock()
|
||||
self.offsets: dict[Path, int] = {}
|
||||
self.entries: deque[dict] = deque(maxlen=STREAM_KEEP)
|
||||
self.crons: dict[str, dict] = {}
|
||||
@@ -89,7 +90,8 @@ class Transcripts:
|
||||
self.scanned = -math.inf
|
||||
|
||||
def refresh(self) -> None:
|
||||
with self.lock:
|
||||
# Trip-wire: a first scan reads hundreds of MiB; only the per-file add takes self.lock, which /api/state also needs.
|
||||
with self.scanning:
|
||||
if time.monotonic() - self.scanned < STREAM_SCAN:
|
||||
return
|
||||
self.scanned = time.monotonic()
|
||||
@@ -98,7 +100,10 @@ class Transcripts:
|
||||
except OSError:
|
||||
paths = []
|
||||
for path in paths:
|
||||
self.read(path)
|
||||
rows = self.read(path)
|
||||
with self.lock:
|
||||
for row in rows:
|
||||
self.add(path, row)
|
||||
|
||||
def start(self, path: Path, info: os.stat_result) -> int:
|
||||
offset = self.offsets.get(path)
|
||||
@@ -107,7 +112,7 @@ class Transcripts:
|
||||
return max(info.st_size - STREAM_FIRST, 0) if recent else info.st_size
|
||||
return 0 if info.st_size < offset else offset
|
||||
|
||||
def read(self, path: Path) -> None:
|
||||
def read(self, path: Path) -> list[dict]:
|
||||
try:
|
||||
info = path.stat()
|
||||
begin = self.start(path, info)
|
||||
@@ -115,13 +120,11 @@ class Transcripts:
|
||||
handle.seek(begin)
|
||||
data = handle.read(max(info.st_size - begin, 0))
|
||||
except OSError:
|
||||
return
|
||||
return []
|
||||
end = data.rfind(b"\n") + 1
|
||||
self.offsets[path] = begin + end
|
||||
for raw in data[:end].splitlines():
|
||||
row = parse_line(raw.decode("utf-8", errors="replace"))
|
||||
if row is not None:
|
||||
self.add(path, row)
|
||||
rows = (parse_line(raw.decode("utf-8", errors="replace")) for raw in data[:end].splitlines())
|
||||
return [row for row in rows if row is not None]
|
||||
|
||||
def add(self, path: Path, row: dict) -> None:
|
||||
ts = iso_epoch(row.get("timestamp"))
|
||||
|
||||
@@ -75,12 +75,17 @@ class View:
|
||||
return begin, end
|
||||
|
||||
def window(self, query: dict[str, list[str]]) -> dict:
|
||||
now = time.time()
|
||||
begin, end = self.span(query, now)
|
||||
sampled = single(query, "track")
|
||||
if sampled not in (None, "1"):
|
||||
raise ToolError("track must be 1 or left out")
|
||||
self.tree.beat()
|
||||
if "from" in query or "to" in query:
|
||||
return self.built(query, sampled)
|
||||
return self.live_state.shared(sampled, lambda: self.built(query, sampled))
|
||||
|
||||
def built(self, query: dict[str, list[str]], sampled: str | None) -> dict:
|
||||
now = time.time()
|
||||
begin, end = self.span(query, now)
|
||||
self.sweep_sealed()
|
||||
return self.state(begin, end, now, TRACK_STEPS if sampled else 0)
|
||||
|
||||
@@ -121,7 +126,7 @@ class View:
|
||||
grant_rows = [{**grant, "target": directory.key(grant["to"], grant["ts"])} for grant in grants(history, (begin, end), claims, tasks)]
|
||||
self.watch.see(tasks, now)
|
||||
jobs = [{**{key: value for key, value in job.items() if key != "proof"}, "output": job["output"][-OUTPUT_LIMIT:]} for job in self.jobs.listed()[-JOBS_SHOWN:]]
|
||||
drawn = graph(radio, grant_rows, tasks, claims, end, directory)
|
||||
drawn = graph(radio, grant_rows, tasks, claims, (begin, end), directory)
|
||||
first = self.first(news, history, claims, now)
|
||||
drawn["track"] = track(drawn["nodes"], radio, tasks, claims, (min(max(begin, first), end), end), directory, steps)
|
||||
people = {node["id"]: {"name": node["label"], "now": directory.now(node["id"]) or node["label"], "names": node["names"]} for node in drawn["nodes"] if node["kind"] != "subagent"}
|
||||
@@ -215,6 +220,23 @@ class View:
|
||||
streams.append({**stream, "via": via})
|
||||
return {**settings, "streams": streams}, problems
|
||||
|
||||
def retry_stations(self) -> list[str]:
|
||||
"""Returns:
|
||||
Why a station's redirect chain still does not resolve. A station
|
||||
whose via is None is resolved again once Redirects.due allows,
|
||||
without waiting for a save; a save that lands meanwhile wins.
|
||||
"""
|
||||
saved = self.settings.get()
|
||||
if not any(stream["via"] is None and self.redirects.due(stream["url"]) for stream in saved["streams"]):
|
||||
return []
|
||||
resolved, problems = self.resolved(saved, saved)
|
||||
if resolved != saved:
|
||||
try:
|
||||
self.settings.put(resolved, saved)
|
||||
except ToolError as error:
|
||||
problems.append(str(error))
|
||||
return problems
|
||||
|
||||
def sign_settings(self, body: bytes) -> dict:
|
||||
saved = self.settings.get()
|
||||
return self.challenge_for(self.signed_changes(valid_settings(completed(decode(body), saved)), saved))
|
||||
|
||||
@@ -77,9 +77,10 @@ from kvb_orchestrator.engines import Engines
|
||||
from kvb_orchestrator.errors import ToolError
|
||||
from kvb_orchestrator.exchange import HubServer
|
||||
from kvb_orchestrator.handler import Handler
|
||||
from kvb_orchestrator.hub import Places, open_hub, sighting
|
||||
from kvb_orchestrator.hub import Places, open_hub, sighting, stations
|
||||
from kvb_orchestrator.keys import key_from
|
||||
from kvb_orchestrator.language import Model
|
||||
from kvb_orchestrator.livestate import LIVE_TTL
|
||||
from kvb_orchestrator.redirects import TIMEOUT, Redirects, dial
|
||||
from kvb_orchestrator.skill_scripts import Scripts
|
||||
from kvb_orchestrator.speechservice import Service
|
||||
@@ -119,12 +120,14 @@ def main() -> int:
|
||||
),
|
||||
approvers=args.approvers,
|
||||
redirects=Redirects(tls=ssl.create_default_context(), dial=dial, timeout=TIMEOUT),
|
||||
live_ttl=LIVE_TTL,
|
||||
)
|
||||
except ToolError as error:
|
||||
parser.error(str(error))
|
||||
server = HubServer((args.bind, args.listen_port), Handler)
|
||||
server.hub = hub
|
||||
sighting(hub, threading.Event())
|
||||
stations(hub, threading.Event())
|
||||
print(f"{NAME}: listening on {args.bind}:{args.listen_port}, public port {args.public_port}", flush=True)
|
||||
server.serve_forever()
|
||||
return 0
|
||||
|
||||
@@ -51,9 +51,7 @@
|
||||
}
|
||||
|
||||
function poll() {
|
||||
if (!loadsInFlight) {
|
||||
refresh();
|
||||
}
|
||||
refresh();
|
||||
}
|
||||
|
||||
const SELECTION_HOLD = 20000;
|
||||
@@ -62,7 +60,7 @@
|
||||
function refresh() {
|
||||
const selected = String(window.getSelection()).length > 0;
|
||||
selectedSince = selected ? selectedSince || Date.now() : 0;
|
||||
if (!ui.busy && !ui.playing && !document.hidden && (!selected || Date.now() - selectedSince >= SELECTION_HOLD)) {
|
||||
if (!loadsInFlight && !ui.busy && !ui.playing && !document.hidden && (!selected || Date.now() - selectedSince >= SELECTION_HOLD)) {
|
||||
load();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -53,7 +53,7 @@
|
||||
|
||||
function redirectNote(stream) {
|
||||
if (stream && stream.via === null) {
|
||||
return "The server has not resolved this station's redirect yet; saving after ten minutes tries again.";
|
||||
return "The hub has not resolved this station's redirect yet; it tries again every ten minutes by itself, the hub log names why it failed.";
|
||||
}
|
||||
const released = stream ? stream.via : [];
|
||||
const unloaded = released.filter((origin) => !pageMedia.has(origin));
|
||||
|
||||
@@ -77,6 +77,7 @@ class HubCase(unittest.TestCase):
|
||||
engines=self.engines,
|
||||
approvers=self.approvers,
|
||||
redirects=self.redirects(),
|
||||
live_ttl=self.live_ttl(),
|
||||
)
|
||||
self.httpd = httpd
|
||||
self.origin = signing.origin_of(self.port)
|
||||
@@ -90,6 +91,13 @@ class HubCase(unittest.TestCase):
|
||||
def redirects(self) -> Redirects:
|
||||
return offline_redirects()
|
||||
|
||||
def live_ttl(self) -> float:
|
||||
"""Returns:
|
||||
0, so a test that writes council files directly reads them in the
|
||||
next state; a case testing the live cache overrides it.
|
||||
"""
|
||||
return 0.0
|
||||
|
||||
def call(self, method: str, path: str, body: bytes | None, headers: dict[str, str]) -> tuple[int, dict, bytes]:
|
||||
connection = http.client.HTTPConnection("127.0.0.1", self.port, timeout=30)
|
||||
try:
|
||||
|
||||
@@ -0,0 +1,85 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import threading
|
||||
import time
|
||||
import unittest
|
||||
|
||||
import hub_support # noqa: F401
|
||||
from hub_case import HubCase
|
||||
from kvb_orchestrator.livestate import LiveState
|
||||
|
||||
|
||||
def counting(pause: float) -> tuple[list[int], object]:
|
||||
calls: list[int] = []
|
||||
|
||||
def build() -> dict:
|
||||
calls.append(1)
|
||||
time.sleep(pause)
|
||||
return {"build": len(calls)}
|
||||
|
||||
return calls, build
|
||||
|
||||
|
||||
class TestLiveState(unittest.TestCase):
|
||||
def test_callers_arriving_during_a_build_wait_for_it_and_share_it(self):
|
||||
live = LiveState(60.0)
|
||||
calls, build = counting(0.3)
|
||||
results: list[dict] = []
|
||||
threads = [threading.Thread(target=lambda: results.append(live.shared("live", build))) for _ in range(4)]
|
||||
for thread in threads:
|
||||
thread.start()
|
||||
for thread in threads:
|
||||
thread.join(5)
|
||||
self.assertEqual((len(calls), len(results)), (1, 4))
|
||||
self.assertTrue(all(result is results[0] for result in results))
|
||||
|
||||
def test_a_zero_ttl_rebuilds_and_other_keys_build_on_their_own(self):
|
||||
live = LiveState(0.0)
|
||||
calls, build = counting(0.0)
|
||||
live.shared("live", build)
|
||||
live.shared("live", build)
|
||||
live.shared("track", build)
|
||||
self.assertEqual(len(calls), 3)
|
||||
|
||||
def test_invalidate_ends_the_shared_build_at_once(self):
|
||||
live = LiveState(60.0)
|
||||
_, build = counting(0.0)
|
||||
first = live.shared("live", build)
|
||||
self.assertIs(live.shared("live", build), first)
|
||||
live.invalidate()
|
||||
self.assertEqual(live.shared("live", build), {"build": 2})
|
||||
|
||||
def test_a_build_overtaken_by_invalidate_is_not_served_again(self):
|
||||
live = LiveState(60.0)
|
||||
started = threading.Event()
|
||||
|
||||
def slow() -> dict:
|
||||
started.set()
|
||||
time.sleep(0.2)
|
||||
return {"stale": True}
|
||||
|
||||
thread = threading.Thread(target=lambda: live.shared("live", slow))
|
||||
thread.start()
|
||||
started.wait(5)
|
||||
live.invalidate()
|
||||
thread.join(5)
|
||||
self.assertEqual(live.shared("live", lambda: {"stale": False}), {"stale": False})
|
||||
|
||||
|
||||
class TestLiveStateRoute(HubCase):
|
||||
def live_ttl(self) -> float:
|
||||
return 60.0
|
||||
|
||||
def test_polls_share_one_state_until_a_post_or_a_window_query(self):
|
||||
first = self.snapshot()
|
||||
self.assertEqual(self.snapshot()["now"], first["now"])
|
||||
status, _, body = self.get(f"/api/state?to={first['now']}")
|
||||
self.assertEqual(status, 200, body)
|
||||
self.assertGreater(json.loads(body)["now"], first["now"])
|
||||
self.post_json("/api/inbox/mark", {}, self.guard(self.host))
|
||||
self.assertGreater(self.snapshot()["now"], first["now"])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -27,5 +27,37 @@ class TestRenameOnce(unittest.TestCase):
|
||||
self.assertEqual([row["title"] for row in names.Directory(entries).renames(0, 10)], ["A is now called B", "B is now called A"])
|
||||
|
||||
|
||||
def flips(count: int) -> list[dict]:
|
||||
return [{"session": "s1", "name": "AB"[index % 2], "first_seen": float(index), "source": "claim"} for index in range(count)]
|
||||
|
||||
|
||||
class TestDirectoryLookups(unittest.TestCase):
|
||||
def test_at_names_the_last_entry_seen_at_or_before_the_moment_and_the_first_one_before_any(self):
|
||||
directory = names.Directory(flips(6))
|
||||
scanned = flips(6)
|
||||
for ts in (-1.0, 0.0, 0.5, 1.0, 2.9, 3.0, 5.0, 99.0):
|
||||
valid = [entry for entry in scanned if entry["first_seen"] <= ts]
|
||||
self.assertEqual(directory.at("s1", ts), (valid[-1] if valid else scanned[0])["name"], ts)
|
||||
self.assertEqual(directory.at("s9", 1.0), "")
|
||||
|
||||
def test_a_windowed_timeline_keeps_the_birth_and_every_name_from_the_last_one_before_the_window(self):
|
||||
directory = names.Directory(flips(10))
|
||||
self.assertEqual([entry["since"] for entry in directory.timeline("s1", 6.5)], [0.0, 6.0, 7.0, 8.0, 9.0])
|
||||
self.assertEqual(directory.timeline("s1", -5.0), directory.timeline("s1"))
|
||||
self.assertEqual(directory.timeline("s1", 0.0), directory.timeline("s1"))
|
||||
self.assertEqual(directory.timeline("s9", 3.0), [])
|
||||
|
||||
def test_the_directory_is_rebuilt_only_after_a_new_entry(self):
|
||||
with tempfile.TemporaryDirectory() as folder:
|
||||
book = names.Names(Path(folder) / "names.json")
|
||||
book.observe([("s1", "A", 1.0, "claim")], 10.0)
|
||||
first = book.directory()
|
||||
book.observe([("s1", "A", 2.0, "claim")], 20.0)
|
||||
self.assertIs(book.directory(), first)
|
||||
book.observe([("s1", "B", 3.0, "claim")], 30.0)
|
||||
self.assertIsNot(book.directory(), first)
|
||||
self.assertEqual(book.directory().now("s1"), "B")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -351,6 +351,24 @@ class TestRedirectSettings(HubCase):
|
||||
self.assertEqual([stream["via"] for stream in retried["settings"]["streams"]], [["https://*.cdn.test"], ["https://*.cdn.test"]])
|
||||
self.assertEqual(retried["problems"], [])
|
||||
|
||||
def test_an_unresolved_station_heals_without_a_save_once_due(self):
|
||||
saved = self.signed_and_saved([self.station(STATION, [])])["settings"]
|
||||
cdn().httpd.broken.add("station.radio.test")
|
||||
self.addCleanup(cdn().httpd.broken.discard, "station.radio.test")
|
||||
other = "https://station.radio.test/other"
|
||||
self.signed_and_saved([*saved["streams"], self.station(other, [])])
|
||||
hub = self.httpd.hub
|
||||
before = self.dials()
|
||||
self.assertEqual((hub.retry_stations(), self.dials() - before), ([], 0))
|
||||
hub.redirects.tried[other] -= RETRY_SECONDS
|
||||
self.assertEqual(hub.retry_stations(), ["DLF: answers 302 without a Location"])
|
||||
self.assertIsNone(hub.settings.get()["streams"][1]["via"])
|
||||
cdn().httpd.broken.discard("station.radio.test")
|
||||
hub.redirects.tried[other] -= RETRY_SECONDS
|
||||
self.assertEqual(hub.retry_stations(), [])
|
||||
self.assertEqual([stream["via"] for stream in hub.settings.get()["streams"]], [["https://*.cdn.test"], ["https://*.cdn.test"]])
|
||||
self.assertIn("https://*.cdn.test", self.media_src())
|
||||
|
||||
def test_a_save_that_lands_while_another_resolves_makes_that_one_fail(self):
|
||||
saved = self.signed_and_saved([self.station(STATION, [])])["settings"]
|
||||
slow = [*saved["streams"], self.station("https://drip.radio.test/", [])]
|
||||
|
||||
@@ -579,7 +579,7 @@ class TestHubView(HubCase):
|
||||
council=self.council, pages=self.pages, public_port=self.port, state=self.state, host_state=HOST_STATE, transcripts=self.projects, scripts=self.scripts,
|
||||
templates=HUB / "templates",
|
||||
places=server.Places(tree_file=self.tree_file, timers=self.timers, vendor=HUB / "vendor", cleanup=self.cleanup, offices=self.offices, kept=self.kept),
|
||||
model=server.Model(url="", model="", key=""), engines=self.engines, approvers=self.approvers, redirects=self.redirects(),
|
||||
model=server.Model(url="", model="", key=""), engines=self.engines, approvers=self.approvers, redirects=self.redirects(), live_ttl=self.live_ttl(),
|
||||
)
|
||||
|
||||
def test_a_get_and_a_patch_round_trip_and_the_file_is_private(self):
|
||||
|
||||
Reference in new issue
Block a user