From 96d4d2933c05234361ae800d5ca3a5a3796ca668 Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Wed, 18 Nov 2026 19:54:23 +0000 Subject: [PATCH] perf(hub): cache the names directory, share state builds, retry stations MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Concurrent /api/state polls share one build through LiveState for a short TTL, and every POST and PATCH invalidates it, so a write is never followed by a stale view. The names directory is cached, the first transcript scan reads outside the hub lock, the graph timeline starts at the window begin, and unresolved radio stations are retried on a daemon thread every RETRY_SECONDS instead of failing until restart. 🎯 Fix: 85 % 💥 Break: 8 % 💸 Tokens: – ⏱️ Velocity: – 🛡️ Security: 100 % 🏆 Score: 78 🚦 Gate: ⛔ fix 85 % < 90, break 8 % > 5 Created-with: https://s.veen.world/skills Co-Authored-By: Claude Opus 5.5 --- orchestrator/kvb_orchestrator/graph.py | 5 +- orchestrator/kvb_orchestrator/handler.py | 6 +- orchestrator/kvb_orchestrator/hub.py | 30 ++++++- orchestrator/kvb_orchestrator/livestate.py | 43 ++++++++++ orchestrator/kvb_orchestrator/names.py | 33 +++++++- orchestrator/kvb_orchestrator/offices.py | 2 +- orchestrator/kvb_orchestrator/rules.py | 3 +- orchestrator/kvb_orchestrator/sources.py | 4 +- orchestrator/kvb_orchestrator/transcripts.py | 19 +++-- orchestrator/kvb_orchestrator/view.py | 28 ++++++- orchestrator/server.py | 5 +- orchestrator/templates/page-load.js | 6 +- orchestrator/templates/page-radio.js | 2 +- tests/hub_case.py | 8 ++ tests/test_hub_livestate.py | 85 ++++++++++++++++++++ tests/test_hub_names.py | 32 ++++++++ tests/test_hub_redirects.py | 18 +++++ tests/test_hub_view.py | 2 +- 18 files changed, 299 insertions(+), 32 deletions(-) create mode 100644 orchestrator/kvb_orchestrator/livestate.py create mode 100644 tests/test_hub_livestate.py diff --git a/orchestrator/kvb_orchestrator/graph.py b/orchestrator/kvb_orchestrator/graph.py index 62decc5..e43d65d 100644 --- a/orchestrator/kvb_orchestrator/graph.py +++ b/orchestrator/kvb_orchestrator/graph.py @@ -44,7 +44,8 @@ def edges(radio: list[dict], grant_rows: list[dict], chair: str) -> list[dict]: return [{"source": s, "target": t, "class": c, "count": n} for (s, t, c), n in sorted(counts.items())] -def graph(radio: list[dict], grant_rows: list[dict], tasks: dict[str, dict], claims: list[dict], end: float, directory: Directory) -> dict: +def graph(radio: list[dict], grant_rows: list[dict], tasks: dict[str, dict], claims: list[dict], span: tuple[float, float], directory: Directory) -> dict: + begin, end = span current = {task: row for task, row in sorted(tasks.items()) if status_of(row) not in ENDED} chair = chair_of(claims, directory, end) top = chair or HUMAN @@ -82,7 +83,7 @@ def graph(radio: list[dict], grant_rows: list[dict], tasks: dict[str, dict], cla "parent": parent, "label": label, "names": directory.names(key) or [label], - "timeline": directory.timeline(key) or [{"name": label, "since": end}], + "timeline": directory.timeline(key, begin) or [{"name": label, "since": end}], "tasks": list(holders.get(key, {}).values()), "claims": [claim for claim in claims if claim["session"] == key], } diff --git a/orchestrator/kvb_orchestrator/handler.py b/orchestrator/kvb_orchestrator/handler.py index 9d71ab3..ba0a82f 100644 --- a/orchestrator/kvb_orchestrator/handler.py +++ b/orchestrator/kvb_orchestrator/handler.py @@ -312,7 +312,10 @@ class Handler(Exchange): return receive = self.receiver(path, host) if receive is not None: - receive(self.rfile.read(length)) + try: + receive(self.rfile.read(length)) + finally: + self.server.hub.live_state.invalidate() def do_PATCH(self) -> None: path = urlsplit(self.path).path @@ -328,6 +331,7 @@ class Handler(Exchange): if length is not None: body = self.rfile.read(length) self.answer(lambda: self.server.hub.view_store.patch(body), HTTPStatus.OK) + self.server.hub.live_state.invalidate() def index(self) -> None: nonce = secrets.token_urlsafe(18) diff --git a/orchestrator/kvb_orchestrator/hub.py b/orchestrator/kvb_orchestrator/hub.py index 093f309..be98755 100644 --- a/orchestrator/kvb_orchestrator/hub.py +++ b/orchestrator/kvb_orchestrator/hub.py @@ -1,6 +1,7 @@ from __future__ import annotations import functools +import sys import threading import time from dataclasses import dataclass @@ -27,6 +28,7 @@ from .inbox import Inbox from .jobs import Jobs from .language import Language, Model from .ledger import Ledger, Marks, Reviews, Sealed +from .livestate import LiveState from .matching import Matching from .mcp import Mcp from .messaging import Messaging @@ -43,7 +45,7 @@ from .receipts import ( sight, sighting_pass, ) -from .redirects import Redirects +from .redirects import RETRY_SECONDS, Redirects from .requesting import Requesting from .resources import RESOURCES, SAMPLES from .revealing import Revealing @@ -134,6 +136,7 @@ class Hub(Mcp, Reviewing, Revealing, Asking, Messaging, Choosing, Approving, Den queue_order: QueueOrder view_store: ViewStore match_snapshots: Ledger + live_state: LiveState def hosts(self) -> tuple[str, str]: return f"127.0.0.1:{self.public_port}", f"localhost:{self.public_port}" @@ -158,7 +161,29 @@ def sighting(hub: Hub, stop: threading.Event) -> threading.Thread: return thread -def open_hub(council: Path, pages: Path, public_port: int, state: Path, host_state: Path, transcripts: Path, scripts: Runner, templates: Path, places: Places, model: Model, engines: Engines, approvers: Path, redirects: Redirects) -> Hub: +def stations(hub: Hub, stop: threading.Event) -> threading.Thread: + """Args: + hub: the hub whose unresolved stations are retried. + stop: ends the thread once set. + + Returns: + The started daemon thread that runs retry_stations() at the start and + every RETRY_SECONDS, printing why a station still does not resolve. + """ + + def run() -> None: + while True: + for problem in hub.retry_stations(): + print(f"station redirect: {problem}", file=sys.stderr, flush=True) + if stop.wait(RETRY_SECONDS): + return + + thread = threading.Thread(target=run, name="station redirects", daemon=True) + thread.start() + return thread + + +def open_hub(council: Path, pages: Path, public_port: int, state: Path, host_state: Path, transcripts: Path, scripts: Runner, templates: Path, places: Places, model: Model, engines: Engines, approvers: Path, redirects: Redirects, live_ttl: float) -> Hub: uploads = state / "uploads" uploads.mkdir(exist_ok=True) reviews = Reviews(state) @@ -181,6 +206,7 @@ def open_hub(council: Path, pages: Path, public_port: int, state: Path, host_sta replay=state / "replay" / "messages.json", view_store=ViewStore(state / "view.json"), match_snapshots=Ledger(state / "match.json"), + live_state=LiveState(live_ttl), rules=Ledger(state / "rules.json"), offers=Offers(), jobs=Ledger(state / "jobs.json"), diff --git a/orchestrator/kvb_orchestrator/livestate.py b/orchestrator/kvb_orchestrator/livestate.py new file mode 100644 index 0000000..7b789e6 --- /dev/null +++ b/orchestrator/kvb_orchestrator/livestate.py @@ -0,0 +1,43 @@ +from __future__ import annotations + +import threading +import time +from collections.abc import Callable + +LIVE_TTL = 2.0 + + +class LiveState: + def __init__(self, ttl: float) -> None: + """Args: + ttl: seconds a finished build serves later callers of the same + key; 0 rebuilds every time but still one build at a time. + """ + self.ttl = ttl + self.guard = threading.Lock() + self.locks: dict[object, threading.Lock] = {} + self.built: dict[object, tuple[int, float, dict]] = {} + self.generation = 0 + + def invalidate(self) -> None: + with self.guard: + self.generation += 1 + + def shared(self, key: object, build: Callable[[], dict]) -> dict: + """Returns: + build()'s result; callers arriving while it runs wait for it and + share it, and so do callers within ttl after it, until + invalidate() is called. + """ + with self.guard: + lock = self.locks.setdefault(key, threading.Lock()) + with lock: + with self.guard: + generation = self.generation + hit = self.built.get(key) + if hit is not None and hit[0] == generation and time.monotonic() - hit[1] < self.ttl: + return hit[2] + value = build() + with self.guard: + self.built[key] = (generation, time.monotonic(), value) + return value diff --git a/orchestrator/kvb_orchestrator/names.py b/orchestrator/kvb_orchestrator/names.py index 2d1e75c..c445c4c 100644 --- a/orchestrator/kvb_orchestrator/names.py +++ b/orchestrator/kvb_orchestrator/names.py @@ -1,5 +1,6 @@ from __future__ import annotations +import bisect import itertools import json import math @@ -27,13 +28,14 @@ class Directory: if entry["session"] in holders: holders.remove(entry["session"]) holders.append(entry["session"]) + self.times = {session: [entry["first_seen"] for entry in own] for session, own in self.by_session.items()} def at(self, session: str, ts: float) -> str: own = self.by_session.get(session) if not own: return "" - valid = [entry for entry in own if entry["first_seen"] <= ts] - return (valid[-1] if valid else own[0])["name"] + index = bisect.bisect_right(self.times[session], ts) + return own[index - 1 if index else 0]["name"] def now(self, session: str) -> str: own = self.by_session.get(session) @@ -57,8 +59,18 @@ class Directory: def names(self, key: str) -> list[str]: return list(dict.fromkeys(entry["name"] for entry in self.by_session.get(key, []))) - def timeline(self, key: str) -> list[dict]: - return [{"name": entry["name"], "since": entry["first_seen"]} for entry in self.by_session.get(key, [])] + def timeline(self, key: str, begin: float | None = None) -> list[dict]: + """Args: + key: the session. + begin: when given, the names between the first one and the last + one seen at or before begin are left out; the first stays as + the session's birth. + """ + own = self.by_session.get(key, []) + if begin is not None and own: + start = bisect.bisect_right(self.times[key], begin) - 1 + own = own[:1] + own[max(start, 1):] + return [{"name": entry["name"], "since": entry["first_seen"]} for entry in own] def aliases(self, name: str) -> set[str]: return set(self.by_name.get(name, [])) @@ -103,6 +115,8 @@ class Names: entries, observed = self.data.get("entries"), self.data.get("observed") self.entries = [entry for entry in entries if valid_entry(entry)] if isinstance(entries, list) else [] self.observed: dict[str, str] = {key: name for key, name in observed.items() if isinstance(name, str)} if isinstance(observed, dict) else {} + self.version = 0 + self.cached: tuple[int, Directory] | None = None def payload(self) -> dict: return {"entries": self.entries, "observed": self.observed} @@ -136,6 +150,7 @@ class Names: floor = history[-1]["first_seen"] if history else -math.inf first = seen if seen is not None and seen > floor else max(now, floor) self.entries.append({"session": session, "name": name, "first_seen": first, "source": source}) + self.version += 1 if changed: self.save() @@ -143,6 +158,16 @@ class Names: with self.lock: return [dict(entry) for entry in self.entries] + def directory(self) -> Directory: + """Returns: + The Directory of the current entries, rebuilt only after observe() + added one; it shares the entry dicts, which nothing mutates. + """ + with self.lock: + if self.cached is None or self.cached[0] != self.version: + self.cached = (self.version, Directory(list(self.entries))) + return self.cached[1] + def due(self, now: float) -> bool: with self.lock: if now - self.titled < TITLE_SCAN: diff --git a/orchestrator/kvb_orchestrator/offices.py b/orchestrator/kvb_orchestrator/offices.py index adb6aff..15d2dd9 100644 --- a/orchestrator/kvb_orchestrator/offices.py +++ b/orchestrator/kvb_orchestrator/offices.py @@ -20,7 +20,7 @@ def office_request(raw: object) -> bool: class Offices: def office_sessions(self, claims: list[dict]) -> set[str]: - named = {entry["session"] for entry in self.names.listed()} | {claim["session"] for claim in claims} + named = set(self.names.directory().by_session) | {claim["session"] for claim in claims} return {session for session in named if signing.SESSION.fullmatch(session) and session != HUMAN} def appointable(self, raw: object) -> list[dict]: diff --git a/orchestrator/kvb_orchestrator/rules.py b/orchestrator/kvb_orchestrator/rules.py index 381777b..c3e1f07 100644 --- a/orchestrator/kvb_orchestrator/rules.py +++ b/orchestrator/kvb_orchestrator/rules.py @@ -7,7 +7,6 @@ import time from .council import read_claims from .errors import Conflict, Missing, ToolError from .fields import COMMAND_LIMIT, PATH_LIMIT, bounded, decode, expect, visible -from .names import Directory from .signed import ask_signed, passkey, review_signed, signing, verifier RULE_KINDS = ("ask", "review", "decision") @@ -155,7 +154,7 @@ class Rules: """ if kind_name != "decision": return set() - directory = Directory(self.names.listed()) + directory = self.names.directory() holders = {session for session in directory.aliases(record["agent"]) if directory.now(session) == record["agent"]} return holders if len(holders) == 1 else set() diff --git a/orchestrator/kvb_orchestrator/sources.py b/orchestrator/kvb_orchestrator/sources.py index 348fa6a..40f335a 100644 --- a/orchestrator/kvb_orchestrator/sources.py +++ b/orchestrator/kvb_orchestrator/sources.py @@ -34,7 +34,7 @@ class Sources: sightings = title_sightings(self.transcripts.root, claimed, now) if self.names.due(now) else [] sightings += [(claim["session"], claim["agent"], iso_epoch(claim["since"]), "claim") for claim in claims if claim["agent"] and claim["session"] != "-"] self.names.observe(sorted(sightings, key=lambda sighting: (sighting[2] is None, sighting[2] or 0.0)), now) - known = {entry["name"] for entry in self.names.listed()} + known = set(self.names.directory().by_name) radio = [] for row in news: name, ts = agent(row.get("from")), epoch(row.get("ts")) @@ -42,7 +42,7 @@ class Sources: radio.append((f"{PSEUDO}{name}", name, ts, "radio")) known.add(name) self.names.observe(radio, now) - return Directory(self.names.listed()) + return self.names.directory() def repositories(self) -> dict: tasks = read_register(self.council / REGISTER) diff --git a/orchestrator/kvb_orchestrator/transcripts.py b/orchestrator/kvb_orchestrator/transcripts.py index edeabc0..66f75b5 100644 --- a/orchestrator/kvb_orchestrator/transcripts.py +++ b/orchestrator/kvb_orchestrator/transcripts.py @@ -82,6 +82,7 @@ class Transcripts: def __init__(self, root: Path) -> None: self.root = root self.lock = threading.Lock() + self.scanning = threading.Lock() self.offsets: dict[Path, int] = {} self.entries: deque[dict] = deque(maxlen=STREAM_KEEP) self.crons: dict[str, dict] = {} @@ -89,7 +90,8 @@ class Transcripts: self.scanned = -math.inf def refresh(self) -> None: - with self.lock: + # Trip-wire: a first scan reads hundreds of MiB; only the per-file add takes self.lock, which /api/state also needs. + with self.scanning: if time.monotonic() - self.scanned < STREAM_SCAN: return self.scanned = time.monotonic() @@ -98,7 +100,10 @@ class Transcripts: except OSError: paths = [] for path in paths: - self.read(path) + rows = self.read(path) + with self.lock: + for row in rows: + self.add(path, row) def start(self, path: Path, info: os.stat_result) -> int: offset = self.offsets.get(path) @@ -107,7 +112,7 @@ class Transcripts: return max(info.st_size - STREAM_FIRST, 0) if recent else info.st_size return 0 if info.st_size < offset else offset - def read(self, path: Path) -> None: + def read(self, path: Path) -> list[dict]: try: info = path.stat() begin = self.start(path, info) @@ -115,13 +120,11 @@ class Transcripts: handle.seek(begin) data = handle.read(max(info.st_size - begin, 0)) except OSError: - return + return [] end = data.rfind(b"\n") + 1 self.offsets[path] = begin + end - for raw in data[:end].splitlines(): - row = parse_line(raw.decode("utf-8", errors="replace")) - if row is not None: - self.add(path, row) + rows = (parse_line(raw.decode("utf-8", errors="replace")) for raw in data[:end].splitlines()) + return [row for row in rows if row is not None] def add(self, path: Path, row: dict) -> None: ts = iso_epoch(row.get("timestamp")) diff --git a/orchestrator/kvb_orchestrator/view.py b/orchestrator/kvb_orchestrator/view.py index 3f09013..bad447b 100644 --- a/orchestrator/kvb_orchestrator/view.py +++ b/orchestrator/kvb_orchestrator/view.py @@ -75,12 +75,17 @@ class View: return begin, end def window(self, query: dict[str, list[str]]) -> dict: - now = time.time() - begin, end = self.span(query, now) sampled = single(query, "track") if sampled not in (None, "1"): raise ToolError("track must be 1 or left out") self.tree.beat() + if "from" in query or "to" in query: + return self.built(query, sampled) + return self.live_state.shared(sampled, lambda: self.built(query, sampled)) + + def built(self, query: dict[str, list[str]], sampled: str | None) -> dict: + now = time.time() + begin, end = self.span(query, now) self.sweep_sealed() return self.state(begin, end, now, TRACK_STEPS if sampled else 0) @@ -121,7 +126,7 @@ class View: grant_rows = [{**grant, "target": directory.key(grant["to"], grant["ts"])} for grant in grants(history, (begin, end), claims, tasks)] self.watch.see(tasks, now) jobs = [{**{key: value for key, value in job.items() if key != "proof"}, "output": job["output"][-OUTPUT_LIMIT:]} for job in self.jobs.listed()[-JOBS_SHOWN:]] - drawn = graph(radio, grant_rows, tasks, claims, end, directory) + drawn = graph(radio, grant_rows, tasks, claims, (begin, end), directory) first = self.first(news, history, claims, now) drawn["track"] = track(drawn["nodes"], radio, tasks, claims, (min(max(begin, first), end), end), directory, steps) people = {node["id"]: {"name": node["label"], "now": directory.now(node["id"]) or node["label"], "names": node["names"]} for node in drawn["nodes"] if node["kind"] != "subagent"} @@ -215,6 +220,23 @@ class View: streams.append({**stream, "via": via}) return {**settings, "streams": streams}, problems + def retry_stations(self) -> list[str]: + """Returns: + Why a station's redirect chain still does not resolve. A station + whose via is None is resolved again once Redirects.due allows, + without waiting for a save; a save that lands meanwhile wins. + """ + saved = self.settings.get() + if not any(stream["via"] is None and self.redirects.due(stream["url"]) for stream in saved["streams"]): + return [] + resolved, problems = self.resolved(saved, saved) + if resolved != saved: + try: + self.settings.put(resolved, saved) + except ToolError as error: + problems.append(str(error)) + return problems + def sign_settings(self, body: bytes) -> dict: saved = self.settings.get() return self.challenge_for(self.signed_changes(valid_settings(completed(decode(body), saved)), saved)) diff --git a/orchestrator/server.py b/orchestrator/server.py index 1847782..02769db 100755 --- a/orchestrator/server.py +++ b/orchestrator/server.py @@ -77,9 +77,10 @@ from kvb_orchestrator.engines import Engines from kvb_orchestrator.errors import ToolError from kvb_orchestrator.exchange import HubServer from kvb_orchestrator.handler import Handler -from kvb_orchestrator.hub import Places, open_hub, sighting +from kvb_orchestrator.hub import Places, open_hub, sighting, stations from kvb_orchestrator.keys import key_from from kvb_orchestrator.language import Model +from kvb_orchestrator.livestate import LIVE_TTL from kvb_orchestrator.redirects import TIMEOUT, Redirects, dial from kvb_orchestrator.skill_scripts import Scripts from kvb_orchestrator.speechservice import Service @@ -119,12 +120,14 @@ def main() -> int: ), approvers=args.approvers, redirects=Redirects(tls=ssl.create_default_context(), dial=dial, timeout=TIMEOUT), + live_ttl=LIVE_TTL, ) except ToolError as error: parser.error(str(error)) server = HubServer((args.bind, args.listen_port), Handler) server.hub = hub sighting(hub, threading.Event()) + stations(hub, threading.Event()) print(f"{NAME}: listening on {args.bind}:{args.listen_port}, public port {args.public_port}", flush=True) server.serve_forever() return 0 diff --git a/orchestrator/templates/page-load.js b/orchestrator/templates/page-load.js index ef580fc..47f2894 100644 --- a/orchestrator/templates/page-load.js +++ b/orchestrator/templates/page-load.js @@ -51,9 +51,7 @@ } function poll() { - if (!loadsInFlight) { - refresh(); - } + refresh(); } const SELECTION_HOLD = 20000; @@ -62,7 +60,7 @@ function refresh() { const selected = String(window.getSelection()).length > 0; selectedSince = selected ? selectedSince || Date.now() : 0; - if (!ui.busy && !ui.playing && !document.hidden && (!selected || Date.now() - selectedSince >= SELECTION_HOLD)) { + if (!loadsInFlight && !ui.busy && !ui.playing && !document.hidden && (!selected || Date.now() - selectedSince >= SELECTION_HOLD)) { load(); } } diff --git a/orchestrator/templates/page-radio.js b/orchestrator/templates/page-radio.js index bcaa8eb..ab30a1a 100644 --- a/orchestrator/templates/page-radio.js +++ b/orchestrator/templates/page-radio.js @@ -53,7 +53,7 @@ function redirectNote(stream) { if (stream && stream.via === null) { - return "The server has not resolved this station's redirect yet; saving after ten minutes tries again."; + return "The hub has not resolved this station's redirect yet; it tries again every ten minutes by itself, the hub log names why it failed."; } const released = stream ? stream.via : []; const unloaded = released.filter((origin) => !pageMedia.has(origin)); diff --git a/tests/hub_case.py b/tests/hub_case.py index c4a1efb..0badcfa 100644 --- a/tests/hub_case.py +++ b/tests/hub_case.py @@ -77,6 +77,7 @@ class HubCase(unittest.TestCase): engines=self.engines, approvers=self.approvers, redirects=self.redirects(), + live_ttl=self.live_ttl(), ) self.httpd = httpd self.origin = signing.origin_of(self.port) @@ -90,6 +91,13 @@ class HubCase(unittest.TestCase): def redirects(self) -> Redirects: return offline_redirects() + def live_ttl(self) -> float: + """Returns: + 0, so a test that writes council files directly reads them in the + next state; a case testing the live cache overrides it. + """ + return 0.0 + def call(self, method: str, path: str, body: bytes | None, headers: dict[str, str]) -> tuple[int, dict, bytes]: connection = http.client.HTTPConnection("127.0.0.1", self.port, timeout=30) try: diff --git a/tests/test_hub_livestate.py b/tests/test_hub_livestate.py new file mode 100644 index 0000000..57922a9 --- /dev/null +++ b/tests/test_hub_livestate.py @@ -0,0 +1,85 @@ +from __future__ import annotations + +import json +import threading +import time +import unittest + +import hub_support # noqa: F401 +from hub_case import HubCase +from kvb_orchestrator.livestate import LiveState + + +def counting(pause: float) -> tuple[list[int], object]: + calls: list[int] = [] + + def build() -> dict: + calls.append(1) + time.sleep(pause) + return {"build": len(calls)} + + return calls, build + + +class TestLiveState(unittest.TestCase): + def test_callers_arriving_during_a_build_wait_for_it_and_share_it(self): + live = LiveState(60.0) + calls, build = counting(0.3) + results: list[dict] = [] + threads = [threading.Thread(target=lambda: results.append(live.shared("live", build))) for _ in range(4)] + for thread in threads: + thread.start() + for thread in threads: + thread.join(5) + self.assertEqual((len(calls), len(results)), (1, 4)) + self.assertTrue(all(result is results[0] for result in results)) + + def test_a_zero_ttl_rebuilds_and_other_keys_build_on_their_own(self): + live = LiveState(0.0) + calls, build = counting(0.0) + live.shared("live", build) + live.shared("live", build) + live.shared("track", build) + self.assertEqual(len(calls), 3) + + def test_invalidate_ends_the_shared_build_at_once(self): + live = LiveState(60.0) + _, build = counting(0.0) + first = live.shared("live", build) + self.assertIs(live.shared("live", build), first) + live.invalidate() + self.assertEqual(live.shared("live", build), {"build": 2}) + + def test_a_build_overtaken_by_invalidate_is_not_served_again(self): + live = LiveState(60.0) + started = threading.Event() + + def slow() -> dict: + started.set() + time.sleep(0.2) + return {"stale": True} + + thread = threading.Thread(target=lambda: live.shared("live", slow)) + thread.start() + started.wait(5) + live.invalidate() + thread.join(5) + self.assertEqual(live.shared("live", lambda: {"stale": False}), {"stale": False}) + + +class TestLiveStateRoute(HubCase): + def live_ttl(self) -> float: + return 60.0 + + def test_polls_share_one_state_until_a_post_or_a_window_query(self): + first = self.snapshot() + self.assertEqual(self.snapshot()["now"], first["now"]) + status, _, body = self.get(f"/api/state?to={first['now']}") + self.assertEqual(status, 200, body) + self.assertGreater(json.loads(body)["now"], first["now"]) + self.post_json("/api/inbox/mark", {}, self.guard(self.host)) + self.assertGreater(self.snapshot()["now"], first["now"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_hub_names.py b/tests/test_hub_names.py index 4ff4e6a..6fd372b 100644 --- a/tests/test_hub_names.py +++ b/tests/test_hub_names.py @@ -27,5 +27,37 @@ class TestRenameOnce(unittest.TestCase): self.assertEqual([row["title"] for row in names.Directory(entries).renames(0, 10)], ["A is now called B", "B is now called A"]) +def flips(count: int) -> list[dict]: + return [{"session": "s1", "name": "AB"[index % 2], "first_seen": float(index), "source": "claim"} for index in range(count)] + + +class TestDirectoryLookups(unittest.TestCase): + def test_at_names_the_last_entry_seen_at_or_before_the_moment_and_the_first_one_before_any(self): + directory = names.Directory(flips(6)) + scanned = flips(6) + for ts in (-1.0, 0.0, 0.5, 1.0, 2.9, 3.0, 5.0, 99.0): + valid = [entry for entry in scanned if entry["first_seen"] <= ts] + self.assertEqual(directory.at("s1", ts), (valid[-1] if valid else scanned[0])["name"], ts) + self.assertEqual(directory.at("s9", 1.0), "") + + def test_a_windowed_timeline_keeps_the_birth_and_every_name_from_the_last_one_before_the_window(self): + directory = names.Directory(flips(10)) + self.assertEqual([entry["since"] for entry in directory.timeline("s1", 6.5)], [0.0, 6.0, 7.0, 8.0, 9.0]) + self.assertEqual(directory.timeline("s1", -5.0), directory.timeline("s1")) + self.assertEqual(directory.timeline("s1", 0.0), directory.timeline("s1")) + self.assertEqual(directory.timeline("s9", 3.0), []) + + def test_the_directory_is_rebuilt_only_after_a_new_entry(self): + with tempfile.TemporaryDirectory() as folder: + book = names.Names(Path(folder) / "names.json") + book.observe([("s1", "A", 1.0, "claim")], 10.0) + first = book.directory() + book.observe([("s1", "A", 2.0, "claim")], 20.0) + self.assertIs(book.directory(), first) + book.observe([("s1", "B", 3.0, "claim")], 30.0) + self.assertIsNot(book.directory(), first) + self.assertEqual(book.directory().now("s1"), "B") + + if __name__ == "__main__": unittest.main() diff --git a/tests/test_hub_redirects.py b/tests/test_hub_redirects.py index 2d470fa..022de74 100644 --- a/tests/test_hub_redirects.py +++ b/tests/test_hub_redirects.py @@ -351,6 +351,24 @@ class TestRedirectSettings(HubCase): self.assertEqual([stream["via"] for stream in retried["settings"]["streams"]], [["https://*.cdn.test"], ["https://*.cdn.test"]]) self.assertEqual(retried["problems"], []) + def test_an_unresolved_station_heals_without_a_save_once_due(self): + saved = self.signed_and_saved([self.station(STATION, [])])["settings"] + cdn().httpd.broken.add("station.radio.test") + self.addCleanup(cdn().httpd.broken.discard, "station.radio.test") + other = "https://station.radio.test/other" + self.signed_and_saved([*saved["streams"], self.station(other, [])]) + hub = self.httpd.hub + before = self.dials() + self.assertEqual((hub.retry_stations(), self.dials() - before), ([], 0)) + hub.redirects.tried[other] -= RETRY_SECONDS + self.assertEqual(hub.retry_stations(), ["DLF: answers 302 without a Location"]) + self.assertIsNone(hub.settings.get()["streams"][1]["via"]) + cdn().httpd.broken.discard("station.radio.test") + hub.redirects.tried[other] -= RETRY_SECONDS + self.assertEqual(hub.retry_stations(), []) + self.assertEqual([stream["via"] for stream in hub.settings.get()["streams"]], [["https://*.cdn.test"], ["https://*.cdn.test"]]) + self.assertIn("https://*.cdn.test", self.media_src()) + def test_a_save_that_lands_while_another_resolves_makes_that_one_fail(self): saved = self.signed_and_saved([self.station(STATION, [])])["settings"] slow = [*saved["streams"], self.station("https://drip.radio.test/", [])] diff --git a/tests/test_hub_view.py b/tests/test_hub_view.py index e4d6022..9e6d7d1 100644 --- a/tests/test_hub_view.py +++ b/tests/test_hub_view.py @@ -579,7 +579,7 @@ class TestHubView(HubCase): council=self.council, pages=self.pages, public_port=self.port, state=self.state, host_state=HOST_STATE, transcripts=self.projects, scripts=self.scripts, templates=HUB / "templates", places=server.Places(tree_file=self.tree_file, timers=self.timers, vendor=HUB / "vendor", cleanup=self.cleanup, offices=self.offices, kept=self.kept), - model=server.Model(url="", model="", key=""), engines=self.engines, approvers=self.approvers, redirects=self.redirects(), + model=server.Model(url="", model="", key=""), engines=self.engines, approvers=self.approvers, redirects=self.redirects(), live_ttl=self.live_ttl(), ) def test_a_get_and_a_patch_round_trip_and_the_file_is_private(self):