fix(orchestrator): release a station's redirect targets in media-src

Deutschlandfunk stayed silent: its signed URL answers 302 to
f111.rndfnk.com, an origin the signed media-src never listed, so the
browser blocked the redirected load and reported only the station's own
URL in securitypolicyviolation.

The hub now resolves each https stream's redirect chain server side when
a settings save adds or changes it and stores the extra origins as the
stream's via list. media-src lists them next to the signed origins. A CDN
host of three or more labels widens to its siblings (*.rndfnk.com); an
IP, a two-label host and a name under a country code's generic second
level stay exact.

Hardening:

- Only the signed URL itself may lie at a non-global address; every
  redirect hop dials a global address only, checked on the address
  actually dialed (NAT64 and IPv4-compatible IPv6 count by the embedded
  IPv4 address).
- A resolve answers within three socket timeouts even for a server that
  drips its headers; a late probe goes on in a daemon thread and a url
  still in flight is not probed twice.
- A failed resolution stores via null and retries on a save ten minutes
  later; the radio page says so instead of blaming the CSP.
- A save whose settings changed while it was resolved fails instead of
  overwriting them (compare and set under the settings lock).
- A new stream URL at an already signed origin takes the passkey too.
- Settings written before via existed load as via [] and never dial.

Residuals:

- The passkey proof is spent before the resolve, so a save racing
  another one fails and the operator signs again.
- A save that hits a due retry blocks up to twelve seconds and may
  reload the page.
- A page loaded before the upgrade throws on the missing via until it
  reloads.
- A private multi-tenant suffix such as github.io still widens.
- getaddrinfo has no timeout; it runs inside the daemon thread.

Operator step after install: remove Deutschlandfunk in Radio, Sender
bearbeiten, save, add it again and save with the passkey.

🎯 Fix: 92 %
💥 Break: 4 %
💸 Cost: 100 %
⏱️ Time: –
🛡️ Security: 100 %
🏆 Score: 88
🚦 Gate: ✅ 90/5, cost and time <= 100, security 100 %

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

orchestrator/kvb_orchestrator/hub.py

@@ -24,6 +24,7 @@ from .mcp import Mcp
 from .messaging import Messaging
 from .names import Names
 from .offices import Offices
+from .redirects import Redirects
 from .requesting import Requesting
 from .reviewing import Reviewing
 from .rules import Rules
@@ -86,6 +87,7 @@ class Hub(Mcp, Reviewing, Asking, Messaging, Approving, Denying, Requesting, Cle
     model: Model
     timers: Path
     settings: Settings
+    redirects: Redirects
     speech: Speech
     approvers: Path
     vendor: Path
@@ -103,7 +105,7 @@ class Hub(Mcp, Reviewing, Asking, Messaging, Approving, Denying, Requesting, Cle
         return f"{signing.origin_of(self.public_port)}/"
 
 
-def open_hub(council: Path, pages: Path, public_port: int, state: Path, host_state: Path, transcripts: Path, scripts: Runner, templates: Path, places: Places, model: Model, engines: Engines, approvers: Path) -> Hub:
+def open_hub(council: Path, pages: Path, public_port: int, state: Path, host_state: Path, transcripts: Path, scripts: Runner, templates: Path, places: Places, model: Model, engines: Engines, approvers: Path, redirects: Redirects) -> Hub:
     uploads = state / "uploads"
     uploads.mkdir(exist_ok=True)
     reviews = Reviews(state)
@@ -126,6 +128,7 @@ def open_hub(council: Path, pages: Path, public_port: int, state: Path, host_sta
         batches=Pending(),
         names=Names(state / "names.json"),
         settings=Settings(state / "settings.json"),
+        redirects=redirects,
         watch=Watch(),
         tree=Tree(file=places.tree_file, heartbeat=state / TREE_HEARTBEAT, lock=threading.Lock(), cached=None, stamp=None),
         model=model,

orchestrator/kvb_orchestrator/redirects.py

@@ -0,0 +1,204 @@
+from __future__ import annotations
+
+import http.client
+import ipaddress
+import socket
+import ssl
+import threading
+import time
+from collections.abc import Callable
+from urllib.parse import urljoin, urlsplit
+
+from . import NAME
+from .errors import ToolError
+from .settings import stream_origin
+
+HOP_LIMIT = 5
+TIMEOUT = 4.0
+DEADLINE_HOPS = 3
+RETRY_SECONDS = 600
+MOVED = frozenset({301, 302, 303, 307, 308})
+HEADERS = {"User-Agent": NAME, "Accept": "*/*"}
+EMBEDDING = (ipaddress.ip_network("64:ff9b::/96"), ipaddress.ip_network("::/96"))
+SECOND_LEVELS = frozenset({"ac", "co", "com", "edu", "go", "gov", "ne", "net", "or", "org"})
+
+Dial = Callable[[str, int, float, bool], socket.socket]
+
+
+def global_address(text: str) -> bool:
+    """Returns:
+        Whether the address is global; a NAT64 or IPv4-compatible IPv6
+        address counts by the IPv4 address it embeds.
+    """
+    address = ipaddress.ip_address(text)
+    if address.version == 6 and any(address in network for network in EMBEDDING):
+        return ipaddress.IPv4Address(int(address) & 0xFFFFFFFF).is_global
+    return address.is_global
+
+
+def dial(host: str, port: int, timeout: float, public: bool) -> socket.socket:
+    """Args:
+        public: connect only to a global address, as for a redirect target
+            the operator never signed; the address checked is the one
+            dialed, so a second lookup cannot swap it. Each allowed address
+            is tried in turn.
+    """
+    failure: Exception = ToolError(f"{host} has no public address")
+    for _, _, _, _, address in socket.getaddrinfo(host, port, type=socket.SOCK_STREAM):
+        if public and not global_address(address[0]):
+            continue
+        try:
+            return socket.create_connection(address[:2], timeout)
+        except OSError as error:
+            failure = error
+    raise failure
+
+
+def media_source(url: str) -> str:
+    """Returns:
+        The media-src source a redirect hop to the url needs. A host of three
+        or more labels widens to its siblings (https://*.cdn.example): a CDN
+        such as the ARD's rndfnk.com picks another node per request, so the
+        exact host the server saw is not the one the browser gets. An IP
+        address, a two-label host and a name right under a country code's
+        generic second level (bbc.co.uk, SECOND_LEVELS) stay exact; a
+        private multi-tenant suffix such as github.io is not known and
+        widens.
+    """
+    origin = stream_origin(url)
+    parts = urlsplit(origin)
+    host = parts.hostname or ""
+    labels = host.split(".")
+    try:
+        ipaddress.ip_address(host)
+        exact = True
+    except ValueError:
+        exact = len(labels) < 3 or (len(labels) == 3 and len(labels[2]) == 2 and labels[1] in SECOND_LEVELS)
+    shown = host if exact else "*." + ".".join(labels[1:])
+    return f"https://{shown}" if parts.port is None else f"https://{shown}:{parts.port}"
+
+
+class Hop(http.client.HTTPConnection):
+    default_port = 443
+
+    def __init__(self, host: str, port: int, tls: ssl.SSLContext, opener: Dial, timeout: float, public: bool) -> None:
+        super().__init__(host, port, timeout=timeout)
+        self.tls = tls
+        self.opener = opener
+        self.public = public
+
+    def connect(self) -> None:
+        self.sock = self.tls.wrap_socket(self.opener(self.host, self.port, self.timeout, self.public), server_hostname=self.host)
+
+
+class Redirects:
+    """Args:
+        tls: the context every hop's certificate is verified against.
+        dial: opens the TCP connection to a host and port within a timeout,
+            to a global address only when told so.
+        timeout: seconds each socket operation of a hop may take; a whole
+            resolve answers within DEADLINE_HOPS times that.
+    """
+
+    def __init__(self, tls: ssl.SSLContext, dial: Dial, timeout: float) -> None:
+        self.tls = tls
+        self.dial = dial
+        self.timeout = timeout
+        self.tried: dict[str, float] = {}
+        self.running: set[str] = set()
+        self.lock = threading.Lock()
+
+    def due(self, url: str) -> bool:
+        return time.monotonic() - self.tried.get(url, -RETRY_SECONDS) >= RETRY_SECONDS
+
+    def hop(self, url: str, public: bool) -> str | None:
+        """Returns:
+            The absolute URL the url redirects to, or None when it answers
+            without a redirect. A GET that reads no byte of the body, since a
+            live stream never ends and some stream servers refuse HEAD.
+        """
+        parts = urlsplit(url)
+        connection = Hop(parts.hostname or "", parts.port or Hop.default_port, self.tls, self.dial, self.timeout, public)
+        try:
+            connection.request("GET", (parts.path or "/") + (f"?{parts.query}" if parts.query else ""), headers=HEADERS)
+            with connection.getresponse() as response:
+                status, location = response.status, response.getheader("Location")
+        finally:
+            connection.close()
+        if status not in MOVED:
+            return None
+        if not location:
+            raise ToolError(f"answers {status} without a Location")
+        return urljoin(url, location)
+
+    def chain(self, url: str) -> list[str]:
+        """Returns:
+            Every URL the url redirects through, in order, after the url.
+
+        Raises:
+            ToolError when a hop leaves https, the chain is longer than
+            HOP_LIMIT, or a hop names no valid URL. Only the url itself, which
+            the operator signed, may lie at a non-global address.
+        """
+        hops: list[str] = []
+        target = self.hop(url, False)
+        while target is not None:
+            origin = stream_origin(target)
+            if urlsplit(origin).scheme != "https":
+                raise ToolError(f"redirects to {origin}, which is not https")
+            if len(hops) == HOP_LIMIT:
+                raise ToolError(f"redirects more than {HOP_LIMIT} times")
+            hops.append(target)
+            target = self.hop(target, True)
+        return hops
+
+    def via(self, url: str) -> list[str]:
+        own = stream_origin(url)
+        return list(dict.fromkeys(media_source(hop) for hop in self.chain(url) if stream_origin(hop) != own))
+
+    def outcome(self, url: str) -> tuple[list[str] | None, str]:
+        try:
+            return self.via(url), ""
+        except (OSError, http.client.HTTPException, ToolError, ValueError) as error:
+            return None, str(error) or type(error).__name__
+
+    def probe(self, url: str, outcomes: dict[str, tuple[list[str] | None, str]]) -> None:
+        try:
+            outcomes[url] = self.outcome(url)
+        finally:
+            with self.lock:
+                self.running.discard(url)
+
+    def claim(self, urls: list[str]) -> tuple[list[str], list[str]]:
+        """Returns:
+            The https urls among urls, and those of them no earlier probe
+            still resolves, now marked running and tried.
+        """
+        wanted = sorted({url for url in urls if urlsplit(url).scheme == "https"})
+        with self.lock:
+            probed = [url for url in wanted if url not in self.running]
+            self.running.update(probed)
+        self.tried.update(dict.fromkeys(probed, time.monotonic()))
+        return wanted, probed
+
+    def resolve(self, urls: list[str]) -> dict[str, tuple[list[str] | None, str]]:
+        """Returns:
+            Per url the origins its redirect chain needs besides its own, or
+            None and why the chain could not be resolved, within the deadline
+            even when a station answers byte by byte. A late chain goes on in
+            its daemon thread until its socket timeouts end it, and a url
+            whose earlier probe still runs is not probed again, so at most one
+            thread per url lingers. Only https urls are probed; any other maps
+            to no origins. Records each probe for due().
+        """
+        wanted, probed = self.claim(urls)
+        limit = self.timeout * DEADLINE_HOPS
+        deadline = time.monotonic() + limit
+        found: dict[str, tuple[list[str] | None, str]] = {}
+        threads = [threading.Thread(target=self.probe, args=(url, found), name=f"redirects {url}", daemon=True) for url in probed]
+        for thread in threads:
+            thread.start()
+        for thread in threads:
+            thread.join(max(0.0, deadline - time.monotonic()))
+        outcomes = {url: (None, "an earlier save still resolves it") for url in wanted} | {url: (None, f"no answer within {limit:g} s") for url in probed} | dict(found)
+        return {url: outcomes.get(url, ([], "")) for url in urls}

orchestrator/kvb_orchestrator/settings.py

@@ -21,6 +21,8 @@ MIDNIGHT = "24:00"
 STREAMS_LIMIT = 20
 STREAM_URL = re.compile(r"https?://[A-Za-z0-9._~:/?#\[\]@!$&()*+,;=%-]{1,2040}")
 HOSTNAME = re.compile(r"(?=.{1,253}$)[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?(?:\.[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?)*")
+VIA = re.compile(r"https://(?P<wild>\*\.)?(?P<host>[a-z0-9.-]{1,253})(?::(?P<port>[1-9][0-9]{0,4}))?")
+VIA_LIMIT = 8
 DEFAULT_PORTS = {"http": 80, "https": 443}
 SETTINGS = {
     "latitude": 0.0,
@@ -93,6 +95,21 @@ def stream_origin(url: object) -> str:
     return f"{parts.scheme}://{host}" if port in (None, DEFAULT_PORTS[parts.scheme]) else f"{parts.scheme}://{host}:{port}"
 
 
+def via_origin(value: object, name: str) -> str:
+    found = VIA.fullmatch(value) if isinstance(value, str) else None
+    host = found is not None and HOSTNAME.fullmatch(found["host"]) is not None and (found["wild"] is None or "." in found["host"])
+    port = found is not None and (found["port"] is None or (found["port"] != "443" and int(found["port"]) <= 65535))
+    if not (host and port):
+        raise ToolError(f"{name} must hold https origins like https://cdn.example or https://*.cdn.example, without the default port")
+    return value
+
+
+def via_list(value: object, name: str) -> list[str]:
+    if not isinstance(value, list) or len(value) > VIA_LIMIT or len(set(map(str, value))) != len(value):
+        raise ToolError(f"{name} must list at most {VIA_LIMIT} distinct origins")
+    return [via_origin(origin, name) for origin in value]
+
+
 def clock_minutes(value: object, name: str, end: bool) -> int:
     if end and value == MIDNIGHT:
         return 24 * 60
@@ -102,9 +119,18 @@ def clock_minutes(value: object, name: str, end: bool) -> int:
     return int(hours) * 60 + int(minutes)
 
 
+def stream_fields(raw: object, name: str) -> dict:
+    """Returns:
+        The stream's fields with their names checked and its via valid; a
+        missing via and None both mean not resolved yet.
+    """
+    entry = section({"via": None, **raw} if isinstance(raw, dict) else raw, name, ("name", "url", "from", "to", "days", "on", "via"))
+    return {**entry, "via": None if entry["via"] is None else via_list(entry["via"], f"{name}.via")}
+
+
 def stream_entry(raw: object, index: int) -> dict:
     name = f"streams[{index}]"
-    entry = section(raw, name, ("name", "url", "from", "to", "days", "on"))
+    entry = stream_fields(raw, name)
     label, days = entry["name"], entry["days"]
     if not isinstance(label, str) or not FIELDS["label"][0].fullmatch(label):
         raise ToolError(f"{name}.name must be one line of 1 to {NAME_LIMIT} characters")
@@ -113,7 +139,10 @@ def stream_entry(raw: object, index: int) -> dict:
         raise ToolError(f"{name} must end at another time than it starts")
     if not isinstance(days, list) or not days or len(set(map(str, days))) != len(days) or any(isinstance(day, bool) or not isinstance(day, int) or not 0 <= day <= 6 for day in days):
         raise ToolError(f"{name}.days must list distinct weekdays from 0 (Monday) to 6 (Sunday)")
-    return {"name": label, "url": entry["url"], "from": entry["from"], "to": entry["to"], "days": sorted(days), "on": flag(entry["on"], f"{name}.on")}
+    return {
+        "name": label, "url": entry["url"], "from": entry["from"], "to": entry["to"], "days": sorted(days), "on": flag(entry["on"], f"{name}.on"),
+        "via": entry["via"],
+    }
 
 
 def tone_setting(raw: object, kind_name: str) -> dict:
@@ -173,7 +202,7 @@ def valid_settings(raw: object) -> dict:
     }
 
 
-def migrated(stored: object) -> object:
+def speech_renamed(stored: object) -> object:
     speech = stored.get("speech") if isinstance(stored, dict) else None
     classes = speech.get("classes") if isinstance(speech, dict) else None
     if not isinstance(classes, list) or not any(name in LEGACY_CLASSES for name in classes):
@@ -182,10 +211,28 @@ def migrated(stored: object) -> object:
     return {**stored, "speech": {**speech, "classes": list(dict.fromkeys(renamed))}}
 
 
-def media_origins(settings: dict) -> list[str]:
+def migrated(stored: object) -> object:
+    """Returns:
+        The stored settings in the current shape: legacy speech classes
+        renamed, and a stream saved before redirect resolution with via []:
+        it may have been saved without the passkey, so the hub never dials
+        it unsigned; re-adding it with the passkey resolves its redirects.
+    """
+    stored = speech_renamed(stored)
+    streams = stored.get("streams") if isinstance(stored, dict) else None
+    if not isinstance(streams, list):
+        return stored
+    return {**stored, "streams": [{**stream, "via": []} if isinstance(stream, dict) and "via" not in stream else stream for stream in streams]}
+
+
+def station_origins(settings: dict) -> list[str]:
     return sorted({stream_origin(stream["url"]) for stream in settings["streams"]})
 
 
+def media_origins(settings: dict) -> list[str]:
+    return sorted({*station_origins(settings), *(origin for stream in settings["streams"] for origin in stream["via"] or ())})
+
+
 class Settings:
     def __init__(self, file: Path) -> None:
         self.file = file
@@ -200,9 +247,16 @@ class Settings:
         with self.lock:
             return copy.deepcopy(self.current)
 
-    def put(self, raw: object) -> dict:
+    def put(self, raw: object, expected: dict) -> dict:
+        """Args:
+            raw: the settings to save.
+            expected: the settings the save was judged against; another save
+                that landed meanwhile refuses this one.
+        """
         settings = valid_settings(raw)
         with self.lock:
+            if self.current != expected:
+                raise ToolError("the settings changed while this save was checked; save again")
             write_atomic(self.file, json.dumps(settings, indent=1, sort_keys=True).encode("utf-8"))
             self.current = settings
             return copy.deepcopy(settings)

orchestrator/kvb_orchestrator/view.py

@@ -33,7 +33,7 @@ from .register import (
     stored_order,
 )
 from .reviewing import review_row
-from .settings import media_origins, valid_settings
+from .settings import media_origins, station_origins, valid_settings
 from .signed import signing
 from .skill_scripts import OUTPUT_LIMIT
 from .tasks import task_rows
@@ -138,25 +138,64 @@ class View:
             "graph": drawn,
         }
 
-    def stream_change(self, raw: dict) -> dict | None:
-        """Returns:
-            The item the operator's passkey signs when the settings let the
-            page load audio from an origin the saved ones do not, else None.
+    def stream_change(self, settings: dict, saved: dict) -> dict | None:
+        """Args:
+            settings: valid settings the page wants to save.
+            saved: the settings saved now.
+
+        Returns:
+            The item the operator's passkey signs when the settings name a
+            stream url the saved ones do not, else None: the station origins
+            and those urls. The hub dials no url before it is signed, and
+            resolves its redirect targets only after the signature.
         """
-        origins = media_origins(valid_settings(raw))
-        return signing.stream_item(origins) if set(origins) - set(media_origins(self.settings.get())) else None
+        urls = sorted({stream["url"] for stream in settings["streams"]} - {stream["url"] for stream in saved["streams"]})
+        return signing.stream_item(station_origins(settings), urls) if urls else None
+
+    def resolved(self, settings: dict, saved: dict) -> tuple[dict, list[str]]:
+        """Args:
+            settings: valid settings whose new stream urls the operator
+                signed in this request.
+            saved: the settings saved now.
+
+        Returns:
+            The settings with the via of each new url taken from its redirect
+            chain, and of each saved url whose via is still None (unresolved)
+            once Redirects.due allows a retry; every other stream keeps its
+            saved via without a dial, and the page's via is never used. Also
+            one line per stream whose chain did not resolve; its via stays
+            None.
+        """
+        known = {stream["url"]: stream["via"] for stream in saved["streams"]}
+        urls = {stream["url"] for stream in settings["streams"]}
+        outcomes = self.redirects.resolve(sorted(url for url in urls if url not in known or (known[url] is None and self.redirects.due(url))))
+        streams, problems = [], []
+        for stream in settings["streams"]:
+            via, problem = outcomes.get(stream["url"], (known.get(stream["url"]), ""))
+            if problem:
+                problems.append(f"{stream['name']}: {problem}")
+            streams.append({**stream, "via": via})
+        return {**settings, "streams": streams}, problems
 
     def sign_settings(self, body: bytes) -> dict:
-        change = self.stream_change(decode(body))
+        change = self.stream_change(valid_settings(decode(body)), self.settings.get())
         return self.challenge_for([] if change is None else [change])
 
     def save_settings(self, body: bytes) -> dict:
+        """Returns:
+            {"settings", "problems"}: the saved settings, their via resolved
+            once the proof holds, and why a chain did not resolve. A save
+            that lands while this one resolves makes this one fail.
+        """
         payload = decode(body)
         proof = payload.pop("proof", None)
-        change = self.stream_change(payload)
+        settings = valid_settings(payload)
+        saved = self.settings.get()
+        change = self.stream_change(settings, saved)
         if change is not None:
             self.operator_proven(proof, change)
-        return self.settings.put(payload)
+        resolved, problems = self.resolved(settings, saved)
+        return {"settings": self.settings.put(resolved, saved), "problems": problems}
 
     def render(self, nonce: str) -> tuple[str, str]:
         now = time.time()