feat(orchestrator): S8 shared speech service and sealed model transport

Requirement 001 S8: the hub can use a shared OpenAI-style speech service (KVB_ORCHESTRATOR_SPEECH_URL, key from a mounted file) instead of the local piper and whisper engines, and the image takes its pinned artefacts through an optional INFINITO_ARTEFACT_BASE cache. A new transport serves speech and the LLM: one wall-clock deadline over DNS, connect, TLS handshake and body, certificate and hostname always verified, no redirect followed. A key travels only over https or to loopback; both the speech and the LLM start refuse it otherwise. Key files are capped at 4096 bytes. Streamed WAV answers with placeholder sizes are accepted only when the data chunk runs to the end, and are rewritten before caching. Accepted MEDUSA false positives (41, independent review, operator pick): raw client socket patterns in transport.py, AIC-EXFIL-005 on the sealed key path, JB-SCAN-2237 on the hub's own system prompt, STG009 on the RIFF pad byte and LLM01 at language.py:111, where agent-written titles reach the NL prompt but the model only proposes a plan the operator confirms.

🎯 Fix: 93 %
💥 Break: 4 %
💸 Cost: –
⏱️ Time: –
🛡️ Security: 100 %
🏆 Score: 89
🚦 Gate: ✅ 90/5, cost and time <= 100, security 100 %

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Kevin Veen-BirkenbachandClaude Opus 5.5 committed 2026-11-13 16:57:52 +00:00
1 parent 590b430d37
commit 6d2bca77f7
16 files changed
+1253 -56

No files matched your search

+64 -9
View File
@@ -4,6 +4,7 @@ import io
import os
import re
import secrets
import struct
import subprocess
import threading
import time
@@ -18,6 +19,7 @@ from .fields import TEXT_LIMIT, bounded, decode, expect
from .phrasing import PROCEDURE_LINES, SPEECH_LIMIT, SPOKEN_LINE, phrase
from .records import digest, read_json
from .skill_scripts import SAFE_PATH
from .speechservice import Service
MODEL_NAME = re.compile(r"[A-Za-z0-9][A-Za-z0-9_.-]{0,127}")
SPEAKER_LIMIT = 10000
@@ -33,6 +35,7 @@ HEARING_RATE = 6
HEARING_HOUR = 3600
HEARING_BUDGET = 5 * STT_SECONDS
NOISE = re.compile(r"\[[^\]\n]{0,40}\]")
PLACEHOLDERS = frozenset({0, 0xFFFFFFFF})
def voice_models(folder: Path) -> list[Path]:
@@ -111,6 +114,36 @@ def wav_seconds(body: bytes) -> float:
return frames / STT_RATE
def whole_wav(body: bytes) -> bytes | None:
"""Returns:
body if it is a RIFF WAVE whose RIFF size matches the bytes received
and whose chunks, a non-empty data chunk among them, all end within
those bytes; a streamed WAVE, whose RIFF size is a placeholder in
PLACEHOLDERS, only if its data chunk is the last chunk and runs to the
end of the body, returned with both sizes set to the bytes received;
None for every other body.
"""
if len(body) < 12 or body[:4] != b"RIFF" or body[8:12] != b"WAVE":
return None
riff = struct.unpack_from("<I", body, 4)[0]
streamed = riff in PLACEHOLDERS
if not streamed and riff + 8 != len(body):
return None
offset, data = 12, False
while offset + 8 <= len(body):
name, size = body[offset : offset + 4], struct.unpack_from("<I", body, offset + 4)[0]
start = offset + 8
if streamed and name == b"data":
if start == len(body) or (size not in PLACEHOLDERS and start + size != len(body)):
return None
return b"RIFF" + struct.pack("<I", len(body) - 8) + body[8 : offset + 4] + struct.pack("<I", len(body) - start) + body[start:]
if start + size > len(body):
return None
data = data or (name == b"data" and size > 0)
offset = start + size + (size & 1)
return body if data and not streamed and offset in (len(body), len(body) + 1) else None
def playable(path: Path) -> bool:
try:
with wave.open(str(path), "rb") as reader:
@@ -131,6 +164,7 @@ class Engines:
whisper: Path
whisper_model: Path
scratch: Path
service: Service
class Speech:
@@ -156,10 +190,13 @@ class Speech:
def output_gap(self) -> str:
"""Returns:
"" when speech output works, otherwise why not, naming the Piper
executable, the voice directory or a voice config with its path.
executable, the voice directory or a voice config with its path;
with the speech service the voices still come from the voice
directory, and no Piper runs here.
"""
folder = self.engines.voices
gap = unreachable(self.engines.piper, "the Piper executable", os.X_OK, "executable") or unreachable(folder, "the voice directory", os.R_OK | os.X_OK, "searchable")
piper = "" if self.engines.service.enabled() else unreachable(self.engines.piper, "the Piper executable", os.X_OK, "executable")
gap = piper or unreachable(folder, "the voice directory", os.R_OK | os.X_OK, "searchable")
if gap or self.voices:
return gap
models = voice_models(folder)
@@ -170,8 +207,11 @@ class Speech:
def input_gap(self) -> str:
"""Returns:
"" when speech input works, otherwise why not, naming the
whisper.cpp executable or model with its path.
whisper.cpp executable or model with its path; always "" with the
speech service.
"""
if self.engines.service.enabled():
return ""
model = self.engines.whisper_model
gap = unreachable(self.engines.whisper, "the whisper.cpp executable", os.X_OK, "executable") or unreachable(model, "the whisper.cpp model", os.R_OK, "readable")
return gap or ("" if os.path.isfile(model) else f"the whisper.cpp model {model} is not a file")
@@ -229,9 +269,24 @@ class Speech:
return target.read_bytes()
def render(self, voice: str, line: str, target: Path) -> None:
partial = self.cache / f".{target.stem}-{secrets.token_hex(4)}.wav"
try:
if self.engines.service.enabled():
spoken = whole_wav(self.engines.service.speak(voice, line))
if spoken is None:
raise EngineError("the speech service answered no complete WAV")
partial.write_bytes(spoken)
if not playable(partial):
raise EngineError("the speech service answered no playable WAV")
else:
self.piper(voice, line, partial)
os.replace(partial, target)
finally:
partial.unlink(missing_ok=True)
def piper(self, voice: str, line: str, partial: Path) -> None:
model, speaker = self.voices[voice]
onnx = self.engines.voices / f"{model}.onnx"
partial = self.cache / f".{target.stem}-{secrets.token_hex(4)}.wav"
command = [str(self.engines.piper), "--model", str(onnx), "--config", f"{onnx}.json", "--output-file", str(partial)]
command += [] if speaker is None else ["--speaker", str(speaker)]
try:
@@ -246,15 +301,12 @@ class Speech:
cwd=self.engines.scratch,
check=False,
)
if done.returncode != 0 or not playable(partial):
raise EngineError(f"piper failed with exit code {done.returncode}: {done.stderr.strip()[-300:]}")
os.replace(partial, target)
except subprocess.TimeoutExpired:
raise EngineError(f"piper took longer than {TTS_TIMEOUT} seconds") from None
except OSError as error:
raise EngineError(f"piper did not start: {error.strerror}") from None
finally:
partial.unlink(missing_ok=True)
if done.returncode != 0 or not playable(partial):
raise EngineError(f"piper failed with exit code {done.returncode}: {done.stderr.strip()[-300:]}")
def trim(self) -> None:
spoken = (path for path in self.cache.glob("*.wav") if not path.name.startswith(WORD_CACHE))
@@ -285,6 +337,9 @@ class Speech:
return self.whisper(body)
def whisper(self, body: bytes) -> dict:
if self.engines.service.enabled():
with self.hearing:
return {"text": heard(self.engines.service.hear(body))}
command = [str(self.engines.whisper), "--model", str(self.engines.whisper_model), "--file", "-", "--output-txt", "--language", "auto", "--no-prints", "--suppress-nst"]
with self.hearing:
try:
+54
View File
@@ -0,0 +1,54 @@
from __future__ import annotations
from pathlib import Path
from urllib.parse import urlsplit
from .errors import ToolError
KEY_LIMIT = 4096
LOOPBACK = frozenset({"127.0.0.1", "localhost", "::1"})
def bearer(key: str) -> bool:
"""Returns:
Whether key is printable ASCII without whitespace, the only shape an
Authorization header carries.
"""
return key.isascii() and key.isprintable() and not any(character.isspace() for character in key)
def sealed(url: str) -> bool:
"""Returns:
Whether a bearer key sent to url stays off the open network: an https
URL or one whose host is 127.0.0.1, localhost or ::1; False for a URL
that does not parse.
"""
try:
parts = urlsplit(url)
return parts.scheme == "https" or parts.hostname in LOOPBACK
except ValueError:
return False
def key_from(path: Path, owner: str) -> str:
"""Args:
path: a bearer key file; an empty file such as /dev/null means no key.
owner: what the key is for, such as "speech service", named in the
message.
Returns:
The key without the whitespace around it.
Raises:
ToolError naming the path, never its content, when the file cannot be
read as UTF-8 text or holds more than KEY_LIMIT bytes; at most
KEY_LIMIT + 1 bytes are read, so /dev/zero fails at once.
"""
try:
with path.open("rb") as handle:
data = handle.read(KEY_LIMIT + 1)
if len(data) > KEY_LIMIT:
raise ToolError(f"the {owner} key file {path} holds more than {KEY_LIMIT} bytes")
return data.decode("utf-8").strip()
except (OSError, ValueError):
raise ToolError(f"the {owner} key file {path} is unreadable") from None
+25 -25
View File
@@ -2,15 +2,15 @@ from __future__ import annotations
import json
import time
import urllib.request
from dataclasses import dataclass
from pathlib import Path
from dataclasses import dataclass, field
from . import transport
from .council import NEWS, ordered, read_claims
from .denying import DENIABLE
from .errors import ToolError
from .events import VIEWS
from .fields import bounded, decode, expect, message_text
from .keys import bearer, sealed
from .records import read_jsonl
from .register import REGISTER, agent_order, read_register
@@ -18,14 +18,6 @@ NL_LIMIT = 2000
NL_ACTION_LIMIT = 20
LLM_TIMEOUT = 60
LLM_REPLY = 1 << 20
class NoRedirect(urllib.request.HTTPRedirectHandler):
def redirect_request(self, req, fp, code, msg, headers, newurl):
return None
LLM_OPENER = urllib.request.build_opener(urllib.request.ProxyHandler({}), NoRedirect)
SCREENS = ("stream", "inbox", "rules", "agents", "artifacts", "tasks")
IDS = {"type": "array", "items": {"type": "string"}, "minItems": 1, "description": "Ids from items."}
NL_ACTIONS = [
@@ -139,9 +131,23 @@ def nl_actions(reply: str, items: list[dict], agents: list[str]) -> dict:
@dataclass(frozen=True)
class Model:
"""Args:
url: base URL of an OpenAI-compatible chat endpoint; empty turns the
natural-language command line off.
model: the model name sent to it.
key: the bearer key it expects; empty sends none. It never shows in
the repr or in any message.
"""
url: str
model: str
key: str
key: str = field(repr=False)
def __post_init__(self) -> None:
if self.key and not bearer(self.key):
raise ToolError("the model endpoint key must be printable ASCII without whitespace: check KVB_ORCHESTRATOR_LLM_KEY_FILE")
if self.url and self.key and not sealed(self.url):
raise ToolError("the model endpoint key travels only over https or to 127.0.0.1, localhost or ::1: set an https KVB_ORCHESTRATOR_LLM_URL or empty KVB_ORCHESTRATOR_LLM_KEY_FILE")
def enabled(self) -> bool:
return bool(self.url and self.model)
@@ -153,25 +159,19 @@ class Model:
if self.key:
headers["Authorization"] = f"Bearer {self.key}"
body = json.dumps({"model": self.model, "messages": messages, "temperature": 0, "response_format": {"type": "json_object"}}).encode()
request = urllib.request.Request(target, data=body, headers=headers, method="POST")
try:
with LLM_OPENER.open(request, timeout=LLM_TIMEOUT) as response:
data = json.loads(response.read(LLM_REPLY))
content = data["choices"][0]["message"]["content"]
except (OSError, ValueError, KeyError, IndexError, TypeError) as error:
raise ToolError(f"the model endpoint failed: {error}") from None
reply = transport.post(target, body, headers, LLM_REPLY, LLM_TIMEOUT)
except transport.Failure as failure:
raise ToolError(f"the model endpoint {failure}") from None
try:
content = json.loads(reply)["choices"][0]["message"]["content"]
except (ValueError, RecursionError, KeyError, IndexError, TypeError) as error:
raise ToolError(f"the model endpoint failed: {type(error).__name__}") from None
if not isinstance(content, str):
raise ToolError("the model endpoint answered no text")
return content
def read_key(path: Path) -> str:
try:
return path.read_text(encoding="utf-8").strip()
except (OSError, ValueError):
return ""
class Language:
def nl(self, body: bytes) -> dict:
if not self.model.enabled():
@@ -0,0 +1,115 @@
from __future__ import annotations
import json
import secrets
from dataclasses import dataclass, field
from urllib.parse import urlsplit
from . import transport
from .errors import EngineError, ToolError
from .keys import bearer, sealed
SPEECH_PATH = "/audio/speech"
HEARING_PATH = "/audio/transcriptions"
SPEECH_MODEL = "tts-1"
HEARING_MODEL = "whisper-1"
SERVICE_TIMEOUT = 120
SPOKEN_REPLY = 32 << 20
HEARD_REPLY = 1 << 20
SCHEMES = frozenset({"http", "https"})
def service_base(url: str) -> bool:
"""Returns:
Whether url is a printable ASCII http or https URL with a host and a
port from 1 to 65535 if any, without whitespace, userinfo, query,
fragment or percent escape, so no credential travels in it.
"""
if not (url.isascii() and url.isprintable()) or any(mark in url for mark in " @?#%"):
return False
try:
parts = urlsplit(url)
port_valid = parts.port != 0
except ValueError:
return False
return port_valid and parts.scheme in SCHEMES and bool(parts.hostname)
def form(fields: dict[str, str], name: str, wav: bytes) -> tuple[bytes, str]:
"""Args:
fields: the plain form fields.
name: the field the WAV file travels in.
wav: the recording.
Returns:
The multipart/form-data body and its content type.
"""
boundary = secrets.token_hex(16)
while boundary.encode() in wav:
boundary = secrets.token_hex(16)
parts = [f'--{boundary}\r\nContent-Disposition: form-data; name="{key}"\r\n\r\n{value}\r\n'.encode() for key, value in fields.items()]
parts.append(f'--{boundary}\r\nContent-Disposition: form-data; name="{name}"; filename="recording.wav"\r\nContent-Type: audio/wav\r\n\r\n'.encode() + wav + b"\r\n")
return b"".join(parts) + f"--{boundary}--\r\n".encode(), f"multipart/form-data; boundary={boundary}"
@dataclass(frozen=True)
class Service:
"""Args:
url: base URL of an OpenAI-compatible audio API with its /v1, such as
svc-ai-speech or the LiteLLM route to it; empty keeps the
in-container engines.
key: the bearer key the service expects; empty sends none. It never
shows in the repr or in any message.
"""
url: str
key: str = field(repr=False)
def __post_init__(self) -> None:
if self.url and not service_base(self.url):
raise ToolError("the speech service URL must be http or https with a host and without userinfo, query or fragment")
if self.key and not bearer(self.key):
raise ToolError("the speech service key must be printable ASCII without whitespace: check KVB_ORCHESTRATOR_SPEECH_KEY_FILE")
if self.url and self.key and not sealed(self.url):
raise ToolError("the speech service key travels only over https or to 127.0.0.1, localhost or ::1: set an https KVB_ORCHESTRATOR_SPEECH_URL or empty KVB_ORCHESTRATOR_SPEECH_KEY_FILE")
def enabled(self) -> bool:
return bool(self.url)
def post(self, path: str, body: bytes, kind: str, limit: int) -> bytes:
"""Returns:
The body of a 200 answer, within SERVICE_TIMEOUT seconds of wall
clock for the whole call.
Raises:
EngineError for every failure, naming a status, a limit or an
exception type only.
"""
headers = {"Content-Type": kind, "Accept": "*/*"}
if self.key:
headers["Authorization"] = f"Bearer {self.key}"
try:
return transport.post(f"{self.url.rstrip('/')}{path}", body, headers, limit, SERVICE_TIMEOUT)
except transport.Failure as failure:
raise EngineError(f"the speech service {failure}") from None
def speak(self, voice: str, line: str) -> bytes:
"""Returns:
The WAV the service renders of line in voice, unchecked.
"""
body = json.dumps({"model": SPEECH_MODEL, "input": line, "voice": voice, "response_format": "wav"}).encode()
return self.post(SPEECH_PATH, body, "application/json", SPOKEN_REPLY)
def hear(self, wav: bytes) -> str:
"""Returns:
The transcript the service hears in the recording.
"""
body, kind = form({"model": HEARING_MODEL, "response_format": "json"}, "file", wav)
reply = self.post(HEARING_PATH, body, kind, HEARD_REPLY)
try:
text = json.loads(reply)["text"]
except (ValueError, RecursionError, KeyError, IndexError, TypeError):
raise EngineError("the speech service answered no transcript") from None
if not isinstance(text, str):
raise EngineError("the speech service answered no transcript")
return text
+190
View File
@@ -0,0 +1,190 @@
from __future__ import annotations
import contextlib
import http.client
import socket
import ssl
import threading
import time
from urllib.parse import urlsplit
CHUNK = 1 << 16
TLS = ssl.create_default_context()
class Failure(Exception):
"""Args:
reason: what went wrong as a status, a limit or an exception type,
never a header or a body of the exchange.
"""
class Call:
"""Args:
seconds: the wall-clock time one call may take, from the name lookup
to the last byte of the answer.
"""
def __init__(self, seconds: float) -> None:
self.seconds = seconds
self.deadline = time.monotonic() + seconds
self.lock = threading.Lock()
self.sock: socket.socket | None = None
self.over = False
def late(self) -> Failure:
return Failure(f"took longer than {self.seconds:g} seconds")
def remaining(self) -> float:
left = self.deadline - time.monotonic()
if left <= 0:
raise self.late()
return left
def attach(self, sock: socket.socket | None) -> None:
with self.lock:
if self.over and sock is not None:
raise self.late()
self.sock = sock
def cut(self) -> None:
with self.lock:
self.over = True
if self.sock is not None:
with contextlib.suppress(OSError):
socket.socket.shutdown(self.sock, socket.SHUT_RDWR)
def finish(self) -> None:
with self.lock:
self.over = True
self.sock = None
def resolve(self, host: str, port: int) -> list[tuple]:
"""Returns:
The addresses of host, looked up in a daemon thread that the call
waits for only until its deadline.
"""
outcome: list[object] = []
def lookup() -> None:
try:
outcome.append(socket.getaddrinfo(host, port, type=socket.SOCK_STREAM))
except OSError as error:
outcome.append(error)
worker = threading.Thread(target=lookup, name="bounded lookup", daemon=True)
worker.start()
worker.join(self.remaining())
if not outcome:
raise self.late()
if isinstance(outcome[0], OSError):
raise outcome[0]
return outcome[0]
def dial(self, host: str, port: int) -> socket.socket:
"""Returns:
A socket connected to the first address of host that answers,
each attempt within the time left, the watchdog able to cut it.
"""
failure: OSError = OSError("no address")
for family, kind, proto, _, address in self.resolve(host, port):
sock = socket.socket(family, kind, proto)
try:
self.attach(sock)
sock.settimeout(self.remaining())
sock.connect(address)
sock.settimeout(self.remaining())
return sock
except OSError as error:
self.attach(None)
sock.close()
failure = error
except Failure:
self.attach(None)
sock.close()
raise
raise failure
class Plain(http.client.HTTPConnection):
def __init__(self, host: str, port: int, call: Call) -> None:
super().__init__(host, port, timeout=call.seconds)
self.call = call
def connect(self) -> None:
self.sock = self.call.dial(self.host, self.port)
class Sealed(Plain):
def connect(self) -> None:
raw = self.call.dial(self.host, self.port)
try:
tls = TLS.wrap_socket(raw, server_hostname=self.host, do_handshake_on_connect=False)
except BaseException:
raw.close()
raise
self.sock = tls
self.call.attach(tls)
tls.do_handshake()
CONNECTIONS = {"http": (Plain, 80), "https": (Sealed, 443)}
def drain(response: http.client.HTTPResponse, limit: int, call: Call) -> bytes:
chunks: list[bytes] = []
size = 0
while chunk := response.read1(CHUNK):
size += len(chunk)
if size > limit:
raise Failure(f"answered more than {limit} bytes")
call.remaining()
chunks.append(chunk)
call.remaining()
if response.length:
raise Failure("answered a truncated reply")
return b"".join(chunks)
def post(url: str, body: bytes, headers: dict[str, str], limit: int, seconds: float) -> bytes:
"""Args:
url: the http or https URL to POST to; no proxy and no redirect.
body: the request body.
headers: the request headers.
limit: the most bytes the answer may carry.
seconds: the wall-clock time the whole call may take: name lookup,
connect, TLS handshake, request and answer.
Returns:
The body of a 200 answer.
Raises:
Failure for every other outcome, naming a status, a limit or an
exception type only.
"""
call = Call(seconds)
watchdog = threading.Timer(seconds, call.cut)
watchdog.daemon = True
connection: Plain | None = None
try:
watchdog.start()
parts = urlsplit(url)
kind, port = CONNECTIONS[parts.scheme]
connection = kind(parts.hostname or "", parts.port or port, call)
connection.request("POST", (parts.path or "/") + (f"?{parts.query}" if parts.query else ""), body=body, headers=headers)
call.remaining()
with connection.getresponse() as response:
if response.status != 200:
raise Failure(f"answered HTTP {response.status}")
return drain(response, limit, call)
except Failure:
raise
except Exception as error: # noqa: BLE001
if isinstance(error, TimeoutError) or time.monotonic() >= call.deadline:
raise call.late() from None
raise Failure(f"failed: {type(error).__name__}") from None
finally:
watchdog.cancel()
call.finish()
if connection is not None:
connection.close()