feat(orchestrator): speak into every text field of the hub (requirement 013)

Requirement 013 and its implementation: page-mic.js puts a microphone on every textarea and text input, records with MediaRecorder and inserts the transcript at the cursor; Escape cancels. /api/stt transcribes with the local whisper.cpp model ggml-small-q8_0 (pinned to commit 5359861c and its sha256) in a per-request temp dir, capped at 60 s and 5 MiB, and returns German or English only, retranscribing a clip detected as another language in both and keeping the surer one. Without a model the button is disabled and names the install row. Tests run once on unify/skills-20261114 after the merge (operator order 2026-11-14).

🎯 Fix: 72 %
💥 Break: 12 %
💸 Tokens: –
⏱️ Velocity: –
🛡️ Security: 100 %
🏆 Score: 63
🚦 Gate: ⛔ fix 72 % < 90, break 12 % > 5

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Kevin Veen-BirkenbachandClaude Opus 5.5 committed 2026-11-14 21:37:26 +00:00
1 parent a2a03995cb
commit a438a2f54e
27 files changed
+861 -149

No files matched your search

+74 -17
View File
@@ -1,11 +1,13 @@
from __future__ import annotations
import io
import json
import os
import re
import secrets
import struct
import subprocess
import tempfile
import threading
import time
import wave
@@ -30,6 +32,12 @@ STT_RATE = 16000
STT_SECONDS = 120
STT_LIMIT = 44 + STT_RATE * 2 * STT_SECONDS
STT_TIMEOUT = 120
INPUT_SECONDS = 60
INPUT_LIMIT = 5 << 20
MODEL_ROW = "make orchestrator-install"
HEARING_DIR = "stt-"
LANGUAGES = ("de", "en")
SPECIAL = "[_"
HEARING_WINDOW = 60
HEARING_RATE = 6
HEARING_HOUR = 3600
@@ -100,7 +108,15 @@ def voice_of(key: str, pool: list[str], operator: str) -> str:
return pool[int(digest(key), 16) % len(pool)]
def wav_seconds(body: bytes) -> float:
def wav_seconds(body: bytes, seconds: int) -> float:
"""Args:
body: the recording.
seconds: the longest recording accepted.
Returns:
Its length in seconds once it is 16 kHz mono 16-bit PCM WAV lasting
more than 0 and at most seconds; otherwise it raises ToolError.
"""
try:
with wave.open(io.BytesIO(body), "rb") as reader:
shape = (reader.getnchannels(), reader.getsampwidth(), reader.getframerate(), reader.getcomptype())
@@ -109,8 +125,8 @@ def wav_seconds(body: bytes) -> float:
raise ToolError("the recording must be a WAV file") from None
if shape != (1, 2, STT_RATE, "NONE"):
raise ToolError(f"the recording must be {STT_RATE} Hz mono 16-bit PCM")
if not 0 < frames <= STT_RATE * STT_SECONDS:
raise ToolError(f"the recording must last more than 0 and at most {STT_SECONDS} seconds")
if not 0 < frames <= STT_RATE * seconds:
raise ToolError(f"the recording must last more than 0 and at most {seconds} seconds")
return frames / STT_RATE
@@ -156,6 +172,27 @@ def heard(output: str) -> str:
return " ".join(NOISE.sub(" ", output).split())[:TEXT_LIMIT]
def transcript(path: Path) -> tuple[str, str, float]:
"""Args:
path: the JSON whisper.cpp writes with --output-json-full.
Returns:
(the language it names, its segment texts joined, the mean
probability of its text tokens, 0.0 without any); a file that is
missing or not of that shape raises EngineError.
"""
try:
data = json.loads(path.read_bytes().decode("utf-8", errors="replace"))
language, segments = data["result"]["language"], data["transcription"]
texts = [segment["text"] for segment in segments]
sure = [token["p"] for segment in segments for token in segment.get("tokens", []) if not str(token["text"]).startswith(SPECIAL)]
except (OSError, ValueError, RecursionError, KeyError, IndexError, TypeError, AttributeError):
raise EngineError("whisper.cpp wrote no readable JSON transcript") from None
if not isinstance(language, str) or not all(isinstance(text, str) for text in texts) or not all(isinstance(p, int | float) and not isinstance(p, bool) for p in sure):
raise EngineError("whisper.cpp wrote no readable JSON transcript")
return language, " ".join(texts), sum(sure) / len(sure) if sure else 0.0
@dataclass(frozen=True)
class Engines:
piper: Path
@@ -207,14 +244,15 @@ class Speech:
def input_gap(self) -> str:
"""Returns:
"" when speech input works, otherwise why not, naming the
whisper.cpp executable or model with its path; always "" with the
speech service.
whisper.cpp executable or model with its path and the operator row
MODEL_ROW that installs both; always "" with the speech service.
"""
if self.engines.service.enabled():
return ""
model = self.engines.whisper_model
gap = unreachable(self.engines.whisper, "the whisper.cpp executable", os.X_OK, "executable") or unreachable(model, "the whisper.cpp model", os.R_OK, "readable")
return gap or ("" if os.path.isfile(model) else f"the whisper.cpp model {model} is not a file")
gap = gap or ("" if os.path.isfile(model) else f"the whisper.cpp model {model} is not a file")
return f"{gap}; the operator row {MODEL_ROW} installs it" if gap else ""
def voice_of(self, key: str) -> str:
"""Args:
@@ -233,7 +271,7 @@ class Speech:
silent, deaf = self.output_gap(), self.input_gap()
return {
"tts": not silent, "stt": not deaf, "tts_off": silent, "stt_off": deaf, "voices": len(self.voices), "operator": operator, "sample": sample,
"limit": SPEECH_LIMIT, "rate": STT_RATE, "seconds": STT_SECONDS,
"limit": SPEECH_LIMIT, "rate": STT_RATE, "seconds": STT_SECONDS, "input": INPUT_SECONDS,
}
def synthesize(self, body: bytes, admit: Callable[[str], None]) -> bytes:
@@ -333,21 +371,40 @@ class Speech:
gap = self.input_gap()
if gap:
raise Unavailable(f"speech input is off: {gap}")
self.admit(wav_seconds(body))
self.admit(wav_seconds(body, INPUT_SECONDS))
return self.whisper(body)
def whisper(self, body: bytes) -> dict:
"""Returns:
{"text"} from the speech service, otherwise {"text", "language"}
in one of LANGUAGES.
"""
if self.engines.service.enabled():
with self.hearing:
return {"text": heard(self.engines.service.hear(body))}
command = [str(self.engines.whisper), "--model", str(self.engines.whisper_model), "--file", "-", "--output-txt", "--language", "auto", "--no-prints", "--suppress-nst"]
with self.hearing:
try:
done = subprocess.run(command, input=body, capture_output=True, timeout=STT_TIMEOUT, env=self.env(), cwd=self.engines.scratch, check=False)
except subprocess.TimeoutExpired:
raise EngineError(f"whisper.cpp took longer than {STT_TIMEOUT} seconds") from None
except OSError as error:
raise EngineError(f"whisper.cpp did not start: {error.strerror}") from None
with self.hearing, tempfile.TemporaryDirectory(prefix=HEARING_DIR, dir=self.engines.scratch) as folder:
language, text, _ = self.listen(body, Path(folder), "auto")
if language not in LANGUAGES:
language, text, _ = max((self.listen(body, Path(folder), choice) for choice in LANGUAGES), key=lambda found: found[2])
return {"text": heard(text), "language": language}
def listen(self, body: bytes, folder: Path, language: str) -> tuple[str, str, float]:
"""Args:
body: the recording.
folder: the request's own directory, cwd and TMPDIR of the run.
language: "auto" to detect it, otherwise the language to hear.
Returns:
transcript() of the JSON whisper.cpp writes into folder.
"""
target = folder / language
command = [str(self.engines.whisper), "--model", str(self.engines.whisper_model), "--file", "-", "--output-json", "--output-json-full", "--output-file", str(target), "--language", language, "--no-prints", "--suppress-nst"]
try:
done = subprocess.run(command, input=body, capture_output=True, timeout=STT_TIMEOUT, env={**self.env(), "TMPDIR": str(folder)}, cwd=folder, check=False)
except subprocess.TimeoutExpired:
raise EngineError(f"whisper.cpp took longer than {STT_TIMEOUT} seconds") from None
except OSError as error:
raise EngineError(f"whisper.cpp did not start: {error.strerror}") from None
if done.returncode != 0:
raise EngineError(f"whisper.cpp failed with exit code {done.returncode}: {done.stderr.decode('utf-8', errors='replace').strip()[-300:]}")
return {"text": heard(done.stdout.decode("utf-8", errors="replace"))}
return transcript(target.with_name(f"{language}.json"))
+2 -2
View File
@@ -8,7 +8,7 @@ from http import HTTPStatus
from urllib.parse import urlsplit
from .diffs import DIFF_BODY_LIMIT
from .engines import STT_LIMIT
from .engines import INPUT_LIMIT, STT_LIMIT
from .exchange import PLAIN_CSP, Exchange
from .resources import RESOURCES, sampled
from .scores import scored
@@ -285,7 +285,7 @@ class Handler(Exchange):
def do_POST(self) -> None:
path = urlsplit(self.path).path
length = self.body_length({UPLOADS: UPLOAD_LIMIT, STT_ROUTE: STT_LIMIT, VOICE_ROUTE: STT_LIMIT, "/mcp": DIFF_BODY_LIMIT, "/api/asks": DIFF_BODY_LIMIT}.get(path, BODY_LIMIT))
length = self.body_length({UPLOADS: UPLOAD_LIMIT, STT_ROUTE: INPUT_LIMIT, VOICE_ROUTE: STT_LIMIT, "/mcp": DIFF_BODY_LIMIT, "/api/asks": DIFF_BODY_LIMIT}.get(path, BODY_LIMIT))
if length is None:
return
host = self.host()
+3 -2
View File
@@ -10,7 +10,8 @@ from collections import Counter
from collections.abc import Callable, Iterator
from pathlib import Path
from .engines import Speech, wav_seconds
from . import engines
from .engines import Speech
from .errors import EngineError, Missing, ToolError, Unavailable
from .records import write_atomic
@@ -44,7 +45,7 @@ class VoiceNotes:
gap = self.speech.input_gap()
if gap:
raise Unavailable(f"speech input is off: {gap}")
seconds = wav_seconds(body)
seconds = engines.wav_seconds(body, engines.STT_SECONDS)
with self.lock:
if sum(1 for note in self.notes.values() if note["status"] == "pending") >= HEARING_LIMIT:
raise Unavailable(f"{HEARING_LIMIT} recordings are already waiting for their transcript")