Merge branch 'feature/hub-voice-input' into unify/skills-20261114

# Conflicts:
#	README.md
#	docs/requirements/README.md
This commit is contained in:
Kevin Veen-Birkenbach committed 2026-11-14 22:03:42 +00:00
commit b95e305db0
27 files changed
+861 -149

No files matched your search

+238
View File
@@ -0,0 +1,238 @@
from __future__ import annotations
import base64
import dataclasses
import json
import os
import shutil
import sys
import unittest
from pathlib import Path
from unittest import mock
HERE = Path(__file__).resolve().parent
sys.path[:0] = [str(HERE.parent), str(HERE)]
import test_orchestrator_e2e as base
from hub_case import HubCase
from kvb_orchestrator import engines
OUT = Path(os.environ.get("E2E_OUT", "/tmp/voice-input-shots"))
WIDTHS = ((1920, 1080), (390, 844))
FIXTURE = HERE.parent / "fixtures" / "voice" / "hello-world.wav"
HEARD = "hello from the mic second line"
FAKE_MIC = """() => {
const wav = Uint8Array.from(atob("@WAV@"), (char) => char.charCodeAt(0));
const posted = [];
window.__voice = {posted, opened: 0};
const fetched = window.fetch.bind(window);
window.fetch = (path, init) => {
if (String(path) === "/api/stt") {
posted.push(Date.now());
}
return fetched(path, init);
};
if (navigator.mediaDevices) {
navigator.mediaDevices.getUserMedia = async () => {
window.__voice.opened += 1;
const sound = new AudioContext();
sound.resume();
const source = sound.createBufferSource();
source.buffer = await sound.decodeAudioData(wav.slice().buffer);
source.loop = true;
const sink = sound.createMediaStreamDestination();
source.connect(sink);
source.start();
return sink.stream;
};
}
}"""
MIC = """
const FIELDS = 'textarea, input:not([type]), input[type="text"], input[type="search"]';
const micOf = (field) => (field && field.id ? document.querySelector(`button.mic[aria-controls="${CSS.escape(field.id)}"]`) : null);
const bare = () => all(FIELDS).filter((field) => !field.readOnly && !micOf(field)).map((field) => field.getAttribute("aria-label") || field.placeholder || field.outerHTML.slice(0, 80));
const composer = () => document.querySelector('#radio-spot textarea[aria-label="Message"]');
const live = () => {
const found = document.getElementById("mic-live");
return found && !found.hidden ? text(found) : null;
};
const look = (button) => {
const box = button.getBoundingClientRect();
return {pressed: button.getAttribute("aria-pressed"), disabled: button.disabled, title: button.title, width: box.width, height: box.height, live: live()};
};
const escape = () => document.dispatchEvent(new KeyboardEvent("keydown", {key: "Escape", bubbles: true, cancelable: true}));
const openComposer = async () => {
const spot = document.getElementById("radio-spot");
if (spot.hidden) {
document.getElementById("radio-open").click();
}
const field = await until(() => composer(), 4000);
field.scrollIntoView({block: "center"});
return field;
};
"""
SCAN = """(async () => {@HELPERS@@MIC@
const missing = {};
let scanned = 0;
for (const screen of @SCREENS@) {
location.hash = `#${screen}`;
await screenShown(8000);
await sleep(250);
scanned += all(FIELDS).length;
const found = bare();
if (found.length) {
missing[screen] = found;
}
}
for (const [name, opener] of [["settings", "settings-open"], ["palette", "palette-open"]]) {
document.getElementById(opener).click();
await sleep(500);
scanned += all(FIELDS).length;
const found = bare();
if (found.length) {
missing[name] = found;
}
escape();
await sleep(300);
}
location.hash = "#stream";
await screenShown(8000);
const field = await openComposer();
field.value = "start end";
field.focus();
field.setSelectionRange(6, 6);
await sleep(400);
return {missing, scanned, idle: look(micOf(field))};
})()"""
RECORD = """(async () => {@HELPERS@@MIC@
const button = micOf(composer());
button.click();
await until(() => button.getAttribute("aria-pressed") === "true" && live(), 8000);
await sleep(300);
return {...look(button), opened: window.__voice.opened};
})()"""
STOP = """(async () => {@HELPERS@@MIC@
const field = composer();
const button = micOf(field);
button.click();
await until(() => field.value.includes("@HEARD@"), 20000);
await sleep(200);
return {value: field.value, caret: field.selectionStart, focused: document.activeElement === field, posts: window.__voice.posted.length, ...look(button)};
})()"""
CANCEL = """(async () => {@HELPERS@@MIC@
const field = composer();
const button = micOf(field);
const before = [field.value, window.__voice.posted.length];
button.click();
const recording = Boolean(await until(() => button.getAttribute("aria-pressed") === "true" && live(), 8000));
await sleep(800);
escape();
await until(() => button.getAttribute("aria-pressed") === "false" && !live(), 8000);
await sleep(1500);
return {recording, same: field.value === before[0], posts: window.__voice.posted.length - before[1], ...look(button)};
})()"""
DEAF = """(async () => {@HELPERS@@MIC@
await screenShown(8000);
const field = await openComposer();
const button = micOf(field);
button.click();
await sleep(800);
return {...look(button), opened: window.__voice.opened, posts: window.__voice.posted.length};
})()"""
def probe(source: str) -> str:
return source.replace("@HELPERS@", base.HELPERS).replace("@MIC@", MIC).replace("@SCREENS@", json.dumps(base.SCREENS)).replace("@HEARD@", HEARD)
def fake_mic() -> str:
return FAKE_MIC.replace("@WAV@", base64.b64encode(FIXTURE.read_bytes()).decode())
@unittest.skipUnless(shutil.which(os.environ.get("CHROMIUM", "chromium")), "no Chromium")
class TestVoiceInputE2E(HubCase):
drive = base.TestOrchestratorE2E.drive
def setUp(self):
super().setUp()
OUT.mkdir(parents=True, exist_ok=True)
def steps(self, browser: str) -> list[dict]:
steps = []
for width, height in WIDTHS:
url = f"http://localhost:{self.port}/?{width}#stream"
steps += [
{"width": width, "height": height, "url": url, "wait": 1500, "probe": probe(SCAN), "shot": str(OUT / f"{browser}-{width}-idle.png"), "gesture": True},
{"width": width, "height": height, "wait": 200, "probe": probe(RECORD), "shot": str(OUT / f"{browser}-{width}-recording.png"), "gesture": True},
{"width": width, "height": height, "wait": 1200, "probe": probe(STOP), "gesture": True},
{"width": width, "height": height, "wait": 200, "probe": probe(CANCEL), "gesture": True},
]
return steps
def deaf_steps(self) -> list[dict]:
return [{"width": width, "height": height, "url": f"http://localhost:{self.port}/?{width}#stream", "wait": 1500, "probe": probe(DEAF), "gesture": True} for width, height in WIDTHS]
def deafen(self) -> None:
speech = self.httpd.hub.speech
patch = mock.patch.object(speech, "engines", dataclasses.replace(speech.engines, whisper_model=self.outside / "ggml-missing.bin"))
patch.start()
self.addCleanup(patch.stop)
def errors(self, data: dict) -> list[str]:
return [error for error in data["errors"] if "/favicon.ico" not in error]
def check(self, data: dict, browser: str) -> None:
self.assertEqual(self.errors(data), [])
for index, (width, _) in enumerate(WIDTHS):
scan, recording, stopped, cancelled = data["results"][index * 4 : index * 4 + 4]
with self.subTest(browser=browser, width=width):
for found in (scan, recording, stopped, cancelled):
self.assertNotIn("error", found, found)
self.assertEqual(scan["missing"], {}, scan)
self.assertGreater(scan["scanned"], 0, scan)
idle = scan["idle"]
self.assertEqual((idle["pressed"], idle["disabled"], idle["live"]), ("false", False, None), idle)
self.assertEqual((recording["pressed"], recording["opened"]), ("true", 1), recording)
self.assertIn("Recording", recording["live"] or "", recording)
self.assertEqual(stopped["value"], f"start {HEARD} end", stopped)
self.assertEqual((stopped["caret"], stopped["focused"], stopped["posts"], stopped["pressed"], stopped["live"]), (len(f"start {HEARD}"), True, 1, "false", None), stopped)
self.assertEqual((cancelled["recording"], cancelled["same"], cancelled["posts"], cancelled["pressed"], cancelled["live"]), (True, True, 0, "false", None), cancelled)
if width < 640:
for shown in (idle, recording):
self.assertGreaterEqual(min(shown["width"], shown["height"]), 44, shown)
def check_deaf(self, data: dict, browser: str) -> None:
self.assertEqual(self.errors(data), [])
for (width, _), found in zip(WIDTHS, data["results"], strict=True):
with self.subTest(browser=browser, width=width):
self.assertNotIn("error", found, found)
self.assertEqual((found["disabled"], found["pressed"], found["opened"], found["posts"], found["live"]), (True, "false", 0, 0, None), found)
self.assertIn(f"the operator row {engines.MODEL_ROW} installs it", found["title"], found)
def test_chromium_records_from_a_fake_microphone_and_inserts_the_transcript_at_the_cursor_at_1920_and_390(self):
data = self.drive(self.steps("chromium"), None, init=fake_mic())
(OUT / "chromium-voice-input.json").write_text(json.dumps(data, indent=1), encoding="utf-8")
self.check(data, "chromium")
@unittest.skipUnless(shutil.which(os.environ.get("FIREFOX", "firefox")), "no Firefox")
def test_firefox_records_from_a_fake_microphone_and_inserts_the_transcript_at_the_cursor_at_1920_and_390(self):
data = self.drive(self.steps("firefox"), None, "bidi.mjs", os.environ.get("FIREFOX", "firefox"), init=fake_mic())
(OUT / "firefox-voice-input.json").write_text(json.dumps(data, indent=1), encoding="utf-8")
self.check(data, "firefox")
def test_chromium_disables_the_microphone_and_names_the_operator_row_without_a_model(self):
self.deafen()
data = self.drive(self.deaf_steps(), None, init=fake_mic())
(OUT / "chromium-voice-input-deaf.json").write_text(json.dumps(data, indent=1), encoding="utf-8")
self.check_deaf(data, "chromium")
@unittest.skipUnless(shutil.which(os.environ.get("FIREFOX", "firefox")), "no Firefox")
def test_firefox_disables_the_microphone_and_names_the_operator_row_without_a_model(self):
self.deafen()
data = self.drive(self.deaf_steps(), None, "bidi.mjs", os.environ.get("FIREFOX", "firefox"), init=fake_mic())
(OUT / "firefox-voice-input-deaf.json").write_text(json.dumps(data, indent=1), encoding="utf-8")
self.check_deaf(data, "firefox")
if __name__ == "__main__":
unittest.main()
Binary file not shown.
Binary file not shown.
+7 -4
View File
@@ -31,13 +31,16 @@ with wave.open(args.output_file, "wb") as out:
"""
FAKE_WHISPER = """
import json, os, sys
argv = sys.argv[1:]
data = sys.stdin.buffer.read()
with open(os.path.join(os.environ["HOME"], "whisper.jsonl"), "a", encoding="utf-8") as log:
log.write(json.dumps({"argv": sys.argv[1:], "sha256": __import__("hashlib").sha256(data).hexdigest()}) + "\\n")
log.write(json.dumps({"argv": argv, "sha256": __import__("hashlib").sha256(data).hexdigest()}) + "\\n")
if b"FAIL" in data:
sys.exit(4)
print(" hello from the [BLANK_AUDIO] mic")
print("second line")
language = argv[argv.index("--language") + 1]
segments = [{"text": " hello from the [BLANK_AUDIO] mic", "tokens": [{"text": "[_BEG_]", "p": 0.1}, {"text": " hello", "p": 0.9}]}, {"text": "second line", "tokens": [{"text": " second", "p": 0.8}]}]
with open(argv[argv.index("--output-file") + 1] + ".json", "w", encoding="utf-8") as out:
json.dump({"result": {"language": "en" if language == "auto" else language}, "transcription": segments}, out)
"""
@@ -53,7 +56,7 @@ def write_engines(base: Path) -> server.Engines:
(voices / f"{model}.onnx").write_bytes(b"onnx")
(voices / f"{model}.onnx.json").write_text(json.dumps(config), encoding="utf-8")
(voices / "broken.onnx").write_bytes(b"onnx")
model = base / "ggml-base.bin"
model = base / "ggml-small-q8_0.bin"
model.write_bytes(b"ggml")
return server.Engines(piper=bin_dir / "piper", voices=voices, operator_voice=OPERATOR_VOICE, whisper=bin_dir / "whisper-cli", whisper_model=model, scratch=scratch, service=server.Service(url="", key=""))
+14 -9
View File
@@ -18,6 +18,8 @@ from hub_engines import OPERATOR_VOICE, VOICES, log_rows, wav_bytes
from hub_support import HOST_STATE, HUB, server
from kvb_orchestrator import council, engines, exchange, handler, phrasing
ROW = f"; the operator row {engines.MODEL_ROW} installs it"
class TestHubSpeech(HubCase):
def test_voices_come_from_the_models_and_stick_to_their_session(self):
@@ -52,9 +54,9 @@ class TestHubSpeech(HubCase):
({"piper": self.outside / "piper"}, "tts_off", f"the Piper executable {self.outside / 'piper'} is missing or not executable"),
({"voices": self.outside}, "tts_off", f"the voice directory {self.outside} holds no <model>.onnx voice"),
({"voices": lone}, "tts_off", f"the voice config {lone / 'solo.onnx'}.json is missing, unreadable or names no 1 to {engines.SPEAKER_LIMIT} speakers"),
({"whisper": self.outside / "whisper-cli"}, "stt_off", f"the whisper.cpp executable {self.outside / 'whisper-cli'} is missing or not executable"),
({"whisper_model": self.outside / "model.bin"}, "stt_off", f"the whisper.cpp model {self.outside / 'model.bin'} is missing or not readable"),
({"whisper_model": self.outside}, "stt_off", f"the whisper.cpp model {self.outside} is not a file"),
({"whisper": self.outside / "whisper-cli"}, "stt_off", f"the whisper.cpp executable {self.outside / 'whisper-cli'} is missing or not executable{ROW}"),
({"whisper_model": self.outside / "model.bin"}, "stt_off", f"the whisper.cpp model {self.outside / 'model.bin'} is missing or not readable{ROW}"),
({"whisper_model": self.outside}, "stt_off", f"the whisper.cpp model {self.outside} is not a file{ROW}"),
)
for number, (change, field, message) in enumerate(cases):
with self.subTest(change=change):
@@ -70,7 +72,7 @@ class TestHubSpeech(HubCase):
self.addCleanup(folder.chmod, 0o755)
image = engines.Speech(dataclasses.replace(self.engines, whisper=whisper), self.state / "image").describe()
self.assertEqual({key: image[key] for key in ("tts", "voices", "operator", "tts_off")}, {"tts": False, "voices": 0, "operator": "", "tts_off": f"the voice directory {voices} is missing or not searchable"})
self.assertEqual((image["stt"], image["stt_off"]), (False, f"the whisper.cpp executable {whisper} lies under {whisper.parent}, which the hub may not search"))
self.assertEqual((image["stt"], image["stt_off"]), (False, f"the whisper.cpp executable {whisper} lies under {whisper.parent}, which the hub may not search{ROW}"))
status, _, body = self.post_json("/api/tts", {"text": "radio check", "voice": OPERATOR_VOICE}, self.guard(self.host))
self.assertEqual((status, body.decode()), (503, f"speech output is off: the voice directory {voices} is missing or not searchable\n"))
@@ -147,7 +149,7 @@ class TestHubSpeech(HubCase):
with self.subTest(content_type=content_type):
self.assertEqual(self.call("POST", "/api/stt", clip, {**good, "Content-Type": content_type})[0], 415)
self.assertEqual(self.call("POST", "/api/stt", clip, good)[0], 415)
self.assertEqual(self.call("POST", "/api/stt", b"RIFF", {**good, "Content-Type": "audio/wav", "Content-Length": str(engines.STT_LIMIT + 1)})[0], 413)
self.assertEqual(self.call("POST", "/api/stt", b"RIFF", {**good, "Content-Type": "audio/wav", "Content-Length": str(engines.INPUT_LIMIT + 1)})[0], 413)
for broken in (
b"not a wav",
wav_bytes(1, 2, 44100, b"\x00\x00" * 100),
@@ -157,14 +159,17 @@ class TestHubSpeech(HubCase):
):
with self.subTest(broken=broken[:40]):
self.assertEqual(post(broken, good)[0], 400)
with mock.patch.object(engines, "STT_SECONDS", 1):
with mock.patch.object(engines, "INPUT_SECONDS", 1):
self.assertEqual(post(wav_bytes(1, 2, engines.STT_RATE, b"\x00\x00" * (engines.STT_RATE + 1)), good)[0], 400)
self.assertEqual(log_rows(calls), [])
status, headers, body = post(clip, good)
self.assertEqual((status, headers["content-type"]), (200, "application/json"), body)
self.assertEqual(json.loads(body), {"text": "hello from the mic second line"})
argv = ["--model", str(self.engines.whisper_model), "--file", "-", "--output-txt", "--language", "auto", "--no-prints", "--suppress-nst"]
self.assertEqual(log_rows(calls), [{"argv": argv, "sha256": hashlib.sha256(clip).hexdigest()}])
self.assertEqual(json.loads(body), {"text": "hello from the mic second line", "language": "en"})
[row] = log_rows(calls)
target = PurePosixPath(row["argv"][row["argv"].index("--output-file") + 1])
argv = ["--model", str(self.engines.whisper_model), "--file", "-", "--output-json", "--output-json-full", "--output-file", str(target), "--language", "auto", "--no-prints", "--suppress-nst"]
self.assertEqual(row, {"argv": argv, "sha256": hashlib.sha256(clip).hexdigest()})
self.assertEqual((target.name, str(target.parent.parent)), ("auto", str(self.engines.scratch)))
long = wav_bytes(1, 2, engines.STT_RATE, b"\x00\x00" * engines.STT_RATE * 40)
self.assertGreater(len(long), handler.BODY_LIMIT)
self.assertEqual(post(long, good)[0], 200)
+1 -1
View File
@@ -169,7 +169,7 @@ class TestHubState(HubCase):
self.assertTrue(all(row["voice"] in VOICES and row["voice"] != OPERATOR_VOICE for row in state["radio"]))
self.assertEqual(state["speech"], {
"tts": True, "stt": True, "tts_off": "", "stt_off": "", "voices": len(VOICES), "operator": OPERATOR_VOICE, "sample": "all stations, this is Operator, radio check, out",
"limit": phrasing.SPEECH_LIMIT, "rate": engines.STT_RATE, "seconds": engines.STT_SECONDS, "gate": {"threshold": 50, "cpu": None, "open": False},
"limit": phrasing.SPEECH_LIMIT, "rate": engines.STT_RATE, "seconds": engines.STT_SECONDS, "input": engines.INPUT_SECONDS, "gate": {"threshold": 50, "cpu": None, "open": False},
})
self.assertAlmostEqual(state["now"] - state["since"], view.RADIO_WINDOW)
self.assertEqual((state["night"], state["silence"]), ("day", None))
+14 -1
View File
@@ -74,12 +74,25 @@ class TestVoiceModule(unittest.TestCase):
def test_the_recording_is_encoded_as_16_khz_mono_pcm(self):
body = base64.b64decode(self.data["wav"])
self.assertEqual(engines.wav_seconds(body), 7 / engines.STT_RATE)
self.assertEqual(engines.wav_seconds(body, engines.STT_SECONDS), 7 / engines.STT_RATE)
with wave.open(io.BytesIO(body)) as reader:
self.assertEqual((reader.getnchannels(), reader.getsampwidth(), reader.getframerate(), reader.getnframes()), (1, 2, 16000, 7))
self.assertEqual(struct.unpack("<7h", reader.readframes(7)), (0, 16383, -16384, 32767, -32768, 32767, -32768))
self.assertEqual(self.data["frames"], [24000, 16000 * 120, 0])
def test_the_transcript_lands_at_the_cursor_with_one_space_on_each_side(self):
cases = [["start end", 6, 6, "hello"], ["", 0, 0, " hello "], ["abc", 3, 3, "x"], ["ab cd", 1, 4, "X"], ["keep", 2, 2, " "], ["a\nb", 2, 2, "line"]]
probe = "const V = require(process.argv[1]); console.log(JSON.stringify(JSON.parse(process.argv[2]).map((one) => V.spliced(...one))));"
done = subprocess.run(["node", "-e", probe, str(VOICE), json.dumps(cases)], capture_output=True, text=True, timeout=60, check=True)
self.assertEqual(json.loads(done.stdout), [
{"value": "start hello end", "caret": 11},
{"value": "hello", "caret": 5},
{"value": "abc x", "caret": 5},
{"value": "a X d", "caret": 3},
{"value": "keep", "caret": 2},
{"value": "a\nline b", "caret": 6},
])
if __name__ == "__main__":
unittest.main()
+150
View File
@@ -0,0 +1,150 @@
from __future__ import annotations
import dataclasses
import hashlib
import json
import re
import sys
import unittest
from pathlib import Path
from unittest import mock
from hub_case import HubCase
from hub_engines import log_rows, wav_bytes
from kvb_orchestrator import engines
FIXTURES = Path(__file__).resolve().parent / "fixtures" / "voice"
DUTCH = wav_bytes(1, 2, engines.STT_RATE, b"\x02\x00" * engines.STT_RATE)
FRENCH = wav_bytes(1, 2, engines.STT_RATE, b"\x03\x00" * engines.STT_RATE)
HEARD = {
"hallo-welt.wav": {"auto": ["de", "Hallo Welt."], "de": [0.9, "Hallo Welt."], "en": [0.3, "Hollow felt."]},
"hello-world.wav": {"auto": ["en", "Hello world."], "de": [0.2, "Hello Wörld."], "en": [0.9, "Hello world."]},
"dutch": {"auto": ["nl", "Hallo wereld."], "de": [0.7, "Hallo Welt."], "en": [0.4, "Hello world."]},
"french": {"auto": ["fr", "Bonjour le monde."], "de": [0.2, "Bonschur."], "en": [0.6, "Bonjour world."]},
}
FAKE_WHISPER = """
import hashlib, json, os, sys
argv = sys.argv[1:]
data = sys.stdin.buffer.read()
home = os.environ["HOME"]
with open(os.path.join(home, "heard.json"), encoding="utf-8") as table:
clip = json.load(table).get(hashlib.sha256(data).hexdigest(), {"auto": ["en", ""], "de": [0.0, ""], "en": [0.0, ""]})
with open(os.path.join(home, "whisper.jsonl"), "a", encoding="utf-8") as log:
log.write(json.dumps({"argv": argv, "cwd": os.getcwd(), "tmp": os.environ["TMPDIR"]}) + "\\n")
with open(os.path.join(os.environ["TMPDIR"], "engine-spill.wav"), "wb") as spill:
spill.write(data)
if b"FAIL" in data:
sys.exit(4)
language = argv[argv.index("--language") + 1]
detected, text, sure = (*clip["auto"], 0.5) if language == "auto" else (language, clip[language][1], clip[language][0])
tokens = [{"text": "[_BEG_]", "p": 0.01}, *({"text": f" {word}", "p": sure} for word in text.split())]
with open(argv[argv.index("--output-file") + 1] + ".json", "w", encoding="utf-8") as out:
json.dump({"result": {"language": detected}, "transcription": [{"text": text, "tokens": tokens}]}, out)
"""
KEPT = ("heard.json", "whisper.jsonl")
def spoken(text: str) -> str:
return " ".join(re.sub(r"[^\w\s]", " ", text).lower().split())
class TestVoiceInput(HubCase):
def setUp(self):
super().setUp()
self.engines.whisper.write_text(f"#!{sys.executable}\n{FAKE_WHISPER}", encoding="utf-8")
clips = {"dutch": DUTCH, "french": FRENCH}
heard = {hashlib.sha256(clips[name] if name in clips else (FIXTURES / name).read_bytes()).hexdigest(): table for name, table in HEARD.items()}
(self.engines.scratch / "heard.json").write_text(json.dumps(heard), encoding="utf-8")
self.calls = self.engines.scratch / "whisper.jsonl"
def post(self, body: bytes, headers: dict[str, str] | None = None, length: int | None = None) -> tuple[int, dict, bytes]:
merged = {**(self.guard(self.host) if headers is None else headers), "Content-Type": "audio/wav"}
if length is not None:
merged["Content-Length"] = str(length)
return self.call("POST", "/api/stt", body, merged)
def leftovers(self) -> list[str]:
return sorted(path.name for path in self.engines.scratch.iterdir() if path.name not in KEPT)
def languages(self) -> list[str]:
return [row["argv"][row["argv"].index("--language") + 1] for row in log_rows(self.calls)]
def test_the_german_and_the_english_fixture_come_back_as_their_words_with_the_language_detected(self):
for name, language, words in (("hallo-welt.wav", "de", "hallo welt"), ("hello-world.wav", "en", "hello world")):
with self.subTest(name=name):
status, _, body = self.post((FIXTURES / name).read_bytes())
self.assertEqual(status, 200, body)
answer = json.loads(body)
self.assertEqual((answer["language"], spoken(answer["text"])), (language, words))
self.assertEqual(self.languages(), ["auto", "auto"])
for row in log_rows(self.calls):
self.assertEqual(row["argv"].count("--language"), 1, row)
def test_a_third_language_is_heard_again_in_the_surer_of_german_and_english(self):
for clip, language, words in ((DUTCH, "de", "hallo welt"), (FRENCH, "en", "bonjour world")):
with self.subTest(language=language):
status, _, body = self.post(clip)
self.assertEqual(status, 200, body)
answer = json.loads(body)
self.assertEqual((answer["language"], spoken(answer["text"])), (language, words))
self.assertEqual(self.languages(), ["auto", "de", "en", "auto", "de", "en"])
self.assertEqual(self.leftovers(), [])
def test_the_temp_directory_is_empty_after_every_request_including_a_failed_one(self):
self.assertEqual(self.post((FIXTURES / "hello-world.wav").read_bytes())[0], 200)
self.assertEqual(self.leftovers(), [])
status, _, body = self.post(wav_bytes(1, 2, engines.STT_RATE, b"FAIL" * 400))
self.assertEqual(status, 502, body)
self.assertEqual(self.leftovers(), [])
rows = log_rows(self.calls)
self.assertEqual(len(rows), 2)
for row in rows:
used = Path(row["tmp"])
self.assertEqual((Path(row["cwd"]), used.parent), (used, self.engines.scratch), row)
self.assertFalse(used.exists(), row)
def test_a_recording_over_60_seconds_or_5_mb_is_refused_and_not_transcribed(self):
rate = engines.STT_RATE
self.assertEqual((engines.INPUT_SECONDS, engines.INPUT_LIMIT), (60, 5 << 20))
self.assertEqual(self.httpd.hub.speech.describe()["input"], engines.INPUT_SECONDS)
self.assertEqual(self.post(wav_bytes(1, 2, rate, b"\x01\x00" * rate * 60))[0], 200)
self.assertEqual(len(log_rows(self.calls)), 1)
status, _, body = self.post(wav_bytes(1, 2, rate, b"\x01\x00" * (rate * 60 + 1)))
self.assertEqual((status, body), (400, b"the recording must last more than 0 and at most 60 seconds\n"))
self.assertEqual(self.post(b"\x00" * engines.INPUT_LIMIT)[0], 400)
self.assertEqual(self.post(b"RIFF", length=engines.INPUT_LIMIT + 1)[0], 413)
self.assertEqual(len(log_rows(self.calls)), 1)
self.assertEqual(self.leftovers(), [])
def test_another_origin_host_or_a_missing_csrf_header_is_refused_and_not_transcribed(self):
clip = (FIXTURES / "hallo-welt.wav").read_bytes()
good = self.guard(self.host)
refused = (
{key: value for key, value in good.items() if key != "X-Kvb-Orchestrator"},
{key: value for key, value in good.items() if key != "Origin"},
{**good, "Origin": "http://evil.example"},
{**good, "Origin": f"http://localhost:{self.port + 1}"},
{**good, "Host": "evil.example"},
)
for headers in refused:
with self.subTest(headers=headers):
self.assertEqual(self.post(clip, headers)[0], 403)
self.assertEqual(log_rows(self.calls), [])
self.assertEqual(self.post(clip, good)[0], 200)
def test_without_a_model_the_endpoint_and_the_button_state_name_the_operator_row(self):
missing = self.outside / "ggml-missing.bin"
speech = self.httpd.hub.speech
with mock.patch.object(speech, "engines", dataclasses.replace(self.engines, whisper_model=missing)):
state = speech.describe()
status, _, body = self.post((FIXTURES / "hallo-welt.wav").read_bytes())
self.assertFalse(state["stt"])
self.assertEqual(state["stt_off"], f"the whisper.cpp model {missing} is missing or not readable; the operator row {engines.MODEL_ROW} installs it")
self.assertEqual(status, 503, body)
self.assertIn(f"the operator row {engines.MODEL_ROW} installs it".encode(), body)
self.assertEqual(log_rows(self.calls), [])
self.assertEqual(engines.MODEL_ROW, "make orchestrator-install")
if __name__ == "__main__":
unittest.main()
+36
View File
@@ -0,0 +1,36 @@
from __future__ import annotations
import dataclasses
import json
import os
import unittest
from pathlib import Path
from unittest import mock
from hub_case import HubCase
from test_hub_voice_input import FIXTURES, spoken
WHISPER = Path(os.environ.get("KVB_WHISPER_CLI", "/nonexistent/whisper-cli"))
MODEL = Path(os.environ.get("KVB_WHISPER_MODEL", "/nonexistent/ggml-small-q8_0.bin"))
QUEUED = os.environ.get("KVB_VOICE_INTEGRATION") == "1"
READY = os.access(WHISPER, os.X_OK) and MODEL.is_file()
REASON = (
"real Whisper run: needs KVB_WHISPER_CLI and KVB_WHISPER_MODEL from the operator rows of requirement 013 "
"and KVB_VOICE_INTEGRATION=1, which only the queue.py hold command line sets"
)
@unittest.skipUnless(QUEUED and READY, REASON)
class TestVoiceModel(HubCase):
def test_the_real_model_transcribes_the_german_and_the_english_fixture(self):
speech = self.httpd.hub.speech
with mock.patch.object(speech, "engines", dataclasses.replace(speech.engines, whisper=WHISPER, whisper_model=MODEL)):
for name, words in (("hallo-welt.wav", "hallo welt"), ("hello-world.wav", "hello world")):
with self.subTest(name=name):
status, _, body = self.call("POST", "/api/stt", (FIXTURES / name).read_bytes(), {**self.guard(self.host), "Content-Type": "audio/wav"})
self.assertEqual(status, 200, body)
self.assertEqual(spoken(json.loads(body)["text"]), words)
if __name__ == "__main__":
unittest.main()
+1 -1
View File
@@ -12,7 +12,7 @@ VOICE = "huggingface.co/rhasspy/piper-voices/resolve/e7229f7d33a10e145e109b886d6
PINNED = {
VOICE: "9127a559e11603f10b366d1a20ac7426826081dbc521de4c2420c57728d73f0f",
f"{VOICE}.json": "2efdc6d7f954588b8180132cbd9b8001933fdd00932c92bc92fd0d2028a9eb3d",
"huggingface.co/ggerganov/whisper.cpp/resolve/5359861c739e955e79d9a303bcbc70fb988958b1/ggml-base-q8_0.bin": "c577b9a86e7e048a0b7eada054f4dd79a56bbfa911fbdacf900ac5b567cbb7d9",
"huggingface.co/ggerganov/whisper.cpp/resolve/5359861c739e955e79d9a303bcbc70fb988958b1/ggml-small-q8_0.bin": "49c8fb02b65e6049d5fa6c04f81f53b867b5ec9540406812c643f177317f779f",
}
ARCHIVE = (f"codeload.github.com/ggml-org/whisper.cpp/tar.gz/{COMMIT}", "645445da884df6dbd52b30f82d7be7c01d1767c938714e477e786b88d529e9f4")
CLONE = ("clone", "https://github.com/ggml-org/whisper.cpp", f"v1.9.5@{COMMIT}")