2026-11-14 21:37:26 +00:00
|
|
|
from __future__ import annotations
|
|
|
|
|
|
|
|
|
|
import base64
|
|
|
|
|
import dataclasses
|
|
|
|
|
import json
|
|
|
|
|
import os
|
|
|
|
|
import shutil
|
|
|
|
|
import sys
|
|
|
|
|
import unittest
|
|
|
|
|
from pathlib import Path
|
|
|
|
|
from unittest import mock
|
|
|
|
|
|
|
|
|
|
HERE = Path(__file__).resolve().parent
|
|
|
|
|
sys.path[:0] = [str(HERE.parent), str(HERE)]
|
|
|
|
|
|
2026-11-15 06:54:36 +00:00
|
|
|
import hub_e2e_kit as base
|
2026-11-14 21:37:26 +00:00
|
|
|
from hub_case import HubCase
|
|
|
|
|
from kvb_orchestrator import engines
|
|
|
|
|
|
|
|
|
|
OUT = Path(os.environ.get("E2E_OUT", "/tmp/voice-input-shots"))
|
|
|
|
|
WIDTHS = ((1920, 1080), (390, 844))
|
|
|
|
|
FIXTURE = HERE.parent / "fixtures" / "voice" / "hello-world.wav"
|
|
|
|
|
HEARD = "hello from the mic second line"
|
|
|
|
|
FAKE_MIC = """() => {
|
|
|
|
|
const wav = Uint8Array.from(atob("@WAV@"), (char) => char.charCodeAt(0));
|
|
|
|
|
const posted = [];
|
|
|
|
|
window.__voice = {posted, opened: 0};
|
|
|
|
|
const fetched = window.fetch.bind(window);
|
|
|
|
|
window.fetch = (path, init) => {
|
|
|
|
|
if (String(path) === "/api/stt") {
|
|
|
|
|
posted.push(Date.now());
|
|
|
|
|
}
|
|
|
|
|
return fetched(path, init);
|
|
|
|
|
};
|
|
|
|
|
if (navigator.mediaDevices) {
|
|
|
|
|
navigator.mediaDevices.getUserMedia = async () => {
|
|
|
|
|
window.__voice.opened += 1;
|
|
|
|
|
const sound = new AudioContext();
|
|
|
|
|
sound.resume();
|
|
|
|
|
const source = sound.createBufferSource();
|
|
|
|
|
source.buffer = await sound.decodeAudioData(wav.slice().buffer);
|
|
|
|
|
source.loop = true;
|
|
|
|
|
const sink = sound.createMediaStreamDestination();
|
|
|
|
|
source.connect(sink);
|
|
|
|
|
source.start();
|
|
|
|
|
return sink.stream;
|
|
|
|
|
};
|
|
|
|
|
}
|
|
|
|
|
}"""
|
|
|
|
|
MIC = """
|
|
|
|
|
const FIELDS = 'textarea, input:not([type]), input[type="text"], input[type="search"]';
|
|
|
|
|
const micOf = (field) => (field && field.id ? document.querySelector(`button.mic[aria-controls="${CSS.escape(field.id)}"]`) : null);
|
|
|
|
|
const bare = () => all(FIELDS).filter((field) => !field.readOnly && !micOf(field)).map((field) => field.getAttribute("aria-label") || field.placeholder || field.outerHTML.slice(0, 80));
|
|
|
|
|
const composer = () => document.querySelector('#radio-spot textarea[aria-label="Message"]');
|
|
|
|
|
const live = () => {
|
|
|
|
|
const found = document.getElementById("mic-live");
|
|
|
|
|
return found && !found.hidden ? text(found) : null;
|
|
|
|
|
};
|
|
|
|
|
const look = (button) => {
|
|
|
|
|
const box = button.getBoundingClientRect();
|
|
|
|
|
return {pressed: button.getAttribute("aria-pressed"), disabled: button.disabled, title: button.title, width: box.width, height: box.height, live: live()};
|
|
|
|
|
};
|
|
|
|
|
const escape = () => document.dispatchEvent(new KeyboardEvent("keydown", {key: "Escape", bubbles: true, cancelable: true}));
|
|
|
|
|
const openComposer = async () => {
|
|
|
|
|
const spot = document.getElementById("radio-spot");
|
|
|
|
|
if (spot.hidden) {
|
|
|
|
|
document.getElementById("radio-open").click();
|
|
|
|
|
}
|
|
|
|
|
const field = await until(() => composer(), 4000);
|
|
|
|
|
field.scrollIntoView({block: "center"});
|
|
|
|
|
return field;
|
|
|
|
|
};
|
|
|
|
|
"""
|
|
|
|
|
SCAN = """(async () => {@HELPERS@@MIC@
|
|
|
|
|
const missing = {};
|
|
|
|
|
let scanned = 0;
|
|
|
|
|
for (const screen of @SCREENS@) {
|
|
|
|
|
location.hash = `#${screen}`;
|
|
|
|
|
await screenShown(8000);
|
|
|
|
|
await sleep(250);
|
|
|
|
|
scanned += all(FIELDS).length;
|
|
|
|
|
const found = bare();
|
|
|
|
|
if (found.length) {
|
|
|
|
|
missing[screen] = found;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
for (const [name, opener] of [["settings", "settings-open"], ["palette", "palette-open"]]) {
|
|
|
|
|
document.getElementById(opener).click();
|
|
|
|
|
await sleep(500);
|
|
|
|
|
scanned += all(FIELDS).length;
|
|
|
|
|
const found = bare();
|
|
|
|
|
if (found.length) {
|
|
|
|
|
missing[name] = found;
|
|
|
|
|
}
|
|
|
|
|
escape();
|
|
|
|
|
await sleep(300);
|
|
|
|
|
}
|
|
|
|
|
location.hash = "#stream";
|
|
|
|
|
await screenShown(8000);
|
|
|
|
|
const field = await openComposer();
|
|
|
|
|
field.value = "start end";
|
|
|
|
|
field.focus();
|
|
|
|
|
field.setSelectionRange(6, 6);
|
|
|
|
|
await sleep(400);
|
|
|
|
|
return {missing, scanned, idle: look(micOf(field))};
|
|
|
|
|
})()"""
|
|
|
|
|
RECORD = """(async () => {@HELPERS@@MIC@
|
|
|
|
|
const button = micOf(composer());
|
|
|
|
|
button.click();
|
|
|
|
|
await until(() => button.getAttribute("aria-pressed") === "true" && live(), 8000);
|
|
|
|
|
await sleep(300);
|
|
|
|
|
return {...look(button), opened: window.__voice.opened};
|
|
|
|
|
})()"""
|
|
|
|
|
STOP = """(async () => {@HELPERS@@MIC@
|
|
|
|
|
const field = composer();
|
|
|
|
|
const button = micOf(field);
|
|
|
|
|
button.click();
|
|
|
|
|
await until(() => field.value.includes("@HEARD@"), 20000);
|
|
|
|
|
await sleep(200);
|
|
|
|
|
return {value: field.value, caret: field.selectionStart, focused: document.activeElement === field, posts: window.__voice.posted.length, ...look(button)};
|
|
|
|
|
})()"""
|
|
|
|
|
CANCEL = """(async () => {@HELPERS@@MIC@
|
|
|
|
|
const field = composer();
|
|
|
|
|
const button = micOf(field);
|
|
|
|
|
const before = [field.value, window.__voice.posted.length];
|
|
|
|
|
button.click();
|
|
|
|
|
const recording = Boolean(await until(() => button.getAttribute("aria-pressed") === "true" && live(), 8000));
|
|
|
|
|
await sleep(800);
|
|
|
|
|
escape();
|
|
|
|
|
await until(() => button.getAttribute("aria-pressed") === "false" && !live(), 8000);
|
|
|
|
|
await sleep(1500);
|
|
|
|
|
return {recording, same: field.value === before[0], posts: window.__voice.posted.length - before[1], ...look(button)};
|
|
|
|
|
})()"""
|
|
|
|
|
DEAF = """(async () => {@HELPERS@@MIC@
|
|
|
|
|
await screenShown(8000);
|
|
|
|
|
const field = await openComposer();
|
|
|
|
|
const button = micOf(field);
|
|
|
|
|
button.click();
|
|
|
|
|
await sleep(800);
|
|
|
|
|
return {...look(button), opened: window.__voice.opened, posts: window.__voice.posted.length};
|
|
|
|
|
})()"""
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def probe(source: str) -> str:
|
|
|
|
|
return source.replace("@HELPERS@", base.HELPERS).replace("@MIC@", MIC).replace("@SCREENS@", json.dumps(base.SCREENS)).replace("@HEARD@", HEARD)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def fake_mic() -> str:
|
|
|
|
|
return FAKE_MIC.replace("@WAV@", base64.b64encode(FIXTURE.read_bytes()).decode())
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
@unittest.skipUnless(shutil.which(os.environ.get("CHROMIUM", "chromium")), "no Chromium")
|
|
|
|
|
class TestVoiceInputE2E(HubCase):
|
2026-11-15 06:54:36 +00:00
|
|
|
drive = base.OrchestratorE2ECase.drive
|
2026-11-14 21:37:26 +00:00
|
|
|
|
|
|
|
|
def setUp(self):
|
|
|
|
|
super().setUp()
|
|
|
|
|
OUT.mkdir(parents=True, exist_ok=True)
|
|
|
|
|
|
|
|
|
|
def steps(self, browser: str) -> list[dict]:
|
|
|
|
|
steps = []
|
|
|
|
|
for width, height in WIDTHS:
|
|
|
|
|
url = f"http://localhost:{self.port}/?{width}#stream"
|
|
|
|
|
steps += [
|
|
|
|
|
{"width": width, "height": height, "url": url, "wait": 1500, "probe": probe(SCAN), "shot": str(OUT / f"{browser}-{width}-idle.png"), "gesture": True},
|
|
|
|
|
{"width": width, "height": height, "wait": 200, "probe": probe(RECORD), "shot": str(OUT / f"{browser}-{width}-recording.png"), "gesture": True},
|
|
|
|
|
{"width": width, "height": height, "wait": 1200, "probe": probe(STOP), "gesture": True},
|
|
|
|
|
{"width": width, "height": height, "wait": 200, "probe": probe(CANCEL), "gesture": True},
|
|
|
|
|
]
|
|
|
|
|
return steps
|
|
|
|
|
|
|
|
|
|
def deaf_steps(self) -> list[dict]:
|
|
|
|
|
return [{"width": width, "height": height, "url": f"http://localhost:{self.port}/?{width}#stream", "wait": 1500, "probe": probe(DEAF), "gesture": True} for width, height in WIDTHS]
|
|
|
|
|
|
|
|
|
|
def deafen(self) -> None:
|
|
|
|
|
speech = self.httpd.hub.speech
|
|
|
|
|
patch = mock.patch.object(speech, "engines", dataclasses.replace(speech.engines, whisper_model=self.outside / "ggml-missing.bin"))
|
|
|
|
|
patch.start()
|
|
|
|
|
self.addCleanup(patch.stop)
|
|
|
|
|
|
|
|
|
|
def errors(self, data: dict) -> list[str]:
|
|
|
|
|
return [error for error in data["errors"] if "/favicon.ico" not in error]
|
|
|
|
|
|
|
|
|
|
def check(self, data: dict, browser: str) -> None:
|
|
|
|
|
self.assertEqual(self.errors(data), [])
|
|
|
|
|
for index, (width, _) in enumerate(WIDTHS):
|
|
|
|
|
scan, recording, stopped, cancelled = data["results"][index * 4 : index * 4 + 4]
|
|
|
|
|
with self.subTest(browser=browser, width=width):
|
|
|
|
|
for found in (scan, recording, stopped, cancelled):
|
|
|
|
|
self.assertNotIn("error", found, found)
|
|
|
|
|
self.assertEqual(scan["missing"], {}, scan)
|
|
|
|
|
self.assertGreater(scan["scanned"], 0, scan)
|
|
|
|
|
idle = scan["idle"]
|
|
|
|
|
self.assertEqual((idle["pressed"], idle["disabled"], idle["live"]), ("false", False, None), idle)
|
|
|
|
|
self.assertEqual((recording["pressed"], recording["opened"]), ("true", 1), recording)
|
|
|
|
|
self.assertIn("Recording", recording["live"] or "", recording)
|
|
|
|
|
self.assertEqual(stopped["value"], f"start {HEARD} end", stopped)
|
|
|
|
|
self.assertEqual((stopped["caret"], stopped["focused"], stopped["posts"], stopped["pressed"], stopped["live"]), (len(f"start {HEARD}"), True, 1, "false", None), stopped)
|
|
|
|
|
self.assertEqual((cancelled["recording"], cancelled["same"], cancelled["posts"], cancelled["pressed"], cancelled["live"]), (True, True, 0, "false", None), cancelled)
|
|
|
|
|
if width < 640:
|
|
|
|
|
for shown in (idle, recording):
|
|
|
|
|
self.assertGreaterEqual(min(shown["width"], shown["height"]), 44, shown)
|
|
|
|
|
|
|
|
|
|
def check_deaf(self, data: dict, browser: str) -> None:
|
|
|
|
|
self.assertEqual(self.errors(data), [])
|
|
|
|
|
for (width, _), found in zip(WIDTHS, data["results"], strict=True):
|
|
|
|
|
with self.subTest(browser=browser, width=width):
|
|
|
|
|
self.assertNotIn("error", found, found)
|
|
|
|
|
self.assertEqual((found["disabled"], found["pressed"], found["opened"], found["posts"], found["live"]), (True, "false", 0, 0, None), found)
|
|
|
|
|
self.assertIn(f"the operator row {engines.MODEL_ROW} installs it", found["title"], found)
|
|
|
|
|
|
|
|
|
|
def test_chromium_records_from_a_fake_microphone_and_inserts_the_transcript_at_the_cursor_at_1920_and_390(self):
|
|
|
|
|
data = self.drive(self.steps("chromium"), None, init=fake_mic())
|
|
|
|
|
(OUT / "chromium-voice-input.json").write_text(json.dumps(data, indent=1), encoding="utf-8")
|
|
|
|
|
self.check(data, "chromium")
|
|
|
|
|
|
|
|
|
|
@unittest.skipUnless(shutil.which(os.environ.get("FIREFOX", "firefox")), "no Firefox")
|
|
|
|
|
def test_firefox_records_from_a_fake_microphone_and_inserts_the_transcript_at_the_cursor_at_1920_and_390(self):
|
|
|
|
|
data = self.drive(self.steps("firefox"), None, "bidi.mjs", os.environ.get("FIREFOX", "firefox"), init=fake_mic())
|
|
|
|
|
(OUT / "firefox-voice-input.json").write_text(json.dumps(data, indent=1), encoding="utf-8")
|
|
|
|
|
self.check(data, "firefox")
|
|
|
|
|
|
|
|
|
|
def test_chromium_disables_the_microphone_and_names_the_operator_row_without_a_model(self):
|
|
|
|
|
self.deafen()
|
|
|
|
|
data = self.drive(self.deaf_steps(), None, init=fake_mic())
|
|
|
|
|
(OUT / "chromium-voice-input-deaf.json").write_text(json.dumps(data, indent=1), encoding="utf-8")
|
|
|
|
|
self.check_deaf(data, "chromium")
|
|
|
|
|
|
|
|
|
|
@unittest.skipUnless(shutil.which(os.environ.get("FIREFOX", "firefox")), "no Firefox")
|
|
|
|
|
def test_firefox_disables_the_microphone_and_names_the_operator_row_without_a_model(self):
|
|
|
|
|
self.deafen()
|
|
|
|
|
data = self.drive(self.deaf_steps(), None, "bidi.mjs", os.environ.get("FIREFOX", "firefox"), init=fake_mic())
|
|
|
|
|
(OUT / "firefox-voice-input-deaf.json").write_text(json.dumps(data, indent=1), encoding="utf-8")
|
|
|
|
|
self.check_deaf(data, "firefox")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
if __name__ == "__main__":
|
|
|
|
|
unittest.main()
|