Merge branch 'feature/hub-voice-input' into unify/skills-20261114
# Conflicts: # README.md # docs/requirements/README.md
This commit is contained in:
commit
b95e305db0
27 files changed
+861
-149
No files matched your search
@@ -0,0 +1,238 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import dataclasses
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import sys
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from unittest import mock
|
||||
|
||||
HERE = Path(__file__).resolve().parent
|
||||
sys.path[:0] = [str(HERE.parent), str(HERE)]
|
||||
|
||||
import test_orchestrator_e2e as base
|
||||
from hub_case import HubCase
|
||||
from kvb_orchestrator import engines
|
||||
|
||||
OUT = Path(os.environ.get("E2E_OUT", "/tmp/voice-input-shots"))
|
||||
WIDTHS = ((1920, 1080), (390, 844))
|
||||
FIXTURE = HERE.parent / "fixtures" / "voice" / "hello-world.wav"
|
||||
HEARD = "hello from the mic second line"
|
||||
FAKE_MIC = """() => {
|
||||
const wav = Uint8Array.from(atob("@WAV@"), (char) => char.charCodeAt(0));
|
||||
const posted = [];
|
||||
window.__voice = {posted, opened: 0};
|
||||
const fetched = window.fetch.bind(window);
|
||||
window.fetch = (path, init) => {
|
||||
if (String(path) === "/api/stt") {
|
||||
posted.push(Date.now());
|
||||
}
|
||||
return fetched(path, init);
|
||||
};
|
||||
if (navigator.mediaDevices) {
|
||||
navigator.mediaDevices.getUserMedia = async () => {
|
||||
window.__voice.opened += 1;
|
||||
const sound = new AudioContext();
|
||||
sound.resume();
|
||||
const source = sound.createBufferSource();
|
||||
source.buffer = await sound.decodeAudioData(wav.slice().buffer);
|
||||
source.loop = true;
|
||||
const sink = sound.createMediaStreamDestination();
|
||||
source.connect(sink);
|
||||
source.start();
|
||||
return sink.stream;
|
||||
};
|
||||
}
|
||||
}"""
|
||||
MIC = """
|
||||
const FIELDS = 'textarea, input:not([type]), input[type="text"], input[type="search"]';
|
||||
const micOf = (field) => (field && field.id ? document.querySelector(`button.mic[aria-controls="${CSS.escape(field.id)}"]`) : null);
|
||||
const bare = () => all(FIELDS).filter((field) => !field.readOnly && !micOf(field)).map((field) => field.getAttribute("aria-label") || field.placeholder || field.outerHTML.slice(0, 80));
|
||||
const composer = () => document.querySelector('#radio-spot textarea[aria-label="Message"]');
|
||||
const live = () => {
|
||||
const found = document.getElementById("mic-live");
|
||||
return found && !found.hidden ? text(found) : null;
|
||||
};
|
||||
const look = (button) => {
|
||||
const box = button.getBoundingClientRect();
|
||||
return {pressed: button.getAttribute("aria-pressed"), disabled: button.disabled, title: button.title, width: box.width, height: box.height, live: live()};
|
||||
};
|
||||
const escape = () => document.dispatchEvent(new KeyboardEvent("keydown", {key: "Escape", bubbles: true, cancelable: true}));
|
||||
const openComposer = async () => {
|
||||
const spot = document.getElementById("radio-spot");
|
||||
if (spot.hidden) {
|
||||
document.getElementById("radio-open").click();
|
||||
}
|
||||
const field = await until(() => composer(), 4000);
|
||||
field.scrollIntoView({block: "center"});
|
||||
return field;
|
||||
};
|
||||
"""
|
||||
SCAN = """(async () => {@HELPERS@@MIC@
|
||||
const missing = {};
|
||||
let scanned = 0;
|
||||
for (const screen of @SCREENS@) {
|
||||
location.hash = `#${screen}`;
|
||||
await screenShown(8000);
|
||||
await sleep(250);
|
||||
scanned += all(FIELDS).length;
|
||||
const found = bare();
|
||||
if (found.length) {
|
||||
missing[screen] = found;
|
||||
}
|
||||
}
|
||||
for (const [name, opener] of [["settings", "settings-open"], ["palette", "palette-open"]]) {
|
||||
document.getElementById(opener).click();
|
||||
await sleep(500);
|
||||
scanned += all(FIELDS).length;
|
||||
const found = bare();
|
||||
if (found.length) {
|
||||
missing[name] = found;
|
||||
}
|
||||
escape();
|
||||
await sleep(300);
|
||||
}
|
||||
location.hash = "#stream";
|
||||
await screenShown(8000);
|
||||
const field = await openComposer();
|
||||
field.value = "start end";
|
||||
field.focus();
|
||||
field.setSelectionRange(6, 6);
|
||||
await sleep(400);
|
||||
return {missing, scanned, idle: look(micOf(field))};
|
||||
})()"""
|
||||
RECORD = """(async () => {@HELPERS@@MIC@
|
||||
const button = micOf(composer());
|
||||
button.click();
|
||||
await until(() => button.getAttribute("aria-pressed") === "true" && live(), 8000);
|
||||
await sleep(300);
|
||||
return {...look(button), opened: window.__voice.opened};
|
||||
})()"""
|
||||
STOP = """(async () => {@HELPERS@@MIC@
|
||||
const field = composer();
|
||||
const button = micOf(field);
|
||||
button.click();
|
||||
await until(() => field.value.includes("@HEARD@"), 20000);
|
||||
await sleep(200);
|
||||
return {value: field.value, caret: field.selectionStart, focused: document.activeElement === field, posts: window.__voice.posted.length, ...look(button)};
|
||||
})()"""
|
||||
CANCEL = """(async () => {@HELPERS@@MIC@
|
||||
const field = composer();
|
||||
const button = micOf(field);
|
||||
const before = [field.value, window.__voice.posted.length];
|
||||
button.click();
|
||||
const recording = Boolean(await until(() => button.getAttribute("aria-pressed") === "true" && live(), 8000));
|
||||
await sleep(800);
|
||||
escape();
|
||||
await until(() => button.getAttribute("aria-pressed") === "false" && !live(), 8000);
|
||||
await sleep(1500);
|
||||
return {recording, same: field.value === before[0], posts: window.__voice.posted.length - before[1], ...look(button)};
|
||||
})()"""
|
||||
DEAF = """(async () => {@HELPERS@@MIC@
|
||||
await screenShown(8000);
|
||||
const field = await openComposer();
|
||||
const button = micOf(field);
|
||||
button.click();
|
||||
await sleep(800);
|
||||
return {...look(button), opened: window.__voice.opened, posts: window.__voice.posted.length};
|
||||
})()"""
|
||||
|
||||
|
||||
def probe(source: str) -> str:
|
||||
return source.replace("@HELPERS@", base.HELPERS).replace("@MIC@", MIC).replace("@SCREENS@", json.dumps(base.SCREENS)).replace("@HEARD@", HEARD)
|
||||
|
||||
|
||||
def fake_mic() -> str:
|
||||
return FAKE_MIC.replace("@WAV@", base64.b64encode(FIXTURE.read_bytes()).decode())
|
||||
|
||||
|
||||
@unittest.skipUnless(shutil.which(os.environ.get("CHROMIUM", "chromium")), "no Chromium")
|
||||
class TestVoiceInputE2E(HubCase):
|
||||
drive = base.TestOrchestratorE2E.drive
|
||||
|
||||
def setUp(self):
|
||||
super().setUp()
|
||||
OUT.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
def steps(self, browser: str) -> list[dict]:
|
||||
steps = []
|
||||
for width, height in WIDTHS:
|
||||
url = f"http://localhost:{self.port}/?{width}#stream"
|
||||
steps += [
|
||||
{"width": width, "height": height, "url": url, "wait": 1500, "probe": probe(SCAN), "shot": str(OUT / f"{browser}-{width}-idle.png"), "gesture": True},
|
||||
{"width": width, "height": height, "wait": 200, "probe": probe(RECORD), "shot": str(OUT / f"{browser}-{width}-recording.png"), "gesture": True},
|
||||
{"width": width, "height": height, "wait": 1200, "probe": probe(STOP), "gesture": True},
|
||||
{"width": width, "height": height, "wait": 200, "probe": probe(CANCEL), "gesture": True},
|
||||
]
|
||||
return steps
|
||||
|
||||
def deaf_steps(self) -> list[dict]:
|
||||
return [{"width": width, "height": height, "url": f"http://localhost:{self.port}/?{width}#stream", "wait": 1500, "probe": probe(DEAF), "gesture": True} for width, height in WIDTHS]
|
||||
|
||||
def deafen(self) -> None:
|
||||
speech = self.httpd.hub.speech
|
||||
patch = mock.patch.object(speech, "engines", dataclasses.replace(speech.engines, whisper_model=self.outside / "ggml-missing.bin"))
|
||||
patch.start()
|
||||
self.addCleanup(patch.stop)
|
||||
|
||||
def errors(self, data: dict) -> list[str]:
|
||||
return [error for error in data["errors"] if "/favicon.ico" not in error]
|
||||
|
||||
def check(self, data: dict, browser: str) -> None:
|
||||
self.assertEqual(self.errors(data), [])
|
||||
for index, (width, _) in enumerate(WIDTHS):
|
||||
scan, recording, stopped, cancelled = data["results"][index * 4 : index * 4 + 4]
|
||||
with self.subTest(browser=browser, width=width):
|
||||
for found in (scan, recording, stopped, cancelled):
|
||||
self.assertNotIn("error", found, found)
|
||||
self.assertEqual(scan["missing"], {}, scan)
|
||||
self.assertGreater(scan["scanned"], 0, scan)
|
||||
idle = scan["idle"]
|
||||
self.assertEqual((idle["pressed"], idle["disabled"], idle["live"]), ("false", False, None), idle)
|
||||
self.assertEqual((recording["pressed"], recording["opened"]), ("true", 1), recording)
|
||||
self.assertIn("Recording", recording["live"] or "", recording)
|
||||
self.assertEqual(stopped["value"], f"start {HEARD} end", stopped)
|
||||
self.assertEqual((stopped["caret"], stopped["focused"], stopped["posts"], stopped["pressed"], stopped["live"]), (len(f"start {HEARD}"), True, 1, "false", None), stopped)
|
||||
self.assertEqual((cancelled["recording"], cancelled["same"], cancelled["posts"], cancelled["pressed"], cancelled["live"]), (True, True, 0, "false", None), cancelled)
|
||||
if width < 640:
|
||||
for shown in (idle, recording):
|
||||
self.assertGreaterEqual(min(shown["width"], shown["height"]), 44, shown)
|
||||
|
||||
def check_deaf(self, data: dict, browser: str) -> None:
|
||||
self.assertEqual(self.errors(data), [])
|
||||
for (width, _), found in zip(WIDTHS, data["results"], strict=True):
|
||||
with self.subTest(browser=browser, width=width):
|
||||
self.assertNotIn("error", found, found)
|
||||
self.assertEqual((found["disabled"], found["pressed"], found["opened"], found["posts"], found["live"]), (True, "false", 0, 0, None), found)
|
||||
self.assertIn(f"the operator row {engines.MODEL_ROW} installs it", found["title"], found)
|
||||
|
||||
def test_chromium_records_from_a_fake_microphone_and_inserts_the_transcript_at_the_cursor_at_1920_and_390(self):
|
||||
data = self.drive(self.steps("chromium"), None, init=fake_mic())
|
||||
(OUT / "chromium-voice-input.json").write_text(json.dumps(data, indent=1), encoding="utf-8")
|
||||
self.check(data, "chromium")
|
||||
|
||||
@unittest.skipUnless(shutil.which(os.environ.get("FIREFOX", "firefox")), "no Firefox")
|
||||
def test_firefox_records_from_a_fake_microphone_and_inserts_the_transcript_at_the_cursor_at_1920_and_390(self):
|
||||
data = self.drive(self.steps("firefox"), None, "bidi.mjs", os.environ.get("FIREFOX", "firefox"), init=fake_mic())
|
||||
(OUT / "firefox-voice-input.json").write_text(json.dumps(data, indent=1), encoding="utf-8")
|
||||
self.check(data, "firefox")
|
||||
|
||||
def test_chromium_disables_the_microphone_and_names_the_operator_row_without_a_model(self):
|
||||
self.deafen()
|
||||
data = self.drive(self.deaf_steps(), None, init=fake_mic())
|
||||
(OUT / "chromium-voice-input-deaf.json").write_text(json.dumps(data, indent=1), encoding="utf-8")
|
||||
self.check_deaf(data, "chromium")
|
||||
|
||||
@unittest.skipUnless(shutil.which(os.environ.get("FIREFOX", "firefox")), "no Firefox")
|
||||
def test_firefox_disables_the_microphone_and_names_the_operator_row_without_a_model(self):
|
||||
self.deafen()
|
||||
data = self.drive(self.deaf_steps(), None, "bidi.mjs", os.environ.get("FIREFOX", "firefox"), init=fake_mic())
|
||||
(OUT / "firefox-voice-input-deaf.json").write_text(json.dumps(data, indent=1), encoding="utf-8")
|
||||
self.check_deaf(data, "firefox")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in new issue
Block a user