Merge branch 'feature/hub-voice-input' into unify/skills-20261114
# Conflicts: # README.md # docs/requirements/README.md
This commit is contained in:
commit
b95e305db0
27 files changed
+861
-149
No files matched your search
@@ -50,9 +50,9 @@ ADD --checksum=sha256:2efdc6d7f954588b8180132cbd9b8001933fdd00932c92bc92fd0d2028
|
||||
${INFINITO_ARTEFACT_BASE:-https:/}/huggingface.co/rhasspy/piper-voices/resolve/e7229f7d33a10e145e109b886d661984201ac3c4/en/en_US/libritts/high/en_US-libritts-high.onnx.json \
|
||||
/opt/voices/en_US-libritts-high.onnx.json
|
||||
# hadolint ignore=DL3020
|
||||
ADD --checksum=sha256:c577b9a86e7e048a0b7eada054f4dd79a56bbfa911fbdacf900ac5b567cbb7d9 --chmod=0644 \
|
||||
${INFINITO_ARTEFACT_BASE:-https:/}/huggingface.co/ggerganov/whisper.cpp/resolve/5359861c739e955e79d9a303bcbc70fb988958b1/ggml-base-q8_0.bin \
|
||||
/opt/whisper/ggml-base-q8_0.bin
|
||||
ADD --checksum=sha256:49c8fb02b65e6049d5fa6c04f81f53b867b5ec9540406812c643f177317f779f --chmod=0644 \
|
||||
${INFINITO_ARTEFACT_BASE:-https:/}/huggingface.co/ggerganov/whisper.cpp/resolve/5359861c739e955e79d9a303bcbc70fb988958b1/ggml-small-q8_0.bin \
|
||||
/opt/whisper/ggml-small-q8_0.bin
|
||||
COPY --from=whisper-cli /out/whisper-cli /opt/whisper/whisper-cli
|
||||
# ADD --chmod=0644 also gives the directories it creates mode 0644; without the search bit the non-root hub finds no voice and no whisper.cpp
|
||||
RUN chmod 0755 /opt/voices /opt/whisper
|
||||
|
||||
@@ -77,7 +77,7 @@ services:
|
||||
- --whisper
|
||||
- /opt/whisper/whisper-cli
|
||||
- --whisper-model
|
||||
- /opt/whisper/ggml-base-q8_0.bin
|
||||
- /opt/whisper/ggml-small-q8_0.bin
|
||||
- --scratch
|
||||
- /scratch
|
||||
- --speech-url
|
||||
|
||||
@@ -1,11 +1,13 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import io
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import secrets
|
||||
import struct
|
||||
import subprocess
|
||||
import tempfile
|
||||
import threading
|
||||
import time
|
||||
import wave
|
||||
@@ -30,6 +32,12 @@ STT_RATE = 16000
|
||||
STT_SECONDS = 120
|
||||
STT_LIMIT = 44 + STT_RATE * 2 * STT_SECONDS
|
||||
STT_TIMEOUT = 120
|
||||
INPUT_SECONDS = 60
|
||||
INPUT_LIMIT = 5 << 20
|
||||
MODEL_ROW = "make orchestrator-install"
|
||||
HEARING_DIR = "stt-"
|
||||
LANGUAGES = ("de", "en")
|
||||
SPECIAL = "[_"
|
||||
HEARING_WINDOW = 60
|
||||
HEARING_RATE = 6
|
||||
HEARING_HOUR = 3600
|
||||
@@ -100,7 +108,15 @@ def voice_of(key: str, pool: list[str], operator: str) -> str:
|
||||
return pool[int(digest(key), 16) % len(pool)]
|
||||
|
||||
|
||||
def wav_seconds(body: bytes) -> float:
|
||||
def wav_seconds(body: bytes, seconds: int) -> float:
|
||||
"""Args:
|
||||
body: the recording.
|
||||
seconds: the longest recording accepted.
|
||||
|
||||
Returns:
|
||||
Its length in seconds once it is 16 kHz mono 16-bit PCM WAV lasting
|
||||
more than 0 and at most seconds; otherwise it raises ToolError.
|
||||
"""
|
||||
try:
|
||||
with wave.open(io.BytesIO(body), "rb") as reader:
|
||||
shape = (reader.getnchannels(), reader.getsampwidth(), reader.getframerate(), reader.getcomptype())
|
||||
@@ -109,8 +125,8 @@ def wav_seconds(body: bytes) -> float:
|
||||
raise ToolError("the recording must be a WAV file") from None
|
||||
if shape != (1, 2, STT_RATE, "NONE"):
|
||||
raise ToolError(f"the recording must be {STT_RATE} Hz mono 16-bit PCM")
|
||||
if not 0 < frames <= STT_RATE * STT_SECONDS:
|
||||
raise ToolError(f"the recording must last more than 0 and at most {STT_SECONDS} seconds")
|
||||
if not 0 < frames <= STT_RATE * seconds:
|
||||
raise ToolError(f"the recording must last more than 0 and at most {seconds} seconds")
|
||||
return frames / STT_RATE
|
||||
|
||||
|
||||
@@ -156,6 +172,27 @@ def heard(output: str) -> str:
|
||||
return " ".join(NOISE.sub(" ", output).split())[:TEXT_LIMIT]
|
||||
|
||||
|
||||
def transcript(path: Path) -> tuple[str, str, float]:
|
||||
"""Args:
|
||||
path: the JSON whisper.cpp writes with --output-json-full.
|
||||
|
||||
Returns:
|
||||
(the language it names, its segment texts joined, the mean
|
||||
probability of its text tokens, 0.0 without any); a file that is
|
||||
missing or not of that shape raises EngineError.
|
||||
"""
|
||||
try:
|
||||
data = json.loads(path.read_bytes().decode("utf-8", errors="replace"))
|
||||
language, segments = data["result"]["language"], data["transcription"]
|
||||
texts = [segment["text"] for segment in segments]
|
||||
sure = [token["p"] for segment in segments for token in segment.get("tokens", []) if not str(token["text"]).startswith(SPECIAL)]
|
||||
except (OSError, ValueError, RecursionError, KeyError, IndexError, TypeError, AttributeError):
|
||||
raise EngineError("whisper.cpp wrote no readable JSON transcript") from None
|
||||
if not isinstance(language, str) or not all(isinstance(text, str) for text in texts) or not all(isinstance(p, int | float) and not isinstance(p, bool) for p in sure):
|
||||
raise EngineError("whisper.cpp wrote no readable JSON transcript")
|
||||
return language, " ".join(texts), sum(sure) / len(sure) if sure else 0.0
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Engines:
|
||||
piper: Path
|
||||
@@ -207,14 +244,15 @@ class Speech:
|
||||
def input_gap(self) -> str:
|
||||
"""Returns:
|
||||
"" when speech input works, otherwise why not, naming the
|
||||
whisper.cpp executable or model with its path; always "" with the
|
||||
speech service.
|
||||
whisper.cpp executable or model with its path and the operator row
|
||||
MODEL_ROW that installs both; always "" with the speech service.
|
||||
"""
|
||||
if self.engines.service.enabled():
|
||||
return ""
|
||||
model = self.engines.whisper_model
|
||||
gap = unreachable(self.engines.whisper, "the whisper.cpp executable", os.X_OK, "executable") or unreachable(model, "the whisper.cpp model", os.R_OK, "readable")
|
||||
return gap or ("" if os.path.isfile(model) else f"the whisper.cpp model {model} is not a file")
|
||||
gap = gap or ("" if os.path.isfile(model) else f"the whisper.cpp model {model} is not a file")
|
||||
return f"{gap}; the operator row {MODEL_ROW} installs it" if gap else ""
|
||||
|
||||
def voice_of(self, key: str) -> str:
|
||||
"""Args:
|
||||
@@ -233,7 +271,7 @@ class Speech:
|
||||
silent, deaf = self.output_gap(), self.input_gap()
|
||||
return {
|
||||
"tts": not silent, "stt": not deaf, "tts_off": silent, "stt_off": deaf, "voices": len(self.voices), "operator": operator, "sample": sample,
|
||||
"limit": SPEECH_LIMIT, "rate": STT_RATE, "seconds": STT_SECONDS,
|
||||
"limit": SPEECH_LIMIT, "rate": STT_RATE, "seconds": STT_SECONDS, "input": INPUT_SECONDS,
|
||||
}
|
||||
|
||||
def synthesize(self, body: bytes, admit: Callable[[str], None]) -> bytes:
|
||||
@@ -333,21 +371,40 @@ class Speech:
|
||||
gap = self.input_gap()
|
||||
if gap:
|
||||
raise Unavailable(f"speech input is off: {gap}")
|
||||
self.admit(wav_seconds(body))
|
||||
self.admit(wav_seconds(body, INPUT_SECONDS))
|
||||
return self.whisper(body)
|
||||
|
||||
def whisper(self, body: bytes) -> dict:
|
||||
"""Returns:
|
||||
{"text"} from the speech service, otherwise {"text", "language"}
|
||||
in one of LANGUAGES.
|
||||
"""
|
||||
if self.engines.service.enabled():
|
||||
with self.hearing:
|
||||
return {"text": heard(self.engines.service.hear(body))}
|
||||
command = [str(self.engines.whisper), "--model", str(self.engines.whisper_model), "--file", "-", "--output-txt", "--language", "auto", "--no-prints", "--suppress-nst"]
|
||||
with self.hearing:
|
||||
try:
|
||||
done = subprocess.run(command, input=body, capture_output=True, timeout=STT_TIMEOUT, env=self.env(), cwd=self.engines.scratch, check=False)
|
||||
except subprocess.TimeoutExpired:
|
||||
raise EngineError(f"whisper.cpp took longer than {STT_TIMEOUT} seconds") from None
|
||||
except OSError as error:
|
||||
raise EngineError(f"whisper.cpp did not start: {error.strerror}") from None
|
||||
with self.hearing, tempfile.TemporaryDirectory(prefix=HEARING_DIR, dir=self.engines.scratch) as folder:
|
||||
language, text, _ = self.listen(body, Path(folder), "auto")
|
||||
if language not in LANGUAGES:
|
||||
language, text, _ = max((self.listen(body, Path(folder), choice) for choice in LANGUAGES), key=lambda found: found[2])
|
||||
return {"text": heard(text), "language": language}
|
||||
|
||||
def listen(self, body: bytes, folder: Path, language: str) -> tuple[str, str, float]:
|
||||
"""Args:
|
||||
body: the recording.
|
||||
folder: the request's own directory, cwd and TMPDIR of the run.
|
||||
language: "auto" to detect it, otherwise the language to hear.
|
||||
|
||||
Returns:
|
||||
transcript() of the JSON whisper.cpp writes into folder.
|
||||
"""
|
||||
target = folder / language
|
||||
command = [str(self.engines.whisper), "--model", str(self.engines.whisper_model), "--file", "-", "--output-json", "--output-json-full", "--output-file", str(target), "--language", language, "--no-prints", "--suppress-nst"]
|
||||
try:
|
||||
done = subprocess.run(command, input=body, capture_output=True, timeout=STT_TIMEOUT, env={**self.env(), "TMPDIR": str(folder)}, cwd=folder, check=False)
|
||||
except subprocess.TimeoutExpired:
|
||||
raise EngineError(f"whisper.cpp took longer than {STT_TIMEOUT} seconds") from None
|
||||
except OSError as error:
|
||||
raise EngineError(f"whisper.cpp did not start: {error.strerror}") from None
|
||||
if done.returncode != 0:
|
||||
raise EngineError(f"whisper.cpp failed with exit code {done.returncode}: {done.stderr.decode('utf-8', errors='replace').strip()[-300:]}")
|
||||
return {"text": heard(done.stdout.decode("utf-8", errors="replace"))}
|
||||
return transcript(target.with_name(f"{language}.json"))
|
||||
@@ -8,7 +8,7 @@ from http import HTTPStatus
|
||||
from urllib.parse import urlsplit
|
||||
|
||||
from .diffs import DIFF_BODY_LIMIT
|
||||
from .engines import STT_LIMIT
|
||||
from .engines import INPUT_LIMIT, STT_LIMIT
|
||||
from .exchange import PLAIN_CSP, Exchange
|
||||
from .rates import rated
|
||||
from .resources import RESOURCES, sampled
|
||||
@@ -294,7 +294,7 @@ class Handler(Exchange):
|
||||
|
||||
def do_POST(self) -> None:
|
||||
path = urlsplit(self.path).path
|
||||
length = self.body_length({UPLOADS: UPLOAD_LIMIT, STT_ROUTE: STT_LIMIT, VOICE_ROUTE: STT_LIMIT, "/mcp": DIFF_BODY_LIMIT, "/api/asks": DIFF_BODY_LIMIT}.get(path, BODY_LIMIT))
|
||||
length = self.body_length({UPLOADS: UPLOAD_LIMIT, STT_ROUTE: INPUT_LIMIT, VOICE_ROUTE: STT_LIMIT, "/mcp": DIFF_BODY_LIMIT, "/api/asks": DIFF_BODY_LIMIT}.get(path, BODY_LIMIT))
|
||||
if length is None:
|
||||
return
|
||||
host = self.host()
|
||||
|
||||
@@ -10,7 +10,8 @@ from collections import Counter
|
||||
from collections.abc import Callable, Iterator
|
||||
from pathlib import Path
|
||||
|
||||
from .engines import Speech, wav_seconds
|
||||
from . import engines
|
||||
from .engines import Speech
|
||||
from .errors import EngineError, Missing, ToolError, Unavailable
|
||||
from .records import write_atomic
|
||||
|
||||
@@ -44,7 +45,7 @@ class VoiceNotes:
|
||||
gap = self.speech.input_gap()
|
||||
if gap:
|
||||
raise Unavailable(f"speech input is off: {gap}")
|
||||
seconds = wav_seconds(body)
|
||||
seconds = engines.wav_seconds(body, engines.STT_SECONDS)
|
||||
with self.lock:
|
||||
if sum(1 for note in self.notes.values() if note["status"] == "pending") >= HEARING_LIMIT:
|
||||
raise Unavailable(f"{HEARING_LIMIT} recordings are already waiting for their transcript")
|
||||
|
||||
@@ -320,7 +320,16 @@ textarea { width: 100%; min-height: 84px; padding: 8px 10px; border: 1px solid v
|
||||
.palette-box input { width: 100%; padding: 14px 16px; border: 0; background: transparent; font-size: var(--fs-lg); outline: none; }
|
||||
.palette-row { display: flex; align-items: center; gap: 6px; padding-right: 10px; border-bottom: 1px solid var(--line); }
|
||||
.palette-row input { flex: 1; min-width: 0; }
|
||||
.mic[aria-pressed="true"] { background: var(--bad-soft); color: var(--bad); }
|
||||
.mic { flex: none; }
|
||||
.mic[aria-pressed="true"] { background: var(--bad-soft); color: var(--bad); animation: mic-pulse 1.2s ease-in-out infinite; }
|
||||
.mic-auto { margin-left: 4px; vertical-align: middle; }
|
||||
textarea + .mic-auto { display: flex; margin: 4px 0 0 auto; }
|
||||
.mic-live { position: fixed; left: 50%; bottom: calc(18px + env(safe-area-inset-bottom, 0px)); z-index: 60; display: flex; align-items: center; gap: 8px; max-width: calc(100vw - 32px); padding: 8px 14px; transform: translateX(-50%); border-radius: 999px; background: var(--bad); color: #fff; font-weight: 650; box-shadow: var(--shadow-lift); }
|
||||
.mic-live[hidden] { display: none; }
|
||||
.mic-live::before { content: ""; flex: none; width: 10px; height: 10px; border-radius: 50%; background: #fff; animation: mic-pulse 1s ease-in-out infinite; }
|
||||
@keyframes mic-pulse { 50% { opacity: .45; } }
|
||||
@media (max-width: 640px) { .mic { min-width: 44px; min-height: 44px; } }
|
||||
@media (prefers-reduced-motion: reduce) { .mic[aria-pressed="true"], .mic-live::before { animation: none; } }
|
||||
.palette-list { max-height: 55vh; margin: 0; padding: 6px; overflow: auto; list-style: none; }
|
||||
.palette-list li { display: flex; align-items: center; gap: 10px; padding: 8px 10px; border-radius: 8px; cursor: pointer; }
|
||||
.palette-list li[aria-selected="true"] { background: var(--accent-soft); color: var(--accent); }
|
||||
@@ -617,6 +626,8 @@ pre.notes { margin: 0; padding: 8px 10px; background: var(--sunken); border-radi
|
||||
|
||||
{% include "page-speech.js" %}
|
||||
|
||||
{% include "page-mic.js" %}
|
||||
|
||||
{% include "page-bubbles.js" %}
|
||||
|
||||
{% include "page-alerts.js" %}
|
||||
|
||||
@@ -94,7 +94,7 @@
|
||||
const classes = choice("to-one", TO_ONE, "post");
|
||||
const form = h("form", {class: "popup-form"}, text, file, classes,
|
||||
h("p", {class: "muted"}, "Post arrives at the end of the turn, Call at once with a read receipt; your passkey signs, verifiable with operator_message."),
|
||||
h("div", {class: "actions"}, h("span", {class: "spacer"}), micButton(text), button));
|
||||
h("div", {class: "actions"}, h("span", {class: "spacer"}), micButton(text, false), button));
|
||||
form.addEventListener("submit", async (event) => {
|
||||
event.preventDefault();
|
||||
if (!text.value.trim()) {
|
||||
|
||||
@@ -39,6 +39,7 @@
|
||||
window.addEventListener("resize", placePopup);
|
||||
window.addEventListener("resize", placeStations);
|
||||
skyPanel();
|
||||
micWatch();
|
||||
radioSpot();
|
||||
radio.element = radioElement();
|
||||
drawSky();
|
||||
|
||||
@@ -78,7 +78,7 @@
|
||||
throw new Error("no recording");
|
||||
}
|
||||
parts.status.textContent = "Converting …";
|
||||
const stored = await send("/api/voice", await wavOf(blob, sound), "audio/wav");
|
||||
const stored = await send("/api/voice", await wavOf(blob, sound, state.speech.seconds), "audio/wav");
|
||||
if (!stored.ok || !stored.data) {
|
||||
throw new Error(`${stored.status} ${stored.text}`.trim());
|
||||
}
|
||||
|
||||
@@ -0,0 +1,185 @@
|
||||
const MIC_HINT = "Voice input: tap to record, tap again to insert the text at the cursor; Escape cancels";
|
||||
const MIC_NODES = `${Voice.FIELDS}, button.mic`;
|
||||
const micButtons = new WeakMap();
|
||||
const micFields = new WeakMap();
|
||||
let micCount = 0;
|
||||
|
||||
function micLive() {
|
||||
let live = document.getElementById("mic-live");
|
||||
if (!live) {
|
||||
live = h("div", {id: "mic-live", class: "mic-live", role: "status", "aria-live": "polite", hidden: true});
|
||||
document.body.append(live);
|
||||
}
|
||||
return live;
|
||||
}
|
||||
|
||||
function micState(button) {
|
||||
const reason = micGap();
|
||||
button.disabled = Boolean(reason);
|
||||
button.title = reason || MIC_HINT;
|
||||
}
|
||||
|
||||
function micLook(button, look) {
|
||||
const recording = look === "recording";
|
||||
button.setAttribute("aria-pressed", String(recording));
|
||||
button.setAttribute("aria-label", recording ? "Stop voice input" : "Voice input");
|
||||
button.replaceChildren(icon(recording ? "bi-stop-fill" : look === "busy" ? "bi-hourglass-split" : "bi-mic"));
|
||||
micLive().hidden = !recording;
|
||||
if (look === "busy") {
|
||||
button.disabled = true;
|
||||
} else {
|
||||
micState(button);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {HTMLInputElement|HTMLTextAreaElement} field the text field the transcript lands in.
|
||||
* @param {boolean} auto whether the shared component placed the button right after the field and keeps it there.
|
||||
* @returns {HTMLButtonElement} the microphone button of the field.
|
||||
*/
|
||||
function micButton(field, auto) {
|
||||
if (!field.id) {
|
||||
micCount += 1;
|
||||
field.id = `mic-field-${micCount}`;
|
||||
}
|
||||
const button = h("button", {type: "button", class: auto ? "btn small ghost icon-only mic mic-auto" : "btn small ghost icon-only mic", "aria-controls": field.id}, icon("bi-mic"));
|
||||
micButtons.set(field, button);
|
||||
micFields.set(button, field);
|
||||
field.dataset.mic = "on";
|
||||
micLook(button, "idle");
|
||||
button.addEventListener("click", () => toggleMic(field, button));
|
||||
return button;
|
||||
}
|
||||
|
||||
function micSweep() {
|
||||
for (const field of document.querySelectorAll(Voice.FIELDS)) {
|
||||
const button = micButtons.get(field);
|
||||
if (field.readOnly || (button && !button.classList.contains("mic-auto"))) {
|
||||
continue;
|
||||
}
|
||||
if (!button) {
|
||||
field.after(micButton(field, true));
|
||||
} else if (button.previousElementSibling !== field) {
|
||||
field.after(button);
|
||||
}
|
||||
}
|
||||
for (const button of document.querySelectorAll("button.mic-auto")) {
|
||||
if (!micFields.get(button).isConnected) {
|
||||
button.remove();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function micTouched(records) {
|
||||
return records.some((record) => [...record.addedNodes, ...record.removedNodes].some((node) => node instanceof Element && (node.matches(MIC_NODES) || Boolean(node.querySelector(MIC_NODES)))));
|
||||
}
|
||||
|
||||
function micWatch() {
|
||||
micSweep();
|
||||
new MutationObserver((records) => micTouched(records) && micSweep()).observe(document.body, {childList: true, subtree: true});
|
||||
window.addEventListener("keydown", (event) => {
|
||||
if (event.key === "Escape" && mic.button && !mic.busy) {
|
||||
event.preventDefault();
|
||||
event.stopImmediatePropagation();
|
||||
stopMic(true);
|
||||
}
|
||||
}, true);
|
||||
}
|
||||
|
||||
function stopMic(cancel) {
|
||||
mic.cancel = cancel;
|
||||
window.clearInterval(mic.timer);
|
||||
if (mic.recorder && mic.recorder.state === "recording") {
|
||||
mic.recorder.stop();
|
||||
}
|
||||
}
|
||||
|
||||
function micTick() {
|
||||
const seconds = Math.floor((Date.now() - mic.started) / 1000);
|
||||
micLive().textContent = `Recording ${seconds} s of at most ${state.speech.input} s: tap the microphone again to insert, Escape cancels`;
|
||||
if (seconds >= state.speech.input) {
|
||||
stopMic(false);
|
||||
}
|
||||
}
|
||||
|
||||
function micDone() {
|
||||
Object.assign(mic, {recorder: null, button: null, field: null, busy: false, cancel: false});
|
||||
}
|
||||
|
||||
async function toggleMic(field, button) {
|
||||
if (mic.button) {
|
||||
if (mic.button === button && mic.recorder) {
|
||||
stopMic(false);
|
||||
}
|
||||
return;
|
||||
}
|
||||
Object.assign(mic, {button, field, cancel: false});
|
||||
let media = null;
|
||||
try {
|
||||
media = await navigator.mediaDevices.getUserMedia({audio: {channelCount: 1, echoCancellation: true, noiseSuppression: true}});
|
||||
} catch (error) {
|
||||
micDone();
|
||||
toast("Microphone not available", [String(error.message || error)], false);
|
||||
return;
|
||||
}
|
||||
if (mic.cancel) {
|
||||
media.getTracks().forEach((track) => track.stop());
|
||||
micDone();
|
||||
return;
|
||||
}
|
||||
const recorder = new MediaRecorder(media);
|
||||
const chunks = [];
|
||||
recorder.addEventListener("dataavailable", (event) => {
|
||||
if (event.data.size) {
|
||||
chunks.push(event.data);
|
||||
}
|
||||
});
|
||||
recorder.addEventListener("stop", () => {
|
||||
media.getTracks().forEach((track) => track.stop());
|
||||
window.clearInterval(mic.timer);
|
||||
const cancelled = mic.cancel;
|
||||
micLook(button, "idle");
|
||||
if (cancelled) {
|
||||
micDone();
|
||||
return;
|
||||
}
|
||||
mic.recorder = null;
|
||||
micTranscribe(new Blob(chunks, {type: recorder.mimeType}), field, button);
|
||||
}, {once: true});
|
||||
Object.assign(mic, {recorder, started: Date.now()});
|
||||
recorder.start();
|
||||
micLook(button, "recording");
|
||||
micTick();
|
||||
mic.timer = window.setInterval(micTick, 250);
|
||||
}
|
||||
|
||||
function micInsert(field, text) {
|
||||
const start = field.selectionStart ?? field.value.length;
|
||||
const spliced = Voice.spliced(field.value, start, field.selectionEnd ?? start, text);
|
||||
field.value = spliced.value;
|
||||
field.focus();
|
||||
field.setSelectionRange(spliced.caret, spliced.caret);
|
||||
field.dispatchEvent(new Event("input", {bubbles: true}));
|
||||
}
|
||||
|
||||
async function micTranscribe(blob, field, button) {
|
||||
const sound = context();
|
||||
if (!sound || !blob.size) {
|
||||
micDone();
|
||||
return;
|
||||
}
|
||||
mic.busy = true;
|
||||
micLook(button, "busy");
|
||||
try {
|
||||
const result = await send("/api/stt", await wavOf(blob, sound, state.speech.input), "audio/wav");
|
||||
if (!result.ok || !result.data) {
|
||||
throw new Error(`${result.status} ${result.text}`.trim());
|
||||
}
|
||||
micInsert(field, result.data.text || "");
|
||||
} catch (error) {
|
||||
toast("Voice input failed", [String(error.message || error)], false);
|
||||
} finally {
|
||||
micDone();
|
||||
micLook(button, "idle");
|
||||
}
|
||||
}
|
||||
@@ -142,7 +142,7 @@
|
||||
runPalette(palette.index);
|
||||
}
|
||||
});
|
||||
box.replaceChildren(h("div", {class: "palette-box", role: "dialog", "aria-label": "Commands"}, h("div", {class: "palette-row"}, palette.input, micButton(palette.input)), palette.list));
|
||||
box.replaceChildren(h("div", {class: "palette-box", role: "dialog", "aria-label": "Commands"}, h("div", {class: "palette-row"}, palette.input, micButton(palette.input, false)), palette.list));
|
||||
box.hidden = false;
|
||||
drawPalette();
|
||||
palette.input.focus();
|
||||
|
||||
@@ -309,84 +309,10 @@
|
||||
: !window.MediaRecorder ? "This browser cannot record" : !state.speech.stt ? `Voice input is off: ${state.speech.stt_off}` : "";
|
||||
}
|
||||
|
||||
function micButton(field) {
|
||||
const button = h("button", {type: "button", class: "btn small ghost icon-only mic", "aria-pressed": "false", ...named("Voice input", "Voice input: hold and speak; the text lands in the field and is not sent")}, icon("bi-mic"));
|
||||
const reason = micGap();
|
||||
if (reason) {
|
||||
button.disabled = true;
|
||||
button.title = `Voice input: ${reason}`;
|
||||
return button;
|
||||
}
|
||||
const press = (event) => {
|
||||
event.preventDefault();
|
||||
record(field, button);
|
||||
};
|
||||
button.addEventListener("pointerdown", press);
|
||||
for (const name of ["pointerup", "pointerleave", "pointercancel"]) {
|
||||
button.addEventListener(name, release);
|
||||
}
|
||||
button.addEventListener("keydown", (event) => {
|
||||
if ((event.key === " " || event.key === "Enter") && !event.repeat) {
|
||||
press(event);
|
||||
}
|
||||
});
|
||||
button.addEventListener("keyup", (event) => {
|
||||
if (event.key === " " || event.key === "Enter") {
|
||||
event.preventDefault();
|
||||
release();
|
||||
}
|
||||
});
|
||||
return button;
|
||||
}
|
||||
|
||||
async function record(field, button) {
|
||||
if (mic.held || mic.recorder || mic.busy) {
|
||||
return;
|
||||
}
|
||||
mic.held = true;
|
||||
let media = null;
|
||||
try {
|
||||
media = await navigator.mediaDevices.getUserMedia({audio: {channelCount: 1, echoCancellation: true, noiseSuppression: true}});
|
||||
} catch (error) {
|
||||
mic.held = false;
|
||||
toast("Microphone not available", [String(error.message || error)], false);
|
||||
return;
|
||||
}
|
||||
if (!mic.held) {
|
||||
media.getTracks().forEach((track) => track.stop());
|
||||
return;
|
||||
}
|
||||
const recorder = new MediaRecorder(media);
|
||||
const chunks = [];
|
||||
recorder.addEventListener("dataavailable", (event) => {
|
||||
if (event.data.size) {
|
||||
chunks.push(event.data);
|
||||
}
|
||||
});
|
||||
recorder.addEventListener("stop", () => {
|
||||
media.getTracks().forEach((track) => track.stop());
|
||||
mic.recorder = null;
|
||||
button.setAttribute("aria-pressed", "false");
|
||||
transcribe(new Blob(chunks, {type: recorder.mimeType}), field, button);
|
||||
}, {once: true});
|
||||
mic.recorder = recorder;
|
||||
recorder.start();
|
||||
button.setAttribute("aria-pressed", "true");
|
||||
mic.timer = window.setTimeout(release, state.speech.seconds * 1000);
|
||||
}
|
||||
|
||||
function release() {
|
||||
mic.held = false;
|
||||
window.clearTimeout(mic.timer);
|
||||
if (mic.recorder && mic.recorder.state === "recording") {
|
||||
mic.recorder.stop();
|
||||
}
|
||||
}
|
||||
|
||||
async function wavOf(blob, sound) {
|
||||
async function wavOf(blob, sound, seconds) {
|
||||
const decoded = await sound.decodeAudioData(await blob.arrayBuffer());
|
||||
const rate = state.speech.rate;
|
||||
const length = Voice.frames(decoded.duration, rate, state.speech.seconds);
|
||||
const length = Voice.frames(decoded.duration, rate, seconds);
|
||||
if (length < rate / 4) {
|
||||
throw new Error("too short: speak for at least a quarter second");
|
||||
}
|
||||
@@ -398,28 +324,3 @@
|
||||
const rendered = await offline.startRendering();
|
||||
return Voice.wav(rendered.getChannelData(0), rate);
|
||||
}
|
||||
|
||||
async function transcribe(blob, field, button) {
|
||||
const sound = context();
|
||||
if (!sound || !blob.size) {
|
||||
return;
|
||||
}
|
||||
mic.busy = true;
|
||||
button.disabled = true;
|
||||
try {
|
||||
const result = await send("/api/stt", await wavOf(blob, sound), "audio/wav");
|
||||
if (!result.ok || !result.data) {
|
||||
throw new Error(`${result.status} ${result.text}`.trim());
|
||||
}
|
||||
if (result.data.text) {
|
||||
field.value = [field.value.trim(), result.data.text].filter(Boolean).join(" ");
|
||||
field.dispatchEvent(new Event("input", {bubbles: true}));
|
||||
}
|
||||
field.focus();
|
||||
} catch (error) {
|
||||
toast("Voice input failed", [String(error.message || error)], false);
|
||||
} finally {
|
||||
mic.busy = false;
|
||||
button.disabled = false;
|
||||
}
|
||||
}
|
||||
@@ -47,7 +47,7 @@
|
||||
const radio = {element: null, url: "", level: 0, timer: 0, playing: false, waiting: false, index: -1, pick: null, sig: "", state: "", short: "", note: "", told: null};
|
||||
const speech = {pending: [], busy: false, warned: 0, current: null, source: null, loading: null, timer: 0, item: null};
|
||||
let radioSet = RadioSet.valid(stored(RadioSet.KEY, null));
|
||||
const mic = {recorder: null, held: false, busy: false, timer: 0};
|
||||
const mic = {recorder: null, button: null, field: null, busy: false, cancel: false, timer: 0, started: 0};
|
||||
const space = {
|
||||
views: {graph: {...state.settings.space.graph}, sky: {...state.settings.space.sky}}, glides: {graph: null, sky: null}, edits: 0, saved: 0, timer: 0, look: false, compass: null,
|
||||
};
|
||||
|
||||
@@ -3,6 +3,7 @@ const Voice = (() => {
|
||||
const RANKS = {mayday: 0, "pan-pan": 1, security: 2};
|
||||
const ALARMS = new Set(["mayday", "pan-pan"]);
|
||||
const ROUTINE = 3;
|
||||
const FIELDS = 'textarea, input:not([type]), input[type="text"], input[type="search"]';
|
||||
|
||||
function toneOf(row, human) {
|
||||
if (ALARMS.has(row.class)) {
|
||||
@@ -59,7 +60,26 @@ const Voice = (() => {
|
||||
return buffer;
|
||||
}
|
||||
|
||||
return {toneOf, loudness, beep, enqueue, frames, wav};
|
||||
/**
|
||||
* @param {string} value the text of the field.
|
||||
* @param {number} start the start of its selection.
|
||||
* @param {number} end the end of its selection.
|
||||
* @param {string} text the transcript.
|
||||
* @returns {{value: string, caret: number}} value with the trimmed transcript in place of the selection, one space on each side where a neighbour is no whitespace, and the caret right after the transcript.
|
||||
*/
|
||||
function spliced(value, start, end, text) {
|
||||
const words = String(text).trim();
|
||||
if (!words) {
|
||||
return {value, caret: end};
|
||||
}
|
||||
const before = value.slice(0, start);
|
||||
const after = value.slice(end);
|
||||
const lead = before && !/\s$/.test(before) ? " " : "";
|
||||
const tail = after && !/^\s/.test(after) ? " " : "";
|
||||
return {value: `${before}${lead}${words}${tail}${after}`, caret: before.length + lead.length + words.length};
|
||||
}
|
||||
|
||||
return {FIELDS, toneOf, loudness, beep, enqueue, frames, wav, spliced};
|
||||
})();
|
||||
if (typeof module === "object" && module.exports) {
|
||||
module.exports = Voice;
|
||||
|
||||
Reference in new issue
Block a user