Merge branch 'feature/hub-voice-input' into unify/skills-20261114

# Conflicts:
#	README.md
#	docs/requirements/README.md
This commit is contained in:
Kevin Veen-Birkenbach committed 2026-11-14 22:03:42 +00:00
commit b95e305db0
27 files changed
+861 -149

No files matched your search

+12 -1
View File
@@ -320,7 +320,16 @@ textarea { width: 100%; min-height: 84px; padding: 8px 10px; border: 1px solid v
.palette-box input { width: 100%; padding: 14px 16px; border: 0; background: transparent; font-size: var(--fs-lg); outline: none; }
.palette-row { display: flex; align-items: center; gap: 6px; padding-right: 10px; border-bottom: 1px solid var(--line); }
.palette-row input { flex: 1; min-width: 0; }
.mic[aria-pressed="true"] { background: var(--bad-soft); color: var(--bad); }
.mic { flex: none; }
.mic[aria-pressed="true"] { background: var(--bad-soft); color: var(--bad); animation: mic-pulse 1.2s ease-in-out infinite; }
.mic-auto { margin-left: 4px; vertical-align: middle; }
textarea + .mic-auto { display: flex; margin: 4px 0 0 auto; }
.mic-live { position: fixed; left: 50%; bottom: calc(18px + env(safe-area-inset-bottom, 0px)); z-index: 60; display: flex; align-items: center; gap: 8px; max-width: calc(100vw - 32px); padding: 8px 14px; transform: translateX(-50%); border-radius: 999px; background: var(--bad); color: #fff; font-weight: 650; box-shadow: var(--shadow-lift); }
.mic-live[hidden] { display: none; }
.mic-live::before { content: ""; flex: none; width: 10px; height: 10px; border-radius: 50%; background: #fff; animation: mic-pulse 1s ease-in-out infinite; }
@keyframes mic-pulse { 50% { opacity: .45; } }
@media (max-width: 640px) { .mic { min-width: 44px; min-height: 44px; } }
@media (prefers-reduced-motion: reduce) { .mic[aria-pressed="true"], .mic-live::before { animation: none; } }
.palette-list { max-height: 55vh; margin: 0; padding: 6px; overflow: auto; list-style: none; }
.palette-list li { display: flex; align-items: center; gap: 10px; padding: 8px 10px; border-radius: 8px; cursor: pointer; }
.palette-list li[aria-selected="true"] { background: var(--accent-soft); color: var(--accent); }
@@ -617,6 +626,8 @@ pre.notes { margin: 0; padding: 8px 10px; background: var(--sunken); border-radi
{% include "page-speech.js" %}
{% include "page-mic.js" %}
{% include "page-bubbles.js" %}
{% include "page-alerts.js" %}
+1 -1
View File
@@ -94,7 +94,7 @@
const classes = choice("to-one", TO_ONE, "post");
const form = h("form", {class: "popup-form"}, text, file, classes,
h("p", {class: "muted"}, "Post arrives at the end of the turn, Call at once with a read receipt; your passkey signs, verifiable with operator_message."),
h("div", {class: "actions"}, h("span", {class: "spacer"}), micButton(text), button));
h("div", {class: "actions"}, h("span", {class: "spacer"}), micButton(text, false), button));
form.addEventListener("submit", async (event) => {
event.preventDefault();
if (!text.value.trim()) {
+1
View File
@@ -39,6 +39,7 @@
window.addEventListener("resize", placePopup);
window.addEventListener("resize", placeStations);
skyPanel();
micWatch();
radioSpot();
radio.element = radioElement();
drawSky();
+1 -1
View File
@@ -78,7 +78,7 @@
throw new Error("no recording");
}
parts.status.textContent = "Converting …";
const stored = await send("/api/voice", await wavOf(blob, sound), "audio/wav");
const stored = await send("/api/voice", await wavOf(blob, sound, state.speech.seconds), "audio/wav");
if (!stored.ok || !stored.data) {
throw new Error(`${stored.status} ${stored.text}`.trim());
}
+185
View File
@@ -0,0 +1,185 @@
const MIC_HINT = "Voice input: tap to record, tap again to insert the text at the cursor; Escape cancels";
const MIC_NODES = `${Voice.FIELDS}, button.mic`;
const micButtons = new WeakMap();
const micFields = new WeakMap();
let micCount = 0;
function micLive() {
let live = document.getElementById("mic-live");
if (!live) {
live = h("div", {id: "mic-live", class: "mic-live", role: "status", "aria-live": "polite", hidden: true});
document.body.append(live);
}
return live;
}
function micState(button) {
const reason = micGap();
button.disabled = Boolean(reason);
button.title = reason || MIC_HINT;
}
function micLook(button, look) {
const recording = look === "recording";
button.setAttribute("aria-pressed", String(recording));
button.setAttribute("aria-label", recording ? "Stop voice input" : "Voice input");
button.replaceChildren(icon(recording ? "bi-stop-fill" : look === "busy" ? "bi-hourglass-split" : "bi-mic"));
micLive().hidden = !recording;
if (look === "busy") {
button.disabled = true;
} else {
micState(button);
}
}
/**
* @param {HTMLInputElement|HTMLTextAreaElement} field the text field the transcript lands in.
* @param {boolean} auto whether the shared component placed the button right after the field and keeps it there.
* @returns {HTMLButtonElement} the microphone button of the field.
*/
function micButton(field, auto) {
if (!field.id) {
micCount += 1;
field.id = `mic-field-${micCount}`;
}
const button = h("button", {type: "button", class: auto ? "btn small ghost icon-only mic mic-auto" : "btn small ghost icon-only mic", "aria-controls": field.id}, icon("bi-mic"));
micButtons.set(field, button);
micFields.set(button, field);
field.dataset.mic = "on";
micLook(button, "idle");
button.addEventListener("click", () => toggleMic(field, button));
return button;
}
function micSweep() {
for (const field of document.querySelectorAll(Voice.FIELDS)) {
const button = micButtons.get(field);
if (field.readOnly || (button && !button.classList.contains("mic-auto"))) {
continue;
}
if (!button) {
field.after(micButton(field, true));
} else if (button.previousElementSibling !== field) {
field.after(button);
}
}
for (const button of document.querySelectorAll("button.mic-auto")) {
if (!micFields.get(button).isConnected) {
button.remove();
}
}
}
function micTouched(records) {
return records.some((record) => [...record.addedNodes, ...record.removedNodes].some((node) => node instanceof Element && (node.matches(MIC_NODES) || Boolean(node.querySelector(MIC_NODES)))));
}
function micWatch() {
micSweep();
new MutationObserver((records) => micTouched(records) && micSweep()).observe(document.body, {childList: true, subtree: true});
window.addEventListener("keydown", (event) => {
if (event.key === "Escape" && mic.button && !mic.busy) {
event.preventDefault();
event.stopImmediatePropagation();
stopMic(true);
}
}, true);
}
function stopMic(cancel) {
mic.cancel = cancel;
window.clearInterval(mic.timer);
if (mic.recorder && mic.recorder.state === "recording") {
mic.recorder.stop();
}
}
function micTick() {
const seconds = Math.floor((Date.now() - mic.started) / 1000);
micLive().textContent = `Recording ${seconds} s of at most ${state.speech.input} s: tap the microphone again to insert, Escape cancels`;
if (seconds >= state.speech.input) {
stopMic(false);
}
}
function micDone() {
Object.assign(mic, {recorder: null, button: null, field: null, busy: false, cancel: false});
}
async function toggleMic(field, button) {
if (mic.button) {
if (mic.button === button && mic.recorder) {
stopMic(false);
}
return;
}
Object.assign(mic, {button, field, cancel: false});
let media = null;
try {
media = await navigator.mediaDevices.getUserMedia({audio: {channelCount: 1, echoCancellation: true, noiseSuppression: true}});
} catch (error) {
micDone();
toast("Microphone not available", [String(error.message || error)], false);
return;
}
if (mic.cancel) {
media.getTracks().forEach((track) => track.stop());
micDone();
return;
}
const recorder = new MediaRecorder(media);
const chunks = [];
recorder.addEventListener("dataavailable", (event) => {
if (event.data.size) {
chunks.push(event.data);
}
});
recorder.addEventListener("stop", () => {
media.getTracks().forEach((track) => track.stop());
window.clearInterval(mic.timer);
const cancelled = mic.cancel;
micLook(button, "idle");
if (cancelled) {
micDone();
return;
}
mic.recorder = null;
micTranscribe(new Blob(chunks, {type: recorder.mimeType}), field, button);
}, {once: true});
Object.assign(mic, {recorder, started: Date.now()});
recorder.start();
micLook(button, "recording");
micTick();
mic.timer = window.setInterval(micTick, 250);
}
function micInsert(field, text) {
const start = field.selectionStart ?? field.value.length;
const spliced = Voice.spliced(field.value, start, field.selectionEnd ?? start, text);
field.value = spliced.value;
field.focus();
field.setSelectionRange(spliced.caret, spliced.caret);
field.dispatchEvent(new Event("input", {bubbles: true}));
}
async function micTranscribe(blob, field, button) {
const sound = context();
if (!sound || !blob.size) {
micDone();
return;
}
mic.busy = true;
micLook(button, "busy");
try {
const result = await send("/api/stt", await wavOf(blob, sound, state.speech.input), "audio/wav");
if (!result.ok || !result.data) {
throw new Error(`${result.status} ${result.text}`.trim());
}
micInsert(field, result.data.text || "");
} catch (error) {
toast("Voice input failed", [String(error.message || error)], false);
} finally {
micDone();
micLook(button, "idle");
}
}
+1 -1
View File
@@ -142,7 +142,7 @@
runPalette(palette.index);
}
});
box.replaceChildren(h("div", {class: "palette-box", role: "dialog", "aria-label": "Commands"}, h("div", {class: "palette-row"}, palette.input, micButton(palette.input)), palette.list));
box.replaceChildren(h("div", {class: "palette-box", role: "dialog", "aria-label": "Commands"}, h("div", {class: "palette-row"}, palette.input, micButton(palette.input, false)), palette.list));
box.hidden = false;
drawPalette();
palette.input.focus();
+2 -101
View File
@@ -309,84 +309,10 @@
: !window.MediaRecorder ? "This browser cannot record" : !state.speech.stt ? `Voice input is off: ${state.speech.stt_off}` : "";
}
function micButton(field) {
const button = h("button", {type: "button", class: "btn small ghost icon-only mic", "aria-pressed": "false", ...named("Voice input", "Voice input: hold and speak; the text lands in the field and is not sent")}, icon("bi-mic"));
const reason = micGap();
if (reason) {
button.disabled = true;
button.title = `Voice input: ${reason}`;
return button;
}
const press = (event) => {
event.preventDefault();
record(field, button);
};
button.addEventListener("pointerdown", press);
for (const name of ["pointerup", "pointerleave", "pointercancel"]) {
button.addEventListener(name, release);
}
button.addEventListener("keydown", (event) => {
if ((event.key === " " || event.key === "Enter") && !event.repeat) {
press(event);
}
});
button.addEventListener("keyup", (event) => {
if (event.key === " " || event.key === "Enter") {
event.preventDefault();
release();
}
});
return button;
}
async function record(field, button) {
if (mic.held || mic.recorder || mic.busy) {
return;
}
mic.held = true;
let media = null;
try {
media = await navigator.mediaDevices.getUserMedia({audio: {channelCount: 1, echoCancellation: true, noiseSuppression: true}});
} catch (error) {
mic.held = false;
toast("Microphone not available", [String(error.message || error)], false);
return;
}
if (!mic.held) {
media.getTracks().forEach((track) => track.stop());
return;
}
const recorder = new MediaRecorder(media);
const chunks = [];
recorder.addEventListener("dataavailable", (event) => {
if (event.data.size) {
chunks.push(event.data);
}
});
recorder.addEventListener("stop", () => {
media.getTracks().forEach((track) => track.stop());
mic.recorder = null;
button.setAttribute("aria-pressed", "false");
transcribe(new Blob(chunks, {type: recorder.mimeType}), field, button);
}, {once: true});
mic.recorder = recorder;
recorder.start();
button.setAttribute("aria-pressed", "true");
mic.timer = window.setTimeout(release, state.speech.seconds * 1000);
}
function release() {
mic.held = false;
window.clearTimeout(mic.timer);
if (mic.recorder && mic.recorder.state === "recording") {
mic.recorder.stop();
}
}
async function wavOf(blob, sound) {
async function wavOf(blob, sound, seconds) {
const decoded = await sound.decodeAudioData(await blob.arrayBuffer());
const rate = state.speech.rate;
const length = Voice.frames(decoded.duration, rate, state.speech.seconds);
const length = Voice.frames(decoded.duration, rate, seconds);
if (length < rate / 4) {
throw new Error("too short: speak for at least a quarter second");
}
@@ -398,28 +324,3 @@
const rendered = await offline.startRendering();
return Voice.wav(rendered.getChannelData(0), rate);
}
async function transcribe(blob, field, button) {
const sound = context();
if (!sound || !blob.size) {
return;
}
mic.busy = true;
button.disabled = true;
try {
const result = await send("/api/stt", await wavOf(blob, sound), "audio/wav");
if (!result.ok || !result.data) {
throw new Error(`${result.status} ${result.text}`.trim());
}
if (result.data.text) {
field.value = [field.value.trim(), result.data.text].filter(Boolean).join(" ");
field.dispatchEvent(new Event("input", {bubbles: true}));
}
field.focus();
} catch (error) {
toast("Voice input failed", [String(error.message || error)], false);
} finally {
mic.busy = false;
button.disabled = false;
}
}
+1 -1
View File
@@ -47,7 +47,7 @@
const radio = {element: null, url: "", level: 0, timer: 0, playing: false, waiting: false, index: -1, pick: null, sig: "", state: "", short: "", note: "", told: null};
const speech = {pending: [], busy: false, warned: 0, current: null, source: null, loading: null, timer: 0, item: null};
let radioSet = RadioSet.valid(stored(RadioSet.KEY, null));
const mic = {recorder: null, held: false, busy: false, timer: 0};
const mic = {recorder: null, button: null, field: null, busy: false, cancel: false, timer: 0, started: 0};
const space = {
views: {graph: {...state.settings.space.graph}, sky: {...state.settings.space.sky}}, glides: {graph: null, sky: null}, edits: 0, saved: 0, timer: 0, look: false, compass: null,
};
+21 -1
View File
@@ -3,6 +3,7 @@ const Voice = (() => {
const RANKS = {mayday: 0, "pan-pan": 1, security: 2};
const ALARMS = new Set(["mayday", "pan-pan"]);
const ROUTINE = 3;
const FIELDS = 'textarea, input:not([type]), input[type="text"], input[type="search"]';
function toneOf(row, human) {
if (ALARMS.has(row.class)) {
@@ -59,7 +60,26 @@ const Voice = (() => {
return buffer;
}
return {toneOf, loudness, beep, enqueue, frames, wav};
/**
* @param {string} value the text of the field.
* @param {number} start the start of its selection.
* @param {number} end the end of its selection.
* @param {string} text the transcript.
* @returns {{value: string, caret: number}} value with the trimmed transcript in place of the selection, one space on each side where a neighbour is no whitespace, and the caret right after the transcript.
*/
function spliced(value, start, end, text) {
const words = String(text).trim();
if (!words) {
return {value, caret: end};
}
const before = value.slice(0, start);
const after = value.slice(end);
const lead = before && !/\s$/.test(before) ? " " : "";
const tail = after && !/^\s/.test(after) ? " " : "";
return {value: `${before}${lead}${words}${tail}${after}`, caret: before.length + lead.length + words.length};
}
return {FIELDS, toneOf, loudness, beep, enqueue, frames, wav, spliced};
})();
if (typeof module === "object" && module.exports) {
module.exports = Voice;