From 0f32d2a47314a8c87c9377af96e44780f08afa4b Mon Sep 17 00:00:00 2001 From: reng Date: Tue, 22 Sep 2026 17:44:14 +0800 Subject: [PATCH] !!! use OpenAI TTS --- vite/public/default.json | 12 +- vite/src/pages/flow_free.jsx | 105 +++++---- vite/src/pages/settings.jsx | 4 +- vite/src/util/multipart.js | 37 +++ vite/src/util/stt.js | 274 ++++++++++++++++++++++ vite/src/util/system_prompt.js | 9 + vite/src/util/useSpeechInput.jsx | 384 +++++++++++++++++++++++++++++++ 7 files changed, 780 insertions(+), 45 deletions(-) create mode 100644 vite/src/util/multipart.js create mode 100644 vite/src/util/stt.js create mode 100644 vite/src/util/useSpeechInput.jsx diff --git a/vite/public/default.json b/vite/public/default.json index 0c46ddd..3756385 100644 --- a/vite/public/default.json +++ b/vite/public/default.json @@ -5,5 +5,13 @@ "summary_prompt": "請將這段口白的核心情感，轉化為一句不超過 50 字的抽象化描述。這句話應保有距離感，只勾勒出情感的輪廓，同時暗示著一種持續前行、未完待續的狀態，語氣平實。", "speech_idle_time": "4000", "sd_prompt_prefix": "a luminous impression of a {{", - "sd_prompt_suffix": "}}, a whispered Taiwanese memory, an iridescent wash of colors, shimmering light, ethereal, optimistic tone, fading contours, a gentle touch, dreamlike clarity, (bright ambient light), (subtle lens flare), (high key lighting), peaceful and hopeful." -} \ No newline at end of file + "sd_prompt_suffix": "}}, a whispered Taiwanese memory, an iridescent wash of colors, shimmering light, ethereal, optimistic tone, fading contours, a gentle touch, dreamlike clarity, (bright ambient light), (subtle lens flare), (high key lighting), peaceful and hopeful.", + "stt_mode": "realtime", + "stt_model": "gpt-4o-transcribe", + "stt_language": "zh-tw", + "stt_prompt": "以台灣繁體中文輸出，保留口語停頓。", + "vad_threshold": "0.5", + "vad_silence_ms": "500", + "stt_max_segment_ms": "8000", + "speech_flush_lead": "3000" +} diff --git a/vite/src/pages/flow_free.jsx b/vite/src/pages/flow_free.jsx index 23e7e3a..c9785af 100644 --- a/vite/src/pages/flow_free.jsx +++ b/vite/src/pages/flow_free.jsx @@ -1,6 +1,6 @@ import { invoke } from '@tauri-apps/api/core'; import { useEffect, useRef, useState } from "react"; -import SpeechRecognition, { useSpeechRecognition } from 'react-speech-recognition'; +import useSpeechInput from '../util/useSpeechInput'; import { Countdown } from "../comps/timer"; @@ -18,6 +18,7 @@ import { useUser } from "../util/useUser"; const CUELIST_FILE = 'cuelist_1009.json'; const AUDIO_FADE_TIME=3000; // in ms +const BLUR=false; const EmojiType={ phone: '📞', @@ -102,9 +103,14 @@ export function FreeFlow(){ finalTranscript, listening, resetTranscript, - browserSupportsSpeechRecognition, isMicrophoneAvailable, - }=useSpeechRecognition(); + start: startRecognition, + stop: stopRecognition, + flush: flushSpeech, + }=useSpeechInput(data, { + onSpeechStart: handleSpeechStart, + onSpeechStop: handleSpeechStop, + }); function resetData() { @@ -532,6 +538,24 @@ export function FreeFlow(){ sendOsc(OSC_ADDRESS.SPEECH, 'stop'); refSpeaking.current=false; } + + // VAD 說「開始講了」。這個訊號很重要：realtime 模式下講話中途完全沒有文字更新 + // （delta 要等 VAD 斷句並 commit 之後才吐），靠 transcript 變動清靜音計時器是清不到的。 + function handleSpeechStart(){ + sendSpeechStart(); + + if(refPauseTimer.current) clearTimeout(refPauseTimer.current); + } + + // VAD 說「停了」。這才是「一輪講完」的真正訊號。 + function handleSpeechStop(){ + if(refCurrentCue.current?.type!='chat') return; + + sendSpeechEnd(); + + if(chatStatus!=ChatStatus.User) return; + setPauseTimer(); + } function onCueEnd() { refTimer.current?.stop(); // Stop the timer when cue ends @@ -632,11 +656,25 @@ export function FreeFlow(){ let timeleft=refCurrentCue.current?.chatInterval || 0; const endTime=new Date().getTime()+timeleft*1000; + const flushLead=Number(data?.speech_flush_lead) || 2000; + let flushed=false; function tick(){ const now=new Date().getTime(); const timeLeft=endTime-now; + // 硬上限快到了，先把還沒斷句的尾巴逼出來（實測 commit 到文字落地約 1.2 秒）， + // 否則 processSpeech 讀到的 textarea 會少掉最後一整段。 + // + // 注意 lead 要多留 0.9 秒：下面的 timeleft 是上一個 tick 用 Math.floor 算的秒數， + // 所以硬上限實際上比 endTime 早約 0.8-0.9 秒觸發，那段是從 lead 裡扣掉的。 + // 也就是實際可用視窗 = speech_flush_lead - 900ms。 + if(!flushed && timeLeft<=flushLead){ + flushed=true; + console.log('~~~ chat timer near end, flush speech tail'); + flushSpeech(); + } + if(timeleft<=0){ console.log('~~~ chat timer ended, process speech'); clearInterval(refChatTimer.current); @@ -745,18 +783,16 @@ export function FreeFlow(){ function onSpeechEnd(){ - - if(currentCue?.type!='chat') return; // Only process if current cue is user input if(chatStatus!=ChatStatus.User) return; // Only process if chat status is User - sendSpeechEnd(); - + // 人還在講就不要起算。實測連續語音可以 12 秒不斷句， + // 而 speech_idle_time 只有 4 秒；這裡若無條件起算，第一句斷完四秒後 + // 就會在人講到一半把訊息送出去。停止講話的判斷交給 handleSpeechStop。 + if(refSpeaking.current) return; + console.log('~~~ on speech end, start pause timer',data.speech_idle_time); - // refSpeechPaused.current=true; setPauseTimer(); - - } function processSpeech(){ @@ -821,46 +857,30 @@ export function FreeFlow(){ },[finalTranscript]); - function startRecognition() { - - SpeechRecognition.startListening({ continuous: true, language: 'zh-TW' }).then(() => { - console.log("Speech recognition started."); - }).catch(error => { - console.error("Error starting speech recognition:", error); - }); - } - function blurText(text) { + if(!BLUR) return text; + if(!text) return ''; return text.replace(/./g, '*'); } useEffect(()=>{ - if(audioInput && isMicrophoneAvailable) { - - startRecognition(); - - const recognition= SpeechRecognition.getRecognition(); - - recognition.onspeechstart=(e)=>{ - - console.log('Speech start:', e); - sendSpeechStart(); - - }; - // recognition.onspeechend=(e)=>{ - // console.log('Speech end:', e); - // startRecognition(); - // }; - + // 只在會用到 transcript 的 cue 期間連線，Realtime 是按分鐘計費的。 + // 不拆到 chatStatus 這層，否則每輪都要重連，開頭幾個字會被吃掉。 + // 這裡刻意不看 isMicrophoneAvailable：它是 start 失敗後才變 false 的狀態旗標， + // 放進守衛會和 stop() 互相觸發成無限重試。麥克風壞掉就讓 start() 自己失敗、 + // 記一筆 log，下一個 chat cue 再試一次。 + const wantsAudio = audioInput + && (currentCue?.type=='chat' || currentCue?.type=='user_input'); + if(wantsAudio) { + startRecognition(); }else{ - console.log('Stopping speech recognition...'); - SpeechRecognition.stopListening(); + stopRecognition(); } - },[audioInput]); + },[audioInput, currentCue, startRecognition, stopRecognition]); useEffect(()=>{ @@ -1088,7 +1108,7 @@ export function FreeFlow(){
-
+
{history?.map((msg, index) => (
{blurText(msg.content)}
@@ -1098,7 +1118,7 @@ export function FreeFlow(){ {summary &&
{summary}
}
@@ -1109,6 +1129,9 @@ export function FreeFlow(){ setAudioInput(e.target.checked)} /> + {!isMicrophoneAvailable && ( +
mic_unavailable
+ )} setAutoSend(e.target.checked)} /> diff --git a/vite/src/pages/settings.jsx b/vite/src/pages/settings.jsx index e34c875..1a9c788 100644 --- a/vite/src/pages/settings.jsx +++ b/vite/src/pages/settings.jsx @@ -34,8 +34,8 @@ export function Settings(){ {data && Object.entries(data).map(([key, value], index) => (
- {key=="speech_idle_time" ? ( - + {["speech_idle_time","vad_threshold","vad_silence_ms","stt_max_segment_ms","speech_flush_lead"].includes(key) ? ( + ):( )} diff --git a/vite/src/util/multipart.js b/vite/src/util/multipart.js new file mode 100644 index 0000000..f779da6 --- /dev/null +++ b/vite/src/util/multipart.js @@ -0,0 +1,37 @@ +// tauri 的 fetch 不保證吃得下 FormData，multipart 自己組成 bytes 最穩。 +// 這個檔案刻意不 import 任何 tauri 模組，才能在 node 裡直接跑測試。 + +/** + * @param {Record} fields 文字欄位，undefined/空字串會被略過 + * @param {{filename: string, type: string, bytes: Uint8Array}} file + * @returns {{boundary: string, body: Uint8Array}} + */ +export function buildMultipart(fields, file) { + const boundary = `----tipsy${Math.random().toString(16).slice(2)}`; + const encoder = new TextEncoder(); + const parts = []; + + for (const [name, value] of Object.entries(fields)) { + if (value === undefined || value === null || value === '') continue; + parts.push(encoder.encode( + `--${boundary}\r\nContent-Disposition: form-data; name="${name}"\r\n\r\n${value}\r\n` + )); + } + + parts.push(encoder.encode( + `--${boundary}\r\nContent-Disposition: form-data; name="file"; filename="${file.filename}"\r\n` + + `Content-Type: ${file.type}\r\n\r\n` + )); + parts.push(file.bytes); + parts.push(encoder.encode(`\r\n--${boundary}--\r\n`)); + + const size = parts.reduce((total, part) => total + part.byteLength, 0); + const body = new Uint8Array(size); + let offset = 0; + for (const part of parts) { + body.set(part, offset); + offset += part.byteLength; + } + + return { boundary, body }; +} diff --git a/vite/src/util/stt.js b/vite/src/util/stt.js new file mode 100644 index 0000000..85152c2 --- /dev/null +++ b/vite/src/util/stt.js @@ -0,0 +1,274 @@ +import { fetch } from '@tauri-apps/plugin-http'; +import { invoke } from '@tauri-apps/api/core'; +import { buildMultipart } from './multipart'; + +// OpenAI speech-to-text 的傳輸層。兩條路： +// - realtime: WebSocket 長連線，邊說邊吐 delta，斷句由 server VAD 決定 +// - batch: 錄成一段 webm 再整段丟 /v1/audio/transcriptions +// 兩者都由 useSpeechInput 包成跟舊的 useSpeechRecognition 同形的介面。 + +const REALTIME_URL = 'wss://api.openai.com/v1/realtime?intent=transcription'; +const SAMPLE_RATE = 24000; + +// AudioWorklet 只能用 URL 載入，走 Blob 比丟進 public/ 少一個部署時會忘記的檔案。 +// AudioContext 開在 24kHz，重採樣交給瀏覽器，這裡只負責 Float32 -> Int16。 +const PCM_WORKLET_SRC = ` +class PCMWorklet extends AudioWorkletProcessor { + process(inputs) { + const input = inputs[0]?.[0]; + if (!input) return true; + + const pcm = new Int16Array(input.length); + for (let i = 0; i < input.length; i++) { + const s = Math.max(-1, Math.min(1, input[i])); + pcm[i] = s < 0 ? s * 0x8000 : s * 0x7fff; + } + this.port.postMessage(pcm.buffer, [pcm.buffer]); + return true; + } +} +registerProcessor('pcm-worklet', PCMWorklet); +`; + +async function getOpenAIToken() { + return invoke('get_env', { name: 'OPENAI_API_KEY' }); +} + +function encodeBase64(buffer) { + const bytes = new Uint8Array(buffer); + let binary = ''; + // 分段避免 String.fromCharCode 參數過多爆堆疊 + for (let i = 0; i < bytes.length; i += 0x8000) { + binary += String.fromCharCode(...bytes.subarray(i, i + 0x8000)); + } + return btoa(binary); +} + +function sttModel(data) { + return data?.stt_model?.trim() || 'gpt-4o-transcribe'; +} + +// gpt-live-transcribe 自己決定斷句，帶 turn_detection 會被 400 擋掉。 +function supportsTurnDetection(model) { + return model !== 'gpt-live-transcribe'; +} + +// whisper-1 帶 chunking_strategy 會 400："chunking_strategy is not supported with this model"。 +function supportsChunkingStrategy(model) { + return model !== 'whisper-1'; +} + +// gpt-4o-transcribe / gpt-live-transcribe 收得下 zh-tw 這種區域碼， +// whisper-1 只收 ISO-639-1，給它 zh-tw 會 400，得把後綴削掉靠 prompt 顧繁體。 +function sttLanguage(data, model) { + const language = data?.stt_language?.trim() || 'zh-tw'; + return model === 'whisper-1' ? language.split('-')[0] : language; +} + +function transcriptionSession(data) { + const model = sttModel(data); + const input = { + format: { type: 'audio/pcm', rate: SAMPLE_RATE }, + transcription: { + model, + language: sttLanguage(data, model), + }, + noise_reduction: { type: 'near_field' }, + }; + + const prompt = data?.stt_prompt?.trim(); + if (prompt) input.transcription.prompt = prompt; + + if (supportsTurnDetection(model)) { + input.turn_detection = { + type: 'server_vad', + threshold: Number(data?.vad_threshold) || 0.5, + silence_duration_ms: Number(data?.vad_silence_ms) || 500, + prefix_padding_ms: 300, + }; + } + + return { type: 'transcription', audio: { input } }; +} + +async function createEphemeralToken(data) { + const token = await getOpenAIToken(); + + const response = await fetch('https://api.openai.com/v1/realtime/client_secrets', { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + 'Authorization': `Bearer ${token}`, + }, + body: JSON.stringify({ session: transcriptionSession(data) }), + }); + + if (!response.ok) { + const text = await response.text(); + console.error('Error response:', text); + throw new Error(`HTTP error! status: ${response.status}`); + } + + const result = await response.json(); + return result.value; +} + +/** + * 開一條即時轉錄連線：麥克風 -> worklet -> WebSocket -> delta / completed 回呼。 + * + * @param {MediaStream} stream getUserMedia 拿到的麥克風串流 + * @param {object} data 設定檔（stt_model / stt_language / stt_prompt / vad_*） + * @param {{onSpeechStart, onSpeechStop, onDelta, onCompleted, onError, onOpen}} on + * @returns {Promise<{close: () => void}>} + */ +export async function connectRealtimeTranscription(stream, data, on = {}) { + const ek = await createEphemeralToken(data); + + // 瀏覽器的 WebSocket 不能自訂 header，金鑰只能搭 subprotocol 送。 + const ws = new WebSocket(REALTIME_URL, ['realtime', `openai-insecure-api-key.${ek}`]); + + const audioContext = new AudioContext({ sampleRate: SAMPLE_RATE }); + const workletUrl = URL.createObjectURL(new Blob([PCM_WORKLET_SRC], { type: 'application/javascript' })); + + let closed = false; + let source; + let worklet; + + function close() { + if (closed) return; + closed = true; + + try { worklet?.disconnect(); } catch { /* 已經斷了 */ } + try { source?.disconnect(); } catch { /* 已經斷了 */ } + try { audioContext.close(); } catch { /* 已經關了 */ } + URL.revokeObjectURL(workletUrl); + if (ws.readyState === WebSocket.OPEN || ws.readyState === WebSocket.CONNECTING) ws.close(); + } + + ws.onopen = async () => { + console.log('[stt] realtime session open'); + + try { + await audioContext.audioWorklet.addModule(workletUrl); + if (closed) return; + + source = audioContext.createMediaStreamSource(stream); + worklet = new AudioWorkletNode(audioContext, 'pcm-worklet'); + worklet.port.onmessage = (event) => { + if (closed || ws.readyState !== WebSocket.OPEN) return; + ws.send(JSON.stringify({ + type: 'input_audio_buffer.append', + audio: encodeBase64(event.data), + })); + }; + source.connect(worklet); + // 不接到 destination，避免把麥克風繞回喇叭 + } catch (error) { + console.error('[stt] worklet setup failed:', error); + on.onError?.(error); + close(); + return; + } + + on.onOpen?.(); + }; + + ws.onmessage = (event) => { + let message; + try { + message = JSON.parse(event.data); + } catch { + return; + } + + switch (message.type) { + case 'input_audio_buffer.speech_started': + on.onSpeechStart?.(); + break; + // 前提：speech_stopped 一定先於下一段的 speech_started（實測如此）。 + // 若哪天倒過來，頁面的 refSpeaking 會停在 false，下一個 completed 就會誤起靜音計時器。 + case 'input_audio_buffer.speech_stopped': + on.onSpeechStop?.(); + break; + case 'conversation.item.input_audio_transcription.delta': + on.onDelta?.(message.delta ?? ''); + break; + case 'conversation.item.input_audio_transcription.completed': + on.onCompleted?.(message.transcript ?? ''); + break; + case 'error': + console.error('[stt] realtime error:', message.error); + on.onError?.(new Error(message.error?.message || 'realtime error')); + break; + default: + break; + } + }; + + ws.onerror = (event) => { + console.error('[stt] websocket error:', event); + on.onError?.(new Error('websocket error')); + }; + + ws.onclose = (event) => { + console.log(`[stt] realtime session closed: ${event.code} ${event.reason}`); + close(); + }; + + // 把目前還留在 buffer 裡、還沒被 server VAD 斷句的那一段逼出來。 + // 實測：server_vad 開著也接受手動 commit，commit 到文字落地約 1.2 秒。 + function commit() { + if (closed || ws.readyState !== WebSocket.OPEN) return; + ws.send(JSON.stringify({ type: 'input_audio_buffer.commit' })); + } + + return { close, commit }; +} + +/** + * 批次轉錄：把 MediaRecorder 錄下來的一整段音訊送去 /v1/audio/transcriptions。 + * + * @param {Blob} blob MediaRecorder 的輸出（webm/opus 在支援格式清單內） + * @param {object} data 設定檔 + * @param {number} durationMs 錄了多久。用來決定要不要要 server 分段， + * 不能拿 byteLength 推——webm/opus 約 4 KB/s，推出來差了四倍。 + * @returns {Promise} 轉錄文字 + */ +export async function transcribeBlob(blob, data, durationMs = 0) { + const token = await getOpenAIToken(); + const model = sttModel(data); + const bytes = new Uint8Array(await blob.arrayBuffer()); + + const fields = { + model, + language: sttLanguage(data, model), + prompt: data?.stt_prompt?.trim(), + response_format: 'json', + }; + // 超過 30 秒的音訊官方建議交給 server 端 VAD 分段 + if (durationMs > 30000 && supportsChunkingStrategy(model)) fields.chunking_strategy = 'auto'; + + const { boundary, body } = buildMultipart(fields, { + filename: 'speech.webm', + type: blob.type || 'audio/webm', + bytes, + }); + + const response = await fetch('https://api.openai.com/v1/audio/transcriptions', { + method: 'POST', + headers: { + 'Content-Type': `multipart/form-data; boundary=${boundary}`, + 'Authorization': `Bearer ${token}`, + }, + body, + }); + + if (!response.ok) { + const text = await response.text(); + console.error('Error response:', text); + throw new Error(`HTTP error! status: ${response.status}`); + } + + const result = await response.json(); + return result.text ?? ''; +} diff --git a/vite/src/util/system_prompt.js b/vite/src/util/system_prompt.js index cc107c9..d7b1e41 100644 --- a/vite/src/util/system_prompt.js +++ b/vite/src/util/system_prompt.js @@ -11,6 +11,15 @@ export const DefaultParams={ voice_prompt:`Voice Affect: Low, hushed, and suspenseful; convey tension and intrigue.\n\nTone: Deeply serious and mysterious, maintaining an undercurrent of unease throughout.\n\nPacing: Slow, deliberate, pausing slightly after suspenseful moments to heighten drama.\n\nEmotion: Restrained yet intense—voice should subtly tremble or tighten at key suspenseful points.\n\nEmphasis: Highlight sensory descriptions (\"footsteps echoed,\" \"heart hammering,\" \"shadows melting into darkness\") to amplify atmosphere.\n\nPronunciation: Slightly elongated vowels and softened consonants for an eerie, haunting effect.\n\nPauses: Insert meaningful pauses after phrases like \"only shadows melting into darkness,\" and especially before the final line, to enhance suspense dramatically.`, voice:"onyx", + stt_mode:"realtime", // realtime | batch + stt_model:"gpt-4o-transcribe", // 別改 gpt-live-transcribe：它不支援 server VAD，連帶的 OSC /speech start 也會沒了 + stt_language:"zh-tw", + stt_prompt:`以台灣繁體中文輸出，保留口語停頓。`, + vad_threshold:"0.5", // batch 模式的音量門檻 0–1 + vad_silence_ms:"500", // 一句講完的靜音長度 + stt_max_segment_ms:"8000", // 講不停時每隔這麼久強制切一段（batch） + speech_flush_lead:"3000", // chatInterval 剩這麼多時把尾巴逼出來（實際視窗要再扣 0.9 秒） + summary_prompt:`幫我把以下一段話整理成一段文字，以第一人稱視角作為當事人的文字紀念，文字內容 50 字以內：`, } diff --git a/vite/src/util/useSpeechInput.jsx b/vite/src/util/useSpeechInput.jsx new file mode 100644 index 0000000..4d8fef4 --- /dev/null +++ b/vite/src/util/useSpeechInput.jsx @@ -0,0 +1,384 @@ +import { useCallback, useEffect, useRef, useState } from 'react'; +import { connectRealtimeTranscription, transcribeBlob } from './stt'; + +// 取代 react-speech-recognition 的 useSpeechRecognition()。 +// 回傳值刻意跟舊 hook 同形（transcript / finalTranscript / listening / resetTranscript / +// isMicrophoneAvailable），讓頁面端的時序邏輯一行都不用改： +// transcript = 已確定的句子 + 當下還在講的片段（給 textarea 與 OSC 即時顯示） +// finalTranscript = 只有已確定的句子（變動時代表「一句講完了」，頁面用它起靜音計時） +// +// mode 由設定檔的 stt_mode 決定： +// realtime — WebSocket 串流，逐字 delta，斷句由 OpenAI server VAD 判 +// batch — 本地音量 VAD 判斷靜音後，整段送 /v1/audio/transcriptions +// +// 註：batch 模式下使用者說話當下 transcript 是空的，要等整段轉錄回來才會出現； +// 出現時一樣走 commitSentence 的兩段式，先進 transcript 一拍，再收進 finalTranscript。 + +const VOLUME_INTERVAL = 100; +const RECORDER_TIMESLICE = 1000; // 讓 MediaRecorder 逐步吐資料，而不是全壓在 stop 那一刻 + +// 一句話轉錄好之後，隔多久才把它收進 finalTranscript。 +// 這段空檔是給頁面把文字寫進 textarea 用的，詳見 commitSentence。 +const COMMIT_DELAY = 50; + +export default function useSpeechInput(data, { onSpeechStart, onSpeechStop } = {}) { + + const [transcript, setTranscript] = useState(''); + const [finalTranscript, setFinalTranscript] = useState(''); + const [listening, setListening] = useState(false); + const [isMicrophoneAvailable, setIsMicrophoneAvailable] = useState(true); + + const refStream = useRef(); + const refSession = useRef(); // realtime 連線 + const refFinal = useRef(''); // 已確定句子，避免 setState 非同步造成拼接錯亂 + const refInterim = useRef(''); // 當下未確定的片段 + const refStarting = useRef(false); // 擋住 start() 重入 + const refGeneration = useRef(0); // 每次 stop() 就換代，讓還在 await 的 start() 知道自己過期了 + + // batch 模式用 + const refRecorder = useRef(); + const refChunks = useRef([]); + const refAudioContext = useRef(); + const refAnalyser = useRef(); + const refVolumes = useRef(); + const refVolumeInterval = useRef(); + const refSpeaking = useRef(false); + const refSilenceSince = useRef(0); + const refSegmentSince = useRef(0); // 目前這一段錄音是何時開始的 + const refFlushResolve = useRef(); // 正在等 recorder.onstop 的 resolver + + const refPending = useRef(''); // 已轉錄但還沒收進 final 的那一句 + const refCommitTimer = useRef(); + + // 回呼放進 ref，這樣連線中途頁面重繪也不用重建整條 pipeline + const refCallbacks = useRef({ onSpeechStart, onSpeechStop }); + refCallbacks.current = { onSpeechStart, onSpeechStop }; + + const refData = useRef(data); + refData.current = data; + + const mode = data?.stt_mode?.trim() === 'batch' ? 'batch' : 'realtime'; + + function syncTranscript() { + setFinalTranscript(refFinal.current); + setTranscript(refFinal.current + refInterim.current); + } + + // 把還懸著的那一句收進 final。不負責 syncTranscript，呼叫端自己決定何時同步。 + function settlePending() { + if (refCommitTimer.current) { + clearTimeout(refCommitTimer.current); + refCommitTimer.current = undefined; + } + + const sentence = refPending.current; + refPending.current = ''; + if (sentence.length === 0) return; + + // interim 可能已經長出下一句的開頭，只削掉收進 final 的那一段，別整段清空。 + // else 那支在現行呼叫圖下走不到（interim 必定以 sentence 開頭），純防禦。 + refInterim.current = refInterim.current.startsWith(sentence) + ? refInterim.current.slice(sentence.length) + : ''; + refFinal.current = refFinal.current.length > 0 + ? `${refFinal.current} ${sentence}` + : sentence; + } + + // 注意：這裡必須把懸著的 commit 一併取消。否則 cue 切換清空之後的下一拍， + // 舊句子會被寫回 finalTranscript——頁面會把它當成新的一句話送出去。 + const resetTranscript = useCallback(() => { + if (refCommitTimer.current) { + clearTimeout(refCommitTimer.current); + refCommitTimer.current = undefined; + } + refPending.current = ''; + + refFinal.current = ''; + refInterim.current = ''; + setFinalTranscript(''); + setTranscript(''); + }, []); + + // 兩段式：先把整句放進 interim（此時 transcript > finalTranscript，頁面會把它寫進 textarea）， + // 隔一拍再收進 final（兩者相等，頁面才會走靜音計時器）。 + // + // 這是 realtime 天然的 delta -> completed 時序，batch 也得裝一次： + // flow_free 的 `if(transcript != finalTranscript)` 是 textarea 的唯一入口， + // 少了第一段，batch 會正常跑完但每一輪都送出空訊息，而且不會報錯。 + // 順帶一個好處：realtime 的 completed 會把加了標點的修正版本追進 textarea。 + function commitSentence(text) { + settlePending(); // 50ms 內連來第二句時，先把第一句收好，不要弄丟 + + const sentence = text.trim(); + + if (sentence.length === 0) { + refInterim.current = ''; + syncTranscript(); + return; + } + + refPending.current = sentence; + refInterim.current = sentence; + syncTranscript(); + + refCommitTimer.current = setTimeout(() => { + settlePending(); + syncTranscript(); + }, COMMIT_DELAY); + } + + // --- batch 模式：用音量門檻自己做 VAD --------------------------------- + + function averageVolume() { + const volumes = refVolumes.current; + refAnalyser.current.getByteFrequencyData(volumes); + + let volumeSum = 0; + for (const volume of volumes) volumeSum += volume; + return volumeSum / volumes.length / 127.0; + } + + async function flushRecording() { + const recorder = refRecorder.current; + if (!recorder || recorder.state === 'inactive') return; + + // 要在 startRecorder() 把 refSegmentSince 覆寫之前先算 + const durationMs = refSegmentSince.current ? Date.now() - refSegmentSince.current : 0; + + const chunks = await new Promise((resolve) => { + // stopBatch() 可能在 onstop 跑到之前就把 recorder 拆了，留個手把讓它把這條 promise 收掉 + refFlushResolve.current = resolve; + recorder.onstop = () => { + const collected = refChunks.current; + refChunks.current = []; + refFlushResolve.current = undefined; + resolve(collected); + }; + recorder.stop(); + }); + + // 錄完馬上重開，使用者接著講不會漏掉 + if (refStream.current) startRecorder(); + + if (chunks.length === 0) return; + + const generation = refGeneration.current; + try { + const text = await transcribeBlob(new Blob(chunks, { type: recorder.mimeType }), refData.current, durationMs); + if (refGeneration.current !== generation) return; // 轉錄期間被 stop() 掉了 + commitSentence(text); + } catch (error) { + console.error('[stt] batch transcription failed:', error); + } + } + + function startRecorder() { + const recorder = new MediaRecorder(refStream.current, { mimeType: 'audio/webm' }); + refChunks.current = []; + recorder.ondataavailable = (event) => { + if (event.data.size > 0) refChunks.current.push(event.data); + }; + recorder.start(RECORDER_TIMESLICE); + refRecorder.current = recorder; + refSegmentSince.current = Date.now(); + } + + function watchVolume() { + const threshold = Number(refData.current?.vad_threshold) || 0.5; + const silenceMs = Number(refData.current?.vad_silence_ms) || 500; + const level = averageVolume(); + + if (level >= threshold) { + refSilenceSince.current = 0; + if (!refSpeaking.current) { + refSpeaking.current = true; + refCallbacks.current.onSpeechStart?.(); + } + + // 一直講不停的人永遠不會觸發靜音分段，不定期切一刀的話， + // 講完之前 textarea 一個字也不會出現，硬上限到時整段話都沒了。 + const maxSegment = Number(refData.current?.stt_max_segment_ms) || 8000; + if (refSegmentSince.current && Date.now() - refSegmentSince.current >= maxSegment) { + flushRecording(); + } + return; + } + + if (!refSpeaking.current) return; + + if (refSilenceSince.current === 0) { + refSilenceSince.current = Date.now(); + return; + } + + if (Date.now() - refSilenceSince.current >= silenceMs) { + refSpeaking.current = false; + refSilenceSince.current = 0; + refCallbacks.current.onSpeechStop?.(); + flushRecording(); + } + } + + function startBatch() { + const audioContext = new AudioContext(); + const source = audioContext.createMediaStreamSource(refStream.current); + const analyser = audioContext.createAnalyser(); + analyser.fftSize = 512; + analyser.minDecibels = -127; + analyser.maxDecibels = 0; + analyser.smoothingTimeConstant = 0.4; + source.connect(analyser); + + refAudioContext.current = audioContext; + refAnalyser.current = analyser; + refVolumes.current = new Uint8Array(analyser.frequencyBinCount); + refSpeaking.current = false; + refSilenceSince.current = 0; + + startRecorder(); + refVolumeInterval.current = setInterval(watchVolume, VOLUME_INTERVAL); + } + + function stopBatch() { + clearInterval(refVolumeInterval.current); + refVolumeInterval.current = undefined; + + const recorder = refRecorder.current; + if (recorder && recorder.state !== 'inactive') { + recorder.onstop = null; + recorder.stop(); + } + refRecorder.current = undefined; + refChunks.current = []; + + refFlushResolve.current?.([]); + refFlushResolve.current = undefined; + + try { refAudioContext.current?.close(); } catch { /* 已經關了 */ } + refAudioContext.current = undefined; + refAnalyser.current = undefined; + refSpeaking.current = false; + refSegmentSince.current = 0; + } + + // --- 開關 ------------------------------------------------------------- + + const stop = useCallback(() => { + refStarting.current = false; + refGeneration.current += 1; + + if (refCommitTimer.current) { + clearTimeout(refCommitTimer.current); + refCommitTimer.current = undefined; + } + refPending.current = ''; + + refSession.current?.close(); + refSession.current = undefined; + stopBatch(); + + refStream.current?.getTracks().forEach((track) => track.stop()); + refStream.current = undefined; + + setListening(false); + }, []); + + const start = useCallback(async () => { + if (refStarting.current || refStream.current) return; + refStarting.current = true; + + // 這輪 start 的世代。中途被 stop() 打斷的話世代就對不上， + // 後面每個 await 回來都要先核一次，否則會留下沒人關得掉的連線。 + const generation = refGeneration.current; + const expired = () => refGeneration.current !== generation; + + let stream; + try { + stream = await navigator.mediaDevices.getUserMedia({ + audio: { echoCancellation: true }, + video: false, + }); + setIsMicrophoneAvailable(true); + } catch (error) { + console.error('[stt] microphone unavailable:', error); + setIsMicrophoneAvailable(false); + if (!expired()) refStarting.current = false; + return; + } + + if (expired()) { + // 旗標不是我的了：stop() 已經清過一次，我前面若又有人 start() 那是他的。 + // 在這裡雞婆清掉，下一個 start() 會穿過守衛，多開一組沒人關得掉的連線。 + stream.getTracks().forEach((track) => track.stop()); + return; + } + refStream.current = stream; + + try { + if (refData.current?.stt_mode?.trim() === 'batch') { + startBatch(); + } else { + const session = await connectRealtimeTranscription( + refStream.current, + refData.current, + { + onSpeechStart: () => refCallbacks.current.onSpeechStart?.(), + onSpeechStop: () => refCallbacks.current.onSpeechStop?.(), + onDelta: (delta) => { + refInterim.current += delta; + syncTranscript(); + }, + onCompleted: (text) => commitSentence(text), + onError: (error) => console.error('[stt] session error:', error), + }, + ); + + if (expired()) { + session.close(); + return; + } + refSession.current = session; + } + setListening(true); + } catch (error) { + console.error('[stt] failed to start listening:', error); + stop(); + } finally { + if (!expired()) refStarting.current = false; + } + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [stop]); + + // 把還沒斷句的尾巴逼出來。頁面在硬上限快到時呼叫， + // 否則 processSpeech 讀到的 textarea 會少掉最後一整段。 + const flush = useCallback(() => { + if (refSession.current) { + refSession.current.commit(); + return; + } + if (refRecorder.current) flushRecording(); + // eslint-disable-next-line react-hooks/exhaustive-deps + }, []); + + // 換模式（realtime <-> batch）時重啟，不然舊 pipeline 會一直掛著 + useEffect(() => { + if (!refStream.current) return; + stop(); + start(); + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [mode]); + + useEffect(() => stop, [stop]); + + return { + transcript, + finalTranscript, + listening, + resetTranscript, + isMicrophoneAvailable, + start, + stop, + flush, + }; +}