parent
2f3d96702c
commit
0f32d2a473
7 changed files with 780 additions and 45 deletions
@ -0,0 +1,37 @@ |
||||
// tauri 的 fetch 不保證吃得下 FormData，multipart 自己組成 bytes 最穩。
|
||||
// 這個檔案刻意不 import 任何 tauri 模組，才能在 node 裡直接跑測試。
|
||||
|
||||
/** |
||||
* @param {Record<string, string|undefined>} fields 文字欄位，undefined/空字串會被略過 |
||||
* @param {{filename: string, type: string, bytes: Uint8Array}} file |
||||
* @returns {{boundary: string, body: Uint8Array}} |
||||
*/ |
||||
export function buildMultipart(fields, file) { |
||||
const boundary = `----tipsy${Math.random().toString(16).slice(2)}`; |
||||
const encoder = new TextEncoder(); |
||||
const parts = []; |
||||
|
||||
for (const [name, value] of Object.entries(fields)) { |
||||
if (value === undefined || value === null || value === '') continue; |
||||
parts.push(encoder.encode( |
||||
`--${boundary}\r\nContent-Disposition: form-data; name="${name}"\r\n\r\n${value}\r\n` |
||||
)); |
||||
} |
||||
|
||||
parts.push(encoder.encode( |
||||
`--${boundary}\r\nContent-Disposition: form-data; name="file"; filename="${file.filename}"\r\n` + |
||||
`Content-Type: ${file.type}\r\n\r\n` |
||||
)); |
||||
parts.push(file.bytes); |
||||
parts.push(encoder.encode(`\r\n--${boundary}--\r\n`)); |
||||
|
||||
const size = parts.reduce((total, part) => total + part.byteLength, 0); |
||||
const body = new Uint8Array(size); |
||||
let offset = 0; |
||||
for (const part of parts) { |
||||
body.set(part, offset); |
||||
offset += part.byteLength; |
||||
} |
||||
|
||||
return { boundary, body }; |
||||
} |
||||
@ -0,0 +1,274 @@ |
||||
import { fetch } from '@tauri-apps/plugin-http'; |
||||
import { invoke } from '@tauri-apps/api/core'; |
||||
import { buildMultipart } from './multipart'; |
||||
|
||||
// OpenAI speech-to-text 的傳輸層。兩條路：
|
||||
// - realtime: WebSocket 長連線，邊說邊吐 delta，斷句由 server VAD 決定
|
||||
// - batch: 錄成一段 webm 再整段丟 /v1/audio/transcriptions
|
||||
// 兩者都由 useSpeechInput 包成跟舊的 useSpeechRecognition 同形的介面。
|
||||
|
||||
const REALTIME_URL = 'wss://api.openai.com/v1/realtime?intent=transcription'; |
||||
const SAMPLE_RATE = 24000; |
||||
|
||||
// AudioWorklet 只能用 URL 載入，走 Blob 比丟進 public/ 少一個部署時會忘記的檔案。
|
||||
// AudioContext 開在 24kHz，重採樣交給瀏覽器，這裡只負責 Float32 -> Int16。
|
||||
const PCM_WORKLET_SRC = ` |
||||
class PCMWorklet extends AudioWorkletProcessor { |
||||
process(inputs) { |
||||
const input = inputs[0]?.[0]; |
||||
if (!input) return true; |
||||
|
||||
const pcm = new Int16Array(input.length); |
||||
for (let i = 0; i < input.length; i++) { |
||||
const s = Math.max(-1, Math.min(1, input[i])); |
||||
pcm[i] = s < 0 ? s * 0x8000 : s * 0x7fff; |
||||
} |
||||
this.port.postMessage(pcm.buffer, [pcm.buffer]); |
||||
return true; |
||||
} |
||||
} |
||||
registerProcessor('pcm-worklet', PCMWorklet); |
||||
`;
|
||||
|
||||
async function getOpenAIToken() { |
||||
return invoke('get_env', { name: 'OPENAI_API_KEY' }); |
||||
} |
||||
|
||||
function encodeBase64(buffer) { |
||||
const bytes = new Uint8Array(buffer); |
||||
let binary = ''; |
||||
// 分段避免 String.fromCharCode 參數過多爆堆疊
|
||||
for (let i = 0; i < bytes.length; i += 0x8000) { |
||||
binary += String.fromCharCode(...bytes.subarray(i, i + 0x8000)); |
||||
} |
||||
return btoa(binary); |
||||
} |
||||
|
||||
function sttModel(data) { |
||||
return data?.stt_model?.trim() || 'gpt-4o-transcribe'; |
||||
} |
||||
|
||||
// gpt-live-transcribe 自己決定斷句，帶 turn_detection 會被 400 擋掉。
|
||||
function supportsTurnDetection(model) { |
||||
return model !== 'gpt-live-transcribe'; |
||||
} |
||||
|
||||
// whisper-1 帶 chunking_strategy 會 400："chunking_strategy is not supported with this model"。
|
||||
function supportsChunkingStrategy(model) { |
||||
return model !== 'whisper-1'; |
||||
} |
||||
|
||||
// gpt-4o-transcribe / gpt-live-transcribe 收得下 zh-tw 這種區域碼，
|
||||
// whisper-1 只收 ISO-639-1，給它 zh-tw 會 400，得把後綴削掉靠 prompt 顧繁體。
|
||||
function sttLanguage(data, model) { |
||||
const language = data?.stt_language?.trim() || 'zh-tw'; |
||||
return model === 'whisper-1' ? language.split('-')[0] : language; |
||||
} |
||||
|
||||
function transcriptionSession(data) { |
||||
const model = sttModel(data); |
||||
const input = { |
||||
format: { type: 'audio/pcm', rate: SAMPLE_RATE }, |
||||
transcription: { |
||||
model, |
||||
language: sttLanguage(data, model), |
||||
}, |
||||
noise_reduction: { type: 'near_field' }, |
||||
}; |
||||
|
||||
const prompt = data?.stt_prompt?.trim(); |
||||
if (prompt) input.transcription.prompt = prompt; |
||||
|
||||
if (supportsTurnDetection(model)) { |
||||
input.turn_detection = { |
||||
type: 'server_vad', |
||||
threshold: Number(data?.vad_threshold) || 0.5, |
||||
silence_duration_ms: Number(data?.vad_silence_ms) || 500, |
||||
prefix_padding_ms: 300, |
||||
}; |
||||
} |
||||
|
||||
return { type: 'transcription', audio: { input } }; |
||||
} |
||||
|
||||
async function createEphemeralToken(data) { |
||||
const token = await getOpenAIToken(); |
||||
|
||||
const response = await fetch('https://api.openai.com/v1/realtime/client_secrets', { |
||||
method: 'POST', |
||||
headers: { |
||||
'Content-Type': 'application/json', |
||||
'Authorization': `Bearer ${token}`, |
||||
}, |
||||
body: JSON.stringify({ session: transcriptionSession(data) }), |
||||
}); |
||||
|
||||
if (!response.ok) { |
||||
const text = await response.text(); |
||||
console.error('Error response:', text); |
||||
throw new Error(`HTTP error! status: ${response.status}`); |
||||
} |
||||
|
||||
const result = await response.json(); |
||||
return result.value; |
||||
} |
||||
|
||||
/** |
||||
* 開一條即時轉錄連線：麥克風 -> worklet -> WebSocket -> delta / completed 回呼。 |
||||
* |
||||
* @param {MediaStream} stream getUserMedia 拿到的麥克風串流 |
||||
* @param {object} data 設定檔（stt_model / stt_language / stt_prompt / vad_*） |
||||
* @param {{onSpeechStart, onSpeechStop, onDelta, onCompleted, onError, onOpen}} on |
||||
* @returns {Promise<{close: () => void}>} |
||||
*/ |
||||
export async function connectRealtimeTranscription(stream, data, on = {}) { |
||||
const ek = await createEphemeralToken(data); |
||||
|
||||
// 瀏覽器的 WebSocket 不能自訂 header，金鑰只能搭 subprotocol 送。
|
||||
const ws = new WebSocket(REALTIME_URL, ['realtime', `openai-insecure-api-key.${ek}`]); |
||||
|
||||
const audioContext = new AudioContext({ sampleRate: SAMPLE_RATE }); |
||||
const workletUrl = URL.createObjectURL(new Blob([PCM_WORKLET_SRC], { type: 'application/javascript' })); |
||||
|
||||
let closed = false; |
||||
let source; |
||||
let worklet; |
||||
|
||||
function close() { |
||||
if (closed) return; |
||||
closed = true; |
||||
|
||||
try { worklet?.disconnect(); } catch { /* 已經斷了 */ } |
||||
try { source?.disconnect(); } catch { /* 已經斷了 */ } |
||||
try { audioContext.close(); } catch { /* 已經關了 */ } |
||||
URL.revokeObjectURL(workletUrl); |
||||
if (ws.readyState === WebSocket.OPEN || ws.readyState === WebSocket.CONNECTING) ws.close(); |
||||
} |
||||
|
||||
ws.onopen = async () => { |
||||
console.log('[stt] realtime session open'); |
||||
|
||||
try { |
||||
await audioContext.audioWorklet.addModule(workletUrl); |
||||
if (closed) return; |
||||
|
||||
source = audioContext.createMediaStreamSource(stream); |
||||
worklet = new AudioWorkletNode(audioContext, 'pcm-worklet'); |
||||
worklet.port.onmessage = (event) => { |
||||
if (closed || ws.readyState !== WebSocket.OPEN) return; |
||||
ws.send(JSON.stringify({ |
||||
type: 'input_audio_buffer.append', |
||||
audio: encodeBase64(event.data), |
||||
})); |
||||
}; |
||||
source.connect(worklet); |
||||
// 不接到 destination，避免把麥克風繞回喇叭
|
||||
} catch (error) { |
||||
console.error('[stt] worklet setup failed:', error); |
||||
on.onError?.(error); |
||||
close(); |
||||
return; |
||||
} |
||||
|
||||
on.onOpen?.(); |
||||
}; |
||||
|
||||
ws.onmessage = (event) => { |
||||
let message; |
||||
try { |
||||
message = JSON.parse(event.data); |
||||
} catch { |
||||
return; |
||||
} |
||||
|
||||
switch (message.type) { |
||||
case 'input_audio_buffer.speech_started': |
||||
on.onSpeechStart?.(); |
||||
break; |
||||
// 前提：speech_stopped 一定先於下一段的 speech_started（實測如此）。
|
||||
// 若哪天倒過來，頁面的 refSpeaking 會停在 false，下一個 completed 就會誤起靜音計時器。
|
||||
case 'input_audio_buffer.speech_stopped': |
||||
on.onSpeechStop?.(); |
||||
break; |
||||
case 'conversation.item.input_audio_transcription.delta': |
||||
on.onDelta?.(message.delta ?? ''); |
||||
break; |
||||
case 'conversation.item.input_audio_transcription.completed': |
||||
on.onCompleted?.(message.transcript ?? ''); |
||||
break; |
||||
case 'error': |
||||
console.error('[stt] realtime error:', message.error); |
||||
on.onError?.(new Error(message.error?.message || 'realtime error')); |
||||
break; |
||||
default: |
||||
break; |
||||
} |
||||
}; |
||||
|
||||
ws.onerror = (event) => { |
||||
console.error('[stt] websocket error:', event); |
||||
on.onError?.(new Error('websocket error')); |
||||
}; |
||||
|
||||
ws.onclose = (event) => { |
||||
console.log(`[stt] realtime session closed: ${event.code} ${event.reason}`); |
||||
close(); |
||||
}; |
||||
|
||||
// 把目前還留在 buffer 裡、還沒被 server VAD 斷句的那一段逼出來。
|
||||
// 實測：server_vad 開著也接受手動 commit，commit 到文字落地約 1.2 秒。
|
||||
function commit() { |
||||
if (closed || ws.readyState !== WebSocket.OPEN) return; |
||||
ws.send(JSON.stringify({ type: 'input_audio_buffer.commit' })); |
||||
} |
||||
|
||||
return { close, commit }; |
||||
} |
||||
|
||||
/** |
||||
* 批次轉錄：把 MediaRecorder 錄下來的一整段音訊送去 /v1/audio/transcriptions。 |
||||
* |
||||
* @param {Blob} blob MediaRecorder 的輸出（webm/opus 在支援格式清單內） |
||||
* @param {object} data 設定檔 |
||||
* @param {number} durationMs 錄了多久。用來決定要不要要 server 分段， |
||||
* 不能拿 byteLength 推——webm/opus 約 4 KB/s，推出來差了四倍。 |
||||
* @returns {Promise<string>} 轉錄文字 |
||||
*/ |
||||
export async function transcribeBlob(blob, data, durationMs = 0) { |
||||
const token = await getOpenAIToken(); |
||||
const model = sttModel(data); |
||||
const bytes = new Uint8Array(await blob.arrayBuffer()); |
||||
|
||||
const fields = { |
||||
model, |
||||
language: sttLanguage(data, model), |
||||
prompt: data?.stt_prompt?.trim(), |
||||
response_format: 'json', |
||||
}; |
||||
// 超過 30 秒的音訊官方建議交給 server 端 VAD 分段
|
||||
if (durationMs > 30000 && supportsChunkingStrategy(model)) fields.chunking_strategy = 'auto'; |
||||
|
||||
const { boundary, body } = buildMultipart(fields, { |
||||
filename: 'speech.webm', |
||||
type: blob.type || 'audio/webm', |
||||
bytes, |
||||
}); |
||||
|
||||
const response = await fetch('https://api.openai.com/v1/audio/transcriptions', { |
||||
method: 'POST', |
||||
headers: { |
||||
'Content-Type': `multipart/form-data; boundary=${boundary}`, |
||||
'Authorization': `Bearer ${token}`, |
||||
}, |
||||
body, |
||||
}); |
||||
|
||||
if (!response.ok) { |
||||
const text = await response.text(); |
||||
console.error('Error response:', text); |
||||
throw new Error(`HTTP error! status: ${response.status}`); |
||||
} |
||||
|
||||
const result = await response.json(); |
||||
return result.text ?? ''; |
||||
} |
||||
@ -0,0 +1,384 @@ |
||||
import { useCallback, useEffect, useRef, useState } from 'react'; |
||||
import { connectRealtimeTranscription, transcribeBlob } from './stt'; |
||||
|
||||
// 取代 react-speech-recognition 的 useSpeechRecognition()。 |
||||
// 回傳值刻意跟舊 hook 同形（transcript / finalTranscript / listening / resetTranscript / |
||||
// isMicrophoneAvailable），讓頁面端的時序邏輯一行都不用改： |
||||
// transcript = 已確定的句子 + 當下還在講的片段（給 textarea 與 OSC 即時顯示） |
||||
// finalTranscript = 只有已確定的句子（變動時代表「一句講完了」，頁面用它起靜音計時） |
||||
// |
||||
// mode 由設定檔的 stt_mode 決定： |
||||
// realtime — WebSocket 串流，逐字 delta，斷句由 OpenAI server VAD 判 |
||||
// batch — 本地音量 VAD 判斷靜音後，整段送 /v1/audio/transcriptions |
||||
// |
||||
// 註：batch 模式下使用者說話當下 transcript 是空的，要等整段轉錄回來才會出現； |
||||
// 出現時一樣走 commitSentence 的兩段式，先進 transcript 一拍，再收進 finalTranscript。 |
||||
|
||||
const VOLUME_INTERVAL = 100; |
||||
const RECORDER_TIMESLICE = 1000; // 讓 MediaRecorder 逐步吐資料，而不是全壓在 stop 那一刻 |
||||
|
||||
// 一句話轉錄好之後，隔多久才把它收進 finalTranscript。 |
||||
// 這段空檔是給頁面把文字寫進 textarea 用的，詳見 commitSentence。 |
||||
const COMMIT_DELAY = 50; |
||||
|
||||
export default function useSpeechInput(data, { onSpeechStart, onSpeechStop } = {}) { |
||||
|
||||
const [transcript, setTranscript] = useState(''); |
||||
const [finalTranscript, setFinalTranscript] = useState(''); |
||||
const [listening, setListening] = useState(false); |
||||
const [isMicrophoneAvailable, setIsMicrophoneAvailable] = useState(true); |
||||
|
||||
const refStream = useRef(); |
||||
const refSession = useRef(); // realtime 連線 |
||||
const refFinal = useRef(''); // 已確定句子，避免 setState 非同步造成拼接錯亂 |
||||
const refInterim = useRef(''); // 當下未確定的片段 |
||||
const refStarting = useRef(false); // 擋住 start() 重入 |
||||
const refGeneration = useRef(0); // 每次 stop() 就換代，讓還在 await 的 start() 知道自己過期了 |
||||
|
||||
// batch 模式用 |
||||
const refRecorder = useRef(); |
||||
const refChunks = useRef([]); |
||||
const refAudioContext = useRef(); |
||||
const refAnalyser = useRef(); |
||||
const refVolumes = useRef(); |
||||
const refVolumeInterval = useRef(); |
||||
const refSpeaking = useRef(false); |
||||
const refSilenceSince = useRef(0); |
||||
const refSegmentSince = useRef(0); // 目前這一段錄音是何時開始的 |
||||
const refFlushResolve = useRef(); // 正在等 recorder.onstop 的 resolver |
||||
|
||||
const refPending = useRef(''); // 已轉錄但還沒收進 final 的那一句 |
||||
const refCommitTimer = useRef(); |
||||
|
||||
// 回呼放進 ref，這樣連線中途頁面重繪也不用重建整條 pipeline |
||||
const refCallbacks = useRef({ onSpeechStart, onSpeechStop }); |
||||
refCallbacks.current = { onSpeechStart, onSpeechStop }; |
||||
|
||||
const refData = useRef(data); |
||||
refData.current = data; |
||||
|
||||
const mode = data?.stt_mode?.trim() === 'batch' ? 'batch' : 'realtime'; |
||||
|
||||
function syncTranscript() { |
||||
setFinalTranscript(refFinal.current); |
||||
setTranscript(refFinal.current + refInterim.current); |
||||
} |
||||
|
||||
// 把還懸著的那一句收進 final。不負責 syncTranscript，呼叫端自己決定何時同步。 |
||||
function settlePending() { |
||||
if (refCommitTimer.current) { |
||||
clearTimeout(refCommitTimer.current); |
||||
refCommitTimer.current = undefined; |
||||
} |
||||
|
||||
const sentence = refPending.current; |
||||
refPending.current = ''; |
||||
if (sentence.length === 0) return; |
||||
|
||||
// interim 可能已經長出下一句的開頭，只削掉收進 final 的那一段，別整段清空。 |
||||
// else 那支在現行呼叫圖下走不到（interim 必定以 sentence 開頭），純防禦。 |
||||
refInterim.current = refInterim.current.startsWith(sentence) |
||||
? refInterim.current.slice(sentence.length) |
||||
: ''; |
||||
refFinal.current = refFinal.current.length > 0 |
||||
? `${refFinal.current} ${sentence}` |
||||
: sentence; |
||||
} |
||||
|
||||
// 注意：這裡必須把懸著的 commit 一併取消。否則 cue 切換清空之後的下一拍， |
||||
// 舊句子會被寫回 finalTranscript——頁面會把它當成新的一句話送出去。 |
||||
const resetTranscript = useCallback(() => { |
||||
if (refCommitTimer.current) { |
||||
clearTimeout(refCommitTimer.current); |
||||
refCommitTimer.current = undefined; |
||||
} |
||||
refPending.current = ''; |
||||
|
||||
refFinal.current = ''; |
||||
refInterim.current = ''; |
||||
setFinalTranscript(''); |
||||
setTranscript(''); |
||||
}, []); |
||||
|
||||
// 兩段式：先把整句放進 interim（此時 transcript > finalTranscript，頁面會把它寫進 textarea）， |
||||
// 隔一拍再收進 final（兩者相等，頁面才會走靜音計時器）。 |
||||
// |
||||
// 這是 realtime 天然的 delta -> completed 時序，batch 也得裝一次： |
||||
// flow_free 的 `if(transcript != finalTranscript)` 是 textarea 的唯一入口， |
||||
// 少了第一段，batch 會正常跑完但每一輪都送出空訊息，而且不會報錯。 |
||||
// 順帶一個好處：realtime 的 completed 會把加了標點的修正版本追進 textarea。 |
||||
function commitSentence(text) { |
||||
settlePending(); // 50ms 內連來第二句時，先把第一句收好，不要弄丟 |
||||
|
||||
const sentence = text.trim(); |
||||
|
||||
if (sentence.length === 0) { |
||||
refInterim.current = ''; |
||||
syncTranscript(); |
||||
return; |
||||
} |
||||
|
||||
refPending.current = sentence; |
||||
refInterim.current = sentence; |
||||
syncTranscript(); |
||||
|
||||
refCommitTimer.current = setTimeout(() => { |
||||
settlePending(); |
||||
syncTranscript(); |
||||
}, COMMIT_DELAY); |
||||
} |
||||
|
||||
// --- batch 模式：用音量門檻自己做 VAD --------------------------------- |
||||
|
||||
function averageVolume() { |
||||
const volumes = refVolumes.current; |
||||
refAnalyser.current.getByteFrequencyData(volumes); |
||||
|
||||
let volumeSum = 0; |
||||
for (const volume of volumes) volumeSum += volume; |
||||
return volumeSum / volumes.length / 127.0; |
||||
} |
||||
|
||||
async function flushRecording() { |
||||
const recorder = refRecorder.current; |
||||
if (!recorder || recorder.state === 'inactive') return; |
||||
|
||||
// 要在 startRecorder() 把 refSegmentSince 覆寫之前先算 |
||||
const durationMs = refSegmentSince.current ? Date.now() - refSegmentSince.current : 0; |
||||
|
||||
const chunks = await new Promise((resolve) => { |
||||
// stopBatch() 可能在 onstop 跑到之前就把 recorder 拆了，留個手把讓它把這條 promise 收掉 |
||||
refFlushResolve.current = resolve; |
||||
recorder.onstop = () => { |
||||
const collected = refChunks.current; |
||||
refChunks.current = []; |
||||
refFlushResolve.current = undefined; |
||||
resolve(collected); |
||||
}; |
||||
recorder.stop(); |
||||
}); |
||||
|
||||
// 錄完馬上重開，使用者接著講不會漏掉 |
||||
if (refStream.current) startRecorder(); |
||||
|
||||
if (chunks.length === 0) return; |
||||
|
||||
const generation = refGeneration.current; |
||||
try { |
||||
const text = await transcribeBlob(new Blob(chunks, { type: recorder.mimeType }), refData.current, durationMs); |
||||
if (refGeneration.current !== generation) return; // 轉錄期間被 stop() 掉了 |
||||
commitSentence(text); |
||||
} catch (error) { |
||||
console.error('[stt] batch transcription failed:', error); |
||||
} |
||||
} |
||||
|
||||
function startRecorder() { |
||||
const recorder = new MediaRecorder(refStream.current, { mimeType: 'audio/webm' }); |
||||
refChunks.current = []; |
||||
recorder.ondataavailable = (event) => { |
||||
if (event.data.size > 0) refChunks.current.push(event.data); |
||||
}; |
||||
recorder.start(RECORDER_TIMESLICE); |
||||
refRecorder.current = recorder; |
||||
refSegmentSince.current = Date.now(); |
||||
} |
||||
|
||||
function watchVolume() { |
||||
const threshold = Number(refData.current?.vad_threshold) || 0.5; |
||||
const silenceMs = Number(refData.current?.vad_silence_ms) || 500; |
||||
const level = averageVolume(); |
||||
|
||||
if (level >= threshold) { |
||||
refSilenceSince.current = 0; |
||||
if (!refSpeaking.current) { |
||||
refSpeaking.current = true; |
||||
refCallbacks.current.onSpeechStart?.(); |
||||
} |
||||
|
||||
// 一直講不停的人永遠不會觸發靜音分段，不定期切一刀的話， |
||||
// 講完之前 textarea 一個字也不會出現，硬上限到時整段話都沒了。 |
||||
const maxSegment = Number(refData.current?.stt_max_segment_ms) || 8000; |
||||
if (refSegmentSince.current && Date.now() - refSegmentSince.current >= maxSegment) { |
||||
flushRecording(); |
||||
} |
||||
return; |
||||
} |
||||
|
||||
if (!refSpeaking.current) return; |
||||
|
||||
if (refSilenceSince.current === 0) { |
||||
refSilenceSince.current = Date.now(); |
||||
return; |
||||
} |
||||
|
||||
if (Date.now() - refSilenceSince.current >= silenceMs) { |
||||
refSpeaking.current = false; |
||||
refSilenceSince.current = 0; |
||||
refCallbacks.current.onSpeechStop?.(); |
||||
flushRecording(); |
||||
} |
||||
} |
||||
|
||||
function startBatch() { |
||||
const audioContext = new AudioContext(); |
||||
const source = audioContext.createMediaStreamSource(refStream.current); |
||||
const analyser = audioContext.createAnalyser(); |
||||
analyser.fftSize = 512; |
||||
analyser.minDecibels = -127; |
||||
analyser.maxDecibels = 0; |
||||
analyser.smoothingTimeConstant = 0.4; |
||||
source.connect(analyser); |
||||
|
||||
refAudioContext.current = audioContext; |
||||
refAnalyser.current = analyser; |
||||
refVolumes.current = new Uint8Array(analyser.frequencyBinCount); |
||||
refSpeaking.current = false; |
||||
refSilenceSince.current = 0; |
||||
|
||||
startRecorder(); |
||||
refVolumeInterval.current = setInterval(watchVolume, VOLUME_INTERVAL); |
||||
} |
||||
|
||||
function stopBatch() { |
||||
clearInterval(refVolumeInterval.current); |
||||
refVolumeInterval.current = undefined; |
||||
|
||||
const recorder = refRecorder.current; |
||||
if (recorder && recorder.state !== 'inactive') { |
||||
recorder.onstop = null; |
||||
recorder.stop(); |
||||
} |
||||
refRecorder.current = undefined; |
||||
refChunks.current = []; |
||||
|
||||
refFlushResolve.current?.([]); |
||||
refFlushResolve.current = undefined; |
||||
|
||||
try { refAudioContext.current?.close(); } catch { /* 已經關了 */ } |
||||
refAudioContext.current = undefined; |
||||
refAnalyser.current = undefined; |
||||
refSpeaking.current = false; |
||||
refSegmentSince.current = 0; |
||||
} |
||||
|
||||
// --- 開關 ------------------------------------------------------------- |
||||
|
||||
const stop = useCallback(() => { |
||||
refStarting.current = false; |
||||
refGeneration.current += 1; |
||||
|
||||
if (refCommitTimer.current) { |
||||
clearTimeout(refCommitTimer.current); |
||||
refCommitTimer.current = undefined; |
||||
} |
||||
refPending.current = ''; |
||||
|
||||
refSession.current?.close(); |
||||
refSession.current = undefined; |
||||
stopBatch(); |
||||
|
||||
refStream.current?.getTracks().forEach((track) => track.stop()); |
||||
refStream.current = undefined; |
||||
|
||||
setListening(false); |
||||
}, []); |
||||
|
||||
const start = useCallback(async () => { |
||||
if (refStarting.current || refStream.current) return; |
||||
refStarting.current = true; |
||||
|
||||
// 這輪 start 的世代。中途被 stop() 打斷的話世代就對不上， |
||||
// 後面每個 await 回來都要先核一次，否則會留下沒人關得掉的連線。 |
||||
const generation = refGeneration.current; |
||||
const expired = () => refGeneration.current !== generation; |
||||
|
||||
let stream; |
||||
try { |
||||
stream = await navigator.mediaDevices.getUserMedia({ |
||||
audio: { echoCancellation: true }, |
||||
video: false, |
||||
}); |
||||
setIsMicrophoneAvailable(true); |
||||
} catch (error) { |
||||
console.error('[stt] microphone unavailable:', error); |
||||
setIsMicrophoneAvailable(false); |
||||
if (!expired()) refStarting.current = false; |
||||
return; |
||||
} |
||||
|
||||
if (expired()) { |
||||
// 旗標不是我的了：stop() 已經清過一次，我前面若又有人 start() 那是他的。 |
||||
// 在這裡雞婆清掉，下一個 start() 會穿過守衛，多開一組沒人關得掉的連線。 |
||||
stream.getTracks().forEach((track) => track.stop()); |
||||
return; |
||||
} |
||||
refStream.current = stream; |
||||
|
||||
try { |
||||
if (refData.current?.stt_mode?.trim() === 'batch') { |
||||
startBatch(); |
||||
} else { |
||||
const session = await connectRealtimeTranscription( |
||||
refStream.current, |
||||
refData.current, |
||||
{ |
||||
onSpeechStart: () => refCallbacks.current.onSpeechStart?.(), |
||||
onSpeechStop: () => refCallbacks.current.onSpeechStop?.(), |
||||
onDelta: (delta) => { |
||||
refInterim.current += delta; |
||||
syncTranscript(); |
||||
}, |
||||
onCompleted: (text) => commitSentence(text), |
||||
onError: (error) => console.error('[stt] session error:', error), |
||||
}, |
||||
); |
||||
|
||||
if (expired()) { |
||||
session.close(); |
||||
return; |
||||
} |
||||
refSession.current = session; |
||||
} |
||||
setListening(true); |
||||
} catch (error) { |
||||
console.error('[stt] failed to start listening:', error); |
||||
stop(); |
||||
} finally { |
||||
if (!expired()) refStarting.current = false; |
||||
} |
||||
// eslint-disable-next-line react-hooks/exhaustive-deps |
||||
}, [stop]); |
||||
|
||||
// 把還沒斷句的尾巴逼出來。頁面在硬上限快到時呼叫， |
||||
// 否則 processSpeech 讀到的 textarea 會少掉最後一整段。 |
||||
const flush = useCallback(() => { |
||||
if (refSession.current) { |
||||
refSession.current.commit(); |
||||
return; |
||||
} |
||||
if (refRecorder.current) flushRecording(); |
||||
// eslint-disable-next-line react-hooks/exhaustive-deps |
||||
}, []); |
||||
|
||||
// 換模式（realtime <-> batch）時重啟，不然舊 pipeline 會一直掛著 |
||||
useEffect(() => { |
||||
if (!refStream.current) return; |
||||
stop(); |
||||
start(); |
||||
// eslint-disable-next-line react-hooks/exhaustive-deps |
||||
}, [mode]); |
||||
|
||||
useEffect(() => stop, [stop]); |
||||
|
||||
return { |
||||
transcript, |
||||
finalTranscript, |
||||
listening, |
||||
resetTranscript, |
||||
isMicrophoneAvailable, |
||||
start, |
||||
stop, |
||||
flush, |
||||
}; |
||||
} |
||||
Loading…
Reference in new issue