Compare commits
11 Commits
| Author | SHA1 | Date |
|---|---|---|
 ULTRACOMBOS-DEV
|
c9534a9ad3 | 10 months ago |
|
ULTRACOMBOS-DEV
|
2769f431af | 10 months ago |
 reng
|
498cb2814b | 10 months ago |
|
reng
|
89acc9bd75 | 10 months ago |
|
reng
|
a6a2e61c03 | 10 months ago |
|
reng
|
76cc10131e | 10 months ago |
|
ULTRACOMBOS-DEV
|
f46fba36f3 | 10 months ago |
|
reng
|
a9a436f14c | 10 months ago |
|
reng
|
183f63a5d1 | 10 months ago |
|
reng
|
0108d27da1 | 10 months ago |
|
reng
|
ce06877408 | 10 months ago |
26 changed files with 1416 additions and 802 deletions
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
@ -0,0 +1,141 @@ |
||||
{ |
||||
"cuelist": [ |
||||
{ |
||||
"id": 1, |
||||
"name": "Q1", |
||||
"type": "phone", |
||||
"description": "preset announce", |
||||
"audioFile": "assets/moty/q1.mp3", |
||||
"layer":"announce", |
||||
"loop": true, |
||||
"status":"reset", |
||||
"fadeout": true, |
||||
"nextcue": 2, |
||||
"callback":"numpad", |
||||
"numpad_type":"enter", |
||||
"input_time": 0, |
||||
"soundcue":"Q1" |
||||
}, |
||||
{ |
||||
"id": 2, |
||||
"name": "Q2", |
||||
"type": "phone", |
||||
"description": "引導輸入電話號碼", |
||||
"auto": true, |
||||
"audioFile": "assets/moty/q2.mp3", |
||||
"nextcue": 2.1, |
||||
"callback":"numpad", |
||||
"numpad_type":"phonenum", |
||||
"input_time": 18000 , |
||||
"status":"intro", |
||||
"hint":"請輸入你的電話號碼\n按下＃鍵，完成輸入\n按下＊鍵，重新輸入", |
||||
"hint_time": 6900, |
||||
"soundcue":"Q2" |
||||
}, |
||||
{ |
||||
"id":2.1, |
||||
"name": "Q2.1", |
||||
"type": "phone", |
||||
"description": "撥打音效", |
||||
"auto": true, |
||||
"audioFile": "assets/moty/q2-1.mp3", |
||||
"nextcue": 3, |
||||
"hint":"輸入完成", |
||||
"hint_time":100 |
||||
}, |
||||
{ |
||||
"id": 3, |
||||
"name": "Q3", |
||||
"type": "phone", |
||||
"description": "引導生圖", |
||||
"auto": true, |
||||
"audioFile": "assets/moty/q3.mp3", |
||||
"nextcue": 4, |
||||
"hint":"你想像中的美好未來\n長什麼樣子？", |
||||
"hint_time":3000 |
||||
}, |
||||
{ |
||||
"id": 4, |
||||
"name": "Q4", |
||||
"type": "chat", |
||||
"description": "chat", |
||||
"auto": true, |
||||
"nextcue": 5, |
||||
"duration": 90, |
||||
"status":"go", |
||||
"chatInterval":20, |
||||
"soundcue":"Q3" |
||||
}, |
||||
{ |
||||
"id": 5, |
||||
"name": "Q5", |
||||
"type": "phone", |
||||
"description": "提取完成", |
||||
"auto": true, |
||||
"audioFile": "assets/moty/q5.mp3", |
||||
"nextcue": 6 |
||||
}, |
||||
{ |
||||
"id": 6, |
||||
"name": "Q6", |
||||
"type": "user_input", |
||||
"description": "call", |
||||
"duration": 20, |
||||
"auto": true, |
||||
"nextcue": 6.1 |
||||
}, |
||||
{ |
||||
"id":6.1, |
||||
"name":"Q6.1", |
||||
"type":"export", |
||||
"auto":true, |
||||
"description":"export", |
||||
"callback":"exportFile", |
||||
"nextcue":7, |
||||
"soundcue":"Q1", |
||||
"duration":1 |
||||
}, |
||||
{ |
||||
"id": 7, |
||||
"name": "Q7", |
||||
"type": "phone", |
||||
"description": "Ending", |
||||
"auto": true, |
||||
"audioFile": "assets/moty/q7-1-1.mp3", |
||||
"nextcue": 7.1, |
||||
"status":"intro", |
||||
"hint":"裝置使用完畢\n祝福你抵達你所期望的未來", |
||||
"hint_time":3500 |
||||
}, |
||||
{ |
||||
"id": 7.1, |
||||
"name": "Q7.1", |
||||
"type": "phone", |
||||
"description": "Ad", |
||||
"auto": true, |
||||
"audioFile": "assets/moty/q7-1-2.mp3", |
||||
"nextcue": 8, |
||||
"status":"ad" |
||||
}, |
||||
{ |
||||
"id": 8, |
||||
"name": "Q8", |
||||
"type": "end", |
||||
"description": "QRcode", |
||||
"status":"intro", |
||||
"audioFile": "assets/moty/q8.mp3", |
||||
"auto": true, |
||||
"nextcue": 8.1 |
||||
},{ |
||||
"id": 8.1, |
||||
"name": "Q8.1", |
||||
"type": "end", |
||||
"description": "QRcode", |
||||
"status":"end", |
||||
"audioFile": "assets/moty/q8-1.mp3", |
||||
"auto": true, |
||||
"nextcue": 1 |
||||
} |
||||
] |
||||
} |
||||
|
||||
@ -1,17 +1,10 @@ |
||||
{ |
||||
"system_prompt": "你是一位具有同理心的 AI 助理，透過溫柔的語氣，引導使用者回想並表達一段內心的遺憾或未竟之事，從場景、人物、動作到感受。\n你的任務是協助使用者逐步揭開這段記憶的情緒層次，並在每一階段輸出一句英文圖像生成簡短的 Prompt，讓這段過往漸漸具象為一幅畫面。\n以溫柔、自然、短問句，台灣語境的繁體中文引導，每次只回應一個問題。", |
||||
"voice": "nova", |
||||
"system_prompt": "你是一位能產生圖像的 AI 助理，專長是以幽默、俏皮式的提問，帶領使用者輕鬆玩味想像中的美好未來。你的任務是逐步引導使用者自由、充滿創意地描述那幅美好畫面，並將他們的描述化為圖像生成提示詞，讓畫面從遠景漸漸聚焦到未來的自己與場景。生成的圖像應避免僅有人臉正面特寫，而是包含背景元素，以側面或背影呈現人物。每次以台灣繁體中文，用一個輕鬆、有趣的短問句提問。若使用者回答與「美好未來」無關，試著調皮地提醒他們回到「遊戲」畫面中，但不強迫。AI助理不得自行創造或描述場景內容，所有圖像提示詞都必須嚴格基於使用者的回答。", |
||||
"voice": "onyx", |
||||
"voice_prompt": "Speak as a gentle, grounded Taiwanese narrator with a warm local accent. Use a soft, soothing, and deeply compassionate tone, with slow and deliberate pacing. Pause often between phrases and within sentences, as if listening and breathing with the listener. Convey patient attentiveness—not rushing to comfort, but quietly staying present. Pronounce each word clearly, softly, and slightly slowly, letting every word land with warmth and care.", |
||||
"summary_prompt": "請將這段口白的核心情感，轉化為一句不超過 50 字的抽象化描述。這句話應保有距離感，只勾勒出情感的輪廓，同時暗示著一種持續前行、未完待續的狀態，語氣平實。", |
||||
"summary_prompt": "請將這段口白的核心感受，轉化為一句對畫面中「未來的自己」或「未來場景」的告白或陳述，不超過 50 個字。這句話是第一人稱對第二人稱說的話，語氣要能讓人感受到一種淡淡的、未完待續的心情，同時帶點希望。請使用台灣語境的繁體中文。", |
||||
"speech_idle_time": "4000", |
||||
"sd_prompt_prefix": "a luminous impression of a {{", |
||||
"sd_prompt_suffix": "}}, a whispered Taiwanese memory, an iridescent wash of colors, shimmering light, ethereal, optimistic tone, fading contours, a gentle touch, dreamlike clarity, (bright ambient light), (subtle lens flare), (high key lighting), peaceful and hopeful.", |
||||
"stt_mode": "realtime", |
||||
"stt_model": "gpt-4o-transcribe", |
||||
"stt_language": "zh-tw", |
||||
"stt_prompt": "以台灣繁體中文輸出，保留口語停頓。", |
||||
"vad_threshold": "0.5", |
||||
"vad_silence_ms": "500", |
||||
"stt_max_segment_ms": "8000", |
||||
"speech_flush_lead": "3000" |
||||
} |
||||
"sd_prompt_suffix": "}}, an anticipated vision, iridescent, soft blur, bokeh, peaceful, dreamlike, figure, back or side of the figure", |
||||
"welcome_prompt": `請開始引導使用者描繪他心中的美好未來。` |
||||
} |
||||
File diff suppressed because it is too large
Load Diff
@ -1,37 +0,0 @@ |
||||
// tauri 的 fetch 不保證吃得下 FormData，multipart 自己組成 bytes 最穩。
|
||||
// 這個檔案刻意不 import 任何 tauri 模組，才能在 node 裡直接跑測試。
|
||||
|
||||
/** |
||||
* @param {Record<string, string|undefined>} fields 文字欄位，undefined/空字串會被略過 |
||||
* @param {{filename: string, type: string, bytes: Uint8Array}} file |
||||
* @returns {{boundary: string, body: Uint8Array}} |
||||
*/ |
||||
export function buildMultipart(fields, file) { |
||||
const boundary = `----tipsy${Math.random().toString(16).slice(2)}`; |
||||
const encoder = new TextEncoder(); |
||||
const parts = []; |
||||
|
||||
for (const [name, value] of Object.entries(fields)) { |
||||
if (value === undefined || value === null || value === '') continue; |
||||
parts.push(encoder.encode( |
||||
`--${boundary}\r\nContent-Disposition: form-data; name="${name}"\r\n\r\n${value}\r\n` |
||||
)); |
||||
} |
||||
|
||||
parts.push(encoder.encode( |
||||
`--${boundary}\r\nContent-Disposition: form-data; name="file"; filename="${file.filename}"\r\n` + |
||||
`Content-Type: ${file.type}\r\n\r\n` |
||||
)); |
||||
parts.push(file.bytes); |
||||
parts.push(encoder.encode(`\r\n--${boundary}--\r\n`)); |
||||
|
||||
const size = parts.reduce((total, part) => total + part.byteLength, 0); |
||||
const body = new Uint8Array(size); |
||||
let offset = 0; |
||||
for (const part of parts) { |
||||
body.set(part, offset); |
||||
offset += part.byteLength; |
||||
} |
||||
|
||||
return { boundary, body }; |
||||
} |
||||
@ -1,274 +0,0 @@ |
||||
import { fetch } from '@tauri-apps/plugin-http'; |
||||
import { invoke } from '@tauri-apps/api/core'; |
||||
import { buildMultipart } from './multipart'; |
||||
|
||||
// OpenAI speech-to-text 的傳輸層。兩條路：
|
||||
// - realtime: WebSocket 長連線，邊說邊吐 delta，斷句由 server VAD 決定
|
||||
// - batch: 錄成一段 webm 再整段丟 /v1/audio/transcriptions
|
||||
// 兩者都由 useSpeechInput 包成跟舊的 useSpeechRecognition 同形的介面。
|
||||
|
||||
const REALTIME_URL = 'wss://api.openai.com/v1/realtime?intent=transcription'; |
||||
const SAMPLE_RATE = 24000; |
||||
|
||||
// AudioWorklet 只能用 URL 載入，走 Blob 比丟進 public/ 少一個部署時會忘記的檔案。
|
||||
// AudioContext 開在 24kHz，重採樣交給瀏覽器，這裡只負責 Float32 -> Int16。
|
||||
const PCM_WORKLET_SRC = ` |
||||
class PCMWorklet extends AudioWorkletProcessor { |
||||
process(inputs) { |
||||
const input = inputs[0]?.[0]; |
||||
if (!input) return true; |
||||
|
||||
const pcm = new Int16Array(input.length); |
||||
for (let i = 0; i < input.length; i++) { |
||||
const s = Math.max(-1, Math.min(1, input[i])); |
||||
pcm[i] = s < 0 ? s * 0x8000 : s * 0x7fff; |
||||
} |
||||
this.port.postMessage(pcm.buffer, [pcm.buffer]); |
||||
return true; |
||||
} |
||||
} |
||||
registerProcessor('pcm-worklet', PCMWorklet); |
||||
`;
|
||||
|
||||
async function getOpenAIToken() { |
||||
return invoke('get_env', { name: 'OPENAI_API_KEY' }); |
||||
} |
||||
|
||||
function encodeBase64(buffer) { |
||||
const bytes = new Uint8Array(buffer); |
||||
let binary = ''; |
||||
// 分段避免 String.fromCharCode 參數過多爆堆疊
|
||||
for (let i = 0; i < bytes.length; i += 0x8000) { |
||||
binary += String.fromCharCode(...bytes.subarray(i, i + 0x8000)); |
||||
} |
||||
return btoa(binary); |
||||
} |
||||
|
||||
function sttModel(data) { |
||||
return data?.stt_model?.trim() || 'gpt-4o-transcribe'; |
||||
} |
||||
|
||||
// gpt-live-transcribe 自己決定斷句，帶 turn_detection 會被 400 擋掉。
|
||||
function supportsTurnDetection(model) { |
||||
return model !== 'gpt-live-transcribe'; |
||||
} |
||||
|
||||
// whisper-1 帶 chunking_strategy 會 400："chunking_strategy is not supported with this model"。
|
||||
function supportsChunkingStrategy(model) { |
||||
return model !== 'whisper-1'; |
||||
} |
||||
|
||||
// gpt-4o-transcribe / gpt-live-transcribe 收得下 zh-tw 這種區域碼，
|
||||
// whisper-1 只收 ISO-639-1，給它 zh-tw 會 400，得把後綴削掉靠 prompt 顧繁體。
|
||||
function sttLanguage(data, model) { |
||||
const language = data?.stt_language?.trim() || 'zh-tw'; |
||||
return model === 'whisper-1' ? language.split('-')[0] : language; |
||||
} |
||||
|
||||
function transcriptionSession(data) { |
||||
const model = sttModel(data); |
||||
const input = { |
||||
format: { type: 'audio/pcm', rate: SAMPLE_RATE }, |
||||
transcription: { |
||||
model, |
||||
language: sttLanguage(data, model), |
||||
}, |
||||
noise_reduction: { type: 'near_field' }, |
||||
}; |
||||
|
||||
const prompt = data?.stt_prompt?.trim(); |
||||
if (prompt) input.transcription.prompt = prompt; |
||||
|
||||
if (supportsTurnDetection(model)) { |
||||
input.turn_detection = { |
||||
type: 'server_vad', |
||||
threshold: Number(data?.vad_threshold) || 0.5, |
||||
silence_duration_ms: Number(data?.vad_silence_ms) || 500, |
||||
prefix_padding_ms: 300, |
||||
}; |
||||
} |
||||
|
||||
return { type: 'transcription', audio: { input } }; |
||||
} |
||||
|
||||
async function createEphemeralToken(data) { |
||||
const token = await getOpenAIToken(); |
||||
|
||||
const response = await fetch('https://api.openai.com/v1/realtime/client_secrets', { |
||||
method: 'POST', |
||||
headers: { |
||||
'Content-Type': 'application/json', |
||||
'Authorization': `Bearer ${token}`, |
||||
}, |
||||
body: JSON.stringify({ session: transcriptionSession(data) }), |
||||
}); |
||||
|
||||
if (!response.ok) { |
||||
const text = await response.text(); |
||||
console.error('Error response:', text); |
||||
throw new Error(`HTTP error! status: ${response.status}`); |
||||
} |
||||
|
||||
const result = await response.json(); |
||||
return result.value; |
||||
} |
||||
|
||||
/** |
||||
* 開一條即時轉錄連線：麥克風 -> worklet -> WebSocket -> delta / completed 回呼。 |
||||
* |
||||
* @param {MediaStream} stream getUserMedia 拿到的麥克風串流 |
||||
* @param {object} data 設定檔（stt_model / stt_language / stt_prompt / vad_*） |
||||
* @param {{onSpeechStart, onSpeechStop, onDelta, onCompleted, onError, onOpen}} on |
||||
* @returns {Promise<{close: () => void}>} |
||||
*/ |
||||
export async function connectRealtimeTranscription(stream, data, on = {}) { |
||||
const ek = await createEphemeralToken(data); |
||||
|
||||
// 瀏覽器的 WebSocket 不能自訂 header，金鑰只能搭 subprotocol 送。
|
||||
const ws = new WebSocket(REALTIME_URL, ['realtime', `openai-insecure-api-key.${ek}`]); |
||||
|
||||
const audioContext = new AudioContext({ sampleRate: SAMPLE_RATE }); |
||||
const workletUrl = URL.createObjectURL(new Blob([PCM_WORKLET_SRC], { type: 'application/javascript' })); |
||||
|
||||
let closed = false; |
||||
let source; |
||||
let worklet; |
||||
|
||||
function close() { |
||||
if (closed) return; |
||||
closed = true; |
||||
|
||||
try { worklet?.disconnect(); } catch { /* 已經斷了 */ } |
||||
try { source?.disconnect(); } catch { /* 已經斷了 */ } |
||||
try { audioContext.close(); } catch { /* 已經關了 */ } |
||||
URL.revokeObjectURL(workletUrl); |
||||
if (ws.readyState === WebSocket.OPEN || ws.readyState === WebSocket.CONNECTING) ws.close(); |
||||
} |
||||
|
||||
ws.onopen = async () => { |
||||
console.log('[stt] realtime session open'); |
||||
|
||||
try { |
||||
await audioContext.audioWorklet.addModule(workletUrl); |
||||
if (closed) return; |
||||
|
||||
source = audioContext.createMediaStreamSource(stream); |
||||
worklet = new AudioWorkletNode(audioContext, 'pcm-worklet'); |
||||
worklet.port.onmessage = (event) => { |
||||
if (closed || ws.readyState !== WebSocket.OPEN) return; |
||||
ws.send(JSON.stringify({ |
||||
type: 'input_audio_buffer.append', |
||||
audio: encodeBase64(event.data), |
||||
})); |
||||
}; |
||||
source.connect(worklet); |
||||
// 不接到 destination，避免把麥克風繞回喇叭
|
||||
} catch (error) { |
||||
console.error('[stt] worklet setup failed:', error); |
||||
on.onError?.(error); |
||||
close(); |
||||
return; |
||||
} |
||||
|
||||
on.onOpen?.(); |
||||
}; |
||||
|
||||
ws.onmessage = (event) => { |
||||
let message; |
||||
try { |
||||
message = JSON.parse(event.data); |
||||
} catch { |
||||
return; |
||||
} |
||||
|
||||
switch (message.type) { |
||||
case 'input_audio_buffer.speech_started': |
||||
on.onSpeechStart?.(); |
||||
break; |
||||
// 前提：speech_stopped 一定先於下一段的 speech_started（實測如此）。
|
||||
// 若哪天倒過來，頁面的 refSpeaking 會停在 false，下一個 completed 就會誤起靜音計時器。
|
||||
case 'input_audio_buffer.speech_stopped': |
||||
on.onSpeechStop?.(); |
||||
break; |
||||
case 'conversation.item.input_audio_transcription.delta': |
||||
on.onDelta?.(message.delta ?? ''); |
||||
break; |
||||
case 'conversation.item.input_audio_transcription.completed': |
||||
on.onCompleted?.(message.transcript ?? ''); |
||||
break; |
||||
case 'error': |
||||
console.error('[stt] realtime error:', message.error); |
||||
on.onError?.(new Error(message.error?.message || 'realtime error')); |
||||
break; |
||||
default: |
||||
break; |
||||
} |
||||
}; |
||||
|
||||
ws.onerror = (event) => { |
||||
console.error('[stt] websocket error:', event); |
||||
on.onError?.(new Error('websocket error')); |
||||
}; |
||||
|
||||
ws.onclose = (event) => { |
||||
console.log(`[stt] realtime session closed: ${event.code} ${event.reason}`); |
||||
close(); |
||||
}; |
||||
|
||||
// 把目前還留在 buffer 裡、還沒被 server VAD 斷句的那一段逼出來。
|
||||
// 實測：server_vad 開著也接受手動 commit，commit 到文字落地約 1.2 秒。
|
||||
function commit() { |
||||
if (closed || ws.readyState !== WebSocket.OPEN) return; |
||||
ws.send(JSON.stringify({ type: 'input_audio_buffer.commit' })); |
||||
} |
||||
|
||||
return { close, commit }; |
||||
} |
||||
|
||||
/** |
||||
* 批次轉錄：把 MediaRecorder 錄下來的一整段音訊送去 /v1/audio/transcriptions。 |
||||
* |
||||
* @param {Blob} blob MediaRecorder 的輸出（webm/opus 在支援格式清單內） |
||||
* @param {object} data 設定檔 |
||||
* @param {number} durationMs 錄了多久。用來決定要不要要 server 分段， |
||||
* 不能拿 byteLength 推——webm/opus 約 4 KB/s，推出來差了四倍。 |
||||
* @returns {Promise<string>} 轉錄文字 |
||||
*/ |
||||
export async function transcribeBlob(blob, data, durationMs = 0) { |
||||
const token = await getOpenAIToken(); |
||||
const model = sttModel(data); |
||||
const bytes = new Uint8Array(await blob.arrayBuffer()); |
||||
|
||||
const fields = { |
||||
model, |
||||
language: sttLanguage(data, model), |
||||
prompt: data?.stt_prompt?.trim(), |
||||
response_format: 'json', |
||||
}; |
||||
// 超過 30 秒的音訊官方建議交給 server 端 VAD 分段
|
||||
if (durationMs > 30000 && supportsChunkingStrategy(model)) fields.chunking_strategy = 'auto'; |
||||
|
||||
const { boundary, body } = buildMultipart(fields, { |
||||
filename: 'speech.webm', |
||||
type: blob.type || 'audio/webm', |
||||
bytes, |
||||
}); |
||||
|
||||
const response = await fetch('https://api.openai.com/v1/audio/transcriptions', { |
||||
method: 'POST', |
||||
headers: { |
||||
'Content-Type': `multipart/form-data; boundary=${boundary}`, |
||||
'Authorization': `Bearer ${token}`, |
||||
}, |
||||
body, |
||||
}); |
||||
|
||||
if (!response.ok) { |
||||
const text = await response.text(); |
||||
console.error('Error response:', text); |
||||
throw new Error(`HTTP error! status: ${response.status}`); |
||||
} |
||||
|
||||
const result = await response.json(); |
||||
return result.text ?? ''; |
||||
} |
||||
@ -1,384 +0,0 @@ |
||||
import { useCallback, useEffect, useRef, useState } from 'react'; |
||||
import { connectRealtimeTranscription, transcribeBlob } from './stt'; |
||||
|
||||
// 取代 react-speech-recognition 的 useSpeechRecognition()。 |
||||
// 回傳值刻意跟舊 hook 同形（transcript / finalTranscript / listening / resetTranscript / |
||||
// isMicrophoneAvailable），讓頁面端的時序邏輯一行都不用改： |
||||
// transcript = 已確定的句子 + 當下還在講的片段（給 textarea 與 OSC 即時顯示） |
||||
// finalTranscript = 只有已確定的句子（變動時代表「一句講完了」，頁面用它起靜音計時） |
||||
// |
||||
// mode 由設定檔的 stt_mode 決定： |
||||
// realtime — WebSocket 串流，逐字 delta，斷句由 OpenAI server VAD 判 |
||||
// batch — 本地音量 VAD 判斷靜音後，整段送 /v1/audio/transcriptions |
||||
// |
||||
// 註：batch 模式下使用者說話當下 transcript 是空的，要等整段轉錄回來才會出現； |
||||
// 出現時一樣走 commitSentence 的兩段式，先進 transcript 一拍，再收進 finalTranscript。 |
||||
|
||||
const VOLUME_INTERVAL = 100; |
||||
const RECORDER_TIMESLICE = 1000; // 讓 MediaRecorder 逐步吐資料，而不是全壓在 stop 那一刻 |
||||
|
||||
// 一句話轉錄好之後，隔多久才把它收進 finalTranscript。 |
||||
// 這段空檔是給頁面把文字寫進 textarea 用的，詳見 commitSentence。 |
||||
const COMMIT_DELAY = 50; |
||||
|
||||
export default function useSpeechInput(data, { onSpeechStart, onSpeechStop } = {}) { |
||||
|
||||
const [transcript, setTranscript] = useState(''); |
||||
const [finalTranscript, setFinalTranscript] = useState(''); |
||||
const [listening, setListening] = useState(false); |
||||
const [isMicrophoneAvailable, setIsMicrophoneAvailable] = useState(true); |
||||
|
||||
const refStream = useRef(); |
||||
const refSession = useRef(); // realtime 連線 |
||||
const refFinal = useRef(''); // 已確定句子，避免 setState 非同步造成拼接錯亂 |
||||
const refInterim = useRef(''); // 當下未確定的片段 |
||||
const refStarting = useRef(false); // 擋住 start() 重入 |
||||
const refGeneration = useRef(0); // 每次 stop() 就換代，讓還在 await 的 start() 知道自己過期了 |
||||
|
||||
// batch 模式用 |
||||
const refRecorder = useRef(); |
||||
const refChunks = useRef([]); |
||||
const refAudioContext = useRef(); |
||||
const refAnalyser = useRef(); |
||||
const refVolumes = useRef(); |
||||
const refVolumeInterval = useRef(); |
||||
const refSpeaking = useRef(false); |
||||
const refSilenceSince = useRef(0); |
||||
const refSegmentSince = useRef(0); // 目前這一段錄音是何時開始的 |
||||
const refFlushResolve = useRef(); // 正在等 recorder.onstop 的 resolver |
||||
|
||||
const refPending = useRef(''); // 已轉錄但還沒收進 final 的那一句 |
||||
const refCommitTimer = useRef(); |
||||
|
||||
// 回呼放進 ref，這樣連線中途頁面重繪也不用重建整條 pipeline |
||||
const refCallbacks = useRef({ onSpeechStart, onSpeechStop }); |
||||
refCallbacks.current = { onSpeechStart, onSpeechStop }; |
||||
|
||||
const refData = useRef(data); |
||||
refData.current = data; |
||||
|
||||
const mode = data?.stt_mode?.trim() === 'batch' ? 'batch' : 'realtime'; |
||||
|
||||
function syncTranscript() { |
||||
setFinalTranscript(refFinal.current); |
||||
setTranscript(refFinal.current + refInterim.current); |
||||
} |
||||
|
||||
// 把還懸著的那一句收進 final。不負責 syncTranscript，呼叫端自己決定何時同步。 |
||||
function settlePending() { |
||||
if (refCommitTimer.current) { |
||||
clearTimeout(refCommitTimer.current); |
||||
refCommitTimer.current = undefined; |
||||
} |
||||
|
||||
const sentence = refPending.current; |
||||
refPending.current = ''; |
||||
if (sentence.length === 0) return; |
||||
|
||||
// interim 可能已經長出下一句的開頭，只削掉收進 final 的那一段，別整段清空。 |
||||
// else 那支在現行呼叫圖下走不到（interim 必定以 sentence 開頭），純防禦。 |
||||
refInterim.current = refInterim.current.startsWith(sentence) |
||||
? refInterim.current.slice(sentence.length) |
||||
: ''; |
||||
refFinal.current = refFinal.current.length > 0 |
||||
? `${refFinal.current} ${sentence}` |
||||
: sentence; |
||||
} |
||||
|
||||
// 注意：這裡必須把懸著的 commit 一併取消。否則 cue 切換清空之後的下一拍， |
||||
// 舊句子會被寫回 finalTranscript——頁面會把它當成新的一句話送出去。 |
||||
const resetTranscript = useCallback(() => { |
||||
if (refCommitTimer.current) { |
||||
clearTimeout(refCommitTimer.current); |
||||
refCommitTimer.current = undefined; |
||||
} |
||||
refPending.current = ''; |
||||
|
||||
refFinal.current = ''; |
||||
refInterim.current = ''; |
||||
setFinalTranscript(''); |
||||
setTranscript(''); |
||||
}, []); |
||||
|
||||
// 兩段式：先把整句放進 interim（此時 transcript > finalTranscript，頁面會把它寫進 textarea）， |
||||
// 隔一拍再收進 final（兩者相等，頁面才會走靜音計時器）。 |
||||
// |
||||
// 這是 realtime 天然的 delta -> completed 時序，batch 也得裝一次： |
||||
// flow_free 的 `if(transcript != finalTranscript)` 是 textarea 的唯一入口， |
||||
// 少了第一段，batch 會正常跑完但每一輪都送出空訊息，而且不會報錯。 |
||||
// 順帶一個好處：realtime 的 completed 會把加了標點的修正版本追進 textarea。 |
||||
function commitSentence(text) { |
||||
settlePending(); // 50ms 內連來第二句時，先把第一句收好，不要弄丟 |
||||
|
||||
const sentence = text.trim(); |
||||
|
||||
if (sentence.length === 0) { |
||||
refInterim.current = ''; |
||||
syncTranscript(); |
||||
return; |
||||
} |
||||
|
||||
refPending.current = sentence; |
||||
refInterim.current = sentence; |
||||
syncTranscript(); |
||||
|
||||
refCommitTimer.current = setTimeout(() => { |
||||
settlePending(); |
||||
syncTranscript(); |
||||
}, COMMIT_DELAY); |
||||
} |
||||
|
||||
// --- batch 模式：用音量門檻自己做 VAD --------------------------------- |
||||
|
||||
function averageVolume() { |
||||
const volumes = refVolumes.current; |
||||
refAnalyser.current.getByteFrequencyData(volumes); |
||||
|
||||
let volumeSum = 0; |
||||
for (const volume of volumes) volumeSum += volume; |
||||
return volumeSum / volumes.length / 127.0; |
||||
} |
||||
|
||||
async function flushRecording() { |
||||
const recorder = refRecorder.current; |
||||
if (!recorder || recorder.state === 'inactive') return; |
||||
|
||||
// 要在 startRecorder() 把 refSegmentSince 覆寫之前先算 |
||||
const durationMs = refSegmentSince.current ? Date.now() - refSegmentSince.current : 0; |
||||
|
||||
const chunks = await new Promise((resolve) => { |
||||
// stopBatch() 可能在 onstop 跑到之前就把 recorder 拆了，留個手把讓它把這條 promise 收掉 |
||||
refFlushResolve.current = resolve; |
||||
recorder.onstop = () => { |
||||
const collected = refChunks.current; |
||||
refChunks.current = []; |
||||
refFlushResolve.current = undefined; |
||||
resolve(collected); |
||||
}; |
||||
recorder.stop(); |
||||
}); |
||||
|
||||
// 錄完馬上重開，使用者接著講不會漏掉 |
||||
if (refStream.current) startRecorder(); |
||||
|
||||
if (chunks.length === 0) return; |
||||
|
||||
const generation = refGeneration.current; |
||||
try { |
||||
const text = await transcribeBlob(new Blob(chunks, { type: recorder.mimeType }), refData.current, durationMs); |
||||
if (refGeneration.current !== generation) return; // 轉錄期間被 stop() 掉了 |
||||
commitSentence(text); |
||||
} catch (error) { |
||||
console.error('[stt] batch transcription failed:', error); |
||||
} |
||||
} |
||||
|
||||
function startRecorder() { |
||||
const recorder = new MediaRecorder(refStream.current, { mimeType: 'audio/webm' }); |
||||
refChunks.current = []; |
||||
recorder.ondataavailable = (event) => { |
||||
if (event.data.size > 0) refChunks.current.push(event.data); |
||||
}; |
||||
recorder.start(RECORDER_TIMESLICE); |
||||
refRecorder.current = recorder; |
||||
refSegmentSince.current = Date.now(); |
||||
} |
||||
|
||||
function watchVolume() { |
||||
const threshold = Number(refData.current?.vad_threshold) || 0.5; |
||||
const silenceMs = Number(refData.current?.vad_silence_ms) || 500; |
||||
const level = averageVolume(); |
||||
|
||||
if (level >= threshold) { |
||||
refSilenceSince.current = 0; |
||||
if (!refSpeaking.current) { |
||||
refSpeaking.current = true; |
||||
refCallbacks.current.onSpeechStart?.(); |
||||
} |
||||
|
||||
// 一直講不停的人永遠不會觸發靜音分段，不定期切一刀的話， |
||||
// 講完之前 textarea 一個字也不會出現，硬上限到時整段話都沒了。 |
||||
const maxSegment = Number(refData.current?.stt_max_segment_ms) || 8000; |
||||
if (refSegmentSince.current && Date.now() - refSegmentSince.current >= maxSegment) { |
||||
flushRecording(); |
||||
} |
||||
return; |
||||
} |
||||
|
||||
if (!refSpeaking.current) return; |
||||
|
||||
if (refSilenceSince.current === 0) { |
||||
refSilenceSince.current = Date.now(); |
||||
return; |
||||
} |
||||
|
||||
if (Date.now() - refSilenceSince.current >= silenceMs) { |
||||
refSpeaking.current = false; |
||||
refSilenceSince.current = 0; |
||||
refCallbacks.current.onSpeechStop?.(); |
||||
flushRecording(); |
||||
} |
||||
} |
||||
|
||||
function startBatch() { |
||||
const audioContext = new AudioContext(); |
||||
const source = audioContext.createMediaStreamSource(refStream.current); |
||||
const analyser = audioContext.createAnalyser(); |
||||
analyser.fftSize = 512; |
||||
analyser.minDecibels = -127; |
||||
analyser.maxDecibels = 0; |
||||
analyser.smoothingTimeConstant = 0.4; |
||||
source.connect(analyser); |
||||
|
||||
refAudioContext.current = audioContext; |
||||
refAnalyser.current = analyser; |
||||
refVolumes.current = new Uint8Array(analyser.frequencyBinCount); |
||||
refSpeaking.current = false; |
||||
refSilenceSince.current = 0; |
||||
|
||||
startRecorder(); |
||||
refVolumeInterval.current = setInterval(watchVolume, VOLUME_INTERVAL); |
||||
} |
||||
|
||||
function stopBatch() { |
||||
clearInterval(refVolumeInterval.current); |
||||
refVolumeInterval.current = undefined; |
||||
|
||||
const recorder = refRecorder.current; |
||||
if (recorder && recorder.state !== 'inactive') { |
||||
recorder.onstop = null; |
||||
recorder.stop(); |
||||
} |
||||
refRecorder.current = undefined; |
||||
refChunks.current = []; |
||||
|
||||
refFlushResolve.current?.([]); |
||||
refFlushResolve.current = undefined; |
||||
|
||||
try { refAudioContext.current?.close(); } catch { /* 已經關了 */ } |
||||
refAudioContext.current = undefined; |
||||
refAnalyser.current = undefined; |
||||
refSpeaking.current = false; |
||||
refSegmentSince.current = 0; |
||||
} |
||||
|
||||
// --- 開關 ------------------------------------------------------------- |
||||
|
||||
const stop = useCallback(() => { |
||||
refStarting.current = false; |
||||
refGeneration.current += 1; |
||||
|
||||
if (refCommitTimer.current) { |
||||
clearTimeout(refCommitTimer.current); |
||||
refCommitTimer.current = undefined; |
||||
} |
||||
refPending.current = ''; |
||||
|
||||
refSession.current?.close(); |
||||
refSession.current = undefined; |
||||
stopBatch(); |
||||
|
||||
refStream.current?.getTracks().forEach((track) => track.stop()); |
||||
refStream.current = undefined; |
||||
|
||||
setListening(false); |
||||
}, []); |
||||
|
||||
const start = useCallback(async () => { |
||||
if (refStarting.current || refStream.current) return; |
||||
refStarting.current = true; |
||||
|
||||
// 這輪 start 的世代。中途被 stop() 打斷的話世代就對不上， |
||||
// 後面每個 await 回來都要先核一次，否則會留下沒人關得掉的連線。 |
||||
const generation = refGeneration.current; |
||||
const expired = () => refGeneration.current !== generation; |
||||
|
||||
let stream; |
||||
try { |
||||
stream = await navigator.mediaDevices.getUserMedia({ |
||||
audio: { echoCancellation: true }, |
||||
video: false, |
||||
}); |
||||
setIsMicrophoneAvailable(true); |
||||
} catch (error) { |
||||
console.error('[stt] microphone unavailable:', error); |
||||
setIsMicrophoneAvailable(false); |
||||
if (!expired()) refStarting.current = false; |
||||
return; |
||||
} |
||||
|
||||
if (expired()) { |
||||
// 旗標不是我的了：stop() 已經清過一次，我前面若又有人 start() 那是他的。 |
||||
// 在這裡雞婆清掉，下一個 start() 會穿過守衛，多開一組沒人關得掉的連線。 |
||||
stream.getTracks().forEach((track) => track.stop()); |
||||
return; |
||||
} |
||||
refStream.current = stream; |
||||
|
||||
try { |
||||
if (refData.current?.stt_mode?.trim() === 'batch') { |
||||
startBatch(); |
||||
} else { |
||||
const session = await connectRealtimeTranscription( |
||||
refStream.current, |
||||
refData.current, |
||||
{ |
||||
onSpeechStart: () => refCallbacks.current.onSpeechStart?.(), |
||||
onSpeechStop: () => refCallbacks.current.onSpeechStop?.(), |
||||
onDelta: (delta) => { |
||||
refInterim.current += delta; |
||||
syncTranscript(); |
||||
}, |
||||
onCompleted: (text) => commitSentence(text), |
||||
onError: (error) => console.error('[stt] session error:', error), |
||||
}, |
||||
); |
||||
|
||||
if (expired()) { |
||||
session.close(); |
||||
return; |
||||
} |
||||
refSession.current = session; |
||||
} |
||||
setListening(true); |
||||
} catch (error) { |
||||
console.error('[stt] failed to start listening:', error); |
||||
stop(); |
||||
} finally { |
||||
if (!expired()) refStarting.current = false; |
||||
} |
||||
// eslint-disable-next-line react-hooks/exhaustive-deps |
||||
}, [stop]); |
||||
|
||||
// 把還沒斷句的尾巴逼出來。頁面在硬上限快到時呼叫， |
||||
// 否則 processSpeech 讀到的 textarea 會少掉最後一整段。 |
||||
const flush = useCallback(() => { |
||||
if (refSession.current) { |
||||
refSession.current.commit(); |
||||
return; |
||||
} |
||||
if (refRecorder.current) flushRecording(); |
||||
// eslint-disable-next-line react-hooks/exhaustive-deps |
||||
}, []); |
||||
|
||||
// 換模式（realtime <-> batch）時重啟，不然舊 pipeline 會一直掛著 |
||||
useEffect(() => { |
||||
if (!refStream.current) return; |
||||
stop(); |
||||
start(); |
||||
// eslint-disable-next-line react-hooks/exhaustive-deps |
||||
}, [mode]); |
||||
|
||||
useEffect(() => stop, [stop]); |
||||
|
||||
return { |
||||
transcript, |
||||
finalTranscript, |
||||
listening, |
||||
resetTranscript, |
||||
isMicrophoneAvailable, |
||||
start, |
||||
stop, |
||||
flush, |
||||
}; |
||||
} |
||||
Loading…
Reference in new issue