diff --git a/vite/public/assets/ai/ai-01-1.mp3 b/vite/public/assets/ai/ai-01-1.mp3 new file mode 100644 index 0000000..b24e82a Binary files /dev/null and b/vite/public/assets/ai/ai-01-1.mp3 differ diff --git a/vite/public/assets/ai/ai-01-2.mp3 b/vite/public/assets/ai/ai-01-2.mp3 new file mode 100644 index 0000000..5566e6e Binary files /dev/null and b/vite/public/assets/ai/ai-01-2.mp3 differ diff --git a/vite/public/assets/ai/sfx-03-ai-03.mp3 b/vite/public/assets/ai/sfx-03-ai-03.mp3 new file mode 100644 index 0000000..f833ee0 Binary files /dev/null and b/vite/public/assets/ai/sfx-03-ai-03.mp3 differ diff --git a/vite/public/assets/ai/sfx-06-ai-04-record-03-ai-05-sfx-07.mp3 b/vite/public/assets/ai/sfx-06-ai-04-record-03-ai-05-sfx-07.mp3 new file mode 100644 index 0000000..cc53297 Binary files /dev/null and b/vite/public/assets/ai/sfx-06-ai-04-record-03-ai-05-sfx-07.mp3 differ diff --git a/vite/public/assets/ai/sfx-08-record-04-ai-06.mp3 b/vite/public/assets/ai/sfx-08-record-04-ai-06.mp3 new file mode 100644 index 0000000..f7742f1 Binary files /dev/null and b/vite/public/assets/ai/sfx-08-record-04-ai-06.mp3 differ diff --git a/vite/public/assets/record/record-05.mp3 b/vite/public/assets/record/record-05.mp3 index 50b9d37..89acc85 100644 Binary files a/vite/public/assets/record/record-05.mp3 and b/vite/public/assets/record/record-05.mp3 differ diff --git a/vite/public/cuelist_demo2.json b/vite/public/cuelist_demo2.json index a582914..c3fd2ae 100644 --- a/vite/public/cuelist_demo2.json +++ b/vite/public/cuelist_demo2.json @@ -33,22 +33,21 @@ "name": "Q4", "type": "phone", "description": "引導撥號", - "auto": false, - "audioFile": "assets/ai/ai-01.mp3", - "nextcue": 4.01, - "callback":"numpad", - "numpad_type":"userid", - "hint":"輸入兩位數號碼\n輸入完成送出,請按井字\n取消輸入,請按米字" + "auto": true, + "audioFile": "assets/ai/ai-01-1.mp3", + "nextcue": 4.01 }, { "id": 4.01, "name": "Q4.01", "type": "phone", - "description": "撥接音效", - "auto": true, - "audioFile": "assets/sfx/sfx-03.mp3", + "description": "撥號", + "auto": false, + "audioFile": "assets/ai/ai-01-2.mp3", "nextcue": 4.1, - "hint":"輸入完成" + "callback":"numpad", + "numpad_type":"userid", + "hint":"輸入兩位數號碼\n輸入完成送出,請按井字\n取消輸入,請按米字" }, { "id": 4.1, @@ -56,7 +55,7 @@ "type": "phone", "description": "輸入完成,請描述腦中的記憶畫面", "auto": true, - "audioFile": "assets/ai/ai-03.mp3", + "audioFile": "assets/ai/sfx-03-ai-03.mp3", "nextcue": 4.11 }, { @@ -76,56 +75,20 @@ "type": "chat", "description": "chat", "auto": true, - "nextcue": 4.21, - "duration": 90, + "nextcue": 4.3, + "duration": 20, "status":"go" }, - { - "id": 4.21, - "name": "Q4.21", - "type": "phone", - "description": "裝置完成音效", - "auto": true, - "audioFile": "assets/sfx/sfx-06.mp3", - "nextcue": 4.3 - }, { "id": 4.3, "name": "Q4.3", "type": "phone", "description": "記憶提取完成", "auto": true, - "audioFile": "assets/ai/ai-04.mp3", - "nextcue": 4.4 - }, - { - "id": 4.4, - "name": "Q4.4", - "type": "phone", - "description": "準備打電話", - "audioFile": "assets/record/record-03.mp3", - "auto": true, - "nextcue": 4.5 - }, - { - "id": 4.5, - "name": "Q4.5", - "type": "phone", - "description": "電話將自動接通", - "audioFile": "assets/ai/ai-05.mp3", - "auto": true, - "nextcue": 4.51 - }, - { - "id": 4.51, - "name": "Q4.51", - "type": "phone", - "description": "接通音效", - "audioFile": "assets/sfx/sfx-07.mp3", - "auto": true, + "audioFile": "assets/ai/sfx-06-ai-04-record-03-ai-05-sfx-07.mp3", "nextcue": 5.1, - "hint":"電話已接通,請留言" - }, + "hint":"電話已接通" + }, { "id": 5.1, "name": "Q5.1", @@ -133,33 +96,15 @@ "description": "call", "duration": 30, "auto": true, - "nextcue": 5.11 - }, - { - "id": 5.11, - "name": "Q5.11", - "type": "summary", - "description": "結束逼", - "auto": true, - "audioFile": "assets/sfx/sfx-08.mp3", "nextcue": 5.2, - "hint":"通話結束" + "callback":"summary" }, { "id": 5.2, "name": "Q5.2", "type": "phone", - "description": "保留刪除", - "auto": true, - "nextcue": 5.3, - "audioFile": "assets/record/record-04.mp3" - }, - { - "id": 5.3, - "name": "Q5.3", - "type": "phone", "description": "保留刪除操作說明", - "audioFile": "assets/ai/ai-06.mp3", + "audioFile": "assets/ai/sfx-08-record-04-ai-06.mp3", "callback":"numpad", "numpad_type":"choice", "auto": false, diff --git a/vite/src/comps/numpad.jsx b/vite/src/comps/numpad.jsx index eaf7eaf..e53e0c2 100644 --- a/vite/src/comps/numpad.jsx +++ b/vite/src/comps/numpad.jsx @@ -62,9 +62,9 @@ export default function NumPad({onSend, disabled, type}){ } break; case NUMPAD_TYPE.PASSWORD: - if(num>=0 && num<=999){ - onSend(num.toString().padStart(3, '0')); - refAudio.current[KEY_ENTER]?.play(); + if(refInput.current.length==3){ + onSend(refInput.current); + refAudio.current[KEY_ENTER]?.play(); }else{ refAudio.current['error']?.play(); } diff --git a/vite/src/pages/flow_free.jsx b/vite/src/pages/flow_free.jsx index fe352e0..394bbd4 100644 --- a/vite/src/pages/flow_free.jsx +++ b/vite/src/pages/flow_free.jsx @@ -33,6 +33,7 @@ const ChatStatus={ Clear: 'clear', End: 'end', Playing: 'playing', + Message: 'message', } const Voice={ @@ -50,10 +51,12 @@ export function FreeFlow(){ const [audioInput, setAudioInput] = useState(true); const [autoSend, setAutoSend] = useState(true); const [chatStatus, setChatStatus] = useState(ChatStatus.System); // System, User, Processing - const { userId, setUserId, getFileId, setPassword, reset:resetUser, uploadHistory, setSummary, summary,setChoice } = useUser(); + const { userId, setUserId, getFileId, setPassword, reset:resetUser, uploadHistory, setSummary, summary,setChoice, getUploadFolder,getDataId } = useUser(); const refTimer=useRef(); const refAudio=useRef(); + const refAudioPrompt=useRef(); + const refInput=useRef(); const refLight=useRef(); @@ -91,12 +94,12 @@ export function FreeFlow(){ //TODO: if cue end, don't play audio if(refCurrentCue.current?.type=='chat'){ - if(refChatCueEnd.current) { - console.log('Chat cue has ended, not playing audio:', url); - setChatStatus(ChatStatus.Clear); // Reset chat status to Clear - onCueEnd(); - return; - } + // if(refChatCueEnd.current) { + // console.log('Chat cue has ended, not playing audio:', url); + // setChatStatus(ChatStatus.Clear); // Reset chat status to Clear + // onCueEnd(); + // return; + // } // if audio time larger than cue remaining time, don't play audio } @@ -152,12 +155,17 @@ export function FreeFlow(){ updatePrompt(prompt); sendOsc(OSC_ADDRESS.PROMPT, prompt); - } + // play audio for prompt + refAudioPrompt.current?.play().catch(error => { + console.error("Audio prompt playback error:", error); + }); - if(refChatCueEnd.current){ - console.log('Talking ended, ending current cue'); - onCueEnd(); // End the current cue if chat cue has ended } + + // if(refChatCueEnd.current){ + // console.log('Talking ended, ending current cue'); + // onCueEnd(); // End the current cue if chat cue has ended + // } } } @@ -186,6 +194,9 @@ export function FreeFlow(){ } + // clear unity hint + sendOsc(OSC_ADDRESS.HINT, ''); // Clear hint message + sendOsc(OSC_ADDRESS.INPUT, ''); // Clear input message switch(cue.type){ case 'chat': @@ -204,31 +215,30 @@ export function FreeFlow(){ setChatWelcome(false); setChatStatus(ChatStatus.Clear); break; - case 'summary': - console.log('Getting summary...'); - - setChatStatus(ChatStatus.Clear); // Set chat status to Processing - getSummary(history.map(el=>`${el.role}:${el.content}`).join('\n'), data).then(summary_ => { + // case 'summary': + // console.log('Getting summary...'); - console.log('Summary:', summary_); - onCueEnd(); // End the current cue after getting summary + // setChatStatus(ChatStatus.Clear); // Set chat status to Processing + // getSummary(history.map(el=>`${el.role}:${el.content}`).join('\n'), data).then(summary_ => { - setSummary(summary_?.result); - refContainer.current.scrollTop = refContainer.current.scrollHeight; // Scroll to bottom + // console.log('Summary:', summary_); + // onCueEnd(); // End the current cue after getting summary - // sendOsc(OSC_ADDRESS.SUMMARY, summary_?.result); - // sendOsc(OSC_ADDRESS.EXPORT, `${getFileId()}#${summary_?.result}#${password}`); // Save summary with file ID - - }).catch(error => { - console.error('Error getting summary:', error); - }); + // setSummary(summary_?.result); + // refContainer.current.scrollTop = refContainer.current.scrollHeight; // Scroll to bottom + + // }).catch(error => { + // console.error('Error getting summary:', error); + // }); - break; + // break; case 'user_input': - console.log('User input cue, setting chat status to User'); - setChatStatus(ChatStatus.Clear); // Set chat status to User for user input cues + setChatStatus(ChatStatus.Message); // Set chat status to User resetTranscript(); // Reset transcript for user input - break; + break; + default: + setChatStatus(ChatStatus.Clear); + break; } @@ -275,17 +285,19 @@ export function FreeFlow(){ const cue= refCurrentCue.current; // Get the current cue from ref if(cue.type=='chat'){ - if(chatStatus==ChatStatus.System) { - console.log('Still talking...'); - refChatCueEnd.current=true; - return; - } + // if(chatStatus==ChatStatus.System) { + // console.log('Still talking...'); + // refChatCueEnd.current=true; + // return; + // } + console.log('save chat history:', history); uploadHistory(history); // Save chat history when cue ends } if(cue.callback==OSC_ADDRESS.DISCARD) { sendOsc(OSC_ADDRESS.CHOICE, OSC_ADDRESS.DISCARD); // Send OSC discard message setPassword(); + sendOsc(OSC_ADDRESS.EXPORT, `${getUploadFolder()}#${getFileId()}#${summary}#${mess}`); } if(cue.hint!=null){ @@ -294,7 +306,23 @@ export function FreeFlow(){ sendOsc(OSC_ADDRESS.HINT, ''); // Clear hint message } - refAudio.current?.pause(); // Pause any playing audio + if(cue.callback=='summary') { + console.log('Getting summary...'); + + getSummary(history.map(el=>`${el.role}:${el.content}`).join('\n'), data).then(summary_ => { + + console.log('Summary:', summary_); + onCueEnd(); // End the current cue after getting summary + + setSummary(summary_?.result); + refContainer.current.scrollTop = refContainer.current.scrollHeight; // Scroll to bottom + + }).catch(error => { + console.error('Error getting summary:', error); + }); + } + + refAudio.current?.pause(); // Pause any playing audio console.log('onCueEnd:', cue.id); @@ -334,12 +362,12 @@ export function FreeFlow(){ break; case NUMPAD_TYPE.CHOICE: next=cue.branch[mess.toString()].nextcue; - setChoice(cue.branch[mess.toString()].description); // Set choice for user input + setChoice(cue.branch[mess.toString()].description); // Set choice for user input break; case NUMPAD_TYPE.PASSWORD: setPassword(()=>mess); // sendOsc(OSC_ADDRESS.PASSWORD, mess); // Send OSC password message - sendOsc(OSC_ADDRESS.EXPORT, `${getFileId()}#${summary}#${mess}`); + sendOsc(OSC_ADDRESS.EXPORT, `${getUploadFolder()}#${getDataId()}#${summary}#${getFileId(mess)}`); sendOsc(OSC_ADDRESS.CHOICE, OSC_ADDRESS.SAVE); // Send OSC save choice message break; } @@ -474,7 +502,27 @@ export function FreeFlow(){ useEffect(()=>{ resetTranscript(); - sendOsc(OSC_ADDRESS.INPUT, chatStatus); + + let text=''; + switch(chatStatus) { + case ChatStatus.System: + text = '等我一下\n換我說囉'; + break; + case ChatStatus.User: + text = '換你說了'; + break; + case ChatStatus.Processing: + text = '記憶讀取中'; + break; + case ChatStatus.Message: + text = '請留言'; + break; + case ChatStatus.Clear: + default: + text = ''; + break; + } + sendOsc(OSC_ADDRESS.INPUT, text); },[chatStatus]); @@ -506,6 +554,9 @@ export function FreeFlow(){ console.error('Error fetching cuelist:', error); }); + + refAudioPrompt.current = new Audio('assets/sfx/sfx-05.mp3'); // Load audio prompt if available + },[]); @@ -591,7 +642,7 @@ export function FreeFlow(){ + disabled={chatStatus!=ChatStatus.User && chatStatus!=ChatStatus.Message}>