diff --git a/vite/public/assets/ai/ai-01-1.mp3 b/vite/public/assets/ai/ai-01-1.mp3 new file mode 100644 index 0000000..b24e82a Binary files /dev/null and b/vite/public/assets/ai/ai-01-1.mp3 differ diff --git a/vite/public/assets/ai/ai-01-2.mp3 b/vite/public/assets/ai/ai-01-2.mp3 new file mode 100644 index 0000000..5566e6e Binary files /dev/null and b/vite/public/assets/ai/ai-01-2.mp3 differ diff --git a/vite/public/assets/ai/sfx-03-ai-03.mp3 b/vite/public/assets/ai/sfx-03-ai-03.mp3 new file mode 100644 index 0000000..f833ee0 Binary files /dev/null and b/vite/public/assets/ai/sfx-03-ai-03.mp3 differ diff --git a/vite/public/assets/ai/sfx-06-ai-04-record-03-ai-05-sfx-07.mp3 b/vite/public/assets/ai/sfx-06-ai-04-record-03-ai-05-sfx-07.mp3 new file mode 100644 index 0000000..cc53297 Binary files /dev/null and b/vite/public/assets/ai/sfx-06-ai-04-record-03-ai-05-sfx-07.mp3 differ diff --git a/vite/public/assets/ai/sfx-08-record-04-ai-06.mp3 b/vite/public/assets/ai/sfx-08-record-04-ai-06.mp3 new file mode 100644 index 0000000..f7742f1 Binary files /dev/null and b/vite/public/assets/ai/sfx-08-record-04-ai-06.mp3 differ diff --git a/vite/public/assets/record/record-05.mp3 b/vite/public/assets/record/record-05.mp3 index 50b9d37..89acc85 100644 Binary files a/vite/public/assets/record/record-05.mp3 and b/vite/public/assets/record/record-05.mp3 differ diff --git a/vite/public/cuelist_demo2.json b/vite/public/cuelist_demo2.json index a582914..c3fd2ae 100644 --- a/vite/public/cuelist_demo2.json +++ b/vite/public/cuelist_demo2.json @@ -33,22 +33,21 @@ "name": "Q4", "type": "phone", "description": "引導撥號", - "auto": false, - "audioFile": "assets/ai/ai-01.mp3", - "nextcue": 4.01, - "callback":"numpad", - "numpad_type":"userid", - "hint":"輸入兩位數號碼\n輸入完成送出,請按井字\n取消輸入,請按米字" + "auto": true, + "audioFile": "assets/ai/ai-01-1.mp3", + "nextcue": 4.01 }, { "id": 4.01, "name": "Q4.01", "type": "phone", - "description": "撥接音效", - "auto": true, - "audioFile": "assets/sfx/sfx-03.mp3", + "description": "撥號", + "auto": false, + "audioFile": "assets/ai/ai-01-2.mp3", "nextcue": 4.1, - "hint":"輸入完成" + "callback":"numpad", + "numpad_type":"userid", + "hint":"輸入兩位數號碼\n輸入完成送出,請按井字\n取消輸入,請按米字" }, { "id": 4.1, @@ -56,7 +55,7 @@ "type": "phone", "description": "輸入完成,請描述腦中的記憶畫面", "auto": true, - "audioFile": "assets/ai/ai-03.mp3", + "audioFile": "assets/ai/sfx-03-ai-03.mp3", "nextcue": 4.11 }, { @@ -76,56 +75,20 @@ "type": "chat", "description": "chat", "auto": true, - "nextcue": 4.21, - "duration": 90, + "nextcue": 4.3, + "duration": 20, "status":"go" }, - { - "id": 4.21, - "name": "Q4.21", - "type": "phone", - "description": "裝置完成音效", - "auto": true, - "audioFile": "assets/sfx/sfx-06.mp3", - "nextcue": 4.3 - }, { "id": 4.3, "name": "Q4.3", "type": "phone", "description": "記憶提取完成", "auto": true, - "audioFile": "assets/ai/ai-04.mp3", - "nextcue": 4.4 - }, - { - "id": 4.4, - "name": "Q4.4", - "type": "phone", - "description": "準備打電話", - "audioFile": "assets/record/record-03.mp3", - "auto": true, - "nextcue": 4.5 - }, - { - "id": 4.5, - "name": "Q4.5", - "type": "phone", - "description": "電話將自動接通", - "audioFile": "assets/ai/ai-05.mp3", - "auto": true, - "nextcue": 4.51 - }, - { - "id": 4.51, - "name": "Q4.51", - "type": "phone", - "description": "接通音效", - "audioFile": "assets/sfx/sfx-07.mp3", - "auto": true, + "audioFile": "assets/ai/sfx-06-ai-04-record-03-ai-05-sfx-07.mp3", "nextcue": 5.1, - "hint":"電話已接通,請留言" - }, + "hint":"電話已接通" + }, { "id": 5.1, "name": "Q5.1", @@ -133,33 +96,15 @@ "description": "call", "duration": 30, "auto": true, - "nextcue": 5.11 - }, - { - "id": 5.11, - "name": "Q5.11", - "type": "summary", - "description": "結束逼", - "auto": true, - "audioFile": "assets/sfx/sfx-08.mp3", "nextcue": 5.2, - "hint":"通話結束" + "callback":"summary" }, { "id": 5.2, "name": "Q5.2", "type": "phone", - "description": "保留刪除", - "auto": true, - "nextcue": 5.3, - "audioFile": "assets/record/record-04.mp3" - }, - { - "id": 5.3, - "name": "Q5.3", - "type": "phone", "description": "保留刪除操作說明", - "audioFile": "assets/ai/ai-06.mp3", + "audioFile": "assets/ai/sfx-08-record-04-ai-06.mp3", "callback":"numpad", "numpad_type":"choice", "auto": false, diff --git a/vite/src/comps/numpad.jsx b/vite/src/comps/numpad.jsx index eaf7eaf..e53e0c2 100644 --- a/vite/src/comps/numpad.jsx +++ b/vite/src/comps/numpad.jsx @@ -62,9 +62,9 @@ export default function NumPad({onSend, disabled, type}){ } break; case NUMPAD_TYPE.PASSWORD: - if(num>=0 && num<=999){ - onSend(num.toString().padStart(3, '0')); - refAudio.current[KEY_ENTER]?.play(); + if(refInput.current.length==3){ + onSend(refInput.current); + refAudio.current[KEY_ENTER]?.play(); }else{ refAudio.current['error']?.play(); } diff --git a/vite/src/pages/flow_free.jsx b/vite/src/pages/flow_free.jsx index fe352e0..394bbd4 100644 --- a/vite/src/pages/flow_free.jsx +++ b/vite/src/pages/flow_free.jsx @@ -33,6 +33,7 @@ const ChatStatus={ Clear: 'clear', End: 'end', Playing: 'playing', + Message: 'message', } const Voice={ @@ -50,10 +51,12 @@ export function FreeFlow(){ const [audioInput, setAudioInput] = useState(true); const [autoSend, setAutoSend] = useState(true); const [chatStatus, setChatStatus] = useState(ChatStatus.System); // System, User, Processing - const { userId, setUserId, getFileId, setPassword, reset:resetUser, uploadHistory, setSummary, summary,setChoice } = useUser(); + const { userId, setUserId, getFileId, setPassword, reset:resetUser, uploadHistory, setSummary, summary,setChoice, getUploadFolder,getDataId } = useUser(); const refTimer=useRef(); const refAudio=useRef(); + const refAudioPrompt=useRef(); + const refInput=useRef(); const refLight=useRef(); @@ -91,12 +94,12 @@ export function FreeFlow(){ //TODO: if cue end, don't play audio if(refCurrentCue.current?.type=='chat'){ - if(refChatCueEnd.current) { - console.log('Chat cue has ended, not playing audio:', url); - setChatStatus(ChatStatus.Clear); // Reset chat status to Clear - onCueEnd(); - return; - } + // if(refChatCueEnd.current) { + // console.log('Chat cue has ended, not playing audio:', url); + // setChatStatus(ChatStatus.Clear); // Reset chat status to Clear + // onCueEnd(); + // return; + // } // if audio time larger than cue remaining time, don't play audio } @@ -152,12 +155,17 @@ export function FreeFlow(){ updatePrompt(prompt); sendOsc(OSC_ADDRESS.PROMPT, prompt); - } + // play audio for prompt + refAudioPrompt.current?.play().catch(error => { + console.error("Audio prompt playback error:", error); + }); - if(refChatCueEnd.current){ - console.log('Talking ended, ending current cue'); - onCueEnd(); // End the current cue if chat cue has ended } + + // if(refChatCueEnd.current){ + // console.log('Talking ended, ending current cue'); + // onCueEnd(); // End the current cue if chat cue has ended + // } } } @@ -186,6 +194,9 @@ export function FreeFlow(){ } + // clear unity hint + sendOsc(OSC_ADDRESS.HINT, ''); // Clear hint message + sendOsc(OSC_ADDRESS.INPUT, ''); // Clear input message switch(cue.type){ case 'chat': @@ -204,31 +215,30 @@ export function FreeFlow(){ setChatWelcome(false); setChatStatus(ChatStatus.Clear); break; - case 'summary': - console.log('Getting summary...'); - - setChatStatus(ChatStatus.Clear); // Set chat status to Processing - getSummary(history.map(el=>`${el.role}:${el.content}`).join('\n'), data).then(summary_ => { + // case 'summary': + // console.log('Getting summary...'); - console.log('Summary:', summary_); - onCueEnd(); // End the current cue after getting summary + // setChatStatus(ChatStatus.Clear); // Set chat status to Processing + // getSummary(history.map(el=>`${el.role}:${el.content}`).join('\n'), data).then(summary_ => { - setSummary(summary_?.result); - refContainer.current.scrollTop = refContainer.current.scrollHeight; // Scroll to bottom + // console.log('Summary:', summary_); + // onCueEnd(); // End the current cue after getting summary - // sendOsc(OSC_ADDRESS.SUMMARY, summary_?.result); - // sendOsc(OSC_ADDRESS.EXPORT, `${getFileId()}#${summary_?.result}#${password}`); // Save summary with file ID - - }).catch(error => { - console.error('Error getting summary:', error); - }); + // setSummary(summary_?.result); + // refContainer.current.scrollTop = refContainer.current.scrollHeight; // Scroll to bottom + + // }).catch(error => { + // console.error('Error getting summary:', error); + // }); - break; + // break; case 'user_input': - console.log('User input cue, setting chat status to User'); - setChatStatus(ChatStatus.Clear); // Set chat status to User for user input cues + setChatStatus(ChatStatus.Message); // Set chat status to User resetTranscript(); // Reset transcript for user input - break; + break; + default: + setChatStatus(ChatStatus.Clear); + break; } @@ -275,17 +285,19 @@ export function FreeFlow(){ const cue= refCurrentCue.current; // Get the current cue from ref if(cue.type=='chat'){ - if(chatStatus==ChatStatus.System) { - console.log('Still talking...'); - refChatCueEnd.current=true; - return; - } + // if(chatStatus==ChatStatus.System) { + // console.log('Still talking...'); + // refChatCueEnd.current=true; + // return; + // } + console.log('save chat history:', history); uploadHistory(history); // Save chat history when cue ends } if(cue.callback==OSC_ADDRESS.DISCARD) { sendOsc(OSC_ADDRESS.CHOICE, OSC_ADDRESS.DISCARD); // Send OSC discard message setPassword(); + sendOsc(OSC_ADDRESS.EXPORT, `${getUploadFolder()}#${getFileId()}#${summary}#${mess}`); } if(cue.hint!=null){ @@ -294,7 +306,23 @@ export function FreeFlow(){ sendOsc(OSC_ADDRESS.HINT, ''); // Clear hint message } - refAudio.current?.pause(); // Pause any playing audio + if(cue.callback=='summary') { + console.log('Getting summary...'); + + getSummary(history.map(el=>`${el.role}:${el.content}`).join('\n'), data).then(summary_ => { + + console.log('Summary:', summary_); + onCueEnd(); // End the current cue after getting summary + + setSummary(summary_?.result); + refContainer.current.scrollTop = refContainer.current.scrollHeight; // Scroll to bottom + + }).catch(error => { + console.error('Error getting summary:', error); + }); + } + + refAudio.current?.pause(); // Pause any playing audio console.log('onCueEnd:', cue.id); @@ -334,12 +362,12 @@ export function FreeFlow(){ break; case NUMPAD_TYPE.CHOICE: next=cue.branch[mess.toString()].nextcue; - setChoice(cue.branch[mess.toString()].description); // Set choice for user input + setChoice(cue.branch[mess.toString()].description); // Set choice for user input break; case NUMPAD_TYPE.PASSWORD: setPassword(()=>mess); // sendOsc(OSC_ADDRESS.PASSWORD, mess); // Send OSC password message - sendOsc(OSC_ADDRESS.EXPORT, `${getFileId()}#${summary}#${mess}`); + sendOsc(OSC_ADDRESS.EXPORT, `${getUploadFolder()}#${getDataId()}#${summary}#${getFileId(mess)}`); sendOsc(OSC_ADDRESS.CHOICE, OSC_ADDRESS.SAVE); // Send OSC save choice message break; } @@ -474,7 +502,27 @@ export function FreeFlow(){ useEffect(()=>{ resetTranscript(); - sendOsc(OSC_ADDRESS.INPUT, chatStatus); + + let text=''; + switch(chatStatus) { + case ChatStatus.System: + text = '等我一下\n換我說囉'; + break; + case ChatStatus.User: + text = '換你說了'; + break; + case ChatStatus.Processing: + text = '記憶讀取中'; + break; + case ChatStatus.Message: + text = '請留言'; + break; + case ChatStatus.Clear: + default: + text = ''; + break; + } + sendOsc(OSC_ADDRESS.INPUT, text); },[chatStatus]); @@ -506,6 +554,9 @@ export function FreeFlow(){ console.error('Error fetching cuelist:', error); }); + + refAudioPrompt.current = new Audio('assets/sfx/sfx-05.mp3'); // Load audio prompt if available + },[]); @@ -591,7 +642,7 @@ export function FreeFlow(){ + disabled={chatStatus!=ChatStatus.User && chatStatus!=ChatStatus.Message}>
@@ -606,6 +657,7 @@ export function FreeFlow(){ setAutoSend(e.target.checked)} /> +
api_status= {status}
chat_status= {chatStatus}
diff --git a/vite/src/util/osc.js b/vite/src/util/osc.js index 869de3c..8c12139 100644 --- a/vite/src/util/osc.js +++ b/vite/src/util/osc.js @@ -25,9 +25,9 @@ export async function sendOsc(key, message){ console.warn('sendOsc: message is empty, skipping'); return; } - if(key!=OSC_ADDRESS.HINT && message === '') { - return; - } + // if(key!=OSC_ADDRESS.HINT && message === '') { + // return; + // } try{ console.log(`Sending OSC message: ${key} -> ${message}`); diff --git a/vite/src/util/useUser.jsx b/vite/src/util/useUser.jsx index 9b06a00..a64b686 100644 --- a/vite/src/util/useUser.jsx +++ b/vite/src/util/useUser.jsx @@ -4,7 +4,7 @@ import { updateUser } from "./backend"; const userContext=createContext(); -const SesstionTime={ +const SessionTime={ A:["12:00", "13:30"], B:["13:30", "15:00"], C:["15:00", "17:30"], @@ -23,14 +23,25 @@ export function UserProvider({children}) { const [summary, setSummary] = useState(null); - function getFileId(){ - if(!userId) return `test-${moment().format("YYYYMMDDmmss")}-${sessionId}`; - return `${moment().format("YYYYMMDD").substring(2)}-${sessionId}-${userId}`; + function getFileId(pass){ + if(!userId) return `${password||moment().format('hhmmss')}_testuser`; + return `${password||pass||''}_${userId}`; } + function getDataId(){ + if(!userId) return `${moment().format('YYYYMM')}/${moment().format("MMDD")}/${sessionId==null? SessionTime[sessionId][0]:'no-sesion'}/testuser_${moment().format('hhmmss')}`; + return `${moment().format('YYYYMM')}/${moment().format("MMDD")}/${sessionId==null? SessionTime[sessionId][0]:'no-sesion'}/${userId}`; + } + + function getUploadFolder(){ + return `${moment().format("YYYYMM")}/${moment().format("MMDD")}/${sessionId==null? SessionTime[sessionId][0]:'no-session'}`; + } + function reset(){ setUserId(null); setSessionId(null); setPassword(null); + setChoice(null); + setSummary(null); } function saveHistory(history) { @@ -39,7 +50,7 @@ export function UserProvider({children}) { const data={ history, } - updateUser(getFileId(), data) + updateUser('user/'+getDataId(), data) .then(() => { console.log("History saved successfully"); }) @@ -55,17 +66,17 @@ export function UserProvider({children}) { const data={ userId, - sessionId, + // sessionId, password, choice, summary, - date: moment().format("YYYY-MM-DD"), + date: moment().format("YYYY-MM-DD hh:mm:ss"), fileId: getFileId(), }; console.log("Updating user data:", data); - updateUser(getFileId(), data) + updateUser('user/'+getDataId(), data) .then(() => { console.log("User data updated successfully"); }) @@ -73,13 +84,13 @@ export function UserProvider({children}) { console.error("Error updating user data:", error); }); - },[password, sessionId, choice, summary]); + },[password, choice, summary]); useEffect(() => { let symbol=''; - for(const [key, value] of Object.entries(SesstionTime)){ + for(const [key, value] of Object.entries(SessionTime)){ const start=moment(value[0], "HH:mm"); const end=moment(value[1], "HH:mm"); @@ -95,11 +106,11 @@ export function UserProvider({children}) { return ( - {children} + {children} ); }