diff --git a/vite/public/assets/moty/q1.mp3 b/vite/public/assets/moty/q1.mp3 new file mode 100644 index 0000000..f9ffc4d Binary files /dev/null and b/vite/public/assets/moty/q1.mp3 differ diff --git a/vite/public/assets/moty/q2-1.mp3 b/vite/public/assets/moty/q2-1.mp3 new file mode 100644 index 0000000..dc618e0 Binary files /dev/null and b/vite/public/assets/moty/q2-1.mp3 differ diff --git a/vite/public/assets/moty/q2.mp3 b/vite/public/assets/moty/q2.mp3 new file mode 100644 index 0000000..ddc175c Binary files /dev/null and b/vite/public/assets/moty/q2.mp3 differ diff --git a/vite/public/assets/moty/q3.mp3 b/vite/public/assets/moty/q3.mp3 new file mode 100644 index 0000000..0a9e3d7 Binary files /dev/null and b/vite/public/assets/moty/q3.mp3 differ diff --git a/vite/public/assets/moty/q5.mp3 b/vite/public/assets/moty/q5.mp3 new file mode 100644 index 0000000..86c43d2 Binary files /dev/null and b/vite/public/assets/moty/q5.mp3 differ diff --git a/vite/public/assets/moty/q7.mp3 b/vite/public/assets/moty/q7.mp3 new file mode 100644 index 0000000..a25e01f Binary files /dev/null and b/vite/public/assets/moty/q7.mp3 differ diff --git a/vite/public/cuelist_moty.json b/vite/public/cuelist_moty.json new file mode 100644 index 0000000..1ec7f67 --- /dev/null +++ b/vite/public/cuelist_moty.json @@ -0,0 +1,99 @@ +{ + "cuelist": [ + { + "id": 1, + "name": "Q1", + "type": "phone", + "description": "preset announce", + "audioFile": "assets/moty/q1.mp3", + "layer":"announce", + "loop": true, + "status":"reset", + "fadeout": true, + "nextcue": 2, + "auto": true + }, + { + "id": 2, + "name": "Q2", + "type": "phone", + "description": "引導輸入電話號碼", + "auto": true, + "audioFile": "assets/moty/q2.mp3", + "nextcue": 3, + "callback":"numpad", + "numpad_type":"userid", + "input_time": 26000 + }, + { + "id":2.1, + "name": "Q2.1", + "type": "phone", + "description": "撥打音效", + "auto": false, + "audioFile": "assets/moty/q2-1.mp3", + "loop": true, + "nextcue": 3, + "fadeout": true + }, + { + "id": 3, + "name": "Q3", + "type": "phone", + "description": "引導生圖", + "auto": true, + "audioFile": "assets/moty/q3.mp3", + "nextcue": 4, + "status":"intro", + "status_delay": 3000 + }, + { + "id": 4, + "name": "Q4", + "type": "chat", + "description": "chat", + "auto": true, + "nextcue": 5, + "duration": 90, + "status":"go", + "chatInterval":20 + }, + { + "id": 5, + "name": "Q5", + "type": "phone", + "description": "提取完成", + "auto": true, + "audioFile": "assets/moty/q5.mp3", + "nextcue": 6 + }, + { + "id": 6, + "name": "Q6", + "type": "user_input", + "description": "call", + "duration": 30, + "auto": true, + "nextcue": 6.1 + }, + { + "id":6.1, + "name":"Q6.1", + "type":"summary", + "auto":true, + "description":"summary", + "nextcue":7 + }, + { + "id": 7, + "name": "Q7", + "type": "phone", + "description": "Ending", + "auto": true, + "audioFile": "assets/moty/q7.mp3", + "status":"end", + "soundcue":"Q3" + } + ] +} + \ No newline at end of file diff --git a/vite/src/main.jsx b/vite/src/main.jsx index 9d070ab..ef2a116 100644 --- a/vite/src/main.jsx +++ b/vite/src/main.jsx @@ -6,6 +6,7 @@ import './index.css' import App from './App.jsx' import { Settings } from './pages/settings.jsx'; import { Flow } from './pages/flow.jsx'; +import { FlowMoty } from './pages/flow_moty.jsx'; import { Conversation } from './pages/conversation.jsx'; import { ChatProvider } from './util/useChat.jsx'; import { DataProvider } from './util/useData.jsx'; @@ -20,10 +21,11 @@ createRoot(document.getElementById('root')).render( - } /> + } /> } /> } /> } /> + } /> diff --git a/vite/src/pages/flow_moty.jsx b/vite/src/pages/flow_moty.jsx new file mode 100644 index 0000000..e217ed3 --- /dev/null +++ b/vite/src/pages/flow_moty.jsx @@ -0,0 +1,1125 @@ +import { invoke } from '@tauri-apps/api/core'; +import { useEffect, useRef, useState } from "react"; +import SpeechRecognition, { useSpeechRecognition } from 'react-speech-recognition'; + +import { Countdown } from "../comps/timer"; + +import { Status, useChat } from "../util/useChat"; +import { getSummary } from "../util/chat"; +import { saveHistory } from "../util/output"; +import NumPad, { NUMPAD_TYPE } from "../comps/numpad"; +import { Light } from "../comps/light"; +import { useData } from "../util/useData"; +import VoiceAnalysis from "../comps/voiceanalysis"; +import { sendOsc, OSC_ADDRESS, updatePrompt, onOscMessageReceived, sendOscStatus } from "../util/osc"; +import { DebugControl, TEST_PROMPT } from "../comps/debug"; +import { useUser } from "../util/useUser"; + + +const CUELIST_FILE = 'cuelist_moty.json'; +const AUDIO_FADE_TIME=3000; // in ms + +const EmojiType={ + phone: '📞', + headphone: '🎧', + speaker: '🔊', + chat: '🤖', + chat_end: '🤖', + user_input: '💬', + announce: '📢', + debug: '🐞', +} + +const ChatStatus={ + System: 'system', + User: 'user', + Processing: 'processing', + Clear: 'clear', + End: 'end', + Playing: 'playing', + Message: 'message', +} + +const Voice={ + ONYX: 'onyx', + SHIMMER: 'shimmer', +} + +export function FlowMoty(){ + + const { data }=useData(); + + const [cuelist, setCuelist] = useState([]); + const [currentCue, setCurrentCue] = useState(null); + const [nextCue, setNextCue] = useState(null); + const [localIP, setLocalIP] = useState(null); + + const [chatWelcome, setChatWelcome] = useState(null); + const [audioInput, setAudioInput] = useState(true); + const [autoSend, setAutoSend] = useState(true); + const [chatStatus, setChatStatus] = useState(ChatStatus.System); // System, User, Processing + + const [padInput, setPadInput] = useState(null); + + const [inputReady, setInputReady] = useState(false); + + const { userId, setUserId, getFileId, setPassword, reset:resetUser, uploadHistory, setSummary, summary,setChoice,choice, getUploadFolder,getDataId, writeSheet } = useUser(); + + const refTimer=useRef(); + const refAudio=useRef(); + const refAudioPrompt=useRef(); + + const refAudioAnnounce=useRef(); + + const refInput=useRef(); + // const refLight=useRef(); + + const refPauseTimer=useRef(); + const refChatCueEnd=useRef(false); + + const refContainer=useRef(); + + const refCurrentCue= useRef(null); + const refData=useRef(data); + + const refHintTimeout=useRef(); + const refInputTimeout=useRef(); + + const refFadeOutInterval=useRef(); + const refVolDownInterval=useRef(); + + const refChatTimer=useRef(); + const refSpeaking=useRef(false); + + const [lastOsc, setLastOsc]=useState(); + + const { history, status, reset, sendMessage, setStatus, audioOutput, setAudioOutput, stop:stopChat, + audioUrl }=useChat(); + + + const { + transcript, + finalTranscript, + listening, + resetTranscript, + browserSupportsSpeechRecognition, + isMicrophoneAvailable, + }=useSpeechRecognition(); + + + function resetData() { + setSummary(null); + + reset(); + resetUser(); + sendOsc(OSC_ADDRESS.CHOICE, 'reset'); + sendSpeechEnd(); + + setPadInput(); + setChoice(); + + if(refChatTimer.current) clearInterval(refChatTimer.current); + + } + + function onOsc(payload){ + console.log('onOsc', payload); + const address=payload.addr; + const message=payload.args[0]; + const params=message.split('#'); + + if(params[0]!='all'){ + console.log('set lastOsc', {address, params}); + setLastOsc(()=>({address, params})); + return; + } + + switch(address){ + case OSC_ADDRESS.PLAY_CUE: + setNextCue(()=>params[1]); + break; + case OSC_ADDRESS.STOP_CUE: + onStop(); + break; + case OSC_ADDRESS.RESET_CUE: + sendOsc(OSC_ADDRESS.STATUS, 'reset'); + onStop(); + resetData(); + break; + case OSC_ADDRESS.TEST_EXPORT: + exportFile(true); + break; + } + + // Handle OSC messages here + } + + function sendPrompt(){ + // send prompt + let raw_prompt=history[history.length-1]?.prompt || ''; + + if(raw_prompt && raw_prompt.trim() !== '') { + const prompt = `${data?.sd_prompt_prefix || ''}${raw_prompt}${data?.sd_prompt_suffix || ''}`; + + updatePrompt(prompt); + sendOsc(OSC_ADDRESS.PROMPT, prompt); + // play audio for prompt + refAudioPrompt.current?.play().catch(error => { + console.error("Audio prompt playback error:", error); + }); + + refAudioPrompt.current.onended = () => { + console.log('Audio prompt ended, setting chat status to User'); + setChatStatus(ChatStatus.User); // Set chat status to User after audio ends + + const user_input = history.find(msg => msg.role === 'user'); + if(user_input && user_input.content.trim() !== '') { + sendOsc(OSC_ADDRESS.STATUS, 'go'); // Send OSC status message + } + + } + + }else{ + setChatStatus(()=>ChatStatus.User); // Reset chat status to User after audio ends + } + + } + + + function playAudio(url){ + if(!url) return; + + console.log('Playing audio:', url); + + if(refCurrentCue.current?.layer=='announce'){ + if(refAudioAnnounce.current) { + refAudioAnnounce.current.pause(); // Stop any currently playing announce audio + } + refAudioAnnounce.current = new Audio(url); + refAudioAnnounce.current.loop=refCurrentCue.current?.loop || false; // Set loop if defined in cue + refAudioAnnounce.current.play().catch(error => { + console.error("Audio announce playback error:", error); + }); + + + // lower audio + if(refAudio.current) { + // fade out current audio + if(refVolDownInterval.current){ + clearInterval(refVolDownInterval.current); + } + + const dest=0.2; + let fadeOutInterval = setInterval(() => { + if (refAudio.current.volume > dest) { + refAudio.current.volume =Math.max(dest, refAudio.current.volume - (1.0-dest)/(AUDIO_FADE_TIME/100)); // Decrease volume gradually + //console.log('Fading out audio volume:', refAudio.current.volume); + } else { + clearInterval(fadeOutInterval); + } + }, 100); + refVolDownInterval.current=fadeOutInterval; + } + + return; + } + + + if(refAudioAnnounce.current) { + refAudioAnnounce.current.pause(); // Stop any currently playing announce audio + refAudioAnnounce.current = null; + } + + if(refAudio.current) { + refAudio.current.pause(); // Stop any currently playing audio + } + + let audioUrl = url; + // if(voice==Voice.SHIMMER) audioUrl = url.replace(Voice.ONYX, Voice.SHIMMER); + // console.log('Using voice:', voice, 'for audio:', audioUrl); + + const audio = new Audio(audioUrl); + + + //TODO: if cue end, don't play audio + if(refCurrentCue.current?.type=='chat'){ + // if(refChatCueEnd.current) { + // console.log('Chat cue has ended, not playing audio:', url); + // setChatStatus(ChatStatus.Clear); // Reset chat status to Clear + // onCueEnd(); + // return; + // } + } + + + audio.loop=refCurrentCue.current?.loop || false; // Set loop if defined in cue + + + audio.addEventListener("loadedmetadata", () => { + if(refCurrentCue.current?.type!='chat' && refCurrentCue.current?.type!='user_input') { + refTimer.current?.restart(audio.duration*1000 || 0); + audio.play().catch(error => { + console.error("Audio playback error:", error); + }); + + }else{ + if(refCurrentCue.current?.type=='chat'){ + + if(refTimer.current?.remainingTime < audio.duration*1000) { + console.log('Audio duration is longer than remaining cue time, not playing audio:', url); + // send propmpt + sendPrompt(); + return; + }else{ + setChatStatus(()=>ChatStatus.System); + audio.play().catch(error => { + console.error("Audio playback error:", error); + }); + } + + }else{ + setChatStatus(()=>ChatStatus.Playing); + + audio.play().catch(error => { + console.error("Audio playback error:", error); + }); + + } + } + }); + + audio.onended = () => { + if(refCurrentCue.current?.type!='chat'){ + setChatStatus(ChatStatus.End); + onCueEnd(); + console.log('Audio ended, ending current cue'); + }else{ + + // if history contains user input, send it + const last_user_input = history.slice().reverse().find(msg => msg.role === 'user'); + console.log('Audio ended, checking for user input in history:', last_user_input); + if(last_user_input.content!='...'){ + sendPrompt(); + }else{ + setChatStatus(ChatStatus.User); // Reset chat status to Clear + } + + } + } + + refAudio.current = audio; // Store the new audio reference + + } + function fadeOutAudio(callback){ + + if(refVolDownInterval.current) clearInterval(refVolDownInterval.current); + + if(refAudio.current || refAudioAnnounce.current){ + console.log('Fading out audio'); + + let audio = refAudio.current; + let announce = refAudioAnnounce.current; + + if(refFadeOutInterval.current){ + clearInterval(refFadeOutInterval.current); + refFadeOutInterval.current=null; + } + + let fadeOutInterval = setInterval(() => { + if(audio){ + if (audio.volume > 0) { + audio.volume =Math.max(0, audio.volume - 1.0/(AUDIO_FADE_TIME/100)); // Decrease volume gradually + } else { + audio.pause(); + audio.volume = 0; // Reset volume for next play + + } + } + + if(announce){ + if (announce.volume > 0) { + announce.volume = Math.max(0, announce.volume - 1.0/(AUDIO_FADE_TIME/100)); // Decrease volume gradually + } else { + //clearInterval(fadeOutInterval); + announce.pause(); + announce.volume = 0; // Reset volume for next play + } + } + + if((audio==null || audio.volume==0) && (announce==null || announce.volume==0)){ + clearInterval(fadeOutInterval); + if(callback) callback(); + } + }, 100); // Decrease volume every 100ms + + refFadeOutInterval.current=fadeOutInterval; + + }else{ + if(callback) callback(); + } + } + + function playCue(cue) { + + if(!cue) return; + console.log('Playing cue:', cue); + + // stop audio + // if(refAudio.current) refAudio.current.pause(); + + + + setCurrentCue(cue); + refCurrentCue.current = cue; // Store the current cue in ref + + + + if(parseFloat(cue.id)<=4.2){ + // Special case for starting a conversation + console.log('clear conversation...'); + reset(); + + const prompt = `${data?.sd_prompt_prefix || ''}${TEST_PROMPT}${data?.sd_prompt_suffix || ''}`; + updatePrompt(prompt); + } + + + // clear unity hint + if(refHintTimeout.current) clearTimeout(refHintTimeout.current); + + sendOsc(OSC_ADDRESS.HINT, ''); // Clear hint message + sendOsc(OSC_ADDRESS.INPUT, ''); // Clear input message + sendSpeechEnd(); + + setPadInput(); + + + switch(cue.type){ + case 'chat': + // Special case for starting a conversation + refChatCueEnd.current=false; + resetTranscript(); + console.log('Starting conversation...'); + setChatStatus(ChatStatus.User); + //sendMessage(null, false, false, null); // Send initial message with voice + //setChatWelcome(true); + //resetData(); // Reset data for new conversation + break; + + case 'summary': + console.log('Getting summary...'); + + setChatStatus(ChatStatus.Clear); // Set chat status to Processing + + let message=refInput.current?.value?.trim() || history.map(el=>`${el.role}:${el.content}`).join('\n'); + console.log('Summary input message:', message); + + if(!message || message.length==0) { + + setSummary(); + console.log('no message input, clear summary'); + onCueEnd(); // End the current cue after getting summary + + }else{ + + getSummary(message, data).then(summary_ => { + + console.log('Summary:', summary_); + onCueEnd(); // End the current cue after getting summary + + setSummary(()=>summary_?.result); + refContainer.current.scrollTop = refContainer.current.scrollHeight; // Scroll to bottom + + }).catch(error => { + console.error('Error getting summary:', error); + }); + } + + + break; + case 'user_input': + setChatStatus(ChatStatus.Message); // Set chat status to User + resetTranscript(); // Reset transcript for user input + break; + default: + setChatStatus(ChatStatus.Clear); + break; + } + + + // if(cue.callback=='fade_in_light') refLight.current.fadeIn(); // Fade in light for conversation start + // if(cue.callback=='fade_out_light') refLight.current.fadeOut(); // Fade out light for conversation end + + if(cue.hint!=null && cue.hint_time!=null){ + refHintTimeout.current=setTimeout(()=>{ + sendOsc(OSC_ADDRESS.HINT, cue.hint); // Send OSC hint message + + }, cue.hint_time); + } + + setInputReady(false); + + if(cue.input_time!=null){ + if(refInputTimeout.current) clearTimeout(refInputTimeout.current); + refInputTimeout.current=setTimeout(()=>{ + setInputReady(()=>true); + }, cue.input_time); + } + + if(cue.audioFile){ + playAudio(cue.audioFile); + } + + if(cue.duration){ + refTimer.current.restart(cue.duration*1000, ()=>{ + onCueEnd(cue); + }); + } + + switch(cue.callback){ + case 'exportFile': + sendOsc(OSC_ADDRESS.HINT,''); + exportFile(cue.type=='debug'); + break; + // case 'fadeout': + // fadeOutAudio(); + // break; + } + + + // control unity + if(cue.status && cue.status!='go') { + if(cue.status_delay) { + setTimeout(()=>{ + sendOsc(OSC_ADDRESS.STATUS, cue.status); // Send OSC status message + }, cue.status_delay); + }else{ + sendOsc(OSC_ADDRESS.STATUS, cue.status); // Send OSC status message + } + if(cue.status=='reset') { + // refLight.current.set(1); + resetData(); + } + } + if(cue.type=='user_input' || cue.type=='chat'){ + sendOsc(OSC_ADDRESS.COUNTDOWN, cue.duration || '0'); // Send OSC countdown message + }else{ + sendOsc(OSC_ADDRESS.COUNTDOWN, '0'); // Reset countdown for non-chat cues + } + if(cue.numpad_type=='choice'){ + setChoice(); + } + + sendOscStatus(OSC_ADDRESS.CLIENT_STATUS,`${data.id}#playcue#${cue.id}`); + + console.log('~~~~ clear pause timer'); + if(refPauseTimer.current) clearTimeout(refPauseTimer.current); + // refSpeechPaused.current=false; + } + function sendSpeechStart(){ + + if(refCurrentCue.current?.type!='chat') return; + + console.log('------- speech start -------'); + sendOsc(OSC_ADDRESS.SPEECH, 'start'); + refSpeaking.current=true; + } + function sendSpeechEnd(){ + + // if(refCurrentCue.current?.type!='chat' || chatStatus!=ChatStatus.User) return; + + console.log('------- speech end -------'); + sendOsc(OSC_ADDRESS.SPEECH, 'stop'); + refSpeaking.current=false; + } + function onCueEnd() { + + refTimer.current?.stop(); // Stop the timer when cue ends + + if(!refCurrentCue.current) return; + const cue= refCurrentCue.current; // Get the current cue from ref + + if(cue.type=='chat'){ + // if(chatStatus==ChatStatus.System) { + // console.log('Still talking...'); + // refChatCueEnd.current=true; + // return; + // } + console.log('save chat history:', history); + uploadHistory(history); // Save chat history when cue ends + } + + + + sendOsc(OSC_ADDRESS.HINT, ''); // Clear hint message + sendSpeechEnd(); + + + if(cue.numpad_type=='choice'){ + if(!choice){ + console.log('set default choice to save'); + setChoice('save'); + sendOsc(OSC_ADDRESS.CHOICE, 'save'); // Send OSC save choice message + }else{ + // sendOsc(OSC_ADDRESS.CHOICE, choice); // Send OSC save choice message + } + } + + refAudio.current?.pause(); // Pause any playing audio + console.log('onCueEnd:', cue.id); + + + resetTranscript(); // Reset transcript after cue ends + sendOscStatus(OSC_ADDRESS.CLIENT_STATUS, `${data.id}#endcue#${cue.id}`); + + if(cue.auto || cue.callback=='numpad'){ + playCue(cuelist.find(c => c.id === cue.nextcue)); + } + + } + + function onStop(){ + console.log('Stopping current cue'); + if(refAudio.current) { + refAudio.current.pause(); + refAudio.current = null; + } + if(refAudioAnnounce.current) { + refAudioAnnounce.current.pause(); + refAudioAnnounce.current = null; + } + + setCurrentCue(null); + refCurrentCue.current = null; // Clear the current cue reference + + refTimer.current.restart(0); + + stopChat(); // Stop chat processing + } + + + function onNumpad(mess){ + console.log('onNumPad',mess); + setPadInput(()=>mess); + } + + function exportFile(isTest=false){ + const user_input = history.find(msg => msg.role === 'user' && msg.content!='...'); + const default_image=!(user_input && user_input.content.trim() !== ''); + + const order=parseInt(data?.id)||Math.floor(Math.random()*10+5); + const delay=order*2000; + console.log('export delay time:', delay); + + setTimeout(()=>{ + if(isTest) { + sendOsc(OSC_ADDRESS.EXPORT, `${getUploadFolder()}#${getDataId()}#${summary||''}#${getFileId(padInput)}#${choice||''}#generated`); // Send OSC export message + }else{ + sendOsc(OSC_ADDRESS.EXPORT, `${getUploadFolder()}#${getDataId()}#${summary||''}#${getFileId(padInput)}#${choice||''}#${default_image?'default':'generated'}`); // Send OSC export message + } + + + }, delay); // random delay between 0.5s to 5.5s + + writeSheet(); + } + + function startChatTimer(){ + // sendOsc(OSC_ADDRESS.COUNTDOWN, refCurrentCue.current?.chatInterval || '0'); // Send OSC countdown message + if(refChatTimer.current) clearInterval(refChatTimer.current); + if(refCurrentCue.current?.type!='chat') return; + + let timeleft=refCurrentCue.current?.chatInterval || 0; + + const endTime=new Date().getTime()+timeleft*1000; + + function tick(){ + const now=new Date().getTime(); + const timeLeft=endTime-now; + + if(timeleft<=0){ + console.log('~~~ chat timer ended, process speech'); + clearInterval(refChatTimer.current); + // send message + if(chatStatus==ChatStatus.User) processSpeech(); + + return; + } + timeleft=Math.max(0, Math.floor(timeLeft/1000)); + + refChatTimer.current=setTimeout(tick, 100); + } + + refChatTimer.current=setTimeout(tick, 100); + + } + + useEffect(()=>{ + + if(!lastOsc) return; + console.log('Process last OSC:', lastOsc); + + if(lastOsc.params[0]!=data.id) return; + + switch(lastOsc.address){ + case OSC_ADDRESS.PLAY_CUE: + setNextCue(()=>lastOsc.params[1]); + break; + case OSC_ADDRESS.STOP_CUE: + onStop(); + break; + case OSC_ADDRESS.RESET_CUE: + sendOsc(OSC_ADDRESS.STATUS, 'reset'); + onStop(); + resetData(); + break; + case OSC_ADDRESS.TEST_EXPORT: + exportFile(true); + break; + } + + },[lastOsc]); + + useEffect(()=>{ + + + if(padInput==null) return; + + console.log('Numpad input:', padInput); + if(refCurrentCue.current?.callback!='numpad') return; + + let cue=refCurrentCue.current; + let next=cue.nextcue; + switch(cue.numpad_type){ + case NUMPAD_TYPE.USERID: + console.log('set id', padInput); + setUserId(()=>padInput); + break; + case NUMPAD_TYPE.CHOICE: + next=cue.branch[padInput.toString()].nextcue; + setChoice(()=>cue.branch[padInput.toString()].description); // Set choice for user input + sendOsc(OSC_ADDRESS.CHOICE, cue.branch[padInput.toString()].description); + break; + case NUMPAD_TYPE.PASSWORD: + setPassword(padInput); + // sendOsc(OSC_ADDRESS.PASSWORD, mess); // Send OSC password message + // sendOsc(OSC_ADDRESS.CHOICE, choice); // Send OSC save choice message + break; + } + + if(next){ + onStop(); + console.log('Finish enter number, next cue:', next); + playCue(cuelist.find(c => c.id === next)); + } + + + },[padInput]); + + useEffect(()=>{ + + if(userId>=1 && userId<=24) { + console.log('User ID set:', userId); + //playCue(cuelist.find(c => c.id === refCurrentCue.current.nextcue)); // Play cue 5 when userId is set + } + + },[userId]); + + function setPauseTimer(){ + + if(refPauseTimer.current) clearTimeout(refPauseTimer.current); + + // sendOsc(OSC_ADDRESS.SPEECH_PAUSE, data.speech_idle_time.toString()); + refPauseTimer.current=setTimeout(()=>{ + + // sendSpeechEnd(); + + console.log('~~~ pause timer ended, process speech'); + // if(refSpeechPaused.current) + processSpeech(); + // sendOsc(OSC_ADDRESS.SPEECH_PAUSE, '0'); + + }, data.speech_idle_time); + } + + + function onSpeechEnd(){ + + + + if(currentCue?.type!='chat') return; // Only process if current cue is user input + if(chatStatus!=ChatStatus.User) return; // Only process if chat status is User + + sendSpeechEnd(); + + console.log('~~~ on speech end, start pause timer',data.speech_idle_time); + // refSpeechPaused.current=true; + setPauseTimer(); + + + } + function processSpeech(){ + + + if(refPauseTimer.current) clearTimeout(refPauseTimer.current); + if(refChatTimer.current) clearInterval(refChatTimer.current); + // sendOsc(OSC_ADDRESS.COUNTDOWN, '0'); // Reset countdown + + // sendSpeechEnd(); + + if(currentCue?.type!='chat') return; // Only process if current cue is user input + + + console.log('processSpeech:', finalTranscript); + + if(refChatCueEnd.current) { + console.log('Chat cue has ended, do not processing speech'); + onCueEnd(); + return; + } + if(autoSend) { + + const message= refInput.current?.value?.trim(); + + if(message && message.length>0) { + console.log('Ending conversation with message:', message); + sendMessage(message, false, false, null); + setChatWelcome(false); + + setChatStatus(ChatStatus.Processing); // Set chat status to Processing + }else{ + console.log('No message input'); + sendMessage('...', false, false, null); + setChatWelcome(false); + setChatStatus(ChatStatus.Processing); // Set chat status to Processing + } + resetTranscript(); + } + } + function manualSendMessage() { + if(currentCue?.type!='chat') return; // Only process if current cue is user input + if(chatStatus!=ChatStatus.User) return; // Only process if chat status is User + + const message= refInput.current?.value?.trim(); + if(message && message.length>0) { + console.log('Manual sending message:', message); + sendMessage(message, false, false, null); + setChatWelcome(false); + setChatStatus(ChatStatus.Processing); // Set chat status to Processing + + } + resetTranscript(); + + } + + useEffect(()=>{ + + console.log('Final transcript changed:', finalTranscript); + if(finalTranscript.trim().length > 0) { + onSpeechEnd(); + } + + },[finalTranscript]); + + function startRecognition() { + + SpeechRecognition.startListening({ continuous: true, language: 'zh-TW' }).then(() => { + console.log("Speech recognition started."); + }).catch(error => { + console.error("Error starting speech recognition:", error); + }); + } + + function blurText(text) { + if(!text) return ''; + return text.replace(/./g, '*'); + } + + useEffect(()=>{ + if(audioInput && isMicrophoneAvailable) { + + startRecognition(); + + const recognition= SpeechRecognition.getRecognition(); + + recognition.onspeechstart=(e)=>{ + + console.log('Speech start:', e); + sendSpeechStart(); + + }; + // recognition.onspeechend=(e)=>{ + // console.log('Speech end:', e); + // startRecognition(); + // }; + + + + }else{ + console.log('Stopping speech recognition...'); + SpeechRecognition.stopListening(); + } + + },[audioInput]); + + + useEffect(()=>{ + + + + if((currentCue?.type=='chat' && chatStatus==ChatStatus.User) || currentCue?.type=='user_input') { + + // console.log('transcript state changed:', transcript); + + if(transcript!=finalTranscript){ + refInput.current.value = transcript; + + + // clear pause timer + // console.log('~~~~ clear pause timer'); + if(refPauseTimer.current) clearTimeout(refPauseTimer.current); + } + + if(transcript.length>0){ + // sendSpeechStart(); + // if(refChatTimer.current) clearInterval(refChatTimer.current); + } + + sendOscStatus(OSC_ADDRESS.CLIENT_INPUT, `${data.id}#${transcript}`); + // Send current input via OSC + + + } + + },[transcript]); + + + useEffect(()=>{ + + if(refCurrentCue.current?.type!='chat') return; + + if(audioUrl){ + if(refSpeaking.current) { + console.log('Still speaking, do not play AI audio yet'); + sendPrompt(); + return; + }else{ + // play ai audio + console.log('AI audio ready, play it:', audioUrl); + playAudio(audioUrl); + } + } + + },[audioUrl]); + useEffect(()=>{ + + if(refCurrentCue.current?.type!='chat' && refCurrentCue.current?.type!='user_input') return; + resetTranscript(); + + let text=''; + switch(chatStatus) { + case ChatStatus.System: + text = '等我一下\n換我說囉'; + break; + case ChatStatus.User: + text = '換你說了'; + startChatTimer(); + break; + case ChatStatus.Processing: + text = '記憶讀取中'; + break; + case ChatStatus.Message: + text = '請留言'; + break; + case ChatStatus.Clear: + default: + text = ''; + break; + } + + + sendOsc(OSC_ADDRESS.INPUT, text); + // sendSpeechEnd(); + + },[chatStatus]); + + useEffect(()=>{ + switch(status) { + case Status.SUCCESS: + console.log('Success!'); + + setStatus(Status.IDLE); + refInput.current.value = ''; + resetTranscript(); + + refContainer.current.scrollTop = refContainer.current.scrollHeight; + + break; + } + + },[status]); + + useEffect(()=>{ + + if(!nextCue) return; + console.log('Next cue:', nextCue); + + const next=cuelist.find(c => c.name === nextCue); + if(currentCue?.fadeout){ + // fade out audio + fadeOutAudio(()=>{ + console.log('fade out then play next cue:', next); + playCue(next); + setNextCue(()=>null); + }); + + + }else{ + + playCue(next); + setNextCue(null); + } + + + },[nextCue]); + + + useEffect(()=>{ + fetch(CUELIST_FILE) + .then(response => response.json()) + .then(data => { + console.log('Cuelist data:', data); + setCuelist(data.cuelist); + }) + .catch(error => { + console.error('Error fetching cuelist:', error); + }); + + + refAudioPrompt.current = new Audio('assets/sfx/sfx-05.mp3'); // Load audio prompt if available + + onOscMessageReceived(onOsc); // Set up OSC message listener + + invoke('get_ip').then((ip)=>{ + console.log('Local IP address:', ip); + setLocalIP(ip); + }); + + },[]); + + + return ( +
+
+
+ PC {data?.id} + {localIP || '...'} +
+ +
+
+ {refCurrentCue.current?.name} +
+ + + + {/* */} + + {/* */} + {/* */} + + {/*
+ + + +
*/} + +
+ + + UserId{userId} + FileId{getFileId()} + Choice{choice || ''} +
+ + +
+
+ + + + {/* */} + + + + + + + + + + + + {cuelist?.map(({id, name, description, type, auto, audioFile,...props}, index) => ( + + {/* */} + + + + + + + + + + ))} + +
IDNameDescriptionTypeAutoAudioDurationAction
{id} + + {name}{description}{EmojiType[type]}{auto ? '⤵️' : ''}{audioFile || ""}{props.duration || ''}{props.callback && `<${props.callback}>`}{props.status && `(${props.status})`}
+
+
+
+
+ {history?.map((msg, index) => ( +
+
{blurText(msg.content)}
+ {msg.prompt &&
{blurText(msg.prompt)}
} +
+ ))} + {summary &&
{summary}
} +
+ +
+ + + setAudioOutput(e.target.checked)} /> + + + + setAudioInput(e.target.checked)} /> + + + + setAutoSend(e.target.checked)} /> + + + + +
api_status= {status}
+
chat_status= {chatStatus}
+
+
+
+ ); +} \ No newline at end of file