|
|
|
|
@ -33,6 +33,7 @@ const ChatStatus={ |
|
|
|
|
Clear: 'clear', |
|
|
|
|
End: 'end', |
|
|
|
|
Playing: 'playing', |
|
|
|
|
Message: 'message', |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
const Voice={ |
|
|
|
|
@ -50,10 +51,12 @@ export function FreeFlow(){ |
|
|
|
|
const [audioInput, setAudioInput] = useState(true); |
|
|
|
|
const [autoSend, setAutoSend] = useState(true); |
|
|
|
|
const [chatStatus, setChatStatus] = useState(ChatStatus.System); // System, User, Processing |
|
|
|
|
const { userId, setUserId, getFileId, setPassword, reset:resetUser, uploadHistory, setSummary, summary,setChoice } = useUser(); |
|
|
|
|
const { userId, setUserId, getFileId, setPassword, reset:resetUser, uploadHistory, setSummary, summary,setChoice, getUploadFolder,getDataId } = useUser(); |
|
|
|
|
|
|
|
|
|
const refTimer=useRef(); |
|
|
|
|
const refAudio=useRef(); |
|
|
|
|
const refAudioPrompt=useRef(); |
|
|
|
|
|
|
|
|
|
const refInput=useRef(); |
|
|
|
|
const refLight=useRef(); |
|
|
|
|
|
|
|
|
|
@ -91,12 +94,12 @@ export function FreeFlow(){ |
|
|
|
|
|
|
|
|
|
//TODO: if cue end, don't play audio |
|
|
|
|
if(refCurrentCue.current?.type=='chat'){ |
|
|
|
|
if(refChatCueEnd.current) { |
|
|
|
|
console.log('Chat cue has ended, not playing audio:', url); |
|
|
|
|
setChatStatus(ChatStatus.Clear); // Reset chat status to Clear |
|
|
|
|
onCueEnd(); |
|
|
|
|
return; |
|
|
|
|
} |
|
|
|
|
// if(refChatCueEnd.current) { |
|
|
|
|
// console.log('Chat cue has ended, not playing audio:', url); |
|
|
|
|
// setChatStatus(ChatStatus.Clear); // Reset chat status to Clear |
|
|
|
|
// onCueEnd(); |
|
|
|
|
// return; |
|
|
|
|
// } |
|
|
|
|
// if audio time larger than cue remaining time, don't play audio |
|
|
|
|
|
|
|
|
|
} |
|
|
|
|
@ -152,12 +155,17 @@ export function FreeFlow(){ |
|
|
|
|
|
|
|
|
|
updatePrompt(prompt); |
|
|
|
|
sendOsc(OSC_ADDRESS.PROMPT, prompt); |
|
|
|
|
} |
|
|
|
|
// play audio for prompt |
|
|
|
|
refAudioPrompt.current?.play().catch(error => { |
|
|
|
|
console.error("Audio prompt playback error:", error); |
|
|
|
|
}); |
|
|
|
|
|
|
|
|
|
if(refChatCueEnd.current){ |
|
|
|
|
console.log('Talking ended, ending current cue'); |
|
|
|
|
onCueEnd(); // End the current cue if chat cue has ended |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
// if(refChatCueEnd.current){ |
|
|
|
|
// console.log('Talking ended, ending current cue'); |
|
|
|
|
// onCueEnd(); // End the current cue if chat cue has ended |
|
|
|
|
// } |
|
|
|
|
} |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
@ -186,6 +194,9 @@ export function FreeFlow(){ |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
// clear unity hint |
|
|
|
|
sendOsc(OSC_ADDRESS.HINT, ''); // Clear hint message |
|
|
|
|
sendOsc(OSC_ADDRESS.INPUT, ''); // Clear input message |
|
|
|
|
|
|
|
|
|
switch(cue.type){ |
|
|
|
|
case 'chat': |
|
|
|
|
@ -204,31 +215,30 @@ export function FreeFlow(){ |
|
|
|
|
setChatWelcome(false); |
|
|
|
|
setChatStatus(ChatStatus.Clear); |
|
|
|
|
break; |
|
|
|
|
case 'summary': |
|
|
|
|
console.log('Getting summary...'); |
|
|
|
|
|
|
|
|
|
setChatStatus(ChatStatus.Clear); // Set chat status to Processing |
|
|
|
|
getSummary(history.map(el=>`${el.role}:${el.content}`).join('\n'), data).then(summary_ => { |
|
|
|
|
// case 'summary': |
|
|
|
|
// console.log('Getting summary...'); |
|
|
|
|
|
|
|
|
|
console.log('Summary:', summary_); |
|
|
|
|
onCueEnd(); // End the current cue after getting summary |
|
|
|
|
// setChatStatus(ChatStatus.Clear); // Set chat status to Processing |
|
|
|
|
// getSummary(history.map(el=>`${el.role}:${el.content}`).join('\n'), data).then(summary_ => { |
|
|
|
|
|
|
|
|
|
setSummary(summary_?.result); |
|
|
|
|
refContainer.current.scrollTop = refContainer.current.scrollHeight; // Scroll to bottom |
|
|
|
|
// console.log('Summary:', summary_); |
|
|
|
|
// onCueEnd(); // End the current cue after getting summary |
|
|
|
|
|
|
|
|
|
// sendOsc(OSC_ADDRESS.SUMMARY, summary_?.result); |
|
|
|
|
// sendOsc(OSC_ADDRESS.EXPORT, `${getFileId()}#${summary_?.result}#${password}`); // Save summary with file ID |
|
|
|
|
// setSummary(summary_?.result); |
|
|
|
|
// refContainer.current.scrollTop = refContainer.current.scrollHeight; // Scroll to bottom |
|
|
|
|
|
|
|
|
|
}).catch(error => { |
|
|
|
|
console.error('Error getting summary:', error); |
|
|
|
|
}); |
|
|
|
|
// }).catch(error => { |
|
|
|
|
// console.error('Error getting summary:', error); |
|
|
|
|
// }); |
|
|
|
|
|
|
|
|
|
break; |
|
|
|
|
// break; |
|
|
|
|
case 'user_input': |
|
|
|
|
console.log('User input cue, setting chat status to User'); |
|
|
|
|
setChatStatus(ChatStatus.Clear); // Set chat status to User for user input cues |
|
|
|
|
setChatStatus(ChatStatus.Message); // Set chat status to User |
|
|
|
|
resetTranscript(); // Reset transcript for user input |
|
|
|
|
break; |
|
|
|
|
default: |
|
|
|
|
setChatStatus(ChatStatus.Clear); |
|
|
|
|
break; |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
@ -275,17 +285,19 @@ export function FreeFlow(){ |
|
|
|
|
const cue= refCurrentCue.current; // Get the current cue from ref |
|
|
|
|
|
|
|
|
|
if(cue.type=='chat'){ |
|
|
|
|
if(chatStatus==ChatStatus.System) { |
|
|
|
|
console.log('Still talking...'); |
|
|
|
|
refChatCueEnd.current=true; |
|
|
|
|
return; |
|
|
|
|
} |
|
|
|
|
// if(chatStatus==ChatStatus.System) { |
|
|
|
|
// console.log('Still talking...'); |
|
|
|
|
// refChatCueEnd.current=true; |
|
|
|
|
// return; |
|
|
|
|
// } |
|
|
|
|
console.log('save chat history:', history); |
|
|
|
|
uploadHistory(history); // Save chat history when cue ends |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
if(cue.callback==OSC_ADDRESS.DISCARD) { |
|
|
|
|
sendOsc(OSC_ADDRESS.CHOICE, OSC_ADDRESS.DISCARD); // Send OSC discard message |
|
|
|
|
setPassword(); |
|
|
|
|
sendOsc(OSC_ADDRESS.EXPORT, `${getUploadFolder()}#${getFileId()}#${summary}#${mess}`); |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
if(cue.hint!=null){ |
|
|
|
|
@ -294,6 +306,22 @@ export function FreeFlow(){ |
|
|
|
|
sendOsc(OSC_ADDRESS.HINT, ''); // Clear hint message |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
if(cue.callback=='summary') { |
|
|
|
|
console.log('Getting summary...'); |
|
|
|
|
|
|
|
|
|
getSummary(history.map(el=>`${el.role}:${el.content}`).join('\n'), data).then(summary_ => { |
|
|
|
|
|
|
|
|
|
console.log('Summary:', summary_); |
|
|
|
|
onCueEnd(); // End the current cue after getting summary |
|
|
|
|
|
|
|
|
|
setSummary(summary_?.result); |
|
|
|
|
refContainer.current.scrollTop = refContainer.current.scrollHeight; // Scroll to bottom |
|
|
|
|
|
|
|
|
|
}).catch(error => { |
|
|
|
|
console.error('Error getting summary:', error); |
|
|
|
|
}); |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
refAudio.current?.pause(); // Pause any playing audio |
|
|
|
|
console.log('onCueEnd:', cue.id); |
|
|
|
|
|
|
|
|
|
@ -339,7 +367,7 @@ export function FreeFlow(){ |
|
|
|
|
case NUMPAD_TYPE.PASSWORD: |
|
|
|
|
setPassword(()=>mess); |
|
|
|
|
// sendOsc(OSC_ADDRESS.PASSWORD, mess); // Send OSC password message |
|
|
|
|
sendOsc(OSC_ADDRESS.EXPORT, `${getFileId()}#${summary}#${mess}`); |
|
|
|
|
sendOsc(OSC_ADDRESS.EXPORT, `${getUploadFolder()}#${getDataId()}#${summary}#${getFileId(mess)}`); |
|
|
|
|
sendOsc(OSC_ADDRESS.CHOICE, OSC_ADDRESS.SAVE); // Send OSC save choice message |
|
|
|
|
break; |
|
|
|
|
} |
|
|
|
|
@ -474,7 +502,27 @@ export function FreeFlow(){ |
|
|
|
|
useEffect(()=>{ |
|
|
|
|
|
|
|
|
|
resetTranscript(); |
|
|
|
|
sendOsc(OSC_ADDRESS.INPUT, chatStatus); |
|
|
|
|
|
|
|
|
|
let text=''; |
|
|
|
|
switch(chatStatus) { |
|
|
|
|
case ChatStatus.System: |
|
|
|
|
text = '等我一下\n換我說囉'; |
|
|
|
|
break; |
|
|
|
|
case ChatStatus.User: |
|
|
|
|
text = '換你說了'; |
|
|
|
|
break; |
|
|
|
|
case ChatStatus.Processing: |
|
|
|
|
text = '記憶讀取中'; |
|
|
|
|
break; |
|
|
|
|
case ChatStatus.Message: |
|
|
|
|
text = '請留言'; |
|
|
|
|
break; |
|
|
|
|
case ChatStatus.Clear: |
|
|
|
|
default: |
|
|
|
|
text = ''; |
|
|
|
|
break; |
|
|
|
|
} |
|
|
|
|
sendOsc(OSC_ADDRESS.INPUT, text); |
|
|
|
|
|
|
|
|
|
},[chatStatus]); |
|
|
|
|
|
|
|
|
|
@ -506,6 +554,9 @@ export function FreeFlow(){ |
|
|
|
|
console.error('Error fetching cuelist:', error); |
|
|
|
|
}); |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
refAudioPrompt.current = new Audio('assets/sfx/sfx-05.mp3'); // Load audio prompt if available |
|
|
|
|
|
|
|
|
|
},[]); |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
@ -591,7 +642,7 @@ export function FreeFlow(){ |
|
|
|
|
</div> |
|
|
|
|
<textarea ref={refInput} name="message" rows={2} |
|
|
|
|
className={`w-full border-1 resize-none p-2 disabled:bg-gray-500`} |
|
|
|
|
disabled={chatStatus!=ChatStatus.User}></textarea> |
|
|
|
|
disabled={chatStatus!=ChatStatus.User && chatStatus!=ChatStatus.Message}></textarea> |
|
|
|
|
<div className="flex flex-row justify-end gap-2 flex-wrap"> |
|
|
|
|
<span className="flex flex-row gap-1"> |
|
|
|
|
<label>audio_output</label> |
|
|
|
|
@ -606,6 +657,7 @@ export function FreeFlow(){ |
|
|
|
|
<input type='checkbox' checked={autoSend} onChange={(e) => setAutoSend(e.target.checked)} /> |
|
|
|
|
</span> |
|
|
|
|
<button onClick={resetTranscript}>reset transcript</button> |
|
|
|
|
<button onClick={sendMessage} disabled={chatStatus!=ChatStatus.User && chatStatus!=ChatStatus.Message}>Send</button> |
|
|
|
|
|
|
|
|
|
<div className="rounded-2xl bg-gray-300 self-end px-4 tracking-widest">api_status= {status}</div> |
|
|
|
|
<div className="rounded-2xl bg-gray-300 self-end px-4 tracking-widest">chat_status= {chatStatus}</div> |
|
|
|
|
|
