You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
 
 
 
 

814 lines
29 KiB

import { invoke } from '@tauri-apps/api/core';
import { useEffect, useRef, useState } from "react";
import SpeechRecognition, { useSpeechRecognition } from 'react-speech-recognition';
import { Countdown } from "../comps/timer";
import { Status, useChat } from "../util/useChat";
import { getSummary } from "../util/chat";
import { saveHistory } from "../util/output";
import NumPad, { NUMPAD_TYPE } from "../comps/numpad";
import { Light } from "../comps/light";
import { useData } from "../util/useData";
import VoiceAnalysis from "../comps/voiceanalysis";
import { sendOsc, OSC_ADDRESS, updatePrompt, onOscMessageReceived, sendOscStatus } from "../util/osc";
import { DebugControl, TEST_PROMPT } from "../comps/debug";
import { useUser } from "../util/useUser";
import { set } from 'zod/v4';
const CUELIST_FILE = 'cuelist_demo3.json';
const AUDIO_FADE_TIME=3000; // in ms
const EmojiType={
phone: '📞',
headphone: '🎧',
speaker: '🔊',
chat: '🤖',
chat_end: '🤖',
user_input: '💬',
}
const ChatStatus={
System: 'system',
User: 'user',
Processing: 'processing',
Clear: 'clear',
End: 'end',
Playing: 'playing',
Message: 'message',
}
const Voice={
ONYX: 'onyx',
SHIMMER: 'shimmer',
}
export function FreeFlow(){
const { data }=useData();
const [cuelist, setCuelist] = useState([]);
const [currentCue, setCurrentCue] = useState(null);
const [nextCue, setNextCue] = useState(null);
const [localIP, setLocalIP] = useState(null);
const [chatWelcome, setChatWelcome] = useState(null);
const [audioInput, setAudioInput] = useState(true);
const [autoSend, setAutoSend] = useState(true);
const [chatStatus, setChatStatus] = useState(ChatStatus.System); // System, User, Processing
const [padInput, setPadInput] = useState(null);
const { userId, setUserId, getFileId, setPassword, reset:resetUser, uploadHistory, setSummary, summary,setChoice,choice, getUploadFolder,getDataId } = useUser();
const refTimer=useRef();
const refAudio=useRef();
const refAudioPrompt=useRef();
const refInput=useRef();
// const refLight=useRef();
const refPauseTimer=useRef();
const refSpeechPaused=useRef(false);
const refChatCueEnd=useRef(false);
const refContainer=useRef();
const refCurrentCue= useRef(null);
const refData=useRef(data);
const { history, status, reset, sendMessage, setStatus, audioOutput, setAudioOutput, stop:stopChat,
audioUrl }=useChat();
const {
transcript,
finalTranscript,
listening,
resetTranscript,
browserSupportsSpeechRecognition,
isMicrophoneAvailable,
}=useSpeechRecognition();
function resetData() {
setSummary(null);
reset();
sendOsc(OSC_ADDRESS.CHOICE, 'reset');
sendOsc(OSC_ADDRESS.SPEECH, 'stop');
}
function onOsc(payload){
console.log('onOsc', payload);
const address=payload.addr;
const message=payload.args[0];
const params=message.split('#');
switch(address){
case OSC_ADDRESS.PLAY_CUE:
if(params[0]=='all' || params[0]==refData.current?.id)
setNextCue(()=>params[1]);
break;
case OSC_ADDRESS.STOP_CUE:
if(params[0]=='all' || params[0]==refData.current?.id)
onStop();
break;
case OSC_ADDRESS.RESET_CUE:
if(params[0]=='all' || params[0]==refData.current?.id){
sendOsc(OSC_ADDRESS.STATUS, 'reset');
onStop();
resetData();
}
break;
}
// Handle OSC messages here
}
function playAudio(url){
if(!url) return;
//TODO: if cue end, don't play audio
if(refCurrentCue.current?.type=='chat'){
// if(refChatCueEnd.current) {
// console.log('Chat cue has ended, not playing audio:', url);
// setChatStatus(ChatStatus.Clear); // Reset chat status to Clear
// onCueEnd();
// return;
// }
// if audio time larger than cue remaining time, don't play audio
}
console.log('Playing audio:', url);
if(refAudio.current) {
refAudio.current.pause(); // Stop any currently playing audio
}
let audioUrl = url;
// if(voice==Voice.SHIMMER) audioUrl = url.replace(Voice.ONYX, Voice.SHIMMER);
// console.log('Using voice:', voice, 'for audio:', audioUrl);
const audio = new Audio(audioUrl);
audio.loop=refCurrentCue.current?.loop || false; // Set loop if defined in cue
audio.play().catch(error => {
console.error("Audio playback error:", error);
});
audio.addEventListener("loadedmetadata", () => {
if(refCurrentCue.current?.type!='chat' && refCurrentCue.current?.type!='user_input') {
refTimer.current?.restart(audio.duration*1000 || 0);
}else{
if(refCurrentCue.current?.type=='chat') setChatStatus(()=>ChatStatus.System);
else setChatStatus(()=>ChatStatus.Playing);
}
});
audio.onended = () => {
if(refCurrentCue.current?.type!='chat'){
setChatStatus(ChatStatus.End);
onCueEnd();
console.log('Audio ended, ending current cue');
}else{
// if history contains user input, send it
const user_input = history.find(msg => msg.role === 'user');
if(user_input && user_input.content.trim() !== '') {
sendOsc(OSC_ADDRESS.STATUS, 'go'); // Send OSC status message
}
// send prompt
let raw_prompt=history[history.length-1]?.prompt || '';
if(raw_prompt && raw_prompt.trim() !== '') {
const prompt = `${data?.sd_prompt_prefix || ''}${raw_prompt}${data?.sd_prompt_suffix || ''}`;
updatePrompt(prompt);
sendOsc(OSC_ADDRESS.PROMPT, prompt);
// play audio for prompt
refAudioPrompt.current?.play().catch(error => {
console.error("Audio prompt playback error:", error);
});
refAudioPrompt.current.onended = () => {
console.log('Audio prompt ended, setting chat status to User');
setChatStatus(ChatStatus.User); // Set chat status to User after audio ends
}
}else{
setChatStatus(()=>ChatStatus.User); // Reset chat status to User after audio ends
}
}
}
refAudio.current = audio; // Store the new audio reference
}
function fadeOutAudio(callback){
if(refAudio.current) {
console.log('Fading out audio');
let audio = refAudio.current;
let fadeOutInterval = setInterval(() => {
if (audio.volume > 0) {
audio.volume =Math.max(0, audio.volume - 1.0/(AUDIO_FADE_TIME/100)); // Decrease volume gradually
} else {
clearInterval(fadeOutInterval);
audio.pause();
audio.volume = 0; // Reset volume for next play
if(callback) callback();
}
}, 100); // Decrease volume every 100ms
}else{
if(callback) callback();
}
}
function playCue(cue) {
if(!cue) return;
console.log('Playing cue:', cue);
// stop audio
if(refAudio.current) refAudio.current.pause();
setCurrentCue(cue);
refCurrentCue.current = cue; // Store the current cue in ref
if(parseFloat(cue.id)<=4.2){
// Special case for starting a conversation
console.log('clear conversation...');
reset();
const prompt = `${data?.sd_prompt_prefix || ''}${TEST_PROMPT}${data?.sd_prompt_suffix || ''}`;
updatePrompt(prompt);
}
// clear unity hint
sendOsc(OSC_ADDRESS.HINT, ''); // Clear hint message
sendOsc(OSC_ADDRESS.INPUT, ''); // Clear input message
sendOsc(OSC_ADDRESS.SPEECH, 'stop');
switch(cue.type){
case 'chat':
// Special case for starting a conversation
refChatCueEnd.current=false;
resetTranscript();
console.log('Starting conversation...');
setChatStatus(ChatStatus.User);
//sendMessage(null, false, false, null); // Send initial message with voice
//setChatWelcome(true);
resetData(); // Reset data for new conversation
break;
// case 'chat_end':
// const message= refInput.current?.value?.trim();
// console.log('Ending conversation with message:', message);
// sendMessage(message, false, true, null);
// setChatWelcome(false);
// setChatStatus(ChatStatus.Clear);
// break;
case 'summary':
console.log('Getting summary...');
setChatStatus(ChatStatus.Clear); // Set chat status to Processing
let message=refInput.current?.value?.trim() || history.map(el=>`${el.role}:${el.content}`).join('\n');
console.log('Summary input message:', message);
if(!message || message.length==0) {
setSummary();
console.log('no message input, clear summary');
onCueEnd(); // End the current cue after getting summary
}else{
getSummary(message, data).then(summary_ => {
console.log('Summary:', summary_);
onCueEnd(); // End the current cue after getting summary
setSummary(()=>summary_?.result);
refContainer.current.scrollTop = refContainer.current.scrollHeight; // Scroll to bottom
}).catch(error => {
console.error('Error getting summary:', error);
});
}
break;
case 'user_input':
setChatStatus(ChatStatus.Message); // Set chat status to User
resetTranscript(); // Reset transcript for user input
break;
default:
setChatStatus(ChatStatus.Clear);
break;
}
// if(cue.callback=='fade_in_light') refLight.current.fadeIn(); // Fade in light for conversation start
// if(cue.callback=='fade_out_light') refLight.current.fadeOut(); // Fade out light for conversation end
if(cue.audioFile){
playAudio(cue.audioFile);
}
if(cue.duration){
refTimer.current.restart(cue.duration*1000, ()=>{
onCueEnd(cue);
});
}
// control unity
if(cue.status && cue.status!='go') {
sendOsc(OSC_ADDRESS.STATUS, cue.status); // Send OSC status message
if(cue.status=='reset') {
// refLight.current.set(1);
resetData();
}
}
if(cue.type=='chat' || cue.type=='user_input') {
sendOsc(OSC_ADDRESS.COUNTDOWN, cue.duration || '0'); // Send OSC countdown message
}else{
sendOsc(OSC_ADDRESS.COUNTDOWN, '0'); // Reset countdown for non-chat cues
}
sendOscStatus(OSC_ADDRESS.CLIENT_STATUS,`${data.id}#playcue#${cue.id}`);
console.log('~~~~ clear pause timer');
if(refPauseTimer.current) clearTimeout(refPauseTimer.current);
// refSpeechPaused.current=false;
}
function onCueEnd() {
refTimer.current?.stop(); // Stop the timer when cue ends
if(!refCurrentCue.current) return;
const cue= refCurrentCue.current; // Get the current cue from ref
if(cue.type=='chat'){
if(chatStatus==ChatStatus.System) {
console.log('Still talking...');
refChatCueEnd.current=true;
return;
}
console.log('save chat history:', history);
uploadHistory(history); // Save chat history when cue ends
}
if(cue.hint!=null){
sendOsc(OSC_ADDRESS.HINT, cue.hint); // Send OSC hint message
}else{
sendOsc(OSC_ADDRESS.HINT, ''); // Clear hint message
}
sendOsc(OSC_ADDRESS.SPEECH, 'stop');
refAudio.current?.pause(); // Pause any playing audio
console.log('onCueEnd:', cue.id);
resetTranscript(); // Reset transcript after cue ends
sendOscStatus(OSC_ADDRESS.CLIENT_STATUS, `${data.id}#endcue#${cue.id}`);
if(cue.auto || cue.callback=='numpad'){
playCue(cuelist.find(c => c.id === cue.nextcue));
}
}
function onStop(){
console.log('Stopping current cue');
if(refAudio.current) {
refAudio.current.pause();
refAudio.current = null;
}
setCurrentCue(null);
refCurrentCue.current = null; // Clear the current cue reference
refTimer.current.restart(0);
stopChat(); // Stop chat processing
}
function onNumpad(mess){
setPadInput(mess);
}
useEffect(()=>{
if(padInput==null) return;
console.log('Numpad input:', padInput);
if(refCurrentCue.current?.callback!='numpad') return;
let cue=refCurrentCue.current;
let next=cue.nextcue;
switch(cue.numpad_type){
case NUMPAD_TYPE.USERID:
setUserId(()=>padInput);
break;
case NUMPAD_TYPE.CHOICE:
next=cue.branch[padInput.toString()].nextcue;
setChoice(()=>cue.branch[padInput.toString()].description); // Set choice for user input
break;
case NUMPAD_TYPE.PASSWORD:
setPassword(()=>padInput);
// sendOsc(OSC_ADDRESS.PASSWORD, mess); // Send OSC password message
const user_input = history.find(msg => msg.role === 'user');
const default_image=!(user_input && user_input.content.trim() !== '');
sendOsc(OSC_ADDRESS.EXPORT, `${getUploadFolder()}#${getDataId()}#${summary||''}#${getFileId(padInput)}#${choice||''}#${default_image?'default':'generated'}`); // Send OSC export message
sendOsc(OSC_ADDRESS.CHOICE, choice); // Send OSC save choice message
break;
}
if(next){
onStop();
console.log('Finish enter number, next cue:', next);
playCue(cuelist.find(c => c.id === next));
}
},[padInput]);
useEffect(()=>{
if(userId>=1 && userId<=24) {
console.log('User ID set:', userId);
//playCue(cuelist.find(c => c.id === refCurrentCue.current.nextcue)); // Play cue 5 when userId is set
}
},[userId]);
function onSpeechEnd(){
sendOsc(OSC_ADDRESS.SPEECH, 'stop');
if(currentCue?.type!='chat') return; // Only process if current cue is user input
if(chatStatus!=ChatStatus.User) return; // Only process if chat status is User
console.log('~~~ on speech end, start pause timer',data.speech_idle_time);
// refSpeechPaused.current=true;
if(refPauseTimer.current) clearTimeout(refPauseTimer.current);
refPauseTimer.current=setTimeout(()=>{
console.log('~~~ pause timer ended, process speech');
// if(refSpeechPaused.current)
processSpeech();
}, data.speech_idle_time);
}
function processSpeech(){
if(currentCue?.type!='chat') return; // Only process if current cue is user input
console.log('processSpeech:', finalTranscript);
if(refChatCueEnd.current) {
console.log('Chat cue has ended, do not processing speech');
onCueEnd();
return;
}
if(autoSend && transcript.trim().length > 0) {
console.log('Auto sending transcript:', transcript);
// onCueEnd();
const message= refInput.current?.value?.trim();
if(message && message.length>0) {
console.log('Ending conversation with message:', message);
sendMessage(message, false, false, null);
setChatWelcome(false);
setChatStatus(ChatStatus.Processing); // Set chat status to Processing
}
resetTranscript();
}
}
function manualSendMessage() {
if(currentCue?.type!='chat') return; // Only process if current cue is user input
if(chatStatus!=ChatStatus.User) return; // Only process if chat status is User
const message= refInput.current?.value?.trim();
if(message && message.length>0) {
console.log('Manual sending message:', message);
sendMessage(message, false, false, null);
setChatWelcome(false);
setChatStatus(ChatStatus.Processing); // Set chat status to Processing
}
resetTranscript();
}
useEffect(()=>{
// console.log('Final transcript changed:', finalTranscript);
if(finalTranscript.trim().length > 0) {
onSpeechEnd();
}
},[finalTranscript]);
function startRecognition() {
SpeechRecognition.startListening({ continuous: true, language: 'zh-TW' }).then(() => {
console.log("Speech recognition started.");
}).catch(error => {
console.error("Error starting speech recognition:", error);
});
}
useEffect(()=>{
if(audioInput && isMicrophoneAvailable) {
startRecognition();
const recognition= SpeechRecognition.getRecognition();
recognition.onspeechstart=(e)=>{
console.log('Speech start:', e);
};
// recognition.onspeechend=(e)=>{
// console.log('Speech end:', e);
// startRecognition();
// };
}else{
console.log('Stopping speech recognition...');
SpeechRecognition.stopListening();
}
},[audioInput]);
useEffect(()=>{
if((currentCue?.type=='chat' && chatStatus==ChatStatus.User) || currentCue?.type=='user_input') {
// console.log('transcript state changed:', transcript);
if(transcript!=finalTranscript){
refInput.current.value = transcript;
// clear pause timer
// console.log('~~~~ clear pause timer');
if(refPauseTimer.current) clearTimeout(refPauseTimer.current);
refSpeechPaused.current=false;
}
sendOsc(OSC_ADDRESS.SPEECH, 'start');
sendOscStatus(OSC_ADDRESS.CLIENT_INPUT, `${data.id}#${transcript}`);
// Send current input via OSC
}
},[transcript]);
useEffect(()=>{
if(audioUrl) playAudio(audioUrl);
},[audioUrl]);
useEffect(()=>{
resetTranscript();
let text='';
switch(chatStatus) {
case ChatStatus.System:
text = '等我一下\n換我說囉';
break;
case ChatStatus.User:
text = '換你說了';
break;
case ChatStatus.Processing:
text = '記憶讀取中';
break;
case ChatStatus.Message:
text = '請留言';
break;
case ChatStatus.Clear:
default:
text = '';
break;
}
sendOsc(OSC_ADDRESS.SPEECH, 'stop');
sendOsc(OSC_ADDRESS.INPUT, text);
},[chatStatus]);
useEffect(()=>{
switch(status) {
case Status.SUCCESS:
console.log('Success!');
setStatus(Status.IDLE);
refInput.current.value = '';
resetTranscript();
refContainer.current.scrollTop = refContainer.current.scrollHeight;
break;
}
},[status]);
useEffect(()=>{
if(!nextCue) return;
console.log('Next cue:', nextCue);
const next=cuelist.find(c => c.name === nextCue);
if(currentCue?.fadeout){
// fade out audio
fadeOutAudio(()=>{
console.log('fade out then play next cue:', next);
playCue(next);
setNextCue(()=>null);
});
}else{
playCue(next);
setNextCue(null);
}
},[nextCue]);
useEffect(()=>{
fetch(CUELIST_FILE)
.then(response => response.json())
.then(data => {
console.log('Cuelist data:', data);
setCuelist(data.cuelist);
})
.catch(error => {
console.error('Error fetching cuelist:', error);
});
refAudioPrompt.current = new Audio('assets/sfx/sfx-05.mp3'); // Load audio prompt if available
onOscMessageReceived(onOsc); // Set up OSC message listener
invoke('get_ip').then((ip)=>{
console.log('Local IP address:', ip);
setLocalIP(ip);
});
},[]);
return (
<main className="items-start">
<section className="flex-1 flex flex-col gap-2 self-stretch overflow-hidden">
<div className="bg-purple-300 text-3xl flex flex-row justify-between px-4 items-center gap-1">
<span className='font-bold'>PC {data?.id}</span>
<span className="">{localIP || '...'}</span>
</div>
<DebugControl/>
<div className="w-full p-2 grid grid-cols-4 gap-2 items-stretch justify-center *:max-h-[5rem]">
<div className="bg-gray-100 text-4xl font-bold mb-4 flex justify-center items-center">
{refCurrentCue.current?.name}
</div>
<Countdown ref={refTimer} clientId={data?.id}/>
<button className="!bg-red-300" onClick={onStop}>Stop</button>
{/* <button onClick={saveImage}>Save image</button> */}
<NumPad onSend={onNumpad}
disabled={currentCue?.callback !== 'numpad'}
type={currentCue?.numpad_type}
clientId={data?.id}
/>
{/* <Light ref={refLight} /> */}
<VoiceAnalysis/>
{/* <div className="flex flex-col">
<label className="text-center">Voice</label>
<button className={`${voice==Voice.ONYX && 'bg-gray-300'}`} onClick={() => setVoice(Voice.ONYX)}>Onyx</button>
<button className={`${voice==Voice.SHIMMER && 'bg-gray-300'}`} onClick={() => setVoice(Voice.SHIMMER)}>Shimmer</button>
</div> */}
<div className="grid grid-cols-[1fr_2fr] relative text-xs">
<label className="col-span-1">user</label>
<span></span>
<span>UserId</span><span>{userId}</span>
<span>FileId</span><span>{getFileId()}</span>
<span>Choice</span><span>{choice || ''}</span>
</div>
</div>
<div className="flex-1 overflow-y-auto">
<table className="border-collapse **:border-y w-full **:p-2 text-sm">
<thead>
<tr className="text-left">
{/* <th>ID</th> */}
<th></th>
<th>Name</th>
<th>Description</th>
<th>Type</th>
<th>Auto</th>
<th>Audio</th>
<th>Duration</th>
<th>Action</th>
</tr>
</thead>
<tbody>
{cuelist?.map(({id, name, description, type, auto, audioFile,...props}, index) => (
<tr key={id}>
{/* <td>{id}</td> */}
<td>
<button className="rounded-full !bg-green-200"
onClick={()=>{
playCue({id, name, description, type, auto, audioFile, ...props});
}}>go</button>
</td>
<td>{name}</td>
<td>{description}</td>
<td>{EmojiType[type]}</td>
<td>{auto ? '⤵' : ''}</td>
<td>{audioFile || ""}</td>
<td>{props.duration || ''}</td>
<td>{props.callback && `<${props.callback}>`}{props.status && `(${props.status})`}</td>
</tr>
))}
</tbody>
</table>
</div>
</section>
<section className="flex-1 self-stretch overflow-y-auto flex flex-col justify-end gap-2">
<div ref={refContainer} className="flex-1 flex flex-col overflow-y-auto gap-2">
{history?.map((msg, index) => (
<div key={index} className={`w-5/6 ${msg.role=='user'? 'self-end':''}`}>
<div className={`${msg.role=='user'? 'bg-green-300':'bg-pink-300'} px-2`}>{msg.content}</div>
{msg.prompt && <div className="text-xs bg-gray-200">{msg.prompt}</div>}
</div>
))}
{summary && <div className="w-full self-center bg-blue-200 px-2">{summary}</div>}
</div>
<textarea ref={refInput} name="message" rows={2}
className={`w-full border-1 resize-none p-2 disabled:bg-gray-500`}
disabled={chatStatus!=ChatStatus.User && chatStatus!=ChatStatus.Message}></textarea>
<div className="flex flex-row justify-end gap-2 flex-wrap">
<span className="flex flex-row gap-1">
<label>audio_output</label>
<input type='checkbox' checked={audioOutput} onChange={(e) => setAudioOutput(e.target.checked)} />
</span>
<span className="flex flex-row gap-1">
<label>audio_input</label>
<input type='checkbox' checked={audioInput} onChange={(e) => setAudioInput(e.target.checked)} />
</span>
<span className="flex flex-row gap-1">
<label>auto_send</label>
<input type='checkbox' checked={autoSend} onChange={(e) => setAutoSend(e.target.checked)} />
</span>
<button onClick={resetTranscript}>reset transcript</button>
<button onClick={manualSendMessage} disabled={chatStatus!=ChatStatus.User && chatStatus!=ChatStatus.Message}>Send</button>
<div className="rounded-2xl bg-gray-300 self-end px-4 tracking-widest">api_status= {status}</div>
<div className="rounded-2xl bg-gray-300 self-end px-4 tracking-widest">chat_status= {chatStatus}</div>
</div>
</section>
</main>
);
}