diff --git a/vite/package.json b/vite/package.json index 9133816..07a23df 100644 --- a/vite/package.json +++ b/vite/package.json @@ -7,7 +7,8 @@ "dev": "vite", "build": "vite build", "lint": "eslint .", - "preview": "vite preview" + "preview": "vite preview", + "deploy":"tauri build && copy /Y src-tauri\\target\\release\\app.exe C:\\uc\\24070-Sync\\app.exe" }, "dependencies": { "@tailwindcss/vite": "^4.1.8", diff --git a/vite/public/assets/ai/sfx-06-ai-04-record-03-ai-05-sfx-07.mp3 b/vite/public/assets/ai/sfx-06-ai-04-record-03-ai-05-sfx-07.mp3 index f42f701..90a153a 100644 Binary files a/vite/public/assets/ai/sfx-06-ai-04-record-03-ai-05-sfx-07.mp3 and b/vite/public/assets/ai/sfx-06-ai-04-record-03-ai-05-sfx-07.mp3 differ diff --git a/vite/src/comps/debug.jsx b/vite/src/comps/debug.jsx index 304c8bf..5ba89b2 100644 --- a/vite/src/comps/debug.jsx +++ b/vite/src/comps/debug.jsx @@ -1,7 +1,7 @@ import { sendOsc, OSC_ADDRESS, updatePrompt } from "../util/osc" import { useData } from '../util/useData.jsx'; -export const TEST_PROMPT='A distant, ambient setting with gentle light'; +export const TEST_PROMPT='a window frame, beach outside, island far away'; export function DebugControl({refLight}){ @@ -24,7 +24,7 @@ export function DebugControl({refLight}){ {/*
*/} - + {/*
*/} diff --git a/vite/src/pages/flow_free.jsx b/vite/src/pages/flow_free.jsx index 6a8741c..65e87e8 100644 --- a/vite/src/pages/flow_free.jsx +++ b/vite/src/pages/flow_free.jsx @@ -14,6 +14,7 @@ import VoiceAnalysis from "../comps/voiceanalysis"; import { sendOsc, OSC_ADDRESS, updatePrompt, onOscMessageReceived, sendOscStatus } from "../util/osc"; import { DebugControl, TEST_PROMPT } from "../comps/debug"; import { useUser } from "../util/useUser"; +import { set } from 'zod/v4'; const CUELIST_FILE = 'cuelist_demo3.json'; @@ -56,6 +57,9 @@ export function FreeFlow(){ const [audioInput, setAudioInput] = useState(true); const [autoSend, setAutoSend] = useState(true); const [chatStatus, setChatStatus] = useState(ChatStatus.System); // System, User, Processing + + const [padInput, setPadInput] = useState(null); + const { userId, setUserId, getFileId, setPassword, reset:resetUser, uploadHistory, setSummary, summary,setChoice,choice, getUploadFolder,getDataId } = useUser(); const refTimer=useRef(); @@ -93,6 +97,7 @@ export function FreeFlow(){ reset(); sendOsc(OSC_ADDRESS.CHOICE, 'reset'); + sendOsc(OSC_ADDRESS.SPEECH, 'stop'); } function onOsc(payload){ @@ -233,6 +238,9 @@ export function FreeFlow(){ if(!cue) return; console.log('Playing cue:', cue); + + // stop audio + if(refAudio.current) refAudio.current.pause(); @@ -254,6 +262,7 @@ export function FreeFlow(){ // clear unity hint sendOsc(OSC_ADDRESS.HINT, ''); // Clear hint message sendOsc(OSC_ADDRESS.INPUT, ''); // Clear input message + sendOsc(OSC_ADDRESS.SPEECH, 'stop'); switch(cue.type){ case 'chat': @@ -280,17 +289,28 @@ export function FreeFlow(){ setChatStatus(ChatStatus.Clear); // Set chat status to Processing let message=refInput.current?.value?.trim() || history.map(el=>`${el.role}:${el.content}`).join('\n'); - getSummary(message, data).then(summary_ => { + console.log('Summary input message:', message); + + if(!message || message.length==0) { - console.log('Summary:', summary_); + setSummary(); + console.log('no message input, clear summary'); onCueEnd(); // End the current cue after getting summary - setSummary(()=>summary_?.result); - refContainer.current.scrollTop = refContainer.current.scrollHeight; // Scroll to bottom - - }).catch(error => { - console.error('Error getting summary:', error); - }); + }else{ + + getSummary(message, data).then(summary_ => { + + console.log('Summary:', summary_); + onCueEnd(); // End the current cue after getting summary + + setSummary(()=>summary_?.result); + refContainer.current.scrollTop = refContainer.current.scrollHeight; // Scroll to bottom + + }).catch(error => { + console.error('Error getting summary:', error); + }); + } break; @@ -363,7 +383,7 @@ export function FreeFlow(){ }else{ sendOsc(OSC_ADDRESS.HINT, ''); // Clear hint message } - + sendOsc(OSC_ADDRESS.SPEECH, 'stop'); refAudio.current?.pause(); // Pause any playing audio console.log('onCueEnd:', cue.id); @@ -395,23 +415,34 @@ export function FreeFlow(){ function onNumpad(mess){ - console.log('Numpad input:', mess); + setPadInput(mess); + } + + useEffect(()=>{ + + + if(padInput==null) return; + + console.log('Numpad input:', padInput); if(refCurrentCue.current?.callback!='numpad') return; let cue=refCurrentCue.current; let next=cue.nextcue; switch(cue.numpad_type){ case NUMPAD_TYPE.USERID: - setUserId(()=>mess); + setUserId(()=>padInput); break; case NUMPAD_TYPE.CHOICE: - next=cue.branch[mess.toString()].nextcue; - setChoice(cue.branch[mess.toString()].description); // Set choice for user input + next=cue.branch[padInput.toString()].nextcue; + setChoice(()=>cue.branch[padInput.toString()].description); // Set choice for user input break; case NUMPAD_TYPE.PASSWORD: - setPassword(()=>mess); + setPassword(()=>padInput); // sendOsc(OSC_ADDRESS.PASSWORD, mess); // Send OSC password message - sendOsc(OSC_ADDRESS.EXPORT, `${getUploadFolder()}#${getDataId()}#${summary}#${getFileId(mess)}`); + const user_input = history.find(msg => msg.role === 'user'); + const default_image=!(user_input && user_input.content.trim() !== ''); + + sendOsc(OSC_ADDRESS.EXPORT, `${getUploadFolder()}#${getDataId()}#${summary||''}#${getFileId(padInput)}#${choice||''}#${default_image?'default':'generated'}`); // Send OSC export message sendOsc(OSC_ADDRESS.CHOICE, choice); // Send OSC save choice message break; } @@ -421,10 +452,9 @@ export function FreeFlow(){ console.log('Finish enter number, next cue:', next); playCue(cuelist.find(c => c.id === next)); } - } - function saveImage(){ - sendOsc('/export', 'output/test.png'); // Send OSC message to save image - } + + + },[padInput]); useEffect(()=>{ @@ -437,6 +467,8 @@ export function FreeFlow(){ function onSpeechEnd(){ + sendOsc(OSC_ADDRESS.SPEECH, 'stop'); + if(currentCue?.type!='chat') return; // Only process if current cue is user input if(chatStatus!=ChatStatus.User) return; // Only process if chat status is User @@ -555,8 +587,13 @@ export function FreeFlow(){ if(refPauseTimer.current) clearTimeout(refPauseTimer.current); refSpeechPaused.current=false; } + + sendOsc(OSC_ADDRESS.SPEECH, 'start'); - sendOscStatus(OSC_ADDRESS.CLIENT_INPUT, `${data.id}#${transcript}`); // Send current input via OSC + sendOscStatus(OSC_ADDRESS.CLIENT_INPUT, `${data.id}#${transcript}`); + // Send current input via OSC + + } },[transcript]); @@ -590,6 +627,7 @@ export function FreeFlow(){ text = ''; break; } + sendOsc(OSC_ADDRESS.SPEECH, 'stop'); sendOsc(OSC_ADDRESS.INPUT, text); },[chatStatus]); @@ -626,6 +664,8 @@ export function FreeFlow(){ }else{ + + playCue(next); setNextCue(null); } @@ -693,9 +733,7 @@ export function FreeFlow(){ UserId{userId} FileId{getFileId()} - + Choice{choice || ''} diff --git a/vite/src/util/osc.js b/vite/src/util/osc.js index 695b65b..d76ec01 100644 --- a/vite/src/util/osc.js +++ b/vite/src/util/osc.js @@ -17,6 +17,7 @@ export const OSC_ADDRESS={ DISCARD: 'discard', PASSWORD: '/password', HINT:'/hint', + SPEECH:'/speech', CLIENT_STATUS:'/client', CLIENT_INPUT:'/client_input', diff --git a/vite/src/util/useUser.jsx b/vite/src/util/useUser.jsx index 2afcdb4..6830689 100644 --- a/vite/src/util/useUser.jsx +++ b/vite/src/util/useUser.jsx @@ -1,4 +1,4 @@ -import { createContext, useState, useEffect, useContext, use } from "react"; +import { createContext, useState, useEffect, useContext, use, useRef } from "react"; import moment from "moment"; import { updateUser } from "./backend"; import { useData } from "./useData"; @@ -23,6 +23,8 @@ export function UserProvider({children}) { const [choice, setChoice] = useState(null); const [summary, setSummary] = useState(null); + const refChoice=useRef(); + const {data}=useData(); @@ -114,11 +116,16 @@ export function UserProvider({children}) { console.log("Session ID changed:", sessionId); }, [sessionId]); + // useEffect(()=>{ + // refChoice.current=choice; + // console.log("Choice changed:", choice); + // }, [choice]); return ( {children}