diff --git a/vite/package.json b/vite/package.json
index 9133816..07a23df 100644
--- a/vite/package.json
+++ b/vite/package.json
@@ -7,7 +7,8 @@
"dev": "vite",
"build": "vite build",
"lint": "eslint .",
- "preview": "vite preview"
+ "preview": "vite preview",
+ "deploy":"tauri build && copy /Y src-tauri\\target\\release\\app.exe C:\\uc\\24070-Sync\\app.exe"
},
"dependencies": {
"@tailwindcss/vite": "^4.1.8",
diff --git a/vite/public/assets/ai/sfx-06-ai-04-record-03-ai-05-sfx-07.mp3 b/vite/public/assets/ai/sfx-06-ai-04-record-03-ai-05-sfx-07.mp3
index f42f701..90a153a 100644
Binary files a/vite/public/assets/ai/sfx-06-ai-04-record-03-ai-05-sfx-07.mp3 and b/vite/public/assets/ai/sfx-06-ai-04-record-03-ai-05-sfx-07.mp3 differ
diff --git a/vite/src/comps/debug.jsx b/vite/src/comps/debug.jsx
index 304c8bf..5ba89b2 100644
--- a/vite/src/comps/debug.jsx
+++ b/vite/src/comps/debug.jsx
@@ -1,7 +1,7 @@
import { sendOsc, OSC_ADDRESS, updatePrompt } from "../util/osc"
import { useData } from '../util/useData.jsx';
-export const TEST_PROMPT='A distant, ambient setting with gentle light';
+export const TEST_PROMPT='a window frame, beach outside, island far away';
export function DebugControl({refLight}){
@@ -24,7 +24,7 @@ export function DebugControl({refLight}){
{/*
*/}
-
+
{/*
*/}
diff --git a/vite/src/pages/flow_free.jsx b/vite/src/pages/flow_free.jsx
index 6a8741c..65e87e8 100644
--- a/vite/src/pages/flow_free.jsx
+++ b/vite/src/pages/flow_free.jsx
@@ -14,6 +14,7 @@ import VoiceAnalysis from "../comps/voiceanalysis";
import { sendOsc, OSC_ADDRESS, updatePrompt, onOscMessageReceived, sendOscStatus } from "../util/osc";
import { DebugControl, TEST_PROMPT } from "../comps/debug";
import { useUser } from "../util/useUser";
+import { set } from 'zod/v4';
const CUELIST_FILE = 'cuelist_demo3.json';
@@ -56,6 +57,9 @@ export function FreeFlow(){
const [audioInput, setAudioInput] = useState(true);
const [autoSend, setAutoSend] = useState(true);
const [chatStatus, setChatStatus] = useState(ChatStatus.System); // System, User, Processing
+
+ const [padInput, setPadInput] = useState(null);
+
const { userId, setUserId, getFileId, setPassword, reset:resetUser, uploadHistory, setSummary, summary,setChoice,choice, getUploadFolder,getDataId } = useUser();
const refTimer=useRef();
@@ -93,6 +97,7 @@ export function FreeFlow(){
reset();
sendOsc(OSC_ADDRESS.CHOICE, 'reset');
+ sendOsc(OSC_ADDRESS.SPEECH, 'stop');
}
function onOsc(payload){
@@ -233,6 +238,9 @@ export function FreeFlow(){
if(!cue) return;
console.log('Playing cue:', cue);
+
+ // stop audio
+ if(refAudio.current) refAudio.current.pause();
@@ -254,6 +262,7 @@ export function FreeFlow(){
// clear unity hint
sendOsc(OSC_ADDRESS.HINT, ''); // Clear hint message
sendOsc(OSC_ADDRESS.INPUT, ''); // Clear input message
+ sendOsc(OSC_ADDRESS.SPEECH, 'stop');
switch(cue.type){
case 'chat':
@@ -280,17 +289,28 @@ export function FreeFlow(){
setChatStatus(ChatStatus.Clear); // Set chat status to Processing
let message=refInput.current?.value?.trim() || history.map(el=>`${el.role}:${el.content}`).join('\n');
- getSummary(message, data).then(summary_ => {
+ console.log('Summary input message:', message);
+
+ if(!message || message.length==0) {
- console.log('Summary:', summary_);
+ setSummary();
+ console.log('no message input, clear summary');
onCueEnd(); // End the current cue after getting summary
- setSummary(()=>summary_?.result);
- refContainer.current.scrollTop = refContainer.current.scrollHeight; // Scroll to bottom
-
- }).catch(error => {
- console.error('Error getting summary:', error);
- });
+ }else{
+
+ getSummary(message, data).then(summary_ => {
+
+ console.log('Summary:', summary_);
+ onCueEnd(); // End the current cue after getting summary
+
+ setSummary(()=>summary_?.result);
+ refContainer.current.scrollTop = refContainer.current.scrollHeight; // Scroll to bottom
+
+ }).catch(error => {
+ console.error('Error getting summary:', error);
+ });
+ }
break;
@@ -363,7 +383,7 @@ export function FreeFlow(){
}else{
sendOsc(OSC_ADDRESS.HINT, ''); // Clear hint message
}
-
+ sendOsc(OSC_ADDRESS.SPEECH, 'stop');
refAudio.current?.pause(); // Pause any playing audio
console.log('onCueEnd:', cue.id);
@@ -395,23 +415,34 @@ export function FreeFlow(){
function onNumpad(mess){
- console.log('Numpad input:', mess);
+ setPadInput(mess);
+ }
+
+ useEffect(()=>{
+
+
+ if(padInput==null) return;
+
+ console.log('Numpad input:', padInput);
if(refCurrentCue.current?.callback!='numpad') return;
let cue=refCurrentCue.current;
let next=cue.nextcue;
switch(cue.numpad_type){
case NUMPAD_TYPE.USERID:
- setUserId(()=>mess);
+ setUserId(()=>padInput);
break;
case NUMPAD_TYPE.CHOICE:
- next=cue.branch[mess.toString()].nextcue;
- setChoice(cue.branch[mess.toString()].description); // Set choice for user input
+ next=cue.branch[padInput.toString()].nextcue;
+ setChoice(()=>cue.branch[padInput.toString()].description); // Set choice for user input
break;
case NUMPAD_TYPE.PASSWORD:
- setPassword(()=>mess);
+ setPassword(()=>padInput);
// sendOsc(OSC_ADDRESS.PASSWORD, mess); // Send OSC password message
- sendOsc(OSC_ADDRESS.EXPORT, `${getUploadFolder()}#${getDataId()}#${summary}#${getFileId(mess)}`);
+ const user_input = history.find(msg => msg.role === 'user');
+ const default_image=!(user_input && user_input.content.trim() !== '');
+
+ sendOsc(OSC_ADDRESS.EXPORT, `${getUploadFolder()}#${getDataId()}#${summary||''}#${getFileId(padInput)}#${choice||''}#${default_image?'default':'generated'}`); // Send OSC export message
sendOsc(OSC_ADDRESS.CHOICE, choice); // Send OSC save choice message
break;
}
@@ -421,10 +452,9 @@ export function FreeFlow(){
console.log('Finish enter number, next cue:', next);
playCue(cuelist.find(c => c.id === next));
}
- }
- function saveImage(){
- sendOsc('/export', 'output/test.png'); // Send OSC message to save image
- }
+
+
+ },[padInput]);
useEffect(()=>{
@@ -437,6 +467,8 @@ export function FreeFlow(){
function onSpeechEnd(){
+ sendOsc(OSC_ADDRESS.SPEECH, 'stop');
+
if(currentCue?.type!='chat') return; // Only process if current cue is user input
if(chatStatus!=ChatStatus.User) return; // Only process if chat status is User
@@ -555,8 +587,13 @@ export function FreeFlow(){
if(refPauseTimer.current) clearTimeout(refPauseTimer.current);
refSpeechPaused.current=false;
}
+
+ sendOsc(OSC_ADDRESS.SPEECH, 'start');
- sendOscStatus(OSC_ADDRESS.CLIENT_INPUT, `${data.id}#${transcript}`); // Send current input via OSC
+ sendOscStatus(OSC_ADDRESS.CLIENT_INPUT, `${data.id}#${transcript}`);
+ // Send current input via OSC
+
+
}
},[transcript]);
@@ -590,6 +627,7 @@ export function FreeFlow(){
text = '';
break;
}
+ sendOsc(OSC_ADDRESS.SPEECH, 'stop');
sendOsc(OSC_ADDRESS.INPUT, text);
},[chatStatus]);
@@ -626,6 +664,8 @@ export function FreeFlow(){
}else{
+
+
playCue(next);
setNextCue(null);
}
@@ -693,9 +733,7 @@ export function FreeFlow(){
UserId{userId}
FileId{getFileId()}
-
+ Choice{choice || ''}
diff --git a/vite/src/util/osc.js b/vite/src/util/osc.js
index 695b65b..d76ec01 100644
--- a/vite/src/util/osc.js
+++ b/vite/src/util/osc.js
@@ -17,6 +17,7 @@ export const OSC_ADDRESS={
DISCARD: 'discard',
PASSWORD: '/password',
HINT:'/hint',
+ SPEECH:'/speech',
CLIENT_STATUS:'/client',
CLIENT_INPUT:'/client_input',
diff --git a/vite/src/util/useUser.jsx b/vite/src/util/useUser.jsx
index 2afcdb4..6830689 100644
--- a/vite/src/util/useUser.jsx
+++ b/vite/src/util/useUser.jsx
@@ -1,4 +1,4 @@
-import { createContext, useState, useEffect, useContext, use } from "react";
+import { createContext, useState, useEffect, useContext, use, useRef } from "react";
import moment from "moment";
import { updateUser } from "./backend";
import { useData } from "./useData";
@@ -23,6 +23,8 @@ export function UserProvider({children}) {
const [choice, setChoice] = useState(null);
const [summary, setSummary] = useState(null);
+ const refChoice=useRef();
+
const {data}=useData();
@@ -114,11 +116,16 @@ export function UserProvider({children}) {
console.log("Session ID changed:", sessionId);
}, [sessionId]);
+ // useEffect(()=>{
+ // refChoice.current=choice;
+ // console.log("Choice changed:", choice);
+ // }, [choice]);
return (
{children}