Merge branch 'main' of https://git.ultracombos.com/24070-TheGreatTipsy/Source-Web-24070-TheGreatTipsy

vite/package.json

@@ -7,7 +7,8 @@
     "dev": "vite",
     "build": "vite build",
     "lint": "eslint .",
-    "preview": "vite preview"
+    "preview": "vite preview",
+    "deploy":"tauri build && copy /Y src-tauri\\target\\release\\app.exe C:\\uc\\24070-Sync\\app.exe"
   },
   "dependencies": {
     "@tailwindcss/vite": "^4.1.8",

vite/src/comps/debug.jsx

@@ -1,7 +1,7 @@
 import { sendOsc, OSC_ADDRESS, updatePrompt } from "../util/osc"
 import { useData } from '../util/useData.jsx';
 
-export const TEST_PROMPT='A distant, ambient setting with gentle light';
+export const TEST_PROMPT='a window frame, beach outside, island far away';
 
 export function DebugControl({refLight}){
     
@@ -24,7 +24,7 @@ export function DebugControl({refLight}){
             <button onClick={() => sendOsc(OSC_ADDRESS.STATUS, 'end')}>end</button>
 
             {/* <div className="flex flex-col gap-1"> */}
-                <button className="btn btn-success" onClick={() => sendPrompt('a dog')}>Test Prompt #1</button>  
+                <button className="btn btn-success" onClick={() => sendPrompt('a telephone on table with light')}>Test Prompt #1</button>  
                 <button className="btn btn-success" onClick={() => sendPrompt(TEST_PROMPT)}>Test Prompt #2</button>  
             {/* </div> */}
         </div>

vite/src/pages/flow_free.jsx

@@ -14,6 +14,7 @@ import VoiceAnalysis from "../comps/voiceanalysis";
 import { sendOsc, OSC_ADDRESS, updatePrompt, onOscMessageReceived, sendOscStatus } from "../util/osc";
 import { DebugControl, TEST_PROMPT } from "../comps/debug";
 import { useUser } from "../util/useUser";
+import { set } from 'zod/v4';
 
 
 const CUELIST_FILE = 'cuelist_demo3.json';
@@ -56,6 +57,9 @@ export function FreeFlow(){
     const [audioInput, setAudioInput] = useState(true); 
     const [autoSend, setAutoSend] = useState(true);
     const [chatStatus, setChatStatus] = useState(ChatStatus.System); // System, User, Processing
+
+    const [padInput, setPadInput] = useState(null);
+
     const { userId, setUserId, getFileId, setPassword, reset:resetUser, uploadHistory, setSummary, summary,setChoice,choice, getUploadFolder,getDataId } = useUser();
 
     const refTimer=useRef();
@@ -93,6 +97,7 @@ export function FreeFlow(){
       
         reset();
         sendOsc(OSC_ADDRESS.CHOICE, 'reset');
+        sendOsc(OSC_ADDRESS.SPEECH, 'stop');
     }
 
     function onOsc(payload){
@@ -233,6 +238,9 @@ export function FreeFlow(){
 
         if(!cue) return;
         console.log('Playing cue:', cue);
+        
+        // stop audio
+        if(refAudio.current) refAudio.current.pause();
 
         
         
@@ -254,6 +262,7 @@ export function FreeFlow(){
         // clear unity hint
         sendOsc(OSC_ADDRESS.HINT, ''); // Clear hint message
         sendOsc(OSC_ADDRESS.INPUT, ''); // Clear input message
+        sendOsc(OSC_ADDRESS.SPEECH, 'stop');
 
         switch(cue.type){
             case 'chat':
@@ -280,17 +289,28 @@ export function FreeFlow(){
                 setChatStatus(ChatStatus.Clear); // Set chat status to Processing
                 
                 let message=refInput.current?.value?.trim() || history.map(el=>`${el.role}:${el.content}`).join('\n');
-                getSummary(message, data).then(summary_ => {
+                console.log('Summary input message:', message);
+
+                if(!message || message.length==0) {
 
-                    console.log('Summary:', summary_);
+                    setSummary();
+                    console.log('no message input, clear summary');
                     onCueEnd(); // End the current cue after getting summary
 
-                    setSummary(()=>summary_?.result);
-                    refContainer.current.scrollTop = refContainer.current.scrollHeight; // Scroll to bottom
-            
-                }).catch(error => {
-                    console.error('Error getting summary:', error);
-                });
+                }else{
+
+                    getSummary(message, data).then(summary_ => {
+
+                        console.log('Summary:', summary_);
+                        onCueEnd(); // End the current cue after getting summary
+
+                        setSummary(()=>summary_?.result);
+                        refContainer.current.scrollTop = refContainer.current.scrollHeight; // Scroll to bottom
+                
+                    }).catch(error => {
+                        console.error('Error getting summary:', error);
+                    });
+                }
                 
                 
                 break;
@@ -363,7 +383,7 @@ export function FreeFlow(){
         }else{
             sendOsc(OSC_ADDRESS.HINT, ''); // Clear hint message
         }
-
+        sendOsc(OSC_ADDRESS.SPEECH, 'stop');
 
         refAudio.current?.pause(); // Pause any playing audio
         console.log('onCueEnd:', cue.id);
@@ -395,23 +415,34 @@ export function FreeFlow(){
 
 
     function onNumpad(mess){
-        console.log('Numpad input:', mess);        
+       setPadInput(mess);
+    }
+
+    useEffect(()=>{
+
+
+        if(padInput==null) return;
+
+        console.log('Numpad input:', padInput);        
         if(refCurrentCue.current?.callback!='numpad') return;        
         
         let cue=refCurrentCue.current;
         let next=cue.nextcue;
         switch(cue.numpad_type){
             case NUMPAD_TYPE.USERID:
-                setUserId(()=>mess);
+                setUserId(()=>padInput);
                 break;
             case NUMPAD_TYPE.CHOICE:
-                next=cue.branch[mess.toString()].nextcue;          
-                setChoice(cue.branch[mess.toString()].description); // Set choice for user input            
+                next=cue.branch[padInput.toString()].nextcue;          
+                setChoice(()=>cue.branch[padInput.toString()].description); // Set choice for user input            
                 break;
             case NUMPAD_TYPE.PASSWORD:
-                setPassword(()=>mess);                
+                setPassword(()=>padInput);                
                 // sendOsc(OSC_ADDRESS.PASSWORD, mess); // Send OSC password message
-                sendOsc(OSC_ADDRESS.EXPORT, `${getUploadFolder()}#${getDataId()}#${summary}#${getFileId(mess)}`);
+                const user_input = history.find(msg => msg.role === 'user');
+                const default_image=!(user_input && user_input.content.trim() !== '');
+
+                sendOsc(OSC_ADDRESS.EXPORT, `${getUploadFolder()}#${getDataId()}#${summary||''}#${getFileId(padInput)}#${choice||''}#${default_image?'default':'generated'}`); // Send OSC export message
                 sendOsc(OSC_ADDRESS.CHOICE, choice); // Send OSC save choice message
                 break;
         }
@@ -421,10 +452,9 @@ export function FreeFlow(){
             console.log('Finish enter number, next cue:', next);
             playCue(cuelist.find(c => c.id === next));
         }
-    }
-    function saveImage(){
-        sendOsc('/export', 'output/test.png'); // Send OSC message to save image
-    }
+
+
+    },[padInput]);
 
     useEffect(()=>{
 
@@ -437,6 +467,8 @@ export function FreeFlow(){
 
     function onSpeechEnd(){
 
+        sendOsc(OSC_ADDRESS.SPEECH, 'stop');
+
         if(currentCue?.type!='chat') return; // Only process if current cue is user input        
         if(chatStatus!=ChatStatus.User) return; // Only process if chat status is User
        
@@ -555,8 +587,13 @@ export function FreeFlow(){
                 if(refPauseTimer.current) clearTimeout(refPauseTimer.current);
                 refSpeechPaused.current=false;
             }
+            
+            sendOsc(OSC_ADDRESS.SPEECH, 'start');
 
-            sendOscStatus(OSC_ADDRESS.CLIENT_INPUT, `${data.id}#${transcript}`); // Send current input via OSC
+            sendOscStatus(OSC_ADDRESS.CLIENT_INPUT, `${data.id}#${transcript}`);
+            // Send current input via OSC
+            
+            
         }        
 
     },[transcript]);
@@ -590,6 +627,7 @@ export function FreeFlow(){
                 text = '';
                 break;
         }
+        sendOsc(OSC_ADDRESS.SPEECH, 'stop');
         sendOsc(OSC_ADDRESS.INPUT, text);
 
     },[chatStatus]);
@@ -626,6 +664,8 @@ export function FreeFlow(){
 
 
         }else{
+
+           
             playCue(next);
             setNextCue(null);
         }
@@ -693,9 +733,7 @@ export function FreeFlow(){
                         <span></span>
                         <span>UserId</span><span>{userId}</span>
                         <span>FileId</span><span>{getFileId()}</span>
-                        <button className="!bg-yellow-300 col-span-2" onClick={()=>{
-                            saveHistory(history);
-                        }}>Save Log</button>  
+                        <span>Choice</span><span>{choice || ''}</span>
                     </div>
 
                     

vite/src/util/osc.js

@@ -17,6 +17,7 @@ export const OSC_ADDRESS={
     DISCARD: 'discard',
     PASSWORD: '/password',
     HINT:'/hint',
+    SPEECH:'/speech',
 
     CLIENT_STATUS:'/client',
     CLIENT_INPUT:'/client_input',

vite/src/util/useUser.jsx

@@ -1,4 +1,4 @@
-import { createContext, useState, useEffect, useContext, use } from "react";
+import { createContext, useState, useEffect, useContext, use, useRef } from "react";
 import moment from "moment";
 import { updateUser } from "./backend";
 import { useData } from "./useData";
@@ -23,6 +23,8 @@ export function UserProvider({children}) {
     const [choice, setChoice] = useState(null);
     const [summary, setSummary] = useState(null);
 
+    const refChoice=useRef();
+
     const {data}=useData();
 
 
@@ -114,11 +116,16 @@ export function UserProvider({children}) {
         console.log("Session ID changed:", sessionId);
 
     }, [sessionId]);
+    // useEffect(()=>{
+    //     refChoice.current=choice;
+    //     console.log("Choice changed:", choice);
+    // }, [choice]);
 
     return (
         <userContext.Provider value={{ 
             userId, setUserId, getFileId, getUploadFolder,
-            reset, uploadHistory: saveHistory, setChoice, choice,
+            reset, uploadHistory: saveHistory, setChoice, 
+            choice,
             setSummary, summary, getDataId,
             setPassword}}>
                 {children}