update

vite/public/cuelist.json

@@ -6,7 +6,8 @@
             "type": "space",
             "description": "Annonce",
             "audioFile": "assets/q1.mp3",
-            "loop": true
+            "loop": true,
+            "status":"reset"
         },
         { 
             "id": 2, 
@@ -42,7 +43,8 @@
             "description": "Guide to construct scene",
             "auto": true,
             "audioFile": "assets/q4-1.mp3",
-            "nextcue": 4.2
+            "nextcue": 4.2,
+            "status":"go"
         },
         { 
             "id": 4.2, 
@@ -136,7 +138,8 @@
             "name": "Q6", 
             "type": "space",
             "description": "Ending",
-            "audioFile": "assets/q6.mp3"         
+            "audioFile": "assets/q6.mp3",
+            "status":"end"
         }
     ]
 }

vite/public/cuelist_free.json

@@ -6,7 +6,8 @@
             "type": "space",
             "description": "Annonce",
             "audioFile": "assets/q1.mp3",
-            "loop": true
+            "loop": true,
+            "status":"reset"
         },
         { 
             "id": 2, 
@@ -42,7 +43,8 @@
             "description": "Guide to construct scene",
             "auto": true,
             "audioFile": "assets/q4-1.mp3",
-            "nextcue": 4.2
+            "nextcue": 4.2,
+            "status":"go"
         },
         { 
             "id": 4.2, 
@@ -84,7 +86,8 @@
             "name": "Q6", 
             "type": "space",
             "description": "Ending",
-            "audioFile": "assets/q6.mp3"         
+            "audioFile": "assets/q6.mp3",
+            "status":"end"
         }
     ]
 }

vite/src-tauri/capabilities/default.json

@@ -17,6 +17,8 @@
       "allow": [
         {
           "url": "https://*.openai.com"
+        },{
+          "url":"http://localhost:34800"
         }
       ]
     },    

vite/src-tauri/src/lib.rs

@@ -50,6 +50,7 @@ async fn send_osc_message(
     Ok(())
 }
 
+
 #[tauri::command]
 fn reset_permission(origin: &str, app: AppHandle) {
     let webview = app.get_webview_window("main").unwrap();

vite/src/comps/debug.jsx

@@ -0,0 +1,20 @@
+import { sendOsc, OSC_ADDRESS, updatePrompt } from "../util/osc"
+
+
+const TEST_PROMPT='a hazy memory of a {{ Scene }}, seen through soft atmospheric blur, distant silhouettes and faded contours, pastel light and cinematic haze, (analog film texture), (shallow depth of field:1.3), shallow depth of field, memory fragment effect, light leak, subtle grain, chromatic aberration, surreal glow, in muted warm tones, cinematic framing,';
+
+export function DebugControl(){
+    
+    return (
+        <div className="flex flex-row gap-2 [&>button]:rounded-full [&>button]:bg-white bg-gray-200 p-2 w-full justify-center">
+            <button onClick={() => sendOsc(OSC_ADDRESS.STATUS, 'reset')}>reset</button>
+            <button onClick={() => sendOsc(OSC_ADDRESS.STATUS, 'go')}>go</button>
+            <button onClick={() => sendOsc(OSC_ADDRESS.STATUS, 'end')}>end</button>
+
+            <div className="flex flex-col gap-1">
+                <button className="btn btn-success" onClick={() => updatePrompt('Random dogs')}>Test Prompt</button>  
+                <button className="btn btn-success" onClick={() => updatePrompt(TEST_PROMPT)}>Test Prompt</button>  
+            </div>
+        </div>
+    )
+}

vite/src/comps/light.jsx

@@ -1,6 +1,6 @@
 import gsap from "gsap"
 import { forwardRef, useEffect, useImperativeHandle, useRef } from "react"
-import { sendOsc } from "../util/osc";
+import { OSC_ADDRESS, sendOsc } from "../util/osc";
 
 const FADE_TIME=3;
 
@@ -18,7 +18,7 @@ export const Light=forwardRef((props, ref)=>{
             duration: time,
             onUpdate: () => {
                 // console.log(refVal.current.val);
-                sendOsc('/light', refVal.current.val.toString());
+                sendOsc(OSC_ADDRESS.LIGHT, refVal.current.val.toString());
                 if(refContainer.current)
                     refContainer.current.style.background= `rgba(0, 255, 0, ${refVal.current.val})`; // Update background color based on value
             },

vite/src/index.css

@@ -3,7 +3,7 @@
 
 @layer base{
   button{
-    @apply rounded-full border-4 px-2 !bg-orange-200 cursor-pointer font-bold;
+    @apply rounded-full border-2 px-2 bg-orange-200 cursor-pointer font-bold;
   }
 
 }

vite/src/pages/conversation.jsx

@@ -75,7 +75,7 @@ export function Conversation() {
         
       }else{
         console.log('create speech:', data.output_text);
-        textToSpeech(data.output_text, params?.voice_prompt).then(audioUrl => {
+        textToSpeech(data.output_text, params?.voice, params?.voice_prompt).then(audioUrl => {
           const audio = new Audio(audioUrl);
           
           console.log('play audio...', new Date(Date.now()-startTime).toISOString().slice(11, 19));
@@ -183,7 +183,7 @@ export function Conversation() {
       }else{
         // tts
         console.log('create speech:', data.output_text);
-        textToSpeech(data.output_text, params?.voice_prompt).then(audioUrl => {
+        textToSpeech(data.output_text, params?.voice, params?.voice_prompt).then(audioUrl => {
           const audio = new Audio(audioUrl);
           
           console.log('play audio...', new Date(Date.now()-startTime).toISOString().slice(11, 19));

vite/src/pages/flow.jsx

@@ -10,7 +10,7 @@ import NumPad from "../comps/numpad";
 import { Light } from "../comps/light";
 import { useData } from "../util/useData";
 import VoiceAnalysis from "../comps/voiceanalysis";
-import { sendOsc } from "../util/osc";
+import { sendOsc, OSC_ADDRESS } from "../util/osc";
 
 
 const EmojiType={
@@ -108,6 +108,9 @@ export function Flow(){
             }
         }
 
+        if(cue.status){
+            sendOsc(OSC_ADDRESS.STATUS, cue.status); // Send OSC status message
+        }
 
         if(cue.audioFile){            
             playAudio(cue.audioFile);
@@ -177,6 +180,7 @@ export function Flow(){
     function onSpeechEnd(){
         console.log('onSpeechEnd:', finalTranscript);
         if(currentCue?.type!='user_input') return; // Only process if current cue is user input
+        
         if(autoSend && transcript.trim().length > 0) {
             console.log('Auto sending transcript:', transcript);
             onCueEnd(); 

vite/src/pages/flow_free.jsx

@@ -10,8 +10,8 @@ import NumPad from "../comps/numpad";
 import { Light } from "../comps/light";
 import { useData } from "../util/useData";
 import VoiceAnalysis from "../comps/voiceanalysis";
-import { sendOsc } from "../util/osc";
+import { sendOsc, OSC_ADDRESS } from "../util/osc";
-import { set } from "zod/v4";
+import { DebugControl } from "../comps/debug";
 
 
 const EmojiType={
@@ -138,6 +138,8 @@ export function FreeFlow(){
         }
 
         
+
+
         if(cue.audioFile){            
             playAudio(cue.audioFile);
         }
@@ -147,6 +149,17 @@ export function FreeFlow(){
                 onCueEnd(cue);
             });
         }
+
+
+        // control unity
+        if(cue.status){
+            sendOsc(OSC_ADDRESS.STATUS, cue.status); // Send OSC status message
+        }
+        if(cue.type=='chat'){
+            sendOsc(OSC_ADDRESS.COUNTDOWN, cue.duration || 0); // Send OSC countdown message
+        }else{
+            sendOsc(OSC_ADDRESS.COUNTDOWN, 0); // Reset countdown for non-chat cues
+        }
     }
 
     function onCueEnd() {
@@ -272,6 +285,7 @@ export function FreeFlow(){
     useEffect(()=>{
         
         resetTranscript();
+        sendOsc(OSC_ADDRESS.INPUT, chatStatus);
 
     },[chatStatus]);
 
@@ -308,7 +322,7 @@ export function FreeFlow(){
 
     return (
         <main className="items-center">
-
+            <DebugControl/>
             <div className="w-full p-2 flex flex-row justify-center gap-2 *:w-[12vw] *:h-[12vw]">
                 <div className="bg-gray-100 text-4xl font-bold mb-4 flex justify-center items-center">
                     {refCurrentCue.current?.name}
@@ -364,7 +378,7 @@ export function FreeFlow(){
                             {msg.prompt && <div className="text-xs bg-gray-200">{msg.prompt}</div>}
                         </div>
                     ))}
-                    {summary && <div className="w-full self-center bg-blue-200 px-2">{summary}</div>}
+                    {summary && <div className="w-full self-center bg-blue-200 px-2">{JSON.stringify(summary)}</div>}
                 </div>
                 <textarea ref={refInput} name="message" rows={2} 
                     className={`w-full border-1 resize-none p-2 disabled:bg-gray-500`}

vite/src/util/chat.js

@@ -1,6 +1,6 @@
 import { fetch } from '@tauri-apps/plugin-http';
 import { params } from './system_prompt';
-import { sendOsc } from './osc';
+import { sendOsc, OSC_ADDRESS, updatePrompt } from './osc';
 import { invoke } from '@tauri-apps/api/core';
 import { useData } from './useData';
 
@@ -97,9 +97,12 @@ export async function sendChatMessage(messages, data, isLastMessage = false) {
     // console.log("Generated response:", choice.message);
     const result=JSON.parse(choice.message.content);
     
-    // send to tauri
+    // send to python & unity
-    await sendOsc('/prompt', result.prompt?.replaceAll('"', ''));
+    const prompt = result.prompt?.replaceAll('"', '');
-    await sendOsc('/output_text', result.output_text?.replaceAll('"', ''));
+    await updatePrompt(prompt);
+    await sendOsc(OSC_ADDRESS.PROMPT, prompt);    
+
+    // TODO: send to python
 
 
     return {
@@ -153,7 +156,7 @@ export async function getSummary(messages, data) {
     const result=choice.message.content;
     
     // send to tauri
-    await sendOsc('/summary', result);
+    await sendOsc(OSC_ADDRESS.SUMMARY, result);
 
 
     return {

vite/src/util/osc.js

@@ -1,4 +1,18 @@
 import { invoke } from '@tauri-apps/api/core';
+import { fetch } from '@tauri-apps/plugin-http';
+
+export const OSC_ADDRESS={
+    LIGHT: '/light',
+    STATUS: '/status',
+    INPUT: '/input',
+    COUNTDOWN: '/countdown',
+    VOLUME: '/volume',
+    SCRIPT: '/script',
+    SUMMARY: '/summary',
+    PROMPT: '/prompt',
+    
+}
+
 
 export async function sendOsc(key, message){
 
@@ -6,11 +20,30 @@ export async function sendOsc(key, message){
         console.warn('sendOsc: message is empty, skipping');
         return;
     }
-    // console.log(`Sending OSC message: ${key} -> ${message}`);
+    console.log(`Sending OSC message: ${key} -> ${message}`);
     await invoke('send_osc_message', {
         key: key,
-        message: message,
+        message: message.toString(),
         host:`0.0.0.0:0`,
-        target: '127.0.0.1:8787',
+        target: '127.0.0.1:9000',
+    });
+}
+
+
+// send to python fastapi
+export async function updatePrompt(prompt) {
+    console.log(`Updating prompt: ${prompt}`);
+    
+    try{
+        await fetch('http://localhost:34800/api/update/prompt', {
+            method: 'POST',
+            headers: {
+                'Content-Type': 'application/json'
+            },
+            body: JSON.stringify({ prompt })
         });
+    }catch(error){
+        console.error('Error updating prompt:', error);
+        throw error;
+    }
 }

vite/src/util/system_prompt.js

@@ -6,12 +6,14 @@ export const params={
     last_prompt:`請用一句話為這段對話簡短的收尾，並邀請使用者在 60 秒的時間內，說出對遺憾對象想說的話`,
 
     welcome_prompt:`請開始引導使用者回想一段內心的遺憾或未竟之事。`,
-    voice_prompt:`使用台灣語境的繁體中文，有同理心，柔和的口氣，不要有質問感，語速自然流暢，稍微慢。用冥想的方式，引導使用者回想記憶。`,
+    
-    voice:"nova",
+    voice_prompt:`Voice Affect: Low, hushed, and suspenseful; convey tension and intrigue.\n\nTone: Deeply serious and mysterious, maintaining an undercurrent of unease throughout.\n\nPacing: Slow, deliberate, pausing slightly after suspenseful moments to heighten drama.\n\nEmotion: Restrained yet intense—voice should subtly tremble or tighten at key suspenseful points.\n\nEmphasis: Highlight sensory descriptions (\"footsteps echoed,\" \"heart hammering,\" \"shadows melting into darkness\") to amplify atmosphere.\n\nPronunciation: Slightly elongated vowels and softened consonants for an eerie, haunting effect.\n\nPauses: Insert meaningful pauses after phrases like \"only shadows melting into darkness,\" and especially before the final line, to enhance suspense dramatically.`,
+    voice:"onyx",
+
     summary_prompt:`幫我把以下一段話整理成一段文字，以第一人稱視角作為當事人的文字紀念，文字內容 50 字以內：`,
 }
 
-export const ParamKeys= Object.keys(params);
+export const ParamKeys=Object.keys(params);
 
 
 // export const welcome_prompt="請開始引導使用者回想一段內心的遺憾或未竟之事。";

vite/src/util/tts.js

@@ -2,7 +2,7 @@ import { fetch } from '@tauri-apps/plugin-http';
 import { invoke } from '@tauri-apps/api/core';
 // import { voice_prompt } from './system_prompt';
 
-export async function textToSpeech(text, voice_prompt) {
+export async function textToSpeech(text, voice, voice_prompt) {
 
     const token = await invoke('get_env', { name: 'OPENAI_API_KEY' });
 
@@ -15,7 +15,7 @@ export async function textToSpeech(text, voice_prompt) {
         body: JSON.stringify({
             model: 'gpt-4o-mini-tts',//'tts-1',
             input: text,
-            voice: "nova",
+            voice: voice,
             instructions: voice_prompt,
             response_format:'opus'
         }),

vite/src/util/useChat.jsx

@@ -60,7 +60,7 @@ export function ChatProvider({children}){
 
             if(response.output_text && (!force_no_audio && audioOutput)){
                 setStatus(Status.PROCESSING_AUDIO);
-                textToSpeech(response.output_text, data?.voice_prompt).then(url => {
+                textToSpeech(response.output_text, data?.voice, data?.voice_prompt).then(url => {
                     setStatus(Status.SUCCESS);                
                     setAudioUrl(url); // Store the audio URL