diff --git a/vite/public/cuelist.json b/vite/public/cuelist.json index c3faadc..0052a42 100644 --- a/vite/public/cuelist.json +++ b/vite/public/cuelist.json @@ -6,7 +6,8 @@ "type": "space", "description": "Annonce", "audioFile": "assets/q1.mp3", - "loop": true + "loop": true, + "status":"reset" }, { "id": 2, @@ -42,7 +43,8 @@ "description": "Guide to construct scene", "auto": true, "audioFile": "assets/q4-1.mp3", - "nextcue": 4.2 + "nextcue": 4.2, + "status":"go" }, { "id": 4.2, @@ -136,7 +138,8 @@ "name": "Q6", "type": "space", "description": "Ending", - "audioFile": "assets/q6.mp3" + "audioFile": "assets/q6.mp3", + "status":"end" } ] } diff --git a/vite/public/cuelist_free.json b/vite/public/cuelist_free.json index 75bfd67..e18bdaa 100644 --- a/vite/public/cuelist_free.json +++ b/vite/public/cuelist_free.json @@ -6,7 +6,8 @@ "type": "space", "description": "Annonce", "audioFile": "assets/q1.mp3", - "loop": true + "loop": true, + "status":"reset" }, { "id": 2, @@ -42,7 +43,8 @@ "description": "Guide to construct scene", "auto": true, "audioFile": "assets/q4-1.mp3", - "nextcue": 4.2 + "nextcue": 4.2, + "status":"go" }, { "id": 4.2, @@ -84,7 +86,8 @@ "name": "Q6", "type": "space", "description": "Ending", - "audioFile": "assets/q6.mp3" + "audioFile": "assets/q6.mp3", + "status":"end" } ] } diff --git a/vite/src-tauri/capabilities/default.json b/vite/src-tauri/capabilities/default.json index ce12897..efeb1ce 100644 --- a/vite/src-tauri/capabilities/default.json +++ b/vite/src-tauri/capabilities/default.json @@ -17,6 +17,8 @@ "allow": [ { "url": "https://*.openai.com" + },{ + "url":"http://localhost:34800" } ] }, diff --git a/vite/src-tauri/src/lib.rs b/vite/src-tauri/src/lib.rs index eaf72b7..027e46c 100644 --- a/vite/src-tauri/src/lib.rs +++ b/vite/src-tauri/src/lib.rs @@ -50,6 +50,7 @@ async fn send_osc_message( Ok(()) } + #[tauri::command] fn reset_permission(origin: &str, app: AppHandle) { let webview = app.get_webview_window("main").unwrap(); diff --git a/vite/src/comps/debug.jsx b/vite/src/comps/debug.jsx new file mode 100644 index 0000000..f8bf88b --- /dev/null +++ b/vite/src/comps/debug.jsx @@ -0,0 +1,20 @@ +import { sendOsc, OSC_ADDRESS, updatePrompt } from "../util/osc" + + +const TEST_PROMPT='a hazy memory of a {{ Scene }}, seen through soft atmospheric blur, distant silhouettes and faded contours, pastel light and cinematic haze, (analog film texture), (shallow depth of field:1.3), shallow depth of field, memory fragment effect, light leak, subtle grain, chromatic aberration, surreal glow, in muted warm tones, cinematic framing,'; + +export function DebugControl(){ + + return ( +
+ + + + +
+ + +
+
+ ) +} \ No newline at end of file diff --git a/vite/src/comps/light.jsx b/vite/src/comps/light.jsx index e575c05..53bed0c 100644 --- a/vite/src/comps/light.jsx +++ b/vite/src/comps/light.jsx @@ -1,6 +1,6 @@ import gsap from "gsap" import { forwardRef, useEffect, useImperativeHandle, useRef } from "react" -import { sendOsc } from "../util/osc"; +import { OSC_ADDRESS, sendOsc } from "../util/osc"; const FADE_TIME=3; @@ -18,7 +18,7 @@ export const Light=forwardRef((props, ref)=>{ duration: time, onUpdate: () => { // console.log(refVal.current.val); - sendOsc('/light', refVal.current.val.toString()); + sendOsc(OSC_ADDRESS.LIGHT, refVal.current.val.toString()); if(refContainer.current) refContainer.current.style.background= `rgba(0, 255, 0, ${refVal.current.val})`; // Update background color based on value }, diff --git a/vite/src/index.css b/vite/src/index.css index 3391b53..064cec5 100644 --- a/vite/src/index.css +++ b/vite/src/index.css @@ -3,7 +3,7 @@ @layer base{ button{ - @apply rounded-full border-4 px-2 !bg-orange-200 cursor-pointer font-bold; + @apply rounded-full border-2 px-2 bg-orange-200 cursor-pointer font-bold; } } \ No newline at end of file diff --git a/vite/src/pages/conversation.jsx b/vite/src/pages/conversation.jsx index 29a21b4..f25f0e3 100644 --- a/vite/src/pages/conversation.jsx +++ b/vite/src/pages/conversation.jsx @@ -75,7 +75,7 @@ export function Conversation() { }else{ console.log('create speech:', data.output_text); - textToSpeech(data.output_text, params?.voice_prompt).then(audioUrl => { + textToSpeech(data.output_text, params?.voice, params?.voice_prompt).then(audioUrl => { const audio = new Audio(audioUrl); console.log('play audio...', new Date(Date.now()-startTime).toISOString().slice(11, 19)); @@ -183,7 +183,7 @@ export function Conversation() { }else{ // tts console.log('create speech:', data.output_text); - textToSpeech(data.output_text, params?.voice_prompt).then(audioUrl => { + textToSpeech(data.output_text, params?.voice, params?.voice_prompt).then(audioUrl => { const audio = new Audio(audioUrl); console.log('play audio...', new Date(Date.now()-startTime).toISOString().slice(11, 19)); diff --git a/vite/src/pages/flow.jsx b/vite/src/pages/flow.jsx index efe4777..5dcb2f7 100644 --- a/vite/src/pages/flow.jsx +++ b/vite/src/pages/flow.jsx @@ -10,7 +10,7 @@ import NumPad from "../comps/numpad"; import { Light } from "../comps/light"; import { useData } from "../util/useData"; import VoiceAnalysis from "../comps/voiceanalysis"; -import { sendOsc } from "../util/osc"; +import { sendOsc, OSC_ADDRESS } from "../util/osc"; const EmojiType={ @@ -108,6 +108,9 @@ export function Flow(){ } } + if(cue.status){ + sendOsc(OSC_ADDRESS.STATUS, cue.status); // Send OSC status message + } if(cue.audioFile){ playAudio(cue.audioFile); @@ -177,6 +180,7 @@ export function Flow(){ function onSpeechEnd(){ console.log('onSpeechEnd:', finalTranscript); if(currentCue?.type!='user_input') return; // Only process if current cue is user input + if(autoSend && transcript.trim().length > 0) { console.log('Auto sending transcript:', transcript); onCueEnd(); diff --git a/vite/src/pages/flow_free.jsx b/vite/src/pages/flow_free.jsx index c628ea8..5f3f32f 100644 --- a/vite/src/pages/flow_free.jsx +++ b/vite/src/pages/flow_free.jsx @@ -10,8 +10,8 @@ import NumPad from "../comps/numpad"; import { Light } from "../comps/light"; import { useData } from "../util/useData"; import VoiceAnalysis from "../comps/voiceanalysis"; -import { sendOsc } from "../util/osc"; -import { set } from "zod/v4"; +import { sendOsc, OSC_ADDRESS } from "../util/osc"; +import { DebugControl } from "../comps/debug"; const EmojiType={ @@ -112,7 +112,7 @@ export function FreeFlow(){ resetTranscript(); console.log('Starting conversation...'); sendMessage(); - setChatWelcome(true); + setChatWelcome(true); break; case 'chat_end': const message= refInput.current?.value?.trim(); @@ -137,6 +137,8 @@ export function FreeFlow(){ break; } + + if(cue.audioFile){ playAudio(cue.audioFile); @@ -147,6 +149,17 @@ export function FreeFlow(){ onCueEnd(cue); }); } + + + // control unity + if(cue.status){ + sendOsc(OSC_ADDRESS.STATUS, cue.status); // Send OSC status message + } + if(cue.type=='chat'){ + sendOsc(OSC_ADDRESS.COUNTDOWN, cue.duration || 0); // Send OSC countdown message + }else{ + sendOsc(OSC_ADDRESS.COUNTDOWN, 0); // Reset countdown for non-chat cues + } } function onCueEnd() { @@ -272,6 +285,7 @@ export function FreeFlow(){ useEffect(()=>{ resetTranscript(); + sendOsc(OSC_ADDRESS.INPUT, chatStatus); },[chatStatus]); @@ -308,7 +322,7 @@ export function FreeFlow(){ return (
- +
{refCurrentCue.current?.name} @@ -364,7 +378,7 @@ export function FreeFlow(){ {msg.prompt &&
{msg.prompt}
}
))} - {summary &&
{summary}
} + {summary &&
{JSON.stringify(summary)}
}