diff --git a/vite/index.html b/vite/index.html index f156110..61699b1 100644 --- a/vite/index.html +++ b/vite/index.html @@ -4,6 +4,10 @@ + + 24070-Conversation diff --git a/vite/public/assets/ai/sfx-06-ai-04-record-03-ai-05-sfx-07.mp3 b/vite/public/assets/ai/sfx-06-ai-04-record-03-ai-05-sfx-07.mp3 index d7ff697..babf257 100644 Binary files a/vite/public/assets/ai/sfx-06-ai-04-record-03-ai-05-sfx-07.mp3 and b/vite/public/assets/ai/sfx-06-ai-04-record-03-ai-05-sfx-07.mp3 differ diff --git a/vite/public/assets/ai/sfx-08-record-04-ai-06.mp3 b/vite/public/assets/ai/sfx-08-record-04-ai-06.mp3 index ded52cd..e6d79c5 100644 Binary files a/vite/public/assets/ai/sfx-08-record-04-ai-06.mp3 and b/vite/public/assets/ai/sfx-08-record-04-ai-06.mp3 differ diff --git a/vite/public/assets/bg-01 (小束袋）.mp3 b/vite/public/assets/bg-01 (小束袋）.mp3 new file mode 100644 index 0000000..10b0e59 Binary files /dev/null and b/vite/public/assets/bg-01 (小束袋）.mp3 differ diff --git a/vite/public/assets/bg-01 (香料瓶）.mp3 b/vite/public/assets/bg-01 (香料瓶）.mp3 new file mode 100644 index 0000000..dd5c6dd Binary files /dev/null and b/vite/public/assets/bg-01 (香料瓶）.mp3 differ diff --git a/vite/public/assets/bg-02.mp3 b/vite/public/assets/bg-02.mp3 new file mode 100644 index 0000000..88adf1a Binary files /dev/null and b/vite/public/assets/bg-02.mp3 differ diff --git a/vite/public/assets/bg-03.mp3 b/vite/public/assets/bg-03.mp3 deleted file mode 100644 index d1f1b7c..0000000 Binary files a/vite/public/assets/bg-03.mp3 and /dev/null differ diff --git a/vite/public/assets/bg-04.mp3 b/vite/public/assets/bg-04.mp3 new file mode 100644 index 0000000..7664b70 Binary files /dev/null and b/vite/public/assets/bg-04.mp3 differ diff --git a/vite/public/assets/record/record-01（小束袋）.mp3 b/vite/public/assets/record/record-01（小束袋）.mp3 new file mode 100644 index 0000000..869c641 Binary files /dev/null and b/vite/public/assets/record/record-01（小束袋）.mp3 differ diff --git a/vite/public/assets/record/record-01（香料瓶）.mp3 b/vite/public/assets/record/record-01（香料瓶）.mp3 new file mode 100644 index 0000000..7be906f Binary files /dev/null and b/vite/public/assets/record/record-01（香料瓶）.mp3 differ diff --git a/vite/public/assets/record/record-05（小束袋）.mp3 b/vite/public/assets/record/record-05（小束袋）.mp3 new file mode 100644 index 0000000..48b7cd1 Binary files /dev/null and b/vite/public/assets/record/record-05（小束袋）.mp3 differ diff --git a/vite/public/assets/sfx/sfx-12.mp3 b/vite/public/assets/sfx/sfx-12.mp3 new file mode 100644 index 0000000..145311b Binary files /dev/null and b/vite/public/assets/sfx/sfx-12.mp3 differ diff --git a/vite/public/cuelist_demo2.json b/vite/public/cuelist_demo2.json index 658ca99..36ec592 100644 --- a/vite/public/cuelist_demo2.json +++ b/vite/public/cuelist_demo2.json @@ -5,7 +5,7 @@ "name": "Q1", "type": "space", "description": "Annonce", - "audioFile": "assets/bg-01 (小束袋）.mp3", + "audioFile": "assets/bg-02.mp3", "loop": true, "status":"reset" }, diff --git a/vite/src-tauri/src/lib.rs b/vite/src-tauri/src/lib.rs index 12a244d..7a883f2 100644 --- a/vite/src-tauri/src/lib.rs +++ b/vite/src-tauri/src/lib.rs @@ -2,13 +2,20 @@ use dotenv::dotenv; use rosc::{encoder, OscMessage, OscPacket, OscType}; use std::env; use std::{net::SocketAddrV4, str::FromStr}; -use tauri::{AppHandle, Manager}; +use tauri::{AppHandle, Manager, Emitter}; use tokio::net::UdpSocket; use webview2_com::Microsoft::Web::WebView2::Win32::{ ICoreWebView2Profile4, ICoreWebView2_13, COREWEBVIEW2_PERMISSION_KIND_MICROPHONE, COREWEBVIEW2_PERMISSION_STATE_DEFAULT, }; use windows::core::{Interface, PCWSTR}; +use serde::Serialize; + +#[derive(Serialize, Clone)] +struct OscEvent { + addr: String, + args: Vec, +} #[tauri::command] fn get_env(name: &str) -> String { @@ -97,10 +104,58 @@ pub fn run() { .build(), )?; } + + tauri::async_runtime::spawn(setup_osc_server(app.handle().clone())); Ok(()) }) .run(tauri::generate_context!()) .expect("error while running tauri application"); + + +} + +async fn setup_osc_server(app_handle: AppHandle) { + + println!("Setting up OSC server..."); + // setup osc sever + let addr = "0.0.0.0:8000"; + let sock = UdpSocket::bind(addr).await.unwrap(); + println!("Listening to {}", addr); + + let mut buf = [0u8; rosc::decoder::MTU]; + + loop { + match sock.recv_from(&mut buf).await { + Ok((size, addr)) => { + println!("Received packet with size {} from: {}", size, addr); + if let Ok((_, packet)) = rosc::decoder::decode_udp(&buf[..size]) { + if let OscPacket::Message(msg) = packet { + println!("OSC message: {:?}", msg); + + app_handle.emit( + "osc_message", + OscEvent { + addr: msg.addr.clone(), + args: msg.args.iter().map(|arg| match arg { + OscType::Int(i) => i.to_string(), + OscType::Float(f) => f.to_string(), + OscType::String(s) => s.clone(), + OscType::Bool(b) => b.to_string(), + _ => format!("{:?}", arg), + }).collect(), + }, + ).unwrap(); + } + } else { + println!("Failed to decode OSC packet"); + } + } + Err(e) => { + println!("Error receiving from socket: {}", e); + break; + } + } + } } static mut LIGHT: Option = None; diff --git a/vite/src/App.css b/vite/src/App.css index 2d6ed67..ca0bd58 100644 --- a/vite/src/App.css +++ b/vite/src/App.css @@ -1,5 +1,8 @@ @import "tailwindcss"; +html{ + font-family: "Sora", sans-serif; +} #root{ @apply flex flex-col h-screen; } diff --git a/vite/src/comps/numpad.jsx b/vite/src/comps/numpad.jsx index e53e0c2..d63d666 100644 --- a/vite/src/comps/numpad.jsx +++ b/vite/src/comps/numpad.jsx @@ -62,7 +62,7 @@ export default function NumPad({onSend, disabled, type}){ } break; case NUMPAD_TYPE.PASSWORD: - if(refInput.current.length==3){ + if(refInput.current.length=3){ onSend(refInput.current); refAudio.current[KEY_ENTER]?.play(); }else{ @@ -88,7 +88,7 @@ export default function NumPad({onSend, disabled, type}){ return; }else if(e.key===KEY_BACKSPACE || e.key==='Backspace' || e.key==='Delete'){ - setInput((prev)=>''); + setInput(()=>''); refAudio.current[KEY_BACKSPACE]?.play(); return; diff --git a/vite/src/pages/flow_free.jsx b/vite/src/pages/flow_free.jsx index bbd3202..60492eb 100644 --- a/vite/src/pages/flow_free.jsx +++ b/vite/src/pages/flow_free.jsx @@ -10,9 +10,10 @@ import NumPad, { NUMPAD_TYPE } from "../comps/numpad"; import { Light } from "../comps/light"; import { useData } from "../util/useData"; import VoiceAnalysis from "../comps/voiceanalysis"; -import { sendOsc, OSC_ADDRESS, updatePrompt } from "../util/osc"; +import { sendOsc, OSC_ADDRESS, updatePrompt, onOscMessageReceived } from "../util/osc"; import { DebugControl, TEST_PROMPT } from "../comps/debug"; import { useUser } from "../util/useUser"; +import { set } from "zod/v4"; const CUELIST_FILE = 'cuelist_demo2.json'; @@ -47,6 +48,8 @@ export function FreeFlow(){ const [cuelist, setCuelist] = useState([]); const [currentCue, setCurrentCue] = useState(null); + const [nextCue, setNextCue] = useState(null); + const [chatWelcome, setChatWelcome] = useState(null); const [audioInput, setAudioInput] = useState(true); const [autoSend, setAutoSend] = useState(true); @@ -89,6 +92,24 @@ export function FreeFlow(){ sendOsc(OSC_ADDRESS.CHOICE, 'reset'); } + function onOsc(payload){ + console.log('onOsc', payload); + const address=payload.addr; + const message=payload.args[0]; + + switch(address){ + case '/playcue': + // playCue(cuelist.find(c => c.id === message)); + setNextCue(()=>message); + break; + case '/stopcue': + onStop(); + break; + } + + // Handle OSC messages here + } + function playAudio(url){ if(!url) return; @@ -542,6 +563,18 @@ export function FreeFlow(){ },[status]); + useEffect(()=>{ + + if(!nextCue) return; + + console.log('Next cue:', nextCue); + playCue(cuelist.find(c => c.name === nextCue)); // Play the next cue when it changes + + // Reset next cue after playing + setNextCue(null); + + },[nextCue]); + useEffect(()=>{ fetch(CUELIST_FILE) @@ -557,6 +590,8 @@ export function FreeFlow(){ refAudioPrompt.current = new Audio('assets/sfx/sfx-05.mp3'); // Load audio prompt if available + onOscMessageReceived(onOsc); // Set up OSC message listener + },[]); diff --git a/vite/src/util/osc.js b/vite/src/util/osc.js index 8c12139..7d6c06d 100644 --- a/vite/src/util/osc.js +++ b/vite/src/util/osc.js @@ -1,4 +1,5 @@ import { invoke } from '@tauri-apps/api/core'; +import { listen } from '@tauri-apps/api/event'; import { fetch } from '@tauri-apps/plugin-http'; export const OSC_ADDRESS={ @@ -58,4 +59,16 @@ export async function updatePrompt(prompt) { }catch(error){ console.error('Error updating prompt:', error); } +} + + +export function onOscMessageReceived(callback) { + try{ + listen('osc_message', (event) => { + console.log(`Received OSC message: ${event.payload}`); + callback(event.payload); + }); + }catch(error){ + console.error('Error setting up OSC message listener:', error); + } } \ No newline at end of file