update

4 months ago · 5353f5406e
parent a4661e1dbd
commit 5353f5406e
18 changed files with 115 additions and 5 deletions
--- a/vite/index.html
+++ b/vite/index.html
@ -4,6 +4,10 @@
    <meta charset="UTF-8" />
    <link rel="icon" href="/favicon.ico" />
    <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+      <link
+    href="https://fonts.googleapis.com/css2?family=Noto+Serif+TC:wght@200..900&family=Sora:wght@100..800&display=swap"
+    rel="stylesheet">
+
    <title>24070-Conversation</title>
  </head>
  <body>
--- a/vite/public/assets/ai/sfx-06-ai-04-record-03-ai-05-sfx-07.mp3
+++ b/vite/public/assets/ai/sfx-06-ai-04-record-03-ai-05-sfx-07.mp3
--- a/vite/public/assets/ai/sfx-08-record-04-ai-06.mp3
+++ b/vite/public/assets/ai/sfx-08-record-04-ai-06.mp3
--- a/vite/public/assets/bg-01
+++ b/vite/public/assets/bg-01
--- a/vite/public/assets/bg-01
+++ b/vite/public/assets/bg-01
--- a/vite/public/assets/bg-02.mp3
+++ b/vite/public/assets/bg-02.mp3
--- a/vite/public/assets/bg-03.mp3
+++ b/vite/public/assets/bg-03.mp3
--- a/vite/public/assets/bg-04.mp3
+++ b/vite/public/assets/bg-04.mp3
--- a/vite/public/assets/record/record-01（小束袋）.mp3
+++ b/vite/public/assets/record/record-01（小束袋）.mp3
--- a/vite/public/assets/record/record-01（香料瓶）.mp3
+++ b/vite/public/assets/record/record-01（香料瓶）.mp3
--- a/vite/public/assets/record/record-05（小束袋）.mp3
+++ b/vite/public/assets/record/record-05（小束袋）.mp3
--- a/vite/public/assets/sfx/sfx-12.mp3
+++ b/vite/public/assets/sfx/sfx-12.mp3
--- a/vite/public/cuelist_demo2.json
+++ b/vite/public/cuelist_demo2.json
@ -5,7 +5,7 @@
            "name": "Q1", 
            "type": "space",
            "description": "Annonce",
-            "audioFile": "assets/bg-01 (小束袋）.mp3",
+            "audioFile": "assets/bg-02.mp3",
            "loop": true,
            "status":"reset"
        },
--- a/vite/src-tauri/src/lib.rs
+++ b/vite/src-tauri/src/lib.rs
@ -2,13 +2,20 @@ use dotenv::dotenv;
 use rosc::{encoder, OscMessage, OscPacket, OscType};
 use std::env;
 use std::{net::SocketAddrV4, str::FromStr};
-use tauri::{AppHandle, Manager};
+use tauri::{AppHandle, Manager, Emitter};
 use tokio::net::UdpSocket;
 use webview2_com::Microsoft::Web::WebView2::Win32::{
    ICoreWebView2Profile4, ICoreWebView2_13, COREWEBVIEW2_PERMISSION_KIND_MICROPHONE,
    COREWEBVIEW2_PERMISSION_STATE_DEFAULT,
 };
 use windows::core::{Interface, PCWSTR};
+use serde::Serialize;
+
+#[derive(Serialize, Clone)]
+struct OscEvent {
+    addr: String,
+    args: Vec<String>,
+}

 #[tauri::command]
 fn get_env(name: &str) -> String {
@ -97,10 +104,58 @@ pub fn run() {
                        .build(),
                )?;
            }
+
+            tauri::async_runtime::spawn(setup_osc_server(app.handle().clone()));
            Ok(())
        })
        .run(tauri::generate_context!())
        .expect("error while running tauri application");
+
+    
+}
+
+async fn setup_osc_server(app_handle: AppHandle) {
+
+    println!("Setting up OSC server...");
+    // setup osc sever
+    let addr = "0.0.0.0:8000";
+    let sock = UdpSocket::bind(addr).await.unwrap();
+    println!("Listening to {}", addr);
+
+    let mut buf = [0u8; rosc::decoder::MTU];
+
+    loop {
+        match sock.recv_from(&mut buf).await {
+            Ok((size, addr)) => {
+                println!("Received packet with size {} from: {}", size, addr);
+                if let Ok((_, packet)) = rosc::decoder::decode_udp(&buf[..size]) {
+                    if let OscPacket::Message(msg) = packet {
+                        println!("OSC message: {:?}", msg);
+                        
+                        app_handle.emit(
+                            "osc_message",
+                            OscEvent {
+                                addr: msg.addr.clone(),
+                                args: msg.args.iter().map(|arg| match arg {
+                                    OscType::Int(i) => i.to_string(),
+                                    OscType::Float(f) => f.to_string(),
+                                    OscType::String(s) => s.clone(),
+                                    OscType::Bool(b) => b.to_string(),
+                                    _ => format!("{:?}", arg),
+                                }).collect(),
+                            },
+                        ).unwrap();
+                    }
+                } else {
+                    println!("Failed to decode OSC packet");
+                }
+            }
+            Err(e) => {
+                println!("Error receiving from socket: {}", e);
+                break;
+            }
+        }
+    }
 }

 static mut LIGHT: Option<enttecopendmx::EnttecOpenDMX> = None;
--- a/vite/src/App.css
+++ b/vite/src/App.css
@ -1,5 +1,8 @@
@import "tailwindcss";

+html{
+    font-family: "Sora", sans-serif;
+}
 #root{
    @apply flex flex-col h-screen;
 }
--- a/vite/src/comps/numpad.jsx
+++ b/vite/src/comps/numpad.jsx
@ -62,7 +62,7 @@ export default function NumPad({onSend, disabled, type}){
                        }
                        break;
                    case NUMPAD_TYPE.PASSWORD:
-                        if(refInput.current.length==3){
+                        if(refInput.current.length=3){
                            onSend(refInput.current);
                            refAudio.current[KEY_ENTER]?.play();
                        }else{
@ -88,7 +88,7 @@ export default function NumPad({onSend, disabled, type}){

            return;
        }else if(e.key===KEY_BACKSPACE || e.key==='Backspace' || e.key==='Delete'){
-            setInput((prev)=>'');
+            setInput(()=>'');

            refAudio.current[KEY_BACKSPACE]?.play(); 
            return;
--- a/vite/src/pages/flow_free.jsx
+++ b/vite/src/pages/flow_free.jsx
@ -10,9 +10,10 @@ import NumPad, { NUMPAD_TYPE } from "../comps/numpad";
 import { Light } from "../comps/light";
 import { useData } from "../util/useData";
 import VoiceAnalysis from "../comps/voiceanalysis";
-import { sendOsc, OSC_ADDRESS, updatePrompt } from "../util/osc";
+import { sendOsc, OSC_ADDRESS, updatePrompt, onOscMessageReceived } from "../util/osc";
 import { DebugControl, TEST_PROMPT } from "../comps/debug";
 import { useUser } from "../util/useUser";
+import { set } from "zod/v4";


 const CUELIST_FILE = 'cuelist_demo2.json';
@ -47,6 +48,8 @@ export function FreeFlow(){

    const [cuelist, setCuelist] = useState([]);
    const [currentCue, setCurrentCue] = useState(null);
+    const [nextCue, setNextCue] = useState(null);
+
    const [chatWelcome, setChatWelcome] = useState(null);
    const [audioInput, setAudioInput] = useState(true); 
    const [autoSend, setAutoSend] = useState(true);
@ -89,6 +92,24 @@ export function FreeFlow(){
        sendOsc(OSC_ADDRESS.CHOICE, 'reset');
    }

+    function onOsc(payload){
+        console.log('onOsc', payload);
+        const address=payload.addr;
+        const message=payload.args[0];
+
+        switch(address){
+            case '/playcue':
+                // playCue(cuelist.find(c => c.id === message));
+                setNextCue(()=>message);
+                break;
+            case '/stopcue':
+                onStop();
+                break;
+        }
+
+        // Handle OSC messages here
+    }
+
    function playAudio(url){
        if(!url) return;

@ -542,6 +563,18 @@ export function FreeFlow(){

    },[status]);

+    useEffect(()=>{
+
+        if(!nextCue) return;
+
+        console.log('Next cue:', nextCue);
+        playCue(cuelist.find(c => c.name === nextCue)); // Play the next cue when it changes
+
+        // Reset next cue after playing
+        setNextCue(null);
+
+    },[nextCue]);
+

    useEffect(()=>{
        fetch(CUELIST_FILE)
@ -557,6 +590,8 @@ export function FreeFlow(){

        refAudioPrompt.current = new Audio('assets/sfx/sfx-05.mp3'); // Load audio prompt if available

+        onOscMessageReceived(onOsc); // Set up OSC message listener
+
    },[]);


--- a/vite/src/util/osc.js
+++ b/vite/src/util/osc.js
@ -1,4 +1,5 @@
 import { invoke } from '@tauri-apps/api/core';
+import { listen } from '@tauri-apps/api/event';
 import { fetch } from '@tauri-apps/plugin-http';

 export const OSC_ADDRESS={
@ -59,3 +60,15 @@ export async function updatePrompt(prompt) {
        console.error('Error updating prompt:', error);        
    }
 }
+
+
+export function onOscMessageReceived(callback) {
+    try{
+        listen('osc_message', (event) => {
+            console.log(`Received OSC message: ${event.payload}`);
+            callback(event.payload);
+        });
+    }catch(error){
+        console.error('Error setting up OSC message listener:', error);
+    }
+}