add speech

vite/package-lock.json

@@ -10,8 +10,10 @@
       "dependencies": {
         "@tailwindcss/vite": "^4.1.8",
         "@tauri-apps/plugin-http": "^2.4.4",
+        "gsap": "^3.13.0",
         "react": "^19.1.0",
         "react-dom": "^19.1.0",
+        "react-speech-recognition": "^4.0.1",
         "tailwindcss": "^4.1.8"
       },
       "devDependencies": {
@@ -2177,6 +2179,11 @@
       "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.11.tgz",
       "integrity": "sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ=="
     },
+    "node_modules/gsap": {
+      "version": "3.13.0",
+      "resolved": "https://registry.npmjs.org/gsap/-/gsap-3.13.0.tgz",
+      "integrity": "sha512-QL7MJ2WMjm1PHWsoFrAQH/J8wUeqZvMtHO58qdekHpCfhvhSL4gSiz6vJf5EeMP0LOn3ZCprL2ki/gjED8ghVw=="
+    },
     "node_modules/has-flag": {
       "version": "4.0.0",
       "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz",
@@ -2539,6 +2546,11 @@
         "url": "https://github.com/sponsors/sindresorhus"
       }
     },
+    "node_modules/lodash.debounce": {
+      "version": "4.0.8",
+      "resolved": "https://registry.npmjs.org/lodash.debounce/-/lodash.debounce-4.0.8.tgz",
+      "integrity": "sha512-FT1yDzDYEoYWhnSGnpE/4Kj1fLZkDFyqRb7fNt6FdYOSxlUWAtp42Eh6Wb0rGIv/m9Bgo7x4GhQbm5Ys4SG5ow=="
+    },
     "node_modules/lodash.merge": {
       "version": "4.6.2",
       "resolved": "https://registry.npmjs.org/lodash.merge/-/lodash.merge-4.6.2.tgz",
@@ -2784,6 +2796,17 @@
         "react": "^19.1.0"
       }
     },
+    "node_modules/react-speech-recognition": {
+      "version": "4.0.1",
+      "resolved": "https://registry.npmjs.org/react-speech-recognition/-/react-speech-recognition-4.0.1.tgz",
+      "integrity": "sha512-0fIqzLtfY8vuYA6AmJVK7qiabZx0oFKOO+rbiBgFI3COWVGREy0A+gdU16hWXmFebeyrI8JsOLYsWk6WaHUXRw==",
+      "dependencies": {
+        "lodash.debounce": "^4.0.8"
+      },
+      "peerDependencies": {
+        "react": ">=16.8.0"
+      }
+    },
     "node_modules/resolve-from": {
       "version": "4.0.0",
       "resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-4.0.0.tgz",
@@ -4289,6 +4312,11 @@
       "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.11.tgz",
       "integrity": "sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ=="
     },
+    "gsap": {
+      "version": "3.13.0",
+      "resolved": "https://registry.npmjs.org/gsap/-/gsap-3.13.0.tgz",
+      "integrity": "sha512-QL7MJ2WMjm1PHWsoFrAQH/J8wUeqZvMtHO58qdekHpCfhvhSL4gSiz6vJf5EeMP0LOn3ZCprL2ki/gjED8ghVw=="
+    },
     "has-flag": {
       "version": "4.0.0",
       "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz",
@@ -4476,6 +4504,11 @@
         "p-locate": "^5.0.0"
       }
     },
+    "lodash.debounce": {
+      "version": "4.0.8",
+      "resolved": "https://registry.npmjs.org/lodash.debounce/-/lodash.debounce-4.0.8.tgz",
+      "integrity": "sha512-FT1yDzDYEoYWhnSGnpE/4Kj1fLZkDFyqRb7fNt6FdYOSxlUWAtp42Eh6Wb0rGIv/m9Bgo7x4GhQbm5Ys4SG5ow=="
+    },
     "lodash.merge": {
       "version": "4.6.2",
       "resolved": "https://registry.npmjs.org/lodash.merge/-/lodash.merge-4.6.2.tgz",
@@ -4632,6 +4665,14 @@
         "scheduler": "^0.26.0"
       }
     },
+    "react-speech-recognition": {
+      "version": "4.0.1",
+      "resolved": "https://registry.npmjs.org/react-speech-recognition/-/react-speech-recognition-4.0.1.tgz",
+      "integrity": "sha512-0fIqzLtfY8vuYA6AmJVK7qiabZx0oFKOO+rbiBgFI3COWVGREy0A+gdU16hWXmFebeyrI8JsOLYsWk6WaHUXRw==",
+      "requires": {
+        "lodash.debounce": "^4.0.8"
+      }
+    },
     "resolve-from": {
       "version": "4.0.0",
       "resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-4.0.0.tgz",

vite/package.json

@@ -12,8 +12,10 @@
   "dependencies": {
     "@tailwindcss/vite": "^4.1.8",
     "@tauri-apps/plugin-http": "^2.4.4",
+    "gsap": "^3.13.0",
     "react": "^19.1.0",
     "react-dom": "^19.1.0",
+    "react-speech-recognition": "^4.0.1",
     "tailwindcss": "^4.1.8"
   },
   "devDependencies": {

vite/src-tauri/Cargo.toml

@@ -21,7 +21,7 @@ tauri-build = { version = "2.2.0", features = [] }
 serde_json = "1.0"
 serde = { version = "1.0", features = ["derive"] }
 log = "0.4"
-tauri = { version = "2.5.0", features = [] }
+tauri = { version = "2.5.0", features = ["devtools"] }
 tauri-plugin-log = "2.0.0-rc"
 tauri-plugin-http = "2"
 dotenv = "0.15.0"

vite/src/App.css

@@ -0,0 +1 @@
+@import "tailwindcss";

vite/src/App.jsx

@@ -1,6 +1,12 @@
 import { useEffect, useRef, useState } from 'react';
 import './App.css'
 import { sendChatMessage } from './util/chat';
+import SpeechRecognition, { useSpeechRecognition } from 'react-speech-recognition';
+import { textToSpeech } from './util/tts';
+import { gsap } from "gsap";
+import { SplitText } from 'gsap/SplitText';
+import { set } from 'zod';
+gsap.registerPlugin(SplitText);
 
 
 const BASE_URL='http://localhost:3333';
@@ -9,81 +15,103 @@ function App() {
   
   const [history, setHistory] = useState([]);
   const [processing, setProcessing] = useState(false);
+  const [showProcessing, setShowProcessing] = useState(false);
+
   const [prompt, setPrompt] = useState([]);
   
   const refHistoryContainer= useRef(null);
   const refPrompContainer= useRef(null);
+  const refInput=useRef(null);
 
-  function onSubmitToNode(event) {
+  const {
-    event.preventDefault();
+    transcript,
-    const input = event.target.elements.input.value;
+    finalTranscript,
-    console.log("Submitted:", input);
+    listening,
+    resetTranscript,
+  }=useSpeechRecognition();
 
-    // setProcessing(true);
 
+  function restart(){
+    console.log("Restarting...");
+    setHistory([]);
+    setPrompt([]);
+    refInput.current.value = '';
+    resetTranscript();
+    SpeechRecognition.stopListening();
 
-    fetch(`${BASE_URL}/generate`, {
+    // create start message
-      method: 'POST',
+    const startTime=Date.now();
-      headers: {
+    setProcessing(true);
-        'Content-Type': 'application/json',
+    sendChatMessage([]).then(response => {
-      },
-      body: JSON.stringify({ 
-        input:[
-          ...history,
-          {
-            role:'user',
-            content:[{
-              type:'input_text',
-              text: input 
-            }]
-          }
-        ]
-      }),
-    }).then(response => {
       if (!response.ok) {
         throw new Error('Network response was not ok');
       }
-      response.json().then(data => {;
+    
-        console.log(data);
+      let data=response;
+      console.log('get reply: ', data, new Date(Date.now()-startTime).toISOString().slice(11, 19));
+      
 
       // add to history
       setHistory(prev => [...prev, {
-          role: 'user',
-          content: [{
-              type:'input_text',
-              text: input
-          }]
-        }, {
         role: 'assistant',
-          content: [{
+        content: data.output_text,
-              type:'output_text',
-              text: data.output_text
-          }]
       }]);
-
       setPrompt([
         ...prompt,
         data.prompt,
       ]);
 
-        // clear input
+      // tts
-        event.target.elements.input.value = '';
+      console.log('create speech:', data.output_text);
-        setProcessing(false);        
+      textToSpeech(data.output_text).then(audioUrl => {
+        const audio = new Audio(audioUrl);
+        
+        console.log('play audio...', new Date(Date.now()-startTime).toISOString().slice(11, 19));
 
+        audio.play().catch(error => {
+          console.error('Audio playback failed:', error);
+        });
+      }).catch(error => {
+        console.error('TTS error:', error);
       });
 
+      setProcessing(false);        
+
     });
+  }
 
+  function toggleAudio() {
+    // console.log("onclickAudio");
+    if(!listening){
+      SpeechRecognition.startListening({ continuous: true, language: 'zh-TW' });   
 
+    }else{
+      SpeechRecognition.stopListening();
+    }
   }
 
+
+
+
   function onSubmit(event) {    
     event.preventDefault();
+
+    if(processing) {
+      console.warn("Already processing, ignoring submission.");
+      return;
+    }
+    setProcessing(true);
+    setShowProcessing(true);
+
     const input = event.target.elements.input.value;
-    console.log("Submitted:", input);
 
-    // setProcessing(true);
+    if(!input.trim()?.length) {
+      console.warn("Input is empty, ignoring submission.");
+      return;
+    }
 
+    const startTime=Date.now();
+    console.log("Submit reply:", input);
     
     sendChatMessage([
         ...history,
@@ -94,16 +122,14 @@ function App() {
     ]).then(response => {
       if (!response.ok) {
         throw new Error('Network response was not ok');
+        setProcessing(false);
       }
       
       let data=response;
-      console.log(data);
+      console.log('get reply: ', data, new Date(Date.now()-startTime).toISOString().slice(11, 19));
 
       // add to history
       setHistory(prev => [...prev, {
-        role: 'user',
-        content:input,
-      }, {
         role: 'assistant',
         content: data.output_text,
       }]);              
@@ -113,21 +139,108 @@ function App() {
         data.prompt,
       ]);
       
-      // clear input
+      setShowProcessing(false);
-      event.target.elements.input.value = '';
+
-      setProcessing(false);        
+      // tts
+      console.log('create speech:', data.output_text);
+      textToSpeech(data.output_text).then(audioUrl => {
+        const audio = new Audio(audioUrl);
+        
+        console.log('play audio...', new Date(Date.now()-startTime).toISOString().slice(11, 19));
+
+        audio.play().catch(error => {
+          console.error('Audio playback failed:', error);
+        });
+        
+        audio.addEventListener('ended',() => {
+          console.log('Audio playback ended');
+          setProcessing(()=>false);
+        });
+
+      }).catch(error => {
+        console.error('TTS error:', error);
+        setProcessing(()=>false);
+      });
+    
 
     });
 
+    // clear input
+    event.target.elements.input.value = '';
+    // setProcessing(()=>false);        
+    setHistory(prev => [...prev, {
+      role: 'user',
+      content:input,
+    }]);
+
 
   }
   useEffect(()=>{    
     refHistoryContainer.current.scrollTop = refHistoryContainer.current.scrollHeight;    
+
+    // Animate the history items
+    if(history.length === 0) return;
+
+    let last_item=document.querySelector('.last_history');
+    console.log('last_item', last_item);
+    if(!last_item) return;
+
+    let split=SplitText.create(last_item, {
+      type: "chars",
+      aria:'hidden'      
+    });
+    console.log('split', split);
+    gsap.fromTo(split.chars, {
+      opacity: 0,      
+    }, {
+      opacity: 1,
+      y: 0,
+      duration: 1,
+      ease: "steps(1)",
+      stagger: 0.05
+    });
+
+
+
   },[history]);
   useEffect(()=>{    
     refPrompContainer.current.scrollTop = refPrompContainer.current.scrollHeight;    
   },[prompt]);
   
+
+  useEffect(()=>{
+
+    if(listening){
+      refInput.current.value = transcript;
+    }
+
+  },[transcript]);
+
+
+  useEffect(()=>{
+    if(finalTranscript){
+      refInput.current.value = finalTranscript;
+      console.log('Final Transcript:', finalTranscript);
+      
+      if(processing) return; // Prevent submission if already processing
+
+      
+      // Submit the final transcript
+      onSubmit({
+        preventDefault: () => {},
+        target: {
+          elements: {
+            input: refInput.current
+          }
+        }
+      });
+      resetTranscript(); // Clear the transcript after submission
+
+    }
+  },[finalTranscript]);
+
+
+
   return (
     <main className='h-screen flex flex-col gap-8 justify-end p-8'>
       <div ref={refPrompContainer} className='flex-1 flex flex-col gap-2 border-4 overflow-y-auto'>        
@@ -143,21 +256,35 @@ function App() {
       </div>      
       <div ref={refHistoryContainer} className='flex-1 overflow-y-auto'>
         <div className='flex flex-col justify-end gap-2'>
-          {history?.length==0? (
+          {history?.length==0 && !showProcessing? (
             <div className='p-2'>History will appear here...</div>
           ):(
             history.map((item, index) => (
               <div key={index} className={`p-2 rounded border-4 ${item.role === 'user' ? 'bg-gray-100' : 'bg-yellow-100'}`}>
-                <p className='text-lg whitespace-pre-wrap'>{item.content}</p>                
+                <p className={`text-lg whitespace-pre-wrap history_item ${index==history?.length-1 && 'last_history'}`}>{item.content}</p>                
               </div>
             ))
           )}
+          {showProcessing && (
+            <div className='p-2 rounded border-4 bg-yellow-100'>
+              <span className='animate-pulse'>...</span>
+            </div>
+          )}
+        </div>
       </div>      
+      <div className='flex flex-col gap-2'>
+        <div className='flex flex-row justify-end gap-2 '>
+          <button className='self-end' onClick={restart}>Restart</button>
+          <span className='flex-1'></span>
+          <button className='' onClick={()=>{
+            refInput.current.value=''
+            resetTranscript();
+          }}>clear</button>
+          <button onClick={toggleAudio} className={`${listening? '!bg-red-200':'!bg-gray-200'}`}>{listening? 'AudioIn On':'AudioIn Off'}</button>
         </div>
-      <div className=''>
         <form className='flex flex-col justify-center *:border-4 gap-4' onSubmit={onSubmit} autoComplete="off">
-          <input id="input" name="input" required className='self-stretch p-2' autoComplete="off"/>
+          <textarea ref={refInput} id="input" name="input" required className='self-stretch p-2 resize-none' rows={3} autoComplete="off"/>          
-          <button type="submit" className='rounded-full uppercase' disabled={processing}>Send</button>
+          <button type="submit" className='uppercase' disabled={processing}>Send</button>
         </form>
       </div>
     </main>

vite/src/index.css

@@ -1 +1,9 @@
 @import "tailwindcss";
+
+
+@layer base{
+  button{
+    @apply rounded-full border-4 px-2 bg-slate-200;
+  }
+
+}

vite/src/util/chat.js

@@ -1,6 +1,6 @@
 import { fetch } from '@tauri-apps/plugin-http';
 import { invoke } from '@tauri-apps/api/core';
-import { system_prompt } from './system_prompt';
+import { system_prompt, welcome_prompt } from './system_prompt';
 
 async function getOpenAIToken() {
     return invoke('get_env',{name:'OPENAI_API_KEY'});
@@ -20,10 +20,11 @@ export async function sendChatMessage(messages) {
         body: JSON.stringify({
             model: 'gpt-4o',
             messages: [
-                {
+                // {
-                    role: "system",
+                //     role: "system",
-                    content:system_prompt,                    
+                //     content:system_prompt,                    
-                },
+                // },
+                ...welcome_prompt,
                 ...messages
             ],
             response_format: {
@@ -59,13 +60,13 @@ export async function sendChatMessage(messages) {
     const output= await response.json();
     const choice= output.choices[0];
 
-    console.log("Generated response:", choice.message);
+    // console.log("Generated response:", choice.message);
     const result=JSON.parse(choice.message.content);
     
     // send to tauri
     await invoke('send_osc_message', { 
         key:'/prompt',        
-        message: result.prompt.replaceAll('"', '"'), // escape quotes for OSC
+        message: result.prompt.replaceAll('"', ''), // escape quotes for OSC
         host:`0.0.0.0:0`,
         target: '127.0.0.1:8787'
     });

vite/src/util/system_prompt.js

@@ -60,3 +60,17 @@ Prompt 5：“A boy sits beside a sleeping figure, imagining the summer he never
 
 ❗ 禁止在對話中提及「Prompt」、「畫面」、「圖像生成」或任何 AI 正在進行輸出的技術細節。請務必以自然的對話方式與使用者互動，讓生成的英文句子看起來像是內在的文字敘述，而非指令或轉換的結果。
 ❗ 英文描述不會出現在中文回答之中`;
+
+
+export const welcome_prompt=[
+    {
+        "role": "system",
+        "content": system_prompt
+    },
+    {
+        "role": "system",
+        "content": "請開始引導使用者回想一段內心的遺憾或未竟之事。"
+    }
+]
+
+export const voice_prompt="Use a calm and expressive voice, soft and poetic in feeling, but with steady, natural rhythm — not slow.";

vite/src/util/tts.js

@@ -0,0 +1,55 @@
+import { fetch } from '@tauri-apps/plugin-http';
+import { invoke } from '@tauri-apps/api/core';
+import { voice_prompt } from './system_prompt';
+
+export async function textToSpeech(text) {
+
+    const token = await invoke('get_env', { name: 'OPENAI_API_KEY' });
+
+    const response = await fetch('https://api.openai.com/v1/audio/speech', {
+        method: 'POST',
+        headers: {
+            'Content-Type': 'application/json',
+            'Authorization': `Bearer ${token}`
+        },
+        body: JSON.stringify({
+            model: 'gpt-4o-mini-tts',//'tts-1',
+            input: text,
+            voice: "fable",
+            instructions: voice_prompt,
+            response_format:'opus'
+        }),
+    });
+
+    if (!response.ok) {
+        const text = await response.text();
+        console.error("Error response:", text);
+        throw new Error(`HTTP error! status: ${response.status}`);
+    }
+
+    const reader = response.body.getReader();
+    const stream=new ReadableStream({
+        start(controller) {
+            function push() {
+                reader.read().then(({ done, value }) => {
+                    if (done) {
+                        controller.close();
+                        return;
+                    }
+                    controller.enqueue(value);
+                    push();
+                }).catch(error => {
+                    console.error('Stream reading error:', error);
+                    controller.error(error);
+                });
+            }
+            push();
+        }
+    });
+
+    const audioBlob = await new Response(stream).blob();
+    const audioUrl = URL.createObjectURL(audioBlob);
+
+    return audioUrl;
+
+}