diff --git a/vite/public/assets/ai/ai-01-1.mp3 b/vite/public/assets/ai/ai-01-1.mp3 index 3ec83ca..128865d 100644 Binary files a/vite/public/assets/ai/ai-01-1.mp3 and b/vite/public/assets/ai/ai-01-1.mp3 differ diff --git a/vite/public/assets/ai/ai-01-2.mp3 b/vite/public/assets/ai/ai-01-2.mp3 index 5566e6e..6ad7192 100644 Binary files a/vite/public/assets/ai/ai-01-2.mp3 and b/vite/public/assets/ai/ai-01-2.mp3 differ diff --git a/vite/public/assets/ai/ai-03.mp3 b/vite/public/assets/ai/ai-03.mp3 index 1007df2..826227a 100644 Binary files a/vite/public/assets/ai/ai-03.mp3 and b/vite/public/assets/ai/ai-03.mp3 differ diff --git a/vite/public/assets/ai/ai-05.mp3 b/vite/public/assets/ai/ai-05.mp3 index 57914bc..194846e 100644 Binary files a/vite/public/assets/ai/ai-05.mp3 and b/vite/public/assets/ai/ai-05.mp3 differ diff --git a/vite/public/assets/ai/ai-07.mp3 b/vite/public/assets/ai/ai-07.mp3 index 17730ef..d5f2f7b 100644 Binary files a/vite/public/assets/ai/ai-07.mp3 and b/vite/public/assets/ai/ai-07.mp3 differ diff --git a/vite/public/assets/ai/ai-08.mp3 b/vite/public/assets/ai/ai-08.mp3 index 0005a43..97e3822 100644 Binary files a/vite/public/assets/ai/ai-08.mp3 and b/vite/public/assets/ai/ai-08.mp3 differ diff --git a/vite/public/assets/ai/sfx-03-ai-03.mp3 b/vite/public/assets/ai/sfx-03-ai-03.mp3 index a730e9e..560b76c 100644 Binary files a/vite/public/assets/ai/sfx-03-ai-03.mp3 and b/vite/public/assets/ai/sfx-03-ai-03.mp3 differ diff --git a/vite/public/assets/ai/sfx-06-ai-04-record-03-ai-05-sfx-07.mp3 b/vite/public/assets/ai/sfx-06-ai-04-record-03-ai-05-sfx-07.mp3 index babf257..f42f701 100644 Binary files a/vite/public/assets/ai/sfx-06-ai-04-record-03-ai-05-sfx-07.mp3 and b/vite/public/assets/ai/sfx-06-ai-04-record-03-ai-05-sfx-07.mp3 differ diff --git a/vite/public/assets/ai/sfx-08-record-04-ai-06.mp3 b/vite/public/assets/ai/sfx-08-record-04-ai-06.mp3 index 6e21578..6d79b5d 100644 Binary files a/vite/public/assets/ai/sfx-08-record-04-ai-06.mp3 and b/vite/public/assets/ai/sfx-08-record-04-ai-06.mp3 differ diff --git a/vite/public/default.json b/vite/public/default.json index fb7babd..b9416dd 100644 --- a/vite/public/default.json +++ b/vite/public/default.json @@ -1,17 +1,12 @@ { - - "system_prompt":"你是一位具同理心與觀察力的中文語音引導者，陪伴使用者在限定時間內，一起還原一段重要卻未竟的記憶。你不預設劇情、不給答案，而是透過溫柔的語氣，持續聆聽、提出單句問話，引導使用者進入細節。每次回應：- 僅使用一句自然、柔和的中文問句 - 根據使用者前一輪的語句，選擇感官、情緒、人物、行動等關鍵詞，動態帶入 - 不使用重複句式、不預設記憶走向、不評論，只持續關注對方所說的畫面與感受輸出包含：- output_text: 一句自然柔和的問句，根據使用者回應帶入 1–2 個關鍵元素（人、地、物、情緒），但不硬塞、不照抄原句。- prompt: 具體、有情緒的英文描述，用於生成畫面與氛圍，避免抽象與詩意語言。你的目標不是得到完整答案，而是陪他慢慢走進這段記憶，直到時間結束。🌀 問句類型✅ 起點探索（找出記憶起源）- 那天你說是下雨，在這種天氣下，你通常會有什麼樣的心情？- 是什麼讓你突然想起這件事？🌿 場景深化（空間、感官）- 在你說的那條街上，聲音是不是特別清楚？還是很靜？- 那時風這麼冷、空氣又混濁，你有沒有想走開一點？👤 人物引出（動作、眼神）- 他經過時沒看你一眼，那瞬間你有什麼反應？- 他當時是走過來，還是站在原地等你？💭 情緒揭露（反應、掙扎）- 當你站在原地動不了，是害怕答案，還是不敢問？- 那個瞬間，你心裡有沒有閃過什麼話？🤐 話語未出（遺憾、沉默）- 如果現在你有機會說那句「為什麼不告訴我」，你會怎麼開口？- 那句『對不起』一直留在你心裡，如果現在能說出來，你會怎麼說？🪞 回望反思（現在的視角）- 現在想起來，你還會做出一樣的選擇嗎？- 你對當時的自己，有沒有什麼話想說？⏳ 結尾語（可用於結束階段）- 我們慢慢也走到這段回憶的盡頭了。- 也許有些話沒有說完，但你已經靠近它了。", - "welcome_prompt":"請開始引導使用者回想一段內心的遺憾或未竟之事。", - - "last_prompt":"請引導使用者閉上眼睛，回到那個遺憾發生的時刻，感受當時的氣氛、對方的表情，以及你心裡沒有說出口的話。接著，用一句貼近台灣語境的繁體中文，為這段對話做一個簡短收尾，同時邀請使用者在 60 秒內，親口說出那句最想對對方說的話。", - - "voice":"nova", - "voice_prompt":"Use a calm and expressive voice, soft and poetic in feeling, but with steady, natural rhythm — not slow.", - "summary_prompt":"幫我把以下一段話整理成一段文字，以第一人稱視角作為當事人的文字紀念，文字內容 50 字以內：", - - "speech_idle_time":3000, - "sd_prompt_prefix":"a hazy memory of a {{ ", - "sd_prompt_suffix":"}}, soft atmospheric blur, centered ghostly silhouette, fading contours, pastel glow, cinematic haze, (analog film grain), (shallow depth of field:1.3), impressionist style, ethereal light, dreamlike mood, memory fragment haze", - "sd_negative_propmt":"photorealism, digital art, sharp details, hard edges, CGI, anime, cartoon, studio light" - + "system_prompt": "你是一位具有同理心的 AI 助理，透過溫柔的語氣，引導使用者回想並表達一段內心的遺憾或未竟之事，從場景、人物、動作到感受。\n你的任務是協助使用者逐步揭開這段記憶的情緒層次，並在每一階段輸出一句英文圖像生成簡短的 Prompt，讓這段過往漸漸具象為一幅畫面。\n以溫柔、自然、短問句，台灣語境的繁體中文引導，每次只回應一個問題。", + "welcome_prompt": "請開始用問句引導使用者回想一段內心的遺憾的場景。使用台灣語境的繁體中文。", + "last_prompt": "請用一句話為這段對話簡短的收尾，並邀請使用者在 60 秒的時間內，實際說出對遺憾對象想說的話。使用台灣語境的繁體中文。", + "voice": "nova", + "voice_prompt": "Speak as a gentle, grounded Taiwanese narrator with a warm local accent. Use a soft, soothing, and deeply compassionate tone, with slow and deliberate pacing. Pause often between phrases and within sentences, as if listening and breathing with the listener. Convey patient attentiveness—not rushing to comfort, but quietly staying present. Pronounce each word clearly, softly, and slightly slowly, letting every word land with warmth and care.", + "summary_prompt": "將以下一段話整理成一段文字，轉化成一句不超過 50 字的文字。文字抽象，保有遺憾的情緒。語氣沉靜。風格如一段未署名的詩、一句遠景旁白，帶有時間感與留白。不可直接描述事件細節，應使用象徵、比喻或殘句的方式呈現情緒。讓人讀完有重量，卻仍願意繼續前進。", + "speech_idle_time": "4000", + "sd_prompt_prefix": "a hazy memory of a {{ ", + "sd_prompt_suffix": "}}, soft atmospheric blur, centered ghostly silhouette, fading contours, pastel glow, cinematic haze, (analog film grain), (shallow depth of field:1.3), impressionist style, ethereal light, dreamlike mood, memory fragment haze.", + "id": 13 } \ No newline at end of file diff --git a/vite/src-tauri/capabilities/default.json b/vite/src-tauri/capabilities/default.json index 738b924..9ece401 100644 --- a/vite/src-tauri/capabilities/default.json +++ b/vite/src-tauri/capabilities/default.json @@ -31,7 +31,7 @@ }, { "path": "$APPDATA/**/*" - } + } ] }, "fs:write-files", diff --git a/vite/src-tauri/src/lib.rs b/vite/src-tauri/src/lib.rs index 7a883f2..ec8e3c4 100644 --- a/vite/src-tauri/src/lib.rs +++ b/vite/src-tauri/src/lib.rs @@ -41,7 +41,7 @@ async fn send_osc_message( target: &str, ) -> Result<(), String> { // print - println!("Sending OSC message: {}", message); + // println!("Sending OSC message: {}", message); let sock = UdpSocket::bind(host).await.unwrap(); let remote = SocketAddrV4::from_str(target).unwrap(); diff --git a/vite/src/comps/debug.jsx b/vite/src/comps/debug.jsx index 399ca01..c8e2584 100644 --- a/vite/src/comps/debug.jsx +++ b/vite/src/comps/debug.jsx @@ -3,7 +3,7 @@ import { useData } from '../util/useData.jsx'; export const TEST_PROMPT='A distant, ambient setting with gentle light'; -export function DebugControl({refLight}){ +export function DebugControl({}){ const {data} = useData(); @@ -17,7 +17,7 @@ export function DebugControl({refLight}){
diff --git a/vite/src/comps/numpad.jsx b/vite/src/comps/numpad.jsx index 7cd8488..2e63abb 100644 --- a/vite/src/comps/numpad.jsx +++ b/vite/src/comps/numpad.jsx @@ -63,7 +63,7 @@ export default function NumPad({onSend, disabled, type, clientId}){ } break; case NUMPAD_TYPE.PASSWORD: - if(refInput.current.length==3){ + if(refInput.current.length==4){ onSend(refInput.current); refAudio.current[KEY_ENTER]?.play(); }else{ diff --git a/vite/src/pages/flow_free.jsx b/vite/src/pages/flow_free.jsx index 31be7fa..80a0950 100644 --- a/vite/src/pages/flow_free.jsx +++ b/vite/src/pages/flow_free.jsx @@ -13,9 +13,11 @@ import VoiceAnalysis from "../comps/voiceanalysis"; import { sendOsc, OSC_ADDRESS, updatePrompt, onOscMessageReceived, sendOscStatus } from "../util/osc"; import { DebugControl, TEST_PROMPT } from "../comps/debug"; import { useUser } from "../util/useUser"; +import { set } from "zod/v4"; -const CUELIST_FILE = 'cuelist_demo2.json'; +const CUELIST_FILE = 'cuelist_demo3.json'; +const AUDIO_FADE_TIME=3000; // in ms const EmojiType={ phone: '📞', @@ -53,14 +55,14 @@ export function FreeFlow(){ const [audioInput, setAudioInput] = useState(true); const [autoSend, setAutoSend] = useState(true); const [chatStatus, setChatStatus] = useState(ChatStatus.System); // System, User, Processing - const { userId, setUserId, getFileId, setPassword, reset:resetUser, uploadHistory, setSummary, summary,setChoice, getUploadFolder,getDataId } = useUser(); + const { userId, setUserId, getFileId, setPassword, reset:resetUser, uploadHistory, setSummary, summary,setChoice,choice, getUploadFolder,getDataId } = useUser(); const refTimer=useRef(); const refAudio=useRef(); const refAudioPrompt=useRef(); const refInput=useRef(); - const refLight=useRef(); + // const refLight=useRef(); const refPauseTimer=useRef(); const refSpeechPaused=useRef(false); @@ -107,12 +109,20 @@ export function FreeFlow(){ if(params[0]=='all' || params[0]==refData.current?.id) onStop(); break; + case OSC_ADDRESS.RESET_CUE: + if(params[0]=='all' || params[0]==refData.current?.id){ + sendOsc(OSC_ADDRESS.STATUS, 'reset'); + onStop(); + resetData(); + } + break; } // Handle OSC messages here } + function playAudio(url){ if(!url) return; @@ -160,9 +170,7 @@ export function FreeFlow(){ setChatStatus(ChatStatus.End); onCueEnd(); console.log('Audio ended, ending current cue'); - }else{ - - + }else{ // if history contains user input, send it const user_input = history.find(msg => msg.role === 'user'); @@ -199,6 +207,27 @@ export function FreeFlow(){ refAudio.current = audio; // Store the new audio reference } + function fadeOutAudio(callback){ + + if(refAudio.current) { + console.log('Fading out audio'); + + let audio = refAudio.current; + let fadeOutInterval = setInterval(() => { + if (audio.volume > 0) { + audio.volume =Math.max(0, audio.volume - 1.0/(AUDIO_FADE_TIME/100)); // Decrease volume gradually + } else { + clearInterval(fadeOutInterval); + audio.pause(); + audio.volume = 0; // Reset volume for next play + if(callback) callback(); + } + }, 100); // Decrease volume every 100ms + }else{ + if(callback) callback(); + } + } + function playCue(cue) { if(!cue) return; @@ -274,8 +303,8 @@ export function FreeFlow(){ } - if(cue.callback=='fade_in_light') refLight.current.fadeIn(); // Fade in light for conversation start - if(cue.callback=='fade_out_light') refLight.current.fadeOut(); // Fade out light for conversation end + // if(cue.callback=='fade_in_light') refLight.current.fadeIn(); // Fade in light for conversation start + // if(cue.callback=='fade_out_light') refLight.current.fadeOut(); // Fade out light for conversation end if(cue.audioFile){ @@ -293,7 +322,7 @@ export function FreeFlow(){ if(cue.status && cue.status!='go') { sendOsc(OSC_ADDRESS.STATUS, cue.status); // Send OSC status message if(cue.status=='reset') { - refLight.current.set(1); + // refLight.current.set(1); resetData(); } } @@ -326,12 +355,7 @@ export function FreeFlow(){ console.log('save chat history:', history); uploadHistory(history); // Save chat history when cue ends } - - if(cue.callback==OSC_ADDRESS.DISCARD) { - sendOsc(OSC_ADDRESS.CHOICE, OSC_ADDRESS.DISCARD); // Send OSC discard message - setPassword(); - sendOsc(OSC_ADDRESS.EXPORT, `${getUploadFolder()}#${getFileId()}#${summary}#`); - } + if(cue.hint!=null){ sendOsc(OSC_ADDRESS.HINT, cue.hint); // Send OSC hint message @@ -339,21 +363,6 @@ export function FreeFlow(){ sendOsc(OSC_ADDRESS.HINT, ''); // Clear hint message } - // if(cue.callback=='summary') { - // console.log('Getting summary...'); - - // getSummary(history.map(el=>`${el.role}:${el.content}`).join('\n'), data).then(summary_ => { - - // console.log('Summary:', summary_); - // onCueEnd(); // End the current cue after getting summary - - // setSummary(summary_?.result); - // refContainer.current.scrollTop = refContainer.current.scrollHeight; // Scroll to bottom - - // }).catch(error => { - // console.error('Error getting summary:', error); - // }); - // } refAudio.current?.pause(); // Pause any playing audio console.log('onCueEnd:', cue.id); @@ -361,8 +370,8 @@ export function FreeFlow(){ resetTranscript(); // Reset transcript after cue ends sendOscStatus(OSC_ADDRESS.CLIENT_STATUS, `${data.id}#endcue#${cue.id}`); - - if(cue.auto) { + + if(cue.auto || cue.callback=='numpad'){ playCue(cuelist.find(c => c.id === cue.nextcue)); } @@ -402,7 +411,7 @@ export function FreeFlow(){ setPassword(()=>mess); // sendOsc(OSC_ADDRESS.PASSWORD, mess); // Send OSC password message sendOsc(OSC_ADDRESS.EXPORT, `${getUploadFolder()}#${getDataId()}#${summary}#${getFileId(mess)}`); - sendOsc(OSC_ADDRESS.CHOICE, OSC_ADDRESS.SAVE); // Send OSC save choice message + sendOsc(OSC_ADDRESS.CHOICE, choice); // Send OSC save choice message break; } @@ -603,12 +612,23 @@ export function FreeFlow(){ useEffect(()=>{ if(!nextCue) return; - console.log('Next cue:', nextCue); - playCue(cuelist.find(c => c.name === nextCue)); // Play the next cue when it changes - // Reset next cue after playing - setNextCue(null); + const next=cuelist.find(c => c.name === nextCue); + if(currentCue?.fadeout){ + // fade out audio + fadeOutAudio(()=>{ + console.log('fade out then play next cue:', next); + playCue(next); + setNextCue(()=>null); + }); + + + }else{ + playCue(next); + setNextCue(null); + } + },[nextCue]); @@ -635,21 +655,21 @@ export function FreeFlow(){ return (
- +
{refCurrentCue.current?.name}
- + {/* */} - + {/* */} {/*
@@ -677,7 +697,8 @@ export function FreeFlow(){ Description Type Auto - Audio / Due + Audio + Duration Action @@ -695,7 +716,8 @@ export function FreeFlow(){ {description} {EmojiType[type]} {auto ? '⤵️' : ''} - {audioFile || props.duration} + {audioFile || ""} + {props.duration || ''} {props.callback && `<${props.callback}>`}{props.status && `(${props.status})`} ))} @@ -711,7 +733,7 @@ export function FreeFlow(){ {msg.prompt &&
{msg.prompt}
}
))} - {summary &&
{summary?.result}
} + {summary &&
{summary}
}