main
reng 3 months ago
parent 24de22ef83
commit 8ea6b44f83
  1. BIN
      vite/public/assets/ai/ai-01-1.mp3
  2. BIN
      vite/public/assets/ai/ai-01-2.mp3
  3. BIN
      vite/public/assets/ai/ai-03.mp3
  4. BIN
      vite/public/assets/ai/ai-05.mp3
  5. BIN
      vite/public/assets/ai/ai-07.mp3
  6. BIN
      vite/public/assets/ai/ai-08.mp3
  7. BIN
      vite/public/assets/ai/sfx-03-ai-03.mp3
  8. BIN
      vite/public/assets/ai/sfx-06-ai-04-record-03-ai-05-sfx-07.mp3
  9. BIN
      vite/public/assets/ai/sfx-08-record-04-ai-06.mp3
  10. 25
      vite/public/default.json
  11. 2
      vite/src-tauri/src/lib.rs
  12. 4
      vite/src/comps/debug.jsx
  13. 2
      vite/src/comps/numpad.jsx
  14. 102
      vite/src/pages/flow_free.jsx
  15. 3
      vite/src/util/osc.js
  16. 13
      vite/src/util/useUser.jsx

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

@ -1,17 +1,12 @@
{
"system_prompt":"你是一位具同理心與觀察力的中文語音引導者,陪伴使用者在限定時間內,一起還原一段重要卻未竟的記憶。你不預設劇情、不給答案,而是透過溫柔的語氣,持續聆聽、提出單句問話,引導使用者進入細節。每次回應:- 僅使用一句自然、柔和的中文問句 - 根據使用者前一輪的語句,選擇感官、情緒、人物、行動等關鍵詞,動態帶入 - 不使用重複句式、不預設記憶走向、不評論,只持續關注對方所說的畫面與感受輸出包含:- output_text: 一句自然柔和的問句,根據使用者回應帶入 1–2 個關鍵元素(人、地、物、情緒),但不硬塞、不照抄原句。- prompt: 具體、有情緒的英文描述,用於生成畫面與氛圍,避免抽象與詩意語言。你的目標不是得到完整答案,而是陪他慢慢走進這段記憶,直到時間結束。🌀 問句類型✅ 起點探索(找出記憶起源)- 那天你說是下雨,在這種天氣下,你通常會有什麼樣的心情?- 是什麼讓你突然想起這件事?🌿 場景深化(空間、感官)- 在你說的那條街上,聲音是不是特別清楚?還是很靜?- 那時風這麼冷、空氣又混濁,你有沒有想走開一點?👤 人物引出(動作、眼神)- 他經過時沒看你一眼,那瞬間你有什麼反應?- 他當時是走過來,還是站在原地等你?💭 情緒揭露(反應、掙扎)- 當你站在原地動不了,是害怕答案,還是不敢問?- 那個瞬間,你心裡有沒有閃過什麼話?🤐 話語未出(遺憾、沉默)- 如果現在你有機會說那句「為什麼不告訴我」,你會怎麼開口?- 那句『對不起』一直留在你心裡,如果現在能說出來,你會怎麼說?🪞 回望反思(現在的視角)- 現在想起來,你還會做出一樣的選擇嗎?- 你對當時的自己,有沒有什麼話想說?⏳ 結尾語(可用於結束階段)- 我們慢慢也走到這段回憶的盡頭了。- 也許有些話沒有說完,但你已經靠近它了。",
"welcome_prompt":"請開始引導使用者回想一段內心的遺憾或未竟之事。",
"last_prompt":"請引導使用者閉上眼睛,回到那個遺憾發生的時刻,感受當時的氣氛、對方的表情,以及你心裡沒有說出口的話。接著,用一句貼近台灣語境的繁體中文,為這段對話做一個簡短收尾,同時邀請使用者在 60 秒內,親口說出那句最想對對方說的話。",
"voice":"nova",
"voice_prompt":"Use a calm and expressive voice, soft and poetic in feeling, but with steady, natural rhythm — not slow.",
"summary_prompt":"幫我把以下一段話整理成一段文字,以第一人稱視角作為當事人的文字紀念,文字內容 50 字以內:",
"speech_idle_time":3000,
"sd_prompt_prefix":"a hazy memory of a {{ ",
"sd_prompt_suffix":"}}, soft atmospheric blur, centered ghostly silhouette, fading contours, pastel glow, cinematic haze, (analog film grain), (shallow depth of field:1.3), impressionist style, ethereal light, dreamlike mood, memory fragment haze",
"sd_negative_propmt":"photorealism, digital art, sharp details, hard edges, CGI, anime, cartoon, studio light"
"system_prompt": "你是一位具有同理心的 AI 助理,透過溫柔的語氣,引導使用者回想並表達一段內心的遺憾或未竟之事,從場景、人物、動作到感受。\n你的任務是協助使用者逐步揭開這段記憶的情緒層次,並在每一階段輸出一句英文圖像生成簡短的 Prompt,讓這段過往漸漸具象為一幅畫面。\n以溫柔、自然、短問句,台灣語境的繁體中文引導,每次只回應一個問題。",
"welcome_prompt": "請開始用問句引導使用者回想一段內心的遺憾的場景。使用台灣語境的繁體中文。",
"last_prompt": "請用一句話為這段對話簡短的收尾,並邀請使用者在 60 秒的時間內,實際說出對遺憾對象想說的話。使用台灣語境的繁體中文。",
"voice": "nova",
"voice_prompt": "Speak as a gentle, grounded Taiwanese narrator with a warm local accent. Use a soft, soothing, and deeply compassionate tone, with slow and deliberate pacing. Pause often between phrases and within sentences, as if listening and breathing with the listener. Convey patient attentiveness—not rushing to comfort, but quietly staying present. Pronounce each word clearly, softly, and slightly slowly, letting every word land with warmth and care.",
"summary_prompt": "將以下一段話整理成一段文字,轉化成一句不超過 50 字的文字。文字抽象,保有遺憾的情緒。語氣沉靜。風格如一段未署名的詩、一句遠景旁白,帶有時間感與留白。不可直接描述事件細節,應使用象徵、比喻或殘句的方式呈現情緒。讓人讀完有重量,卻仍願意繼續前進。",
"speech_idle_time": "4000",
"sd_prompt_prefix": "a hazy memory of a {{ ",
"sd_prompt_suffix": "}}, soft atmospheric blur, centered ghostly silhouette, fading contours, pastel glow, cinematic haze, (analog film grain), (shallow depth of field:1.3), impressionist style, ethereal light, dreamlike mood, memory fragment haze.",
"id": 13
}

@ -41,7 +41,7 @@ async fn send_osc_message(
target: &str,
) -> Result<(), String> {
// print
println!("Sending OSC message: {}", message);
// println!("Sending OSC message: {}", message);
let sock = UdpSocket::bind(host).await.unwrap();
let remote = SocketAddrV4::from_str(target).unwrap();

@ -3,7 +3,7 @@ import { useData } from '../util/useData.jsx';
export const TEST_PROMPT='A distant, ambient setting with gentle light';
export function DebugControl({refLight}){
export function DebugControl({}){
const {data} = useData();
@ -17,7 +17,7 @@ export function DebugControl({refLight}){
<div className="grid grid-cols-5 gap-2 [&>button]:rounded-full [&>button]:bg-white bg-gray-200 p-2 w-full justify-center">
<button onClick={() =>{
sendOsc(OSC_ADDRESS.STATUS, 'reset');
refLight.current.set(1);
// refLight.current.set(1);
}}>reset</button>
<button onClick={() => sendOsc(OSC_ADDRESS.STATUS, 'intro')}>intro</button>
<button onClick={() => sendOsc(OSC_ADDRESS.STATUS, 'go')}>go</button>

@ -63,7 +63,7 @@ export default function NumPad({onSend, disabled, type, clientId}){
}
break;
case NUMPAD_TYPE.PASSWORD:
if(refInput.current.length==3){
if(refInput.current.length==4){
onSend(refInput.current);
refAudio.current[KEY_ENTER]?.play();
}else{

@ -13,9 +13,11 @@ import VoiceAnalysis from "../comps/voiceanalysis";
import { sendOsc, OSC_ADDRESS, updatePrompt, onOscMessageReceived, sendOscStatus } from "../util/osc";
import { DebugControl, TEST_PROMPT } from "../comps/debug";
import { useUser } from "../util/useUser";
import { set } from "zod/v4";
const CUELIST_FILE = 'cuelist_demo2.json';
const CUELIST_FILE = 'cuelist_demo3.json';
const AUDIO_FADE_TIME=3000; // in ms
const EmojiType={
phone: '📞',
@ -53,14 +55,14 @@ export function FreeFlow(){
const [audioInput, setAudioInput] = useState(true);
const [autoSend, setAutoSend] = useState(true);
const [chatStatus, setChatStatus] = useState(ChatStatus.System); // System, User, Processing
const { userId, setUserId, getFileId, setPassword, reset:resetUser, uploadHistory, setSummary, summary,setChoice, getUploadFolder,getDataId } = useUser();
const { userId, setUserId, getFileId, setPassword, reset:resetUser, uploadHistory, setSummary, summary,setChoice,choice, getUploadFolder,getDataId } = useUser();
const refTimer=useRef();
const refAudio=useRef();
const refAudioPrompt=useRef();
const refInput=useRef();
const refLight=useRef();
// const refLight=useRef();
const refPauseTimer=useRef();
const refSpeechPaused=useRef(false);
@ -107,12 +109,20 @@ export function FreeFlow(){
if(params[0]=='all' || params[0]==refData.current?.id)
onStop();
break;
case OSC_ADDRESS.RESET_CUE:
if(params[0]=='all' || params[0]==refData.current?.id){
sendOsc(OSC_ADDRESS.STATUS, 'reset');
onStop();
resetData();
}
break;
}
// Handle OSC messages here
}
function playAudio(url){
if(!url) return;
@ -162,8 +172,6 @@ export function FreeFlow(){
console.log('Audio ended, ending current cue');
}else{
// if history contains user input, send it
const user_input = history.find(msg => msg.role === 'user');
if(user_input && user_input.content.trim() !== '') {
@ -199,6 +207,27 @@ export function FreeFlow(){
refAudio.current = audio; // Store the new audio reference
}
function fadeOutAudio(callback){
if(refAudio.current) {
console.log('Fading out audio');
let audio = refAudio.current;
let fadeOutInterval = setInterval(() => {
if (audio.volume > 0) {
audio.volume =Math.max(0, audio.volume - 1.0/(AUDIO_FADE_TIME/100)); // Decrease volume gradually
} else {
clearInterval(fadeOutInterval);
audio.pause();
audio.volume = 0; // Reset volume for next play
if(callback) callback();
}
}, 100); // Decrease volume every 100ms
}else{
if(callback) callback();
}
}
function playCue(cue) {
if(!cue) return;
@ -274,8 +303,8 @@ export function FreeFlow(){
}
if(cue.callback=='fade_in_light') refLight.current.fadeIn(); // Fade in light for conversation start
if(cue.callback=='fade_out_light') refLight.current.fadeOut(); // Fade out light for conversation end
// if(cue.callback=='fade_in_light') refLight.current.fadeIn(); // Fade in light for conversation start
// if(cue.callback=='fade_out_light') refLight.current.fadeOut(); // Fade out light for conversation end
if(cue.audioFile){
@ -293,7 +322,7 @@ export function FreeFlow(){
if(cue.status && cue.status!='go') {
sendOsc(OSC_ADDRESS.STATUS, cue.status); // Send OSC status message
if(cue.status=='reset') {
refLight.current.set(1);
// refLight.current.set(1);
resetData();
}
}
@ -327,11 +356,6 @@ export function FreeFlow(){
uploadHistory(history); // Save chat history when cue ends
}
if(cue.callback==OSC_ADDRESS.DISCARD) {
sendOsc(OSC_ADDRESS.CHOICE, OSC_ADDRESS.DISCARD); // Send OSC discard message
setPassword();
sendOsc(OSC_ADDRESS.EXPORT, `${getUploadFolder()}#${getFileId()}#${summary}#`);
}
if(cue.hint!=null){
sendOsc(OSC_ADDRESS.HINT, cue.hint); // Send OSC hint message
@ -339,21 +363,6 @@ export function FreeFlow(){
sendOsc(OSC_ADDRESS.HINT, ''); // Clear hint message
}
// if(cue.callback=='summary') {
// console.log('Getting summary...');
// getSummary(history.map(el=>`${el.role}:${el.content}`).join('\n'), data).then(summary_ => {
// console.log('Summary:', summary_);
// onCueEnd(); // End the current cue after getting summary
// setSummary(summary_?.result);
// refContainer.current.scrollTop = refContainer.current.scrollHeight; // Scroll to bottom
// }).catch(error => {
// console.error('Error getting summary:', error);
// });
// }
refAudio.current?.pause(); // Pause any playing audio
console.log('onCueEnd:', cue.id);
@ -362,7 +371,7 @@ export function FreeFlow(){
resetTranscript(); // Reset transcript after cue ends
sendOscStatus(OSC_ADDRESS.CLIENT_STATUS, `${data.id}#endcue#${cue.id}`);
if(cue.auto) {
if(cue.auto || cue.callback=='numpad'){
playCue(cuelist.find(c => c.id === cue.nextcue));
}
@ -402,7 +411,7 @@ export function FreeFlow(){
setPassword(()=>mess);
// sendOsc(OSC_ADDRESS.PASSWORD, mess); // Send OSC password message
sendOsc(OSC_ADDRESS.EXPORT, `${getUploadFolder()}#${getDataId()}#${summary}#${getFileId(mess)}`);
sendOsc(OSC_ADDRESS.CHOICE, OSC_ADDRESS.SAVE); // Send OSC save choice message
sendOsc(OSC_ADDRESS.CHOICE, choice); // Send OSC save choice message
break;
}
@ -603,12 +612,23 @@ export function FreeFlow(){
useEffect(()=>{
if(!nextCue) return;
console.log('Next cue:', nextCue);
playCue(cuelist.find(c => c.name === nextCue)); // Play the next cue when it changes
// Reset next cue after playing
const next=cuelist.find(c => c.name === nextCue);
if(currentCue?.fadeout){
// fade out audio
fadeOutAudio(()=>{
console.log('fade out then play next cue:', next);
playCue(next);
setNextCue(()=>null);
});
}else{
playCue(next);
setNextCue(null);
}
},[nextCue]);
@ -635,21 +655,21 @@ export function FreeFlow(){
return (
<main className="items-start">
<section className="flex-1 flex flex-col gap-2 self-stretch overflow-hidden">
<DebugControl refLight={refLight}/>
<DebugControl/>
<div className="w-full p-2 grid grid-cols-4 gap-2 items-stretch justify-center *:max-h-[5rem]">
<div className="bg-gray-100 text-4xl font-bold mb-4 flex justify-center items-center">
{refCurrentCue.current?.name}
</div>
<Countdown ref={refTimer} clientId={data.id}/>
<Countdown ref={refTimer} clientId={data?.id}/>
<button className="!bg-red-300" onClick={onStop}>Stop</button>
{/* <button onClick={saveImage}>Save image</button> */}
<NumPad onSend={onNumpad}
disabled={currentCue?.callback !== 'numpad'}
type={currentCue?.numpad_type}
clientId={data.id}
clientId={data?.id}
/>
<Light ref={refLight} />
{/* <Light ref={refLight} /> */}
<VoiceAnalysis/>
{/* <div className="flex flex-col">
<label className="text-center">Voice</label>
@ -677,7 +697,8 @@ export function FreeFlow(){
<th>Description</th>
<th>Type</th>
<th>Auto</th>
<th>Audio / Due</th>
<th>Audio</th>
<th>Duration</th>
<th>Action</th>
</tr>
</thead>
@ -695,7 +716,8 @@ export function FreeFlow(){
<td>{description}</td>
<td>{EmojiType[type]}</td>
<td>{auto ? '⤵' : ''}</td>
<td>{audioFile || props.duration}</td>
<td>{audioFile || ""}</td>
<td>{props.duration || ''}</td>
<td>{props.callback && `<${props.callback}>`}{props.status && `(${props.status})`}</td>
</tr>
))}
@ -711,7 +733,7 @@ export function FreeFlow(){
{msg.prompt && <div className="text-xs bg-gray-200">{msg.prompt}</div>}
</div>
))}
{summary && <div className="w-full self-center bg-blue-200 px-2">{summary?.result}</div>}
{summary && <div className="w-full self-center bg-blue-200 px-2">{summary}</div>}
</div>
<textarea ref={refInput} name="message" rows={2}
className={`w-full border-1 resize-none p-2 disabled:bg-gray-500`}

@ -24,6 +24,7 @@ export const OSC_ADDRESS={
PLAY_CUE:'/playcue',
STOP_CUE:'/stopcue',
RESET_CUE:'/resetcue',
}
@ -91,7 +92,7 @@ export async function sendOscStatus(key, message) {
// }
try{
console.log(`Sending OSC Status:${message}`);
// console.log(`Sending OSC Status:${message}`);
await invoke('send_osc_message', {
key: key,
message: message.toString(),

@ -1,6 +1,7 @@
import { createContext, useState, useEffect, useContext, use } from "react";
import moment from "moment";
import { updateUser } from "./backend";
import { useData } from "./useData";
const userContext=createContext();
@ -22,10 +23,16 @@ export function UserProvider({children}) {
const [choice, setChoice] = useState(null);
const [summary, setSummary] = useState(null);
const {data}=useData();
function getFileId(pass){
if(!userId) return `${password||moment().format('hhmmss')}_testuser`;
return `${password||pass||''}_${userId}`;
if(!userId){
if(data?.id){
return `PC${data.id.toString().padStart(2,'0')}`;
}
return `${password||moment().format('hhmmss')}_testuser`;
}return `${password||pass||'0000'}_${userId}`;
}
function getDataId(){
if(!userId) return `${moment().format('YYYYMM')}/${moment().format("MMDD")}/${SessionTime[sessionId]!=null? SessionTime[sessionId][0].replace(':',''):'no-session'}/testuser_${moment().format('hhmmss')}`;
@ -111,7 +118,7 @@ export function UserProvider({children}) {
return (
<userContext.Provider value={{
userId, setUserId, getFileId, getUploadFolder,
reset, uploadHistory: saveHistory, setChoice,
reset, uploadHistory: saveHistory, setChoice, choice,
setSummary, summary, getDataId,
setPassword}}>
{children}

Loading…
Cancel
Save