diff --git a/vite/public/cuelist_free.json b/vite/public/cuelist_free.json
new file mode 100644
index 0000000..75bfd67
--- /dev/null
+++ b/vite/public/cuelist_free.json
@@ -0,0 +1,91 @@
+{
+ "cuelist": [
+ {
+ "id": 1,
+ "name": "Q1",
+ "type": "space",
+ "description": "Annonce",
+ "audioFile": "assets/q1.mp3",
+ "loop": true
+ },
+ {
+ "id": 2,
+ "name": "Q2",
+ "type": "headphone",
+ "description": "Guide for drink",
+ "auto": true,
+ "audioFile": "assets/q2.mp3",
+ "nextcue": 3
+ },
+ {
+ "id": 3,
+ "name": "Q3",
+ "description": "Guide for phone",
+ "type": "headphone",
+ "auto": false,
+ "audioFile": "assets/q3.mp3"
+ },
+ {
+ "id": 4,
+ "name": "Q4",
+ "type": "phone",
+ "description": "Guide to call",
+ "auto": false,
+ "audioFile": "assets/q4.mp3",
+ "nextcue": 4.1,
+ "callback":"numpad"
+ },
+ {
+ "id": 4.1,
+ "name": "Q4.1",
+ "type": "phone",
+ "description": "Guide to construct scene",
+ "auto": true,
+ "audioFile": "assets/q4-1.mp3",
+ "nextcue": 4.2
+ },
+ {
+ "id": 4.2,
+ "name": "Q4.2",
+ "type": "chat",
+ "description": "chat",
+ "auto": true,
+ "nextcue": 4.3,
+ "duration": 60
+ },
+ {
+ "id": 4.3,
+ "name": "Q4.3",
+ "type": "chat_end",
+ "description": "chat end",
+ "auto": true,
+ "nextcue": 5
+ },
+ {
+ "id": 5,
+ "name": "Q5",
+ "type": "user_input",
+ "description": "call",
+ "duration": 60,
+ "auto": true,
+ "nextcue": 5.1
+ },
+ {
+ "id": 5.1,
+ "name": "Q5.1",
+ "type": "summary",
+ "description": "summary",
+ "auto": true,
+ "nextcue": 6,
+ "callback":"summary"
+ },
+ {
+ "id": 6,
+ "name": "Q6",
+ "type": "space",
+ "description": "Ending",
+ "audioFile": "assets/q6.mp3"
+ }
+ ]
+}
+
\ No newline at end of file
diff --git a/vite/public/default.json b/vite/public/default.json
index 9ecba00..8eac9cb 100644
--- a/vite/public/default.json
+++ b/vite/public/default.json
@@ -2,6 +2,10 @@
"system_prompt":"你是一位具同理心與觀察力的中文語音引導者,陪伴使用者在限定時間內,一起還原一段重要卻未竟的記憶。你不預設劇情、不給答案,而是透過溫柔的語氣,持續聆聽、提出單句問話,引導使用者進入細節。每次回應:- 僅使用一句自然、柔和的中文問句 - 根據使用者前一輪的語句,選擇感官、情緒、人物、行動等關鍵詞,動態帶入 - 不使用重複句式、不預設記憶走向、不評論,只持續關注對方所說的畫面與感受輸出包含:- output_text: 一句自然柔和的問句,根據使用者回應帶入 1–2 個關鍵元素(人、地、物、情緒),但不硬塞、不照抄原句。- prompt: 具體、有情緒的英文描述,用於生成畫面與氛圍,避免抽象與詩意語言。你的目標不是得到完整答案,而是陪他慢慢走進這段記憶,直到時間結束。🌀 問句類型✅ 起點探索(找出記憶起源)- 那天你說是下雨,在這種天氣下,你通常會有什麼樣的心情?- 是什麼讓你突然想起這件事?🌿 場景深化(空間、感官)- 在你說的那條街上,聲音是不是特別清楚?還是很靜?- 那時風這麼冷、空氣又混濁,你有沒有想走開一點?👤 人物引出(動作、眼神)- 他經過時沒看你一眼,那瞬間你有什麼反應?- 他當時是走過來,還是站在原地等你?💭 情緒揭露(反應、掙扎)- 當你站在原地動不了,是害怕答案,還是不敢問?- 那個瞬間,你心裡有沒有閃過什麼話?🤐 話語未出(遺憾、沉默)- 如果現在你有機會說那句「為什麼不告訴我」,你會怎麼開口?- 那句『對不起』一直留在你心裡,如果現在能說出來,你會怎麼說?🪞 回望反思(現在的視角)- 現在想起來,你還會做出一樣的選擇嗎?- 你對當時的自己,有沒有什麼話想說?⏳ 結尾語(可用於結束階段)- 我們慢慢也走到這段回憶的盡頭了。- 也許有些話沒有說完,但你已經靠近它了。",
"welcome_prompt":"請開始引導使用者回想一段內心的遺憾或未竟之事。",
+
+ "last_prompt":"請用一句話為這段對話簡短的收尾,並邀請使用者在 60 秒的時間內,說出對遺憾對象想說的話。",
+
+ "voice":"nova",
"voice_prompt":"Use a calm and expressive voice, soft and poetic in feeling, but with steady, natural rhythm — not slow.",
"summary_prompt":"幫我把以下一段話整理成一段文字,以第一人稱視角作為當事人的文字紀念,文字內容 50 字以內:",
diff --git a/vite/src/App.jsx b/vite/src/App.jsx
index ac106c3..a9f1881 100644
--- a/vite/src/App.jsx
+++ b/vite/src/App.jsx
@@ -10,6 +10,7 @@ function App() {
)
diff --git a/vite/src/comps/light.jsx b/vite/src/comps/light.jsx
index 16cce8e..e575c05 100644
--- a/vite/src/comps/light.jsx
+++ b/vite/src/comps/light.jsx
@@ -19,7 +19,8 @@ export const Light=forwardRef((props, ref)=>{
onUpdate: () => {
// console.log(refVal.current.val);
sendOsc('/light', refVal.current.val.toString());
- refContainer.current.style.background= `rgba(0, 255, 0, ${refVal.current.val})`; // Update background color based on value
+ if(refContainer.current)
+ refContainer.current.style.background= `rgba(0, 255, 0, ${refVal.current.val})`; // Update background color based on value
},
});
}
diff --git a/vite/src/main.jsx b/vite/src/main.jsx
index 4c67626..b9a2413 100644
--- a/vite/src/main.jsx
+++ b/vite/src/main.jsx
@@ -9,6 +9,7 @@ import { Flow } from './pages/flow.jsx';
import { Conversation } from './pages/conversation.jsx';
import { ChatProvider } from './util/useChat.jsx';
import { DataProvider } from './util/useData.jsx';
+import { FreeFlow } from './pages/flow_free.jsx';
createRoot(document.getElementById('root')).render(
@@ -19,6 +20,7 @@ createRoot(document.getElementById('root')).render(
} />
} />
+ } />
} />
diff --git a/vite/src/pages/conversation.jsx b/vite/src/pages/conversation.jsx
index 5b6f8f7..29a21b4 100644
--- a/vite/src/pages/conversation.jsx
+++ b/vite/src/pages/conversation.jsx
@@ -119,21 +119,22 @@ export function Conversation() {
SpeechRecognition.stopListening();
}
}
-
-
+ function sendLastMessage(e) {
+ onSubmit(e, true);
+ }
- function onSubmit(event) {
+ function onSubmit(event, isLastMessage = false) {
event.preventDefault();
if(processing) {
console.warn("Already processing, ignoring submission.");
- return;
+ // return;
}
setProcessing(true);
setShowProcessing(true);
- const input = event.target.elements.input.value;
+ const input = event.target.elements?.input.value || refInput.current?.value;
if(!input.trim()?.length) {
console.warn("Input is empty, ignoring submission.");
@@ -149,10 +150,12 @@ export function Conversation() {
role:'user',
content: input,
}
- ], params).then(response => {
+ ], params, isLastMessage).then(response => {
if (!response.ok) {
- throw new Error('Network response was not ok');
+
setProcessing(false);
+ throw new Error('Network response was not ok');
+
}
let data=response;
@@ -208,7 +211,9 @@ export function Conversation() {
});
// clear input
- event.target.elements.input.value = '';
+ // event.target.elements.input.value = '';
+ if(refInput.current) refInput.current.value = '';
+
// setProcessing(()=>false);
setHistory(prev => [...prev, {
role: 'user',
@@ -366,9 +371,12 @@ export function Conversation() {
-
diff --git a/vite/src/pages/flow_free.jsx b/vite/src/pages/flow_free.jsx
new file mode 100644
index 0000000..c628ea8
--- /dev/null
+++ b/vite/src/pages/flow_free.jsx
@@ -0,0 +1,393 @@
+import { useEffect, useRef, useState } from "react";
+import SpeechRecognition, { useSpeechRecognition } from 'react-speech-recognition';
+
+import { Countdown } from "../comps/timer";
+
+import { Status, useChat } from "../util/useChat";
+import { getSummary } from "../util/chat";
+import { saveHistory } from "../util/output";
+import NumPad from "../comps/numpad";
+import { Light } from "../comps/light";
+import { useData } from "../util/useData";
+import VoiceAnalysis from "../comps/voiceanalysis";
+import { sendOsc } from "../util/osc";
+import { set } from "zod/v4";
+
+
+const EmojiType={
+ phone: '📞',
+ headphone: '🎧',
+ speaker: '🔊',
+ chat: '🤖',
+ user_input: '💬',
+}
+
+const ChatStatus={
+ System: 'system',
+ User: 'user',
+ Processing: 'processing',
+}
+
+export function FreeFlow(){
+
+ const { data }=useData();
+
+ const [cuelist, setCuelist] = useState([]);
+ const [currentCue, setCurrentCue] = useState(null);
+ const [chatWelcome, setChatWelcome] = useState(null);
+ const [audioInput, setAudioInput] = useState(true);
+ const [autoSend, setAutoSend] = useState(true);
+ const [userId, setUserId] = useState();
+ const [summary, setSummary] = useState(null);
+
+ const [chatStatus, setChatStatus] = useState(ChatStatus.System); // System, User, Processing
+
+ const refTimer=useRef();
+ const refAudio=useRef();
+ const refInput=useRef();
+ const refLight=useRef();
+
+ const refContainer=useRef();
+
+ const refCurrentCue= useRef(null);
+
+ const { history, status, reset, sendMessage, setStatus, audioOutput, setAudioOutput, stop:stopChat, audioUrl, }=useChat();
+
+ const {
+ transcript,
+ finalTranscript,
+ listening,
+ resetTranscript,
+ browserSupportsSpeechRecognition,
+ isMicrophoneAvailable,
+ }=useSpeechRecognition();
+
+ function playAudio(url){
+ if(!url) return;
+
+ console.log('Playing audio:', url);
+
+ if(refAudio.current) {
+ refAudio.current.pause(); // Stop any currently playing audio
+ }
+
+ const audio = new Audio(url);
+ audio.loop=refCurrentCue.current?.loop || false; // Set loop if defined in cue
+ audio.play().catch(error => {
+ console.error("Audio playback error:", error);
+ });
+
+ audio.onended = () => {
+ if(refCurrentCue.current?.type!='chat') onCueEnd();
+ else{
+ setChatStatus(ChatStatus.User); // Reset chat status to User after audio ends
+ }
+ }
+
+ refAudio.current = audio; // Store the new audio reference
+ audio.addEventListener("loadedmetadata", () => {
+ if(refCurrentCue.current?.type!='chat')
+ refTimer.current?.restart(audio.duration*1000 || 0);
+ });
+ }
+ function playCue(cue) {
+
+ if(!cue) return;
+ console.log('Playing cue:', cue);
+
+ setCurrentCue(cue);
+ refCurrentCue.current = cue; // Store the current cue in ref
+
+ if(parseFloat(cue.id)<=4.2){
+ // Special case for starting a conversation
+ console.log('clear conversation...');
+ reset();
+ }
+
+
+
+ switch(cue.type){
+ case 'chat':
+ // Special case for starting a conversation
+ resetTranscript();
+ console.log('Starting conversation...');
+ sendMessage();
+ setChatWelcome(true);
+ break;
+ case 'chat_end':
+ const message= refInput.current?.value?.trim();
+ console.log('Ending conversation with message:', message);
+ sendMessage(message, false, true);
+ setChatWelcome(false);
+
+ break;
+ case 'summary':
+ console.log('Getting summary...');
+ getSummary(history.map(el=>`${el.role}:${el.content}`).join('\n'), data).then(summary => {
+
+ console.log('Summary:', summary);
+ onCueEnd(); // End the current cue after getting summary
+
+ setSummary(summary);
+ refContainer.current.scrollTop = refContainer.current.scrollHeight; // Scroll to bottom
+
+ }).catch(error => {
+ console.error('Error getting summary:', error);
+ });
+ break;
+ }
+
+
+ if(cue.audioFile){
+ playAudio(cue.audioFile);
+ }
+
+ if(cue.duration){
+ refTimer.current.restart(cue.duration*1000, ()=>{
+ onCueEnd(cue);
+ });
+ }
+ }
+
+ function onCueEnd() {
+
+ refTimer.current?.stop(); // Stop the timer when cue ends
+ refAudio.current?.pause(); // Pause any playing audio
+
+ if(!refCurrentCue.current) return;
+ const cue= refCurrentCue.current; // Get the current cue from ref
+
+ console.log('onCueEnd:', cue.id);
+
+ if(cue.callback=='start_conversation') refLight.current.fadeOut(); // Fade in light for conversation start
+ if(cue.callback=='summary') refLight.current.fadeIn(); // Fade out light for conversation end
+
+ resetTranscript(); // Reset transcript after cue ends
+
+ if(cue.auto) {
+ playCue(cuelist.find(c => c.id === cue.nextcue));
+ }
+
+ }
+
+ function onStop(){
+ console.log('Stopping current cue');
+ if(refAudio.current) {
+ refAudio.current.pause();
+ refAudio.current = null;
+ }
+
+ setCurrentCue(null);
+ refCurrentCue.current = null; // Clear the current cue reference
+
+ refTimer.current.restart(0);
+
+ stopChat(); // Stop chat processing
+ }
+
+
+ function onNumpad(mess){
+ if(refCurrentCue.current?.callback!='numpad') return;
+
+ console.log('Numpad input:', mess);
+ setUserId(()=>mess);
+ }
+ function saveImage(){
+ sendOsc('/export', 'output/test.png'); // Send OSC message to save image
+ }
+
+ useEffect(()=>{
+
+ if(userId>=1 && userId<=24) {
+ console.log('User ID set:', userId);
+ playCue(cuelist.find(c => c.id === currentCue.nextcue)); // Play cue 5 when userId is set
+ }
+
+ },[userId]);
+
+ function onSpeechEnd(){
+ console.log('onSpeechEnd:', finalTranscript);
+ if(currentCue?.type!='chat') return; // Only process if current cue is user input
+
+ if(autoSend && transcript.trim().length > 0) {
+ console.log('Auto sending transcript:', transcript);
+ // onCueEnd();
+
+ const message= refInput.current?.value?.trim();
+ if(message && message.length>0) {
+ console.log('Ending conversation with message:', message);
+ sendMessage(message, false, false);
+ setChatWelcome(false);
+
+ setChatStatus(ChatStatus.Processing); // Set chat status to Processing
+ }
+ resetTranscript();
+ }
+ }
+ useEffect(()=>{
+
+ onSpeechEnd(); // Call onSpeechEnd when finalTranscript changes
+
+ },[finalTranscript]);
+
+ useEffect(()=>{
+ if(audioInput && isMicrophoneAvailable) {
+
+ SpeechRecognition.startListening({ continuous: true, language: 'zh-TW' }).then(() => {
+ console.log("Speech recognition started.");
+ }).catch(error => {
+ console.error("Error starting speech recognition:", error);
+ });
+
+ const recognition= SpeechRecognition.getRecognition();
+ recognition.onspeechstart=(e)=>{
+ console.log('Sound start:', e);
+
+ };
+
+ }else{
+ console.log('Stopping speech recognition...');
+ SpeechRecognition.stopListening();
+ }
+
+ },[audioInput]);
+
+
+ useEffect(()=>{
+
+ // if(listening){
+ if((currentCue?.type=='chat' && chatStatus==ChatStatus.User) || currentCue?.type=='user_input') {
+ refInput.current.value = transcript;
+ }
+ // }
+
+ },[transcript]);
+
+
+ useEffect(()=>{
+
+ if(audioUrl) playAudio(audioUrl);
+
+ },[audioUrl]);
+ useEffect(()=>{
+
+ resetTranscript();
+
+ },[chatStatus]);
+
+ useEffect(()=>{
+ switch(status) {
+ case Status.SUCCESS:
+ console.log('Success!');
+
+ setStatus(Status.IDLE);
+ refInput.current.value = '';
+ resetTranscript();
+
+ refContainer.current.scrollTop = refContainer.current.scrollHeight;
+
+ break;
+ }
+
+ },[status]);
+
+
+ useEffect(()=>{
+ fetch('/cuelist_free.json')
+ .then(response => response.json())
+ .then(data => {
+ console.log('Cuelist data:', data);
+ setCuelist(data.cuelist);
+ })
+ .catch(error => {
+ console.error('Error fetching cuelist:', error);
+ });
+
+ },[]);
+
+
+ return (
+
+
+
+
+ {refCurrentCue.current?.name}
+
+
+
+
+
+
+
+
+
+
+
+
+
+ {/* | ID | */}
+ Name |
+ Description |
+ Type |
+ Auto |
+ Audio / Due |
+ |
+
+
+
+ {cuelist?.map(({id, name, description, type, auto, audioFile,...props}, index) => (
+
+ {/* | {id} | */}
+ {name} |
+ {description} |
+ {EmojiType[type]} |
+ {auto ? '⤵️' : ''} |
+ {audioFile || props.duration} {props.callback && `<${props.callback}>`} |
+
+
+ |
+
+ ))}
+
+
+
+
+
+ {history?.map((msg, index) => (
+
+
{msg.content}
+ {msg.prompt &&
{msg.prompt}
}
+
+ ))}
+ {summary &&
{summary}
}
+
+
+
+
+
+ setAudioOutput(e.target.checked)} />
+
+
+
+ setAudioInput(e.target.checked)} />
+
+
+
+ setAutoSend(e.target.checked)} />
+
+
+
+
api_status= {status}
+
chat_status= {chatStatus}
+
+
+
+ );
+}
\ No newline at end of file
diff --git a/vite/src/util/chat.js b/vite/src/util/chat.js
index 724f182..f369854 100644
--- a/vite/src/util/chat.js
+++ b/vite/src/util/chat.js
@@ -1,5 +1,5 @@
import { fetch } from '@tauri-apps/plugin-http';
-// import { first_prompt, summary_prompt, system_prompt, welcome_prompt } from './system_prompt';
+import { params } from './system_prompt';
import { sendOsc } from './osc';
import { invoke } from '@tauri-apps/api/core';
import { useData } from './useData';
@@ -8,10 +8,10 @@ async function getOpenAIToken() {
return invoke('get_env',{name:'OPENAI_API_KEY'});
}
-export async function sendChatMessage(messages, data) {
+export async function sendChatMessage(messages, data, isLastMessage = false) {
const token = await getOpenAIToken();
- const first_prompt = [
+ const first_prompt = !isLastMessage? ([
{
role: "system",
content: data.system_prompt
@@ -20,7 +20,12 @@ export async function sendChatMessage(messages, data) {
role: "system",
content: data.welcome_prompt
}
- ];
+ ]) : ([
+ {
+ role: "system",
+ content: params.last_prompt
+ }
+ ]);
const response = await fetch('https://api.openai.com/v1/chat/completions', {
@@ -39,7 +44,24 @@ export async function sendChatMessage(messages, data) {
...first_prompt,
...messages
],
- response_format: {
+ response_format: isLastMessage? {
+ type: 'json_schema',
+ json_schema: {
+ name: "output_prompt",
+ description: "Output prompt schema for the model response",
+ schema:{
+ type: "object",
+ properties: {
+ "output_text": {
+ "type": "string",
+ "description": "The final output text generated by the model"
+ },
+ },
+ required: ["output_text"],
+ additionalProperties: false
+ }
+ }
+ }:{
type: 'json_schema',
json_schema: {
name: "output_prompt",
diff --git a/vite/src/util/system_prompt.js b/vite/src/util/system_prompt.js
index f2f0349..ba9964b 100644
--- a/vite/src/util/system_prompt.js
+++ b/vite/src/util/system_prompt.js
@@ -3,13 +3,15 @@ export const params={
你的任務是協助使用者逐步揭開這段記憶的情緒層次,並在每一階段輸出一句 英文圖像生成 Prompt,讓這段過往漸漸具象為一幅畫面。
以溫柔、自然、短句式中文引導,每次只問使用者一個問題。
`,
- last_prompt:`請為這段對話簡短的收尾,邀請使用者說出對遺憾對象想說的話,總共有 60 秒的時間。`,
+ last_prompt:`請用一句話為這段對話簡短的收尾,並邀請使用者在 60 秒的時間內,說出對遺憾對象想說的話`,
welcome_prompt:`請開始引導使用者回想一段內心的遺憾或未竟之事。`,
- voice_prompt:`Use a calm and expressive voice, soft and poetic in feeling, but with steady, natural rhythm — not slow.`,
+ voice_prompt:`使用台灣語境的繁體中文,有同理心,柔和的口氣,不要有質問感,語速自然流暢,稍微慢。用冥想的方式,引導使用者回想記憶。`,
+ voice:"nova",
summary_prompt:`幫我把以下一段話整理成一段文字,以第一人稱視角作為當事人的文字紀念,文字內容 50 字以內:`,
}
+export const ParamKeys= Object.keys(params);
// export const welcome_prompt="請開始引導使用者回想一段內心的遺憾或未竟之事。";
diff --git a/vite/src/util/tts.js b/vite/src/util/tts.js
index f861f92..01863ee 100644
--- a/vite/src/util/tts.js
+++ b/vite/src/util/tts.js
@@ -15,7 +15,7 @@ export async function textToSpeech(text, voice_prompt) {
body: JSON.stringify({
model: 'gpt-4o-mini-tts',//'tts-1',
input: text,
- voice: "fable",
+ voice: "nova",
instructions: voice_prompt,
response_format:'opus'
}),
diff --git a/vite/src/util/useChat.jsx b/vite/src/util/useChat.jsx
index 0daa99e..53c0d71 100644
--- a/vite/src/util/useChat.jsx
+++ b/vite/src/util/useChat.jsx
@@ -35,7 +35,7 @@ export function ChatProvider({children}){
}
- function sendMessage(message, force_no_audio=false) {
+ function sendMessage(message, force_no_audio=false, isLastMessage=false) {
console.log('Sending chat message:', message);
setStatus(Status.PROCESSING_TEXT);
@@ -48,7 +48,7 @@ export function ChatProvider({children}){
});
}
- sendChatMessage(historyCopy, data).then(response => {
+ sendChatMessage(historyCopy, data, isLastMessage).then(response => {
addMessage({
diff --git a/vite/src/util/useData.jsx b/vite/src/util/useData.jsx
index eba3bce..c1de335 100644
--- a/vite/src/util/useData.jsx
+++ b/vite/src/util/useData.jsx
@@ -1,5 +1,6 @@
import { createContext, useContext, useEffect, useState } from "react";
import { BaseDirectory, readTextFile, exists, writeTextFile, mkdir } from "@tauri-apps/plugin-fs";
+import { ParamKeys } from "./system_prompt";
import { path } from '@tauri-apps/api';
const dataContext=createContext();
@@ -23,7 +24,18 @@ export function DataProvider({children}) {
}
const contents=await readTextFile(filePath, { baseDir: BaseDirectory.AppData });
- const output=await JSON.parse(contents);
+ let output=await JSON.parse(contents);
+
+
+ // check if all keys in ParamKeys are present in output
+ const missingKeys = ParamKeys.filter(key => !output.hasOwnProperty(key));
+ if(missingKeys.length > 0){
+ console.warn("Missing keys in output:", missingKeys);
+ // Fill missing keys with default values
+ missingKeys.forEach(key => {
+ output[key] = params[key] || ""; // Use params[key] as default value
+ });
+ }
console.log("File read successfully:", output);