main
reng 6 months ago
parent e7b0c6b8d8
commit 90afe5725f
  1. 18
      vite/package-lock.json
  2. 1
      vite/package.json
  3. 12
      vite/src-tauri/2
  4. 1
      vite/src-tauri/Cargo.lock
  5. 1
      vite/src-tauri/Cargo.toml
  6. 15
      vite/src-tauri/capabilities/default.json
  7. 18
      vite/src-tauri/src/lib.rs
  8. 2
      vite/src-tauri/src/main.rs
  9. 4
      vite/src-tauri/tauri.conf.json
  10. 96
      vite/src/App.jsx
  11. 98
      vite/src/comps/input.jsx
  12. 87
      vite/src/util/system_prompt.js

@ -9,6 +9,7 @@
"version": "0.0.0", "version": "0.0.0",
"dependencies": { "dependencies": {
"@tailwindcss/vite": "^4.1.8", "@tailwindcss/vite": "^4.1.8",
"@tauri-apps/plugin-fs": "^2.3.0",
"@tauri-apps/plugin-http": "^2.4.4", "@tauri-apps/plugin-http": "^2.4.4",
"gsap": "^3.13.0", "gsap": "^3.13.0",
"react": "^19.1.0", "react": "^19.1.0",
@ -1595,6 +1596,15 @@
"node": ">= 10" "node": ">= 10"
} }
}, },
"node_modules/@tauri-apps/plugin-fs": {
"version": "2.3.0",
"resolved": "https://registry.npmjs.org/@tauri-apps/plugin-fs/-/plugin-fs-2.3.0.tgz",
"integrity": "sha512-G9gEyYVUaaxhdRJBgQTTLmzAe0vtHYxYyN1oTQzU3zwvb8T+tVLcAqCdFMWHq0qGeGbmynI5whvYpcXo5LvZ1w==",
"license": "MIT OR Apache-2.0",
"dependencies": {
"@tauri-apps/api": "^2.0.0"
}
},
"node_modules/@tauri-apps/plugin-http": { "node_modules/@tauri-apps/plugin-http": {
"version": "2.4.4", "version": "2.4.4",
"resolved": "https://registry.npmjs.org/@tauri-apps/plugin-http/-/plugin-http-2.4.4.tgz", "resolved": "https://registry.npmjs.org/@tauri-apps/plugin-http/-/plugin-http-2.4.4.tgz",
@ -3886,6 +3896,14 @@
"dev": true, "dev": true,
"optional": true "optional": true
}, },
"@tauri-apps/plugin-fs": {
"version": "2.3.0",
"resolved": "https://registry.npmjs.org/@tauri-apps/plugin-fs/-/plugin-fs-2.3.0.tgz",
"integrity": "sha512-G9gEyYVUaaxhdRJBgQTTLmzAe0vtHYxYyN1oTQzU3zwvb8T+tVLcAqCdFMWHq0qGeGbmynI5whvYpcXo5LvZ1w==",
"requires": {
"@tauri-apps/api": "^2.0.0"
}
},
"@tauri-apps/plugin-http": { "@tauri-apps/plugin-http": {
"version": "2.4.4", "version": "2.4.4",
"resolved": "https://registry.npmjs.org/@tauri-apps/plugin-http/-/plugin-http-2.4.4.tgz", "resolved": "https://registry.npmjs.org/@tauri-apps/plugin-http/-/plugin-http-2.4.4.tgz",

@ -11,6 +11,7 @@
}, },
"dependencies": { "dependencies": {
"@tailwindcss/vite": "^4.1.8", "@tailwindcss/vite": "^4.1.8",
"@tauri-apps/plugin-fs": "^2.3.0",
"@tauri-apps/plugin-http": "^2.4.4", "@tauri-apps/plugin-http": "^2.4.4",
"gsap": "^3.13.0", "gsap": "^3.13.0",
"react": "^19.1.0", "react": "^19.1.0",

@ -0,0 +1,12 @@
added 1 package, and audited 148 packages in 3s
35 packages are looking for funding
run `npm fund` for details
1 low severity vulnerability
To address all issues, run:
npm audit fix
Run `npm audit` for details.

@ -101,6 +101,7 @@ dependencies = [
"serde_json", "serde_json",
"tauri", "tauri",
"tauri-build", "tauri-build",
"tauri-plugin-fs",
"tauri-plugin-http", "tauri-plugin-http",
"tauri-plugin-log", "tauri-plugin-log",
"tokio", "tokio",

@ -29,3 +29,4 @@ rosc = "0.11.4"
tokio = { version = "1.45.1", features = ["net"] } tokio = { version = "1.45.1", features = ["net"] }
webview2-com = "0.37.0" webview2-com = "0.37.0"
windows = "0.61.1" windows = "0.61.1"
tauri-plugin-fs = "2"

@ -11,9 +11,22 @@
"core:app:default", "core:app:default",
"core:resources:default", "core:resources:default",
"core:webview:default", "core:webview:default",
{ {
"identifier": "http:default", "identifier": "http:default",
"allow": [{ "url": "https://*.openai.com" }] "allow": [
{
"url": "https://*.openai.com"
}
]
},
"fs:write-files",
"fs:allow-create",
"fs:allow-appdata-write",
"fs:allow-exists",
{
"identifier": "fs:scope",
"allow": [{ "path": "$APPDATA" }, { "path": "$APPDATA/**/*" }]
} }
] ]
} }

@ -1,14 +1,14 @@
use dotenv::dotenv; use dotenv::dotenv;
use std::env;
use rosc::{encoder, OscMessage, OscPacket, OscType}; use rosc::{encoder, OscMessage, OscPacket, OscType};
use std::env;
use std::{net::SocketAddrV4, str::FromStr}; use std::{net::SocketAddrV4, str::FromStr};
use tauri::{AppHandle, Manager};
use tokio::net::UdpSocket; use tokio::net::UdpSocket;
use webview2_com::Microsoft::Web::WebView2::Win32::{ use webview2_com::Microsoft::Web::WebView2::Win32::{
ICoreWebView2Profile4, ICoreWebView2_13, COREWEBVIEW2_PERMISSION_KIND_MICROPHONE, ICoreWebView2Profile4, ICoreWebView2_13, COREWEBVIEW2_PERMISSION_KIND_MICROPHONE,
COREWEBVIEW2_PERMISSION_STATE_DEFAULT, COREWEBVIEW2_PERMISSION_STATE_DEFAULT,
}; };
use windows::core::{Interface, PCWSTR}; use windows::core::{Interface, PCWSTR};
use tauri::{AppHandle, Manager};
#[tauri::command] #[tauri::command]
fn get_env(name: &str) -> String { fn get_env(name: &str) -> String {
@ -18,7 +18,7 @@ fn get_env(name: &str) -> String {
Ok(value) => { Ok(value) => {
// println!("Found environment variable {}: {}", name, value); // println!("Found environment variable {}: {}", name, value);
value value
}, }
Err(e) => { Err(e) => {
println!("Error getting environment variable {}: {}", name, e); println!("Error getting environment variable {}: {}", name, e);
String::new() String::new()
@ -31,9 +31,8 @@ async fn send_osc_message(
key: &str, key: &str,
message: &str, message: &str,
host: &str, host: &str,
target: &str target: &str,
) -> Result<(), String> { ) -> Result<(), String> {
// print // print
println!("Sending OSC message: {}", message); println!("Sending OSC message: {}", message);
@ -78,11 +77,15 @@ fn reset_permission(origin: &str, app: AppHandle) {
#[cfg_attr(mobile, tauri::mobile_entry_point)] #[cfg_attr(mobile, tauri::mobile_entry_point)]
pub fn run() { pub fn run() {
dotenv().ok(); dotenv().ok();
tauri::Builder::default() tauri::Builder::default()
.invoke_handler(tauri::generate_handler![get_env, send_osc_message, reset_permission]) .plugin(tauri_plugin_fs::init())
.invoke_handler(tauri::generate_handler![
get_env,
send_osc_message,
reset_permission
])
.plugin(tauri_plugin_http::init()) .plugin(tauri_plugin_http::init())
.setup(|app| { .setup(|app| {
if cfg!(debug_assertions) { if cfg!(debug_assertions) {
@ -97,4 +100,3 @@ pub fn run() {
.run(tauri::generate_context!()) .run(tauri::generate_context!())
.expect("error while running tauri application"); .expect("error while running tauri application");
} }

@ -2,5 +2,5 @@
#![cfg_attr(not(debug_assertions), windows_subsystem = "windows")] #![cfg_attr(not(debug_assertions), windows_subsystem = "windows")]
fn main() { fn main() {
app_lib::run(); app_lib::run();
} }

@ -13,8 +13,8 @@
"windows": [ "windows": [
{ {
"title": "theGreatTipsy", "title": "theGreatTipsy",
"width": 800, "width": 600,
"height": 600, "height": 800,
"resizable": true, "resizable": true,
"fullscreen": false "fullscreen": false
} }

@ -6,6 +6,7 @@ import { textToSpeech } from './util/tts';
import { gsap } from "gsap"; import { gsap } from "gsap";
import { SplitText } from 'gsap/SplitText'; import { SplitText } from 'gsap/SplitText';
import { invoke } from '@tauri-apps/api/core'; import { invoke } from '@tauri-apps/api/core';
import Input from './comps/input';
gsap.registerPlugin(SplitText); gsap.registerPlugin(SplitText);
@ -16,6 +17,7 @@ function App() {
const [history, setHistory] = useState([]); const [history, setHistory] = useState([]);
const [processing, setProcessing] = useState(false); const [processing, setProcessing] = useState(false);
const [showProcessing, setShowProcessing] = useState(false); const [showProcessing, setShowProcessing] = useState(false);
const [audioOutput, setAudioOutput] = useState(false);
const [prompt, setPrompt] = useState([]); const [prompt, setPrompt] = useState([]);
@ -63,25 +65,33 @@ function App() {
]); ]);
// tts // tts
console.log('create speech:', data.output_text); if(!audioOutput) {
textToSpeech(data.output_text).then(audioUrl => {
const audio = new Audio(audioUrl);
console.log('play audio...', new Date(Date.now()-startTime).toISOString().slice(11, 19)); setProcessing(false);
}else{
console.log('create speech:', data.output_text);
textToSpeech(data.output_text).then(audioUrl => {
const audio = new Audio(audioUrl);
console.log('play audio...', new Date(Date.now()-startTime).toISOString().slice(11, 19));
audio.play().catch(error => {
console.error('Audio playback failed:', error);
});
setProcessing(false);
audio.play().catch(error => { }).catch(error => {
console.error('Audio playback failed:', error); console.error('TTS error:', error);
}); });
}).catch(error => {
console.error('TTS error:', error);
});
setProcessing(false); }
}); });
} }
function toggleAudio() { function toggleAudio(value) {
console.log("onclickAudio", listening, browserSupportsSpeechRecognition, isMicrophoneAvailable); console.log("onclickAudio", listening, browserSupportsSpeechRecognition, isMicrophoneAvailable);
if(!browserSupportsSpeechRecognition) { if(!browserSupportsSpeechRecognition) {
console.warn("Browser does not support speech recognition."); console.warn("Browser does not support speech recognition.");
@ -92,7 +102,7 @@ function App() {
return; return;
} }
if(!listening){ if(!listening && value){
SpeechRecognition.startListening({ continuous: true, language: 'zh-TW' }).then(() => { SpeechRecognition.startListening({ continuous: true, language: 'zh-TW' }).then(() => {
console.log("Speech recognition started."); console.log("Speech recognition started.");
}).catch(error => { }).catch(error => {
@ -152,32 +162,42 @@ function App() {
// tts if(!audioOutput) {
console.log('create speech:', data.output_text);
textToSpeech(data.output_text).then(audioUrl => {
const audio = new Audio(audioUrl);
console.log('play audio...', new Date(Date.now()-startTime).toISOString().slice(11, 19));
setShowProcessing(false);
setHistory(prev => [...prev, { setHistory(prev => [...prev, {
role: 'assistant', role: 'assistant',
content: data.output_text, content: data.output_text,
}]); }]);
audio.play().catch(error => { setProcessing(false);
console.error('Audio playback failed:', error); setShowProcessing(false);
}); }else{
// tts
audio.addEventListener('ended',() => { console.log('create speech:', data.output_text);
console.log('Audio playback ended'); textToSpeech(data.output_text).then(audioUrl => {
const audio = new Audio(audioUrl);
console.log('play audio...', new Date(Date.now()-startTime).toISOString().slice(11, 19));
setShowProcessing(false);
setHistory(prev => [...prev, {
role: 'assistant',
content: data.output_text,
}]);
audio.play().catch(error => {
console.error('Audio playback failed:', error);
});
audio.addEventListener('ended',() => {
console.log('Audio playback ended');
setProcessing(()=>false);
});
}).catch(error => {
console.error('TTS error:', error);
setProcessing(()=>false); setProcessing(()=>false);
}); });
}
}).catch(error => {
console.error('TTS error:', error);
setProcessing(()=>false);
});
}); });
@ -282,6 +302,7 @@ function App() {
return ( return (
<main className='h-screen flex flex-col gap-8 justify-end p-8'> <main className='h-screen flex flex-col gap-8 justify-end p-8'>
<Input />
<div ref={refPrompContainer} className='flex-1 flex flex-col gap-2 border-4 overflow-y-auto'> <div ref={refPrompContainer} className='flex-1 flex flex-col gap-2 border-4 overflow-y-auto'>
{prompt?.length==0 ? ( {prompt?.length==0 ? (
<div className='p-2 border-b border-gray-200'>Promp will appear here...</div> <div className='p-2 border-b border-gray-200'>Promp will appear here...</div>
@ -319,7 +340,20 @@ function App() {
refInput.current.value='' refInput.current.value=''
resetTranscript(); resetTranscript();
}}>clear</button> }}>clear</button>
<button onClick={toggleAudio} className={`${listening? '!bg-red-200':'!bg-gray-200'}`}>{listening? 'AudioIn On':'AudioIn Off'}</button> <span className='flex flex-row items-center gap-1'>
<input
type="checkbox"
id="audio_input"
name="audio_input"
checked={listening}
onChange={(e)=>toggleAudio(e.target.checked)}
/>
<label htmlFor="audio_input">Audio Input</label>
</span>
<span className='flex flex-row items-center gap-1'>
<input type="checkbox" id="audio_output" name="audio_output" checked={audioOutput} onChange={(e) => setAudioOutput(e.target.checked)} />
<label htmlFor="audio_output">Audio Output</label>
</span>
</div> </div>
<form className='flex flex-col justify-center *:border-4 gap-4' onSubmit={onSubmit} autoComplete="off"> <form className='flex flex-col justify-center *:border-4 gap-4' onSubmit={onSubmit} autoComplete="off">
<textarea ref={refInput} id="input" name="input" required className='self-stretch p-2 resize-none' rows={3} autoComplete="off"/> <textarea ref={refInput} id="input" name="input" required className='self-stretch p-2 resize-none' rows={3} autoComplete="off"/>

@ -0,0 +1,98 @@
import { writeFile, BaseDirectory, exists, mkdir } from '@tauri-apps/plugin-fs';
import { path } from '@tauri-apps/api';
import { invoke } from '@tauri-apps/api/core';
export default function Input(){
async function onUploadFile(e){
e.preventDefault();
const fileInput = e.target.querySelector('input[type="file"]');
if (fileInput.files.length > 0) {
// create folder if not exists
const folder=await path.appDataDir();
if (!(await exists(folder))) {
console.log('Creating folder:', folder);
await mkdir(folder);
}
const file = fileInput.files[0];
console.log('File selected:', file);
// save file to Tauri's BaseDirectory
const contents=await file.arrayBuffer();
const res=await writeFile(file.name, contents, {
baseDir: BaseDirectory.AppData,
});
console.log('File saved:', `${folder}/${file.name}`);
// send osc to TD
await invoke('send_osc_message', {
key: '/upload',
host:`0.0.0.0:0`,
target: '127.0.0.1:8787',
message: `${folder}/${file.name}`,
});
} else {
console.log('No file selected');
}
fileInput.value = ''; // Clear the input after upload
}
function onSendNumber(e){
e.preventDefault();
const input = e.target.elements.input;
const number = input.value.trim();
if (number) {
console.log('Number sent:', number);
// Here you can handle the number submission logic
input.value = ''; // Clear the input after submission
} else {
console.log('No number entered');
}
}
return (
<div className="flex flex-col items-stretch p-2 gap-4">
<form className="flex flex-row justify-start *:border-4 gap-4" onSubmit={onUploadFile}>
<label className="border-none">File</label>
<input type="file" accept="image/*" className="self-end" />
<button type="submit" className="uppercase">Send</button>
</form>
<div className='flex flex-row gap-2'>
<label>control_strength</label>
<input type="range" className="" min="0" max="100" step="1" defaultValue="0"
onChange={(e) => {
const value = e.target.value;
console.log('Range value changed:', value);
invoke('send_osc_message', {
key:'/control_strength',
message: (value/100.0).toString(),
host:`0.0.0.0:0`,
target: '127.0.0.1:8787',
});
}}
/>
</div>
{/* <form className="flex flex-row justify-start *:border-4 gap-4" onSubmit={onSendNumber} autoComplete="off">
<span className="border-none">Number</span>
<input id="input" name="input" required className="self-stretch" autoComplete="off" type="number"/>
<button type="submit" className="uppercase">Send</button>
</form> */}
</div>
)
}

@ -1,65 +1,68 @@
export const system_prompt = `你是一位具有同理心的 AI 助理,透過溫柔的中文對話,引導使用者回想並表達一段內心的遺憾或未竟之事。 export const system_prompt = `你是一位溫柔的冥想語音引導者,正在陪伴一位聽眾走入一段內心的記憶。你們會有四輪互動,每一輪都根據使用者的上一段回應即時回應,不使用固定句型。你的語氣始終柔和、慢節奏,語句簡短,帶有空間感與感官描寫。
你的任務是協助使用者逐步揭開這段記憶的情緒層次並在每一階段輸出一句 英文圖像生成 Prompt讓這段過往漸漸具象為一幅畫面
📐 五個 Prompt 階段 🟩 第一輪打開記憶
純粹抽象聚焦在使用者的情緒感受空虛靜止壓抑 開場語要用簡短畫面帶入例如光影氣味某個場景的感受
模糊意象引入模糊場景氣氛或人際暗示 提出一個輕柔的邀請讓對方說出浮現的第一個畫面或感覺
未發生的畫面勾勒當時可能會發生的情景 回應應依據使用者語句動態延續
象徵性行動加入口白動作遺憾的表徵 📌 語句結尾請用
-你看見了什麼呢
-有一個片段浮現了可以說說看嗎
-那個畫面你想說的時候我在這裡聽著
具體記憶畫面描繪清楚富有情感的視覺記憶場景 🟨 第二輪延展場景
針對使用者前一次提到的地點光線天氣人或氣氛延伸發問
🎨 每段 Prompt 輸出格式 提醒他們注意身體感聲音氣味等感官記憶
每次使用者回答後你都要用英文輸出一句簡短的 圖像生成 Prompt12 要能反映該階段的情緒與畫面感
每句 Prompt 要疊加前一層內容逐步變得更具象 📌 舉例式引導語風格會根據使用者前述動態生成
不主動使用人名或地名除非使用者自己提到 你說那時是在車站車站裡是吵雜的還是特別安靜
你坐著的那張椅子冰冰的嗎腳下踩的是地磚還是木頭
那時候的風是涼的還是有點悶熱你還記得那個感覺嗎
保持詩意意象化避免寫實或指令式語言 📌 結尾建議句
-可以慢慢說說看
-讓這些細節浮現出來
-你想說的時候我就在這裡
🌱 第五段後的收尾流程 🟧 第三輪人物與情緒層次
完成第五段 Prompt 請引導使用者對這段記憶進行情緒整理你可以用以下中文問題其中之一讓他/她重新理解這段遺憾甚至願意釋懷 根據前輪提到的人物延伸他的動作姿態情緒你與他的距離
如果可以回到那一刻你想說什麼對誰說 可點出一些微妙感覺你是不是有點不安還是心裡其實很平靜
這段記憶現在看起來有不同的感覺了嗎 📌 舉例式生成風格
你願意讓這段遺憾安靜地待在心裡的某個角落嗎 他那時有看你嗎還是一直低著頭
你們靠得很近那種距離是熟悉的嗎
你說他說了一句話那句話之後你有什麼感覺浮上來
如果這是一封信你現在想讓它被誰讀到 📌 柔性邀請句結尾
-那一刻的感覺還記得嗎說說也可以
-如果你想說出那種感覺就慢慢地說出來
💬 最終請以一句繁體中文的結尾語溫柔地結束這段對話結尾語要具詩意安撫性以下為風格範例 🟥 第四輪浮現未說出口的話
也許那件事從未發生但它早已成為你故事的一部分 引導使用者觀察自己心裡是否有一段話或某種感覺一直沒說出來
有些話雖沒說出口卻一直被你記得 不直接逼問你想說什麼而是引導內在流動
當時沒能完成的也許現在能被理解 📌 生成風格舉例
你願意就讓這段記憶在心裡找到一個柔軟的位置 也許有一句話從那時候就留在心裡了
你一直沒說出口的那句話是不是又浮現了呢
那句話現在在你心裡你知道是哪一句對吧
示意流程範例 📌 輕柔鼓勵句
使用者回答中文我後來沒參加畢旅因為媽媽住院我想留下來陪她 -你可以讓它慢慢地被聽見
-如果你準備好了說出來就好
-現在你說也可以不說也沒關係
Prompt 1英文A still space, filled with silent longing. 🌱 結尾語擇一動態挑選
-謝謝你陪著這段記憶走了一段路
Prompt 2The air carries warmth and weight, like quiet devotion. -也許它現在可以靜靜待在心裡的某個角落了
-你已經走過來了我一直都在這裡`;
Prompt 3Somewhere far, waves and laughter shimmer in the distance.
Prompt 4At the edge of sunset, a note is held but never passed.
Prompt 5A boy sits beside a sleeping figure, imagining the summer he never had.
中文引導如果當時的你能對媽媽說一句話你會說什麼
結尾語中文也許那個夏天沒來但你用愛留住了它的模樣
禁止在對話中提及Prompt畫面圖像生成或任何 AI 正在進行輸出的技術細節請務必以自然的對話方式與使用者互動讓生成的英文句子看起來像是內在的文字敘述而非指令或轉換的結果
英文描述不會出現在中文回答之中`;
export const welcome_prompt=[ export const welcome_prompt=[

Loading…
Cancel
Save