You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
375 lines
10 KiB
375 lines
10 KiB
import { useEffect, useRef, useState } from 'react';
|
|
import SpeechRecognition, { useSpeechRecognition } from 'react-speech-recognition';
|
|
import { gsap } from "gsap";
|
|
import { SplitText } from 'gsap/SplitText';
|
|
|
|
import Input from '../comps/input';
|
|
|
|
import { sendChatMessage } from '../util/chat';
|
|
import { textToSpeech } from '../util/tts';
|
|
import { useData } from '../util/useData';
|
|
|
|
gsap.registerPlugin(SplitText);
|
|
|
|
|
|
|
|
export function Conversation() {
|
|
|
|
const { data: params}=useData();
|
|
|
|
const [history, setHistory] = useState([]);
|
|
const [processing, setProcessing] = useState(false);
|
|
const [showProcessing, setShowProcessing] = useState(false);
|
|
const [audioOutput, setAudioOutput] = useState(false);
|
|
|
|
const [prompt, setPrompt] = useState([]);
|
|
|
|
const refHistoryContainer= useRef(null);
|
|
const refPrompContainer= useRef(null);
|
|
const refInput=useRef(null);
|
|
|
|
const {
|
|
transcript,
|
|
finalTranscript,
|
|
listening,
|
|
resetTranscript,
|
|
browserSupportsSpeechRecognition,
|
|
isMicrophoneAvailable,
|
|
}=useSpeechRecognition();
|
|
|
|
|
|
function restart(){
|
|
console.log("Restarting...");
|
|
setHistory([]);
|
|
setPrompt([]);
|
|
refInput.current.value = '';
|
|
resetTranscript();
|
|
SpeechRecognition.stopListening();
|
|
|
|
// create start message
|
|
const startTime=Date.now();
|
|
setProcessing(true);
|
|
sendChatMessage([], params).then(response => {
|
|
if (!response.ok) {
|
|
throw new Error('Network response was not ok');
|
|
}
|
|
|
|
let data=response;
|
|
console.log('get reply: ', data, new Date(Date.now()-startTime).toISOString().slice(11, 19));
|
|
|
|
|
|
// add to history
|
|
setHistory(() => [{
|
|
role: 'assistant',
|
|
content: data.output_text,
|
|
}]);
|
|
setPrompt(()=>[
|
|
data.prompt,
|
|
]);
|
|
|
|
// tts
|
|
if(!audioOutput) {
|
|
|
|
setProcessing(false);
|
|
|
|
|
|
}else{
|
|
console.log('create speech:', data.output_text);
|
|
textToSpeech(data.output_text, params?.voice_prompt).then(audioUrl => {
|
|
const audio = new Audio(audioUrl);
|
|
|
|
console.log('play audio...', new Date(Date.now()-startTime).toISOString().slice(11, 19));
|
|
|
|
audio.play().catch(error => {
|
|
console.error('Audio playback failed:', error);
|
|
});
|
|
|
|
setProcessing(false);
|
|
|
|
|
|
}).catch(error => {
|
|
console.error('TTS error:', error);
|
|
});
|
|
|
|
}
|
|
|
|
});
|
|
}
|
|
|
|
function toggleAudio(value) {
|
|
console.log("onclickAudio", listening, browserSupportsSpeechRecognition, isMicrophoneAvailable);
|
|
if(!browserSupportsSpeechRecognition) {
|
|
console.warn("Browser does not support speech recognition.");
|
|
return;
|
|
}
|
|
if(!isMicrophoneAvailable) {
|
|
console.warn("Microphone is not available.");
|
|
return;
|
|
}
|
|
|
|
if(!listening && value){
|
|
SpeechRecognition.startListening({ continuous: true, language: 'zh-TW' }).then(() => {
|
|
console.log("Speech recognition started.");
|
|
}).catch(error => {
|
|
console.error("Error starting speech recognition:", error);
|
|
});
|
|
|
|
}else{
|
|
SpeechRecognition.stopListening();
|
|
}
|
|
}
|
|
|
|
|
|
|
|
|
|
function onSubmit(event) {
|
|
event.preventDefault();
|
|
|
|
if(processing) {
|
|
console.warn("Already processing, ignoring submission.");
|
|
return;
|
|
}
|
|
setProcessing(true);
|
|
setShowProcessing(true);
|
|
|
|
const input = event.target.elements.input.value;
|
|
|
|
if(!input.trim()?.length) {
|
|
console.warn("Input is empty, ignoring submission.");
|
|
return;
|
|
}
|
|
|
|
const startTime=Date.now();
|
|
console.log("Submit reply:", input);
|
|
|
|
sendChatMessage([
|
|
...history,
|
|
{
|
|
role:'user',
|
|
content: input,
|
|
}
|
|
], params).then(response => {
|
|
if (!response.ok) {
|
|
throw new Error('Network response was not ok');
|
|
setProcessing(false);
|
|
}
|
|
|
|
let data=response;
|
|
console.log('get reply: ', data, new Date(Date.now()-startTime).toISOString().slice(11, 19));
|
|
|
|
// add to history
|
|
|
|
|
|
setPrompt([
|
|
...prompt,
|
|
data.prompt,
|
|
]);
|
|
|
|
|
|
|
|
if(!audioOutput) {
|
|
|
|
setHistory(prev => [...prev, {
|
|
role: 'assistant',
|
|
content: data.output_text,
|
|
}]);
|
|
|
|
setProcessing(false);
|
|
setShowProcessing(false);
|
|
}else{
|
|
// tts
|
|
console.log('create speech:', data.output_text);
|
|
textToSpeech(data.output_text, params?.voice_prompt).then(audioUrl => {
|
|
const audio = new Audio(audioUrl);
|
|
|
|
console.log('play audio...', new Date(Date.now()-startTime).toISOString().slice(11, 19));
|
|
setShowProcessing(false);
|
|
setHistory(prev => [...prev, {
|
|
role: 'assistant',
|
|
content: data.output_text,
|
|
}]);
|
|
|
|
audio.play().catch(error => {
|
|
console.error('Audio playback failed:', error);
|
|
});
|
|
|
|
audio.addEventListener('ended',() => {
|
|
console.log('Audio playback ended');
|
|
setProcessing(()=>false);
|
|
});
|
|
|
|
}).catch(error => {
|
|
console.error('TTS error:', error);
|
|
setProcessing(()=>false);
|
|
});
|
|
}
|
|
|
|
});
|
|
|
|
// clear input
|
|
event.target.elements.input.value = '';
|
|
// setProcessing(()=>false);
|
|
setHistory(prev => [...prev, {
|
|
role: 'user',
|
|
content:input,
|
|
}]);
|
|
|
|
|
|
}
|
|
useEffect(()=>{
|
|
refHistoryContainer.current.scrollTop = refHistoryContainer.current.scrollHeight;
|
|
|
|
// Animate the history items
|
|
if(history.length === 0) return;
|
|
|
|
let last_item=document.querySelector('.last_history');
|
|
|
|
if(!last_item) return;
|
|
if(last_item.classList.contains('user')) return;
|
|
// console.log('last_item', last_item);
|
|
|
|
let split=SplitText.create(last_item, {
|
|
type: "chars",
|
|
aria:'hidden'
|
|
});
|
|
console.log('split', split);
|
|
gsap.fromTo(split.chars, {
|
|
opacity: 0,
|
|
}, {
|
|
opacity: 1,
|
|
y: 0,
|
|
duration: 0.5,
|
|
ease: "steps(1)",
|
|
stagger: 0.1,
|
|
onComplete:()=>{
|
|
|
|
}
|
|
});
|
|
|
|
|
|
|
|
},[history]);
|
|
useEffect(()=>{
|
|
refPrompContainer.current.scrollTop = refPrompContainer.current.scrollHeight;
|
|
},[prompt]);
|
|
|
|
|
|
useEffect(()=>{
|
|
|
|
if(listening){
|
|
refInput.current.value = transcript;
|
|
}
|
|
|
|
},[transcript]);
|
|
|
|
|
|
useEffect(()=>{
|
|
if(finalTranscript){
|
|
refInput.current.value = finalTranscript;
|
|
console.log('Final Transcript:', finalTranscript);
|
|
|
|
if(processing) return; // Prevent submission if already processing
|
|
|
|
|
|
// Submit the final transcript
|
|
onSubmit({
|
|
preventDefault: () => {},
|
|
target: {
|
|
elements: {
|
|
input: refInput.current
|
|
}
|
|
}
|
|
});
|
|
resetTranscript(); // Clear the transcript after submission
|
|
|
|
}
|
|
},[finalTranscript]);
|
|
|
|
useEffect(()=>{
|
|
|
|
console.log('window.SpeechRecognition=', window.SpeechRecognition || window.webkitSpeechRecognition);
|
|
|
|
// if (navigator.getUserMedia){
|
|
|
|
// navigator.getUserMedia({audio:true},
|
|
// function(stream) {
|
|
// // start_microphone(stream);
|
|
// console.log('Microphone access granted.');
|
|
// },
|
|
// function(e) {
|
|
// alert('Error capturing audio.');
|
|
// }
|
|
// );
|
|
|
|
// } else { alert('getUserMedia not supported in this browser.'); }
|
|
|
|
|
|
},[]);
|
|
|
|
|
|
|
|
return (
|
|
<main className=''>
|
|
<div className='flex flex-row items-center justify-between'>
|
|
<Input />
|
|
</div>
|
|
<div ref={refPrompContainer} className='flex-1 flex flex-col gap-2 border-4 overflow-y-auto'>
|
|
{prompt?.length==0 ? (
|
|
<div className='p-2 border-b border-gray-200'>Promp will appear here...</div>
|
|
):(
|
|
prompt?.map((item, index) => (
|
|
<div key={index} className='p-2 border-b border-gray-500 bg-pink-200'>
|
|
<p className='text-lg'>{item}</p>
|
|
</div>
|
|
))
|
|
)}
|
|
</div>
|
|
<div ref={refHistoryContainer} className='flex-1 overflow-y-auto'>
|
|
<div className='flex flex-col justify-end gap-2'>
|
|
{history?.length==0 && !showProcessing? (
|
|
<div className='p-2'>History will appear here...</div>
|
|
):(
|
|
history.map((item, index) => (
|
|
<div key={index} className={`p-2 rounded border-4 ${item.role === 'user' ? 'bg-gray-100' : 'bg-yellow-100'}`}>
|
|
<p className={`text-lg whitespace-pre-wrap history_item ${index==history?.length-1 && item.role!='user' && 'last_history'}`}>{item.content}</p>
|
|
</div>
|
|
))
|
|
)}
|
|
{showProcessing && (
|
|
<div className='p-2 rounded border-4 bg-yellow-100'>
|
|
<span className='animate-pulse'>...</span>
|
|
</div>
|
|
)}
|
|
</div>
|
|
</div>
|
|
<div className='flex flex-col gap-2'>
|
|
<div className='flex flex-row justify-end gap-2 '>
|
|
<button className='self-end' onClick={restart}>Restart</button>
|
|
<span className='flex-1'></span>
|
|
<button className='' onClick={()=>{
|
|
refInput.current.value=''
|
|
resetTranscript();
|
|
}}>clear</button>
|
|
<span className='checkbox'>
|
|
<input
|
|
type="checkbox"
|
|
id="audio_input"
|
|
name="audio_input"
|
|
checked={listening}
|
|
onChange={(e)=>toggleAudio(e.target.checked)}
|
|
/>
|
|
<label htmlFor="audio_input">Audio Input</label>
|
|
</span>
|
|
<span className='checkbox'>
|
|
<input type="checkbox" id="audio_output" name="audio_output" checked={audioOutput} onChange={(e) => setAudioOutput(e.target.checked)} />
|
|
<label htmlFor="audio_output">Audio Output</label>
|
|
</span>
|
|
</div>
|
|
<form className='flex flex-col justify-center *:border-4 gap-4' onSubmit={onSubmit} autoComplete="off">
|
|
<textarea ref={refInput} id="input" name="input" required className='self-stretch p-2 resize-none' autoComplete="off"/>
|
|
<button type="submit" className='' disabled={processing}>Send</button>
|
|
</form>
|
|
</div>
|
|
</main>
|
|
)
|
|
}
|
|
|