@ -13,9 +13,11 @@ import VoiceAnalysis from "../comps/voiceanalysis";
import { sendOsc , OSC _ADDRESS , updatePrompt , onOscMessageReceived , sendOscStatus } from "../util/osc" ; import { sendOsc , OSC _ADDRESS , updatePrompt , onOscMessageReceived , sendOscStatus } from "../util/osc" ;
import { DebugControl , TEST _PROMPT } from "../comps/debug" ; import { DebugControl , TEST _PROMPT } from "../comps/debug" ;
import { useUser } from "../util/useUser" ; import { useUser } from "../util/useUser" ;
import { set } from "zod/v4" ;
const CUELIST _FILE = 'cuelist_demo2.json' ; const CUELIST _FILE = 'cuelist_demo3.json' ;
const AUDIO _FADE _TIME = 3000 ; / / i n m s
const EmojiType = { const EmojiType = {
phone : '📞' , phone : '📞' ,
@ -53,14 +55,14 @@ export function FreeFlow(){
const [ audioInput , setAudioInput ] = useState ( true ) ; const [ audioInput , setAudioInput ] = useState ( true ) ;
const [ autoSend , setAutoSend ] = useState ( true ) ; const [ autoSend , setAutoSend ] = useState ( true ) ;
const [ chatStatus , setChatStatus ] = useState ( ChatStatus . System ) ; / / S y s t e m , U s e r , P r o c e s s i n g const [ chatStatus , setChatStatus ] = useState ( ChatStatus . System ) ; / / S y s t e m , U s e r , P r o c e s s i n g
const { userId , setUserId , getFileId , setPassword , reset : resetUser , uploadHistory , setSummary , summary , setChoice , getUploadFolder , getDataId } = useUser ( ) ; const { userId , setUserId , getFileId , setPassword , reset : resetUser , uploadHistory , setSummary , summary , setChoice , choice , getUploadFolder , getDataId } = useUser ( ) ;
const refTimer = useRef ( ) ; const refTimer = useRef ( ) ;
const refAudio = useRef ( ) ; const refAudio = useRef ( ) ;
const refAudioPrompt = useRef ( ) ; const refAudioPrompt = useRef ( ) ;
const refInput = useRef ( ) ; const refInput = useRef ( ) ;
const refLight = useRef ( ) ; / / c o n s t r e f L i g h t = u s e R e f ( ) ;
const refPauseTimer = useRef ( ) ; const refPauseTimer = useRef ( ) ;
const refSpeechPaused = useRef ( false ) ; const refSpeechPaused = useRef ( false ) ;
@ -107,12 +109,20 @@ export function FreeFlow(){
if ( params [ 0 ] == 'all' || params [ 0 ] == refData . current ? . id ) if ( params [ 0 ] == 'all' || params [ 0 ] == refData . current ? . id )
onStop ( ) ; onStop ( ) ;
break ; break ;
case OSC _ADDRESS . RESET _CUE :
if ( params [ 0 ] == 'all' || params [ 0 ] == refData . current ? . id ) {
sendOsc ( OSC _ADDRESS . STATUS , 'reset' ) ;
onStop ( ) ;
resetData ( ) ;
}
break ;
} }
/ / H a n d l e O S C m e s s a g e s h e r e / / H a n d l e O S C m e s s a g e s h e r e
} }
function playAudio ( url ) { function playAudio ( url ) {
if ( ! url ) return ; if ( ! url ) return ;
@ -160,9 +170,7 @@ export function FreeFlow(){
setChatStatus ( ChatStatus . End ) ; setChatStatus ( ChatStatus . End ) ;
onCueEnd ( ) ; onCueEnd ( ) ;
console . log ( 'Audio ended, ending current cue' ) ; console . log ( 'Audio ended, ending current cue' ) ;
} else { } else {
/ / i f h i s t o r y c o n t a i n s u s e r i n p u t , s e n d i t / / i f h i s t o r y c o n t a i n s u s e r i n p u t , s e n d i t
const user _input = history . find ( msg => msg . role === 'user' ) ; const user _input = history . find ( msg => msg . role === 'user' ) ;
@ -199,6 +207,27 @@ export function FreeFlow(){
refAudio . current = audio ; / / S t o r e t h e n e w a u d i o r e f e r e n c e refAudio . current = audio ; / / S t o r e t h e n e w a u d i o r e f e r e n c e
} }
function fadeOutAudio ( callback ) {
if ( refAudio . current ) {
console . log ( 'Fading out audio' ) ;
let audio = refAudio . current ;
let fadeOutInterval = setInterval ( ( ) => {
if ( audio . volume > 0 ) {
audio . volume = Math . max ( 0 , audio . volume - 1.0 / ( AUDIO _FADE _TIME / 100 ) ) ; / / D e c r e a s e v o l u m e g r a d u a l l y
} else {
clearInterval ( fadeOutInterval ) ;
audio . pause ( ) ;
audio . volume = 0 ; / / R e s e t v o l u m e f o r n e x t p l a y
if ( callback ) callback ( ) ;
}
} , 100 ) ; / / D e c r e a s e v o l u m e e v e r y 1 0 0 m s
} else {
if ( callback ) callback ( ) ;
}
}
function playCue ( cue ) { function playCue ( cue ) {
if ( ! cue ) return ; if ( ! cue ) return ;
@ -274,8 +303,8 @@ export function FreeFlow(){
} }
if ( cue . callback == 'fade_in_light' ) refLight . current . fadeIn ( ) ; / / F a d e i n l i g h t f o r c o n v e r s a t i o n s t a r t / / i f ( c u e . c a l l b a c k = = ' f a d e _ i n _ l i g h t ' ) r e f L i g h t . c u r r e n t . f a d e I n ( ) ; / / F a d e i n l i g h t f o r c o n v e r s a t i o n s t a r t
if ( cue . callback == 'fade_out_light' ) refLight . current . fadeOut ( ) ; / / F a d e o u t l i g h t f o r c o n v e r s a t i o n e n d / / i f ( c u e . c a l l b a c k = = ' f a d e _ o u t _ l i g h t ' ) r e f L i g h t . c u r r e n t . f a d e O u t ( ) ; / / F a d e o u t l i g h t f o r c o n v e r s a t i o n e n d
if ( cue . audioFile ) { if ( cue . audioFile ) {
@ -293,7 +322,7 @@ export function FreeFlow(){
if ( cue . status && cue . status != 'go' ) { if ( cue . status && cue . status != 'go' ) {
sendOsc ( OSC _ADDRESS . STATUS , cue . status ) ; / / S e n d O S C s t a t u s m e s s a g e sendOsc ( OSC _ADDRESS . STATUS , cue . status ) ; / / S e n d O S C s t a t u s m e s s a g e
if ( cue . status == 'reset' ) { if ( cue . status == 'reset' ) {
refLight . current . set ( 1 ) ; / / r e f L i g h t . c u r r e n t . s e t ( 1 ) ;
resetData ( ) ; resetData ( ) ;
} }
} }
@ -326,12 +355,7 @@ export function FreeFlow(){
console . log ( 'save chat history:' , history ) ; console . log ( 'save chat history:' , history ) ;
uploadHistory ( history ) ; / / S a v e c h a t h i s t o r y w h e n c u e e n d s uploadHistory ( history ) ; / / S a v e c h a t h i s t o r y w h e n c u e e n d s
} }
if ( cue . callback == OSC _ADDRESS . DISCARD ) {
sendOsc ( OSC _ADDRESS . CHOICE , OSC _ADDRESS . DISCARD ) ; / / S e n d O S C d i s c a r d m e s s a g e
setPassword ( ) ;
sendOsc ( OSC _ADDRESS . EXPORT , ` ${ getUploadFolder ( ) } # ${ getFileId ( ) } # ${ summary } # ` ) ;
}
if ( cue . hint != null ) { if ( cue . hint != null ) {
sendOsc ( OSC _ADDRESS . HINT , cue . hint ) ; / / S e n d O S C h i n t m e s s a g e sendOsc ( OSC _ADDRESS . HINT , cue . hint ) ; / / S e n d O S C h i n t m e s s a g e
@ -339,21 +363,6 @@ export function FreeFlow(){
sendOsc ( OSC _ADDRESS . HINT , '' ) ; / / C l e a r h i n t m e s s a g e sendOsc ( OSC _ADDRESS . HINT , '' ) ; / / C l e a r h i n t m e s s a g e
} }
/ / i f ( c u e . c a l l b a c k = = ' s u m m a r y ' ) {
/ / c o n s o l e . l o g ( ' G e t t i n g s u m m a r y . . . ' ) ;
/ / g e t S u m m a r y ( h i s t o r y . m a p ( e l = > ` $ { e l . r o l e } : $ { e l . c o n t e n t } ` ) . j o i n ( ' \ n ' ) , d a t a ) . t h e n ( s u m m a r y _ = > {
/ / c o n s o l e . l o g ( ' S u m m a r y : ' , s u m m a r y _ ) ;
/ / o n C u e E n d ( ) ; / / E n d t h e c u r r e n t c u e a f t e r g e t t i n g s u m m a r y
/ / s e t S u m m a r y ( s u m m a r y _ ? . r e s u l t ) ;
/ / r e f C o n t a i n e r . c u r r e n t . s c r o l l T o p = r e f C o n t a i n e r . c u r r e n t . s c r o l l H e i g h t ; / / S c r o l l t o b o t t o m
/ / } ) . c a t c h ( e r r o r = > {
/ / c o n s o l e . e r r o r ( ' E r r o r g e t t i n g s u m m a r y : ' , e r r o r ) ;
/ / } ) ;
/ / }
refAudio . current ? . pause ( ) ; / / P a u s e a n y p l a y i n g a u d i o refAudio . current ? . pause ( ) ; / / P a u s e a n y p l a y i n g a u d i o
console . log ( 'onCueEnd:' , cue . id ) ; console . log ( 'onCueEnd:' , cue . id ) ;
@ -361,8 +370,8 @@ export function FreeFlow(){
resetTranscript ( ) ; / / R e s e t t r a n s c r i p t a f t e r c u e e n d s resetTranscript ( ) ; / / R e s e t t r a n s c r i p t a f t e r c u e e n d s
sendOscStatus ( OSC _ADDRESS . CLIENT _STATUS , ` ${ data . id } #endcue# ${ cue . id } ` ) ; sendOscStatus ( OSC _ADDRESS . CLIENT _STATUS , ` ${ data . id } #endcue# ${ cue . id } ` ) ;
if ( cue . auto ) { if ( cue . auto || cue . callback == 'numpad' ) {
playCue ( cuelist . find ( c => c . id === cue . nextcue ) ) ; playCue ( cuelist . find ( c => c . id === cue . nextcue ) ) ;
} }
@ -402,7 +411,7 @@ export function FreeFlow(){
setPassword ( ( ) => mess ) ; setPassword ( ( ) => mess ) ;
/ / s e n d O s c ( O S C _ A D D R E S S . P A S S W O R D , m e s s ) ; / / S e n d O S C p a s s w o r d m e s s a g e / / s e n d O s c ( O S C _ A D D R E S S . P A S S W O R D , m e s s ) ; / / S e n d O S C p a s s w o r d m e s s a g e
sendOsc ( OSC _ADDRESS . EXPORT , ` ${ getUploadFolder ( ) } # ${ getDataId ( ) } # ${ summary } # ${ getFileId ( mess ) } ` ) ; sendOsc ( OSC _ADDRESS . EXPORT , ` ${ getUploadFolder ( ) } # ${ getDataId ( ) } # ${ summary } # ${ getFileId ( mess ) } ` ) ;
sendOsc ( OSC _ADDRESS . CHOICE , OSC _ADDRESS . SAVE ) ; / / S e n d O S C s a v e c h o i c e m e s s a g e sendOsc ( OSC _ADDRESS . CHOICE , choice ) ; / / S e n d O S C s a v e c h o i c e m e s s a g e
break ; break ;
} }
@ -603,12 +612,23 @@ export function FreeFlow(){
useEffect ( ( ) => { useEffect ( ( ) => {
if ( ! nextCue ) return ; if ( ! nextCue ) return ;
console . log ( 'Next cue:' , nextCue ) ; console . log ( 'Next cue:' , nextCue ) ;
playCue ( cuelist . find ( c => c . name === nextCue ) ) ; / / P l a y t h e n e x t c u e w h e n i t c h a n g e s
/ / R e s e t n e x t c u e a f t e r p l a y i n g const next = cuelist . find ( c => c . name === nextCue ) ;
setNextCue ( null ) ; if ( currentCue ? . fadeout ) {
/ / f a d e o u t a u d i o
fadeOutAudio ( ( ) => {
console . log ( 'fade out then play next cue:' , next ) ;
playCue ( next ) ;
setNextCue ( ( ) => null ) ;
} ) ;
} else {
playCue ( next ) ;
setNextCue ( null ) ;
}
} , [ nextCue ] ) ; } , [ nextCue ] ) ;
@ -635,21 +655,21 @@ export function FreeFlow(){
return ( return (
< main className = "items-start" > < main className = "items-start" >
< section className = "flex-1 flex flex-col gap-2 self-stretch overflow-hidden" > < section className = "flex-1 flex flex-col gap-2 self-stretch overflow-hidden" >
< DebugControl refLight = { refLight } / > < DebugControl / >
< div className = "w-full p-2 grid grid-cols-4 gap-2 items-stretch justify-center *:max-h-[5rem]" > < div className = "w-full p-2 grid grid-cols-4 gap-2 items-stretch justify-center *:max-h-[5rem]" >
< div className = "bg-gray-100 text-4xl font-bold mb-4 flex justify-center items-center" > < div className = "bg-gray-100 text-4xl font-bold mb-4 flex justify-center items-center" >
{ refCurrentCue . current ? . name } { refCurrentCue . current ? . name }
< / div > < / div >
< Countdown ref = { refTimer } clientId = { data . id } / > < Countdown ref = { refTimer } clientId = { data ? . id } / >
< button className = "!bg-red-300" onClick = { onStop } > Stop < / button > < button className = "!bg-red-300" onClick = { onStop } > Stop < / button >
{ /* <button onClick={saveImage}>Save image</button> */ } { /* <button onClick={saveImage}>Save image</button> */ }
< NumPad onSend = { onNumpad } < NumPad onSend = { onNumpad }
disabled = { currentCue ? . callback !== 'numpad' } disabled = { currentCue ? . callback !== 'numpad' }
type = { currentCue ? . numpad _type } type = { currentCue ? . numpad _type }
clientId = { data . id } clientId = { data ? . id }
/ > / >
< Light ref = { refLight } / > { /* <Light ref={refLight} /> */ }
< VoiceAnalysis / > < VoiceAnalysis / >
{ / * < d i v c l a s s N a m e = " f l e x f l e x - c o l " > { / * < d i v c l a s s N a m e = " f l e x f l e x - c o l " >
< label className = "text-center" > Voice < / label > < label className = "text-center" > Voice < / label >
@ -677,7 +697,8 @@ export function FreeFlow(){
< th > Description < / th > < th > Description < / th >
< th > Type < / th > < th > Type < / th >
< th > Auto < / th > < th > Auto < / th >
< th > Audio / Due < / th > < th > Audio < / th >
< th > Duration < / th >
< th > Action < / th > < th > Action < / th >
< / tr > < / tr >
< / thead > < / thead >
@ -695,7 +716,8 @@ export function FreeFlow(){
< td > { description } < / td > < td > { description } < / td >
< td > { EmojiType [ type ] } < / td > < td > { EmojiType [ type ] } < / td >
< td > { auto ? '⤵️ : '' } < / td > < td > { auto ? '⤵️ : '' } < / td >
< td > { audioFile || props . duration } < / td > < td > { audioFile || "" } < / td >
< td > { props . duration || '' } < / td >
< td > { props . callback && ` < ${ props . callback } > ` } { props . status && ` ( ${ props . status } ) ` } < / td > < td > { props . callback && ` < ${ props . callback } > ` } { props . status && ` ( ${ props . status } ) ` } < / td >
< / tr > < / tr >
) ) } ) ) }
@ -711,7 +733,7 @@ export function FreeFlow(){
{ msg . prompt && < div className = "text-xs bg-gray-200" > { msg . prompt } < / div > } { msg . prompt && < div className = "text-xs bg-gray-200" > { msg . prompt } < / div > }
< / div > < / div >
) ) } ) ) }
{ summary && < div className = "w-full self-center bg-blue-200 px-2" > { summary ? . result } < / div > } { summary && < div className = "w-full self-center bg-blue-200 px-2" > { summary } < / div > }
< / div > < / div >
< textarea ref = { refInput } name = "message" rows = { 2 } < textarea ref = { refInput } name = "message" rows = { 2 }
className = { ` w-full border-1 resize-none p-2 disabled:bg-gray-500 ` } className = { ` w-full border-1 resize-none p-2 disabled:bg-gray-500 ` }
