|
|
|
|
@ -1,6 +1,12 @@ |
|
|
|
|
import { useEffect, useRef, useState } from 'react'; |
|
|
|
|
import './App.css' |
|
|
|
|
import { sendChatMessage } from './util/chat'; |
|
|
|
|
import SpeechRecognition, { useSpeechRecognition } from 'react-speech-recognition'; |
|
|
|
|
import { textToSpeech } from './util/tts'; |
|
|
|
|
import { gsap } from "gsap"; |
|
|
|
|
import { SplitText } from 'gsap/SplitText'; |
|
|
|
|
import { set } from 'zod'; |
|
|
|
|
gsap.registerPlugin(SplitText); |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
const BASE_URL='http://localhost:3333'; |
|
|
|
|
@ -9,81 +15,103 @@ function App() { |
|
|
|
|
|
|
|
|
|
const [history, setHistory] = useState([]); |
|
|
|
|
const [processing, setProcessing] = useState(false); |
|
|
|
|
const [showProcessing, setShowProcessing] = useState(false); |
|
|
|
|
|
|
|
|
|
const [prompt, setPrompt] = useState([]); |
|
|
|
|
|
|
|
|
|
const refHistoryContainer= useRef(null); |
|
|
|
|
const refPrompContainer= useRef(null); |
|
|
|
|
const refInput=useRef(null); |
|
|
|
|
|
|
|
|
|
function onSubmitToNode(event) { |
|
|
|
|
event.preventDefault(); |
|
|
|
|
const input = event.target.elements.input.value; |
|
|
|
|
console.log("Submitted:", input); |
|
|
|
|
const { |
|
|
|
|
transcript, |
|
|
|
|
finalTranscript, |
|
|
|
|
listening, |
|
|
|
|
resetTranscript, |
|
|
|
|
}=useSpeechRecognition(); |
|
|
|
|
|
|
|
|
|
// setProcessing(true); |
|
|
|
|
|
|
|
|
|
function restart(){ |
|
|
|
|
console.log("Restarting..."); |
|
|
|
|
setHistory([]); |
|
|
|
|
setPrompt([]); |
|
|
|
|
refInput.current.value = ''; |
|
|
|
|
resetTranscript(); |
|
|
|
|
SpeechRecognition.stopListening(); |
|
|
|
|
|
|
|
|
|
fetch(`${BASE_URL}/generate`, { |
|
|
|
|
method: 'POST', |
|
|
|
|
headers: { |
|
|
|
|
'Content-Type': 'application/json', |
|
|
|
|
}, |
|
|
|
|
body: JSON.stringify({ |
|
|
|
|
input:[ |
|
|
|
|
...history, |
|
|
|
|
{ |
|
|
|
|
role:'user', |
|
|
|
|
content:[{ |
|
|
|
|
type:'input_text', |
|
|
|
|
text: input |
|
|
|
|
}] |
|
|
|
|
} |
|
|
|
|
] |
|
|
|
|
}), |
|
|
|
|
}).then(response => { |
|
|
|
|
// create start message |
|
|
|
|
const startTime=Date.now(); |
|
|
|
|
setProcessing(true); |
|
|
|
|
sendChatMessage([]).then(response => { |
|
|
|
|
if (!response.ok) { |
|
|
|
|
throw new Error('Network response was not ok'); |
|
|
|
|
} |
|
|
|
|
response.json().then(data => {; |
|
|
|
|
console.log(data); |
|
|
|
|
|
|
|
|
|
let data=response; |
|
|
|
|
console.log('get reply: ', data, new Date(Date.now()-startTime).toISOString().slice(11, 19)); |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
// add to history |
|
|
|
|
setHistory(prev => [...prev, { |
|
|
|
|
role: 'user', |
|
|
|
|
content: [{ |
|
|
|
|
type:'input_text', |
|
|
|
|
text: input |
|
|
|
|
}] |
|
|
|
|
}, { |
|
|
|
|
role: 'assistant', |
|
|
|
|
content: [{ |
|
|
|
|
type:'output_text', |
|
|
|
|
text: data.output_text |
|
|
|
|
}] |
|
|
|
|
content: data.output_text, |
|
|
|
|
}]); |
|
|
|
|
|
|
|
|
|
setPrompt([ |
|
|
|
|
...prompt, |
|
|
|
|
data.prompt, |
|
|
|
|
]); |
|
|
|
|
|
|
|
|
|
// clear input |
|
|
|
|
event.target.elements.input.value = ''; |
|
|
|
|
setProcessing(false); |
|
|
|
|
// tts |
|
|
|
|
console.log('create speech:', data.output_text); |
|
|
|
|
textToSpeech(data.output_text).then(audioUrl => { |
|
|
|
|
const audio = new Audio(audioUrl); |
|
|
|
|
|
|
|
|
|
console.log('play audio...', new Date(Date.now()-startTime).toISOString().slice(11, 19)); |
|
|
|
|
|
|
|
|
|
audio.play().catch(error => { |
|
|
|
|
console.error('Audio playback failed:', error); |
|
|
|
|
}); |
|
|
|
|
}).catch(error => { |
|
|
|
|
console.error('TTS error:', error); |
|
|
|
|
}); |
|
|
|
|
|
|
|
|
|
setProcessing(false); |
|
|
|
|
|
|
|
|
|
}); |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
function toggleAudio() { |
|
|
|
|
// console.log("onclickAudio"); |
|
|
|
|
if(!listening){ |
|
|
|
|
SpeechRecognition.startListening({ continuous: true, language: 'zh-TW' }); |
|
|
|
|
|
|
|
|
|
}else{ |
|
|
|
|
SpeechRecognition.stopListening(); |
|
|
|
|
} |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
function onSubmit(event) { |
|
|
|
|
event.preventDefault(); |
|
|
|
|
|
|
|
|
|
if(processing) { |
|
|
|
|
console.warn("Already processing, ignoring submission."); |
|
|
|
|
return; |
|
|
|
|
} |
|
|
|
|
setProcessing(true); |
|
|
|
|
setShowProcessing(true); |
|
|
|
|
|
|
|
|
|
const input = event.target.elements.input.value; |
|
|
|
|
console.log("Submitted:", input); |
|
|
|
|
|
|
|
|
|
// setProcessing(true); |
|
|
|
|
if(!input.trim()?.length) { |
|
|
|
|
console.warn("Input is empty, ignoring submission."); |
|
|
|
|
return; |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
const startTime=Date.now(); |
|
|
|
|
console.log("Submit reply:", input); |
|
|
|
|
|
|
|
|
|
sendChatMessage([ |
|
|
|
|
...history, |
|
|
|
|
@ -94,16 +122,14 @@ function App() { |
|
|
|
|
]).then(response => { |
|
|
|
|
if (!response.ok) { |
|
|
|
|
throw new Error('Network response was not ok'); |
|
|
|
|
setProcessing(false); |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
let data=response; |
|
|
|
|
console.log(data); |
|
|
|
|
console.log('get reply: ', data, new Date(Date.now()-startTime).toISOString().slice(11, 19)); |
|
|
|
|
|
|
|
|
|
// add to history |
|
|
|
|
setHistory(prev => [...prev, { |
|
|
|
|
role: 'user', |
|
|
|
|
content:input, |
|
|
|
|
}, { |
|
|
|
|
role: 'assistant', |
|
|
|
|
content: data.output_text, |
|
|
|
|
}]); |
|
|
|
|
@ -113,21 +139,108 @@ function App() { |
|
|
|
|
data.prompt, |
|
|
|
|
]); |
|
|
|
|
|
|
|
|
|
// clear input |
|
|
|
|
event.target.elements.input.value = ''; |
|
|
|
|
setProcessing(false); |
|
|
|
|
setShowProcessing(false); |
|
|
|
|
|
|
|
|
|
// tts |
|
|
|
|
console.log('create speech:', data.output_text); |
|
|
|
|
textToSpeech(data.output_text).then(audioUrl => { |
|
|
|
|
const audio = new Audio(audioUrl); |
|
|
|
|
|
|
|
|
|
console.log('play audio...', new Date(Date.now()-startTime).toISOString().slice(11, 19)); |
|
|
|
|
|
|
|
|
|
audio.play().catch(error => { |
|
|
|
|
console.error('Audio playback failed:', error); |
|
|
|
|
}); |
|
|
|
|
|
|
|
|
|
audio.addEventListener('ended',() => { |
|
|
|
|
console.log('Audio playback ended'); |
|
|
|
|
setProcessing(()=>false); |
|
|
|
|
}); |
|
|
|
|
|
|
|
|
|
}).catch(error => { |
|
|
|
|
console.error('TTS error:', error); |
|
|
|
|
setProcessing(()=>false); |
|
|
|
|
}); |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
}); |
|
|
|
|
|
|
|
|
|
// clear input |
|
|
|
|
event.target.elements.input.value = ''; |
|
|
|
|
// setProcessing(()=>false); |
|
|
|
|
setHistory(prev => [...prev, { |
|
|
|
|
role: 'user', |
|
|
|
|
content:input, |
|
|
|
|
}]); |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
} |
|
|
|
|
useEffect(()=>{ |
|
|
|
|
refHistoryContainer.current.scrollTop = refHistoryContainer.current.scrollHeight; |
|
|
|
|
|
|
|
|
|
// Animate the history items |
|
|
|
|
if(history.length === 0) return; |
|
|
|
|
|
|
|
|
|
let last_item=document.querySelector('.last_history'); |
|
|
|
|
console.log('last_item', last_item); |
|
|
|
|
if(!last_item) return; |
|
|
|
|
|
|
|
|
|
let split=SplitText.create(last_item, { |
|
|
|
|
type: "chars", |
|
|
|
|
aria:'hidden' |
|
|
|
|
}); |
|
|
|
|
console.log('split', split); |
|
|
|
|
gsap.fromTo(split.chars, { |
|
|
|
|
opacity: 0, |
|
|
|
|
}, { |
|
|
|
|
opacity: 1, |
|
|
|
|
y: 0, |
|
|
|
|
duration: 1, |
|
|
|
|
ease: "steps(1)", |
|
|
|
|
stagger: 0.05 |
|
|
|
|
}); |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
},[history]); |
|
|
|
|
useEffect(()=>{ |
|
|
|
|
refPrompContainer.current.scrollTop = refPrompContainer.current.scrollHeight; |
|
|
|
|
},[prompt]); |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
useEffect(()=>{ |
|
|
|
|
|
|
|
|
|
if(listening){ |
|
|
|
|
refInput.current.value = transcript; |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
},[transcript]); |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
useEffect(()=>{ |
|
|
|
|
if(finalTranscript){ |
|
|
|
|
refInput.current.value = finalTranscript; |
|
|
|
|
console.log('Final Transcript:', finalTranscript); |
|
|
|
|
|
|
|
|
|
if(processing) return; // Prevent submission if already processing |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
// Submit the final transcript |
|
|
|
|
onSubmit({ |
|
|
|
|
preventDefault: () => {}, |
|
|
|
|
target: { |
|
|
|
|
elements: { |
|
|
|
|
input: refInput.current |
|
|
|
|
} |
|
|
|
|
} |
|
|
|
|
}); |
|
|
|
|
resetTranscript(); // Clear the transcript after submission |
|
|
|
|
|
|
|
|
|
} |
|
|
|
|
},[finalTranscript]); |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
return ( |
|
|
|
|
<main className='h-screen flex flex-col gap-8 justify-end p-8'> |
|
|
|
|
<div ref={refPrompContainer} className='flex-1 flex flex-col gap-2 border-4 overflow-y-auto'> |
|
|
|
|
@ -143,21 +256,35 @@ function App() { |
|
|
|
|
</div> |
|
|
|
|
<div ref={refHistoryContainer} className='flex-1 overflow-y-auto'> |
|
|
|
|
<div className='flex flex-col justify-end gap-2'> |
|
|
|
|
{history?.length==0? ( |
|
|
|
|
{history?.length==0 && !showProcessing? ( |
|
|
|
|
<div className='p-2'>History will appear here...</div> |
|
|
|
|
):( |
|
|
|
|
history.map((item, index) => ( |
|
|
|
|
<div key={index} className={`p-2 rounded border-4 ${item.role === 'user' ? 'bg-gray-100' : 'bg-yellow-100'}`}> |
|
|
|
|
<p className='text-lg whitespace-pre-wrap'>{item.content}</p> |
|
|
|
|
<p className={`text-lg whitespace-pre-wrap history_item ${index==history?.length-1 && 'last_history'}`}>{item.content}</p> |
|
|
|
|
</div> |
|
|
|
|
)) |
|
|
|
|
)} |
|
|
|
|
{showProcessing && ( |
|
|
|
|
<div className='p-2 rounded border-4 bg-yellow-100'> |
|
|
|
|
<span className='animate-pulse'>...</span> |
|
|
|
|
</div> |
|
|
|
|
)} |
|
|
|
|
</div> |
|
|
|
|
</div> |
|
|
|
|
<div className='flex flex-col gap-2'> |
|
|
|
|
<div className='flex flex-row justify-end gap-2 '> |
|
|
|
|
<button className='self-end' onClick={restart}>Restart</button> |
|
|
|
|
<span className='flex-1'></span> |
|
|
|
|
<button className='' onClick={()=>{ |
|
|
|
|
refInput.current.value='' |
|
|
|
|
resetTranscript(); |
|
|
|
|
}}>clear</button> |
|
|
|
|
<button onClick={toggleAudio} className={`${listening? '!bg-red-200':'!bg-gray-200'}`}>{listening? 'AudioIn On':'AudioIn Off'}</button> |
|
|
|
|
</div> |
|
|
|
|
<div className=''> |
|
|
|
|
<form className='flex flex-col justify-center *:border-4 gap-4' onSubmit={onSubmit} autoComplete="off"> |
|
|
|
|
<input id="input" name="input" required className='self-stretch p-2' autoComplete="off"/> |
|
|
|
|
<button type="submit" className='rounded-full uppercase' disabled={processing}>Send</button> |
|
|
|
|
<textarea ref={refInput} id="input" name="input" required className='self-stretch p-2 resize-none' rows={3} autoComplete="off"/> |
|
|
|
|
<button type="submit" className='uppercase' disabled={processing}>Send</button> |
|
|
|
|
</form> |
|
|
|
|
</div> |
|
|
|
|
</main> |
|
|
|
|
|
