diff --git a/vite/package-lock.json b/vite/package-lock.json index db1938a..d790696 100644 --- a/vite/package-lock.json +++ b/vite/package-lock.json @@ -10,8 +10,10 @@ "dependencies": { "@tailwindcss/vite": "^4.1.8", "@tauri-apps/plugin-http": "^2.4.4", + "gsap": "^3.13.0", "react": "^19.1.0", "react-dom": "^19.1.0", + "react-speech-recognition": "^4.0.1", "tailwindcss": "^4.1.8" }, "devDependencies": { @@ -2177,6 +2179,11 @@ "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.11.tgz", "integrity": "sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==" }, + "node_modules/gsap": { + "version": "3.13.0", + "resolved": "https://registry.npmjs.org/gsap/-/gsap-3.13.0.tgz", + "integrity": "sha512-QL7MJ2WMjm1PHWsoFrAQH/J8wUeqZvMtHO58qdekHpCfhvhSL4gSiz6vJf5EeMP0LOn3ZCprL2ki/gjED8ghVw==" + }, "node_modules/has-flag": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz", @@ -2539,6 +2546,11 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/lodash.debounce": { + "version": "4.0.8", + "resolved": "https://registry.npmjs.org/lodash.debounce/-/lodash.debounce-4.0.8.tgz", + "integrity": "sha512-FT1yDzDYEoYWhnSGnpE/4Kj1fLZkDFyqRb7fNt6FdYOSxlUWAtp42Eh6Wb0rGIv/m9Bgo7x4GhQbm5Ys4SG5ow==" + }, "node_modules/lodash.merge": { "version": "4.6.2", "resolved": "https://registry.npmjs.org/lodash.merge/-/lodash.merge-4.6.2.tgz", @@ -2784,6 +2796,17 @@ "react": "^19.1.0" } }, + "node_modules/react-speech-recognition": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/react-speech-recognition/-/react-speech-recognition-4.0.1.tgz", + "integrity": "sha512-0fIqzLtfY8vuYA6AmJVK7qiabZx0oFKOO+rbiBgFI3COWVGREy0A+gdU16hWXmFebeyrI8JsOLYsWk6WaHUXRw==", + "dependencies": { + "lodash.debounce": "^4.0.8" + }, + "peerDependencies": { + "react": ">=16.8.0" + } + }, "node_modules/resolve-from": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-4.0.0.tgz", @@ -4289,6 +4312,11 @@ "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.11.tgz", "integrity": "sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==" }, + "gsap": { + "version": "3.13.0", + "resolved": "https://registry.npmjs.org/gsap/-/gsap-3.13.0.tgz", + "integrity": "sha512-QL7MJ2WMjm1PHWsoFrAQH/J8wUeqZvMtHO58qdekHpCfhvhSL4gSiz6vJf5EeMP0LOn3ZCprL2ki/gjED8ghVw==" + }, "has-flag": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz", @@ -4476,6 +4504,11 @@ "p-locate": "^5.0.0" } }, + "lodash.debounce": { + "version": "4.0.8", + "resolved": "https://registry.npmjs.org/lodash.debounce/-/lodash.debounce-4.0.8.tgz", + "integrity": "sha512-FT1yDzDYEoYWhnSGnpE/4Kj1fLZkDFyqRb7fNt6FdYOSxlUWAtp42Eh6Wb0rGIv/m9Bgo7x4GhQbm5Ys4SG5ow==" + }, "lodash.merge": { "version": "4.6.2", "resolved": "https://registry.npmjs.org/lodash.merge/-/lodash.merge-4.6.2.tgz", @@ -4632,6 +4665,14 @@ "scheduler": "^0.26.0" } }, + "react-speech-recognition": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/react-speech-recognition/-/react-speech-recognition-4.0.1.tgz", + "integrity": "sha512-0fIqzLtfY8vuYA6AmJVK7qiabZx0oFKOO+rbiBgFI3COWVGREy0A+gdU16hWXmFebeyrI8JsOLYsWk6WaHUXRw==", + "requires": { + "lodash.debounce": "^4.0.8" + } + }, "resolve-from": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-4.0.0.tgz", diff --git a/vite/package.json b/vite/package.json index 7be0735..d328682 100644 --- a/vite/package.json +++ b/vite/package.json @@ -12,8 +12,10 @@ "dependencies": { "@tailwindcss/vite": "^4.1.8", "@tauri-apps/plugin-http": "^2.4.4", + "gsap": "^3.13.0", "react": "^19.1.0", "react-dom": "^19.1.0", + "react-speech-recognition": "^4.0.1", "tailwindcss": "^4.1.8" }, "devDependencies": { diff --git a/vite/src-tauri/Cargo.toml b/vite/src-tauri/Cargo.toml index 98209a8..bf35552 100644 --- a/vite/src-tauri/Cargo.toml +++ b/vite/src-tauri/Cargo.toml @@ -21,7 +21,7 @@ tauri-build = { version = "2.2.0", features = [] } serde_json = "1.0" serde = { version = "1.0", features = ["derive"] } log = "0.4" -tauri = { version = "2.5.0", features = [] } +tauri = { version = "2.5.0", features = ["devtools"] } tauri-plugin-log = "2.0.0-rc" tauri-plugin-http = "2" dotenv = "0.15.0" diff --git a/vite/src/App.css b/vite/src/App.css index e69de29..a461c50 100644 --- a/vite/src/App.css +++ b/vite/src/App.css @@ -0,0 +1 @@ +@import "tailwindcss"; \ No newline at end of file diff --git a/vite/src/App.jsx b/vite/src/App.jsx index b513a1c..bd85dad 100644 --- a/vite/src/App.jsx +++ b/vite/src/App.jsx @@ -1,6 +1,12 @@ import { useEffect, useRef, useState } from 'react'; import './App.css' import { sendChatMessage } from './util/chat'; +import SpeechRecognition, { useSpeechRecognition } from 'react-speech-recognition'; +import { textToSpeech } from './util/tts'; +import { gsap } from "gsap"; +import { SplitText } from 'gsap/SplitText'; +import { set } from 'zod'; +gsap.registerPlugin(SplitText); const BASE_URL='http://localhost:3333'; @@ -9,82 +15,104 @@ function App() { const [history, setHistory] = useState([]); const [processing, setProcessing] = useState(false); + const [showProcessing, setShowProcessing] = useState(false); + const [prompt, setPrompt] = useState([]); const refHistoryContainer= useRef(null); const refPrompContainer= useRef(null); + const refInput=useRef(null); - function onSubmitToNode(event) { - event.preventDefault(); - const input = event.target.elements.input.value; - console.log("Submitted:", input); - - // setProcessing(true); - - - fetch(`${BASE_URL}/generate`, { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - }, - body: JSON.stringify({ - input:[ - ...history, - { - role:'user', - content:[{ - type:'input_text', - text: input - }] - } - ] - }), - }).then(response => { + const { + transcript, + finalTranscript, + listening, + resetTranscript, + }=useSpeechRecognition(); + + + function restart(){ + console.log("Restarting..."); + setHistory([]); + setPrompt([]); + refInput.current.value = ''; + resetTranscript(); + SpeechRecognition.stopListening(); + + // create start message + const startTime=Date.now(); + setProcessing(true); + sendChatMessage([]).then(response => { if (!response.ok) { throw new Error('Network response was not ok'); } - response.json().then(data => {; - console.log(data); - - // add to history - setHistory(prev => [...prev, { - role: 'user', - content: [{ - type:'input_text', - text: input - }] - }, { - role: 'assistant', - content: [{ - type:'output_text', - text: data.output_text - }] - }]); - - setPrompt([ - ...prompt, - data.prompt, - ]); - - // clear input - event.target.elements.input.value = ''; - setProcessing(false); + + let data=response; + console.log('get reply: ', data, new Date(Date.now()-startTime).toISOString().slice(11, 19)); + + + // add to history + setHistory(prev => [...prev, { + role: 'assistant', + content: data.output_text, + }]); + setPrompt([ + ...prompt, + data.prompt, + ]); + // tts + console.log('create speech:', data.output_text); + textToSpeech(data.output_text).then(audioUrl => { + const audio = new Audio(audioUrl); + + console.log('play audio...', new Date(Date.now()-startTime).toISOString().slice(11, 19)); + + audio.play().catch(error => { + console.error('Audio playback failed:', error); + }); + }).catch(error => { + console.error('TTS error:', error); }); + setProcessing(false); + }); + } + function toggleAudio() { + // console.log("onclickAudio"); + if(!listening){ + SpeechRecognition.startListening({ continuous: true, language: 'zh-TW' }); + }else{ + SpeechRecognition.stopListening(); + } } - function onSubmit(event) { + + + + function onSubmit(event) { event.preventDefault(); + + if(processing) { + console.warn("Already processing, ignoring submission."); + return; + } + setProcessing(true); + setShowProcessing(true); + const input = event.target.elements.input.value; - console.log("Submitted:", input); - - // setProcessing(true); + if(!input.trim()?.length) { + console.warn("Input is empty, ignoring submission."); + return; + } + const startTime=Date.now(); + console.log("Submit reply:", input); + sendChatMessage([ ...history, { @@ -94,16 +122,14 @@ function App() { ]).then(response => { if (!response.ok) { throw new Error('Network response was not ok'); + setProcessing(false); } let data=response; - console.log(data); + console.log('get reply: ', data, new Date(Date.now()-startTime).toISOString().slice(11, 19)); // add to history setHistory(prev => [...prev, { - role: 'user', - content:input, - }, { role: 'assistant', content: data.output_text, }]); @@ -112,21 +138,108 @@ function App() { ...prompt, data.prompt, ]); + + setShowProcessing(false); - // clear input - event.target.elements.input.value = ''; - setProcessing(false); + // tts + console.log('create speech:', data.output_text); + textToSpeech(data.output_text).then(audioUrl => { + const audio = new Audio(audioUrl); + + console.log('play audio...', new Date(Date.now()-startTime).toISOString().slice(11, 19)); + + audio.play().catch(error => { + console.error('Audio playback failed:', error); + }); + + audio.addEventListener('ended',() => { + console.log('Audio playback ended'); + setProcessing(()=>false); + }); + + }).catch(error => { + console.error('TTS error:', error); + setProcessing(()=>false); + }); + }); + // clear input + event.target.elements.input.value = ''; + // setProcessing(()=>false); + setHistory(prev => [...prev, { + role: 'user', + content:input, + }]); + } useEffect(()=>{ refHistoryContainer.current.scrollTop = refHistoryContainer.current.scrollHeight; + + // Animate the history items + if(history.length === 0) return; + + let last_item=document.querySelector('.last_history'); + console.log('last_item', last_item); + if(!last_item) return; + + let split=SplitText.create(last_item, { + type: "chars", + aria:'hidden' + }); + console.log('split', split); + gsap.fromTo(split.chars, { + opacity: 0, + }, { + opacity: 1, + y: 0, + duration: 1, + ease: "steps(1)", + stagger: 0.05 + }); + + + },[history]); useEffect(()=>{ refPrompContainer.current.scrollTop = refPrompContainer.current.scrollHeight; },[prompt]); + + + useEffect(()=>{ + + if(listening){ + refInput.current.value = transcript; + } + + },[transcript]); + + + useEffect(()=>{ + if(finalTranscript){ + refInput.current.value = finalTranscript; + console.log('Final Transcript:', finalTranscript); + + if(processing) return; // Prevent submission if already processing + + + // Submit the final transcript + onSubmit({ + preventDefault: () => {}, + target: { + elements: { + input: refInput.current + } + } + }); + resetTranscript(); // Clear the transcript after submission + + } + },[finalTranscript]); + + return (
@@ -143,21 +256,35 @@ function App() {
- {history?.length==0? ( + {history?.length==0 && !showProcessing? (
History will appear here...
):( history.map((item, index) => (
-

{item.content}

+

{item.content}

)) )} + {showProcessing && ( +
+ ... +
+ )}
-
+
+
+ + + + +
- - +