diff --git a/README.md b/README.md index 24a90a7..8b62fa9 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,9 @@ -# +# Counting-App on NX - +✨ This is Virginia Tech's Echolab [Counting app research project](https://github.com/echo-lab/Counting-App) but within an [NX](https://nx.dev/getting-started/tutorials/react-monorepo-tutorial?utm_source=nx_project&utm_medium=readme&utm_campaign=nx_projects) workspace. -✨ Your new, shiny [Nx workspace](https://nx.dev) is almost ready ✨. + +shiny [Nx workspace](https://nx.dev) is almost ready ✨. [Learn more about this workspace setup and its capabilities](https://nx.dev/getting-started/tutorials/react-monorepo-tutorial?utm_source=nx_project&utm_medium=readme&utm_campaign=nx_projects) or run `npx nx graph` to visually explore what was created. Now, let's get you up to speed! diff --git a/apps/backend/package.json b/apps/backend/package.json index 5a1d1e5..5661c02 100644 --- a/apps/backend/package.json +++ b/apps/backend/package.json @@ -30,5 +30,8 @@ } } } + }, + "dependencies": { + "wav": "^1.0.2" } } diff --git a/apps/backend/src/server.js b/apps/backend/src/server.js index d875226..3ab1440 100644 --- a/apps/backend/src/server.js +++ b/apps/backend/src/server.js @@ -12,6 +12,8 @@ import fetch from 'node-fetch'; import cors from 'cors'; import path from 'path'; import { fileURLToPath } from 'url'; +import { Writer } from 'wav'; +import { PassThrough } from 'stream'; //import { config } from 'dotenv'; // might move it before importing the db.js file // change from apps/backend/.env.local to .env.local to enable environment variables to be loaded on server @@ -245,7 +247,11 @@ app.post("/register", async (req, res) => { }); const data = await response.json(); - res.json(data); + //res.json(data); + const audioBuffer = Buffer.from(data.audioContent, 'base64'); + res.setHeader('Content-Type', 'audio/mpeg'); + res.end(audioBuffer); + } catch (error) { console.error('Server Error in Google Text-to-Speech:', error); res.status(500).json({ message: error.toString() }); diff --git a/apps/frontend/index.html b/apps/frontend/index.html index 6232af6..04ddf5a 100644 --- a/apps/frontend/index.html +++ b/apps/frontend/index.html @@ -2,14 +2,14 @@ - + - + Counting App diff --git a/apps/frontend/logoecholab.png b/apps/frontend/logoecholab.png new file mode 100644 index 0000000..eea2f15 Binary files /dev/null and b/apps/frontend/logoecholab.png differ diff --git a/apps/frontend/src/App.jsx b/apps/frontend/src/App.jsx index 7987c75..dabbe98 100644 --- a/apps/frontend/src/App.jsx +++ b/apps/frontend/src/App.jsx @@ -33,6 +33,23 @@ function App() { document.removeEventListener("contextmenu", handleContextMenu); }; }, []); + + + useEffect(() => { + const unlockAudio = () => { + const silent = new Audio('data:audio/mp3;base64,SUQzBAAAAAABEVRYWFgAAAAtAAADY29tbWVudABCaWdTb3VuZEJhbmsuY29tIC8gTGFTb25vdGhlcXVlLm9yZwBURU5DAAAAHQAAA1N3aXRjaCBQbHVzIMHumPMHBgAAA'); + silent.play().catch(() => {}); + document.body.removeEventListener('click', unlockAudio); + document.body.removeEventListener('touchend', unlockAudio); + }; + document.body.addEventListener('click', unlockAudio); + document.body.addEventListener('touchend', unlockAudio); + return () => { + document.body.removeEventListener('click', unlockAudio); + document.body.removeEventListener('touchend', unlockAudio); + }; + }, []); + return (
diff --git a/apps/frontend/src/helpers/textToSpeech.js b/apps/frontend/src/helpers/textToSpeech.js index f1de9bb..77c6ca6 100644 --- a/apps/frontend/src/helpers/textToSpeech.js +++ b/apps/frontend/src/helpers/textToSpeech.js @@ -8,6 +8,11 @@ export async function textToSpeech(utterance, onSpeechEnd) { return; } + if (currentAudio) { + currentAudio.pause(); // Stop any currently playing audio + currentAudio = null; + } + try { const requestData = { text: utterance, @@ -27,64 +32,95 @@ export async function textToSpeech(utterance, onSpeechEnd) { }); console.log('Response status:', response.status); + + // expected response is a blob + const audioBlob = await response.blob(); + const audioUrl = URL.createObjectURL(audioBlob); + const audio = new Audio(audioUrl); + currentAudio = audio; + + audio.addEventListener('ended', () => { + currentAudio = null; + // Clean up the temporary URL to release memory + URL.revokeObjectURL(audioUrl); + if (typeof onSpeechEnd === 'function') { + onSpeechEnd(); + } + }); - if (!response.ok) { - const errorText = await response.text(); - console.error('Server error response:', { - status: response.status, - statusText: response.statusText, - errorText - }); - return; - } + audio.addEventListener('error', (e) => { + console.error('Audio element error:', e); + URL.revokeObjectURL(audioUrl); // Also clean up on error + }); + + try { + await audio.play(); + console.log('Playback started successfully'); + } catch (playError){ + console.error('Playback failed:', { + name: playError.name, + message: playError.message + }); + URL.revokeObjectURL(audioUrl); + } + + // if (!response.ok) { + // const errorText = await response.text(); + // console.error('Server error response:', { + // status: response.status, + // statusText: response.statusText, + // errorText + // }); + // return; + // } - const data = await response.json(); - if (currentAudio) { - currentAudio.pause(); - currentAudio = null; - } + // const data = await response.json(); + // if (currentAudio) { + // currentAudio.pause(); + // currentAudio = null; + // } - console.log('Parsed response data:', data); + // console.log('Parsed response data:', data); - // Validate audio content - if (!data || !data.audioContent) { - console.error('No audio content in response', data); - return; - } + // // Validate audio content + // if (!data || !data.audioContent) { + // console.error('No audio content in response', data); + // return; + // } - // Stop any currently playing audio - if (currentAudio) { - currentAudio.pause(); - currentAudio = null; - } + // // Stop any currently playing audio + // if (currentAudio) { + // currentAudio.pause(); + // currentAudio = null; + // } - const audioSrc = `data:audio/mp3;base64,${data.audioContent}`; + // const audioSrc = `data:audio/mp3;base64,${data.audioContent}`; - const audio = new Audio(audioSrc); - currentAudio = audio; + // const audio = new Audio(audioSrc); + // currentAudio = audio; - // Set up event listeners - audio.addEventListener('error', (e) => { - console.error('Audio element error:', e); - currentAudio = null; - }); + // // Set up event listeners + // audio.addEventListener('error', (e) => { + // console.error('Audio element error:', e); + // currentAudio = null; + // }); - audio.addEventListener('ended', () => { - currentAudio = null; - if (typeof onSpeechEnd === 'function') { - onSpeechEnd(); - } - }); + // audio.addEventListener('ended', () => { + // currentAudio = null; + // if (typeof onSpeechEnd === 'function') { + // onSpeechEnd(); + // } + // }); - try { - await audio.play(); - console.log('Playback started successfully'); - } catch (playError) { - console.error('Playback failed:', { - name: playError.name, - message: playError.message - }); - } + // try { + // await audio.play(); + // console.log('Playback started successfully'); + // } catch (playError) { + // console.error('Playback failed:', { + // name: playError.name, + // message: playError.message + // }); + // } } catch (error) { console.error('Comprehensive error in Text-to-Speech:', { diff --git a/apps/frontend/src/pages/TouchTrainingPage.jsx b/apps/frontend/src/pages/TouchTrainingPage.jsx index 5dd76c9..ed1efe8 100644 --- a/apps/frontend/src/pages/TouchTrainingPage.jsx +++ b/apps/frontend/src/pages/TouchTrainingPage.jsx @@ -248,21 +248,21 @@ const TouchTrainingPage = () => { const handleTrayClick = (trayType) => { setSelectedTray(trayType); // if the correct tray has been clicked - if (trayType === "greenTray" && sectionTrainData.pages[currentPage].greenTray[0].biscuits.length === sectionTrainData.pages[currentPage].cookies.length) { - textToSpeech("Green is correct, Good job!"); - } - else if (trayType === "purpleTray" && sectionTrainData.pages[currentPage].purpleTray[0].biscuits.length === sectionTrainData.pages[currentPage].cookies.length){ - textToSpeech("Purple is correct, Well done!"); - } - // if the wrong tray has been clicked - else if (trayType === "greenTray" && sectionTrainData.pages[currentPage].greenTray[0].biscuits.length !== sectionTrainData.pages[currentPage].cookies.length) { - const explanation = `No, ${trayType} has ${sectionTrainData.pages[currentPage].greenTray[0].biscuits.length} cookies. Try again!`; - textToSpeech(explanation); - } - else{ - const explanation = `Wrong answer, ${trayType} has ${sectionTrainData.pages[currentPage].purpleTray[0].biscuits.length} cookies. Try again!`; - textToSpeech(explanation); - } + // if (trayType === "greenTray" && sectionTrainData.pages[currentPage].greenTray[0].biscuits.length === sectionTrainData.pages[currentPage].cookies.length) { + // textToSpeech("Green is correct, Good job!"); + // } + // else if (trayType === "purpleTray" && sectionTrainData.pages[currentPage].purpleTray[0].biscuits.length === sectionTrainData.pages[currentPage].cookies.length){ + // textToSpeech("Purple is correct, Well done!"); + // } + // // if the wrong tray has been clicked + // else if (trayType === "greenTray" && sectionTrainData.pages[currentPage].greenTray[0].biscuits.length !== sectionTrainData.pages[currentPage].cookies.length) { + // const explanation = `No, ${trayType} has ${sectionTrainData.pages[currentPage].greenTray[0].biscuits.length} cookies. Try again!`; + // textToSpeech(explanation); + // } + // else{ + // const explanation = `Wrong answer, ${trayType} has ${sectionTrainData.pages[currentPage].purpleTray[0].biscuits.length} cookies. Try again!`; + // textToSpeech(explanation); + // } storeAnswer(currentPage, trayType); }; diff --git a/apps/frontend/src/pages/basePage.jsx b/apps/frontend/src/pages/basePage.jsx index 615632b..bfc90bc 100644 --- a/apps/frontend/src/pages/basePage.jsx +++ b/apps/frontend/src/pages/basePage.jsx @@ -85,21 +85,21 @@ function basePage() { const handleTrayClick = (trayType) => { setSelectedTray(trayType); // if the correct tray has been clicked - if (trayType === "greenTray" && baseData.pages[currentPage].greenTray[0].biscuits.length === baseData.pages[currentPage].cookies.length) { - textToSpeech("Green is correct, Good job!"); - } - else if (trayType === "purpleTray" && baseData.pages[currentPage].purpleTray[0].biscuits.length === baseData.pages[currentPage].cookies.length){ - textToSpeech("Purple is correct, Well done!"); - } - // if the wrong tray has been clicked - else if (trayType === "greenTray" && baseData.pages[currentPage].greenTray[0].biscuits.length !== baseData.pages[currentPage].cookies.length) { - const explanation = `No, ${trayType} has ${baseData.pages[currentPage].greenTray[0].biscuits.length} cookies. Try again!`; - textToSpeech(explanation); - } - else{ - const explanation = `Wrong answer, ${trayType} has ${baseData.pages[currentPage].purpleTray[0].biscuits.length} cookies. Try again!`; - textToSpeech(explanation); - } + // if (trayType === "greenTray" && baseData.pages[currentPage].greenTray[0].biscuits.length === baseData.pages[currentPage].cookies.length) { + // textToSpeech("Green is correct, Good job!"); + // } + // else if (trayType === "purpleTray" && baseData.pages[currentPage].purpleTray[0].biscuits.length === baseData.pages[currentPage].cookies.length){ + // textToSpeech("Purple is correct, Well done!"); + // } + // // if the wrong tray has been clicked + // else if (trayType === "greenTray" && baseData.pages[currentPage].greenTray[0].biscuits.length !== baseData.pages[currentPage].cookies.length) { + // const explanation = `No, ${trayType} has ${baseData.pages[currentPage].greenTray[0].biscuits.length} cookies. Try again!`; + // textToSpeech(explanation); + // } + // else{ + // const explanation = `Wrong answer, ${trayType} has ${baseData.pages[currentPage].purpleTray[0].biscuits.length} cookies. Try again!`; + // textToSpeech(explanation); + // } storeAnswer(currentPage, trayType); }; diff --git a/package-lock.json b/package-lock.json index aadb3aa..75dc1f1 100644 --- a/package-lock.json +++ b/package-lock.json @@ -98,7 +98,10 @@ } }, "apps/backend": { - "version": "0.0.1" + "version": "0.0.1", + "dependencies": { + "wav": "^1.0.2" + } }, "apps/backend-e2e": { "version": "0.0.1" @@ -11155,6 +11158,22 @@ "ieee754": "^1.2.1" } }, + "node_modules/buffer-alloc": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/buffer-alloc/-/buffer-alloc-1.2.0.tgz", + "integrity": "sha512-CFsHQgjtW1UChdXgbyJGtnm+O/uLQeZdtbDo8mfUgYXCHSM1wgrVxXm6bSyrUuErEb+4sYVGCzASBRot7zyrow==", + "license": "MIT", + "dependencies": { + "buffer-alloc-unsafe": "^1.1.0", + "buffer-fill": "^1.0.0" + } + }, + "node_modules/buffer-alloc-unsafe": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/buffer-alloc-unsafe/-/buffer-alloc-unsafe-1.1.0.tgz", + "integrity": "sha512-TEM2iMIEQdJ2yjPJoSIsldnleVaAk1oW3DBVUykyOLsEsFmEc9kn+SFFPz+gl54KQNxlDnAwCXosOS9Okx2xAg==", + "license": "MIT" + }, "node_modules/buffer-builder": { "version": "0.2.0", "resolved": "https://registry.npmjs.org/buffer-builder/-/buffer-builder-0.2.0.tgz", @@ -11178,11 +11197,16 @@ "integrity": "sha512-zRpUiDwd/xk6ADqPMATG8vc9VPrkck7T07OIx0gnjmJAnHnTVXNQG3vfvWNuiZIkwu9KrKdA1iJKfsfTVxE6NA==", "license": "BSD-3-Clause" }, + "node_modules/buffer-fill": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/buffer-fill/-/buffer-fill-1.0.0.tgz", + "integrity": "sha512-T7zexNBwiiaCOGDg9xNX9PBmjrubblRkENuptryuI64URkXDFum9il/JGL8Lm8wYfAXpredVXXZz7eMHilimiQ==", + "license": "MIT" + }, "node_modules/buffer-from": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.2.tgz", "integrity": "sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==", - "dev": true, "license": "MIT" }, "node_modules/buffer-xor": { @@ -23687,6 +23711,30 @@ "readable-stream": "^3.5.0" } }, + "node_modules/stream-parser": { + "version": "0.3.1", + "resolved": "https://registry.npmjs.org/stream-parser/-/stream-parser-0.3.1.tgz", + "integrity": "sha512-bJ/HgKq41nlKvlhccD5kaCr/P+Hu0wPNKPJOH7en+YrJu/9EgqUF+88w5Jb6KNcjOFMhfX4B2asfeAtIGuHObQ==", + "license": "MIT", + "dependencies": { + "debug": "2" + } + }, + "node_modules/stream-parser/node_modules/debug": { + "version": "2.6.9", + "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", + "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", + "license": "MIT", + "dependencies": { + "ms": "2.0.0" + } + }, + "node_modules/stream-parser/node_modules/ms": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", + "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==", + "license": "MIT" + }, "node_modules/streamroller": { "version": "3.1.5", "resolved": "https://registry.npmjs.org/streamroller/-/streamroller-3.1.5.tgz", @@ -25761,6 +25809,58 @@ "node": ">=10.13.0" } }, + "node_modules/wav": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/wav/-/wav-1.0.2.tgz", + "integrity": "sha512-viHtz3cDd/Tcr/HbNqzQCofKdF6kWUymH9LGDdskfWFoIy/HJ+RTihgjEcHfnsy1PO4e9B+y4HwgTwMrByquhg==", + "license": "MIT", + "dependencies": { + "buffer-alloc": "^1.1.0", + "buffer-from": "^1.0.0", + "debug": "^2.2.0", + "readable-stream": "^1.1.14", + "stream-parser": "^0.3.1" + } + }, + "node_modules/wav/node_modules/debug": { + "version": "2.6.9", + "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", + "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", + "license": "MIT", + "dependencies": { + "ms": "2.0.0" + } + }, + "node_modules/wav/node_modules/isarray": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-0.0.1.tgz", + "integrity": "sha512-D2S+3GLxWH+uhrNEcoh/fnmYeP8E8/zHl644d/jdA0g2uyXvy3sb0qxotE+ne0LtccHknQzWwZEzhak7oJ0COQ==", + "license": "MIT" + }, + "node_modules/wav/node_modules/ms": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", + "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==", + "license": "MIT" + }, + "node_modules/wav/node_modules/readable-stream": { + "version": "1.1.14", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-1.1.14.tgz", + "integrity": "sha512-+MeVjFf4L44XUkhM1eYbD8fyEsxcV81pqMSR5gblfcLCHfZvbrqy4/qYHE+/R5HoBUT11WV5O08Cr1n3YXkWVQ==", + "license": "MIT", + "dependencies": { + "core-util-is": "~1.0.0", + "inherits": "~2.0.1", + "isarray": "0.0.1", + "string_decoder": "~0.10.x" + } + }, + "node_modules/wav/node_modules/string_decoder": { + "version": "0.10.31", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-0.10.31.tgz", + "integrity": "sha512-ev2QzSzWPYmy9GuqfIVildA4OdcGLeFZQrq5ys6RtiuF+RQQiZWr8TZNyAcuVXyQRYfEO+MsoB/1BuQVhOJuoQ==", + "license": "MIT" + }, "node_modules/wbuf": { "version": "1.7.3", "resolved": "https://registry.npmjs.org/wbuf/-/wbuf-1.7.3.tgz",