From c37168f2bdf76e3ef271a6b471bae2ab9a404ae7 Mon Sep 17 00:00:00 2001 From: kwkr Date: Thu, 5 Feb 2026 15:32:52 +0100 Subject: [PATCH] add initial tts using open ai whisper --- caido.config.ts | 3 +- packages/backend/package.json | 4 +- packages/frontend/package.json | 4 +- packages/frontend/src/actions/actions.ts | 50 ++++ packages/frontend/src/index.ts | 3 + packages/frontend/src/register-tts.ts | 124 ++++++++++ packages/frontend/src/views/Settings.vue | 283 +++++++++++++++++++++++ pnpm-lock.yaml | 64 +++-- 8 files changed, 512 insertions(+), 23 deletions(-) create mode 100644 packages/frontend/src/register-tts.ts create mode 100644 packages/frontend/src/views/Settings.vue diff --git a/caido.config.ts b/caido.config.ts index 5029a7c..e563181 100644 --- a/caido.config.ts +++ b/caido.config.ts @@ -34,10 +34,11 @@ export default defineConfig({ plugins: [vue()], build: { rollupOptions: { - external: ["@caido/frontend-sdk"], + external: ["@caido/frontend-sdk", "vue"], }, }, resolve: { + dedupe: ["vue"], alias: [ { find: "@", diff --git a/packages/backend/package.json b/packages/backend/package.json index 073fab2..ae32f8d 100644 --- a/packages/backend/package.json +++ b/packages/backend/package.json @@ -7,8 +7,8 @@ "typecheck": "tsc --noEmit" }, "devDependencies": { - "@caido/sdk-backend": "^0.48.0", - "@caido/sdk-frontend": "0.48.0", + "@caido/sdk-backend": "^0.55.2", + "@caido/sdk-frontend": "0.55.2", "shared": "workspace:*" }, "dependencies": { diff --git a/packages/frontend/package.json b/packages/frontend/package.json index 2dd4683..395cbe5 100644 --- a/packages/frontend/package.json +++ b/packages/frontend/package.json @@ -37,8 +37,8 @@ "vue": "3.5.13" }, "devDependencies": { - "@caido/sdk-backend": "^0.48.0", - "@caido/sdk-frontend": "^0.48.0", + "@caido/sdk-backend": "^0.55.2", + "@caido/sdk-frontend": "0.55.2", "backend": "workspace:*", "shared": "workspace:*", "vue-tsc": "2.2.10" diff --git a/packages/frontend/src/actions/actions.ts b/packages/frontend/src/actions/actions.ts index 3c752f2..6865657 100644 --- a/packages/frontend/src/actions/actions.ts +++ b/packages/frontend/src/actions/actions.ts @@ -95,6 +95,56 @@ export const sendSelectedTextToNote = async (sdk: FrontendSDK) => { } }; +/** + * Sends raw text to the currently open note + */ +export const sendTextToNote = async (sdk: FrontendSDK, text: string) => { + const notesStore = useNotesStore(); + const textToSend = text; + if (!textToSend) { + sdk.window.showToast("No text to send", { variant: "warning" }); + return; + } + + if (!notesStore.currentNotePath) { + sdk.window.showToast( + "No note is currently open. Please open a note first.", + { variant: "warning" }, + ); + return; + } + + try { + await notesStore.loadNote(notesStore.currentNotePath); + + if (notesStore.currentNote) { + const paragraph = createTextParagraph(textToSend); + const updatedContent = addParagraphToContent( + notesStore.currentNote.content, + paragraph, + ); + + await notesStore.updateNoteContent( + notesStore.currentNotePath, + updatedContent, + ); + + sdk.window.showToast( + `Text added to note ${notesStore.currentNotePath}`, + { + variant: "success", + }, + ); + + await notesStore.refreshTree(); + } + } catch (error) { + sdk.window.showToast(`Error adding text to note: ${error}`, { + variant: "error", + }); + } +}; + /** * Sends the replay session to the currently open note */ diff --git a/packages/frontend/src/index.ts b/packages/frontend/src/index.ts index 89a4c11..5771036 100644 --- a/packages/frontend/src/index.ts +++ b/packages/frontend/src/index.ts @@ -16,6 +16,7 @@ import { } from "@/actions/actions"; import { emitter } from "@/utils/eventBus"; import { convertMarkdownToTipTap } from "@/utils/markdownToJSON"; +import { registerVoiceNotes } from "./register-tts"; export const init = (sdk: FrontendSDK) => { const pinia = createPinia(); @@ -95,6 +96,8 @@ export const init = (sdk: FrontendSDK) => { icon: "fas fa-file-alt", }); + registerVoiceNotes(sdk); + emitter.on("confirmMigration", (notes) => { migrateNotes(sdk, notes); }); diff --git a/packages/frontend/src/register-tts.ts b/packages/frontend/src/register-tts.ts new file mode 100644 index 0000000..a3a9b87 --- /dev/null +++ b/packages/frontend/src/register-tts.ts @@ -0,0 +1,124 @@ +import { sendTextToNote } from "./actions/actions"; +import type { FrontendSDK } from "./types"; +import Settings from "./views/Settings.vue"; + +export const registerVoiceNotes = (sdk: FrontendSDK) => { + const COMMAND_KEY = "notesplusplus:tts-toggle-recording"; + const MAX_RECORDING_TIME = 60000; // 1 minute + + sdk.settings.addToSlot("plugins-section", { + type: "Custom", + name: "TTS", + definition: { component: Settings, props: { sdk } }, + }); + + let isRecording = false; + let mediaRecorder: any = null; + let audioChunks: any[] = []; + let autoStopTimer: ReturnType | null = null; + + const stopRecordingAction = () => { + if (autoStopTimer) { + clearTimeout(autoStopTimer); + autoStopTimer = null; + } + + if (mediaRecorder && mediaRecorder.state !== "inactive") { + mediaRecorder.stop(); + } + isRecording = false; + }; + + sdk.commands.register(COMMAND_KEY, { + name: "Toggle Voice Recording", + run: async () => { + if (isRecording) { + stopRecordingAction(); + sdk.window.showToast("Recording stopped. Transcribing...", { + variant: "info", + }); + return; + } + + try { + const stream = await window.navigator.mediaDevices.getUserMedia({ + audio: true, + }); + audioChunks = []; + mediaRecorder = new MediaRecorder(stream); + + mediaRecorder.ondataavailable = (event: any) => { + if (event.data.size > 0) audioChunks.push(event.data); + }; + + mediaRecorder.onstop = async () => { + // STOP HARDWARE + if (mediaRecorder.stream) { + mediaRecorder.stream + .getTracks() + .forEach((track: { stop: () => void }) => track.stop()); + } + + const audioBlob = new Blob(audioChunks, { type: "audio/webm" }); + + const storage = sdk.storage.get() as Record; + const apiKey = storage?.["tts-plugin-api-key"]; + + if (!apiKey) { + sdk.window.showToast("API Key missing! Check settings.", { + variant: "error", + }); + return; + } + + const formData = new FormData(); + formData.append("file", audioBlob, "recording.webm"); + formData.append("model", "whisper-1"); + + try { + const response = await fetch( + "https://api.openai.com/v1/audio/transcriptions", + { + method: "POST", + headers: { Authorization: `Bearer ${apiKey}` }, + body: formData, + }, + ); + + const data = await response.json(); + if (data.text) { + sendTextToNote(sdk, data.text); + } else { + throw new Error(data.error?.message || "Unknown error"); + } + } catch (err: any) { + console.error("OpenAI Whisper Error:", err); + sdk.window.showToast(`Transcription failed: ${err.message}`, { + variant: "error", + }); + } + }; + + mediaRecorder.start(); + isRecording = true; + sdk.window.showToast("Recording started...", { variant: "info" }); + + // AUTO-STOP AFTER 1 MINUTE + autoStopTimer = setTimeout(() => { + if (isRecording) { + sdk.window.showToast("Time limit reached (1 min).", { + variant: "info", + }); + stopRecordingAction(); + } + }, MAX_RECORDING_TIME); + } catch (err) { + console.error("Microphone Access Error:", err); + sdk.window.showToast("Microphone access denied.", { variant: "error" }); + } + }, + }); + + sdk.commandPalette.register(COMMAND_KEY); + sdk.shortcuts.register(COMMAND_KEY, ["alt", "shift", "V"]); +}; diff --git a/packages/frontend/src/views/Settings.vue b/packages/frontend/src/views/Settings.vue new file mode 100644 index 0000000..5e9221b --- /dev/null +++ b/packages/frontend/src/views/Settings.vue @@ -0,0 +1,283 @@ + + + diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 7305de4..d2de218 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -43,11 +43,11 @@ importers: version: 3.24.4 devDependencies: '@caido/sdk-backend': - specifier: ^0.48.0 - version: 0.48.0 + specifier: ^0.55.2 + version: 0.55.2 '@caido/sdk-frontend': - specifier: 0.48.0 - version: 0.48.0(@codemirror/state@6.5.2)(@codemirror/view@6.39.3) + specifier: 0.55.2 + version: 0.55.2(@ai-sdk/provider@3.0.7)(@codemirror/state@6.5.2)(@codemirror/view@6.39.3)(vue@3.5.13(typescript@5.8.3)) shared: specifier: workspace:* version: link:../shared @@ -143,11 +143,11 @@ importers: version: 3.5.13(typescript@5.8.3) devDependencies: '@caido/sdk-backend': - specifier: ^0.48.0 - version: 0.48.0 + specifier: ^0.55.2 + version: 0.55.2 '@caido/sdk-frontend': - specifier: ^0.48.0 - version: 0.48.0(@codemirror/state@6.5.2)(@codemirror/view@6.39.3) + specifier: 0.55.2 + version: 0.55.2(@ai-sdk/provider@3.0.7)(@codemirror/state@6.5.2)(@codemirror/view@6.39.3)(vue@3.5.13(typescript@5.8.3)) backend: specifier: workspace:* version: link:../backend @@ -159,6 +159,10 @@ importers: packages: + '@ai-sdk/provider@3.0.7': + resolution: {integrity: sha512-VkPLrutM6VdA924/mG8OS+5frbVTcu6e046D2bgDo00tehBANR1QBJ/mPcZ9tXMFOsVcm6SQArOregxePzTFPw==} + engines: {node: '>=18'} + '@alloc/quick-lru@5.2.0': resolution: {integrity: sha512-UrcABB+4bUrFABwbluTIBErXwvbsU/V7TZWfmbgJfbkwiBuziS9gxdODUyuiecfdGQ85jglMW6juS3+z5TsKLw==} engines: {node: '>=10'} @@ -199,17 +203,19 @@ packages: peerDependencies: primevue: 4.1.0 - '@caido/quickjs-types@0.18.0': - resolution: {integrity: sha512-hRXUVdDvlhEhvkBoWWytoVS2j1KDVZa8dx2Q/KvWUQTR57U8EMSYE9iFgvPhu78gS8z+RF42Zcb7moNx4SDMlw==} + '@caido/quickjs-types@0.25.4': + resolution: {integrity: sha512-U7Cvi4aHoCN8T2JB+Az3ArqQ3MUK0PRCtBm5Sq1rfqaJ9NymAkh3O29q77Keh3G/zINq2WTkKZq5qJlvrQNl+Q==} - '@caido/sdk-backend@0.48.0': - resolution: {integrity: sha512-eB22xt3GtjoE0paCNlNIGDkw4dcfMm04L4G2W7HEmfkSQi4dMkxvlHd0qfe2ALx9PI5C/KEycZ1rZhucL1sSkw==} + '@caido/sdk-backend@0.55.2': + resolution: {integrity: sha512-GQXeFQamq0WJAJgjuw7yA8AXZweaKx4RYKpM8xp+Wpc96okxgURBALHlw2tPogPU28Y8zkOu4klJcwMoX97TMg==} - '@caido/sdk-frontend@0.48.0': - resolution: {integrity: sha512-qhKPFmrJBZK0zlxM4OhMEwy8w9TcQN5TtpivIR6s5rFa5u/+lzuCAUSd3ZfAxshDdi5C3SmRrlptGk3KfKXrRg==} + '@caido/sdk-frontend@0.55.2': + resolution: {integrity: sha512-jHPUUvvs51hcVq3J6vVWitXU7lgPr90RpRfvxR3NchEDOG1/Uv2lCKTZzlDqoDjuO5KulgRzEKZNgKh6bTPeqQ==} peerDependencies: + '@ai-sdk/provider': ^3.0.1 '@codemirror/state': ^6.0.0 '@codemirror/view': ^6.0.0 + vue: ^3.0.0 '@caido/sdk-shared@0.1.1': resolution: {integrity: sha512-JAV5ajUqxZdXYPTmDEvIKBZon8I5uHq44ATj0Nj3BVpllRDUGY9kcBd+PXMD50+3lv1CvhR3/f6q24T0+4aVJQ==} @@ -596,56 +602,67 @@ packages: resolution: {integrity: sha512-mz5POx5Zu58f2xAG5RaRRhp3IZDK7zXGk5sdEDj4o96HeaXhlUwmLFzNlc4hCQi5sGdR12VDgEUqVSHer0lI9g==} cpu: [arm] os: [linux] + libc: [glibc] '@rollup/rollup-linux-arm-musleabihf@4.39.0': resolution: {integrity: sha512-+YDwhM6gUAyakl0CD+bMFpdmwIoRDzZYaTWV3SDRBGkMU/VpIBYXXEvkEcTagw/7VVkL2vA29zU4UVy1mP0/Yw==} cpu: [arm] os: [linux] + libc: [musl] '@rollup/rollup-linux-arm64-gnu@4.39.0': resolution: {integrity: sha512-EKf7iF7aK36eEChvlgxGnk7pdJfzfQbNvGV/+l98iiMwU23MwvmV0Ty3pJ0p5WQfm3JRHOytSIqD9LB7Bq7xdQ==} cpu: [arm64] os: [linux] + libc: [glibc] '@rollup/rollup-linux-arm64-musl@4.39.0': resolution: {integrity: sha512-vYanR6MtqC7Z2SNr8gzVnzUul09Wi1kZqJaek3KcIlI/wq5Xtq4ZPIZ0Mr/st/sv/NnaPwy/D4yXg5x0B3aUUA==} cpu: [arm64] os: [linux] + libc: [musl] '@rollup/rollup-linux-loongarch64-gnu@4.39.0': resolution: {integrity: sha512-NMRUT40+h0FBa5fb+cpxtZoGAggRem16ocVKIv5gDB5uLDgBIwrIsXlGqYbLwW8YyO3WVTk1FkFDjMETYlDqiw==} cpu: [loong64] os: [linux] + libc: [glibc] '@rollup/rollup-linux-powerpc64le-gnu@4.39.0': resolution: {integrity: sha512-0pCNnmxgduJ3YRt+D+kJ6Ai/r+TaePu9ZLENl+ZDV/CdVczXl95CbIiwwswu4L+K7uOIGf6tMo2vm8uadRaICQ==} cpu: [ppc64] os: [linux] + libc: [glibc] '@rollup/rollup-linux-riscv64-gnu@4.39.0': resolution: {integrity: sha512-t7j5Zhr7S4bBtksT73bO6c3Qa2AV/HqiGlj9+KB3gNF5upcVkx+HLgxTm8DK4OkzsOYqbdqbLKwvGMhylJCPhQ==} cpu: [riscv64] os: [linux] + libc: [glibc] '@rollup/rollup-linux-riscv64-musl@4.39.0': resolution: {integrity: sha512-m6cwI86IvQ7M93MQ2RF5SP8tUjD39Y7rjb1qjHgYh28uAPVU8+k/xYWvxRO3/tBN2pZkSMa5RjnPuUIbrwVxeA==} cpu: [riscv64] os: [linux] + libc: [musl] '@rollup/rollup-linux-s390x-gnu@4.39.0': resolution: {integrity: sha512-iRDJd2ebMunnk2rsSBYlsptCyuINvxUfGwOUldjv5M4tpa93K8tFMeYGpNk2+Nxl+OBJnBzy2/JCscGeO507kA==} cpu: [s390x] os: [linux] + libc: [glibc] '@rollup/rollup-linux-x64-gnu@4.39.0': resolution: {integrity: sha512-t9jqYw27R6Lx0XKfEFe5vUeEJ5pF3SGIM6gTfONSMb7DuG6z6wfj2yjcoZxHg129veTqU7+wOhY6GX8wmf90dA==} cpu: [x64] os: [linux] + libc: [glibc] '@rollup/rollup-linux-x64-musl@4.39.0': resolution: {integrity: sha512-ThFdkrFDP55AIsIZDKSBWEt/JcWlCzydbZHinZ0F/r1h83qbGeenCt/G/wG2O0reuENDD2tawfAj2s8VK7Bugg==} cpu: [x64] os: [linux] + libc: [musl] '@rollup/rollup-win32-arm64-msvc@4.39.0': resolution: {integrity: sha512-jDrLm6yUtbOg2TYB3sBF3acUnAwsIksEYjLeHL+TJv9jg+TmTwdyjnDex27jqEMakNKf3RwwPahDIt7QXCSqRQ==} @@ -1910,6 +1927,9 @@ packages: json-schema-traverse@1.0.0: resolution: {integrity: sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==} + json-schema@0.4.0: + resolution: {integrity: sha512-es94M3nTIfsEPisRafak+HDLfHXnKBhV3vU5eqPcS3flIWqcxJWgXHXiey3YrpaNsanY5ei1VoYEbOzijuq9BA==} + json-stable-stringify-without-jsonify@1.0.1: resolution: {integrity: sha512-Bdboy+l7tA3OGW6FjyFHWkP5LuByj1Tk33Ljyq0axyzdk9//JSi2u3fP1QSmd1KNwq6VOKYGlAu87CisVir6Pw==} @@ -2926,6 +2946,10 @@ packages: snapshots: + '@ai-sdk/provider@3.0.7': + dependencies: + json-schema: 0.4.0 + '@alloc/quick-lru@5.2.0': {} '@babel/helper-string-parser@7.25.9': {} @@ -3003,17 +3027,19 @@ snapshots: dependencies: primevue: 4.1.0(vue@3.5.13(typescript@5.8.3)) - '@caido/quickjs-types@0.18.0': {} + '@caido/quickjs-types@0.25.4': {} - '@caido/sdk-backend@0.48.0': + '@caido/sdk-backend@0.55.2': dependencies: - '@caido/quickjs-types': 0.18.0 + '@caido/quickjs-types': 0.25.4 '@caido/sdk-shared': 0.1.1 - '@caido/sdk-frontend@0.48.0(@codemirror/state@6.5.2)(@codemirror/view@6.39.3)': + '@caido/sdk-frontend@0.55.2(@ai-sdk/provider@3.0.7)(@codemirror/state@6.5.2)(@codemirror/view@6.39.3)(vue@3.5.13(typescript@5.8.3))': dependencies: + '@ai-sdk/provider': 3.0.7 '@codemirror/state': 6.5.2 '@codemirror/view': 6.39.3 + vue: 3.5.13(typescript@5.8.3) '@caido/sdk-shared@0.1.1': {} @@ -4934,6 +4960,8 @@ snapshots: json-schema-traverse@1.0.0: {} + json-schema@0.4.0: {} + json-stable-stringify-without-jsonify@1.0.1: {} json5@1.0.2: