feat: agent mode tool-calling via Vercel AI SDK

.github/workflows/build-and-notarize.yml

@@ -54,6 +54,11 @@ jobs:
         env:
           GITHUB_TOKEN: ${{ secrets.GH_TOKEN || github.token }}
 
+      - name: Download Qdrant binary
+        run: node scripts/download-qdrant.js --current
+        env:
+          GITHUB_TOKEN: ${{ secrets.GH_TOKEN || github.token }}
+
       - name: Cache Electron
         uses: actions/cache@v4
         with:
@@ -124,6 +129,11 @@ jobs:
         env:
           GITHUB_TOKEN: ${{ secrets.GH_TOKEN || github.token }}
 
+      - name: Download Qdrant binary
+        run: node scripts/download-qdrant.js --current
+        env:
+          GITHUB_TOKEN: ${{ secrets.GH_TOKEN || github.token }}
+
       - name: Download nircmd.exe
         run: node scripts/download-nircmd.js
         env:
@@ -132,12 +142,17 @@ jobs:
       - name: Download windows-fast-paste.exe
         run: node scripts/download-windows-fast-paste.js
         env:
-          GITHUB_TOKEN: ${{ secrets.GH_TOKEN }}
+          GITHUB_TOKEN: ${{ secrets.GH_TOKEN || github.token }}
 
       - name: Download windows-key-listener.exe
         run: node scripts/download-windows-key-listener.js
         env:
-          GITHUB_TOKEN: ${{ secrets.GH_TOKEN }}
+          GITHUB_TOKEN: ${{ secrets.GH_TOKEN || github.token }}
+
+      - name: Download windows-mic-listener.exe
+        run: node scripts/download-windows-mic-listener.js
+        env:
+          GITHUB_TOKEN: ${{ secrets.GH_TOKEN || github.token }}
 
       - name: Cache Electron
         uses: actions/cache@v4
@@ -152,13 +167,16 @@ jobs:
           echo "GOOGLE_CALENDAR_CLIENT_ID=${{ secrets.GOOGLE_CALENDAR_CLIENT_ID }}" >> .env
           echo "GOOGLE_CALENDAR_CLIENT_SECRET=${{ secrets.GOOGLE_CALENDAR_CLIENT_SECRET }}" >> .env
 
-      - name: Build Application
+      - name: Build and Sign Application
         run: npm run build:win -- --publish never
         env:
           GH_TOKEN: ${{ secrets.GH_TOKEN || github.token }}
           VITE_NEON_AUTH_URL: ${{ vars.VITE_NEON_AUTH_URL }}
           VITE_OPENWHISPR_API_URL: ${{ vars.VITE_OPENWHISPR_API_URL }}
           VITE_OPENWHISPR_OAUTH_CALLBACK_URL: ${{ vars.VITE_OPENWHISPR_OAUTH_CALLBACK_URL }}
+          AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }}
+          AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }}
+          AZURE_CLIENT_SECRET: ${{ secrets.AZURE_CLIENT_SECRET }}
 
       - name: Upload Artifacts
         uses: actions/upload-artifact@v4
@@ -212,6 +230,11 @@ jobs:
         env:
           GITHUB_TOKEN: ${{ secrets.GH_TOKEN || github.token }}
 
+      - name: Download Qdrant binary
+        run: node scripts/download-qdrant.js --current --platform darwin --arch ${{ matrix.arch }}
+        env:
+          GITHUB_TOKEN: ${{ secrets.GH_TOKEN || github.token }}
+
       - name: Cache Electron
         uses: actions/cache@v4
         with:

.github/workflows/release.yml

@@ -53,6 +53,11 @@ jobs:
         env:
           GITHUB_TOKEN: ${{ secrets.GH_TOKEN }}
 
+      - name: Download Qdrant binary
+        run: node scripts/download-qdrant.js --current
+        env:
+          GITHUB_TOKEN: ${{ secrets.GH_TOKEN }}
+
       - name: Cache Electron
         uses: actions/cache@v4
         with:
@@ -116,6 +121,11 @@ jobs:
         env:
           GITHUB_TOKEN: ${{ secrets.GH_TOKEN }}
 
+      - name: Download Qdrant binary
+        run: node scripts/download-qdrant.js --current
+        env:
+          GITHUB_TOKEN: ${{ secrets.GH_TOKEN }}
+
       - name: Download nircmd.exe
         run: node scripts/download-nircmd.js
         env:
@@ -131,6 +141,11 @@ jobs:
         env:
           GITHUB_TOKEN: ${{ secrets.GH_TOKEN }}
 
+      - name: Download windows-mic-listener.exe
+        run: node scripts/download-windows-mic-listener.js
+        env:
+          GITHUB_TOKEN: ${{ secrets.GH_TOKEN }}
+
       - name: Cache Electron
         uses: actions/cache@v4
         with:
@@ -144,13 +159,16 @@ jobs:
           echo "GOOGLE_CALENDAR_CLIENT_ID=${{ secrets.GOOGLE_CALENDAR_CLIENT_ID }}" >> .env
           echo "GOOGLE_CALENDAR_CLIENT_SECRET=${{ secrets.GOOGLE_CALENDAR_CLIENT_SECRET }}" >> .env
 
-      - name: Build Application
+      - name: Build and Sign Application
         run: npm run build:win -- --publish always
         env:
           GH_TOKEN: ${{ secrets.GH_TOKEN }}
           VITE_NEON_AUTH_URL: ${{ vars.VITE_NEON_AUTH_URL }}
           VITE_OPENWHISPR_API_URL: ${{ vars.VITE_OPENWHISPR_API_URL }}
           VITE_OPENWHISPR_OAUTH_CALLBACK_URL: ${{ vars.VITE_OPENWHISPR_OAUTH_CALLBACK_URL }}
+          AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }}
+          AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }}
+          AZURE_CLIENT_SECRET: ${{ secrets.AZURE_CLIENT_SECRET }}
 
   build-macos:
     runs-on: macos-latest
@@ -197,6 +215,11 @@ jobs:
         env:
           GITHUB_TOKEN: ${{ secrets.GH_TOKEN }}
 
+      - name: Download Qdrant binary
+        run: node scripts/download-qdrant.js --current --platform darwin --arch ${{ matrix.arch }}
+        env:
+          GITHUB_TOKEN: ${{ secrets.GH_TOKEN }}
+
       - name: Cache Electron
         uses: actions/cache@v4
         with:

.gitignore

@@ -314,3 +314,4 @@ Thumbs.db
 .macos-fast-paste.hash
 .macos-fast-paste.*.hash
 .linux-fast-paste.hash
+.qdrant-initialized

CHANGELOG.md

@@ -7,6 +7,58 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+## [1.6.7] - 2026-04-02
+
+### Added
+
+- **Save Notes as Files**: Export notes to the local filesystem as Markdown files, mirroring folder hierarchy
+- **Responsive Settings Dialog**: Settings dialog adapts to narrow windows — sidebar collapses to icon rail, rows stack vertically, plan grid reflows
+- **Chat Sidebar**: Full sidebar chat tab with conversation history, cloud sync, and semantic search
+- **Chat UX Polish**: Empty state with illustration, shimmer thinking/streaming indicator, stop button, action buttons and search dialog
+- **Local Semantic Search**: Always-on Qdrant vector DB sidecar for offline semantic search across notes — hybrid FTS5 + vector with Reciprocal Rank Fusion
+- **Agent Tool Calling**: Agentic tool-calling system with note management tools (get, create, update, search), cloud agent support with NDJSON streaming, and local model tool calling with RAG context injection
+- **Embedded Chat in Notes**: Embedded chat panel in the note editor with floating and sidebar modes
+- **Per-GPU Device Selector**: Choose a specific GPU for transcription and intelligence processing (#539)
+- **Settings Keyboard Shortcut**: Cmd+, / Ctrl+, keyboard shortcut to open Settings
+- **Notes Actions Button**: Actions sidebar button with redesigned action editor dialog
+- **Notes Folder Picker**: Folder picker in the note metadata row with cleaned-up input styles
+- **Notes Sidebar Buttons**: New note and search notes buttons in the sidebar
+- **Meeting Echo Cancellation**: Echo cancellation on mic input and note metadata chips in meeting view
+- **Linux Wrapper Script**: Wrapper script to force XWayland and support user flags (#507)
+
+### Changed
+
+- **Vercel AI SDK Migration**: Agent mode migrated from raw API calls to Vercel AI SDK
+- **Notes Bottom Bar Redesign**: Redesigned bottom bar with compact action picker
+- **Dialog Design System Alignment**: All dialogs aligned with design system guidelines
+- **Removed Note Word Count**: Removed word count from note editor
+- **Cloud Agent Streaming**: Stream cloud agent responses directly from the renderer via IPC
+
+### Fixed
+
+- **Meeting Auto-Detection**: Fix auto-detection not firing for browser meetings
+- **Meeting Transcription Provider**: Use local transcription provider for notes/meeting recording (#530)
+- **Meeting Partial Transcript Spam**: Prevent partial transcript spam and duplicate final segments
+- **Meeting Notification Timing**: Resolve notification popup timing and detection lifecycle bugs
+- **Folder/Note Race Conditions**: Resolve race conditions when switching folders quickly, prevent meeting view from exiting when changing folder, fix rapid delete/switch state management
+- **Clipboard Preservation**: Preserve images and HTML in clipboard during paste-and-restore (#381)
+- **Transcription Retry Provider**: Retry transcription uses configured provider instead of forcing Parakeet
+- **JSON Parse Validation**: Validate JSON.parse result type before calling .replace() in prompts (#541)
+- **GPU Selector Polish**: Address code review feedback, rename Intelligence GPU label, fix dropdown chevron padding (#539)
+- **Chat Duplicate Conversations**: Fix duplicate conversations — includeArchived filter returned all instead of only archived
+- **Linux Wayland Fixes**: Force XWayland on KDE/GNOME Wayland, fix hotkey startup race; use uinput before portal on GNOME Wayland (#468, #494)
+- **Mic Permission Gate**: Remove mic permission gate, fix system audio detection
+- **Windows Build Signing**: Fix Windows build signing on PRs, add missing mic-listener download, add missing publisherName to Azure signing config
+- **Dead optimizeAudio Crash**: Remove dead optimizeAudio call that crashes on recordings over 90 seconds (#524)
+- **Download URL Logging**: Remove URL truncation from download log and add failure logging (#540)
+
+### Security
+
+- **picomatch**: Bump to 4.0.4
+- **brace-expansion**: Bump to 1.1.13 (security backport)
+- **yaml**: Bump to 2.8.3
+- **tar**: Bump to 7.5.13
+
 ## [1.6.6] - 2026-03-19
 
 ### Added

CLAUDE.md

@@ -104,6 +104,9 @@ OpenWhispr is an Electron-based desktop dictation application that uses whisper.
 - **whisper.js**: Local whisper.cpp integration and model management
 - **parakeet.js**: NVIDIA Parakeet model management via sherpa-onnx
 - **parakeetServer.js**: sherpa-onnx CLI wrapper for transcription
+- **qdrantManager.js**: Qdrant vector DB sidecar process lifecycle (spawn, health check, shutdown)
+- **localEmbeddings.js**: Local text embedding via ONNX Runtime + all-MiniLM-L6-v2 (384-dim vectors)
+- **vectorIndex.js**: Qdrant collection management — upsert, delete, search, batch reindex
 - **windowConfig.js**: Centralized window configuration
 - **windowManager.js**: Window creation and lifecycle management
 
@@ -161,6 +164,32 @@ OpenWhispr is an Electron-based desktop dictation application that uses whisper.
 
 - **Download URLs**: Models from sherpa-onnx ASR models release on GitHub
 
+### Local Semantic Search (Qdrant + MiniLM)
+
+Always-on offline semantic search that finds notes by meaning, not just keywords. Used by the AI agent's `search_notes` tool. Qdrant starts automatically on app launch; embedding model auto-downloads on first run if missing.
+
+**Architecture**:
+- **Qdrant sidecar**: Rust binary spawned as child process (`qdrantManager.js`), port 6333–6350
+- **Embedding model**: `all-MiniLM-L6-v2` via ONNX Runtime (`localEmbeddings.js`), 384-dim vectors
+- **Vector index**: Qdrant collection management (`vectorIndex.js`), cosine distance
+- **Hybrid search**: FTS5 + Qdrant in parallel → Reciprocal Rank Fusion (K=60) with 0.3 cosine score threshold
+
+**Pipeline**:
+1. App launches → Qdrant binary starts → collection created. Embedding model auto-downloads if missing (~22MB)
+2. Note create/update/delete → SQLite write → background vector upsert/delete via `_asyncVectorUpsert()`/`_asyncVectorDelete()`
+3. Agent searches → `db-semantic-search-notes` IPC → parallel FTS5 + vector search → RRF merge → ranked results
+
+**Search fallback chain** (in `searchNotesTool.ts`): cloud search → local semantic → FTS5 keyword
+
+**Storage**:
+- Qdrant data: `~/.cache/openwhispr/qdrant-data/`
+- Qdrant binary: `resources/bin/qdrant-{platform}-{arch}` (bundled — downloaded during `prebuild` / `predev`)
+- Embedding model: `~/.cache/openwhispr/embedding-models/all-MiniLM-L6-v2/` (auto-downloaded on first launch)
+
+**Dependencies**: `@qdrant/js-client-rest`, `onnxruntime-node`
+
+**Dev setup**: The Qdrant binary downloads automatically via `predev`/`prestart`. The embedding model auto-downloads on first app launch. To manually download: `npm run download:qdrant` and `npm run download:embedding-model`.
+
 ### Build Scripts (scripts/)
 
 - **download-whisper-cpp.js**: Downloads whisper.cpp binaries from GitHub releases
@@ -169,6 +198,8 @@ OpenWhispr is an Electron-based desktop dictation application that uses whisper.
 - **download-windows-key-listener.js**: Downloads prebuilt Windows key listener binary
 - **download-windows-mic-listener.js**: Downloads prebuilt Windows mic listener binary
 - **download-sherpa-onnx.js**: Downloads sherpa-onnx binaries for Parakeet support
+- **download-qdrant.js**: Downloads Qdrant vector DB binary for local semantic search
+- **download-minilm.js**: Downloads all-MiniLM-L6-v2 ONNX model + tokenizer for local embeddings
 - **build-globe-listener.js**: Compiles macOS Globe key listener from Swift source
 - **build-macos-mic-listener.js**: Compiles macOS mic listener from Swift source
 - **build-windows-key-listener.js**: Compiles Windows key listener (for local development)
@@ -230,9 +261,9 @@ CREATE TABLE transcriptions (
 Settings stored in localStorage with these keys:
 - `whisperModel`: Selected Whisper model
 - `useLocalWhisper`: Boolean for local vs cloud
-- `openaiApiKey`: Encrypted API key
-- `anthropicApiKey`: Encrypted API key
-- `geminiApiKey`: Encrypted API key
+- `openaiApiKey`: API key (plaintext)
+- `anthropicApiKey`: API key (plaintext)
+- `geminiApiKey`: API key (plaintext)
 - `language`: Selected language code
 - `agentName`: User's custom agent name
 - `reasoningModel`: Selected AI model for processing
@@ -520,7 +551,8 @@ const { t } = useTranslation();
 2. **New Setting**: Update useSettings.ts and SettingsPage.tsx
 3. **New UI Component**: Follow shadcn/ui patterns in src/components/ui
 4. **New Manager**: Create in src/helpers/, initialize in main.js
-5. **New UI Strings**: Add translation keys to all 9 language files (see i18n section above)
+5. **New UI Strings**: Add translation keys to all 10 language files (see i18n section above)
+6. **New Sidecar Binary**: Add download script in `scripts/`, add to `prebuild*` scripts in package.json, add manager in `src/helpers/`, initialize in `main.js`, shutdown in `will-quit` handler
 
 ### Testing Checklist
 
@@ -539,6 +571,9 @@ const { t } = useTranslation();
 - [ ] Verify meeting detection works with event-driven mode (check debug logs for "event-driven")
 - [ ] Test meeting notification suppression during recording
 - [ ] Test post-recording cooldown (notifications shouldn't flash immediately)
+- [ ] Create a note about "quarterly revenue projections", search via agent for "financial forecast" — should match semantically
+- [ ] Verify Qdrant starts on app launch (check debug logs for "qdrant started successfully")
+- [ ] Kill Qdrant process manually — verify FTS5 keyword search still works as fallback
 
 ### Common Issues and Solutions
 
@@ -583,6 +618,14 @@ const { t } = useTranslation();
    - Linux: Verify `pactl` is installed (`pulseaudio-utils` or `pipewire-pulse` package)
    - If event-driven binary is missing, detection falls back to polling automatically
 
+7. **Local Semantic Search Not Working**:
+   - Qdrant binary should be in `resources/bin/qdrant-{platform}-{arch}` (auto-downloaded during `predev`/`prebuild`)
+   - Embedding model should be in `~/.cache/openwhispr/embedding-models/all-MiniLM-L6-v2/model.onnx` (auto-downloaded on first app launch)
+   - Run `npm run download:qdrant` and `npm run download:embedding-model` manually if missing
+   - Check debug logs for "qdrant" entries (port, health check, errors)
+   - If Qdrant fails to start, search still works via FTS5 keyword fallback
+   - Semantic search is only available through the AI agent's `search_notes` tool, not the manual search UI
+
 ### Platform-Specific Notes
 
 **macOS**:
@@ -650,7 +693,7 @@ const { t } = useTranslation();
 
 ## Security Considerations
 
-- API keys stored in system keychain when possible
+- API keys stored in plaintext in userData `.env` and localStorage (not yet encrypted — see #532)
 - Context isolation enabled
 - No remote code execution
 - Sanitized file paths

README.md

@@ -45,6 +45,7 @@ This project is licensed under the MIT License - see the [LICENSE](LICENSE) file
   - **Google**: Gemini 3.1 Pro, Gemini 3 Flash, Gemini 2.5 Flash Lite
   - **Groq**: Ultra-fast inference with Llama and Mixtral models
   - **Local**: Qwen, LLaMA, Mistral, Gemma models via llama.cpp
+- 🛠️ **Agent Tool Calling**: Agentic tools for searching, creating, and updating notes — with RAG context injection for local models
 - 🤖 **AI Actions**: Apply AI-powered actions to notes with customizable processing templates
 
 ### Meeting Transcription
@@ -59,8 +60,11 @@ This project is licensed under the MIT License - see the [LICENSE](LICENSE) file
 
 - 📝 **Notes System**: Create, edit, and organize notes with folders, audio upload, and real-time dictation
 - 🔎 **Full-Text Search**: FTS5-powered search across all note content with Cmd+K command palette
+- 🧠 **Local Semantic Search**: Always-on Qdrant vector DB with MiniLM embeddings for offline meaning-based search across notes
 - ☁️ **Cloud Sync**: Local-first storage with cloud backup and semantic search
+- 💬 **Embedded Chat**: Chat panel in the note editor with floating and sidebar modes
 - 📁 **Folder Organization**: Organize notes into custom folders with drag-and-drop
+- 💾 **Save Notes as Files**: Export notes to the local filesystem as Markdown files, mirroring your folder hierarchy
 
 ### Cloud & Account
 
@@ -73,6 +77,7 @@ This project is licensed under the MIT License - see the [LICENSE](LICENSE) file
 
 - 🌐 **Cross-Platform**: Works on macOS, Windows, and Linux
 - 🎨 **Modern UI**: Built with React 19, TypeScript, and Tailwind CSS v4
+- 📱 **Responsive Settings**: Settings dialog adapts to narrow windows with collapsible icon-rail sidebar and stacked layouts
 - 🖱️ **Draggable Interface**: Move the dictation panel anywhere on your screen with configurable start position
 - 🌐 **Globe Key Toggle (macOS)**: Optional Fn/Globe key listener for a hardware-level dictation trigger — no Input Monitoring permission required
 - ⌨️ **Compound Hotkeys**: Support for multi-key combinations like `Cmd+Shift+K`
@@ -849,7 +854,7 @@ A: No. As of v1.6.0, OpenWhispr uses NSEvent monitors instead of CGEvent taps, e
 
 ## Project Status
 
-OpenWhispr is actively maintained and ready for production use. Current version: 1.6.6
+OpenWhispr is actively maintained and ready for production use. Current version: 1.6.7
 
 - ✅ Core dictation with local and cloud processing
 - ✅ Cross-platform support (macOS, Windows, Linux)

SECURITY.md

@@ -39,8 +39,11 @@ Out of scope:
 - **Local-first audio processing** — Audio is transcribed on-device using
   whisper.cpp or nvidia parakeet. Recordings are not sent to external servers unless explicitly
   configured by the user.
-- **Credential storage** — Sensitive credentials are stored using Electron's
-  `safeStorage` API where available.
+- **Credential storage** — API keys provided by users (BYOK) are stored in
+  plaintext in the app's `userData` directory (`.env` file and Electron
+  `localStorage`). They are readable by any process running as the current OS
+  user. Migrating to Electron's `safeStorage` API for platform-native
+  encryption is tracked in [#532](https://github.com/OpenWhispr/openwhispr/issues/532).
 - **Native binaries** — Platform-specific helpers (key listeners, paste
   utilities) are compiled from source during the build process.
 - **Context isolation** — The Electron renderer runs with context isolation