diff --git a/.dockerignore b/.dockerignore
index f369e1e7..10ec6b5a 100644
--- a/.dockerignore
+++ b/.dockerignore
@@ -1,33 +1,8 @@
-# Ignore secure files (permission issues)
-backend/secure_files
-backend/data
-backend/uploads
-backend/logs
-
-# Git
+node_modules
+npm-debug.log
+release
.git
.gitignore
-
-# Node modules
-**/node_modules
-
-# Build artifacts
-**/dist
-**/*.log
-
-# IDE
-.vscode
-.idea
-
-# Env files
-.env
-.env.local
-
-# Docker
-docker-compose*.yml
-!docker-compose.yml
-
-# Docs
-docs/
*.md
-!README.md
+.env
+.env.*
diff --git a/.env.default b/.env.default
deleted file mode 100644
index 98b34e1c..00000000
--- a/.env.default
+++ /dev/null
@@ -1,150 +0,0 @@
-# ============================================================================
-# ClaraVerse Environment Configuration - DEFAULT TEMPLATE
-# ============================================================================
-# This file contains sensible defaults for quick setup.
-# The quickstart.sh/quickstart.bat scripts will:
-# 1. Copy this file to .env
-# 2. Auto-generate secure encryption keys
-# 3. You're ready to go!
-#
-# For manual setup:
-# 1. Copy this file: cp .env.default .env
-# 2. Generate keys: openssl rand -hex 32 (for ENCRYPTION_MASTER_KEY)
-# openssl rand -hex 64 (for JWT_SECRET)
-# 3. Replace the placeholder values below
-# 4. Run: docker compose up -d
-#
-# For advanced configuration, see .env.example for detailed documentation
-# ============================================================================
-
-# ================================
-# AUTO-GENERATED KEYS
-# ================================
-# These will be automatically generated by quickstart scripts
-# DO NOT set these manually unless you know what you're doing
-
-# Encryption key for user data (conversations, credentials)
-# WARNING: Losing this key = losing access to encrypted data!
-ENCRYPTION_MASTER_KEY=auto-generated-on-first-run
-
-# JWT secret for authentication tokens
-# Used to sign and verify JWT tokens
-JWT_SECRET=auto-generated-on-first-run
-
-# ================================
-# ENVIRONMENT MODE
-# ================================
-# Options: development, production
-ENVIRONMENT=development
-
-# ================================
-# FRONTEND URLs (for building)
-# ================================
-# Development defaults - work out-of-box for local Docker setup
-VITE_API_BASE_URL=http://localhost:3001
-VITE_WS_URL=ws://localhost:3001
-VITE_APP_NAME=ClaraVerse
-VITE_APP_VERSION=2.0.0
-VITE_ENABLE_ANALYTICS=false
-
-# ================================
-# BACKEND CONFIGURATION
-# ================================
-# CORS: Comma-separated allowed origins
-# These defaults work for local development
-ALLOWED_ORIGINS=http://localhost,http://localhost:80,http://localhost:5173,http://localhost:5174,http://localhost:3000,http://localhost:8080
-
-# Frontend URL (for payment redirects)
-FRONTEND_URL=http://localhost:80
-
-# Backend public URL (for generating download URLs)
-BACKEND_URL=http://localhost:3001
-
-# ================================
-# DATABASE CONFIGURATION
-# ================================
-# MySQL - for providers, models, and capabilities
-MYSQL_ROOT_PASSWORD=claraverse_root_2024
-MYSQL_PASSWORD=claraverse_pass_2024
-
-# MongoDB - for conversations, workflows, and data persistence
-MONGODB_URI=mongodb://mongodb:27017/claraverse
-
-# Redis - for job scheduling and WebSocket pub/sub
-REDIS_URL=redis://redis:6379
-
-# ================================
-# SEARCH ENGINE
-# ================================
-# SearXNG (local search engine, no API key needed)
-SEARXNG_URLS=http://searxng:8080
-
-# ================================
-# CODE EXECUTION (E2B)
-# ================================
-# E2B Local Mode - No API key needed!
-# This runs code execution locally using Docker
-E2B_MODE=local
-E2B_LOCAL_USE_DOCKER=true
-E2B_SANDBOX_POOL_SIZE=3
-E2B_EXECUTION_TIMEOUT=30000
-E2B_RATE_LIMIT_PER_MIN=20
-
-# ================================
-# OPTIONAL SERVICES
-# ================================
-# These are optional - leave empty to skip
-# You can add these later in Settings UI
-
-# Supabase Authentication (optional - for production auth)
-# Get from: https://app.supabase.com/project/_/settings/api
-VITE_SUPABASE_URL=
-VITE_SUPABASE_ANON_KEY=
-SUPABASE_URL=
-SUPABASE_KEY=
-
-# E2B Cloud Mode (optional - if you have API key)
-# Get from: https://e2b.dev/dashboard
-# Comment out E2B_MODE=local above and uncomment these:
-# E2B_MODE=production
-# E2B_API_KEY=
-
-# Composio Integrations (optional)
-# Get from: https://app.composio.dev
-COMPOSIO_API_KEY=
-COMPOSIO_GOOGLESHEETS_AUTH_CONFIG_ID=
-COMPOSIO_GMAIL_AUTH_CONFIG_ID=
-
-# Cloudflare Turnstile (optional - for bot protection)
-# Get from: https://dash.cloudflare.com/turnstile
-VITE_TURNSTILE_SITE_KEY=
-
-# ================================
-# ADMIN CONFIGURATION
-# ================================
-# Comma-separated list of user IDs with superadmin access
-# Leave empty - first registered user becomes admin automatically
-SUPERADMIN_USER_IDS=
-
-# Development API key for testing (only works when ENVIRONMENT != production)
-DEV_API_KEY=claraverse-dev-key-2024
-
-# JWT Token Expiry
-JWT_ACCESS_TOKEN_EXPIRY=15m
-JWT_REFRESH_TOKEN_EXPIRY=168h
-
-# ================================
-# RATE LIMITING (requests per minute)
-# ================================
-# Protect your instance from abuse with these sensible defaults
-RATE_LIMIT_GLOBAL_API=200
-RATE_LIMIT_PUBLIC_READ=120
-RATE_LIMIT_AUTHENTICATED=60
-RATE_LIMIT_WEBSOCKET=20
-RATE_LIMIT_IMAGE_PROXY=60
-
-# ================================
-# DOCKER BUILD CONFIGURATION
-# ================================
-# Skip tests during Docker build for faster builds
-SKIP_TESTS=false
diff --git a/.env.example b/.env.example
index 45de836e..a198cc96 100644
--- a/.env.example
+++ b/.env.example
@@ -1,127 +1,6 @@
-# ============================================================================
-# ClaraVerse Environment Configuration
-# ============================================================================
-# This is the SINGLE SOURCE OF TRUTH for Docker Compose deployments.
-# Docker Compose automatically reads this file.
-#
-# Setup Instructions:
-# 1. Copy this file: cp .env.example .env
-# 2. Fill in required values (marked with TODO)
-# 3. Run: ./dev-docker.sh up
-#
-# For production:
-# 1. Copy this file to your server
-# 2. Update values for production
-# 3. Run: docker-compose -f docker-compose.yml -f docker-compose.production.yml up -d --build
-# ============================================================================
+# Supabase Configuration
+VITE_SUPABASE_URL=your_supabase_url_here
+VITE_SUPABASE_ANON_KEY=your_anon_key_here
-# ================================
-# ENVIRONMENT MODE
-# ================================
-# Options: development, production
-ENVIRONMENT=development
-
-# ================================
-# FRONTEND URLs (for building)
-# ================================
-# Development: http://localhost:3001 / ws://localhost:3001
-# Production: https://api.yourdomain.com / wss://api.yourdomain.com
-VITE_API_BASE_URL=http://localhost:3001
-VITE_WS_URL=ws://localhost:3001
-
-# ================================
-# SUPABASE AUTHENTICATION
-# ================================
-# TODO: Get these from https://app.supabase.com/project/_/settings/api
-# Required for user authentication (optional for development)
-VITE_SUPABASE_URL=
-VITE_SUPABASE_ANON_KEY=
-
-# Backend (same values as above)
-SUPABASE_URL=
-SUPABASE_KEY=
-
-# ================================
-# BACKEND CONFIGURATION
-# ================================
-# CORS: Comma-separated allowed origins
-# Development: http://localhost,http://localhost:5173,http://localhost:5174
-# Production: https://yourdomain.com
-ALLOWED_ORIGINS=http://localhost,http://localhost:5173,http://localhost:5174,http://localhost:3000,http://localhost:8080
-
-# Encryption key for user data (MongoDB conversations, credentials)
-# TODO: Generate with: openssl rand -hex 32
-# WARNING: Losing this key = losing access to encrypted data!
-ENCRYPTION_MASTER_KEY=
-
-# Frontend URL (for payment redirects)
-FRONTEND_URL=http://localhost:5173
-
-# Backend public URL (for generating download URLs)
-BACKEND_URL=http://localhost:3001
-
-# Cloudflare Turnstile (optional - for bot protection)
-VITE_TURNSTILE_SITE_KEY=
-
-# ================================
-# MYSQL DATABASE
-# ================================
-# MySQL root password (default: claraverse_root_2024)
-MYSQL_ROOT_PASSWORD=claraverse_root_2024
-
-# MySQL user password (default: claraverse_pass_2024)
-MYSQL_PASSWORD=claraverse_pass_2024
-
-# ================================
-# MONGODB & REDIS
-# ================================
-# MongoDB - for conversations, workflows, and data persistence
-# Docker: mongodb://mongodb:27017/claraverse
-# Atlas: mongodb+srv://user:pass@cluster.mongodb.net/claraverse
-# Self-host: mongodb://user:pass@host:27017/claraverse
-MONGODB_URI=mongodb://mongodb:27017/claraverse
-
-# Redis - for job scheduling and WebSocket pub/sub
-REDIS_URL=redis://redis:6379
-
-# ================================
-# EXTERNAL SERVICES
-# ================================
-# E2B Code Interpreter (optional - for code execution)
-# Get API key from: https://e2b.dev/dashboard
-E2B_API_KEY=
-
-# SearXNG (internal Docker URL, or comma-separated for load balancing)
-SEARXNG_URLS=http://searxng:8080
-
-# Composio (optional - for integrations)
-COMPOSIO_API_KEY=
-COMPOSIO_GOOGLESHEETS_AUTH_CONFIG_ID=
-COMPOSIO_GMAIL_AUTH_CONFIG_ID=
-
-
-
-# ================================
-# ADMIN (Optional)
-# ================================
-# Comma-separated list of Supabase user IDs with superadmin access
-# Example: user-id-1,user-id-2,user-id-3
-SUPERADMIN_USER_IDS=
-
-# ================================
-# RATE LIMITING (requests per minute)
-# ================================
-# Global API rate limit (all /api/* routes)
-RATE_LIMIT_GLOBAL_API=200
-
-# Public read-only endpoints
-RATE_LIMIT_PUBLIC_READ=120
-
-# Authenticated user requests
-RATE_LIMIT_AUTHENTICATED=60
-
-# WebSocket connections
-RATE_LIMIT_WEBSOCKET=20
-
-# Image proxy (to prevent bandwidth abuse)
-RATE_LIMIT_IMAGE_PROXY=60
+# Only for server-side usage, do not use in client-side code
+VITE_SUPABASE_SERVICE_KEY=your_service_key_here_server_only
diff --git a/.gitattributes b/.gitattributes
deleted file mode 100644
index 6a0223c6..00000000
--- a/.gitattributes
+++ /dev/null
@@ -1,44 +0,0 @@
-# Auto-detect text files and perform LF normalization
-* text=auto
-
-# Shell scripts should always use LF (Unix-style)
-*.sh text eol=lf
-
-# Windows batch files should always use CRLF (Windows-style)
-*.bat text eol=crlf
-
-# Docker files should always use LF
-Dockerfile* text eol=lf
-docker-compose*.yml text eol=lf
-.dockerignore text eol=lf
-
-# Configuration files
-*.yml text eol=lf
-*.yaml text eol=lf
-*.json text eol=lf
-*.toml text eol=lf
-
-# Source code
-*.ts text eol=lf
-*.tsx text eol=lf
-*.js text eol=lf
-*.jsx text eol=lf
-*.go text eol=lf
-*.py text eol=lf
-
-# Documentation
-*.md text eol=lf
-
-# Environment files
-.env* text eol=lf
-
-# Binary files
-*.png binary
-*.jpg binary
-*.jpeg binary
-*.gif binary
-*.ico binary
-*.webp binary
-*.pdf binary
-*.woff binary
-*.woff2 binary
diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml
new file mode 100644
index 00000000..ac9aae77
--- /dev/null
+++ b/.github/FUNDING.yml
@@ -0,0 +1,15 @@
+# These are supported funding model platforms
+
+github:
+patreon: # Replace with a single Patreon username
+open_collective: # Replace with a single Open Collective username
+ko_fi: # Replace with a single Ko-fi username
+tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel
+community_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry
+liberapay: # Replace with a single Liberapay username
+issuehunt: # Replace with a single IssueHunt username
+lfx_crowdfunding: # Replace with a single LFX Crowdfunding project-name e.g., cloud-foundry
+polar: # Replace with a single Polar username
+buy_me_a_coffee: claraverse
+thanks_dev: # Replace with a single thanks.dev username
+custom: # Replace with up to 4 custom sponsorship URLs e.g., ['link1', 'link2']
diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml
new file mode 100644
index 00000000..d9ad1aac
--- /dev/null
+++ b/.github/workflows/docker-publish.yml
@@ -0,0 +1,60 @@
+name: Build and Publish Docker Image
+
+on:
+ push:
+ tags:
+ - 'v*'
+ workflow_dispatch:
+
+env:
+ REGISTRY: docker.io
+ IMAGE_NAME: clara17verse/clara-backend
+ DOCKER_USERNAME: clara17verse
+ # The token should be stored as a GitHub secret, not directly in the workflow
+
+jobs:
+ build-and-push:
+ runs-on: ubuntu-latest
+ environment: production
+ permissions:
+ contents: read
+ packages: write
+
+ steps:
+ - name: Checkout repository
+ uses: actions/checkout@v4
+
+ - name: Set up QEMU
+ uses: docker/setup-qemu-action@v3
+
+ - name: Set up Docker Buildx
+ uses: docker/setup-buildx-action@v3
+
+ - name: Log in to Docker Hub
+ uses: docker/login-action@v3
+ with:
+ username: ${{ env.DOCKER_USERNAME }}
+ password: ${{ secrets.DOCKER_PAT }}
+
+ - name: Extract metadata (tags, labels)
+ id: meta
+ uses: docker/metadata-action@v5
+ with:
+ images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+ tags: |
+ type=semver,pattern={{version}}
+ type=semver,pattern={{major}}.{{minor}}
+ type=sha,prefix=
+ type=raw,value=latest,enable={{is_default_branch}}
+
+ - name: Build and push Docker image
+ uses: docker/build-push-action@v5
+ with:
+ context: ./py_backend
+ file: ./py_backend/Dockerfile
+ platforms: linux/amd64,linux/arm64
+ push: true
+ tags: ${{ steps.meta.outputs.tags }}
+ labels: ${{ steps.meta.outputs.labels }}
+ cache-from: type=gha
+ cache-to: type=gha,mode=max
\ No newline at end of file
diff --git a/.gitignore b/.gitignore
index 60df40a6..73d46109 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,38 +1,160 @@
-frontend/.env
+# Logs
+logs
+*.log
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*
+pnpm-debug.log*
+lerna-debug.log*
+
+node_modules
+dist
+dist-ssr
+*.local
+
+# Editor directories and files
+.vscode/*
+!.vscode/extensions.json
+.idea
+.DS_Store
+*.suo
+*.ntvs*
+*.njsproj
+*.sln
+*.sw?
.env
-*.csv
-*.xlsx
-*.xls
-*.pdf
-*.docx
-*.doc
-*.pptx
-*.ppt
-*.odt
-*.odp
-
-# Ignore txt files except requirements.txt
-*.txt
-!requirements.txt
-
-# Ignore markdown except README
-*.md
-!README.md
-
-# Ignore JSON files except essential ones
-*.json
-!package.json
-!package-lock.json
-!tsconfig.json
-!tsconfig.*.json
-!eslint.config.json
-!**/fixtures/**/*.json
-!providers.example.json
-.claude/settings.local.json
-__pycache__/
-frontend/.claude/settings.local.json
-comfyui
-_bmad
-_bmad*
-.agentvibes
-.claude
\ No newline at end of file
+.env.*
+!.env.example
+/tools/.env
+
+# Security files
+secret_key.txt
+*_key.txt
+*.pem
+*.key
+
+# Electron specific
+release
+.electron-builder
+electron-builder.yml
+*.dmg
+*.exe
+*.deb
+*.AppImage
+*.snap
+*.blockmap
+*.zip
+latest-mac.yml
+latest-linux.yml
+latest-win.yml
+.bolt
+__pycache__
+
+
+# Code signing files
+package-mac-signed.sh
+certificate.p12
+
+./Support
+python-env
+.clara
+
+./py_backend/__pycache__
+./py_backend/cache
+./py_backend/python-env
+./py_backend/python-env/cache
+./py_backend/python-env/pip-cache
+./py_backend/python-env/python-env
+
+.cursor
+
+# any files in tools
+tools/*
+
+/clara_interpreter_dockerstuff
+
+electron/llamacpp-binaries/models/*
+
+testing_sdk/*
+
+sdk/node_modules/*
+
+clara-sdk-docs/*
+
+electron/llamacpp-binaries/config.yaml
+
+sdk_examples/*
+electron/llamacpp-binaries/config.yaml
+battery-report.html
+electron/llamacpp-binaries/config.yaml
+landing/*
+issues.md
+# Local Netlify folder
+.netlify
+
+# LLAMACPP BINARIES - Exclude only large binary files from platform directories
+# Keep directory structure and config files, exclude only binaries
+
+# Exclude binary files from specific platform directories
+electron/llamacpp-binaries/darwin-arm64/*.dylib
+# electron/llamacpp-binaries/darwin-arm64/llama-*
+electron/llamacpp-binaries/darwin-arm64/lib*
+electron/llamacpp-binaries/darwin-arm64/*-server
+electron/llamacpp-binaries/darwin-arm64/ggml-*
+
+electron/llamacpp-binaries/linux-x64/*.so
+# electron/llamacpp-binaries/linux-x64/llama-*
+electron/llamacpp-binaries/linux-x64/lib*
+electron/llamacpp-binaries/linux-x64/*-server
+electron/llamacpp-binaries/linux-x64/ggml-*
+electron/llamacpp-binaries/linux-x64/rpc-server
+electron/llamacpp-binaries/linux-x64/vulkan-shaders-gen
+
+# electron/llamacpp-binaries/win32-x64/*.dll
+# # electron/llamacpp-binaries/win32-x64/llama-*
+# electron/llamacpp-binaries/win32-x64/lib*
+# electron/llamacpp-binaries/win32-x64/*-server
+# electron/llamacpp-binaries/win32-x64/ggml-*
+
+# Keep config files, documentation, and scripts
+!electron/llamacpp-binaries/*.md
+!electron/llamacpp-binaries/*.json
+!electron/llamacpp-binaries/*.yaml
+!electron/llamacpp-binaries/*.sh
+!electron/llamacpp-binaries/LICENSE*
+py_backend/cache/*
+py_backend/.venv/*
+py_backend/light_rag_examples/*
+electron/llamacpp-binaries/darwin-arm64-backup*
+notarization.env
+# anything with .env in the name
+.env
+scripts/notarize-production.sh
+notarize-production.sh
+setup-notarization-env.sh
+mcp_workspace/*
+electron/llamacpp-binaries/*
+
+.bmad-core
+.claude
+.github
+.vscode
+web-bundles
+docs-internal-agents
+searXNG-Clara
+searxng-config
+.bmad-infrastructure-devops
+clara-mcp/mcp_workspace/*
+llama-swap/*
+dev_docs/*
+config.yaml
+model_folders.json
+settings.json
+/binaries/*
+downloads/*
+config.yaml.backup.*
+electron/claracore/progress_state.*
+package-lock.json
+
+# Agent deployment container testing
+test_agent_container/
diff --git a/CHANGELOG.md b/CHANGELOG.md
new file mode 100644
index 00000000..636d7b1c
--- /dev/null
+++ b/CHANGELOG.md
@@ -0,0 +1,290 @@
+# Changelog
+
+All notable changes to Clara will be documented in this file.
+
+The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
+and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+
+## [0.1.4] - 2025-08-28
+
+### 🚀 Major Features Added
+
+#### 🔹 Agents
+- **MCP Autonomy**: Agents can now use MCP to operate fully autonomously — automate almost any PC task (except Paint, Excel, Word for now)
+- **Speech Capabilities**: Agents can talk once tasks are done for more natural interactions
+- **Scheduled Automation**: Automate with time. Example: Agents that log in/out of apps daily — works ~90% of the time on first try, retries if needed
+
+#### 🔹 RAG (Retrieval-Augmented Generation)
+- **Improved Performance**: Enhanced RAG engine with better retrieval accuracy
+- **Redesigned UI**: Clean, intuitive interface for better user experience
+- **Task Histories**: Easier tracking of previous operations and results
+- **Document Retry**: Graceful error handling for failed document processing
+
+#### 🔹 Chat
+- **Redesigned Chat Input**: Simpler interface with no need for advanced MCP settings
+- **Voice UX Improvements**: Smoother voice interactions and better speech recognition
+- **Clara Memories Enhancements**: More efficient and user-friendly memory management
+
+#### 🔹 Tasks
+- **Automate & Monitor**: Run tasks in the background, schedule them, monitor progress, and collect results
+- **Background Execution**: Set-and-forget task automation
+- **Progress Tracking**: Real-time monitoring of task execution
+- **Results Collection**: Automated gathering and organization of task outputs
+
+### ✨ What You Can Try Today
+- Set up an agent to auto-login to apps every morning
+- Use voice-enabled chat to summarize your daily reports
+- Schedule a workflow to clean up folders or back up files nightly
+- Build a RAG-powered assistant to search across your company docs
+- Automate email checks or notifications without relying on cloud services
+
+### 🛠️ Technical Improvements
+- Enhanced automation reliability and retry mechanisms
+- Improved voice processing and speech synthesis
+- Better memory management for Clara Memories
+- Optimized task scheduling and background processing
+
+### 🎯 Release Focus
+This release is about making automation practical, reliable, and personal — right from your desktop. No cloud dependencies, full privacy control, and enterprise-grade task automation capabilities.
+
+---
+
+## [Unreleased] - 0.1.5
+
+### 🚧 In Development
+- Multi-user support with authentication
+- Cloud deployment templates for AWS/GCP/Azure
+- Plugin system for custom AI tools
+- Enhanced N8N workflow integration
+
+---
+
+## [0.1.3] - 2024-12-22
+
+### 🎉 Major Milestone: Complete Docker Transformation
+ClaraVerse has undergone a **revolutionary transformation** from an Electron desktop application to a **Docker-powered web application** similar to OpenWebUI, while maintaining complete privacy and local execution.
+
+### 🚀 Major Features Added
+- **🐳 Docker-First Architecture**: Complete Docker Compose setup with 7 integrated services
+- **🔧 LumaUI: Complete Web Development Environment**: WebContainer integration with Monaco Editor
+- **🧠 Enhanced AI Capabilities**: Dynamic token allocation (16k-128k) with autonomous execution
+- **🎨 Advanced Preview System**: Dual preview modes with console integration
+
+### ✨ New Features
+- **Docker Services Stack**: Clara Web UI, Backend API, LlamaSwap, N8N, ComfyUI, Redis, PostgreSQL
+- **LumaUI Development**: Project templates, AI code generation, terminal integration, file management
+- **Smart AI Integration**: Precision editing modes, tool call limits, error recovery
+
+### 🛠️ Technical Improvements
+- **Performance & Reliability**: Fixed race conditions, optimized token usage, enhanced error handling
+- **Developer Experience**: TypeScript integration, hot reload, debugging tools
+- **Architecture Enhancements**: Service isolation, health monitoring, scalability
+
+### 🐛 Critical Bug Fixes
+- **Docker Conversion**: Removed Electron dependencies, fixed console errors
+- **LumaUI Stability**: Fixed WebContainer remounting, auto mode loops, file sync issues
+- **AI Integration**: Fixed tool schema validation, token limits, conversation history
+
+### 🔧 Breaking Changes
+- **Migration from Electron to Docker**: Desktop app discontinued, now web application
+- **New Installation**: Use Docker Compose instead of app installers
+- **LumaUI Interface**: WebContainer-based projects, enhanced AI chat, auto-save behavior
+
+---
+
+## [0.1.2] - 2024-05-30
+
+### 🚀 Major Features Added
+- **Custom Model Path Management**: Added support for custom download paths for model downloads
+- **Enhanced Local Storage Management**: Improved storage handling and configuration
+- **SDK for Users**: Added comprehensive SDK for developers to build on Clara
+- **Granular Configuration System**: Enhanced settings with more detailed configuration options
+- **Multi-Platform Optimizations**: Tested and optimized for Linux, improved Windows compatibility
+- **Server Management Integration**: Moved servers to settings for better organization
+
+### ✨ New Features
+- **Custom Download Path Support**: Users can now specify custom paths for model downloads
+- **Enhanced MCP Diagnosis**: Added support for nvm node versions in PATH for MCP diagnosis
+- **Linux 64-bit CPU Binaries**: Added dedicated binaries for Linux 64-bit systems
+- **Windows CUDA Binaries**: Added CUDA support for Windows users
+- **Call, TTS, and STT Integration**: Added text-to-speech and speech-to-text capabilities
+- **Enhanced Python Backend**: Improved stability and performance of the Python backend
+- **Provider Management**: Added comprehensive provider management functionality in settings
+
+### 🛠️ Improvements
+- **Security Enhancements**: Fixed security issues with exposed API keys and vulnerable dependencies
+- **UI/UX Improvements**: Multiple quality-of-life improvements across the interface
+- **Performance Optimizations**: Enhanced performance across multiple components
+- **Documentation Updates**: Updated README and documentation for better clarity
+- **Build System Improvements**: Enhanced build processes and dependency management
+
+### 🐛 Bug Fixes
+- **Dependency Vulnerabilities**: Fixed multiple security vulnerabilities in dependencies
+- **API Key Exposure**: Resolved issues with exposed API keys
+- **Model Management**: Fixed various bugs in model downloading and management
+- **UI Responsiveness**: Fixed various UI responsiveness issues
+- **Cross-Platform Compatibility**: Resolved platform-specific issues
+
+### 🔧 Technical Improvements
+- **Code Quality**: Refactored multiple components for better maintainability
+- **Build Process**: Enhanced build and deployment processes
+- **Testing**: Improved testing coverage and reliability
+- **Documentation**: Enhanced code documentation and user guides
+
+---
+
+## [0.1.1] - 2024-05-20
+
+### 🚀 Major Features Added
+- **Electron Integration**: Full desktop application support with native features
+- **Image Generation Support**: Comprehensive image generation capabilities
+- **Node-Based Workflow System**: Visual workflow builder with drag-and-drop functionality
+- **App Creator Enhancement**: Complete refactoring of the node registration mechanism
+
+### ✨ New Features
+- **Clipboard Node**: Added clipboard functionality for workflows
+- **Concatenation Tool**: New tool for string concatenation in workflows
+- **Visual App Runner**: Enhanced app runner with chat UI and horizontal image+text inputs
+- **Image Handling**: Improved image handling in nodes with runtime image replacement
+- **Auto-Save Functionality**: Added automatic saving for user work
+- **Template System**: Added templates for image generation
+
+### 🛠️ Improvements
+- **UI/UX Enhancements**: Multiple quality-of-life improvements
+- **Code Highlighting**: Removed syntax highlighting and border styling from code blocks for cleaner appearance
+- **App Deletion Process**: Moved app deletion to AppCreator with improved deletion process
+- **Workflow Integration**: Enhanced workflow system with better node management
+
+### 🐛 Bug Fixes
+- **Image Node Issues**: Fixed image handling bugs in workflow nodes
+- **UI Responsiveness**: Resolved various UI layout issues
+- **Workflow Execution**: Fixed bugs in workflow execution engine
+
+### 🔧 Technical Improvements
+- **Code Refactoring**: Major refactoring of the complete node register mechanism
+- **Component Architecture**: Improved component structure for better maintainability
+- **Build System**: Enhanced build processes for Electron integration
+
+---
+
+## [0.1.0] - 2024-05-01
+
+### 🎉 Initial Release
+- **Core Chat Interface**: Basic AI chat functionality with local LLM support
+- **Privacy-First Architecture**: Complete local processing with no cloud dependencies
+- **Multi-Provider Support**: Support for various AI model providers
+- **Basic UI Framework**: Initial user interface with essential features
+- **Local Storage**: Client-side data storage system
+- **Open Source Foundation**: MIT licensed with full source code availability
+
+### ✨ Initial Features
+- **Local AI Chat**: Chat with AI models running locally
+- **Model Management**: Basic model loading and management
+- **Responsive Design**: Mobile and desktop responsive interface
+- **Settings System**: Basic configuration and settings management
+- **File Handling**: Initial file upload and processing capabilities
+
+### 🔧 Technical Foundation
+- **React Frontend**: Built with modern React and TypeScript
+- **Electron Support**: Desktop application framework
+- **Vite Build System**: Fast development and build processes
+- **Local Storage API**: IndexedDB integration for local data persistence
+- **Modular Architecture**: Component-based architecture for extensibility
+
+---
+
+## Installation & Upgrade Guide
+
+### Fresh Installation
+```bash
+# Clone the repository
+git clone https://github.com/badboysm890/ClaraVerse.git
+cd ClaraVerse
+
+# Install dependencies
+npm install
+
+# Run development server
+npm run dev
+
+# Or run desktop application
+npm run electron:dev
+```
+
+### Upgrading from Previous Versions
+```bash
+# Pull latest changes
+git pull origin main
+
+# Update dependencies
+npm install
+
+# Rebuild application
+npm run build
+```
+
+### Docker Installation
+```bash
+# Run with Docker
+docker run -p 8069:8069 clara-ollama:latest
+```
+
+---
+
+## Breaking Changes
+
+### From 0.1.0 to 0.1.1
+- **Node System**: Complete refactoring of the node registration system
+- **Image Handling**: Changes to image processing pipeline
+- **App Creator**: Significant changes to app creation workflow
+
+### From 0.1.1 to 0.1.2
+- **Settings Structure**: Server settings moved to new location
+- **Model Paths**: Custom model path configuration added
+- **Storage Management**: Enhanced local storage structure
+
+---
+
+## Migration Guide
+
+### Migrating to 0.1.2
+1. **Settings Update**: Check your server settings as they have been reorganized
+2. **Model Paths**: Configure custom model download paths if needed
+3. **Dependencies**: Update all dependencies using `npm install`
+4. **Storage**: Clear local storage if experiencing issues (use clear_storage.js)
+
+---
+
+## Known Issues
+
+### Current Known Issues (0.1.2)
+- Some legacy workflow configurations may need manual updating
+- Windows users may need to run as administrator for certain model downloads
+- macOS users need to manually approve unsigned applications
+
+### Workarounds
+- **macOS App Damage Warning**: Right-click app and select "Open", then approve in System Preferences
+- **Windows Admin Rights**: Run as administrator if model downloads fail
+- **Linux Permissions**: Ensure proper permissions for model storage directories
+
+---
+
+## Support & Feedback
+
+- **Email**: [praveensm890@gmail.com](mailto:praveensm890@gmail.com)
+- **GitHub Issues**: [Report bugs and request features](https://github.com/badboysm890/ClaraVerse/issues)
+- **Discussions**: [Join community discussions](https://github.com/badboysm890/ClaraVerse/discussions)
+
+---
+
+## Contributing
+
+We welcome contributions! Please see our [Contributing Guide](CONTRIBUTING.md) for details on how to:
+- Report bugs
+- Suggest features
+- Submit pull requests
+- Improve documentation
+
+---
+
+*For more information, visit the [Clara Documentation](https://github.com/badboysm890/ClaraVerse) or join our community discussions.*
\ No newline at end of file
diff --git a/Dockerfile b/Dockerfile
new file mode 100644
index 00000000..33b34542
--- /dev/null
+++ b/Dockerfile
@@ -0,0 +1,13 @@
+FROM node:20-alpine AS builder
+
+WORKDIR /app
+COPY package*.json ./
+RUN npm install
+COPY . .
+RUN npm run build
+
+FROM nginx:alpine
+COPY --from=builder /app/dist /usr/share/nginx/html
+COPY nginx.conf /etc/nginx/conf.d/default.conf
+EXPOSE 8069
+CMD ["nginx", "-g", "daemon off;"]
diff --git a/LICENSE b/LICENSE
index 981aec6b..79443301 100644
--- a/LICENSE
+++ b/LICENSE
@@ -1,702 +1,21 @@
-ClaraVerse - Privacy-First AI Workspace
-Copyright (C) 2025 claraverse-space
-
-This program is free software: you can redistribute it and/or modify
-it under the terms of the GNU Affero General Public License as published
-by the Free Software Foundation, either version 3 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-GNU Affero General Public License for more details.
-
-You should have received a copy of the GNU Affero General Public License
-along with this program. If not, see .
-
-Additional Terms (AGPL Section 7):
-- Network Use: If you run a modified version of ClaraVerse as a network service,
- you must make the complete source code available to users of that service.
-- Attribution: You must retain all copyright notices and give appropriate credit
- to the ClaraVerse developers.
-
-DUAL LICENSING OPTIONS:
-
-ClaraVerse is available under dual licensing:
-
-1. AGPL-3.0 (This License - Free & Open Source)
- - Free for everyone
- - Must share all modifications
- - Network copyleft applies
-
-2. Commercial License (Alternative Licensing)
- - Small companies (<1000 employees): FREE upon request
- - Large enterprises (1000+ employees): Paid license with enterprise support
- - No requirement to share modifications
- - Can integrate into proprietary software
-
-Contact: support@claraverse.space for commercial licensing inquiries
-
-================================================================================
-
- GNU AFFERO GENERAL PUBLIC LICENSE
- Version 3, 19 November 2007
-
- Copyright (C) 2007 Free Software Foundation, Inc.
- Everyone is permitted to copy and distribute verbatim copies
- of this license document, but changing it is not allowed.
-
- Preamble
-
- The GNU Affero General Public License is a free, copyleft license for
-software and other kinds of works, specifically designed to ensure
-cooperation with the community in the case of network server software.
-
- The licenses for most software and other practical works are designed
-to take away your freedom to share and change the works. By contrast,
-our General Public Licenses are intended to guarantee your freedom to
-share and change all versions of a program--to make sure it remains free
-software for all its users.
-
- When we speak of free software, we are referring to freedom, not
-price. Our General Public Licenses are designed to make sure that you
-have the freedom to distribute copies of free software (and charge for
-them if you wish), that you receive source code or can get it if you
-want it, that you can change the software or use pieces of it in new
-free programs, and that you know you can do these things.
-
- Developers that use our General Public Licenses protect your rights
-with two steps: (1) assert copyright on the software, and (2) offer
-you this License which gives you legal permission to copy, distribute
-and/or modify the software.
-
- A secondary benefit of defending all users' freedom is that
-improvements made in alternate versions of the program, if they
-receive widespread use, become available for other developers to
-incorporate. Many developers of free software are heartened and
-encouraged by the resulting cooperation. However, in the case of
-software used on network servers, this result may fail to come about.
-The GNU General Public License permits making a modified version and
-letting the public access it on a server without ever releasing its
-source code to the public.
-
- The GNU Affero General Public License is designed specifically to
-ensure that, in such cases, the modified source code becomes available
-to the community. It requires the operator of a network server to
-provide the source code of the modified version running there to the
-users of that server. Therefore, public use of a modified version, on
-a publicly accessible server, gives the public access to the source
-code of the modified version.
-
- An older license, called the Affero General Public License and
-published by Affero, was designed to accomplish similar goals. This is
-a different license, not a version of the Affero GPL, but Affero has
-released a new version of the Affero GPL which permits relicensing under
-this license.
-
- The precise terms and conditions for copying, distribution and
-modification follow.
-
- TERMS AND CONDITIONS
-
- 0. Definitions.
-
- "This License" refers to version 3 of the GNU Affero General Public License.
-
- "Copyright" also means copyright-like laws that apply to other kinds of
-works, such as semiconductor masks.
-
- "The Program" refers to any copyrightable work licensed under this
-License. Each licensee is addressed as "you". "Licensees" and
-"recipients" may be individuals or organizations.
-
- To "modify" a work means to copy from or adapt all or part of the work
-in a fashion requiring copyright permission, other than the making of an
-exact copy. The resulting work is called a "modified version" of the
-earlier work or a work "based on" the earlier work.
-
- A "covered work" means either the unmodified Program or a work based
-on the Program.
-
- To "propagate" a work means to do anything with it that, without
-permission, would make you directly or secondarily liable for
-infringement under applicable copyright law, except executing it on a
-computer or modifying a private copy. Propagation includes copying,
-distribution (with or without modification), making available to the
-public, and in some countries other activities as well.
-
- To "convey" a work means any kind of propagation that enables other
-parties to make or receive copies. Mere interaction with a user through
-a computer network, with no transfer of a copy, is not conveying.
-
- An interactive user interface displays "Appropriate Legal Notices"
-to the extent that it includes a convenient and prominently visible
-feature that (1) displays an appropriate copyright notice, and (2)
-tells the user that there is no warranty for the work (except to the
-extent that warranties are provided), that licensees may convey the
-work under this License, and how to view a copy of this License. If
-the interface presents a list of user commands or options, such as a
-menu, a prominent item in the list meets this criterion.
-
- 1. Source Code.
-
- The "source code" for a work means the preferred form of the work
-for making modifications to it. "Object code" means any non-source
-form of a work.
-
- A "Standard Interface" means an interface that either is an official
-standard defined by a recognized standards body, or, in the case of
-interfaces specified for a particular programming language, one that
-is widely used among developers working in that language.
-
- The "System Libraries" of an executable work include anything, other
-than the work as a whole, that (a) is included in the normal form of
-packaging a Major Component, but which is not part of that Major
-Component, and (b) serves only to enable use of the work with that
-Major Component, or to implement a Standard Interface for which an
-implementation is available to the public in source code form. A
-"Major Component", in this context, means a major essential component
-(kernel, window system, and so on) of the specific operating system
-(if any) on which the executable work runs, or a compiler used to
-produce the work, or an object code interpreter used to run it.
-
- The "Corresponding Source" for a work in object code form means all
-the source code needed to generate, install, and (for an executable
-work) run the object code and to modify the work, including scripts to
-control those activities. However, it does not include the work's
-System Libraries, or general-purpose tools or generally available free
-programs which are used unmodified in performing those activities but
-which are not part of the work. For example, Corresponding Source
-includes interface definition files associated with source files for
-the work, and the source code for shared libraries and dynamically
-linked subprograms that the work is specifically designed to require,
-such as by intimate data communication or control flow between those
-subprograms and other parts of the work.
-
- The Corresponding Source need not include anything that users
-can regenerate automatically from other parts of the Corresponding
-Source.
-
- The Corresponding Source for a work in source code form is that
-same work.
-
- 2. Basic Permissions.
-
- All rights granted under this License are granted for the term of
-copyright on the Program, and are irrevocable provided the stated
-conditions are met. This License explicitly affirms your unlimited
-permission to run the unmodified Program. The output from running a
-covered work is covered by this License only if the output, given its
-content, constitutes a covered work. This License acknowledges your
-rights of fair use or other equivalent, as provided by copyright law.
-
- You may make, run and propagate covered works that you do not
-convey, without conditions so long as your license otherwise remains
-in force. You may convey covered works to others for the sole purpose
-of having them make modifications exclusively for you, or provide you
-with facilities for running those works, provided that you comply with
-the terms of this License in conveying all material for which you do
-not control copyright. Those thus making or running the covered works
-for you must do so exclusively on your behalf, under your direction
-and control, on terms that prohibit them from making any copies of
-your copyrighted material outside their relationship with you.
-
- Conveying under any other circumstances is permitted solely under
-the conditions stated below. Sublicensing is not allowed; section 10
-makes it unnecessary.
-
- 3. Protecting Users' Legal Rights From Anti-Circumvention Law.
-
- No covered work shall be deemed part of an effective technological
-measure under any applicable law fulfilling obligations under article
-11 of the WIPO copyright treaty adopted on 20 December 1996, or
-similar laws prohibiting or restricting circumvention of such
-measures.
-
- When you convey a covered work, you waive any legal power to forbid
-circumvention of technological measures to the extent such circumvention
-is effected by exercising rights under this License with respect to
-the covered work, and you disclaim any intention to limit operation or
-modification of the work as a means of enforcing, against the work's
-users, your or third parties' legal rights to forbid circumvention of
-technological measures.
-
- 4. Conveying Verbatim Copies.
-
- You may convey verbatim copies of the Program's source code as you
-receive it, in any medium, provided that you conspicuously and
-appropriately publish on each copy an appropriate copyright notice;
-keep intact all notices stating that this License and any
-non-permissive terms added in accord with section 7 apply to the code;
-keep intact all notices of the absence of any warranty; and give all
-recipients a copy of this License along with the Program.
-
- You may charge any price or no price for each copy that you convey,
-and you may offer support or warranty protection for a fee.
-
- 5. Conveying Modified Source Versions.
-
- You may convey a work based on the Program, or the modifications to
-produce it from the Program, in the form of source code under the
-terms of section 4, provided that you also meet all of these conditions:
-
- a) The work must carry prominent notices stating that you modified
- it, and giving a relevant date.
-
- b) The work must carry prominent notices stating that it is
- released under this License and any conditions added under section
- 7. This requirement modifies the requirement in section 4 to
- "keep intact all notices".
-
- c) You must license the entire work, as a whole, under this
- License to anyone who comes into possession of a copy. This
- License will therefore apply, along with any applicable section 7
- additional terms, to the whole of the work, and all its parts,
- regardless of how they are packaged. This License gives no
- permission to license the work in any other way, but it does not
- invalidate such permission if you have separately received it.
-
- d) If the work has interactive user interfaces, each must display
- Appropriate Legal Notices; however, if the Program has interactive
- interfaces that do not display Appropriate Legal Notices, your
- work need not make them do so.
-
- A compilation of a covered work with other separate and independent
-works, which are not by their nature extensions of the covered work,
-and which are not combined with it such as to form a larger program,
-in or on a volume of a storage or distribution medium, is called an
-"aggregate" if the compilation and its resulting copyright are not
-used to limit the access or legal rights of the compilation's users
-beyond what the individual works permit. Inclusion of a covered work
-in an aggregate does not cause this License to apply to the other
-parts of the aggregate.
-
- 6. Conveying Non-Source Forms.
-
- You may convey a covered work in object code form under the terms
-of sections 4 and 5, provided that you also convey the
-machine-readable Corresponding Source under the terms of this License,
-in one of these ways:
-
- a) Convey the object code in, or embodied in, a physical product
- (including a physical distribution medium), accompanied by the
- Corresponding Source fixed on a durable physical medium
- customarily used for software interchange.
-
- b) Convey the object code in, or embodied in, a physical product
- (including a physical distribution medium), accompanied by a
- written offer, valid for at least three years and valid for as
- long as you offer spare parts or customer support for that product
- model, to give anyone who possesses the object code either (1) a
- copy of the Corresponding Source for all the software in the
- product that is covered by this License, on a durable physical
- medium customarily used for software interchange, for a price no
- more than your reasonable cost of physically performing this
- conveying of source, or (2) access to copy the
- Corresponding Source from a network server at no charge.
-
- c) Convey individual copies of the object code with a copy of the
- written offer to provide the Corresponding Source. This
- alternative is allowed only occasionally and noncommercially, and
- only if you received the object code with such an offer, in accord
- with subsection 6b.
-
- d) Convey the object code by offering access from a designated
- place (gratis or for a charge), and offer equivalent access to the
- Corresponding Source in the same way through the same place at no
- further charge. You need not require recipients to copy the
- Corresponding Source along with the object code. If the place to
- copy the object code is a network server, the Corresponding Source
- may be on a different server (operated by you or a third party)
- that supports equivalent copying facilities, provided you maintain
- clear directions next to the object code saying where to find the
- Corresponding Source. Regardless of what server hosts the
- Corresponding Source, you remain obligated to ensure that it is
- available for as long as needed to satisfy these requirements.
-
- e) Convey the object code using peer-to-peer transmission, provided
- you inform other peers where the object code and Corresponding
- Source of the work are being offered to the general public at no
- charge under subsection 6d.
-
- A separable portion of the object code, whose source code is excluded
-from the Corresponding Source as a System Library, need not be
-included in conveying the object code work.
-
- A "User Product" is either (1) a "consumer product", which means any
-tangible personal property which is normally used for personal, family,
-or household purposes, or (2) anything designed or sold for incorporation
-into a dwelling. In determining whether a product is a consumer product,
-doubtful cases shall be resolved in favor of coverage. For a particular
-product received by a particular user, "normally used" refers to a
-typical or common use of that class of product, regardless of the status
-of the particular user or of the way in which the particular user
-actually uses, or expects or is expected to use, the product. A product
-is a consumer product regardless of whether the product has substantial
-commercial, industrial or non-consumer uses, unless such uses represent
-the only significant mode of use of the product.
-
- "Installation Information" for a User Product means any methods,
-procedures, authorization keys, or other information required to install
-and execute modified versions of a covered work in that User Product from
-a modified version of its Corresponding Source. The information must
-suffice to ensure that the continued functioning of the modified object
-code is in no case prevented or interfered with solely because
-modification has been made.
-
- If you convey an object code work under this section in, or with, or
-specifically for use in, a User Product, and the conveying occurs as
-part of a transaction in which the right of possession and use of the
-User Product is transferred to the recipient in perpetuity or for a
-fixed term (regardless of how the transaction is characterized), the
-Corresponding Source conveyed under this section must be accompanied
-by the Installation Information. But this requirement does not apply
-if neither you nor any third party retains the ability to install
-modified object code on the User Product (for example, the work has
-been installed in ROM).
-
- The requirement to provide Installation Information does not include a
-requirement to continue to provide support service, warranty, or updates
-for a work that has been modified or installed by the recipient, or for
-the User Product in which it has been modified or installed. Access to a
-network may be denied when the modification itself materially and
-adversely affects the operation of the network or violates the rules and
-protocols for communication across the network.
-
- Corresponding Source conveyed, and Installation Information provided,
-in accord with this section must be in a format that is publicly
-documented (and with an implementation available to the public in
-source code form), and must require no special password or key for
-unpacking, reading or copying.
-
- 7. Additional Terms.
-
- "Additional permissions" are terms that supplement the terms of this
-License by making exceptions from one or more of its conditions.
-Additional permissions that are applicable to the entire Program shall
-be treated as though they were included in this License, to the extent
-that they are valid under applicable law. If additional permissions
-apply only to part of the Program, that part may be used separately
-under those permissions, but the entire Program remains governed by
-this License without regard to the additional permissions.
-
- When you convey a copy of a covered work, you may at your option
-remove any additional permissions from that copy, or from any part of
-it. (Additional permissions may be written to require their own
-removal in certain cases when you modify the work.) You may place
-additional permissions on material, added by you to a covered work,
-for which you have or can give appropriate copyright permission.
-
- Notwithstanding any other provision of this License, for material you
-add to a covered work, you may (if authorized by the copyright holders of
-that material) supplement the terms of this License with terms:
-
- a) Disclaiming warranty or limiting liability differently from the
- terms of sections 15 and 16 of this License; or
-
- b) Requiring preservation of specified reasonable legal notices or
- author attributions in that material or in the Appropriate Legal
- Notices displayed by works containing it; or
-
- c) Prohibiting misrepresentation of the origin of that material, or
- requiring that modified versions of such material be marked in
- reasonable ways as different from the original version; or
-
- d) Limiting the use for publicity purposes of names of licensors or
- authors of the material; or
-
- e) Declining to grant rights under trademark law for use of some
- trade names, trademarks, or service marks; or
-
- f) Requiring indemnification of licensors and authors of that
- material by anyone who conveys the material (or modified versions of
- it) with contractual assumptions of liability to the recipient, for
- any liability that these contractual assumptions directly impose on
- those licensors and authors.
-
- All other non-permissive additional terms are considered "further
-restrictions" within the meaning of section 10. If the Program as you
-received it, or any part of it, contains a notice stating that it is
-governed by this License along with a term that is a further
-restriction, you may remove that term. If a license document contains
-a further restriction but permits relicensing or conveying under this
-License, you may add to a covered work material governed by the terms
-of that license document, provided that the further restriction does
-not survive such relicensing or conveying.
-
- If you add terms to a covered work in accord with this section, you
-must place, in the relevant source files, a statement of the
-additional terms that apply to those files, or a notice indicating
-where to find the applicable terms.
-
- Additional terms, permissive or non-permissive, may be stated in the
-form of a separately written license, or stated as exceptions;
-the above requirements apply either way.
-
- 8. Termination.
-
- You may not propagate or modify a covered work except as expressly
-provided under this License. Any attempt otherwise to propagate or
-modify it is void, and will automatically terminate your rights under
-this License (including any patent licenses granted under the third
-paragraph of section 11).
-
- However, if you cease all violation of this License, then your
-license from a particular copyright holder is reinstated (a)
-provisionally, unless and until the copyright holder explicitly and
-finally terminates your license, and (b) permanently, if the copyright
-holder fails to notify you of the violation by some reasonable means
-prior to 60 days after the cessation.
-
- Moreover, your license from a particular copyright holder is
-reinstated permanently if the copyright holder notifies you of the
-violation by some reasonable means, this is the first time you have
-received notice of violation of this License (for any work) from that
-copyright holder, and you cure the violation prior to 30 days after
-your receipt of the notice.
-
- Termination of your rights under this section does not terminate the
-licenses of parties who have received copies or rights from you under
-this License. If your rights have been terminated and not permanently
-reinstated, you do not qualify to receive new licenses for the same
-material under section 10.
-
- 9. Acceptance Not Required for Having Copies.
-
- You are not required to accept this License in order to receive or
-run a copy of the Program. Ancillary propagation of a covered work
-occurring solely as a consequence of using peer-to-peer transmission
-to receive a copy likewise does not require acceptance. However,
-nothing other than this License grants you permission to propagate or
-modify any covered work. These actions infringe copyright if you do
-not accept this License. Therefore, by modifying or propagating a
-covered work, you indicate your acceptance of this License to do so.
-
- 10. Automatic Licensing of Downstream Recipients.
-
- Each time you convey a covered work, the recipient automatically
-receives a license from the original licensors, to run, modify and
-propagate that work, subject to this License. You are not responsible
-for enforcing compliance by third parties with this License.
-
- An "entity transaction" is a transaction transferring control of an
-organization, or substantially all assets of one, or subdividing an
-organization, or merging organizations. If propagation of a covered
-work results from an entity transaction, each party to that
-transaction who receives a copy of the work also receives whatever
-licenses to the work the party's predecessor in interest had or could
-give under the previous paragraph, plus a right to possession of the
-Corresponding Source of the work from the predecessor in interest, if
-the predecessor has it or can get it with reasonable efforts.
-
- You may not impose any further restrictions on the exercise of the
-rights granted or affirmed under this License. For example, you may
-not impose a license fee, royalty, or other charge for exercise of
-rights granted under this License, and you may not initiate litigation
-(including a cross-claim or counterclaim in a lawsuit) alleging that
-any patent claim is infringed by making, using, selling, offering for
-sale, or importing the Program or any portion of it.
-
- 11. Patents.
-
- A "contributor" is a copyright holder who authorizes use under this
-License of the Program or a work on which the Program is based. The
-work thus licensed is called the contributor's "contributor version".
-
- A contributor's "essential patent claims" are all patent claims
-owned or controlled by the contributor, whether already acquired or
-hereafter acquired, that would be infringed by some manner, permitted
-by this License, of making, using, or selling its contributor version,
-but do not include claims that would be infringed only as a
-consequence of further modification of the contributor version. For
-purposes of this definition, "control" includes the right to grant
-patent sublicenses in a manner consistent with the requirements of
-this License.
-
- Each contributor grants you a non-exclusive, worldwide, royalty-free
-patent license under the contributor's essential patent claims, to
-make, use, sell, offer for sale, import and otherwise run, modify and
-propagate the contents of its contributor version.
-
- In the following three paragraphs, a "patent license" is any express
-agreement or commitment, however denominated, not to enforce a patent
-(such as an express permission to practice a patent or covenant not to
-sue for patent infringement). To "grant" such a patent license to a
-party means to make such an agreement or commitment not to enforce a
-patent against the party.
-
- If you convey a covered work, knowingly relying on a patent license,
-and the Corresponding Source of the work is not available for anyone
-to copy, free of charge and under the terms of this License, through a
-publicly available network server or other readily accessible means,
-then you must either (1) cause the Corresponding Source to be so
-available, or (2) arrange to deprive yourself of the benefit of the
-patent license for this particular work, or (3) arrange, in a manner
-consistent with the requirements of this License, to extend the patent
-license to downstream recipients. "Knowingly relying" means you have
-actual knowledge that, but for the patent license, your conveying the
-covered work in a country, or your recipient's use of the covered work
-in a country, would infringe one or more identifiable patents in that
-country that you have reason to believe are valid.
-
- If, pursuant to or in connection with a single transaction or
-arrangement, you convey, or propagate by procuring conveyance of, a
-covered work, and grant a patent license to some of the parties
-receiving the covered work authorizing them to use, propagate, modify
-or convey a specific copy of the covered work, then the patent license
-you grant is automatically extended to all recipients of the covered
-work and works based on it.
-
- A patent license is "discriminatory" if it does not include within
-the scope of its coverage, prohibits the exercise of, or is
-conditioned on the non-exercise of one or more of the rights that are
-specifically granted under this License. You may not convey a covered
-work if you are a party to an arrangement with a third party that is
-in the business of distributing software, under which you make payment
-to the third party based on the extent of your activity of conveying
-the work, and under which the third party grants, to any of the
-parties who would receive the covered work from you, a discriminatory
-patent license (a) in connection with copies of the covered work
-conveyed by you (or copies made from those copies), or (b) primarily
-for and in connection with specific products or compilations that
-contain the covered work, unless you entered into that arrangement,
-or that patent license was granted, prior to 28 March 2007.
-
- Nothing in this License shall be construed as excluding or limiting
-any implied license or other defenses to infringement that may
-otherwise be available to you under applicable patent law.
-
- 12. No Surrender of Others' Freedom.
-
- If conditions are imposed on you (whether by court order, agreement or
-otherwise) that contradict the conditions of this License, they do not
-excuse you from the conditions of this License. If you cannot convey a
-covered work so as to satisfy simultaneously your obligations under this
-License and any other pertinent obligations, then as a consequence you may
-not convey it at all. For example, if you agree to terms that obligate you
-to collect a royalty for further conveying from those to whom you convey
-the Program, the only way you could satisfy both those terms and this
-License would be to refrain entirely from conveying the Program.
-
- 13. Remote Network Interaction; Use with the GNU General Public License.
-
- Notwithstanding any other provision of this License, if you modify the
-Program, your modified version must prominently offer all users
-interacting with it remotely through a computer network (if your version
-supports such interaction) an opportunity to receive the Corresponding
-Source of your version by providing access to the Corresponding Source
-from a network server at no charge, through some standard or customary
-means of facilitating copying of software. This Corresponding Source
-shall include the Corresponding Source for any work covered by version 3
-of the GNU General Public License that is incorporated pursuant to the
-following paragraph.
-
- Notwithstanding any other provision of this License, you have
-permission to link or combine any covered work with a work licensed
-under version 3 of the GNU General Public License into a single
-combined work, and to convey the resulting work. The terms of this
-License will continue to apply to the part which is the covered work,
-but the work with which it is combined will remain governed by version
-3 of the GNU General Public License.
-
- 14. Revised Versions of this License.
-
- The Free Software Foundation may publish revised and/or new versions of
-the GNU Affero General Public License from time to time. Such new versions
-will be similar in spirit to the present version, but may differ in detail to
-address new problems or concerns.
-
- Each version is given a distinguishing version number. If the
-Program specifies that a certain numbered version of the GNU Affero General
-Public License "or any later version" applies to it, you have the
-option of following the terms and conditions either of that numbered
-version or of any later version published by the Free Software
-Foundation. If the Program does not specify a version number of the
-GNU Affero General Public License, you may choose any version ever published
-by the Free Software Foundation.
-
- If the Program specifies that a proxy can decide which future
-versions of the GNU Affero General Public License can be used, that proxy's
-public statement of acceptance of a version permanently authorizes you
-to choose that version for the Program.
-
- Later license versions may give you additional or different
-permissions. However, no additional obligations are imposed on any
-author or copyright holder as a result of your choosing to follow a
-later version.
-
- 15. Disclaimer of Warranty.
-
- THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY
-APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT
-HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY
-OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO,
-THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
-PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM
-IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF
-ALL NECESSARY SERVICING, REPAIR OR CORRECTION.
-
- 16. Limitation of Liability.
-
- IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
-WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS
-THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY
-GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE
-USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF
-DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD
-PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS),
-EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF
-SUCH DAMAGES.
-
- 17. Interpretation of Sections 15 and 16.
-
- If the disclaimer of warranty and limitation of liability provided
-above cannot be given local legal effect according to their terms,
-reviewing courts shall apply local law that most closely approximates
-an absolute waiver of all civil liability in connection with the
-Program, unless a warranty or assumption of liability accompanies a
-copy of the Program in return for a fee.
-
- END OF TERMS AND CONDITIONS
-
- How to Apply These Terms to Your New Programs
-
- If you develop a new program, and you want it to be of the greatest
-possible use to the public, the best way to achieve this is to make it
-free software which everyone can redistribute and change under these terms.
-
- To do so, attach the following notices to the program. It is safest
-to attach them to the start of each source file to most effectively
-state the exclusion of warranty; and each file should have at least
-the "copyright" line and a pointer to where the full notice is found.
-
-
- Copyright (C)
-
- This program is free software: you can redistribute it and/or modify
- it under the terms of the GNU Affero General Public License as published by
- the Free Software Foundation, either version 3 of the License, or
- (at your option) any later version.
-
- This program is distributed in the hope that it will be useful,
- but WITHOUT ANY WARRANTY; without even the implied warranty of
- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- GNU Affero General Public License for more details.
-
- You should have received a copy of the GNU Affero General Public License
- along with this program. If not, see .
-
-Also add information on how to contact you by electronic and paper mail.
-
- If your software can interact with users remotely through a computer
-network, you should also make sure that it provides a way for users to
-get its source. For example, if your program is a web application, its
-interface could display a "Source" link that leads users to an archive
-of the code. There are many ways you could offer source, and different
-solutions will be better for different programs; see section 13 for the
-specific requirements.
-
- You should also get your employer (if you work as a programmer) or school,
-if any, to sign a "copyright disclaimer" for the program, if necessary.
-For more information on this, and how to apply and follow the GNU AGPL, see
-.
+MIT License
+
+Copyright (c) 2025 ClaraVerse Labs and Contributors
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
diff --git a/README.md b/README.md
index faf3c5c8..7542832e 100644
--- a/README.md
+++ b/README.md
@@ -1,708 +1,917 @@
---
-## 🚀 Quick Start
+
-**Install CLI:**
-```bash
-curl -fsSL https://get.claraverse.app | bash
-```
+## 💭 **The Story Behind ClaraVerse**
-**Start ClaraVerse:**
-```bash
-claraverse init
-```
+> **"Why can't everything be in a single app? Why do we need to jump between different AI tools and pay multiple subscriptions?"**
-Open **http://localhost** → Register → Add AI provider → Start chatting!
+**Here's what happened to us (and probably you too):**
-
-Other options
+We found ourselves constantly jumping between different AI apps — tired of paying for Claude subscriptions, then installing LM Studio for local models, then N8N for automation, then Ollama for model management, then OpenWebUI for a better chat interface..
-**Docker (no CLI):**
-```bash
-docker run -d -p 80:80 -p 3001:3001 -v claraverse-data:/data claraverseoss/claraverse:latest
-```
+When using Ollama, downloading models was the only way to get started — but then we were stuck with limited chat features. LM Studio was good for models, but the chat experience felt basic. We'd end up running everything in the background while building UIs and generating images separately.
-**Clone & Run:**
-```bash
-git clone https://github.com/claraverse-space/ClaraVerseAI.git && cd ClaraVerseAI && ./quickstart.sh
-```
+**That's when it hit us: Why can't everything be in a single app?**
-
+ClaraVerse is our answer to that question.
----
+> ClaraVerse is not another OpenWebUI (which is excellent for chat) or Ollama - everything is just scattered around and I'm trying to build a single app where people can just install it and use it.
-## ✨ What's Included
+
-Everything runs locally - no external APIs needed:
+## 🎯 **The Problem**
-| Service | Purpose |
-|---------|---------|
-| **Frontend** | React app on port 80 |
-| **Backend** | Go API on port 3001 |
-| **MongoDB** | Conversations & workflows |
-| **MySQL** | Providers & models |
-| **Redis** | Job scheduling |
-| **SearXNG** | Web search (no API key!) |
-| **E2B** | Code execution (no API key!) |
+
**Total: $960/year** 😱 | • Chat in Claude → Code in VScode • Prompt in LLM → Use it in ComfyUI • Deploy with Ollama → Run in OpenWebUI
**Lost context with every switch** 🤦 |
-## Why ClaraVerse?
+
+
+
-**Self-hosting isn't enough.** Most "privacy-focused" chat UIs still store your conversations in MongoDB or PostgreSQL. ClaraVerse goes further with **browser-local storage**—even the server admin can't read your chats.
+## ✨ **The Solution: ClaraVerse**
-| Feature | ClaraVerse | ChatGPT/Claude | Open WebUI | LibreChat |
-|---------|------------|----------------|------------|-----------|
-| **Browser-Local Storage** | ✅ Never touches server | ❌ Cloud-only | ❌ Stored in MongoDB | ❌ Stored in MongoDB |
-| **Server Can't Read Chats** | ✅ Zero-knowledge architecture | ❌ Full access | ❌ Admin has full access | ❌ Admin has full access |
-| **Self-Hosting** | ✅ Optional | ❌ Cloud-only | ✅ Required | ✅ Required |
-| **Works Offline** | ✅ Full offline mode | ❌ Internet required | ⚠️ Server required | ⚠️ Server required |
-| **Multi-Provider** | ✅ OpenAI, Claude, Gemini, local | ❌ Single provider | ✅ Multi-provider | ✅ Multi-provider |
-| **Visual Workflow Builder** | ✅ Chat + n8n combined | ❌ | ❌ | ❌ |
-| **Interactive Prompts** | ✅ AI asks questions mid-chat | ❌ | ⚠️ Pre-defined only | ❌ |
+
-> **50,000+ downloads** | The only AI platform where conversations never touch the server—even when self-hosted
+### **One App. Six Tools. Zero Compromises.**
-
-📋 Advanced Setup & Troubleshooting
+
-### Prerequisites
-- Docker & Docker Compose installed
-- 4GB RAM minimum (8GB recommended)
+
-# 4. Verify
-docker compose ps
+```diff
++ 100% Local Processing Your data never leaves your machine
++ Zero Telemetry We can't see what you're doing
++ Open Source Every line of code is auditable
++ Works Offline No internet? No problem!
```
-### Troubleshooting
+
+
+### **Option 2: Development Build**
```bash
-# Run diagnostics
-./diagnose.sh # Linux/Mac
-diagnose.bat # Windows
+# Clone the repository
+git clone https://github.com/badboysm890/ClaraVerse.git
+cd ClaraVerse
-# View logs
-docker compose logs -f backend
+# Install dependencies
+npm install
+
+# Run in development mode
+npm run electron:dev:hot
+```
-# Restart
-docker compose restart
+### **Option 3: Docker** 🐳
-# Fresh start
-docker compose down -v && docker compose up -d
+```bash
+# Coming soon!
```
-
+
----
+## 📊 **Why ClaraVerse?**
-## Browser-Local Storage: True Zero-Knowledge Privacy
+
-**The Problem with Traditional Self-Hosted Chat UIs:**
+| Feature | ClaraVerse | Others |
+|:--------|:----------:|:------:|
+| **All-in-One Platform** | ✅ | ❌ |
+| **100% Local Processing** | ✅ | ❌ |
+| **No Subscriptions** | ✅ | ❌ |
+| **Context Sharing** | ✅ | ❌ |
+| **Community Hub** | ✅ | ❌ |
+| **Autonomous Agents** | ✅ | ⚠️ |
+| **MCP Tool Ecosystem** | ✅ | ❌ |
+| **Open Source** | ✅ | ⚠️ |
+| **Offline Mode** | ✅ | ❌ |
+| **Custom Models** | ✅ | ⚠️ |
+| **Enterprise Ready** | ⌛ | 💰 |
-When you self-host Open WebUI or LibreChat, conversations are stored in your MongoDB database. You control the server, but the data still exists in a queryable database.
+
-```python
-# Traditional self-hosted architecture
-User → Server → MongoDB
- ↓
- db.conversations.find({user_id: "123"}) # Admin can read everything
-```
+
-**ClaraVerse's Zero-Knowledge Architecture:**
+## 🏗️ **Architecture**
-Conversations stay in your browser's IndexedDB and **never touch the server or database**. The server only proxies API calls to LLM providers—it never sees or stores message content.
+
-```python
-# ClaraVerse browser-local mode
-User → IndexedDB (browser only, never leaves device)
- → Server (API proxy only, doesn't log or store)
+```mermaid
+graph LR
+ A[🎨 React UI] --> B[⚡ Unified API]
+ B --> C[🧠 Clara Core]
+ B --> D[🔧 LumaUI]
+ B --> E[🎨 ComfyUI]
+ B --> F[🔄 N8N]
+ B --> G[🤖 Agent Studio]
+ B --> H[📊 Widgets]
+ B --> I[👥 Community]
+ B --> J[🔗 MCP Ecosystem]
+
+ C --> K[Llama.cpp]
+ C --> L[Vision]
+ C --> M[Voice]
+
+ G --> N[Autonomous Agents]
+ G --> O[Visual Designer]
+
+ J --> P[Desktop Automation]
+ J --> Q[Browser Control]
+ J --> R[File System]
+
+ style A fill:#4A90E2
+ style C fill:#FF6B6B
+ style E fill:#4ECDC4
+ style G fill:#9B59B6
+ style I fill:#E67E22
+ style J fill:#27AE60
```
-### Why This Matters
+
-✅ **Host for Teams Without Liability**: Even as server admin, you **cannot** access user conversations
-✅ **True Compliance**: No server-side message retention = simplified GDPR/HIPAA compliance
-✅ **No Database Bloat**: Messages aren't stored in MongoDB—database only holds accounts and settings
-✅ **Air-Gap Capable**: Browser caches conversations; works completely offline after initial load
-✅ **Zero Backup Exposure**: Database backups don't contain sensitive chat content
+
-### Per-Conversation Privacy Control
+## 🌟 **Features Roadmap**
-Unlike competitors that force you into one mode, ClaraVerse lets you choose **per conversation**:
+
-- **Work Projects**: Browser-local mode (100% offline, zero server access)
-- **Personal Chats**: Cloud-sync mode (encrypted backup for mobile access)
-- **Switch Anytime**: Toggle privacy mode without losing conversation history
+| Status | Feature | ETA |
+|:-------|:--------|:----|
+| ✅ | Clara AI Assistant | **Released** |
+| ✅ | LumaUI Code Builder | **Released** |
+| ✅ | ComfyUI Integration | **Released** |
+| ✅ | N8N Workflows | **Released** |
+| ✅ | Agent Studio (Advanced) | **Released** |
+| ✅ | Community Hub | **Released** |
+| ✅ | MCP Ecosystem (20+ Servers) | **Released** |
+| 🚧 | Docker Image for Remote servers | Q3 2025 |
+| 🚧 | Mobile App with Offline Support | Q3 2025 |
+| 🚧 | Cloud Sync (Optional) | Q4 2025 |
+| 📋 | Plugin Marketplace | Q4 2025 |
+| 📋 | Team Collaboration | Q4 2025 |
-**This is privacy-first architecture done right.**
+
+
+> **🚧 Development Status**
+> ClaraVerse is actively evolving! While core features are stable, some components may change as we work toward v1.0. We prioritize stability but welcome your feedback on improvements and new features. Join our [Discord](https://discord.gg/j633fsrAne) to stay updated! 🚀
----
+
-## Feature Showcase in a Nutshell
+## 🤝 **Community & Support**
-### Natural Chat Interface
-
+[](https://discord.gg/j633fsrAne)
+[](https://www.reddit.com/r/claraverse/)
+[](https://twitter.com/claraverse)
+[](https://youtube.com/@claraverse)
-*Chat naturally with GPT-4, Claude, Gemini, and more - all in one unified interface*
+### **📚 Resources**
+
+[Documentation](https://github.com/badboysm890/ClaraVerse/tree/main/docs) •
+[API Reference](https://github.com/badboysm890/ClaraVerse/tree/main/docs/api) •
+[Tutorials](https://github.com/badboysm890/ClaraVerse/tree/main/docs/tutorials) •
+[FAQ](https://github.com/badboysm890/ClaraVerse/wiki/FAQ)
+
+
-### Clara Memory - Context-Aware Conversations
-
+## 💝 **Contributors**
-*Clara remembers your preferences and conversation context across sessions*
-
Clara's memory system: She can remember which is needed in Short Term Memory and Archive rest of the Memories that's not used very often
+
-
+
+
+
+
+
+
+### **How to Contribute**
-### Smart Multi-Agent Orchestration, Chat with Clara to Create your crew of agents
-
+We love contributions! Check our [Contributing Guide](CONTRIBUTING.md) to get started.
-*Coordinate multiple specialized AI agents for complex workflows*
-
Clara's Integrated Architecture allows Chat and Agents to use and share Integration and Automate the chat workflow automatically
+```mermaid
+graph LR
+ A[🍴 Fork] --> B[🔧 Code]
+ B --> C[✅ Test]
+ C --> D[📤 PR]
+ D --> E[🎉 Merge]
+```
+
-### Private AI Processing - Nothing needs to be stored on the server
-
+## 🙏 **Acknowledgments**
+
+
+
+**Built on the shoulders of giants:**
-*Browser-local storage ensures your data stays private - even server admins can't access your conversations*
+[llama.cpp](https://github.com/ggml-org/llama.cpp) •
+[llama-swap](https://github.com/mostlygeek/llama-swap) •
+[faster-whisper](https://github.com/SYSTRAN/faster-whisper) •
+[ComfyUI](https://github.com/comfyanonymous/ComfyUI) •
+[N8N](https://github.com/n8n-io/n8n)
+
+### **Special Thanks**
+
+☕ **Coffee Supporters** • 🌟 **Star Gazers** • 🐛 **Bug Hunters** • 💡 **Feature Suggesters**
----
+
+
+## 📄 **License**
+
+
-## Key Features
-
-### **Privacy & Security First**
-- **Browser-Local Storage**: Conversations stored in IndexedDB, never touch server—even admins can't read your chats
-- **Zero-Knowledge Architecture**: Server only proxies LLM API calls, doesn't log or store message content
-- **Per-Conversation Privacy**: Choose browser-local (100% offline) or cloud-sync (encrypted backup) per chat
-- **Local JWT Authentication**: Secure authentication with Argon2id password hashing
-- **True Offline Mode**: Works completely air-gapped after initial load—no server dependency
-
-### **Universal AI Access**
-- **Multi-Provider Support**: OpenAI, Anthropic Claude, Google Gemini, and any OpenAI-compatible endpoint
-- **Bring Your Own Key (BYOK)**: Use existing API accounts or free local models
-- **400+ Models Available**: From GPT-4o to Llama, Mistral, and specialized models
-- **Unified Interface**: One workspace for all your AI needs
-
-### **Advanced Capabilities**
-- **Visual Workflow Builder**: Drag-and-drop workflow designer with auto-layout—chat to create, visual editor to refine
-- **Hybrid Block Architecture**: Variable blocks, LLM blocks, and Code blocks (execute tools without LLM overhead)
-- **Interactive Prompts**: AI asks clarifying questions mid-conversation with typed forms (text, select, checkbox)
-- **Real-Time Streaming**: WebSocket-based chat with automatic reconnection and conversation resume
-- **Tool Execution**: Code generation, image creation, web search, file analysis with real-time status tracking
-- **Response Versioning**: Generate, compare, and track multiple versions (add details, make concise, no search)
-
-### **Cross-Platform & Flexible**
-- **Desktop Apps**: Native Windows, macOS, and Linux applications
-- **Web Interface**: Browser-based access via React frontend
-- **Mobile Ready**: Responsive design for tablets and phones
-- **P2P Sync**: Device-to-device synchronization without cloud storage
-- **Enterprise Deployment**: Self-host for complete organizational control
-
-### **Developer-Friendly**
-- **MCP Bridge**: Native Model Context Protocol support—connect any MCP-compatible tool seamlessly
-- **Open API**: RESTful + WebSocket APIs for custom integrations
-- **Plugin System**: Extend functionality with custom tools and connectors
-- **Docker Support**: One-command deployment with `docker compose`
-- **GitHub, Slack, Notion Integration**: Pre-built connectors for your workflow
-- **Database Connections**: Query and analyze data with AI assistance
+**MIT License** - Use it, modify it, sell it, we don't mind! (If want you to give credit, that would be awesome but not mandatory 😉)
+
+See [LICENSE](LICENSE) for details.
+
+
---
-## Our Mission
+
-**Building the best AI interface experience—without compromising your privacy.**
+
-While other AI tools force you to choose between features and privacy, ClaraVerse refuses that trade-off. We believe you deserve both: a powerful, intuitive interface AND complete data sovereignty.
+### **🚀 Ready to revolutionize your AI workflow?**
-### Why ClaraVerse is Different
+
-Most "privacy-focused" AI tools sacrifice usability for security. Open WebUI and others offer self-hosting, but you're still limited to basic chat interfaces. ClaraVerse goes further:
+
+
+
- **Best-in-Class Interface**
-- Intuitive, polished UI that rivals ChatGPT and Claude
-- Real-time streaming with automatic reconnection
-- Smart context management across sessions
-- Multi-modal support (text, images, code, files)
-- Clara Memory: Remembers what matters, archives what doesn't
+
+
- **Privacy WITHOUT Compromise**
-- **Browser-local storage**: Conversations in IndexedDB, never touch server/database
-- **Zero-knowledge architecture**: Server admins cannot read user chats—even in self-hosted deployments
-- **Per-conversation privacy**: Toggle browser-local (offline) vs cloud-sync (encrypted) per chat
-- **Air-gap capable**: Works 100% offline after initial load, no server dependency
-- **Local authentication**: JWT with Argon2id password hashing, no external auth services
-- **Open source (AGPL-3.0)**: Verify and audit security yourself
+**⭐ Star us** • **🍴 Fork us** • **💬 Join us**
- **Extensibility That Matters**
-- **MCP Bridge**: Native Model Context Protocol integration for seamless tool connections
-- Multi-agent orchestration: Coordinate specialized AI agents for complex workflows
-- 400+ models: OpenAI, Anthropic, Google, Gemini, and any OpenAI-compatible endpoint
-- BYOK: Use your own API keys or completely free local models
-- Plugin ecosystem: GitHub, Slack, Notion, databases, and custom integrations
+
- **All-in-One Platform**
-- Replaces ChatGPT (conversations), Midjourney (image generation), n8n (workflows)
-- **Visual workflow builder + chat** in one interface—chat to design, visual editor to execute
-- **Interactive prompts**: AI asks clarifying questions mid-conversation with typed forms
-- **Memory auto-archival**: Active memory management—keeps context focused without manual cleanup
-- Cross-platform: Desktop apps, web interface, mobile-ready
-- P2P sync: Device-to-device synchronization without cloud dependencies
+Made with ❤️ by developers who believe privacy is a right, not a privilege
-### Our Promise
+
-**Privacy-first doesn't mean features-last.** Every interface decision, every feature, every line of code is designed with this dual commitment:
+
-1. **Security by Default**: Your data, your keys, your control
-2. **Excellence by Design**: Experience that makes privacy feel effortless
+
+
-### Built For
+
-- **Individuals**: Super-powered AI workspace without surveillance
-- **Developers**: Open API, MCP bridge, plugin system, complete source access
-- **Teams**: Collaborate with AI while keeping confidential data on-premises
-- **Enterprises**: Deploy infrastructure that complies with strictest data sovereignty requirements (GDPR, HIPAA, SOC2)
+## 💭 **A Note From The Developer regarding incomplete features, bugs and docs**
-> **50,000+ downloads worldwide** | Join developers and privacy advocates who refuse to compromise
+
----
+
-## Documentation
+### **Building ClaraVerse**
-| Resource | Description |
-|----------|-------------|
-| [ Architecture Guide](backend/docs/ARCHITECTURE.md) | System design and component overview |
-| [ API Reference](backend/docs/API_REFERENCE.md) | REST and WebSocket API documentation |
-| [ Docker Guide](docs/DOCKER.md) | Comprehensive Docker deployment |
-| [ Security Guide](backend/docs/FINAL_SECURITY_INSPECTION.md) | Security features and best practices |
-| [ Admin Guide](backend/docs/ADMIN_GUIDE.md) | System administration and configuration |
-| [ Developer Guide](backend/docs/DEVELOPER_GUIDE.md) | Contributing and local development |
-| [ Quick Reference](backend/docs/QUICK_REFERENCE.md) | Common commands and workflows |
+
----
+
+
+
-## Architecture
+
-ClaraVerse is built with modern, production-ready technologies:
+### **Solo Developer**
-```
-┌─────────────────────────────────────────────────────────────┐
-│ Frontend Layer │
-│ React 19 + TypeScript + Tailwind CSS 4 │
-│ Zustand State + React Router 7 │
-└────────────────────┬────────────────────────────────────────┘
- │ WebSocket + REST API
-┌────────────────────▼────────────────────────────────────────┐
-│ Backend Layer │
-│ Go 1.24 + Fiber Framework │
-│ Real-time Streaming + Tool Execution │
-└────────────────────┬────────────────────────────────────────┘
- │
- ┌────────────┼────────────┐
- │ │ │
- ┌────▼───┐ ┌───▼────┐ ┌───▼────┐
- │MongoDB │ │ Redis │ │SearXNG │
- │Storage │ │ Jobs │ │ Search │
- └────────┘ └────────┘ └────────┘
-```
+`9-6 Day Job` `Night Coding`
+`Weekend Builds` `3 AM Commits`
-**Technology Stack:**
-- **Frontend**: React 19, TypeScript, Vite 7, Tailwind CSS 4, Zustand 5
-- **Backend**: Go 1.24, Fiber (web framework), WebSocket streaming
-- **Database**: MongoDB for persistence, MySQL for models/providers, Redis for caching/jobs
-- **Services**: SearXNG (search), E2B Local Docker (code execution - no API key!)
-- **Deployment**: Docker Compose, Nginx reverse proxy
-- **Auth**: Local JWT with Argon2id password hashing (v2.0 - fully local, no Supabase)
+**One person. Three platforms.**
+**Just having fun in doing this.**
----
+
+
-## Features in Detail
-
-### Real-Time Streaming Chat
-
-Experience instant AI responses with our WebSocket-based architecture:
-
-- **Chunked Streaming**: See responses as they're generated
-- **Connection Recovery**: Automatic reconnection with conversation resume
-- **Heartbeat System**: Maintains stable connections through proxies
-- **Multi-User Support**: Concurrent conversations without interference
-
-### Tool Execution Engine
-
-Extend AI capabilities beyond text:
-
-| Tool | Description | Example |
-|------|-------------|---------|
-| **Code Generation** | Execute Python, JavaScript, Go code in sandboxed E2B environment | "Write and run a script to analyze this CSV" |
-| **Image Generation** | Create images with DALL-E, Stable Diffusion, or local models | "Generate a logo for my startup" |
-| **Web Search** | Real-time internet search via SearXNG | "What are the latest AI developments?" |
-| **File Analysis** | Process PDFs, images, documents with vision models | "Summarize this 50-page report" |
-| **Data Query** | Connect to databases and run SQL queries | "Show sales trends from our PostgreSQL" |
-
-### Bring Your Own Key (BYOK)
-
-Use your existing AI subscriptions:
-
-1. Add your API keys in `backend/providers.json`
-2. Configure model preferences and rate limits
-3. Switch between providers seamlessly
-4. Or use completely free local models (Ollama, LM Studio)
-
-```json
-{
- "providers": [
- {
- "name": "OpenAI",
- "api_key": "sk-your-key",
- "models": ["gpt-4o", "gpt-4o-mini"]
- },
- {
- "name": "Anthropic",
- "api_key": "your-key",
- "models": ["claude-3-5-sonnet", "claude-3-opus"]
- }
- ]
-}
-```
+
-### Multi-Agent Orchestration
+### **Community Heroes**
-Coordinate multiple AI agents for complex workflows:
+`Discord Testers` `Bug Hunters`
+`Coffee Supporters` `PR Contributors`
-- **Specialized Agents**: Create agents with specific roles (researcher, coder, analyst)
-- **Agent Collaboration**: Agents can communicate and share context
-- **Workflow Automation**: Chain agent tasks for multi-step processes
-- **Custom Instructions**: Define agent behavior with natural language
+**You make this possible.**
+**Every single day.**
----
+
+
+
-## Use Cases
-
-### For Developers
-- **Code Review & Debugging**: Get instant feedback on code quality
-- **Documentation Generation**: Auto-generate docs from codebases
-- **API Integration**: Connect ClaraVerse to your development workflow
-- **Database Analysis**: Query and visualize data with AI assistance
-
-### For Businesses
-- **Zero-Liability Hosting**: Host for teams without server-side chat storage—admins can't access conversations
-- **True Data Sovereignty**: Browser-local mode means data never leaves employee devices, even when self-hosted
-- **Simplified Compliance**: No message retention in database = easier GDPR/HIPAA compliance
-- **Team Collaboration**: Shared AI workspace with access control and privacy guarantees
-- **Custom Integrations**: Connect to Slack, Notion, GitHub, CRMs via visual workflow builder
-- **Cost Control**: BYOK means you control AI spending with your own API keys
-
-### For Privacy Advocates
-- **Browser-Local Storage**: Conversations never touch server—even when self-hosted, admins can't read chats
-- **Zero-Knowledge Architecture**: Server only proxies API calls, doesn't log or store message content
-- **Per-Conversation Privacy**: Choose offline browser-local or encrypted cloud-sync per conversation
-- **Air-Gapped Operation**: Works 100% offline after initial load—no server dependency
-- **Open Source**: Verify zero-knowledge claims yourself or hire security auditors
-- **No Database Retention**: Messages not stored in MongoDB—simplified compliance for GDPR/HIPAA
-
-### For Researchers
-- **Experiment with Models**: Test 400+ models in one interface
-- **Dataset Analysis**: Process large datasets with AI assistance
-- **Literature Review**: Search and summarize academic papers
-- **Reproducible Workflows**: Save and share AI-assisted research processes
+
----
+
+📖 Read the Full Story
-## Roadmap
-
-### ✅ Completed (v2.0 - Current Version)
-- [x] **Browser-local storage** (IndexedDB) with zero-knowledge architecture
-- [x] **Visual workflow builder** with drag-and-drop interface and auto-layout
-- [x] **Interactive prompts** (AI asks questions mid-conversation with typed forms)
-- [x] **Per-conversation privacy toggle** (browser-local vs cloud-sync)
-- [x] Multi-provider LLM support (OpenAI, Anthropic, Google, OpenAI-compatible)
-- [x] Real-time WebSocket streaming with automatic reconnection
-- [x] Tool execution (code, image generation, web search)
-- [x] Response versioning (regenerate, add details, make concise, etc.)
-- [x] Memory system with auto-archival and scoring
-- [x] Hybrid block architecture (Variable, LLM, Code blocks)
-- [x] Docker-based deployment
-- [x] BYOK (Bring Your Own Key) functionality
-- [x] MongoDB + MySQL + Redis infrastructure
-- [x] **Local JWT authentication** (v2.0 - replaced Supabase, fully offline)
-- [x] **E2B Local Docker mode** (v2.0 - code execution without API key)
-- [x] **Removed payment processing** (v2.0 - all users Pro tier by default)
-- [x] **Removed CAPTCHA** (v2.0 - rate limiting only)
-- [x] **100% offline core functionality** (v2.0 - no external service dependencies)
-- [x] File upload support with previews (images, PDFs, documents, CSV, audio)
-- [x] Markdown rendering with reasoning extraction
-
-### 🚧 In Progress (v1.1)
-- [ ] Desktop applications (Windows, macOS, Linux)
-- [ ] Mobile apps (iOS, Android)
-- [ ] P2P device synchronization
-- [ ] Enhanced multi-agent orchestration
-- [ ] Plugin marketplace
-
-### 🔮 Planned (v2.0 and beyond)
-- [ ] Local LLM integration (Ollama, LM Studio native support)
-- [ ] Voice input/output
-- [ ] Advanced RAG (Retrieval-Augmented Generation)
-- [ ] Workspace collaboration features
-- [ ] Browser extension
-- [ ] Kubernetes deployment templates
-- [ ] Enterprise SSO integration
-
-[View full roadmap →](https://github.com/claraverse-space/ClaraVerseAI/projects)
+
----
+> **"Why are some docs incomplete? Why does this feature need work?"**
-## Contributing
+Here's the honest truth...
-We welcome contributions from developers of all skill levels! ClaraVerse is built by the community, for the community.
+I'm a **single developer** building ClaraVerse during nights and weekends while working a **9-6 day job**. I'm maintaining **3 platforms**, fixing bugs, adding features, answering Discord questions, reviewing PRs, and somehow trying to keep documentation updated.
-### How to Contribute
+
-1. **Fork** the repository
-2. **Create** a feature branch: `git checkout -b feature/amazing-feature`
-3. **Make** your changes and add tests
-4. **Run** linting: `npm run lint && go vet ./...`
-5. **Commit** with clear messages: `git commit -m 'Add amazing feature'`
-6. **Push** to your fork: `git push origin feature/amazing-feature`
-7. **Open** a Pull Request with a detailed description
+### **🌟 The Amazing People Making This Possible**
-### Contribution Areas
+
+
+
-We especially welcome help in these areas:
+**💬 Discord Community**
+
+UI Testing • Feature Ideas • Bug Reports
-- **Bug Fixes**: Check [open issues](https://github.com/claraverse-space/ClaraVerseAI/issues)
-- **Documentation**: Improve guides, add examples, fix typos
-- **Translations**: Help us reach non-English speakers
-- **UI/UX**: Design improvements and accessibility
-- **Testing**: Add unit tests, integration tests, E2E tests
-- **Integrations**: Build connectors for new tools and services
-- **Models**: Add support for new LLM providers
+
+
-### Development Setup
+**👨💻 @aruntemme**
+
+ClaraCore Dev • Notebook Features
-See [DEVELOPER_GUIDE.md](backend/docs/DEVELOPER_GUIDE.md) for detailed instructions.
+
+
-Quick start for contributors:
+**☕ Coffee Supporters**
+
+Keeping 3 AM Sessions Alive
-```bash
-# Install dependencies
-make install
+
+
-# Start development environment with hot reload
-./dev.sh
+**🐛 Issue Reporters**
+
+Detailed Reports • Patience
-# Run tests
-cd frontend && npm run test
-cd backend && go test ./...
+
+
+
-# Check code quality
-npm run lint && npm run format
-go vet ./... && go fmt ./...
-```
+
-### Code of Conduct
+**The Reality Check:**
+- Docs written with **Claude & GPT** during my **2-3 hours of free time**
+- Some nights too exhausted to write proper commit messages
+- Every feature request matters, but time is limited
+- Your patience literally keeps this project alive
-We are committed to providing a welcoming and inclusive environment. Please read our [Code of Conduct](CODE_OF_CONDUCT.md) before participating.
+
----
+
-## Community
+**🙏 Thank you for being part of this crazy journey**
-Join thousands of privacy-conscious developers and AI enthusiasts:
+*Building an AI platform alone is insane.*
+*Your support makes it possible.*
-- **[Discord](https://discord.com/invite/j633fsrAne)**: Real-time chat and support
-- **[Twitter/X](https://x.com/clara_verse_)**: Updates and announcements
-- **[TikTok](https://www.tiktok.com/@claraversehq)**: Short-form content and demos
-- **[Newsletter](https://claraverse.space/newsletter)**: Monthly updates and tips
-- **[YouTube](https://www.youtube.com/@ClaraVerseAI)**: Tutorials and demos
-- **[LinkedIn](https://linkedin.com/company/claraverse)**: Professional updates
+— @badboysm890, probably debugging at 3 AM ☕
-### Show Your Support
+
-If ClaraVerse has helped you, consider:
+
-- **Star** this repository
-- **Report bugs** and suggest features
-- **Share** with colleagues and on social media
-- **Sponsor** development ([GitHub Sponsors](https://github.com/sponsors/claraverse-space))
-- **Contribute** code, docs, or designs
+
+
+
---
+
-## License
+## ❓ **Frequently Asked Questions**
-ClaraVerse is licensed under the **GNU Affero General Public License v3.0 (AGPL-3.0)** - see the [LICENSE](LICENSE) file for details.
+
-### What This Means:
+### **Quick Answers to Common Questions**
-**You ARE free to:**
-- ✅ **Use commercially** - Host ClaraVerse as a service, even for profit
-- ✅ **Modify** - Customize and improve the software
-- ✅ **Distribute** - Share with others
-- ✅ **Private use** - Use internally in your organization
+
-**BUT you MUST:**
-- 📤 **Share modifications** - Any changes must be open-sourced under AGPL-3.0
-- 🌐 **Network copyleft** - If you host ClaraVerse as a service, users must have access to your source code
-- 📝 **Credit developers** - Preserve copyright and license notices
-- 🔓 **Give back to the community** - Improvements benefit everyone
+
-### Why AGPL-3.0?
+
-We chose AGPL-3.0 to ensure that:
-1. **ClaraVerse remains free forever** - No one can take it private
-2. **The community benefits from all improvements** - Even from hosted/SaaS deployments
-3. **Developers get credit** - Your contributions are always attributed
-4. **Big tech gives back** - Companies using ClaraVerse must contribute improvements
+
+
+ 🚀 Installation & Setup
+
-**ClaraVerse is and will remain free and open-source forever.**
+
----
+
+
+
-## Acknowledgments
+### **System Requirements**
-ClaraVerse is built on the shoulders of giants. Special thanks to:
+| Platform | Requirements |
+|:---------|:------------|
+| **Windows** | Win 10/11 • 8GB RAM |
+| **macOS** | 10.15+ • M1/Intel |
+| **Linux** | Ubuntu 18.04+ • FUSE |
-- **[Go Fiber](https://gofiber.io/)** - Lightning-fast web framework
-- **[React](https://react.dev/)** - UI library
-- **[Anthropic](https://anthropic.com/)**, **[OpenAI](https://openai.com/)**, **[Google](https://ai.google.dev/)** - AI model providers
-- **[SearXNG](https://github.com/searxng/searxng)** - Privacy-respecting search
-- **[E2B](https://e2b.dev/)** - Code execution sandboxes (now running in local Docker mode!)
-- **[Argon2](https://github.com/P-H-C/phc-winner-argon2)** - Password hashing library
-- All our [contributors](https://github.com/claraverse-space/ClaraVerseAI/graphs/contributors) and community members
+
+
-**Note**: v2.0 moved from Supabase to local JWT authentication for complete offline capability
+### **Common Issues**
----
+**🛡️ Windows Protection Warning?**
+```
+More Info → Run Anyway
+(App is safe but unsigned)
+```
-## Troubleshooting
+**🍎 macOS Won't Open?**
+```bash
+sudo xattr -r -d com.apple.quarantine \
+ /Applications/ClaraVerse.app
+```
-### Common Issues
+
+
+
+💡 Pro Tip: Multi-Tool Usage
+
+
+
+You can run **all tools simultaneously** - that's the magic of ClaraVerse!
+Clara AI + LumaUI + ComfyUI can all share context and work together.
+
+
-1. Ensure file exists (copy from `providers.example.json`)
-2. Check API keys are valid
-3. Verify `enabled: true` for each provider
-4. Restart backend: `docker compose restart backend`
-Build failures
+
+ 🤖 Autonomous Agents & MCP
+
+
+
-**Solution**: Ensure you have the correct versions:
+
+
+
-```bash
-go version # Should be 1.24+
-node --version # Should be 20+
-python --version # Should be 3.11+
-```
+### **Autonomous Agent Features**
+
+✅ **Self-Executing Workflows**
+- Autonomous task execution
+- Multi-step reasoning
+- Error self-correction
+- Chain-of-thought processing
+
+✅ **MCP Tool Integration**
+- 20+ available MCP servers
+- Desktop automation
+- Browser control
+- File system access
+
+
-**Built with ❤️ by the ClaraVerse Community**
+### **🆘 Still Need Help?**
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
-*Pioneering the new age of private, powerful AI for the Super Individual*
+
-[⬆ Back to Top](#️-claraverse)
+
+
+Can't find your answer? Join our Discord for real-time help!
+
+
diff --git a/assets/entitlements.mac.plist b/assets/entitlements.mac.plist
new file mode 100644
index 00000000..8d30e7bc
--- /dev/null
+++ b/assets/entitlements.mac.plist
@@ -0,0 +1,22 @@
+
+
+
+
+ com.apple.security.cs.allow-jit
+
+ com.apple.security.cs.allow-unsigned-executable-memory
+
+ com.apple.security.cs.allow-dyld-environment-variables
+
+ com.apple.security.network.client
+
+ com.apple.security.network.server
+
+ com.apple.security.files.user-selected.read-write
+
+ com.apple.security.inherit
+
+ com.apple.security.automation.apple-events
+
+
+
diff --git a/assets/icons/128x128.png b/assets/icons/128x128.png
new file mode 100644
index 00000000..6de372a7
Binary files /dev/null and b/assets/icons/128x128.png differ
diff --git a/assets/icons/16x16.png b/assets/icons/16x16.png
new file mode 100644
index 00000000..7e46356d
Binary files /dev/null and b/assets/icons/16x16.png differ
diff --git a/assets/icons/24x24.png b/assets/icons/24x24.png
new file mode 100644
index 00000000..0fcb05cf
Binary files /dev/null and b/assets/icons/24x24.png differ
diff --git a/assets/icons/256x256.png b/assets/icons/256x256.png
new file mode 100644
index 00000000..87b906d2
Binary files /dev/null and b/assets/icons/256x256.png differ
diff --git a/assets/icons/32x32.png b/assets/icons/32x32.png
new file mode 100644
index 00000000..75fb4899
Binary files /dev/null and b/assets/icons/32x32.png differ
diff --git a/assets/icons/48x48.png b/assets/icons/48x48.png
new file mode 100644
index 00000000..d5c0c4ec
Binary files /dev/null and b/assets/icons/48x48.png differ
diff --git a/assets/icons/512x512.png b/assets/icons/512x512.png
new file mode 100644
index 00000000..5da5eb41
Binary files /dev/null and b/assets/icons/512x512.png differ
diff --git a/assets/icons/64x64.png b/assets/icons/64x64.png
new file mode 100644
index 00000000..61f78c45
Binary files /dev/null and b/assets/icons/64x64.png differ
diff --git a/docs/images/logo.png b/assets/icons/logo.png
similarity index 100%
rename from docs/images/logo.png
rename to assets/icons/logo.png
diff --git a/assets/icons/png/128x128.png b/assets/icons/png/128x128.png
new file mode 100644
index 00000000..6de372a7
Binary files /dev/null and b/assets/icons/png/128x128.png differ
diff --git a/assets/icons/png/16x16.png b/assets/icons/png/16x16.png
new file mode 100644
index 00000000..7e46356d
Binary files /dev/null and b/assets/icons/png/16x16.png differ
diff --git a/assets/icons/png/24x24.png b/assets/icons/png/24x24.png
new file mode 100644
index 00000000..0fcb05cf
Binary files /dev/null and b/assets/icons/png/24x24.png differ
diff --git a/assets/icons/png/256x256.png b/assets/icons/png/256x256.png
new file mode 100644
index 00000000..87b906d2
Binary files /dev/null and b/assets/icons/png/256x256.png differ
diff --git a/assets/icons/png/32x32.png b/assets/icons/png/32x32.png
new file mode 100644
index 00000000..75fb4899
Binary files /dev/null and b/assets/icons/png/32x32.png differ
diff --git a/assets/icons/png/48x48.png b/assets/icons/png/48x48.png
new file mode 100644
index 00000000..d5c0c4ec
Binary files /dev/null and b/assets/icons/png/48x48.png differ
diff --git a/assets/icons/png/512x512.png b/assets/icons/png/512x512.png
new file mode 100644
index 00000000..5da5eb41
Binary files /dev/null and b/assets/icons/png/512x512.png differ
diff --git a/assets/icons/png/64x64.png b/assets/icons/png/64x64.png
new file mode 100644
index 00000000..61f78c45
Binary files /dev/null and b/assets/icons/png/64x64.png differ
diff --git a/assets/icons/png/README.md b/assets/icons/png/README.md
new file mode 100644
index 00000000..06635b6a
--- /dev/null
+++ b/assets/icons/png/README.md
@@ -0,0 +1,29 @@
+# Icon Requirements for Linux
+
+For proper icon display in Linux distributions, the following icon sizes are required:
+
+- 16x16.png
+- 24x24.png
+- 32x32.png
+- 48x48.png
+- 64x64.png
+- 128x128.png
+- 256x256.png
+- 512x512.png
+
+Please ensure all PNG icons are properly optimized and follow these standard sizes.
+
+You can create these from your original logo.png using tools like ImageMagick:
+
+```bash
+convert logo.png -resize 16x16 16x16.png
+convert logo.png -resize 24x24 24x24.png
+convert logo.png -resize 32x32 32x32.png
+convert logo.png -resize 48x48 48x48.png
+convert logo.png -resize 64x64 64x64.png
+convert logo.png -resize 128x128 128x128.png
+convert logo.png -resize 256x256 256x256.png
+convert logo.png -resize 512x512 512x512.png
+```
+
+This directory structure is specifically referenced in the electron-builder configuration for Linux builds.
diff --git a/frontend/src/assets/logo.png b/assets/icons/png/logo.png
similarity index 100%
rename from frontend/src/assets/logo.png
rename to assets/icons/png/logo.png
diff --git a/assets/icons/win/icon.ico b/assets/icons/win/icon.ico
new file mode 100644
index 00000000..763db1ec
Binary files /dev/null and b/assets/icons/win/icon.ico differ
diff --git a/backend/.air.toml b/backend/.air.toml
deleted file mode 100644
index 01128182..00000000
--- a/backend/.air.toml
+++ /dev/null
@@ -1,44 +0,0 @@
-root = "."
-testdata_dir = "testdata"
-tmp_dir = "tmp"
-
-[build]
- args_bin = []
- bin = "./tmp/main"
- cmd = "go build -o ./tmp/main ./cmd/server"
- delay = 1000
- exclude_dir = ["assets", "tmp", "vendor", "testdata"]
- exclude_file = []
- exclude_regex = ["_test.go"]
- exclude_unchanged = false
- follow_symlink = false
- full_bin = ""
- include_dir = []
- include_ext = ["go", "tpl", "tmpl", "html"]
- include_file = []
- kill_delay = "0s"
- log = "build-errors.log"
- poll = false
- poll_interval = 0
- rerun = false
- rerun_delay = 500
- send_interrupt = false
- stop_on_error = false
-
-[color]
- app = ""
- build = "yellow"
- main = "magenta"
- runner = "green"
- watcher = "cyan"
-
-[log]
- main_only = false
- time = false
-
-[misc]
- clean_on_exit = false
-
-[screen]
- clear_on_rebuild = false
- keep_scroll = true
diff --git a/backend/.dockerignore b/backend/.dockerignore
deleted file mode 100644
index 76cfcce8..00000000
--- a/backend/.dockerignore
+++ /dev/null
@@ -1,66 +0,0 @@
-# Git
-.git
-.gitignore
-.gitattributes
-
-# Documentation
-*.md
-docs/
-README.md
-
-# Environment files (will be mounted at runtime)
-.env
-.env.local
-.env.*.local
-
-# Database files (will be in persistent volumes)
-*.db
-*.db-shm
-*.db-wal
-
-# Secure files (not for container)
-secure_files/
-
-# Provider configuration (will be mounted at runtime)
-providers.json
-
-# Uploads directory (will be in persistent volumes)
-uploads/
-*.tmp
-
-# Logs
-*.log
-logs/
-
-# Build artifacts
-claraverse
-*.exe
-*.exe~
-*.so
-*.dylib
-dist/
-build/
-
-# MCP bridge builds (separate binary)
-mcp-bridge/mcp-client
-mcp-bridge/mcp-client.exe
-mcp-bridge/*.exe~
-
-# IDE and editor files
-.vscode/
-.idea/
-*.swp
-*.swo
-*~
-.DS_Store
-
-# Testing
-*_test.go
-test/
-coverage.txt
-*.out
-
-# Temporary files
-tmp/
-temp/
-*.tmp
diff --git a/backend/.env.example b/backend/.env.example
deleted file mode 100644
index b296b730..00000000
--- a/backend/.env.example
+++ /dev/null
@@ -1,74 +0,0 @@
-# Server Configuration
-PORT=3001
-ENVIRONMENT=development # Options: development, testing, production (REQUIRED)
-
-# MongoDB Configuration
-# Flexible deployment - use any MongoDB instance:
-# - Local Docker: mongodb://localhost:27017/claraverse
-# - MongoDB Atlas: mongodb+srv://user:pass@cluster.mongodb.net/claraverse
-# - Self-hosted: mongodb://user:pass@your-server:27017/claraverse
-MONGODB_URI=mongodb://localhost:27017/claraverse
-
-# Encryption Configuration
-# Generate a secure 32-byte hex key: openssl rand -hex 32
-# This key is used to encrypt user data (workflows, conversations, executions)
-# WARNING: Losing this key means losing access to all encrypted data!
-ENCRYPTION_MASTER_KEY=
-
-# Provider Configuration
-PROVIDERS_FILE=providers.json
-
-# Supabase Authentication (REQUIRED in production, optional in development)
-# Get these from https://app.supabase.com/project/_/settings/api
-# SUPABASE_URL: Your project URL (e.g., https://xxxxx.supabase.co)
-# SUPABASE_KEY: Your service role key (not anon key) for backend validation
-# WARNING: Leaving these empty in production will cause the server to terminate
-SUPABASE_URL=
-SUPABASE_KEY=
-
-# SearXNG Web Search (Optional)
-# Docker: Use http://searxng:8080 (automatically provided in docker-compose)
-# Local dev: Set up your own SearXNG instance at http://localhost:8080
-SEARXNG_URL=http://searxng:8080
-
-# File Upload Configuration
-# Directory for storing uploaded files
-UPLOAD_DIR=./uploads
-# Maximum file size in bytes (20MB default)
-MAX_FILE_SIZE=20971520
-
-# Security Configuration
-# CORS: Comma-separated list of allowed origins (REQUIRED in production)
-# Development: http://localhost:5173,http://localhost:3000
-# Production: https://yourdomain.com,https://www.yourdomain.com
-ALLOWED_ORIGINS=http://localhost:5173,http://localhost:3000
-
-# FRONTEND_URL: The URL of your frontend (REQUIRED for payment redirects)
-# This is where users will be redirected after completing payment
-# Development: http://localhost:5173 or http://localhost:3001
-# Production: https://yourdomain.com
-# Ngrok: https://your-ngrok-url.ngrok-free.app
-FRONTEND_URL=http://localhost:5173
-# Backend Public URL (for generating absolute download URLs)
-# Used by tools to generate full URLs that LLMs can use directly
-# Docker: http://localhost:3001 (or your public domain in production)
-# Production: https://api.yourdomain.com
-BACKEND_URL=http://localhost:3001
-
-# E2B Code Interpreter Service (REQUIRED for Python code execution)
-# Get your API key from https://e2b.dev/dashboard
-# Example: E2B_API_KEY=e2b_xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
-E2B_API_KEY=
-# E2B Service URL (automatically set in Docker, use http://localhost:8001 for local dev)
-E2B_SERVICE_URL=http://e2b-service:8001
-
-# Promotional Campaign Configuration
-# Enable this to give new signups a temporary pro plan during the promotional period
-# Example: January 2025 - give all new users 30 days of pro plan
-PROMO_ENABLED=false
-# Start date in RFC3339 format (UTC timezone)
-PROMO_START_DATE=2025-01-01T00:00:00Z
-# End date in RFC3339 format (UTC timezone) - exclusive
-PROMO_END_DATE=2025-02-01T00:00:00Z
-# Duration of promotional pro plan in days (from signup date)
-PROMO_DURATION_DAYS=30
diff --git a/backend/.gitignore b/backend/.gitignore
deleted file mode 100644
index a01d610f..00000000
--- a/backend/.gitignore
+++ /dev/null
@@ -1,33 +0,0 @@
-# Configuration files with secrets
-providers.json
-.env
-
-# Database
-model_capabilities.db
-*.db
-
-# Logs
-*.log
-
-# IDE
-.vscode/
-.idea/
-.claude/
-
-# Build artifacts
-claraverse-server
-claraverse-server.exe
-*.exe
-
-# OS
-.DS_Store
-Thumbs.db
-
-# Air hot reload
-tmp/
-*.md
-
-*.venv/
-# Air hot reload temp directory
-tmp/
-build-errors.log
diff --git a/backend/Dockerfile b/backend/Dockerfile
deleted file mode 100644
index 21a5f9c5..00000000
--- a/backend/Dockerfile
+++ /dev/null
@@ -1,112 +0,0 @@
-# Stage 1: Builder
-FROM golang:1.25.5-alpine AS builder
-
-# Install build dependencies for CGO (required for SQLite)
-RUN apk add --no-cache gcc musl-dev
-
-# Set working directory
-WORKDIR /build
-
-# Copy dependency files first (for better layer caching)
-COPY go.mod go.sum ./
-
-# Download dependencies
-RUN go mod download
-
-# Copy source code
-COPY . .
-
-# Run critical tests before building (fail fast if tests fail)
-# This ensures no broken code makes it into production
-ARG SKIP_TESTS=false
-RUN if [ "$SKIP_TESTS" = "false" ]; then \
- echo "=== Running test suite ===" && \
- echo "--- Testing database ---" && \
- CGO_ENABLED=1 go test -race -timeout 120s ./internal/database/... && \
- echo "--- Testing models ---" && \
- CGO_ENABLED=1 go test -race -timeout 120s ./internal/models/... && \
- echo "--- Testing services ---" && \
- CGO_ENABLED=1 go test -race -timeout 120s ./internal/services/... && \
- echo "--- Testing tools (file I/O) ---" && \
- CGO_ENABLED=1 go test -race -timeout 120s ./internal/tools/... && \
- echo "--- Testing execution (workflows) ---" && \
- CGO_ENABLED=1 go test -race -timeout 120s ./internal/execution/... && \
- echo "--- Testing filecache ---" && \
- CGO_ENABLED=1 go test -race -timeout 120s ./internal/filecache/... && \
- echo "--- Testing audio service ---" && \
- CGO_ENABLED=1 go test -race -timeout 120s ./internal/audio/... && \
- echo "--- Testing vision service ---" && \
- CGO_ENABLED=1 go test -race -timeout 120s ./internal/vision/... && \
- echo "--- Testing securefile service ---" && \
- CGO_ENABLED=1 go test -race -timeout 120s ./internal/securefile/... && \
- echo "--- Testing preflight ---" && \
- CGO_ENABLED=1 go test -race -timeout 120s ./internal/preflight/... && \
- echo "--- Testing integration ---" && \
- CGO_ENABLED=1 go test -race -timeout 120s ./tests/... && \
- echo "=== All tests passed ==="; \
- else echo "=== Skipping tests ==="; fi
-
-# Build the application with CGO enabled (required for modernc.org/sqlite)
-# -ldflags "-s -w" strips debug info to reduce binary size
-RUN CGO_ENABLED=1 GOOS=linux go build -ldflags "-s -w" -o claraverse ./cmd/server
-
-# Stage 2: Runtime
-FROM alpine:latest
-
-# Install runtime dependencies including Chromium for PDF generation
-# Includes emoji fonts and extended Unicode symbol support for PDF rendering
-RUN apk add --no-cache \
- ca-certificates \
- tzdata \
- wget \
- chromium \
- nss \
- freetype \
- harfbuzz \
- ttf-freefont \
- font-noto \
- font-noto-emoji \
- font-noto-cjk \
- fontconfig
-
-# Rebuild font cache to register all installed fonts
-RUN fc-cache -f -v
-
-# Create non-root user and group
-RUN addgroup -g 1000 claraverse && \
- adduser -D -u 1000 -G claraverse claraverse
-
-# Set working directory
-WORKDIR /app
-
-# Create necessary directories with proper permissions
-RUN mkdir -p /app/data /app/config /app/uploads /app/logs /app/generated /app/secure_files && \
- chown -R claraverse:claraverse /app
-
-# Set environment variables for Chromium
-ENV CHROME_BIN=/usr/bin/chromium-browser \
- CHROME_PATH=/usr/lib/chromium/
-
-# Copy binary from builder stage
-COPY --from=builder --chown=claraverse:claraverse /build/claraverse /app/claraverse
-
-# Copy example configuration files (if they exist)
-COPY --chown=claraverse:claraverse providers.example.json /app/providers.example.json
-
-# Copy and set permissions for entrypoint script
-COPY --chown=claraverse:claraverse docker-entrypoint.sh /app/docker-entrypoint.sh
-RUN sed -i 's/\r$//' /app/docker-entrypoint.sh && chmod +x /app/docker-entrypoint.sh
-
-# Switch to non-root user
-USER claraverse
-
-# Expose port
-EXPOSE 3001
-
-# Health check
-HEALTHCHECK --interval=30s --timeout=3s --start-period=10s --retries=3 \
- CMD wget --no-verbose --tries=1 --spider http://localhost:3001/health || exit 1
-
-# Set entrypoint and command
-ENTRYPOINT ["/app/docker-entrypoint.sh"]
-CMD ["/app/claraverse"]
diff --git a/backend/Dockerfile.dev b/backend/Dockerfile.dev
deleted file mode 100644
index c0673fc1..00000000
--- a/backend/Dockerfile.dev
+++ /dev/null
@@ -1,22 +0,0 @@
-FROM golang:1.25.5-alpine
-
-# Install air for hot reload
-RUN go install github.com/air-verse/air@latest
-
-# Install build dependencies
-RUN apk add --no-cache gcc musl-dev
-
-WORKDIR /app
-
-# Copy go mod files
-COPY go.mod go.sum ./
-RUN go mod download
-
-# Copy source
-COPY . .
-
-# Expose port
-EXPOSE 3001
-
-# Run with air for hot reload
-CMD ["air", "-c", ".air.toml"]
diff --git a/backend/cmd/server/main.go b/backend/cmd/server/main.go
deleted file mode 100644
index 571ac8cf..00000000
--- a/backend/cmd/server/main.go
+++ /dev/null
@@ -1,1686 +0,0 @@
-package main
-
-import (
- "claraverse/internal/config"
- "claraverse/internal/crypto"
- "claraverse/internal/database"
- "claraverse/internal/document"
- "claraverse/internal/execution"
- "claraverse/internal/filecache"
- "claraverse/internal/handlers"
- "claraverse/internal/jobs"
- "claraverse/internal/middleware"
- "claraverse/internal/models"
- "claraverse/internal/preflight"
- "claraverse/internal/services"
- "claraverse/internal/tools"
- "claraverse/pkg/auth"
- "context"
- "fmt"
- "log"
- "os"
- "os/signal"
- "path/filepath"
- "strings"
- "syscall"
- "time"
-
- "github.com/ansrivas/fiberprometheus/v2"
- "github.com/fsnotify/fsnotify"
- "github.com/gofiber/contrib/websocket"
- "github.com/gofiber/fiber/v2"
- "github.com/gofiber/fiber/v2/middleware/cors"
- "github.com/gofiber/fiber/v2/middleware/limiter"
- "github.com/gofiber/fiber/v2/middleware/logger"
- "github.com/gofiber/fiber/v2/middleware/recover"
- "github.com/joho/godotenv"
-)
-
-func main() {
- log.SetFlags(log.LstdFlags | log.Lshortfile)
- log.Println("🚀 Starting ClaraVerse Server...")
-
- // Load .env file (ignore error if file doesn't exist)
- if err := godotenv.Load(); err != nil {
- log.Printf("⚠️ No .env file found or error loading it: %v", err)
- } else {
- log.Println("✅ .env file loaded successfully")
- }
-
- // Load configuration
- cfg := config.Load()
- log.Printf("📋 Configuration loaded (Port: %s, DB: MySQL)", cfg.Port)
-
- // Initialize MySQL database
- if cfg.DatabaseURL == "" {
- log.Fatal("❌ DATABASE_URL environment variable is required (mysql://user:pass@host:port/dbname?parseTime=true)")
- }
- db, err := database.New(cfg.DatabaseURL)
- if err != nil {
- log.Fatalf("❌ Failed to connect to database: %v", err)
- }
- defer db.Close()
-
- if err := db.Initialize(); err != nil {
- log.Fatalf("❌ Failed to initialize database: %v", err)
- }
-
- // Initialize MongoDB (optional - for builder conversations and user data)
- var mongoDB *database.MongoDB
- var encryptionService *crypto.EncryptionService
- var userService *services.UserService
- var builderConvService *services.BuilderConversationService
-
- mongoURI := os.Getenv("MONGODB_URI")
- if mongoURI != "" {
- log.Println("🔗 Connecting to MongoDB...")
- var err error
- mongoDB, err = database.NewMongoDB(mongoURI)
- if err != nil {
- log.Printf("⚠️ Failed to connect to MongoDB: %v (builder features disabled)", err)
- } else {
- defer mongoDB.Close(context.Background())
- log.Println("✅ MongoDB connected successfully")
-
- // Initialize encryption service
- masterKey := os.Getenv("ENCRYPTION_MASTER_KEY")
- if masterKey != "" {
- encryptionService, err = crypto.NewEncryptionService(masterKey)
- if err != nil {
- log.Printf("⚠️ Failed to initialize encryption: %v", err)
- } else {
- log.Println("✅ Encryption service initialized")
- }
- } else {
- // SECURITY: In production, encryption is required when MongoDB is enabled
- environment := os.Getenv("ENVIRONMENT")
- if environment == "production" {
- log.Fatal("❌ CRITICAL SECURITY ERROR: ENCRYPTION_MASTER_KEY is required in production when MongoDB is enabled. Generate with: openssl rand -hex 32")
- }
- log.Println("⚠️ ENCRYPTION_MASTER_KEY not set - conversation encryption disabled (development mode only)")
- }
-
- // Initialize user service
- userService = services.NewUserService(mongoDB, cfg, nil) // usageLimiter set later
- log.Println("✅ User service initialized")
-
- // Initialize builder conversation service
- if encryptionService != nil {
- builderConvService = services.NewBuilderConversationService(mongoDB, encryptionService)
- log.Println("✅ Builder conversation service initialized")
- }
- }
- } else {
- log.Println("⚠️ MONGODB_URI not set - builder conversation persistence disabled")
- }
-
- // Initialize chat sync service (requires MongoDB + EncryptionService)
- var chatSyncService *services.ChatSyncService
- if mongoDB != nil && encryptionService != nil {
- chatSyncService = services.NewChatSyncService(mongoDB, encryptionService)
- // Ensure indexes
- if err := chatSyncService.EnsureIndexes(context.Background()); err != nil {
- log.Printf("⚠️ Failed to ensure chat sync indexes: %v", err)
- }
- log.Println("✅ Chat sync service initialized (encrypted cloud storage)")
- }
-
- // Initialize Redis service (for scheduler + pub/sub)
- var redisService *services.RedisService
- var schedulerService *services.SchedulerService
- var executionLimiter *middleware.ExecutionLimiter
-
- if cfg.RedisURL != "" {
- log.Println("🔗 Connecting to Redis...")
- var err error
- redisService, err = services.NewRedisService(cfg.RedisURL)
- if err != nil {
- log.Printf("⚠️ Failed to connect to Redis: %v (scheduler disabled)", err)
- } else {
- log.Println("✅ Redis connected successfully")
- }
- } else {
- log.Println("⚠️ REDIS_URL not set - scheduler disabled")
- }
-
- // Run preflight checks
- checker := preflight.NewChecker(db)
- results := checker.RunAll()
-
- // Exit if critical checks failed
- if preflight.HasFailures(results) {
- log.Println("\n❌ Pre-flight checks failed. Please fix the issues above before starting the server.")
- os.Exit(1)
- }
-
- log.Println("✅ All pre-flight checks passed")
-
- // Initialize services
- providerService := services.NewProviderService(db)
- modelService := services.NewModelService(db)
- connManager := services.NewConnectionManager()
-
- // Initialize Prometheus metrics
- services.InitMetrics(connManager)
- log.Println("✅ Prometheus metrics initialized")
-
- // Initialize MCP bridge service
- mcpBridge := services.NewMCPBridgeService(db, tools.GetRegistry())
- log.Println("✅ MCP bridge service initialized")
-
- chatService := services.NewChatService(db, providerService, mcpBridge, nil) // toolService set later after credential service init
-
- // Initialize agent service (requires MongoDB for scalable storage)
- var agentService *services.AgentService
- if mongoDB != nil {
- agentService = services.NewAgentService(mongoDB)
- // Ensure indexes for agents, workflows, and workflow_versions
- if err := agentService.EnsureIndexes(context.Background()); err != nil {
- log.Printf("⚠️ Failed to ensure agent indexes: %v", err)
- }
- log.Println("✅ Agent service initialized (MongoDB)")
- } else {
- log.Println("⚠️ MongoDB not available - agent builder features disabled")
- }
-
- workflowGeneratorService := services.NewWorkflowGeneratorService(db, providerService, chatService)
- log.Println("✅ Workflow generator service initialized")
-
- workflowGeneratorV2Service := services.NewWorkflowGeneratorV2Service(db, providerService, chatService)
- log.Println("✅ Workflow generator v2 service initialized (multi-step with tool selection)")
-
- // Initialize tier service (requires MongoDB)
- var tierService *services.TierService
- if mongoDB != nil {
- tierService = services.NewTierService(mongoDB)
- log.Println("✅ Tier service initialized")
- }
-
- // Initialize execution limiter (requires TierService + Redis)
- if tierService != nil && redisService != nil {
- executionLimiter = middleware.NewExecutionLimiter(tierService, redisService.Client())
- log.Println("✅ Execution limiter initialized")
- } else {
- log.Println("⚠️ Execution limiter disabled (requires TierService and Redis)")
- }
-
- // Initialize usage limiter service (requires TierService + Redis + MongoDB)
- var usageLimiter *services.UsageLimiterService
- if tierService != nil && redisService != nil && mongoDB != nil {
- usageLimiter = services.NewUsageLimiterService(tierService, redisService.Client(), mongoDB)
- log.Println("✅ Usage limiter service initialized")
-
- // Inject usage limiter into user service for promo user counter reset
- if userService != nil {
- userService.SetUsageLimiter(usageLimiter)
- log.Println("✅ Usage limiter injected into user service")
- }
- } else {
- log.Println("⚠️ Usage limiter disabled (requires TierService, Redis, and MongoDB)")
- }
-
- // Initialize execution service (requires MongoDB + TierService)
- var executionService *services.ExecutionService
- if mongoDB != nil {
- executionService = services.NewExecutionService(mongoDB, tierService)
- // Ensure indexes
- if err := executionService.EnsureIndexes(context.Background()); err != nil {
- log.Printf("⚠️ Failed to ensure execution indexes: %v", err)
- }
- log.Println("✅ Execution service initialized")
- }
-
- // Initialize analytics service (minimal, non-invasive usage tracking)
- var analyticsService *services.AnalyticsService
- if mongoDB != nil {
- analyticsService = services.NewAnalyticsService(mongoDB)
- // Ensure indexes
- if err := analyticsService.EnsureIndexes(context.Background()); err != nil {
- log.Printf("⚠️ Failed to ensure analytics indexes: %v", err)
- }
- log.Println("✅ Analytics service initialized (minimal tracking)")
- }
-
- // Initialize API key service (requires MongoDB + TierService)
- var apiKeyService *services.APIKeyService
- if mongoDB != nil {
- apiKeyService = services.NewAPIKeyService(mongoDB, tierService)
- // Ensure indexes
- if err := apiKeyService.EnsureIndexes(context.Background()); err != nil {
- log.Printf("⚠️ Failed to ensure API key indexes: %v", err)
- }
- log.Println("✅ API key service initialized")
- }
-
- // Initialize credential service (requires MongoDB + EncryptionService)
- var credentialService *services.CredentialService
- if mongoDB != nil && encryptionService != nil {
- credentialService = services.NewCredentialService(mongoDB, encryptionService)
- // Ensure indexes
- if err := credentialService.EnsureIndexes(context.Background()); err != nil {
- log.Printf("⚠️ Failed to ensure credential indexes: %v", err)
- }
- log.Println("✅ Credential service initialized")
- }
-
- // Initialize tool service (provides credential-filtered tools)
- toolService := services.NewToolService(tools.GetRegistry(), credentialService)
- log.Println("✅ Tool service initialized")
-
- // Set tool service on chat service (was initialized with nil earlier)
- chatService.SetToolService(toolService)
-
- // Initialize and set tool predictor service for dynamic tool selection
- toolPredictorService := services.NewToolPredictorService(db, providerService, chatService)
- chatService.SetToolPredictorService(toolPredictorService)
- log.Println("✅ Tool predictor service initialized")
-
- // Initialize memory services (requires MongoDB + EncryptionService)
- var memoryStorageService *services.MemoryStorageService
- var memoryExtractionService *services.MemoryExtractionService
- var memorySelectionService *services.MemorySelectionService
- var memoryDecayService *services.MemoryDecayService
- var memoryModelPool *services.MemoryModelPool
- if mongoDB != nil && encryptionService != nil {
- memoryStorageService = services.NewMemoryStorageService(mongoDB, encryptionService)
- log.Println("✅ Memory storage service initialized")
-
- // Initialize model pool for dynamic memory model selection
- var err error
- memoryModelPool, err = services.NewMemoryModelPool(chatService, db.DB)
- if err != nil {
- log.Printf("⚠️ Failed to initialize memory model pool: %v", err)
- log.Println("⚠️ Memory extraction/selection services disabled (requires valid memory models)")
- } else {
- log.Println("✅ Memory model pool initialized")
-
- memoryExtractionService = services.NewMemoryExtractionService(
- mongoDB,
- encryptionService,
- providerService,
- memoryStorageService,
- chatService,
- memoryModelPool,
- )
- log.Println("✅ Memory extraction service initialized")
-
- memorySelectionService = services.NewMemorySelectionService(
- mongoDB,
- encryptionService,
- providerService,
- memoryStorageService,
- chatService,
- memoryModelPool,
- )
- log.Println("✅ Memory selection service initialized")
-
- // Set memory services on chat service
- chatService.SetMemoryExtractionService(memoryExtractionService)
- chatService.SetMemorySelectionService(memorySelectionService)
- chatService.SetUserService(userService)
- }
-
- memoryDecayService = services.NewMemoryDecayService(mongoDB)
- log.Println("✅ Memory decay service initialized")
- } else {
- log.Println("⚠️ Memory services disabled (requires MongoDB + EncryptionService)")
- }
-
- // Initialize scheduler service (requires Redis + MongoDB + AgentService + ExecutionService)
- if redisService != nil && mongoDB != nil {
- var err error
- schedulerService, err = services.NewSchedulerService(mongoDB, redisService, agentService, executionService)
- if err != nil {
- log.Printf("⚠️ Failed to initialize scheduler: %v", err)
- } else {
- log.Println("✅ Scheduler service initialized")
- }
- }
-
- // Initialize PubSub service (requires Redis)
- var pubsubService *services.PubSubService
- if redisService != nil {
- instanceID := fmt.Sprintf("instance-%d", time.Now().UnixNano()%10000)
- pubsubService = services.NewPubSubService(redisService, instanceID)
- if err := pubsubService.Start(); err != nil {
- log.Printf("⚠️ Failed to start PubSub service: %v", err)
- } else {
- log.Printf("✅ PubSub service initialized (instance: %s)", instanceID)
- }
- }
-
- // Initialize workflow execution engine with block checker support
- executorRegistry := execution.NewExecutorRegistry(chatService, providerService, tools.GetRegistry(), credentialService)
- workflowEngine := execution.NewWorkflowEngineWithChecker(executorRegistry, providerService)
- log.Println("✅ Workflow execution engine initialized (with block checker)")
-
- // Set workflow executor on scheduler and start it
- if schedulerService != nil {
- // Create a workflow executor callback that wraps the workflow engine
- workflowExecutor := func(workflow *models.Workflow, inputs map[string]interface{}) (*models.WorkflowExecuteResult, error) {
- // Create a dummy status channel (scheduled jobs don't need real-time updates)
- statusChan := make(chan models.ExecutionUpdate, 100)
- go func() {
- for range statusChan {
- // Drain channel - in Phase 4, this will publish to Redis pub/sub
- }
- }()
-
- result, err := workflowEngine.Execute(context.Background(), workflow, inputs, statusChan)
- close(statusChan)
-
- if err != nil {
- return &models.WorkflowExecuteResult{
- Status: "failed",
- Error: err.Error(),
- }, err
- }
- return &models.WorkflowExecuteResult{
- Status: result.Status,
- Output: result.Output,
- BlockStates: result.BlockStates,
- Error: result.Error,
- }, nil
- }
-
- schedulerService.SetWorkflowExecutor(workflowExecutor)
- if err := schedulerService.Start(context.Background()); err != nil {
- log.Printf("⚠️ Failed to start scheduler: %v", err)
- } else {
- log.Println("✅ Scheduler started successfully")
- }
- }
-
- // Initialize authentication (Local JWT - v2.0)
- var jwtAuth *auth.LocalJWTAuth
- jwtSecret := os.Getenv("JWT_SECRET")
- if jwtSecret == "" {
- environment := os.Getenv("ENVIRONMENT")
- if environment == "production" {
- log.Fatal("❌ CRITICAL SECURITY ERROR: JWT_SECRET is required in production. Generate with: openssl rand -hex 64")
- }
- log.Println("⚠️ JWT_SECRET not set - authentication disabled (development mode)")
- } else {
- // Parse JWT expiry durations from environment variables
- accessTokenExpiry := 15 * time.Minute // Default: 15 minutes
- refreshTokenExpiry := 7 * 24 * time.Hour // Default: 7 days
-
- if accessExpiryStr := os.Getenv("JWT_ACCESS_TOKEN_EXPIRY"); accessExpiryStr != "" {
- if parsed, err := time.ParseDuration(accessExpiryStr); err == nil {
- accessTokenExpiry = parsed
- } else {
- log.Printf("⚠️ Invalid JWT_ACCESS_TOKEN_EXPIRY: %v, using default 15m", err)
- }
- }
-
- if refreshExpiryStr := os.Getenv("JWT_REFRESH_TOKEN_EXPIRY"); refreshExpiryStr != "" {
- if parsed, err := time.ParseDuration(refreshExpiryStr); err == nil {
- refreshTokenExpiry = parsed
- } else {
- log.Printf("⚠️ Invalid JWT_REFRESH_TOKEN_EXPIRY: %v, using default 7d", err)
- }
- }
-
- var err error
- jwtAuth, err = auth.NewLocalJWTAuth(jwtSecret, accessTokenExpiry, refreshTokenExpiry)
- if err != nil {
- log.Fatalf("❌ Failed to initialize JWT authentication: %v", err)
- }
- log.Printf("✅ Local JWT authentication initialized (access: %v, refresh: %v)", accessTokenExpiry, refreshTokenExpiry)
- }
-
- // Try loading configuration from database first
- _, err = loadConfigFromDatabase(modelService, chatService, providerService)
- if err != nil {
- log.Printf("⚠️ Warning: Could not load config from database: %v", err)
- }
-
- // Database starts empty - use admin UI to add providers and models
-
- // Initialize vision service (for describe_image tool)
- // Must be after provider sync so model aliases are available
- services.SetVisionDependencies(providerService, db)
- services.InitVisionService()
-
- // Initialize audio service (for transcribe_audio tool)
- // Uses the same provider service dependency set above
- services.InitAudioService()
-
- // NOTE: providers.json file watcher removed - all provider management now in MySQL
-
- // Start background model refresh job (refreshes from database)
- go startModelRefreshJob(providerService, modelService, chatService)
-
- // Run startup cleanup to delete orphaned files from previous runs
- // This ensures zero retention policy is enforced even after server restarts
- uploadDir := "./uploads"
- fileCache := filecache.GetService()
- fileCache.RunStartupCleanup(uploadDir)
-
- // Start background image cleanup job (also cleans orphaned files)
- go startImageCleanupJob(uploadDir)
-
- // Start background document cleanup job
- go startDocumentCleanupJob()
-
- // Start memory extraction worker (requires memory extraction service)
- if memoryExtractionService != nil {
- go startMemoryExtractionWorker(memoryExtractionService)
- }
-
- // Start memory decay worker (requires memory decay service)
- if memoryDecayService != nil {
- go startMemoryDecayWorker(memoryDecayService)
- }
-
- // Initialize Fiber app
- app := fiber.New(fiber.Config{
- AppName: "ClaraVerse v1.0",
- ReadTimeout: 360 * time.Second, // 6 minutes to handle long tool executions
- WriteTimeout: 360 * time.Second, // 6 minutes to handle long tool executions
- IdleTimeout: 360 * time.Second, // 6 minutes to handle long tool executions
- BodyLimit: 50 * 1024 * 1024, // 50MB limit for chat messages with images and large conversations
- })
-
- // Middleware
- app.Use(recover.New())
- app.Use(logger.New())
-
- // Prometheus metrics middleware
- prometheus := fiberprometheus.New("claraverse")
- prometheus.RegisterAt(app, "/metrics")
- app.Use(prometheus.Middleware)
- log.Println("📊 Prometheus metrics endpoint enabled at /metrics")
-
- // Load rate limiting configuration
- rateLimitConfig := middleware.LoadRateLimitConfig()
- log.Printf("🛡️ [RATE-LIMIT] Loaded config: Global=%d/min, Public=%d/min, Auth=%d/min, WS=%d/min",
- rateLimitConfig.GlobalAPIMax,
- rateLimitConfig.PublicReadMax,
- rateLimitConfig.AuthenticatedMax,
- rateLimitConfig.WebSocketMax,
- )
-
- // CORS configuration with environment-based origins
- allowedOrigins := os.Getenv("ALLOWED_ORIGINS")
- if allowedOrigins == "" {
- // Default to localhost for development
- allowedOrigins = "http://localhost:5173,http://localhost:3000"
- log.Println("⚠️ ALLOWED_ORIGINS not set, using development defaults")
- }
-
- app.Use(cors.New(cors.Config{
- AllowOrigins: allowedOrigins,
- AllowMethods: "GET,POST,PUT,DELETE,OPTIONS",
- AllowHeaders: "Origin,Content-Type,Accept,Authorization",
- AllowCredentials: true, // Required for cookies (JWT refresh tokens)
- // Skip CORS check for external access endpoints - they have their own permissive CORS
- Next: func(c *fiber.Ctx) bool {
- path := c.Path()
- return strings.HasPrefix(path, "/api/trigger") || strings.HasPrefix(path, "/api/external")
- },
- }))
-
- log.Printf("🔒 [SECURITY] CORS allowed origins: %s (excluding /api/trigger and /api/external)", allowedOrigins)
-
- // Global API rate limiter - first line of DDoS defense
- // Applies to all /api/* routes, excludes health checks and metrics
- app.Use("/api", middleware.GlobalAPIRateLimiter(rateLimitConfig))
- log.Println("🛡️ [RATE-LIMIT] Global API rate limiter enabled")
-
- // Initialize handlers
- healthHandler := handlers.NewHealthHandler(connManager)
- providerHandler := handlers.NewProviderHandler(providerService)
- modelHandler := handlers.NewModelHandler(modelService)
- uploadHandler := handlers.NewUploadHandler("./uploads", usageLimiter)
- downloadHandler := handlers.NewDownloadHandler()
- secureDownloadHandler := handlers.NewSecureDownloadHandler()
- conversationHandler := handlers.NewConversationHandler(chatService, builderConvService)
- userHandler := handlers.NewUserHandler(chatService, userService)
- wsHandler := handlers.NewWebSocketHandler(connManager, chatService, analyticsService, usageLimiter)
-
- // Initialize local auth handler (v2.0)
- var localAuthHandler *handlers.LocalAuthHandler
- if jwtAuth != nil && mongoDB != nil && userService != nil {
- localAuthHandler = handlers.NewLocalAuthHandler(jwtAuth, userService)
- log.Println("✅ Local auth handler initialized")
- }
-
- // Initialize memory handler (requires memory services)
- var memoryHandler *handlers.MemoryHandler
- if memoryStorageService != nil && memoryExtractionService != nil {
- memoryHandler = handlers.NewMemoryHandler(memoryStorageService, memoryExtractionService, chatService)
- log.Println("✅ Memory handler initialized")
- }
-
- // Inject usage limiter into chat service for tool execution
- if usageLimiter != nil {
- chatService.SetUsageLimiter(usageLimiter)
- }
- mcpWSHandler := handlers.NewMCPWebSocketHandler(mcpBridge)
- configHandler := handlers.NewConfigHandler()
- // Initialize agent handler (requires agentService)
- var agentHandler *handlers.AgentHandler
- var workflowWSHandler *handlers.WorkflowWebSocketHandler
- if agentService != nil {
- agentHandler = handlers.NewAgentHandler(agentService, workflowGeneratorService)
- // Wire up builder conversation service for sync endpoint
- if builderConvService != nil {
- agentHandler.SetBuilderConversationService(builderConvService)
- }
- // Wire up v2 workflow generator service (multi-step with tool selection)
- agentHandler.SetWorkflowGeneratorV2Service(workflowGeneratorV2Service)
- // Wire up provider service for Ask mode
- agentHandler.SetProviderService(providerService)
- workflowWSHandler = handlers.NewWorkflowWebSocketHandler(agentService, workflowEngine, executionLimiter)
- // Wire up execution service for workflow execution tracking
- if executionService != nil {
- workflowWSHandler.SetExecutionService(executionService)
- }
- log.Println("✅ Agent handler initialized")
- }
- toolsHandler := handlers.NewToolsHandler(tools.GetRegistry(), toolService)
- imageProxyHandler := handlers.NewImageProxyHandler()
- audioHandler := handlers.NewAudioHandler()
- log.Println("✅ Audio handler initialized")
-
- // Initialize schedule handler (requires scheduler service)
- var scheduleHandler *handlers.ScheduleHandler
- if schedulerService != nil {
- scheduleHandler = handlers.NewScheduleHandler(schedulerService, agentService)
- log.Println("✅ Schedule handler initialized")
- }
-
- // Initialize execution handler (requires execution service)
- var executionHandler *handlers.ExecutionHandler
- if executionService != nil {
- executionHandler = handlers.NewExecutionHandler(executionService)
- log.Println("✅ Execution handler initialized")
- }
-
- // Initialize API key handler (requires API key service)
- var apiKeyHandler *handlers.APIKeyHandler
- if apiKeyService != nil {
- apiKeyHandler = handlers.NewAPIKeyHandler(apiKeyService)
- log.Println("✅ API key handler initialized")
- }
-
- // Initialize trigger handler (requires agent service + execution service + workflow engine)
- var triggerHandler *handlers.TriggerHandler
- if executionService != nil {
- triggerHandler = handlers.NewTriggerHandler(agentService, executionService, workflowEngine)
- log.Println("✅ Trigger handler initialized")
- }
-
- // Initialize credential handler (requires credential service)
- var credentialHandler *handlers.CredentialHandler
- var composioAuthHandler *handlers.ComposioAuthHandler
- if credentialService != nil {
- credentialHandler = handlers.NewCredentialHandler(credentialService)
- log.Println("✅ Credential handler initialized")
-
- // Initialize Composio OAuth handler
- composioAuthHandler = handlers.NewComposioAuthHandler(credentialService)
- log.Println("✅ Composio OAuth handler initialized")
- }
-
- // Initialize chat sync handler (requires chat sync service)
- var chatSyncHandler *handlers.ChatSyncHandler
- if chatSyncService != nil {
- chatSyncHandler = handlers.NewChatSyncHandler(chatSyncService)
- log.Println("✅ Chat sync handler initialized")
- }
-
- // Initialize user preferences handler (requires userService)
- var userPreferencesHandler *handlers.UserPreferencesHandler
- if userService != nil {
- userPreferencesHandler = handlers.NewUserPreferencesHandler(userService)
- log.Println("✅ User preferences handler initialized")
- }
-
- // Payment service removed in v2.0 - all users default to Pro tier
-
- // Wire up GDPR services for complete account deletion
- userHandler.SetGDPRServices(
- agentService,
- executionService,
- apiKeyService,
- credentialService,
- chatSyncService,
- schedulerService,
- builderConvService,
- )
- log.Println("✅ GDPR services wired up for account deletion")
-
- // Routes
-
- // Health check (public)
- app.Get("/health", healthHandler.Handle)
-
- // Rate limiter for upload endpoint (10 uploads per minute per user)
- uploadLimiter := limiter.New(limiter.Config{
- Max: 10,
- Expiration: 1 * time.Minute,
- KeyGenerator: func(c *fiber.Ctx) string {
- // Rate limit by user ID
- userID, ok := c.Locals("user_id").(string)
- if !ok || userID == "" {
- // Fallback to IP if no user ID
- return c.IP()
- }
- return "upload:" + userID
- },
- LimitReached: func(c *fiber.Ctx) error {
- log.Printf("⚠️ [RATE-LIMIT] Upload limit reached for user: %v", c.Locals("user_id"))
- return c.Status(fiber.StatusTooManyRequests).JSON(fiber.Map{
- "error": "Too many upload requests. Please wait before uploading again.",
- })
- },
- })
-
- // Create endpoint-specific rate limiters
- publicReadLimiter := middleware.PublicReadRateLimiter(rateLimitConfig)
- imageProxyLimiter := middleware.ImageProxyRateLimiter(rateLimitConfig)
- transcribeLimiter := middleware.TranscribeRateLimiter(rateLimitConfig)
-
- // API routes (public read-only)
- api := app.Group("/api")
- {
- // Authentication routes (v2.0 - Local JWT)
- if localAuthHandler != nil {
- auth := api.Group("/auth")
- auth.Get("/status", localAuthHandler.GetStatus) // Public - check if users exist
- auth.Post("/register", localAuthHandler.Register)
- auth.Post("/login", localAuthHandler.Login)
- auth.Post("/refresh", localAuthHandler.RefreshToken)
- auth.Post("/logout", middleware.LocalAuthMiddleware(jwtAuth), localAuthHandler.Logout)
- auth.Get("/me", middleware.LocalAuthMiddleware(jwtAuth), localAuthHandler.GetCurrentUser)
- log.Println("✅ Local auth routes registered (/api/auth/*)")
- }
-
- // Image proxy endpoint (public - used by frontend for image search results)
- // Has its own rate limiter to prevent bandwidth abuse
- api.Get("/proxy/image", imageProxyLimiter, imageProxyHandler.ProxyImage)
-
- // Public read-only endpoints with rate limiting
- api.Get("/providers", publicReadLimiter, providerHandler.List)
- api.Get("/models", publicReadLimiter, middleware.OptionalLocalAuthMiddleware(jwtAuth), modelHandler.List)
- api.Get("/models/tool-predictors", publicReadLimiter, modelHandler.ListToolPredictorModels) // Specific routes before parameterized ones
- api.Get("/models/provider/:id", publicReadLimiter, modelHandler.ListByProvider)
-
- // Configuration endpoints (public)
- api.Get("/config/recommended-models", publicReadLimiter, configHandler.GetRecommendedModels)
-
- // Conversation status check (public)
- api.Get("/conversations/:id/status", publicReadLimiter, conversationHandler.GetStatus)
-
- // File upload (requires authentication via JWT or API key + rate limiting)
- // API key users need "upload" scope to upload files
- if apiKeyService != nil {
- api.Post("/upload",
- middleware.APIKeyOrJWTMiddleware(apiKeyService, middleware.OptionalLocalAuthMiddleware(jwtAuth)),
- middleware.RequireScope("upload"),
- uploadLimiter,
- uploadHandler.Upload,
- )
- } else {
- api.Post("/upload",
- middleware.OptionalLocalAuthMiddleware(jwtAuth),
- uploadLimiter,
- uploadHandler.Upload,
- )
- }
- api.Delete("/upload/:id", middleware.OptionalLocalAuthMiddleware(jwtAuth), uploadHandler.Delete)
-
- // File status check endpoint (for pre-execution validation)
- api.Get("/upload/:id/status", middleware.OptionalLocalAuthMiddleware(jwtAuth), uploadHandler.CheckFileStatus)
-
- // Audio transcription endpoint (requires authentication + rate limiting for expensive GPU operation)
- api.Post("/audio/transcribe", middleware.OptionalLocalAuthMiddleware(jwtAuth), transcribeLimiter, audioHandler.Transcribe)
-
- // Document download (requires authentication for access control)
- api.Get("/download/:id", middleware.OptionalLocalAuthMiddleware(jwtAuth), downloadHandler.Download)
-
- // Secure file downloads (access code based - no auth required for download)
- api.Get("/files/:id", secureDownloadHandler.Download) // Download with access code
- api.Get("/files/:id/info", secureDownloadHandler.GetInfo) // Get file info with access code
- api.Get("/files", middleware.LocalAuthMiddleware(jwtAuth), secureDownloadHandler.ListUserFiles) // List user's files
- api.Delete("/files/:id", middleware.LocalAuthMiddleware(jwtAuth), secureDownloadHandler.Delete) // Delete file (owner only)
-
- // User preferences endpoints (requires authentication)
- api.Get("/user/preferences", middleware.LocalAuthMiddleware(jwtAuth), userHandler.GetPreferences)
- api.Put("/user/preferences", middleware.LocalAuthMiddleware(jwtAuth), userHandler.UpdatePreferences)
- api.Post("/user/welcome-popup-seen", middleware.LocalAuthMiddleware(jwtAuth), userHandler.MarkWelcomePopupSeen)
-
- // GDPR Compliance endpoints (requires authentication)
- api.Get("/user/data", middleware.LocalAuthMiddleware(jwtAuth), userHandler.ExportData)
- api.Delete("/user/account", middleware.LocalAuthMiddleware(jwtAuth), userHandler.DeleteAccount)
-
- // Privacy policy (public)
- api.Get("/privacy-policy", userHandler.GetPrivacyPolicy)
-
- // Memory management routes (requires authentication + memory services)
- if memoryHandler != nil {
- memories := api.Group("/memories", middleware.LocalAuthMiddleware(jwtAuth))
- memories.Get("/", memoryHandler.ListMemories)
- memories.Get("/stats", memoryHandler.GetMemoryStats) // Must be before /:id to avoid route conflict
- memories.Get("/:id", memoryHandler.GetMemory)
- memories.Post("/", memoryHandler.CreateMemory)
- memories.Put("/:id", memoryHandler.UpdateMemory)
- memories.Delete("/:id", memoryHandler.DeleteMemory)
- memories.Post("/:id/archive", memoryHandler.ArchiveMemory)
- memories.Post("/:id/unarchive", memoryHandler.UnarchiveMemory)
-
- // Conversation memory extraction (manual trigger)
- api.Post("/conversations/:id/extract-memories", middleware.LocalAuthMiddleware(jwtAuth), memoryHandler.TriggerMemoryExtraction)
- }
-
- // Agent builder routes (requires authentication + MongoDB)
- if agentHandler != nil {
- agents := api.Group("/agents", middleware.LocalAuthMiddleware(jwtAuth))
- agents.Post("/", agentHandler.Create)
- agents.Get("/", agentHandler.List)
- agents.Get("/recent", agentHandler.ListRecent) // Must be before /:id to avoid route conflict
- agents.Post("/ask", agentHandler.Ask) // Ask mode - must be before /:id to avoid route conflict
- agents.Get("/:id", agentHandler.Get)
- agents.Put("/:id", agentHandler.Update)
- agents.Delete("/:id", agentHandler.Delete)
- agents.Post("/:id/sync", agentHandler.SyncAgent) // Sync local agent to backend
-
- // Workflow version routes - MUST be before /:id/workflow to avoid route conflict
- agents.Get("/:id/workflow/versions", agentHandler.ListWorkflowVersions)
- agents.Get("/:id/workflow/versions/:version", agentHandler.GetWorkflowVersion)
- agents.Post("/:id/workflow/restore/:version", agentHandler.RestoreWorkflowVersion)
-
- // Workflow routes (less specific, must come after /versions routes)
- agents.Put("/:id/workflow", agentHandler.SaveWorkflow)
- agents.Get("/:id/workflow", agentHandler.GetWorkflow)
- agents.Post("/:id/generate-workflow", agentHandler.GenerateWorkflow)
- agents.Post("/:id/generate-workflow-v2", agentHandler.GenerateWorkflowV2) // Multi-step with tool selection
- agents.Post("/:id/select-tools", agentHandler.SelectTools) // Tool selection only (step 1)
- agents.Post("/:id/generate-with-tools", agentHandler.GenerateWithTools) // Generate with pre-selected tools (step 2)
- agents.Post("/:id/generate-sample-input", agentHandler.GenerateSampleInput) // Generate sample JSON input for testing
-
- // Builder conversation routes (under agents)
- agents.Get("/:id/conversations", conversationHandler.ListBuilderConversations)
- agents.Post("/:id/conversations", conversationHandler.CreateBuilderConversation)
- agents.Get("/:id/conversations/current", conversationHandler.GetOrCreateBuilderConversation)
- agents.Get("/:id/conversations/:convId", conversationHandler.GetBuilderConversation)
- agents.Delete("/:id/conversations/:convId", conversationHandler.DeleteBuilderConversation)
- agents.Post("/:id/conversations/:convId/messages", conversationHandler.AddBuilderMessage)
-
- // Schedule routes (under agents) - requires scheduler service
- if scheduleHandler != nil {
- agents.Post("/:id/schedule", scheduleHandler.Create)
- agents.Get("/:id/schedule", scheduleHandler.Get)
- agents.Put("/:id/schedule", scheduleHandler.Update)
- agents.Delete("/:id/schedule", scheduleHandler.Delete)
- agents.Post("/:id/schedule/run", scheduleHandler.TriggerNow)
- }
-
- // Execution routes (under agents)
- if executionHandler != nil {
- agents.Get("/:id/executions", executionHandler.ListByAgent)
- agents.Get("/:id/executions/stats", executionHandler.GetStats)
- }
- }
-
- // Execution routes (top-level, authenticated) - MongoDB only
- if executionHandler != nil {
- executions := api.Group("/executions", middleware.LocalAuthMiddleware(jwtAuth))
- executions.Get("/", executionHandler.ListAll)
- executions.Get("/:id", executionHandler.GetByID)
- }
-
- // Schedule routes (top-level, authenticated) - for usage stats
- if scheduleHandler != nil {
- schedules := api.Group("/schedules", middleware.LocalAuthMiddleware(jwtAuth))
- schedules.Get("/usage", scheduleHandler.GetUsage)
- }
-
- // Tool routes (requires authentication)
- tools := api.Group("/tools", middleware.LocalAuthMiddleware(jwtAuth))
- tools.Get("/", toolsHandler.ListTools)
- tools.Get("/available", toolsHandler.GetAvailableTools) // Returns tools filtered by user's credentials
- tools.Post("/recommend", toolsHandler.RecommendTools)
- if agentHandler != nil {
- tools.Get("/registry", agentHandler.GetToolRegistry) // Tool registry for workflow builder
- }
-
- // API Key management routes (requires authentication)
- if apiKeyHandler != nil {
- keys := api.Group("/keys", middleware.LocalAuthMiddleware(jwtAuth))
- keys.Post("/", apiKeyHandler.Create)
- keys.Get("/", apiKeyHandler.List)
- keys.Get("/:id", apiKeyHandler.Get)
- keys.Post("/:id/revoke", apiKeyHandler.Revoke)
- keys.Delete("/:id", apiKeyHandler.Delete)
- }
-
- // Credential management routes (requires authentication)
- if credentialHandler != nil {
- // Integration registry (public read)
- api.Get("/integrations", credentialHandler.GetIntegrations)
- api.Get("/integrations/:id", credentialHandler.GetIntegration)
-
- // Credential CRUD (authenticated)
- credentials := api.Group("/credentials", middleware.LocalAuthMiddleware(jwtAuth))
- credentials.Post("/", credentialHandler.Create)
- credentials.Get("/", credentialHandler.List)
- credentials.Get("/by-integration", credentialHandler.GetCredentialsByIntegration)
- credentials.Get("/references", credentialHandler.GetCredentialReferences)
- credentials.Get("/:id", credentialHandler.Get)
- credentials.Put("/:id", credentialHandler.Update)
- credentials.Delete("/:id", credentialHandler.Delete)
- credentials.Post("/:id/test", credentialHandler.Test)
-
- // Composio OAuth routes (authenticated)
- if composioAuthHandler != nil {
- composio := api.Group("/integrations/composio", middleware.LocalAuthMiddleware(jwtAuth))
- composio.Get("/googlesheets/authorize", composioAuthHandler.InitiateGoogleSheetsAuth)
- composio.Get("/gmail/authorize", composioAuthHandler.InitiateGmailAuth)
- composio.Get("/connected-account", composioAuthHandler.GetConnectedAccount)
- composio.Post("/complete-setup", composioAuthHandler.CompleteComposioSetup)
-
- // Callback endpoint (unauthenticated - Composio calls this)
- api.Get("/integrations/composio/callback", composioAuthHandler.HandleComposioCallback)
- }
- }
-
- // Chat sync routes (requires authentication + chat sync service)
- if chatSyncHandler != nil {
- chats := api.Group("/chats", middleware.LocalAuthMiddleware(jwtAuth))
- chats.Get("/sync", chatSyncHandler.SyncAll) // Get all chats for initial sync (must be before /:id)
- chats.Post("/sync", chatSyncHandler.BulkSync) // Bulk upload chats
- chats.Get("/", chatSyncHandler.List) // List chats (paginated)
- chats.Post("/", chatSyncHandler.CreateOrUpdate) // Create or update a chat
- chats.Get("/:id", chatSyncHandler.Get) // Get single chat
- chats.Put("/:id", chatSyncHandler.Update) // Partial update
- chats.Delete("/:id", chatSyncHandler.Delete) // Delete single chat
- chats.Post("/:id/messages", chatSyncHandler.AddMessage) // Add message to chat
- chats.Delete("/", chatSyncHandler.DeleteAll) // Delete all chats (GDPR)
- log.Println("✅ Chat sync routes registered")
- }
-
- // User preferences routes (requires authentication + userService)
- if userPreferencesHandler != nil {
- prefs := api.Group("/preferences", middleware.LocalAuthMiddleware(jwtAuth))
- prefs.Get("/", userPreferencesHandler.Get) // Get preferences
- prefs.Put("/", userPreferencesHandler.Update) // Update preferences
- log.Println("✅ User preferences routes registered")
- }
-
- // Subscription routes removed in v2.0 - no payment processing
-
- // Admin routes (protected by admin middleware - superadmin only)
- if userService != nil && tierService != nil {
- adminHandler := handlers.NewAdminHandler(userService, tierService, analyticsService, providerService, modelService)
- adminRoutes := api.Group("/admin", middleware.LocalAuthMiddleware(jwtAuth), middleware.AdminMiddleware(cfg))
-
- // Admin status
- adminRoutes.Get("/me", adminHandler.GetAdminStatus)
-
- // User management
- adminRoutes.Get("/users/:userID", adminHandler.GetUserDetails)
- adminRoutes.Post("/users/:userID/overrides", adminHandler.SetLimitOverrides)
- adminRoutes.Delete("/users/:userID/overrides", adminHandler.RemoveAllOverrides)
- adminRoutes.Get("/users", adminHandler.ListUsers)
-
- // Analytics
- adminRoutes.Get("/analytics/overview", adminHandler.GetOverviewAnalytics)
- adminRoutes.Get("/analytics/providers", adminHandler.GetProviderAnalytics)
- adminRoutes.Get("/analytics/chats", adminHandler.GetChatAnalytics)
- adminRoutes.Get("/analytics/models", adminHandler.GetModelAnalytics)
- adminRoutes.Get("/analytics/agents", adminHandler.GetAgentAnalytics)
- adminRoutes.Post("/analytics/migrate-timestamps", adminHandler.MigrateChatSessionTimestamps)
-
- // Provider management (CRUD)
- adminRoutes.Get("/providers", adminHandler.GetProviders)
- adminRoutes.Post("/providers", adminHandler.CreateProvider)
- adminRoutes.Put("/providers/:id", adminHandler.UpdateProvider)
- adminRoutes.Delete("/providers/:id", adminHandler.DeleteProvider)
- adminRoutes.Put("/providers/:id/toggle", adminHandler.ToggleProvider)
-
- // Model management (CRUD, testing, benchmarking, aliases)
- if modelService != nil && providerService != nil {
- modelMgmtService := services.NewModelManagementService(db)
- modelMgmtHandler := handlers.NewModelManagementHandler(modelMgmtService, modelService, providerService)
-
- // Bulk operations (MUST be before parameterized routes to avoid :modelId matching)
- adminRoutes.Post("/models/import-aliases", modelMgmtHandler.ImportAliasesFromJSON)
- adminRoutes.Put("/models/bulk/agents-enabled", modelMgmtHandler.BulkUpdateAgentsEnabled)
- adminRoutes.Put("/models/bulk/visibility", modelMgmtHandler.BulkUpdateVisibility)
-
- // Model CRUD (list and create don't conflict with specific paths)
- adminRoutes.Get("/models", modelMgmtHandler.GetAllModels)
- adminRoutes.Post("/models", modelMgmtHandler.CreateModel)
-
- // Global tier management (specific paths before :modelId)
- adminRoutes.Get("/tiers", modelMgmtHandler.GetTiers)
-
- // Model fetching from provider API
- adminRoutes.Post("/providers/:providerId/fetch", modelMgmtHandler.FetchModelsFromProvider)
- adminRoutes.Post("/providers/:providerId/sync", modelMgmtHandler.SyncProviderToJSON)
-
- // Parameterized model routes (MUST come after all specific /models/* paths)
- adminRoutes.Put("/models/:modelId", modelMgmtHandler.UpdateModel)
- adminRoutes.Delete("/models/:modelId", modelMgmtHandler.DeleteModel)
- adminRoutes.Post("/models/:modelId/tier", modelMgmtHandler.SetModelTier)
- adminRoutes.Delete("/models/:modelId/tier", modelMgmtHandler.ClearModelTier)
-
- // Model testing
- adminRoutes.Post("/models/:modelId/test/connection", modelMgmtHandler.TestModelConnection)
- adminRoutes.Post("/models/:modelId/test/capability", modelMgmtHandler.TestModelCapability)
- adminRoutes.Post("/models/:modelId/benchmark", modelMgmtHandler.RunModelBenchmark)
- adminRoutes.Get("/models/:modelId/test-results", modelMgmtHandler.GetModelTestResults)
-
- // Alias management (parameterized routes must come after specific paths)
- adminRoutes.Get("/models/:modelId/aliases", modelMgmtHandler.GetModelAliases)
- adminRoutes.Post("/models/:modelId/aliases", modelMgmtHandler.CreateModelAlias)
- adminRoutes.Put("/models/:modelId/aliases/:alias", modelMgmtHandler.UpdateModelAlias)
- adminRoutes.Delete("/models/:modelId/aliases/:alias", modelMgmtHandler.DeleteModelAlias)
-
- log.Println("✅ Model management routes registered (CRUD, testing, tiers, aliases)")
- }
-
- // Legacy stats endpoint
- adminRoutes.Get("/stats", adminHandler.GetSystemStats)
-
- log.Println("✅ Admin routes registered (status, analytics, user management, providers)")
- }
-
- // Webhook endpoint removed in v2.0 - no payment processing
-
- }
-
- // Trigger endpoints (API key authenticated, CORS open for external access)
- // These are meant to be called from anywhere (webhooks, external services, etc.)
- if triggerHandler != nil && apiKeyService != nil {
- trigger := app.Group("/api/trigger")
-
- // Apply permissive CORS for trigger endpoints only
- trigger.Use(cors.New(cors.Config{
- AllowOrigins: "*",
- AllowMethods: "GET,POST,OPTIONS",
- AllowHeaders: "Origin,Content-Type,Accept,Authorization,X-API-Key",
- AllowCredentials: false,
- }))
-
- // Apply API key authentication
- trigger.Use(middleware.APIKeyMiddleware(apiKeyService))
-
- trigger.Post("/:agentId", triggerHandler.TriggerAgent)
- trigger.Get("/status/:executionId", triggerHandler.GetExecutionStatus)
-
- log.Println("✅ Trigger endpoints registered with open CORS (external access enabled)")
- }
-
- // External upload endpoint (API key authenticated, CORS open for external access)
- // Allows external services to upload files before triggering agents
- if apiKeyService != nil {
- externalUpload := app.Group("/api/external")
-
- // Apply permissive CORS for external upload
- externalUpload.Use(cors.New(cors.Config{
- AllowOrigins: "*",
- AllowMethods: "POST,OPTIONS",
- AllowHeaders: "Origin,Content-Type,Accept,Authorization,X-API-Key",
- AllowCredentials: false,
- }))
-
- // Apply API key authentication with upload scope requirement
- externalUpload.Use(middleware.APIKeyMiddleware(apiKeyService))
- externalUpload.Use(middleware.RequireScope("upload"))
-
- externalUpload.Post("/upload", uploadLimiter, uploadHandler.Upload)
-
- log.Println("✅ External upload endpoint registered with open CORS (/api/external/upload)")
- }
-
- // Serve uploaded files (authenticated - replaced static serving for security)
- app.Get("/uploads/:filename", middleware.OptionalLocalAuthMiddleware(jwtAuth), func(c *fiber.Ctx) error {
- filename := c.Params("filename")
-
- // Get user ID from auth middleware
- userID, ok := c.Locals("user_id").(string)
- if !ok || userID == "" || userID == "anonymous" {
- return c.Status(fiber.StatusUnauthorized).JSON(fiber.Map{
- "error": "Authentication required to access files",
- })
- }
-
- // Extract file ID from filename (UUID before extension)
- fileID := strings.TrimSuffix(filename, filepath.Ext(filename))
-
- // Get file from cache and verify ownership
- fileCache := filecache.GetService()
- file, found := fileCache.Get(fileID)
-
- if !found {
- log.Printf("⚠️ [FILE-ACCESS] File not found or expired: %s (user: %s)", fileID, userID)
- return c.Status(fiber.StatusNotFound).JSON(fiber.Map{
- "error": "File not found or has expired",
- })
- }
-
- // Verify ownership
- if file.UserID != userID {
- log.Printf("🚫 [SECURITY] User %s denied access to file %s (owned by %s)", userID, fileID, file.UserID)
- return c.Status(fiber.StatusForbidden).JSON(fiber.Map{
- "error": "Access denied to this file",
- })
- }
-
- // Verify file exists on disk
- if file.FilePath == "" || !strings.HasSuffix(file.FilePath, filename) {
- return c.Status(fiber.StatusNotFound).JSON(fiber.Map{
- "error": "File not found",
- })
- }
-
- log.Printf("✅ [FILE-ACCESS] User %s accessing file %s", userID, filename)
-
- // Serve file
- return c.SendFile(file.FilePath)
- })
-
- // WebSocket route (requires auth)
- app.Use("/ws", func(c *fiber.Ctx) error {
- if websocket.IsWebSocketUpgrade(c) {
- c.Locals("allowed", true)
- return c.Next()
- }
- return fiber.ErrUpgradeRequired
- })
-
- // Rate limiter for WebSocket connections (configurable via RATE_LIMIT_WEBSOCKET env var)
- wsConnectionLimiter := middleware.WebSocketRateLimiter(rateLimitConfig)
-
- app.Use("/ws/chat", wsConnectionLimiter)
- app.Use("/ws/chat", middleware.OptionalLocalAuthMiddleware(jwtAuth))
- app.Get("/ws/chat", websocket.New(wsHandler.Handle))
-
- // MCP WebSocket endpoint (requires authentication)
- app.Use("/mcp/connect", func(c *fiber.Ctx) error {
- if websocket.IsWebSocketUpgrade(c) {
- c.Locals("allowed", true)
- return c.Next()
- }
- return fiber.ErrUpgradeRequired
- })
-
- // Rate limiter for MCP WebSocket connections (uses same config as WebSocket)
- mcpConnectionLimiter := middleware.WebSocketRateLimiter(rateLimitConfig)
-
- app.Use("/mcp/connect", mcpConnectionLimiter)
- app.Use("/mcp/connect", middleware.OptionalLocalAuthMiddleware(jwtAuth))
- app.Get("/mcp/connect", websocket.New(mcpWSHandler.HandleConnection))
-
- // Workflow execution WebSocket endpoint (requires authentication + MongoDB)
- if workflowWSHandler != nil {
- app.Use("/ws/workflow", func(c *fiber.Ctx) error {
- if websocket.IsWebSocketUpgrade(c) {
- c.Locals("allowed", true)
- return c.Next()
- }
- return fiber.ErrUpgradeRequired
- })
-
- app.Use("/ws/workflow", wsConnectionLimiter)
- app.Use("/ws/workflow", middleware.LocalAuthMiddleware(jwtAuth))
- app.Get("/ws/workflow", websocket.New(workflowWSHandler.Handle))
- }
-
- // Initialize background jobs
- var jobScheduler *jobs.JobScheduler
- if mongoDB != nil && tierService != nil && userService != nil {
- jobScheduler = jobs.NewJobScheduler()
-
- // Register retention cleanup job (runs daily at 2 AM UTC)
- retentionJob := jobs.NewRetentionCleanupJob(mongoDB, tierService)
- jobScheduler.Register("retention_cleanup", retentionJob)
-
- // Register grace period checker (runs hourly)
- gracePeriodJob := jobs.NewGracePeriodChecker(mongoDB, userService, tierService, 7) // 7 day grace period
- jobScheduler.Register("grace_period_check", gracePeriodJob)
-
- // Register promo expiration checker (runs hourly)
- promoExpirationJob := jobs.NewPromoExpirationChecker(mongoDB, userService, tierService)
- jobScheduler.Register("promo_expiration_check", promoExpirationJob)
-
- // Start job scheduler
- if err := jobScheduler.Start(); err != nil {
- log.Printf("⚠️ Failed to start job scheduler: %v", err)
- } else {
- log.Println("✅ Background job scheduler started")
- }
- } else {
- log.Println("⚠️ Background jobs disabled (requires MongoDB, TierService, UserService)")
- }
-
- // Start server
- log.Printf("✅ Server ready on port %s", cfg.Port)
- log.Printf("🔗 WebSocket endpoint: ws://localhost:%s/ws/chat", cfg.Port)
- log.Printf("🔌 MCP endpoint: ws://localhost:%s/mcp/connect", cfg.Port)
- log.Printf("⚡ Workflow endpoint: ws://localhost:%s/ws/workflow", cfg.Port)
- log.Printf("📡 Health check: http://localhost:%s/health", cfg.Port)
- if schedulerService != nil {
- log.Printf("⏰ Scheduler enabled with Redis")
- }
- if jobScheduler != nil {
- log.Printf("🕐 Background jobs: retention cleanup (daily 2 AM), grace period check (hourly)")
- }
-
- // Handle graceful shutdown
- go func() {
- sigChan := make(chan os.Signal, 1)
- signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM)
- <-sigChan
-
- log.Println("\n🛑 Shutting down server...")
-
- // Stop background jobs
- if jobScheduler != nil {
- jobScheduler.Stop()
- }
-
- // Stop scheduler first
- if schedulerService != nil {
- if err := schedulerService.Stop(); err != nil {
- log.Printf("⚠️ Error stopping scheduler: %v", err)
- }
- }
-
- // Stop PubSub service
- if pubsubService != nil {
- if err := pubsubService.Stop(); err != nil {
- log.Printf("⚠️ Error stopping PubSub: %v", err)
- }
- }
-
- // Shutdown Fiber
- if err := app.Shutdown(); err != nil {
- log.Printf("⚠️ Error shutting down server: %v", err)
- }
- }()
-
- if err := app.Listen(":" + cfg.Port); err != nil {
- log.Fatalf("❌ Failed to start server: %v", err)
- }
-}
-
-// syncProviders syncs providers from JSON file to database
-func syncProviders(filePath string, providerService *services.ProviderService, modelService *services.ModelService, chatService *services.ChatService) error {
- log.Println("🔄 Syncing providers from providers.json...")
-
- providersConfig, err := config.LoadProviders(filePath)
- if err != nil {
- return fmt.Errorf("failed to load providers config: %w", err)
- }
-
- log.Printf("📋 Syncing %d providers from providers.json...", len(providersConfig.Providers))
-
- // Build a set of provider names from config
- configProviderNames := make(map[string]bool)
- for _, providerConfig := range providersConfig.Providers {
- configProviderNames[providerConfig.Name] = true
- }
-
- // Clean up stale providers that are no longer in providers.json
- existingProviders, err := providerService.GetAllIncludingDisabled()
- if err != nil {
- log.Printf("⚠️ Could not check for stale providers: %v", err)
- } else {
- for _, existingProvider := range existingProviders {
- if !configProviderNames[existingProvider.Name] {
- log.Printf(" 🗑️ Removing stale provider: %s (ID %d) - no longer in providers.json", existingProvider.Name, existingProvider.ID)
- if err := providerService.Delete(existingProvider.ID); err != nil {
- log.Printf(" ⚠️ Failed to delete stale provider %s: %v", existingProvider.Name, err)
- } else {
- log.Printf(" ✅ Deleted stale provider: %s and its models", existingProvider.Name)
- }
- }
- }
- }
-
- for _, providerConfig := range providersConfig.Providers {
- // Check if provider exists
- existingProvider, err := providerService.GetByName(providerConfig.Name)
- if err != nil {
- return fmt.Errorf("failed to check provider: %w", err)
- }
-
- var provider *models.Provider
- if existingProvider == nil {
- // Create new provider
- log.Printf(" ➕ Creating provider: %s", providerConfig.Name)
- provider, err = providerService.Create(providerConfig)
- if err != nil {
- return fmt.Errorf("failed to create provider: %w", err)
- }
- } else {
- // Update existing provider
- log.Printf(" ♻️ Updating provider: %s (ID %d)", providerConfig.Name, existingProvider.ID)
- if err := providerService.Update(existingProvider.ID, providerConfig); err != nil {
- return fmt.Errorf("failed to update provider: %w", err)
- }
- provider = existingProvider
- }
-
- // Get config service instance
- configService := services.GetConfigService()
-
- // Load model aliases into both chat service and config service
- if len(providerConfig.ModelAliases) > 0 {
- log.Printf(" 🔄 Loading %d model aliases for %s...", len(providerConfig.ModelAliases), providerConfig.Name)
- chatService.SetModelAliases(provider.ID, providerConfig.ModelAliases)
- configService.SetModelAliases(provider.ID, providerConfig.ModelAliases)
-
- // Save aliases to database
- if err := modelService.SaveAliasesToDB(provider.ID, providerConfig.ModelAliases); err != nil {
- log.Printf(" ⚠️ Failed to save aliases to database for %s: %v", providerConfig.Name, err)
- }
- }
-
- // Store recommended models
- if providerConfig.RecommendedModels != nil {
- configService.SetRecommendedModels(provider.ID, providerConfig.RecommendedModels)
-
- // Save recommended models to database
- if err := modelService.SaveRecommendedModelsToDB(provider.ID, providerConfig.RecommendedModels); err != nil {
- log.Printf(" ⚠️ Failed to save recommended models to database for %s: %v", providerConfig.Name, err)
- }
- }
-
- // Store provider security flag
- configService.SetProviderSecure(provider.ID, providerConfig.Secure)
-
- // Sync filters
- if len(providerConfig.Filters) > 0 {
- log.Printf(" 🔧 Syncing %d filters for %s...", len(providerConfig.Filters), providerConfig.Name)
- if err := providerService.SyncFilters(provider.ID, providerConfig.Filters); err != nil {
- return fmt.Errorf("failed to sync filters: %w", err)
- }
- }
-
- // Fetch models if provider is enabled
- if providerConfig.Enabled {
- if err := modelService.FetchFromProvider(provider); err != nil {
- log.Printf(" ⚠️ Failed to fetch models for %s: %v", provider.Name, err)
- } else {
- // Sync model alias metadata to database (smart_tool_router, agents, etc.)
- if len(providerConfig.ModelAliases) > 0 {
- if err := modelService.SyncModelAliasMetadata(provider.ID, providerConfig.ModelAliases); err != nil {
- log.Printf(" ⚠️ Failed to sync model alias metadata for %s: %v", provider.Name, err)
- }
- }
-
- // Apply filters
- if err := providerService.ApplyFilters(provider.ID); err != nil {
- log.Printf(" ⚠️ Failed to apply filters for %s: %v", provider.Name, err)
- }
- }
- }
- }
-
- // After syncing providers to database, load image providers from database
- log.Println("🎨 Loading image providers from database...")
- allProviders, err := providerService.GetAll()
- if err != nil {
- log.Printf("⚠️ Failed to load providers from database: %v", err)
- } else {
- // Convert database Provider models to ProviderConfig
- var providerConfigs []models.ProviderConfig
- for _, p := range allProviders {
- providerConfigs = append(providerConfigs, models.ProviderConfig{
- Name: p.Name,
- BaseURL: p.BaseURL,
- APIKey: p.APIKey,
- Enabled: p.Enabled,
- Secure: p.Secure,
- AudioOnly: p.AudioOnly,
- ImageOnly: p.ImageOnly,
- ImageEditOnly: p.ImageEditOnly,
- DefaultModel: p.DefaultModel,
- SystemPrompt: p.SystemPrompt,
- Favicon: p.Favicon,
- })
- }
-
- // Load image providers into the image provider service
- imageProviderService := services.GetImageProviderService()
- imageProviderService.LoadFromProviders(providerConfigs)
-
- // Load image edit providers into the image edit provider service
- imageEditProviderService := services.GetImageEditProviderService()
- imageEditProviderService.LoadFromProviders(providerConfigs)
- }
-
- log.Println("✅ Provider sync completed")
- return nil
-}
-
-// loadConfigFromDatabase loads model aliases and recommended models from database
-// Returns true if data was successfully loaded, false if database is empty (first run)
-func loadConfigFromDatabase(modelService *services.ModelService, chatService *services.ChatService, providerService *services.ProviderService) (bool, error) {
- log.Println("🔄 Loading configuration from database...")
-
- // Load aliases from database
- aliases, err := modelService.LoadAllAliasesFromDB()
- if err != nil {
- return false, fmt.Errorf("failed to load aliases from database: %w", err)
- }
-
- // Load recommended models from database
- recommendedModels, err := modelService.LoadAllRecommendedModelsFromDB()
- if err != nil {
- return false, fmt.Errorf("failed to load recommended models from database: %w", err)
- }
-
- // If database is empty, that's fine - admin will configure via UI
- if len(aliases) == 0 && len(recommendedModels) == 0 {
- log.Println("📋 Database is empty - use admin UI to configure providers and models")
- return false, nil
- }
-
- // Load into ConfigService
- configService := services.GetConfigService()
-
- // Load all providers to get security flags
- _, err = providerService.GetAllIncludingDisabled()
- if err != nil {
- return false, fmt.Errorf("failed to load providers: %w", err)
- }
-
- // Load aliases into both chat service and config service
- for providerID, providerAliases := range aliases {
- if len(providerAliases) > 0 {
- chatService.SetModelAliases(providerID, providerAliases)
- configService.SetModelAliases(providerID, providerAliases)
- log.Printf(" ✅ Loaded %d aliases for provider %d", len(providerAliases), providerID)
- }
- }
-
- // Load recommended models into config service
- for providerID, recommended := range recommendedModels {
- configService.SetRecommendedModels(providerID, recommended)
- log.Printf(" ✅ Loaded recommended models for provider %d", providerID)
- }
-
- // Load image providers from database
- log.Println("🎨 Loading image providers from database...")
- allProviders, err := providerService.GetAll()
- if err != nil {
- log.Printf("⚠️ Failed to load providers from database: %v", err)
- } else {
- // Convert database Provider models to ProviderConfig
- var providerConfigs []models.ProviderConfig
- for _, p := range allProviders {
- providerConfigs = append(providerConfigs, models.ProviderConfig{
- Name: p.Name,
- BaseURL: p.BaseURL,
- APIKey: p.APIKey,
- Enabled: p.Enabled,
- Secure: p.Secure,
- AudioOnly: p.AudioOnly,
- ImageOnly: p.ImageOnly,
- ImageEditOnly: p.ImageEditOnly,
- DefaultModel: p.DefaultModel,
- SystemPrompt: p.SystemPrompt,
- Favicon: p.Favicon,
- })
- }
-
- // Load image providers into the image provider service
- imageProviderService := services.GetImageProviderService()
- imageProviderService.LoadFromProviders(providerConfigs)
-
- // Load image edit providers into the image edit provider service
- imageEditProviderService := services.GetImageEditProviderService()
- imageEditProviderService.LoadFromProviders(providerConfigs)
- }
-
- log.Printf("✅ Loaded configuration from database: %d provider aliases, %d recommended model sets",
- len(aliases), len(recommendedModels))
- return true, nil
-}
-
-// startModelRefreshJob starts a background job to refresh models every 24 hours
-func startModelRefreshJob(providerService *services.ProviderService, modelService *services.ModelService, chatService *services.ChatService) {
- ticker := time.NewTicker(24 * time.Hour)
- defer ticker.Stop()
-
- log.Println("⏰ Model refresh job started (every 24 hours)")
-
- for range ticker.C {
- log.Println("🔄 Running scheduled model refresh...")
-
- // Reload aliases from database to ensure they stay in sync
- log.Println("🔄 Reloading model aliases from database...")
- if err := reloadModelAliases(providerService, modelService, chatService); err != nil {
- log.Printf("⚠️ Failed to reload model aliases: %v", err)
- } else {
- log.Println("✅ Model aliases reloaded successfully")
- }
-
- providers, err := providerService.GetAll()
- if err != nil {
- log.Printf("❌ Failed to get providers for refresh: %v", err)
- continue
- }
-
- for _, provider := range providers {
- if !provider.Enabled {
- continue
- }
-
- if err := modelService.FetchFromProvider(&provider); err != nil {
- log.Printf("❌ Error refreshing models for %s: %v", provider.Name, err)
- } else {
- // Apply filters after refresh
- if err := providerService.ApplyFilters(provider.ID); err != nil {
- log.Printf("⚠️ Failed to apply filters for %s: %v", provider.Name, err)
- }
- }
- }
-
- log.Println("✅ Scheduled model refresh completed")
- }
-}
-
-// reloadModelAliases reloads model aliases from database into memory
-// This is called by the background refresh job to keep in-memory cache fresh
-func reloadModelAliases(providerService *services.ProviderService, modelService *services.ModelService, chatService *services.ChatService) error {
- log.Println("🔄 [ALIAS-RELOAD] Loading model aliases from database...")
-
- // Load aliases from database
- aliases, err := modelService.LoadAllAliasesFromDB()
- if err != nil {
- return fmt.Errorf("failed to load aliases from database: %w", err)
- }
-
- // Load recommended models from database
- recommendedModels, err := modelService.LoadAllRecommendedModelsFromDB()
- if err != nil {
- return fmt.Errorf("failed to load recommended models from database: %w", err)
- }
-
- configService := services.GetConfigService()
-
- // Load aliases into both chat service and config service
- for providerID, providerAliases := range aliases {
- if len(providerAliases) > 0 {
- chatService.SetModelAliases(providerID, providerAliases)
- configService.SetModelAliases(providerID, providerAliases)
- log.Printf(" ✅ Reloaded %d aliases for provider %d", len(providerAliases), providerID)
- }
- }
-
- // Load recommended models into config service
- for providerID, recommended := range recommendedModels {
- configService.SetRecommendedModels(providerID, recommended)
- }
-
- log.Printf("✅ [ALIAS-RELOAD] Configuration reloaded from database")
- return nil
-}
-
-// startImageCleanupJob starts a background job to clean up expired images every 10 minutes
-func startImageCleanupJob(uploadDir string) {
- ticker := time.NewTicker(10 * time.Minute)
- defer ticker.Stop()
-
- log.Println("⏰ File cleanup job started (every 10 minutes) - handles images, CSV, Excel, JSON, etc.")
-
- for range ticker.C {
- log.Println("🧹 Running scheduled file cleanup...")
-
- // Get file cache service
- fileCache := filecache.GetService()
-
- // Cleanup expired files tracked in cache (images, CSV, Excel, JSON, etc.)
- fileCache.CleanupExpiredFiles()
-
- // Cleanup orphaned files on disk (not in cache - e.g., from server restarts)
- // Max age of 1 hour matches our retention policy
- fileCache.CleanupOrphanedFiles(uploadDir, 1*time.Hour)
-
- log.Println("✅ Scheduled file cleanup completed")
- }
-}
-
-// startDocumentCleanupJob starts a background job to clean up downloaded documents every 5 minutes
-func startDocumentCleanupJob() {
- ticker := time.NewTicker(5 * time.Minute)
- defer ticker.Stop()
-
- log.Println("⏰ Document cleanup job started (every 5 minutes)")
-
- for range ticker.C {
- log.Println("🧹 Running scheduled document cleanup...")
-
- // Get document service
- documentService := document.GetService()
-
- // Cleanup downloaded documents
- documentService.CleanupDownloadedDocuments()
-
- log.Println("✅ Scheduled document cleanup completed")
- }
-}
-
-// startMemoryExtractionWorker processes pending memory extraction jobs
-func startMemoryExtractionWorker(memoryExtractionService *services.MemoryExtractionService) {
- ticker := time.NewTicker(30 * time.Second)
- defer ticker.Stop()
-
- log.Println("⏰ Memory extraction worker started (every 30 seconds)")
-
- for range ticker.C {
- ctx, cancel := context.WithTimeout(context.Background(), 25*time.Second)
- if err := memoryExtractionService.ProcessPendingJobs(ctx); err != nil {
- log.Printf("⚠️ [MEMORY-WORKER] Failed to process jobs: %v", err)
- }
- cancel()
- }
-}
-
-func startMemoryDecayWorker(memoryDecayService *services.MemoryDecayService) {
- // Run immediately on startup
- log.Println("🔄 [MEMORY-DECAY] Running initial decay job")
- ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
- if err := memoryDecayService.RunDecayJob(ctx); err != nil {
- log.Printf("⚠️ [MEMORY-DECAY] Initial decay job failed: %v", err)
- }
- cancel()
-
- // Then run every 6 hours
- ticker := time.NewTicker(6 * time.Hour)
- defer ticker.Stop()
-
- log.Println("⏰ Memory decay worker started (every 6 hours)")
-
- for range ticker.C {
- ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
- if err := memoryDecayService.RunDecayJob(ctx); err != nil {
- log.Printf("⚠️ [MEMORY-DECAY] Decay job failed: %v", err)
- }
- cancel()
- }
-}
-
-// startProvidersFileWatcher watches providers.json for changes and auto-syncs
-func startProvidersFileWatcher(
- filePath string,
- providerService *services.ProviderService,
- modelService *services.ModelService,
- chatService *services.ChatService,
-) {
- watcher, err := fsnotify.NewWatcher()
- if err != nil {
- log.Printf("⚠️ Failed to create file watcher: %v", err)
- return
- }
-
- // Get absolute path for the file
- absPath, err := filepath.Abs(filePath)
- if err != nil {
- log.Printf("⚠️ Failed to get absolute path for %s: %v", filePath, err)
- watcher.Close()
- return
- }
-
- // Watch the directory containing the file (more reliable than watching the file directly)
- dir := filepath.Dir(absPath)
- filename := filepath.Base(absPath)
-
- if err := watcher.Add(dir); err != nil {
- log.Printf("⚠️ Failed to watch directory %s: %v", dir, err)
- watcher.Close()
- return
- }
-
- log.Printf("👁️ Watching %s for changes (hot-reload enabled)", filePath)
-
- // Debounce timer to avoid multiple syncs for rapid file changes
- var debounceTimer *time.Timer
- debounceDuration := 500 * time.Millisecond
-
- for {
- select {
- case event, ok := <-watcher.Events:
- if !ok {
- return
- }
-
- // Only react to changes to our specific file
- if filepath.Base(event.Name) != filename {
- continue
- }
-
- // React to write and create events
- if event.Op&fsnotify.Write == fsnotify.Write || event.Op&fsnotify.Create == fsnotify.Create {
- // Debounce: cancel previous timer and set a new one
- if debounceTimer != nil {
- debounceTimer.Stop()
- }
-
- debounceTimer = time.AfterFunc(debounceDuration, func() {
- log.Printf("🔄 Detected changes in %s, re-syncing providers...", filePath)
-
- if err := syncProviders(filePath, providerService, modelService, chatService); err != nil {
- log.Printf("❌ Failed to sync providers after file change: %v", err)
- } else {
- log.Printf("✅ Providers synced successfully from %s", filePath)
- }
- })
- }
-
- case err, ok := <-watcher.Errors:
- if !ok {
- return
- }
- log.Printf("⚠️ File watcher error: %v", err)
- }
- }
-}
diff --git a/backend/docker-entrypoint.sh b/backend/docker-entrypoint.sh
deleted file mode 100755
index 8147260f..00000000
--- a/backend/docker-entrypoint.sh
+++ /dev/null
@@ -1,67 +0,0 @@
-#!/bin/sh
-set -e
-
-# ClaraVerse Backend Docker Entrypoint Script
-# This script handles initialization before starting the application
-
-echo "=== ClaraVerse Backend Starting ==="
-
-# Check if .env file exists
-if [ ! -f "/app/.env" ]; then
- echo "WARNING: .env file not found. Using example configuration."
- echo "Please create .env file from .env.example before running in production."
-
- # Create .env from example if available
- if [ -f "/app/.env.example" ]; then
- cp /app/.env.example /app/.env
- echo "Created .env from .env.example"
- fi
-fi
-
-# Check if providers.json exists at configured location
-PROVIDERS_PATH="${PROVIDERS_FILE:-/app/providers.json}"
-if [ ! -f "$PROVIDERS_PATH" ]; then
- echo "WARNING: providers.json not found at $PROVIDERS_PATH. Using example configuration."
-
- # Create providers.json from example if available
- if [ -f "/app/providers.example.json" ]; then
- cp /app/providers.example.json "$PROVIDERS_PATH"
- echo "Created providers.json from providers.example.json"
- else
- echo "ERROR: No providers configuration found!"
- echo "Please mount providers.json or create one from providers.example.json"
- exit 1
- fi
-fi
-
-# Ensure data directories exist with proper permissions
-mkdir -p /app/data /app/uploads /app/logs
-
-# Display configuration summary
-echo ""
-echo "Configuration:"
-echo " - Database: ${DATABASE_PATH:-model_capabilities.db}"
-echo " - Providers: ${PROVIDERS_FILE:-providers.json}"
-echo " - Upload Dir: ${UPLOAD_DIR:-./uploads}"
-echo " - Environment: ${ENVIRONMENT:-development}"
-echo " - Port: ${PORT:-3001}"
-echo ""
-
-# Check for required environment variables in production
-if [ "$ENVIRONMENT" = "production" ]; then
- echo "Running in PRODUCTION mode"
-
- if [ -z "$SUPABASE_URL" ] || [ -z "$SUPABASE_KEY" ]; then
- echo "WARNING: Supabase configuration is missing!"
- echo "SUPABASE_URL and SUPABASE_KEY are required in production mode."
- echo "The server will terminate if authentication is not properly configured."
- fi
-fi
-
-echo ""
-echo "Starting ClaraVerse backend..."
-echo "==================================="
-echo ""
-
-# Execute the main application
-exec "$@"
diff --git a/backend/e2b-service/.dockerignore b/backend/e2b-service/.dockerignore
deleted file mode 100644
index 582f6978..00000000
--- a/backend/e2b-service/.dockerignore
+++ /dev/null
@@ -1,18 +0,0 @@
-__pycache__
-*.pyc
-*.pyo
-*.pyd
-.Python
-*.so
-*.egg
-*.egg-info
-dist
-build
-.pytest_cache
-.coverage
-htmlcov
-.tox
-.env
-.venv
-venv
-ENV
diff --git a/backend/e2b-service/Dockerfile b/backend/e2b-service/Dockerfile
deleted file mode 100644
index ffb80752..00000000
--- a/backend/e2b-service/Dockerfile
+++ /dev/null
@@ -1,20 +0,0 @@
-FROM python:3.11-slim
-
-WORKDIR /app
-
-# Install dependencies
-COPY requirements.txt .
-RUN pip install --no-cache-dir -r requirements.txt
-
-# Copy application
-COPY main.py .
-
-# Expose port
-EXPOSE 8001
-
-# Health check
-HEALTHCHECK --interval=30s --timeout=10s --start-period=5s --retries=3 \
- CMD python -c "import requests; requests.get('http://localhost:8001/health')"
-
-# Run the application
-CMD ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "8001", "--log-level", "info"]
diff --git a/backend/e2b-service/main.py b/backend/e2b-service/main.py
deleted file mode 100644
index 61304f28..00000000
--- a/backend/e2b-service/main.py
+++ /dev/null
@@ -1,475 +0,0 @@
-#!/usr/bin/env python3
-"""
-E2B Code Executor Microservice for ClaraVerse
-Now using E2B in LOCAL mode - no cloud API required!
-Provides REST API for executing Python code in E2B sandboxes
-"""
-
-import os
-import base64
-from typing import List, Optional, Dict, Any
-from fastapi import FastAPI, HTTPException, UploadFile, File, Form
-from fastapi.middleware.cors import CORSMiddleware
-from pydantic import BaseModel
-from e2b_code_interpreter import Sandbox
-import logging
-
-# Configure logging
-logging.basicConfig(level=logging.INFO)
-logger = logging.getLogger(__name__)
-
-# Configure E2B for local execution (v2.0 - no API key needed!)
-E2B_MODE = os.getenv("E2B_MODE", "local") # local or production
-os.environ['E2B_MODE'] = E2B_MODE
-
-if E2B_MODE == "local":
- logger.info("🐳 E2B running in LOCAL Docker mode - no API key required!")
- os.environ['E2B_LOCAL_USE_DOCKER'] = os.getenv("E2B_LOCAL_USE_DOCKER", "true")
- os.environ['E2B_SANDBOX_POOL_SIZE'] = os.getenv("E2B_SANDBOX_POOL_SIZE", "3")
- os.environ['E2B_EXECUTION_TIMEOUT'] = os.getenv("E2B_EXECUTION_TIMEOUT", "30000")
- os.environ['E2B_RATE_LIMIT_PER_MIN'] = os.getenv("E2B_RATE_LIMIT_PER_MIN", "20")
- logger.info(f" Pool size: {os.environ['E2B_SANDBOX_POOL_SIZE']} warm sandboxes")
- logger.info(f" Timeout: {os.environ['E2B_EXECUTION_TIMEOUT']}ms")
-else:
- # Production mode - requires E2B API key
- E2B_API_KEY = os.getenv("E2B_API_KEY")
- if not E2B_API_KEY:
- logger.error("E2B_API_KEY environment variable required for production mode")
- raise RuntimeError("E2B_API_KEY is required when E2B_MODE=production")
- logger.info("☁️ E2B running in CLOUD mode with API key")
-
-app = FastAPI(
- title="E2B Code Executor Service",
- description="Microservice for executing Python code in isolated E2B sandboxes",
- version="1.0.0"
-)
-
-# CORS middleware
-app.add_middleware(
- CORSMiddleware,
- allow_origins=["*"], # In production, restrict to backend service
- allow_credentials=True,
- allow_methods=["*"],
- allow_headers=["*"],
-)
-
-
-# Request/Response Models
-class ExecuteRequest(BaseModel):
- code: str
- timeout: Optional[int] = 30 # seconds
-
-
-class PlotResult(BaseModel):
- format: str # "png", "svg", etc.
- data: str # base64 encoded
-
-
-class ExecuteResponse(BaseModel):
- success: bool
- stdout: str
- stderr: str
- error: Optional[str] = None
- plots: List[PlotResult] = []
- execution_time: Optional[float] = None
-
-
-class FileUploadRequest(BaseModel):
- code: str
- timeout: Optional[int] = 30
-
-
-# Advanced execution models (with dependencies and output files)
-class AdvancedExecuteRequest(BaseModel):
- code: str
- timeout: Optional[int] = 30
- dependencies: List[str] = [] # pip packages to install
- output_files: List[str] = [] # files to retrieve after execution
-
-
-class FileResult(BaseModel):
- filename: str
- data: str # base64 encoded
- size: int
-
-
-class AdvancedExecuteResponse(BaseModel):
- success: bool
- stdout: str
- stderr: str
- error: Optional[str] = None
- plots: List[PlotResult] = []
- files: List[FileResult] = []
- execution_time: Optional[float] = None
- install_output: str = ""
-
-
-# Health check endpoint
-@app.get("/health")
-async def health_check():
- """Health check endpoint"""
- return {
- "status": "healthy",
- "service": "e2b-executor",
- "mode": E2B_MODE,
- "e2b_api_key_set": bool(os.getenv("E2B_API_KEY")) if E2B_MODE == "production" else False
- }
-
-
-# Execute Python code endpoint
-@app.post("/execute", response_model=ExecuteResponse)
-async def execute_code(request: ExecuteRequest):
- """
- Execute Python code in an E2B sandbox (local Docker mode)
-
- Returns:
- - stdout: Standard output
- - stderr: Standard error
- - plots: List of generated plots (base64 encoded)
- - error: Error message if execution failed
- """
- logger.info(f"Executing code in {E2B_MODE.upper()} mode (length: {len(request.code)} chars)")
-
- try:
- # Create sandbox
- with Sandbox.create() as sandbox:
- # Run code
- execution = sandbox.run_code(request.code)
-
- # Collect stdout
- stdout = ""
- if execution.logs.stdout:
- stdout = "\n".join(execution.logs.stdout)
-
- # Collect stderr
- stderr = ""
- if execution.logs.stderr:
- stderr = "\n".join(execution.logs.stderr)
-
- # Check for execution errors
- error_msg = None
- if execution.error:
- error_msg = str(execution.error)
- logger.warning(f"Execution error: {error_msg}")
-
- # Collect plots and text results
- plots = []
- result_texts = []
- for i, result in enumerate(execution.results):
- if hasattr(result, 'png') and result.png:
- plots.append(PlotResult(
- format="png",
- data=result.png # Already base64 encoded
- ))
- logger.info(f"Found plot {i}: {len(result.png)} bytes (base64)")
- elif hasattr(result, 'text') and result.text:
- result_texts.append(result.text)
- logger.info(f"Found text result {i}: {result.text[:100]}...")
-
- # Append result texts to stdout (captures last expression like Jupyter)
- if result_texts:
- result_output = "\n".join(result_texts)
- if stdout:
- stdout = stdout + "\n" + result_output
- else:
- stdout = result_output
-
- response = ExecuteResponse(
- success=error_msg is None,
- stdout=stdout,
- stderr=stderr,
- error=error_msg,
- plots=plots
- )
-
- logger.info(f"Execution completed: success={response.success}, plots={len(plots)}")
- return response
-
- except Exception as e:
- logger.error(f"Sandbox execution failed: {str(e)}")
- raise HTTPException(
- status_code=500,
- detail=f"Sandbox execution failed: {str(e)}"
- )
-
-
-# Execute with file upload endpoint
-@app.post("/execute-with-files", response_model=ExecuteResponse)
-async def execute_with_files(
- code: str = Form(...),
- files: List[UploadFile] = File(...),
- timeout: int = Form(30)
-):
- """
- Execute Python code with uploaded files
-
- Files are uploaded to the sandbox and can be accessed by filename in the code
- """
- logger.info(f"Executing code with {len(files)} files")
-
- try:
- # Create sandbox
- with Sandbox.create() as sandbox:
- # Upload files to sandbox
- for file in files:
- content = await file.read()
- sandbox.files.write(file.filename, content)
- logger.info(f"Uploaded file: {file.filename} ({len(content)} bytes)")
-
- # Run code
- execution = sandbox.run_code(code)
-
- # Collect stdout
- stdout = ""
- if execution.logs.stdout:
- stdout = "\n".join(execution.logs.stdout)
-
- # Collect stderr
- stderr = ""
- if execution.logs.stderr:
- stderr = "\n".join(execution.logs.stderr)
-
- # Check for errors
- error_msg = None
- if execution.error:
- error_msg = str(execution.error)
- logger.warning(f"Execution error: {error_msg}")
-
- # Collect plots
- plots = []
- for i, result in enumerate(execution.results):
- if hasattr(result, 'png') and result.png:
- plots.append(PlotResult(
- format="png",
- data=result.png
- ))
- logger.info(f"Found plot {i}")
-
- response = ExecuteResponse(
- success=error_msg is None,
- stdout=stdout,
- stderr=stderr,
- error=error_msg,
- plots=plots
- )
-
- logger.info(f"Execution with files completed: success={response.success}")
- return response
-
- except Exception as e:
- logger.error(f"Sandbox execution failed: {str(e)}")
- raise HTTPException(
- status_code=500,
- detail=f"Sandbox execution failed: {str(e)}"
- )
-
-
-# Execute with dependencies and output file retrieval
-@app.post("/execute-advanced", response_model=AdvancedExecuteResponse)
-async def execute_advanced(request: AdvancedExecuteRequest):
- """
- Execute Python code with pip dependencies and output file retrieval.
-
- - Install pip packages before running code
- - Run user code (max 30 seconds)
- - Auto-detect and retrieve ALL generated files (plus any explicitly specified)
- """
- import time
-
- logger.info(f"Advanced execution: code={len(request.code)} chars, deps={request.dependencies}, output_files={request.output_files}")
-
- try:
- with Sandbox.create() as sandbox:
- start_time = time.time()
- install_output = ""
-
- # 1. Install dependencies (if any)
- if request.dependencies:
- deps_str = " ".join(request.dependencies)
- logger.info(f"Installing dependencies: {deps_str}")
- try:
- result = sandbox.commands.run(f"pip install -q {deps_str}", timeout=60)
- install_output = (result.stdout or "") + (result.stderr or "")
- logger.info(f"Dependencies installed: {install_output[:200]}")
- except Exception as e:
- logger.error(f"Dependency installation failed: {e}")
- return AdvancedExecuteResponse(
- success=False,
- stdout="",
- stderr="",
- error=f"Failed to install dependencies: {str(e)}",
- plots=[],
- files=[],
- execution_time=time.time() - start_time,
- install_output=str(e)
- )
-
- # 2. List files BEFORE execution to detect new files later
- files_before = set()
- try:
- result = sandbox.commands.run("find /home/user -maxdepth 2 -type f 2>/dev/null || ls -la /home/user", timeout=10)
- if result.stdout:
- for line in result.stdout.strip().split('\n'):
- line = line.strip()
- if line and not line.startswith('total'):
- # Handle both find output (full paths) and ls output
- if line.startswith('/'):
- files_before.add(line)
- else:
- # ls -la format: permissions links owner group size date name
- parts = line.split()
- if len(parts) >= 9:
- files_before.add(parts[-1])
- logger.info(f"Files before execution: {len(files_before)}")
- except Exception as e:
- logger.warning(f"Could not list files before execution: {e}")
-
- # 3. Run user code
- execution = sandbox.run_code(request.code)
-
- # Collect stdout
- stdout = ""
- if execution.logs.stdout:
- stdout = "\n".join(execution.logs.stdout)
-
- # Collect stderr
- stderr = ""
- if execution.logs.stderr:
- stderr = "\n".join(execution.logs.stderr)
-
- # Check for errors
- error_msg = None
- if execution.error:
- error_msg = str(execution.error)
- logger.warning(f"Execution error: {error_msg}")
-
- # Collect plots and text results from execution.results
- # E2B results contain last expression value (like Jupyter)
- plots = []
- result_texts = []
- for i, result in enumerate(execution.results):
- if hasattr(result, 'png') and result.png:
- plots.append(PlotResult(
- format="png",
- data=result.png
- ))
- logger.info(f"Found plot {i}")
- elif hasattr(result, 'text') and result.text:
- # Capture text output from last expression (like Jupyter Out[])
- result_texts.append(result.text)
- logger.info(f"Found text result {i}: {result.text[:100]}...")
-
- # Append result texts to stdout if no explicit print was used
- if result_texts:
- result_output = "\n".join(result_texts)
- if stdout:
- stdout = stdout + "\n" + result_output
- else:
- stdout = result_output
-
- # 4. List files AFTER execution to detect new files
- files_after = set()
- new_files = []
- try:
- result = sandbox.commands.run("find /home/user -maxdepth 2 -type f 2>/dev/null || ls -la /home/user", timeout=10)
- if result.stdout:
- for line in result.stdout.strip().split('\n'):
- line = line.strip()
- if line and not line.startswith('total'):
- if line.startswith('/'):
- files_after.add(line)
- else:
- parts = line.split()
- if len(parts) >= 9:
- files_after.add(parts[-1])
- # Find new files (created during execution)
- new_files = list(files_after - files_before)
- # Filter out common unwanted files
- excluded_patterns = ['.pyc', '__pycache__', '.ipynb_checkpoints', '.cache']
- new_files = [f for f in new_files if not any(p in f for p in excluded_patterns)]
- logger.info(f"Files after execution: {len(files_after)}, new files detected: {new_files}")
- except Exception as e:
- logger.warning(f"Could not list files after execution: {e}")
-
- # 5. Collect output files (auto-detected + explicitly requested)
- files = []
- collected_filenames = set()
-
- # First collect explicitly requested files
- for filepath in request.output_files:
- try:
- content = sandbox.files.read(filepath)
- # Handle both string and bytes
- if isinstance(content, str):
- content = content.encode('utf-8')
- filename = os.path.basename(filepath)
- files.append(FileResult(
- filename=filename,
- data=base64.b64encode(content).decode('utf-8'),
- size=len(content)
- ))
- collected_filenames.add(filename)
- logger.info(f"Retrieved requested file: {filepath} ({len(content)} bytes)")
- except Exception as e:
- logger.warning(f"Could not retrieve file {filepath}: {e}")
-
- # Then collect auto-detected new files (if not already collected)
- for filepath in new_files:
- filename = os.path.basename(filepath)
- if filename in collected_filenames:
- continue # Already collected
- try:
- # Try both the full path and just the filename
- content = None
- for try_path in [filepath, f"/home/user/{filename}", filename]:
- try:
- content = sandbox.files.read(try_path)
- break
- except:
- continue
-
- if content is not None:
- if isinstance(content, str):
- content = content.encode('utf-8')
- files.append(FileResult(
- filename=filename,
- data=base64.b64encode(content).decode('utf-8'),
- size=len(content)
- ))
- collected_filenames.add(filename)
- logger.info(f"Retrieved auto-detected file: {filename} ({len(content)} bytes)")
- except Exception as e:
- logger.warning(f"Could not retrieve auto-detected file {filepath}: {e}")
-
- execution_time = time.time() - start_time
-
- response = AdvancedExecuteResponse(
- success=error_msg is None,
- stdout=stdout,
- stderr=stderr,
- error=error_msg,
- plots=plots,
- files=files,
- execution_time=execution_time,
- install_output=install_output
- )
-
- logger.info(f"Advanced execution completed: success={response.success}, plots={len(plots)}, files={len(files)}, time={execution_time:.2f}s")
- return response
-
- except Exception as e:
- logger.error(f"Advanced sandbox execution failed: {str(e)}")
- raise HTTPException(
- status_code=500,
- detail=f"Sandbox execution failed: {str(e)}"
- )
-
-
-if __name__ == "__main__":
- import uvicorn
- uvicorn.run(
- app,
- host="0.0.0.0",
- port=8001,
- log_level="info"
- )
diff --git a/backend/e2b-service/main_simple.py b/backend/e2b-service/main_simple.py
deleted file mode 100644
index a302cdb0..00000000
--- a/backend/e2b-service/main_simple.py
+++ /dev/null
@@ -1,417 +0,0 @@
-#!/usr/bin/env python3
-"""
-Simple Python Code Executor for ClaraVerse All-in-One Docker Image
-
-This is a lightweight alternative to the E2B-based executor that runs Python code
-directly using subprocess. It's designed for the all-in-one Docker image where
-Docker-in-Docker isn't available for E2B local mode.
-
-Features:
-- Subprocess-based Python execution with timeout
-- Basic sandboxing via restricted builtins and resource limits
-- Support for matplotlib plots (auto-saved as PNG)
-- File upload and retrieval support
-- No external dependencies (E2B API key or Docker)
-
-Note: This is less secure than E2B sandboxes. For production use with untrusted
-code, use the regular E2B service with proper sandboxing.
-"""
-
-import os
-import sys
-import base64
-import tempfile
-import subprocess
-import shutil
-import signal
-from typing import List, Optional
-from pathlib import Path
-from fastapi import FastAPI, HTTPException, UploadFile, File, Form
-from fastapi.middleware.cors import CORSMiddleware
-from pydantic import BaseModel
-import logging
-import time
-import uuid
-
-# Configure logging
-logging.basicConfig(level=logging.INFO)
-logger = logging.getLogger(__name__)
-
-logger.info("🚀 Starting Simple Python Executor (All-in-One Mode)")
-logger.info(" This executor runs Python code directly without E2B sandboxes")
-
-app = FastAPI(
- title="Simple Python Executor Service",
- description="Lightweight Python code executor for ClaraVerse All-in-One",
- version="1.0.0"
-)
-
-# CORS middleware
-app.add_middleware(
- CORSMiddleware,
- allow_origins=["*"],
- allow_credentials=True,
- allow_methods=["*"],
- allow_headers=["*"],
-)
-
-# Execution settings
-EXECUTION_TIMEOUT = int(os.getenv("EXECUTION_TIMEOUT", "30")) # seconds
-MAX_OUTPUT_SIZE = 100 * 1024 # 100KB max output
-WORK_DIR = Path(os.getenv("WORK_DIR", "/tmp/code-executor"))
-
-# Ensure work directory exists
-WORK_DIR.mkdir(parents=True, exist_ok=True)
-
-
-# Request/Response Models
-class ExecuteRequest(BaseModel):
- code: str
- timeout: Optional[int] = 30
-
-
-class PlotResult(BaseModel):
- format: str
- data: str # base64 encoded
-
-
-class ExecuteResponse(BaseModel):
- success: bool
- stdout: str
- stderr: str
- error: Optional[str] = None
- plots: List[PlotResult] = []
- execution_time: Optional[float] = None
-
-
-class AdvancedExecuteRequest(BaseModel):
- code: str
- timeout: Optional[int] = 30
- dependencies: List[str] = []
- output_files: List[str] = []
-
-
-class FileResult(BaseModel):
- filename: str
- data: str # base64 encoded
- size: int
-
-
-class AdvancedExecuteResponse(BaseModel):
- success: bool
- stdout: str
- stderr: str
- error: Optional[str] = None
- plots: List[PlotResult] = []
- files: List[FileResult] = []
- execution_time: Optional[float] = None
- install_output: str = ""
-
-
-def create_wrapper_code(user_code: str, plot_dir: str) -> str:
- """
- Wrap user code with matplotlib backend setup for headless plot generation.
- """
- wrapper = f'''
-import sys
-import os
-
-# Set matplotlib to use non-interactive backend BEFORE importing pyplot
-import matplotlib
-matplotlib.use('Agg')
-
-# Configure plot output directory
-_plot_dir = {repr(plot_dir)}
-_plot_counter = [0]
-
-# Patch plt.show() to save plots instead
-import matplotlib.pyplot as plt
-_original_show = plt.show
-
-def _patched_show(*args, **kwargs):
- _plot_counter[0] += 1
- plot_path = os.path.join(_plot_dir, f"plot_{{_plot_counter[0]}}.png")
- plt.savefig(plot_path, format='png', dpi=100, bbox_inches='tight')
- print(f"[PLOT_SAVED]{{plot_path}}")
- plt.close()
-
-plt.show = _patched_show
-
-# Also save figures when plt.savefig is called
-_original_savefig = plt.savefig
-
-def _patched_savefig(fname, *args, **kwargs):
- # Call original savefig
- result = _original_savefig(fname, *args, **kwargs)
- # Also save to our plot dir if it's a new file
- if not str(fname).startswith(_plot_dir):
- _plot_counter[0] += 1
- copy_path = os.path.join(_plot_dir, f"plot_{{_plot_counter[0]}}.png")
- _original_savefig(copy_path, format='png', dpi=100, bbox_inches='tight')
- print(f"[PLOT_SAVED]{{copy_path}}")
- return result
-
-# Don't patch savefig - let user control where files go
-# plt.savefig = _patched_savefig
-
-# Change to work directory
-os.chdir({repr(plot_dir)})
-
-# Execute user code
-{user_code}
-'''
- return wrapper
-
-
-def execute_python_code(code: str, timeout: int, work_dir: Path, dependencies: List[str] = None) -> dict:
- """
- Execute Python code in a subprocess with timeout.
- Returns dict with stdout, stderr, error, plots, files.
- """
- start_time = time.time()
- result = {
- "stdout": "",
- "stderr": "",
- "error": None,
- "plots": [],
- "files": [],
- "install_output": "",
- "execution_time": 0
- }
-
- # Create temporary directory for this execution
- exec_id = str(uuid.uuid4())[:8]
- exec_dir = work_dir / exec_id
- exec_dir.mkdir(parents=True, exist_ok=True)
- plot_dir = exec_dir / "plots"
- plot_dir.mkdir(exist_ok=True)
-
- try:
- # Install dependencies if requested
- if dependencies:
- logger.info(f"Installing dependencies: {dependencies}")
- try:
- install_result = subprocess.run(
- [sys.executable, "-m", "pip", "install", "-q"] + dependencies,
- capture_output=True,
- text=True,
- timeout=60
- )
- result["install_output"] = install_result.stdout + install_result.stderr
- if install_result.returncode != 0:
- result["error"] = f"Failed to install dependencies: {result['install_output']}"
- result["execution_time"] = time.time() - start_time
- return result
- logger.info(f"Dependencies installed: {result['install_output'][:200]}")
- except subprocess.TimeoutExpired:
- result["error"] = "Dependency installation timed out"
- result["execution_time"] = time.time() - start_time
- return result
-
- # Wrap code with matplotlib setup
- wrapped_code = create_wrapper_code(code, str(exec_dir))
-
- # Write code to temp file
- code_file = exec_dir / "script.py"
- code_file.write_text(wrapped_code)
-
- # Execute with timeout
- try:
- proc = subprocess.run(
- [sys.executable, str(code_file)],
- capture_output=True,
- text=True,
- timeout=timeout,
- cwd=str(exec_dir)
- )
-
- result["stdout"] = proc.stdout[:MAX_OUTPUT_SIZE] if proc.stdout else ""
- result["stderr"] = proc.stderr[:MAX_OUTPUT_SIZE] if proc.stderr else ""
-
- if proc.returncode != 0:
- # Extract just the error message, not the full traceback if possible
- stderr = result["stderr"]
- if "Error:" in stderr:
- result["error"] = stderr.split("\n")[-2] if stderr else "Execution failed"
- else:
- result["error"] = stderr or "Execution failed with non-zero exit code"
-
- except subprocess.TimeoutExpired:
- result["error"] = f"Execution timed out after {timeout} seconds"
- result["stderr"] = f"TimeoutError: Code execution exceeded {timeout} second limit"
-
- # Collect plots
- plot_files = list(plot_dir.glob("*.png")) + list(exec_dir.glob("*.png"))
- for plot_file in plot_files:
- try:
- with open(plot_file, "rb") as f:
- plot_data = base64.b64encode(f.read()).decode("utf-8")
- result["plots"].append({
- "format": "png",
- "data": plot_data
- })
- logger.info(f"Collected plot: {plot_file.name}")
- except Exception as e:
- logger.warning(f"Failed to read plot {plot_file}: {e}")
-
- # Remove [PLOT_SAVED] messages from stdout
- if result["stdout"]:
- lines = result["stdout"].split("\n")
- result["stdout"] = "\n".join(
- line for line in lines if not line.startswith("[PLOT_SAVED]")
- )
-
- # Collect any other generated files (excluding script.py and plots)
- for file_path in exec_dir.iterdir():
- if file_path.is_file() and file_path.name != "script.py":
- if file_path.suffix.lower() not in [".png", ".pyc"]:
- try:
- with open(file_path, "rb") as f:
- file_data = f.read()
- result["files"].append({
- "filename": file_path.name,
- "data": base64.b64encode(file_data).decode("utf-8"),
- "size": len(file_data)
- })
- logger.info(f"Collected file: {file_path.name} ({len(file_data)} bytes)")
- except Exception as e:
- logger.warning(f"Failed to read file {file_path}: {e}")
-
- finally:
- # Clean up execution directory
- try:
- shutil.rmtree(exec_dir)
- except Exception as e:
- logger.warning(f"Failed to cleanup exec dir: {e}")
-
- result["execution_time"] = time.time() - start_time
-
- return result
-
-
-# Health check endpoint
-@app.get("/health")
-async def health_check():
- """Health check endpoint"""
- return {
- "status": "healthy",
- "service": "simple-python-executor",
- "mode": "subprocess",
- "e2b_api_key_set": False
- }
-
-
-# Execute Python code endpoint
-@app.post("/execute", response_model=ExecuteResponse)
-async def execute_code(request: ExecuteRequest):
- """
- Execute Python code using subprocess.
- """
- logger.info(f"Executing code (length: {len(request.code)} chars)")
-
- timeout = min(request.timeout or EXECUTION_TIMEOUT, EXECUTION_TIMEOUT)
- result = execute_python_code(request.code, timeout, WORK_DIR)
-
- response = ExecuteResponse(
- success=result["error"] is None,
- stdout=result["stdout"],
- stderr=result["stderr"],
- error=result["error"],
- plots=[PlotResult(**p) for p in result["plots"]],
- execution_time=result["execution_time"]
- )
-
- logger.info(f"Execution completed: success={response.success}, plots={len(response.plots)}")
- return response
-
-
-# Execute with file upload endpoint
-@app.post("/execute-with-files", response_model=ExecuteResponse)
-async def execute_with_files(
- code: str = Form(...),
- files: List[UploadFile] = File(...),
- timeout: int = Form(30)
-):
- """
- Execute Python code with uploaded files.
- """
- logger.info(f"Executing code with {len(files)} files")
-
- # Create temp directory and save uploaded files
- exec_id = str(uuid.uuid4())[:8]
- exec_dir = WORK_DIR / exec_id
- exec_dir.mkdir(parents=True, exist_ok=True)
-
- try:
- # Save uploaded files
- for file in files:
- content = await file.read()
- file_path = exec_dir / file.filename
- file_path.write_bytes(content)
- logger.info(f"Uploaded file: {file.filename} ({len(content)} bytes)")
-
- # Prepend code to change to the directory with uploaded files
- full_code = f"import os; os.chdir({repr(str(exec_dir))})\n{code}"
-
- timeout = min(timeout, EXECUTION_TIMEOUT)
- result = execute_python_code(full_code, timeout, exec_dir)
-
- response = ExecuteResponse(
- success=result["error"] is None,
- stdout=result["stdout"],
- stderr=result["stderr"],
- error=result["error"],
- plots=[PlotResult(**p) for p in result["plots"]],
- execution_time=result["execution_time"]
- )
-
- logger.info(f"Execution with files completed: success={response.success}")
- return response
-
- finally:
- # Cleanup
- try:
- shutil.rmtree(exec_dir)
- except:
- pass
-
-
-# Advanced execution endpoint
-@app.post("/execute-advanced", response_model=AdvancedExecuteResponse)
-async def execute_advanced(request: AdvancedExecuteRequest):
- """
- Execute Python code with pip dependencies and output file retrieval.
- """
- logger.info(f"Advanced execution: code={len(request.code)} chars, deps={request.dependencies}, output_files={request.output_files}")
-
- timeout = min(request.timeout or EXECUTION_TIMEOUT, EXECUTION_TIMEOUT)
- result = execute_python_code(
- request.code,
- timeout,
- WORK_DIR,
- dependencies=request.dependencies
- )
-
- response = AdvancedExecuteResponse(
- success=result["error"] is None,
- stdout=result["stdout"],
- stderr=result["stderr"],
- error=result["error"],
- plots=[PlotResult(**p) for p in result["plots"]],
- files=[FileResult(**f) for f in result["files"]],
- execution_time=result["execution_time"],
- install_output=result["install_output"]
- )
-
- logger.info(f"Advanced execution completed: success={response.success}, plots={len(response.plots)}, files={len(response.files)}")
- return response
-
-
-if __name__ == "__main__":
- import uvicorn
- uvicorn.run(
- app,
- host="0.0.0.0",
- port=8001,
- log_level="info"
- )
diff --git a/backend/e2b-service/requirements.txt b/backend/e2b-service/requirements.txt
deleted file mode 100644
index baf49fea..00000000
--- a/backend/e2b-service/requirements.txt
+++ /dev/null
@@ -1,11 +0,0 @@
-fastapi>=0.109.0
-uvicorn>=0.27.0
-pydantic>=2.5.0
-e2b-code-interpreter>=0.0.11
-python-multipart>=0.0.6
-requests>=2.31.0
-# For simple executor (all-in-one mode) - plot generation
-matplotlib>=3.8.0
-numpy>=1.26.0
-pandas>=2.1.0
-
diff --git a/backend/go.mod b/backend/go.mod
deleted file mode 100644
index 6d4affc0..00000000
--- a/backend/go.mod
+++ /dev/null
@@ -1,118 +0,0 @@
-module claraverse
-
-go 1.25.5
-
-require (
- github.com/ansrivas/fiberprometheus/v2 v2.14.0
- github.com/chromedp/cdproto v0.0.0-20250724212937-08a3db8b4327
- github.com/chromedp/chromedp v0.14.2
- github.com/dodopayments/dodopayments-go v1.70.0
- github.com/fsnotify/fsnotify v1.9.0
- github.com/go-co-op/gocron/v2 v2.14.0
- github.com/gofiber/contrib/websocket v1.3.4
- github.com/gofiber/fiber/v2 v2.52.9
- github.com/google/uuid v1.6.0
- github.com/invopop/jsonschema v0.13.0
- github.com/joho/godotenv v1.5.1
- github.com/ledongthuc/pdf v0.0.0-20250511090121-5959a4027728
- github.com/markusmobius/go-trafilatura v1.12.2
- github.com/patrickmn/go-cache v2.1.0+incompatible
- github.com/prometheus/client_golang v1.23.2
- github.com/redis/go-redis/v9 v9.7.0
- github.com/robfig/cron/v3 v3.0.1
- github.com/sirupsen/logrus v1.9.3
- github.com/temoto/robotstxt v1.1.2
- github.com/xuri/excelize/v2 v2.10.0
- github.com/yuin/goldmark v1.7.13
- go.mongodb.org/mongo-driver v1.17.1
- golang.org/x/crypto v0.47.0
- golang.org/x/time v0.14.0
- modernc.org/sqlite v1.40.1
-)
-
-require (
- filippo.io/edwards25519 v1.1.0 // indirect
- github.com/RadhiFadlillah/whatlanggo v0.0.0-20240916001553-aac1f0f737fc // indirect
- github.com/andybalholm/brotli v1.2.0 // indirect
- github.com/andybalholm/cascadia v1.3.3 // indirect
- github.com/araddon/dateparse v0.0.0-20210429162001-6b43995a97de // indirect
- github.com/bahlo/generic-list-go v0.2.0 // indirect
- github.com/beorn7/perks v1.0.1 // indirect
- github.com/buger/jsonparser v1.1.1 // indirect
- github.com/cespare/xxhash/v2 v2.3.0 // indirect
- github.com/chromedp/sysutil v1.1.0 // indirect
- github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
- github.com/dustin/go-humanize v1.0.1 // indirect
- github.com/elliotchance/pie/v2 v2.9.0 // indirect
- github.com/fasthttp/websocket v1.5.8 // indirect
- github.com/forPelevin/gomoji v1.2.0 // indirect
- github.com/go-json-experiment/json v0.0.0-20250725192818-e39067aee2d2 // indirect
- github.com/go-shiori/dom v0.0.0-20230515143342-73569d674e1c // indirect
- github.com/go-shiori/go-readability v0.0.0-20241012063810-92284fa8a71f // indirect
- github.com/go-sql-driver/mysql v1.9.3 // indirect
- github.com/gobwas/httphead v0.1.0 // indirect
- github.com/gobwas/pool v0.2.1 // indirect
- github.com/gobwas/ws v1.4.0 // indirect
- github.com/gogs/chardet v0.0.0-20211120154057-b7413eaefb8f // indirect
- github.com/golang-jwt/jwt/v5 v5.3.0 // indirect
- github.com/golang/snappy v0.0.4 // indirect
- github.com/hablullah/go-hijri v1.0.2 // indirect
- github.com/hablullah/go-juliandays v1.0.0 // indirect
- github.com/jalaali/go-jalaali v0.0.0-20210801064154-80525e88d958 // indirect
- github.com/jonboulle/clockwork v0.4.0 // indirect
- github.com/klauspost/compress v1.18.0 // indirect
- github.com/mailru/easyjson v0.7.7 // indirect
- github.com/markusmobius/go-dateparser v1.2.3 // indirect
- github.com/markusmobius/go-domdistiller v0.0.0-20240926050704-25b8d046ffb4 // indirect
- github.com/markusmobius/go-htmldate v1.9.1 // indirect
- github.com/mattn/go-colorable v0.1.13 // indirect
- github.com/mattn/go-isatty v0.0.20 // indirect
- github.com/mattn/go-runewidth v0.0.16 // indirect
- github.com/montanaflynn/stats v0.7.1 // indirect
- github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
- github.com/ncruces/go-strftime v1.0.0 // indirect
- github.com/philhofer/fwd v1.1.3-0.20240916144458-20a13a1f6b7c // indirect
- github.com/prometheus/client_model v0.6.2 // indirect
- github.com/prometheus/common v0.66.1 // indirect
- github.com/prometheus/procfs v0.16.1 // indirect
- github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
- github.com/richardlehane/mscfb v1.0.4 // indirect
- github.com/richardlehane/msoleps v1.0.4 // indirect
- github.com/rivo/uniseg v0.4.7 // indirect
- github.com/rs/zerolog v1.33.0 // indirect
- github.com/savsgio/gotils v0.0.0-20240303185622-093b76447511 // indirect
- github.com/sergi/go-diff v1.4.0 // indirect
- github.com/standard-webhooks/standard-webhooks/libraries v0.0.0-20251210175704-b03a68fe8b19 // indirect
- github.com/tetratelabs/wazero v1.8.1 // indirect
- github.com/tidwall/gjson v1.14.4 // indirect
- github.com/tidwall/match v1.1.1 // indirect
- github.com/tidwall/pretty v1.2.1 // indirect
- github.com/tidwall/sjson v1.2.5 // indirect
- github.com/tiendc/go-deepcopy v1.7.1 // indirect
- github.com/tinylib/msgp v1.2.5 // indirect
- github.com/valyala/bytebufferpool v1.0.0 // indirect
- github.com/valyala/fasthttp v1.65.0 // indirect
- github.com/wasilibs/go-re2 v1.7.0 // indirect
- github.com/wasilibs/wazero-helpers v0.0.0-20240620070341-3dff1577cd52 // indirect
- github.com/wk8/go-ordered-map/v2 v2.1.8 // indirect
- github.com/xdg-go/pbkdf2 v1.0.0 // indirect
- github.com/xdg-go/scram v1.1.2 // indirect
- github.com/xdg-go/stringprep v1.0.4 // indirect
- github.com/xuri/efp v0.0.1 // indirect
- github.com/xuri/nfp v0.0.2-0.20250530014748-2ddeb826f9a9 // indirect
- github.com/yosssi/gohtml v0.0.0-20201013000340-ee4748c638f4 // indirect
- github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 // indirect
- go.opentelemetry.io/otel v1.37.0 // indirect
- go.opentelemetry.io/otel/trace v1.37.0 // indirect
- go.yaml.in/yaml/v2 v2.4.2 // indirect
- golang.org/x/exp v0.0.0-20251023183803-a4bb9ffd2546 // indirect
- golang.org/x/net v0.48.0 // indirect
- golang.org/x/sync v0.19.0 // indirect
- golang.org/x/sys v0.40.0 // indirect
- golang.org/x/text v0.33.0 // indirect
- google.golang.org/protobuf v1.36.8 // indirect
- gopkg.in/yaml.v3 v3.0.1 // indirect
- modernc.org/libc v1.66.10 // indirect
- modernc.org/mathutil v1.7.1 // indirect
- modernc.org/memory v1.11.0 // indirect
-)
diff --git a/backend/go.sum b/backend/go.sum
deleted file mode 100644
index f267d970..00000000
--- a/backend/go.sum
+++ /dev/null
@@ -1,396 +0,0 @@
-filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA=
-filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4=
-github.com/RadhiFadlillah/whatlanggo v0.0.0-20240916001553-aac1f0f737fc h1:6aA31zw7fnfJ/G1ebisIesCDl44slkIVFqk3YTSadd8=
-github.com/RadhiFadlillah/whatlanggo v0.0.0-20240916001553-aac1f0f737fc/go.mod h1:PgrPWaMBxL1lyq1k5DEMqC0Y67R3pG1vEsHzxFXeDxc=
-github.com/andybalholm/brotli v1.2.0 h1:ukwgCxwYrmACq68yiUqwIWnGY0cTPox/M94sVwToPjQ=
-github.com/andybalholm/brotli v1.2.0/go.mod h1:rzTDkvFWvIrjDXZHkuS16NPggd91W3kUSvPlQ1pLaKY=
-github.com/andybalholm/cascadia v1.3.3 h1:AG2YHrzJIm4BZ19iwJ/DAua6Btl3IwJX+VI4kktS1LM=
-github.com/andybalholm/cascadia v1.3.3/go.mod h1:xNd9bqTn98Ln4DwST8/nG+H0yuB8Hmgu1YHNnWw0GeA=
-github.com/ansrivas/fiberprometheus/v2 v2.14.0 h1:4DhjAk+zA2cRA8VSlZBLjCms40AITc9Cbs8Y/ovq/SU=
-github.com/ansrivas/fiberprometheus/v2 v2.14.0/go.mod h1:sekqW4C04j0fWHXrimsTTX7ZUbPnX0d/8w+E5SxHTeg=
-github.com/araddon/dateparse v0.0.0-20210429162001-6b43995a97de h1:FxWPpzIjnTlhPwqqXc4/vE0f7GvRjuAsbW+HOIe8KnA=
-github.com/araddon/dateparse v0.0.0-20210429162001-6b43995a97de/go.mod h1:DCaWoUhZrYW9p1lxo/cm8EmUOOzAPSEZNGF2DK1dJgw=
-github.com/bahlo/generic-list-go v0.2.0 h1:5sz/EEAK+ls5wF+NeqDpk5+iNdMDXrh3z3nPnH1Wvgk=
-github.com/bahlo/generic-list-go v0.2.0/go.mod h1:2KvAjgMlE5NNynlg/5iLrrCCZ2+5xWbdbCW3pNTGyYg=
-github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
-github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
-github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs=
-github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c=
-github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA=
-github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0=
-github.com/buger/jsonparser v1.1.1 h1:2PnMjfWD7wBILjqQbt530v576A/cAbQvEW9gGIpYMUs=
-github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0=
-github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
-github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
-github.com/chromedp/cdproto v0.0.0-20250724212937-08a3db8b4327 h1:UQ4AU+BGti3Sy/aLU8KVseYKNALcX9UXY6DfpwQ6J8E=
-github.com/chromedp/cdproto v0.0.0-20250724212937-08a3db8b4327/go.mod h1:NItd7aLkcfOA/dcMXvl8p1u+lQqioRMq/SqDp71Pb/k=
-github.com/chromedp/chromedp v0.14.2 h1:r3b/WtwM50RsBZHMUm9fsNhhzRStTHrKdr2zmwbZSzM=
-github.com/chromedp/chromedp v0.14.2/go.mod h1:rHzAv60xDE7VNy/MYtTUrYreSc0ujt2O1/C3bzctYBo=
-github.com/chromedp/sysutil v1.1.0 h1:PUFNv5EcprjqXZD9nJb9b/c9ibAbxiYo4exNWZyipwM=
-github.com/chromedp/sysutil v1.1.0/go.mod h1:WiThHUdltqCNKGc4gaU50XgYjwjYIhKWoHGPTUfWTJ8=
-github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
-github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
-github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78=
-github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc=
-github.com/dodopayments/dodopayments-go v1.70.0 h1:ejsToCzKxWwPRp/uAmPIyulhoYxKjHZNB4CVudlZIno=
-github.com/dodopayments/dodopayments-go v1.70.0/go.mod h1:8ZBB5JQSIA5r3jLLVZ+rbIYDaidg/+8oSzCj6FZuVTM=
-github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
-github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
-github.com/elliotchance/pie/v2 v2.9.0 h1:BkEhh8b/avGCSpXpABSjNuytxlI/S2snkjT3vtVORjw=
-github.com/elliotchance/pie/v2 v2.9.0/go.mod h1:18t0dgGFH006g4eVdDtWfgFZPQEgl10IoEO8YWEq3Og=
-github.com/fasthttp/websocket v1.5.8 h1:k5DpirKkftIF/w1R8ZzjSgARJrs54Je9YJK37DL/Ah8=
-github.com/fasthttp/websocket v1.5.8/go.mod h1:d08g8WaT6nnyvg9uMm8K9zMYyDjfKyj3170AtPRuVU0=
-github.com/forPelevin/gomoji v1.2.0 h1:9k4WVSSkE1ARO/BWywxgEUBvR/jMnao6EZzrql5nxJ8=
-github.com/forPelevin/gomoji v1.2.0/go.mod h1:8+Z3KNGkdslmeGZBC3tCrwMrcPy5GRzAD+gL9NAwMXg=
-github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k=
-github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0=
-github.com/go-co-op/gocron/v2 v2.14.0 h1:bWPJeIdd4ioqiEpLLD1BVSTrtae7WABhX/WaVJbKVqg=
-github.com/go-co-op/gocron/v2 v2.14.0/go.mod h1:ZF70ZwEqz0OO4RBXE1sNxnANy/zvwLcattWEFsqpKig=
-github.com/go-json-experiment/json v0.0.0-20250725192818-e39067aee2d2 h1:iizUGZ9pEquQS5jTGkh4AqeeHCMbfbjeb0zMt0aEFzs=
-github.com/go-json-experiment/json v0.0.0-20250725192818-e39067aee2d2/go.mod h1:TiCD2a1pcmjd7YnhGH0f/zKNcCD06B029pHhzV23c2M=
-github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI=
-github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
-github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
-github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
-github.com/go-shiori/dom v0.0.0-20230515143342-73569d674e1c h1:wpkoddUomPfHiOziHZixGO5ZBS73cKqVzZipfrLmO1w=
-github.com/go-shiori/dom v0.0.0-20230515143342-73569d674e1c/go.mod h1:oVDCh3qjJMLVUSILBRwrm+Bc6RNXGZYtoh9xdvf1ffM=
-github.com/go-shiori/go-readability v0.0.0-20241012063810-92284fa8a71f h1:cypj7SJh+47G9J3VCPdMzT3uWcXWAWDJA54ErTfOigI=
-github.com/go-shiori/go-readability v0.0.0-20241012063810-92284fa8a71f/go.mod h1:YWa00ashoPZMAOElrSn4E1cJErhDVU6PWAll4Hxzn+w=
-github.com/go-sql-driver/mysql v1.9.3 h1:U/N249h2WzJ3Ukj8SowVFjdtZKfu9vlLZxjPXV1aweo=
-github.com/go-sql-driver/mysql v1.9.3/go.mod h1:qn46aNg1333BRMNU69Lq93t8du/dwxI64Gl8i5p1WMU=
-github.com/gobwas/httphead v0.1.0 h1:exrUm0f4YX0L7EBwZHuCF4GDp8aJfVeBrlLQrs6NqWU=
-github.com/gobwas/httphead v0.1.0/go.mod h1:O/RXo79gxV8G+RqlR/otEwx4Q36zl9rqC5u12GKvMCM=
-github.com/gobwas/pool v0.2.1 h1:xfeeEhW7pwmX8nuLVlqbzVc7udMDrwetjEv+TZIz1og=
-github.com/gobwas/pool v0.2.1/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw=
-github.com/gobwas/ws v1.4.0 h1:CTaoG1tojrh4ucGPcoJFiAQUAsEWekEWvLy7GsVNqGs=
-github.com/gobwas/ws v1.4.0/go.mod h1:G3gNqMNtPppf5XUz7O4shetPpcZ1VJ7zt18dlUeakrc=
-github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
-github.com/gofiber/contrib/websocket v1.3.4 h1:tWeBdbJ8q0WFQXariLN4dBIbGH9KBU75s0s7YXplOSg=
-github.com/gofiber/contrib/websocket v1.3.4/go.mod h1:kTFBPC6YENCnKfKx0BoOFjgXxdz7E85/STdkmZPEmPs=
-github.com/gofiber/fiber/v2 v2.52.9 h1:YjKl5DOiyP3j0mO61u3NTmK7or8GzzWzCFzkboyP5cw=
-github.com/gofiber/fiber/v2 v2.52.9/go.mod h1:YEcBbO/FB+5M1IZNBP9FO3J9281zgPAreiI1oqg8nDw=
-github.com/gogs/chardet v0.0.0-20211120154057-b7413eaefb8f h1:3BSP1Tbs2djlpprl7wCLuiqMaUh5SJkkzI2gDs+FgLs=
-github.com/gogs/chardet v0.0.0-20211120154057-b7413eaefb8f/go.mod h1:Pcatq5tYkCW2Q6yrR2VRHlbHpZ/R4/7qyL1TCF7vl14=
-github.com/golang-jwt/jwt/v5 v5.3.0 h1:pv4AsKCKKZuqlgs5sUmn4x8UlGa0kEVt/puTpKx9vvo=
-github.com/golang-jwt/jwt/v5 v5.3.0/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE=
-github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM=
-github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
-github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
-github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
-github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
-github.com/google/pprof v0.0.0-20250317173921-a4b03ec1a45e h1:ijClszYn+mADRFY17kjQEVQ1XRhq2/JR1M3sGqeJoxs=
-github.com/google/pprof v0.0.0-20250317173921-a4b03ec1a45e/go.mod h1:boTsfXsheKC2y+lKOCMpSfarhxDeIzfZG1jqGcPl3cA=
-github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
-github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
-github.com/hablullah/go-hijri v1.0.2 h1:drT/MZpSZJQXo7jftf5fthArShcaMtsal0Zf/dnmp6k=
-github.com/hablullah/go-hijri v1.0.2/go.mod h1:OS5qyYLDjORXzK4O1adFw9Q5WfhOcMdAKglDkcTxgWQ=
-github.com/hablullah/go-juliandays v1.0.0 h1:A8YM7wIj16SzlKT0SRJc9CD29iiaUzpBLzh5hr0/5p0=
-github.com/hablullah/go-juliandays v1.0.0/go.mod h1:0JOYq4oFOuDja+oospuc61YoX+uNEn7Z6uHYTbBzdGc=
-github.com/invopop/jsonschema v0.13.0 h1:KvpoAJWEjR3uD9Kbm2HWJmqsEaHt8lBUpd0qHcIi21E=
-github.com/invopop/jsonschema v0.13.0/go.mod h1:ffZ5Km5SWWRAIN6wbDXItl95euhFz2uON45H2qjYt+0=
-github.com/jalaali/go-jalaali v0.0.0-20210801064154-80525e88d958 h1:qxLoi6CAcXVzjfvu+KXIXJOAsQB62LXjsfbOaErsVzE=
-github.com/jalaali/go-jalaali v0.0.0-20210801064154-80525e88d958/go.mod h1:Wqfu7mjUHj9WDzSSPI5KfBclTTEnLveRUFr/ujWnTgE=
-github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0=
-github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4=
-github.com/jonboulle/clockwork v0.4.0 h1:p4Cf1aMWXnXAUh8lVfewRBx1zaTSYKrKMF2g3ST4RZ4=
-github.com/jonboulle/clockwork v0.4.0/go.mod h1:xgRqUGwRcjKCO1vbZUEtSLrqKoPSsUpK7fnezOII0kc=
-github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
-github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo=
-github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ=
-github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
-github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
-github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
-github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
-github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
-github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
-github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
-github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
-github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
-github.com/ledongthuc/pdf v0.0.0-20250511090121-5959a4027728 h1:QwWKgMY28TAXaDl+ExRDqGQltzXqN/xypdKP86niVn8=
-github.com/ledongthuc/pdf v0.0.0-20250511090121-5959a4027728/go.mod h1:1fEHWurg7pvf5SG6XNE5Q8UZmOwex51Mkx3SLhrW5B4=
-github.com/magefile/mage v1.15.1-0.20230912152418-9f54e0f83e2a h1:tdPcGgyiH0K+SbsJBBm2oPyEIOTAvLBwD9TuUwVtZho=
-github.com/magefile/mage v1.15.1-0.20230912152418-9f54e0f83e2a/go.mod h1:z5UZb/iS3GoOSn0JgWuiw7dxlurVYTu+/jHXqQg881A=
-github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0=
-github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
-github.com/markusmobius/go-dateparser v1.2.3 h1:TvrsIvr5uk+3v6poDjaicnAFJ5IgtFHgLiuMY2Eb7Nw=
-github.com/markusmobius/go-dateparser v1.2.3/go.mod h1:cMwQRrBUQlK1UI5TIFHEcvpsMbkWrQLXuaPNMFzuYLk=
-github.com/markusmobius/go-domdistiller v0.0.0-20240926050704-25b8d046ffb4 h1:+7kfF1+dmSXV469sqjeNC+eKJF7xDuS5mvZA3DFVLLY=
-github.com/markusmobius/go-domdistiller v0.0.0-20240926050704-25b8d046ffb4/go.mod h1:E7PoeC3nd4GqtxP1A64v7JDBxpAbpTSnhlq9/DHmQ28=
-github.com/markusmobius/go-htmldate v1.9.1 h1:0kfVz0wdxGCBaotWNzdtIZKhy7+8ClBlzvANQ67Rlt8=
-github.com/markusmobius/go-htmldate v1.9.1/go.mod h1:fLls4rjQDxYR+Pxhf0YR6Ht8dEeHd4SxK/NPaVqhMa8=
-github.com/markusmobius/go-trafilatura v1.12.2 h1:JgEto0kDjwTuyXFl6TB+psrs1QGJqTdYJEbLhDy1vrw=
-github.com/markusmobius/go-trafilatura v1.12.2/go.mod h1:2WnYLuvGBgJAarHaAQnsvofihEojt2xDDrtVJU5UXZI=
-github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
-github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
-github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
-github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
-github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
-github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
-github.com/mattn/go-runewidth v0.0.10/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk=
-github.com/mattn/go-runewidth v0.0.16 h1:E5ScNMtiwvlvB5paMFdw9p4kSQzbXFikJ5SQO6TULQc=
-github.com/mattn/go-runewidth v0.0.16/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
-github.com/montanaflynn/stats v0.7.1 h1:etflOAAHORrCC44V+aR6Ftzort912ZU+YLiSTuV8eaE=
-github.com/montanaflynn/stats v0.7.1/go.mod h1:etXPPgVO6n31NxCd9KQUMvCM+ve0ruNzt6R8Bnaayow=
-github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
-github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
-github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOFAw7w=
-github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls=
-github.com/orisano/pixelmatch v0.0.0-20220722002657-fb0b55479cde h1:x0TT0RDC7UhAVbbWWBzr41ElhJx5tXPWkIHA2HWPRuw=
-github.com/orisano/pixelmatch v0.0.0-20220722002657-fb0b55479cde/go.mod h1:nZgzbfBr3hhjoZnS66nKrHmduYNpc34ny7RK4z5/HM0=
-github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc=
-github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ=
-github.com/philhofer/fwd v1.1.3-0.20240916144458-20a13a1f6b7c h1:dAMKvw0MlJT1GshSTtih8C2gDs04w8dReiOGXrGLNoY=
-github.com/philhofer/fwd v1.1.3-0.20240916144458-20a13a1f6b7c/go.mod h1:RqIHx9QI14HlwKwm98g9Re5prTQ6LdeRQn+gXJFxsJM=
-github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
-github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
-github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
-github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h0RJWRi/o0o=
-github.com/prometheus/client_golang v1.23.2/go.mod h1:Tb1a6LWHB3/SPIzCoaDXI4I8UHKeFTEQ1YCr+0Gyqmg=
-github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk=
-github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE=
-github.com/prometheus/common v0.66.1 h1:h5E0h5/Y8niHc5DlaLlWLArTQI7tMrsfQjHV+d9ZoGs=
-github.com/prometheus/common v0.66.1/go.mod h1:gcaUsgf3KfRSwHY4dIMXLPV0K/Wg1oZ8+SbZk/HH/dA=
-github.com/prometheus/procfs v0.16.1 h1:hZ15bTNuirocR6u0JZ6BAHHmwS1p8B4P6MRqxtzMyRg=
-github.com/prometheus/procfs v0.16.1/go.mod h1:teAbpZRB1iIAJYREa1LsoWUXykVXA1KlTmWl8x/U+Is=
-github.com/redis/go-redis/v9 v9.7.0 h1:HhLSs+B6O021gwzl+locl0zEDnyNkxMtf/Z3NNBMa9E=
-github.com/redis/go-redis/v9 v9.7.0/go.mod h1:f6zhXITC7JUJIlPEiBOTXxJgPLdZcA93GewI7inzyWw=
-github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE=
-github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
-github.com/richardlehane/mscfb v1.0.4 h1:WULscsljNPConisD5hR0+OyZjwK46Pfyr6mPu5ZawpM=
-github.com/richardlehane/mscfb v1.0.4/go.mod h1:YzVpcZg9czvAuhk9T+a3avCpcFPMUWm7gK3DypaEsUk=
-github.com/richardlehane/msoleps v1.0.1/go.mod h1:BWev5JBpU9Ko2WAgmZEuiz4/u3ZYTKbjLycmwiWUfWg=
-github.com/richardlehane/msoleps v1.0.4 h1:WuESlvhX3gH2IHcd8UqyCuFY5yiq/GR/yqaSM/9/g00=
-github.com/richardlehane/msoleps v1.0.4/go.mod h1:BWev5JBpU9Ko2WAgmZEuiz4/u3ZYTKbjLycmwiWUfWg=
-github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
-github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
-github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ=
-github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
-github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs=
-github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro=
-github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ=
-github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog=
-github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=
-github.com/rs/zerolog v1.33.0 h1:1cU2KZkvPxNyfgEmhHAz/1A9Bz+llsdYzklWFzgp0r8=
-github.com/rs/zerolog v1.33.0/go.mod h1:/7mN4D5sKwJLZQ2b/znpjC3/GQWY/xaDXUM0kKWRHss=
-github.com/savsgio/gotils v0.0.0-20240303185622-093b76447511 h1:KanIMPX0QdEdB4R3CiimCAbxFrhB3j7h0/OvpYGVQa8=
-github.com/savsgio/gotils v0.0.0-20240303185622-093b76447511/go.mod h1:sM7Mt7uEoCeFSCBM+qBrqvEo+/9vdmj19wzp3yzUhmg=
-github.com/scylladb/termtables v0.0.0-20191203121021-c4c0b6d42ff4/go.mod h1:C1a7PQSMz9NShzorzCiG2fk9+xuCgLkPeCvMHYR2OWg=
-github.com/sergi/go-diff v1.4.0 h1:n/SP9D5ad1fORl+llWyN+D6qoUETXNZARKjyY2/KVCw=
-github.com/sergi/go-diff v1.4.0/go.mod h1:A0bzQcvG0E7Rwjx0REVgAGH58e96+X0MeOfepqsbeW4=
-github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ=
-github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
-github.com/standard-webhooks/standard-webhooks/libraries v0.0.0-20251210175704-b03a68fe8b19 h1:8rMUmsyom6y/10iTAgqkfv8zHVKxVQxFwlOb42V23cA=
-github.com/standard-webhooks/standard-webhooks/libraries v0.0.0-20251210175704-b03a68fe8b19/go.mod h1:L1MQhA6x4dn9r007T033lsaZMv9EmBAdXyU/+EF40fo=
-github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
-github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
-github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
-github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
-github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
-github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
-github.com/temoto/robotstxt v1.1.2 h1:W2pOjSJ6SWvldyEuiFXNxz3xZ8aiWX5LbfDiOFd7Fxg=
-github.com/temoto/robotstxt v1.1.2/go.mod h1:+1AmkuG3IYkh1kv0d2qEB9Le88ehNO0zwOr3ujewlOo=
-github.com/tetratelabs/wazero v1.8.1 h1:NrcgVbWfkWvVc4UtT4LRLDf91PsOzDzefMdwhLfA550=
-github.com/tetratelabs/wazero v1.8.1/go.mod h1:yAI0XTsMBhREkM/YDAK/zNou3GoiAce1P6+rp/wQhjs=
-github.com/tidwall/gjson v1.14.2/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
-github.com/tidwall/gjson v1.14.4 h1:uo0p8EbA09J7RQaflQ1aBRffTR7xedD2bcIVSYxLnkM=
-github.com/tidwall/gjson v1.14.4/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
-github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA=
-github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM=
-github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU=
-github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4=
-github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU=
-github.com/tidwall/sjson v1.2.5 h1:kLy8mja+1c9jlljvWTlSazM7cKDRfJuR/bOJhcY5NcY=
-github.com/tidwall/sjson v1.2.5/go.mod h1:Fvgq9kS/6ociJEDnK0Fk1cpYF4FIW6ZF7LAe+6jwd28=
-github.com/tiendc/go-deepcopy v1.7.1 h1:LnubftI6nYaaMOcaz0LphzwraqN8jiWTwm416sitff4=
-github.com/tiendc/go-deepcopy v1.7.1/go.mod h1:4bKjNC2r7boYOkD2IOuZpYjmlDdzjbpTRyCx+goBCJQ=
-github.com/tinylib/msgp v1.2.5 h1:WeQg1whrXRFiZusidTQqzETkRpGjFjcIhW6uqWH09po=
-github.com/tinylib/msgp v1.2.5/go.mod h1:ykjzy2wzgrlvpDCRc4LA8UXy6D8bzMSuAF3WD57Gok0=
-github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
-github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
-github.com/valyala/fasthttp v1.65.0 h1:j/u3uzFEGFfRxw79iYzJN+TteTJwbYkru9uDp3d0Yf8=
-github.com/valyala/fasthttp v1.65.0/go.mod h1:P/93/YkKPMsKSnATEeELUCkG8a7Y+k99uxNHVbKINr4=
-github.com/wasilibs/go-re2 v1.7.0 h1:bYhl8gn+a9h01dxwotNycxkiFPTiSgwUrIz8KZJ90Lc=
-github.com/wasilibs/go-re2 v1.7.0/go.mod h1:sUsZMLflgl+LNivDE229omtmvjICmOseT9xOy199VDU=
-github.com/wasilibs/nottinygc v0.4.0 h1:h1TJMihMC4neN6Zq+WKpLxgd9xCFMw7O9ETLwY2exJQ=
-github.com/wasilibs/nottinygc v0.4.0/go.mod h1:oDcIotskuYNMpqMF23l7Z8uzD4TC0WXHK8jetlB3HIo=
-github.com/wasilibs/wazero-helpers v0.0.0-20240620070341-3dff1577cd52 h1:OvLBa8SqJnZ6P+mjlzc2K7PM22rRUPE1x32G9DTPrC4=
-github.com/wasilibs/wazero-helpers v0.0.0-20240620070341-3dff1577cd52/go.mod h1:jMeV4Vpbi8osrE/pKUxRZkVaA0EX7NZN0A9/oRzgpgY=
-github.com/wk8/go-ordered-map/v2 v2.1.8 h1:5h/BUHu93oj4gIdvHHHGsScSTMijfx5PeYkE/fJgbpc=
-github.com/wk8/go-ordered-map/v2 v2.1.8/go.mod h1:5nJHM5DyteebpVlHnWMV0rPz6Zp7+xBAnxjb1X5vnTw=
-github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c=
-github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI=
-github.com/xdg-go/scram v1.1.2 h1:FHX5I5B4i4hKRVRBCFRxq1iQRej7WO3hhBuJf+UUySY=
-github.com/xdg-go/scram v1.1.2/go.mod h1:RT/sEzTbU5y00aCK8UOx6R7YryM0iF1N2MOmC3kKLN4=
-github.com/xdg-go/stringprep v1.0.4 h1:XLI/Ng3O1Atzq0oBs3TWm+5ZVgkq2aqdlvP9JtoZ6c8=
-github.com/xdg-go/stringprep v1.0.4/go.mod h1:mPGuuIYwz7CmR2bT9j4GbQqutWS1zV24gijq1dTyGkM=
-github.com/xuri/efp v0.0.1 h1:fws5Rv3myXyYni8uwj2qKjVaRP30PdjeYe2Y6FDsCL8=
-github.com/xuri/efp v0.0.1/go.mod h1:ybY/Jr0T0GTCnYjKqmdwxyxn2BQf2RcQIIvex5QldPI=
-github.com/xuri/excelize/v2 v2.10.0 h1:8aKsP7JD39iKLc6dH5Tw3dgV3sPRh8uRVXu/fMstfW4=
-github.com/xuri/excelize/v2 v2.10.0/go.mod h1:SC5TzhQkaOsTWpANfm+7bJCldzcnU/jrhqkTi/iBHBU=
-github.com/xuri/nfp v0.0.2-0.20250530014748-2ddeb826f9a9 h1:+C0TIdyyYmzadGaL/HBLbf3WdLgC29pgyhTjAT/0nuE=
-github.com/xuri/nfp v0.0.2-0.20250530014748-2ddeb826f9a9/go.mod h1:WwHg+CVyzlv/TX9xqBFXEZAuxOPxn2k1GNHwG41IIUQ=
-github.com/xyproto/randomstring v1.0.5 h1:YtlWPoRdgMu3NZtP45drfy1GKoojuR7hmRcnhZqKjWU=
-github.com/xyproto/randomstring v1.0.5/go.mod h1:rgmS5DeNXLivK7YprL0pY+lTuhNQW3iGxZ18UQApw/E=
-github.com/yosssi/gohtml v0.0.0-20201013000340-ee4748c638f4 h1:0sw0nJM544SpsihWx1bkXdYLQDlzRflMgFJQ4Yih9ts=
-github.com/yosssi/gohtml v0.0.0-20201013000340-ee4748c638f4/go.mod h1:+ccdNT0xMY1dtc5XBxumbYfOUhmduiGudqaDgD2rVRE=
-github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 h1:ilQV1hzziu+LLM3zUTJ0trRztfwgjqKnBWNtSRkbmwM=
-github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78/go.mod h1:aL8wCCfTfSfmXjznFBSZNN13rSJjlIOI1fUNAtF7rmI=
-github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
-github.com/yuin/goldmark v1.7.13 h1:GPddIs617DnBLFFVJFgpo1aBfe/4xcvMc3SB5t/D0pA=
-github.com/yuin/goldmark v1.7.13/go.mod h1:ip/1k0VRfGynBgxOz0yCqHrbZXhcjxyuS66Brc7iBKg=
-go.mongodb.org/mongo-driver v1.17.1 h1:Wic5cJIwJgSpBhe3lx3+/RybR5PiYRMpVFgO7cOHyIM=
-go.mongodb.org/mongo-driver v1.17.1/go.mod h1:wwWm/+BuOddhcq3n68LKRmgk2wXzmF6s0SFOa0GINL4=
-go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA=
-go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A=
-go.opentelemetry.io/otel v1.37.0 h1:9zhNfelUvx0KBfu/gb+ZgeAfAgtWrfHJZcAqFC228wQ=
-go.opentelemetry.io/otel v1.37.0/go.mod h1:ehE/umFRLnuLa/vSccNq9oS1ErUlkkK71gMcN34UG8I=
-go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.37.0 h1:SNhVp/9q4Go/XHBkQ1/d5u9P/U+L1yaGPoi0x+mStaI=
-go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.37.0/go.mod h1:tx8OOlGH6R4kLV67YaYO44GFXloEjGPZuMjEkaaqIp4=
-go.opentelemetry.io/otel/metric v1.37.0 h1:mvwbQS5m0tbmqML4NqK+e3aDiO02vsf/WgbsdpcPoZE=
-go.opentelemetry.io/otel/metric v1.37.0/go.mod h1:04wGrZurHYKOc+RKeye86GwKiTb9FKm1WHtO+4EVr2E=
-go.opentelemetry.io/otel/sdk v1.37.0 h1:ItB0QUqnjesGRvNcmAcU0LyvkVyGJ2xftD29bWdDvKI=
-go.opentelemetry.io/otel/sdk v1.37.0/go.mod h1:VredYzxUvuo2q3WRcDnKDjbdvmO0sCzOvVAiY+yUkAg=
-go.opentelemetry.io/otel/trace v1.37.0 h1:HLdcFNbRQBE2imdSEgm/kwqmQj1Or1l/7bW6mxVK7z4=
-go.opentelemetry.io/otel/trace v1.37.0/go.mod h1:TlgrlQ+PtQO5XFerSPUYG0JSgGyryXewPGyayAWSBS0=
-go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
-go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
-go.yaml.in/yaml/v2 v2.4.2 h1:DzmwEr2rDGHl7lsFgAHxmNz/1NlQ7xLIrlN2h5d1eGI=
-go.yaml.in/yaml/v2 v2.4.2/go.mod h1:081UH+NErpNdqlCXm3TtEran0rJZGxAYx9hb/ELlsPU=
-golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
-golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
-golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc=
-golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU=
-golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8=
-golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk=
-golang.org/x/crypto v0.45.0 h1:jMBrvKuj23MTlT0bQEOBcAE0mjg8mK9RXFhRH6nyF3Q=
-golang.org/x/crypto v0.45.0/go.mod h1:XTGrrkGJve7CYK7J8PEww4aY7gM3qMCElcJQ8n8JdX4=
-golang.org/x/crypto v0.47.0 h1:V6e3FRj+n4dbpw86FJ8Fv7XVOql7TEwpHapKoMJ/GO8=
-golang.org/x/crypto v0.47.0/go.mod h1:ff3Y9VzzKbwSSEzWqJsJVBnWmRwRSHt/6Op5n9bQc4A=
-golang.org/x/exp v0.0.0-20251023183803-a4bb9ffd2546 h1:mgKeJMpvi0yx/sU5GsxQ7p6s2wtOnGAHZWCHUM4KGzY=
-golang.org/x/exp v0.0.0-20251023183803-a4bb9ffd2546/go.mod h1:j/pmGrbnkbPtQfxEe5D0VQhZC6qKbfKifgD0oM7sR70=
-golang.org/x/image v0.25.0 h1:Y6uW6rH1y5y/LK1J8BPWZtr6yZ7hrsy6hFrXjgsc2fQ=
-golang.org/x/image v0.25.0/go.mod h1:tCAmOEGthTtkalusGp1g3xa2gke8J6c2N565dTyl9Rs=
-golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
-golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
-golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
-golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
-golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
-golang.org/x/mod v0.29.0 h1:HV8lRxZC4l2cr3Zq1LvtOsi/ThTgWnUk/y64QSs8GwA=
-golang.org/x/mod v0.29.0/go.mod h1:NyhrlYXJ2H4eJiRy/WDBO6HMqZQ6q9nk4JzS3NuCK+w=
-golang.org/x/mod v0.31.0 h1:HaW9xtz0+kOcWKwli0ZXy79Ix+UW/vOfmWI5QVd2tgI=
-golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
-golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
-golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
-golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
-golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk=
-golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
-golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM=
-golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4=
-golang.org/x/net v0.47.0 h1:Mx+4dIFzqraBXUugkia1OOvlD6LemFo1ALMHjrXDOhY=
-golang.org/x/net v0.47.0/go.mod h1:/jNxtkgq5yWUGYkaZGqo27cfGZ1c5Nen03aYrrKpVRU=
-golang.org/x/net v0.48.0 h1:zyQRTTrjc33Lhh0fBgT/H3oZq9WuvRR5gPC70xpDiQU=
-golang.org/x/net v0.48.0/go.mod h1:+ndRgGjkh8FGtu1w1FGbEC31if4VrNVMuKTgcAAnQRY=
-golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y=
-golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
-golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
-golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
-golang.org/x/sync v0.18.0 h1:kr88TuHDroi+UVf+0hZnirlk8o8T+4MrK6mr60WkH/I=
-golang.org/x/sync v0.18.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI=
-golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4=
-golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI=
-golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
-golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
-golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
-golang.org/x/sys v0.38.0 h1:3yZWxaJjBmCWXqhN1qh02AkOnCQ1poK6oF+a7xWL6Gc=
-golang.org/x/sys v0.38.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
-golang.org/x/sys v0.40.0 h1:DBZZqJ2Rkml6QMQsZywtnjnnGvHza6BTfYFWY9kjEWQ=
-golang.org/x/sys v0.40.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
-golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE=
-golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
-golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
-golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
-golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo=
-golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU=
-golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk=
-golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY=
-golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM=
-golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
-golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
-golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
-golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ=
-golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
-golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
-golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
-golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
-golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
-golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ=
-golang.org/x/text v0.31.0 h1:aC8ghyu4JhP8VojJ2lEHBnochRno1sgL6nEi9WGFGMM=
-golang.org/x/text v0.31.0/go.mod h1:tKRAlv61yKIjGGHX/4tP1LTbc13YSec1pxVEWXzfoeM=
-golang.org/x/text v0.33.0 h1:B3njUFyqtHDUI5jMn1YIr5B0IE2U0qck04r6d4KPAxE=
-golang.org/x/text v0.33.0/go.mod h1:LuMebE6+rBincTi9+xWTY8TztLzKHc/9C1uBCG27+q8=
-golang.org/x/time v0.14.0 h1:MRx4UaLrDotUKUdCIqzPC48t1Y9hANFKIRpNx+Te8PI=
-golang.org/x/time v0.14.0/go.mod h1:eL/Oa2bBBK0TkX57Fyni+NgnyQQN4LitPmob2Hjnqw4=
-golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
-golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
-golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
-golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58=
-golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk=
-golang.org/x/tools v0.38.0 h1:Hx2Xv8hISq8Lm16jvBZ2VQf+RLmbd7wVUsALibYI/IQ=
-golang.org/x/tools v0.38.0/go.mod h1:yEsQ/d/YK8cjh0L6rZlY8tgtlKiBNTL14pGDJPJpYQs=
-golang.org/x/tools v0.40.0 h1:yLkxfA+Qnul4cs9QA3KnlFu0lVmd8JJfoq+E41uSutA=
-golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-google.golang.org/protobuf v1.36.8 h1:xHScyCOEuuwZEc6UtSOvPbAT4zRh0xcNRYekJwfqyMc=
-google.golang.org/protobuf v1.36.8/go.mod h1:fuxRtAxBytpl4zzqUh6/eyUujkJdNiuEkXntxiD/uRU=
-gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
-gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
-gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
-gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
-gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
-gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
-modernc.org/cc/v4 v4.26.5 h1:xM3bX7Mve6G8K8b+T11ReenJOT+BmVqQj0FY5T4+5Y4=
-modernc.org/cc/v4 v4.26.5/go.mod h1:uVtb5OGqUKpoLWhqwNQo/8LwvoiEBLvZXIQ/SmO6mL0=
-modernc.org/ccgo/v4 v4.28.1 h1:wPKYn5EC/mYTqBO373jKjvX2n+3+aK7+sICCv4Fjy1A=
-modernc.org/ccgo/v4 v4.28.1/go.mod h1:uD+4RnfrVgE6ec9NGguUNdhqzNIeeomeXf6CL0GTE5Q=
-modernc.org/fileutil v1.3.40 h1:ZGMswMNc9JOCrcrakF1HrvmergNLAmxOPjizirpfqBA=
-modernc.org/fileutil v1.3.40/go.mod h1:HxmghZSZVAz/LXcMNwZPA/DRrQZEVP9VX0V4LQGQFOc=
-modernc.org/gc/v2 v2.6.5 h1:nyqdV8q46KvTpZlsw66kWqwXRHdjIlJOhG6kxiV/9xI=
-modernc.org/gc/v2 v2.6.5/go.mod h1:YgIahr1ypgfe7chRuJi2gD7DBQiKSLMPgBQe9oIiito=
-modernc.org/goabi0 v0.2.0 h1:HvEowk7LxcPd0eq6mVOAEMai46V+i7Jrj13t4AzuNks=
-modernc.org/goabi0 v0.2.0/go.mod h1:CEFRnnJhKvWT1c1JTI3Avm+tgOWbkOu5oPA8eH8LnMI=
-modernc.org/libc v1.66.10 h1:yZkb3YeLx4oynyR+iUsXsybsX4Ubx7MQlSYEw4yj59A=
-modernc.org/libc v1.66.10/go.mod h1:8vGSEwvoUoltr4dlywvHqjtAqHBaw0j1jI7iFBTAr2I=
-modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU=
-modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg=
-modernc.org/memory v1.11.0 h1:o4QC8aMQzmcwCK3t3Ux/ZHmwFPzE6hf2Y5LbkRs+hbI=
-modernc.org/memory v1.11.0/go.mod h1:/JP4VbVC+K5sU2wZi9bHoq2MAkCnrt2r98UGeSK7Mjw=
-modernc.org/opt v0.1.4 h1:2kNGMRiUjrp4LcaPuLY2PzUfqM/w9N23quVwhKt5Qm8=
-modernc.org/opt v0.1.4/go.mod h1:03fq9lsNfvkYSfxrfUhZCWPk1lm4cq4N+Bh//bEtgns=
-modernc.org/sortutil v1.2.1 h1:+xyoGf15mM3NMlPDnFqrteY07klSFxLElE2PVuWIJ7w=
-modernc.org/sortutil v1.2.1/go.mod h1:7ZI3a3REbai7gzCLcotuw9AC4VZVpYMjDzETGsSMqJE=
-modernc.org/sqlite v1.40.1 h1:VfuXcxcUWWKRBuP8+BR9L7VnmusMgBNNnBYGEe9w/iY=
-modernc.org/sqlite v1.40.1/go.mod h1:9fjQZ0mB1LLP0GYrp39oOJXx/I2sxEnZtzCmEQIKvGE=
-modernc.org/strutil v1.2.1 h1:UneZBkQA+DX2Rp35KcM69cSsNES9ly8mQWD71HKlOA0=
-modernc.org/strutil v1.2.1/go.mod h1:EHkiggD70koQxjVdSBM3JKM7k6L0FbGE5eymy9i3B9A=
-modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y=
-modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM=
diff --git a/backend/internal/audio/service.go b/backend/internal/audio/service.go
deleted file mode 100644
index 30d0b045..00000000
--- a/backend/internal/audio/service.go
+++ /dev/null
@@ -1,277 +0,0 @@
-package audio
-
-import (
- "bytes"
- "encoding/json"
- "fmt"
- "io"
- "log"
- "mime/multipart"
- "net/http"
- "os"
- "path/filepath"
- "sync"
- "time"
-)
-
-// Provider represents a minimal provider interface for audio
-type Provider struct {
- ID int
- Name string
- BaseURL string
- APIKey string
- Enabled bool
-}
-
-// ProviderGetter is a function type to get a provider
-type ProviderGetter func() (*Provider, error)
-
-// Service handles audio transcription using Whisper API (Groq or OpenAI)
-type Service struct {
- httpClient *http.Client
- groqProviderGetter ProviderGetter
- openaiProviderGetter ProviderGetter
- mu sync.RWMutex
-}
-
-var (
- instance *Service
- once sync.Once
-)
-
-// GetService returns the singleton audio service
-func GetService() *Service {
- return instance
-}
-
-// InitService initializes the audio service with dependencies
-// Priority: Groq (cheaper) -> OpenAI (fallback)
-func InitService(groqProviderGetter, openaiProviderGetter ProviderGetter) *Service {
- once.Do(func() {
- instance = &Service{
- httpClient: &http.Client{
- Timeout: 120 * time.Second, // Whisper can take a while for long audio
- },
- groqProviderGetter: groqProviderGetter,
- openaiProviderGetter: openaiProviderGetter,
- }
- })
- return instance
-}
-
-// TranscribeRequest contains parameters for audio transcription
-type TranscribeRequest struct {
- AudioPath string
- Language string // Optional language code (e.g., "en", "es", "fr")
- Prompt string // Optional prompt to guide transcription
- TranslateToEnglish bool // If true, translates non-English audio to English
-}
-
-// TranscribeResponse contains the result of transcription
-type TranscribeResponse struct {
- Text string `json:"text"`
- Language string `json:"language,omitempty"`
- Duration float64 `json:"duration,omitempty"`
- Provider string `json:"provider,omitempty"` // Which provider was used
-}
-
-// Transcribe transcribes audio to text using Whisper API
-// Tries Groq first (cheaper), falls back to OpenAI
-// If TranslateToEnglish is true, uses the translation endpoint to output English
-func (s *Service) Transcribe(req *TranscribeRequest) (*TranscribeResponse, error) {
- s.mu.RLock()
- defer s.mu.RUnlock()
-
- action := "Transcribing"
- if req.TranslateToEnglish {
- action = "Translating to English"
- }
- log.Printf("🎵 [AUDIO] %s audio: %s", action, req.AudioPath)
-
- // Try Groq first (much cheaper: $0.04/hour vs OpenAI $0.36/hour)
- // Note: Groq supports transcription but translation support may be limited
- if s.groqProviderGetter != nil && !req.TranslateToEnglish {
- provider, err := s.groqProviderGetter()
- if err == nil && provider != nil && provider.APIKey != "" {
- log.Printf("🚀 [AUDIO] Using Groq Whisper (whisper-large-v3)")
- resp, err := s.transcribeWithGroq(req, provider)
- if err == nil {
- return resp, nil
- }
- log.Printf("⚠️ [AUDIO] Groq transcription failed, trying OpenAI: %v", err)
- }
- }
-
- // Use OpenAI for translation or as fallback for transcription
- if s.openaiProviderGetter != nil {
- provider, err := s.openaiProviderGetter()
- if err == nil && provider != nil && provider.APIKey != "" {
- if req.TranslateToEnglish {
- log.Printf("🌐 [AUDIO] Using OpenAI Whisper Translation (whisper-1)")
- return s.translateWithOpenAI(req, provider)
- }
- log.Printf("🔄 [AUDIO] Using OpenAI Whisper (whisper-1)")
- return s.transcribeWithOpenAI(req, provider)
- }
- }
-
- return nil, fmt.Errorf("no audio provider configured. Please add Groq or OpenAI API key")
-}
-
-// transcribeWithGroq uses Groq's Whisper API (whisper-large-v3)
-func (s *Service) transcribeWithGroq(req *TranscribeRequest, provider *Provider) (*TranscribeResponse, error) {
- return s.transcribeWithProvider(req, provider, "https://api.groq.com/openai/v1/audio/transcriptions", "whisper-large-v3", "Groq")
-}
-
-// transcribeWithOpenAI uses OpenAI's Whisper API (whisper-1)
-func (s *Service) transcribeWithOpenAI(req *TranscribeRequest, provider *Provider) (*TranscribeResponse, error) {
- return s.transcribeWithProvider(req, provider, "https://api.openai.com/v1/audio/transcriptions", "whisper-1", "OpenAI")
-}
-
-// translateWithOpenAI uses OpenAI's Whisper Translation API to translate audio to English
-func (s *Service) translateWithOpenAI(req *TranscribeRequest, provider *Provider) (*TranscribeResponse, error) {
- return s.transcribeWithProvider(req, provider, "https://api.openai.com/v1/audio/translations", "whisper-1", "OpenAI-Translation")
-}
-
-// transcribeWithProvider is the common transcription logic for any Whisper-compatible API
-func (s *Service) transcribeWithProvider(req *TranscribeRequest, provider *Provider, apiURL, model, providerName string) (*TranscribeResponse, error) {
- // Open audio file
- audioFile, err := os.Open(req.AudioPath)
- if err != nil {
- return nil, fmt.Errorf("failed to open audio file: %w", err)
- }
- defer audioFile.Close()
-
- // Get file info
- fileInfo, err := audioFile.Stat()
- if err != nil {
- return nil, fmt.Errorf("failed to stat audio file: %w", err)
- }
-
- log.Printf("🔄 [AUDIO] Sending audio to %s Whisper API (%d bytes, model: %s)", providerName, fileInfo.Size(), model)
-
- // Create multipart form
- body := &bytes.Buffer{}
- writer := multipart.NewWriter(body)
-
- // Add file field
- filename := filepath.Base(req.AudioPath)
- part, err := writer.CreateFormFile("file", filename)
- if err != nil {
- return nil, fmt.Errorf("failed to create form file: %w", err)
- }
-
- if _, err := io.Copy(part, audioFile); err != nil {
- return nil, fmt.Errorf("failed to copy audio data: %w", err)
- }
-
- // Add model field
- if err := writer.WriteField("model", model); err != nil {
- return nil, fmt.Errorf("failed to write model field: %w", err)
- }
-
- // Add optional language
- if req.Language != "" {
- if err := writer.WriteField("language", req.Language); err != nil {
- return nil, fmt.Errorf("failed to write language field: %w", err)
- }
- }
-
- // Add optional prompt
- if req.Prompt != "" {
- if err := writer.WriteField("prompt", req.Prompt); err != nil {
- return nil, fmt.Errorf("failed to write prompt field: %w", err)
- }
- }
-
- // Add response format
- if err := writer.WriteField("response_format", "verbose_json"); err != nil {
- return nil, fmt.Errorf("failed to write response_format field: %w", err)
- }
-
- if err := writer.Close(); err != nil {
- return nil, fmt.Errorf("failed to close multipart writer: %w", err)
- }
-
- // Create request
- httpReq, err := http.NewRequest("POST", apiURL, body)
- if err != nil {
- return nil, fmt.Errorf("failed to create request: %w", err)
- }
-
- httpReq.Header.Set("Content-Type", writer.FormDataContentType())
- httpReq.Header.Set("Authorization", fmt.Sprintf("Bearer %s", provider.APIKey))
-
- // Make request
- resp, err := s.httpClient.Do(httpReq)
- if err != nil {
- return nil, fmt.Errorf("API request failed: %w", err)
- }
- defer resp.Body.Close()
-
- respBody, err := io.ReadAll(resp.Body)
- if err != nil {
- return nil, fmt.Errorf("failed to read response: %w", err)
- }
-
- if resp.StatusCode != http.StatusOK {
- log.Printf("❌ [AUDIO] %s Whisper API error: %d - %s", providerName, resp.StatusCode, string(respBody))
-
- // Try to parse error message
- var errorResp struct {
- Error struct {
- Message string `json:"message"`
- Type string `json:"type"`
- } `json:"error"`
- }
- if err := json.Unmarshal(respBody, &errorResp); err == nil && errorResp.Error.Message != "" {
- return nil, fmt.Errorf("%s Whisper API error: %s", providerName, errorResp.Error.Message)
- }
-
- return nil, fmt.Errorf("%s Whisper API error: %d", providerName, resp.StatusCode)
- }
-
- // Parse response
- var apiResp struct {
- Text string `json:"text"`
- Language string `json:"language"`
- Duration float64 `json:"duration"`
- }
-
- if err := json.Unmarshal(respBody, &apiResp); err != nil {
- return nil, fmt.Errorf("failed to parse response: %w", err)
- }
-
- log.Printf("✅ [AUDIO] %s transcription successful (%d chars, %.1fs duration)", providerName, len(apiResp.Text), apiResp.Duration)
-
- return &TranscribeResponse{
- Text: apiResp.Text,
- Language: apiResp.Language,
- Duration: apiResp.Duration,
- Provider: providerName,
- }, nil
-}
-
-// GetSupportedFormats returns the list of supported audio formats
-func GetSupportedFormats() []string {
- return []string{
- "mp3", "mp4", "mpeg", "mpga", "m4a", "wav", "webm", "ogg", "flac",
- }
-}
-
-// IsSupportedFormat checks if a MIME type is supported for transcription
-func IsSupportedFormat(mimeType string) bool {
- supportedTypes := map[string]bool{
- "audio/mpeg": true,
- "audio/mp3": true,
- "audio/mp4": true,
- "audio/x-m4a": true,
- "audio/wav": true,
- "audio/x-wav": true,
- "audio/wave": true,
- "audio/webm": true,
- "audio/ogg": true,
- "audio/flac": true,
- }
- return supportedTypes[mimeType]
-}
diff --git a/backend/internal/audio/service_test.go b/backend/internal/audio/service_test.go
deleted file mode 100644
index 7a5c6956..00000000
--- a/backend/internal/audio/service_test.go
+++ /dev/null
@@ -1,153 +0,0 @@
-package audio
-
-import (
- "testing"
-)
-
-// TestSupportedFormats verifies all expected audio formats are supported
-func TestSupportedFormats(t *testing.T) {
- supportedMimeTypes := []string{
- "audio/mpeg",
- "audio/mp3",
- "audio/wav",
- "audio/x-wav",
- "audio/wave",
- "audio/mp4",
- "audio/x-m4a",
- "audio/webm",
- "audio/ogg",
- "audio/flac",
- }
-
- for _, mimeType := range supportedMimeTypes {
- if !IsSupportedFormat(mimeType) {
- t.Errorf("MIME type %s should be supported", mimeType)
- }
- }
-}
-
-// TestUnsupportedFormats verifies unsupported formats are rejected
-func TestUnsupportedFormats(t *testing.T) {
- unsupportedMimeTypes := []string{
- "video/mp4",
- "image/jpeg",
- "application/pdf",
- "text/plain",
- "audio/midi",
- "audio/aiff",
- }
-
- for _, mimeType := range unsupportedMimeTypes {
- if IsSupportedFormat(mimeType) {
- t.Errorf("MIME type %s should NOT be supported", mimeType)
- }
- }
-}
-
-// TestGetSupportedFormats verifies the list of supported formats
-func TestGetSupportedFormats(t *testing.T) {
- formats := GetSupportedFormats()
-
- if len(formats) == 0 {
- t.Error("GetSupportedFormats should return non-empty list")
- }
-
- // Check some expected formats are in the list (file extensions, not MIME types)
- expectedFormats := map[string]bool{
- "mp3": false,
- "wav": false,
- "mp4": false,
- "ogg": false,
- "webm": false,
- "flac": false,
- }
-
- for _, format := range formats {
- if _, ok := expectedFormats[format]; ok {
- expectedFormats[format] = true
- }
- }
-
- for format, found := range expectedFormats {
- if !found {
- t.Errorf("Expected format %s in supported formats list", format)
- }
- }
-}
-
-// TestTranscribeRequestValidation tests request validation
-func TestTranscribeRequestValidation(t *testing.T) {
- // Service requires initialization, so we test the request structure
- req := &TranscribeRequest{
- AudioPath: "/path/to/audio.mp3",
- Language: "en",
- Prompt: "Test transcription",
- }
-
- if req.AudioPath == "" {
- t.Error("AudioPath should be set")
- }
- if req.Language != "en" {
- t.Errorf("Language should be 'en', got %s", req.Language)
- }
-}
-
-// TestTranscribeResponseStructure tests response structure
-func TestTranscribeResponseStructure(t *testing.T) {
- resp := &TranscribeResponse{
- Text: "Hello world",
- Language: "en",
- Duration: 5.5,
- }
-
- if resp.Text == "" {
- t.Error("Text should not be empty")
- }
- if resp.Duration <= 0 {
- t.Error("Duration should be positive")
- }
-}
-
-// TestProviderStructure tests provider structure
-func TestProviderStructure(t *testing.T) {
- provider := &Provider{
- ID: 1,
- Name: "openai",
- BaseURL: "https://api.openai.com/v1",
- APIKey: "test-key",
- Enabled: true,
- }
-
- if provider.Name != "openai" {
- t.Errorf("Expected provider name 'openai', got %s", provider.Name)
- }
- if !provider.Enabled {
- t.Error("Provider should be enabled")
- }
-}
-
-// TestGetServiceSingleton verifies singleton pattern
-func TestGetServiceSingleton(t *testing.T) {
- // GetService should return nil if not initialized
- svc := GetService()
- // Note: This may return non-nil if InitService was called elsewhere
- // The test mainly verifies no panic occurs
- _ = svc
-}
-
-// Benchmark tests
-func BenchmarkIsSupportedFormat(b *testing.B) {
- testCases := []string{
- "audio/mpeg",
- "audio/wav",
- "video/mp4",
- "image/jpeg",
- }
-
- b.ResetTimer()
- for i := 0; i < b.N; i++ {
- for _, tc := range testCases {
- IsSupportedFormat(tc)
- }
- }
-}
diff --git a/backend/internal/config/config.go b/backend/internal/config/config.go
deleted file mode 100644
index 82f2a8e9..00000000
--- a/backend/internal/config/config.go
+++ /dev/null
@@ -1,127 +0,0 @@
-package config
-
-import (
- "encoding/json"
- "fmt"
- "os"
- "strconv"
- "strings"
- "time"
-
- "claraverse/internal/models"
-)
-
-// Config holds all application configuration
-type Config struct {
- Port string
- DatabaseURL string // MySQL DSN: mysql://user:pass@host:port/dbname?parseTime=true
- SupabaseURL string
- SupabaseKey string
- SearXNGURL string
- RedisURL string
-
- // DodoPayments configuration
- DodoAPIKey string
- DodoWebhookSecret string
- DodoBusinessID string
- DodoEnvironment string // "live" or "test"
-
- // Promotional campaign configuration
- PromoEnabled bool
- PromoStartDate time.Time
- PromoEndDate time.Time
- PromoDuration int // days
-
- // Superadmin configuration
- SuperadminUserIDs []string // List of Supabase user IDs with superadmin access
-}
-
-// Load loads configuration from environment variables with defaults
-func Load() *Config {
- // Parse superadmin user IDs (comma-separated)
- superadminEnv := getEnv("SUPERADMIN_USER_IDS", "")
- var superadminUserIDs []string
- if superadminEnv != "" {
- superadminUserIDs = strings.Split(superadminEnv, ",")
- // Trim whitespace from each ID
- for i := range superadminUserIDs {
- superadminUserIDs[i] = strings.TrimSpace(superadminUserIDs[i])
- }
- }
-
- return &Config{
- Port: getEnv("PORT", "3001"),
- DatabaseURL: getEnv("DATABASE_URL", ""),
- SupabaseURL: getEnv("SUPABASE_URL", ""),
- SupabaseKey: getEnv("SUPABASE_KEY", ""),
- SearXNGURL: getEnv("SEARXNG_URL", "http://localhost:8080"),
- RedisURL: getEnv("REDIS_URL", "redis://localhost:6379"),
-
- // DodoPayments configuration
- DodoAPIKey: getEnv("DODO_API_KEY", ""),
- DodoWebhookSecret: getEnv("DODO_WEBHOOK_SECRET", ""),
- DodoBusinessID: getEnv("DODO_BUSINESS_ID", ""),
- DodoEnvironment: getEnv("DODO_ENVIRONMENT", "test"),
-
- // Promotional campaign configuration
- PromoEnabled: getBoolEnv("PROMO_ENABLED", true),
- PromoStartDate: getTimeEnv("PROMO_START_DATE", "2026-01-01T00:00:00Z"),
- PromoEndDate: getTimeEnv("PROMO_END_DATE", "2026-02-01T00:00:00Z"),
- PromoDuration: getIntEnv("PROMO_DURATION_DAYS", 30),
-
- // Superadmin configuration
- SuperadminUserIDs: superadminUserIDs,
- }
-}
-
-// LoadProviders loads providers configuration from JSON file
-func LoadProviders(filePath string) (*models.ProvidersConfig, error) {
- data, err := os.ReadFile(filePath)
- if err != nil {
- return nil, fmt.Errorf("failed to read providers file: %w", err)
- }
-
- var config models.ProvidersConfig
- if err := json.Unmarshal(data, &config); err != nil {
- return nil, fmt.Errorf("failed to parse providers JSON: %w", err)
- }
-
- return &config, nil
-}
-
-func getEnv(key, defaultValue string) string {
- if value := os.Getenv(key); value != "" {
- return value
- }
- return defaultValue
-}
-
-func getBoolEnv(key string, defaultValue bool) bool {
- if value := os.Getenv(key); value != "" {
- parsed, err := strconv.ParseBool(value)
- if err == nil {
- return parsed
- }
- }
- return defaultValue
-}
-
-func getIntEnv(key string, defaultValue int) int {
- if value := os.Getenv(key); value != "" {
- parsed, err := strconv.Atoi(value)
- if err == nil {
- return parsed
- }
- }
- return defaultValue
-}
-
-func getTimeEnv(key string, defaultValue string) time.Time {
- value := getEnv(key, defaultValue)
- parsed, err := time.Parse(time.RFC3339, value)
- if err != nil {
- // If parsing fails, return zero time
- return time.Time{}
- }
- return parsed
-}
diff --git a/backend/internal/crypto/encryption.go b/backend/internal/crypto/encryption.go
deleted file mode 100644
index e25a6f65..00000000
--- a/backend/internal/crypto/encryption.go
+++ /dev/null
@@ -1,177 +0,0 @@
-package crypto
-
-import (
- "crypto/aes"
- "crypto/cipher"
- "crypto/rand"
- "encoding/base64"
- "encoding/hex"
- "errors"
- "fmt"
- "io"
-
- "golang.org/x/crypto/hkdf"
- "crypto/sha256"
-)
-
-// EncryptionService handles encryption/decryption of user data
-type EncryptionService struct {
- masterKey []byte
-}
-
-// NewEncryptionService creates a new encryption service with the given master key
-// masterKey should be a 32-byte hex-encoded string (64 characters)
-func NewEncryptionService(masterKeyHex string) (*EncryptionService, error) {
- if masterKeyHex == "" {
- return nil, errors.New("encryption master key is required")
- }
-
- masterKey, err := hex.DecodeString(masterKeyHex)
- if err != nil {
- return nil, fmt.Errorf("invalid master key format (must be hex): %w", err)
- }
-
- if len(masterKey) != 32 {
- return nil, fmt.Errorf("master key must be 32 bytes (64 hex characters), got %d bytes", len(masterKey))
- }
-
- return &EncryptionService{
- masterKey: masterKey,
- }, nil
-}
-
-// DeriveUserKey derives a unique encryption key for a specific user
-// using HKDF (HMAC-based Key Derivation Function)
-func (e *EncryptionService) DeriveUserKey(userID string) ([]byte, error) {
- if userID == "" {
- return nil, errors.New("user ID is required for key derivation")
- }
-
- // Use HKDF to derive a user-specific key
- hkdfReader := hkdf.New(sha256.New, e.masterKey, []byte(userID), []byte("claraverse-user-encryption"))
-
- userKey := make([]byte, 32) // AES-256 requires 32-byte key
- if _, err := io.ReadFull(hkdfReader, userKey); err != nil {
- return nil, fmt.Errorf("failed to derive user key: %w", err)
- }
-
- return userKey, nil
-}
-
-// Encrypt encrypts plaintext using AES-256-GCM with a user-specific key
-// Returns base64-encoded ciphertext (nonce prepended)
-func (e *EncryptionService) Encrypt(userID string, plaintext []byte) (string, error) {
- if len(plaintext) == 0 {
- return "", nil // Return empty string for empty input
- }
-
- // Derive user-specific key
- userKey, err := e.DeriveUserKey(userID)
- if err != nil {
- return "", err
- }
-
- // Create AES cipher
- block, err := aes.NewCipher(userKey)
- if err != nil {
- return "", fmt.Errorf("failed to create cipher: %w", err)
- }
-
- // Create GCM mode
- gcm, err := cipher.NewGCM(block)
- if err != nil {
- return "", fmt.Errorf("failed to create GCM: %w", err)
- }
-
- // Generate random nonce
- nonce := make([]byte, gcm.NonceSize())
- if _, err := io.ReadFull(rand.Reader, nonce); err != nil {
- return "", fmt.Errorf("failed to generate nonce: %w", err)
- }
-
- // Encrypt and prepend nonce
- ciphertext := gcm.Seal(nonce, nonce, plaintext, nil)
-
- // Return as base64
- return base64.StdEncoding.EncodeToString(ciphertext), nil
-}
-
-// Decrypt decrypts base64-encoded ciphertext using AES-256-GCM
-func (e *EncryptionService) Decrypt(userID string, ciphertextB64 string) ([]byte, error) {
- if ciphertextB64 == "" {
- return nil, nil // Return nil for empty input
- }
-
- // Decode base64
- ciphertext, err := base64.StdEncoding.DecodeString(ciphertextB64)
- if err != nil {
- return nil, fmt.Errorf("failed to decode ciphertext: %w", err)
- }
-
- // Derive user-specific key
- userKey, err := e.DeriveUserKey(userID)
- if err != nil {
- return nil, err
- }
-
- // Create AES cipher
- block, err := aes.NewCipher(userKey)
- if err != nil {
- return nil, fmt.Errorf("failed to create cipher: %w", err)
- }
-
- // Create GCM mode
- gcm, err := cipher.NewGCM(block)
- if err != nil {
- return nil, fmt.Errorf("failed to create GCM: %w", err)
- }
-
- // Extract nonce
- nonceSize := gcm.NonceSize()
- if len(ciphertext) < nonceSize {
- return nil, errors.New("ciphertext too short")
- }
-
- nonce, ciphertext := ciphertext[:nonceSize], ciphertext[nonceSize:]
-
- // Decrypt
- plaintext, err := gcm.Open(nil, nonce, ciphertext, nil)
- if err != nil {
- return nil, fmt.Errorf("failed to decrypt: %w", err)
- }
-
- return plaintext, nil
-}
-
-// EncryptString is a convenience method for encrypting strings
-func (e *EncryptionService) EncryptString(userID string, plaintext string) (string, error) {
- return e.Encrypt(userID, []byte(plaintext))
-}
-
-// DecryptString is a convenience method for decrypting to strings
-func (e *EncryptionService) DecryptString(userID string, ciphertext string) (string, error) {
- plaintext, err := e.Decrypt(userID, ciphertext)
- if err != nil {
- return "", err
- }
- return string(plaintext), nil
-}
-
-// EncryptJSON encrypts a JSON byte slice
-func (e *EncryptionService) EncryptJSON(userID string, jsonData []byte) (string, error) {
- return e.Encrypt(userID, jsonData)
-}
-
-// DecryptJSON decrypts to a JSON byte slice
-func (e *EncryptionService) DecryptJSON(userID string, ciphertext string) ([]byte, error) {
- return e.Decrypt(userID, ciphertext)
-}
-
-// GenerateMasterKey generates a new random 32-byte master key (for setup)
-func GenerateMasterKey() (string, error) {
- key := make([]byte, 32)
- if _, err := io.ReadFull(rand.Reader, key); err != nil {
- return "", fmt.Errorf("failed to generate key: %w", err)
- }
- return hex.EncodeToString(key), nil
-}
diff --git a/backend/internal/database/database.go b/backend/internal/database/database.go
deleted file mode 100644
index 6565a1c4..00000000
--- a/backend/internal/database/database.go
+++ /dev/null
@@ -1,229 +0,0 @@
-package database
-
-import (
- "database/sql"
- "fmt"
- "log"
- "os"
- "strings"
-
- _ "github.com/go-sql-driver/mysql"
-)
-
-// DB wraps the SQL database connection
-type DB struct {
- *sql.DB
-}
-
-// New creates a new database connection
-// Supports both MySQL DSN (mysql://...) and legacy SQLite path for backwards compatibility
-func New(dsn string) (*DB, error) {
- var db *sql.DB
- var err error
-
- // Detect database type from DSN
- if strings.HasPrefix(dsn, "mysql://") {
- // MySQL DSN format: mysql://user:pass@host:port/dbname?parseTime=true
- // Convert to Go MySQL driver format: user:pass@tcp(host:port)/dbname?parseTime=true
- dsn = strings.TrimPrefix(dsn, "mysql://")
-
- // Parse the DSN to add tcp() wrapper around host:port
- // Format: user:pass@host:port/dbname -> user:pass@tcp(host:port)/dbname
- parts := strings.SplitN(dsn, "@", 2)
- if len(parts) == 2 {
- hostAndRest := parts[1]
- // Find the '/' that separates host:port from dbname
- slashIdx := strings.Index(hostAndRest, "/")
- if slashIdx > 0 {
- host := hostAndRest[:slashIdx]
- rest := hostAndRest[slashIdx:]
- dsn = parts[0] + "@tcp(" + host + ")" + rest
- }
- }
-
- db, err = sql.Open("mysql", dsn)
- } else {
- // Legacy SQLite path (for backwards compatibility during migration)
- return nil, fmt.Errorf("SQLite no longer supported - please use DATABASE_URL with MySQL DSN")
- }
-
- if err != nil {
- return nil, fmt.Errorf("failed to open database: %w", err)
- }
-
- // Configure connection pool
- db.SetMaxOpenConns(25)
- db.SetMaxIdleConns(5)
-
- if err := db.Ping(); err != nil {
- return nil, fmt.Errorf("failed to ping database: %w", err)
- }
-
- log.Println("✅ MySQL database connected")
-
- return &DB{db}, nil
-}
-
-// Initialize creates all required tables
-// NOTE: MySQL schema is created via migrations/001_initial_schema.sql on first run
-// This function only runs additional migrations for schema evolution
-func (db *DB) Initialize() error {
- log.Println("🔍 Checking database schema...")
-
- // Run migrations for existing databases
- if err := db.runMigrations(); err != nil {
- return fmt.Errorf("failed to run migrations: %w", err)
- }
-
- log.Println("✅ Database initialized successfully")
- return nil
-}
-
-// runMigrations runs database migrations for schema updates
-// Uses INFORMATION_SCHEMA to check for column existence (MySQL-compatible)
-func (db *DB) runMigrations() error {
- dbName := os.Getenv("MYSQL_DATABASE")
- if dbName == "" {
- dbName = "claraverse" // default
- }
-
- // Helper function to check if column exists
- columnExists := func(tableName, columnName string) (bool, error) {
- var count int
- query := `
- SELECT COUNT(*)
- FROM INFORMATION_SCHEMA.COLUMNS
- WHERE TABLE_SCHEMA = ? AND TABLE_NAME = ? AND COLUMN_NAME = ?
- `
- err := db.QueryRow(query, dbName, tableName, columnName).Scan(&count)
- if err != nil {
- return false, err
- }
- return count > 0, nil
- }
-
- // Helper function to check if table exists
- tableExists := func(tableName string) (bool, error) {
- var count int
- query := `
- SELECT COUNT(*)
- FROM INFORMATION_SCHEMA.TABLES
- WHERE TABLE_SCHEMA = ? AND TABLE_NAME = ?
- `
- err := db.QueryRow(query, dbName, tableName).Scan(&count)
- if err != nil {
- return false, err
- }
- return count > 0, nil
- }
-
- // Migration: Add audio_only column to providers table (if missing)
- if exists, _ := tableExists("providers"); exists {
- if colExists, _ := columnExists("providers", "audio_only"); !colExists {
- log.Println("📦 Running migration: Adding audio_only to providers table")
- if _, err := db.Exec("ALTER TABLE providers ADD COLUMN audio_only BOOLEAN DEFAULT FALSE"); err != nil {
- return fmt.Errorf("failed to add audio_only to providers: %w", err)
- }
- log.Println("✅ Migration completed: providers.audio_only added")
- }
- }
-
- // Migration: Add image_only column to providers table (if missing)
- if exists, _ := tableExists("providers"); exists {
- if colExists, _ := columnExists("providers", "image_only"); !colExists {
- log.Println("📦 Running migration: Adding image_only to providers table")
- if _, err := db.Exec("ALTER TABLE providers ADD COLUMN image_only BOOLEAN DEFAULT FALSE"); err != nil {
- return fmt.Errorf("failed to add image_only to providers: %w", err)
- }
- log.Println("✅ Migration completed: providers.image_only added")
- }
- }
-
- // Migration: Add image_edit_only column to providers table (if missing)
- if exists, _ := tableExists("providers"); exists {
- if colExists, _ := columnExists("providers", "image_edit_only"); !colExists {
- log.Println("📦 Running migration: Adding image_edit_only to providers table")
- if _, err := db.Exec("ALTER TABLE providers ADD COLUMN image_edit_only BOOLEAN DEFAULT FALSE"); err != nil {
- return fmt.Errorf("failed to add image_edit_only to providers: %w", err)
- }
- log.Println("✅ Migration completed: providers.image_edit_only added")
- }
- }
-
- // Migration: Add secure column to providers table (if missing)
- if exists, _ := tableExists("providers"); exists {
- if colExists, _ := columnExists("providers", "secure"); !colExists {
- log.Println("📦 Running migration: Adding secure to providers table")
- if _, err := db.Exec("ALTER TABLE providers ADD COLUMN secure BOOLEAN DEFAULT FALSE COMMENT 'Privacy-focused provider'"); err != nil {
- return fmt.Errorf("failed to add secure to providers: %w", err)
- }
- log.Println("✅ Migration completed: providers.secure added")
- }
- }
-
- // Migration: Add default_model column to providers table (if missing)
- if exists, _ := tableExists("providers"); exists {
- if colExists, _ := columnExists("providers", "default_model"); !colExists {
- log.Println("📦 Running migration: Adding default_model to providers table")
- if _, err := db.Exec("ALTER TABLE providers ADD COLUMN default_model VARCHAR(255)"); err != nil {
- return fmt.Errorf("failed to add default_model to providers: %w", err)
- }
- log.Println("✅ Migration completed: providers.default_model added")
- }
- }
-
- // Migration: Add smart_tool_router column to models table (if missing)
- if exists, _ := tableExists("models"); exists {
- if colExists, _ := columnExists("models", "smart_tool_router"); !colExists {
- log.Println("📦 Running migration: Adding smart_tool_router to models table")
- if _, err := db.Exec("ALTER TABLE models ADD COLUMN smart_tool_router BOOLEAN DEFAULT FALSE COMMENT 'Can predict tool usage'"); err != nil {
- return fmt.Errorf("failed to add smart_tool_router to models: %w", err)
- }
- log.Println("✅ Migration completed: models.smart_tool_router added")
- }
- }
-
- // Migration: Add agents_enabled column to models table (if missing)
- if exists, _ := tableExists("models"); exists {
- if colExists, _ := columnExists("models", "agents_enabled"); !colExists {
- log.Println("📦 Running migration: Adding agents_enabled to models table")
- if _, err := db.Exec("ALTER TABLE models ADD COLUMN agents_enabled BOOLEAN DEFAULT FALSE COMMENT 'Available in agent builder'"); err != nil {
- return fmt.Errorf("failed to add agents_enabled to models: %w", err)
- }
- log.Println("✅ Migration completed: models.agents_enabled added")
- }
- }
-
- // Migration: Add created_at and updated_at timestamps to models (if missing)
- if exists, _ := tableExists("models"); exists {
- if colExists, _ := columnExists("models", "created_at"); !colExists {
- log.Println("📦 Running migration: Adding created_at to models table")
- if _, err := db.Exec("ALTER TABLE models ADD COLUMN created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP"); err != nil {
- return fmt.Errorf("failed to add created_at to models: %w", err)
- }
- log.Println("✅ Migration completed: models.created_at added")
- }
-
- if colExists, _ := columnExists("models", "updated_at"); !colExists {
- log.Println("📦 Running migration: Adding updated_at to models table")
- if _, err := db.Exec("ALTER TABLE models ADD COLUMN updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP"); err != nil {
- return fmt.Errorf("failed to add updated_at to models: %w", err)
- }
- log.Println("✅ Migration completed: models.updated_at added")
- }
- }
-
- // Migration: Add smart_tool_router column to model_aliases table (if missing)
- if exists, _ := tableExists("model_aliases"); exists {
- if colExists, _ := columnExists("model_aliases", "smart_tool_router"); !colExists {
- log.Println("📦 Running migration: Adding smart_tool_router to model_aliases table")
- if _, err := db.Exec("ALTER TABLE model_aliases ADD COLUMN smart_tool_router BOOLEAN DEFAULT FALSE"); err != nil {
- return fmt.Errorf("failed to add smart_tool_router to model_aliases: %w", err)
- }
- log.Println("✅ Migration completed: model_aliases.smart_tool_router added")
- }
- }
-
- log.Println("✅ All migrations completed")
- return nil
-}
diff --git a/backend/internal/database/mongodb.go b/backend/internal/database/mongodb.go
deleted file mode 100644
index c53e432f..00000000
--- a/backend/internal/database/mongodb.go
+++ /dev/null
@@ -1,264 +0,0 @@
-package database
-
-import (
- "context"
- "fmt"
- "log"
- "time"
-
- "go.mongodb.org/mongo-driver/bson"
- "go.mongodb.org/mongo-driver/mongo"
- "go.mongodb.org/mongo-driver/mongo/options"
- "go.mongodb.org/mongo-driver/mongo/readpref"
-)
-
-// MongoDB wraps the MongoDB client and database
-type MongoDB struct {
- client *mongo.Client
- database *mongo.Database
- dbName string
-}
-
-// Collection names
-const (
- CollectionUsers = "users"
- CollectionAgents = "agents"
- CollectionWorkflows = "workflows"
- CollectionBuilderConversations = "builder_conversations"
- CollectionExecutions = "executions"
- CollectionProviders = "providers"
- CollectionModels = "models"
- CollectionMCPConnections = "mcp_connections"
- CollectionMCPTools = "mcp_tools"
- CollectionMCPAuditLog = "mcp_audit_log"
- CollectionCredentials = "credentials"
- CollectionChats = "chats"
-
- // Memory system collections
- CollectionMemories = "memories"
- CollectionMemoryExtractionJobs = "memory_extraction_jobs"
- CollectionConversationEngagement = "conversation_engagement"
-)
-
-// NewMongoDB creates a new MongoDB connection with connection pooling
-func NewMongoDB(uri string) (*MongoDB, error) {
- ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
- defer cancel()
-
- // Configure client options with connection pooling
- clientOptions := options.Client().
- ApplyURI(uri).
- SetMaxPoolSize(100).
- SetMinPoolSize(10).
- SetMaxConnIdleTime(30 * time.Second).
- SetServerSelectionTimeout(5 * time.Second).
- SetConnectTimeout(10 * time.Second)
-
- // Connect to MongoDB
- client, err := mongo.Connect(ctx, clientOptions)
- if err != nil {
- return nil, fmt.Errorf("failed to connect to MongoDB: %w", err)
- }
-
- // Ping to verify connection
- if err := client.Ping(ctx, readpref.Primary()); err != nil {
- return nil, fmt.Errorf("failed to ping MongoDB: %w", err)
- }
-
- // Extract database name from URI or use default
- dbName := extractDBName(uri)
- if dbName == "" {
- dbName = "claraverse"
- }
-
- db := &MongoDB{
- client: client,
- database: client.Database(dbName),
- dbName: dbName,
- }
-
- log.Printf("✅ Connected to MongoDB database: %s", dbName)
-
- return db, nil
-}
-
-// extractDBName extracts the database name from MongoDB URI
-func extractDBName(uri string) string {
- // Simple extraction - works for standard MongoDB URIs
- // mongodb://localhost:27017/claraverse -> claraverse
- // mongodb+srv://user:pass@cluster/claraverse -> claraverse
- opts := options.Client().ApplyURI(uri)
- if opts.Auth != nil && opts.Auth.AuthSource != "" {
- return opts.Auth.AuthSource
- }
- // Default fallback
- return "claraverse"
-}
-
-// Initialize creates indexes for all collections
-func (m *MongoDB) Initialize(ctx context.Context) error {
- log.Println("📦 Initializing MongoDB indexes...")
-
- // Users collection indexes
- if err := m.createIndexes(ctx, CollectionUsers, []mongo.IndexModel{
- {Keys: bson.D{{Key: "supabaseUserId", Value: 1}}, Options: options.Index().SetUnique(true)},
- {Keys: bson.D{{Key: "email", Value: 1}}},
- }); err != nil {
- return fmt.Errorf("failed to create users indexes: %w", err)
- }
-
- // Agents collection indexes
- if err := m.createIndexes(ctx, CollectionAgents, []mongo.IndexModel{
- {Keys: bson.D{{Key: "userId", Value: 1}, {Key: "updatedAt", Value: -1}}},
- {Keys: bson.D{{Key: "status", Value: 1}}},
- }); err != nil {
- return fmt.Errorf("failed to create agents indexes: %w", err)
- }
-
- // Workflows collection indexes
- if err := m.createIndexes(ctx, CollectionWorkflows, []mongo.IndexModel{
- {Keys: bson.D{{Key: "agentId", Value: 1}, {Key: "version", Value: -1}}},
- }); err != nil {
- return fmt.Errorf("failed to create workflows indexes: %w", err)
- }
-
- // Builder conversations collection indexes
- if err := m.createIndexes(ctx, CollectionBuilderConversations, []mongo.IndexModel{
- {Keys: bson.D{{Key: "agentId", Value: 1}}},
- {Keys: bson.D{{Key: "userId", Value: 1}}},
- {Keys: bson.D{{Key: "expiresAt", Value: 1}}, Options: options.Index().SetExpireAfterSeconds(0)},
- }); err != nil {
- return fmt.Errorf("failed to create builder_conversations indexes: %w", err)
- }
-
- // Executions collection indexes
- if err := m.createIndexes(ctx, CollectionExecutions, []mongo.IndexModel{
- {Keys: bson.D{{Key: "agentId", Value: 1}, {Key: "startedAt", Value: -1}}},
- {Keys: bson.D{{Key: "status", Value: 1}}},
- }); err != nil {
- return fmt.Errorf("failed to create executions indexes: %w", err)
- }
-
- // Providers collection indexes
- if err := m.createIndexes(ctx, CollectionProviders, []mongo.IndexModel{
- {Keys: bson.D{{Key: "name", Value: 1}}, Options: options.Index().SetUnique(true)},
- }); err != nil {
- return fmt.Errorf("failed to create providers indexes: %w", err)
- }
-
- // Models collection indexes
- if err := m.createIndexes(ctx, CollectionModels, []mongo.IndexModel{
- {Keys: bson.D{{Key: "providerId", Value: 1}}},
- {Keys: bson.D{{Key: "isVisible", Value: 1}}},
- }); err != nil {
- return fmt.Errorf("failed to create models indexes: %w", err)
- }
-
- // MCP connections indexes
- if err := m.createIndexes(ctx, CollectionMCPConnections, []mongo.IndexModel{
- {Keys: bson.D{{Key: "userId", Value: 1}, {Key: "isActive", Value: 1}}},
- {Keys: bson.D{{Key: "clientId", Value: 1}}, Options: options.Index().SetUnique(true)},
- }); err != nil {
- return fmt.Errorf("failed to create mcp_connections indexes: %w", err)
- }
-
- // MCP tools indexes
- if err := m.createIndexes(ctx, CollectionMCPTools, []mongo.IndexModel{
- {Keys: bson.D{{Key: "userId", Value: 1}}},
- {Keys: bson.D{{Key: "userId", Value: 1}, {Key: "toolName", Value: 1}}, Options: options.Index().SetUnique(true)},
- }); err != nil {
- return fmt.Errorf("failed to create mcp_tools indexes: %w", err)
- }
-
- // MCP audit log indexes
- if err := m.createIndexes(ctx, CollectionMCPAuditLog, []mongo.IndexModel{
- {Keys: bson.D{{Key: "userId", Value: 1}, {Key: "executedAt", Value: -1}}},
- }); err != nil {
- return fmt.Errorf("failed to create mcp_audit_log indexes: %w", err)
- }
-
- // Chats collection indexes (for cloud sync)
- if err := m.createIndexes(ctx, CollectionChats, []mongo.IndexModel{
- {Keys: bson.D{{Key: "userId", Value: 1}, {Key: "updatedAt", Value: -1}}}, // List user's chats sorted by recent
- {Keys: bson.D{{Key: "userId", Value: 1}, {Key: "chatId", Value: 1}}, Options: options.Index().SetUnique(true)}, // Unique chat per user
- {Keys: bson.D{{Key: "userId", Value: 1}, {Key: "isStarred", Value: 1}}}, // Filter starred chats
- }); err != nil {
- return fmt.Errorf("failed to create chats indexes: %w", err)
- }
-
- // Memories collection indexes
- if err := m.createIndexes(ctx, CollectionMemories, []mongo.IndexModel{
- {Keys: bson.D{{Key: "userId", Value: 1}, {Key: "isArchived", Value: 1}, {Key: "score", Value: -1}}}, // Get top active memories
- {Keys: bson.D{{Key: "userId", Value: 1}, {Key: "contentHash", Value: 1}}, Options: options.Index().SetUnique(true)}, // Deduplication
- {Keys: bson.D{{Key: "userId", Value: 1}, {Key: "category", Value: 1}}}, // Filter by category
- {Keys: bson.D{{Key: "userId", Value: 1}, {Key: "tags", Value: 1}}}, // Tag-based lookup
- {Keys: bson.D{{Key: "userId", Value: 1}, {Key: "lastAccessedAt", Value: -1}}}, // Recency tracking
- }); err != nil {
- return fmt.Errorf("failed to create memories indexes: %w", err)
- }
-
- // Memory extraction jobs collection indexes
- if err := m.createIndexes(ctx, CollectionMemoryExtractionJobs, []mongo.IndexModel{
- {Keys: bson.D{{Key: "userId", Value: 1}, {Key: "status", Value: 1}}}, // Find pending jobs
- {Keys: bson.D{{Key: "createdAt", Value: 1}}, Options: options.Index().SetExpireAfterSeconds(86400)}, // TTL: cleanup after 24h
- }); err != nil {
- return fmt.Errorf("failed to create memory_extraction_jobs indexes: %w", err)
- }
-
- // Conversation engagement collection indexes
- if err := m.createIndexes(ctx, CollectionConversationEngagement, []mongo.IndexModel{
- {Keys: bson.D{{Key: "userId", Value: 1}, {Key: "conversationId", Value: 1}}, Options: options.Index().SetUnique(true)}, // Unique per user+conversation
- }); err != nil {
- return fmt.Errorf("failed to create conversation_engagement indexes: %w", err)
- }
-
- log.Println("✅ MongoDB indexes initialized successfully")
- return nil
-}
-
-// createIndexes creates indexes for a collection
-func (m *MongoDB) createIndexes(ctx context.Context, collectionName string, indexes []mongo.IndexModel) error {
- collection := m.database.Collection(collectionName)
- _, err := collection.Indexes().CreateMany(ctx, indexes)
- return err
-}
-
-// Collection returns a collection handle
-func (m *MongoDB) Collection(name string) *mongo.Collection {
- return m.database.Collection(name)
-}
-
-// Client returns the underlying MongoDB client
-func (m *MongoDB) Client() *mongo.Client {
- return m.client
-}
-
-// Database returns the underlying MongoDB database
-func (m *MongoDB) Database() *mongo.Database {
- return m.database
-}
-
-// Close closes the MongoDB connection
-func (m *MongoDB) Close(ctx context.Context) error {
- log.Println("🔌 Closing MongoDB connection...")
- return m.client.Disconnect(ctx)
-}
-
-// Ping checks if the database connection is alive
-func (m *MongoDB) Ping(ctx context.Context) error {
- return m.client.Ping(ctx, readpref.Primary())
-}
-
-// WithTransaction executes a function within a transaction
-func (m *MongoDB) WithTransaction(ctx context.Context, fn func(sessCtx mongo.SessionContext) error) error {
- session, err := m.client.StartSession()
- if err != nil {
- return fmt.Errorf("failed to start session: %w", err)
- }
- defer session.EndSession(ctx)
-
- _, err = session.WithTransaction(ctx, func(sessCtx mongo.SessionContext) (interface{}, error) {
- return nil, fn(sessCtx)
- })
- return err
-}
diff --git a/backend/internal/document/service.go b/backend/internal/document/service.go
deleted file mode 100644
index 52ad4455..00000000
--- a/backend/internal/document/service.go
+++ /dev/null
@@ -1,475 +0,0 @@
-package document
-
-import (
- "bytes"
- "context"
- "fmt"
- "log"
- "os"
- "path/filepath"
- "sync"
- "time"
-
- "github.com/chromedp/chromedp"
- "github.com/chromedp/cdproto/page"
- "github.com/google/uuid"
- "github.com/yuin/goldmark"
- "github.com/yuin/goldmark/extension"
-)
-
-// GeneratedDocument represents a generated document
-type GeneratedDocument struct {
- DocumentID string
- UserID string
- ConversationID string
- Filename string
- FilePath string
- Size int64
- DownloadURL string
- ContentType string // MIME type for download (e.g., "application/pdf", "text/plain")
- CreatedAt time.Time
- Downloaded bool
- DownloadedAt *time.Time
-}
-
-// Service handles document generation and management
-type Service struct {
- outputDir string
- documents map[string]*GeneratedDocument
- mu sync.RWMutex
-}
-
-var (
- serviceInstance *Service
- serviceOnce sync.Once
-)
-
-// GetService returns the singleton document service
-func GetService() *Service {
- serviceOnce.Do(func() {
- outputDir := "./generated"
- if err := os.MkdirAll(outputDir, 0700); err != nil {
- log.Printf("⚠️ Warning: Could not create generated directory: %v", err)
- }
- serviceInstance = &Service{
- outputDir: outputDir,
- documents: make(map[string]*GeneratedDocument),
- }
- })
- return serviceInstance
-}
-
-// GenerateDocumentFromHTML creates a PDF document from custom HTML content
-func (s *Service) GenerateDocumentFromHTML(htmlContent, filename, title, userID, conversationID string) (*GeneratedDocument, error) {
- // Wrap HTML in complete document structure if not already present
- fullHTML := htmlContent
-
- // Check if HTML is a complete document (has )
- hasDoctype := bytes.Contains([]byte(htmlContent), []byte("
-
-
-
-
- %s
-
-
-
-%s
-
-`, title, htmlContent)
- }
-
- // Generate unique document ID and filename
- documentID := uuid.New().String()
- safeFilename := filename + ".pdf"
- filePath := filepath.Join(s.outputDir, documentID+".pdf")
-
- // Convert HTML to PDF using chromedp
- if err := s.generatePDF(fullHTML, filePath); err != nil {
- return nil, fmt.Errorf("failed to generate PDF: %w", err)
- }
-
- // Get file size
- fileInfo, err := os.Stat(filePath)
- if err != nil {
- return nil, fmt.Errorf("failed to get file info: %w", err)
- }
-
- // Create document record
- doc := &GeneratedDocument{
- DocumentID: documentID,
- UserID: userID,
- ConversationID: conversationID,
- Filename: safeFilename,
- FilePath: filePath,
- Size: fileInfo.Size(),
- DownloadURL: fmt.Sprintf("/api/download/%s", documentID),
- ContentType: "application/pdf",
- CreatedAt: time.Now(),
- Downloaded: false,
- }
-
- // Store document
- s.mu.Lock()
- s.documents[documentID] = doc
- s.mu.Unlock()
-
- log.Printf("📄 [DOCUMENT-SERVICE] Generated custom HTML PDF: %s (%d bytes)", safeFilename, fileInfo.Size())
-
- return doc, nil
-}
-
-// GenerateDocument creates a PDF document from markdown content (deprecated - use GenerateDocumentFromHTML)
-// Kept for backward compatibility with existing code
-func (s *Service) GenerateDocument(content, filename, title, userID, conversationID string) (*GeneratedDocument, error) {
- // Convert markdown to HTML with GFM extensions
- var htmlBuf bytes.Buffer
- md := goldmark.New(
- goldmark.WithExtensions(
- extension.GFM, // GitHub Flavored Markdown (includes Table, Strikethrough, Linkify, TaskList)
- ),
- )
- if err := md.Convert([]byte(content), &htmlBuf); err != nil {
- return nil, fmt.Errorf("failed to convert markdown: %w", err)
- }
-
- // Wrap in HTML template with basic styling
- fullHTML := fmt.Sprintf(`
-
-
-
- %s
-
-
-
- %s
-
-`, title, htmlBuf.String())
-
- // Generate unique document ID and filename
- documentID := uuid.New().String()
- safeFilename := filename + ".pdf"
- filePath := filepath.Join(s.outputDir, documentID+".pdf")
-
- // Convert HTML to PDF using chromedp
- if err := s.generatePDF(fullHTML, filePath); err != nil {
- return nil, fmt.Errorf("failed to generate PDF: %w", err)
- }
-
- // Get file size
- fileInfo, err := os.Stat(filePath)
- if err != nil {
- return nil, fmt.Errorf("failed to get file info: %w", err)
- }
-
- // Create document record
- doc := &GeneratedDocument{
- DocumentID: documentID,
- UserID: userID,
- ConversationID: conversationID,
- Filename: safeFilename,
- FilePath: filePath,
- Size: fileInfo.Size(),
- DownloadURL: fmt.Sprintf("/api/download/%s", documentID),
- ContentType: "application/pdf",
- CreatedAt: time.Now(),
- Downloaded: false,
- }
-
- // Store document
- s.mu.Lock()
- s.documents[documentID] = doc
- s.mu.Unlock()
-
- log.Printf("📄 [DOCUMENT-SERVICE] Generated PDF from markdown: %s (%d bytes)", safeFilename, fileInfo.Size())
-
- return doc, nil
-}
-
-// generatePDF converts HTML to PDF using chromedp
-func (s *Service) generatePDF(htmlContent, outputPath string) error {
- // Create allocator options for headless Chrome
- opts := append(chromedp.DefaultExecAllocatorOptions[:],
- chromedp.ExecPath("/usr/bin/chromium-browser"),
- chromedp.NoSandbox,
- chromedp.DisableGPU,
- chromedp.Flag("disable-dev-shm-usage", true),
- chromedp.Flag("no-first-run", true),
- chromedp.Flag("no-default-browser-check", true),
- )
-
- // Create allocator context
- allocCtx, allocCancel := chromedp.NewExecAllocator(context.Background(), opts...)
- defer allocCancel()
-
- // Create context
- ctx, cancel := chromedp.NewContext(allocCtx)
- defer cancel()
-
- // Set timeout
- ctx, cancel = context.WithTimeout(ctx, 30*time.Second)
- defer cancel()
-
- // Generate PDF
- var pdfBuffer []byte
- if err := chromedp.Run(ctx,
- chromedp.Navigate("about:blank"),
- chromedp.ActionFunc(func(ctx context.Context) error {
- frameTree, err := page.GetFrameTree().Do(ctx)
- if err != nil {
- return err
- }
- return page.SetDocumentContent(frameTree.Frame.ID, htmlContent).Do(ctx)
- }),
- chromedp.ActionFunc(func(ctx context.Context) error {
- var err error
- pdfBuffer, _, err = page.PrintToPDF().
- WithPrintBackground(true).
- WithDisplayHeaderFooter(false).
- WithMarginTop(0).
- WithMarginBottom(0).
- WithMarginLeft(0).
- WithMarginRight(0).
- WithPaperWidth(8.27). // A4 width in inches
- WithPaperHeight(11.69). // A4 height in inches
- WithScale(1.0). // 100% scale, no shrinking
- Do(ctx)
- return err
- }),
- ); err != nil {
- return err
- }
-
- // Write PDF to file with restricted permissions (owner read/write only for security)
- if err := os.WriteFile(outputPath, pdfBuffer, 0600); err != nil {
- return err
- }
-
- return nil
-}
-
-// GetDocument retrieves a document by ID
-func (s *Service) GetDocument(documentID string) (*GeneratedDocument, bool) {
- s.mu.RLock()
- defer s.mu.RUnlock()
- doc, exists := s.documents[documentID]
- return doc, exists
-}
-
-// MarkDownloaded marks a document as downloaded
-func (s *Service) MarkDownloaded(documentID string) {
- s.mu.Lock()
- defer s.mu.Unlock()
-
- if doc, exists := s.documents[documentID]; exists {
- now := time.Now()
- doc.Downloaded = true
- doc.DownloadedAt = &now
- log.Printf("✅ [DOCUMENT-SERVICE] Document downloaded: %s", doc.Filename)
- }
-}
-
-// CleanupDownloadedDocuments deletes documents that have been downloaded
-func (s *Service) CleanupDownloadedDocuments() {
- s.mu.Lock()
- defer s.mu.Unlock()
-
- now := time.Now()
- cleanedCount := 0
-
- for docID, doc := range s.documents {
- shouldDelete := false
-
- // Delete if downloaded AND 5 minutes passed (fast path)
- if doc.Downloaded && doc.DownloadedAt != nil {
- if now.Sub(*doc.DownloadedAt) > 5*time.Minute {
- shouldDelete = true
- log.Printf("🗑️ [DOCUMENT-SERVICE] Deleting downloaded document: %s (downloaded %v ago)",
- doc.Filename, now.Sub(*doc.DownloadedAt))
- }
- }
-
- // Delete if created over 10 minutes ago (main TTL - privacy-first)
- if now.Sub(doc.CreatedAt) > 10*time.Minute {
- shouldDelete = true
- log.Printf("🗑️ [DOCUMENT-SERVICE] Deleting expired document: %s (created %v ago)",
- doc.Filename, now.Sub(doc.CreatedAt))
- }
-
- if shouldDelete {
- // Delete file from disk
- if err := os.Remove(doc.FilePath); err != nil && !os.IsNotExist(err) {
- log.Printf("⚠️ Failed to delete document file %s: %v", doc.FilePath, err)
- }
-
- // Remove from map
- delete(s.documents, docID)
- cleanedCount++
- }
- }
-
- if cleanedCount > 0 {
- log.Printf("✅ [DOCUMENT-SERVICE] Cleaned up %d documents", cleanedCount)
- }
-}
-
-// GenerateTextFile creates a text-based file with the given content and extension
-func (s *Service) GenerateTextFile(content, filename, extension, userID, conversationID string) (*GeneratedDocument, error) {
- // Sanitize extension (remove leading dot if present)
- if len(extension) > 0 && extension[0] == '.' {
- extension = extension[1:]
- }
-
- // Generate unique document ID and filename
- documentID := uuid.New().String()
- safeFilename := filename + "." + extension
- filePath := filepath.Join(s.outputDir, documentID+"."+extension)
-
- // Write content to file with restricted permissions (owner read/write only for security)
- if err := os.WriteFile(filePath, []byte(content), 0600); err != nil {
- return nil, fmt.Errorf("failed to write text file: %w", err)
- }
-
- // Get file size
- fileInfo, err := os.Stat(filePath)
- if err != nil {
- return nil, fmt.Errorf("failed to get file info: %w", err)
- }
-
- // Get appropriate content type
- contentType := getContentTypeForExtension(extension)
-
- // Create document record
- doc := &GeneratedDocument{
- DocumentID: documentID,
- UserID: userID,
- ConversationID: conversationID,
- Filename: safeFilename,
- FilePath: filePath,
- Size: fileInfo.Size(),
- DownloadURL: fmt.Sprintf("/api/download/%s", documentID),
- ContentType: contentType,
- CreatedAt: time.Now(),
- Downloaded: false,
- }
-
- // Store document
- s.mu.Lock()
- s.documents[documentID] = doc
- s.mu.Unlock()
-
- log.Printf("📝 [DOCUMENT-SERVICE] Generated text file: %s (%d bytes)", safeFilename, fileInfo.Size())
-
- return doc, nil
-}
-
-// getContentTypeForExtension returns the MIME type for a given file extension
-func getContentTypeForExtension(ext string) string {
- contentTypes := map[string]string{
- // Text formats
- "txt": "text/plain",
- "text": "text/plain",
- "log": "text/plain",
-
- // Data formats
- "json": "application/json",
- "yaml": "application/x-yaml",
- "yml": "application/x-yaml",
- "xml": "application/xml",
- "csv": "text/csv",
- "tsv": "text/tab-separated-values",
-
- // Config formats
- "ini": "text/plain",
- "toml": "application/toml",
- "env": "text/plain",
- "conf": "text/plain",
- "cfg": "text/plain",
-
- // Web formats
- "html": "text/html",
- "htm": "text/html",
- "css": "text/css",
- "js": "application/javascript",
- "mjs": "application/javascript",
- "ts": "application/typescript",
- "tsx": "application/typescript",
- "jsx": "text/jsx",
-
- // Markdown
- "md": "text/markdown",
- "markdown": "text/markdown",
-
- // Programming languages
- "py": "text/x-python",
- "go": "text/x-go",
- "rs": "text/x-rust",
- "java": "text/x-java",
- "c": "text/x-c",
- "cpp": "text/x-c++",
- "h": "text/x-c",
- "hpp": "text/x-c++",
- "cs": "text/x-csharp",
- "rb": "text/x-ruby",
- "php": "text/x-php",
- "swift": "text/x-swift",
- "kt": "text/x-kotlin",
- "scala": "text/x-scala",
- "r": "text/x-r",
-
- // Shell scripts
- "sh": "application/x-sh",
- "bash": "application/x-sh",
- "zsh": "application/x-sh",
- "ps1": "application/x-powershell",
- "bat": "application/x-msdos-program",
- "cmd": "application/x-msdos-program",
-
- // Database
- "sql": "application/sql",
-
- // Other
- "diff": "text/x-diff",
- "patch": "text/x-diff",
- }
-
- if contentType, ok := contentTypes[ext]; ok {
- return contentType
- }
-
- // Default to text/plain for unknown extensions
- return "text/plain"
-}
diff --git a/backend/internal/e2b/executor.go b/backend/internal/e2b/executor.go
deleted file mode 100644
index 740fa9aa..00000000
--- a/backend/internal/e2b/executor.go
+++ /dev/null
@@ -1,334 +0,0 @@
-package e2b
-
-import (
- "bytes"
- "context"
- "encoding/json"
- "fmt"
- "io"
- "mime/multipart"
- "net/http"
- "os"
- "time"
-
- "github.com/sirupsen/logrus"
-)
-
-// E2BExecutorService handles communication with the E2B Python microservice
-type E2BExecutorService struct {
- baseURL string
- httpClient *http.Client
- logger *logrus.Logger
-}
-
-// ExecuteRequest represents a code execution request
-type ExecuteRequest struct {
- Code string `json:"code"`
- Timeout int `json:"timeout,omitempty"` // seconds
-}
-
-// PlotResult represents a generated plot
-type PlotResult struct {
- Format string `json:"format"` // "png", "svg", etc.
- Data string `json:"data"` // base64 encoded
-}
-
-// ExecuteResponse represents the response from code execution
-type ExecuteResponse struct {
- Success bool `json:"success"`
- Stdout string `json:"stdout"`
- Stderr string `json:"stderr"`
- Error *string `json:"error"`
- Plots []PlotResult `json:"plots"`
- ExecutionTime *float64 `json:"execution_time"`
-}
-
-// AdvancedExecuteRequest represents a request with dependencies and output files
-type AdvancedExecuteRequest struct {
- Code string `json:"code"`
- Timeout int `json:"timeout,omitempty"`
- Dependencies []string `json:"dependencies,omitempty"`
- OutputFiles []string `json:"output_files,omitempty"`
-}
-
-// FileResult represents a file retrieved from the sandbox
-type FileResult struct {
- Filename string `json:"filename"`
- Data string `json:"data"` // base64 encoded
- Size int `json:"size"`
-}
-
-// AdvancedExecuteResponse represents the response with files
-type AdvancedExecuteResponse struct {
- Success bool `json:"success"`
- Stdout string `json:"stdout"`
- Stderr string `json:"stderr"`
- Error *string `json:"error"`
- Plots []PlotResult `json:"plots"`
- Files []FileResult `json:"files"`
- ExecutionTime *float64 `json:"execution_time"`
- InstallOutput string `json:"install_output"`
-}
-
-var (
- e2bExecutorServiceInstance *E2BExecutorService
-)
-
-// GetE2BExecutorService returns the singleton instance of E2BExecutorService
-func GetE2BExecutorService() *E2BExecutorService {
- if e2bExecutorServiceInstance == nil {
- logger := logrus.New()
- logger.SetFormatter(&logrus.JSONFormatter{})
-
- // Get E2B service URL from environment
- baseURL := os.Getenv("E2B_SERVICE_URL")
- if baseURL == "" {
- baseURL = "http://e2b-service:8001" // Default for Docker Compose
- }
-
- e2bExecutorServiceInstance = &E2BExecutorService{
- baseURL: baseURL,
- httpClient: &http.Client{
- Timeout: 330 * time.Second, // 5.5 minutes to allow 5 min execution + overhead
- },
- logger: logger,
- }
-
- e2bExecutorServiceInstance.logger.WithField("baseURL", baseURL).Info("E2B Executor Service initialized")
- }
- return e2bExecutorServiceInstance
-}
-
-// HealthCheck checks if the E2B service is healthy
-func (s *E2BExecutorService) HealthCheck(ctx context.Context) error {
- url := fmt.Sprintf("%s/health", s.baseURL)
-
- req, err := http.NewRequestWithContext(ctx, "GET", url, nil)
- if err != nil {
- return fmt.Errorf("failed to create health check request: %w", err)
- }
-
- resp, err := s.httpClient.Do(req)
- if err != nil {
- return fmt.Errorf("health check request failed: %w", err)
- }
- defer resp.Body.Close()
-
- if resp.StatusCode != http.StatusOK {
- body, _ := io.ReadAll(resp.Body)
- return fmt.Errorf("health check failed: status=%d, body=%s", resp.StatusCode, string(body))
- }
-
- s.logger.Info("E2B service health check passed")
- return nil
-}
-
-// Execute runs Python code in an E2B sandbox
-func (s *E2BExecutorService) Execute(ctx context.Context, code string, timeout int) (*ExecuteResponse, error) {
- url := fmt.Sprintf("%s/execute", s.baseURL)
-
- // Prepare request
- reqBody := ExecuteRequest{
- Code: code,
- Timeout: timeout,
- }
-
- jsonData, err := json.Marshal(reqBody)
- if err != nil {
- return nil, fmt.Errorf("failed to marshal request: %w", err)
- }
-
- s.logger.WithFields(logrus.Fields{
- "code_length": len(code),
- "timeout": timeout,
- }).Info("Executing code in E2B sandbox")
-
- // Create HTTP request
- req, err := http.NewRequestWithContext(ctx, "POST", url, bytes.NewBuffer(jsonData))
- if err != nil {
- return nil, fmt.Errorf("failed to create request: %w", err)
- }
- req.Header.Set("Content-Type", "application/json")
-
- // Execute request
- resp, err := s.httpClient.Do(req)
- if err != nil {
- return nil, fmt.Errorf("request failed: %w", err)
- }
- defer resp.Body.Close()
-
- // Read response
- body, err := io.ReadAll(resp.Body)
- if err != nil {
- return nil, fmt.Errorf("failed to read response: %w", err)
- }
-
- // Check status code
- if resp.StatusCode != http.StatusOK {
- return nil, fmt.Errorf("execution failed: status=%d, body=%s", resp.StatusCode, string(body))
- }
-
- // Parse response
- var result ExecuteResponse
- if err := json.Unmarshal(body, &result); err != nil {
- return nil, fmt.Errorf("failed to parse response: %w", err)
- }
-
- s.logger.WithFields(logrus.Fields{
- "success": result.Success,
- "plot_count": len(result.Plots),
- "has_stdout": len(result.Stdout) > 0,
- "has_stderr": len(result.Stderr) > 0,
- }).Info("Code execution completed")
-
- return &result, nil
-}
-
-// ExecuteWithFiles runs Python code with uploaded files
-func (s *E2BExecutorService) ExecuteWithFiles(ctx context.Context, code string, files map[string][]byte, timeout int) (*ExecuteResponse, error) {
- url := fmt.Sprintf("%s/execute-with-files", s.baseURL)
-
- // Create multipart form
- body := &bytes.Buffer{}
- writer := multipart.NewWriter(body)
-
- // Add code field
- if err := writer.WriteField("code", code); err != nil {
- return nil, fmt.Errorf("failed to write code field: %w", err)
- }
-
- // Add timeout field
- if err := writer.WriteField("timeout", fmt.Sprintf("%d", timeout)); err != nil {
- return nil, fmt.Errorf("failed to write timeout field: %w", err)
- }
-
- // Add files
- for filename, content := range files {
- part, err := writer.CreateFormFile("files", filename)
- if err != nil {
- return nil, fmt.Errorf("failed to create form file %s: %w", filename, err)
- }
-
- if _, err := part.Write(content); err != nil {
- return nil, fmt.Errorf("failed to write file %s: %w", filename, err)
- }
-
- s.logger.WithFields(logrus.Fields{
- "filename": filename,
- "size": len(content),
- }).Info("Added file to request")
- }
-
- if err := writer.Close(); err != nil {
- return nil, fmt.Errorf("failed to close multipart writer: %w", err)
- }
-
- s.logger.WithFields(logrus.Fields{
- "code_length": len(code),
- "file_count": len(files),
- "timeout": timeout,
- }).Info("Executing code with files in E2B sandbox")
-
- // Create HTTP request
- req, err := http.NewRequestWithContext(ctx, "POST", url, body)
- if err != nil {
- return nil, fmt.Errorf("failed to create request: %w", err)
- }
- req.Header.Set("Content-Type", writer.FormDataContentType())
-
- // Execute request
- resp, err := s.httpClient.Do(req)
- if err != nil {
- return nil, fmt.Errorf("request failed: %w", err)
- }
- defer resp.Body.Close()
-
- // Read response
- respBody, err := io.ReadAll(resp.Body)
- if err != nil {
- return nil, fmt.Errorf("failed to read response: %w", err)
- }
-
- // Check status code
- if resp.StatusCode != http.StatusOK {
- return nil, fmt.Errorf("execution failed: status=%d, body=%s", resp.StatusCode, string(respBody))
- }
-
- // Parse response
- var result ExecuteResponse
- if err := json.Unmarshal(respBody, &result); err != nil {
- return nil, fmt.Errorf("failed to parse response: %w", err)
- }
-
- s.logger.WithFields(logrus.Fields{
- "success": result.Success,
- "plot_count": len(result.Plots),
- "has_stdout": len(result.Stdout) > 0,
- "has_stderr": len(result.Stderr) > 0,
- }).Info("Code execution with files completed")
-
- return &result, nil
-}
-
-// ExecuteAdvanced runs Python code with dependencies and retrieves output files
-func (s *E2BExecutorService) ExecuteAdvanced(ctx context.Context, req AdvancedExecuteRequest) (*AdvancedExecuteResponse, error) {
- url := fmt.Sprintf("%s/execute-advanced", s.baseURL)
-
- // Set default timeout
- if req.Timeout == 0 {
- req.Timeout = 30
- }
-
- jsonData, err := json.Marshal(req)
- if err != nil {
- return nil, fmt.Errorf("failed to marshal request: %w", err)
- }
-
- s.logger.WithFields(logrus.Fields{
- "code_length": len(req.Code),
- "timeout": req.Timeout,
- "dependencies": req.Dependencies,
- "output_files": req.OutputFiles,
- }).Info("Executing advanced code in E2B sandbox")
-
- // Create HTTP request
- httpReq, err := http.NewRequestWithContext(ctx, "POST", url, bytes.NewBuffer(jsonData))
- if err != nil {
- return nil, fmt.Errorf("failed to create request: %w", err)
- }
- httpReq.Header.Set("Content-Type", "application/json")
-
- // Execute request
- resp, err := s.httpClient.Do(httpReq)
- if err != nil {
- return nil, fmt.Errorf("request failed: %w", err)
- }
- defer resp.Body.Close()
-
- // Read response
- body, err := io.ReadAll(resp.Body)
- if err != nil {
- return nil, fmt.Errorf("failed to read response: %w", err)
- }
-
- // Check status code
- if resp.StatusCode != http.StatusOK {
- return nil, fmt.Errorf("execution failed: status=%d, body=%s", resp.StatusCode, string(body))
- }
-
- // Parse response
- var result AdvancedExecuteResponse
- if err := json.Unmarshal(body, &result); err != nil {
- return nil, fmt.Errorf("failed to parse response: %w", err)
- }
-
- s.logger.WithFields(logrus.Fields{
- "success": result.Success,
- "plot_count": len(result.Plots),
- "file_count": len(result.Files),
- "has_stdout": len(result.Stdout) > 0,
- "has_stderr": len(result.Stderr) > 0,
- }).Info("Advanced code execution completed")
-
- return &result, nil
-}
diff --git a/backend/internal/examples/interactive_prompt_example.go b/backend/internal/examples/interactive_prompt_example.go
deleted file mode 100644
index 7c6418c7..00000000
--- a/backend/internal/examples/interactive_prompt_example.go
+++ /dev/null
@@ -1,180 +0,0 @@
-package examples
-
-import (
- "github.com/google/uuid"
-
- "claraverse/internal/models"
-)
-
-// ExampleSimplePrompt shows how to create a simple interactive prompt
-// with basic question types (text, number)
-func ExampleSimplePrompt(conversationID string) models.ServerMessage {
- return models.ServerMessage{
- Type: "interactive_prompt",
- PromptID: uuid.New().String(),
- ConversationID: conversationID,
- Title: "Need More Information",
- Description: "To help you better, I need a few more details.",
- Questions: []models.InteractiveQuestion{
- {
- ID: "name",
- Type: "text",
- Label: "What's your name?",
- Placeholder: "Enter your name...",
- Required: true,
- },
- {
- ID: "age",
- Type: "number",
- Label: "How old are you?",
- Required: false,
- Validation: &models.QuestionValidation{
- Min: floatPtr(0),
- Max: floatPtr(150),
- },
- },
- },
- AllowSkip: boolPtr(true),
- }
-}
-
-// ExampleComplexPrompt shows how to create a complex prompt
-// with all question types and validation
-func ExampleComplexPrompt(conversationID string) models.ServerMessage {
- return models.ServerMessage{
- Type: "interactive_prompt",
- PromptID: uuid.New().String(),
- ConversationID: conversationID,
- Title: "Create a New Project",
- Description: "To create your project, I need some information about your requirements.",
- Questions: []models.InteractiveQuestion{
- {
- ID: "language",
- Type: "select",
- Label: "What programming language do you want to use?",
- Required: true,
- Options: []string{"Python", "JavaScript", "TypeScript", "Java", "Go"},
- AllowOther: true,
- },
- {
- ID: "features",
- Type: "multi-select",
- Label: "Which features do you need?",
- Required: true,
- Options: []string{"Authentication", "Database", "API", "Testing"},
- AllowOther: true,
- },
- {
- ID: "complexity",
- Type: "number",
- Label: "Complexity level (1-10)",
- Required: true,
- Validation: &models.QuestionValidation{
- Min: floatPtr(1),
- Max: floatPtr(10),
- },
- },
- {
- ID: "async",
- Type: "checkbox",
- Label: "Use async/await?",
- Required: false,
- },
- {
- ID: "description",
- Type: "text",
- Label: "Project description",
- Placeholder: "Describe your project...",
- Required: false,
- Validation: &models.QuestionValidation{
- MinLength: intPtr(10),
- MaxLength: intPtr(200),
- },
- },
- },
- AllowSkip: boolPtr(false), // User must answer
- }
-}
-
-// ExampleEmailValidation shows how to create a prompt with email validation
-func ExampleEmailValidation(conversationID string) models.ServerMessage {
- return models.ServerMessage{
- Type: "interactive_prompt",
- PromptID: uuid.New().String(),
- ConversationID: conversationID,
- Title: "Email Verification",
- Description: "Please verify your email address to continue.",
- Questions: []models.InteractiveQuestion{
- {
- ID: "email",
- Type: "text",
- Label: "Email address",
- Placeholder: "your@email.com",
- Required: true,
- Validation: &models.QuestionValidation{
- Pattern: `^[^\s@]+@[^\s@]+\.[^\s@]+$`, // Email regex
- },
- },
- {
- ID: "agree",
- Type: "checkbox",
- Label: "I agree to the terms and conditions",
- Required: true,
- },
- },
- AllowSkip: boolPtr(false),
- }
-}
-
-// Example of how to use SendInteractivePrompt in a tool or handler
-//
-// func (h *WebSocketHandler) SomeToolOrHandler(userConn *models.UserConnection) {
-// // Create a prompt
-// prompt := ExampleSimplePrompt(userConn.ConversationID)
-//
-// // Send it to the user
-// success := h.SendInteractivePrompt(userConn, prompt)
-// if !success {
-// log.Printf("Failed to send prompt to user")
-// return
-// }
-//
-// // The response will be received in handleInteractivePromptResponse
-// // You can store the promptID and wait for the response before continuing
-// }
-
-// Example of sending a validation error
-func ExampleValidationError(conversationID, promptID string) models.ServerMessage {
- return models.ServerMessage{
- Type: "prompt_validation_error",
- PromptID: promptID,
- ConversationID: conversationID,
- Errors: map[string]string{
- "email": "Please enter a valid email address",
- "age": "Age must be between 0 and 150",
- },
- }
-}
-
-// Example of sending a timeout message
-func ExampleTimeout(conversationID, promptID string) models.ServerMessage {
- return models.ServerMessage{
- Type: "prompt_timeout",
- PromptID: promptID,
- ConversationID: conversationID,
- ErrorMessage: "Prompt timed out. Please try again.",
- }
-}
-
-// Helper functions
-func boolPtr(b bool) *bool {
- return &b
-}
-
-func floatPtr(f float64) *float64 {
- return &f
-}
-
-func intPtr(i int) *int {
- return &i
-}
diff --git a/backend/internal/execution/agent_block_executor.go b/backend/internal/execution/agent_block_executor.go
deleted file mode 100644
index 39311ba6..00000000
--- a/backend/internal/execution/agent_block_executor.go
+++ /dev/null
@@ -1,2887 +0,0 @@
-package execution
-
-import (
- "bufio"
- "bytes"
- "claraverse/internal/filecache"
- "claraverse/internal/models"
- "claraverse/internal/services"
- "claraverse/internal/tools"
- "context"
- "encoding/base64"
- "encoding/json"
- "fmt"
- "io"
- "log"
- "net/http"
- "os"
- "path/filepath"
- "regexp"
- "strings"
- "time"
-
- "go.mongodb.org/mongo-driver/bson/primitive"
-)
-
-// ExecutionModePrefix is injected before all system prompts during workflow execution
-// This forces the LLM into "action mode" rather than "conversational mode"
-const ExecutionModePrefix = `## WORKFLOW EXECUTION MODE - MANDATORY INSTRUCTIONS
-
-You are operating in WORKFLOW EXECUTION MODE. This is NOT a conversation.
-
-CRITICAL RULES:
-1. DO NOT ask questions - all required data is provided below
-2. DO NOT explain what you're about to do - JUST DO IT immediately
-3. DO NOT hesitate or offer alternatives - execute the primary task NOW
-4. MUST use the available tools to complete your task - tool usage is MANDATORY, not optional
-5. DO NOT generate placeholder/example data - use the ACTUAL data provided in the input
-6. After completing tool calls, provide a brief confirmation and STOP
-7. DO NOT ask for webhook URLs, credentials, or configuration - these are auto-injected
-
-EXECUTION PATTERN:
-1. Read the input data provided
-2. Immediately call the required tool(s) with the actual data
-3. Return a brief confirmation of what was done
-4. STOP - do not continue iterating or ask follow-up questions
-
-`
-
-// ToolUsageError represents a validation error when required tools were not called
-type ToolUsageError struct {
- Type string `json:"type"` // "no_tool_called" or "required_tool_missing"
- Message string `json:"message"`
- EnabledTools []string `json:"enabledTools,omitempty"`
- MissingTools []string `json:"missingTools,omitempty"`
-}
-
-// ToolUsageValidator validates that required tools were called during block execution
-type ToolUsageValidator struct {
- requiredTools []string
- enabledTools []string
- requireAnyTool bool
-}
-
-// NewToolUsageValidator creates a new validator based on block configuration
-func NewToolUsageValidator(config models.AgentBlockConfig) *ToolUsageValidator {
- return &ToolUsageValidator{
- requiredTools: config.RequiredTools,
- enabledTools: config.EnabledTools,
- requireAnyTool: len(config.EnabledTools) > 0 && config.RequireToolUsage,
- }
-}
-
-// Validate checks if the required tools were called
-// Returns nil if validation passes, or a ToolUsageError if it fails
-// IMPORTANT: Tools that were called but failed (e.g., API rate limits) still count as "called"
-// because the LLM correctly attempted to use the tool - we don't want to retry in that case
-func (v *ToolUsageValidator) Validate(toolCalls []models.ToolCallRecord) *ToolUsageError {
- if !v.requireAnyTool && len(v.requiredTools) == 0 {
- return nil // No validation needed
- }
-
- // Build sets of attempted tools (all calls) and successful tools
- attemptedTools := make(map[string]bool)
- successfulTools := make(map[string]bool)
- var failedToolErrors []string
-
- for _, tc := range toolCalls {
- attemptedTools[tc.Name] = true
- if tc.Error == "" {
- successfulTools[tc.Name] = true
- } else {
- // Track tool errors for better error messages
- failedToolErrors = append(failedToolErrors, fmt.Sprintf("%s: %s", tc.Name, tc.Error))
- }
- }
-
- // Check if any tool was ATTEMPTED (when tools are enabled and required)
- // If a tool was called but failed externally (API error), we don't retry - the LLM did its job
- if v.requireAnyTool && len(attemptedTools) == 0 {
- return &ToolUsageError{
- Type: "no_tool_called",
- Message: "Block has tools enabled but none were called. The LLM responded with text only instead of using the available tools.",
- EnabledTools: v.enabledTools,
- }
- }
-
- // If tools were attempted but ALL failed, check if it's a parameter error or external failure
- // Parameter errors (wrong enum, invalid action, etc.) should trigger retry
- // External errors (API down, rate limit, auth failure) should not retry
- if len(attemptedTools) > 0 && len(successfulTools) == 0 && len(failedToolErrors) > 0 {
- // Check if errors are parameter/validation issues that the LLM can fix
- allParameterErrors := true
- for _, errMsg := range failedToolErrors {
- // Parameter errors contain hints like "Did you mean", "is not valid", "unsupported action"
- isParameterError := strings.Contains(errMsg, "Did you mean") ||
- strings.Contains(errMsg, "is not valid") ||
- strings.Contains(errMsg, "unsupported action") ||
- strings.Contains(errMsg, "is required") ||
- strings.Contains(errMsg, "invalid action")
-
- if !isParameterError {
- allParameterErrors = false
- break
- }
- }
-
- // If all errors are parameter errors, treat as validation failure (will retry)
- // If any error is external (API, network, etc.), don't retry
- if !allParameterErrors {
- log.Printf("⚠️ [TOOL-VALIDATOR] Tools were called but failed externally: %v", failedToolErrors)
- return nil // Don't retry - external failure
- }
- // If allParameterErrors is true, fall through to validation logic below
- // This will trigger a retry with the error message feedback
- }
-
- // Check required specific tools - must be ATTEMPTED (not necessarily successful)
- // If a required tool was called but failed, that's an external issue, not a retry case
- var missingTools []string
- for _, required := range v.requiredTools {
- if !attemptedTools[required] {
- missingTools = append(missingTools, required)
- }
- }
-
- if len(missingTools) > 0 {
- return &ToolUsageError{
- Type: "required_tool_missing",
- Message: fmt.Sprintf("Required tools not called: %v. These tools must be used to complete the task.", missingTools),
- MissingTools: missingTools,
- }
- }
-
- return nil
-}
-
-// AgentBlockExecutor executes LLM blocks as mini-agents with tool support
-type AgentBlockExecutor struct {
- chatService *services.ChatService
- providerService *services.ProviderService
- toolRegistry *tools.Registry
- credentialService *services.CredentialService
- httpClient *http.Client
-}
-
-// NewAgentBlockExecutor creates a new agent block executor
-func NewAgentBlockExecutor(
- chatService *services.ChatService,
- providerService *services.ProviderService,
- toolRegistry *tools.Registry,
- credentialService *services.CredentialService,
-) *AgentBlockExecutor {
- return &AgentBlockExecutor{
- chatService: chatService,
- providerService: providerService,
- toolRegistry: toolRegistry,
- credentialService: credentialService,
- httpClient: &http.Client{
- Timeout: 120 * time.Second,
- },
- }
-}
-
-// Execute runs an LLM block as a mini-agent with tool support
-// Uses two-phase approach:
-// - Phase 1: Task execution with tools (no schema concerns)
-// - Phase 2: Schema formatting (dedicated formatting step)
-// Retry logic only handles tool usage validation, not schema errors
-func (e *AgentBlockExecutor) Execute(ctx context.Context, block models.Block, inputs map[string]any) (map[string]any, error) {
- // Parse config with defaults
- config := e.parseConfig(block.Config)
-
- // Create tool usage validator
- validator := NewToolUsageValidator(config)
-
- // Retry loop for tool usage validation ONLY
- // Schema formatting is now handled as a separate phase in executeOnce
- var lastResult map[string]any
- var lastValidationError *ToolUsageError
-
- for attempt := 0; attempt <= config.MaxRetries; attempt++ {
- if attempt > 0 {
- retryReason := fmt.Sprintf("Tool validation failed: %s", lastValidationError.Message)
- log.Printf("🔄 [AGENT-BLOCK] Retry attempt %d for block '%s' - %s",
- attempt, block.Name, retryReason)
- // Inject retry context for stronger prompting
- inputs["_retryAttempt"] = attempt
- inputs["_retryReason"] = retryReason
- }
-
- // Execute the block (includes Phase 1 + Phase 2)
- result, err := e.executeOnce(ctx, block, inputs, config)
- if err != nil {
- return nil, err // Execution error, not validation error - don't retry
- }
-
- // Validate tool usage
- toolCalls, _ := result["toolCalls"].([]models.ToolCallRecord)
- validationError := validator.Validate(toolCalls)
-
- if validationError != nil {
- // Tool validation failed
- lastResult = result
- lastValidationError = validationError
- log.Printf("⚠️ [AGENT-BLOCK] Block '%s' tool validation failed (attempt %d/%d): %s",
- block.Name, attempt+1, config.MaxRetries+1, validationError.Message)
-
- // If this was the last attempt, return with warning
- if attempt == config.MaxRetries {
- log.Printf("⚠️ [AGENT-BLOCK] Block '%s' exhausted all %d retry attempts, returning with tool validation warning",
- block.Name, config.MaxRetries+1)
- result["_toolValidationWarning"] = validationError.Message
- result["_toolValidationType"] = validationError.Type
- delete(result, "_retryAttempt")
- delete(result, "_retryReason")
- return result, nil
- }
-
- // Clear retry-specific state for next attempt
- delete(inputs, "_retryAttempt")
- delete(inputs, "_retryReason")
- continue
- }
-
- // Tool validation passed - schema formatting was already handled in executeOnce Phase 2
- // Clean up and return
- delete(result, "_retryAttempt")
- delete(result, "_retryReason")
- if attempt > 0 {
- log.Printf("✅ [AGENT-BLOCK] Block '%s' succeeded on retry attempt %d", block.Name, attempt)
- }
- return result, nil
- }
-
- // Should never reach here, but return last result as fallback
- return lastResult, nil
-}
-
-// executeOnce performs a single execution attempt of the LLM block
-func (e *AgentBlockExecutor) executeOnce(ctx context.Context, block models.Block, inputs map[string]any, config models.AgentBlockConfig) (map[string]any, error) {
-
- // Check for workflow-level model override (set in Start block)
- if workflowModelID, ok := inputs["_workflowModelId"].(string); ok && workflowModelID != "" {
- log.Printf("🎯 [AGENT-BLOCK] Block '%s': Using workflow model override: %s", block.Name, workflowModelID)
- config.Model = workflowModelID
- }
-
- log.Printf("🤖 [AGENT-BLOCK] Block '%s': model=%s, enabledTools=%v, maxToolCalls=%d",
- block.Name, config.Model, config.EnabledTools, config.MaxToolCalls)
-
- // Resolve model (alias -> direct -> fallback)
- provider, modelID, err := e.resolveModel(config.Model)
- if err != nil {
- return nil, fmt.Errorf("failed to resolve model: %w", err)
- }
-
- log.Printf("✅ [AGENT-BLOCK] Resolved model '%s' -> '%s' (provider: %s)",
- config.Model, modelID, provider.Name)
-
- // Extract data files for context injection (but don't auto-enable tools)
- dataFiles := e.extractDataFileAttachments(inputs)
- if len(dataFiles) > 0 {
- log.Printf("📊 [AGENT-BLOCK] Found %d data file(s) - tools must be explicitly configured", len(dataFiles))
- }
-
- // NOTE: Auto-detection removed - blocks only use explicitly configured tools
- // Users must configure enabledTools in the block settings
-
- // Build initial messages with interpolated prompts
- messages := e.buildMessages(config, inputs)
-
- // Filter tools to only those enabled for this block
- enabledTools := e.filterTools(config.EnabledTools)
-
- log.Printf("🔧 [AGENT-BLOCK] Enabled %d tools for block '%s'", len(enabledTools), block.Name)
-
- // Track tool calls and token usage
- var allToolCalls []models.ToolCallRecord
- var totalTokens models.TokenUsage
- iterations := 0
- var lastContent string // Track last content for timeout handling
-
- // Track generated chart images for auto-injection into Discord/Slack messages
- // The LLM sees sanitized placeholders like [CHART_IMAGE_SAVED] but we need the real base64
- var generatedCharts []string
-
- // PRE-POPULATE generatedCharts with artifacts from previous blocks
- // This allows Discord/Slack blocks to access charts generated by upstream blocks
- generatedCharts = e.extractChartsFromInputs(inputs)
- if len(generatedCharts) > 0 {
- log.Printf("🖼️ [AGENT-BLOCK] Pre-loaded %d chart(s) from previous block artifacts for auto-injection", len(generatedCharts))
- }
-
- // Run agent loop - continues until LLM stops calling tools or timeout
- for {
- iterations++
- log.Printf("🔄 [AGENT-BLOCK] Iteration %d for block '%s'", iterations, block.Name)
-
- // Enforce iteration limit to prevent infinite loops
- maxIterations := 10 // Default safety limit
- if config.MaxToolCalls > 0 {
- maxIterations = config.MaxToolCalls
- }
- if iterations > maxIterations {
- log.Printf("🛑 [AGENT-BLOCK] Block '%s' reached max iterations (%d)", block.Name, maxIterations)
- return e.buildTimeoutResult(inputs, modelID, totalTokens, allToolCalls, lastContent, iterations)
- }
-
- // Track tool calls to detect repetition within this iteration only
- // Reset per iteration to allow same tool across different iterations (legitimate refinement)
- executedToolCalls := make(map[string]bool)
-
- // Check if context is cancelled (timeout)
- select {
- case <-ctx.Done():
- log.Printf("⏱️ [AGENT-BLOCK] Block '%s' timed out after %d iterations", block.Name, iterations)
- return e.buildTimeoutResult(inputs, modelID, totalTokens, allToolCalls, lastContent, iterations)
- default:
- // Continue execution
- }
-
- // Call LLM with retry for transient errors (timeout, rate limit, server errors)
- response, retryAttempts, err := e.callLLMWithRetry(ctx, provider, modelID, messages, enabledTools, config.Temperature, config.RetryPolicy)
- if err != nil {
- // Include retry info in error for debugging
- if len(retryAttempts) > 0 {
- return nil, fmt.Errorf("LLM call failed in iteration %d after %d attempt(s): %w", iterations, len(retryAttempts)+1, err)
- }
- return nil, fmt.Errorf("LLM call failed in iteration %d: %w", iterations, err)
- }
-
- // Track retry attempts for surfacing in output (first iteration only to avoid duplicates)
- if iterations == 1 && len(retryAttempts) > 0 {
- inputs["_llmRetryAttempts"] = retryAttempts
- }
-
- // Accumulate tokens
- totalTokens.Input += response.InputTokens
- totalTokens.Output += response.OutputTokens
-
- // Check finish_reason for explicit stop signal from LLM
- // This is how chat mode knows to stop - agent mode should do the same
- if response.FinishReason == "stop" || response.FinishReason == "end_turn" {
- log.Printf("✅ [AGENT-BLOCK] LLM signaled stop (finish_reason=%s), completing block '%s'",
- response.FinishReason, block.Name)
- // Continue to completion logic below (same as no tool calls)
- }
-
- // Check if there are tool calls
- if len(response.ToolCalls) == 0 || response.FinishReason == "stop" || response.FinishReason == "end_turn" {
- // No more tools - agent is done (Phase 1 complete)
- log.Printf("✅ [AGENT-BLOCK] Block '%s' Phase 1 (task execution) completed after %d iteration(s)", block.Name, iterations)
-
- // Build result starting with all inputs (pass through workflow variables)
- result := make(map[string]any)
- for k, v := range inputs {
- result[k] = v
- }
-
- // Store raw LLM response
- result["rawResponse"] = response.Content
- result["response"] = response.Content // Default, may be overwritten by schema formatting
-
- // CRITICAL: Extract and surface tool results for downstream blocks
- // This solves the data passing problem where tool output was buried in toolCalls
- toolResults := e.extractToolResultsForDownstream(allToolCalls)
- toolResultsMaps := make([]map[string]any, 0)
- if len(toolResults) > 0 {
- // Store parsed tool results for easy access
- result["toolResults"] = toolResults
-
- // Surface key data fields at top level for template access
- // e.g., {{block-name.text}} instead of {{block-name.toolResults.transcribe_audio.text}}
- for _, tr := range toolResults {
- if trMap, ok := tr.(map[string]any); ok {
- toolResultsMaps = append(toolResultsMaps, trMap)
- // Surface common data fields
- for _, key := range []string{"text", "data", "content", "result", "output", "transcription"} {
- if val, exists := trMap[key]; exists && val != nil {
- // Only surface if not already set by LLM response
- if _, alreadySet := result[key]; !alreadySet {
- result[key] = val
- log.Printf("📤 [AGENT-BLOCK] Surfaced tool result field '%s' to top level", key)
- }
- }
- }
- }
- }
- }
-
- // Phase 2: Schema formatting (if schema is defined)
- // This is the key change - we use a dedicated formatting step for structured output
- if config.OutputSchema != nil {
- log.Printf("📐 [AGENT-BLOCK] Block '%s' starting Phase 2 (schema formatting)", block.Name)
-
- // Prepare input for schema formatter
- formatInput := FormatInput{
- RawData: response.Content,
- ToolResults: toolResultsMaps,
- LLMResponse: response.Content,
- Context: config.SystemPrompt,
- }
-
- // Call the dedicated schema formatter
- formatOutput, err := e.FormatToSchema(ctx, formatInput, config.OutputSchema, modelID)
- if err != nil {
- log.Printf("⚠️ [AGENT-BLOCK] Phase 2 schema formatting error: %v", err)
- // Continue with raw output if formatting fails
- result["_formatError"] = err.Error()
- } else if formatOutput != nil {
- if formatOutput.Success {
- log.Printf("✅ [AGENT-BLOCK] Phase 2 schema formatting succeeded")
- // Use the formatted data as the response
- result["response"] = formatOutput.Data
- result["data"] = formatOutput.Data
- result["output"] = formatOutput.Data
- // Also spread the fields at top level for easy access
- for k, v := range formatOutput.Data {
- result[k] = v
- }
- // Track the formatting tokens
- totalTokens.Input += formatOutput.Tokens.Input
- totalTokens.Output += formatOutput.Tokens.Output
- } else {
- log.Printf("⚠️ [AGENT-BLOCK] Phase 2 schema formatting failed: %s", formatOutput.Error)
- result["_formatError"] = formatOutput.Error
- // Fall back to basic parsing without validation
- if parsedOutput, err := e.parseAndValidateOutput(response.Content, nil, false); err == nil {
- for k, v := range parsedOutput {
- result[k] = v
- }
- }
- }
- }
- } else {
- // No schema defined - just parse the response as-is
- output, err := e.parseAndValidateOutput(response.Content, nil, false)
- if err != nil {
- log.Printf("⚠️ [AGENT-BLOCK] Output parsing error (no schema): %v", err)
- } else {
- for k, v := range output {
- result[k] = v
- }
- result["output"] = output
- }
- }
-
- // If LLM response is just a summary but we have tool data, use tool data as response
- if len(allToolCalls) > 0 {
- llmResponse, _ := result["response"].(string)
- // Check if LLM response is a short summary (likely not the actual data)
- if len(llmResponse) < 500 && len(toolResults) > 0 {
- // Check if we have a text field from tools that's more substantial
- if textResult, ok := result["text"].(string); ok && len(textResult) > len(llmResponse) {
- // Keep LLM response as summary, but ensure text is available
- result["summary"] = llmResponse
- log.Printf("📤 [AGENT-BLOCK] Tool 'text' field (%d chars) surfaced; LLM response (%d chars) kept as 'summary'",
- len(textResult), len(llmResponse))
- }
- }
- }
-
- result["model"] = modelID
- result["tokens"] = map[string]int{
- "input": totalTokens.Input,
- "output": totalTokens.Output,
- "total": totalTokens.Input + totalTokens.Output,
- }
- result["toolCalls"] = allToolCalls
- result["iterations"] = iterations
-
- // Check for tool errors and surface them for block checker
- // This helps distinguish between "LLM didn't use tools" vs "LLM used tools but they failed externally"
- var toolErrors []string
- for _, tc := range allToolCalls {
- if tc.Error != "" {
- toolErrors = append(toolErrors, fmt.Sprintf("%s: %s", tc.Name, tc.Error))
- }
- }
- if len(toolErrors) > 0 {
- result["_toolError"] = strings.Join(toolErrors, "; ")
- log.Printf("⚠️ [AGENT-BLOCK] Block has %d tool error(s): %v", len(toolErrors), toolErrors)
- }
-
- // Extract artifacts (charts, images) from tool calls for consistent API access
- artifacts := e.extractArtifactsFromToolCalls(allToolCalls)
- result["artifacts"] = artifacts
-
- // Extract generated files (PDFs, documents) from tool calls
- generatedFiles := e.extractGeneratedFilesFromToolCalls(allToolCalls)
- result["generatedFiles"] = generatedFiles
- // Also expose the first file's download URL directly for easy access
- if len(generatedFiles) > 0 {
- result["file_url"] = generatedFiles[0].DownloadURL
- result["file_name"] = generatedFiles[0].Filename
- }
-
- // Surface retry information for debugging/monitoring
- if retryAttempts, ok := inputs["_llmRetryAttempts"].([]models.RetryAttempt); ok && len(retryAttempts) > 0 {
- result["_retryInfo"] = map[string]any{
- "totalAttempts": len(retryAttempts) + 1,
- "retriedCount": len(retryAttempts),
- "history": retryAttempts,
- }
- log.Printf("📊 [AGENT-BLOCK] Block completed with %d retry attempt(s)", len(retryAttempts))
- }
-
- log.Printf("🔍 [AGENT-BLOCK] Output keys: %v, artifacts: %d, files: %d, toolResults: %d",
- getMapKeys(result), len(artifacts), len(generatedFiles), len(toolResults))
- return result, nil
- }
-
- // Execute tools and add results to messages
- log.Printf("🔧 [AGENT-BLOCK] Executing %d tool call(s) in iteration %d", len(response.ToolCalls), iterations)
-
- // Add assistant message with tool calls
- assistantMsg := map[string]any{
- "role": "assistant",
- "tool_calls": response.ToolCalls,
- }
- if response.Content != "" {
- assistantMsg["content"] = response.Content
- }
- messages = append(messages, assistantMsg)
-
- // Execute each tool call
- repeatDetected := false
- for _, toolCall := range response.ToolCalls {
- toolName := e.getToolName(toolCall)
-
- // Check for repetition - if same tool called twice, it's likely looping
- if executedToolCalls[toolName] {
- log.Printf("⚠️ [AGENT-BLOCK] Detected repeated call to '%s', stopping to prevent loop", toolName)
- repeatDetected = true
- break
- }
- executedToolCalls[toolName] = true
-
- // Extract userID from inputs for credential resolution (uses __user_id__ convention)
- userID, _ := inputs["__user_id__"].(string)
- toolRecord := e.executeToolCall(toolCall, inputs, dataFiles, generatedCharts, userID, config.Credentials)
- allToolCalls = append(allToolCalls, toolRecord)
-
- // Extract any chart images from successful tool results for later injection
- if toolRecord.Error == "" && toolRecord.Result != "" {
- charts := e.extractChartsFromResult(toolRecord.Result)
- if len(charts) > 0 {
- generatedCharts = append(generatedCharts, charts...)
- log.Printf("📊 [AGENT-BLOCK] Extracted %d chart(s) from tool '%s' (total: %d)",
- len(charts), toolName, len(generatedCharts))
- }
- }
-
- // Sanitize tool result for LLM - remove base64 images which are useless as text
- sanitizedResult := e.sanitizeToolResultForLLM(toolRecord.Result)
-
- // Add tool result to messages
- toolResultMsg := map[string]any{
- "role": "tool",
- "tool_call_id": toolCall["id"],
- "name": toolName,
- "content": sanitizedResult,
- }
- if toolRecord.Error != "" {
- toolResultMsg["content"] = fmt.Sprintf("Error: %s", toolRecord.Error)
- }
- messages = append(messages, toolResultMsg)
- }
-
- // If repetition detected, exit loop and return current results
- if repeatDetected {
- log.Printf("🛑 [AGENT-BLOCK] Exiting loop due to repeated tool call")
- return e.buildTimeoutResult(inputs, modelID, totalTokens, allToolCalls, lastContent, iterations)
- }
-
- // Track last content for timeout fallback
- if response.Content != "" {
- lastContent = response.Content
- }
- }
- // Note: Loop only exits via return statements (success or timeout)
-}
-
-// buildTimeoutResult creates a result when the block times out
-// Instead of returning an error, it returns the collected tool call data
-// so downstream blocks can still use the information gathered
-func (e *AgentBlockExecutor) buildTimeoutResult(
- inputs map[string]any,
- modelID string,
- totalTokens models.TokenUsage,
- allToolCalls []models.ToolCallRecord,
- lastContent string,
- iterations int,
-) (map[string]any, error) {
- // Build result starting with all inputs (pass through workflow variables)
- result := make(map[string]any)
- for k, v := range inputs {
- result[k] = v
- }
-
- // Build a summary from tool call results if no meaningful content was generated
- var outputContent string
- trimmedContent := strings.TrimSpace(lastContent)
- if trimmedContent != "" {
- outputContent = lastContent
- } else if len(allToolCalls) > 0 {
- // Compile tool results as the output
- var summaryParts []string
- for _, tc := range allToolCalls {
- if tc.Result != "" && tc.Error == "" {
- summaryParts = append(summaryParts, tc.Result)
- }
- }
- if len(summaryParts) > 0 {
- outputContent = strings.Join(summaryParts, "\n\n")
- }
- }
-
- // Flatten output fields directly into result for consistent access
- // This makes {{block-name.response}} work the same as simple LLM executor
- result["response"] = outputContent
- result["timedOut"] = true
-
- // CRITICAL: Extract and surface tool results for downstream blocks (same as normal completion)
- toolResults := e.extractToolResultsForDownstream(allToolCalls)
- if len(toolResults) > 0 {
- result["toolResults"] = toolResults
-
- // Surface key data fields at top level for template access
- for _, tr := range toolResults {
- if trMap, ok := tr.(map[string]any); ok {
- for _, key := range []string{"text", "data", "content", "result", "output", "transcription"} {
- if val, exists := trMap[key]; exists && val != nil {
- if _, alreadySet := result[key]; !alreadySet {
- result[key] = val
- log.Printf("📤 [AGENT-BLOCK] Surfaced tool result field '%s' to top level (timeout)", key)
- }
- }
- }
- }
- }
- }
-
- // Also keep "output" for backward compatibility
- result["output"] = map[string]any{
- "response": outputContent,
- "timedOut": true,
- "iterations": iterations,
- "toolResults": len(allToolCalls),
- }
- result["rawResponse"] = outputContent
- result["model"] = modelID
- result["tokens"] = map[string]int{
- "input": totalTokens.Input,
- "output": totalTokens.Output,
- "total": totalTokens.Input + totalTokens.Output,
- }
- result["toolCalls"] = allToolCalls
- result["iterations"] = iterations
-
- // Extract artifacts (charts, images) from tool calls for consistent API access
- artifacts := e.extractArtifactsFromToolCalls(allToolCalls)
- result["artifacts"] = artifacts
-
- // Extract generated files (PDFs, documents) from tool calls
- generatedFiles := e.extractGeneratedFilesFromToolCalls(allToolCalls)
- result["generatedFiles"] = generatedFiles
- // Also expose the first file's download URL directly for easy access
- if len(generatedFiles) > 0 {
- result["file_url"] = generatedFiles[0].DownloadURL
- result["file_name"] = generatedFiles[0].Filename
- }
-
- log.Printf("⏱️ [AGENT-BLOCK] Timeout result built with %d tool calls, content length: %d, artifacts: %d, files: %d, toolResults: %d",
- len(allToolCalls), len(outputContent), len(artifacts), len(generatedFiles), len(toolResults))
- log.Printf("🔍 [AGENT-BLOCK] Output keys: %v", getMapKeys(result))
-
- return result, nil
-}
-
-// parseToolsList converts various array types to []string for tool names
-func parseToolsList(raw interface{}) []string {
- var tools []string
-
- switch v := raw.(type) {
- case []interface{}:
- for _, t := range v {
- if toolName, ok := t.(string); ok {
- tools = append(tools, toolName)
- }
- }
- case []string:
- tools = v
- case primitive.A: // BSON array type
- for _, t := range v {
- if toolName, ok := t.(string); ok {
- tools = append(tools, toolName)
- }
- }
- default:
- log.Printf("⚠️ [CONFIG] Unknown enabledTools type: %T", raw)
- }
-
- return tools
-}
-
-// getDefaultModel returns the first available model from database
-func (e *AgentBlockExecutor) getDefaultModel() string {
- // Use chatService to get optimal text model
- provider, modelID, err := e.chatService.GetTextProviderWithModel()
- if err == nil && modelID != "" {
- log.Printf("🎯 [AGENT-EXEC] Using dynamic default model: %s (provider: %s)", modelID, provider.Name)
- return modelID
- }
-
- // If that fails, try to get default provider with model
- provider, modelID, err = e.chatService.GetDefaultProviderWithModel()
- if err == nil && modelID != "" {
- log.Printf("🎯 [AGENT-EXEC] Using fallback default model: %s", modelID)
- return modelID
- }
-
- // Last resort: return empty string (will cause error later if no model specified)
- log.Printf("⚠️ [AGENT-EXEC] No default model available - agent execution will require explicit model")
- return ""
-}
-
-// parseConfig parses block config into AgentBlockConfig with defaults
-func (e *AgentBlockExecutor) parseConfig(config map[string]any) models.AgentBlockConfig {
- // Get dynamic default model from available models
- defaultModel := e.getDefaultModel()
-
- result := models.AgentBlockConfig{
- Model: defaultModel,
- Temperature: 0.7,
- MaxToolCalls: 15, // Increased to allow agents with multiple search iterations
- }
-
- // Model
- if v, ok := config["model"].(string); ok && v != "" {
- result.Model = v
- }
- if v, ok := config["modelId"].(string); ok && v != "" {
- result.Model = v
- }
-
- // Temperature
- if v, ok := config["temperature"].(float64); ok {
- result.Temperature = v
- }
-
- // System prompt
- if v, ok := config["systemPrompt"].(string); ok {
- result.SystemPrompt = v
- }
- if v, ok := config["system_prompt"].(string); ok && result.SystemPrompt == "" {
- result.SystemPrompt = v
- }
-
- // User prompt
- if v, ok := config["userPrompt"].(string); ok {
- result.UserPrompt = v
- }
- if v, ok := config["userPromptTemplate"].(string); ok && result.UserPrompt == "" {
- result.UserPrompt = v
- }
- if v, ok := config["user_prompt"].(string); ok && result.UserPrompt == "" {
- result.UserPrompt = v
- }
-
- // Enabled tools - handle multiple possible types from BSON/JSON
- if enabledToolsRaw, exists := config["enabledTools"]; exists && enabledToolsRaw != nil {
- result.EnabledTools = parseToolsList(enabledToolsRaw)
- log.Printf("🔧 [CONFIG] Parsed enabledTools from config: %v (type was %T)", result.EnabledTools, enabledToolsRaw)
- }
- if len(result.EnabledTools) == 0 {
- if enabledToolsRaw, exists := config["enabled_tools"]; exists && enabledToolsRaw != nil {
- result.EnabledTools = parseToolsList(enabledToolsRaw)
- log.Printf("🔧 [CONFIG] Parsed enabled_tools from config: %v (type was %T)", result.EnabledTools, enabledToolsRaw)
- }
- }
-
- // Max tool calls
- if v, ok := config["maxToolCalls"].(float64); ok {
- result.MaxToolCalls = int(v)
- }
- if v, ok := config["max_tool_calls"].(float64); ok && result.MaxToolCalls == 15 {
- result.MaxToolCalls = int(v)
- }
-
- // Credentials - array of credential IDs configured by user for tool authentication
- if credentialsRaw, exists := config["credentials"]; exists && credentialsRaw != nil {
- result.Credentials = parseToolsList(credentialsRaw) // Reuse the same parser ([]string)
- log.Printf("🔐 [CONFIG] Parsed credentials from config: %v", result.Credentials)
- }
-
- // Output schema
- if v, ok := config["outputSchema"].(map[string]any); ok {
- result.OutputSchema = e.parseJSONSchema(v)
- log.Printf("📋 [CONFIG] Parsed outputSchema with %d required fields: %v", len(result.OutputSchema.Required), result.OutputSchema.Required)
- } else {
- log.Printf("📋 [CONFIG] No outputSchema found in config (outputSchema key: %v)", config["outputSchema"] != nil)
- }
-
- // Strict output
- if v, ok := config["strictOutput"].(bool); ok {
- result.StrictOutput = v
- }
-
- // Execution Mode Configuration (NEW)
- // Parse requireToolUsage - if explicitly set, use that value
- if v, ok := config["requireToolUsage"].(bool); ok {
- result.RequireToolUsage = v
- } else {
- // Default: Auto-enable when tools are present for deterministic execution
- result.RequireToolUsage = len(result.EnabledTools) > 0
- }
-
- // Parse maxRetries - default to 2 for resilience
- result.MaxRetries = 2 // Default
- if v, ok := config["maxRetries"].(float64); ok {
- result.MaxRetries = int(v)
- }
- if v, ok := config["max_retries"].(float64); ok {
- result.MaxRetries = int(v)
- }
-
- // Parse requiredTools - specific tools that MUST be called
- if requiredToolsRaw, exists := config["requiredTools"]; exists && requiredToolsRaw != nil {
- result.RequiredTools = parseToolsList(requiredToolsRaw)
- log.Printf("🔧 [CONFIG] Parsed requiredTools from config: %v", result.RequiredTools)
- }
- if len(result.RequiredTools) == 0 {
- if requiredToolsRaw, exists := config["required_tools"]; exists && requiredToolsRaw != nil {
- result.RequiredTools = parseToolsList(requiredToolsRaw)
- }
- }
-
- // Auto-lower temperature for execution mode when tools are enabled
- // Lower temp = more deterministic tool calling
- if len(result.EnabledTools) > 0 {
- if _, explicitTemp := config["temperature"]; !explicitTemp {
- result.Temperature = 0.3
- log.Printf("🔧 [CONFIG] Auto-lowered temperature to 0.3 for execution mode")
- }
- }
-
- // Parse RetryPolicy for LLM API call retries (transient error handling)
- if retryPolicyRaw, exists := config["retryPolicy"]; exists && retryPolicyRaw != nil {
- if retryMap, ok := retryPolicyRaw.(map[string]any); ok {
- result.RetryPolicy = &models.RetryPolicy{}
-
- if v, ok := retryMap["maxRetries"].(float64); ok {
- result.RetryPolicy.MaxRetries = int(v)
- }
- if v, ok := retryMap["initialDelay"].(float64); ok {
- result.RetryPolicy.InitialDelay = int(v)
- }
- if v, ok := retryMap["maxDelay"].(float64); ok {
- result.RetryPolicy.MaxDelay = int(v)
- }
- if v, ok := retryMap["backoffMultiplier"].(float64); ok {
- result.RetryPolicy.BackoffMultiplier = v
- }
- if v, ok := retryMap["jitterPercent"].(float64); ok {
- result.RetryPolicy.JitterPercent = int(v)
- }
- if retryOn, ok := retryMap["retryOn"].([]interface{}); ok {
- for _, r := range retryOn {
- if s, ok := r.(string); ok {
- result.RetryPolicy.RetryOn = append(result.RetryPolicy.RetryOn, s)
- }
- }
- }
- log.Printf("🔧 [CONFIG] Parsed retryPolicy: maxRetries=%d, initialDelay=%dms",
- result.RetryPolicy.MaxRetries, result.RetryPolicy.InitialDelay)
- }
- }
-
- // Apply default retry policy if not specified (for production resilience)
- if result.RetryPolicy == nil {
- result.RetryPolicy = models.DefaultRetryPolicy()
- }
-
- log.Printf("🔧 [CONFIG] Execution mode: requireToolUsage=%v, maxRetries=%d, requiredTools=%v",
- result.RequireToolUsage, result.MaxRetries, result.RequiredTools)
-
- return result
-}
-
-// parseJSONSchema converts a map to JSONSchema
-func (e *AgentBlockExecutor) parseJSONSchema(schema map[string]any) *models.JSONSchema {
- result := &models.JSONSchema{}
-
- if v, ok := schema["type"].(string); ok {
- result.Type = v
- }
-
- if v, ok := schema["properties"].(map[string]any); ok {
- result.Properties = make(map[string]*models.JSONSchema)
- for key, prop := range v {
- if propMap, ok := prop.(map[string]any); ok {
- result.Properties[key] = e.parseJSONSchema(propMap)
- }
- }
- }
-
- if v, ok := schema["items"].(map[string]any); ok {
- result.Items = e.parseJSONSchema(v)
- }
-
- // Handle required field - support multiple Go types including MongoDB's primitive.A
- // Note: []interface{} and []any are the same type in Go, so only use one
- switch v := schema["required"].(type) {
- case []interface{}:
- log.Printf("📋 [SCHEMA] Found required field ([]interface{}): %v", v)
- for _, r := range v {
- if req, ok := r.(string); ok {
- result.Required = append(result.Required, req)
- }
- }
- case []string:
- log.Printf("📋 [SCHEMA] Found required field ([]string): %v", v)
- result.Required = v
- case primitive.A: // MongoDB BSON array type
- log.Printf("📋 [SCHEMA] Found required field (primitive.A): %v", v)
- for _, r := range v {
- if req, ok := r.(string); ok {
- result.Required = append(result.Required, req)
- }
- }
- default:
- if schema["required"] != nil {
- log.Printf("📋 [SCHEMA] Unhandled required field type: %T", schema["required"])
- }
- }
-
- if v, ok := schema["description"].(string); ok {
- result.Description = v
- }
-
- return result
-}
-
-// jsonSchemaToMap converts a JSONSchema struct to a map for API requests
-// This is used for native structured output (response_format with json_schema)
-func (e *AgentBlockExecutor) jsonSchemaToMap(schema *models.JSONSchema) map[string]interface{} {
- if schema == nil {
- return nil
- }
-
- result := map[string]interface{}{
- "type": schema.Type,
- }
-
- // Convert properties
- if len(schema.Properties) > 0 {
- props := make(map[string]interface{})
- for key, prop := range schema.Properties {
- props[key] = e.jsonSchemaToMap(prop)
- }
- result["properties"] = props
- }
-
- // Convert items (for arrays)
- if schema.Items != nil {
- result["items"] = e.jsonSchemaToMap(schema.Items)
- }
-
- // Add required fields
- if len(schema.Required) > 0 {
- result["required"] = schema.Required
- }
-
- // Add description if present
- if schema.Description != "" {
- result["description"] = schema.Description
- }
-
- // Add enum if present
- if len(schema.Enum) > 0 {
- result["enum"] = schema.Enum
- }
-
- // Strict mode requires additionalProperties: false for objects
- if schema.Type == "object" {
- result["additionalProperties"] = false
- }
-
- return result
-}
-
-// resolveModel resolves model alias to actual model ID and provider
-func (e *AgentBlockExecutor) resolveModel(modelID string) (*models.Provider, string, error) {
- // Step 1: Try direct lookup
- provider, err := e.providerService.GetByModelID(modelID)
- if err == nil {
- return provider, modelID, nil
- }
-
- // Step 2: Try model alias resolution
- log.Printf("🔄 [AGENT-BLOCK] Model '%s' not found directly, trying alias resolution...", modelID)
- if aliasProvider, aliasModel, found := e.chatService.ResolveModelAlias(modelID); found {
- return aliasProvider, aliasModel, nil
- }
-
- // Step 3: Fallback to default provider with model
- log.Printf("⚠️ [AGENT-BLOCK] Model '%s' not found, using default provider", modelID)
- defaultProvider, defaultModel, err := e.chatService.GetDefaultProviderWithModel()
- if err != nil {
- return nil, "", fmt.Errorf("failed to find provider for model %s: %w", modelID, err)
- }
-
- return defaultProvider, defaultModel, nil
-}
-
-// buildMessages creates the initial messages with interpolated prompts
-func (e *AgentBlockExecutor) buildMessages(config models.AgentBlockConfig, inputs map[string]any) []map[string]any {
- messages := []map[string]any{}
-
- log.Printf("🔍 [AGENT-BLOCK] Building messages with inputs: %+v", inputs)
-
- // Check for data file attachments first (needed for system prompt enhancement)
- dataAttachments := e.extractDataFileAttachments(inputs)
-
- // Build the enhanced system prompt with execution mode prefix
- var systemPromptBuilder strings.Builder
-
- // ALWAYS inject execution mode preamble for deterministic behavior
- systemPromptBuilder.WriteString(ExecutionModePrefix)
-
- // Add tool-specific mandatory instructions when tools are enabled
- if len(config.EnabledTools) > 0 {
- systemPromptBuilder.WriteString("## REQUIRED TOOLS FOR THIS TASK\n")
- systemPromptBuilder.WriteString("You MUST use one or more of these tools to complete your task:\n\n")
-
- // Get tool descriptions to help the LLM understand how to use them
- toolDescriptions := e.getToolDescriptions(config.EnabledTools)
- for _, toolDesc := range toolDescriptions {
- systemPromptBuilder.WriteString(toolDesc)
- systemPromptBuilder.WriteString("\n")
- }
-
- systemPromptBuilder.WriteString("\nIMPORTANT: DO NOT respond with text only. You MUST call at least one of the above tools.\n\n")
- }
-
- // Check for retry context and add stronger instructions
- if retryAttempt, ok := inputs["_retryAttempt"].(int); ok && retryAttempt > 0 {
- retryReason, _ := inputs["_retryReason"].(string)
-
- // Determine if this is a schema error or tool error
- if strings.Contains(retryReason, "Schema validation failed") || strings.Contains(retryReason, "schema") {
- // Schema validation retry - guide LLM to fix JSON output format
- systemPromptBuilder.WriteString(fmt.Sprintf(`## ⚠️ SCHEMA VALIDATION RETRY (Attempt %d)
-Your previous response did NOT match the required JSON schema.
-Error: %s
-
-CRITICAL REQUIREMENTS:
-1. Your response MUST be valid JSON that matches the exact schema structure
-2. Include ALL required fields - missing fields cause validation failure
-3. Use the correct data types (strings vs numbers) as defined in the schema
-4. If the schema expects an object with an array property, wrap your array in an object
-5. Do NOT add extra fields not defined in the schema
-
-Fix your response NOW to match the required schema exactly.
-
-`, retryAttempt+1, retryReason))
- } else {
- // Tool usage retry
- systemPromptBuilder.WriteString(fmt.Sprintf(`## RETRY NOTICE (Attempt %d)
-Your previous response did not use the required tools.
-Reason: %s
-
-YOU MUST call the appropriate tool(s) NOW. Do not respond with text only.
-This is your last chance - call the tool immediately.
-
-`, retryAttempt+1, retryReason))
- }
- log.Printf("🔄 [AGENT-BLOCK] Added retry notice for attempt %d (reason: %s)", retryAttempt+1, retryReason)
- }
-
- // Add the user's system prompt
- if config.SystemPrompt != "" {
- systemPromptBuilder.WriteString("## YOUR SPECIFIC TASK\n")
- systemPromptBuilder.WriteString(interpolateTemplate(config.SystemPrompt, inputs))
- systemPromptBuilder.WriteString("\n\n")
- }
-
- // If data files present, add analysis guidelines to system prompt
- if len(dataAttachments) > 0 {
- systemPromptBuilder.WriteString(`
-## Data Analysis Guidelines
-- The data file content is provided in the user message - you can see the structure
-- Use the 'analyze_data' tool to run Python code - data is pre-loaded as pandas DataFrame 'df'
-- Generate all charts/visualizations in ONE comprehensive tool call
-- After receiving results with charts, provide your insights and STOP - do not repeat the analysis
-- Charts are automatically captured - you will see [CHART_IMAGE_SAVED] in the result
-`)
- log.Printf("📊 [AGENT-BLOCK] Added data analysis guidelines to system prompt")
- }
-
- // Add structured data context so LLM knows exactly what data is available
- dataContextSection := e.buildDataContext(inputs)
- if dataContextSection != "" {
- systemPromptBuilder.WriteString(dataContextSection)
- }
-
- // Build the final system prompt
- finalSystemPrompt := systemPromptBuilder.String()
- log.Printf("🔍 [AGENT-BLOCK] System prompt built (%d chars, %d tools enabled)", len(finalSystemPrompt), len(config.EnabledTools))
-
- messages = append(messages, map[string]any{
- "role": "system",
- "content": finalSystemPrompt,
- })
-
- // Add user prompt with potential image attachments
- userPrompt := interpolateTemplate(config.UserPrompt, inputs)
- log.Printf("🔍 [AGENT-BLOCK] User prompt (interpolated): %s", userPrompt)
-
- // Inject data file content into user prompt (dataAttachments already extracted above)
- if len(dataAttachments) > 0 {
- var dataContext strings.Builder
- dataContext.WriteString("\n\n--- Data Files ---\n")
-
- for _, att := range dataAttachments {
- dataContext.WriteString(fmt.Sprintf("\nFile: %s\n", att.Filename))
- dataContext.WriteString(fmt.Sprintf("Type: %s\n", att.MimeType))
- dataContext.WriteString("Content preview (first 100 lines):\n```\n")
- dataContext.WriteString(att.Content)
- dataContext.WriteString("\n```\n")
- }
-
- userPrompt = userPrompt + dataContext.String()
- log.Printf("📊 [AGENT-BLOCK] Injected %d data file(s) into prompt (%d chars added)",
- len(dataAttachments), dataContext.Len())
- }
-
- // Check for image attachments and vision model support
- log.Printf("🔍 [AGENT-BLOCK] Checking for image attachments in %d inputs...", len(inputs))
- imageAttachments := e.extractImageAttachments(inputs)
- isVisionModel := e.isOpenAIVisionModel(config.Model)
- log.Printf("🔍 [AGENT-BLOCK] Found %d image attachments, isVisionModel=%v (model=%s)", len(imageAttachments), isVisionModel, config.Model)
-
- if len(imageAttachments) > 0 && isVisionModel {
- // Build multipart content with text and images
- contentParts := []map[string]any{
- {
- "type": "text",
- "text": userPrompt,
- },
- }
-
- for _, att := range imageAttachments {
- imageURL := e.getImageAsBase64DataURL(att.FileID)
- if imageURL != "" {
- contentParts = append(contentParts, map[string]any{
- "type": "image_url",
- "image_url": map[string]any{
- "url": imageURL,
- "detail": "auto",
- },
- })
- log.Printf("🖼️ [AGENT-BLOCK] Added image attachment: %s", att.Filename)
- }
- }
-
- messages = append(messages, map[string]any{
- "role": "user",
- "content": contentParts,
- })
- } else {
- // Standard text message
- messages = append(messages, map[string]any{
- "role": "user",
- "content": userPrompt,
- })
- }
-
- return messages
-}
-
-// buildDataContext creates a structured data context section for the system prompt
-// This helps the LLM understand exactly what data is available from previous blocks
-func (e *AgentBlockExecutor) buildDataContext(inputs map[string]any) string {
- var builder strings.Builder
- var hasContent bool
-
- // Categorize inputs
- var workflowInputs []string
- blockOutputs := make(map[string]string)
-
- for key, value := range inputs {
- // Skip internal keys
- if strings.HasPrefix(key, "_") || strings.HasPrefix(key, "__") {
- continue
- }
-
- // Skip common passthrough keys
- if key == "input" || key == "value" || key == "start" {
- // Format the main input nicely
- if valueStr := formatValueForContext(value); valueStr != "" {
- workflowInputs = append(workflowInputs, fmt.Sprintf("- **%s**: %s", key, valueStr))
- hasContent = true
- }
- continue
- }
-
- // Check if it's a block output (nested map with response key)
- if m, ok := value.(map[string]any); ok {
- if response, hasResponse := m["response"]; hasResponse {
- if responseStr, ok := response.(string); ok && responseStr != "" {
- // Truncate very long responses for context
- if len(responseStr) > 1500 {
- responseStr = responseStr[:1500] + "... [TRUNCATED - full data in user message]"
- }
- blockOutputs[key] = responseStr
- hasContent = true
- }
- }
- }
- }
-
- // Always include current datetime for time-sensitive queries (safety net)
- builder.WriteString("\n## CURRENT DATE AND TIME\n")
- now := time.Now()
- builder.WriteString(fmt.Sprintf("**Today's Date:** %s\n", now.Format("Monday, January 2, 2006")))
- builder.WriteString(fmt.Sprintf("**Current Time:** %s\n", now.Format("3:04 PM MST")))
- builder.WriteString(fmt.Sprintf("**ISO Format:** %s\n\n", now.Format(time.RFC3339)))
- builder.WriteString("Use this date when searching for 'today', 'recent', 'latest', or 'current' information.\n\n")
-
- if !hasContent {
- // Still return the datetime even if no other content
- return builder.String()
- }
-
- builder.WriteString("## AVAILABLE DATA (Already Resolved)\n")
- builder.WriteString("The following data has been collected from previous steps and is ready for use:\n\n")
-
- // Present workflow inputs
- if len(workflowInputs) > 0 {
- builder.WriteString("### Direct Inputs\n")
- for _, input := range workflowInputs {
- builder.WriteString(input + "\n")
- }
- builder.WriteString("\n")
- }
-
- // Present block outputs
- if len(blockOutputs) > 0 {
- builder.WriteString("### Data from Previous Blocks\n")
- for blockID, response := range blockOutputs {
- builder.WriteString(fmt.Sprintf("**From `%s`:**\n", blockID))
- builder.WriteString("```\n")
- builder.WriteString(response)
- builder.WriteString("\n```\n\n")
- }
- }
-
- builder.WriteString("Use this data directly - DO NOT ask for it or claim you don't have it.\n\n")
-
- return builder.String()
-}
-
-// formatValueForContext formats a value for display in the data context
-func formatValueForContext(value any) string {
- switch v := value.(type) {
- case string:
- if len(v) > 500 {
- return fmt.Sprintf("%q... [%d chars total]", v[:500], len(v))
- }
- if len(v) > 100 {
- return fmt.Sprintf("%q", v[:100]+"...")
- }
- return fmt.Sprintf("%q", v)
- case float64:
- if v == float64(int(v)) {
- return fmt.Sprintf("%d", int(v))
- }
- return fmt.Sprintf("%g", v)
- case int:
- return fmt.Sprintf("%d", v)
- case bool:
- return fmt.Sprintf("%t", v)
- case map[string]any:
- // Check for file reference
- if fileID, ok := v["file_id"].(string); ok && fileID != "" {
- filename, _ := v["filename"].(string)
- return fmt.Sprintf("[File: %s]", filename)
- }
- // For other maps, JSON encode briefly
- jsonBytes, err := json.Marshal(v)
- if err != nil {
- return "[complex object]"
- }
- if len(jsonBytes) > 200 {
- return string(jsonBytes[:200]) + "..."
- }
- return string(jsonBytes)
- default:
- return ""
- }
-}
-
-// FileAttachment represents an image or file attachment
-type FileAttachment struct {
- FileID string `json:"file_id"`
- Filename string `json:"filename"`
- MimeType string `json:"mime_type"`
- Type string `json:"type"` // "image", "document", "audio", "data"
-}
-
-// extractImageAttachments extracts image attachments from inputs
-func (e *AgentBlockExecutor) extractImageAttachments(inputs map[string]any) []FileAttachment {
- var attachments []FileAttachment
-
- // Helper to extract attachment from a map
- extractFromMap := func(attMap map[string]any) *FileAttachment {
- att := FileAttachment{}
- if v, ok := attMap["file_id"].(string); ok {
- att.FileID = v
- } else if v, ok := attMap["fileId"].(string); ok {
- att.FileID = v
- }
- if v, ok := attMap["filename"].(string); ok {
- att.Filename = v
- }
- if v, ok := attMap["mime_type"].(string); ok {
- att.MimeType = v
- } else if v, ok := attMap["mimeType"].(string); ok {
- att.MimeType = v
- }
- if v, ok := attMap["type"].(string); ok {
- att.Type = v
- }
-
- // Only include images
- if att.FileID != "" && (att.Type == "image" || strings.HasPrefix(att.MimeType, "image/")) {
- return &att
- }
- return nil
- }
-
- // Check for "_attachments" or "attachments" in inputs
- var rawAttachments []interface{}
- if att, ok := inputs["_attachments"].([]interface{}); ok {
- rawAttachments = att
- } else if att, ok := inputs["attachments"].([]interface{}); ok {
- rawAttachments = att
- } else if att, ok := inputs["images"].([]interface{}); ok {
- rawAttachments = att
- }
-
- for _, raw := range rawAttachments {
- if attMap, ok := raw.(map[string]interface{}); ok {
- if att := extractFromMap(attMap); att != nil {
- attachments = append(attachments, *att)
- }
- }
- }
-
- // Also check for single image file_id
- if fileID, ok := inputs["image_file_id"].(string); ok && fileID != "" {
- attachments = append(attachments, FileAttachment{
- FileID: fileID,
- Type: "image",
- })
- }
-
- // Check all inputs for file references that are images (e.g., from Start block)
- for key, value := range inputs {
- // Skip internal keys
- if strings.HasPrefix(key, "_") || key == "attachments" || key == "images" {
- continue
- }
-
- // Try map[string]any first
- if attMap, ok := value.(map[string]any); ok {
- log.Printf("🔍 [AGENT-BLOCK] Input '%s' is map[string]any: %+v", key, attMap)
- if att := extractFromMap(attMap); att != nil {
- log.Printf("🖼️ [AGENT-BLOCK] Found image file reference in input '%s': %s", key, att.Filename)
- attachments = append(attachments, *att)
- }
- } else if attMap, ok := value.(map[string]interface{}); ok {
- // Try map[string]interface{} (JSON unmarshaling often produces this)
- log.Printf("🔍 [AGENT-BLOCK] Input '%s' is map[string]interface{}: %+v", key, attMap)
- // Convert to map[string]any
- converted := make(map[string]any)
- for k, v := range attMap {
- converted[k] = v
- }
- if att := extractFromMap(converted); att != nil {
- log.Printf("🖼️ [AGENT-BLOCK] Found image file reference in input '%s': %s", key, att.Filename)
- attachments = append(attachments, *att)
- }
- } else if value != nil {
- log.Printf("🔍 [AGENT-BLOCK] Input '%s' has type %T (not a map)", key, value)
- }
- }
-
- return attachments
-}
-
-// DataFileAttachment represents a data file attachment (CSV, JSON, Excel, etc.)
-type DataFileAttachment struct {
- FileID string `json:"file_id"`
- Filename string `json:"filename"`
- MimeType string `json:"mime_type"`
- Content string `json:"content"` // Preview content (first ~100 lines)
-}
-
-// extractDataFileAttachments extracts data file attachments from inputs
-func (e *AgentBlockExecutor) extractDataFileAttachments(inputs map[string]any) []DataFileAttachment {
- var attachments []DataFileAttachment
-
- // Data file MIME types
- dataTypes := map[string]bool{
- "text/csv": true,
- "application/json": true,
- "application/vnd.ms-excel": true,
- "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet": true,
- "text/plain": true,
- "text/tab-separated-values": true,
- }
-
- // Helper to check if a file is a data file
- isDataFile := func(mimeType, filename string) bool {
- if dataTypes[mimeType] {
- return true
- }
- // Check by extension
- ext := strings.ToLower(filepath.Ext(filename))
- return ext == ".csv" || ext == ".json" || ext == ".xlsx" ||
- ext == ".xls" || ext == ".tsv" || ext == ".txt"
- }
-
- // Check all inputs for file references
- for key, value := range inputs {
- if strings.HasPrefix(key, "_") {
- continue
- }
-
- var attMap map[string]any
- if m, ok := value.(map[string]any); ok {
- attMap = m
- } else if m, ok := value.(map[string]interface{}); ok {
- attMap = make(map[string]any)
- for k, v := range m {
- attMap[k] = v
- }
- }
-
- if attMap == nil {
- continue
- }
-
- fileID, _ := attMap["file_id"].(string)
- if fileID == "" {
- fileID, _ = attMap["fileId"].(string)
- }
- filename, _ := attMap["filename"].(string)
- mimeType, _ := attMap["mime_type"].(string)
- if mimeType == "" {
- mimeType, _ = attMap["mimeType"].(string)
- }
-
- if fileID != "" && isDataFile(mimeType, filename) {
- // Read file content from cache
- content := e.readDataFileContent(fileID, filename)
- if content != "" {
- attachments = append(attachments, DataFileAttachment{
- FileID: fileID,
- Filename: filename,
- MimeType: mimeType,
- Content: content,
- })
- log.Printf("📊 [AGENT-BLOCK] Found data file in input '%s': %s (%d chars)", key, filename, len(content))
- }
- }
- }
-
- return attachments
-}
-
-// readDataFileContent reads content from a data file (CSV, JSON, etc.)
-// Returns a preview (first ~100 lines) suitable for LLM context
-func (e *AgentBlockExecutor) readDataFileContent(fileID, filename string) string {
- fileCacheService := filecache.GetService()
- file, found := fileCacheService.Get(fileID)
- if !found {
- log.Printf("⚠️ [AGENT-BLOCK] Data file not found in cache: %s", fileID)
- return ""
- }
-
- if file.FilePath == "" {
- log.Printf("⚠️ [AGENT-BLOCK] Data file path not available: %s", fileID)
- return ""
- }
-
- // Read file content
- content, err := os.ReadFile(file.FilePath)
- if err != nil {
- log.Printf("❌ [AGENT-BLOCK] Failed to read data file: %v", err)
- return ""
- }
-
- // Convert to string and limit to first 100 lines for context
- lines := strings.Split(string(content), "\n")
- maxLines := 100
- if len(lines) > maxLines {
- lines = lines[:maxLines]
- }
-
- preview := strings.Join(lines, "\n")
- log.Printf("✅ [AGENT-BLOCK] Read data file %s (%d lines, %d bytes)",
- filename, len(lines), len(preview))
-
- return preview
-}
-
-// Artifact represents a generated artifact (chart, image, etc.) from tool execution
-type Artifact struct {
- Type string `json:"type"` // "chart", "image", "file"
- Format string `json:"format"` // "png", "jpeg", "svg", etc.
- Data string `json:"data"` // Base64 encoded data
- Title string `json:"title"` // Optional title/description
-}
-
-// extractArtifactsFromToolCalls extracts all artifacts (charts, images) from tool call results
-// This provides a consistent format for API consumers to access generated visualizations
-func (e *AgentBlockExecutor) extractArtifactsFromToolCalls(toolCalls []models.ToolCallRecord) []Artifact {
- var artifacts []Artifact
-
- for _, tc := range toolCalls {
- if tc.Error != "" || tc.Result == "" {
- continue
- }
-
- // Parse tool result as JSON
- var resultData map[string]any
- if err := json.Unmarshal([]byte(tc.Result), &resultData); err != nil {
- continue
- }
-
- // Look for charts/images in common E2B response formats
- // E2B analyze_data returns: {"plots": [{"data": "base64...", "type": "png"}], ...}
- if plots, ok := resultData["plots"].([]interface{}); ok {
- for i, p := range plots {
- if plot, ok := p.(map[string]interface{}); ok {
- data, _ := plot["data"].(string)
- format, _ := plot["type"].(string)
- if format == "" {
- format = "png"
- }
- if data != "" && len(data) > 100 {
- artifacts = append(artifacts, Artifact{
- Type: "chart",
- Format: format,
- Data: data,
- Title: fmt.Sprintf("Chart %d from %s", i+1, tc.Name),
- })
- }
- }
- }
- }
-
- // Also check for single image/plot fields
- for _, key := range []string{"image", "plot", "chart", "figure", "png", "jpeg"} {
- if data, ok := resultData[key].(string); ok && len(data) > 100 {
- // Determine format from key or data URI
- format := "png"
- if key == "jpeg" {
- format = "jpeg"
- }
- if strings.HasPrefix(data, "data:image/") {
- // Extract format from data URI
- if strings.Contains(data, "jpeg") || strings.Contains(data, "jpg") {
- format = "jpeg"
- } else if strings.Contains(data, "svg") {
- format = "svg"
- }
- }
-
- artifacts = append(artifacts, Artifact{
- Type: "chart",
- Format: format,
- Data: data,
- Title: fmt.Sprintf("Generated %s from %s", key, tc.Name),
- })
- }
- }
-
- // Check for base64_images array (another common E2B format)
- if images, ok := resultData["base64_images"].([]interface{}); ok {
- for i, img := range images {
- if imgData, ok := img.(string); ok && len(imgData) > 100 {
- artifacts = append(artifacts, Artifact{
- Type: "chart",
- Format: "png",
- Data: imgData,
- Title: fmt.Sprintf("Image %d from %s", i+1, tc.Name),
- })
- }
- }
- }
- }
-
- log.Printf("📊 [AGENT-BLOCK] Extracted %d artifacts from tool calls", len(artifacts))
- return artifacts
-}
-
-// GeneratedFile represents a file generated by a tool (PDF, document, etc.)
-type GeneratedFile struct {
- FileID string `json:"file_id"`
- Filename string `json:"filename"`
- DownloadURL string `json:"download_url"`
- AccessCode string `json:"access_code,omitempty"`
- Size int64 `json:"size,omitempty"`
- MimeType string `json:"mime_type,omitempty"`
-}
-
-// extractGeneratedFilesFromToolCalls extracts file references (PDFs, documents) from tool call results
-// This makes download URLs available to subsequent blocks
-func (e *AgentBlockExecutor) extractGeneratedFilesFromToolCalls(toolCalls []models.ToolCallRecord) []GeneratedFile {
- var files []GeneratedFile
-
- for _, tc := range toolCalls {
- if tc.Error != "" || tc.Result == "" {
- continue
- }
-
- // Parse tool result as JSON
- var resultData map[string]any
- if err := json.Unmarshal([]byte(tc.Result), &resultData); err != nil {
- continue
- }
-
- // Look for file reference fields
- fileRef := GeneratedFile{}
-
- if v, ok := resultData["file_id"].(string); ok && v != "" {
- fileRef.FileID = v
- }
- if v, ok := resultData["filename"].(string); ok && v != "" {
- fileRef.Filename = v
- }
- if v, ok := resultData["download_url"].(string); ok && v != "" {
- fileRef.DownloadURL = v
- }
- if v, ok := resultData["access_code"].(string); ok && v != "" {
- fileRef.AccessCode = v
- }
- if v, ok := resultData["size"].(float64); ok {
- fileRef.Size = int64(v)
- }
- if v, ok := resultData["mime_type"].(string); ok && v != "" {
- fileRef.MimeType = v
- }
-
- // Only add if we have meaningful file reference data
- if fileRef.FileID != "" || fileRef.DownloadURL != "" {
- files = append(files, fileRef)
- log.Printf("📄 [AGENT-BLOCK] Extracted file reference: %s (url: %s)", fileRef.Filename, fileRef.DownloadURL)
- }
- }
-
- return files
-}
-
-// sanitizeToolResultForLLM removes base64 image data from tool results
-// Base64 images are huge and useless to the LLM as text - it can't "see" them
-// Instead, we replace them with a placeholder indicating a chart was generated
-func (e *AgentBlockExecutor) sanitizeToolResultForLLM(result string) string {
- if len(result) < 1000 {
- return result // Small results don't need sanitization
- }
-
- chartsGenerated := false
-
- // Pattern to match base64 image data (PNG, JPEG, etc.)
- // Matches: "data:image/png;base64,..." or just long base64 strings
- base64Pattern := regexp.MustCompile(`"data:image/[^;]+;base64,[A-Za-z0-9+/=]{100,}"`)
- if base64Pattern.MatchString(result) {
- chartsGenerated = true
- }
- sanitized := base64Pattern.ReplaceAllString(result, `"[CHART_IMAGE_SAVED]"`)
-
- // Also match standalone base64 blocks that might not have data URI prefix
- // Look for very long strings of base64 characters (>500 chars)
- longBase64Pattern := regexp.MustCompile(`"[A-Za-z0-9+/=]{500,}"`)
- if longBase64Pattern.MatchString(sanitized) {
- chartsGenerated = true
- }
- sanitized = longBase64Pattern.ReplaceAllString(sanitized, `"[CHART_IMAGE_SAVED]"`)
-
- // Also handle base64 in "image" or "plot" fields common in E2B responses
- imageFieldPattern := regexp.MustCompile(`"(image|plot|chart|png|jpeg|figure)":\s*"[A-Za-z0-9+/=]{100,}"`)
- sanitized = imageFieldPattern.ReplaceAllString(sanitized, `"$1": "[CHART_IMAGE_SAVED]"`)
-
- // Truncate if still too long (max 20KB for tool results)
- maxLen := 20000
- if len(sanitized) > maxLen {
- sanitized = sanitized[:maxLen] + "\n... [TRUNCATED - Full result too large for LLM context]"
- }
-
- // Add clear instruction when charts were generated
- if chartsGenerated {
- sanitized = sanitized + "\n\n[CHARTS SUCCESSFULLY GENERATED AND SAVED. Do NOT call analyze_data again. Provide your final summary/insights based on the analysis output above.]"
- }
-
- originalLen := len(result)
- newLen := len(sanitized)
- if originalLen != newLen {
- log.Printf("🧹 [AGENT-BLOCK] Sanitized tool result: %d -> %d chars (removed base64/large data, charts=%v)",
- originalLen, newLen, chartsGenerated)
- }
-
- return sanitized
-}
-
-// extractChartsFromResult extracts base64 chart images from tool results
-// This is used to collect charts for auto-injection into Discord/Slack messages
-func (e *AgentBlockExecutor) extractChartsFromResult(result string) []string {
- var charts []string
-
- // Try to parse as JSON
- var resultData map[string]any
- if err := json.Unmarshal([]byte(result), &resultData); err != nil {
- return charts
- }
-
- // Look for plots array (E2B analyze_data format)
- if plots, ok := resultData["plots"].([]interface{}); ok {
- for _, p := range plots {
- if plot, ok := p.(map[string]interface{}); ok {
- // Check for "data" field containing base64
- if data, ok := plot["data"].(string); ok && len(data) > 100 {
- charts = append(charts, data)
- }
- }
- }
- }
-
- // Also check for single "image" or "chart" field
- for _, key := range []string{"image", "chart", "plot", "figure"} {
- if data, ok := resultData[key].(string); ok && len(data) > 100 {
- charts = append(charts, data)
- }
- }
-
- return charts
-}
-
-// extractChartsFromInputs extracts chart images from previous block artifacts
-// This allows downstream blocks (like Discord Publisher) to access charts generated upstream
-func (e *AgentBlockExecutor) extractChartsFromInputs(inputs map[string]any) []string {
- var charts []string
-
- // Helper function to extract charts from artifacts slice
- extractFromArtifacts := func(artifacts []Artifact) {
- for _, artifact := range artifacts {
- if artifact.Type == "chart" && artifact.Data != "" && len(artifact.Data) > 100 {
- charts = append(charts, artifact.Data)
- log.Printf("🖼️ [AGENT-BLOCK] Found chart artifact: %s (format: %s, %d bytes)",
- artifact.Title, artifact.Format, len(artifact.Data))
- }
- }
- }
-
- // Helper to try converting interface to []Artifact
- tryConvertArtifacts := func(v interface{}) bool {
- // Direct type assertion for []Artifact
- if artifacts, ok := v.([]Artifact); ok {
- extractFromArtifacts(artifacts)
- return true
- }
- // Try []execution.Artifact (same type, different reference)
- if artifacts, ok := v.([]interface{}); ok {
- for _, a := range artifacts {
- if artifact, ok := a.(Artifact); ok {
- if artifact.Type == "chart" && artifact.Data != "" && len(artifact.Data) > 100 {
- charts = append(charts, artifact.Data)
- }
- } else if artifactMap, ok := a.(map[string]interface{}); ok {
- // Handle map representation of artifact
- artifactType, _ := artifactMap["type"].(string)
- artifactData, _ := artifactMap["data"].(string)
- if artifactType == "chart" && artifactData != "" && len(artifactData) > 100 {
- charts = append(charts, artifactData)
- }
- }
- }
- return true
- }
- return false
- }
-
- // 1. Check direct "artifacts" key in inputs
- if artifacts, ok := inputs["artifacts"]; ok {
- tryConvertArtifacts(artifacts)
- }
-
- // 2. Check previous block outputs (e.g., inputs["data-analyzer"]["artifacts"])
- // These are stored as map[string]any with block IDs as keys
- for key, value := range inputs {
- // Skip non-block keys
- if key == "artifacts" || key == "input" || key == "value" || key == "start" ||
- strings.HasPrefix(key, "_") || strings.HasPrefix(key, "__") {
- continue
- }
-
- // Check if value is a map (previous block output)
- if blockOutput, ok := value.(map[string]any); ok {
- // Look for artifacts in this block's output
- if artifacts, ok := blockOutput["artifacts"]; ok {
- tryConvertArtifacts(artifacts)
- }
-
- // Also check for nested output.artifacts
- if output, ok := blockOutput["output"].(map[string]any); ok {
- if artifacts, ok := output["artifacts"]; ok {
- tryConvertArtifacts(artifacts)
- }
- }
-
- // Check toolCalls for chart results (some tools return charts in result)
- if toolCalls, ok := blockOutput["toolCalls"].([]models.ToolCallRecord); ok {
- for _, tc := range toolCalls {
- if tc.Result != "" {
- extractedCharts := e.extractChartsFromResult(tc.Result)
- charts = append(charts, extractedCharts...)
- }
- }
- }
- // Also try interface{} slice for toolCalls
- if toolCalls, ok := blockOutput["toolCalls"].([]interface{}); ok {
- for _, tc := range toolCalls {
- if tcMap, ok := tc.(map[string]interface{}); ok {
- if result, ok := tcMap["Result"].(string); ok && result != "" {
- extractedCharts := e.extractChartsFromResult(result)
- charts = append(charts, extractedCharts...)
- }
- }
- }
- }
- }
- }
-
- return charts
-}
-
-// NOTE: detectToolsFromContext was removed - blocks now only use explicitly configured tools
-// This ensures predictable behavior where users must configure enabledTools in the block settings
-
-// extractToolResultsForDownstream parses tool call results and extracts data for downstream blocks
-// This solves the problem where tool output was buried in toolCalls and not accessible to next blocks
-func (e *AgentBlockExecutor) extractToolResultsForDownstream(toolCalls []models.ToolCallRecord) map[string]any {
- results := make(map[string]any)
-
- for _, tc := range toolCalls {
- if tc.Error != "" || tc.Result == "" {
- continue
- }
-
- // Try to parse result as JSON
- var parsed map[string]any
- if err := json.Unmarshal([]byte(tc.Result), &parsed); err != nil {
- // Not JSON - store as raw string
- results[tc.Name] = tc.Result
- continue
- }
-
- // Store parsed result under tool name
- results[tc.Name] = parsed
-
- log.Printf("📦 [AGENT-BLOCK] Extracted tool result for '%s': %d fields", tc.Name, len(parsed))
- }
-
- return results
-}
-
-// isOpenAIVisionModel checks if the model is an OpenAI vision-capable model
-func (e *AgentBlockExecutor) isOpenAIVisionModel(modelID string) bool {
- // OpenAI vision-capable models
- visionModels := []string{
- "gpt-4o",
- "gpt-4o-mini",
- "gpt-4-turbo",
- "gpt-4-vision-preview",
- "gpt-4-turbo-preview",
- "gpt-5", // GPT-5 series (all variants support vision)
- "gpt-5.1", // GPT-5.1 series
- "o1", // o1 models
- "o1-preview",
- "o1-mini",
- "o3", // o3 models
- "o3-mini",
- "o4-mini", // o4-mini (if released)
- }
-
- modelLower := strings.ToLower(modelID)
- for _, vm := range visionModels {
- if strings.Contains(modelLower, vm) {
- return true
- }
- }
-
- // Also check model aliases that might map to vision models
- // Common patterns: any 4o variant or 5.x variant
- if strings.Contains(modelLower, "gpt-4o") || strings.Contains(modelLower, "4o") ||
- strings.Contains(modelLower, "gpt-5") || strings.Contains(modelLower, "5.1") {
- return true
- }
-
- return false
-}
-
-// getImageAsBase64DataURL converts an image file to a base64 data URL
-func (e *AgentBlockExecutor) getImageAsBase64DataURL(fileID string) string {
- // Get file from cache
- fileCacheService := filecache.GetService()
- file, found := fileCacheService.Get(fileID)
- if !found {
- log.Printf("⚠️ [AGENT-BLOCK] Image file not found: %s", fileID)
- return ""
- }
-
- // Verify it's an image
- if !strings.HasPrefix(file.MimeType, "image/") {
- log.Printf("⚠️ [AGENT-BLOCK] File is not an image: %s (%s)", fileID, file.MimeType)
- return ""
- }
-
- // Read image from disk
- if file.FilePath == "" {
- log.Printf("⚠️ [AGENT-BLOCK] Image file path not available: %s", fileID)
- return ""
- }
-
- imageData, err := os.ReadFile(file.FilePath)
- if err != nil {
- log.Printf("❌ [AGENT-BLOCK] Failed to read image file: %v", err)
- return ""
- }
-
- // Convert to base64 data URL
- base64Image := base64.StdEncoding.EncodeToString(imageData)
- dataURL := fmt.Sprintf("data:%s;base64,%s", file.MimeType, base64Image)
-
- log.Printf("✅ [AGENT-BLOCK] Converted image to base64 (%d bytes)", len(imageData))
- return dataURL
-}
-
-// getToolDescriptions returns human-readable descriptions of enabled tools
-// This helps the LLM understand how to use the tools, including key parameters
-func (e *AgentBlockExecutor) getToolDescriptions(enabledTools []string) []string {
- if len(enabledTools) == 0 {
- return nil
- }
-
- enabledSet := make(map[string]bool)
- for _, name := range enabledTools {
- enabledSet[name] = true
- }
-
- var descriptions []string
- allTools := e.toolRegistry.List()
-
- for _, tool := range allTools {
- if fn, ok := tool["function"].(map[string]interface{}); ok {
- name, _ := fn["name"].(string)
- if !enabledSet[name] {
- continue
- }
-
- desc, _ := fn["description"].(string)
- // Truncate long descriptions but keep key info
- if len(desc) > 500 {
- desc = desc[:500] + "..."
- }
-
- // Build a concise tool summary
- var sb strings.Builder
- sb.WriteString(fmt.Sprintf("### %s\n", name))
- if desc != "" {
- sb.WriteString(fmt.Sprintf("%s\n", desc))
- }
-
- // Extract key parameters to highlight
- if params, ok := fn["parameters"].(map[string]interface{}); ok {
- if props, ok := params["properties"].(map[string]interface{}); ok {
- var keyParams []string
- for paramName, paramDef := range props {
- // Skip internal parameters
- if strings.HasPrefix(paramName, "_") || paramName == "credential_id" || paramName == "api_key" {
- continue
- }
- if paramMap, ok := paramDef.(map[string]interface{}); ok {
- paramDesc, _ := paramMap["description"].(string)
- // Highlight important params like file_url
- if paramName == "file_url" || paramName == "download_url" {
- keyParams = append(keyParams, fmt.Sprintf(" - **%s**: %s", paramName, paramDesc))
- } else if len(keyParams) < 5 { // Limit to 5 params
- shortDesc := paramDesc
- if len(shortDesc) > 100 {
- shortDesc = shortDesc[:100] + "..."
- }
- keyParams = append(keyParams, fmt.Sprintf(" - %s: %s", paramName, shortDesc))
- }
- }
- }
- if len(keyParams) > 0 {
- sb.WriteString("Key parameters:\n")
- for _, p := range keyParams {
- sb.WriteString(p + "\n")
- }
- }
- }
- }
-
- descriptions = append(descriptions, sb.String())
- }
- }
-
- return descriptions
-}
-
-// filterTools returns only the tools that are enabled for this block
-func (e *AgentBlockExecutor) filterTools(enabledTools []string) []map[string]interface{} {
- if len(enabledTools) == 0 {
- // No tools enabled for this block
- return nil
- }
-
- // Get all available tools
- allTools := e.toolRegistry.List()
-
- // Filter to only enabled tools
- var filtered []map[string]interface{}
- enabledSet := make(map[string]bool)
- for _, name := range enabledTools {
- enabledSet[name] = true
- }
-
- for _, tool := range allTools {
- if fn, ok := tool["function"].(map[string]interface{}); ok {
- if name, ok := fn["name"].(string); ok {
- if enabledSet[name] {
- filtered = append(filtered, tool)
- }
- }
- }
- }
-
- return filtered
-}
-
-// LLMResponse represents the response from the LLM
-type LLMResponse struct {
- Content string
- ToolCalls []map[string]any
- FinishReason string // "stop", "tool_calls", "end_turn", etc.
- InputTokens int
- OutputTokens int
-}
-
-// callLLM makes a streaming call to the LLM (required for ClaraVerse API compatibility)
-func (e *AgentBlockExecutor) callLLM(
- ctx context.Context,
- provider *models.Provider,
- modelID string,
- messages []map[string]any,
- tools []map[string]interface{},
- temperature float64,
-) (*LLMResponse, error) {
- return e.callLLMWithSchema(ctx, provider, modelID, messages, tools, temperature, nil)
-}
-
-// callLLMWithSchema calls the LLM with optional native structured output support
-func (e *AgentBlockExecutor) callLLMWithSchema(
- ctx context.Context,
- provider *models.Provider,
- modelID string,
- messages []map[string]any,
- tools []map[string]interface{},
- temperature float64,
- outputSchema *models.JSONSchema,
-) (*LLMResponse, error) {
-
- // Detect provider type by base URL to avoid sending incompatible parameters
- // OpenAI's API is strict and rejects unknown parameters with 400 errors
- isOpenAI := strings.Contains(strings.ToLower(provider.BaseURL), "openai.com")
- isOpenRouter := strings.Contains(strings.ToLower(provider.BaseURL), "openrouter.ai")
- isGLM := strings.Contains(strings.ToLower(provider.BaseURL), "bigmodel.cn") ||
- strings.Contains(strings.ToLower(provider.Name), "glm") ||
- strings.Contains(strings.ToLower(modelID), "glm")
-
- // Check if the model supports native structured output
- // OpenAI GPT-4o models and newer support json_schema response_format
- supportsStructuredOutput := (isOpenAI || isOpenRouter) && outputSchema != nil && len(tools) == 0
-
- // Build request body - use streaming for better compatibility with ClaraVerse API
- requestBody := map[string]interface{}{
- "model": modelID,
- "messages": messages,
- "temperature": temperature,
- "stream": true, // Use streaming - ClaraVerse API works better with streaming
- }
-
- // Use correct token limit parameter based on provider
- // OpenAI newer models (GPT-4o, o1, etc.) require max_completion_tokens instead of max_tokens
- // Most models support 65K+ output tokens, so we use 32768 as a safe high limit
- if isOpenAI {
- requestBody["max_completion_tokens"] = 32768
- } else {
- requestBody["max_tokens"] = 32768
- }
-
- // Add native structured output if supported and no tools are being used
- // Note: Can't use response_format with tools - they're mutually exclusive
- if supportsStructuredOutput {
- // Convert JSONSchema to OpenAI's response_format structure
- schemaMap := e.jsonSchemaToMap(outputSchema)
- requestBody["response_format"] = map[string]interface{}{
- "type": "json_schema",
- "json_schema": map[string]interface{}{
- "name": "structured_output",
- "strict": true,
- "schema": schemaMap,
- },
- }
- log.Printf("📋 [AGENT-BLOCK] Using native structured output (json_schema response_format)")
- }
-
- // Add provider-specific parameters only where supported
- if isGLM {
- // GLM-specific parameters to disable reasoning mode
- requestBody["think"] = false
- requestBody["do_sample"] = true
- requestBody["top_p"] = 0.95
- } else if !isOpenAI && !isOpenRouter {
- // For other non-OpenAI providers, try common parameters that might be supported
- // These providers typically ignore unknown parameters
- requestBody["enable_thinking"] = false
- }
- // Note: OpenAI gets no extra parameters - their API is strict about unknown params
-
- // Only include tools if non-empty
- if len(tools) > 0 {
- requestBody["tools"] = tools
- }
-
- bodyBytes, err := json.Marshal(requestBody)
- if err != nil {
- return nil, fmt.Errorf("failed to marshal request: %w", err)
- }
-
- // Create request
- endpoint := strings.TrimSuffix(provider.BaseURL, "/") + "/chat/completions"
- log.Printf("🌐 [AGENT-BLOCK] Calling LLM: %s (model: %s, streaming: true)", endpoint, modelID)
-
- req, err := http.NewRequestWithContext(ctx, "POST", endpoint, bytes.NewReader(bodyBytes))
- if err != nil {
- return nil, fmt.Errorf("failed to create request: %w", err)
- }
-
- req.Header.Set("Content-Type", "application/json")
- req.Header.Set("Authorization", "Bearer "+provider.APIKey)
-
- // Execute request
- resp, err := e.httpClient.Do(req)
- if err != nil {
- // Classify network/connection errors for retry logic
- return nil, ClassifyError(err)
- }
- defer resp.Body.Close()
-
- if resp.StatusCode != http.StatusOK {
- body, _ := io.ReadAll(resp.Body)
- // Classify HTTP errors for retry logic (429, 5xx are retryable)
- return nil, ClassifyHTTPError(resp.StatusCode, string(body))
- }
-
- // Process SSE stream and accumulate response
- return e.processStreamResponse(resp.Body)
-}
-
-// callLLMWithRetry wraps callLLM with retry logic for transient errors
-// Returns the response, retry attempts history, and any final error
-func (e *AgentBlockExecutor) callLLMWithRetry(
- ctx context.Context,
- provider *models.Provider,
- modelID string,
- messages []map[string]any,
- tools []map[string]interface{},
- temperature float64,
- retryPolicy *models.RetryPolicy,
-) (*LLMResponse, []models.RetryAttempt, error) {
- return e.callLLMWithRetryAndSchema(ctx, provider, modelID, messages, tools, temperature, retryPolicy, nil)
-}
-
-// callLLMWithRetryAndSchema wraps callLLMWithSchema with retry logic for transient errors
-// Returns the response, retry attempts history, and any final error
-func (e *AgentBlockExecutor) callLLMWithRetryAndSchema(
- ctx context.Context,
- provider *models.Provider,
- modelID string,
- messages []map[string]any,
- tools []map[string]interface{},
- temperature float64,
- retryPolicy *models.RetryPolicy,
- outputSchema *models.JSONSchema,
-) (*LLMResponse, []models.RetryAttempt, error) {
-
- // Use default policy if not specified
- if retryPolicy == nil {
- retryPolicy = models.DefaultRetryPolicy()
- }
-
- // Create backoff calculator
- backoff := NewBackoffCalculator(
- retryPolicy.InitialDelay,
- retryPolicy.MaxDelay,
- retryPolicy.BackoffMultiplier,
- retryPolicy.JitterPercent,
- )
-
- var attempts []models.RetryAttempt
-
- for attempt := 0; attempt <= retryPolicy.MaxRetries; attempt++ {
- attemptStart := time.Now()
-
- // Make the LLM call with optional schema
- response, err := e.callLLMWithSchema(ctx, provider, modelID, messages, tools, temperature, outputSchema)
- attemptDuration := time.Since(attemptStart).Milliseconds()
-
- if err == nil {
- // Success!
- if attempt > 0 {
- log.Printf("✅ [AGENT-BLOCK] LLM call succeeded on retry attempt %d", attempt)
- }
- return response, attempts, nil
- }
-
- // Classify the error (may already be classified from callLLM)
- var execErr *ExecutionError
- if e, ok := err.(*ExecutionError); ok {
- execErr = e
- } else {
- execErr = ClassifyError(err)
- }
-
- // Determine error type string for logging and tracking
- errorType := "unknown"
- if execErr.StatusCode == 429 {
- errorType = "rate_limit"
- } else if execErr.StatusCode >= 500 {
- errorType = "server_error"
- } else if strings.Contains(strings.ToLower(execErr.Message), "timeout") ||
- strings.Contains(strings.ToLower(execErr.Message), "deadline") {
- errorType = "timeout"
- } else if strings.Contains(strings.ToLower(execErr.Message), "network") ||
- strings.Contains(strings.ToLower(execErr.Message), "connection") {
- errorType = "network_error"
- }
-
- // Record this attempt
- attempts = append(attempts, models.RetryAttempt{
- Attempt: attempt,
- Error: execErr.Message,
- ErrorType: errorType,
- Timestamp: attemptStart,
- Duration: attemptDuration,
- })
-
- // Check if we should retry
- if attempt < retryPolicy.MaxRetries && ShouldRetry(execErr, retryPolicy.RetryOn) {
- delay := backoff.NextDelay(attempt)
-
- // Use RetryAfter if available and longer (e.g., from 429 response)
- if execErr.RetryAfter > 0 {
- retryAfterDelay := time.Duration(execErr.RetryAfter) * time.Second
- if retryAfterDelay > delay {
- delay = retryAfterDelay
- }
- }
-
- log.Printf("🔄 [AGENT-BLOCK] LLM call failed (attempt %d/%d): %s [%s]. Retrying in %v",
- attempt+1, retryPolicy.MaxRetries+1, execErr.Message, errorType, delay)
-
- // Wait before retry (respecting context cancellation)
- select {
- case <-time.After(delay):
- // Continue to next attempt
- case <-ctx.Done():
- return nil, attempts, &ExecutionError{
- Category: ErrorCategoryTransient,
- Message: "Context cancelled during retry wait",
- Retryable: false,
- Cause: ctx.Err(),
- }
- }
- } else {
- // Not retryable or max retries exceeded
- if attempt >= retryPolicy.MaxRetries {
- log.Printf("❌ [AGENT-BLOCK] LLM call failed after %d attempt(s): %s [%s] (max retries exceeded)",
- attempt+1, execErr.Message, errorType)
- } else {
- log.Printf("❌ [AGENT-BLOCK] LLM call failed: %s [%s] (not retryable)",
- execErr.Message, errorType)
- }
- return nil, attempts, execErr
- }
- }
-
- // Should not reach here, but safety fallback
- return nil, attempts, &ExecutionError{
- Category: ErrorCategoryUnknown,
- Message: "Max retries exceeded",
- Retryable: false,
- }
-}
-
-// processStreamResponse processes SSE stream and returns accumulated response
-func (e *AgentBlockExecutor) processStreamResponse(reader io.Reader) (*LLMResponse, error) {
- response := &LLMResponse{}
- var contentBuilder strings.Builder
-
- // Track tool calls by index to accumulate streaming arguments
- toolCallsMap := make(map[int]*toolCallAccumulator)
-
- scanner := bufio.NewScanner(reader)
- for scanner.Scan() {
- line := scanner.Text()
- if !strings.HasPrefix(line, "data: ") {
- continue
- }
-
- data := strings.TrimPrefix(line, "data: ")
- if data == "[DONE]" {
- break
- }
-
- var chunk map[string]interface{}
- if err := json.Unmarshal([]byte(data), &chunk); err != nil {
- continue
- }
-
- choices, ok := chunk["choices"].([]interface{})
- if !ok || len(choices) == 0 {
- continue
- }
-
- choice := choices[0].(map[string]interface{})
-
- // Capture finish_reason when available (usually in the final chunk)
- if finishReason, ok := choice["finish_reason"].(string); ok && finishReason != "" {
- response.FinishReason = finishReason
- }
-
- delta, ok := choice["delta"].(map[string]interface{})
- if !ok {
- continue
- }
-
- // Accumulate content chunks
- if content, ok := delta["content"].(string); ok {
- contentBuilder.WriteString(content)
- }
-
- // Accumulate tool calls
- if toolCallsData, ok := delta["tool_calls"].([]interface{}); ok {
- for _, tc := range toolCallsData {
- toolCallChunk := tc.(map[string]interface{})
-
- // Get tool call index
- var index int
- if idx, ok := toolCallChunk["index"].(float64); ok {
- index = int(idx)
- }
-
- // Initialize accumulator if needed
- if _, exists := toolCallsMap[index]; !exists {
- toolCallsMap[index] = &toolCallAccumulator{}
- }
-
- acc := toolCallsMap[index]
-
- // Accumulate fields
- if id, ok := toolCallChunk["id"].(string); ok {
- acc.ID = id
- }
- if typ, ok := toolCallChunk["type"].(string); ok {
- acc.Type = typ
- }
- if function, ok := toolCallChunk["function"].(map[string]interface{}); ok {
- if name, ok := function["name"].(string); ok {
- acc.Name = name
- }
- if args, ok := function["arguments"].(string); ok {
- acc.Arguments.WriteString(args)
- }
- }
- }
- }
-
- // Extract token usage from chunk (some APIs include it in each chunk)
- if usage, ok := chunk["usage"].(map[string]interface{}); ok {
- if pt, ok := usage["prompt_tokens"].(float64); ok {
- response.InputTokens = int(pt)
- }
- if ct, ok := usage["completion_tokens"].(float64); ok {
- response.OutputTokens = int(ct)
- }
- }
- }
-
- if err := scanner.Err(); err != nil {
- return nil, fmt.Errorf("error reading stream: %w", err)
- }
-
- // Set accumulated content
- response.Content = contentBuilder.String()
-
- // Convert accumulated tool calls to response format
- for _, acc := range toolCallsMap {
- if acc.Name != "" {
- toolCall := map[string]any{
- "id": acc.ID,
- "type": acc.Type,
- "function": map[string]any{
- "name": acc.Name,
- "arguments": acc.Arguments.String(),
- },
- }
- response.ToolCalls = append(response.ToolCalls, toolCall)
- }
- }
-
- log.Printf("✅ [AGENT-BLOCK] Stream processed: content=%d chars, toolCalls=%d, finishReason=%s",
- len(response.Content), len(response.ToolCalls), response.FinishReason)
-
- return response, nil
-}
-
-// toolCallAccumulator accumulates streaming tool call data
-type toolCallAccumulator struct {
- ID string
- Type string
- Name string
- Arguments strings.Builder
-}
-
-// executeToolCall executes a single tool call and returns the record
-func (e *AgentBlockExecutor) executeToolCall(toolCall map[string]any, blockInputs map[string]any, dataFiles []DataFileAttachment, generatedCharts []string, userID string, credentials []string) models.ToolCallRecord {
- startTime := time.Now()
-
- record := models.ToolCallRecord{
- Arguments: make(map[string]any),
- }
-
- // Extract tool name and arguments
- if fn, ok := toolCall["function"].(map[string]any); ok {
- if name, ok := fn["name"].(string); ok {
- record.Name = name
- }
- if argsStr, ok := fn["arguments"].(string); ok {
- if err := json.Unmarshal([]byte(argsStr), &record.Arguments); err != nil {
- record.Error = fmt.Sprintf("failed to parse arguments: %v", err)
- record.Duration = time.Since(startTime).Milliseconds()
- return record
- }
- }
- }
-
- if record.Name == "" {
- record.Error = "missing tool name"
- record.Duration = time.Since(startTime).Milliseconds()
- return record
- }
-
- // Interpolate template variables in tool arguments
- // This allows tool calls to use {{input}} or other block outputs
- record.Arguments = interpolateMapValues(record.Arguments, blockInputs)
-
- // AUTO-INJECT CSV DATA for analyze_data tool
- // This fixes the issue where LLM uses filename as file_id (which doesn't exist in cache)
- // Instead of relying on file lookup, we inject the already-extracted data directly
- if record.Name == "analyze_data" && len(dataFiles) > 0 {
- // Check if csv_data is already provided and non-empty
- existingCSV, hasCSV := record.Arguments["csv_data"].(string)
- if !hasCSV || existingCSV == "" {
- // No csv_data provided - inject from our extracted data files
- dataFile := dataFiles[0] // Use first data file
- if dataFile.Content != "" {
- record.Arguments["csv_data"] = dataFile.Content
- // Clear file_id to prevent lookup attempts with invalid IDs
- delete(record.Arguments, "file_id")
- log.Printf("📊 [AGENT-BLOCK] Auto-injected csv_data from '%s' (%d chars) - bypassing file cache lookup",
- dataFile.Filename, len(dataFile.Content))
- }
- }
- }
-
- // AUTO-INJECT CHART IMAGES for Discord/Slack messages
- // The LLM may not include image_data at all, or use placeholders - we need the real base64
- if (record.Name == "send_discord_message" || record.Name == "send_slack_message") && len(generatedCharts) > 0 {
- chartToInject := generatedCharts[len(generatedCharts)-1] // Use most recent chart
-
- // Check if image_data exists and is valid
- imageData, hasImageData := record.Arguments["image_data"].(string)
-
- shouldInject := false
- if !hasImageData || imageData == "" {
- // No image_data provided - inject the chart
- shouldInject = true
- log.Printf("🖼️ [AGENT-BLOCK] No image_data in tool call, will auto-inject chart")
- } else if len(imageData) < 100 {
- // Placeholder or short text - not real base64
- shouldInject = true
- log.Printf("🖼️ [AGENT-BLOCK] image_data is placeholder (%d chars), will auto-inject chart", len(imageData))
- } else if strings.Contains(imageData, "[CHART_IMAGE_SAVED]") || strings.Contains(imageData, "[BASE64") {
- // Explicit placeholder text
- shouldInject = true
- log.Printf("🖼️ [AGENT-BLOCK] image_data contains placeholder text, will auto-inject chart")
- }
-
- if shouldInject {
- record.Arguments["image_data"] = chartToInject
- // Also set a filename if not already set
- if _, hasFilename := record.Arguments["image_filename"].(string); !hasFilename {
- record.Arguments["image_filename"] = "chart.png"
- }
- log.Printf("📊 [AGENT-BLOCK] Auto-injected chart image into %s (%d bytes)",
- record.Name, len(chartToInject))
- }
- }
-
- // Inject credential resolver for tools that need authentication
- // The resolver is user-scoped for security - only credentials owned by userID can be accessed
- var resolver tools.CredentialResolver
- if e.credentialService != nil && userID != "" {
- resolver = e.credentialService.CreateCredentialResolver(userID)
- record.Arguments[tools.CredentialResolverKey] = resolver
- record.Arguments[tools.UserIDKey] = userID
- }
-
- // Auto-inject credential_id for tools that need it
- // This allows LLM to NOT know about credentials - we handle it automatically
- toolIntegrationType := tools.GetIntegrationTypeForTool(record.Name)
- if toolIntegrationType != "" && e.credentialService != nil && userID != "" {
- var credentialID string
-
- // First, try to find from explicitly configured credentials
- if len(credentials) > 0 && resolver != nil {
- credentialID = findCredentialForIntegrationType(credentials, toolIntegrationType, resolver)
- if credentialID != "" {
- log.Printf("🔐 [AGENT-BLOCK] Found credential_id=%s from block config for tool=%s",
- credentialID, record.Name)
- }
- }
-
- // If no credential found in block config, try runtime auto-discovery from user's credentials
- if credentialID == "" {
- log.Printf("🔍 [AGENT-BLOCK] No credentials in block config for tool=%s, trying runtime auto-discovery...", record.Name)
- ctx := context.Background()
- userCreds, err := e.credentialService.ListByUserAndType(ctx, userID, toolIntegrationType)
- if err != nil {
- log.Printf("⚠️ [AGENT-BLOCK] Failed to fetch user credentials: %v", err)
- } else if len(userCreds) == 1 {
- // Exactly one credential of this type - auto-use it
- credentialID = userCreds[0].ID
- log.Printf("🔐 [AGENT-BLOCK] Runtime auto-discovered single credential: %s (%s) for tool=%s",
- userCreds[0].Name, credentialID, record.Name)
- } else if len(userCreds) > 1 {
- log.Printf("⚠️ [AGENT-BLOCK] Multiple credentials (%d) found for %s - cannot auto-select. User should configure in Block Settings.",
- len(userCreds), toolIntegrationType)
- } else {
- log.Printf("⚠️ [AGENT-BLOCK] No %s credentials found for user. Please add one in Credentials Manager.",
- toolIntegrationType)
- }
- }
-
- // Inject the credential_id if we found one
- if credentialID != "" {
- record.Arguments["credential_id"] = credentialID
- log.Printf("🔐 [AGENT-BLOCK] Auto-injected credential_id=%s for tool=%s (type=%s)",
- credentialID, record.Name, toolIntegrationType)
- }
- }
-
- // Inject image provider config for generate_image tool
- if record.Name == "generate_image" {
- imageProviderService := services.GetImageProviderService()
- provider := imageProviderService.GetProvider()
- if provider != nil {
- record.Arguments[tools.ImageProviderConfigKey] = &tools.ImageProviderConfig{
- Name: provider.Name,
- BaseURL: provider.BaseURL,
- APIKey: provider.APIKey,
- DefaultModel: provider.DefaultModel,
- }
- log.Printf("🎨 [AGENT-BLOCK] Injected image provider: %s (model: %s)", provider.Name, provider.DefaultModel)
- } else {
- log.Printf("⚠️ [AGENT-BLOCK] No image provider configured for generate_image tool")
- }
- }
-
- log.Printf("🔧 [AGENT-BLOCK] Executing tool: %s with args: %+v", record.Name, record.Arguments)
-
- // Execute the tool
- result, err := e.toolRegistry.Execute(record.Name, record.Arguments)
- if err != nil {
- record.Error = err.Error()
- log.Printf("❌ [AGENT-BLOCK] Tool %s failed: %v", record.Name, err)
- } else {
- record.Result = result
- log.Printf("✅ [AGENT-BLOCK] Tool %s succeeded (result length: %d)", record.Name, len(result))
- }
-
- record.Duration = time.Since(startTime).Milliseconds()
-
- // Clean up internal keys from Arguments before storing
- // These are injected for tool execution but should not be serialized
- delete(record.Arguments, tools.CredentialResolverKey)
- delete(record.Arguments, tools.UserIDKey)
- delete(record.Arguments, tools.ImageProviderConfigKey)
-
- return record
-}
-
-// getToolName extracts the tool name from a tool call
-func (e *AgentBlockExecutor) getToolName(toolCall map[string]any) string {
- if fn, ok := toolCall["function"].(map[string]any); ok {
- if name, ok := fn["name"].(string); ok {
- return name
- }
- }
- return ""
-}
-
-// parseAndValidateOutput parses the LLM response and validates against schema
-func (e *AgentBlockExecutor) parseAndValidateOutput(
- content string,
- schema *models.JSONSchema,
- strict bool,
-) (map[string]any, error) {
- log.Printf("📋 [VALIDATE] parseAndValidateOutput called, schema=%v, strict=%v", schema != nil, strict)
-
- // If no schema provided, try to parse as JSON or return as-is
- if schema == nil {
- log.Printf("📋 [VALIDATE] No schema provided, skipping validation")
- // Try to parse as JSON
- var output map[string]any
- if err := json.Unmarshal([]byte(content), &output); err == nil {
- return output, nil
- }
-
- // Return content as "response" field
- return map[string]any{
- "response": content,
- }, nil
- }
-
- // Extract JSON from content (handle markdown code blocks)
- jsonContent := extractJSON(content)
-
- // Parse JSON - support both objects {} and arrays []
- var output any
- if err := json.Unmarshal([]byte(jsonContent), &output); err != nil {
- if strict {
- return nil, fmt.Errorf("failed to parse output as JSON: %w", err)
- }
- // Non-strict: return content as-is with error
- return map[string]any{
- "response": content,
- "_parseError": err.Error(),
- }, nil
- }
-
- // Validate against schema (basic validation)
- if err := e.validateSchema(output, schema); err != nil {
- log.Printf("❌ [VALIDATE] Schema validation FAILED: %v", err)
- if strict {
- return nil, fmt.Errorf("output validation failed: %w", err)
- }
- // Non-strict: return with validation error at TOP LEVEL for retry loop detection
- log.Printf("⚠️ [AGENT-EXEC] Validation warning (non-strict): %v", err)
- return map[string]any{
- "response": content,
- "data": output,
- "_validationError": err.Error(), // TOP LEVEL for retry loop
- }, nil
- }
-
- log.Printf("✅ [VALIDATE] Schema validation PASSED")
- // Return parsed data as response so downstream blocks can access fields via {{block.response.field}}
- // Also include raw JSON string for debugging
- return map[string]any{
- "response": output, // Parsed JSON object - allows {{block.response.field}} access
- "data": output, // Alias for response
- "rawResponse": content, // Raw JSON string for debugging
- }, nil
-}
-
-// extractJSON extracts JSON from content (handles markdown code blocks)
-func extractJSON(content string) string {
- content = strings.TrimSpace(content)
-
- // Check for markdown JSON code block
- jsonBlockRegex := regexp.MustCompile("```(?:json)?\\s*([\\s\\S]*?)```")
- if matches := jsonBlockRegex.FindStringSubmatch(content); len(matches) > 1 {
- return strings.TrimSpace(matches[1])
- }
-
- // Try to find JSON object or array
- start := strings.IndexAny(content, "{[")
- if start == -1 {
- return content
- }
-
- // Find matching closing bracket
- openBracket := content[start]
- closeBracket := byte('}')
- if openBracket == '[' {
- closeBracket = ']'
- }
-
- depth := 0
- for i := start; i < len(content); i++ {
- if content[i] == openBracket {
- depth++
- } else if content[i] == closeBracket {
- depth--
- if depth == 0 {
- return content[start : i+1]
- }
- }
- }
-
- return content[start:]
-}
-
-// validateSchema performs basic JSON schema validation - supports both objects and arrays
-func (e *AgentBlockExecutor) validateSchema(data any, schema *models.JSONSchema) error {
- if schema == nil {
- return nil
- }
-
- // Handle based on schema type
- if schema.Type == "object" {
- return e.validateObjectSchema(data, schema)
- } else if schema.Type == "array" {
- return e.validateArraySchema(data, schema)
- }
-
- // If no type specified, infer from data
- if schema.Type == "" {
- if _, isMap := data.(map[string]any); isMap {
- return e.validateObjectSchema(data, schema)
- }
- if _, isSlice := data.([]any); isSlice {
- return e.validateArraySchema(data, schema)
- }
- }
-
- return nil
-}
-
-// validateObjectSchema validates object (map) data against schema
-func (e *AgentBlockExecutor) validateObjectSchema(data any, schema *models.JSONSchema) error {
- dataMap, ok := data.(map[string]any)
- if !ok {
- return fmt.Errorf("schema expects object but got %T", data)
- }
-
- // Check required fields
- for _, required := range schema.Required {
- if _, ok := dataMap[required]; !ok {
- return fmt.Errorf("missing required field: %s", required)
- }
- }
-
- // Validate property types (basic)
- for key, propSchema := range schema.Properties {
- if value, ok := dataMap[key]; ok {
- if err := e.validateValue(value, propSchema); err != nil {
- return fmt.Errorf("field %s: %w", key, err)
- }
- }
- }
-
- return nil
-}
-
-// validateArraySchema validates array data against schema
-func (e *AgentBlockExecutor) validateArraySchema(data any, schema *models.JSONSchema) error {
- dataArray, ok := data.([]any)
- if !ok {
- return fmt.Errorf("schema expects array but got %T", data)
- }
-
- // If no items schema, we can't validate items
- if schema.Items == nil {
- return nil
- }
-
- // Validate each item against the items schema
- for i, item := range dataArray {
- if err := e.validateValue(item, schema.Items); err != nil {
- return fmt.Errorf("array[%d]: %w", i, err)
- }
- }
-
- return nil
-}
-
-// validateValue validates a single value against a schema
-func (e *AgentBlockExecutor) validateValue(value any, schema *models.JSONSchema) error {
- if schema == nil {
- return nil
- }
-
- switch schema.Type {
- case "string":
- if _, ok := value.(string); !ok {
- return fmt.Errorf("expected string, got %T", value)
- }
- case "number", "integer":
- switch value.(type) {
- case float64, int, int64:
- // OK
- default:
- return fmt.Errorf("expected number, got %T", value)
- }
- case "boolean":
- if _, ok := value.(bool); !ok {
- return fmt.Errorf("expected boolean, got %T", value)
- }
- case "array":
- if _, ok := value.([]interface{}); !ok {
- return fmt.Errorf("expected array, got %T", value)
- }
- case "object":
- if _, ok := value.(map[string]interface{}); !ok {
- return fmt.Errorf("expected object, got %T", value)
- }
- }
-
- return nil
-}
-
-// findCredentialForIntegrationType finds the first credential matching the integration type
-// from the list of credential IDs configured for the block.
-func findCredentialForIntegrationType(credentialIDs []string, integrationType string, resolver tools.CredentialResolver) string {
- for _, credID := range credentialIDs {
- cred, err := resolver(credID)
- if err != nil {
- // Skip invalid credentials
- continue
- }
- if cred.IntegrationType == integrationType {
- return credID
- }
- }
- return ""
-}
diff --git a/backend/internal/execution/block_checker.go b/backend/internal/execution/block_checker.go
deleted file mode 100644
index e72b751a..00000000
--- a/backend/internal/execution/block_checker.go
+++ /dev/null
@@ -1,574 +0,0 @@
-package execution
-
-import (
- "bytes"
- "claraverse/internal/models"
- "claraverse/internal/services"
- "context"
- "encoding/json"
- "fmt"
- "io"
- "log"
- "net/http"
- "strings"
- "time"
-)
-
-// BlockCheckResult represents the structured output from the block completion checker
-type BlockCheckResult struct {
- Passed bool `json:"passed"` // true if block accomplished its job
- Reason string `json:"reason"` // explanation of why it passed or failed
- ActualOutput string `json:"actual_output"` // truncated actual output for debugging (populated by checker)
-}
-
-// BlockChecker validates whether a block actually accomplished its intended job
-// This prevents workflows from continuing when a block technically "completed"
-// but didn't actually do what it was supposed to do (e.g., tool errors, timeouts)
-type BlockChecker struct {
- providerService *services.ProviderService
- httpClient *http.Client
-}
-
-// NewBlockChecker creates a new block checker
-func NewBlockChecker(providerService *services.ProviderService) *BlockChecker {
- return &BlockChecker{
- providerService: providerService,
- httpClient: &http.Client{
- Timeout: 30 * time.Second,
- },
- }
-}
-
-// CheckBlockCompletion validates if a block actually accomplished its intended task
-// Parameters:
-// - ctx: context for cancellation
-// - workflowGoal: the overall workflow objective (from user's request)
-// - block: the block that just executed
-// - blockInput: what was passed to the block
-// - blockOutput: what the block produced
-// - modelID: the model to use for checking (should be a fast, cheap model)
-//
-// Returns:
-// - BlockCheckResult with passed/failed status and reason
-// - error if the check itself failed
-func (c *BlockChecker) CheckBlockCompletion(
- ctx context.Context,
- workflowGoal string,
- block models.Block,
- blockInput map[string]any,
- blockOutput map[string]any,
- modelID string,
-) (*BlockCheckResult, error) {
- log.Printf("🔍 [BLOCK-CHECKER] Checking completion for block '%s' (type: %s)", block.Name, block.Type)
-
- // Skip checking for Start blocks (variable type with read operation)
- if block.Type == "variable" {
- log.Printf("⏭️ [BLOCK-CHECKER] Skipping Start block '%s'", block.Name)
- return &BlockCheckResult{Passed: true, Reason: "Start block - no validation needed"}, nil
- }
-
- // Build the validation prompt
- prompt := c.buildValidationPrompt(workflowGoal, block, blockInput, blockOutput)
-
- // Get provider for the model
- provider, err := c.providerService.GetByModelID(modelID)
- if err != nil {
- log.Printf("⚠️ [BLOCK-CHECKER] Provider error, defaulting to passed: %v", err)
- return &BlockCheckResult{Passed: true, Reason: "Provider error - defaulting to passed"}, nil
- }
-
- // Build the request with structured output
- requestBody := map[string]interface{}{
- "model": modelID,
- "max_tokens": 200,
- "temperature": 0.1, // Low temperature for consistent validation
- "messages": []map[string]string{
- {
- "role": "system",
- "content": "You are a workflow block validator. Analyze if the block accomplished its intended job based on its purpose, input, and output. Be strict but fair - if there are clear errors, tool failures, or the output doesn't match the intent, it should fail.",
- },
- {
- "role": "user",
- "content": prompt,
- },
- },
- "response_format": map[string]interface{}{
- "type": "json_schema",
- "json_schema": map[string]interface{}{
- "name": "block_check_result",
- "strict": true,
- "schema": map[string]interface{}{
- "type": "object",
- "properties": map[string]interface{}{
- "passed": map[string]interface{}{
- "type": "boolean",
- "description": "true if the block accomplished its intended job, false otherwise",
- },
- "reason": map[string]interface{}{
- "type": "string",
- "description": "Brief explanation (1-2 sentences) of why the block passed or failed",
- },
- },
- "required": []string{"passed", "reason"},
- "additionalProperties": false,
- },
- },
- },
- }
-
- bodyBytes, err := json.Marshal(requestBody)
- if err != nil {
- return nil, fmt.Errorf("failed to marshal request: %w", err)
- }
-
- // Make the API call
- apiURL := fmt.Sprintf("%s/chat/completions", provider.BaseURL)
- req, err := http.NewRequestWithContext(ctx, "POST", apiURL, bytes.NewReader(bodyBytes))
- if err != nil {
- return nil, fmt.Errorf("failed to create request: %w", err)
- }
-
- req.Header.Set("Content-Type", "application/json")
- req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", provider.APIKey))
-
- resp, err := c.httpClient.Do(req)
- if err != nil {
- log.Printf("⚠️ [BLOCK-CHECKER] HTTP error, defaulting to passed: %v", err)
- return &BlockCheckResult{Passed: true, Reason: "HTTP error during check - defaulting to passed"}, nil
- }
- defer resp.Body.Close()
-
- if resp.StatusCode != http.StatusOK {
- body, _ := io.ReadAll(resp.Body)
- log.Printf("⚠️ [BLOCK-CHECKER] API error (status %d): %s, defaulting to passed", resp.StatusCode, string(body))
- return &BlockCheckResult{Passed: true, Reason: "API error during check - defaulting to passed"}, nil
- }
-
- // Parse response
- var apiResp struct {
- Choices []struct {
- Message struct {
- Content string `json:"content"`
- } `json:"message"`
- } `json:"choices"`
- }
-
- if err := json.NewDecoder(resp.Body).Decode(&apiResp); err != nil {
- log.Printf("⚠️ [BLOCK-CHECKER] Decode error, defaulting to passed: %v", err)
- return &BlockCheckResult{Passed: true, Reason: "Response decode error - defaulting to passed"}, nil
- }
-
- if len(apiResp.Choices) == 0 || apiResp.Choices[0].Message.Content == "" {
- log.Printf("⚠️ [BLOCK-CHECKER] Empty response, defaulting to passed")
- return &BlockCheckResult{Passed: true, Reason: "Empty response from checker - defaulting to passed"}, nil
- }
-
- // Parse the structured output
- var result BlockCheckResult
- if err := json.Unmarshal([]byte(apiResp.Choices[0].Message.Content), &result); err != nil {
- log.Printf("⚠️ [BLOCK-CHECKER] JSON parse error, defaulting to passed: %v", err)
- return &BlockCheckResult{Passed: true, Reason: "JSON parse error - defaulting to passed"}, nil
- }
-
- // Always populate ActualOutput with a summary of what the block produced
- // This helps with debugging when the block fails
- result.ActualOutput = c.summarizeOutputForError(blockOutput)
-
- if result.Passed {
- log.Printf("✅ [BLOCK-CHECKER] Block '%s' PASSED: %s", block.Name, result.Reason)
- } else {
- log.Printf("❌ [BLOCK-CHECKER] Block '%s' FAILED: %s\n Actual Output: %s", block.Name, result.Reason, result.ActualOutput)
- }
-
- return &result, nil
-}
-
-// buildValidationPrompt creates the prompt for block validation
-func (c *BlockChecker) buildValidationPrompt(
- workflowGoal string,
- block models.Block,
- blockInput map[string]any,
- blockOutput map[string]any,
-) string {
- // Extract key information from output
- outputSummary := c.summarizeOutput(blockOutput)
-
- // Check for obvious failures
- hasError := false
- errorMessages := []string{}
-
- // Check for tool errors
- if toolCalls, ok := blockOutput["toolCalls"].([]interface{}); ok {
- for _, tc := range toolCalls {
- if tcMap, ok := tc.(map[string]interface{}); ok {
- if errMsg, ok := tcMap["error"].(string); ok && errMsg != "" {
- hasError = true
- errorMessages = append(errorMessages, fmt.Sprintf("Tool '%s' error: %s", tcMap["name"], errMsg))
- }
- }
- }
- }
-
- // Check for timeout
- if timedOut, ok := blockOutput["timedOut"].(bool); ok && timedOut {
- hasError = true
- errorMessages = append(errorMessages, "Block timed out before completing")
- }
-
- // Check for parse errors
- if parseErr, ok := blockOutput["_parseError"].(string); ok && parseErr != "" {
- hasError = true
- errorMessages = append(errorMessages, fmt.Sprintf("Parse error: %s", parseErr))
- }
-
- // Check for aggregated tool errors
- if toolErr, ok := blockOutput["_toolError"].(string); ok && toolErr != "" {
- hasError = true
- errorMessages = append(errorMessages, fmt.Sprintf("Tool error: %s", toolErr))
- }
-
- // Check for empty response - handle both string and object types
- hasResponse := false
- if respStr, ok := blockOutput["response"].(string); ok && respStr != "" {
- hasResponse = true
- } else if respObj, ok := blockOutput["response"].(map[string]any); ok && len(respObj) > 0 {
- hasResponse = true
- }
- if !hasResponse && block.Type == "llm_inference" {
- hasError = true
- errorMessages = append(errorMessages, "Block produced no response")
- }
-
- // Build the prompt - include current date so the model knows what year it is
- currentDate := time.Now().Format("January 2, 2006")
- prompt := fmt.Sprintf(`## IMPORTANT: CURRENT DATE
-**Today's Date:** %s
-(This is the actual current date. Do NOT assume dates in the output are in the future.)
-
-## WORKFLOW CONTEXT
-**Overall Goal:** %s
-
-## BLOCK BEING VALIDATED
-**Block Name:** %s
-**Block Type:** %s
-**Block Description:** %s
-
-## BLOCK INPUT (what it received)
-%s
-
-## BLOCK OUTPUT (what it produced)
-%s
-`,
- currentDate,
- workflowGoal,
- block.Name,
- block.Type,
- block.Description,
- c.formatForPrompt(blockInput),
- outputSummary,
- )
-
- // Add error context if any
- if hasError {
- prompt += fmt.Sprintf(`
-## DETECTED ISSUES
-The following issues were detected in the block output:
-%s
-
-`, c.formatErrors(errorMessages))
- }
-
- prompt += `
-## YOUR TASK
-Analyze if this block accomplished its intended job within the workflow.
-
-CRITICAL DISTINCTION - External Failures vs Block Failures:
-- **External failures** (API rate limits, service unavailable, network errors): The block DID ITS JOB correctly by calling the right tool with correct parameters. The failure is EXTERNAL. Mark as PASSED if the block handled it gracefully (explained the error, provided fallback info).
-- **Block failures** (wrong tool called, missing required data, timeout, empty response): The block FAILED to do its job.
-
-Consider:
-1. Did the block call the correct tool(s) with appropriate parameters?
-2. If a tool returned an external error (rate limit, auth error, service down), did the block handle it gracefully?
-3. Does the response acknowledge and explain what happened?
-4. Is there meaningful information that downstream blocks can use (even if just error context)?
-
-IMPORTANT:
-- External API errors (429 rate limit, 503 service unavailable, etc.) are NOT the block's fault - PASS if handled gracefully
-- Parse errors are formatting issues, not functional failures - PASS if the response content is meaningful
-- If the block called the right tool and got an external error, it PASSED (the tool worked, the API didn't)
-- Only FAIL if: block timed out, produced no response, called wrong tools, or completely failed to attempt its task
-
-Return your judgment as JSON with "passed" (boolean) and "reason" (brief explanation).`
-
- return prompt
-}
-
-// summarizeOutput creates a readable summary of block output for the prompt
-func (c *BlockChecker) summarizeOutput(output map[string]any) string {
- summary := ""
-
- // Response text - handle both string and object types
- if resp, ok := output["response"].(string); ok && resp != "" {
- // Truncate long responses
- if len(resp) > 500 {
- resp = resp[:500] + "... [truncated]"
- }
- summary += fmt.Sprintf("**Response:** %s\n\n", resp)
- } else if respObj, ok := output["response"].(map[string]any); ok && len(respObj) > 0 {
- // Response is a structured object (from schema validation)
- respJSON, err := json.Marshal(respObj)
- if err == nil {
- respStr := string(respJSON)
- if len(respStr) > 500 {
- respStr = respStr[:500] + "... [truncated]"
- }
- summary += fmt.Sprintf("**Response (structured):** %s\n\n", respStr)
- }
- }
-
- // Timeout status
- if timedOut, ok := output["timedOut"].(bool); ok && timedOut {
- summary += "**Status:** TIMED OUT\n\n"
- }
-
- // Iterations
- if iterations, ok := output["iterations"].(int); ok {
- summary += fmt.Sprintf("**Iterations:** %d\n\n", iterations)
- } else if iterations, ok := output["iterations"].(float64); ok {
- summary += fmt.Sprintf("**Iterations:** %.0f\n\n", iterations)
- }
-
- // Tool calls summary
- if toolCalls, ok := output["toolCalls"].([]interface{}); ok && len(toolCalls) > 0 {
- summary += "**Tool Calls:**\n"
- errorCount := 0
- successCount := 0
- for i, tc := range toolCalls {
- if i >= 5 {
- summary += fmt.Sprintf(" ... and %d more tool calls\n", len(toolCalls)-5)
- break
- }
- if tcMap, ok := tc.(map[string]interface{}); ok {
- name, _ := tcMap["name"].(string)
- errMsg, hasErr := tcMap["error"].(string)
- if hasErr && errMsg != "" {
- errorCount++
- summary += fmt.Sprintf(" - %s: ❌ ERROR: %s\n", name, errMsg)
- } else {
- successCount++
- summary += fmt.Sprintf(" - %s: ✓ Success\n", name)
- }
- }
- }
- summary += fmt.Sprintf("\nTotal: %d successful, %d failed\n\n", successCount, errorCount)
- }
-
- // Parse errors
- if parseErr, ok := output["_parseError"].(string); ok && parseErr != "" {
- summary += fmt.Sprintf("**Parse Error:** %s\n\n", parseErr)
- }
-
- // Tool validation warning
- if warning, ok := output["_toolValidationWarning"].(string); ok && warning != "" {
- summary += fmt.Sprintf("**Warning:** %s\n\n", warning)
- }
-
- // Artifacts (images, charts, etc.)
- if artifacts, ok := output["artifacts"].([]interface{}); ok && len(artifacts) > 0 {
- summary += fmt.Sprintf("**Artifacts Generated:** %d artifact(s) created\n", len(artifacts))
- for i, art := range artifacts {
- if i >= 3 {
- summary += fmt.Sprintf(" ... and %d more artifacts\n", len(artifacts)-3)
- break
- }
- if artMap, ok := art.(map[string]interface{}); ok {
- artType, _ := artMap["type"].(string)
- artFormat, _ := artMap["format"].(string)
- if artType != "" {
- summary += fmt.Sprintf(" - Type: %s, Format: %s\n", artType, artFormat)
- } else {
- summary += fmt.Sprintf(" - Format: %s\n", artFormat)
- }
- }
- }
- summary += "\n"
- }
-
- // Generated files
- if files, ok := output["generatedFiles"].([]interface{}); ok && len(files) > 0 {
- summary += fmt.Sprintf("**Generated Files:** %d file(s) created\n", len(files))
- for i, file := range files {
- if i >= 3 {
- summary += fmt.Sprintf(" ... and %d more files\n", len(files)-3)
- break
- }
- if fileMap, ok := file.(map[string]interface{}); ok {
- fileName, _ := fileMap["name"].(string)
- fileType, _ := fileMap["type"].(string)
- summary += fmt.Sprintf(" - %s (type: %s)\n", fileName, fileType)
- }
- }
- summary += "\n"
- }
-
- if summary == "" {
- // Fallback: dump some of the output
- outputBytes, _ := json.MarshalIndent(output, "", " ")
- if len(outputBytes) > 1000 {
- summary = string(outputBytes[:1000]) + "... [truncated]"
- } else {
- summary = string(outputBytes)
- }
- }
-
- return summary
-}
-
-// formatForPrompt formats input data for the prompt
-func (c *BlockChecker) formatForPrompt(data map[string]any) string {
- // For input, just show key names and brief values
- if len(data) == 0 {
- return "(empty)"
- }
-
- result := ""
- for k, v := range data {
- // Skip internal fields
- if k[0] == '_' {
- continue
- }
- valStr := fmt.Sprintf("%v", v)
- if len(valStr) > 200 {
- valStr = valStr[:200] + "..."
- }
- result += fmt.Sprintf("- **%s:** %s\n", k, valStr)
- }
-
- if result == "" {
- return "(internal data only)"
- }
- return result
-}
-
-// formatErrors formats error messages for the prompt
-func (c *BlockChecker) formatErrors(errors []string) string {
- result := ""
- for _, err := range errors {
- result += fmt.Sprintf("- %s\n", err)
- }
- return result
-}
-
-// ShouldCheckBlock determines if a block should be validated
-// Some blocks (like Start/variable blocks) don't need validation
-func ShouldCheckBlock(block models.Block) bool {
- // Skip Start blocks (variable type with read operation)
- if block.Type == "variable" {
- if op, ok := block.Config["operation"].(string); ok && op == "read" {
- return false
- }
- }
-
- // Only check LLM blocks (they're the ones that can fail in complex ways)
- return block.Type == "llm_inference"
-}
-
-// summarizeOutputForError creates a concise summary of block output for error messages
-// This helps developers understand what went wrong when a block fails validation
-func (c *BlockChecker) summarizeOutputForError(output map[string]any) string {
- var parts []string
-
- // Include the response (truncated) - handle both string and object types
- if resp, ok := output["response"].(string); ok && resp != "" {
- truncated := resp
- if len(truncated) > 300 {
- truncated = truncated[:300] + "..."
- }
- parts = append(parts, fmt.Sprintf("Response: %q", truncated))
- } else if respObj, ok := output["response"].(map[string]any); ok && len(respObj) > 0 {
- // Response is a structured object (from schema validation)
- respJSON, err := json.Marshal(respObj)
- if err == nil {
- truncated := string(respJSON)
- if len(truncated) > 300 {
- truncated = truncated[:300] + "..."
- }
- parts = append(parts, fmt.Sprintf("Response: %s", truncated))
- } else {
- parts = append(parts, fmt.Sprintf("Response: (object with %d keys)", len(respObj)))
- }
- } else {
- parts = append(parts, "Response: (empty)")
- }
-
- // Include parse error if present
- if parseErr, ok := output["_parseError"].(string); ok && parseErr != "" {
- parts = append(parts, fmt.Sprintf("Parse Error: %s", parseErr))
- }
-
- // Include tool validation warning if present
- if warning, ok := output["_toolValidationWarning"].(string); ok && warning != "" {
- parts = append(parts, fmt.Sprintf("Tool Warning: %s", warning))
- }
-
- // Summarize tool calls - handle both []models.ToolCallRecord and []interface{}
- toolCallsSummarized := false
- if toolCalls, ok := output["toolCalls"].([]models.ToolCallRecord); ok && len(toolCalls) > 0 {
- successCount := 0
- failCount := 0
- var failedTools []string
- for _, tc := range toolCalls {
- if tc.Error != "" {
- failCount++
- failedTools = append(failedTools, fmt.Sprintf("%s: %s", tc.Name, tc.Error))
- } else {
- successCount++
- }
- }
- parts = append(parts, fmt.Sprintf("Tools: %d called (%d success, %d failed)", len(toolCalls), successCount, failCount))
- if len(failedTools) > 0 && len(failedTools) <= 3 {
- for _, ft := range failedTools {
- parts = append(parts, fmt.Sprintf(" - %s", ft))
- }
- }
- toolCallsSummarized = true
- } else if toolCalls, ok := output["toolCalls"].([]interface{}); ok && len(toolCalls) > 0 {
- // Fallback for []interface{} type (e.g., from JSON unmarshaling)
- successCount := 0
- failCount := 0
- var failedTools []string
- for _, tc := range toolCalls {
- if tcMap, ok := tc.(map[string]interface{}); ok {
- name, _ := tcMap["name"].(string)
- if errMsg, hasErr := tcMap["error"].(string); hasErr && errMsg != "" {
- failCount++
- failedTools = append(failedTools, fmt.Sprintf("%s: %s", name, errMsg))
- } else {
- successCount++
- }
- }
- }
- parts = append(parts, fmt.Sprintf("Tools: %d called (%d success, %d failed)", len(toolCalls), successCount, failCount))
- if len(failedTools) > 0 && len(failedTools) <= 3 {
- for _, ft := range failedTools {
- parts = append(parts, fmt.Sprintf(" - %s", ft))
- }
- }
- toolCallsSummarized = true
- }
- if !toolCallsSummarized {
- parts = append(parts, "Tools: none called")
- }
-
- // Include structured data summary if present
- if data, ok := output["data"]; ok && data != nil {
- dataBytes, _ := json.Marshal(data)
- if len(dataBytes) > 200 {
- parts = append(parts, fmt.Sprintf("Data: %s...", string(dataBytes[:200])))
- } else if len(dataBytes) > 0 {
- parts = append(parts, fmt.Sprintf("Data: %s", string(dataBytes)))
- }
- }
-
- return strings.Join(parts, " | ")
-}
diff --git a/backend/internal/execution/engine.go b/backend/internal/execution/engine.go
deleted file mode 100644
index 7c2e6815..00000000
--- a/backend/internal/execution/engine.go
+++ /dev/null
@@ -1,830 +0,0 @@
-package execution
-
-import (
- "claraverse/internal/models"
- "claraverse/internal/services"
- "context"
- "fmt"
- "log"
- "sync"
- "time"
-)
-
-// WorkflowEngine executes workflows as DAGs with parallel execution
-type WorkflowEngine struct {
- registry *ExecutorRegistry
- blockChecker *BlockChecker
-}
-
-// NewWorkflowEngine creates a new workflow engine
-func NewWorkflowEngine(registry *ExecutorRegistry) *WorkflowEngine {
- return &WorkflowEngine{registry: registry}
-}
-
-// NewWorkflowEngineWithChecker creates a workflow engine with block completion checking
-func NewWorkflowEngineWithChecker(registry *ExecutorRegistry, providerService *services.ProviderService) *WorkflowEngine {
- return &WorkflowEngine{
- registry: registry,
- blockChecker: NewBlockChecker(providerService),
- }
-}
-
-// SetBlockChecker allows setting the block checker after creation
-func (e *WorkflowEngine) SetBlockChecker(checker *BlockChecker) {
- e.blockChecker = checker
-}
-
-// ExecutionResult contains the final result of a workflow execution
-type ExecutionResult struct {
- Status string `json:"status"` // completed, failed, partial
- Output map[string]any `json:"output"`
- BlockStates map[string]*models.BlockState `json:"block_states"`
- Error string `json:"error,omitempty"`
-}
-
-// ExecutionOptions contains optional settings for workflow execution
-type ExecutionOptions struct {
- // WorkflowGoal is the high-level objective of the workflow (used for block checking)
- WorkflowGoal string
- // CheckerModelID is the model to use for block completion checking
- // If empty, block checking is disabled
- CheckerModelID string
- // EnableBlockChecker enables/disables block completion validation
- EnableBlockChecker bool
-}
-
-// Execute runs a workflow and streams updates via the statusChan
-// This is the backwards-compatible version without block checking
-func (e *WorkflowEngine) Execute(
- ctx context.Context,
- workflow *models.Workflow,
- input map[string]any,
- statusChan chan<- models.ExecutionUpdate,
-) (*ExecutionResult, error) {
- return e.ExecuteWithOptions(ctx, workflow, input, statusChan, nil)
-}
-
-// ExecuteWithOptions runs a workflow with optional block completion checking
-func (e *WorkflowEngine) ExecuteWithOptions(
- ctx context.Context,
- workflow *models.Workflow,
- input map[string]any,
- statusChan chan<- models.ExecutionUpdate,
- options *ExecutionOptions,
-) (*ExecutionResult, error) {
- log.Printf("🚀 [ENGINE] Starting workflow execution with %d blocks", len(workflow.Blocks))
-
- // Build block index
- blockIndex := make(map[string]models.Block)
- for _, block := range workflow.Blocks {
- blockIndex[block.ID] = block
- }
-
- // Build dependency graph
- // dependencies[blockID] = list of block IDs that must complete before this block
- dependencies := make(map[string][]string)
- // dependents[blockID] = list of block IDs that depend on this block
- dependents := make(map[string][]string)
-
- for _, block := range workflow.Blocks {
- dependencies[block.ID] = []string{}
- dependents[block.ID] = []string{}
- }
-
- for _, conn := range workflow.Connections {
- // conn.SourceBlockID -> conn.TargetBlockID
- dependencies[conn.TargetBlockID] = append(dependencies[conn.TargetBlockID], conn.SourceBlockID)
- dependents[conn.SourceBlockID] = append(dependents[conn.SourceBlockID], conn.TargetBlockID)
- }
-
- // Find start blocks (no dependencies)
- var startBlocks []string
- for blockID, deps := range dependencies {
- if len(deps) == 0 {
- startBlocks = append(startBlocks, blockID)
- }
- }
-
- if len(startBlocks) == 0 && len(workflow.Blocks) > 0 {
- return nil, fmt.Errorf("workflow has no start blocks (circular dependency?)")
- }
-
- log.Printf("📊 [ENGINE] Found %d start blocks: %v", len(startBlocks), startBlocks)
-
- // Initialize block states and outputs
- blockStates := make(map[string]*models.BlockState)
- blockOutputs := make(map[string]map[string]any)
- var statesMu sync.RWMutex
-
- for _, block := range workflow.Blocks {
- blockStates[block.ID] = &models.BlockState{
- Status: "pending",
- }
- }
-
- // Initialize with workflow variables and input
- globalInputs := make(map[string]any)
- log.Printf("🔍 [ENGINE] Workflow input received: %+v", input)
-
- // First, set workflow variable defaults
- for _, variable := range workflow.Variables {
- if variable.DefaultValue != nil {
- globalInputs[variable.Name] = variable.DefaultValue
- log.Printf("🔍 [ENGINE] Added workflow variable default: %s = %v", variable.Name, variable.DefaultValue)
- }
- }
-
- // Then, override with execution input (takes precedence over defaults)
- for k, v := range input {
- globalInputs[k] = v
- log.Printf("🔍 [ENGINE] Added/overrode from execution input: %s = %v", k, v)
- }
-
- // Extract workflow-level model override from Start block
- for _, block := range workflow.Blocks {
- if block.Type == "variable" {
- if op, ok := block.Config["operation"].(string); ok && op == "read" {
- if varName, ok := block.Config["variableName"].(string); ok && varName == "input" {
- // This is the Start block - check for workflowModelId
- if modelID, ok := block.Config["workflowModelId"].(string); ok && modelID != "" {
- globalInputs["_workflowModelId"] = modelID
- log.Printf("🎯 [ENGINE] Using workflow model override: %s", modelID)
- }
- }
- }
- }
- }
-
- // Track completed blocks for dependency resolution
- completedBlocks := make(map[string]bool)
- failedBlocks := make(map[string]bool)
- var completedMu sync.Mutex
-
- // Error tracking
- var executionErrors []string
- var errorsMu sync.Mutex
-
- // WaitGroup for tracking all goroutines
- var wg sync.WaitGroup
-
- // Recursive function to execute a block and schedule dependents
- var executeBlock func(blockID string)
- executeBlock = func(blockID string) {
- block := blockIndex[blockID]
-
- // Update status to running
- statesMu.Lock()
- blockStates[blockID].Status = "running"
- blockStates[blockID].StartedAt = timePtr(time.Now())
- statesMu.Unlock()
-
- // Send status update (without inputs yet - will send after building them)
- statusChan <- models.ExecutionUpdate{
- Type: "execution_update",
- BlockID: blockID,
- Status: "running",
- }
-
- log.Printf("▶️ [ENGINE] Executing block '%s' (type: %s)", block.Name, block.Type)
-
- // Build inputs for this block from:
- // 1. Global inputs (workflow input + variables)
- // 2. Outputs from upstream blocks
- blockInputs := make(map[string]any)
- log.Printf("🔍 [ENGINE] Block '%s': globalInputs keys: %v", block.Name, getMapKeys(globalInputs))
- for k, v := range globalInputs {
- blockInputs[k] = v
- }
- log.Printf("🔍 [ENGINE] Block '%s': blockInputs after globalInputs: %v", block.Name, getMapKeys(blockInputs))
-
- // Make ALL completed block outputs available for template resolution
- // This allows blocks to reference any upstream block, not just directly connected ones
- // Example: Final block can use {{start.response}}, {{research-overview.response}}, etc.
- statesMu.RLock()
-
- essentialKeys := []string{
- "response", "data", "output", "value", "result",
- "artifacts", "toolResults", "tokens", "model",
- "iterations", "_parseError", "rawResponse",
- "generatedFiles", "toolCalls", "timedOut",
- }
-
- // Track which block is directly connected (for flattening priority)
- directlyConnectedBlockID := ""
- for _, conn := range workflow.Connections {
- if conn.TargetBlockID == blockID {
- directlyConnectedBlockID = conn.SourceBlockID
- break
- }
- }
-
- // Add ALL completed block outputs (for template access like {{block-name.response}})
- for completedBlockID, output := range blockOutputs {
- sourceBlock, exists := blockIndex[completedBlockID]
- if !exists {
- continue
- }
-
- // Create clean output (only essential keys)
- cleanOutput := make(map[string]any)
- for _, key := range essentialKeys {
- if val, exists := output[key]; exists {
- cleanOutput[key] = val
- }
- }
-
- // Store under normalizedId (e.g., "research-overview")
- if sourceBlock.NormalizedID != "" {
- blockInputs[sourceBlock.NormalizedID] = cleanOutput
- }
-
- // Also store under block ID if different
- if sourceBlock.ID != "" && sourceBlock.ID != sourceBlock.NormalizedID {
- blockInputs[sourceBlock.ID] = cleanOutput
- }
- }
-
- // Log available block references
- log.Printf("🔗 [ENGINE] Block '%s' can access %d upstream blocks", block.Name, len(blockOutputs))
-
- // Flatten essential keys from DIRECTLY CONNECTED block only (for {{response}} shorthand)
- if directlyConnectedBlockID != "" {
- if output, ok := blockOutputs[directlyConnectedBlockID]; ok {
- for _, key := range essentialKeys {
- if val, exists := output[key]; exists {
- blockInputs[key] = val
- }
- }
- log.Printf("🔗 [ENGINE] Flattened keys from directly connected block '%s'", blockIndex[directlyConnectedBlockID].Name)
- }
- }
-
- statesMu.RUnlock()
-
- // Store the available inputs in BlockState for debugging
- statesMu.Lock()
- blockStates[blockID].Inputs = blockInputs
- statesMu.Unlock()
-
- log.Printf("🔍 [ENGINE] Block '%s': stored %d input keys for debugging: %v", block.Name, len(blockInputs), getMapKeys(blockInputs))
-
- // Send updated status with inputs for debugging
- statusChan <- models.ExecutionUpdate{
- Type: "execution_update",
- BlockID: blockID,
- Status: "running",
- Inputs: blockInputs,
- }
-
- // Get executor for this block type
- executor, execErr := e.registry.Get(block.Type)
- if execErr != nil {
- handleBlockError(blockID, block.Name, execErr, blockStates, &statesMu, statusChan, &executionErrors, &errorsMu)
- completedMu.Lock()
- failedBlocks[blockID] = true
- completedMu.Unlock()
- return
- }
-
- // Create timeout context
- // Default: 30s for most blocks, 120s for LLM blocks (they need more time for API calls)
- timeout := 30 * time.Second
- if block.Type == "llm_inference" {
- timeout = 120 * time.Second // LLM blocks get 2 minutes by default
- }
- // User-specified timeout can override, but LLM blocks get at least 120s
- if block.Timeout > 0 {
- userTimeout := time.Duration(block.Timeout) * time.Second
- if block.Type == "llm_inference" && userTimeout < 120*time.Second {
- // LLM blocks need at least 120s for reasoning/streaming
- timeout = 120 * time.Second
- } else {
- timeout = userTimeout
- }
- }
- blockCtx, cancel := context.WithTimeout(ctx, timeout)
- defer cancel()
-
- // Execute the block
- output, execErr := executor.Execute(blockCtx, block, blockInputs)
- if execErr != nil {
- handleBlockError(blockID, block.Name, execErr, blockStates, &statesMu, statusChan, &executionErrors, &errorsMu)
- completedMu.Lock()
- failedBlocks[blockID] = true
- completedMu.Unlock()
- return
- }
-
- // Block Completion Check: Validate if block actually accomplished its job
- // This catches cases where a block "completed" but didn't actually succeed
- // (e.g., repeated tool errors, timeouts, empty responses)
- if options != nil && options.EnableBlockChecker && e.blockChecker != nil && ShouldCheckBlock(block) {
- log.Printf("🔍 [ENGINE] Running block completion check for '%s'", block.Name)
-
- checkerModelID := options.CheckerModelID
- if checkerModelID == "" {
- // Default to a fast model for checking
- checkerModelID = "gpt-4.1"
- }
-
- checkResult, checkErr := e.blockChecker.CheckBlockCompletion(
- ctx,
- options.WorkflowGoal,
- block,
- blockInputs,
- output,
- checkerModelID,
- )
-
- if checkErr != nil {
- log.Printf("⚠️ [ENGINE] Block checker error (continuing): %v", checkErr)
- } else if !checkResult.Passed {
- // Block failed the completion check - treat as failure
- log.Printf("❌ [ENGINE] Block '%s' failed completion check: %s\n Actual Output: %s", block.Name, checkResult.Reason, checkResult.ActualOutput)
-
- // Add check failure info to output for visibility
- output["_blockCheckFailed"] = true
- output["_blockCheckReason"] = checkResult.Reason
- output["_blockActualOutput"] = checkResult.ActualOutput
-
- checkError := fmt.Errorf("block did not accomplish its job: %s\n\nActual Output: %s", checkResult.Reason, checkResult.ActualOutput)
- handleBlockError(blockID, block.Name, checkError, blockStates, &statesMu, statusChan, &executionErrors, &errorsMu)
- completedMu.Lock()
- failedBlocks[blockID] = true
- completedMu.Unlock()
- return
- } else {
- log.Printf("✓ [ENGINE] Block '%s' passed completion check: %s", block.Name, checkResult.Reason)
- }
- }
-
- // Store output and mark completed
- statesMu.Lock()
- blockOutputs[blockID] = output
- blockStates[blockID].Status = "completed"
- blockStates[blockID].CompletedAt = timePtr(time.Now())
- blockStates[blockID].Outputs = output
- statesMu.Unlock()
-
- // Send completion update with inputs for debugging
- statusChan <- models.ExecutionUpdate{
- Type: "execution_update",
- BlockID: blockID,
- Status: "completed",
- Inputs: blockInputs,
- Output: output,
- }
-
- log.Printf("✅ [ENGINE] Block '%s' completed", block.Name)
-
- // Mark as completed and check dependents
- completedMu.Lock()
- completedBlocks[blockID] = true
-
- // Check if any dependent blocks can now run
- for _, depBlockID := range dependents[blockID] {
- canRun := true
- for _, reqBlockID := range dependencies[depBlockID] {
- if !completedBlocks[reqBlockID] {
- // Check if the required block failed - if so, we can't run
- if failedBlocks[reqBlockID] {
- canRun = false
- break
- }
- // Required block hasn't completed yet
- canRun = false
- break
- }
- }
- if canRun {
- // Queue this block for execution
- wg.Add(1)
- go func(bid string) {
- defer wg.Done()
- executeBlock(bid)
- }(depBlockID)
- }
- }
- completedMu.Unlock()
- }
-
- // Start execution with start blocks
- for _, blockID := range startBlocks {
- wg.Add(1)
- go func(bid string) {
- defer wg.Done()
- executeBlock(bid)
- }(blockID)
- }
-
- // Wait for all blocks to complete
- wg.Wait()
-
- // Determine final status
- finalStatus := "completed"
- var failedBlockIDs []string
- var completedCount, failedCount int
-
- statesMu.RLock()
- for blockID, state := range blockStates {
- if state.Status == "completed" {
- completedCount++
- } else if state.Status == "failed" {
- failedCount++
- failedBlockIDs = append(failedBlockIDs, blockID)
- }
- }
- statesMu.RUnlock()
-
- if failedCount > 0 {
- if completedCount > 0 {
- finalStatus = "partial"
- } else {
- finalStatus = "failed"
- }
- }
-
- // Collect final output from terminal blocks (blocks with no dependents)
- finalOutput := make(map[string]any)
- statesMu.RLock()
- for blockID, deps := range dependents {
- if len(deps) == 0 {
- if output, ok := blockOutputs[blockID]; ok {
- block := blockIndex[blockID]
- finalOutput[block.Name] = output
- }
- }
- }
- statesMu.RUnlock()
-
- // Build error message if any
- var errorMsg string
- errorsMu.Lock()
- if len(executionErrors) > 0 {
- errorMsg = fmt.Sprintf("%d block(s) failed: %v", len(executionErrors), executionErrors)
- }
- errorsMu.Unlock()
-
- log.Printf("🏁 [ENGINE] Workflow execution %s: %d completed, %d failed",
- finalStatus, completedCount, failedCount)
-
- return &ExecutionResult{
- Status: finalStatus,
- Output: finalOutput,
- BlockStates: blockStates,
- Error: errorMsg,
- }, nil
-}
-
-// handleBlockError handles block execution errors with classification for debugging
-func handleBlockError(
- blockID, blockName string,
- err error,
- blockStates map[string]*models.BlockState,
- statesMu *sync.RWMutex,
- statusChan chan<- models.ExecutionUpdate,
- executionErrors *[]string,
- errorsMu *sync.Mutex,
-) {
- // Try to extract error classification for better debugging
- var errorType string
- var retryable bool
-
- if execErr, ok := err.(*ExecutionError); ok {
- errorType = execErr.Category.String()
- retryable = execErr.Retryable
- log.Printf("❌ [ENGINE] Block '%s' failed: %v [type=%s, retryable=%v]", blockName, err, errorType, retryable)
- } else {
- errorType = "unknown"
- retryable = false
- log.Printf("❌ [ENGINE] Block '%s' failed: %v", blockName, err)
- }
-
- statesMu.Lock()
- blockStates[blockID].Status = "failed"
- blockStates[blockID].CompletedAt = timePtr(time.Now())
- blockStates[blockID].Error = err.Error()
- statesMu.Unlock()
-
- // Include error classification in status update for frontend visibility
- statusChan <- models.ExecutionUpdate{
- Type: "execution_update",
- BlockID: blockID,
- Status: "failed",
- Error: err.Error(),
- Output: map[string]any{
- "errorType": errorType,
- "retryable": retryable,
- },
- }
-
- errorsMu.Lock()
- *executionErrors = append(*executionErrors, fmt.Sprintf("%s: %s", blockName, err.Error()))
- errorsMu.Unlock()
-}
-
-// timePtr returns a pointer to a time.Time
-func timePtr(t time.Time) *time.Time {
- return &t
-}
-
-// getMapKeys returns the keys of a map as a slice
-func getMapKeys(m map[string]any) []string {
- keys := make([]string, 0, len(m))
- for k := range m {
- keys = append(keys, k)
- }
- return keys
-}
-
-// BuildAPIResponse converts an ExecutionResult into a clean, structured API response
-// This provides a standardized output format for API consumers
-func (e *WorkflowEngine) BuildAPIResponse(
- result *ExecutionResult,
- workflow *models.Workflow,
- executionID string,
- durationMs int64,
-) *models.ExecutionAPIResponse {
- response := &models.ExecutionAPIResponse{
- Status: result.Status,
- Artifacts: []models.APIArtifact{},
- Files: []models.APIFile{},
- Blocks: make(map[string]models.APIBlockOutput),
- Metadata: models.ExecutionMetadata{
- ExecutionID: executionID,
- DurationMs: durationMs,
- },
- Error: result.Error,
- }
-
- // Build block index for lookups
- blockIndex := make(map[string]models.Block)
- for _, block := range workflow.Blocks {
- blockIndex[block.ID] = block
- }
-
- // Track totals
- var totalTokens int
- var blocksExecuted, blocksFailed int
-
- // Process block states
- for blockID, state := range result.BlockStates {
- block, exists := blockIndex[blockID]
- if !exists {
- continue
- }
-
- // Create clean block output
- blockOutput := models.APIBlockOutput{
- Name: block.Name,
- Type: block.Type,
- Status: state.Status,
- }
-
- if state.Status == "completed" {
- blocksExecuted++
- } else if state.Status == "failed" {
- blocksFailed++
- blockOutput.Error = state.Error
- }
-
- // Extract response text from outputs
- if state.Outputs != nil {
- // Primary response
- if resp, ok := state.Outputs["response"].(string); ok {
- blockOutput.Response = resp
- }
-
- // Extract tokens (for metadata, but don't expose)
- if tokens, ok := state.Outputs["tokens"].(map[string]any); ok {
- if total, ok := tokens["total"].(int); ok {
- totalTokens += total
- } else if total, ok := tokens["total"].(float64); ok {
- totalTokens += int(total)
- }
- }
-
- // Calculate duration from timestamps
- if state.StartedAt != nil && state.CompletedAt != nil {
- blockOutput.DurationMs = state.CompletedAt.Sub(*state.StartedAt).Milliseconds()
- }
-
- // Extract structured data - filter all outputs except response
- cleanData := make(map[string]any)
- for k, v := range state.Outputs {
- // Skip internal fields and the response (already extracted)
- if !isInternalField(k) && k != "response" {
- // Also check nested output object
- if k == "output" {
- if outputMap, ok := v.(map[string]any); ok {
- for ok, ov := range outputMap {
- if !isInternalField(ok) && ok != "response" {
- cleanData[ok] = ov
- }
- }
- }
- } else {
- cleanData[k] = v
- }
- }
- }
- if len(cleanData) > 0 {
- blockOutput.Data = cleanData
- }
-
- // Extract artifacts from this block
- artifacts := extractArtifactsFromBlockOutput(state.Outputs, block.Name)
- response.Artifacts = append(response.Artifacts, artifacts...)
-
- // Extract files from this block
- files := extractFilesFromBlockOutput(state.Outputs, block.Name)
- response.Files = append(response.Files, files...)
- }
-
- response.Blocks[block.ID] = blockOutput
- }
-
- // Set metadata
- response.Metadata.TotalTokens = totalTokens
- response.Metadata.BlocksExecuted = blocksExecuted
- response.Metadata.BlocksFailed = blocksFailed
- if workflow != nil {
- response.Metadata.WorkflowVersion = workflow.Version
- }
-
- // Extract the primary result and structured data from terminal blocks
- response.Result, response.Data = extractPrimaryResultAndData(result.Output, result.BlockStates)
-
- log.Printf("📦 [ENGINE] Built API response: status=%s, result_length=%d, has_data=%v, artifacts=%d, files=%d",
- response.Status, len(response.Result), response.Data != nil, len(response.Artifacts), len(response.Files))
-
- return response
-}
-
-// extractPrimaryResultAndData gets the main text result AND structured data from the workflow output
-// For structured output blocks, the "data" field contains the parsed JSON which we return separately
-func extractPrimaryResultAndData(output map[string]any, blockStates map[string]*models.BlockState) (string, any) {
- // First, try to get from the final output (terminal blocks)
- for blockName, blockOutput := range output {
- if blockData, ok := blockOutput.(map[string]any); ok {
- var resultStr string
- var structuredData any
-
- // Look for response field (the text/JSON string)
- if resp, ok := blockData["response"].(string); ok && resp != "" {
- resultStr = resp
- log.Printf("📝 [ENGINE] Extracted primary result from block '%s' (%d chars)", blockName, len(resp))
- } else if resp, ok := blockData["rawResponse"].(string); ok && resp != "" {
- // Fallback to rawResponse
- resultStr = resp
- }
-
- // Look for structured data field (parsed JSON from structured output blocks)
- // This is populated when outputFormat="json" and the response was successfully parsed
- if data, ok := blockData["data"]; ok && data != nil {
- structuredData = data
- log.Printf("📊 [ENGINE] Extracted structured data from block '%s'", blockName)
- }
-
- if resultStr != "" {
- return resultStr, structuredData
- }
- }
- }
-
- // Fallback: find the last completed block with a response
- var lastResponse string
- var lastData any
- for _, state := range blockStates {
- if state.Status == "completed" && state.Outputs != nil {
- if resp, ok := state.Outputs["response"].(string); ok && resp != "" {
- lastResponse = resp
- }
- if data, ok := state.Outputs["data"]; ok && data != nil {
- lastData = data
- }
- }
- }
-
- return lastResponse, lastData
-}
-
-// extractArtifactsFromBlockOutput extracts artifacts from a block's output
-func extractArtifactsFromBlockOutput(outputs map[string]any, blockName string) []models.APIArtifact {
- var artifacts []models.APIArtifact
-
- // Check for artifacts array
- if rawArtifacts, ok := outputs["artifacts"]; ok {
- switch arts := rawArtifacts.(type) {
- case []any:
- for _, a := range arts {
- if artMap, ok := a.(map[string]any); ok {
- artifact := models.APIArtifact{
- SourceBlock: blockName,
- }
- if t, ok := artMap["type"].(string); ok {
- artifact.Type = t
- }
- if f, ok := artMap["format"].(string); ok {
- artifact.Format = f
- }
- if d, ok := artMap["data"].(string); ok {
- artifact.Data = d
- }
- if t, ok := artMap["title"].(string); ok {
- artifact.Title = t
- }
- if artifact.Data != "" && len(artifact.Data) > 100 {
- artifacts = append(artifacts, artifact)
- }
- }
- }
- }
- }
-
- return artifacts
-}
-
-// extractFilesFromBlockOutput extracts generated files from a block's output
-func extractFilesFromBlockOutput(outputs map[string]any, blockName string) []models.APIFile {
- var files []models.APIFile
-
- // Check for generatedFiles array
- if rawFiles, ok := outputs["generatedFiles"]; ok {
- switch fs := rawFiles.(type) {
- case []any:
- for _, f := range fs {
- if fileMap, ok := f.(map[string]any); ok {
- file := models.APIFile{
- SourceBlock: blockName,
- }
- if id, ok := fileMap["file_id"].(string); ok {
- file.FileID = id
- }
- if fn, ok := fileMap["filename"].(string); ok {
- file.Filename = fn
- }
- if url, ok := fileMap["download_url"].(string); ok {
- file.DownloadURL = url
- }
- if mt, ok := fileMap["mime_type"].(string); ok {
- file.MimeType = mt
- }
- if sz, ok := fileMap["size"].(float64); ok {
- file.Size = int64(sz)
- }
- if file.FileID != "" || file.DownloadURL != "" {
- files = append(files, file)
- }
- }
- }
- }
- }
-
- // Also check for single file reference
- if fileURL, ok := outputs["file_url"].(string); ok && fileURL != "" {
- file := models.APIFile{
- DownloadURL: fileURL,
- SourceBlock: blockName,
- }
- if fn, ok := outputs["file_name"].(string); ok {
- file.Filename = fn
- }
- files = append(files, file)
- }
-
- return files
-}
-
-// isInternalField checks if a field name is internal and should be hidden from API response
-func isInternalField(key string) bool {
- // Any field starting with _ or __ is internal
- if len(key) > 0 && key[0] == '_' {
- return true
- }
-
- internalFields := map[string]bool{
- // Response duplicates
- "rawResponse": true,
- "output": true, // Duplicate of response
-
- // Execution internals
- "tokens": true,
- "toolCalls": true,
- "iterations": true,
- "model": true, // Internal model ID - never expose
-
- // Already extracted separately
- "artifacts": true,
- "generatedFiles": true,
- "file_url": true,
- "file_name": true,
-
- // Passthrough noise
- "start": true,
- "input": true, // Passthrough from workflow input
- "value": true, // Duplicate of input
- "timedOut": true,
- }
- return internalFields[key]
-}
\ No newline at end of file
diff --git a/backend/internal/execution/errors.go b/backend/internal/execution/errors.go
deleted file mode 100644
index 05127382..00000000
--- a/backend/internal/execution/errors.go
+++ /dev/null
@@ -1,312 +0,0 @@
-package execution
-
-import (
- "fmt"
- "math"
- "math/rand"
- "net/http"
- "strings"
- "time"
-)
-
-// ErrorCategory classifies errors for retry decisions
-type ErrorCategory int
-
-const (
- // ErrorCategoryUnknown - unclassified error, default to not retryable
- ErrorCategoryUnknown ErrorCategory = iota
-
- // ErrorCategoryTransient - temporary failures that may succeed on retry
- // Examples: timeout, rate limit (429), server error (5xx), network error
- ErrorCategoryTransient
-
- // ErrorCategoryPermanent - errors that will not succeed on retry
- // Examples: auth error (401/403), bad request (400), parse error
- ErrorCategoryPermanent
-
- // ErrorCategoryValidation - business logic validation failures
- // Examples: tool not called, required tool missing
- ErrorCategoryValidation
-)
-
-// String returns a human-readable category name
-func (c ErrorCategory) String() string {
- switch c {
- case ErrorCategoryTransient:
- return "transient"
- case ErrorCategoryPermanent:
- return "permanent"
- case ErrorCategoryValidation:
- return "validation"
- default:
- return "unknown"
- }
-}
-
-// ExecutionError wraps errors with classification for retry logic
-type ExecutionError struct {
- Category ErrorCategory
- Message string
- StatusCode int // HTTP status code if applicable
- Retryable bool // Explicit retryable flag
- RetryAfter int // Seconds to wait before retry (from Retry-After header)
- Cause error // Original error
-}
-
-func (e *ExecutionError) Error() string {
- if e.StatusCode > 0 {
- return fmt.Sprintf("[%d] %s", e.StatusCode, e.Message)
- }
- return e.Message
-}
-
-func (e *ExecutionError) Unwrap() error {
- return e.Cause
-}
-
-// IsRetryable determines if an error should be retried
-func (e *ExecutionError) IsRetryable() bool {
- return e.Retryable
-}
-
-// ClassifyHTTPError classifies an HTTP response error
-func ClassifyHTTPError(statusCode int, body string) *ExecutionError {
- err := &ExecutionError{
- StatusCode: statusCode,
- Message: fmt.Sprintf("HTTP %d: %s", statusCode, truncateString(body, 200)),
- }
-
- switch {
- // Rate limiting - always retryable
- case statusCode == http.StatusTooManyRequests:
- err.Category = ErrorCategoryTransient
- err.Retryable = true
- err.RetryAfter = 60 // Default 60 seconds for rate limiting
-
- // Server errors - retryable
- case statusCode >= 500 && statusCode < 600:
- err.Category = ErrorCategoryTransient
- err.Retryable = true
-
- // Request timeout - retryable
- case statusCode == http.StatusRequestTimeout:
- err.Category = ErrorCategoryTransient
- err.Retryable = true
-
- // Gateway errors - retryable
- case statusCode == http.StatusBadGateway ||
- statusCode == http.StatusServiceUnavailable ||
- statusCode == http.StatusGatewayTimeout:
- err.Category = ErrorCategoryTransient
- err.Retryable = true
-
- // Auth errors - NOT retryable
- case statusCode == http.StatusUnauthorized || statusCode == http.StatusForbidden:
- err.Category = ErrorCategoryPermanent
- err.Retryable = false
-
- // Bad request - NOT retryable
- case statusCode == http.StatusBadRequest:
- err.Category = ErrorCategoryPermanent
- err.Retryable = false
-
- // Not found - NOT retryable
- case statusCode == http.StatusNotFound:
- err.Category = ErrorCategoryPermanent
- err.Retryable = false
-
- // Unprocessable entity - NOT retryable
- case statusCode == http.StatusUnprocessableEntity:
- err.Category = ErrorCategoryPermanent
- err.Retryable = false
-
- default:
- err.Category = ErrorCategoryUnknown
- err.Retryable = false
- }
-
- return err
-}
-
-// ClassifyError classifies a general error
-func ClassifyError(err error) *ExecutionError {
- if err == nil {
- return nil
- }
-
- // If already an ExecutionError, return as-is
- if execErr, ok := err.(*ExecutionError); ok {
- return execErr
- }
-
- errStr := err.Error()
-
- // Context timeout/cancellation
- if strings.Contains(errStr, "context deadline exceeded") ||
- strings.Contains(errStr, "context canceled") {
- return &ExecutionError{
- Category: ErrorCategoryTransient,
- Message: "Request timed out",
- Retryable: true,
- Cause: err,
- }
- }
-
- // Network errors - connection issues
- if strings.Contains(errStr, "connection refused") ||
- strings.Contains(errStr, "connection reset") ||
- strings.Contains(errStr, "no such host") ||
- strings.Contains(errStr, "network is unreachable") ||
- strings.Contains(errStr, "i/o timeout") ||
- strings.Contains(errStr, "EOF") {
- return &ExecutionError{
- Category: ErrorCategoryTransient,
- Message: fmt.Sprintf("Network error: %s", truncateString(errStr, 100)),
- Retryable: true,
- Cause: err,
- }
- }
-
- // TLS errors - usually permanent
- if strings.Contains(errStr, "certificate") ||
- strings.Contains(errStr, "tls:") ||
- strings.Contains(errStr, "x509:") {
- return &ExecutionError{
- Category: ErrorCategoryPermanent,
- Message: "TLS/Certificate error",
- Retryable: false,
- Cause: err,
- }
- }
-
- // DNS errors - may be transient
- if strings.Contains(errStr, "no such host") ||
- strings.Contains(errStr, "dns") {
- return &ExecutionError{
- Category: ErrorCategoryTransient,
- Message: "DNS resolution error",
- Retryable: true,
- Cause: err,
- }
- }
-
- // Default: unknown, not retryable
- return &ExecutionError{
- Category: ErrorCategoryUnknown,
- Message: truncateString(errStr, 200),
- Retryable: false,
- Cause: err,
- }
-}
-
-// BackoffCalculator computes retry delays with exponential backoff and jitter
-type BackoffCalculator struct {
- initialDelay time.Duration
- maxDelay time.Duration
- multiplier float64
- jitterPercent int
-}
-
-// NewBackoffCalculator creates a calculator with specified parameters
-func NewBackoffCalculator(initialDelayMs, maxDelayMs int, multiplier float64, jitterPercent int) *BackoffCalculator {
- // Apply defaults if not specified
- if initialDelayMs <= 0 {
- initialDelayMs = 1000 // 1 second default
- }
- if maxDelayMs <= 0 {
- maxDelayMs = 30000 // 30 seconds default
- }
- if multiplier <= 0 {
- multiplier = 2.0 // Double each time
- }
- if jitterPercent < 0 {
- jitterPercent = 20 // 20% jitter default
- }
-
- return &BackoffCalculator{
- initialDelay: time.Duration(initialDelayMs) * time.Millisecond,
- maxDelay: time.Duration(maxDelayMs) * time.Millisecond,
- multiplier: multiplier,
- jitterPercent: jitterPercent,
- }
-}
-
-// NextDelay calculates the delay for the given attempt number (0-indexed)
-func (b *BackoffCalculator) NextDelay(attempt int) time.Duration {
- if attempt < 0 {
- attempt = 0
- }
-
- // Calculate exponential delay: initialDelay * (multiplier ^ attempt)
- delay := float64(b.initialDelay) * math.Pow(b.multiplier, float64(attempt))
-
- // Cap at max delay
- if delay > float64(b.maxDelay) {
- delay = float64(b.maxDelay)
- }
-
- // Add jitter to prevent thundering herd
- if b.jitterPercent > 0 {
- jitterRange := delay * float64(b.jitterPercent) / 100.0
- jitter := (rand.Float64()*2 - 1) * jitterRange // -jitterRange to +jitterRange
- delay += jitter
- }
-
- // Ensure non-negative
- if delay < 0 {
- delay = float64(b.initialDelay)
- }
-
- return time.Duration(delay)
-}
-
-// ShouldRetry determines if the error type should be retried based on policy
-func ShouldRetry(err *ExecutionError, retryOn []string) bool {
- if err == nil || !err.Retryable {
- return false
- }
-
- // If no specific retry types configured, retry all retryable errors
- if len(retryOn) == 0 {
- return err.Retryable
- }
-
- // Map error to type string
- errorType := getErrorType(err)
-
- // Check if this error type is in the retry list
- for _, retryType := range retryOn {
- if retryType == errorType || retryType == "all_transient" {
- return true
- }
- }
-
- return false
-}
-
-// getErrorType maps an ExecutionError to a type string for retry matching
-func getErrorType(err *ExecutionError) string {
- if err.StatusCode == 429 {
- return "rate_limit"
- }
- if err.StatusCode >= 500 {
- return "server_error"
- }
- if strings.Contains(strings.ToLower(err.Message), "timeout") ||
- strings.Contains(strings.ToLower(err.Message), "deadline exceeded") {
- return "timeout"
- }
- if strings.Contains(strings.ToLower(err.Message), "network") ||
- strings.Contains(strings.ToLower(err.Message), "connection") {
- return "network_error"
- }
- return "unknown"
-}
-
-// truncateString truncates a string to maxLen characters
-func truncateString(s string, maxLen int) string {
- if len(s) <= maxLen {
- return s
- }
- return s[:maxLen] + "..."
-}
diff --git a/backend/internal/execution/executor.go b/backend/internal/execution/executor.go
deleted file mode 100644
index a7789141..00000000
--- a/backend/internal/execution/executor.go
+++ /dev/null
@@ -1,58 +0,0 @@
-package execution
-
-import (
- "claraverse/internal/models"
- "claraverse/internal/services"
- "claraverse/internal/tools"
- "context"
- "fmt"
-)
-
-// BlockExecutor interface for all block types
-type BlockExecutor interface {
- Execute(ctx context.Context, block models.Block, inputs map[string]any) (map[string]any, error)
-}
-
-// ExecutorRegistry maps block types to executors
-type ExecutorRegistry struct {
- executors map[string]BlockExecutor
-}
-
-// NewExecutorRegistry creates a new executor registry with all block type executors
-// Hybrid Architecture: Supports variable, llm_inference, and code_block types.
-// - variable: Input/output data handling
-// - llm_inference: AI reasoning with tool access
-// - code_block: Direct tool execution (no LLM, faster & deterministic)
-func NewExecutorRegistry(
- chatService *services.ChatService,
- providerService *services.ProviderService,
- toolRegistry *tools.Registry,
- credentialService *services.CredentialService,
-) *ExecutorRegistry {
- return &ExecutorRegistry{
- executors: map[string]BlockExecutor{
- // Variable blocks handle input/output data
- "variable": NewVariableExecutor(),
- // LLM blocks handle all intelligent actions via tools
- // Tools available: search_web, scrape_web, send_webhook, send_discord_message, send_slack_message, etc.
- "llm_inference": NewAgentBlockExecutor(chatService, providerService, toolRegistry, credentialService),
- // Code blocks execute tools directly without LLM (faster, deterministic)
- // Use for mechanical tasks that don't need AI reasoning
- "code_block": NewToolExecutor(toolRegistry, credentialService),
- },
- }
-}
-
-// Get retrieves an executor for a block type
-func (r *ExecutorRegistry) Get(blockType string) (BlockExecutor, error) {
- exec, ok := r.executors[blockType]
- if !ok {
- return nil, fmt.Errorf("no executor registered for block type: %s", blockType)
- }
- return exec, nil
-}
-
-// Register adds a new executor for a block type
-func (r *ExecutorRegistry) Register(blockType string, executor BlockExecutor) {
- r.executors[blockType] = executor
-}
diff --git a/backend/internal/execution/format_to_schema.go b/backend/internal/execution/format_to_schema.go
deleted file mode 100644
index bf7e6154..00000000
--- a/backend/internal/execution/format_to_schema.go
+++ /dev/null
@@ -1,313 +0,0 @@
-package execution
-
-import (
- "claraverse/internal/models"
- "context"
- "encoding/json"
- "fmt"
- "log"
- "regexp"
- "strings"
-)
-
-// FormatInput represents the input data to be formatted
-type FormatInput struct {
- // RawData is the unstructured data from task execution (can be string, map, slice, etc.)
- RawData any
-
- // ToolResults contains results from tool executions (if any)
- ToolResults []map[string]any
-
- // LLMResponse is the final LLM text response (if any)
- LLMResponse string
-
- // Context provides additional context for formatting (e.g., original task description)
- Context string
-}
-
-// FormatOutput represents the result of schema formatting
-type FormatOutput struct {
- // Data is the validated, schema-compliant structured output
- Data map[string]any
-
- // RawJSON is the raw JSON string returned by the formatter
- RawJSON string
-
- // Model is the model ID used for formatting
- Model string
-
- // Tokens contains token usage information
- Tokens models.TokenUsage
-
- // Success indicates whether formatting succeeded
- Success bool
-
- // Error contains any error message if formatting failed
- Error string
-}
-
-// FormatToSchema formats the given input data into the specified JSON schema
-// This is a method on AgentBlockExecutor so it can reuse the existing LLM call infrastructure
-func (e *AgentBlockExecutor) FormatToSchema(
- ctx context.Context,
- input FormatInput,
- schema *models.JSONSchema,
- modelID string,
-) (*FormatOutput, error) {
- log.Printf("📐 [FORMAT-SCHEMA] Starting schema formatting with model=%s", modelID)
-
- if schema == nil {
- return nil, fmt.Errorf("schema is required for FormatToSchema")
- }
-
- // Resolve model using existing method
- provider, resolvedModelID, err := e.resolveModel(modelID)
- if err != nil {
- return nil, fmt.Errorf("failed to resolve model: %w", err)
- }
-
- log.Printf("📐 [FORMAT-SCHEMA] Resolved model %s -> %s (provider: %s)",
- modelID, resolvedModelID, provider.Name)
-
- // Build the formatting prompt
- systemPrompt, userPrompt := buildFormattingPrompts(input, schema)
-
- // Build messages
- messages := []map[string]any{
- {"role": "system", "content": systemPrompt},
- {"role": "user", "content": userPrompt},
- }
-
- // Make the LLM call with native structured output if supported
- // Use existing callLLMWithRetryAndSchema for consistency
- response, _, err := e.callLLMWithRetryAndSchema(ctx, provider, resolvedModelID, messages, nil, 0.1, nil, schema)
- if err != nil {
- return &FormatOutput{
- Success: false,
- Error: fmt.Sprintf("LLM call failed: %v", err),
- Model: resolvedModelID,
- }, nil
- }
-
- log.Printf("📐 [FORMAT-SCHEMA] LLM response received, length=%d chars", len(response.Content))
-
- // Parse and validate the output
- output, err := parseAndValidateSchema(response.Content, schema)
- if err != nil {
- log.Printf("⚠️ [FORMAT-SCHEMA] Validation failed: %v", err)
- return &FormatOutput{
- Success: false,
- Error: fmt.Sprintf("validation failed: %v", err),
- RawJSON: response.Content,
- Model: resolvedModelID,
- Tokens: models.TokenUsage{
- Input: response.InputTokens,
- Output: response.OutputTokens,
- },
- }, nil
- }
-
- log.Printf("✅ [FORMAT-SCHEMA] Successfully formatted data to schema")
-
- return &FormatOutput{
- Data: output,
- RawJSON: response.Content,
- Model: resolvedModelID,
- Tokens: models.TokenUsage{
- Input: response.InputTokens,
- Output: response.OutputTokens,
- },
- Success: true,
- }, nil
-}
-
-// buildFormattingPrompts creates the system and user prompts for schema formatting
-func buildFormattingPrompts(input FormatInput, schema *models.JSONSchema) (string, string) {
- // Build schema description
- schemaJSON, _ := json.MarshalIndent(schema, "", " ")
-
- // Build data description
- var dataBuilder strings.Builder
-
- // Add tool results if present
- if len(input.ToolResults) > 0 {
- dataBuilder.WriteString("## Tool Execution Results\n")
- for i, tr := range input.ToolResults {
- trJSON, _ := json.MarshalIndent(tr, "", " ")
- dataBuilder.WriteString(fmt.Sprintf("### Tool Result %d\n```json\n%s\n```\n\n", i+1, string(trJSON)))
- }
- }
-
- // Add LLM response if present
- if input.LLMResponse != "" {
- dataBuilder.WriteString("## LLM Response\n")
- dataBuilder.WriteString("```\n")
- dataBuilder.WriteString(input.LLMResponse)
- dataBuilder.WriteString("\n```\n\n")
- }
-
- // Add raw data if present and different from LLM response
- if input.RawData != nil {
- rawStr := fmt.Sprintf("%v", input.RawData)
- if rawStr != input.LLMResponse {
- dataBuilder.WriteString("## Additional Data\n")
- if rawJSON, err := json.MarshalIndent(input.RawData, "", " "); err == nil {
- dataBuilder.WriteString("```json\n")
- dataBuilder.WriteString(string(rawJSON))
- dataBuilder.WriteString("\n```\n\n")
- } else {
- dataBuilder.WriteString("```\n")
- dataBuilder.WriteString(rawStr)
- dataBuilder.WriteString("\n```\n\n")
- }
- }
- }
-
- systemPrompt := fmt.Sprintf(`You are a precise data formatter. Your ONLY task is to extract data from the provided sources and format it as JSON matching the required schema.
-
-## CRITICAL RULES
-1. Respond with ONLY valid JSON - no explanations, no markdown code blocks, no extra text
-2. The JSON must exactly match the required schema structure
-3. Extract ALL relevant data from the provided sources
-4. If data is missing for a required field, use reasonable defaults or null
-5. DO NOT invent or fabricate data - only use what's provided
-6. DO NOT include fields not in the schema
-
-## Required Output Schema
-%s
-
-## Data Fields Explanation
-%s`, string(schemaJSON), buildSchemaFieldsExplanation(schema))
-
- contextNote := ""
- if input.Context != "" {
- contextNote = fmt.Sprintf("\n\n## Context\n%s", input.Context)
- }
-
- userPrompt := fmt.Sprintf(`Format the following data into the required JSON schema.
-
-%s%s
-
-Respond with ONLY the JSON object, nothing else.`, dataBuilder.String(), contextNote)
-
- return systemPrompt, userPrompt
-}
-
-// buildSchemaFieldsExplanation creates a human-readable explanation of schema fields
-func buildSchemaFieldsExplanation(schema *models.JSONSchema) string {
- if schema == nil || schema.Properties == nil {
- return "No specific field requirements."
- }
-
- var builder strings.Builder
- for fieldName, fieldSchema := range schema.Properties {
- builder.WriteString(fmt.Sprintf("- **%s**: ", fieldName))
- if fieldSchema.Description != "" {
- builder.WriteString(fieldSchema.Description)
- } else if fieldSchema.Type != "" {
- builder.WriteString(fmt.Sprintf("(%s)", fieldSchema.Type))
- }
- builder.WriteString("\n")
- }
- return builder.String()
-}
-
-// parseAndValidateSchema parses JSON and validates against schema
-func parseAndValidateSchema(content string, schema *models.JSONSchema) (map[string]any, error) {
- // Extract JSON from content (handle markdown code blocks if any slipped through)
- jsonContent := extractJSONFromContent(content)
-
- // Parse JSON
- var output map[string]any
- if err := json.Unmarshal([]byte(jsonContent), &output); err != nil {
- return nil, fmt.Errorf("failed to parse JSON: %w", err)
- }
-
- // Validate required fields
- for _, required := range schema.Required {
- if _, exists := output[required]; !exists {
- return nil, fmt.Errorf("missing required field: %s", required)
- }
- }
-
- // Validate property types
- for propName, propSchema := range schema.Properties {
- val, exists := output[propName]
- if !exists {
- continue // Not required, skip
- }
-
- if err := validateFieldType(val, propSchema.Type); err != nil {
- return nil, fmt.Errorf("field %s: %w", propName, err)
- }
- }
-
- return output, nil
-}
-
-// extractJSONFromContent extracts JSON from content that may have markdown wrappers
-func extractJSONFromContent(content string) string {
- content = strings.TrimSpace(content)
-
- // Check for markdown JSON code block
- jsonBlockRegex := regexp.MustCompile("```(?:json)?\\s*([\\s\\S]*?)```")
- if matches := jsonBlockRegex.FindStringSubmatch(content); len(matches) > 1 {
- return strings.TrimSpace(matches[1])
- }
-
- // Try to find JSON object
- start := strings.Index(content, "{")
- if start == -1 {
- return content
- }
-
- // Find matching closing brace
- depth := 0
- for i := start; i < len(content); i++ {
- if content[i] == '{' {
- depth++
- } else if content[i] == '}' {
- depth--
- if depth == 0 {
- return content[start : i+1]
- }
- }
- }
-
- return content[start:]
-}
-
-// validateFieldType checks if a value matches the expected JSON schema type
-func validateFieldType(val any, expectedType string) error {
- if val == nil {
- return nil // null is valid for any type
- }
-
- switch expectedType {
- case "string":
- if _, ok := val.(string); !ok {
- return fmt.Errorf("expected string, got %T", val)
- }
- case "number", "integer":
- switch val.(type) {
- case float64, float32, int, int32, int64:
- // OK
- default:
- return fmt.Errorf("expected number, got %T", val)
- }
- case "boolean":
- if _, ok := val.(bool); !ok {
- return fmt.Errorf("expected boolean, got %T", val)
- }
- case "array":
- if _, ok := val.([]any); !ok {
- return fmt.Errorf("expected array, got %T", val)
- }
- case "object":
- if _, ok := val.(map[string]any); !ok {
- return fmt.Errorf("expected object, got %T", val)
- }
- }
-
- return nil
-}
diff --git a/backend/internal/execution/llm_executor.go b/backend/internal/execution/llm_executor.go
deleted file mode 100644
index 26bb613c..00000000
--- a/backend/internal/execution/llm_executor.go
+++ /dev/null
@@ -1,527 +0,0 @@
-package execution
-
-import (
- "bytes"
- "claraverse/internal/models"
- "claraverse/internal/services"
- "context"
- "encoding/json"
- "fmt"
- "io"
- "log"
- "net/http"
- "regexp"
- "strings"
- "time"
-)
-
-// LLMExecutor executes LLM inference blocks
-type LLMExecutor struct {
- chatService *services.ChatService
- providerService *services.ProviderService
- httpClient *http.Client
-}
-
-// NewLLMExecutor creates a new LLM executor
-func NewLLMExecutor(chatService *services.ChatService, providerService *services.ProviderService) *LLMExecutor {
- return &LLMExecutor{
- chatService: chatService,
- providerService: providerService,
- httpClient: &http.Client{
- Timeout: 120 * time.Second,
- },
- }
-}
-
-// getDefaultModel returns the first available model from database
-func (e *LLMExecutor) getDefaultModel() string {
- // Use chatService to get optimal text model
- provider, modelID, err := e.chatService.GetTextProviderWithModel()
- if err == nil && modelID != "" {
- log.Printf("🎯 [LLM-EXEC] Using dynamic default model: %s (provider: %s)", modelID, provider.Name)
- return modelID
- }
-
- // If that fails, try to get default provider with model
- provider, modelID, err = e.chatService.GetDefaultProviderWithModel()
- if err == nil && modelID != "" {
- log.Printf("🎯 [LLM-EXEC] Using fallback default model: %s", modelID)
- return modelID
- }
-
- // Last resort: return empty string (will cause error later if no model specified)
- log.Printf("⚠️ [LLM-EXEC] No default model available - LLM execution will require explicit model")
- return ""
-}
-
-// Execute runs an LLM inference block
-func (e *LLMExecutor) Execute(ctx context.Context, block models.Block, inputs map[string]any) (map[string]any, error) {
- config := block.Config
-
- // Get configuration - support both "model" and "modelId" field names
- modelID := getString(config, "model", "")
- if modelID == "" {
- modelID = getString(config, "modelId", "")
- }
-
- // Check for workflow-level model override (set in Start block)
- if workflowModelID, ok := inputs["_workflowModelId"].(string); ok && workflowModelID != "" {
- log.Printf("🎯 [LLM-EXEC] Block '%s': Using workflow model override: %s", block.Name, workflowModelID)
- modelID = workflowModelID
- }
-
- // Default to a sensible model if not specified
- if modelID == "" {
- modelID = e.getDefaultModel()
- if modelID != "" {
- log.Printf("⚠️ [LLM-EXEC] No model specified for block '%s', using default: %s", block.Name, modelID)
- } else {
- return nil, fmt.Errorf("no model specified and no default model available")
- }
- }
-
- // Support both "systemPrompt" and "system_prompt" field names
- systemPrompt := getString(config, "systemPrompt", "")
- if systemPrompt == "" {
- systemPrompt = getString(config, "system_prompt", "")
- }
-
- // Support "userPrompt", "userPromptTemplate", "user_prompt" field names
- userPromptTemplate := getString(config, "userPrompt", "")
- if userPromptTemplate == "" {
- userPromptTemplate = getString(config, "userPromptTemplate", "")
- }
- if userPromptTemplate == "" {
- userPromptTemplate = getString(config, "user_prompt", "")
- }
- temperature := getFloat(config, "temperature", 0.7)
-
- // Get structured output configuration
- outputFormat := getString(config, "outputFormat", "text")
- var outputSchema map[string]interface{}
- if schema, ok := config["outputSchema"].(map[string]interface{}); ok {
- outputSchema = schema
- }
-
- // Interpolate variables in prompts
- userPrompt := interpolateTemplate(userPromptTemplate, inputs)
- systemPrompt = interpolateTemplate(systemPrompt, inputs)
-
- log.Printf("🤖 [LLM-EXEC] Block '%s': model=%s, prompt_len=%d", block.Name, modelID, len(userPrompt))
-
- // Find provider for this model using multi-step resolution
- var provider *models.Provider
- var actualModelID string
- var err error
-
- // Step 1: Try direct lookup in models table
- provider, err = e.providerService.GetByModelID(modelID)
- if err == nil {
- actualModelID = modelID
- log.Printf("✅ [LLM-EXEC] Found model '%s' via direct lookup", modelID)
- } else {
- // Step 2: Try model alias resolution
- log.Printf("🔄 [LLM-EXEC] Model '%s' not found directly, trying alias resolution...", modelID)
- aliasProvider, aliasModel, found := e.chatService.ResolveModelAlias(modelID)
- if found {
- provider = aliasProvider
- actualModelID = aliasModel
- log.Printf("✅ [LLM-EXEC] Resolved alias '%s' -> '%s'", modelID, actualModelID)
- } else {
- // Step 3: Fallback to default provider with an actual model from the database
- log.Printf("⚠️ [LLM-EXEC] Model '%s' not found, using default provider with default model", modelID)
- defaultProvider, defaultModel, defaultErr := e.chatService.GetDefaultProviderWithModel()
- if defaultErr != nil {
- return nil, fmt.Errorf("failed to find provider for model %s and no default provider available: %w", modelID, defaultErr)
- }
- provider = defaultProvider
- actualModelID = defaultModel
- log.Printf("⚠️ [LLM-EXEC] Using default provider '%s' with model '%s'", provider.Name, actualModelID)
- }
- }
-
- // Use the resolved model ID
- modelID = actualModelID
-
- // Build request
- messages := []map[string]string{
- {"role": "user", "content": userPrompt},
- }
- if systemPrompt != "" {
- messages = append([]map[string]string{{"role": "system", "content": systemPrompt}}, messages...)
- }
-
- requestBody := map[string]interface{}{
- "model": modelID,
- "messages": messages,
- "temperature": temperature,
- "stream": false, // Non-streaming for block execution
- }
-
- // Add structured output if configured (provider-aware implementation)
- if outputFormat == "json" && outputSchema != nil {
- // Detect provider capability for strict schema support
- supportsStrictSchema := supportsStrictJSONSchema(provider.Name, provider.BaseURL)
-
- if supportsStrictSchema {
- // Full support: Use strict JSON schema mode (OpenAI, some OpenRouter models)
- requestBody["response_format"] = map[string]interface{}{
- "type": "json_schema",
- "json_schema": map[string]interface{}{
- "name": fmt.Sprintf("%s_output", block.NormalizedID),
- "strict": true,
- "schema": outputSchema,
- },
- }
- log.Printf("🎯 [LLM-EXEC] Block '%s': Using strict JSON schema mode", block.Name)
- } else {
- // Fallback: JSON mode + schema in system prompt
- requestBody["response_format"] = map[string]interface{}{
- "type": "json_object",
- }
-
- // Add schema to system prompt for better compliance
- schemaJSON, _ := json.Marshal(outputSchema)
- systemPrompt += fmt.Sprintf("\n\nIMPORTANT: Return your response as valid JSON matching this EXACT schema:\n%s\n\nDo not add any extra fields. Include all required fields.", string(schemaJSON))
-
- // Update messages with enhanced prompt
- messages = []map[string]string{
- {"role": "user", "content": userPrompt},
- }
- if systemPrompt != "" {
- messages = append([]map[string]string{{"role": "system", "content": systemPrompt}}, messages...)
- }
- requestBody["messages"] = messages
-
- log.Printf("⚠️ [LLM-EXEC] Block '%s': Using JSON mode with schema in prompt (provider fallback)", block.Name)
- }
- } else if outputFormat == "json" {
- // Fallback to basic JSON mode if no schema provided
- requestBody["response_format"] = map[string]interface{}{
- "type": "json_object",
- }
- log.Printf("🎯 [LLM-EXEC] Block '%s': Using basic JSON output mode", block.Name)
- }
-
- bodyBytes, err := json.Marshal(requestBody)
- if err != nil {
- return nil, fmt.Errorf("failed to marshal request: %w", err)
- }
-
- // Create request
- endpoint := strings.TrimSuffix(provider.BaseURL, "/") + "/chat/completions"
- req, err := http.NewRequestWithContext(ctx, "POST", endpoint, bytes.NewReader(bodyBytes))
- if err != nil {
- return nil, fmt.Errorf("failed to create request: %w", err)
- }
-
- req.Header.Set("Content-Type", "application/json")
- req.Header.Set("Authorization", "Bearer "+provider.APIKey)
-
- // Execute request
- resp, err := e.httpClient.Do(req)
- if err != nil {
- return nil, fmt.Errorf("LLM request failed: %w", err)
- }
- defer resp.Body.Close()
-
- if resp.StatusCode != http.StatusOK {
- body, _ := io.ReadAll(resp.Body)
- return nil, fmt.Errorf("LLM request failed with status %d: %s", resp.StatusCode, string(body))
- }
-
- // Parse response
- var result map[string]interface{}
- if err := json.NewDecoder(resp.Body).Decode(&result); err != nil {
- return nil, fmt.Errorf("failed to parse LLM response: %w", err)
- }
-
- // Extract content from OpenAI-style response
- content := ""
- if choices, ok := result["choices"].([]interface{}); ok && len(choices) > 0 {
- if choice, ok := choices[0].(map[string]interface{}); ok {
- if message, ok := choice["message"].(map[string]interface{}); ok {
- if c, ok := message["content"].(string); ok {
- content = c
- }
- }
- }
- }
-
- // Extract token usage
- var inputTokens, outputTokens int
- if usage, ok := result["usage"].(map[string]interface{}); ok {
- if pt, ok := usage["prompt_tokens"].(float64); ok {
- inputTokens = int(pt)
- }
- if ct, ok := usage["completion_tokens"].(float64); ok {
- outputTokens = int(ct)
- }
- }
-
- log.Printf("✅ [LLM-EXEC] Block '%s': completed, response_len=%d, tokens=%d/%d",
- block.Name, len(content), inputTokens, outputTokens)
-
- // Parse JSON output if structured output was requested
- if outputFormat == "json" {
- var parsedJSON map[string]interface{}
- if err := json.Unmarshal([]byte(content), &parsedJSON); err != nil {
- log.Printf("⚠️ [LLM-EXEC] Block '%s': Failed to parse JSON output: %v", block.Name, err)
- // Return raw content if JSON parsing fails
- return map[string]any{
- "response": content,
- "model": modelID,
- "tokens": map[string]int{
- "input": inputTokens,
- "output": outputTokens,
- },
- "parseError": err.Error(),
- }, nil
- }
-
- log.Printf("✅ [LLM-EXEC] Block '%s': Successfully parsed JSON output with %d keys", block.Name, len(parsedJSON))
-
- // Return both raw and parsed data
- return map[string]any{
- "response": content, // Raw JSON string (for debugging/logging)
- "data": parsedJSON, // Parsed JSON object (for downstream blocks)
- "model": modelID,
- "tokens": map[string]int{
- "input": inputTokens,
- "output": outputTokens,
- },
- }, nil
- }
-
- // Text output (default)
- return map[string]any{
- "response": content,
- "model": modelID,
- "tokens": map[string]int{
- "input": inputTokens,
- "output": outputTokens,
- },
- }, nil
-}
-
-// interpolateTemplate replaces {{variable}} placeholders with actual values
-func interpolateTemplate(template string, inputs map[string]any) string {
- if template == "" {
- return ""
- }
-
- // Match {{path.to.value}} or {{path[0].value}}
- re := regexp.MustCompile(`\{\{([^}]+)\}\}`)
-
- return re.ReplaceAllStringFunc(template, func(match string) string {
- // Extract the path (remove {{ and }})
- path := strings.TrimPrefix(strings.TrimSuffix(match, "}}"), "{{")
- path = strings.TrimSpace(path)
-
- // Debug logging
- log.Printf("🔍 [INTERPOLATE] Resolving '%s' from inputs: %+v", path, inputs)
-
- // Resolve the path in inputs
- value := resolvePath(inputs, path)
- if value == nil {
- log.Printf("⚠️ [INTERPOLATE] Failed to resolve '%s', keeping original", path)
- return match // Keep original if not found
- }
-
- log.Printf("✅ [INTERPOLATE] Resolved '%s' = %v", path, value)
-
- // Convert to string
- switch v := value.(type) {
- case string:
- return v
- case float64:
- if v == float64(int(v)) {
- return fmt.Sprintf("%d", int(v))
- }
- return fmt.Sprintf("%g", v)
- case int:
- return fmt.Sprintf("%d", v)
- case bool:
- return fmt.Sprintf("%t", v)
- default:
- // For complex types, JSON encode
- jsonBytes, err := json.Marshal(v)
- if err != nil {
- return match
- }
- return string(jsonBytes)
- }
- })
-}
-
-// interpolateMapValues recursively interpolates template strings in map values
-func interpolateMapValues(data map[string]any, inputs map[string]any) map[string]any {
- result := make(map[string]any)
-
- for key, value := range data {
- result[key] = interpolateValue(value, inputs)
- }
-
- return result
-}
-
-// interpolateValue interpolates a single value (handles strings, maps, slices)
-func interpolateValue(value any, inputs map[string]any) any {
- switch v := value.(type) {
- case string:
- // Interpolate string templates
- return interpolateTemplate(v, inputs)
- case map[string]any:
- // Recursively interpolate nested maps
- return interpolateMapValues(v, inputs)
- case []any:
- // Interpolate each element in slices
- result := make([]any, len(v))
- for i, elem := range v {
- result[i] = interpolateValue(elem, inputs)
- }
- return result
- default:
- // Return as-is for other types
- return value
- }
-}
-
-// resolvePath resolves a dot-notation path in a map
-// Supports: input.field, input.nested.field, input[0].field
-// Uses exact string matching with normalized block IDs
-func resolvePath(data map[string]any, path string) any {
- parts := strings.Split(path, ".")
- var current any = data
-
- for _, part := range parts {
- if current == nil {
- return nil
- }
-
- // Check for array access: field[0]
- if idx := strings.Index(part, "["); idx != -1 {
- fieldName := part[:idx]
- indexStr := strings.TrimSuffix(part[idx+1:], "]")
-
- // Get the field
- if m, ok := current.(map[string]any); ok {
- current = m[fieldName]
- } else {
- return nil
- }
-
- // Get the array element
- if arr, ok := current.([]any); ok {
- var index int
- fmt.Sscanf(indexStr, "%d", &index)
- if index >= 0 && index < len(arr) {
- current = arr[index]
- } else {
- return nil
- }
- } else {
- return nil
- }
- } else {
- // Simple field access - exact match only
- if m, ok := current.(map[string]any); ok {
- val, exists := m[part]
- if !exists {
- return nil
- }
- current = val
- } else {
- return nil
- }
- }
- }
-
- return current
-}
-
-// Helper functions for config access
-func getString(config map[string]any, key, defaultVal string) string {
- if v, ok := config[key]; ok {
- if s, ok := v.(string); ok {
- return s
- }
- }
- return defaultVal
-}
-
-func getFloat(config map[string]any, key string, defaultVal float64) float64 {
- if v, ok := config[key]; ok {
- switch f := v.(type) {
- case float64:
- return f
- case int:
- return float64(f)
- }
- }
- return defaultVal
-}
-
-func getMap(config map[string]any, key string) map[string]any {
- if v, ok := config[key]; ok {
- if m, ok := v.(map[string]any); ok {
- return m
- }
- }
- return nil
-}
-
-// supportsStrictJSONSchema determines if a provider supports OpenAI's strict JSON schema mode
-// Based on comprehensive testing results (Jan 2026) with 100% compliance validation
-func supportsStrictJSONSchema(providerName, baseURL string) bool {
- // Normalize provider name and base URL for comparison
- name := strings.ToLower(providerName)
- url := strings.ToLower(baseURL)
-
- // ✅ TIER 1: Proven 100% compliance with strict mode
-
- // OpenAI - 100% compliance (tested: gpt-4.1, gpt-4.1-mini)
- // Response time: 3.8-4.1s
- if strings.Contains(name, "openai") || strings.Contains(url, "api.openai.com") {
- return true
- }
-
- // Gemini via OpenRouter - 100% compliance, FASTEST (819ms-1.4s)
- // Models: gemini-3-flash-preview, gemini-2.5-flash-lite-preview
- if strings.Contains(url, "openrouter.ai") {
- // Enable strict mode for OpenRouter - Gemini models have proven 100% compliance
- return true
- }
-
- // ClaraVerse Cloud (private TEE) - Mixed results, use fallback for safety
- // ✅ 100% compliance: Kimi-K2-Thinking-TEE, MiMo-V2-Flash
- // ❌ 0% compliance: GLM-4.7-TEE (accepts strict mode but returns invalid JSON)
- // Decision: Use fallback mode to ensure consistency across all models
- if strings.Contains(url, "llm.chutes.ai") || strings.Contains(url, "chutes.ai") {
- return false // Use fallback mode with prompt-based schema
- }
-
- // ❌ TIER 2: Providers that claim support but fail compliance
-
- // Z.AI - Accepts strict mode but 0% compliance (returns invalid JSON)
- // Models tested: glm-4.5, glm-4.7 (both 0% compliance)
- if strings.Contains(name, "z.ai") || strings.Contains(url, "api.z.ai") {
- return false
- }
-
- // 0G AI - Mixed results, some models 0% compliance
- // Use fallback mode for reliability
- if strings.Contains(url, "13.235.83.18:4002") {
- return false
- }
-
- // Groq - supports json_object but not strict json_schema (as of Jan 2026)
- if strings.Contains(name, "groq") || strings.Contains(url, "groq.com") {
- return false
- }
-
- // Default: Conservative fallback for untested providers
- // Use JSON mode + prompt-based schema enforcement
- return false
-}
-
diff --git a/backend/internal/execution/tool_executor.go b/backend/internal/execution/tool_executor.go
deleted file mode 100644
index bf4e7376..00000000
--- a/backend/internal/execution/tool_executor.go
+++ /dev/null
@@ -1,229 +0,0 @@
-package execution
-
-import (
- "claraverse/internal/models"
- "claraverse/internal/services"
- "claraverse/internal/tools"
- "context"
- "encoding/json"
- "fmt"
- "log"
- "strings"
-)
-
-// ToolExecutor executes tool blocks using the tool registry
-// This executor runs tools directly without LLM involvement for faster, deterministic execution
-type ToolExecutor struct {
- registry *tools.Registry
- credentialService *services.CredentialService
-}
-
-// NewToolExecutor creates a new tool executor
-func NewToolExecutor(registry *tools.Registry, credentialService *services.CredentialService) *ToolExecutor {
- return &ToolExecutor{
- registry: registry,
- credentialService: credentialService,
- }
-}
-
-// deepInterpolate recursively interpolates {{...}} templates in nested structures
-func deepInterpolate(value interface{}, inputs map[string]any) interface{} {
- switch v := value.(type) {
- case string:
- // Handle string interpolation
- if strings.HasPrefix(v, "{{") && strings.HasSuffix(v, "}}") {
- resolvedPath := strings.TrimPrefix(v, "{{")
- resolvedPath = strings.TrimSuffix(resolvedPath, "}}")
- resolvedPath = strings.TrimSpace(resolvedPath)
- resolved := resolvePath(inputs, resolvedPath)
- if resolved != nil {
- return resolved
- }
- }
- return v
- case map[string]interface{}:
- // Handle nested maps
- result := make(map[string]interface{})
- for k, val := range v {
- result[k] = deepInterpolate(val, inputs)
- }
- return result
- case []interface{}:
- // Handle arrays
- result := make([]interface{}, len(v))
- for i, val := range v {
- result[i] = deepInterpolate(val, inputs)
- }
- return result
- default:
- // Return primitives as-is (numbers, booleans, nil)
- return v
- }
-}
-
-// Execute runs a tool block
-func (e *ToolExecutor) Execute(ctx context.Context, block models.Block, inputs map[string]any) (map[string]any, error) {
- config := block.Config
-
- toolName := getString(config, "toolName", "")
- if toolName == "" {
- return nil, fmt.Errorf("toolName is required for tool execution block")
- }
-
- log.Printf("🔧 [TOOL-EXEC] Block '%s': executing tool '%s'", block.Name, toolName)
-
- // Get tool from registry
- tool, exists := e.registry.Get(toolName)
- if !exists {
- return nil, fmt.Errorf("tool not found: %s", toolName)
- }
-
- // Map inputs to tool arguments based on argumentMapping config
- argMapping := getMap(config, "argumentMapping")
- args := make(map[string]interface{})
-
- log.Printf("🔧 [TOOL-EXEC] Block '%s' config: %+v", block.Name, config)
- log.Printf("🔧 [TOOL-EXEC] Block '%s' argumentMapping: %+v", block.Name, argMapping)
- log.Printf("🔧 [TOOL-EXEC] Block '%s' inputs keys: %v", block.Name, getInputKeys(inputs))
-
- if argMapping != nil {
- for argName, inputPath := range argMapping {
- // Use deep interpolation to handle nested structures
- interpolated := deepInterpolate(inputPath, inputs)
-
- // Log the interpolation result
- if pathStr, ok := inputPath.(string); ok && strings.HasPrefix(pathStr, "{{") {
- log.Printf("🔧 [TOOL-EXEC] Interpolated '%s': %v", argName, interpolated)
- } else if _, isMap := inputPath.(map[string]interface{}); isMap {
- log.Printf("🔧 [TOOL-EXEC] Deep interpolated object '%s'", argName)
- } else {
- log.Printf("🔧 [TOOL-EXEC] Using literal value for '%s': %v", argName, inputPath)
- }
-
- if interpolated != nil {
- args[argName] = interpolated
- }
- }
- } else {
- // If no argument mapping, pass all inputs as args
- for k, v := range inputs {
- // Skip internal fields
- if len(k) > 0 && k[0] == '_' {
- continue
- }
- args[k] = v
- }
- }
-
- // Extract userID from inputs for credential resolution (uses __user_id__ convention)
- userID, _ := inputs["__user_id__"].(string)
-
- // Inject credentials for tools that need them
- e.injectCredentials(ctx, toolName, args, userID, config)
-
- log.Printf("🔧 [TOOL-EXEC] Tool '%s' args: %v", toolName, args)
-
- // Execute tool
- result, err := tool.Execute(args)
-
- // Clean up internal keys from args (don't log them)
- delete(args, tools.CredentialResolverKey)
- delete(args, tools.UserIDKey)
-
- if err != nil {
- log.Printf("❌ [TOOL-EXEC] Tool '%s' failed: %v", toolName, err)
- return nil, fmt.Errorf("tool execution failed: %w", err)
- }
-
- log.Printf("✅ [TOOL-EXEC] Tool '%s' completed, result_len=%d", toolName, len(result))
-
- // Try to parse result as JSON for structured output
- var parsedResult any
- if err := json.Unmarshal([]byte(result), &parsedResult); err != nil {
- // Not JSON, use as string
- parsedResult = result
- }
-
- return map[string]any{
- "response": parsedResult, // Primary output key for consistency with other blocks
- "result": parsedResult, // Kept for backwards compatibility
- "data": parsedResult, // For structured data access
- "toolName": toolName,
- "raw": result,
- }, nil
-}
-
-// injectCredentials adds credential resolver and auto-discovers credentials for tools that need them
-func (e *ToolExecutor) injectCredentials(ctx context.Context, toolName string, args map[string]interface{}, userID string, config map[string]any) {
- if e.credentialService == nil || userID == "" {
- return
- }
-
- // Inject credential resolver for tools that need authentication
- // Cast to tools.CredentialResolver type for proper type assertion in credential_helper.go
- resolver := tools.CredentialResolver(e.credentialService.CreateCredentialResolver(userID))
- args[tools.CredentialResolverKey] = resolver
- args[tools.UserIDKey] = userID
-
- // Auto-inject credential_id for tools that need it
- toolIntegrationType := tools.GetIntegrationTypeForTool(toolName)
- if toolIntegrationType == "" {
- return
- }
-
- var credentialID string
-
- // First, try to find from explicitly configured credentials in block config
- if credentials, ok := config["credentials"].([]interface{}); ok && len(credentials) > 0 {
- for _, credID := range credentials {
- if credIDStr, ok := credID.(string); ok {
- cred, err := resolver(credIDStr)
- if err == nil && cred != nil && cred.IntegrationType == toolIntegrationType {
- credentialID = credIDStr
- log.Printf("🔐 [TOOL-EXEC] Found credential_id=%s from block config for tool=%s",
- credentialID, toolName)
- break
- }
- }
- }
- }
-
- // If no credential found in block config, try runtime auto-discovery from user's credentials
- if credentialID == "" {
- log.Printf("🔍 [TOOL-EXEC] No credentials in block config for tool=%s, trying runtime auto-discovery...", toolName)
- userCreds, err := e.credentialService.ListByUserAndType(ctx, userID, toolIntegrationType)
- if err != nil {
- log.Printf("⚠️ [TOOL-EXEC] Failed to fetch user credentials: %v", err)
- } else if len(userCreds) == 1 {
- // Exactly one credential of this type - auto-use it
- credentialID = userCreds[0].ID
- log.Printf("🔐 [TOOL-EXEC] Runtime auto-discovered single credential: %s (%s) for tool=%s",
- userCreds[0].Name, credentialID, toolName)
- } else if len(userCreds) > 1 {
- log.Printf("⚠️ [TOOL-EXEC] Multiple credentials (%d) found for %s - cannot auto-select. Configure in block settings.",
- len(userCreds), toolIntegrationType)
- } else {
- log.Printf("⚠️ [TOOL-EXEC] No %s credentials found for user. Please add one in Credentials Manager.",
- toolIntegrationType)
- }
- }
-
- // Inject the credential_id if we found one
- if credentialID != "" {
- args["credential_id"] = credentialID
- log.Printf("🔐 [TOOL-EXEC] Auto-injected credential_id=%s for tool=%s (type=%s)",
- credentialID, toolName, toolIntegrationType)
- }
-}
-
-// getInputKeys returns sorted keys from inputs map for logging
-func getInputKeys(inputs map[string]any) []string {
- keys := make([]string, 0, len(inputs))
- for k := range inputs {
- // Skip internal fields for cleaner logging
- if !strings.HasPrefix(k, "__") {
- keys = append(keys, k)
- }
- }
- return keys
-}
diff --git a/backend/internal/execution/variable_executor.go b/backend/internal/execution/variable_executor.go
deleted file mode 100644
index 73c7bffe..00000000
--- a/backend/internal/execution/variable_executor.go
+++ /dev/null
@@ -1,318 +0,0 @@
-package execution
-
-import (
- "claraverse/internal/filecache"
- "claraverse/internal/models"
- "claraverse/internal/security"
- "context"
- "fmt"
- "log"
- "os"
- "path/filepath"
- "strings"
- "time"
-)
-
-// FileReference represents a file that can be passed between workflow blocks
-type FileReference struct {
- FileID string `json:"file_id"`
- Filename string `json:"filename"`
- MimeType string `json:"mime_type"`
- Size int64 `json:"size"`
- Type string `json:"type"` // "image", "document", "audio", "data"
-}
-
-// isFileReference checks if a value is a file reference (map with file_id)
-func isFileReference(value any) bool {
- if m, ok := value.(map[string]any); ok {
- _, hasFileID := m["file_id"]
- return hasFileID
- }
- return false
-}
-
-// getFileType determines the file type category from MIME type
-func getFileType(mimeType string) string {
- switch {
- case strings.HasPrefix(mimeType, "image/"):
- return "image"
- case strings.HasPrefix(mimeType, "audio/"):
- return "audio"
- case mimeType == "application/pdf",
- mimeType == "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
- mimeType == "application/vnd.openxmlformats-officedocument.presentationml.presentation",
- mimeType == "application/msword":
- return "document"
- case mimeType == "application/json",
- mimeType == "text/csv",
- mimeType == "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
- strings.HasPrefix(mimeType, "text/"):
- return "data"
- default:
- return "data"
- }
-}
-
-// validateFileReference validates a file reference and enriches it with metadata
-func validateFileReference(value map[string]any, userID string) (*FileReference, error) {
- fileID, ok := value["file_id"].(string)
- if !ok || fileID == "" {
- return nil, fmt.Errorf("invalid file reference: missing file_id")
- }
-
- // SECURITY: Validate fileID to prevent path traversal attacks
- if err := security.ValidateFileID(fileID); err != nil {
- return nil, fmt.Errorf("invalid file reference: %w", err)
- }
-
- // Get file from cache service
- fileCacheService := filecache.GetService()
- file, found := fileCacheService.Get(fileID)
-
- // If not in cache, try to find on disk and restore cache entry
- if !found {
- log.Printf("⚠️ [VAR-EXEC] File %s not in cache, attempting disk recovery...", fileID)
-
- // Try to find the file on disk
- uploadDir := os.Getenv("UPLOAD_DIR")
- if uploadDir == "" {
- uploadDir = "./uploads"
- }
-
- // Try common extensions for data files
- extensions := []string{".csv", ".xlsx", ".xls", ".json", ".txt", ".png", ".jpg", ".jpeg", ""}
- var foundPath string
- var foundFilename string
-
- for _, ext := range extensions {
- testPath := filepath.Join(uploadDir, fileID+ext)
- if info, err := os.Stat(testPath); err == nil {
- foundPath = testPath
- foundFilename = fileID + ext
- log.Printf("✅ [VAR-EXEC] Found file on disk: %s (size: %d bytes)", testPath, info.Size())
-
- // Restore cache entry
- mimeType := getMimeTypeFromExtension(ext)
- cachedFile := &filecache.CachedFile{
- FileID: fileID,
- UserID: userID, // Use current user since original is unknown
- Filename: foundFilename,
- MimeType: mimeType,
- Size: info.Size(),
- FilePath: foundPath,
- UploadedAt: time.Now(),
- }
- fileCacheService.Store(cachedFile)
- file = cachedFile
- found = true
- break
- }
- }
-
- if !found {
- return nil, fmt.Errorf("file not found or has expired: %s (checked disk at %s)", fileID, uploadDir)
- }
- }
-
- // For workflow context, we allow access if userID matches or if no userID check is needed
- if userID != "" && file.UserID != "" && file.UserID != userID {
- return nil, fmt.Errorf("access denied: you don't have permission to access this file")
- }
-
- return &FileReference{
- FileID: file.FileID,
- Filename: file.Filename,
- MimeType: file.MimeType,
- Size: file.Size,
- Type: getFileType(file.MimeType),
- }, nil
-}
-
-// getMimeTypeFromExtension returns MIME type based on file extension
-func getMimeTypeFromExtension(ext string) string {
- switch strings.ToLower(ext) {
- case ".csv":
- return "text/csv"
- case ".xlsx":
- return "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"
- case ".xls":
- return "application/vnd.ms-excel"
- case ".json":
- return "application/json"
- case ".txt":
- return "text/plain"
- case ".png":
- return "image/png"
- case ".jpg", ".jpeg":
- return "image/jpeg"
- case ".gif":
- return "image/gif"
- case ".webp":
- return "image/webp"
- default:
- return "application/octet-stream"
- }
-}
-
-// VariableExecutor executes variable blocks (read/set workflow variables)
-type VariableExecutor struct{}
-
-// NewVariableExecutor creates a new variable executor
-func NewVariableExecutor() *VariableExecutor {
- return &VariableExecutor{}
-}
-
-// Execute runs a variable block
-func (e *VariableExecutor) Execute(ctx context.Context, block models.Block, inputs map[string]any) (map[string]any, error) {
- config := block.Config
-
- operation := getString(config, "operation", "read")
- variableName := getString(config, "variableName", "")
-
- // Extract user context for file validation
- userID, _ := inputs["__user_id__"].(string)
-
- if variableName == "" {
- return nil, fmt.Errorf("variableName is required for variable block")
- }
-
- log.Printf("📦 [VAR-EXEC] Block '%s': %s variable '%s'", block.Name, operation, variableName)
-
- switch operation {
- case "read":
- // Read from inputs (workflow variables are passed as inputs)
- value, ok := inputs[variableName]
- if !ok || value == nil || value == "" {
- // Check inputType in config to determine if we should use defaultValue (text) or fileValue (file)
- inputType := getString(config, "inputType", "text")
-
- if inputType == "file" {
- // Check for fileValue in config (used for Start block file input)
- if fileValue, hasFile := config["fileValue"]; hasFile && fileValue != nil {
- if fileMap, isMap := fileValue.(map[string]any); isMap {
- // Validate and use file reference
- fileRef, err := validateFileReference(fileMap, userID)
- if err != nil {
- log.Printf("⚠️ [VAR-EXEC] File reference validation failed: %v", err)
- } else {
- value = map[string]any{
- "file_id": fileRef.FileID,
- "filename": fileRef.Filename,
- "mime_type": fileRef.MimeType,
- "size": fileRef.Size,
- "type": fileRef.Type,
- }
- log.Printf("📁 [VAR-EXEC] Using fileValue for '%s': %s (%s)", variableName, fileRef.Filename, fileRef.Type)
- output := map[string]any{
- "value": value,
- variableName: value,
- }
- log.Printf("🔍 [VAR-EXEC] Output keys: %v", getKeys(output))
- return output, nil
- }
- }
- }
- }
-
- // Check for defaultValue in config (used for Start block text input)
- defaultValue := getString(config, "defaultValue", "")
- if defaultValue != "" {
- log.Printf("📦 [VAR-EXEC] Using defaultValue for '%s': %s", variableName, defaultValue)
- output := map[string]any{
- "value": defaultValue,
- variableName: defaultValue,
- }
- log.Printf("🔍 [VAR-EXEC] Output keys: %v", getKeys(output))
- return output, nil
- }
- log.Printf("⚠️ [VAR-EXEC] Variable '%s' not found and no defaultValue/fileValue, returning nil", variableName)
- output := map[string]any{
- "value": nil,
- variableName: nil,
- }
- log.Printf("🔍 [VAR-EXEC] Output keys: %v", getKeys(output))
- return output, nil
- }
-
- // Handle file references - validate and enrich with metadata
- if isFileReference(value) {
- fileRef, err := validateFileReference(value.(map[string]any), userID)
- if err != nil {
- log.Printf("⚠️ [VAR-EXEC] File reference validation failed: %v", err)
- // Return the original value but log the warning
- } else {
- // Convert FileReference to map for downstream use
- value = map[string]any{
- "file_id": fileRef.FileID,
- "filename": fileRef.Filename,
- "mime_type": fileRef.MimeType,
- "size": fileRef.Size,
- "type": fileRef.Type,
- }
- log.Printf("📁 [VAR-EXEC] Validated file reference: %s (%s)", fileRef.Filename, fileRef.Type)
- }
- }
-
- log.Printf("✅ [VAR-EXEC] Read variable '%s': %v", variableName, value)
- output := map[string]any{
- "value": value,
- variableName: value,
- }
- log.Printf("🔍 [VAR-EXEC] Output keys: %v", getKeys(output))
- return output, nil
-
- case "set":
- // Set/transform a value
- valueExpr := getString(config, "valueExpression", "")
- var value any
-
- if valueExpr != "" {
- // Resolve value from expression (path in inputs)
- value = resolvePath(inputs, valueExpr)
- } else {
- // Check for a direct value in config
- if v, ok := config["value"]; ok {
- value = v
- }
- }
-
- // Handle file references - validate and enrich with metadata
- if isFileReference(value) {
- fileRef, err := validateFileReference(value.(map[string]any), userID)
- if err != nil {
- log.Printf("⚠️ [VAR-EXEC] File reference validation failed: %v", err)
- // Return the original value but log the warning
- } else {
- // Convert FileReference to map for downstream use
- value = map[string]any{
- "file_id": fileRef.FileID,
- "filename": fileRef.Filename,
- "mime_type": fileRef.MimeType,
- "size": fileRef.Size,
- "type": fileRef.Type,
- }
- log.Printf("📁 [VAR-EXEC] Validated file reference: %s (%s)", fileRef.Filename, fileRef.Type)
- }
- }
-
- log.Printf("✅ [VAR-EXEC] Set variable '%s' = %v", variableName, value)
- output := map[string]any{
- "value": value,
- variableName: value,
- }
- log.Printf("🔍 [VAR-EXEC] Output keys: %v", getKeys(output))
- return output, nil
-
- default:
- return nil, fmt.Errorf("unknown variable operation: %s", operation)
- }
-}
-
-// getKeys returns the keys of a map as a slice
-func getKeys(m map[string]any) []string {
- keys := make([]string, 0, len(m))
- for k := range m {
- keys = append(keys, k)
- }
- return keys
-}
diff --git a/backend/internal/execution/variable_executor_test.go b/backend/internal/execution/variable_executor_test.go
deleted file mode 100644
index 9adad976..00000000
--- a/backend/internal/execution/variable_executor_test.go
+++ /dev/null
@@ -1,421 +0,0 @@
-package execution
-
-import (
- "context"
- "testing"
-
- "claraverse/internal/models"
-)
-
-// TestFileReferenceDetection tests isFileReference helper
-func TestFileReferenceDetection(t *testing.T) {
- testCases := []struct {
- name string
- value any
- expected bool
- }{
- {
- name: "valid file reference",
- value: map[string]any{"file_id": "abc123", "filename": "test.pdf"},
- expected: true,
- },
- {
- name: "file_id only",
- value: map[string]any{"file_id": "abc123"},
- expected: true,
- },
- {
- name: "no file_id",
- value: map[string]any{"filename": "test.pdf"},
- expected: false,
- },
- {
- name: "string value",
- value: "just a string",
- expected: false,
- },
- {
- name: "number value",
- value: 123,
- expected: false,
- },
- {
- name: "nil value",
- value: nil,
- expected: false,
- },
- {
- name: "empty map",
- value: map[string]any{},
- expected: false,
- },
- }
-
- for _, tc := range testCases {
- t.Run(tc.name, func(t *testing.T) {
- result := isFileReference(tc.value)
- if result != tc.expected {
- t.Errorf("isFileReference(%v) = %v, expected %v", tc.value, result, tc.expected)
- }
- })
- }
-}
-
-// TestGetFileType tests MIME type categorization
-func TestGetFileType(t *testing.T) {
- testCases := []struct {
- mimeType string
- expected string
- }{
- // Images
- {"image/jpeg", "image"},
- {"image/png", "image"},
- {"image/gif", "image"},
- {"image/webp", "image"},
- {"image/svg+xml", "image"},
-
- // Audio
- {"audio/mpeg", "audio"},
- {"audio/wav", "audio"},
- {"audio/mp4", "audio"},
- {"audio/ogg", "audio"},
-
- // Documents
- {"application/pdf", "document"},
- {"application/vnd.openxmlformats-officedocument.wordprocessingml.document", "document"},
- {"application/vnd.openxmlformats-officedocument.presentationml.presentation", "document"},
- {"application/msword", "document"},
-
- // Data files
- {"application/json", "data"},
- {"text/csv", "data"},
- {"text/plain", "data"},
- {"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", "data"},
-
- // Unknown defaults to data
- {"application/octet-stream", "data"},
- {"video/mp4", "data"},
- }
-
- for _, tc := range testCases {
- t.Run(tc.mimeType, func(t *testing.T) {
- result := getFileType(tc.mimeType)
- if result != tc.expected {
- t.Errorf("getFileType(%s) = %s, expected %s", tc.mimeType, result, tc.expected)
- }
- })
- }
-}
-
-// TestFileReferenceStruct tests FileReference structure
-func TestFileReferenceStruct(t *testing.T) {
- ref := FileReference{
- FileID: "file-123",
- Filename: "document.pdf",
- MimeType: "application/pdf",
- Size: 12345,
- Type: "document",
- }
-
- if ref.FileID == "" {
- t.Error("FileID should be set")
- }
- if ref.Type != "document" {
- t.Errorf("Type should be 'document', got %s", ref.Type)
- }
-}
-
-// TestVariableExecutorCreation tests executor creation
-func TestVariableExecutorCreation(t *testing.T) {
- executor := NewVariableExecutor()
- if executor == nil {
- t.Fatal("NewVariableExecutor should return non-nil executor")
- }
-}
-
-// TestVariableReadOperation tests read operation
-func TestVariableReadOperation(t *testing.T) {
- executor := NewVariableExecutor()
-
- block := models.Block{
- ID: "var-block",
- Name: "Read Variable",
- Config: map[string]any{
- "operation": "read",
- "variableName": "testVar",
- },
- }
-
- inputs := map[string]any{
- "testVar": "hello world",
- }
-
- result, err := executor.Execute(context.Background(), block, inputs)
- if err != nil {
- t.Fatalf("Execute failed: %v", err)
- }
-
- if result["value"] != "hello world" {
- t.Errorf("Expected 'hello world', got %v", result["value"])
- }
- if result["testVar"] != "hello world" {
- t.Errorf("Expected testVar='hello world', got %v", result["testVar"])
- }
-}
-
-// TestVariableReadWithDefault tests read operation with default value
-func TestVariableReadWithDefault(t *testing.T) {
- executor := NewVariableExecutor()
-
- block := models.Block{
- ID: "var-block",
- Name: "Read Variable",
- Config: map[string]any{
- "operation": "read",
- "variableName": "missingVar",
- "defaultValue": "default value",
- },
- }
-
- inputs := map[string]any{}
-
- result, err := executor.Execute(context.Background(), block, inputs)
- if err != nil {
- t.Fatalf("Execute failed: %v", err)
- }
-
- if result["value"] != "default value" {
- t.Errorf("Expected 'default value', got %v", result["value"])
- }
-}
-
-// TestVariableReadMissing tests read of missing variable without default
-func TestVariableReadMissing(t *testing.T) {
- executor := NewVariableExecutor()
-
- block := models.Block{
- ID: "var-block",
- Name: "Read Variable",
- Config: map[string]any{
- "operation": "read",
- "variableName": "missingVar",
- },
- }
-
- inputs := map[string]any{}
-
- result, err := executor.Execute(context.Background(), block, inputs)
- if err != nil {
- t.Fatalf("Execute failed: %v", err)
- }
-
- // Should return nil for missing variable
- if result["value"] != nil {
- t.Errorf("Expected nil, got %v", result["value"])
- }
-}
-
-// TestVariableSetOperation tests set operation
-func TestVariableSetOperation(t *testing.T) {
- executor := NewVariableExecutor()
-
- block := models.Block{
- ID: "var-block",
- Name: "Set Variable",
- Config: map[string]any{
- "operation": "set",
- "variableName": "newVar",
- "value": "set value",
- },
- }
-
- inputs := map[string]any{}
-
- result, err := executor.Execute(context.Background(), block, inputs)
- if err != nil {
- t.Fatalf("Execute failed: %v", err)
- }
-
- if result["value"] != "set value" {
- t.Errorf("Expected 'set value', got %v", result["value"])
- }
- if result["newVar"] != "set value" {
- t.Errorf("Expected newVar='set value', got %v", result["newVar"])
- }
-}
-
-// TestVariableSetFromExpression tests set operation with expression
-func TestVariableSetFromExpression(t *testing.T) {
- executor := NewVariableExecutor()
-
- block := models.Block{
- ID: "var-block",
- Name: "Set Variable",
- Config: map[string]any{
- "operation": "set",
- "variableName": "result",
- "valueExpression": "sourceData",
- },
- }
-
- inputs := map[string]any{
- "sourceData": "value from source",
- }
-
- result, err := executor.Execute(context.Background(), block, inputs)
- if err != nil {
- t.Fatalf("Execute failed: %v", err)
- }
-
- if result["value"] != "value from source" {
- t.Errorf("Expected 'value from source', got %v", result["value"])
- }
-}
-
-// TestVariableMissingName tests error for missing variable name
-func TestVariableMissingName(t *testing.T) {
- executor := NewVariableExecutor()
-
- block := models.Block{
- ID: "var-block",
- Name: "Bad Block",
- Config: map[string]any{
- "operation": "read",
- // Missing variableName
- },
- }
-
- inputs := map[string]any{}
-
- _, err := executor.Execute(context.Background(), block, inputs)
- if err == nil {
- t.Error("Expected error for missing variableName")
- }
-}
-
-// TestVariableUnknownOperation tests error for unknown operation
-func TestVariableUnknownOperation(t *testing.T) {
- executor := NewVariableExecutor()
-
- block := models.Block{
- ID: "var-block",
- Name: "Bad Block",
- Config: map[string]any{
- "operation": "invalid",
- "variableName": "test",
- },
- }
-
- inputs := map[string]any{}
-
- _, err := executor.Execute(context.Background(), block, inputs)
- if err == nil {
- t.Error("Expected error for unknown operation")
- }
-}
-
-// TestGetKeysHelper tests getKeys helper function
-func TestGetKeysHelper(t *testing.T) {
- m := map[string]any{
- "a": 1,
- "b": 2,
- "c": 3,
- }
-
- keys := getKeys(m)
- if len(keys) != 3 {
- t.Errorf("Expected 3 keys, got %d", len(keys))
- }
-
- // Check all keys are present (order doesn't matter)
- keyMap := make(map[string]bool)
- for _, k := range keys {
- keyMap[k] = true
- }
-
- for _, expected := range []string{"a", "b", "c"} {
- if !keyMap[expected] {
- t.Errorf("Expected key %s not found", expected)
- }
- }
-}
-
-// Benchmark tests
-func BenchmarkVariableRead(b *testing.B) {
- executor := NewVariableExecutor()
-
- block := models.Block{
- ID: "var-block",
- Name: "Read Variable",
- Config: map[string]any{
- "operation": "read",
- "variableName": "testVar",
- },
- }
-
- inputs := map[string]any{
- "testVar": "test value",
- }
-
- ctx := context.Background()
-
- b.ResetTimer()
- for i := 0; i < b.N; i++ {
- executor.Execute(ctx, block, inputs)
- }
-}
-
-func BenchmarkVariableSet(b *testing.B) {
- executor := NewVariableExecutor()
-
- block := models.Block{
- ID: "var-block",
- Name: "Set Variable",
- Config: map[string]any{
- "operation": "set",
- "variableName": "testVar",
- "value": "test value",
- },
- }
-
- inputs := map[string]any{}
- ctx := context.Background()
-
- b.ResetTimer()
- for i := 0; i < b.N; i++ {
- executor.Execute(ctx, block, inputs)
- }
-}
-
-func BenchmarkIsFileReference(b *testing.B) {
- testCases := []any{
- map[string]any{"file_id": "abc123"},
- "just a string",
- 123,
- nil,
- }
-
- b.ResetTimer()
- for i := 0; i < b.N; i++ {
- for _, tc := range testCases {
- isFileReference(tc)
- }
- }
-}
-
-func BenchmarkGetFileType(b *testing.B) {
- mimeTypes := []string{
- "image/jpeg",
- "audio/mpeg",
- "application/pdf",
- "application/json",
- "text/plain",
- }
-
- b.ResetTimer()
- for i := 0; i < b.N; i++ {
- for _, mime := range mimeTypes {
- getFileType(mime)
- }
- }
-}
diff --git a/backend/internal/execution/webhook_executor.go b/backend/internal/execution/webhook_executor.go
deleted file mode 100644
index d0b204bd..00000000
--- a/backend/internal/execution/webhook_executor.go
+++ /dev/null
@@ -1,112 +0,0 @@
-package execution
-
-import (
- "claraverse/internal/models"
- "context"
- "encoding/json"
- "fmt"
- "io"
- "log"
- "net/http"
- "strings"
- "time"
-)
-
-// WebhookExecutor executes webhook blocks (HTTP requests)
-type WebhookExecutor struct {
- client *http.Client
-}
-
-// NewWebhookExecutor creates a new webhook executor
-func NewWebhookExecutor() *WebhookExecutor {
- return &WebhookExecutor{
- client: &http.Client{Timeout: 30 * time.Second},
- }
-}
-
-// Execute runs a webhook block
-func (e *WebhookExecutor) Execute(ctx context.Context, block models.Block, inputs map[string]any) (map[string]any, error) {
- config := block.Config
-
- url := getString(config, "url", "")
- method := strings.ToUpper(getString(config, "method", "GET"))
- headers := getMap(config, "headers")
- bodyTemplate := getString(config, "bodyTemplate", "")
-
- if url == "" {
- return nil, fmt.Errorf("url is required for webhook block")
- }
-
- // Interpolate variables in URL
- url = interpolateTemplate(url, inputs)
-
- // Interpolate variables in body
- body := interpolateTemplate(bodyTemplate, inputs)
-
- log.Printf("🌐 [WEBHOOK-EXEC] Block '%s': %s %s", block.Name, method, url)
-
- // Create request
- var bodyReader io.Reader
- if body != "" {
- bodyReader = strings.NewReader(body)
- }
-
- req, err := http.NewRequestWithContext(ctx, method, url, bodyReader)
- if err != nil {
- return nil, fmt.Errorf("failed to create request: %w", err)
- }
-
- // Set headers
- if headers != nil {
- for key, value := range headers {
- if strVal, ok := value.(string); ok {
- // Interpolate variables in header values (for secrets)
- strVal = interpolateTemplate(strVal, inputs)
- req.Header.Set(key, strVal)
- }
- }
- }
-
- // Default content type for POST/PUT with body
- if body != "" && req.Header.Get("Content-Type") == "" {
- req.Header.Set("Content-Type", "application/json")
- }
-
- // Execute request
- resp, err := e.client.Do(req)
- if err != nil {
- log.Printf("❌ [WEBHOOK-EXEC] Request failed: %v", err)
- return nil, fmt.Errorf("webhook request failed: %w", err)
- }
- defer resp.Body.Close()
-
- // Read response body
- responseBody, err := io.ReadAll(resp.Body)
- if err != nil {
- return nil, fmt.Errorf("failed to read response: %w", err)
- }
-
- log.Printf("✅ [WEBHOOK-EXEC] Block '%s': status=%d, body_len=%d", block.Name, resp.StatusCode, len(responseBody))
-
- // Try to parse response as JSON
- var parsedBody any
- if err := json.Unmarshal(responseBody, &parsedBody); err != nil {
- // Not JSON, use as string
- parsedBody = string(responseBody)
- }
-
- // Convert response headers to map
- respHeaders := make(map[string]string)
- for key, values := range resp.Header {
- if len(values) > 0 {
- respHeaders[key] = values[0]
- }
- }
-
- return map[string]any{
- "status": resp.StatusCode,
- "body": parsedBody,
- "headers": respHeaders,
- "raw": string(responseBody),
- }, nil
-}
diff --git a/backend/internal/execution/workflow_test.go b/backend/internal/execution/workflow_test.go
deleted file mode 100644
index 8e9e583c..00000000
--- a/backend/internal/execution/workflow_test.go
+++ /dev/null
@@ -1,390 +0,0 @@
-package execution
-
-import (
- "claraverse/internal/models"
- "context"
- "testing"
-)
-
-// TestVariableExecutorOutputsCorrectKeys tests that variable blocks output both "value" and the variable name
-func TestVariableExecutorOutputsCorrectKeys(t *testing.T) {
- executor := NewVariableExecutor()
-
- tests := []struct {
- name string
- block models.Block
- inputs map[string]any
- expectedKeys []string
- }{
- {
- name: "Read operation with defaultValue should output both value and variableName",
- block: models.Block{
- Name: "Start",
- Type: "variable",
- Config: map[string]any{
- "operation": "read",
- "variableName": "input",
- "defaultValue": "test value",
- },
- },
- inputs: map[string]any{},
- expectedKeys: []string{"value", "input"},
- },
- {
- name: "Read operation with existing input should output both keys",
- block: models.Block{
- Name: "Read Input",
- Type: "variable",
- Config: map[string]any{
- "operation": "read",
- "variableName": "query",
- },
- },
- inputs: map[string]any{"query": "search term"},
- expectedKeys: []string{"value", "query"},
- },
- {
- name: "Set operation should output both keys",
- block: models.Block{
- Name: "Set Variable",
- Type: "variable",
- Config: map[string]any{
- "operation": "set",
- "variableName": "result",
- "value": "new value",
- },
- },
- inputs: map[string]any{},
- expectedKeys: []string{"value", "result"},
- },
- }
-
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- ctx := context.Background()
- output, err := executor.Execute(ctx, tt.block, tt.inputs)
-
- if err != nil {
- t.Fatalf("Execute failed: %v", err)
- }
-
- // Check that all expected keys are present
- for _, key := range tt.expectedKeys {
- if _, ok := output[key]; !ok {
- t.Errorf("Expected key '%s' not found in output. Got: %+v", key, output)
- }
- }
-
- // For "read" operation, both keys should have the same value
- if tt.block.Config["operation"] == "read" {
- if output["value"] != output[tt.block.Config["variableName"].(string)] {
- t.Errorf("'value' and variable name key should have same value. Got: %+v", output)
- }
- }
- })
- }
-}
-
-// TestInterpolateTemplate tests the template interpolation function
-func TestInterpolateTemplate(t *testing.T) {
- tests := []struct {
- name string
- template string
- inputs map[string]any
- expected string
- }{
- {
- name: "Simple variable interpolation",
- template: "Search for {{input}}",
- inputs: map[string]any{"input": "test query"},
- expected: "Search for test query",
- },
- {
- name: "Multiple variables",
- template: "{{user}} searched for {{query}}",
- inputs: map[string]any{"user": "John", "query": "golang"},
- expected: "John searched for golang",
- },
- {
- name: "Nested object access",
- template: "Result: {{output.response}}",
- inputs: map[string]any{
- "output": map[string]any{
- "response": "success",
- },
- },
- expected: "Result: success",
- },
- {
- name: "Missing variable should keep original",
- template: "Value: {{missing}}",
- inputs: map[string]any{"other": "value"},
- expected: "Value: {{missing}}",
- },
- {
- name: "Number conversion",
- template: "Count: {{count}}",
- inputs: map[string]any{"count": 42},
- expected: "Count: 42",
- },
- {
- name: "Boolean conversion",
- template: "Active: {{active}}",
- inputs: map[string]any{"active": true},
- expected: "Active: true",
- },
- }
-
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- result := interpolateTemplate(tt.template, tt.inputs)
- if result != tt.expected {
- t.Errorf("Expected '%s', got '%s'", tt.expected, result)
- }
- })
- }
-}
-
-// TestInterpolateMapValues tests the map value interpolation function
-func TestInterpolateMapValues(t *testing.T) {
- tests := []struct {
- name string
- data map[string]any
- inputs map[string]any
- expected map[string]any
- }{
- {
- name: "Interpolate string values in map",
- data: map[string]any{
- "query": "{{input}}",
- "type": "search",
- },
- inputs: map[string]any{"input": "test query"},
- expected: map[string]any{
- "query": "test query",
- "type": "search",
- },
- },
- {
- name: "Nested map interpolation",
- data: map[string]any{
- "params": map[string]any{
- "q": "{{query}}",
- },
- },
- inputs: map[string]any{"query": "golang"},
- expected: map[string]any{
- "params": map[string]any{
- "q": "golang",
- },
- },
- },
- {
- name: "Array interpolation",
- data: map[string]any{
- "items": []any{"{{first}}", "{{second}}"},
- },
- inputs: map[string]any{"first": "a", "second": "b"},
- expected: map[string]any{
- "items": []any{"a", "b"},
- },
- },
- }
-
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- result := interpolateMapValues(tt.data, tt.inputs)
-
- // Deep comparison
- if !mapsEqual(result, tt.expected) {
- t.Errorf("Expected %+v, got %+v", tt.expected, result)
- }
- })
- }
-}
-
-// TestWorkflowDataFlow tests end-to-end data flow through a simple workflow
-func TestWorkflowDataFlow(t *testing.T) {
- // Create a simple workflow: Start -> Block A -> Block B
- workflow := &models.Workflow{
- ID: "test-workflow",
- Blocks: []models.Block{
- {
- ID: "start",
- Name: "Start",
- Type: "variable",
- Config: map[string]any{
- "operation": "read",
- "variableName": "input",
- "defaultValue": "test value",
- },
- },
- },
- Connections: []models.Connection{},
- Variables: []models.Variable{},
- }
-
- // Test variable executor output
- varExec := NewVariableExecutor()
- ctx := context.Background()
-
- startOutput, err := varExec.Execute(ctx, workflow.Blocks[0], map[string]any{})
- if err != nil {
- t.Fatalf("Start block failed: %v", err)
- }
-
- t.Logf("Start block output: %+v", startOutput)
-
- // Verify Start block outputs both keys
- if _, ok := startOutput["value"]; !ok {
- t.Error("Start block should output 'value' key")
- }
- if _, ok := startOutput["input"]; !ok {
- t.Error("Start block should output 'input' key")
- }
-
- // Simulate engine passing Start output to next block
- nextBlockInputs := map[string]any{}
- // Copy globalInputs (workflow input)
- nextBlockInputs["input"] = "test value"
- // Add Start block outputs
- for k, v := range startOutput {
- nextBlockInputs[k] = v
- }
-
- t.Logf("Next block would receive inputs: %+v", nextBlockInputs)
-
- // Test interpolation with these inputs
- template := "Process {{input}}"
- result := interpolateTemplate(template, nextBlockInputs)
- expected := "Process test value"
-
- if result != expected {
- t.Errorf("Interpolation failed. Expected '%s', got '%s'", expected, result)
- }
-}
-
-// TestWorkflowEngineBlockInputConstruction tests how engine.go constructs blockInputs
-func TestWorkflowEngineBlockInputConstruction(t *testing.T) {
- // Simulate what engine.go does at lines 129-154
- workflow := &models.Workflow{
- ID: "test",
- Blocks: []models.Block{
- {ID: "start", Name: "Start", Type: "variable"},
- {ID: "block2", Name: "Block 2", Type: "llm_inference"},
- },
- Connections: []models.Connection{
- {
- ID: "conn1",
- SourceBlockID: "start",
- TargetBlockID: "block2",
- SourceOutput: "output",
- TargetInput: "input",
- },
- },
- }
-
- // Initial workflow input
- workflowInput := map[string]any{
- "input": "GUVI HCL Scam",
- }
-
- // Global inputs (what engine.go builds)
- globalInputs := make(map[string]any)
- for k, v := range workflowInput {
- globalInputs[k] = v
- }
-
- // Start block outputs
- startBlockOutput := map[string]any{
- "value": "GUVI HCL Scam",
- "input": "GUVI HCL Scam",
- }
-
- // Block outputs storage
- blockOutputs := map[string]map[string]any{
- "start": startBlockOutput,
- }
-
- // Build inputs for block2 (what engine.go does)
- blockInputs := make(map[string]any)
- for k, v := range globalInputs {
- blockInputs[k] = v
- }
-
- // Add outputs from connected upstream blocks
- for _, conn := range workflow.Connections {
- if conn.TargetBlockID == "block2" {
- if output, ok := blockOutputs[conn.SourceBlockID]; ok {
- // Add under source block name
- blockInputs["Start"] = output
-
- // Also add fields directly
- for k, v := range output {
- blockInputs[k] = v
- }
- }
- }
- }
-
- t.Logf("Block2 inputs: %+v", blockInputs)
-
- // Verify block2 has access to "input" key
- if _, ok := blockInputs["input"]; !ok {
- t.Error("Block2 should have 'input' key in inputs")
- }
-
- if blockInputs["input"] != "GUVI HCL Scam" {
- t.Errorf("Block2 input should be 'GUVI HCL Scam', got: %v", blockInputs["input"])
- }
-
- // Test interpolation would work
- template := "{{input}}"
- result := interpolateTemplate(template, blockInputs)
- if result != "GUVI HCL Scam" {
- t.Errorf("Interpolation should resolve to 'GUVI HCL Scam', got: '%s'", result)
- }
-}
-
-// Helper function for deep map comparison
-func mapsEqual(a, b map[string]any) bool {
- if len(a) != len(b) {
- return false
- }
- for k, v := range a {
- bv, ok := b[k]
- if !ok {
- return false
- }
- // Handle different types
- switch av := v.(type) {
- case map[string]any:
- bvm, ok := bv.(map[string]any)
- if !ok || !mapsEqual(av, bvm) {
- return false
- }
- case []any:
- bva, ok := bv.([]any)
- if !ok || !slicesEqual(av, bva) {
- return false
- }
- default:
- if v != bv {
- return false
- }
- }
- }
- return true
-}
-
-func slicesEqual(a, b []any) bool {
- if len(a) != len(b) {
- return false
- }
- for i := range a {
- if a[i] != b[i] {
- return false
- }
- }
- return true
-}
diff --git a/backend/internal/filecache/filecache.go b/backend/internal/filecache/filecache.go
deleted file mode 100644
index b9d54334..00000000
--- a/backend/internal/filecache/filecache.go
+++ /dev/null
@@ -1,426 +0,0 @@
-package filecache
-
-import (
- "claraverse/internal/security"
- "fmt"
- "log"
- "os"
- "sync"
- "time"
-
- "github.com/patrickmn/go-cache"
-)
-
-// CachedFile represents a file stored in memory cache
-type CachedFile struct {
- FileID string
- UserID string
- ConversationID string
- ExtractedText *security.SecureString // For PDFs
- FileHash security.Hash
- Filename string
- MimeType string
- Size int64
- PageCount int // For PDFs
- WordCount int // For PDFs
- FilePath string // For images (disk location)
- UploadedAt time.Time
-}
-
-// Service manages uploaded files in memory
-type Service struct {
- cache *cache.Cache
- mu sync.RWMutex
-}
-
-var (
- instance *Service
- once sync.Once
-)
-
-// GetService returns the singleton file cache service
-func GetService() *Service {
- once.Do(func() {
- instance = NewService()
- })
- return instance
-}
-
-// NewService creates a new file cache service
-func NewService() *Service {
- c := cache.New(30*time.Minute, 10*time.Minute)
-
- // Set eviction handler for secure wiping
- c.OnEvicted(func(key string, value interface{}) {
- if file, ok := value.(*CachedFile); ok {
- log.Printf("🗑️ [FILE-CACHE] Evicting file %s (%s) - secure wiping memory", file.FileID, file.Filename)
- file.SecureWipe()
- }
- })
-
- return &Service{
- cache: c,
- }
-}
-
-// Store stores a file in the cache
-func (s *Service) Store(file *CachedFile) {
- s.mu.Lock()
- defer s.mu.Unlock()
- s.cache.Set(file.FileID, file, cache.DefaultExpiration)
- log.Printf("📦 [FILE-CACHE] Stored file %s (%s) - %d bytes, %d words",
- file.FileID, file.Filename, file.Size, file.WordCount)
-}
-
-// Get retrieves a file from the cache
-func (s *Service) Get(fileID string) (*CachedFile, bool) {
- s.mu.RLock()
- defer s.mu.RUnlock()
-
- value, found := s.cache.Get(fileID)
- if !found {
- return nil, false
- }
-
- file, ok := value.(*CachedFile)
- if !ok {
- return nil, false
- }
-
- return file, true
-}
-
-// GetByUserAndConversation retrieves a file if it belongs to the user and conversation
-func (s *Service) GetByUserAndConversation(fileID, userID, conversationID string) (*CachedFile, error) {
- file, found := s.Get(fileID)
- if !found {
- return nil, fmt.Errorf("file not found or expired")
- }
-
- // Verify ownership
- if file.UserID != userID {
- return nil, fmt.Errorf("access denied: file belongs to different user")
- }
-
- // Verify conversation
- if file.ConversationID != conversationID {
- return nil, fmt.Errorf("file belongs to different conversation")
- }
-
- return file, nil
-}
-
-// GetByUser retrieves a file if it belongs to the user (ignores conversation)
-func (s *Service) GetByUser(fileID, userID string) (*CachedFile, error) {
- file, found := s.Get(fileID)
- if !found {
- return nil, fmt.Errorf("file not found or expired")
- }
-
- // Verify ownership
- if file.UserID != userID {
- return nil, fmt.Errorf("access denied: file belongs to different user")
- }
-
- return file, nil
-}
-
-// GetFilesForConversation returns all files for a conversation
-func (s *Service) GetFilesForConversation(conversationID string) []*CachedFile {
- s.mu.RLock()
- defer s.mu.RUnlock()
-
- var files []*CachedFile
- for _, item := range s.cache.Items() {
- if file, ok := item.Object.(*CachedFile); ok {
- if file.ConversationID == conversationID {
- files = append(files, file)
- }
- }
- }
-
- return files
-}
-
-// GetConversationFiles returns all file IDs for a conversation
-func (s *Service) GetConversationFiles(conversationID string) []string {
- files := s.GetFilesForConversation(conversationID)
- fileIDs := make([]string, 0, len(files))
- for _, file := range files {
- fileIDs = append(fileIDs, file.FileID)
- }
- return fileIDs
-}
-
-// Delete removes a file from the cache and securely wipes it
-func (s *Service) Delete(fileID string) {
- s.mu.Lock()
- defer s.mu.Unlock()
-
- // Get the file first to wipe it
- if value, found := s.cache.Get(fileID); found {
- if file, ok := value.(*CachedFile); ok {
- log.Printf("🗑️ [FILE-CACHE] Deleting file %s (%s)", file.FileID, file.Filename)
- file.SecureWipe()
- }
- }
-
- s.cache.Delete(fileID)
-}
-
-// DeleteConversationFiles deletes all files for a conversation
-func (s *Service) DeleteConversationFiles(conversationID string) {
- s.mu.Lock()
- defer s.mu.Unlock()
-
- log.Printf("🗑️ [FILE-CACHE] Deleting all files for conversation %s", conversationID)
-
- for key, item := range s.cache.Items() {
- if file, ok := item.Object.(*CachedFile); ok {
- if file.ConversationID == conversationID {
- file.SecureWipe()
- s.cache.Delete(key)
- }
- }
- }
-}
-
-// ExtendTTL extends the TTL of a file to match conversation lifetime
-func (s *Service) ExtendTTL(fileID string, duration time.Duration) {
- s.mu.Lock()
- defer s.mu.Unlock()
-
- if value, found := s.cache.Get(fileID); found {
- s.cache.Set(fileID, value, duration)
- log.Printf("⏰ [FILE-CACHE] Extended TTL for file %s to %v", fileID, duration)
- }
-}
-
-// SecureWipe securely wipes the file's sensitive data
-func (f *CachedFile) SecureWipe() {
- if f.ExtractedText != nil {
- f.ExtractedText.Wipe()
- f.ExtractedText = nil
- }
-
- // Delete physical file if it exists (for images)
- if f.FilePath != "" {
- if err := os.Remove(f.FilePath); err != nil && !os.IsNotExist(err) {
- log.Printf("⚠️ Failed to delete file %s: %v", f.FilePath, err)
- } else {
- log.Printf("🗑️ Deleted file from disk: %s", f.FilePath)
- }
- }
-
- // Wipe hash
- for i := range f.FileHash {
- f.FileHash[i] = 0
- }
-
- // Clear other fields
- f.FileID = ""
- f.UserID = ""
- f.ConversationID = ""
- f.Filename = ""
- f.FilePath = ""
-}
-
-// CleanupExpiredFiles deletes files older than 1 hour
-func (s *Service) CleanupExpiredFiles() {
- s.mu.Lock()
- defer s.mu.Unlock()
-
- now := time.Now()
- expiredCount := 0
-
- for key, item := range s.cache.Items() {
- if file, ok := item.Object.(*CachedFile); ok {
- if file.FilePath != "" {
- if now.Sub(file.UploadedAt) > 1*time.Hour {
- log.Printf("🗑️ [FILE-CACHE] Deleting expired file: %s (uploaded %v ago)",
- file.Filename, now.Sub(file.UploadedAt))
-
- if err := os.Remove(file.FilePath); err != nil && !os.IsNotExist(err) {
- log.Printf("⚠️ Failed to delete expired file %s: %v", file.FilePath, err)
- }
-
- s.cache.Delete(key)
- expiredCount++
- }
- }
- }
- }
-
- if expiredCount > 0 {
- log.Printf("✅ [FILE-CACHE] Cleaned up %d expired files", expiredCount)
- }
-}
-
-// CleanupOrphanedFiles scans the uploads directory and deletes orphaned files
-func (s *Service) CleanupOrphanedFiles(uploadDir string, maxAge time.Duration) {
- s.mu.RLock()
- trackedFiles := make(map[string]bool)
- for _, item := range s.cache.Items() {
- if file, ok := item.Object.(*CachedFile); ok {
- if file.FilePath != "" {
- trackedFiles[file.FilePath] = true
- }
- }
- }
- s.mu.RUnlock()
-
- entries, err := os.ReadDir(uploadDir)
- if err != nil {
- log.Printf("⚠️ [CLEANUP] Failed to read uploads directory: %v", err)
- return
- }
-
- now := time.Now()
- orphanedCount := 0
-
- for _, entry := range entries {
- if entry.IsDir() {
- continue
- }
-
- filePath := fmt.Sprintf("%s/%s", uploadDir, entry.Name())
-
- info, err := entry.Info()
- if err != nil {
- continue
- }
-
- fileAge := now.Sub(info.ModTime())
-
- if !trackedFiles[filePath] {
- if fileAge > 5*time.Minute {
- log.Printf("🗑️ [CLEANUP] Deleting orphaned file: %s (age: %v)", entry.Name(), fileAge)
- if err := os.Remove(filePath); err != nil && !os.IsNotExist(err) {
- log.Printf("⚠️ [CLEANUP] Failed to delete orphaned file %s: %v", entry.Name(), err)
- } else {
- orphanedCount++
- }
- }
- }
- }
-
- if orphanedCount > 0 {
- log.Printf("✅ [CLEANUP] Deleted %d orphaned files", orphanedCount)
- }
-}
-
-// RunStartupCleanup performs initial cleanup when server starts
-func (s *Service) RunStartupCleanup(uploadDir string) {
- log.Printf("🧹 [STARTUP] Running startup file cleanup in %s...", uploadDir)
-
- entries, err := os.ReadDir(uploadDir)
- if err != nil {
- log.Printf("⚠️ [STARTUP] Failed to read uploads directory: %v", err)
- return
- }
-
- now := time.Now()
- deletedCount := 0
-
- for _, entry := range entries {
- if entry.IsDir() {
- continue
- }
-
- filePath := fmt.Sprintf("%s/%s", uploadDir, entry.Name())
-
- info, err := entry.Info()
- if err != nil {
- continue
- }
-
- if now.Sub(info.ModTime()) > 1*time.Hour {
- log.Printf("🗑️ [STARTUP] Deleting stale file: %s (modified: %v ago)",
- entry.Name(), now.Sub(info.ModTime()))
- if err := os.Remove(filePath); err != nil && !os.IsNotExist(err) {
- log.Printf("⚠️ [STARTUP] Failed to delete file %s: %v", entry.Name(), err)
- } else {
- deletedCount++
- }
- }
- }
-
- log.Printf("✅ [STARTUP] Startup cleanup complete: deleted %d stale files", deletedCount)
-}
-
-// GetStats returns cache statistics
-func (s *Service) GetStats() map[string]interface{} {
- s.mu.RLock()
- defer s.mu.RUnlock()
-
- items := s.cache.Items()
- totalSize := int64(0)
- totalWords := 0
-
- for _, item := range items {
- if file, ok := item.Object.(*CachedFile); ok {
- totalSize += file.Size
- totalWords += file.WordCount
- }
- }
-
- return map[string]interface{}{
- "total_files": len(items),
- "total_size": totalSize,
- "total_words": totalWords,
- }
-}
-
-// GetAllFilesByUser returns metadata for all files owned by a user
-func (s *Service) GetAllFilesByUser(userID string) []map[string]interface{} {
- s.mu.RLock()
- defer s.mu.RUnlock()
-
- var fileMetadata []map[string]interface{}
-
- for _, item := range s.cache.Items() {
- if file, ok := item.Object.(*CachedFile); ok {
- if file.UserID == userID {
- metadata := map[string]interface{}{
- "file_id": file.FileID,
- "filename": file.Filename,
- "mime_type": file.MimeType,
- "size": file.Size,
- "uploaded_at": file.UploadedAt.Format(time.RFC3339),
- "conversation_id": file.ConversationID,
- }
-
- if file.MimeType == "application/pdf" {
- metadata["page_count"] = file.PageCount
- metadata["word_count"] = file.WordCount
- }
-
- fileMetadata = append(fileMetadata, metadata)
- }
- }
- }
-
- return fileMetadata
-}
-
-// DeleteAllFilesByUser deletes all files owned by a user
-func (s *Service) DeleteAllFilesByUser(userID string) (int, error) {
- s.mu.Lock()
- defer s.mu.Unlock()
-
- deletedCount := 0
-
- for key, item := range s.cache.Items() {
- if file, ok := item.Object.(*CachedFile); ok {
- if file.UserID == userID {
- log.Printf("🗑️ [GDPR] Deleting file %s (%s) for user %s", file.FileID, file.Filename, userID)
- file.SecureWipe()
- s.cache.Delete(key)
- deletedCount++
- }
- }
- }
-
- log.Printf("✅ [GDPR] Deleted %d files for user %s", deletedCount, userID)
- return deletedCount, nil
-}
diff --git a/backend/internal/filecache/filecache_test.go b/backend/internal/filecache/filecache_test.go
deleted file mode 100644
index ec4d2088..00000000
--- a/backend/internal/filecache/filecache_test.go
+++ /dev/null
@@ -1,501 +0,0 @@
-package filecache
-
-import (
- "claraverse/internal/security"
- "os"
- "path/filepath"
- "testing"
- "time"
-)
-
-// TestNewService verifies service creation
-func TestNewService(t *testing.T) {
- svc := NewService()
- if svc == nil {
- t.Fatal("NewService should return non-nil service")
- }
- if svc.cache == nil {
- t.Error("Service should have cache initialized")
- }
-}
-
-// TestGetServiceSingleton verifies singleton pattern
-func TestGetServiceSingleton(t *testing.T) {
- svc1 := GetService()
- svc2 := GetService()
-
- if svc1 != svc2 {
- t.Error("GetService should return the same instance")
- }
-}
-
-// TestStoreAndGet tests basic store and retrieve
-func TestStoreAndGet(t *testing.T) {
- svc := NewService()
-
- file := &CachedFile{
- FileID: "test-file-123",
- UserID: "user-456",
- ConversationID: "conv-789",
- Filename: "test.pdf",
- MimeType: "application/pdf",
- Size: 1024,
- UploadedAt: time.Now(),
- }
-
- svc.Store(file)
-
- retrieved, found := svc.Get("test-file-123")
- if !found {
- t.Fatal("File should be found after store")
- }
-
- if retrieved.FileID != file.FileID {
- t.Errorf("Expected FileID %s, got %s", file.FileID, retrieved.FileID)
- }
- if retrieved.UserID != file.UserID {
- t.Errorf("Expected UserID %s, got %s", file.UserID, retrieved.UserID)
- }
- if retrieved.Filename != file.Filename {
- t.Errorf("Expected Filename %s, got %s", file.Filename, retrieved.Filename)
- }
-}
-
-// TestGetNotFound tests retrieval of non-existent file
-func TestGetNotFound(t *testing.T) {
- svc := NewService()
-
- _, found := svc.Get("non-existent-file")
- if found {
- t.Error("Non-existent file should not be found")
- }
-}
-
-// TestGetByUser tests user-scoped retrieval
-func TestGetByUser(t *testing.T) {
- svc := NewService()
-
- file := &CachedFile{
- FileID: "file-for-user",
- UserID: "user-123",
- Filename: "user-file.pdf",
- }
- svc.Store(file)
-
- // Same user should be able to retrieve
- retrieved, err := svc.GetByUser("file-for-user", "user-123")
- if err != nil {
- t.Errorf("Same user should retrieve file: %v", err)
- }
- if retrieved == nil {
- t.Fatal("Retrieved file should not be nil")
- }
-
- // Different user should be denied
- _, err = svc.GetByUser("file-for-user", "different-user")
- if err == nil {
- t.Error("Different user should be denied access")
- }
-}
-
-// TestGetByUserAndConversation tests user+conversation scoped retrieval
-func TestGetByUserAndConversation(t *testing.T) {
- svc := NewService()
-
- file := &CachedFile{
- FileID: "conv-file",
- UserID: "user-123",
- ConversationID: "conv-456",
- Filename: "conv-file.pdf",
- }
- svc.Store(file)
-
- // Same user + conversation should work
- retrieved, err := svc.GetByUserAndConversation("conv-file", "user-123", "conv-456")
- if err != nil {
- t.Errorf("Same user+conversation should retrieve file: %v", err)
- }
- if retrieved == nil {
- t.Fatal("Retrieved file should not be nil")
- }
-
- // Same user, different conversation should fail
- _, err = svc.GetByUserAndConversation("conv-file", "user-123", "different-conv")
- if err == nil {
- t.Error("Different conversation should be denied")
- }
-
- // Different user, same conversation should fail
- _, err = svc.GetByUserAndConversation("conv-file", "different-user", "conv-456")
- if err == nil {
- t.Error("Different user should be denied")
- }
-}
-
-// TestDelete tests file deletion
-func TestDelete(t *testing.T) {
- svc := NewService()
-
- file := &CachedFile{
- FileID: "delete-me",
- UserID: "user-123",
- Filename: "to-delete.pdf",
- }
- svc.Store(file)
-
- // Verify it exists
- _, found := svc.Get("delete-me")
- if !found {
- t.Fatal("File should exist before deletion")
- }
-
- // Delete it
- svc.Delete("delete-me")
-
- // Verify it's gone
- _, found = svc.Get("delete-me")
- if found {
- t.Error("File should not exist after deletion")
- }
-}
-
-// TestDeleteConversationFiles tests conversation-level deletion
-func TestDeleteConversationFiles(t *testing.T) {
- svc := NewService()
-
- // Store multiple files for same conversation
- for i := 0; i < 3; i++ {
- svc.Store(&CachedFile{
- FileID: "conv-file-" + string(rune('a'+i)),
- UserID: "user-123",
- ConversationID: "conv-to-delete",
- Filename: "file.pdf",
- })
- }
-
- // Store file for different conversation
- svc.Store(&CachedFile{
- FileID: "other-file",
- UserID: "user-123",
- ConversationID: "other-conv",
- Filename: "other.pdf",
- })
-
- // Delete conversation files
- svc.DeleteConversationFiles("conv-to-delete")
-
- // Conversation files should be gone
- files := svc.GetFilesForConversation("conv-to-delete")
- if len(files) != 0 {
- t.Errorf("Expected 0 files after deletion, got %d", len(files))
- }
-
- // Other conversation's file should remain
- _, found := svc.Get("other-file")
- if !found {
- t.Error("File from other conversation should still exist")
- }
-}
-
-// TestGetFilesForConversation tests conversation-level retrieval
-func TestGetFilesForConversation(t *testing.T) {
- svc := NewService()
-
- targetConv := "target-conv"
-
- // Store files for target conversation
- for i := 0; i < 3; i++ {
- svc.Store(&CachedFile{
- FileID: "target-file-" + string(rune('a'+i)),
- UserID: "user-123",
- ConversationID: targetConv,
- Filename: "file.pdf",
- })
- }
-
- // Store files for other conversation
- svc.Store(&CachedFile{
- FileID: "other-file",
- UserID: "user-123",
- ConversationID: "other-conv",
- Filename: "other.pdf",
- })
-
- files := svc.GetFilesForConversation(targetConv)
- if len(files) != 3 {
- t.Errorf("Expected 3 files for conversation, got %d", len(files))
- }
-
- for _, file := range files {
- if file.ConversationID != targetConv {
- t.Errorf("File %s has wrong conversation %s", file.FileID, file.ConversationID)
- }
- }
-}
-
-// TestGetConversationFiles tests file ID retrieval
-func TestGetConversationFiles(t *testing.T) {
- svc := NewService()
-
- conv := "my-conv"
- expectedIDs := []string{"file-1", "file-2", "file-3"}
-
- for _, id := range expectedIDs {
- svc.Store(&CachedFile{
- FileID: id,
- ConversationID: conv,
- })
- }
-
- fileIDs := svc.GetConversationFiles(conv)
- if len(fileIDs) != len(expectedIDs) {
- t.Errorf("Expected %d file IDs, got %d", len(expectedIDs), len(fileIDs))
- }
-}
-
-// TestExtendTTL tests TTL extension
-func TestExtendTTL(t *testing.T) {
- svc := NewService()
-
- file := &CachedFile{
- FileID: "ttl-file",
- Filename: "ttl.pdf",
- }
- svc.Store(file)
-
- // Extend TTL (should not error)
- svc.ExtendTTL("ttl-file", 1*time.Hour)
-
- // Verify file still exists
- _, found := svc.Get("ttl-file")
- if !found {
- t.Error("File should still exist after TTL extension")
- }
-
- // Extend non-existent file (should not error)
- svc.ExtendTTL("non-existent", 1*time.Hour)
-}
-
-// TestSecureWipe tests secure wiping
-func TestSecureWipe(t *testing.T) {
- // Create temp file
- tmpDir := t.TempDir()
- tmpFile := filepath.Join(tmpDir, "wipe-test.txt")
- if err := os.WriteFile(tmpFile, []byte("test content"), 0644); err != nil {
- t.Fatalf("Failed to create temp file: %v", err)
- }
-
- file := &CachedFile{
- FileID: "wipe-file",
- UserID: "user-123",
- Filename: "wipe-test.txt",
- FilePath: tmpFile,
- ExtractedText: security.NewSecureString("sensitive text"),
- FileHash: security.Hash{1, 2, 3, 4, 5},
- }
-
- file.SecureWipe()
-
- // Verify memory is cleared
- if file.FileID != "" {
- t.Error("FileID should be cleared")
- }
- if file.UserID != "" {
- t.Error("UserID should be cleared")
- }
- if file.ExtractedText != nil {
- t.Error("ExtractedText should be nil")
- }
-
- // Verify hash is zeroed
- for i, b := range file.FileHash {
- if b != 0 {
- t.Errorf("FileHash[%d] should be 0, got %d", i, b)
- }
- }
-
- // Verify physical file is deleted
- if _, err := os.Stat(tmpFile); !os.IsNotExist(err) {
- t.Error("Physical file should be deleted")
- }
-}
-
-// TestGetStats tests statistics retrieval
-func TestGetStats(t *testing.T) {
- svc := NewService()
-
- // Store some files
- svc.Store(&CachedFile{
- FileID: "stats-file-1",
- Size: 1000,
- WordCount: 100,
- })
- svc.Store(&CachedFile{
- FileID: "stats-file-2",
- Size: 2000,
- WordCount: 200,
- })
-
- stats := svc.GetStats()
-
- if stats["total_files"].(int) != 2 {
- t.Errorf("Expected 2 files, got %v", stats["total_files"])
- }
- if stats["total_size"].(int64) != 3000 {
- t.Errorf("Expected total size 3000, got %v", stats["total_size"])
- }
- if stats["total_words"].(int) != 300 {
- t.Errorf("Expected 300 words, got %v", stats["total_words"])
- }
-}
-
-// TestGetAllFilesByUser tests user file listing
-func TestGetAllFilesByUser(t *testing.T) {
- svc := NewService()
-
- targetUser := "target-user"
-
- // Store files for target user
- svc.Store(&CachedFile{
- FileID: "user-file-1",
- UserID: targetUser,
- Filename: "file1.pdf",
- MimeType: "application/pdf",
- Size: 1000,
- UploadedAt: time.Now(),
- })
- svc.Store(&CachedFile{
- FileID: "user-file-2",
- UserID: targetUser,
- Filename: "file2.jpg",
- MimeType: "image/jpeg",
- Size: 2000,
- UploadedAt: time.Now(),
- })
-
- // Store file for different user
- svc.Store(&CachedFile{
- FileID: "other-file",
- UserID: "other-user",
- Filename: "other.pdf",
- })
-
- files := svc.GetAllFilesByUser(targetUser)
- if len(files) != 2 {
- t.Errorf("Expected 2 files for user, got %d", len(files))
- }
-
- for _, metadata := range files {
- if metadata["file_id"] == "other-file" {
- t.Error("Should not return files from other users")
- }
- }
-}
-
-// TestDeleteAllFilesByUser tests GDPR-style user data deletion
-func TestDeleteAllFilesByUser(t *testing.T) {
- svc := NewService()
-
- targetUser := "delete-user"
-
- // Store files for target user
- for i := 0; i < 5; i++ {
- svc.Store(&CachedFile{
- FileID: "delete-file-" + string(rune('a'+i)),
- UserID: targetUser,
- Filename: "file.pdf",
- })
- }
-
- // Store file for different user
- svc.Store(&CachedFile{
- FileID: "keep-file",
- UserID: "other-user",
- Filename: "keep.pdf",
- })
-
- deleted, err := svc.DeleteAllFilesByUser(targetUser)
- if err != nil {
- t.Errorf("DeleteAllFilesByUser should not error: %v", err)
- }
- if deleted != 5 {
- t.Errorf("Expected 5 files deleted, got %d", deleted)
- }
-
- // Verify target user's files are gone
- files := svc.GetAllFilesByUser(targetUser)
- if len(files) != 0 {
- t.Errorf("Expected 0 files for deleted user, got %d", len(files))
- }
-
- // Verify other user's file remains
- _, found := svc.Get("keep-file")
- if !found {
- t.Error("Other user's file should still exist")
- }
-}
-
-// TestCachedFileStructure tests CachedFile struct
-func TestCachedFileStructure(t *testing.T) {
- file := &CachedFile{
- FileID: "test-id",
- UserID: "user-id",
- ConversationID: "conv-id",
- Filename: "test.pdf",
- MimeType: "application/pdf",
- Size: 12345,
- PageCount: 10,
- WordCount: 500,
- FilePath: "/tmp/test.pdf",
- UploadedAt: time.Now(),
- }
-
- if file.FileID == "" {
- t.Error("FileID should be set")
- }
- if file.MimeType != "application/pdf" {
- t.Errorf("MimeType should be application/pdf, got %s", file.MimeType)
- }
-}
-
-// Benchmark tests
-func BenchmarkStore(b *testing.B) {
- svc := NewService()
-
- b.ResetTimer()
- for i := 0; i < b.N; i++ {
- svc.Store(&CachedFile{
- FileID: "bench-file",
- UserID: "user",
- Filename: "bench.pdf",
- })
- }
-}
-
-func BenchmarkGet(b *testing.B) {
- svc := NewService()
- svc.Store(&CachedFile{
- FileID: "bench-get-file",
- UserID: "user",
- Filename: "bench.pdf",
- })
-
- b.ResetTimer()
- for i := 0; i < b.N; i++ {
- svc.Get("bench-get-file")
- }
-}
-
-func BenchmarkGetByUser(b *testing.B) {
- svc := NewService()
- svc.Store(&CachedFile{
- FileID: "bench-user-file",
- UserID: "target-user",
- Filename: "bench.pdf",
- })
-
- b.ResetTimer()
- for i := 0; i < b.N; i++ {
- svc.GetByUser("bench-user-file", "target-user")
- }
-}
diff --git a/backend/internal/handlers/admin.go b/backend/internal/handlers/admin.go
deleted file mode 100644
index 25147f37..00000000
--- a/backend/internal/handlers/admin.go
+++ /dev/null
@@ -1,746 +0,0 @@
-package handlers
-
-import (
- "claraverse/internal/models"
- "claraverse/internal/services"
- "fmt"
- "log"
-
- "github.com/gofiber/fiber/v2"
-)
-
-// AdminHandler handles admin operations
-type AdminHandler struct {
- userService *services.UserService
- tierService *services.TierService
- analyticsService *services.AnalyticsService
- providerService *services.ProviderService
- modelService *services.ModelService
-}
-
-// NewAdminHandler creates a new admin handler
-func NewAdminHandler(userService *services.UserService, tierService *services.TierService, analyticsService *services.AnalyticsService, providerService *services.ProviderService, modelService *services.ModelService) *AdminHandler {
- return &AdminHandler{
- userService: userService,
- tierService: tierService,
- analyticsService: analyticsService,
- providerService: providerService,
- modelService: modelService,
- }
-}
-
-// GetUserDetails returns detailed user information (admin only)
-// GET /api/admin/users/:userID
-func (h *AdminHandler) GetUserDetails(c *fiber.Ctx) error {
- targetUserID := c.Params("userID")
- if targetUserID == "" {
- return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{
- "error": "User ID is required",
- })
- }
-
- adminUserID := c.Locals("user_id").(string)
- log.Printf("🔍 Admin %s viewing details for user %s", adminUserID, targetUserID)
-
- userDetails, err := h.userService.GetAdminUserDetails(c.Context(), targetUserID, h.tierService)
- if err != nil {
- return c.Status(fiber.StatusNotFound).JSON(fiber.Map{
- "error": "User not found",
- })
- }
-
- return c.JSON(userDetails)
-}
-
-// SetLimitOverrides sets tier OR granular limit overrides for a user (admin only)
-// POST /api/admin/users/:userID/overrides
-func (h *AdminHandler) SetLimitOverrides(c *fiber.Ctx) error {
- targetUserID := c.Params("userID")
- if targetUserID == "" {
- return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{
- "error": "User ID is required",
- })
- }
-
- var req models.SetLimitOverridesRequest
- if err := c.BodyParser(&req); err != nil {
- return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{
- "error": "Invalid request body",
- })
- }
-
- // Validate: must provide either tier OR limits
- if req.Tier == nil && req.Limits == nil {
- return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{
- "error": "Must provide either 'tier' or 'limits'",
- })
- }
-
- if req.Tier != nil && req.Limits != nil {
- return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{
- "error": "Cannot set both 'tier' and 'limits' at the same time",
- })
- }
-
- // Validate tier if provided
- if req.Tier != nil {
- validTiers := []string{
- models.TierFree,
- models.TierPro,
- models.TierMax,
- models.TierEnterprise,
- models.TierLegacyUnlimited,
- }
- isValid := false
- for _, validTier := range validTiers {
- if *req.Tier == validTier {
- isValid = true
- break
- }
- }
- if !isValid {
- return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{
- "error": "Invalid tier",
- })
- }
- }
-
- adminUserID := c.Locals("user_id").(string)
-
- err := h.userService.SetLimitOverrides(
- c.Context(),
- targetUserID,
- adminUserID,
- req.Reason,
- req.Tier,
- req.Limits,
- )
- if err != nil {
- return c.Status(fiber.StatusInternalServerError).JSON(fiber.Map{
- "error": err.Error(),
- })
- }
-
- // Invalidate cache
- h.tierService.InvalidateCache(targetUserID)
-
- var message string
- if req.Tier != nil {
- message = fmt.Sprintf("Tier override set to %s", *req.Tier)
- } else {
- message = "Granular limit overrides set successfully"
- }
-
- return c.JSON(fiber.Map{
- "success": true,
- "message": message,
- })
-}
-
-// RemoveAllOverrides removes all overrides (tier and limits) for a user (admin only)
-// DELETE /api/admin/users/:userID/overrides
-func (h *AdminHandler) RemoveAllOverrides(c *fiber.Ctx) error {
- targetUserID := c.Params("userID")
- if targetUserID == "" {
- return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{
- "error": "User ID is required",
- })
- }
-
- adminUserID := c.Locals("user_id").(string)
-
- err := h.userService.RemoveAllOverrides(c.Context(), targetUserID, adminUserID)
- if err != nil {
- return c.Status(fiber.StatusInternalServerError).JSON(fiber.Map{
- "error": err.Error(),
- })
- }
-
- // Invalidate cache
- h.tierService.InvalidateCache(targetUserID)
-
- return c.JSON(fiber.Map{
- "success": true,
- "message": "All overrides removed successfully",
- })
-}
-
-// ListUsers returns a GDPR-compliant paginated list of users (admin only)
-// GET /api/admin/users
-func (h *AdminHandler) ListUsers(c *fiber.Ctx) error {
- if h.analyticsService == nil {
- return c.Status(fiber.StatusServiceUnavailable).JSON(fiber.Map{
- "error": "Analytics service not available",
- })
- }
-
- // Parse query parameters
- page := c.QueryInt("page", 1)
- pageSize := c.QueryInt("page_size", 50)
- tier := c.Query("tier", "")
- search := c.Query("search", "")
-
- // Get aggregated user analytics (GDPR-compliant - no PII)
- users, totalCount, err := h.analyticsService.GetUserListGDPR(c.Context(), page, pageSize, tier, search)
- if err != nil {
- log.Printf("❌ [ADMIN] Failed to get user list: %v", err)
- return c.Status(fiber.StatusInternalServerError).JSON(fiber.Map{
- "error": "Failed to fetch user list",
- })
- }
-
- return c.JSON(fiber.Map{
- "users": users,
- "total_count": totalCount,
- "page": page,
- "page_size": pageSize,
- "gdpr_notice": "This data is aggregated and anonymized. Full email addresses are hashed for privacy. Only domains are shown for trend analysis.",
- })
-}
-
-// GetGDPRPolicy returns the GDPR data policy
-// GET /api/admin/gdpr-policy
-func (h *AdminHandler) GetGDPRPolicy(c *fiber.Ctx) error {
- return c.JSON(fiber.Map{
- "data_collected": []string{
- "User ID (anonymized)",
- "Email domain (for trend analysis only)",
- "Subscription tier",
- "Usage counts (chats, messages, agent runs)",
- "Activity timestamps",
- },
- "data_retention_days": 90,
- "purpose": "Product analytics and performance monitoring",
- "legal_basis": "Legitimate interest (GDPR Art. 6(1)(f))",
- "user_rights": []string{
- "Right to access (Art. 15)",
- "Right to rectification (Art. 16)",
- "Right to erasure (Art. 17)",
- "Right to data portability (Art. 20)",
- "Right to object (Art. 21)",
- },
- })
-}
-
-// GetSystemStats returns system statistics (admin only)
-// GET /api/admin/stats
-func (h *AdminHandler) GetSystemStats(c *fiber.Ctx) error {
- // TODO: Implement stats like user count by tier, active subscriptions, etc.
- return c.JSON(fiber.Map{
- "message": "System stats endpoint - to be implemented",
- })
-}
-
-// GetAdminStatus returns admin status for the authenticated user
-// GET /api/admin/me
-func (h *AdminHandler) GetAdminStatus(c *fiber.Ctx) error {
- userID := c.Locals("user_id").(string)
- email := c.Locals("email")
- if email == nil {
- email = ""
- }
-
- return c.JSON(fiber.Map{
- "is_admin": true, // If this endpoint is reached, user is admin (middleware validated)
- "user_id": userID,
- "email": email,
- })
-}
-
-// GetOverviewAnalytics returns overview analytics
-// GET /api/admin/analytics/overview
-func (h *AdminHandler) GetOverviewAnalytics(c *fiber.Ctx) error {
- if h.analyticsService == nil {
- return c.Status(fiber.StatusServiceUnavailable).JSON(fiber.Map{
- "error": "Analytics service not available",
- })
- }
-
- stats, err := h.analyticsService.GetOverviewStats(c.Context())
- if err != nil {
- log.Printf("❌ [ADMIN] Failed to get overview analytics: %v", err)
- return c.Status(fiber.StatusInternalServerError).JSON(fiber.Map{
- "error": "Failed to fetch overview analytics",
- })
- }
-
- return c.JSON(stats)
-}
-
-// GetProviderAnalytics returns provider usage analytics
-// GET /api/admin/analytics/providers
-func (h *AdminHandler) GetProviderAnalytics(c *fiber.Ctx) error {
- if h.analyticsService == nil {
- return c.Status(fiber.StatusServiceUnavailable).JSON(fiber.Map{
- "error": "Analytics service not available",
- })
- }
-
- analytics, err := h.analyticsService.GetProviderAnalytics(c.Context())
- if err != nil {
- log.Printf("❌ [ADMIN] Failed to get provider analytics: %v", err)
- return c.Status(fiber.StatusInternalServerError).JSON(fiber.Map{
- "error": "Failed to fetch provider analytics",
- })
- }
-
- return c.JSON(analytics)
-}
-
-// GetChatAnalytics returns chat usage analytics
-// GET /api/admin/analytics/chats
-func (h *AdminHandler) GetChatAnalytics(c *fiber.Ctx) error {
- if h.analyticsService == nil {
- return c.Status(fiber.StatusServiceUnavailable).JSON(fiber.Map{
- "error": "Analytics service not available",
- })
- }
-
- analytics, err := h.analyticsService.GetChatAnalytics(c.Context())
- if err != nil {
- log.Printf("❌ [ADMIN] Failed to get chat analytics: %v", err)
- return c.Status(fiber.StatusInternalServerError).JSON(fiber.Map{
- "error": "Failed to fetch chat analytics",
- })
- }
-
- return c.JSON(analytics)
-}
-
-// GetModelAnalytics returns model usage analytics (placeholder)
-// GET /api/admin/analytics/models
-func (h *AdminHandler) GetModelAnalytics(c *fiber.Ctx) error {
- // TODO: Implement model analytics
- return c.JSON([]fiber.Map{})
-}
-
-// GetAgentAnalytics returns comprehensive agent activity analytics
-// GET /api/admin/analytics/agents
-func (h *AdminHandler) GetAgentAnalytics(c *fiber.Ctx) error {
- if h.analyticsService == nil {
- return c.Status(fiber.StatusServiceUnavailable).JSON(fiber.Map{
- "error": "Analytics service not available",
- })
- }
-
- analytics, err := h.analyticsService.GetAgentAnalytics(c.Context())
- if err != nil {
- log.Printf("❌ [ADMIN] Failed to get agent analytics: %v", err)
- return c.Status(fiber.StatusInternalServerError).JSON(fiber.Map{
- "error": "Failed to fetch agent analytics",
- })
- }
-
- return c.JSON(analytics)
-}
-
-// MigrateChatSessionTimestamps fixes existing chat sessions without proper timestamps
-// POST /api/admin/analytics/migrate-timestamps
-func (h *AdminHandler) MigrateChatSessionTimestamps(c *fiber.Ctx) error {
- if h.analyticsService == nil {
- return c.Status(fiber.StatusServiceUnavailable).JSON(fiber.Map{
- "error": "Analytics service not available",
- })
- }
-
- count, err := h.analyticsService.MigrateChatSessionTimestamps(c.Context())
- if err != nil {
- log.Printf("❌ [ADMIN] Failed to migrate chat session timestamps: %v", err)
- return c.Status(fiber.StatusInternalServerError).JSON(fiber.Map{
- "error": "Failed to migrate chat session timestamps",
- "details": err.Error(),
- })
- }
-
- return c.JSON(fiber.Map{
- "success": true,
- "message": fmt.Sprintf("Successfully migrated %d chat sessions", count),
- "sessions_updated": count,
- })
-}
-
-// GetProviders returns all providers from database
-// GET /api/admin/providers
-func (h *AdminHandler) GetProviders(c *fiber.Ctx) error {
- providers, err := h.providerService.GetAllIncludingDisabled()
- if err != nil {
- log.Printf("❌ [ADMIN] Failed to get providers: %v", err)
- return c.Status(fiber.StatusInternalServerError).JSON(fiber.Map{
- "error": "Failed to get providers",
- })
- }
-
- // Get model counts, aliases, and filters for each provider
- var providerViews []fiber.Map
- for _, provider := range providers {
- // Get model count
- models, err := h.modelService.GetByProvider(provider.ID, false)
- modelCount := 0
- if err == nil {
- modelCount = len(models)
- }
-
- // Get aliases
- aliases, err := h.modelService.LoadAllAliasesFromDB()
- providerAliases := make(map[string]interface{})
- if err == nil && aliases[provider.ID] != nil {
- providerAliases = convertAliasesMapToInterface(aliases[provider.ID])
- }
-
- // Get filters
- filters, _ := h.providerService.GetFilters(provider.ID)
-
- // Get recommended models
- recommended, _ := h.modelService.LoadAllRecommendedModelsFromDB()
- var recommendedModels interface{}
- if recommended[provider.ID] != nil {
- recommendedModels = recommended[provider.ID]
- }
-
- providerView := fiber.Map{
- "id": provider.ID,
- "name": provider.Name,
- "base_url": provider.BaseURL,
- "enabled": provider.Enabled,
- "audio_only": provider.AudioOnly,
- "favicon": provider.Favicon,
- "model_count": modelCount,
- "model_aliases": providerAliases,
- "filters": filters,
- "recommended_models": recommendedModels,
- }
-
- providerViews = append(providerViews, providerView)
- }
-
- return c.JSON(fiber.Map{
- "providers": providerViews,
- })
-}
-
-// CreateProvider creates a new provider
-// POST /api/admin/providers
-func (h *AdminHandler) CreateProvider(c *fiber.Ctx) error {
- var req struct {
- Name string `json:"name"`
- BaseURL string `json:"base_url"`
- APIKey string `json:"api_key"`
- Enabled *bool `json:"enabled"`
- AudioOnly *bool `json:"audio_only"`
- ImageOnly *bool `json:"image_only"`
- ImageEditOnly *bool `json:"image_edit_only"`
- Secure *bool `json:"secure"`
- DefaultModel string `json:"default_model"`
- SystemPrompt string `json:"system_prompt"`
- Favicon string `json:"favicon"`
- }
-
- if err := c.BodyParser(&req); err != nil {
- return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{
- "error": "Invalid request body",
- })
- }
-
- // Validate required fields
- if req.Name == "" || req.BaseURL == "" || req.APIKey == "" {
- return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{
- "error": "Name, base_url, and api_key are required",
- })
- }
-
- // Build provider config
- config := models.ProviderConfig{
- Name: req.Name,
- BaseURL: req.BaseURL,
- APIKey: req.APIKey,
- Enabled: req.Enabled != nil && *req.Enabled,
- AudioOnly: req.AudioOnly != nil && *req.AudioOnly,
- ImageOnly: req.ImageOnly != nil && *req.ImageOnly,
- ImageEditOnly: req.ImageEditOnly != nil && *req.ImageEditOnly,
- Secure: req.Secure != nil && *req.Secure,
- DefaultModel: req.DefaultModel,
- SystemPrompt: req.SystemPrompt,
- Favicon: req.Favicon,
- }
-
- provider, err := h.providerService.Create(config)
- if err != nil {
- log.Printf("❌ [ADMIN] Failed to create provider: %v", err)
- return c.Status(fiber.StatusInternalServerError).JSON(fiber.Map{
- "error": fmt.Sprintf("Failed to create provider: %v", err),
- })
- }
-
- log.Printf("✅ [ADMIN] Created provider: %s (ID %d)", provider.Name, provider.ID)
-
- // Reload image providers if this is an image provider
- if config.ImageOnly || config.ImageEditOnly {
- h.reloadImageProviders()
- }
-
- return c.Status(fiber.StatusCreated).JSON(provider)
-}
-
-// UpdateProvider updates an existing provider
-// PUT /api/admin/providers/:id
-func (h *AdminHandler) UpdateProvider(c *fiber.Ctx) error {
- providerID, err := c.ParamsInt("id")
- if err != nil {
- return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{
- "error": "Invalid provider ID",
- })
- }
-
- var req struct {
- Name *string `json:"name"`
- BaseURL *string `json:"base_url"`
- APIKey *string `json:"api_key"`
- Enabled *bool `json:"enabled"`
- AudioOnly *bool `json:"audio_only"`
- ImageOnly *bool `json:"image_only"`
- ImageEditOnly *bool `json:"image_edit_only"`
- Secure *bool `json:"secure"`
- DefaultModel *string `json:"default_model"`
- SystemPrompt *string `json:"system_prompt"`
- Favicon *string `json:"favicon"`
- }
-
- if err := c.BodyParser(&req); err != nil {
- return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{
- "error": "Invalid request body",
- })
- }
-
- // Get existing provider
- existing, err := h.providerService.GetByID(providerID)
- if err != nil {
- return c.Status(fiber.StatusNotFound).JSON(fiber.Map{
- "error": "Provider not found",
- })
- }
-
- // Build update config with existing values as defaults
- config := models.ProviderConfig{
- Name: existing.Name,
- BaseURL: existing.BaseURL,
- APIKey: existing.APIKey,
- Enabled: existing.Enabled,
- AudioOnly: existing.AudioOnly,
- SystemPrompt: existing.SystemPrompt,
- Favicon: existing.Favicon,
- }
-
- // Apply updates
- if req.Name != nil {
- config.Name = *req.Name
- }
- if req.BaseURL != nil {
- config.BaseURL = *req.BaseURL
- }
- if req.APIKey != nil {
- config.APIKey = *req.APIKey
- }
- if req.Enabled != nil {
- config.Enabled = *req.Enabled
- }
- if req.AudioOnly != nil {
- config.AudioOnly = *req.AudioOnly
- }
- if req.ImageOnly != nil {
- config.ImageOnly = *req.ImageOnly
- }
- if req.ImageEditOnly != nil {
- config.ImageEditOnly = *req.ImageEditOnly
- }
- if req.Secure != nil {
- config.Secure = *req.Secure
- }
- if req.DefaultModel != nil {
- config.DefaultModel = *req.DefaultModel
- }
- if req.SystemPrompt != nil {
- config.SystemPrompt = *req.SystemPrompt
- }
- if req.Favicon != nil {
- config.Favicon = *req.Favicon
- }
-
- if err := h.providerService.Update(providerID, config); err != nil {
- log.Printf("❌ [ADMIN] Failed to update provider %d: %v", providerID, err)
- return c.Status(fiber.StatusInternalServerError).JSON(fiber.Map{
- "error": fmt.Sprintf("Failed to update provider: %v", err),
- })
- }
-
- // Get updated provider
- updated, err := h.providerService.GetByID(providerID)
- if err != nil {
- return c.Status(fiber.StatusInternalServerError).JSON(fiber.Map{
- "error": "Failed to retrieve updated provider",
- })
- }
-
- log.Printf("✅ [ADMIN] Updated provider: %s (ID %d)", updated.Name, updated.ID)
-
- // Reload image providers if image flags changed
- if (req.ImageOnly != nil && *req.ImageOnly) || (req.ImageEditOnly != nil && *req.ImageEditOnly) ||
- updated.ImageOnly || updated.ImageEditOnly {
- h.reloadImageProviders()
- }
-
- return c.JSON(updated)
-}
-
-// DeleteProvider deletes a provider
-// DELETE /api/admin/providers/:id
-func (h *AdminHandler) DeleteProvider(c *fiber.Ctx) error {
- providerID, err := c.ParamsInt("id")
- if err != nil {
- return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{
- "error": "Invalid provider ID",
- })
- }
-
- // Get provider before deleting for logging
- provider, err := h.providerService.GetByID(providerID)
- if err != nil {
- return c.Status(fiber.StatusNotFound).JSON(fiber.Map{
- "error": "Provider not found",
- })
- }
-
- if err := h.providerService.Delete(providerID); err != nil {
- log.Printf("❌ [ADMIN] Failed to delete provider %d: %v", providerID, err)
- return c.Status(fiber.StatusInternalServerError).JSON(fiber.Map{
- "error": fmt.Sprintf("Failed to delete provider: %v", err),
- })
- }
-
- log.Printf("✅ [ADMIN] Deleted provider: %s (ID %d)", provider.Name, provider.ID)
- return c.JSON(fiber.Map{
- "message": "Provider deleted successfully",
- })
-}
-
-// ToggleProvider toggles a provider's enabled state
-// PUT /api/admin/providers/:id/toggle
-func (h *AdminHandler) ToggleProvider(c *fiber.Ctx) error {
- providerID, err := c.ParamsInt("id")
- if err != nil {
- return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{
- "error": "Invalid provider ID",
- })
- }
-
- var req struct {
- Enabled bool `json:"enabled"`
- }
-
- if err := c.BodyParser(&req); err != nil {
- return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{
- "error": "Invalid request body",
- })
- }
-
- // Get existing provider
- provider, err := h.providerService.GetByID(providerID)
- if err != nil {
- return c.Status(fiber.StatusNotFound).JSON(fiber.Map{
- "error": "Provider not found",
- })
- }
-
- // Update enabled state
- config := models.ProviderConfig{
- Name: provider.Name,
- BaseURL: provider.BaseURL,
- APIKey: provider.APIKey,
- Enabled: req.Enabled,
- AudioOnly: provider.AudioOnly,
- SystemPrompt: provider.SystemPrompt,
- Favicon: provider.Favicon,
- }
-
- if err := h.providerService.Update(providerID, config); err != nil {
- log.Printf("❌ [ADMIN] Failed to toggle provider %d: %v", providerID, err)
- return c.Status(fiber.StatusInternalServerError).JSON(fiber.Map{
- "error": fmt.Sprintf("Failed to toggle provider: %v", err),
- })
- }
-
- // Get updated provider
- updated, err := h.providerService.GetByID(providerID)
- if err != nil {
- return c.Status(fiber.StatusInternalServerError).JSON(fiber.Map{
- "error": "Failed to retrieve updated provider",
- })
- }
-
- log.Printf("✅ [ADMIN] Toggled provider %s to enabled=%v", updated.Name, updated.Enabled)
- return c.JSON(updated)
-}
-
-// Helper function to convert ModelAlias map to interface{} map for JSON
-func convertAliasesMapToInterface(aliases map[string]models.ModelAlias) map[string]interface{} {
- result := make(map[string]interface{})
- for key, alias := range aliases {
- result[key] = fiber.Map{
- "actual_model": alias.ActualModel,
- "display_name": alias.DisplayName,
- "description": alias.Description,
- "supports_vision": alias.SupportsVision,
- "agents": alias.Agents,
- "smart_tool_router": alias.SmartToolRouter,
- "free_tier": alias.FreeTier,
- "structured_output_support": alias.StructuredOutputSupport,
- "structured_output_compliance": alias.StructuredOutputCompliance,
- "structured_output_warning": alias.StructuredOutputWarning,
- "structured_output_speed_ms": alias.StructuredOutputSpeedMs,
- "structured_output_badge": alias.StructuredOutputBadge,
- "memory_extractor": alias.MemoryExtractor,
- "memory_selector": alias.MemorySelector,
- }
- }
- return result
-}
-
-// reloadImageProviders reloads image providers from the database
-// Called after creating/updating providers with image_only or image_edit_only flags
-func (h *AdminHandler) reloadImageProviders() {
- log.Println("🔄 [ADMIN] Reloading image providers...")
-
- // Get all providers from database
- allProviders, err := h.providerService.GetAll()
- if err != nil {
- log.Printf("⚠️ [ADMIN] Failed to reload providers: %v", err)
- return
- }
-
- // Convert to ProviderConfig format
- var providerConfigs []models.ProviderConfig
- for _, p := range allProviders {
- providerConfigs = append(providerConfigs, models.ProviderConfig{
- Name: p.Name,
- BaseURL: p.BaseURL,
- APIKey: p.APIKey,
- Enabled: p.Enabled,
- Secure: p.Secure,
- AudioOnly: p.AudioOnly,
- ImageOnly: p.ImageOnly,
- ImageEditOnly: p.ImageEditOnly,
- DefaultModel: p.DefaultModel,
- SystemPrompt: p.SystemPrompt,
- Favicon: p.Favicon,
- })
- }
-
- // Reload image providers
- imageProviderService := services.GetImageProviderService()
- imageProviderService.LoadFromProviders(providerConfigs)
-
- // Reload image edit providers
- imageEditProviderService := services.GetImageEditProviderService()
- imageEditProviderService.LoadFromProviders(providerConfigs)
-
- log.Println("✅ [ADMIN] Image providers reloaded")
-}
diff --git a/backend/internal/handlers/agent.go b/backend/internal/handlers/agent.go
deleted file mode 100644
index 8cde2262..00000000
--- a/backend/internal/handlers/agent.go
+++ /dev/null
@@ -1,1243 +0,0 @@
-package handlers
-
-import (
- "bytes"
- "claraverse/internal/models"
- "claraverse/internal/services"
- "encoding/json"
- "fmt"
- "io"
- "log"
- "net/http"
- "strings"
- "time"
-
- "github.com/gofiber/fiber/v2"
-)
-
-// isPlaceholderDescription checks if a description is empty or a placeholder
-func isPlaceholderDescription(desc string) bool {
- if desc == "" {
- return true
- }
- // Normalize for comparison
- lower := strings.ToLower(strings.TrimSpace(desc))
- // Common placeholder patterns
- placeholders := []string{
- "describe what this agent does",
- "description",
- "add a description",
- "enter description",
- "agent description",
- "no description",
- "...",
- "-",
- }
- for _, p := range placeholders {
- if lower == p || strings.HasPrefix(lower, p) {
- return true
- }
- }
- return false
-}
-
-// AgentHandler handles agent-related HTTP requests
-type AgentHandler struct {
- agentService *services.AgentService
- workflowGeneratorService *services.WorkflowGeneratorService
- workflowGeneratorV2Service *services.WorkflowGeneratorV2Service
- builderConvService *services.BuilderConversationService
- providerService *services.ProviderService
-}
-
-// NewAgentHandler creates a new agent handler
-func NewAgentHandler(agentService *services.AgentService, workflowGenerator *services.WorkflowGeneratorService) *AgentHandler {
- return &AgentHandler{
- agentService: agentService,
- workflowGeneratorService: workflowGenerator,
- }
-}
-
-// SetWorkflowGeneratorV2Service sets the v2 workflow generator service
-func (h *AgentHandler) SetWorkflowGeneratorV2Service(svc *services.WorkflowGeneratorV2Service) {
- h.workflowGeneratorV2Service = svc
-}
-
-// SetBuilderConversationService sets the builder conversation service (for sync endpoint)
-func (h *AgentHandler) SetBuilderConversationService(svc *services.BuilderConversationService) {
- h.builderConvService = svc
-}
-
-// SetProviderService sets the provider service (for Ask mode)
-func (h *AgentHandler) SetProviderService(svc *services.ProviderService) {
- h.providerService = svc
-}
-
-// Create creates a new agent
-// POST /api/agents
-func (h *AgentHandler) Create(c *fiber.Ctx) error {
- userID, ok := c.Locals("user_id").(string)
- if !ok || userID == "" {
- return c.Status(fiber.StatusUnauthorized).JSON(fiber.Map{
- "error": "Authentication required",
- })
- }
-
- var req models.CreateAgentRequest
- if err := c.BodyParser(&req); err != nil {
- return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{
- "error": "Invalid request body",
- })
- }
-
- if req.Name == "" {
- return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{
- "error": "Name is required",
- })
- }
-
- log.Printf("🤖 [AGENT] Creating agent '%s' for user %s", req.Name, userID)
-
- agent, err := h.agentService.CreateAgent(userID, req.Name, req.Description)
- if err != nil {
- log.Printf("❌ [AGENT] Failed to create agent: %v", err)
- return c.Status(fiber.StatusInternalServerError).JSON(fiber.Map{
- "error": "Failed to create agent",
- })
- }
-
- log.Printf("✅ [AGENT] Created agent %s", agent.ID)
- return c.Status(fiber.StatusCreated).JSON(agent)
-}
-
-// List returns all agents for the authenticated user with pagination
-// GET /api/agents?limit=20&offset=0
-func (h *AgentHandler) List(c *fiber.Ctx) error {
- userID, ok := c.Locals("user_id").(string)
- if !ok || userID == "" {
- return c.Status(fiber.StatusUnauthorized).JSON(fiber.Map{
- "error": "Authentication required",
- })
- }
-
- limit := c.QueryInt("limit", 20)
- offset := c.QueryInt("offset", 0)
-
- log.Printf("📋 [AGENT] Listing agents for user %s (limit: %d, offset: %d)", userID, limit, offset)
-
- response, err := h.agentService.ListAgentsPaginated(userID, limit, offset)
- if err != nil {
- log.Printf("❌ [AGENT] Failed to list agents: %v", err)
- return c.Status(fiber.StatusInternalServerError).JSON(fiber.Map{
- "error": "Failed to list agents",
- })
- }
-
- // Ensure agents array is not null
- if response.Agents == nil {
- response.Agents = []models.AgentListItem{}
- }
-
- return c.JSON(response)
-}
-
-// ListRecent returns the 10 most recent agents for the landing page
-// GET /api/agents/recent
-func (h *AgentHandler) ListRecent(c *fiber.Ctx) error {
- userID, ok := c.Locals("user_id").(string)
- if !ok || userID == "" {
- return c.Status(fiber.StatusUnauthorized).JSON(fiber.Map{
- "error": "Authentication required",
- })
- }
-
- log.Printf("📋 [AGENT] Getting recent agents for user %s", userID)
-
- response, err := h.agentService.GetRecentAgents(userID)
- if err != nil {
- log.Printf("❌ [AGENT] Failed to get recent agents: %v", err)
- return c.Status(fiber.StatusInternalServerError).JSON(fiber.Map{
- "error": "Failed to get recent agents",
- })
- }
-
- // Ensure agents array is not null
- if response.Agents == nil {
- response.Agents = []models.AgentListItem{}
- }
-
- return c.JSON(response)
-}
-
-// Get returns a single agent by ID
-// GET /api/agents/:id
-func (h *AgentHandler) Get(c *fiber.Ctx) error {
- userID, ok := c.Locals("user_id").(string)
- if !ok || userID == "" {
- return c.Status(fiber.StatusUnauthorized).JSON(fiber.Map{
- "error": "Authentication required",
- })
- }
-
- agentID := c.Params("id")
- if agentID == "" {
- return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{
- "error": "Agent ID is required",
- })
- }
-
- log.Printf("🔍 [AGENT] Getting agent %s for user %s", agentID, userID)
-
- agent, err := h.agentService.GetAgent(agentID, userID)
- if err != nil {
- if err.Error() == "agent not found" {
- return c.Status(fiber.StatusNotFound).JSON(fiber.Map{
- "error": "Agent not found",
- })
- }
- log.Printf("❌ [AGENT] Failed to get agent: %v", err)
- return c.Status(fiber.StatusInternalServerError).JSON(fiber.Map{
- "error": "Failed to get agent",
- })
- }
-
- return c.JSON(agent)
-}
-
-// Update updates an agent's metadata
-// PUT /api/agents/:id
-func (h *AgentHandler) Update(c *fiber.Ctx) error {
- userID, ok := c.Locals("user_id").(string)
- if !ok || userID == "" {
- return c.Status(fiber.StatusUnauthorized).JSON(fiber.Map{
- "error": "Authentication required",
- })
- }
-
- agentID := c.Params("id")
- if agentID == "" {
- return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{
- "error": "Agent ID is required",
- })
- }
-
- var req models.UpdateAgentRequest
- if err := c.BodyParser(&req); err != nil {
- return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{
- "error": "Invalid request body",
- })
- }
-
- log.Printf("✏️ [AGENT] Updating agent %s for user %s", agentID, userID)
-
- // Check if we're deploying and need to auto-generate a description
- if req.Status == "deployed" {
- // Get current agent to check if description is empty or placeholder
- currentAgent, err := h.agentService.GetAgent(agentID, userID)
- if err == nil && currentAgent != nil {
- // Auto-generate description if empty or a placeholder
- if isPlaceholderDescription(currentAgent.Description) {
- log.Printf("🔍 [AGENT] Agent %s has no/placeholder description, generating one on deploy", agentID)
- workflow, err := h.agentService.GetWorkflow(agentID)
- if err == nil && workflow != nil {
- description, err := h.workflowGeneratorService.GenerateDescriptionFromWorkflow(workflow, currentAgent.Name)
- if err != nil {
- log.Printf("⚠️ [AGENT] Failed to generate description (non-fatal): %v", err)
- } else if description != "" {
- req.Description = description
- log.Printf("📝 [AGENT] Auto-generated description for agent %s: %s", agentID, description)
- }
- }
- }
- }
- }
-
- agent, err := h.agentService.UpdateAgent(agentID, userID, &req)
- if err != nil {
- if err.Error() == "agent not found" {
- return c.Status(fiber.StatusNotFound).JSON(fiber.Map{
- "error": "Agent not found",
- })
- }
- log.Printf("❌ [AGENT] Failed to update agent: %v", err)
- return c.Status(fiber.StatusInternalServerError).JSON(fiber.Map{
- "error": "Failed to update agent",
- })
- }
-
- log.Printf("✅ [AGENT] Updated agent %s", agentID)
- return c.JSON(agent)
-}
-
-// Delete deletes an agent
-// DELETE /api/agents/:id
-func (h *AgentHandler) Delete(c *fiber.Ctx) error {
- userID, ok := c.Locals("user_id").(string)
- if !ok || userID == "" {
- return c.Status(fiber.StatusUnauthorized).JSON(fiber.Map{
- "error": "Authentication required",
- })
- }
-
- agentID := c.Params("id")
- if agentID == "" {
- return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{
- "error": "Agent ID is required",
- })
- }
-
- log.Printf("🗑️ [AGENT] Deleting agent %s for user %s", agentID, userID)
-
- err := h.agentService.DeleteAgent(agentID, userID)
- if err != nil {
- if err.Error() == "agent not found" {
- return c.Status(fiber.StatusNotFound).JSON(fiber.Map{
- "error": "Agent not found",
- })
- }
- log.Printf("❌ [AGENT] Failed to delete agent: %v", err)
- return c.Status(fiber.StatusInternalServerError).JSON(fiber.Map{
- "error": "Failed to delete agent",
- })
- }
-
- log.Printf("✅ [AGENT] Deleted agent %s", agentID)
- return c.Status(fiber.StatusNoContent).Send(nil)
-}
-
-// SaveWorkflow saves or updates the workflow for an agent
-// PUT /api/agents/:id/workflow
-func (h *AgentHandler) SaveWorkflow(c *fiber.Ctx) error {
- userID, ok := c.Locals("user_id").(string)
- if !ok || userID == "" {
- return c.Status(fiber.StatusUnauthorized).JSON(fiber.Map{
- "error": "Authentication required",
- })
- }
-
- agentID := c.Params("id")
- if agentID == "" {
- return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{
- "error": "Agent ID is required",
- })
- }
-
- var req models.SaveWorkflowRequest
- if err := c.BodyParser(&req); err != nil {
- return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{
- "error": "Invalid request body",
- })
- }
-
- log.Printf("💾 [AGENT] Saving workflow for agent %s (user: %s, blocks: %d)", agentID, userID, len(req.Blocks))
-
- workflow, err := h.agentService.SaveWorkflow(agentID, userID, &req)
- if err != nil {
- if err.Error() == "agent not found" {
- return c.Status(fiber.StatusNotFound).JSON(fiber.Map{
- "error": "Agent not found",
- })
- }
- log.Printf("❌ [AGENT] Failed to save workflow: %v", err)
- return c.Status(fiber.StatusInternalServerError).JSON(fiber.Map{
- "error": "Failed to save workflow",
- })
- }
-
- log.Printf("✅ [AGENT] Saved workflow for agent %s (version: %d)", agentID, workflow.Version)
- return c.JSON(workflow)
-}
-
-// GetWorkflow returns the workflow for an agent
-// GET /api/agents/:id/workflow
-func (h *AgentHandler) GetWorkflow(c *fiber.Ctx) error {
- userID, ok := c.Locals("user_id").(string)
- if !ok || userID == "" {
- return c.Status(fiber.StatusUnauthorized).JSON(fiber.Map{
- "error": "Authentication required",
- })
- }
-
- agentID := c.Params("id")
- if agentID == "" {
- return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{
- "error": "Agent ID is required",
- })
- }
-
- // Verify agent belongs to user
- _, err := h.agentService.GetAgent(agentID, userID)
- if err != nil {
- if err.Error() == "agent not found" {
- return c.Status(fiber.StatusNotFound).JSON(fiber.Map{
- "error": "Agent not found",
- })
- }
- return c.Status(fiber.StatusInternalServerError).JSON(fiber.Map{
- "error": "Failed to verify agent ownership",
- })
- }
-
- workflow, err := h.agentService.GetWorkflow(agentID)
- if err != nil {
- if err.Error() == "workflow not found" {
- return c.Status(fiber.StatusNotFound).JSON(fiber.Map{
- "error": "Workflow not found",
- })
- }
- log.Printf("❌ [AGENT] Failed to get workflow: %v", err)
- return c.Status(fiber.StatusInternalServerError).JSON(fiber.Map{
- "error": "Failed to get workflow",
- })
- }
-
- return c.JSON(workflow)
-}
-
-// GenerateWorkflow generates or modifies a workflow using AI
-// POST /api/agents/:id/generate-workflow
-func (h *AgentHandler) GenerateWorkflow(c *fiber.Ctx) error {
- userID, ok := c.Locals("user_id").(string)
- if !ok || userID == "" {
- return c.Status(fiber.StatusUnauthorized).JSON(fiber.Map{
- "error": "Authentication required",
- })
- }
-
- agentID := c.Params("id")
- if agentID == "" {
- return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{
- "error": "Agent ID is required",
- })
- }
-
- // Parse request body
- var req models.WorkflowGenerateRequest
- if err := c.BodyParser(&req); err != nil {
- return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{
- "error": "Invalid request body",
- })
- }
-
- if req.UserMessage == "" {
- return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{
- "error": "User message is required",
- })
- }
-
- req.AgentID = agentID
-
- // Get or create the agent - auto-create if it doesn't exist yet
- // This supports the frontend workflow where agent IDs are generated client-side
- agent, err := h.agentService.GetAgent(agentID, userID)
- if err != nil {
- if err.Error() == "agent not found" {
- // Auto-create the agent with a default name (user can rename later)
- log.Printf("🆕 [WORKFLOW-GEN] Agent %s doesn't exist, creating it", agentID)
- agent, err = h.agentService.CreateAgentWithID(agentID, userID, "New Agent", "")
- if err != nil {
- log.Printf("❌ [WORKFLOW-GEN] Failed to auto-create agent: %v", err)
- return c.Status(fiber.StatusInternalServerError).JSON(fiber.Map{
- "error": "Failed to create agent",
- })
- }
- log.Printf("✅ [WORKFLOW-GEN] Auto-created agent %s", agentID)
- } else {
- return c.Status(fiber.StatusInternalServerError).JSON(fiber.Map{
- "error": "Failed to verify agent ownership",
- })
- }
- }
- _ = agent // Agent verified or created
-
- log.Printf("🔧 [WORKFLOW-GEN] Generating workflow for agent %s (user: %s)", agentID, userID)
-
- // Generate the workflow
- response, err := h.workflowGeneratorService.GenerateWorkflow(&req, userID)
- if err != nil {
- log.Printf("❌ [WORKFLOW-GEN] Failed to generate workflow: %v", err)
- return c.Status(fiber.StatusInternalServerError).JSON(fiber.Map{
- "error": "Failed to generate workflow",
- })
- }
-
- if !response.Success {
- log.Printf("⚠️ [WORKFLOW-GEN] Workflow generation failed: %s", response.Error)
- return c.Status(fiber.StatusUnprocessableEntity).JSON(response)
- }
-
- // Generate suggested name and description for new workflows
- // Check if agent still has default name - if so, generate a better one
- shouldGenerateMetadata := response.Action == "create" || (agent != nil && agent.Name == "New Agent")
- log.Printf("🔍 [WORKFLOW-GEN] Checking metadata generation: action=%s, agentName=%s, shouldGenerate=%v", response.Action, agent.Name, shouldGenerateMetadata)
- if shouldGenerateMetadata {
- metadata, err := h.workflowGeneratorService.GenerateAgentMetadata(req.UserMessage)
- if err != nil {
- log.Printf("⚠️ [WORKFLOW-GEN] Failed to generate agent metadata (non-fatal): %v", err)
- } else {
- response.SuggestedName = metadata.Name
- response.SuggestedDescription = metadata.Description
- log.Printf("📝 [WORKFLOW-GEN] Suggested agent: name=%s, desc=%s", metadata.Name, metadata.Description)
-
- // Immediately persist the generated name to the database
- // This ensures the name is saved even if frontend fails to update
- if metadata.Name != "" {
- updateReq := &models.UpdateAgentRequest{
- Name: metadata.Name,
- Description: metadata.Description,
- }
- _, updateErr := h.agentService.UpdateAgent(agentID, userID, updateReq)
- if updateErr != nil {
- log.Printf("⚠️ [WORKFLOW-GEN] Failed to persist agent metadata (non-fatal): %v", updateErr)
- } else {
- log.Printf("💾 [WORKFLOW-GEN] Persisted agent metadata to database: name=%s", metadata.Name)
- }
- }
- }
- }
-
- log.Printf("✅ [WORKFLOW-GEN] Generated workflow for agent %s: %d blocks", agentID, len(response.Workflow.Blocks))
- return c.JSON(response)
-}
-
-// ============================================================================
-// Workflow Version Handlers
-// ============================================================================
-
-// ListWorkflowVersions returns all versions for an agent's workflow
-// GET /api/agents/:id/workflow/versions
-func (h *AgentHandler) ListWorkflowVersions(c *fiber.Ctx) error {
- userID, ok := c.Locals("user_id").(string)
- if !ok || userID == "" {
- return c.Status(fiber.StatusUnauthorized).JSON(fiber.Map{
- "error": "Authentication required",
- })
- }
-
- agentID := c.Params("id")
- if agentID == "" {
- return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{
- "error": "Agent ID is required",
- })
- }
-
- log.Printf("📜 [WORKFLOW] Listing versions for agent %s (user: %s)", agentID, userID)
-
- versions, err := h.agentService.ListWorkflowVersions(agentID, userID)
- if err != nil {
- if err.Error() == "agent not found" {
- return c.Status(fiber.StatusNotFound).JSON(fiber.Map{
- "error": "Agent not found",
- })
- }
- log.Printf("❌ [WORKFLOW] Failed to list versions: %v", err)
- return c.Status(fiber.StatusInternalServerError).JSON(fiber.Map{
- "error": "Failed to list workflow versions",
- })
- }
-
- return c.JSON(fiber.Map{
- "versions": versions,
- "count": len(versions),
- })
-}
-
-// GetWorkflowVersion returns a specific workflow version
-// GET /api/agents/:id/workflow/versions/:version
-func (h *AgentHandler) GetWorkflowVersion(c *fiber.Ctx) error {
- userID, ok := c.Locals("user_id").(string)
- if !ok || userID == "" {
- return c.Status(fiber.StatusUnauthorized).JSON(fiber.Map{
- "error": "Authentication required",
- })
- }
-
- agentID := c.Params("id")
- if agentID == "" {
- return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{
- "error": "Agent ID is required",
- })
- }
-
- version, err := c.ParamsInt("version")
- if err != nil || version <= 0 {
- return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{
- "error": "Valid version number is required",
- })
- }
-
- log.Printf("🔍 [WORKFLOW] Getting version %d for agent %s (user: %s)", version, agentID, userID)
-
- workflow, err := h.agentService.GetWorkflowVersion(agentID, userID, version)
- if err != nil {
- if err.Error() == "agent not found" {
- return c.Status(fiber.StatusNotFound).JSON(fiber.Map{
- "error": "Agent not found",
- })
- }
- if err.Error() == "workflow version not found" {
- return c.Status(fiber.StatusNotFound).JSON(fiber.Map{
- "error": "Workflow version not found",
- })
- }
- log.Printf("❌ [WORKFLOW] Failed to get version: %v", err)
- return c.Status(fiber.StatusInternalServerError).JSON(fiber.Map{
- "error": "Failed to get workflow version",
- })
- }
-
- return c.JSON(workflow)
-}
-
-// RestoreWorkflowVersion restores a workflow to a previous version
-// POST /api/agents/:id/workflow/restore/:version
-func (h *AgentHandler) RestoreWorkflowVersion(c *fiber.Ctx) error {
- userID, ok := c.Locals("user_id").(string)
- if !ok || userID == "" {
- return c.Status(fiber.StatusUnauthorized).JSON(fiber.Map{
- "error": "Authentication required",
- })
- }
-
- agentID := c.Params("id")
- if agentID == "" {
- return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{
- "error": "Agent ID is required",
- })
- }
-
- version, err := c.ParamsInt("version")
- if err != nil || version <= 0 {
- return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{
- "error": "Valid version number is required",
- })
- }
-
- log.Printf("⏪ [WORKFLOW] Restoring version %d for agent %s (user: %s)", version, agentID, userID)
-
- workflow, err := h.agentService.RestoreWorkflowVersion(agentID, userID, version)
- if err != nil {
- if err.Error() == "agent not found" {
- return c.Status(fiber.StatusNotFound).JSON(fiber.Map{
- "error": "Agent not found",
- })
- }
- if err.Error() == "workflow version not found" {
- return c.Status(fiber.StatusNotFound).JSON(fiber.Map{
- "error": "Workflow version not found",
- })
- }
- log.Printf("❌ [WORKFLOW] Failed to restore version: %v", err)
- return c.Status(fiber.StatusInternalServerError).JSON(fiber.Map{
- "error": "Failed to restore workflow version",
- })
- }
-
- log.Printf("✅ [WORKFLOW] Restored version %d for agent %s (new version: %d)", version, agentID, workflow.Version)
- return c.JSON(workflow)
-}
-
-// SyncAgent syncs a local agent to the backend on first message
-// This creates/updates the agent, workflow, and conversation in one call
-// POST /api/agents/:id/sync
-func (h *AgentHandler) SyncAgent(c *fiber.Ctx) error {
- userID, ok := c.Locals("user_id").(string)
- if !ok || userID == "" {
- return c.Status(fiber.StatusUnauthorized).JSON(fiber.Map{
- "error": "Authentication required",
- })
- }
-
- agentID := c.Params("id")
- if agentID == "" {
- return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{
- "error": "Agent ID is required",
- })
- }
-
- var req models.SyncAgentRequest
- if err := c.BodyParser(&req); err != nil {
- return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{
- "error": "Invalid request body",
- })
- }
-
- if req.Name == "" {
- return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{
- "error": "Agent name is required",
- })
- }
-
- log.Printf("🔄 [AGENT] Syncing agent %s for user %s", agentID, userID)
-
- // Sync agent and workflow
- agent, workflow, err := h.agentService.SyncAgent(agentID, userID, &req)
- if err != nil {
- log.Printf("❌ [AGENT] Failed to sync agent: %v", err)
- return c.Status(fiber.StatusInternalServerError).JSON(fiber.Map{
- "error": "Failed to sync agent",
- })
- }
-
- // Create conversation if builder conversation service is available
- var conversationID string
- if h.builderConvService != nil {
- conv, err := h.builderConvService.CreateConversation(c.Context(), agentID, userID, req.ModelID)
- if err != nil {
- log.Printf("⚠️ [AGENT] Failed to create conversation (non-fatal): %v", err)
- // Continue without conversation - not fatal
- } else {
- conversationID = conv.ID
- log.Printf("✅ [AGENT] Created conversation %s for agent %s", conversationID, agentID)
- }
- }
-
- log.Printf("✅ [AGENT] Synced agent %s (workflow v%d, conv: %s)", agentID, workflow.Version, conversationID)
-
- return c.JSON(&models.SyncAgentResponse{
- Agent: agent,
- Workflow: workflow,
- ConversationID: conversationID,
- })
-}
-
-// GenerateWorkflowV2 generates a workflow using multi-step process with tool selection
-// POST /api/agents/:id/generate-workflow-v2
-func (h *AgentHandler) GenerateWorkflowV2(c *fiber.Ctx) error {
- userID, ok := c.Locals("user_id").(string)
- if !ok || userID == "" {
- return c.Status(fiber.StatusUnauthorized).JSON(fiber.Map{
- "error": "Authentication required",
- })
- }
-
- if h.workflowGeneratorV2Service == nil {
- return c.Status(fiber.StatusInternalServerError).JSON(fiber.Map{
- "error": "Workflow generator v2 service not available",
- })
- }
-
- agentID := c.Params("id")
- if agentID == "" {
- return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{
- "error": "Agent ID is required",
- })
- }
-
- // Parse request body
- var req services.MultiStepGenerateRequest
- if err := c.BodyParser(&req); err != nil {
- return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{
- "error": "Invalid request body",
- })
- }
-
- if req.UserMessage == "" {
- return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{
- "error": "User message is required",
- })
- }
-
- req.AgentID = agentID
-
- // Get or create the agent - auto-create if it doesn't exist yet
- agent, err := h.agentService.GetAgent(agentID, userID)
- if err != nil {
- if err.Error() == "agent not found" {
- log.Printf("🆕 [WORKFLOW-GEN-V2] Agent %s doesn't exist, creating it", agentID)
- agent, err = h.agentService.CreateAgentWithID(agentID, userID, "New Agent", "")
- if err != nil {
- log.Printf("❌ [WORKFLOW-GEN-V2] Failed to auto-create agent: %v", err)
- return c.Status(fiber.StatusInternalServerError).JSON(fiber.Map{
- "error": "Failed to create agent",
- })
- }
- log.Printf("✅ [WORKFLOW-GEN-V2] Auto-created agent %s", agentID)
- } else {
- return c.Status(fiber.StatusInternalServerError).JSON(fiber.Map{
- "error": "Failed to verify agent ownership",
- })
- }
- }
-
- log.Printf("🔧 [WORKFLOW-GEN-V2] Starting multi-step generation for agent %s (user: %s)", agentID, userID)
-
- // Generate the workflow using multi-step process
- response, err := h.workflowGeneratorV2Service.GenerateWorkflowMultiStep(&req, userID, nil)
- if err != nil {
- log.Printf("❌ [WORKFLOW-GEN-V2] Failed to generate workflow: %v", err)
- return c.Status(fiber.StatusInternalServerError).JSON(fiber.Map{
- "error": "Failed to generate workflow",
- })
- }
-
- if !response.Success {
- log.Printf("⚠️ [WORKFLOW-GEN-V2] Workflow generation failed: %s", response.Error)
- return c.Status(fiber.StatusUnprocessableEntity).JSON(response)
- }
-
- // Generate suggested name and description for new workflows
- shouldGenerateMetadata := agent != nil && agent.Name == "New Agent"
- if shouldGenerateMetadata && h.workflowGeneratorService != nil {
- metadata, err := h.workflowGeneratorService.GenerateAgentMetadata(req.UserMessage)
- if err != nil {
- log.Printf("⚠️ [WORKFLOW-GEN-V2] Failed to generate agent metadata (non-fatal): %v", err)
- } else if metadata.Name != "" {
- // Persist the generated name
- updateReq := &models.UpdateAgentRequest{
- Name: metadata.Name,
- Description: metadata.Description,
- }
- _, updateErr := h.agentService.UpdateAgent(agentID, userID, updateReq)
- if updateErr != nil {
- log.Printf("⚠️ [WORKFLOW-GEN-V2] Failed to persist agent metadata (non-fatal): %v", updateErr)
- } else {
- log.Printf("💾 [WORKFLOW-GEN-V2] Persisted agent metadata: name=%s", metadata.Name)
- }
- }
- }
-
- log.Printf("✅ [WORKFLOW-GEN-V2] Generated workflow for agent %s: %d blocks, %d tools selected",
- agentID, len(response.Workflow.Blocks), len(response.SelectedTools))
- return c.JSON(response)
-}
-
-// GetToolRegistry returns all available tools and categories for the frontend
-// GET /api/tools/registry
-func (h *AgentHandler) GetToolRegistry(c *fiber.Ctx) error {
- return c.JSON(fiber.Map{
- "tools": services.ToolRegistry,
- "categories": services.ToolCategoryRegistry,
- })
-}
-
-// SelectTools performs just the tool selection step (Step 1 only)
-// POST /api/agents/:id/select-tools
-func (h *AgentHandler) SelectTools(c *fiber.Ctx) error {
- userID, ok := c.Locals("user_id").(string)
- if !ok || userID == "" {
- return c.Status(fiber.StatusUnauthorized).JSON(fiber.Map{
- "error": "Authentication required",
- })
- }
-
- if h.workflowGeneratorV2Service == nil {
- return c.Status(fiber.StatusInternalServerError).JSON(fiber.Map{
- "error": "Workflow generator v2 service not available",
- })
- }
-
- agentID := c.Params("id")
- if agentID == "" {
- return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{
- "error": "Agent ID is required",
- })
- }
-
- // Parse request body
- var req services.MultiStepGenerateRequest
- if err := c.BodyParser(&req); err != nil {
- return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{
- "error": "Invalid request body",
- })
- }
-
- if req.UserMessage == "" {
- return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{
- "error": "User message is required",
- })
- }
-
- req.AgentID = agentID
-
- log.Printf("🔧 [TOOL-SELECT] Selecting tools for agent %s (user: %s)", agentID, userID)
-
- // Perform tool selection only
- result, err := h.workflowGeneratorV2Service.Step1SelectTools(&req, userID)
- if err != nil {
- log.Printf("❌ [TOOL-SELECT] Failed to select tools: %v", err)
- return c.Status(fiber.StatusInternalServerError).JSON(fiber.Map{
- "error": "Failed to select tools",
- })
- }
-
- log.Printf("✅ [TOOL-SELECT] Selected %d tools for agent %s", len(result.SelectedTools), agentID)
- return c.JSON(result)
-}
-
-// GenerateWithToolsRequest is the request for generating a workflow with pre-selected tools
-type GenerateWithToolsRequest struct {
- UserMessage string `json:"user_message"`
- ModelID string `json:"model_id,omitempty"`
- SelectedTools []services.SelectedTool `json:"selected_tools"`
- CurrentWorkflow *models.Workflow `json:"current_workflow,omitempty"`
-}
-
-// GenerateWithTools performs workflow generation with pre-selected tools (Step 2 only)
-// POST /api/agents/:id/generate-with-tools
-func (h *AgentHandler) GenerateWithTools(c *fiber.Ctx) error {
- userID, ok := c.Locals("user_id").(string)
- if !ok || userID == "" {
- return c.Status(fiber.StatusUnauthorized).JSON(fiber.Map{
- "error": "Authentication required",
- })
- }
-
- if h.workflowGeneratorV2Service == nil {
- return c.Status(fiber.StatusInternalServerError).JSON(fiber.Map{
- "error": "Workflow generator v2 service not available",
- })
- }
-
- agentID := c.Params("id")
- if agentID == "" {
- return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{
- "error": "Agent ID is required",
- })
- }
-
- // Parse request body
- var req GenerateWithToolsRequest
- if err := c.BodyParser(&req); err != nil {
- return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{
- "error": "Invalid request body",
- })
- }
-
- if req.UserMessage == "" {
- return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{
- "error": "User message is required",
- })
- }
-
- if len(req.SelectedTools) == 0 {
- return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{
- "error": "Selected tools are required",
- })
- }
-
- log.Printf("🔧 [GENERATE-WITH-TOOLS] Generating workflow for agent %s with %d pre-selected tools (user: %s)",
- agentID, len(req.SelectedTools), userID)
-
- // Build the multi-step request
- multiStepReq := &services.MultiStepGenerateRequest{
- AgentID: agentID,
- UserMessage: req.UserMessage,
- ModelID: req.ModelID,
- CurrentWorkflow: req.CurrentWorkflow,
- }
-
- // Perform workflow generation with pre-selected tools
- result, err := h.workflowGeneratorV2Service.Step2GenerateWorkflow(multiStepReq, req.SelectedTools, userID)
- if err != nil {
- log.Printf("❌ [GENERATE-WITH-TOOLS] Failed to generate workflow: %v", err)
- return c.Status(fiber.StatusInternalServerError).JSON(fiber.Map{
- "error": "Failed to generate workflow",
- "details": err.Error(),
- })
- }
-
- if !result.Success {
- log.Printf("⚠️ [GENERATE-WITH-TOOLS] Workflow generation failed: %s", result.Error)
- return c.Status(fiber.StatusUnprocessableEntity).JSON(result)
- }
-
- // Generate suggested name and description for new workflows
- agent, _ := h.agentService.GetAgent(agentID, userID)
- shouldGenerateMetadata := agent != nil && agent.Name == "New Agent"
- if shouldGenerateMetadata && h.workflowGeneratorService != nil {
- metadata, err := h.workflowGeneratorService.GenerateAgentMetadata(req.UserMessage)
- if err != nil {
- log.Printf("⚠️ [GENERATE-WITH-TOOLS] Failed to generate agent metadata (non-fatal): %v", err)
- } else if metadata.Name != "" {
- result.SuggestedName = metadata.Name
- result.SuggestedDescription = metadata.Description
-
- // Persist the generated name
- updateReq := &models.UpdateAgentRequest{
- Name: metadata.Name,
- Description: metadata.Description,
- }
- _, updateErr := h.agentService.UpdateAgent(agentID, userID, updateReq)
- if updateErr != nil {
- log.Printf("⚠️ [GENERATE-WITH-TOOLS] Failed to persist agent metadata (non-fatal): %v", updateErr)
- } else {
- log.Printf("💾 [GENERATE-WITH-TOOLS] Persisted agent metadata: name=%s", metadata.Name)
- }
- }
- }
-
- log.Printf("✅ [GENERATE-WITH-TOOLS] Generated workflow for agent %s: %d blocks",
- agentID, len(result.Workflow.Blocks))
- return c.JSON(result)
-}
-
-// GenerateSampleInput uses AI to generate sample JSON input for a workflow
-// POST /api/agents/:id/generate-sample-input
-func (h *AgentHandler) GenerateSampleInput(c *fiber.Ctx) error {
- userID, ok := c.Locals("user_id").(string)
- if !ok || userID == "" {
- return c.Status(fiber.StatusUnauthorized).JSON(fiber.Map{
- "error": "Authentication required",
- })
- }
-
- agentID := c.Params("id")
- if agentID == "" {
- return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{
- "error": "Agent ID is required",
- })
- }
-
- // Parse request body
- var req struct {
- ModelID string `json:"model_id"`
- }
- if err := c.BodyParser(&req); err != nil {
- return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{
- "error": "Invalid request body",
- })
- }
-
- if req.ModelID == "" {
- return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{
- "error": "Model ID is required",
- })
- }
-
- // Get the agent and workflow
- agent, err := h.agentService.GetAgent(agentID, userID)
- if err != nil {
- log.Printf("❌ [SAMPLE-INPUT] Failed to get agent: %v", err)
- return c.Status(fiber.StatusNotFound).JSON(fiber.Map{
- "error": "Agent not found",
- })
- }
-
- if agent.Workflow == nil || len(agent.Workflow.Blocks) == 0 {
- return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{
- "error": "Workflow has no blocks",
- })
- }
-
- // Generate sample input using the workflow generator service
- sampleInput, err := h.workflowGeneratorService.GenerateSampleInput(agent.Workflow, req.ModelID, userID)
- if err != nil {
- log.Printf("❌ [SAMPLE-INPUT] Failed to generate sample input: %v", err)
- return c.Status(fiber.StatusInternalServerError).JSON(fiber.Map{
- "error": "Failed to generate sample input",
- "details": err.Error(),
- })
- }
-
- log.Printf("✅ [SAMPLE-INPUT] Generated sample input for agent %s", agentID)
- return c.JSON(fiber.Map{
- "success": true,
- "sample_input": sampleInput,
- })
-}
-
-// Ask handles Ask mode requests - helps users understand their workflow
-// POST /api/agents/ask
-func (h *AgentHandler) Ask(c *fiber.Ctx) error {
- userID, ok := c.Locals("user_id").(string)
- if !ok || userID == "" {
- return c.Status(fiber.StatusUnauthorized).JSON(fiber.Map{
- "error": "Authentication required",
- })
- }
-
- var req struct {
- AgentID string `json:"agent_id"`
- Message string `json:"message"`
- ModelID string `json:"model_id"`
- Context struct {
- Workflow *models.Workflow `json:"workflow"`
- AvailableTools []map[string]string `json:"available_tools"`
- DeploymentExample string `json:"deployment_example"`
- } `json:"context"`
- }
-
- if err := c.BodyParser(&req); err != nil {
- return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{
- "error": "Invalid request body",
- })
- }
-
- if req.AgentID == "" || req.Message == "" {
- return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{
- "error": "agent_id and message are required",
- })
- }
-
- if h.providerService == nil {
- return c.Status(fiber.StatusInternalServerError).JSON(fiber.Map{
- "error": "Provider service not available",
- })
- }
-
- // Get the agent to verify ownership
- agent, err := h.agentService.GetAgent(req.AgentID, userID)
- if err != nil {
- return c.Status(fiber.StatusNotFound).JSON(fiber.Map{
- "error": "Agent not found",
- })
- }
-
- log.Printf("💬 [ASK] User %s asking about agent %s: %s", userID, agent.Name, req.Message)
-
- // Build context from workflow
- var workflowContext string
- if req.Context.Workflow != nil && len(req.Context.Workflow.Blocks) > 0 {
- workflowContext = "\n\n## Current Workflow Structure\n"
- for i, block := range req.Context.Workflow.Blocks {
- desc := block.Description
- if desc == "" {
- desc = "No description"
- }
- workflowContext += fmt.Sprintf("%d. **%s** (%s): %s\n", i+1, block.Name, block.Type, desc)
- }
- }
-
- // Build tools context
- var toolsContext string
- if len(req.Context.AvailableTools) > 0 {
- toolsContext = "\n\n## Available Tools\n"
- for _, tool := range req.Context.AvailableTools {
- toolsContext += fmt.Sprintf("- **%s**: %s (Category: %s)\n",
- tool["name"], tool["description"], tool["category"])
- }
- }
-
- // Build deployment context
- var deploymentContext string
- if req.Context.DeploymentExample != "" {
- deploymentContext = "\n\n## Deployment API Example\n```bash\n" + req.Context.DeploymentExample + "\n```"
- }
-
- // Build system prompt
- systemPrompt := fmt.Sprintf(`You are an AI assistant helping users understand their workflow agent in ClaraVerse.
-
-**Agent Name**: %s
-**Agent Description**: %s
-
-Your role is to:
-1. Answer questions about the workflow structure and how it works
-2. Explain what tools are available and how to use them
-3. Help with deployment and API integration questions
-4. Provide clear, concise explanations
-
-**IMPORTANT**: You are in "Ask" mode, which is for answering questions only. If the user asks you to modify the workflow (add, change, remove blocks), politely tell them to switch to "Builder" mode.
-
-%s%s%s
-
-Be helpful, clear, and concise. If you don't know something, say so.`,
- agent.Name,
- agent.Description,
- workflowContext,
- toolsContext,
- deploymentContext,
- )
-
- // Call LLM with simple chat endpoint
- modelID := req.ModelID
- if modelID == "" {
- modelID = "gpt-4.1" // Default model
- }
-
- // Get provider for model
- provider, err := h.providerService.GetByModelID(modelID)
- if err != nil {
- log.Printf("❌ [ASK] Failed to get provider for model %s: %v", modelID, err)
- return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{
- "error": fmt.Sprintf("Model '%s' not found", modelID),
- })
- }
-
- // Build OpenAI-compatible request
- type Message struct {
- Role string `json:"role"`
- Content string `json:"content"`
- }
- type OpenAIRequest struct {
- Model string `json:"model"`
- Messages []Message `json:"messages"`
- }
-
- reqBody, err := json.Marshal(OpenAIRequest{
- Model: modelID,
- Messages: []Message{
- {Role: "system", Content: systemPrompt},
- {Role: "user", Content: req.Message},
- },
- })
- if err != nil {
- return c.Status(fiber.StatusInternalServerError).JSON(fiber.Map{
- "error": "Failed to prepare request",
- })
- }
-
- // Make HTTP request
- httpReq, err := http.NewRequest("POST", provider.BaseURL+"/chat/completions", bytes.NewBuffer(reqBody))
- if err != nil {
- return c.Status(fiber.StatusInternalServerError).JSON(fiber.Map{
- "error": "Failed to create HTTP request",
- })
- }
-
- httpReq.Header.Set("Content-Type", "application/json")
- httpReq.Header.Set("Authorization", "Bearer "+provider.APIKey)
-
- client := &http.Client{Timeout: 60 * time.Second}
- resp, err := client.Do(httpReq)
- if err != nil {
- log.Printf("❌ [ASK] HTTP request failed: %v", err)
- return c.Status(fiber.StatusInternalServerError).JSON(fiber.Map{
- "error": "Failed to get response from AI",
- })
- }
- defer resp.Body.Close()
-
- // Read response
- body, err := io.ReadAll(resp.Body)
- if err != nil {
- return c.Status(fiber.StatusInternalServerError).JSON(fiber.Map{
- "error": "Failed to read response",
- })
- }
-
- if resp.StatusCode != http.StatusOK {
- log.Printf("⚠️ [ASK] API error: %s", string(body))
- return c.Status(resp.StatusCode).JSON(fiber.Map{
- "error": fmt.Sprintf("AI service error: %s", string(body)),
- })
- }
-
- // Parse response
- var apiResponse struct {
- Choices []struct {
- Message struct {
- Content string `json:"content"`
- } `json:"message"`
- } `json:"choices"`
- }
-
- if err := json.Unmarshal(body, &apiResponse); err != nil {
- return c.Status(fiber.StatusInternalServerError).JSON(fiber.Map{
- "error": "Failed to parse AI response",
- })
- }
-
- if len(apiResponse.Choices) == 0 {
- return c.Status(fiber.StatusInternalServerError).JSON(fiber.Map{
- "error": "No response from AI",
- })
- }
-
- responseText := apiResponse.Choices[0].Message.Content
-
- log.Printf("✅ [ASK] Response generated for agent %s", agent.Name)
- return c.JSON(fiber.Map{
- "response": responseText,
- })
-}
diff --git a/backend/internal/handlers/apikey.go b/backend/internal/handlers/apikey.go
deleted file mode 100644
index 7709b451..00000000
--- a/backend/internal/handlers/apikey.go
+++ /dev/null
@@ -1,175 +0,0 @@
-package handlers
-
-import (
- "claraverse/internal/models"
- "claraverse/internal/services"
- "log"
-
- "github.com/gofiber/fiber/v2"
- "go.mongodb.org/mongo-driver/bson/primitive"
-)
-
-// APIKeyHandler handles API key management endpoints
-type APIKeyHandler struct {
- apiKeyService *services.APIKeyService
-}
-
-// NewAPIKeyHandler creates a new API key handler
-func NewAPIKeyHandler(apiKeyService *services.APIKeyService) *APIKeyHandler {
- return &APIKeyHandler{
- apiKeyService: apiKeyService,
- }
-}
-
-// Create creates a new API key
-// POST /api/keys
-func (h *APIKeyHandler) Create(c *fiber.Ctx) error {
- userID := c.Locals("user_id").(string)
-
- var req models.CreateAPIKeyRequest
- if err := c.BodyParser(&req); err != nil {
- return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{
- "error": "Invalid request body",
- })
- }
-
- // Validate required fields
- if req.Name == "" {
- return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{
- "error": "Name is required",
- })
- }
-
- if len(req.Scopes) == 0 {
- return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{
- "error": "At least one scope is required",
- })
- }
-
- result, err := h.apiKeyService.Create(c.Context(), userID, &req)
- if err != nil {
- log.Printf("❌ [APIKEY] Failed to create API key: %v", err)
- // Check for limit error
- if err.Error()[:14] == "API key limit" {
- return c.Status(fiber.StatusForbidden).JSON(fiber.Map{
- "error": err.Error(),
- })
- }
- return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{
- "error": err.Error(),
- })
- }
-
- return c.Status(fiber.StatusCreated).JSON(result)
-}
-
-// List lists all API keys for the user
-// GET /api/keys
-func (h *APIKeyHandler) List(c *fiber.Ctx) error {
- userID := c.Locals("user_id").(string)
-
- keys, err := h.apiKeyService.ListByUser(c.Context(), userID)
- if err != nil {
- log.Printf("❌ [APIKEY] Failed to list API keys: %v", err)
- return c.Status(fiber.StatusInternalServerError).JSON(fiber.Map{
- "error": "Failed to list API keys",
- })
- }
-
- if keys == nil {
- keys = []*models.APIKeyListItem{}
- }
-
- return c.JSON(fiber.Map{
- "keys": keys,
- })
-}
-
-// Get retrieves a specific API key
-// GET /api/keys/:id
-func (h *APIKeyHandler) Get(c *fiber.Ctx) error {
- userID := c.Locals("user_id").(string)
- keyIDStr := c.Params("id")
-
- keyID, err := primitive.ObjectIDFromHex(keyIDStr)
- if err != nil {
- return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{
- "error": "Invalid key ID",
- })
- }
-
- key, err := h.apiKeyService.GetByIDAndUser(c.Context(), keyID, userID)
- if err != nil {
- if err.Error() == "API key not found" {
- return c.Status(fiber.StatusNotFound).JSON(fiber.Map{
- "error": "API key not found",
- })
- }
- log.Printf("❌ [APIKEY] Failed to get API key: %v", err)
- return c.Status(fiber.StatusInternalServerError).JSON(fiber.Map{
- "error": "Failed to get API key",
- })
- }
-
- return c.JSON(key.ToListItem())
-}
-
-// Revoke revokes an API key (soft delete)
-// POST /api/keys/:id/revoke
-func (h *APIKeyHandler) Revoke(c *fiber.Ctx) error {
- userID := c.Locals("user_id").(string)
- keyIDStr := c.Params("id")
-
- keyID, err := primitive.ObjectIDFromHex(keyIDStr)
- if err != nil {
- return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{
- "error": "Invalid key ID",
- })
- }
-
- if err := h.apiKeyService.Revoke(c.Context(), keyID, userID); err != nil {
- if err.Error() == "API key not found" {
- return c.Status(fiber.StatusNotFound).JSON(fiber.Map{
- "error": "API key not found",
- })
- }
- log.Printf("❌ [APIKEY] Failed to revoke API key: %v", err)
- return c.Status(fiber.StatusInternalServerError).JSON(fiber.Map{
- "error": "Failed to revoke API key",
- })
- }
-
- return c.JSON(fiber.Map{
- "message": "API key revoked successfully",
- })
-}
-
-// Delete permanently deletes an API key
-// DELETE /api/keys/:id
-func (h *APIKeyHandler) Delete(c *fiber.Ctx) error {
- userID := c.Locals("user_id").(string)
- keyIDStr := c.Params("id")
-
- keyID, err := primitive.ObjectIDFromHex(keyIDStr)
- if err != nil {
- return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{
- "error": "Invalid key ID",
- })
- }
-
- if err := h.apiKeyService.Delete(c.Context(), keyID, userID); err != nil {
- if err.Error() == "API key not found" {
- return c.Status(fiber.StatusNotFound).JSON(fiber.Map{
- "error": "API key not found",
- })
- }
- log.Printf("❌ [APIKEY] Failed to delete API key: %v", err)
- return c.Status(fiber.StatusInternalServerError).JSON(fiber.Map{
- "error": "Failed to delete API key",
- })
- }
-
- return c.JSON(fiber.Map{
- "message": "API key deleted successfully",
- })
-}
diff --git a/backend/internal/handlers/audio.go b/backend/internal/handlers/audio.go
deleted file mode 100644
index 67e89739..00000000
--- a/backend/internal/handlers/audio.go
+++ /dev/null
@@ -1,94 +0,0 @@
-package handlers
-
-import (
- "claraverse/internal/audio"
- "fmt"
- "log"
- "os"
- "path/filepath"
-
- "github.com/gofiber/fiber/v2"
- "github.com/google/uuid"
-)
-
-// AudioHandler handles audio-related API requests
-type AudioHandler struct{}
-
-// NewAudioHandler creates a new audio handler
-func NewAudioHandler() *AudioHandler {
- return &AudioHandler{}
-}
-
-// Transcribe handles audio file transcription via OpenAI Whisper
-func (h *AudioHandler) Transcribe(c *fiber.Ctx) error {
- // Get the uploaded file
- file, err := c.FormFile("file")
- if err != nil {
- log.Printf("❌ [AUDIO-API] No file uploaded: %v", err)
- return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{
- "error": "No audio file uploaded",
- })
- }
-
- // Validate file size (max 25MB for Whisper)
- if file.Size > 25*1024*1024 {
- return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{
- "error": "Audio file too large. Maximum size is 25MB",
- })
- }
-
- // Get optional parameters
- language := c.FormValue("language", "")
- prompt := c.FormValue("prompt", "")
-
- // Create temp file to store the upload
- tempDir := os.TempDir()
- ext := filepath.Ext(file.Filename)
- if ext == "" {
- ext = ".webm" // Default extension for browser recordings
- }
- tempFile := filepath.Join(tempDir, fmt.Sprintf("audio_%s%s", uuid.New().String(), ext))
-
- // Save uploaded file to temp location
- if err := c.SaveFile(file, tempFile); err != nil {
- log.Printf("❌ [AUDIO-API] Failed to save temp file: %v", err)
- return c.Status(fiber.StatusInternalServerError).JSON(fiber.Map{
- "error": "Failed to process audio file",
- })
- }
- defer os.Remove(tempFile) // Clean up temp file
-
- // Get audio service
- audioService := audio.GetService()
- if audioService == nil {
- log.Printf("❌ [AUDIO-API] Audio service not initialized")
- return c.Status(fiber.StatusServiceUnavailable).JSON(fiber.Map{
- "error": "Audio transcription service not available. Please configure OpenAI provider.",
- })
- }
-
- // Build transcription request
- req := &audio.TranscribeRequest{
- AudioPath: tempFile,
- Language: language,
- Prompt: prompt,
- }
-
- // Call transcription service
- log.Printf("🎵 [AUDIO-API] Transcribing audio file: %s (%d bytes)", file.Filename, file.Size)
- resp, err := audioService.Transcribe(req)
- if err != nil {
- log.Printf("❌ [AUDIO-API] Transcription failed: %v", err)
- return c.Status(fiber.StatusInternalServerError).JSON(fiber.Map{
- "error": fmt.Sprintf("Transcription failed: %v", err),
- })
- }
-
- log.Printf("✅ [AUDIO-API] Transcription complete: %d chars, language: %s", len(resp.Text), resp.Language)
-
- return c.JSON(fiber.Map{
- "text": resp.Text,
- "language": resp.Language,
- "duration": resp.Duration,
- })
-}
diff --git a/backend/internal/handlers/auth_local.go b/backend/internal/handlers/auth_local.go
deleted file mode 100644
index be4d87d8..00000000
--- a/backend/internal/handlers/auth_local.go
+++ /dev/null
@@ -1,384 +0,0 @@
-package handlers
-
-import (
- "claraverse/internal/models"
- "claraverse/internal/services"
- "claraverse/pkg/auth"
- "context"
- "log"
- "strings"
- "time"
-
- "github.com/gofiber/fiber/v2"
- "go.mongodb.org/mongo-driver/bson"
- "go.mongodb.org/mongo-driver/bson/primitive"
-)
-
-// LocalAuthHandler handles local JWT authentication endpoints
-type LocalAuthHandler struct {
- jwtAuth *auth.LocalJWTAuth
- userService *services.UserService
-}
-
-// NewLocalAuthHandler creates a new local auth handler
-func NewLocalAuthHandler(jwtAuth *auth.LocalJWTAuth, userService *services.UserService) *LocalAuthHandler {
- return &LocalAuthHandler{
- jwtAuth: jwtAuth,
- userService: userService,
- }
-}
-
-// RegisterRequest is the request body for registration
-type RegisterRequest struct {
- Email string `json:"email"`
- Password string `json:"password"`
-}
-
-// LoginRequest is the request body for login
-type LoginRequest struct {
- Email string `json:"email"`
- Password string `json:"password"`
-}
-
-// RefreshTokenRequest is the request body for token refresh
-type RefreshTokenRequest struct {
- RefreshToken string `json:"refresh_token"`
-}
-
-// AuthResponse is the response for successful authentication
-type AuthResponse struct {
- AccessToken string `json:"access_token"`
- RefreshToken string `json:"refresh_token"`
- User models.UserResponse `json:"user"`
- ExpiresIn int `json:"expires_in"` // seconds
-}
-
-// Register creates a new user account
-// POST /api/auth/register
-func (h *LocalAuthHandler) Register(c *fiber.Ctx) error {
- var req RegisterRequest
- if err := c.BodyParser(&req); err != nil {
- return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{
- "error": "Invalid request body",
- })
- }
-
- // Validate email
- req.Email = strings.TrimSpace(strings.ToLower(req.Email))
- if req.Email == "" || !strings.Contains(req.Email, "@") {
- return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{
- "error": "Valid email address is required",
- })
- }
-
- // Validate password
- if err := auth.ValidatePassword(req.Password); err != nil {
- return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{
- "error": err.Error(),
- })
- }
-
- ctx := context.Background()
-
- // Check if user already exists
- existingUser, _ := h.userService.GetUserByEmail(ctx, req.Email)
- if existingUser != nil {
- return c.Status(fiber.StatusConflict).JSON(fiber.Map{
- "error": "User with this email already exists",
- })
- }
-
- // Hash password
- passwordHash, err := h.jwtAuth.HashPassword(req.Password)
- if err != nil {
- log.Printf("❌ Failed to hash password: %v", err)
- return c.Status(fiber.StatusInternalServerError).JSON(fiber.Map{
- "error": "Failed to create account",
- })
- }
-
- // Check if this is the first user (first user becomes admin)
- userCount, err := h.userService.GetUserCount(ctx)
- if err != nil {
- log.Printf("⚠️ Failed to get user count: %v", err)
- userCount = 1 // Default to non-admin if count check fails
- }
-
- // Determine user role (first user = admin, others = user)
- userRole := "user"
- if userCount == 0 {
- userRole = "admin"
- log.Printf("🎉 Creating first user as admin: %s", req.Email)
- }
-
- // Create user
- user := &models.User{
- ID: primitive.NewObjectID(),
- Email: req.Email,
- PasswordHash: passwordHash,
- EmailVerified: true, // Auto-verify in dev mode (no SMTP)
- RefreshTokenVersion: 0,
- Role: userRole,
- CreatedAt: time.Now(),
- LastLoginAt: time.Now(),
- SubscriptionTier: "pro", // Default: all users get Pro tier
- SubscriptionStatus: "active",
- Preferences: models.UserPreferences{
- StoreBuilderChatHistory: true,
- MemoryEnabled: false,
- },
- }
-
- // Save user to database
- if err := h.userService.CreateUser(ctx, user); err != nil {
- log.Printf("❌ Failed to create user: %v", err)
- return c.Status(fiber.StatusInternalServerError).JSON(fiber.Map{
- "error": "Failed to create account",
- })
- }
-
- // Generate tokens
- accessToken, refreshToken, err := h.jwtAuth.GenerateTokens(user.ID.Hex(), user.Email, user.Role)
- if err != nil {
- log.Printf("❌ Failed to generate tokens: %v", err)
- return c.Status(fiber.StatusInternalServerError).JSON(fiber.Map{
- "error": "Failed to generate authentication tokens",
- })
- }
-
- // Set refresh token as httpOnly cookie
- c.Cookie(&fiber.Cookie{
- Name: "refresh_token",
- Value: refreshToken,
- Expires: time.Now().Add(7 * 24 * time.Hour), // 7 days
- HTTPOnly: true,
- Secure: c.Protocol() == "https", // HTTPS only in production
- SameSite: "Strict",
- Path: "/api/auth",
- })
-
- log.Printf("✅ User registered: %s (%s)", user.Email, user.ID.Hex())
-
- return c.Status(fiber.StatusCreated).JSON(AuthResponse{
- AccessToken: accessToken,
- RefreshToken: refreshToken,
- User: user.ToResponse(),
- ExpiresIn: 15 * 60, // 15 minutes in seconds
- })
-}
-
-// Login authenticates a user
-// POST /api/auth/login
-func (h *LocalAuthHandler) Login(c *fiber.Ctx) error {
- var req LoginRequest
- if err := c.BodyParser(&req); err != nil {
- return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{
- "error": "Invalid request body",
- })
- }
-
- req.Email = strings.TrimSpace(strings.ToLower(req.Email))
- ctx := context.Background()
-
- // Get user by email
- user, err := h.userService.GetUserByEmail(ctx, req.Email)
- if err != nil || user == nil {
- // Use constant-time response to prevent email enumeration
- time.Sleep(200 * time.Millisecond)
- return c.Status(fiber.StatusUnauthorized).JSON(fiber.Map{
- "error": "Invalid email or password",
- })
- }
-
- // Verify password
- valid, err := h.jwtAuth.VerifyPassword(user.PasswordHash, req.Password)
- if err != nil || !valid {
- log.Printf("⚠️ Failed login attempt for user: %s", req.Email)
- return c.Status(fiber.StatusUnauthorized).JSON(fiber.Map{
- "error": "Invalid email or password",
- })
- }
-
- // Update last login time
- user.LastLoginAt = time.Now()
- if err := h.userService.UpdateUser(ctx, user); err != nil {
- log.Printf("⚠️ Failed to update last login time: %v", err)
- // Non-critical, continue
- }
-
- // Generate tokens
- accessToken, refreshToken, err := h.jwtAuth.GenerateTokens(user.ID.Hex(), user.Email, user.Role)
- if err != nil {
- log.Printf("❌ Failed to generate tokens: %v", err)
- return c.Status(fiber.StatusInternalServerError).JSON(fiber.Map{
- "error": "Failed to generate authentication tokens",
- })
- }
-
- // Set refresh token as httpOnly cookie
- c.Cookie(&fiber.Cookie{
- Name: "refresh_token",
- Value: refreshToken,
- Expires: time.Now().Add(7 * 24 * time.Hour),
- HTTPOnly: true,
- Secure: c.Protocol() == "https",
- SameSite: "Strict",
- Path: "/api/auth",
- })
-
- log.Printf("✅ User logged in: %s (%s)", user.Email, user.ID.Hex())
-
- return c.JSON(AuthResponse{
- AccessToken: accessToken,
- RefreshToken: refreshToken,
- User: user.ToResponse(),
- ExpiresIn: 15 * 60,
- })
-}
-
-// RefreshToken generates a new access token from a refresh token
-// POST /api/auth/refresh
-func (h *LocalAuthHandler) RefreshToken(c *fiber.Ctx) error {
- // Try to get refresh token from cookie first
- refreshToken := c.Cookies("refresh_token")
-
- // Fallback to request body
- if refreshToken == "" {
- var req RefreshTokenRequest
- if err := c.BodyParser(&req); err == nil {
- refreshToken = req.RefreshToken
- }
- }
-
- if refreshToken == "" {
- return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{
- "error": "Refresh token is required",
- })
- }
-
- // Verify refresh token
- claims, err := h.jwtAuth.VerifyRefreshToken(refreshToken)
- if err != nil {
- return c.Status(fiber.StatusUnauthorized).JSON(fiber.Map{
- "error": "Invalid or expired refresh token",
- })
- }
-
- ctx := context.Background()
-
- // Get user from database
- userID, err := primitive.ObjectIDFromHex(claims.UserID)
- if err != nil {
- return c.Status(fiber.StatusUnauthorized).JSON(fiber.Map{
- "error": "Invalid user ID in token",
- })
- }
-
- user, err := h.userService.GetUserByID(ctx, userID)
- if err != nil || user == nil {
- return c.Status(fiber.StatusUnauthorized).JSON(fiber.Map{
- "error": "User not found",
- })
- }
-
- // Check if refresh token version matches (for revocation)
- // Note: This would require storing token version in claims
- // For now, skip this check - implement later with Redis
-
- // Generate new access token (refresh token remains valid)
- newAccessToken, _, err := h.jwtAuth.GenerateTokens(user.ID.Hex(), user.Email, user.Role)
- if err != nil {
- log.Printf("❌ Failed to generate new access token: %v", err)
- return c.Status(fiber.StatusInternalServerError).JSON(fiber.Map{
- "error": "Failed to refresh token",
- })
- }
-
- return c.JSON(fiber.Map{
- "access_token": newAccessToken,
- "expires_in": 15 * 60,
- })
-}
-
-// Logout invalidates the refresh token
-// POST /api/auth/logout
-func (h *LocalAuthHandler) Logout(c *fiber.Ctx) error {
- userID, ok := c.Locals("user_id").(string)
- if !ok || userID == "" {
- // Allow logout even if not authenticated (clear cookie)
- c.ClearCookie("refresh_token")
- return c.JSON(fiber.Map{
- "message": "Logged out successfully",
- })
- }
-
- ctx := context.Background()
-
- // Increment refresh token version to invalidate all existing tokens
- objID, err := primitive.ObjectIDFromHex(userID)
- if err == nil {
- // Increment token version in database
- _, err = h.userService.Collection().UpdateOne(ctx,
- bson.M{"_id": objID},
- bson.M{"$inc": bson.M{"refreshTokenVersion": 1}},
- )
- if err != nil {
- log.Printf("⚠️ Failed to increment token version: %v", err)
- // Non-critical, continue
- }
- }
-
- // Clear refresh token cookie
- c.ClearCookie("refresh_token")
-
- log.Printf("✅ User logged out: %s", userID)
-
- return c.JSON(fiber.Map{
- "message": "Logged out successfully",
- })
-}
-
-// GetCurrentUser returns the currently authenticated user
-// GET /api/auth/me
-func (h *LocalAuthHandler) GetCurrentUser(c *fiber.Ctx) error {
- userID, ok := c.Locals("user_id").(string)
- if !ok || userID == "" {
- return c.Status(fiber.StatusUnauthorized).JSON(fiber.Map{
- "error": "Authentication required",
- })
- }
-
- ctx := context.Background()
- objID, err := primitive.ObjectIDFromHex(userID)
- if err != nil {
- return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{
- "error": "Invalid user ID",
- })
- }
-
- user, err := h.userService.GetUserByID(ctx, objID)
- if err != nil || user == nil {
- return c.Status(fiber.StatusNotFound).JSON(fiber.Map{
- "error": "User not found",
- })
- }
-
- return c.JSON(user.ToResponse())
-}
-
-// GetStatus returns system status for unauthenticated users
-// GET /api/auth/status
-func (h *LocalAuthHandler) GetStatus(c *fiber.Ctx) error {
- ctx := context.Background()
-
- // Check if any users exist
- userCount, err := h.userService.GetUserCount(ctx)
- if err != nil {
- log.Printf("⚠️ Failed to get user count: %v", err)
- userCount = 0
- }
-
- return c.JSON(fiber.Map{
- "has_users": userCount > 0,
- })
-}
diff --git a/backend/internal/handlers/chat_sync.go b/backend/internal/handlers/chat_sync.go
deleted file mode 100644
index e6ea31b9..00000000
--- a/backend/internal/handlers/chat_sync.go
+++ /dev/null
@@ -1,333 +0,0 @@
-package handlers
-
-import (
- "claraverse/internal/models"
- "claraverse/internal/services"
- "log"
- "strconv"
- "strings"
-
- "github.com/gofiber/fiber/v2"
-)
-
-// ChatSyncHandler handles HTTP requests for chat sync operations
-type ChatSyncHandler struct {
- service *services.ChatSyncService
-}
-
-// NewChatSyncHandler creates a new chat sync handler
-func NewChatSyncHandler(service *services.ChatSyncService) *ChatSyncHandler {
- return &ChatSyncHandler{
- service: service,
- }
-}
-
-// CreateOrUpdate creates a new chat or updates an existing one
-// POST /api/chats
-func (h *ChatSyncHandler) CreateOrUpdate(c *fiber.Ctx) error {
- userID, ok := c.Locals("user_id").(string)
- if !ok || userID == "" {
- return c.Status(fiber.StatusUnauthorized).JSON(fiber.Map{
- "error": "Authentication required",
- })
- }
-
- var req models.CreateChatRequest
- if err := c.BodyParser(&req); err != nil {
- return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{
- "error": "Invalid request body",
- })
- }
-
- if req.ID == "" {
- return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{
- "error": "Chat ID is required",
- })
- }
-
- chat, err := h.service.CreateOrUpdateChat(c.Context(), userID, &req)
- if err != nil {
- log.Printf("❌ Failed to create/update chat: %v", err)
-
- // Check for version conflict (use strings.Contains to avoid panic on short errors)
- errMsg := err.Error()
- if strings.Contains(errMsg, "version conflict") {
- return c.Status(fiber.StatusConflict).JSON(fiber.Map{
- "error": "Version conflict - chat was modified by another device",
- })
- }
-
- return c.Status(fiber.StatusInternalServerError).JSON(fiber.Map{
- "error": "Failed to save chat",
- })
- }
-
- log.Printf("✅ Chat %s saved for user %s (version: %d)", req.ID, userID, chat.Version)
- return c.Status(fiber.StatusOK).JSON(chat)
-}
-
-// Get retrieves a single chat with decrypted messages
-// GET /api/chats/:id
-func (h *ChatSyncHandler) Get(c *fiber.Ctx) error {
- userID, ok := c.Locals("user_id").(string)
- if !ok || userID == "" {
- return c.Status(fiber.StatusUnauthorized).JSON(fiber.Map{
- "error": "Authentication required",
- })
- }
-
- chatID := c.Params("id")
- if chatID == "" {
- return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{
- "error": "Chat ID is required",
- })
- }
-
- chat, err := h.service.GetChat(c.Context(), userID, chatID)
- if err != nil {
- if err.Error() == "chat not found" {
- return c.Status(fiber.StatusNotFound).JSON(fiber.Map{
- "error": "Chat not found",
- })
- }
- log.Printf("❌ Failed to get chat: %v", err)
- return c.Status(fiber.StatusInternalServerError).JSON(fiber.Map{
- "error": "Failed to get chat",
- })
- }
-
- return c.JSON(chat)
-}
-
-// List returns a paginated list of chats
-// GET /api/chats?page=1&page_size=20&starred=true
-func (h *ChatSyncHandler) List(c *fiber.Ctx) error {
- userID, ok := c.Locals("user_id").(string)
- if !ok || userID == "" {
- return c.Status(fiber.StatusUnauthorized).JSON(fiber.Map{
- "error": "Authentication required",
- })
- }
-
- page, _ := strconv.Atoi(c.Query("page", "1"))
- pageSize, _ := strconv.Atoi(c.Query("page_size", "20"))
- starred := c.Query("starred") == "true"
-
- chats, err := h.service.ListChats(c.Context(), userID, page, pageSize, starred)
- if err != nil {
- log.Printf("❌ Failed to list chats: %v", err)
- return c.Status(fiber.StatusInternalServerError).JSON(fiber.Map{
- "error": "Failed to list chats",
- })
- }
-
- return c.JSON(chats)
-}
-
-// Update performs a partial update on a chat
-// PUT /api/chats/:id
-func (h *ChatSyncHandler) Update(c *fiber.Ctx) error {
- userID, ok := c.Locals("user_id").(string)
- if !ok || userID == "" {
- return c.Status(fiber.StatusUnauthorized).JSON(fiber.Map{
- "error": "Authentication required",
- })
- }
-
- chatID := c.Params("id")
- if chatID == "" {
- return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{
- "error": "Chat ID is required",
- })
- }
-
- var req models.UpdateChatRequest
- if err := c.BodyParser(&req); err != nil {
- return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{
- "error": "Invalid request body",
- })
- }
-
- chat, err := h.service.UpdateChat(c.Context(), userID, chatID, &req)
- if err != nil {
- if err.Error() == "chat not found or version conflict" {
- return c.Status(fiber.StatusConflict).JSON(fiber.Map{
- "error": "Chat not found or version conflict",
- })
- }
- log.Printf("❌ Failed to update chat: %v", err)
- return c.Status(fiber.StatusInternalServerError).JSON(fiber.Map{
- "error": "Failed to update chat",
- })
- }
-
- log.Printf("✅ Chat %s updated for user %s", chatID, userID)
- return c.JSON(chat)
-}
-
-// Delete removes a chat
-// DELETE /api/chats/:id
-func (h *ChatSyncHandler) Delete(c *fiber.Ctx) error {
- userID, ok := c.Locals("user_id").(string)
- if !ok || userID == "" {
- return c.Status(fiber.StatusUnauthorized).JSON(fiber.Map{
- "error": "Authentication required",
- })
- }
-
- chatID := c.Params("id")
- if chatID == "" {
- return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{
- "error": "Chat ID is required",
- })
- }
-
- err := h.service.DeleteChat(c.Context(), userID, chatID)
- if err != nil {
- if err.Error() == "chat not found" {
- return c.Status(fiber.StatusNotFound).JSON(fiber.Map{
- "error": "Chat not found",
- })
- }
- log.Printf("❌ Failed to delete chat: %v", err)
- return c.Status(fiber.StatusInternalServerError).JSON(fiber.Map{
- "error": "Failed to delete chat",
- })
- }
-
- log.Printf("✅ Chat %s deleted for user %s", chatID, userID)
- return c.JSON(fiber.Map{
- "success": true,
- "message": "Chat deleted",
- })
-}
-
-// BulkSync uploads multiple chats at once
-// POST /api/chats/sync
-func (h *ChatSyncHandler) BulkSync(c *fiber.Ctx) error {
- userID, ok := c.Locals("user_id").(string)
- if !ok || userID == "" {
- return c.Status(fiber.StatusUnauthorized).JSON(fiber.Map{
- "error": "Authentication required",
- })
- }
-
- var req models.BulkSyncRequest
- if err := c.BodyParser(&req); err != nil {
- return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{
- "error": "Invalid request body",
- })
- }
-
- if len(req.Chats) == 0 {
- return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{
- "error": "No chats provided",
- })
- }
-
- // Limit bulk sync to prevent abuse
- if len(req.Chats) > 100 {
- return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{
- "error": "Maximum 100 chats per bulk sync",
- })
- }
-
- result, err := h.service.BulkSync(c.Context(), userID, &req)
- if err != nil {
- log.Printf("❌ Failed to bulk sync: %v", err)
- return c.Status(fiber.StatusInternalServerError).JSON(fiber.Map{
- "error": "Failed to bulk sync chats",
- })
- }
-
- log.Printf("✅ Bulk sync for user %s: %d synced, %d failed", userID, result.Synced, result.Failed)
- return c.JSON(result)
-}
-
-// SyncAll returns all chats for initial sync
-// GET /api/chats/sync
-func (h *ChatSyncHandler) SyncAll(c *fiber.Ctx) error {
- userID, ok := c.Locals("user_id").(string)
- if !ok || userID == "" {
- return c.Status(fiber.StatusUnauthorized).JSON(fiber.Map{
- "error": "Authentication required",
- })
- }
-
- result, err := h.service.GetAllChats(c.Context(), userID)
- if err != nil {
- log.Printf("❌ Failed to get all chats: %v", err)
- return c.Status(fiber.StatusInternalServerError).JSON(fiber.Map{
- "error": "Failed to get chats",
- })
- }
-
- log.Printf("✅ Sync all for user %s: %d chats", userID, result.TotalCount)
- return c.JSON(result)
-}
-
-// AddMessage adds a single message to a chat
-// POST /api/chats/:id/messages
-func (h *ChatSyncHandler) AddMessage(c *fiber.Ctx) error {
- userID, ok := c.Locals("user_id").(string)
- if !ok || userID == "" {
- return c.Status(fiber.StatusUnauthorized).JSON(fiber.Map{
- "error": "Authentication required",
- })
- }
-
- chatID := c.Params("id")
- if chatID == "" {
- return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{
- "error": "Chat ID is required",
- })
- }
-
- var req models.ChatAddMessageRequest
- if err := c.BodyParser(&req); err != nil {
- return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{
- "error": "Invalid request body",
- })
- }
-
- chat, err := h.service.AddMessage(c.Context(), userID, chatID, &req)
- if err != nil {
- if err.Error() == "chat not found or version conflict" || err.Error() == "version conflict during update" {
- return c.Status(fiber.StatusConflict).JSON(fiber.Map{
- "error": "Version conflict - please refresh and try again",
- })
- }
- log.Printf("❌ Failed to add message: %v", err)
- return c.Status(fiber.StatusInternalServerError).JSON(fiber.Map{
- "error": "Failed to add message",
- })
- }
-
- log.Printf("✅ Message added to chat %s for user %s", chatID, userID)
- return c.JSON(chat)
-}
-
-// DeleteAll deletes all chats for a user (GDPR compliance)
-// DELETE /api/chats
-func (h *ChatSyncHandler) DeleteAll(c *fiber.Ctx) error {
- userID, ok := c.Locals("user_id").(string)
- if !ok || userID == "" {
- return c.Status(fiber.StatusUnauthorized).JSON(fiber.Map{
- "error": "Authentication required",
- })
- }
-
- count, err := h.service.DeleteAllUserChats(c.Context(), userID)
- if err != nil {
- log.Printf("❌ Failed to delete all chats: %v", err)
- return c.Status(fiber.StatusInternalServerError).JSON(fiber.Map{
- "error": "Failed to delete chats",
- })
- }
-
- log.Printf("✅ Deleted %d chats for user %s", count, userID)
- return c.JSON(fiber.Map{
- "success": true,
- "deleted": count,
- })
-}
diff --git a/backend/internal/handlers/chat_sync_test.go b/backend/internal/handlers/chat_sync_test.go
deleted file mode 100644
index 8b15a294..00000000
--- a/backend/internal/handlers/chat_sync_test.go
+++ /dev/null
@@ -1,405 +0,0 @@
-package handlers
-
-import (
- "bytes"
- "claraverse/internal/models"
- "encoding/json"
- "io"
- "net/http/httptest"
- "testing"
-
- "github.com/gofiber/fiber/v2"
-)
-
-// Mock user middleware for testing
-func mockAuthMiddleware(userID string) fiber.Handler {
- return func(c *fiber.Ctx) error {
- c.Locals("user_id", userID)
- return c.Next()
- }
-}
-
-func TestChatSyncHandler_CreateOrUpdate_Validation(t *testing.T) {
- tests := []struct {
- name string
- userID string
- body interface{}
- expectedStatus int
- expectedError string
- }{
- {
- name: "missing user ID",
- userID: "",
- body: models.CreateChatRequest{ID: "chat-1", Title: "Test"},
- expectedStatus: fiber.StatusUnauthorized,
- expectedError: "Authentication required",
- },
- {
- name: "empty chat ID",
- userID: "user-123",
- body: models.CreateChatRequest{ID: "", Title: "Test"},
- expectedStatus: fiber.StatusBadRequest,
- expectedError: "Chat ID is required",
- },
- {
- name: "invalid JSON body",
- userID: "user-123",
- body: "not json",
- expectedStatus: fiber.StatusBadRequest,
- expectedError: "Invalid request body",
- },
- }
-
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- app := fiber.New()
-
- // Add mock auth middleware
- app.Use(mockAuthMiddleware(tt.userID))
-
- // Create handler with nil service (will fail on actual operations but validation should happen first)
- handler := &ChatSyncHandler{service: nil}
- app.Post("/chats", handler.CreateOrUpdate)
-
- var body []byte
- var err error
- if str, ok := tt.body.(string); ok {
- body = []byte(str)
- } else {
- body, err = json.Marshal(tt.body)
- if err != nil {
- t.Fatalf("Failed to marshal body: %v", err)
- }
- }
-
- req := httptest.NewRequest("POST", "/chats", bytes.NewReader(body))
- req.Header.Set("Content-Type", "application/json")
-
- resp, err := app.Test(req, -1)
- if err != nil {
- t.Fatalf("Request failed: %v", err)
- }
-
- if resp.StatusCode != tt.expectedStatus {
- t.Errorf("Expected status %d, got %d", tt.expectedStatus, resp.StatusCode)
- }
-
- // Check error message
- respBody, _ := io.ReadAll(resp.Body)
- var result map[string]string
- json.Unmarshal(respBody, &result)
-
- if result["error"] != tt.expectedError {
- t.Errorf("Expected error %q, got %q", tt.expectedError, result["error"])
- }
- })
- }
-}
-
-func TestChatSyncHandler_Get_Validation(t *testing.T) {
- // Test only auth validation - service calls will panic with nil service
- app := fiber.New()
- app.Use(mockAuthMiddleware(""))
-
- handler := &ChatSyncHandler{service: nil}
- app.Get("/chats/:id", handler.Get)
-
- req := httptest.NewRequest("GET", "/chats/chat-123", nil)
- resp, err := app.Test(req, -1)
- if err != nil {
- t.Fatalf("Request failed: %v", err)
- }
-
- if resp.StatusCode != fiber.StatusUnauthorized {
- t.Errorf("Expected status %d, got %d", fiber.StatusUnauthorized, resp.StatusCode)
- }
-
- respBody, _ := io.ReadAll(resp.Body)
- var result map[string]string
- json.Unmarshal(respBody, &result)
-
- if result["error"] != "Authentication required" {
- t.Errorf("Expected error %q, got %q", "Authentication required", result["error"])
- }
-}
-
-func TestChatSyncHandler_List_Validation(t *testing.T) {
- // Test only auth validation - service calls will panic with nil service
- app := fiber.New()
- app.Use(mockAuthMiddleware(""))
-
- handler := &ChatSyncHandler{service: nil}
- app.Get("/chats", handler.List)
-
- req := httptest.NewRequest("GET", "/chats", nil)
- resp, err := app.Test(req, -1)
- if err != nil {
- t.Fatalf("Request failed: %v", err)
- }
-
- if resp.StatusCode != fiber.StatusUnauthorized {
- t.Errorf("Expected status %d, got %d", fiber.StatusUnauthorized, resp.StatusCode)
- }
-}
-
-func TestChatSyncHandler_Update_Validation(t *testing.T) {
- app := fiber.New()
- app.Use(mockAuthMiddleware(""))
-
- handler := &ChatSyncHandler{service: nil}
- app.Put("/chats/:id", handler.Update)
-
- req := httptest.NewRequest("PUT", "/chats/chat-123", bytes.NewReader([]byte("{}")))
- req.Header.Set("Content-Type", "application/json")
-
- resp, _ := app.Test(req, -1)
- if resp.StatusCode != fiber.StatusUnauthorized {
- t.Errorf("Expected status %d, got %d", fiber.StatusUnauthorized, resp.StatusCode)
- }
-}
-
-func TestChatSyncHandler_Delete_Validation(t *testing.T) {
- app := fiber.New()
- app.Use(mockAuthMiddleware(""))
-
- handler := &ChatSyncHandler{service: nil}
- app.Delete("/chats/:id", handler.Delete)
-
- req := httptest.NewRequest("DELETE", "/chats/chat-123", nil)
- resp, _ := app.Test(req, -1)
-
- if resp.StatusCode != fiber.StatusUnauthorized {
- t.Errorf("Expected status %d, got %d", fiber.StatusUnauthorized, resp.StatusCode)
- }
-}
-
-func TestChatSyncHandler_BulkSync_Validation(t *testing.T) {
- tests := []struct {
- name string
- userID string
- body interface{}
- expectedStatus int
- expectedError string
- }{
- {
- name: "missing user ID",
- userID: "",
- body: models.BulkSyncRequest{Chats: []models.CreateChatRequest{}},
- expectedStatus: fiber.StatusUnauthorized,
- expectedError: "Authentication required",
- },
- {
- name: "empty chats array",
- userID: "user-123",
- body: models.BulkSyncRequest{Chats: []models.CreateChatRequest{}},
- expectedStatus: fiber.StatusBadRequest,
- expectedError: "No chats provided",
- },
- {
- name: "too many chats",
- userID: "user-123",
- body: func() models.BulkSyncRequest {
- chats := make([]models.CreateChatRequest, 101)
- for i := range chats {
- chats[i] = models.CreateChatRequest{ID: "chat-" + string(rune(i))}
- }
- return models.BulkSyncRequest{Chats: chats}
- }(),
- expectedStatus: fiber.StatusBadRequest,
- expectedError: "Maximum 100 chats per bulk sync",
- },
- }
-
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- app := fiber.New()
- app.Use(mockAuthMiddleware(tt.userID))
-
- handler := &ChatSyncHandler{service: nil}
- app.Post("/chats/sync", handler.BulkSync)
-
- body, _ := json.Marshal(tt.body)
- req := httptest.NewRequest("POST", "/chats/sync", bytes.NewReader(body))
- req.Header.Set("Content-Type", "application/json")
-
- resp, err := app.Test(req, -1)
- if err != nil {
- t.Fatalf("Request failed: %v", err)
- }
-
- if resp.StatusCode != tt.expectedStatus {
- t.Errorf("Expected status %d, got %d", tt.expectedStatus, resp.StatusCode)
- }
-
- if tt.expectedError != "" {
- respBody, _ := io.ReadAll(resp.Body)
- var result map[string]string
- json.Unmarshal(respBody, &result)
- if result["error"] != tt.expectedError {
- t.Errorf("Expected error %q, got %q", tt.expectedError, result["error"])
- }
- }
- })
- }
-}
-
-func TestChatSyncHandler_SyncAll_Validation(t *testing.T) {
- app := fiber.New()
- app.Use(mockAuthMiddleware(""))
-
- handler := &ChatSyncHandler{service: nil}
- app.Get("/chats/sync", handler.SyncAll)
-
- req := httptest.NewRequest("GET", "/chats/sync", nil)
- resp, _ := app.Test(req, -1)
-
- if resp.StatusCode != fiber.StatusUnauthorized {
- t.Errorf("Expected status %d, got %d", fiber.StatusUnauthorized, resp.StatusCode)
- }
-}
-
-func TestChatSyncHandler_AddMessage_Validation(t *testing.T) {
- // Test only auth validation
- app := fiber.New()
- app.Use(mockAuthMiddleware(""))
-
- handler := &ChatSyncHandler{service: nil}
- app.Post("/chats/:id/messages", handler.AddMessage)
-
- body, _ := json.Marshal(models.ChatAddMessageRequest{})
- req := httptest.NewRequest("POST", "/chats/chat-123/messages", bytes.NewReader(body))
- req.Header.Set("Content-Type", "application/json")
-
- resp, err := app.Test(req, -1)
- if err != nil {
- t.Fatalf("Request failed: %v", err)
- }
-
- if resp.StatusCode != fiber.StatusUnauthorized {
- t.Errorf("Expected status %d, got %d", fiber.StatusUnauthorized, resp.StatusCode)
- }
-}
-
-func TestChatSyncHandler_DeleteAll_Validation(t *testing.T) {
- app := fiber.New()
- app.Use(mockAuthMiddleware(""))
-
- handler := &ChatSyncHandler{service: nil}
- app.Delete("/chats", handler.DeleteAll)
-
- req := httptest.NewRequest("DELETE", "/chats", nil)
- resp, _ := app.Test(req, -1)
-
- if resp.StatusCode != fiber.StatusUnauthorized {
- t.Errorf("Expected status %d, got %d", fiber.StatusUnauthorized, resp.StatusCode)
- }
-}
-
-// Test request body parsing
-func TestRequestBodyParsing(t *testing.T) {
- tests := []struct {
- name string
- input string
- shouldParse bool
- }{
- {
- name: "valid create chat request",
- input: `{"id":"chat-1","title":"Test","messages":[{"id":"msg-1","role":"user","content":"Hello","timestamp":1700000000000}]}`,
- shouldParse: true,
- },
- {
- name: "valid update request",
- input: `{"title":"New Title","version":5}`,
- shouldParse: true,
- },
- {
- name: "valid bulk sync request",
- input: `{"chats":[{"id":"chat-1","title":"Test","messages":[]}]}`,
- shouldParse: true,
- },
- {
- name: "request with attachments",
- input: `{"id":"chat-1","title":"Test","messages":[{"id":"msg-1","role":"user","content":"Hello","timestamp":1700000000000,"attachments":[{"id":"att-1","name":"file.pdf","type":"application/pdf","size":1024}]}]}`,
- shouldParse: true,
- },
- {
- name: "request with starred",
- input: `{"id":"chat-1","title":"Test","messages":[],"is_starred":true}`,
- shouldParse: true,
- },
- {
- name: "malformed JSON",
- input: `{"id":"chat-1"`,
- shouldParse: false,
- },
- {
- name: "empty object",
- input: `{}`,
- shouldParse: true,
- },
- }
-
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- var req models.CreateChatRequest
- err := json.Unmarshal([]byte(tt.input), &req)
-
- if tt.shouldParse && err != nil {
- t.Errorf("Expected to parse successfully, got error: %v", err)
- }
- if !tt.shouldParse && err == nil {
- t.Error("Expected parse error, got nil")
- }
- })
- }
-}
-
-// Test response format
-func TestResponseFormat(t *testing.T) {
- // Test ChatResponse JSON format
- response := models.ChatResponse{
- ID: "chat-123",
- Title: "Test Chat",
- Messages: []models.ChatMessage{},
- IsStarred: true,
- Version: 1,
- }
-
- jsonData, err := json.Marshal(response)
- if err != nil {
- t.Fatalf("Failed to marshal response: %v", err)
- }
-
- var parsed map[string]interface{}
- json.Unmarshal(jsonData, &parsed)
-
- // Check snake_case field names
- if _, ok := parsed["is_starred"]; !ok {
- t.Error("Expected is_starred field in JSON")
- }
- if _, ok := parsed["created_at"]; !ok {
- t.Error("Expected created_at field in JSON")
- }
-
- // Test ChatListResponse JSON format
- listResponse := models.ChatListResponse{
- Chats: []models.ChatListItem{},
- TotalCount: 10,
- Page: 1,
- PageSize: 20,
- HasMore: false,
- }
-
- jsonData, _ = json.Marshal(listResponse)
- json.Unmarshal(jsonData, &parsed)
-
- if _, ok := parsed["total_count"]; !ok {
- t.Error("Expected total_count field in JSON")
- }
- if _, ok := parsed["page_size"]; !ok {
- t.Error("Expected page_size field in JSON")
- }
- if _, ok := parsed["has_more"]; !ok {
- t.Error("Expected has_more field in JSON")
- }
-}
diff --git a/backend/internal/handlers/composio_auth.go b/backend/internal/handlers/composio_auth.go
deleted file mode 100644
index 69cdb252..00000000
--- a/backend/internal/handlers/composio_auth.go
+++ /dev/null
@@ -1,627 +0,0 @@
-package handlers
-
-import (
- "bytes"
- "claraverse/internal/models"
- "claraverse/internal/security"
- "claraverse/internal/services"
- "encoding/json"
- "fmt"
- "io"
- "log"
- "net/http"
- "net/url"
- "os"
- "strings"
- "time"
-
- "github.com/gofiber/fiber/v2"
-)
-
-// OAuth scopes required for each service
-var requiredScopes = map[string][]string{
- "gmail": {
- "https://www.googleapis.com/auth/gmail.send",
- "https://www.googleapis.com/auth/gmail.readonly",
- "https://www.googleapis.com/auth/gmail.modify",
- },
- "googlesheets": {
- "https://www.googleapis.com/auth/spreadsheets",
- },
-}
-
-// ComposioAuthHandler handles Composio OAuth flow
-type ComposioAuthHandler struct {
- credentialService *services.CredentialService
- httpClient *http.Client
- stateStore *security.OAuthStateStore
-}
-
-// NewComposioAuthHandler creates a new Composio auth handler
-func NewComposioAuthHandler(credentialService *services.CredentialService) *ComposioAuthHandler {
- return &ComposioAuthHandler{
- credentialService: credentialService,
- httpClient: &http.Client{Timeout: 30 * time.Second},
- stateStore: security.NewOAuthStateStore(),
- }
-}
-
-// InitiateGoogleSheetsAuth initiates OAuth flow for Google Sheets via Composio
-// GET /api/integrations/composio/googlesheets/authorize
-func (h *ComposioAuthHandler) InitiateGoogleSheetsAuth(c *fiber.Ctx) error {
- userID := c.Locals("user_id").(string)
-
- composioAPIKey := os.Getenv("COMPOSIO_API_KEY")
- if composioAPIKey == "" {
- log.Printf("❌ [COMPOSIO] COMPOSIO_API_KEY not set")
- return c.Status(fiber.StatusInternalServerError).JSON(fiber.Map{
- "error": "Composio integration not configured",
- })
- }
-
- // Use ClaraVerse user ID as Composio entity ID for simplicity
- entityID := userID
-
- // Validate and sanitize redirect URL
- redirectURL := c.Query("redirect_url")
- if redirectURL == "" {
- // Default to frontend settings page
- frontendURL := os.Getenv("FRONTEND_URL")
- if frontendURL == "" {
- // Only allow localhost fallback in non-production environments
- if os.Getenv("ENVIRONMENT") == "production" {
- log.Printf("❌ [COMPOSIO] FRONTEND_URL not set in production")
- return c.Status(fiber.StatusInternalServerError).JSON(fiber.Map{
- "error": "Configuration error",
- })
- }
- frontendURL = "http://localhost:5173"
- }
- redirectURL = fmt.Sprintf("%s/settings?tab=credentials", frontendURL)
- } else {
- // Validate redirect URL against allowed origins
- if err := validateRedirectURL(redirectURL); err != nil {
- log.Printf("⚠️ [COMPOSIO] Invalid redirect URL: %s", redirectURL)
- return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{
- "error": "Invalid redirect URL",
- })
- }
- }
-
- // Get auth config ID from environment
- // This must be created in Composio dashboard first
- authConfigID := os.Getenv("COMPOSIO_GOOGLESHEETS_AUTH_CONFIG_ID")
- if authConfigID == "" {
- log.Printf("❌ [COMPOSIO] COMPOSIO_GOOGLESHEETS_AUTH_CONFIG_ID not set")
- return c.Status(fiber.StatusInternalServerError).JSON(fiber.Map{
- "error": "Google Sheets auth config not configured. Please set COMPOSIO_GOOGLESHEETS_AUTH_CONFIG_ID in environment.",
- })
- }
-
- // ✅ SECURITY FIX: Generate CSRF state token
- stateToken, err := h.stateStore.GenerateState(userID, "googlesheets")
- if err != nil {
- log.Printf("❌ [COMPOSIO] Failed to generate state token: %v", err)
- return c.Status(fiber.StatusInternalServerError).JSON(fiber.Map{
- "error": "Failed to initiate OAuth",
- })
- }
-
- // Call Composio API v3 to create a link for OAuth
- // v3 uses /link endpoint which returns redirect_url
- payload := map[string]interface{}{
- "auth_config_id": authConfigID,
- "user_id": entityID,
- }
-
- jsonData, err := json.Marshal(payload)
- if err != nil {
- log.Printf("❌ [COMPOSIO] Failed to marshal request: %v", err)
- return c.Status(fiber.StatusInternalServerError).JSON(fiber.Map{
- "error": "Failed to initiate OAuth",
- })
- }
-
- // Create connection link using v3 endpoint
- composioURL := "https://backend.composio.dev/api/v3/connected_accounts/link"
- req, err := http.NewRequest("POST", composioURL, bytes.NewBuffer(jsonData))
- if err != nil {
- log.Printf("❌ [COMPOSIO] Failed to create request: %v", err)
- return c.Status(fiber.StatusInternalServerError).JSON(fiber.Map{
- "error": "Failed to initiate OAuth",
- })
- }
-
- req.Header.Set("Content-Type", "application/json")
- req.Header.Set("x-api-key", composioAPIKey)
-
- resp, err := h.httpClient.Do(req)
- if err != nil {
- log.Printf("❌ [COMPOSIO] Failed to call Composio API: %v", err)
- return c.Status(fiber.StatusInternalServerError).JSON(fiber.Map{
- "error": "Failed to initiate OAuth",
- })
- }
- defer resp.Body.Close()
-
- respBody, _ := io.ReadAll(resp.Body)
-
- if resp.StatusCode >= 400 {
- log.Printf("❌ [COMPOSIO] API error (status %d): %s", resp.StatusCode, string(respBody))
- return c.Status(fiber.StatusInternalServerError).JSON(fiber.Map{
- "error": fmt.Sprintf("Composio API error: %s", string(respBody)),
- })
- }
-
- // Parse response to get redirectUrl
- var composioResp map[string]interface{}
- if err := json.Unmarshal(respBody, &composioResp); err != nil {
- log.Printf("❌ [COMPOSIO] Failed to parse response: %v", err)
- return c.Status(fiber.StatusInternalServerError).JSON(fiber.Map{
- "error": "Failed to parse OAuth response",
- })
- }
-
- // v3 API returns redirect_url (snake_case)
- redirectURLFromComposio, ok := composioResp["redirect_url"].(string)
- if !ok {
- log.Printf("❌ [COMPOSIO] No redirect_url in response: %v", composioResp)
- return c.Status(fiber.StatusInternalServerError).JSON(fiber.Map{
- "error": "Invalid OAuth response from Composio",
- })
- }
-
- // ✅ SECURITY FIX: Append state token to OAuth URL for CSRF protection
- parsedURL, err := url.Parse(redirectURLFromComposio)
- if err != nil {
- log.Printf("❌ [COMPOSIO] Failed to parse OAuth URL: %v", err)
- return c.Status(fiber.StatusInternalServerError).JSON(fiber.Map{
- "error": "Invalid OAuth URL",
- })
- }
- query := parsedURL.Query()
- query.Set("state", stateToken)
- parsedURL.RawQuery = query.Encode()
- authURLWithState := parsedURL.String()
-
- log.Printf("✅ [COMPOSIO] Initiated Google Sheets OAuth for user %s", userID)
-
- // Return the OAuth URL to frontend
- return c.JSON(fiber.Map{
- "authUrl": authURLWithState,
- "entityId": entityID,
- "redirectUrl": redirectURL,
- })
-}
-
-// HandleComposioCallback handles OAuth callback from Composio
-// GET /api/integrations/composio/callback
-func (h *ComposioAuthHandler) HandleComposioCallback(c *fiber.Ctx) error {
- // Get query parameters
- code := c.Query("code")
- state := c.Query("state")
- errorParam := c.Query("error")
-
- // Get frontend URL
- frontendURL := os.Getenv("FRONTEND_URL")
- if frontendURL == "" {
- if os.Getenv("ENVIRONMENT") == "production" {
- log.Printf("❌ [COMPOSIO] FRONTEND_URL not set in production")
- return c.Status(fiber.StatusInternalServerError).JSON(fiber.Map{
- "error": "Configuration error",
- })
- }
- frontendURL = "http://localhost:5173"
- }
-
- if errorParam != "" {
- log.Printf("❌ [COMPOSIO] OAuth error: %s", errorParam)
- return c.Redirect(fmt.Sprintf("%s/settings?tab=credentials&error=%s",
- frontendURL, url.QueryEscape(errorParam)))
- }
-
- if code == "" {
- log.Printf("❌ [COMPOSIO] No code in callback")
- return c.Redirect(fmt.Sprintf("%s/settings?tab=credentials&error=no_code", frontendURL))
- }
-
- // ✅ SECURITY FIX: Validate CSRF state token
- if state == "" {
- log.Printf("❌ [COMPOSIO] Missing state token in callback")
- return c.Redirect(fmt.Sprintf("%s/settings?tab=credentials&error=invalid_state", frontendURL))
- }
-
- userID, service, err := h.stateStore.ValidateState(state)
- if err != nil {
- log.Printf("❌ [COMPOSIO] Invalid state token: %v", err)
- return c.Redirect(fmt.Sprintf("%s/settings?tab=credentials&error=invalid_state", frontendURL))
- }
-
- log.Printf("✅ [COMPOSIO] Valid OAuth callback for user %s, service: %s", userID, service)
-
- // ✅ SECURITY FIX: Store code server-side instead of passing in URL
- // Generate a temporary session token to pass to frontend
- sessionToken, err := h.stateStore.GenerateState(userID, service+"_callback")
- if err != nil {
- log.Printf("❌ [COMPOSIO] Failed to generate session token: %v", err)
- return c.Redirect(fmt.Sprintf("%s/settings?tab=credentials&error=session_error", frontendURL))
- }
-
- // Store the authorization code temporarily (reusing state store for simplicity)
- // In production, you might want a separate session store
- codeStoreKey := "oauth_code:" + sessionToken
- _, err = h.stateStore.GenerateState(codeStoreKey, code) // Store code using state as key
- if err != nil {
- log.Printf("❌ [COMPOSIO] Failed to store authorization code: %v", err)
- return c.Redirect(fmt.Sprintf("%s/settings?tab=credentials&error=session_error", frontendURL))
- }
-
- // ✅ SECURITY FIX: Redirect without exposing authorization code in URL
- redirectURL := fmt.Sprintf("%s/settings?tab=credentials&composio_success=true&service=%s&session=%s",
- frontendURL, url.QueryEscape(service), url.QueryEscape(sessionToken))
-
- log.Printf("✅ [COMPOSIO] OAuth callback successful, redirecting user %s", userID)
- return c.Redirect(redirectURL)
-}
-
-// GetConnectedAccount retrieves Composio connected account for entity
-// GET /api/integrations/composio/connected-account
-func (h *ComposioAuthHandler) GetConnectedAccount(c *fiber.Ctx) error {
- userID := c.Locals("user_id").(string)
- entityID := userID // We use user ID as entity ID
-
- composioAPIKey := os.Getenv("COMPOSIO_API_KEY")
- if composioAPIKey == "" {
- return c.Status(fiber.StatusInternalServerError).JSON(fiber.Map{
- "error": "Composio integration not configured",
- })
- }
-
- // Get connected accounts for entity using v3 API
- baseURL := "https://backend.composio.dev/api/v3/connected_accounts"
- params := url.Values{}
- params.Add("user_ids", entityID)
- fullURL := baseURL + "?" + params.Encode()
- req, err := http.NewRequest("GET", fullURL, nil)
- if err != nil {
- return c.Status(fiber.StatusInternalServerError).JSON(fiber.Map{
- "error": "Failed to fetch connected account",
- })
- }
-
- req.Header.Set("x-api-key", composioAPIKey)
-
- resp, err := h.httpClient.Do(req)
- if err != nil {
- return c.Status(fiber.StatusInternalServerError).JSON(fiber.Map{
- "error": "Failed to fetch connected account",
- })
- }
- defer resp.Body.Close()
-
- respBody, _ := io.ReadAll(resp.Body)
-
- if resp.StatusCode >= 400 {
- return c.Status(fiber.StatusInternalServerError).JSON(fiber.Map{
- "error": string(respBody),
- })
- }
-
- // v3 API returns {items: [...], total_pages, current_page, ...}
- var response struct {
- Items []map[string]interface{} `json:"items"`
- }
- if err := json.Unmarshal(respBody, &response); err != nil {
- return c.Status(fiber.StatusInternalServerError).JSON(fiber.Map{
- "error": "Failed to parse response",
- })
- }
-
- // Find Google Sheets connected account
- // v3 API uses toolkit.slug instead of integrationId
- for _, account := range response.Items {
- if toolkit, ok := account["toolkit"].(map[string]interface{}); ok {
- if slug, ok := toolkit["slug"].(string); ok && slug == "googlesheets" {
- return c.JSON(account)
- }
- }
- }
-
- return c.Status(fiber.StatusNotFound).JSON(fiber.Map{
- "error": "No Google Sheets connection found",
- })
-}
-
-// CompleteComposioSetup creates credential after OAuth success
-// POST /api/integrations/composio/complete-setup
-func (h *ComposioAuthHandler) CompleteComposioSetup(c *fiber.Ctx) error {
- userID := c.Locals("user_id").(string)
-
- var req struct {
- Name string `json:"name"`
- IntegrationType string `json:"integrationType"`
- }
-
- if err := c.BodyParser(&req); err != nil {
- return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{
- "error": "Invalid request body",
- })
- }
-
- if req.Name == "" {
- req.Name = "Google Sheets"
- }
-
- if req.IntegrationType == "" {
- req.IntegrationType = "composio_googlesheets"
- }
-
- // Extract service name from integration type (e.g., "composio_gmail" -> "gmail")
- serviceName := req.IntegrationType
- if len(serviceName) > 9 && serviceName[:9] == "composio_" {
- serviceName = serviceName[9:] // Remove "composio_" prefix
- }
-
- // Entity ID is the same as user ID
- entityID := userID
-
- // Verify the connection exists in Composio
- composioAPIKey := os.Getenv("COMPOSIO_API_KEY")
- if composioAPIKey == "" {
- return c.Status(fiber.StatusInternalServerError).JSON(fiber.Map{
- "error": "Composio integration not configured",
- })
- }
-
- // Get connected accounts to verify using v3 API
- baseURL := "https://backend.composio.dev/api/v3/connected_accounts"
- params := url.Values{}
- params.Add("user_ids", entityID)
- fullURL := baseURL + "?" + params.Encode()
- httpReq, _ := http.NewRequest("GET", fullURL, nil)
- httpReq.Header.Set("x-api-key", composioAPIKey)
-
- resp, err := h.httpClient.Do(httpReq)
- if err != nil {
- log.Printf("❌ [COMPOSIO] Failed to verify connection: %v", err)
- return c.Status(fiber.StatusInternalServerError).JSON(fiber.Map{
- "error": "Failed to verify Composio connection",
- })
- }
- defer resp.Body.Close()
-
- respBody, _ := io.ReadAll(resp.Body)
- var response struct {
- Items []map[string]interface{} `json:"items"`
- }
- if err := json.Unmarshal(respBody, &response); err != nil {
- return c.Status(fiber.StatusInternalServerError).JSON(fiber.Map{
- "error": "Failed to verify connection",
- })
- }
-
- // Check if the service is connected (v3 uses toolkit.slug)
- found := false
- for _, account := range response.Items {
- if toolkit, ok := account["toolkit"].(map[string]interface{}); ok {
- if slug, ok := toolkit["slug"].(string); ok && slug == serviceName {
- found = true
- break
- }
- }
- }
-
- if !found {
- return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{
- "error": fmt.Sprintf("%s not connected in Composio. Please complete OAuth first.", strings.Title(serviceName)),
- })
- }
-
- // Create credential
- credential, err := h.credentialService.Create(c.Context(), userID, &models.CreateCredentialRequest{
- Name: req.Name,
- IntegrationType: req.IntegrationType,
- Data: map[string]interface{}{
- "composio_entity_id": entityID,
- },
- })
-
- if err != nil {
- log.Printf("❌ [COMPOSIO] Failed to create credential: %v", err)
- return c.Status(fiber.StatusInternalServerError).JSON(fiber.Map{
- "error": "Failed to save credential",
- })
- }
-
- log.Printf("✅ [COMPOSIO] Created credential for user %s with entity_id %s", userID, entityID)
-
- return c.Status(fiber.StatusCreated).JSON(credential)
-}
-
-// InitiateGmailAuth initiates OAuth flow for Gmail via Composio
-// GET /api/integrations/composio/gmail/authorize
-func (h *ComposioAuthHandler) InitiateGmailAuth(c *fiber.Ctx) error {
- userID := c.Locals("user_id").(string)
-
- composioAPIKey := os.Getenv("COMPOSIO_API_KEY")
- if composioAPIKey == "" {
- log.Printf("❌ [COMPOSIO] COMPOSIO_API_KEY not set")
- return c.Status(fiber.StatusInternalServerError).JSON(fiber.Map{
- "error": "Composio integration not configured",
- })
- }
-
- // Use ClaraVerse user ID as Composio entity ID
- entityID := userID
-
- // Validate and sanitize redirect URL
- redirectURL := c.Query("redirect_url")
- if redirectURL == "" {
- // Default to frontend settings page
- frontendURL := os.Getenv("FRONTEND_URL")
- if frontendURL == "" {
- // Only allow localhost fallback in non-production environments
- if os.Getenv("ENVIRONMENT") == "production" {
- log.Printf("❌ [COMPOSIO] FRONTEND_URL not set in production")
- return c.Status(fiber.StatusInternalServerError).JSON(fiber.Map{
- "error": "Configuration error",
- })
- }
- frontendURL = "http://localhost:5173"
- }
- redirectURL = fmt.Sprintf("%s/settings?tab=credentials", frontendURL)
- } else {
- // Validate redirect URL against allowed origins
- if err := validateRedirectURL(redirectURL); err != nil {
- log.Printf("⚠️ [COMPOSIO] Invalid redirect URL: %s", redirectURL)
- return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{
- "error": "Invalid redirect URL",
- })
- }
- }
-
- // Get auth config ID from environment
- authConfigID := os.Getenv("COMPOSIO_GMAIL_AUTH_CONFIG_ID")
- if authConfigID == "" {
- log.Printf("❌ [COMPOSIO] COMPOSIO_GMAIL_AUTH_CONFIG_ID not set")
- return c.Status(fiber.StatusInternalServerError).JSON(fiber.Map{
- "error": "Gmail auth config not configured. Please set COMPOSIO_GMAIL_AUTH_CONFIG_ID in environment.",
- })
- }
-
- // ✅ SECURITY FIX: Generate CSRF state token
- stateToken, err := h.stateStore.GenerateState(userID, "gmail")
- if err != nil {
- log.Printf("❌ [COMPOSIO] Failed to generate state token: %v", err)
- return c.Status(fiber.StatusInternalServerError).JSON(fiber.Map{
- "error": "Failed to initiate OAuth",
- })
- }
-
- // Call Composio API v3 to create a link for OAuth
- payload := map[string]interface{}{
- "auth_config_id": authConfigID,
- "user_id": entityID,
- }
-
- jsonData, err := json.Marshal(payload)
- if err != nil {
- log.Printf("❌ [COMPOSIO] Failed to marshal request: %v", err)
- return c.Status(fiber.StatusInternalServerError).JSON(fiber.Map{
- "error": "Failed to initiate OAuth",
- })
- }
-
- // Create connection link using v3 endpoint
- composioURL := "https://backend.composio.dev/api/v3/connected_accounts/link"
- req, err := http.NewRequest("POST", composioURL, bytes.NewBuffer(jsonData))
- if err != nil {
- log.Printf("❌ [COMPOSIO] Failed to create request: %v", err)
- return c.Status(fiber.StatusInternalServerError).JSON(fiber.Map{
- "error": "Failed to initiate OAuth",
- })
- }
-
- req.Header.Set("Content-Type", "application/json")
- req.Header.Set("x-api-key", composioAPIKey)
-
- resp, err := h.httpClient.Do(req)
- if err != nil {
- log.Printf("❌ [COMPOSIO] Failed to call Composio API: %v", err)
- return c.Status(fiber.StatusInternalServerError).JSON(fiber.Map{
- "error": "Failed to initiate OAuth",
- })
- }
- defer resp.Body.Close()
-
- respBody, _ := io.ReadAll(resp.Body)
-
- if resp.StatusCode >= 400 {
- log.Printf("❌ [COMPOSIO] API error (status %d): %s", resp.StatusCode, string(respBody))
- return c.Status(fiber.StatusInternalServerError).JSON(fiber.Map{
- "error": fmt.Sprintf("Composio API error: %s", string(respBody)),
- })
- }
-
- // Parse response to get redirectUrl
- var composioResp map[string]interface{}
- if err := json.Unmarshal(respBody, &composioResp); err != nil {
- log.Printf("❌ [COMPOSIO] Failed to parse response: %v", err)
- return c.Status(fiber.StatusInternalServerError).JSON(fiber.Map{
- "error": "Failed to parse OAuth response",
- })
- }
-
- // v3 API returns redirect_url
- redirectURLFromComposio, ok := composioResp["redirect_url"].(string)
- if !ok {
- log.Printf("❌ [COMPOSIO] No redirect_url in response: %v", composioResp)
- return c.Status(fiber.StatusInternalServerError).JSON(fiber.Map{
- "error": "Invalid OAuth response from Composio",
- })
- }
-
- // ✅ SECURITY FIX: Append state token to OAuth URL for CSRF protection
- gmailOauthURL, err := url.Parse(redirectURLFromComposio)
- if err != nil {
- log.Printf("❌ [COMPOSIO] Failed to parse OAuth URL: %v", err)
- return c.Status(fiber.StatusInternalServerError).JSON(fiber.Map{
- "error": "Invalid OAuth URL",
- })
- }
- gmailQueryParams := gmailOauthURL.Query()
- gmailQueryParams.Set("state", stateToken)
- gmailOauthURL.RawQuery = gmailQueryParams.Encode()
- authURLWithState := gmailOauthURL.String()
-
- log.Printf("✅ [COMPOSIO] Initiated Gmail OAuth for user %s", userID)
-
- // Return the OAuth URL to frontend
- return c.JSON(fiber.Map{
- "authUrl": authURLWithState,
- "entityId": entityID,
- "redirectUrl": redirectURL,
- })
-}
-
-// validateRedirectURL validates that a redirect URL is safe
-func validateRedirectURL(redirectURL string) error {
- parsedURL, err := url.Parse(redirectURL)
- if err != nil {
- return fmt.Errorf("invalid URL format")
- }
-
- // Must use HTTPS in production
- if os.Getenv("ENVIRONMENT") == "production" && parsedURL.Scheme != "https" {
- return fmt.Errorf("redirect URL must use HTTPS in production")
- }
-
- // Allow localhost for development
- if parsedURL.Scheme == "http" && (parsedURL.Hostname() == "localhost" || parsedURL.Hostname() == "127.0.0.1") {
- return nil
- }
-
- // Validate against FRONTEND_URL or ALLOWED_ORIGINS
- frontendURL := os.Getenv("FRONTEND_URL")
- allowedOrigins := os.Getenv("ALLOWED_ORIGINS")
-
- if frontendURL != "" {
- parsedFrontend, err := url.Parse(frontendURL)
- if err == nil && parsedURL.Host == parsedFrontend.Host {
- return nil
- }
- }
-
- if allowedOrigins != "" {
- origins := strings.Split(allowedOrigins, ",")
- for _, origin := range origins {
- origin = strings.TrimSpace(origin)
- parsedOrigin, err := url.Parse(origin)
- if err == nil && parsedURL.Host == parsedOrigin.Host {
- return nil
- }
- }
- }
-
- return fmt.Errorf("redirect URL not in allowed origins")
-}
diff --git a/backend/internal/handlers/config.go b/backend/internal/handlers/config.go
deleted file mode 100644
index 91d2fd4e..00000000
--- a/backend/internal/handlers/config.go
+++ /dev/null
@@ -1,38 +0,0 @@
-package handlers
-
-import (
- "claraverse/internal/services"
-
- "github.com/gofiber/fiber/v2"
-)
-
-// ConfigHandler handles configuration API requests
-type ConfigHandler struct {
- configService *services.ConfigService
-}
-
-// NewConfigHandler creates a new config handler
-func NewConfigHandler() *ConfigHandler {
- return &ConfigHandler{
- configService: services.GetConfigService(),
- }
-}
-
-// GetRecommendedModels returns recommended models for all providers
-func (h *ConfigHandler) GetRecommendedModels(c *fiber.Ctx) error {
- recommended := h.configService.GetAllRecommendedModels()
-
- // Convert to a frontend-friendly format
- response := make(map[string]interface{})
- for providerID, models := range recommended {
- response[string(rune(providerID+'0'))] = fiber.Map{
- "top": models.Top,
- "medium": models.Medium,
- "fastest": models.Fastest,
- }
- }
-
- return c.JSON(fiber.Map{
- "recommended_models": recommended,
- })
-}
diff --git a/backend/internal/handlers/conversation.go b/backend/internal/handlers/conversation.go
deleted file mode 100644
index 858a59ca..00000000
--- a/backend/internal/handlers/conversation.go
+++ /dev/null
@@ -1,286 +0,0 @@
-package handlers
-
-import (
- "claraverse/internal/models"
- "claraverse/internal/services"
- "log"
-
- "github.com/gofiber/fiber/v2"
-)
-
-// ConversationHandler handles conversation-related HTTP requests
-type ConversationHandler struct {
- chatService *services.ChatService
- builderService *services.BuilderConversationService
-}
-
-// NewConversationHandler creates a new conversation handler
-func NewConversationHandler(chatService *services.ChatService, builderService *services.BuilderConversationService) *ConversationHandler {
- return &ConversationHandler{
- chatService: chatService,
- builderService: builderService,
- }
-}
-
-// GetStatus returns the status of a conversation (exists, has files, time until expiration)
-// GET /api/conversations/:id/status
-func (h *ConversationHandler) GetStatus(c *fiber.Ctx) error {
- conversationID := c.Params("id")
-
- if conversationID == "" {
- return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{
- "error": "Conversation ID is required",
- })
- }
-
- // Get user ID from context (set by auth middleware)
- userID, ok := c.Locals("user_id").(string)
- if !ok || userID == "" {
- return c.Status(fiber.StatusUnauthorized).JSON(fiber.Map{
- "error": "Authentication required",
- })
- }
-
- log.Printf("📊 [CONVERSATION] Status check for conversation %s (user: %s)", conversationID, userID)
-
- // Verify conversation ownership
- if !h.chatService.IsConversationOwner(conversationID, userID) {
- return c.Status(fiber.StatusForbidden).JSON(fiber.Map{
- "error": "Access denied to this conversation",
- })
- }
-
- status := h.chatService.GetConversationStatus(conversationID)
-
- return c.JSON(status)
-}
-
-// ListBuilderConversations returns all builder conversations for an agent
-// GET /api/agents/:id/conversations
-func (h *ConversationHandler) ListBuilderConversations(c *fiber.Ctx) error {
- agentID := c.Params("id")
- if agentID == "" {
- return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{
- "error": "Agent ID is required",
- })
- }
-
- userID, ok := c.Locals("user_id").(string)
- if !ok || userID == "" {
- return c.Status(fiber.StatusUnauthorized).JSON(fiber.Map{
- "error": "Authentication required",
- })
- }
-
- if h.builderService == nil {
- return c.Status(fiber.StatusServiceUnavailable).JSON(fiber.Map{
- "error": "Builder conversation service not available",
- })
- }
-
- conversations, err := h.builderService.GetConversationsByAgent(c.Context(), agentID, userID)
- if err != nil {
- log.Printf("❌ Failed to list builder conversations: %v", err)
- return c.Status(fiber.StatusInternalServerError).JSON(fiber.Map{
- "error": "Failed to list conversations",
- })
- }
-
- return c.JSON(conversations)
-}
-
-// GetBuilderConversation returns a specific builder conversation
-// GET /api/agents/:id/conversations/:convId
-func (h *ConversationHandler) GetBuilderConversation(c *fiber.Ctx) error {
- conversationID := c.Params("convId")
- if conversationID == "" {
- return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{
- "error": "Conversation ID is required",
- })
- }
-
- userID, ok := c.Locals("user_id").(string)
- if !ok || userID == "" {
- return c.Status(fiber.StatusUnauthorized).JSON(fiber.Map{
- "error": "Authentication required",
- })
- }
-
- if h.builderService == nil {
- return c.Status(fiber.StatusServiceUnavailable).JSON(fiber.Map{
- "error": "Builder conversation service not available",
- })
- }
-
- conversation, err := h.builderService.GetConversation(c.Context(), conversationID, userID)
- if err != nil {
- log.Printf("❌ Failed to get builder conversation: %v", err)
- return c.Status(fiber.StatusNotFound).JSON(fiber.Map{
- "error": "Conversation not found",
- })
- }
-
- return c.JSON(conversation)
-}
-
-// CreateBuilderConversation creates a new builder conversation for an agent
-// POST /api/agents/:id/conversations
-func (h *ConversationHandler) CreateBuilderConversation(c *fiber.Ctx) error {
- agentID := c.Params("id")
- if agentID == "" {
- return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{
- "error": "Agent ID is required",
- })
- }
-
- userID, ok := c.Locals("user_id").(string)
- if !ok || userID == "" {
- return c.Status(fiber.StatusUnauthorized).JSON(fiber.Map{
- "error": "Authentication required",
- })
- }
-
- if h.builderService == nil {
- return c.Status(fiber.StatusServiceUnavailable).JSON(fiber.Map{
- "error": "Builder conversation service not available",
- })
- }
-
- var req struct {
- ModelID string `json:"model_id"`
- }
- if err := c.BodyParser(&req); err != nil {
- return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{
- "error": "Invalid request body",
- })
- }
-
- conversation, err := h.builderService.CreateConversation(c.Context(), agentID, userID, req.ModelID)
- if err != nil {
- log.Printf("❌ Failed to create builder conversation: %v", err)
- return c.Status(fiber.StatusInternalServerError).JSON(fiber.Map{
- "error": "Failed to create conversation",
- })
- }
-
- log.Printf("✅ Created builder conversation %s for agent %s", conversation.ID, agentID)
- return c.Status(fiber.StatusCreated).JSON(conversation)
-}
-
-// AddBuilderMessage adds a message to a builder conversation
-// POST /api/agents/:id/conversations/:convId/messages
-func (h *ConversationHandler) AddBuilderMessage(c *fiber.Ctx) error {
- conversationID := c.Params("convId")
- if conversationID == "" {
- return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{
- "error": "Conversation ID is required",
- })
- }
-
- userID, ok := c.Locals("user_id").(string)
- if !ok || userID == "" {
- return c.Status(fiber.StatusUnauthorized).JSON(fiber.Map{
- "error": "Authentication required",
- })
- }
-
- if h.builderService == nil {
- return c.Status(fiber.StatusServiceUnavailable).JSON(fiber.Map{
- "error": "Builder conversation service not available",
- })
- }
-
- var req models.AddMessageRequest
- if err := c.BodyParser(&req); err != nil {
- return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{
- "error": "Invalid request body",
- })
- }
-
- if req.Role == "" || req.Content == "" {
- return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{
- "error": "Role and content are required",
- })
- }
-
- message, err := h.builderService.AddMessage(c.Context(), conversationID, userID, &req)
- if err != nil {
- log.Printf("❌ Failed to add message to conversation: %v", err)
- return c.Status(fiber.StatusInternalServerError).JSON(fiber.Map{
- "error": "Failed to add message",
- })
- }
-
- return c.Status(fiber.StatusCreated).JSON(message)
-}
-
-// DeleteBuilderConversation deletes a builder conversation
-// DELETE /api/agents/:id/conversations/:convId
-func (h *ConversationHandler) DeleteBuilderConversation(c *fiber.Ctx) error {
- conversationID := c.Params("convId")
- if conversationID == "" {
- return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{
- "error": "Conversation ID is required",
- })
- }
-
- userID, ok := c.Locals("user_id").(string)
- if !ok || userID == "" {
- return c.Status(fiber.StatusUnauthorized).JSON(fiber.Map{
- "error": "Authentication required",
- })
- }
-
- if h.builderService == nil {
- return c.Status(fiber.StatusServiceUnavailable).JSON(fiber.Map{
- "error": "Builder conversation service not available",
- })
- }
-
- if err := h.builderService.DeleteConversation(c.Context(), conversationID, userID); err != nil {
- log.Printf("❌ Failed to delete builder conversation: %v", err)
- return c.Status(fiber.StatusInternalServerError).JSON(fiber.Map{
- "error": "Failed to delete conversation",
- })
- }
-
- return c.JSON(fiber.Map{
- "message": "Conversation deleted successfully",
- })
-}
-
-// GetOrCreateBuilderConversation gets the most recent conversation or creates a new one
-// GET /api/agents/:id/conversations/current
-func (h *ConversationHandler) GetOrCreateBuilderConversation(c *fiber.Ctx) error {
- agentID := c.Params("id")
- if agentID == "" {
- return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{
- "error": "Agent ID is required",
- })
- }
-
- userID, ok := c.Locals("user_id").(string)
- if !ok || userID == "" {
- return c.Status(fiber.StatusUnauthorized).JSON(fiber.Map{
- "error": "Authentication required",
- })
- }
-
- if h.builderService == nil {
- return c.Status(fiber.StatusServiceUnavailable).JSON(fiber.Map{
- "error": "Builder conversation service not available",
- })
- }
-
- modelID := c.Query("model_id", "")
-
- conversation, err := h.builderService.GetOrCreateConversation(c.Context(), agentID, userID, modelID)
- if err != nil {
- log.Printf("❌ Failed to get/create builder conversation: %v", err)
- return c.Status(fiber.StatusInternalServerError).JSON(fiber.Map{
- "error": "Failed to get conversation",
- })
- }
-
- return c.JSON(conversation)
-}
diff --git a/backend/internal/handlers/credential.go b/backend/internal/handlers/credential.go
deleted file mode 100644
index c7313ae0..00000000
--- a/backend/internal/handlers/credential.go
+++ /dev/null
@@ -1,337 +0,0 @@
-package handlers
-
-import (
- "claraverse/internal/models"
- "claraverse/internal/services"
- "log"
-
- "github.com/gofiber/fiber/v2"
- "go.mongodb.org/mongo-driver/bson/primitive"
-)
-
-// CredentialHandler handles credential management endpoints
-type CredentialHandler struct {
- credentialService *services.CredentialService
- credentialTester *CredentialTester
-}
-
-// NewCredentialHandler creates a new credential handler
-func NewCredentialHandler(credentialService *services.CredentialService) *CredentialHandler {
- return &CredentialHandler{
- credentialService: credentialService,
- credentialTester: NewCredentialTester(credentialService),
- }
-}
-
-// Create creates a new credential
-// POST /api/credentials
-func (h *CredentialHandler) Create(c *fiber.Ctx) error {
- userID := c.Locals("user_id").(string)
-
- var req models.CreateCredentialRequest
- if err := c.BodyParser(&req); err != nil {
- return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{
- "error": "Invalid request body",
- })
- }
-
- // Validate required fields
- if req.Name == "" {
- return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{
- "error": "Name is required",
- })
- }
-
- if req.IntegrationType == "" {
- return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{
- "error": "Integration type is required",
- })
- }
-
- if req.Data == nil || len(req.Data) == 0 {
- return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{
- "error": "Credential data is required",
- })
- }
-
- result, err := h.credentialService.Create(c.Context(), userID, &req)
- if err != nil {
- log.Printf("❌ [CREDENTIAL] Failed to create credential: %v", err)
-
- // Check for validation error
- if _, ok := err.(*models.CredentialValidationError); ok {
- return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{
- "error": err.Error(),
- })
- }
-
- return c.Status(fiber.StatusInternalServerError).JSON(fiber.Map{
- "error": "Failed to create credential",
- })
- }
-
- return c.Status(fiber.StatusCreated).JSON(result)
-}
-
-// List lists all credentials for the user
-// GET /api/credentials
-func (h *CredentialHandler) List(c *fiber.Ctx) error {
- userID := c.Locals("user_id").(string)
- integrationType := c.Query("type") // Optional filter by type
-
- var credentials []*models.CredentialListItem
- var err error
-
- if integrationType != "" {
- credentials, err = h.credentialService.ListByUserAndType(c.Context(), userID, integrationType)
- } else {
- credentials, err = h.credentialService.ListByUser(c.Context(), userID)
- }
-
- if err != nil {
- log.Printf("❌ [CREDENTIAL] Failed to list credentials: %v", err)
- return c.Status(fiber.StatusInternalServerError).JSON(fiber.Map{
- "error": "Failed to list credentials",
- })
- }
-
- return c.JSON(models.GetCredentialsResponse{
- Credentials: credentials,
- Total: len(credentials),
- })
-}
-
-// Get retrieves a specific credential (metadata only)
-// GET /api/credentials/:id
-func (h *CredentialHandler) Get(c *fiber.Ctx) error {
- userID := c.Locals("user_id").(string)
- credIDStr := c.Params("id")
-
- credID, err := primitive.ObjectIDFromHex(credIDStr)
- if err != nil {
- return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{
- "error": "Invalid credential ID",
- })
- }
-
- credential, err := h.credentialService.GetByIDAndUser(c.Context(), credID, userID)
- if err != nil {
- if err.Error() == "credential not found" {
- return c.Status(fiber.StatusNotFound).JSON(fiber.Map{
- "error": "Credential not found",
- })
- }
- log.Printf("❌ [CREDENTIAL] Failed to get credential: %v", err)
- return c.Status(fiber.StatusInternalServerError).JSON(fiber.Map{
- "error": "Failed to get credential",
- })
- }
-
- return c.JSON(credential.ToListItem())
-}
-
-// Update updates a credential
-// PUT /api/credentials/:id
-func (h *CredentialHandler) Update(c *fiber.Ctx) error {
- userID := c.Locals("user_id").(string)
- credIDStr := c.Params("id")
-
- credID, err := primitive.ObjectIDFromHex(credIDStr)
- if err != nil {
- return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{
- "error": "Invalid credential ID",
- })
- }
-
- var req models.UpdateCredentialRequest
- if err := c.BodyParser(&req); err != nil {
- return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{
- "error": "Invalid request body",
- })
- }
-
- // At least one field must be provided
- if req.Name == "" && req.Data == nil {
- return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{
- "error": "At least name or data must be provided",
- })
- }
-
- result, err := h.credentialService.Update(c.Context(), credID, userID, &req)
- if err != nil {
- if err.Error() == "credential not found" {
- return c.Status(fiber.StatusNotFound).JSON(fiber.Map{
- "error": "Credential not found",
- })
- }
- log.Printf("❌ [CREDENTIAL] Failed to update credential: %v", err)
- return c.Status(fiber.StatusInternalServerError).JSON(fiber.Map{
- "error": "Failed to update credential",
- })
- }
-
- return c.JSON(result)
-}
-
-// Delete permanently deletes a credential
-// DELETE /api/credentials/:id
-func (h *CredentialHandler) Delete(c *fiber.Ctx) error {
- userID := c.Locals("user_id").(string)
- credIDStr := c.Params("id")
-
- credID, err := primitive.ObjectIDFromHex(credIDStr)
- if err != nil {
- return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{
- "error": "Invalid credential ID",
- })
- }
-
- if err := h.credentialService.Delete(c.Context(), credID, userID); err != nil {
- if err.Error() == "credential not found" {
- return c.Status(fiber.StatusNotFound).JSON(fiber.Map{
- "error": "Credential not found",
- })
- }
- log.Printf("❌ [CREDENTIAL] Failed to delete credential: %v", err)
- return c.Status(fiber.StatusInternalServerError).JSON(fiber.Map{
- "error": "Failed to delete credential",
- })
- }
-
- return c.JSON(fiber.Map{
- "message": "Credential deleted successfully",
- })
-}
-
-// Test tests a credential by making a real API call
-// POST /api/credentials/:id/test
-func (h *CredentialHandler) Test(c *fiber.Ctx) error {
- userID := c.Locals("user_id").(string)
- credIDStr := c.Params("id")
-
- credID, err := primitive.ObjectIDFromHex(credIDStr)
- if err != nil {
- return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{
- "error": "Invalid credential ID",
- })
- }
-
- // Get and decrypt the credential
- decrypted, err := h.credentialService.GetDecrypted(c.Context(), userID, credID)
- if err != nil {
- if err.Error() == "credential not found" {
- return c.Status(fiber.StatusNotFound).JSON(fiber.Map{
- "error": "Credential not found",
- })
- }
- log.Printf("❌ [CREDENTIAL] Failed to get credential for testing: %v", err)
- return c.Status(fiber.StatusInternalServerError).JSON(fiber.Map{
- "error": "Failed to get credential",
- })
- }
-
- // Test the credential
- result := h.credentialTester.Test(c.Context(), decrypted)
-
- // Update test status
- status := "failed"
- if result.Success {
- status = "success"
- }
- if err := h.credentialService.UpdateTestStatus(c.Context(), credID, userID, status, nil); err != nil {
- log.Printf("⚠️ [CREDENTIAL] Failed to update test status: %v", err)
- }
-
- return c.JSON(result)
-}
-
-// GetIntegrations returns all available integrations
-// GET /api/integrations
-func (h *CredentialHandler) GetIntegrations(c *fiber.Ctx) error {
- categories := models.GetIntegrationsByCategory()
- return c.JSON(models.GetIntegrationsResponse{
- Categories: categories,
- })
-}
-
-// GetIntegration returns a specific integration
-// GET /api/integrations/:id
-func (h *CredentialHandler) GetIntegration(c *fiber.Ctx) error {
- integrationID := c.Params("id")
-
- integration, exists := models.GetIntegration(integrationID)
- if !exists {
- return c.Status(fiber.StatusNotFound).JSON(fiber.Map{
- "error": "Integration not found",
- })
- }
-
- return c.JSON(integration)
-}
-
-// GetCredentialsByIntegration returns credentials grouped by integration type
-// GET /api/credentials/by-integration
-func (h *CredentialHandler) GetCredentialsByIntegration(c *fiber.Ctx) error {
- userID := c.Locals("user_id").(string)
-
- credentials, err := h.credentialService.ListByUser(c.Context(), userID)
- if err != nil {
- log.Printf("❌ [CREDENTIAL] Failed to list credentials: %v", err)
- return c.Status(fiber.StatusInternalServerError).JSON(fiber.Map{
- "error": "Failed to list credentials",
- })
- }
-
- // Group by integration type
- groupedMap := make(map[string]*models.CredentialsByIntegration)
- for _, cred := range credentials {
- if _, exists := groupedMap[cred.IntegrationType]; !exists {
- integration, _ := models.GetIntegration(cred.IntegrationType)
- groupedMap[cred.IntegrationType] = &models.CredentialsByIntegration{
- IntegrationType: cred.IntegrationType,
- Integration: integration,
- Credentials: []*models.CredentialListItem{},
- }
- }
- groupedMap[cred.IntegrationType].Credentials = append(
- groupedMap[cred.IntegrationType].Credentials,
- cred,
- )
- }
-
- // Convert to slice
- var integrations []models.CredentialsByIntegration
- for _, group := range groupedMap {
- integrations = append(integrations, *group)
- }
-
- return c.JSON(models.GetCredentialsByIntegrationResponse{
- Integrations: integrations,
- })
-}
-
-// GetCredentialReferences returns credential references for LLM context
-// GET /api/credentials/references
-func (h *CredentialHandler) GetCredentialReferences(c *fiber.Ctx) error {
- userID := c.Locals("user_id").(string)
-
- // Optional filter by integration types
- var integrationTypes []string
- if types := c.Query("types"); types != "" {
- // Parse comma-separated types
- // For simplicity, just pass nil to get all
- // In production, you'd parse the query param
- }
-
- refs, err := h.credentialService.GetCredentialReferences(c.Context(), userID, integrationTypes)
- if err != nil {
- log.Printf("❌ [CREDENTIAL] Failed to get credential references: %v", err)
- return c.Status(fiber.StatusInternalServerError).JSON(fiber.Map{
- "error": "Failed to get credential references",
- })
- }
-
- return c.JSON(fiber.Map{
- "credentials": refs,
- })
-}
diff --git a/backend/internal/handlers/credential_tester.go b/backend/internal/handlers/credential_tester.go
deleted file mode 100644
index 1cf8ecc0..00000000
--- a/backend/internal/handlers/credential_tester.go
+++ /dev/null
@@ -1,1657 +0,0 @@
-package handlers
-
-import (
- "bytes"
- "claraverse/internal/models"
- "claraverse/internal/services"
- "context"
- "encoding/base64"
- "encoding/json"
- "fmt"
- "io"
- "net/http"
- "os"
- "strings"
- "time"
-)
-
-// CredentialTester handles testing credentials for different integrations
-type CredentialTester struct {
- credentialService *services.CredentialService
- httpClient *http.Client
-}
-
-// NewCredentialTester creates a new credential tester
-func NewCredentialTester(credentialService *services.CredentialService) *CredentialTester {
- return &CredentialTester{
- credentialService: credentialService,
- httpClient: &http.Client{
- Timeout: 10 * time.Second,
- },
- }
-}
-
-// Test tests a credential by making a real API call based on integration type
-func (t *CredentialTester) Test(ctx context.Context, cred *models.DecryptedCredential) *models.TestCredentialResponse {
- switch cred.IntegrationType {
- case "discord":
- return t.testDiscord(ctx, cred.Data)
- case "slack":
- return t.testSlack(ctx, cred.Data)
- case "telegram":
- return t.testTelegram(ctx, cred.Data)
- case "teams":
- return t.testTeams(ctx, cred.Data)
- case "notion":
- return t.testNotion(ctx, cred.Data)
- case "github":
- return t.testGitHub(ctx, cred.Data)
- case "gitlab":
- return t.testGitLab(ctx, cred.Data)
- case "linear":
- return t.testLinear(ctx, cred.Data)
- case "jira":
- return t.testJira(ctx, cred.Data)
- case "airtable":
- return t.testAirtable(ctx, cred.Data)
- case "trello":
- return t.testTrello(ctx, cred.Data)
- case "hubspot":
- return t.testHubSpot(ctx, cred.Data)
- case "sendgrid":
- return t.testSendGrid(ctx, cred.Data)
- case "brevo":
- return t.testBrevo(ctx, cred.Data)
- case "mailchimp":
- return t.testMailchimp(ctx, cred.Data)
- case "openai":
- return t.testOpenAI(ctx, cred.Data)
- case "anthropic":
- return t.testAnthropic(ctx, cred.Data)
- case "google_ai":
- return t.testGoogleAI(ctx, cred.Data)
- case "google_chat":
- return t.testGoogleChat(ctx, cred.Data)
- case "zoom":
- return t.testZoom(ctx, cred.Data)
- case "referralmonk":
- return t.testReferralMonk(ctx, cred.Data)
- case "composio_googlesheets":
- return t.testComposioGoogleSheets(ctx, cred.Data)
- case "composio_gmail":
- return t.testComposioGmail(ctx, cred.Data)
- case "custom_webhook":
- return t.testCustomWebhook(ctx, cred.Data)
- case "rest_api":
- return t.testRestAPI(ctx, cred.Data)
- case "mongodb":
- return t.testMongoDB(ctx, cred.Data)
- case "redis":
- return t.testRedis(ctx, cred.Data)
- default:
- return &models.TestCredentialResponse{
- Success: false,
- Message: "Testing not implemented for this integration type",
- }
- }
-}
-
-// testDiscord tests a Discord webhook by sending a test message
-func (t *CredentialTester) testDiscord(ctx context.Context, data map[string]interface{}) *models.TestCredentialResponse {
- webhookURL, ok := data["webhook_url"].(string)
- if !ok || webhookURL == "" {
- return &models.TestCredentialResponse{
- Success: false,
- Message: "Webhook URL is required",
- }
- }
-
- payload := map[string]string{
- "content": "🔗 **ClaraVerse Test** - Webhook connection verified!",
- }
- body, _ := json.Marshal(payload)
-
- req, _ := http.NewRequestWithContext(ctx, "POST", webhookURL, bytes.NewBuffer(body))
- req.Header.Set("Content-Type", "application/json")
-
- resp, err := t.httpClient.Do(req)
- if err != nil {
- return &models.TestCredentialResponse{
- Success: false,
- Message: "Failed to connect to Discord",
- Details: err.Error(),
- }
- }
- defer resp.Body.Close()
-
- if resp.StatusCode >= 200 && resp.StatusCode < 300 {
- return &models.TestCredentialResponse{
- Success: true,
- Message: "Discord webhook is working! A test message was sent.",
- }
- }
-
- return &models.TestCredentialResponse{
- Success: false,
- Message: fmt.Sprintf("Discord returned status %d", resp.StatusCode),
- }
-}
-
-// testSlack tests a Slack webhook
-func (t *CredentialTester) testSlack(ctx context.Context, data map[string]interface{}) *models.TestCredentialResponse {
- webhookURL, ok := data["webhook_url"].(string)
- if !ok || webhookURL == "" {
- return &models.TestCredentialResponse{
- Success: false,
- Message: "Webhook URL is required",
- }
- }
-
- payload := map[string]string{
- "text": "🔗 *ClaraVerse Test* - Webhook connection verified!",
- }
- body, _ := json.Marshal(payload)
-
- req, _ := http.NewRequestWithContext(ctx, "POST", webhookURL, bytes.NewBuffer(body))
- req.Header.Set("Content-Type", "application/json")
-
- resp, err := t.httpClient.Do(req)
- if err != nil {
- return &models.TestCredentialResponse{
- Success: false,
- Message: "Failed to connect to Slack",
- Details: err.Error(),
- }
- }
- defer resp.Body.Close()
-
- if resp.StatusCode == 200 {
- return &models.TestCredentialResponse{
- Success: true,
- Message: "Slack webhook is working! A test message was sent.",
- }
- }
-
- return &models.TestCredentialResponse{
- Success: false,
- Message: fmt.Sprintf("Slack returned status %d", resp.StatusCode),
- }
-}
-
-// testTelegram tests a Telegram bot token
-func (t *CredentialTester) testTelegram(ctx context.Context, data map[string]interface{}) *models.TestCredentialResponse {
- botToken, ok := data["bot_token"].(string)
- if !ok || botToken == "" {
- return &models.TestCredentialResponse{
- Success: false,
- Message: "Bot token is required",
- }
- }
-
- chatID, ok := data["chat_id"].(string)
- if !ok || chatID == "" {
- return &models.TestCredentialResponse{
- Success: false,
- Message: "Chat ID is required",
- }
- }
-
- url := fmt.Sprintf("https://api.telegram.org/bot%s/sendMessage", botToken)
- payload := map[string]string{
- "chat_id": chatID,
- "text": "🔗 ClaraVerse Test - Bot connection verified!",
- }
- body, _ := json.Marshal(payload)
-
- req, _ := http.NewRequestWithContext(ctx, "POST", url, bytes.NewBuffer(body))
- req.Header.Set("Content-Type", "application/json")
-
- resp, err := t.httpClient.Do(req)
- if err != nil {
- return &models.TestCredentialResponse{
- Success: false,
- Message: "Failed to connect to Telegram",
- Details: err.Error(),
- }
- }
- defer resp.Body.Close()
-
- var result map[string]interface{}
- json.NewDecoder(resp.Body).Decode(&result)
-
- if ok, _ := result["ok"].(bool); ok {
- return &models.TestCredentialResponse{
- Success: true,
- Message: "Telegram bot is working! A test message was sent.",
- }
- }
-
- description, _ := result["description"].(string)
- return &models.TestCredentialResponse{
- Success: false,
- Message: "Telegram API error",
- Details: description,
- }
-}
-
-// testTeams tests a Microsoft Teams webhook
-func (t *CredentialTester) testTeams(ctx context.Context, data map[string]interface{}) *models.TestCredentialResponse {
- webhookURL, ok := data["webhook_url"].(string)
- if !ok || webhookURL == "" {
- return &models.TestCredentialResponse{
- Success: false,
- Message: "Webhook URL is required",
- }
- }
-
- payload := map[string]string{
- "text": "🔗 **ClaraVerse Test** - Webhook connection verified!",
- }
- body, _ := json.Marshal(payload)
-
- req, _ := http.NewRequestWithContext(ctx, "POST", webhookURL, bytes.NewBuffer(body))
- req.Header.Set("Content-Type", "application/json")
-
- resp, err := t.httpClient.Do(req)
- if err != nil {
- return &models.TestCredentialResponse{
- Success: false,
- Message: "Failed to connect to Teams",
- Details: err.Error(),
- }
- }
- defer resp.Body.Close()
-
- if resp.StatusCode == 200 {
- return &models.TestCredentialResponse{
- Success: true,
- Message: "Teams webhook is working! A test message was sent.",
- }
- }
-
- return &models.TestCredentialResponse{
- Success: false,
- Message: fmt.Sprintf("Teams returned status %d", resp.StatusCode),
- }
-}
-
-// testNotion tests a Notion API key
-func (t *CredentialTester) testNotion(ctx context.Context, data map[string]interface{}) *models.TestCredentialResponse {
- apiKey, ok := data["api_key"].(string)
- if !ok || apiKey == "" {
- return &models.TestCredentialResponse{
- Success: false,
- Message: "API key is required",
- }
- }
-
- req, _ := http.NewRequestWithContext(ctx, "GET", "https://api.notion.com/v1/users/me", nil)
- req.Header.Set("Authorization", "Bearer "+apiKey)
- req.Header.Set("Notion-Version", "2022-06-28")
-
- resp, err := t.httpClient.Do(req)
- if err != nil {
- return &models.TestCredentialResponse{
- Success: false,
- Message: "Failed to connect to Notion",
- Details: err.Error(),
- }
- }
- defer resp.Body.Close()
-
- if resp.StatusCode == 200 {
- var result map[string]interface{}
- json.NewDecoder(resp.Body).Decode(&result)
- name, _ := result["name"].(string)
- return &models.TestCredentialResponse{
- Success: true,
- Message: "Notion API key is valid!",
- Details: fmt.Sprintf("Connected as: %s", name),
- }
- }
-
- return &models.TestCredentialResponse{
- Success: false,
- Message: fmt.Sprintf("Notion returned status %d", resp.StatusCode),
- }
-}
-
-// testGitHub tests a GitHub personal access token
-func (t *CredentialTester) testGitHub(ctx context.Context, data map[string]interface{}) *models.TestCredentialResponse {
- token, ok := data["personal_access_token"].(string)
- if !ok || token == "" {
- return &models.TestCredentialResponse{
- Success: false,
- Message: "Personal access token is required",
- }
- }
-
- req, _ := http.NewRequestWithContext(ctx, "GET", "https://api.github.com/user", nil)
- req.Header.Set("Authorization", "Bearer "+token)
- req.Header.Set("Accept", "application/vnd.github.v3+json")
-
- resp, err := t.httpClient.Do(req)
- if err != nil {
- return &models.TestCredentialResponse{
- Success: false,
- Message: "Failed to connect to GitHub",
- Details: err.Error(),
- }
- }
- defer resp.Body.Close()
-
- if resp.StatusCode == 200 {
- var result map[string]interface{}
- json.NewDecoder(resp.Body).Decode(&result)
- login, _ := result["login"].(string)
- return &models.TestCredentialResponse{
- Success: true,
- Message: "GitHub token is valid!",
- Details: fmt.Sprintf("Authenticated as: %s", login),
- }
- }
-
- return &models.TestCredentialResponse{
- Success: false,
- Message: fmt.Sprintf("GitHub returned status %d", resp.StatusCode),
- }
-}
-
-// testGitLab tests a GitLab personal access token
-func (t *CredentialTester) testGitLab(ctx context.Context, data map[string]interface{}) *models.TestCredentialResponse {
- token, ok := data["personal_access_token"].(string)
- if !ok || token == "" {
- return &models.TestCredentialResponse{
- Success: false,
- Message: "Personal access token is required",
- }
- }
-
- baseURL := "https://gitlab.com"
- if url, ok := data["base_url"].(string); ok && url != "" {
- baseURL = strings.TrimSuffix(url, "/")
- }
-
- req, _ := http.NewRequestWithContext(ctx, "GET", baseURL+"/api/v4/user", nil)
- req.Header.Set("PRIVATE-TOKEN", token)
-
- resp, err := t.httpClient.Do(req)
- if err != nil {
- return &models.TestCredentialResponse{
- Success: false,
- Message: "Failed to connect to GitLab",
- Details: err.Error(),
- }
- }
- defer resp.Body.Close()
-
- if resp.StatusCode == 200 {
- var result map[string]interface{}
- json.NewDecoder(resp.Body).Decode(&result)
- username, _ := result["username"].(string)
- return &models.TestCredentialResponse{
- Success: true,
- Message: "GitLab token is valid!",
- Details: fmt.Sprintf("Authenticated as: %s", username),
- }
- }
-
- return &models.TestCredentialResponse{
- Success: false,
- Message: fmt.Sprintf("GitLab returned status %d", resp.StatusCode),
- }
-}
-
-// testLinear tests a Linear API key
-func (t *CredentialTester) testLinear(ctx context.Context, data map[string]interface{}) *models.TestCredentialResponse {
- apiKey, ok := data["api_key"].(string)
- if !ok || apiKey == "" {
- return &models.TestCredentialResponse{
- Success: false,
- Message: "API key is required",
- }
- }
-
- query := `{"query": "{ viewer { id name email } }"}`
- req, _ := http.NewRequestWithContext(ctx, "POST", "https://api.linear.app/graphql", strings.NewReader(query))
- req.Header.Set("Authorization", apiKey)
- req.Header.Set("Content-Type", "application/json")
-
- resp, err := t.httpClient.Do(req)
- if err != nil {
- return &models.TestCredentialResponse{
- Success: false,
- Message: "Failed to connect to Linear",
- Details: err.Error(),
- }
- }
- defer resp.Body.Close()
-
- if resp.StatusCode == 200 {
- var result map[string]interface{}
- json.NewDecoder(resp.Body).Decode(&result)
- if data, ok := result["data"].(map[string]interface{}); ok {
- if viewer, ok := data["viewer"].(map[string]interface{}); ok {
- name, _ := viewer["name"].(string)
- return &models.TestCredentialResponse{
- Success: true,
- Message: "Linear API key is valid!",
- Details: fmt.Sprintf("Authenticated as: %s", name),
- }
- }
- }
- }
-
- return &models.TestCredentialResponse{
- Success: false,
- Message: fmt.Sprintf("Linear returned status %d", resp.StatusCode),
- }
-}
-
-// testJira tests Jira credentials
-func (t *CredentialTester) testJira(ctx context.Context, data map[string]interface{}) *models.TestCredentialResponse {
- email, _ := data["email"].(string)
- apiToken, _ := data["api_token"].(string)
- domain, _ := data["domain"].(string)
-
- if email == "" || apiToken == "" || domain == "" {
- return &models.TestCredentialResponse{
- Success: false,
- Message: "Email, API token, and domain are required",
- }
- }
-
- url := fmt.Sprintf("https://%s/rest/api/3/myself", domain)
- req, _ := http.NewRequestWithContext(ctx, "GET", url, nil)
- req.SetBasicAuth(email, apiToken)
- req.Header.Set("Accept", "application/json")
-
- resp, err := t.httpClient.Do(req)
- if err != nil {
- return &models.TestCredentialResponse{
- Success: false,
- Message: "Failed to connect to Jira",
- Details: err.Error(),
- }
- }
- defer resp.Body.Close()
-
- if resp.StatusCode == 200 {
- var result map[string]interface{}
- json.NewDecoder(resp.Body).Decode(&result)
- displayName, _ := result["displayName"].(string)
- return &models.TestCredentialResponse{
- Success: true,
- Message: "Jira credentials are valid!",
- Details: fmt.Sprintf("Authenticated as: %s", displayName),
- }
- }
-
- return &models.TestCredentialResponse{
- Success: false,
- Message: fmt.Sprintf("Jira returned status %d", resp.StatusCode),
- }
-}
-
-// testAirtable tests an Airtable API key
-func (t *CredentialTester) testAirtable(ctx context.Context, data map[string]interface{}) *models.TestCredentialResponse {
- apiKey, ok := data["api_key"].(string)
- if !ok || apiKey == "" {
- return &models.TestCredentialResponse{
- Success: false,
- Message: "API key is required",
- }
- }
-
- req, _ := http.NewRequestWithContext(ctx, "GET", "https://api.airtable.com/v0/meta/whoami", nil)
- req.Header.Set("Authorization", "Bearer "+apiKey)
-
- resp, err := t.httpClient.Do(req)
- if err != nil {
- return &models.TestCredentialResponse{
- Success: false,
- Message: "Failed to connect to Airtable",
- Details: err.Error(),
- }
- }
- defer resp.Body.Close()
-
- if resp.StatusCode == 200 {
- return &models.TestCredentialResponse{
- Success: true,
- Message: "Airtable API key is valid!",
- }
- }
-
- return &models.TestCredentialResponse{
- Success: false,
- Message: fmt.Sprintf("Airtable returned status %d", resp.StatusCode),
- }
-}
-
-// testTrello tests Trello credentials
-func (t *CredentialTester) testTrello(ctx context.Context, data map[string]interface{}) *models.TestCredentialResponse {
- apiKey, _ := data["api_key"].(string)
- token, _ := data["token"].(string)
-
- if apiKey == "" || token == "" {
- return &models.TestCredentialResponse{
- Success: false,
- Message: "API key and token are required",
- }
- }
-
- url := fmt.Sprintf("https://api.trello.com/1/members/me?key=%s&token=%s", apiKey, token)
- req, _ := http.NewRequestWithContext(ctx, "GET", url, nil)
-
- resp, err := t.httpClient.Do(req)
- if err != nil {
- return &models.TestCredentialResponse{
- Success: false,
- Message: "Failed to connect to Trello",
- Details: err.Error(),
- }
- }
- defer resp.Body.Close()
-
- if resp.StatusCode == 200 {
- var result map[string]interface{}
- json.NewDecoder(resp.Body).Decode(&result)
- username, _ := result["username"].(string)
- return &models.TestCredentialResponse{
- Success: true,
- Message: "Trello credentials are valid!",
- Details: fmt.Sprintf("Authenticated as: %s", username),
- }
- }
-
- return &models.TestCredentialResponse{
- Success: false,
- Message: fmt.Sprintf("Trello returned status %d", resp.StatusCode),
- }
-}
-
-// testHubSpot tests a HubSpot access token
-func (t *CredentialTester) testHubSpot(ctx context.Context, data map[string]interface{}) *models.TestCredentialResponse {
- accessToken, ok := data["access_token"].(string)
- if !ok || accessToken == "" {
- return &models.TestCredentialResponse{
- Success: false,
- Message: "Access token is required",
- }
- }
-
- req, _ := http.NewRequestWithContext(ctx, "GET", "https://api.hubapi.com/crm/v3/objects/contacts?limit=1", nil)
- req.Header.Set("Authorization", "Bearer "+accessToken)
-
- resp, err := t.httpClient.Do(req)
- if err != nil {
- return &models.TestCredentialResponse{
- Success: false,
- Message: "Failed to connect to HubSpot",
- Details: err.Error(),
- }
- }
- defer resp.Body.Close()
-
- if resp.StatusCode == 200 {
- return &models.TestCredentialResponse{
- Success: true,
- Message: "HubSpot access token is valid!",
- }
- }
-
- return &models.TestCredentialResponse{
- Success: false,
- Message: fmt.Sprintf("HubSpot returned status %d", resp.StatusCode),
- }
-}
-
-// testSendGrid tests a SendGrid API key and optionally verifies sender identity
-func (t *CredentialTester) testSendGrid(ctx context.Context, data map[string]interface{}) *models.TestCredentialResponse {
- apiKey, ok := data["api_key"].(string)
- if !ok || apiKey == "" {
- return &models.TestCredentialResponse{
- Success: false,
- Message: "API key is required",
- }
- }
-
- // First, test the API key validity
- req, _ := http.NewRequestWithContext(ctx, "GET", "https://api.sendgrid.com/v3/user/profile", nil)
- req.Header.Set("Authorization", "Bearer "+apiKey)
-
- resp, err := t.httpClient.Do(req)
- if err != nil {
- return &models.TestCredentialResponse{
- Success: false,
- Message: "Failed to connect to SendGrid",
- Details: err.Error(),
- }
- }
- defer resp.Body.Close()
-
- if resp.StatusCode != 200 {
- return &models.TestCredentialResponse{
- Success: false,
- Message: fmt.Sprintf("SendGrid returned status %d - invalid API key", resp.StatusCode),
- }
- }
-
- // API key is valid - now check verified senders if from_email is provided
- fromEmail, hasFromEmail := data["from_email"].(string)
- if !hasFromEmail || fromEmail == "" {
- return &models.TestCredentialResponse{
- Success: true,
- Message: "SendGrid API key is valid!",
- Details: "Note: Add a 'Default From Email' to verify sender identity.",
- }
- }
-
- // Check verified senders
- sendersReq, _ := http.NewRequestWithContext(ctx, "GET", "https://api.sendgrid.com/v3/verified_senders", nil)
- sendersReq.Header.Set("Authorization", "Bearer "+apiKey)
-
- sendersResp, err := t.httpClient.Do(sendersReq)
- if err != nil {
- return &models.TestCredentialResponse{
- Success: true,
- Message: "SendGrid API key is valid!",
- Details: fmt.Sprintf("Could not verify sender '%s' - check SendGrid Sender Identity settings.", fromEmail),
- }
- }
- defer sendersResp.Body.Close()
-
- if sendersResp.StatusCode == 200 {
- var result map[string]interface{}
- json.NewDecoder(sendersResp.Body).Decode(&result)
-
- // Check if the from_email is in the verified senders list
- senderVerified := false
- if results, ok := result["results"].([]interface{}); ok {
- for _, sender := range results {
- if s, ok := sender.(map[string]interface{}); ok {
- if email, ok := s["from_email"].(string); ok {
- if strings.EqualFold(email, fromEmail) {
- if verified, ok := s["verified"].(bool); ok && verified {
- senderVerified = true
- break
- }
- }
- }
- }
- }
- }
-
- if senderVerified {
- return &models.TestCredentialResponse{
- Success: true,
- Message: "SendGrid API key and sender identity verified!",
- Details: fmt.Sprintf("Verified sender: %s", fromEmail),
- }
- }
-
- return &models.TestCredentialResponse{
- Success: true,
- Message: "SendGrid API key is valid, but sender not verified!",
- Details: fmt.Sprintf("'%s' is not a verified sender. Visit https://app.sendgrid.com/settings/sender_auth to verify it.", fromEmail),
- }
- }
-
- return &models.TestCredentialResponse{
- Success: true,
- Message: "SendGrid API key is valid!",
- Details: fmt.Sprintf("Could not check sender verification for '%s'.", fromEmail),
- }
-}
-
-// testBrevo tests a Brevo (SendInBlue) API key
-func (t *CredentialTester) testBrevo(ctx context.Context, data map[string]interface{}) *models.TestCredentialResponse {
- apiKey, ok := data["api_key"].(string)
- if !ok || apiKey == "" {
- return &models.TestCredentialResponse{
- Success: false,
- Message: "API key is required",
- }
- }
-
- // Validate API key format
- if !strings.HasPrefix(apiKey, "xkeysib-") {
- return &models.TestCredentialResponse{
- Success: false,
- Message: "Invalid API key format - Brevo API keys start with 'xkeysib-'",
- }
- }
-
- // Test the API key by getting account info
- req, _ := http.NewRequestWithContext(ctx, "GET", "https://api.brevo.com/v3/account", nil)
- req.Header.Set("api-key", apiKey)
- req.Header.Set("Accept", "application/json")
-
- resp, err := t.httpClient.Do(req)
- if err != nil {
- return &models.TestCredentialResponse{
- Success: false,
- Message: "Failed to connect to Brevo",
- Details: err.Error(),
- }
- }
- defer resp.Body.Close()
-
- if resp.StatusCode != 200 {
- bodyBytes, _ := io.ReadAll(resp.Body)
- return &models.TestCredentialResponse{
- Success: false,
- Message: fmt.Sprintf("Brevo returned status %d - invalid API key", resp.StatusCode),
- Details: string(bodyBytes),
- }
- }
-
- var result map[string]interface{}
- json.NewDecoder(resp.Body).Decode(&result)
-
- // Extract account info
- email, _ := result["email"].(string)
- companyName := ""
- if plan, ok := result["plan"].([]interface{}); ok && len(plan) > 0 {
- if planInfo, ok := plan[0].(map[string]interface{}); ok {
- if name, ok := planInfo["type"].(string); ok {
- companyName = name
- }
- }
- }
-
- details := fmt.Sprintf("Account: %s", email)
- if companyName != "" {
- details += fmt.Sprintf(" (Plan: %s)", companyName)
- }
-
- // Check if from_email is provided and verify sender
- fromEmail, hasFromEmail := data["from_email"].(string)
- if hasFromEmail && fromEmail != "" {
- // Get senders list
- sendersReq, _ := http.NewRequestWithContext(ctx, "GET", "https://api.brevo.com/v3/senders", nil)
- sendersReq.Header.Set("api-key", apiKey)
- sendersReq.Header.Set("Accept", "application/json")
-
- sendersResp, err := t.httpClient.Do(sendersReq)
- if err == nil {
- defer sendersResp.Body.Close()
- if sendersResp.StatusCode == 200 {
- var sendersResult map[string]interface{}
- json.NewDecoder(sendersResp.Body).Decode(&sendersResult)
-
- senderFound := false
- if senders, ok := sendersResult["senders"].([]interface{}); ok {
- for _, sender := range senders {
- if s, ok := sender.(map[string]interface{}); ok {
- if senderEmail, ok := s["email"].(string); ok {
- if strings.EqualFold(senderEmail, fromEmail) {
- senderFound = true
- if active, ok := s["active"].(bool); ok && active {
- details += fmt.Sprintf("\nVerified sender: %s ✓", fromEmail)
- } else {
- details += fmt.Sprintf("\nSender '%s' found but not active", fromEmail)
- }
- break
- }
- }
- }
- }
- }
- if !senderFound {
- details += fmt.Sprintf("\nWarning: '%s' not found in verified senders", fromEmail)
- }
- }
- }
- }
-
- return &models.TestCredentialResponse{
- Success: true,
- Message: "Brevo API key is valid!",
- Details: details,
- }
-}
-
-// testMailchimp tests a Mailchimp API key
-func (t *CredentialTester) testMailchimp(ctx context.Context, data map[string]interface{}) *models.TestCredentialResponse {
- apiKey, ok := data["api_key"].(string)
- if !ok || apiKey == "" {
- return &models.TestCredentialResponse{
- Success: false,
- Message: "API key is required",
- }
- }
-
- // Extract datacenter from API key (format: xxx-usX)
- parts := strings.Split(apiKey, "-")
- if len(parts) < 2 {
- return &models.TestCredentialResponse{
- Success: false,
- Message: "Invalid API key format",
- }
- }
- dc := parts[len(parts)-1]
-
- url := fmt.Sprintf("https://%s.api.mailchimp.com/3.0/", dc)
- req, _ := http.NewRequestWithContext(ctx, "GET", url, nil)
- req.SetBasicAuth("anystring", apiKey)
-
- resp, err := t.httpClient.Do(req)
- if err != nil {
- return &models.TestCredentialResponse{
- Success: false,
- Message: "Failed to connect to Mailchimp",
- Details: err.Error(),
- }
- }
- defer resp.Body.Close()
-
- if resp.StatusCode == 200 {
- var result map[string]interface{}
- json.NewDecoder(resp.Body).Decode(&result)
- accountName, _ := result["account_name"].(string)
- return &models.TestCredentialResponse{
- Success: true,
- Message: "Mailchimp API key is valid!",
- Details: fmt.Sprintf("Account: %s", accountName),
- }
- }
-
- return &models.TestCredentialResponse{
- Success: false,
- Message: fmt.Sprintf("Mailchimp returned status %d", resp.StatusCode),
- }
-}
-
-// testOpenAI tests an OpenAI API key
-func (t *CredentialTester) testOpenAI(ctx context.Context, data map[string]interface{}) *models.TestCredentialResponse {
- apiKey, ok := data["api_key"].(string)
- if !ok || apiKey == "" {
- return &models.TestCredentialResponse{
- Success: false,
- Message: "API key is required",
- }
- }
-
- req, _ := http.NewRequestWithContext(ctx, "GET", "https://api.openai.com/v1/models", nil)
- req.Header.Set("Authorization", "Bearer "+apiKey)
-
- resp, err := t.httpClient.Do(req)
- if err != nil {
- return &models.TestCredentialResponse{
- Success: false,
- Message: "Failed to connect to OpenAI",
- Details: err.Error(),
- }
- }
- defer resp.Body.Close()
-
- if resp.StatusCode == 200 {
- return &models.TestCredentialResponse{
- Success: true,
- Message: "OpenAI API key is valid!",
- }
- }
-
- return &models.TestCredentialResponse{
- Success: false,
- Message: fmt.Sprintf("OpenAI returned status %d", resp.StatusCode),
- }
-}
-
-// testAnthropic tests an Anthropic API key
-func (t *CredentialTester) testAnthropic(ctx context.Context, data map[string]interface{}) *models.TestCredentialResponse {
- apiKey, ok := data["api_key"].(string)
- if !ok || apiKey == "" {
- return &models.TestCredentialResponse{
- Success: false,
- Message: "API key is required",
- }
- }
-
- // Send a minimal message to test the API key
- payload := map[string]interface{}{
- "model": "claude-3-haiku-20240307",
- "max_tokens": 10,
- "messages": []map[string]string{
- {"role": "user", "content": "Hi"},
- },
- }
- body, _ := json.Marshal(payload)
-
- req, _ := http.NewRequestWithContext(ctx, "POST", "https://api.anthropic.com/v1/messages", bytes.NewBuffer(body))
- req.Header.Set("x-api-key", apiKey)
- req.Header.Set("anthropic-version", "2023-06-01")
- req.Header.Set("Content-Type", "application/json")
-
- resp, err := t.httpClient.Do(req)
- if err != nil {
- return &models.TestCredentialResponse{
- Success: false,
- Message: "Failed to connect to Anthropic",
- Details: err.Error(),
- }
- }
- defer resp.Body.Close()
-
- if resp.StatusCode == 200 {
- return &models.TestCredentialResponse{
- Success: true,
- Message: "Anthropic API key is valid!",
- }
- }
-
- // Read error response
- bodyBytes, _ := io.ReadAll(resp.Body)
- return &models.TestCredentialResponse{
- Success: false,
- Message: fmt.Sprintf("Anthropic returned status %d", resp.StatusCode),
- Details: string(bodyBytes),
- }
-}
-
-// testGoogleAI tests a Google AI API key
-func (t *CredentialTester) testGoogleAI(ctx context.Context, data map[string]interface{}) *models.TestCredentialResponse {
- apiKey, ok := data["api_key"].(string)
- if !ok || apiKey == "" {
- return &models.TestCredentialResponse{
- Success: false,
- Message: "API key is required",
- }
- }
-
- url := fmt.Sprintf("https://generativelanguage.googleapis.com/v1/models?key=%s", apiKey)
- req, _ := http.NewRequestWithContext(ctx, "GET", url, nil)
-
- resp, err := t.httpClient.Do(req)
- if err != nil {
- return &models.TestCredentialResponse{
- Success: false,
- Message: "Failed to connect to Google AI",
- Details: err.Error(),
- }
- }
- defer resp.Body.Close()
-
- if resp.StatusCode == 200 {
- return &models.TestCredentialResponse{
- Success: true,
- Message: "Google AI API key is valid!",
- }
- }
-
- return &models.TestCredentialResponse{
- Success: false,
- Message: fmt.Sprintf("Google AI returned status %d", resp.StatusCode),
- }
-}
-
-// testGoogleChat tests a Google Chat webhook
-func (t *CredentialTester) testGoogleChat(ctx context.Context, data map[string]interface{}) *models.TestCredentialResponse {
- webhookURL, ok := data["webhook_url"].(string)
- if !ok || webhookURL == "" {
- return &models.TestCredentialResponse{
- Success: false,
- Message: "Webhook URL is required",
- }
- }
-
- // Validate it's a Google Chat webhook URL
- if !strings.Contains(webhookURL, "chat.googleapis.com") {
- return &models.TestCredentialResponse{
- Success: false,
- Message: "Invalid Google Chat webhook URL - must contain chat.googleapis.com",
- }
- }
-
- // Google Chat webhook payload format
- payload := map[string]string{
- "text": "🔗 *ClaraVerse Test* - Webhook connection verified!",
- }
- body, _ := json.Marshal(payload)
-
- req, _ := http.NewRequestWithContext(ctx, "POST", webhookURL, bytes.NewBuffer(body))
- req.Header.Set("Content-Type", "application/json; charset=UTF-8")
-
- resp, err := t.httpClient.Do(req)
- if err != nil {
- return &models.TestCredentialResponse{
- Success: false,
- Message: "Failed to connect to Google Chat",
- Details: err.Error(),
- }
- }
- defer resp.Body.Close()
-
- if resp.StatusCode >= 200 && resp.StatusCode < 300 {
- return &models.TestCredentialResponse{
- Success: true,
- Message: "Google Chat webhook is working! A test message was sent.",
- }
- }
-
- // Read error response for details
- bodyBytes, _ := io.ReadAll(resp.Body)
- return &models.TestCredentialResponse{
- Success: false,
- Message: fmt.Sprintf("Google Chat returned status %d", resp.StatusCode),
- Details: string(bodyBytes),
- }
-}
-
-// testZoom tests Zoom Server-to-Server OAuth credentials
-func (t *CredentialTester) testZoom(ctx context.Context, data map[string]interface{}) *models.TestCredentialResponse {
- accountID, _ := data["account_id"].(string)
- clientID, _ := data["client_id"].(string)
- clientSecret, _ := data["client_secret"].(string)
-
- if accountID == "" || clientID == "" || clientSecret == "" {
- return &models.TestCredentialResponse{
- Success: false,
- Message: "Account ID, Client ID, and Client Secret are required",
- }
- }
-
- // Try to get an OAuth access token
- tokenURL := "https://zoom.us/oauth/token"
- tokenData := fmt.Sprintf("grant_type=account_credentials&account_id=%s", accountID)
-
- req, _ := http.NewRequestWithContext(ctx, "POST", tokenURL, strings.NewReader(tokenData))
-
- // Basic auth with client_id:client_secret
- auth := base64.StdEncoding.EncodeToString([]byte(clientID + ":" + clientSecret))
- req.Header.Set("Authorization", "Basic "+auth)
- req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
-
- resp, err := t.httpClient.Do(req)
- if err != nil {
- return &models.TestCredentialResponse{
- Success: false,
- Message: "Failed to connect to Zoom OAuth",
- Details: err.Error(),
- }
- }
- defer resp.Body.Close()
-
- bodyBytes, _ := io.ReadAll(resp.Body)
-
- if resp.StatusCode != 200 {
- var errorResp map[string]interface{}
- json.Unmarshal(bodyBytes, &errorResp)
- errorMsg := "Unknown error"
- if reason, ok := errorResp["reason"].(string); ok {
- errorMsg = reason
- } else if errStr, ok := errorResp["error"].(string); ok {
- errorMsg = errStr
- }
- return &models.TestCredentialResponse{
- Success: false,
- Message: fmt.Sprintf("Zoom OAuth failed: %s", errorMsg),
- Details: string(bodyBytes),
- }
- }
-
- var tokenResp map[string]interface{}
- json.Unmarshal(bodyBytes, &tokenResp)
-
- if _, ok := tokenResp["access_token"].(string); !ok {
- return &models.TestCredentialResponse{
- Success: false,
- Message: "Zoom OAuth returned invalid response",
- Details: string(bodyBytes),
- }
- }
-
- // Token obtained successfully - now verify we can list users (basic API test)
- accessToken := tokenResp["access_token"].(string)
-
- userReq, _ := http.NewRequestWithContext(ctx, "GET", "https://api.zoom.us/v2/users/me", nil)
- userReq.Header.Set("Authorization", "Bearer "+accessToken)
-
- userResp, err := t.httpClient.Do(userReq)
- if err != nil {
- return &models.TestCredentialResponse{
- Success: true,
- Message: "Zoom OAuth credentials are valid!",
- Details: "Token generated successfully, but could not verify API access.",
- }
- }
- defer userResp.Body.Close()
-
- if userResp.StatusCode == 200 {
- var userInfo map[string]interface{}
- json.NewDecoder(userResp.Body).Decode(&userInfo)
- email, _ := userInfo["email"].(string)
- firstName, _ := userInfo["first_name"].(string)
- lastName, _ := userInfo["last_name"].(string)
-
- name := strings.TrimSpace(firstName + " " + lastName)
- if name == "" {
- name = email
- }
-
- return &models.TestCredentialResponse{
- Success: true,
- Message: "Zoom credentials verified!",
- Details: fmt.Sprintf("Connected as: %s (%s)", name, email),
- }
- }
-
- return &models.TestCredentialResponse{
- Success: true,
- Message: "Zoom OAuth credentials are valid!",
- Details: "Token generated successfully.",
- }
-}
-
-// testCustomWebhook tests a custom webhook
-func (t *CredentialTester) testCustomWebhook(ctx context.Context, data map[string]interface{}) *models.TestCredentialResponse {
- url, ok := data["url"].(string)
- if !ok || url == "" {
- return &models.TestCredentialResponse{
- Success: false,
- Message: "URL is required",
- }
- }
-
- method := "POST"
- if m, ok := data["method"].(string); ok && m != "" {
- method = m
- }
-
- payload := map[string]string{
- "test": "true",
- "message": "ClaraVerse webhook test",
- }
- body, _ := json.Marshal(payload)
-
- req, _ := http.NewRequestWithContext(ctx, method, url, bytes.NewBuffer(body))
- req.Header.Set("Content-Type", "application/json")
-
- // Add authentication if configured
- if authType, ok := data["auth_type"].(string); ok && authType != "none" {
- authValue, _ := data["auth_value"].(string)
- switch authType {
- case "bearer":
- req.Header.Set("Authorization", "Bearer "+authValue)
- case "basic":
- // authValue should be "user:pass"
- parts := strings.SplitN(authValue, ":", 2)
- if len(parts) == 2 {
- req.SetBasicAuth(parts[0], parts[1])
- }
- case "api_key":
- req.Header.Set("X-API-Key", authValue)
- }
- }
-
- // Add custom headers
- if headers, ok := data["headers"].(string); ok && headers != "" {
- var headerMap map[string]string
- if err := json.Unmarshal([]byte(headers), &headerMap); err == nil {
- for k, v := range headerMap {
- req.Header.Set(k, v)
- }
- }
- }
-
- resp, err := t.httpClient.Do(req)
- if err != nil {
- return &models.TestCredentialResponse{
- Success: false,
- Message: "Failed to connect to webhook",
- Details: err.Error(),
- }
- }
- defer resp.Body.Close()
-
- if resp.StatusCode >= 200 && resp.StatusCode < 300 {
- return &models.TestCredentialResponse{
- Success: true,
- Message: fmt.Sprintf("Webhook responded with status %d", resp.StatusCode),
- }
- }
-
- return &models.TestCredentialResponse{
- Success: false,
- Message: fmt.Sprintf("Webhook returned status %d", resp.StatusCode),
- }
-}
-
-// testRestAPI tests a REST API endpoint
-func (t *CredentialTester) testRestAPI(ctx context.Context, data map[string]interface{}) *models.TestCredentialResponse {
- baseURL, ok := data["base_url"].(string)
- if !ok || baseURL == "" {
- return &models.TestCredentialResponse{
- Success: false,
- Message: "Base URL is required",
- }
- }
-
- req, _ := http.NewRequestWithContext(ctx, "GET", baseURL, nil)
-
- // Add authentication if configured
- if authType, ok := data["auth_type"].(string); ok && authType != "none" {
- authValue, _ := data["auth_value"].(string)
- switch authType {
- case "bearer":
- req.Header.Set("Authorization", "Bearer "+authValue)
- case "basic":
- parts := strings.SplitN(authValue, ":", 2)
- if len(parts) == 2 {
- req.SetBasicAuth(parts[0], parts[1])
- }
- case "api_key_header":
- headerName := "X-API-Key"
- if name, ok := data["auth_header_name"].(string); ok && name != "" {
- headerName = name
- }
- req.Header.Set(headerName, authValue)
- case "api_key_query":
- q := req.URL.Query()
- q.Add("api_key", authValue)
- req.URL.RawQuery = q.Encode()
- }
- }
-
- // Add default headers
- if headers, ok := data["headers"].(string); ok && headers != "" {
- var headerMap map[string]string
- if err := json.Unmarshal([]byte(headers), &headerMap); err == nil {
- for k, v := range headerMap {
- req.Header.Set(k, v)
- }
- }
- }
-
- resp, err := t.httpClient.Do(req)
- if err != nil {
- return &models.TestCredentialResponse{
- Success: false,
- Message: "Failed to connect to API",
- Details: err.Error(),
- }
- }
- defer resp.Body.Close()
-
- if resp.StatusCode >= 200 && resp.StatusCode < 300 {
- return &models.TestCredentialResponse{
- Success: true,
- Message: fmt.Sprintf("API responded with status %d", resp.StatusCode),
- }
- }
-
- return &models.TestCredentialResponse{
- Success: false,
- Message: fmt.Sprintf("API returned status %d", resp.StatusCode),
- }
-}
-
-// testMongoDB tests MongoDB connection credentials
-func (t *CredentialTester) testMongoDB(ctx context.Context, data map[string]interface{}) *models.TestCredentialResponse {
- connectionString, ok := data["connection_string"].(string)
- if !ok || connectionString == "" {
- return &models.TestCredentialResponse{
- Success: false,
- Message: "Connection string is required",
- }
- }
-
- database, _ := data["database"].(string)
- if database == "" {
- return &models.TestCredentialResponse{
- Success: false,
- Message: "Database name is required",
- }
- }
-
- // Import MongoDB driver dynamically to test connection
- // We'll use a simple HTTP-based approach to avoid adding heavy dependencies
- // For production, we use the actual MongoDB driver in the tools
-
- // Validate connection string format
- if !strings.HasPrefix(connectionString, "mongodb://") && !strings.HasPrefix(connectionString, "mongodb+srv://") {
- return &models.TestCredentialResponse{
- Success: false,
- Message: "Invalid connection string format",
- Details: "Connection string must start with 'mongodb://' or 'mongodb+srv://'",
- }
- }
-
- // Use MongoDB Go driver for actual connection test
- // Import is done at runtime via the tool execution
- return t.testMongoDBConnection(ctx, connectionString, database)
-}
-
-// testMongoDBConnection performs the actual MongoDB connection test
-func (t *CredentialTester) testMongoDBConnection(ctx context.Context, connectionString, database string) *models.TestCredentialResponse {
- // We need to use the MongoDB driver here
- // Import: go.mongodb.org/mongo-driver/mongo
- // Since this is a handler and we want to keep it lightweight,
- // we'll call a helper that uses the actual driver
-
- // For now, return a placeholder that will be replaced with actual implementation
- // Using the MongoDB driver directly here
-
- // Create a context with timeout for the connection test
- testCtx, cancel := context.WithTimeout(ctx, 10*time.Second)
- defer cancel()
-
- // Attempt to connect using the MongoDB driver
- // Note: The actual implementation requires importing the MongoDB driver
- // which is already a dependency in this project (used in mongodb_tool.go)
-
- // We'll use a goroutine-based approach to test the connection
- // without blocking the handler for too long
-
- resultChan := make(chan *models.TestCredentialResponse, 1)
-
- go func() {
- result := testMongoDBWithDriver(testCtx, connectionString, database)
- resultChan <- result
- }()
-
- select {
- case result := <-resultChan:
- return result
- case <-testCtx.Done():
- return &models.TestCredentialResponse{
- Success: false,
- Message: "Connection test timed out",
- Details: "MongoDB server did not respond within 10 seconds",
- }
- }
-}
-
-// testRedis tests Redis connection credentials
-func (t *CredentialTester) testRedis(ctx context.Context, data map[string]interface{}) *models.TestCredentialResponse {
- host, _ := data["host"].(string)
- if host == "" {
- host = "localhost"
- }
-
- port, _ := data["port"].(string)
- if port == "" {
- port = "6379"
- }
-
- password, _ := data["password"].(string)
- dbNum, _ := data["database"].(string)
- if dbNum == "" {
- dbNum = "0"
- }
-
- // Create a context with timeout
- testCtx, cancel := context.WithTimeout(ctx, 10*time.Second)
- defer cancel()
-
- resultChan := make(chan *models.TestCredentialResponse, 1)
-
- go func() {
- result := testRedisWithDriver(testCtx, host, port, password, dbNum)
- resultChan <- result
- }()
-
- select {
- case result := <-resultChan:
- return result
- case <-testCtx.Done():
- return &models.TestCredentialResponse{
- Success: false,
- Message: "Connection test timed out",
- Details: "Redis server did not respond within 10 seconds",
- }
- }
-}
-
-// testReferralMonk tests ReferralMonk API credentials by calling their API
-func (t *CredentialTester) testReferralMonk(ctx context.Context, data map[string]interface{}) *models.TestCredentialResponse {
- apiToken, ok := data["api_token"].(string)
- if !ok || apiToken == "" {
- return &models.TestCredentialResponse{
- Success: false,
- Message: "API Token is required",
- }
- }
-
- apiSecret, ok := data["api_secret"].(string)
- if !ok || apiSecret == "" {
- return &models.TestCredentialResponse{
- Success: false,
- Message: "API Secret is required",
- }
- }
-
- // Make a simple API call to verify credentials - using a test endpoint if available
- // For now, we'll verify the credentials are properly formatted
- url := "https://ahaguru.referralmonk.com/api/campaign"
-
- // Create a minimal test payload that won't actually send a message
- // Note: This is a validation check - we're verifying the API responds to our credentials
- testPayload := map[string]interface{}{
- "template_name": "test_validation",
- "channel": "whatsapp",
- "recipients": []map[string]interface{}{},
- }
-
- body, _ := json.Marshal(testPayload)
- req, err := http.NewRequestWithContext(ctx, "POST", url, bytes.NewBuffer(body))
- if err != nil {
- return &models.TestCredentialResponse{
- Success: false,
- Message: "Failed to create request",
- Details: err.Error(),
- }
- }
-
- req.Header.Set("Content-Type", "application/json")
- req.Header.Set("Api-Token", apiToken)
- req.Header.Set("Api-Secret", apiSecret)
-
- resp, err := t.httpClient.Do(req)
- if err != nil {
- return &models.TestCredentialResponse{
- Success: false,
- Message: "Failed to connect to ReferralMonk API",
- Details: err.Error(),
- }
- }
- defer resp.Body.Close()
-
- bodyBytes, _ := io.ReadAll(resp.Body)
-
- // Check for authentication success (even if template doesn't exist, auth should work)
- // 401/403 = bad credentials
- // 400/404 = credentials work but request issue (which is expected for test)
- // 200/201 = success
- if resp.StatusCode == 401 || resp.StatusCode == 403 {
- return &models.TestCredentialResponse{
- Success: false,
- Message: "Invalid API credentials",
- Details: fmt.Sprintf("Authentication failed: %s", string(bodyBytes)),
- }
- }
-
- // If we get 200, 201, 400, or 404, credentials are valid (just the request format might be off)
- if resp.StatusCode < 500 {
- return &models.TestCredentialResponse{
- Success: true,
- Message: "ReferralMonk API credentials verified successfully",
- Details: "API token and secret are valid",
- }
- }
-
- // 500+ errors indicate server issues
- return &models.TestCredentialResponse{
- Success: false,
- Message: "ReferralMonk API server error",
- Details: string(bodyBytes),
- }
-}
-
-// testComposioGoogleSheets tests Composio Google Sheets connection
-func (t *CredentialTester) testComposioGoogleSheets(ctx context.Context, data map[string]interface{}) *models.TestCredentialResponse {
- entityID, ok := data["composio_entity_id"].(string)
- if !ok || entityID == "" {
- return &models.TestCredentialResponse{
- Success: false,
- Message: "Entity ID is required",
- }
- }
-
- composioAPIKey := os.Getenv("COMPOSIO_API_KEY")
- if composioAPIKey == "" {
- return &models.TestCredentialResponse{
- Success: false,
- Message: "Composio integration not configured",
- Details: "COMPOSIO_API_KEY environment variable not set",
- }
- }
-
- // Check if the entity has a connected Google Sheets account using v3 API
- url := fmt.Sprintf("https://backend.composio.dev/api/v3/connected_accounts?user_ids=%s", entityID)
- req, err := http.NewRequestWithContext(ctx, "GET", url, nil)
- if err != nil {
- return &models.TestCredentialResponse{
- Success: false,
- Message: "Failed to create test request",
- }
- }
-
- req.Header.Set("x-api-key", composioAPIKey)
-
- resp, err := t.httpClient.Do(req)
- if err != nil {
- return &models.TestCredentialResponse{
- Success: false,
- Message: "Failed to connect to Composio API",
- Details: err.Error(),
- }
- }
- defer resp.Body.Close()
-
- bodyBytes, _ := io.ReadAll(resp.Body)
-
- if resp.StatusCode >= 400 {
- return &models.TestCredentialResponse{
- Success: false,
- Message: "Failed to verify Composio connection",
- Details: string(bodyBytes),
- }
- }
-
- // Parse response to check for Google Sheets connection
- // v3 API returns {items: [...]}
- var response struct {
- Items []map[string]interface{} `json:"items"`
- }
- if err := json.Unmarshal(bodyBytes, &response); err != nil {
- return &models.TestCredentialResponse{
- Success: false,
- Message: "Failed to parse Composio response",
- }
- }
-
- // Check if Google Sheets is connected (v3 uses toolkit.slug)
- for _, account := range response.Items {
- if toolkit, ok := account["toolkit"].(map[string]interface{}); ok {
- if slug, ok := toolkit["slug"].(string); ok && slug == "googlesheets" {
- return &models.TestCredentialResponse{
- Success: true,
- Message: "Google Sheets connected successfully via Composio",
- Details: fmt.Sprintf("Entity ID: %s", entityID),
- }
- }
- }
- }
-
- return &models.TestCredentialResponse{
- Success: false,
- Message: "No Google Sheets connection found",
- Details: "Please reconnect your Google account",
- }
-}
-
-// testComposioGmail tests Composio Gmail connection
-func (t *CredentialTester) testComposioGmail(ctx context.Context, data map[string]interface{}) *models.TestCredentialResponse {
- entityID, ok := data["composio_entity_id"].(string)
- if !ok || entityID == "" {
- return &models.TestCredentialResponse{
- Success: false,
- Message: "Entity ID is required",
- }
- }
-
- composioAPIKey := os.Getenv("COMPOSIO_API_KEY")
- if composioAPIKey == "" {
- return &models.TestCredentialResponse{
- Success: false,
- Message: "Composio integration not configured",
- Details: "COMPOSIO_API_KEY environment variable not set",
- }
- }
-
- // Check if the entity has a connected Gmail account using v3 API
- url := fmt.Sprintf("https://backend.composio.dev/api/v3/connected_accounts?user_ids=%s", entityID)
- req, err := http.NewRequestWithContext(ctx, "GET", url, nil)
- if err != nil {
- return &models.TestCredentialResponse{
- Success: false,
- Message: "Failed to create test request",
- }
- }
-
- req.Header.Set("x-api-key", composioAPIKey)
-
- resp, err := t.httpClient.Do(req)
- if err != nil {
- return &models.TestCredentialResponse{
- Success: false,
- Message: "Failed to connect to Composio API",
- Details: err.Error(),
- }
- }
- defer resp.Body.Close()
-
- bodyBytes, _ := io.ReadAll(resp.Body)
-
- if resp.StatusCode >= 400 {
- return &models.TestCredentialResponse{
- Success: false,
- Message: "Failed to verify Composio connection",
- Details: string(bodyBytes),
- }
- }
-
- // Parse response to check for Gmail connection
- // v3 API returns {items: [...]}
- var response struct {
- Items []map[string]interface{} `json:"items"`
- }
- if err := json.Unmarshal(bodyBytes, &response); err != nil {
- return &models.TestCredentialResponse{
- Success: false,
- Message: "Failed to parse Composio response",
- }
- }
-
- // Check if Gmail is connected (v3 uses toolkit.slug)
- for _, account := range response.Items {
- if toolkit, ok := account["toolkit"].(map[string]interface{}); ok {
- if slug, ok := toolkit["slug"].(string); ok && slug == "gmail" {
- return &models.TestCredentialResponse{
- Success: true,
- Message: "Gmail connected successfully via Composio",
- Details: fmt.Sprintf("Entity ID: %s", entityID),
- }
- }
- }
- }
-
- return &models.TestCredentialResponse{
- Success: false,
- Message: "No Gmail connection found",
- Details: "Please reconnect your Gmail account",
- }
-}
diff --git a/backend/internal/handlers/database_testers.go b/backend/internal/handlers/database_testers.go
deleted file mode 100644
index d7dfecca..00000000
--- a/backend/internal/handlers/database_testers.go
+++ /dev/null
@@ -1,246 +0,0 @@
-package handlers
-
-import (
- "claraverse/internal/models"
- "context"
- "fmt"
- "strconv"
- "time"
-
- "github.com/redis/go-redis/v9"
- "go.mongodb.org/mongo-driver/bson"
- "go.mongodb.org/mongo-driver/mongo"
- "go.mongodb.org/mongo-driver/mongo/options"
- "go.mongodb.org/mongo-driver/mongo/readpref"
-)
-
-// testMongoDBWithDriver tests MongoDB connection using the official driver
-func testMongoDBWithDriver(ctx context.Context, connectionString, database string) *models.TestCredentialResponse {
- // Set client options with timeout
- clientOptions := options.Client().
- ApplyURI(connectionString).
- SetConnectTimeout(10 * time.Second).
- SetServerSelectionTimeout(10 * time.Second)
-
- // Connect to MongoDB
- client, err := mongo.Connect(ctx, clientOptions)
- if err != nil {
- return &models.TestCredentialResponse{
- Success: false,
- Message: "Failed to create MongoDB client",
- Details: err.Error(),
- }
- }
- defer func() {
- disconnectCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
- defer cancel()
- client.Disconnect(disconnectCtx)
- }()
-
- // Ping the database to verify connection
- if err := client.Ping(ctx, readpref.Primary()); err != nil {
- return &models.TestCredentialResponse{
- Success: false,
- Message: "Failed to connect to MongoDB",
- Details: err.Error(),
- }
- }
-
- // Try to access the specified database and list collections
- db := client.Database(database)
- collections, err := db.ListCollectionNames(ctx, bson.M{})
- if err != nil {
- return &models.TestCredentialResponse{
- Success: false,
- Message: "Connected but failed to access database",
- Details: fmt.Sprintf("Database '%s': %s", database, err.Error()),
- }
- }
-
- // Get server info
- var serverStatus bson.M
- err = db.RunCommand(ctx, bson.D{{Key: "serverStatus", Value: 1}}).Decode(&serverStatus)
-
- details := fmt.Sprintf("Database: %s\nCollections: %d", database, len(collections))
-
- if err == nil {
- if version, ok := serverStatus["version"].(string); ok {
- details = fmt.Sprintf("Server version: %s\n%s", version, details)
- }
- }
-
- if len(collections) > 0 && len(collections) <= 5 {
- details += fmt.Sprintf("\nCollections: %v", collections)
- } else if len(collections) > 5 {
- details += fmt.Sprintf("\nCollections (first 5): %v...", collections[:5])
- }
-
- return &models.TestCredentialResponse{
- Success: true,
- Message: "MongoDB connection successful!",
- Details: details,
- }
-}
-
-// testRedisWithDriver tests Redis connection using the official driver
-func testRedisWithDriver(ctx context.Context, host, port, password, dbNum string) *models.TestCredentialResponse {
- // Parse database number
- db, err := strconv.Atoi(dbNum)
- if err != nil {
- db = 0
- }
-
- // Create Redis client
- addr := fmt.Sprintf("%s:%s", host, port)
- client := redis.NewClient(&redis.Options{
- Addr: addr,
- Password: password,
- DB: db,
- DialTimeout: 10 * time.Second,
- ReadTimeout: 10 * time.Second,
- WriteTimeout: 10 * time.Second,
- })
- defer client.Close()
-
- // Test connection with PING
- pong, err := client.Ping(ctx).Result()
- if err != nil {
- // Provide more helpful error messages
- errMsg := err.Error()
- if password == "" && (contains(errMsg, "NOAUTH") || contains(errMsg, "AUTH")) {
- return &models.TestCredentialResponse{
- Success: false,
- Message: "Redis requires authentication",
- Details: "This Redis server requires a password. Please provide the password in credentials.",
- }
- }
- if contains(errMsg, "connection refused") {
- return &models.TestCredentialResponse{
- Success: false,
- Message: "Connection refused",
- Details: fmt.Sprintf("Could not connect to Redis at %s. Please verify the host and port are correct.", addr),
- }
- }
- if contains(errMsg, "timeout") || contains(errMsg, "deadline") {
- return &models.TestCredentialResponse{
- Success: false,
- Message: "Connection timed out",
- Details: fmt.Sprintf("Redis server at %s did not respond in time.", addr),
- }
- }
- return &models.TestCredentialResponse{
- Success: false,
- Message: "Failed to connect to Redis",
- Details: err.Error(),
- }
- }
-
- // Get server info
- info, err := client.Info(ctx, "server").Result()
- details := fmt.Sprintf("Address: %s\nDatabase: %d\nPing response: %s", addr, db, pong)
-
- if err == nil {
- // Parse server info for version
- version := parseRedisInfoField(info, "redis_version")
- if version != "" {
- details = fmt.Sprintf("Redis version: %s\n%s", version, details)
- }
-
- // Get memory info
- memInfo, memErr := client.Info(ctx, "memory").Result()
- if memErr == nil {
- usedMemory := parseRedisInfoField(memInfo, "used_memory_human")
- if usedMemory != "" {
- details += fmt.Sprintf("\nMemory used: %s", usedMemory)
- }
- }
- }
-
- // Get key count for current database
- dbSize, err := client.DBSize(ctx).Result()
- if err == nil {
- details += fmt.Sprintf("\nKeys in DB %d: %d", db, dbSize)
- }
-
- return &models.TestCredentialResponse{
- Success: true,
- Message: "Redis connection successful!",
- Details: details,
- }
-}
-
-// contains checks if a string contains a substring (case-insensitive)
-func contains(s, substr string) bool {
- return len(s) >= len(substr) && (s == substr || len(s) > 0 && containsIgnoreCase(s, substr))
-}
-
-func containsIgnoreCase(s, substr string) bool {
- for i := 0; i <= len(s)-len(substr); i++ {
- if equalFoldSlice(s[i:i+len(substr)], substr) {
- return true
- }
- }
- return false
-}
-
-func equalFoldSlice(s1, s2 string) bool {
- if len(s1) != len(s2) {
- return false
- }
- for i := 0; i < len(s1); i++ {
- c1, c2 := s1[i], s2[i]
- if c1 >= 'A' && c1 <= 'Z' {
- c1 += 'a' - 'A'
- }
- if c2 >= 'A' && c2 <= 'Z' {
- c2 += 'a' - 'A'
- }
- if c1 != c2 {
- return false
- }
- }
- return true
-}
-
-// parseRedisInfoField extracts a field value from Redis INFO output
-func parseRedisInfoField(info, field string) string {
- lines := splitLines(info)
- prefix := field + ":"
- for _, line := range lines {
- if len(line) > len(prefix) && line[:len(prefix)] == prefix {
- return trimSpace(line[len(prefix):])
- }
- }
- return ""
-}
-
-func splitLines(s string) []string {
- var lines []string
- start := 0
- for i := 0; i < len(s); i++ {
- if s[i] == '\n' {
- line := s[start:i]
- if len(line) > 0 && line[len(line)-1] == '\r' {
- line = line[:len(line)-1]
- }
- lines = append(lines, line)
- start = i + 1
- }
- }
- if start < len(s) {
- lines = append(lines, s[start:])
- }
- return lines
-}
-
-func trimSpace(s string) string {
- start := 0
- end := len(s)
- for start < end && (s[start] == ' ' || s[start] == '\t' || s[start] == '\r' || s[start] == '\n') {
- start++
- }
- for end > start && (s[end-1] == ' ' || s[end-1] == '\t' || s[end-1] == '\r' || s[end-1] == '\n') {
- end--
- }
- return s[start:end]
-}
diff --git a/backend/internal/handlers/download.go b/backend/internal/handlers/download.go
deleted file mode 100644
index 839570b5..00000000
--- a/backend/internal/handlers/download.go
+++ /dev/null
@@ -1,80 +0,0 @@
-package handlers
-
-import (
- "claraverse/internal/document"
- "log"
-
- "github.com/gofiber/fiber/v2"
-)
-
-// DownloadHandler handles file download requests
-type DownloadHandler struct {
- documentService *document.Service
-}
-
-// NewDownloadHandler creates a new download handler
-func NewDownloadHandler() *DownloadHandler {
- return &DownloadHandler{
- documentService: document.GetService(),
- }
-}
-
-// Download serves a generated document and marks it for deletion
-func (h *DownloadHandler) Download(c *fiber.Ctx) error {
- documentID := c.Params("id")
-
- // Get user ID from auth middleware
- userID, ok := c.Locals("user_id").(string)
- if !ok || userID == "" || userID == "anonymous" {
- return c.Status(fiber.StatusUnauthorized).JSON(fiber.Map{
- "error": "Authentication required to download documents",
- })
- }
-
- // Get document
- doc, exists := h.documentService.GetDocument(documentID)
- if !exists {
- log.Printf("⚠️ [DOWNLOAD] Document not found: %s (user: %s)", documentID, userID)
- return c.Status(fiber.StatusNotFound).JSON(fiber.Map{
- "error": "Document not found or already deleted",
- })
- }
-
- // Verify ownership
- if doc.UserID != userID {
- log.Printf("🚫 [SECURITY] User %s denied access to document %s (owned by %s)",
- userID, documentID, doc.UserID)
- return c.Status(fiber.StatusForbidden).JSON(fiber.Map{
- "error": "Access denied to this document",
- })
- }
-
- log.Printf("📥 [DOWNLOAD] Serving document: %s (user: %s, size: %d bytes)",
- doc.Filename, doc.UserID, doc.Size)
-
- // Determine content type
- contentType := doc.ContentType
- if contentType == "" {
- contentType = "application/octet-stream" // Fallback for unknown types
- }
-
- // Set headers for download
- c.Set("Content-Disposition", "attachment; filename=\""+doc.Filename+"\"")
- c.Set("Content-Type", contentType)
-
- // Send file
- err := c.SendFile(doc.FilePath)
- if err != nil {
- log.Printf("❌ [DOWNLOAD] Failed to send file: %v", err)
- return c.Status(fiber.StatusInternalServerError).JSON(fiber.Map{
- "error": "Failed to download file",
- })
- }
-
- // Mark as downloaded (will be deleted in 5 minutes by cleanup job)
- h.documentService.MarkDownloaded(documentID)
-
- log.Printf("✅ [DOWNLOAD] Document downloaded: %s (user: %s)", doc.Filename, userID)
-
- return nil
-}
diff --git a/backend/internal/handlers/execution.go b/backend/internal/handlers/execution.go
deleted file mode 100644
index d408645a..00000000
--- a/backend/internal/handlers/execution.go
+++ /dev/null
@@ -1,126 +0,0 @@
-package handlers
-
-import (
- "claraverse/internal/services"
- "log"
- "strconv"
-
- "github.com/gofiber/fiber/v2"
- "go.mongodb.org/mongo-driver/bson/primitive"
-)
-
-// ExecutionHandler handles execution-related HTTP requests
-type ExecutionHandler struct {
- executionService *services.ExecutionService
-}
-
-// NewExecutionHandler creates a new execution handler
-func NewExecutionHandler(executionService *services.ExecutionService) *ExecutionHandler {
- return &ExecutionHandler{
- executionService: executionService,
- }
-}
-
-// ListByAgent returns paginated executions for a specific agent
-// GET /api/agents/:id/executions
-func (h *ExecutionHandler) ListByAgent(c *fiber.Ctx) error {
- agentID := c.Params("id")
- userID := c.Locals("user_id").(string)
-
- opts := h.parseListOptions(c)
-
- result, err := h.executionService.ListByAgent(c.Context(), agentID, userID, opts)
- if err != nil {
- log.Printf("❌ [EXECUTION] Failed to list agent executions: %v", err)
- return c.Status(fiber.StatusInternalServerError).JSON(fiber.Map{
- "error": "Failed to list executions",
- })
- }
-
- return c.JSON(result)
-}
-
-// ListAll returns paginated executions for the current user
-// GET /api/executions
-func (h *ExecutionHandler) ListAll(c *fiber.Ctx) error {
- userID := c.Locals("user_id").(string)
-
- opts := h.parseListOptions(c)
-
- result, err := h.executionService.ListByUser(c.Context(), userID, opts)
- if err != nil {
- log.Printf("❌ [EXECUTION] Failed to list user executions: %v", err)
- return c.Status(fiber.StatusInternalServerError).JSON(fiber.Map{
- "error": "Failed to list executions",
- })
- }
-
- return c.JSON(result)
-}
-
-// GetByID returns a specific execution
-// GET /api/executions/:id
-func (h *ExecutionHandler) GetByID(c *fiber.Ctx) error {
- executionIDStr := c.Params("id")
- userID := c.Locals("user_id").(string)
-
- executionID, err := primitive.ObjectIDFromHex(executionIDStr)
- if err != nil {
- return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{
- "error": "Invalid execution ID",
- })
- }
-
- execution, err := h.executionService.GetByIDAndUser(c.Context(), executionID, userID)
- if err != nil {
- if err.Error() == "execution not found" {
- return c.Status(fiber.StatusNotFound).JSON(fiber.Map{
- "error": "Execution not found",
- })
- }
- log.Printf("❌ [EXECUTION] Failed to get execution: %v", err)
- return c.Status(fiber.StatusInternalServerError).JSON(fiber.Map{
- "error": "Failed to get execution",
- })
- }
-
- return c.JSON(execution)
-}
-
-// GetStats returns execution statistics for an agent
-// GET /api/agents/:id/executions/stats
-func (h *ExecutionHandler) GetStats(c *fiber.Ctx) error {
- agentID := c.Params("id")
- userID := c.Locals("user_id").(string)
-
- stats, err := h.executionService.GetStats(c.Context(), agentID, userID)
- if err != nil {
- log.Printf("❌ [EXECUTION] Failed to get stats: %v", err)
- return c.Status(fiber.StatusInternalServerError).JSON(fiber.Map{
- "error": "Failed to get execution stats",
- })
- }
-
- return c.JSON(stats)
-}
-
-// parseListOptions extracts pagination and filter options from query params
-func (h *ExecutionHandler) parseListOptions(c *fiber.Ctx) *services.ListExecutionsOptions {
- opts := &services.ListExecutionsOptions{
- Page: 1,
- Limit: 20,
- Status: c.Query("status"),
- TriggerType: c.Query("trigger_type"),
- AgentID: c.Query("agent_id"),
- }
-
- if page, err := strconv.Atoi(c.Query("page")); err == nil && page > 0 {
- opts.Page = page
- }
-
- if limit, err := strconv.Atoi(c.Query("limit")); err == nil && limit > 0 && limit <= 100 {
- opts.Limit = limit
- }
-
- return opts
-}
diff --git a/backend/internal/handlers/handlers_test.go b/backend/internal/handlers/handlers_test.go
deleted file mode 100644
index 8de69f91..00000000
--- a/backend/internal/handlers/handlers_test.go
+++ /dev/null
@@ -1,418 +0,0 @@
-package handlers
-
-import (
- "claraverse/internal/database"
- "claraverse/internal/models"
- "claraverse/internal/services"
- "encoding/json"
- "io"
- "net/http/httptest"
- "os"
- "testing"
- "time"
-
- "github.com/gofiber/fiber/v2"
-)
-
-func setupTestApp(t *testing.T) (*fiber.App, *database.DB, func()) {
- tmpFile := "test_handlers.db"
- db, err := database.New(tmpFile)
- if err != nil {
- t.Fatalf("Failed to create test database: %v", err)
- }
-
- if err := db.Initialize(); err != nil {
- t.Fatalf("Failed to initialize test database: %v", err)
- }
-
- app := fiber.New()
-
- cleanup := func() {
- db.Close()
- os.Remove(tmpFile)
- }
-
- return app, db, cleanup
-}
-
-func createTestProvider(t *testing.T, db *database.DB) *models.Provider {
- providerService := services.NewProviderService(db)
- config := models.ProviderConfig{
- Name: "Test Provider",
- BaseURL: "https://api.test.com/v1",
- APIKey: "test-key",
- Enabled: true,
- }
-
- provider, err := providerService.Create(config)
- if err != nil {
- t.Fatalf("Failed to create test provider: %v", err)
- }
-
- return provider
-}
-
-func insertTestModel(t *testing.T, db *database.DB, model *models.Model) {
- _, err := db.Exec(`
- INSERT OR REPLACE INTO models
- (id, provider_id, name, display_name, is_visible, fetched_at)
- VALUES (?, ?, ?, ?, ?, ?)
- `, model.ID, model.ProviderID, model.Name, model.DisplayName, model.IsVisible, time.Now())
-
- if err != nil {
- t.Fatalf("Failed to insert test model: %v", err)
- }
-}
-
-// TestHealthHandler tests the health check endpoint
-func TestHealthHandler(t *testing.T) {
- app, _, cleanup := setupTestApp(t)
- defer cleanup()
-
- connManager := services.NewConnectionManager()
- handler := NewHealthHandler(connManager)
-
- app.Get("/health", handler.Handle)
-
- req := httptest.NewRequest("GET", "/health", nil)
- resp, err := app.Test(req)
- if err != nil {
- t.Fatalf("Failed to send request: %v", err)
- }
- defer resp.Body.Close()
-
- if resp.StatusCode != fiber.StatusOK {
- t.Errorf("Expected status 200, got %d", resp.StatusCode)
- }
-
- body, err := io.ReadAll(resp.Body)
- if err != nil {
- t.Fatalf("Failed to read response: %v", err)
- }
-
- var result map[string]interface{}
- if err := json.Unmarshal(body, &result); err != nil {
- t.Fatalf("Failed to parse JSON: %v", err)
- }
-
- if result["status"] != "healthy" {
- t.Errorf("Expected status 'healthy', got %v", result["status"])
- }
-
- if result["connections"] == nil {
- t.Error("Expected 'connections' field in response")
- }
-
- if result["timestamp"] == nil {
- t.Error("Expected 'timestamp' field in response")
- }
-}
-
-// TestModelHandler_List tests listing all models
-func TestModelHandler_List(t *testing.T) {
- app, db, cleanup := setupTestApp(t)
- defer cleanup()
-
- modelService := services.NewModelService(db)
- handler := NewModelHandler(modelService)
-
- app.Get("/api/models", handler.List)
-
- // Create test provider and models
- provider := createTestProvider(t, db)
- testModels := []models.Model{
- {ID: "model-1", ProviderID: provider.ID, Name: "Model 1", IsVisible: true},
- {ID: "model-2", ProviderID: provider.ID, Name: "Model 2", IsVisible: true},
- {ID: "model-3", ProviderID: provider.ID, Name: "Model 3", IsVisible: false},
- }
-
- for i := range testModels {
- insertTestModel(t, db, &testModels[i])
- }
-
- // Test with default (visible only)
- req := httptest.NewRequest("GET", "/api/models", nil)
- resp, err := app.Test(req)
- if err != nil {
- t.Fatalf("Failed to send request: %v", err)
- }
- defer resp.Body.Close()
-
- if resp.StatusCode != fiber.StatusOK {
- t.Errorf("Expected status 200, got %d", resp.StatusCode)
- }
-
- body, err := io.ReadAll(resp.Body)
- if err != nil {
- t.Fatalf("Failed to read response: %v", err)
- }
-
- var result map[string]interface{}
- if err := json.Unmarshal(body, &result); err != nil {
- t.Fatalf("Failed to parse JSON: %v", err)
- }
-
- models, ok := result["models"].([]interface{})
- if !ok {
- t.Fatal("Expected 'models' to be an array")
- }
-
- // Should only return visible models
- if len(models) != 2 {
- t.Errorf("Expected 2 visible models, got %d", len(models))
- }
-
- count, ok := result["count"].(float64)
- if !ok {
- t.Fatal("Expected 'count' to be a number")
- }
-
- if int(count) != 2 {
- t.Errorf("Expected count 2, got %d", int(count))
- }
-}
-
-// TestModelHandler_List_AllModels tests listing all models including hidden
-func TestModelHandler_List_AllModels(t *testing.T) {
- app, db, cleanup := setupTestApp(t)
- defer cleanup()
-
- modelService := services.NewModelService(db)
- handler := NewModelHandler(modelService)
-
- app.Get("/api/models", handler.List)
-
- // Create test provider and models
- provider := createTestProvider(t, db)
- testModels := []models.Model{
- {ID: "model-1", ProviderID: provider.ID, Name: "Model 1", IsVisible: true},
- {ID: "model-2", ProviderID: provider.ID, Name: "Model 2", IsVisible: false},
- }
-
- for i := range testModels {
- insertTestModel(t, db, &testModels[i])
- }
-
- // Test with visible_only=false
- req := httptest.NewRequest("GET", "/api/models?visible_only=false", nil)
- resp, err := app.Test(req)
- if err != nil {
- t.Fatalf("Failed to send request: %v", err)
- }
- defer resp.Body.Close()
-
- body, err := io.ReadAll(resp.Body)
- if err != nil {
- t.Fatalf("Failed to read response: %v", err)
- }
-
- var result map[string]interface{}
- if err := json.Unmarshal(body, &result); err != nil {
- t.Fatalf("Failed to parse JSON: %v", err)
- }
-
- models, ok := result["models"].([]interface{})
- if !ok {
- t.Fatal("Expected 'models' to be an array")
- }
-
- // Should return all models
- if len(models) != 2 {
- t.Errorf("Expected 2 models, got %d", len(models))
- }
-}
-
-// TestModelHandler_ListByProvider tests listing models for a specific provider
-func TestModelHandler_ListByProvider(t *testing.T) {
- app, db, cleanup := setupTestApp(t)
- defer cleanup()
-
- modelService := services.NewModelService(db)
- handler := NewModelHandler(modelService)
-
- app.Get("/api/providers/:id/models", handler.ListByProvider)
-
- // Create test providers and models
- provider1 := createTestProvider(t, db)
-
- providerService := services.NewProviderService(db)
- provider2Config := models.ProviderConfig{
- Name: "Provider 2",
- BaseURL: "https://api.provider2.com/v1",
- APIKey: "test-key-2",
- Enabled: true,
- }
- provider2, _ := providerService.Create(provider2Config)
-
- testModels := []models.Model{
- {ID: "model-1", ProviderID: provider1.ID, Name: "Model 1", IsVisible: true},
- {ID: "model-2", ProviderID: provider1.ID, Name: "Model 2", IsVisible: true},
- {ID: "model-3", ProviderID: provider2.ID, Name: "Model 3", IsVisible: true},
- }
-
- for i := range testModels {
- insertTestModel(t, db, &testModels[i])
- }
-
- // Test provider 1 models
- req := httptest.NewRequest("GET", "/api/providers/1/models", nil)
- resp, err := app.Test(req)
- if err != nil {
- t.Fatalf("Failed to send request: %v", err)
- }
- defer resp.Body.Close()
-
- if resp.StatusCode != fiber.StatusOK {
- t.Errorf("Expected status 200, got %d", resp.StatusCode)
- }
-
- body, err := io.ReadAll(resp.Body)
- if err != nil {
- t.Fatalf("Failed to read response: %v", err)
- }
-
- var result map[string]interface{}
- if err := json.Unmarshal(body, &result); err != nil {
- t.Fatalf("Failed to parse JSON: %v", err)
- }
-
- models, ok := result["models"].([]interface{})
- if !ok {
- t.Fatal("Expected 'models' to be an array")
- }
-
- if len(models) != 2 {
- t.Errorf("Expected 2 models for provider 1, got %d", len(models))
- }
-}
-
-// TestModelHandler_ListByProvider_InvalidID tests with invalid provider ID
-func TestModelHandler_ListByProvider_InvalidID(t *testing.T) {
- app, db, cleanup := setupTestApp(t)
- defer cleanup()
-
- modelService := services.NewModelService(db)
- handler := NewModelHandler(modelService)
-
- app.Get("/api/providers/:id/models", handler.ListByProvider)
-
- // Test with invalid ID
- req := httptest.NewRequest("GET", "/api/providers/invalid/models", nil)
- resp, err := app.Test(req)
- if err != nil {
- t.Fatalf("Failed to send request: %v", err)
- }
- defer resp.Body.Close()
-
- if resp.StatusCode != fiber.StatusBadRequest {
- t.Errorf("Expected status 400, got %d", resp.StatusCode)
- }
-
- body, err := io.ReadAll(resp.Body)
- if err != nil {
- t.Fatalf("Failed to read response: %v", err)
- }
-
- var result map[string]interface{}
- if err := json.Unmarshal(body, &result); err != nil {
- t.Fatalf("Failed to parse JSON: %v", err)
- }
-
- if result["error"] == nil {
- t.Error("Expected error message in response")
- }
-}
-
-// TestProviderHandler_List tests listing all providers
-func TestProviderHandler_List(t *testing.T) {
- app, db, cleanup := setupTestApp(t)
- defer cleanup()
-
- providerService := services.NewProviderService(db)
- handler := NewProviderHandler(providerService)
-
- app.Get("/api/providers", handler.List)
-
- // Create test providers
- configs := []models.ProviderConfig{
- {Name: "Provider A", BaseURL: "https://a.com", APIKey: "key-a", Enabled: true},
- {Name: "Provider B", BaseURL: "https://b.com", APIKey: "key-b", Enabled: true},
- {Name: "Provider C", BaseURL: "https://c.com", APIKey: "key-c", Enabled: false},
- }
-
- for _, config := range configs {
- providerService.Create(config)
- }
-
- req := httptest.NewRequest("GET", "/api/providers", nil)
- resp, err := app.Test(req)
- if err != nil {
- t.Fatalf("Failed to send request: %v", err)
- }
- defer resp.Body.Close()
-
- if resp.StatusCode != fiber.StatusOK {
- t.Errorf("Expected status 200, got %d", resp.StatusCode)
- }
-
- body, err := io.ReadAll(resp.Body)
- if err != nil {
- t.Fatalf("Failed to read response: %v", err)
- }
-
- var result map[string]interface{}
- if err := json.Unmarshal(body, &result); err != nil {
- t.Fatalf("Failed to parse JSON: %v", err)
- }
-
- providers, ok := result["providers"].([]interface{})
- if !ok {
- t.Fatal("Expected 'providers' to be an array")
- }
-
- // Should only return enabled providers
- if len(providers) != 2 {
- t.Errorf("Expected 2 enabled providers, got %d", len(providers))
- }
-}
-
-// TestHealthHandler_WithConnections tests health endpoint with active connections
-func TestHealthHandler_WithConnections(t *testing.T) {
- app, _, cleanup := setupTestApp(t)
- defer cleanup()
-
- connManager := services.NewConnectionManager()
- handler := NewHealthHandler(connManager)
-
- app.Get("/health", handler.Handle)
-
- // Simulate adding connections (we can't easily test WebSocket connections here,
- // so we'll just verify the endpoint works)
-
- req := httptest.NewRequest("GET", "/health", nil)
- resp, err := app.Test(req)
- if err != nil {
- t.Fatalf("Failed to send request: %v", err)
- }
- defer resp.Body.Close()
-
- body, err := io.ReadAll(resp.Body)
- if err != nil {
- t.Fatalf("Failed to read response: %v", err)
- }
-
- var result map[string]interface{}
- if err := json.Unmarshal(body, &result); err != nil {
- t.Fatalf("Failed to parse JSON: %v", err)
- }
-
- connections, ok := result["connections"].(float64)
- if !ok {
- t.Fatal("Expected 'connections' to be a number")
- }
-
- // Should be 0 since we haven't added any
- if int(connections) != 0 {
- t.Errorf("Expected 0 connections, got %d", int(connections))
- }
-}
diff --git a/backend/internal/handlers/health.go b/backend/internal/handlers/health.go
deleted file mode 100644
index e47567a4..00000000
--- a/backend/internal/handlers/health.go
+++ /dev/null
@@ -1,27 +0,0 @@
-package handlers
-
-import (
- "claraverse/internal/services"
- "time"
-
- "github.com/gofiber/fiber/v2"
-)
-
-// HealthHandler handles health check requests
-type HealthHandler struct {
- connManager *services.ConnectionManager
-}
-
-// NewHealthHandler creates a new health handler
-func NewHealthHandler(connManager *services.ConnectionManager) *HealthHandler {
- return &HealthHandler{connManager: connManager}
-}
-
-// Handle responds with server health status
-func (h *HealthHandler) Handle(c *fiber.Ctx) error {
- return c.JSON(fiber.Map{
- "status": "healthy",
- "connections": h.connManager.Count(),
- "timestamp": time.Now().Format(time.RFC3339),
- })
-}
diff --git a/backend/internal/handlers/image_proxy.go b/backend/internal/handlers/image_proxy.go
deleted file mode 100644
index 426d47b3..00000000
--- a/backend/internal/handlers/image_proxy.go
+++ /dev/null
@@ -1,283 +0,0 @@
-package handlers
-
-import (
- "claraverse/internal/security"
- "io"
- "log"
- "net/http"
- "net/url"
- "strings"
- "sync"
- "time"
-
- "github.com/gofiber/fiber/v2"
-)
-
-// ImageProxyHandler handles image proxy requests
-type ImageProxyHandler struct {
- client *http.Client
- cache *imageCache
-}
-
-// imageCache provides in-memory caching for proxied images
-type imageCache struct {
- mu sync.RWMutex
- cache map[string]*cachedImage
- maxSize int64 // Max total cache size in bytes
- currSize int64 // Current cache size
-}
-
-type cachedImage struct {
- data []byte
- contentType string
- timestamp time.Time
- size int64
-}
-
-const (
- maxImageSize = 10 * 1024 * 1024 // 10MB max per image
- maxCacheSize = 50 * 1024 * 1024 // 50MB total cache
- cacheTTL = 10 * time.Minute
- requestTimeout = 15 * time.Second
-)
-
-// NewImageProxyHandler creates a new image proxy handler
-func NewImageProxyHandler() *ImageProxyHandler {
- return &ImageProxyHandler{
- client: &http.Client{
- Timeout: requestTimeout,
- // Don't follow redirects automatically - we'll handle them
- CheckRedirect: func(req *http.Request, via []*http.Request) error {
- if len(via) >= 3 {
- return http.ErrUseLastResponse
- }
- return nil
- },
- },
- cache: &imageCache{
- cache: make(map[string]*cachedImage),
- maxSize: maxCacheSize,
- },
- }
-}
-
-// ProxyImage handles GET /api/proxy/image?url={encoded_url}
-func (h *ImageProxyHandler) ProxyImage(c *fiber.Ctx) error {
- // Get URL parameter
- imageURL := c.Query("url")
- if imageURL == "" {
- return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{
- "error": "url parameter is required",
- })
- }
-
- // Validate URL
- parsedURL, err := url.Parse(imageURL)
- if err != nil {
- log.Printf("⚠️ [IMAGE-PROXY] Invalid URL: %s - %v", imageURL, err)
- return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{
- "error": "invalid URL format",
- })
- }
-
- // Only allow http/https schemes
- if parsedURL.Scheme != "http" && parsedURL.Scheme != "https" {
- log.Printf("⚠️ [IMAGE-PROXY] Blocked non-http URL: %s", imageURL)
- return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{
- "error": "only http and https URLs are allowed",
- })
- }
-
- // SSRF protection: block requests to internal/private networks
- if err := security.ValidateURLForSSRF(imageURL); err != nil {
- log.Printf("🚫 [IMAGE-PROXY] SSRF blocked: %s - %v", truncateURL(imageURL), err)
- return c.Status(fiber.StatusForbidden).JSON(fiber.Map{
- "error": "access to internal resources is not allowed",
- })
- }
-
- // Check cache first
- if cached := h.cache.get(imageURL); cached != nil {
- log.Printf("✅ [IMAGE-PROXY] Cache hit for: %s", truncateURL(imageURL))
- c.Set("Content-Type", cached.contentType)
- c.Set("Cache-Control", "public, max-age=3600")
- c.Set("X-Cache", "HIT")
- return c.Send(cached.data)
- }
-
- // Fetch the image
- log.Printf("🖼️ [IMAGE-PROXY] Fetching: %s", truncateURL(imageURL))
-
- req, err := http.NewRequest("GET", imageURL, nil)
- if err != nil {
- log.Printf("❌ [IMAGE-PROXY] Failed to create request: %v", err)
- return c.Status(fiber.StatusInternalServerError).JSON(fiber.Map{
- "error": "failed to create request",
- })
- }
-
- // Set headers to look like a browser
- req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36")
- req.Header.Set("Accept", "image/avif,image/webp,image/apng,image/svg+xml,image/*,*/*;q=0.8")
- req.Header.Set("Accept-Language", "en-US,en;q=0.9")
-
- // Set appropriate referer based on host (some sites like Bing require specific referers)
- referer := parsedURL.Scheme + "://" + parsedURL.Host + "/"
- host := strings.ToLower(parsedURL.Host)
- if strings.Contains(host, "bing.net") || strings.Contains(host, "bing.com") {
- referer = "https://www.bing.com/"
- } else if strings.Contains(host, "google") {
- referer = "https://www.google.com/"
- } else if strings.Contains(host, "duckduckgo") {
- referer = "https://duckduckgo.com/"
- }
- req.Header.Set("Referer", referer)
-
- resp, err := h.client.Do(req)
- if err != nil {
- log.Printf("❌ [IMAGE-PROXY] Fetch failed: %v", err)
- return c.Status(fiber.StatusBadGateway).JSON(fiber.Map{
- "error": "failed to fetch image",
- })
- }
- defer resp.Body.Close()
-
- // Check response status
- if resp.StatusCode != http.StatusOK {
- log.Printf("⚠️ [IMAGE-PROXY] Upstream returned %d for: %s", resp.StatusCode, truncateURL(imageURL))
- return c.Status(fiber.StatusBadGateway).JSON(fiber.Map{
- "error": "upstream server returned error",
- })
- }
-
- // Validate content type
- contentType := resp.Header.Get("Content-Type")
- if !isValidImageContentType(contentType) {
- log.Printf("⚠️ [IMAGE-PROXY] Invalid content type: %s for: %s", contentType, truncateURL(imageURL))
- return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{
- "error": "URL does not point to a valid image",
- })
- }
-
- // Read body with size limit
- limitedReader := io.LimitReader(resp.Body, maxImageSize+1)
- data, err := io.ReadAll(limitedReader)
- if err != nil {
- log.Printf("❌ [IMAGE-PROXY] Failed to read response: %v", err)
- return c.Status(fiber.StatusInternalServerError).JSON(fiber.Map{
- "error": "failed to read image data",
- })
- }
-
- // Check if image exceeds size limit
- if int64(len(data)) > maxImageSize {
- log.Printf("⚠️ [IMAGE-PROXY] Image too large: %d bytes for: %s", len(data), truncateURL(imageURL))
- return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{
- "error": "image exceeds maximum allowed size (10MB)",
- })
- }
-
- // Cache the image
- h.cache.set(imageURL, data, contentType)
-
- log.Printf("✅ [IMAGE-PROXY] Served: %s (%d bytes)", truncateURL(imageURL), len(data))
-
- // Send response
- c.Set("Content-Type", contentType)
- c.Set("Cache-Control", "public, max-age=3600")
- c.Set("X-Cache", "MISS")
- return c.Send(data)
-}
-
-// isValidImageContentType checks if the content type is a valid image type
-func isValidImageContentType(contentType string) bool {
- contentType = strings.ToLower(contentType)
- validTypes := []string{
- "image/jpeg",
- "image/jpg",
- "image/png",
- "image/gif",
- "image/webp",
- "image/svg+xml",
- "image/avif",
- "image/bmp",
- "image/tiff",
- }
-
- for _, vt := range validTypes {
- if strings.HasPrefix(contentType, vt) {
- return true
- }
- }
- return false
-}
-
-// truncateURL truncates URL for logging
-func truncateURL(u string) string {
- if len(u) > 80 {
- return u[:77] + "..."
- }
- return u
-}
-
-// Cache methods
-
-func (c *imageCache) get(key string) *cachedImage {
- c.mu.RLock()
- defer c.mu.RUnlock()
-
- cached, exists := c.cache[key]
- if !exists {
- return nil
- }
-
- // Check TTL
- if time.Since(cached.timestamp) > cacheTTL {
- return nil
- }
-
- return cached
-}
-
-func (c *imageCache) set(key string, data []byte, contentType string) {
- c.mu.Lock()
- defer c.mu.Unlock()
-
- size := int64(len(data))
-
- // If this single image is too large, don't cache it
- if size > c.maxSize/2 {
- return
- }
-
- // Evict old entries if needed
- for c.currSize+size > c.maxSize && len(c.cache) > 0 {
- c.evictOldest()
- }
-
- // Store in cache
- c.cache[key] = &cachedImage{
- data: data,
- contentType: contentType,
- timestamp: time.Now(),
- size: size,
- }
- c.currSize += size
-}
-
-func (c *imageCache) evictOldest() {
- var oldestKey string
- var oldestTime time.Time
-
- for k, v := range c.cache {
- if oldestKey == "" || v.timestamp.Before(oldestTime) {
- oldestKey = k
- oldestTime = v.timestamp
- }
- }
-
- if oldestKey != "" {
- c.currSize -= c.cache[oldestKey].size
- delete(c.cache, oldestKey)
- }
-}
diff --git a/backend/internal/handlers/mcp_websocket.go b/backend/internal/handlers/mcp_websocket.go
deleted file mode 100644
index 6dbbf28a..00000000
--- a/backend/internal/handlers/mcp_websocket.go
+++ /dev/null
@@ -1,201 +0,0 @@
-package handlers
-
-import (
- "encoding/json"
- "fmt"
- "log"
- "time"
-
- "claraverse/internal/models"
- "claraverse/internal/services"
- "github.com/gofiber/contrib/websocket"
- "github.com/gofiber/fiber/v2"
-)
-
-// MCPWebSocketHandler handles MCP client WebSocket connections
-type MCPWebSocketHandler struct {
- mcpService *services.MCPBridgeService
-}
-
-// NewMCPWebSocketHandler creates a new MCP WebSocket handler
-func NewMCPWebSocketHandler(mcpService *services.MCPBridgeService) *MCPWebSocketHandler {
- return &MCPWebSocketHandler{
- mcpService: mcpService,
- }
-}
-
-// HandleConnection handles incoming MCP client WebSocket connections
-func (h *MCPWebSocketHandler) HandleConnection(c *websocket.Conn) {
- // Get user from fiber context (set by auth middleware)
- userID := c.Locals("user_id").(string)
-
- if userID == "" || userID == "anonymous" {
- log.Printf("❌ MCP connection rejected: no authenticated user")
- c.WriteJSON(fiber.Map{
- "type": "error",
- "payload": map[string]interface{}{
- "message": "Authentication required",
- },
- })
- c.Close()
- return
- }
-
- log.Printf("🔌 MCP client connecting: user=%s", userID)
-
- var mcpConn *models.MCPConnection
- var clientID string
-
- // Read loop
- for {
- var msg models.MCPClientMessage
- err := c.ReadJSON(&msg)
- if err != nil {
- if mcpConn != nil {
- log.Printf("MCP client disconnected: %v", err)
- h.mcpService.DisconnectClient(clientID)
- }
- break
- }
-
- switch msg.Type {
- case "register_tools":
- // Parse registration payload
- regData, err := json.Marshal(msg.Payload)
- if err != nil {
- log.Printf("Failed to marshal registration payload: %v", err)
- continue
- }
-
- var registration models.MCPToolRegistration
- err = json.Unmarshal(regData, ®istration)
- if err != nil {
- log.Printf("Failed to unmarshal registration: %v", err)
- c.WriteJSON(models.MCPServerMessage{
- Type: "error",
- Payload: map[string]interface{}{
- "message": "Invalid registration format",
- },
- })
- continue
- }
-
- // Register client
- conn, err := h.mcpService.RegisterClient(userID, ®istration)
- if err != nil {
- log.Printf("Failed to register MCP client: %v", err)
- c.WriteJSON(models.MCPServerMessage{
- Type: "error",
- Payload: map[string]interface{}{
- "message": fmt.Sprintf("Registration failed: %v", err),
- },
- })
- continue
- }
-
- mcpConn = conn
- clientID = registration.ClientID
-
- // Start write loop
- go h.writeLoop(c, conn)
-
- log.Printf("✅ MCP client registered successfully: user=%s, client=%s", userID, clientID)
-
- case "tool_result":
- // Handle tool execution result
- resultData, err := json.Marshal(msg.Payload)
- if err != nil {
- log.Printf("Failed to marshal tool result: %v", err)
- continue
- }
-
- var result models.MCPToolResult
- err = json.Unmarshal(resultData, &result)
- if err != nil {
- log.Printf("Failed to unmarshal tool result: %v", err)
- continue
- }
-
- // Log execution for audit
- execTime := 0 // We don't track this yet, but could add it
- h.mcpService.LogToolExecution(userID, "", "", execTime, result.Success, result.Error)
-
- log.Printf("Tool result received: call_id=%s, success=%v", result.CallID, result.Success)
-
- // Forward result to pending result channel
- if conn, exists := h.mcpService.GetConnection(clientID); exists {
- if resultChan, pending := conn.PendingResults[result.CallID]; pending {
- // Non-blocking send to result channel
- select {
- case resultChan <- result:
- log.Printf("✅ Tool result forwarded to waiting channel: %s", result.CallID)
- default:
- log.Printf("⚠️ Result channel full or closed for call_id: %s", result.CallID)
- }
- } else {
- log.Printf("⚠️ No pending result channel for call_id: %s", result.CallID)
- }
- }
-
- case "heartbeat":
- // Update heartbeat
- if clientID != "" {
- err := h.mcpService.UpdateHeartbeat(clientID)
- if err != nil {
- log.Printf("Failed to update heartbeat: %v", err)
- }
- }
-
- case "disconnect":
- // Client is gracefully disconnecting
- if clientID != "" {
- h.mcpService.DisconnectClient(clientID)
- }
- c.Close()
- return
-
- default:
- log.Printf("Unknown message type from MCP client: %s", msg.Type)
- c.WriteJSON(models.MCPServerMessage{
- Type: "error",
- Payload: map[string]interface{}{
- "message": "Unknown message type",
- },
- })
- }
- }
-}
-
-// writeLoop handles outgoing messages to the MCP client
-func (h *MCPWebSocketHandler) writeLoop(c *websocket.Conn, conn *models.MCPConnection) {
- ticker := time.NewTicker(30 * time.Second)
- defer ticker.Stop()
-
- for {
- select {
- case msg, ok := <-conn.WriteChan:
- if !ok {
- // Channel closed
- return
- }
-
- err := c.WriteJSON(msg)
- if err != nil {
- log.Printf("Failed to write message to MCP client: %v", err)
- return
- }
-
- case <-conn.StopChan:
- // Stop signal received
- return
-
- case <-ticker.C:
- // Send ping to keep connection alive
- err := c.WriteMessage(websocket.PingMessage, []byte{})
- if err != nil {
- log.Printf("Failed to send ping to MCP client: %v", err)
- return
- }
- }
- }
-}
diff --git a/backend/internal/handlers/memory_handler.go b/backend/internal/handlers/memory_handler.go
deleted file mode 100644
index 217953ba..00000000
--- a/backend/internal/handlers/memory_handler.go
+++ /dev/null
@@ -1,578 +0,0 @@
-package handlers
-
-import (
- "context"
- "log"
- "regexp"
- "strconv"
- "strings"
- "time"
-
- "claraverse/internal/models"
- "claraverse/internal/services"
-
- "github.com/gofiber/fiber/v2"
- "go.mongodb.org/mongo-driver/bson/primitive"
-)
-
-// MemoryHandler handles memory-related API endpoints
-type MemoryHandler struct {
- memoryStorageService *services.MemoryStorageService
- memoryExtractionService *services.MemoryExtractionService
- chatService *services.ChatService
-}
-
-// NewMemoryHandler creates a new memory handler
-func NewMemoryHandler(
- memoryStorageService *services.MemoryStorageService,
- memoryExtractionService *services.MemoryExtractionService,
- chatService *services.ChatService,
-) *MemoryHandler {
- return &MemoryHandler{
- memoryStorageService: memoryStorageService,
- memoryExtractionService: memoryExtractionService,
- chatService: chatService,
- }
-}
-
-// ListMemories returns paginated list of memories with optional filters
-// GET /api/v1/memories?category=preferences&tags=ui,theme&includeArchived=false&page=1&pageSize=20
-func (h *MemoryHandler) ListMemories(c *fiber.Ctx) error {
- userID, ok := c.Locals("user_id").(string)
- if !ok || userID == "" {
- return c.Status(fiber.StatusUnauthorized).JSON(fiber.Map{
- "error": "Authentication required",
- })
- }
-
- // Parse query parameters
- category := c.Query("category", "")
- tagsParam := c.Query("tags", "")
- includeArchived := c.Query("includeArchived", "false") == "true"
- page, _ := strconv.Atoi(c.Query("page", "1"))
- pageSize, _ := strconv.Atoi(c.Query("pageSize", "20"))
-
- // Validate pagination
- if page < 1 {
- page = 1
- }
- if pageSize < 1 || pageSize > 100 {
- pageSize = 20
- }
-
- // Parse and sanitize tags (SECURITY: prevent NoSQL injection)
- var tags []string
- if tagsParam != "" {
- // Split by comma
- rawTags := strings.Split(tagsParam, ",")
- for _, rawTag := range rawTags {
- sanitizedTag := sanitizeTag(strings.TrimSpace(rawTag))
- if sanitizedTag != "" && len(sanitizedTag) <= 50 {
- tags = append(tags, sanitizedTag)
- }
- }
-
- // Limit number of tags to prevent abuse
- if len(tags) > 20 {
- tags = tags[:20]
- }
- }
-
- ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
- defer cancel()
-
- // Get memories
- memories, total, err := h.memoryStorageService.ListMemories(
- ctx,
- userID,
- category,
- tags,
- includeArchived,
- page,
- pageSize,
- )
- if err != nil {
- log.Printf("❌ [MEMORY-API] Failed to list memories: %v", err)
- return c.Status(fiber.StatusInternalServerError).JSON(fiber.Map{
- "error": "Failed to retrieve memories",
- })
- }
-
- // Build response
- memoryResponses := make([]fiber.Map, len(memories))
- for i, mem := range memories {
- memoryResponses[i] = buildMemoryResponse(mem)
- }
-
- return c.JSON(fiber.Map{
- "memories": memoryResponses,
- "pagination": fiber.Map{
- "page": page,
- "page_size": pageSize,
- "total": total,
- "total_pages": (total + int64(pageSize) - 1) / int64(pageSize),
- },
- })
-}
-
-// GetMemory returns a single memory by ID
-// GET /api/v1/memories/:id
-func (h *MemoryHandler) GetMemory(c *fiber.Ctx) error {
- userID, ok := c.Locals("user_id").(string)
- if !ok || userID == "" {
- return c.Status(fiber.StatusUnauthorized).JSON(fiber.Map{
- "error": "Authentication required",
- })
- }
- memoryIDParam := c.Params("id")
-
- memoryID, err := primitive.ObjectIDFromHex(memoryIDParam)
- if err != nil {
- return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{
- "error": "Invalid memory ID",
- })
- }
-
- ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
- defer cancel()
-
- memory, err := h.memoryStorageService.GetMemory(ctx, userID, memoryID)
- if err != nil {
- log.Printf("❌ [MEMORY-API] Failed to get memory: %v", err)
- return c.Status(fiber.StatusNotFound).JSON(fiber.Map{
- "error": "Memory not found",
- })
- }
-
- return c.JSON(buildMemoryResponse(*memory))
-}
-
-// CreateMemory creates a new manual memory
-// POST /api/v1/memories
-type CreateMemoryRequest struct {
- Content string `json:"content"`
- Category string `json:"category"`
- Tags []string `json:"tags"`
-}
-
-func (h *MemoryHandler) CreateMemory(c *fiber.Ctx) error {
- userID, ok := c.Locals("user_id").(string)
- if !ok || userID == "" {
- return c.Status(fiber.StatusUnauthorized).JSON(fiber.Map{
- "error": "Authentication required",
- })
- }
-
- var req CreateMemoryRequest
- if err := c.BodyParser(&req); err != nil {
- return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{
- "error": "Invalid request body",
- })
- }
-
- // Validate request (SECURITY: input validation)
- const MaxMemoryContentLength = 10000 // 10KB per memory
- const MaxTagCount = 20
- const MaxTagLength = 50
-
- if req.Content == "" {
- return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{
- "error": "Content is required",
- })
- }
-
- if len(req.Content) > MaxMemoryContentLength {
- return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{
- "error": "Content must be less than 10,000 characters",
- })
- }
-
- if req.Category == "" {
- req.Category = "context"
- }
-
- // Sanitize and validate tags
- if len(req.Tags) > MaxTagCount {
- return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{
- "error": "Maximum 20 tags allowed",
- })
- }
-
- sanitizedTags := make([]string, 0, len(req.Tags))
- for _, tag := range req.Tags {
- sanitized := sanitizeTag(strings.TrimSpace(tag))
- if sanitized != "" && len(sanitized) <= MaxTagLength {
- sanitizedTags = append(sanitizedTags, sanitized)
- }
- }
- req.Tags = sanitizedTags
-
- // Validate category
- validCategories := map[string]bool{
- "personal_info": true,
- "preferences": true,
- "context": true,
- "fact": true,
- "instruction": true,
- }
- if !validCategories[req.Category] {
- return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{
- "error": "Invalid category. Must be one of: personal_info, preferences, context, fact, instruction",
- })
- }
-
- if req.Tags == nil {
- req.Tags = []string{}
- }
-
- ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
- defer cancel()
-
- // Create memory with default engagement score of 0.5 (manually created)
- memory, err := h.memoryStorageService.CreateMemory(
- ctx,
- userID,
- req.Content,
- req.Category,
- req.Tags,
- 0.5, // Default engagement for manual memories
- "", // No conversation ID for manual memories
- )
- if err != nil {
- log.Printf("❌ [MEMORY-API] Failed to create memory: %v", err)
- return c.Status(fiber.StatusInternalServerError).JSON(fiber.Map{
- "error": "Failed to create memory",
- })
- }
-
- // Decrypt for response
- decryptedMemory, err := h.memoryStorageService.GetMemory(ctx, userID, memory.ID)
- if err != nil {
- log.Printf("❌ [MEMORY-API] Failed to decrypt memory: %v", err)
- return c.Status(fiber.StatusInternalServerError).JSON(fiber.Map{
- "error": "Memory created but failed to retrieve",
- })
- }
-
- return c.Status(fiber.StatusCreated).JSON(buildMemoryResponse(*decryptedMemory))
-}
-
-// UpdateMemory updates an existing memory
-// PUT /api/v1/memories/:id
-type UpdateMemoryRequest struct {
- Content *string `json:"content,omitempty"`
- Category *string `json:"category,omitempty"`
- Tags *[]string `json:"tags,omitempty"`
-}
-
-func (h *MemoryHandler) UpdateMemory(c *fiber.Ctx) error {
- userID, ok := c.Locals("user_id").(string)
- if !ok || userID == "" {
- return c.Status(fiber.StatusUnauthorized).JSON(fiber.Map{
- "error": "Authentication required",
- })
- }
- memoryIDParam := c.Params("id")
-
- memoryID, err := primitive.ObjectIDFromHex(memoryIDParam)
- if err != nil {
- return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{
- "error": "Invalid memory ID",
- })
- }
-
- var req UpdateMemoryRequest
- if err := c.BodyParser(&req); err != nil {
- return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{
- "error": "Invalid request body",
- })
- }
-
- ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
- defer cancel()
-
- // Get existing memory to retrieve current values
- existingMemory, err := h.memoryStorageService.GetMemory(ctx, userID, memoryID)
- if err != nil {
- return c.Status(fiber.StatusNotFound).JSON(fiber.Map{
- "error": "Memory not found",
- })
- }
-
- // Validate and sanitize input
- const MaxMemoryContentLength = 10000
- const MaxTagCount = 20
- const MaxTagLength = 50
-
- // Update fields
- content := existingMemory.DecryptedContent
- if req.Content != nil {
- if len(*req.Content) > MaxMemoryContentLength {
- return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{
- "error": "Content must be less than 10,000 characters",
- })
- }
- content = *req.Content
- }
-
- category := existingMemory.Category
- if req.Category != nil {
- category = *req.Category
- // Validate category
- validCategories := map[string]bool{
- "personal_info": true,
- "preferences": true,
- "context": true,
- "fact": true,
- "instruction": true,
- }
- if !validCategories[category] {
- return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{
- "error": "Invalid category",
- })
- }
- }
-
- tags := existingMemory.Tags
- if req.Tags != nil {
- if len(*req.Tags) > MaxTagCount {
- return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{
- "error": "Maximum 20 tags allowed",
- })
- }
-
- // Sanitize tags
- sanitizedTags := make([]string, 0, len(*req.Tags))
- for _, tag := range *req.Tags {
- sanitized := sanitizeTag(strings.TrimSpace(tag))
- if sanitized != "" && len(sanitized) <= MaxTagLength {
- sanitizedTags = append(sanitizedTags, sanitized)
- }
- }
- tags = sanitizedTags
- }
-
- // SECURITY FIX: Use atomic update instead of delete-create to prevent race conditions
- updatedMemory, err := h.memoryStorageService.UpdateMemoryInPlace(
- ctx,
- userID,
- memoryID,
- content,
- category,
- tags,
- existingMemory.SourceEngagement,
- existingMemory.ConversationID,
- )
- if err != nil {
- log.Printf("❌ [MEMORY-API] Failed to update memory: %v", err)
- return c.Status(fiber.StatusInternalServerError).JSON(fiber.Map{
- "error": "Failed to update memory",
- })
- }
-
- // Get decrypted version for response
- decryptedMemory, err := h.memoryStorageService.GetMemory(ctx, userID, updatedMemory.ID)
- if err != nil {
- return c.Status(fiber.StatusInternalServerError).JSON(fiber.Map{
- "error": "Memory updated but failed to retrieve",
- })
- }
-
- return c.JSON(buildMemoryResponse(*decryptedMemory))
-}
-
-// DeleteMemory permanently deletes a memory
-// DELETE /api/v1/memories/:id
-func (h *MemoryHandler) DeleteMemory(c *fiber.Ctx) error {
- userID, ok := c.Locals("user_id").(string)
- if !ok || userID == "" {
- return c.Status(fiber.StatusUnauthorized).JSON(fiber.Map{
- "error": "Authentication required",
- })
- }
- memoryIDParam := c.Params("id")
-
- memoryID, err := primitive.ObjectIDFromHex(memoryIDParam)
- if err != nil {
- return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{
- "error": "Invalid memory ID",
- })
- }
-
- ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
- defer cancel()
-
- err = h.memoryStorageService.DeleteMemory(ctx, userID, memoryID)
- if err != nil {
- log.Printf("❌ [MEMORY-API] Failed to delete memory: %v", err)
- return c.Status(fiber.StatusNotFound).JSON(fiber.Map{
- "error": "Memory not found or already deleted",
- })
- }
-
- return c.JSON(fiber.Map{
- "success": true,
- "message": "Memory deleted successfully",
- })
-}
-
-// ArchiveMemory archives a memory
-// POST /api/v1/memories/:id/archive
-func (h *MemoryHandler) ArchiveMemory(c *fiber.Ctx) error {
- userID, ok := c.Locals("user_id").(string)
- if !ok || userID == "" {
- return c.Status(fiber.StatusUnauthorized).JSON(fiber.Map{
- "error": "Authentication required",
- })
- }
- memoryIDParam := c.Params("id")
-
- memoryID, err := primitive.ObjectIDFromHex(memoryIDParam)
- if err != nil {
- return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{
- "error": "Invalid memory ID",
- })
- }
-
- ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
- defer cancel()
-
- err = h.memoryStorageService.ArchiveMemory(ctx, userID, memoryID)
- if err != nil {
- log.Printf("❌ [MEMORY-API] Failed to archive memory: %v", err)
- return c.Status(fiber.StatusNotFound).JSON(fiber.Map{
- "error": "Memory not found",
- })
- }
-
- return c.JSON(fiber.Map{
- "success": true,
- "message": "Memory archived successfully",
- })
-}
-
-// UnarchiveMemory restores an archived memory
-// POST /api/v1/memories/:id/unarchive
-func (h *MemoryHandler) UnarchiveMemory(c *fiber.Ctx) error {
- userID, ok := c.Locals("user_id").(string)
- if !ok || userID == "" {
- return c.Status(fiber.StatusUnauthorized).JSON(fiber.Map{
- "error": "Authentication required",
- })
- }
- memoryIDParam := c.Params("id")
-
- memoryID, err := primitive.ObjectIDFromHex(memoryIDParam)
- if err != nil {
- return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{
- "error": "Invalid memory ID",
- })
- }
-
- ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
- defer cancel()
-
- err = h.memoryStorageService.UnarchiveMemory(ctx, userID, memoryID)
- if err != nil {
- log.Printf("❌ [MEMORY-API] Failed to unarchive memory: %v", err)
- return c.Status(fiber.StatusNotFound).JSON(fiber.Map{
- "error": "Memory not found",
- })
- }
-
- return c.JSON(fiber.Map{
- "success": true,
- "message": "Memory restored successfully",
- })
-}
-
-// GetMemoryStats returns statistics about user's memories
-// GET /api/v1/memories/stats
-func (h *MemoryHandler) GetMemoryStats(c *fiber.Ctx) error {
- userID, ok := c.Locals("user_id").(string)
- if !ok || userID == "" {
- return c.Status(fiber.StatusUnauthorized).JSON(fiber.Map{
- "error": "Authentication required",
- })
- }
-
- ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
- defer cancel()
-
- stats, err := h.memoryStorageService.GetMemoryStats(ctx, userID)
- if err != nil {
- log.Printf("❌ [MEMORY-API] Failed to get memory stats: %v", err)
- return c.Status(fiber.StatusInternalServerError).JSON(fiber.Map{
- "error": "Failed to retrieve memory statistics",
- })
- }
-
- return c.JSON(stats)
-}
-
-// TriggerMemoryExtraction manually triggers memory extraction for a conversation
-// POST /api/v1/conversations/:id/extract-memories
-func (h *MemoryHandler) TriggerMemoryExtraction(c *fiber.Ctx) error {
- userID, ok := c.Locals("user_id").(string)
- if !ok || userID == "" {
- return c.Status(fiber.StatusUnauthorized).JSON(fiber.Map{
- "error": "Authentication required",
- })
- }
- conversationID := c.Params("id")
-
- ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
- defer cancel()
-
- // Get conversation messages
- messages := h.chatService.GetConversationMessages(conversationID)
- if len(messages) == 0 {
- return c.Status(fiber.StatusNotFound).JSON(fiber.Map{
- "error": "Conversation not found or has no messages",
- })
- }
-
- // Enqueue extraction job
- err := h.memoryExtractionService.EnqueueExtraction(ctx, userID, conversationID, messages)
- if err != nil {
- log.Printf("❌ [MEMORY-API] Failed to trigger extraction: %v", err)
- return c.Status(fiber.StatusInternalServerError).JSON(fiber.Map{
- "error": "Failed to trigger memory extraction",
- })
- }
-
- return c.JSON(fiber.Map{
- "success": true,
- "message": "Memory extraction queued successfully",
- })
-}
-
-// buildMemoryResponse creates a response object from a decrypted memory
-func buildMemoryResponse(mem models.DecryptedMemory) fiber.Map {
- return fiber.Map{
- "id": mem.ID.Hex(),
- "content": mem.DecryptedContent,
- "category": mem.Category,
- "tags": mem.Tags,
- "score": mem.Score,
- "access_count": mem.AccessCount,
- "last_accessed_at": mem.LastAccessedAt,
- "is_archived": mem.IsArchived,
- "archived_at": mem.ArchivedAt,
- "source_engagement": mem.SourceEngagement,
- "conversation_id": mem.ConversationID,
- "created_at": mem.CreatedAt,
- "updated_at": mem.UpdatedAt,
- "version": mem.Version,
- }
-}
-
-// sanitizeTag removes potentially dangerous characters from tags
-// SECURITY: Prevents NoSQL injection via tag parameters
-func sanitizeTag(tag string) string {
- // Only allow alphanumeric characters, hyphens, and underscores
- // This prevents MongoDB operators like $where, $regex, etc.
- reg := regexp.MustCompile(`[^a-zA-Z0-9_-]`)
- sanitized := reg.ReplaceAllString(tag, "")
- return sanitized
-}
diff --git a/backend/internal/handlers/model.go b/backend/internal/handlers/model.go
deleted file mode 100644
index 6adb9135..00000000
--- a/backend/internal/handlers/model.go
+++ /dev/null
@@ -1,313 +0,0 @@
-package handlers
-
-import (
- "claraverse/internal/models"
- "claraverse/internal/services"
- "fmt"
- "log"
- "strconv"
- "strings"
-
- "github.com/gofiber/fiber/v2"
-)
-
-// aliasKey creates a composite key for tracking aliased models by provider+name
-func aliasKey(providerID int, modelName string) string {
- return fmt.Sprintf("%d:%s", providerID, strings.ToLower(modelName))
-}
-
-// ModelHandler handles model-related requests
-type ModelHandler struct {
- modelService *services.ModelService
-}
-
-// NewModelHandler creates a new model handler
-func NewModelHandler(modelService *services.ModelService) *ModelHandler {
- return &ModelHandler{modelService: modelService}
-}
-
-// List returns all available models
-func (h *ModelHandler) List(c *fiber.Ctx) error {
- // Check if we should only return visible models
- visibleOnly := c.Query("visible_only", "true") == "true"
-
- modelsList, err := h.modelService.GetAll(visibleOnly)
- if err != nil {
- return c.Status(fiber.StatusInternalServerError).JSON(fiber.Map{
- "error": "Failed to fetch models",
- })
- }
-
- // Get config service for alias information
- configService := services.GetConfigService()
-
- // Build a map of model alias -> global recommendation tier with full tier info
- // Query global tiers from recommended_models table joined with tier_labels
- type TierInfo struct {
- Tier string
- Label string
- Description string
- Icon string
- }
- modelRecommendationTier := make(map[string]TierInfo) // model_alias -> tier info
-
- rows, err := h.modelService.GetDB().Query(`
- SELECT r.tier, r.model_alias, r.provider_id,
- t.label, t.description, t.icon
- FROM recommended_models r
- JOIN tier_labels t ON r.tier = t.tier
- `)
- if err != nil {
- log.Printf("⚠️ [MODEL-HANDLER] Failed to load global tiers: %v", err)
- } else {
- defer rows.Close()
- for rows.Next() {
- var tier, modelAlias, label, description, icon string
- var providerID int
- if err := rows.Scan(&tier, &modelAlias, &providerID, &label, &description, &icon); err != nil {
- log.Printf("⚠️ [MODEL-HANDLER] Failed to scan tier row: %v", err)
- continue
- }
- // Use lowercase alias for case-insensitive matching
- key := fmt.Sprintf("%d:%s", providerID, strings.ToLower(modelAlias))
- modelRecommendationTier[key] = TierInfo{
- Tier: tier,
- Label: label,
- Description: description,
- Icon: icon,
- }
- log.Printf("🎯 [MODEL-HANDLER] Loaded global tier: %s -> %s (%s - %s) (provider %d)", modelAlias, tier, label, icon, providerID)
- }
- }
-
- // Build enriched models with alias structure
- // If a model has an alias, we expose ONLY the alias to the frontend
- enrichedModels := make([]interface{}, 0)
- aliasedModels := make(map[string]bool) // Track which original models have been aliased (key: providerID:lowercase_name)
-
- // First pass: Add all aliased models
- allAliases := configService.GetAllModelAliases()
- for providerID, aliases := range allAliases {
- for aliasName, aliasInfo := range aliases {
- // Find the original model to get its capabilities
- // Use case-insensitive comparison for model name OR ID matching
- var foundModel *models.Model
- actualModelLower := strings.ToLower(aliasInfo.ActualModel)
- for i := range modelsList {
- if modelsList[i].ProviderID == providerID &&
- (strings.ToLower(modelsList[i].Name) == actualModelLower ||
- strings.ToLower(modelsList[i].ID) == actualModelLower) {
- foundModel = &modelsList[i]
- // Use composite key (providerID:lowercase_name) to track aliased models
- aliasedModels[aliasKey(providerID, modelsList[i].Name)] = true
- aliasedModels[aliasKey(providerID, modelsList[i].ID)] = true
- break
- }
- }
-
- if foundModel == nil {
- log.Printf("⚠️ [MODEL-ALIAS] Could not find model '%s' for alias '%s' (provider %d)", aliasInfo.ActualModel, aliasName, providerID)
- continue
- }
-
- // Determine supports_vision: use alias override if set, otherwise use model's value
- supportsVision := foundModel.SupportsVision
- if aliasInfo.SupportsVision != nil {
- supportsVision = *aliasInfo.SupportsVision
- }
-
- // Determine agents_enabled: use alias Agents flag if set, otherwise default to true
- agentsEnabled := true // Default to true (all models available for agents)
- if aliasInfo.Agents != nil {
- agentsEnabled = *aliasInfo.Agents
- }
-
- // Get provider security status
- isProviderSecure := configService.IsProviderSecure(providerID)
-
- // Create model entry using alias as the ID
- modelMap := map[string]interface{}{
- "id": aliasName, // Alias name becomes the ID
- "provider_id": providerID,
- "provider_name": foundModel.ProviderName,
- "name": aliasInfo.DisplayName, // Use alias display name
- "display_name": aliasInfo.DisplayName, // Use alias display name
- "supports_tools": foundModel.SupportsTools,
- "supports_streaming": foundModel.SupportsStreaming,
- "supports_vision": supportsVision,
- "agents_enabled": agentsEnabled,
- "provider_secure": isProviderSecure,
- "is_visible": foundModel.IsVisible,
- "fetched_at": foundModel.FetchedAt,
- }
-
- // Check if this model (by alias name) is in the recommendation tier
- recommendationKey := fmt.Sprintf("%d:%s", providerID, strings.ToLower(aliasName))
- var tierDescription string
- if tierInfo, exists := modelRecommendationTier[recommendationKey]; exists {
- modelMap["recommendation_tier"] = map[string]interface{}{
- "tier": tierInfo.Tier,
- "label": tierInfo.Label,
- "description": tierInfo.Description,
- "icon": tierInfo.Icon,
- }
- tierDescription = tierInfo.Description
- log.Printf("✅ [MODEL-HANDLER] Added tier '%s' (%s %s) to alias '%s'", tierInfo.Tier, tierInfo.Icon, tierInfo.Label, aliasName)
- }
-
- // Add description - use tier description as fallback if model description is empty
- if aliasInfo.Description != "" {
- modelMap["description"] = aliasInfo.Description
- } else if tierDescription != "" {
- modelMap["description"] = tierDescription
- }
-
- if foundModel.ProviderFavicon != "" {
- modelMap["provider_favicon"] = foundModel.ProviderFavicon
- }
- if aliasInfo.StructuredOutputSupport != "" {
- modelMap["structured_output_support"] = aliasInfo.StructuredOutputSupport
- }
- if aliasInfo.StructuredOutputCompliance != nil {
- modelMap["structured_output_compliance"] = *aliasInfo.StructuredOutputCompliance
- }
- if aliasInfo.StructuredOutputWarning != "" {
- modelMap["structured_output_warning"] = aliasInfo.StructuredOutputWarning
- }
- if aliasInfo.StructuredOutputSpeedMs != nil {
- modelMap["structured_output_speed_ms"] = *aliasInfo.StructuredOutputSpeedMs
- }
- if aliasInfo.StructuredOutputBadge != "" {
- modelMap["structured_output_badge"] = aliasInfo.StructuredOutputBadge
- }
-
- enrichedModels = append(enrichedModels, modelMap)
- }
- }
-
- // Second pass: Add non-aliased models
- for _, model := range modelsList {
- // Use composite key (providerID:lowercase_name) to check if model is aliased
- if !aliasedModels[aliasKey(model.ProviderID, model.Name)] {
- // Get provider security status
- isProviderSecure := configService.IsProviderSecure(model.ProviderID)
-
- modelMap := map[string]interface{}{
- "id": model.ID,
- "provider_id": model.ProviderID,
- "provider_name": model.ProviderName,
- "name": model.Name,
- "display_name": model.DisplayName,
- "supports_tools": model.SupportsTools,
- "supports_streaming": model.SupportsStreaming,
- "supports_vision": model.SupportsVision,
- "agents_enabled": model.AgentsEnabled, // Use model's AgentsEnabled field (defaults to false for non-aliased)
- "provider_secure": isProviderSecure,
- "is_visible": model.IsVisible,
- "fetched_at": model.FetchedAt,
- }
- // Check if this model is in the recommendation tier
- recommendationKey := fmt.Sprintf("%d:%s", model.ProviderID, strings.ToLower(model.ID))
- var tierDescription string
- if tierInfo, exists := modelRecommendationTier[recommendationKey]; exists {
- modelMap["recommendation_tier"] = map[string]interface{}{
- "tier": tierInfo.Tier,
- "label": tierInfo.Label,
- "description": tierInfo.Description,
- "icon": tierInfo.Icon,
- }
- tierDescription = tierInfo.Description
- log.Printf("✅ [MODEL-HANDLER] Added tier '%s' (%s %s) to model '%s'", tierInfo.Tier, tierInfo.Icon, tierInfo.Label, model.ID)
- }
-
- // Add description - use tier description as fallback if model description is empty
- if tierDescription != "" {
- modelMap["description"] = tierDescription
- }
-
- // Add optional provider favicon if present
- if model.ProviderFavicon != "" {
- modelMap["provider_favicon"] = model.ProviderFavicon
- }
-
- enrichedModels = append(enrichedModels, modelMap)
- }
- }
-
- // Check user authentication status
- userID, ok := c.Locals("user_id").(string)
- if !ok || userID == "" {
- userID = "anonymous"
- }
-
- // Filter models based on authentication
- if userID == "anonymous" {
- // Anonymous users only get free tier models
- var filteredModels []interface{}
- for _, model := range enrichedModels {
- if modelMap, ok := model.(map[string]interface{}); ok {
- modelID, _ := modelMap["id"].(string)
-
- // Check if this model is marked as free tier
- if h.modelService.IsFreeTier(modelID) {
- filteredModels = append(filteredModels, model)
- }
- }
- }
-
- log.Printf("🔒 Anonymous user - filtered to %d free tier models", len(filteredModels))
- return c.JSON(fiber.Map{
- "models": filteredModels,
- "count": len(filteredModels),
- "tier": "anonymous",
- })
- }
-
- // Authenticated users get all models
- log.Printf("✅ Authenticated user (%s) - showing all %d models", userID, len(enrichedModels))
- return c.JSON(fiber.Map{
- "models": enrichedModels,
- "count": len(enrichedModels),
- "tier": "authenticated",
- })
-}
-
-// ListByProvider returns models for a specific provider
-func (h *ModelHandler) ListByProvider(c *fiber.Ctx) error {
- providerID, err := strconv.Atoi(c.Params("id"))
- if err != nil {
- return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{
- "error": "Invalid provider ID",
- })
- }
-
- visibleOnly := c.Query("visible_only", "true") == "true"
-
- models, err := h.modelService.GetByProvider(providerID, visibleOnly)
- if err != nil {
- return c.Status(fiber.StatusInternalServerError).JSON(fiber.Map{
- "error": "Failed to fetch models",
- })
- }
-
- return c.JSON(fiber.Map{
- "models": models,
- "count": len(models),
- })
-}
-
-// ListToolPredictorModels returns only models that can be used as tool predictors
-// GET /api/models/tool-predictors
-func (h *ModelHandler) ListToolPredictorModels(c *fiber.Ctx) error {
- models, err := h.modelService.GetToolPredictorModels()
- if err != nil {
- return c.Status(fiber.StatusInternalServerError).JSON(fiber.Map{
- "error": "Failed to fetch tool predictor models",
- })
- }
-
- return c.JSON(fiber.Map{
- "models": models,
- "count": len(models),
- })
-}
diff --git a/backend/internal/handlers/model_management.go b/backend/internal/handlers/model_management.go
deleted file mode 100644
index 0a83c9c9..00000000
--- a/backend/internal/handlers/model_management.go
+++ /dev/null
@@ -1,787 +0,0 @@
-package handlers
-
-import (
- "claraverse/internal/services"
- "fmt"
- "log"
- "net/url"
- "strconv"
-
- "github.com/gofiber/fiber/v2"
-)
-
-// ModelManagementHandler handles model management operations for admin
-type ModelManagementHandler struct {
- modelMgmtService *services.ModelManagementService
- modelService *services.ModelService
- providerService *services.ProviderService
-}
-
-// Helper function to decode URL-encoded model IDs (handles slashes and special characters)
-func decodeModelID(encodedID string) (string, error) {
- return url.QueryUnescape(encodedID)
-}
-
-// NewModelManagementHandler creates a new model management handler
-func NewModelManagementHandler(
- modelMgmtService *services.ModelManagementService,
- modelService *services.ModelService,
- providerService *services.ProviderService,
-) *ModelManagementHandler {
- return &ModelManagementHandler{
- modelMgmtService: modelMgmtService,
- modelService: modelService,
- providerService: providerService,
- }
-}
-
-// ================== MODEL CRUD ENDPOINTS ==================
-
-// GetAllModels returns all models with metadata
-// GET /api/admin/models
-func (h *ModelManagementHandler) GetAllModels(c *fiber.Ctx) error {
- adminUserID := c.Locals("user_id").(string)
- log.Printf("🔍 Admin %s fetching all models", adminUserID)
-
- // Get all models (including hidden ones)
- models, err := h.modelService.GetAll(false)
- if err != nil {
- return c.Status(fiber.StatusInternalServerError).JSON(fiber.Map{
- "error": "Failed to fetch models",
- })
- }
-
- return c.JSON(models)
-}
-
-// CreateModel creates a new model manually
-// POST /api/admin/models
-func (h *ModelManagementHandler) CreateModel(c *fiber.Ctx) error {
- adminUserID := c.Locals("user_id").(string)
-
- var req services.CreateModelRequest
- if err := c.BodyParser(&req); err != nil {
- return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{
- "error": "Invalid request body",
- })
- }
-
- // Validate required fields
- if req.ModelID == "" || req.ProviderID == 0 || req.Name == "" {
- return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{
- "error": "model_id, provider_id, and name are required",
- })
- }
-
- log.Printf("📝 Admin %s creating model: %s (provider %d)", adminUserID, req.ModelID, req.ProviderID)
-
- model, err := h.modelMgmtService.CreateModel(c.Context(), &req)
- if err != nil {
- log.Printf("❌ Failed to create model: %v", err)
- return c.Status(fiber.StatusInternalServerError).JSON(fiber.Map{
- "error": "Failed to create model: " + err.Error(),
- })
- }
-
- return c.Status(fiber.StatusCreated).JSON(model)
-}
-
-// UpdateModel updates an existing model's metadata
-// PUT /api/admin/models/:modelId
-func (h *ModelManagementHandler) UpdateModel(c *fiber.Ctx) error {
- adminUserID := c.Locals("user_id").(string)
- encodedModelID := c.Params("modelId")
-
- if encodedModelID == "" {
- return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{
- "error": "Model ID is required",
- })
- }
-
- // Decode URL-encoded model ID
- modelID, err := decodeModelID(encodedModelID)
- if err != nil {
- return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{
- "error": "Invalid model ID encoding",
- })
- }
-
- var req services.UpdateModelRequest
- if err := c.BodyParser(&req); err != nil {
- return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{
- "error": "Invalid request body",
- })
- }
-
- // WORKAROUND: Fiber's BodyParser doesn't handle *bool correctly when value is false
- // Manually parse boolean fields from raw JSON if present
- var rawBody map[string]interface{}
- if err := c.BodyParser(&rawBody); err == nil {
- log.Printf("[DEBUG] Raw request body: %+v", rawBody)
-
- if val, exists := rawBody["is_visible"]; exists {
- if boolVal, ok := val.(bool); ok {
- req.IsVisible = &boolVal
- log.Printf("[DEBUG] Manually parsed is_visible: %v", boolVal)
- }
- }
-
- if val, exists := rawBody["smart_tool_router"]; exists {
- if boolVal, ok := val.(bool); ok {
- req.SmartToolRouter = &boolVal
- log.Printf("[DEBUG] Manually parsed smart_tool_router: %v", boolVal)
- }
- }
-
- if val, exists := rawBody["supports_tools"]; exists {
- if boolVal, ok := val.(bool); ok {
- req.SupportsTools = &boolVal
- log.Printf("[DEBUG] Manually parsed supports_tools: %v", boolVal)
- }
- }
-
- if val, exists := rawBody["supports_vision"]; exists {
- if boolVal, ok := val.(bool); ok {
- req.SupportsVision = &boolVal
- log.Printf("[DEBUG] Manually parsed supports_vision: %v", boolVal)
- }
- }
-
- if val, exists := rawBody["supports_streaming"]; exists {
- if boolVal, ok := val.(bool); ok {
- req.SupportsStreaming = &boolVal
- log.Printf("[DEBUG] Manually parsed supports_streaming: %v", boolVal)
- }
- }
-
- if val, exists := rawBody["free_tier"]; exists {
- if boolVal, ok := val.(bool); ok {
- req.FreeTier = &boolVal
- log.Printf("[DEBUG] Manually parsed free_tier: %v", boolVal)
- }
- }
- }
-
- log.Printf("📝 Admin %s updating model: %s", adminUserID, modelID)
-
- model, err := h.modelMgmtService.UpdateModel(c.Context(), modelID, &req)
- if err != nil {
- log.Printf("❌ Failed to update model: %v", err)
- return c.Status(fiber.StatusInternalServerError).JSON(fiber.Map{
- "error": "Failed to update model: " + err.Error(),
- })
- }
-
- return c.JSON(model)
-}
-
-// DeleteModel deletes a model
-// DELETE /api/admin/models/:modelId
-func (h *ModelManagementHandler) DeleteModel(c *fiber.Ctx) error {
- adminUserID := c.Locals("user_id").(string)
- encodedModelID := c.Params("modelId")
-
- if encodedModelID == "" {
- return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{
- "error": "Model ID is required",
- })
- }
-
- // Decode URL-encoded model ID
- modelID, err := decodeModelID(encodedModelID)
- if err != nil {
- return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{
- "error": "Invalid model ID encoding",
- })
- }
-
- log.Printf("🗑️ Admin %s deleting model: %s", adminUserID, modelID)
-
- if err := h.modelMgmtService.DeleteModel(c.Context(), modelID); err != nil {
- log.Printf("❌ Failed to delete model: %v", err)
- return c.Status(fiber.StatusInternalServerError).JSON(fiber.Map{
- "error": "Failed to delete model: " + err.Error(),
- })
- }
-
- return c.JSON(fiber.Map{
- "success": true,
- "message": "Model deleted successfully",
- })
-}
-
-// ================== MODEL FETCHING ENDPOINTS ==================
-
-// FetchModelsFromProvider fetches models from a provider's API
-// POST /api/admin/providers/:providerId/fetch
-func (h *ModelManagementHandler) FetchModelsFromProvider(c *fiber.Ctx) error {
- adminUserID := c.Locals("user_id").(string)
- providerIDStr := c.Params("providerId")
-
- providerID, err := strconv.Atoi(providerIDStr)
- if err != nil {
- return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{
- "error": "Invalid provider ID",
- })
- }
-
- log.Printf("🔄 Admin %s fetching models from provider %d", adminUserID, providerID)
-
- count, err := h.modelMgmtService.FetchModelsFromProvider(c.Context(), providerID)
- if err != nil {
- log.Printf("❌ Failed to fetch models: %v", err)
- return c.Status(fiber.StatusInternalServerError).JSON(fiber.Map{
- "error": "Failed to fetch models: " + err.Error(),
- })
- }
-
- return c.JSON(fiber.Map{
- "success": true,
- "models_fetched": count,
- "message": "Models fetched and stored successfully",
- })
-}
-
-// SyncProviderToJSON forces sync of a provider's models to providers.json
-// POST /api/admin/providers/:providerId/sync
-func (h *ModelManagementHandler) SyncProviderToJSON(c *fiber.Ctx) error {
- adminUserID := c.Locals("user_id").(string)
- providerIDStr := c.Params("providerId")
-
- providerID, err := strconv.Atoi(providerIDStr)
- if err != nil {
- return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{
- "error": "Invalid provider ID",
- })
- }
-
- log.Printf("🔄 Admin %s syncing provider %d to JSON", adminUserID, providerID)
-
- // The sync is handled automatically by the service, just trigger it
- _, err = h.modelMgmtService.FetchModelsFromProvider(c.Context(), providerID)
- if err != nil {
- log.Printf("❌ Failed to sync provider: %v", err)
- return c.Status(fiber.StatusInternalServerError).JSON(fiber.Map{
- "error": "Failed to sync provider: " + err.Error(),
- })
- }
-
- return c.JSON(fiber.Map{
- "success": true,
- "message": "Provider synced to providers.json successfully",
- })
-}
-
-// ================== MODEL TESTING ENDPOINTS ==================
-
-// TestModelConnection tests basic connection to a model
-// POST /api/admin/models/:modelId/test/connection
-func (h *ModelManagementHandler) TestModelConnection(c *fiber.Ctx) error {
- adminUserID := c.Locals("user_id").(string)
- encodedModelID := c.Params("modelId")
-
- if encodedModelID == "" {
- return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{
- "error": "Model ID is required",
- })
- }
-
- // Decode URL-encoded model ID
- modelID, err := decodeModelID(encodedModelID)
- if err != nil {
- return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{
- "error": "Invalid model ID encoding",
- })
- }
-
- log.Printf("🔌 Admin %s testing connection for model: %s", adminUserID, modelID)
-
- result, err := h.modelMgmtService.TestModelConnection(c.Context(), modelID)
- if err != nil {
- log.Printf("❌ Connection test failed: %v", err)
- return c.Status(fiber.StatusInternalServerError).JSON(fiber.Map{
- "error": "Connection test failed: " + err.Error(),
- })
- }
-
- if result.Passed {
- return c.JSON(fiber.Map{
- "success": true,
- "passed": result.Passed,
- "latency_ms": result.LatencyMs,
- "message": "Connection test passed",
- })
- }
-
- return c.JSON(fiber.Map{
- "success": false,
- "passed": result.Passed,
- "latency_ms": result.LatencyMs,
- "error": result.Error,
- "message": "Connection test failed",
- })
-}
-
-// TestModelCapability tests model capabilities (tools, vision, streaming)
-// POST /api/admin/models/:modelId/test/capability
-func (h *ModelManagementHandler) TestModelCapability(c *fiber.Ctx) error {
- adminUserID := c.Locals("user_id").(string)
- encodedModelID := c.Params("modelId")
-
- if encodedModelID == "" {
- return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{
- "error": "Model ID is required",
- })
- }
-
- // Decode URL-encoded model ID
- modelID, err := decodeModelID(encodedModelID)
- if err != nil {
- return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{
- "error": "Invalid model ID encoding",
- })
- }
-
- log.Printf("🧪 Admin %s testing capabilities for model: %s", adminUserID, modelID)
-
- // TODO: Implement capability testing
- // This would test tools, vision, streaming support
-
- return c.JSON(fiber.Map{
- "message": "Capability testing not yet implemented",
- })
-}
-
-// RunModelBenchmark runs comprehensive benchmark suite on a model
-// POST /api/admin/models/:modelId/benchmark
-func (h *ModelManagementHandler) RunModelBenchmark(c *fiber.Ctx) error {
- adminUserID := c.Locals("user_id").(string)
- encodedModelID := c.Params("modelId")
-
- if encodedModelID == "" {
- return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{
- "error": "Model ID is required",
- })
- }
-
- // Decode URL-encoded model ID
- modelID, err := decodeModelID(encodedModelID)
- if err != nil {
- return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{
- "error": "Invalid model ID encoding",
- })
- }
-
- // URL decode the model ID (handles IDs with slashes)
- decodedModelID, err := url.QueryUnescape(modelID)
- if err != nil {
- log.Printf("❌ Failed to decode model ID: %v", err)
- decodedModelID = modelID // Fallback to original
- }
-
- log.Printf("📊 Admin %s running benchmark for model: %s (decoded: %s)", adminUserID, modelID, decodedModelID)
-
- results, err := h.modelMgmtService.RunBenchmark(c.Context(), decodedModelID)
- if err != nil {
- log.Printf("❌ Benchmark failed for model %s: %v", decodedModelID, err)
- return c.Status(fiber.StatusInternalServerError).JSON(fiber.Map{
- "error": "Benchmark failed: " + err.Error(),
- })
- }
-
- log.Printf("✅ Benchmark completed for model %s. Results: connection=%v, structured=%v, performance=%v",
- decodedModelID,
- results.ConnectionTest != nil,
- results.StructuredOutput != nil,
- results.Performance != nil)
-
- return c.JSON(results)
-}
-
-// GetModelTestResults retrieves latest test results for a model
-// GET /api/admin/models/:modelId/test-results
-func (h *ModelManagementHandler) GetModelTestResults(c *fiber.Ctx) error {
- adminUserID := c.Locals("user_id").(string)
- encodedModelID := c.Params("modelId")
-
- if encodedModelID == "" {
- return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{
- "error": "Model ID is required",
- })
- }
-
- // Decode URL-encoded model ID
- modelID, err := decodeModelID(encodedModelID)
- if err != nil {
- return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{
- "error": "Invalid model ID encoding",
- })
- }
-
- log.Printf("🔍 Admin %s fetching test results for model: %s", adminUserID, modelID)
-
- // TODO: Query model_capabilities table for test results
-
- return c.JSON(fiber.Map{
- "message": "Test results retrieval not yet implemented",
- })
-}
-
-// ================== ALIAS MANAGEMENT ENDPOINTS ==================
-
-// GetModelAliases retrieves all aliases for a model
-// GET /api/admin/models/:modelId/aliases
-func (h *ModelManagementHandler) GetModelAliases(c *fiber.Ctx) error {
- adminUserID := c.Locals("user_id").(string)
- encodedModelID := c.Params("modelId")
-
- if encodedModelID == "" {
- return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{
- "error": "Model ID is required",
- })
- }
-
- // Decode URL-encoded model ID
- modelID, err := decodeModelID(encodedModelID)
- if err != nil {
- return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{
- "error": "Invalid model ID encoding",
- })
- }
-
- // URL decode the model ID (handles IDs with slashes)
- decodedModelID, err := url.QueryUnescape(modelID)
- if err != nil {
- log.Printf("❌ Failed to decode model ID: %v", err)
- decodedModelID = modelID // Fallback to original
- }
-
- log.Printf("🔍 Admin %s fetching aliases for model: %s (decoded: %s)", adminUserID, modelID, decodedModelID)
-
- aliases, err := h.modelMgmtService.GetAliases(c.Context(), decodedModelID)
- if err != nil {
- log.Printf("❌ Failed to get aliases: %v", err)
- return c.Status(fiber.StatusInternalServerError).JSON(fiber.Map{
- "error": "Failed to get aliases: " + err.Error(),
- })
- }
-
- log.Printf("✅ Returning %d aliases for model %s", len(aliases), decodedModelID)
- return c.JSON(aliases)
-}
-
-// CreateModelAlias creates a new alias for a model
-// POST /api/admin/models/:modelId/aliases
-func (h *ModelManagementHandler) CreateModelAlias(c *fiber.Ctx) error {
- adminUserID := c.Locals("user_id").(string)
- encodedModelID := c.Params("modelId")
-
- if encodedModelID == "" {
- return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{
- "error": "Model ID is required",
- })
- }
-
- // Decode URL-encoded model ID
- modelID, err := decodeModelID(encodedModelID)
- if err != nil {
- return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{
- "error": "Invalid model ID encoding",
- })
- }
-
- // URL decode the model ID (handles IDs with slashes)
- decodedModelID, err := url.QueryUnescape(modelID)
- if err != nil {
- log.Printf("❌ Failed to decode model ID: %v", err)
- decodedModelID = modelID // Fallback to original
- }
-
- var req services.CreateAliasRequest
- if err := c.BodyParser(&req); err != nil {
- return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{
- "error": "Invalid request body",
- })
- }
-
- // Set model ID from URL parameter
- req.ModelID = decodedModelID
-
- // Validate required fields
- if req.AliasName == "" || req.ProviderID == 0 {
- return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{
- "error": "alias_name and provider_id are required",
- })
- }
-
- log.Printf("📝 Admin %s creating alias %s for model: %s", adminUserID, req.AliasName, modelID)
-
- if err := h.modelMgmtService.CreateAlias(c.Context(), &req); err != nil {
- log.Printf("❌ Failed to create alias: %v", err)
- return c.Status(fiber.StatusInternalServerError).JSON(fiber.Map{
- "error": "Failed to create alias: " + err.Error(),
- })
- }
-
- return c.Status(fiber.StatusCreated).JSON(fiber.Map{
- "success": true,
- "message": "Alias created successfully",
- })
-}
-
-// UpdateModelAlias updates an existing alias
-// PUT /api/admin/models/:modelId/aliases/:alias
-func (h *ModelManagementHandler) UpdateModelAlias(c *fiber.Ctx) error {
- adminUserID := c.Locals("user_id").(string)
- modelID := c.Params("modelId")
- aliasName := c.Params("alias")
-
- if modelID == "" || aliasName == "" {
- return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{
- "error": "Model ID and alias name are required",
- })
- }
-
- log.Printf("📝 Admin %s updating alias %s for model: %s", adminUserID, aliasName, modelID)
-
- // TODO: Implement alias update
-
- return c.JSON(fiber.Map{
- "message": "Alias update not yet implemented",
- })
-}
-
-// DeleteModelAlias deletes an alias
-// DELETE /api/admin/models/:modelId/aliases/:alias
-func (h *ModelManagementHandler) DeleteModelAlias(c *fiber.Ctx) error {
- adminUserID := c.Locals("user_id").(string)
- modelID := c.Params("modelId")
- aliasName := c.Params("alias")
-
- if modelID == "" || aliasName == "" {
- return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{
- "error": "Model ID and alias name are required",
- })
- }
-
- // Get provider ID from query parameter or request body
- providerIDStr := c.Query("provider_id")
- if providerIDStr == "" {
- return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{
- "error": "provider_id query parameter is required",
- })
- }
-
- providerID, err := strconv.Atoi(providerIDStr)
- if err != nil {
- return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{
- "error": "Invalid provider ID",
- })
- }
-
- log.Printf("🗑️ Admin %s deleting alias %s for model: %s", adminUserID, aliasName, modelID)
-
- if err := h.modelMgmtService.DeleteAlias(c.Context(), aliasName, providerID); err != nil {
- log.Printf("❌ Failed to delete alias: %v", err)
- return c.Status(fiber.StatusInternalServerError).JSON(fiber.Map{
- "error": "Failed to delete alias: " + err.Error(),
- })
- }
-
- return c.JSON(fiber.Map{
- "success": true,
- "message": "Alias deleted successfully",
- })
-}
-
-// ImportAliasesFromJSON imports all aliases from providers.json into the database
-// POST /api/admin/models/import-aliases
-func (h *ModelManagementHandler) ImportAliasesFromJSON(c *fiber.Ctx) error {
- adminUserID := c.Locals("user_id").(string)
-
- log.Printf("📥 Admin %s triggering alias import from providers.json", adminUserID)
-
- if err := h.modelMgmtService.ImportAliasesFromJSON(c.Context()); err != nil {
- log.Printf("❌ Failed to import aliases: %v", err)
- return c.Status(fiber.StatusInternalServerError).JSON(fiber.Map{
- "error": "Failed to import aliases: " + err.Error(),
- })
- }
-
- return c.JSON(fiber.Map{
- "success": true,
- "message": "Aliases imported successfully from providers.json",
- })
-}
-
-// ================== BULK OPERATIONS ENDPOINTS ==================
-
-// BulkUpdateAgentsEnabled bulk enables/disables models for agent builder
-// PUT /api/admin/models/bulk/agents-enabled
-func (h *ModelManagementHandler) BulkUpdateAgentsEnabled(c *fiber.Ctx) error {
- adminUserID := c.Locals("user_id").(string)
-
- var req struct {
- ModelIDs []string `json:"model_ids"`
- Enabled bool `json:"enabled"`
- }
-
- if err := c.BodyParser(&req); err != nil {
- return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{
- "error": "Invalid request body",
- })
- }
-
- if len(req.ModelIDs) == 0 {
- return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{
- "error": "model_ids array is required",
- })
- }
-
- log.Printf("📝 Admin %s bulk updating agents_enabled for %d models", adminUserID, len(req.ModelIDs))
-
- if err := h.modelMgmtService.BulkUpdateAgentsEnabled(req.ModelIDs, req.Enabled); err != nil {
- log.Printf("❌ Failed to bulk update agents_enabled: %v", err)
- return c.Status(fiber.StatusInternalServerError).JSON(fiber.Map{
- "error": fmt.Sprintf("Failed to update models: %v", err),
- })
- }
-
- return c.JSON(fiber.Map{
- "message": fmt.Sprintf("Updated agents_enabled=%v for %d models", req.Enabled, len(req.ModelIDs)),
- })
-}
-
-// BulkUpdateVisibility bulk shows/hides models
-// PUT /api/admin/models/bulk/visibility
-func (h *ModelManagementHandler) BulkUpdateVisibility(c *fiber.Ctx) error {
- adminUserID := c.Locals("user_id").(string)
-
- var req struct {
- ModelIDs []string `json:"model_ids"`
- Visible bool `json:"visible"`
- }
-
- if err := c.BodyParser(&req); err != nil {
- return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{
- "error": "Invalid request body",
- })
- }
-
- if len(req.ModelIDs) == 0 {
- return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{
- "error": "model_ids array is required",
- })
- }
-
- log.Printf("📝 Admin %s bulk updating visibility for %d models", adminUserID, len(req.ModelIDs))
-
- if err := h.modelMgmtService.BulkUpdateVisibility(req.ModelIDs, req.Visible); err != nil {
- log.Printf("❌ Failed to bulk update visibility: %v", err)
- return c.Status(fiber.StatusInternalServerError).JSON(fiber.Map{
- "error": fmt.Sprintf("Failed to update models: %v", err),
- })
- }
-
- return c.JSON(fiber.Map{
- "message": fmt.Sprintf("Updated is_visible=%v for %d models", req.Visible, len(req.ModelIDs)),
- })
-}
-
-// ================== GLOBAL TIER MANAGEMENT ==================
-
-// SetModelTier assigns a model to a global tier (tier1-tier5)
-// POST /api/admin/models/:modelId/tier
-func (h *ModelManagementHandler) SetModelTier(c *fiber.Ctx) error {
- modelID := c.Params("modelId")
- if modelID == "" {
- return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{
- "error": "Model ID is required",
- })
- }
-
- // URL decode the model ID (handles slashes and other special characters)
- decodedModelID, err := url.QueryUnescape(modelID)
- if err != nil {
- log.Printf("❌ Failed to decode model ID '%s': %v", modelID, err)
- return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{
- "error": "Invalid model ID encoding",
- })
- }
- modelID = decodedModelID
-
- var req struct {
- ProviderID int `json:"provider_id"`
- Tier string `json:"tier"` // "tier1", "tier2", "tier3", "tier4", "tier5"
- }
-
- if err := c.BodyParser(&req); err != nil {
- return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{
- "error": "Invalid request body",
- })
- }
-
- if req.Tier == "" || req.ProviderID == 0 {
- return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{
- "error": "tier and provider_id are required",
- })
- }
-
- if err := h.modelMgmtService.SetGlobalTier(modelID, req.ProviderID, req.Tier); err != nil {
- log.Printf("❌ Failed to set tier: %v", err)
- return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{
- "error": fmt.Sprintf("Failed to set tier: %v", err),
- })
- }
-
- return c.JSON(fiber.Map{
- "message": fmt.Sprintf("Model assigned to %s", req.Tier),
- })
-}
-
-// ClearModelTier removes a model from its tier
-// DELETE /api/admin/models/:modelId/tier
-func (h *ModelManagementHandler) ClearModelTier(c *fiber.Ctx) error {
- var req struct {
- Tier string `json:"tier"`
- }
-
- if err := c.BodyParser(&req); err != nil {
- return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{
- "error": "Invalid request body",
- })
- }
-
- if req.Tier == "" {
- return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{
- "error": "tier is required",
- })
- }
-
- if err := h.modelMgmtService.ClearTier(req.Tier); err != nil {
- log.Printf("❌ Failed to clear tier: %v", err)
- return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{
- "error": fmt.Sprintf("Failed to clear tier: %v", err),
- })
- }
-
- return c.JSON(fiber.Map{
- "message": fmt.Sprintf("Tier %s cleared", req.Tier),
- })
-}
-
-// GetTiers retrieves all global tier assignments
-// GET /api/admin/tiers
-func (h *ModelManagementHandler) GetTiers(c *fiber.Ctx) error {
- tiers, err := h.modelMgmtService.GetGlobalTiers()
- if err != nil {
- log.Printf("❌ Failed to get tiers: %v", err)
- return c.Status(fiber.StatusInternalServerError).JSON(fiber.Map{
- "error": "Failed to retrieve tiers",
- })
- }
-
- return c.JSON(fiber.Map{
- "tiers": tiers,
- })
-}
diff --git a/backend/internal/handlers/provider.go b/backend/internal/handlers/provider.go
deleted file mode 100644
index 998025c4..00000000
--- a/backend/internal/handlers/provider.go
+++ /dev/null
@@ -1,67 +0,0 @@
-package handlers
-
-import (
- "claraverse/internal/services"
- "strconv"
-
- "github.com/gofiber/fiber/v2"
-)
-
-// ProviderHandler handles provider-related requests
-type ProviderHandler struct {
- providerService *services.ProviderService
-}
-
-// NewProviderHandler creates a new provider handler
-func NewProviderHandler(providerService *services.ProviderService) *ProviderHandler {
- return &ProviderHandler{providerService: providerService}
-}
-
-// List returns all enabled providers (names only, no credentials)
-func (h *ProviderHandler) List(c *fiber.Ctx) error {
- providers, err := h.providerService.GetAll()
- if err != nil {
- return c.Status(fiber.StatusInternalServerError).JSON(fiber.Map{
- "error": "Failed to fetch providers",
- })
- }
-
- // Hide sensitive information (API keys, base URLs)
- type PublicProvider struct {
- ID int `json:"id"`
- Name string `json:"name"`
- }
-
- publicProviders := make([]PublicProvider, len(providers))
- for i, p := range providers {
- publicProviders[i] = PublicProvider{
- ID: p.ID,
- Name: p.Name,
- }
- }
-
- return c.JSON(fiber.Map{
- "providers": publicProviders,
- "count": len(publicProviders),
- })
-}
-
-// GetModels returns models for a specific provider
-func (h *ProviderHandler) GetModels(c *fiber.Ctx) error {
- providerID, err := strconv.Atoi(c.Params("id"))
- if err != nil {
- return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{
- "error": "Invalid provider ID",
- })
- }
-
- // Get provider to verify it exists
- _, err = h.providerService.GetByID(providerID)
- if err != nil {
- return c.Status(fiber.StatusNotFound).JSON(fiber.Map{
- "error": "Provider not found",
- })
- }
-
- return c.Next()
-}
diff --git a/backend/internal/handlers/schedule.go b/backend/internal/handlers/schedule.go
deleted file mode 100644
index 52db26ad..00000000
--- a/backend/internal/handlers/schedule.go
+++ /dev/null
@@ -1,298 +0,0 @@
-package handlers
-
-import (
- "claraverse/internal/models"
- "claraverse/internal/services"
- "log"
-
- "github.com/gofiber/fiber/v2"
-)
-
-// ScheduleHandler handles schedule-related HTTP requests
-type ScheduleHandler struct {
- schedulerService *services.SchedulerService
- agentService *services.AgentService
-}
-
-// NewScheduleHandler creates a new schedule handler
-func NewScheduleHandler(schedulerService *services.SchedulerService, agentService *services.AgentService) *ScheduleHandler {
- return &ScheduleHandler{
- schedulerService: schedulerService,
- agentService: agentService,
- }
-}
-
-// Create creates a new schedule for an agent
-// POST /api/agents/:id/schedule
-func (h *ScheduleHandler) Create(c *fiber.Ctx) error {
- userID, ok := c.Locals("user_id").(string)
- if !ok || userID == "" {
- return c.Status(fiber.StatusUnauthorized).JSON(fiber.Map{
- "error": "Authentication required",
- })
- }
-
- agentID := c.Params("id")
- if agentID == "" {
- return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{
- "error": "Agent ID is required",
- })
- }
-
- // Verify agent exists and belongs to user
- agent, err := h.agentService.GetAgent(agentID, userID)
- if err != nil {
- return c.Status(fiber.StatusNotFound).JSON(fiber.Map{
- "error": "Agent not found",
- })
- }
-
- // Check if workflow has file inputs (which expire in 30 minutes)
- if hasFileInputs(agent.Workflow) {
- log.Printf("🚫 [SCHEDULE] Cannot schedule agent %s: workflow has file inputs", agentID)
- return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{
- "error": "Cannot schedule workflows with file inputs",
- "reason": "Uploaded files expire after 30 minutes and won't be available at scheduled execution time",
- "suggestion": "Use the API trigger endpoint instead: POST /api/trigger/" + agentID,
- })
- }
-
- var req models.CreateScheduleRequest
- if err := c.BodyParser(&req); err != nil {
- return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{
- "error": "Invalid request body",
- })
- }
-
- // Validate required fields
- if req.CronExpression == "" {
- return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{
- "error": "cronExpression is required",
- })
- }
- if req.Timezone == "" {
- return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{
- "error": "timezone is required",
- })
- }
-
- log.Printf("📅 [SCHEDULE] Creating schedule for agent %s (user: %s, cron: %s)", agentID, userID, req.CronExpression)
-
- schedule, err := h.schedulerService.CreateSchedule(c.Context(), agentID, userID, &req)
- if err != nil {
- log.Printf("❌ [SCHEDULE] Failed to create schedule: %v", err)
-
- // Check for specific errors
- if err.Error() == "agent already has a schedule" {
- return c.Status(fiber.StatusConflict).JSON(fiber.Map{
- "error": err.Error(),
- })
- }
- if err.Error()[:14] == "schedule limit" {
- return c.Status(fiber.StatusForbidden).JSON(fiber.Map{
- "error": err.Error(),
- })
- }
-
- return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{
- "error": err.Error(),
- })
- }
-
- log.Printf("✅ [SCHEDULE] Created schedule %s for agent %s", schedule.ID.Hex(), agentID)
- return c.Status(fiber.StatusCreated).JSON(schedule.ToResponse())
-}
-
-// Get retrieves the schedule for an agent
-// GET /api/agents/:id/schedule
-func (h *ScheduleHandler) Get(c *fiber.Ctx) error {
- userID, ok := c.Locals("user_id").(string)
- if !ok || userID == "" {
- return c.Status(fiber.StatusUnauthorized).JSON(fiber.Map{
- "error": "Authentication required",
- })
- }
-
- agentID := c.Params("id")
- if agentID == "" {
- return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{
- "error": "Agent ID is required",
- })
- }
-
- schedule, err := h.schedulerService.GetScheduleByAgentID(c.Context(), agentID, userID)
- if err != nil {
- if err.Error() == "schedule not found" {
- return c.Status(fiber.StatusNotFound).JSON(fiber.Map{
- "error": "No schedule found for this agent",
- })
- }
- log.Printf("❌ [SCHEDULE] Failed to get schedule: %v", err)
- return c.Status(fiber.StatusInternalServerError).JSON(fiber.Map{
- "error": "Failed to get schedule",
- })
- }
-
- return c.JSON(schedule.ToResponse())
-}
-
-// Update updates the schedule for an agent
-// PUT /api/agents/:id/schedule
-func (h *ScheduleHandler) Update(c *fiber.Ctx) error {
- userID, ok := c.Locals("user_id").(string)
- if !ok || userID == "" {
- return c.Status(fiber.StatusUnauthorized).JSON(fiber.Map{
- "error": "Authentication required",
- })
- }
-
- agentID := c.Params("id")
- if agentID == "" {
- return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{
- "error": "Agent ID is required",
- })
- }
-
- // Get existing schedule
- existingSchedule, err := h.schedulerService.GetScheduleByAgentID(c.Context(), agentID, userID)
- if err != nil {
- return c.Status(fiber.StatusNotFound).JSON(fiber.Map{
- "error": "No schedule found for this agent",
- })
- }
-
- var req models.UpdateScheduleRequest
- if err := c.BodyParser(&req); err != nil {
- return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{
- "error": "Invalid request body",
- })
- }
-
- log.Printf("📝 [SCHEDULE] Updating schedule %s for agent %s", existingSchedule.ID.Hex(), agentID)
-
- schedule, err := h.schedulerService.UpdateSchedule(c.Context(), existingSchedule.ID.Hex(), userID, &req)
- if err != nil {
- log.Printf("❌ [SCHEDULE] Failed to update schedule: %v", err)
- return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{
- "error": err.Error(),
- })
- }
-
- log.Printf("✅ [SCHEDULE] Updated schedule %s", schedule.ID.Hex())
- return c.JSON(schedule.ToResponse())
-}
-
-// Delete deletes the schedule for an agent
-// DELETE /api/agents/:id/schedule
-func (h *ScheduleHandler) Delete(c *fiber.Ctx) error {
- userID, ok := c.Locals("user_id").(string)
- if !ok || userID == "" {
- return c.Status(fiber.StatusUnauthorized).JSON(fiber.Map{
- "error": "Authentication required",
- })
- }
-
- agentID := c.Params("id")
- if agentID == "" {
- return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{
- "error": "Agent ID is required",
- })
- }
-
- // Get existing schedule
- existingSchedule, err := h.schedulerService.GetScheduleByAgentID(c.Context(), agentID, userID)
- if err != nil {
- return c.Status(fiber.StatusNotFound).JSON(fiber.Map{
- "error": "No schedule found for this agent",
- })
- }
-
- log.Printf("🗑️ [SCHEDULE] Deleting schedule %s for agent %s", existingSchedule.ID.Hex(), agentID)
-
- if err := h.schedulerService.DeleteSchedule(c.Context(), existingSchedule.ID.Hex(), userID); err != nil {
- log.Printf("❌ [SCHEDULE] Failed to delete schedule: %v", err)
- return c.Status(fiber.StatusInternalServerError).JSON(fiber.Map{
- "error": "Failed to delete schedule",
- })
- }
-
- log.Printf("✅ [SCHEDULE] Deleted schedule for agent %s", agentID)
- return c.Status(fiber.StatusNoContent).Send(nil)
-}
-
-// TriggerNow triggers an immediate execution of the schedule
-// POST /api/agents/:id/schedule/run
-func (h *ScheduleHandler) TriggerNow(c *fiber.Ctx) error {
- userID, ok := c.Locals("user_id").(string)
- if !ok || userID == "" {
- return c.Status(fiber.StatusUnauthorized).JSON(fiber.Map{
- "error": "Authentication required",
- })
- }
-
- agentID := c.Params("id")
- if agentID == "" {
- return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{
- "error": "Agent ID is required",
- })
- }
-
- // Get existing schedule
- existingSchedule, err := h.schedulerService.GetScheduleByAgentID(c.Context(), agentID, userID)
- if err != nil {
- return c.Status(fiber.StatusNotFound).JSON(fiber.Map{
- "error": "No schedule found for this agent",
- })
- }
-
- log.Printf("▶️ [SCHEDULE] Triggering immediate run for schedule %s (agent: %s)", existingSchedule.ID.Hex(), agentID)
-
- if err := h.schedulerService.TriggerNow(c.Context(), existingSchedule.ID.Hex(), userID); err != nil {
- log.Printf("❌ [SCHEDULE] Failed to trigger schedule: %v", err)
- return c.Status(fiber.StatusInternalServerError).JSON(fiber.Map{
- "error": "Failed to trigger schedule",
- })
- }
-
- return c.JSON(fiber.Map{
- "message": "Schedule triggered successfully",
- })
-}
-
-// GetUsage returns the user's schedule usage stats
-// GET /api/schedules/usage
-func (h *ScheduleHandler) GetUsage(c *fiber.Ctx) error {
- userID, ok := c.Locals("user_id").(string)
- if !ok || userID == "" {
- return c.Status(fiber.StatusUnauthorized).JSON(fiber.Map{
- "error": "Authentication required",
- })
- }
-
- usage, err := h.schedulerService.GetScheduleUsage(c.Context(), userID)
- if err != nil {
- log.Printf("❌ [SCHEDULE] Failed to get schedule usage: %v", err)
- return c.Status(fiber.StatusInternalServerError).JSON(fiber.Map{
- "error": "Failed to get schedule usage",
- })
- }
-
- return c.JSON(usage)
-}
-
-// hasFileInputs checks if a workflow has any variable blocks with file input type
-// File inputs cannot be scheduled because uploaded files expire after 30 minutes
-func hasFileInputs(workflow *models.Workflow) bool {
- if workflow == nil {
- return false
- }
- for _, block := range workflow.Blocks {
- if block.Type == "variable" {
- // Config is map[string]any
- if inputType, exists := block.Config["inputType"]; exists && inputType == "file" {
- return true
- }
- }
- }
- return false
-}
diff --git a/backend/internal/handlers/secure_download.go b/backend/internal/handlers/secure_download.go
deleted file mode 100644
index 2b10ada3..00000000
--- a/backend/internal/handlers/secure_download.go
+++ /dev/null
@@ -1,181 +0,0 @@
-package handlers
-
-import (
- "claraverse/internal/securefile"
- "fmt"
- "log"
-
- "github.com/gofiber/fiber/v2"
-)
-
-// SecureDownloadHandler handles secure file downloads with access codes
-type SecureDownloadHandler struct {
- secureFileService *securefile.Service
-}
-
-// NewSecureDownloadHandler creates a new secure download handler
-func NewSecureDownloadHandler() *SecureDownloadHandler {
- return &SecureDownloadHandler{
- secureFileService: securefile.GetService(),
- }
-}
-
-// Download handles file downloads with access code validation
-// GET /api/files/:id?code=ACCESS_CODE
-func (h *SecureDownloadHandler) Download(c *fiber.Ctx) error {
- fileID := c.Params("id")
- accessCode := c.Query("code")
-
- if fileID == "" {
- return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{
- "error": "file_id is required",
- })
- }
-
- if accessCode == "" {
- return c.Status(fiber.StatusUnauthorized).JSON(fiber.Map{
- "error": "access code is required",
- })
- }
-
- log.Printf("📥 [SECURE-DOWNLOAD] Download request for file %s", fileID)
-
- // Get file with access code verification
- file, content, err := h.secureFileService.GetFile(fileID, accessCode)
- if err != nil {
- log.Printf("❌ [SECURE-DOWNLOAD] Failed to get file %s: %v", fileID, err)
-
- if err.Error() == "invalid access code" {
- return c.Status(fiber.StatusUnauthorized).JSON(fiber.Map{
- "error": "invalid access code",
- })
- }
-
- return c.Status(fiber.StatusNotFound).JSON(fiber.Map{
- "error": "file not found or expired",
- })
- }
-
- // Set headers for download
- c.Set("Content-Type", file.MimeType)
- c.Set("Content-Disposition", fmt.Sprintf("attachment; filename=\"%s\"", file.Filename))
- c.Set("Content-Length", fmt.Sprintf("%d", file.Size))
- c.Set("X-File-ID", file.ID)
- c.Set("X-Expires-At", file.ExpiresAt.Format("2006-01-02T15:04:05Z07:00"))
-
- log.Printf("✅ [SECURE-DOWNLOAD] Serving file %s (%s, %d bytes)", file.ID, file.Filename, file.Size)
-
- return c.Send(content)
-}
-
-// GetInfo returns file metadata without downloading
-// GET /api/files/:id/info?code=ACCESS_CODE
-func (h *SecureDownloadHandler) GetInfo(c *fiber.Ctx) error {
- fileID := c.Params("id")
- accessCode := c.Query("code")
-
- if fileID == "" {
- return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{
- "error": "file_id is required",
- })
- }
-
- if accessCode == "" {
- return c.Status(fiber.StatusUnauthorized).JSON(fiber.Map{
- "error": "access code is required",
- })
- }
-
- file, err := h.secureFileService.GetFileInfo(fileID, accessCode)
- if err != nil {
- if err.Error() == "invalid access code" {
- return c.Status(fiber.StatusUnauthorized).JSON(fiber.Map{
- "error": "invalid access code",
- })
- }
-
- return c.Status(fiber.StatusNotFound).JSON(fiber.Map{
- "error": "file not found or expired",
- })
- }
-
- return c.JSON(fiber.Map{
- "id": file.ID,
- "filename": file.Filename,
- "mime_type": file.MimeType,
- "size": file.Size,
- "created_at": file.CreatedAt,
- "expires_at": file.ExpiresAt,
- })
-}
-
-// Delete removes a file (requires authentication and ownership)
-// DELETE /api/files/:id
-func (h *SecureDownloadHandler) Delete(c *fiber.Ctx) error {
- fileID := c.Params("id")
- userID := c.Locals("user_id")
-
- if fileID == "" {
- return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{
- "error": "file_id is required",
- })
- }
-
- if userID == nil {
- return c.Status(fiber.StatusUnauthorized).JSON(fiber.Map{
- "error": "authentication required",
- })
- }
-
- err := h.secureFileService.DeleteFile(fileID, userID.(string))
- if err != nil {
- if err.Error() == "access denied" {
- return c.Status(fiber.StatusForbidden).JSON(fiber.Map{
- "error": "you don't have permission to delete this file",
- })
- }
-
- return c.Status(fiber.StatusNotFound).JSON(fiber.Map{
- "error": "file not found",
- })
- }
-
- log.Printf("✅ [SECURE-DOWNLOAD] File %s deleted by user %s", fileID, userID)
-
- return c.JSON(fiber.Map{
- "success": true,
- "message": "file deleted",
- })
-}
-
-// ListUserFiles returns all files for the authenticated user
-// GET /api/files
-func (h *SecureDownloadHandler) ListUserFiles(c *fiber.Ctx) error {
- userID := c.Locals("user_id")
-
- if userID == nil {
- return c.Status(fiber.StatusUnauthorized).JSON(fiber.Map{
- "error": "authentication required",
- })
- }
-
- files := h.secureFileService.ListUserFiles(userID.(string))
-
- // Convert to response format (without sensitive data)
- response := make([]fiber.Map, 0, len(files))
- for _, file := range files {
- response = append(response, fiber.Map{
- "id": file.ID,
- "filename": file.Filename,
- "mime_type": file.MimeType,
- "size": file.Size,
- "created_at": file.CreatedAt,
- "expires_at": file.ExpiresAt,
- })
- }
-
- return c.JSON(fiber.Map{
- "files": response,
- "count": len(response),
- })
-}
diff --git a/backend/internal/handlers/subscription.go b/backend/internal/handlers/subscription.go
deleted file mode 100644
index 6d6e2620..00000000
--- a/backend/internal/handlers/subscription.go
+++ /dev/null
@@ -1,363 +0,0 @@
-package handlers
-
-import (
- "claraverse/internal/services"
- "context"
- "log"
-
- "github.com/gofiber/fiber/v2"
-)
-
-// SubscriptionHandler handles subscription-related endpoints
-type SubscriptionHandler struct {
- paymentService *services.PaymentService
- userService *services.UserService
-}
-
-// NewSubscriptionHandler creates a new subscription handler
-func NewSubscriptionHandler(paymentService *services.PaymentService, userService *services.UserService) *SubscriptionHandler {
- return &SubscriptionHandler{
- paymentService: paymentService,
- userService: userService,
- }
-}
-
-// ListPlans returns all available subscription plans
-// GET /api/subscriptions/plans
-func (h *SubscriptionHandler) ListPlans(c *fiber.Ctx) error {
- plans := h.paymentService.GetAvailablePlans()
- return c.JSON(fiber.Map{
- "plans": plans,
- })
-}
-
-// GetCurrent returns the user's current subscription
-// GET /api/subscriptions/current
-func (h *SubscriptionHandler) GetCurrent(c *fiber.Ctx) error {
- userID, ok := c.Locals("user_id").(string)
- if !ok || userID == "" {
- return c.Status(fiber.StatusUnauthorized).JSON(fiber.Map{
- "error": "Authentication required",
- })
- }
-
- // Get email from auth middleware
- email, _ := c.Locals("user_email").(string)
-
- ctx := context.Background()
-
- // Sync user from Supabase (creates user if not exists, applies promo)
- user, err := h.userService.SyncUserFromSupabase(ctx, userID, email)
- if err != nil {
- log.Printf("⚠️ Failed to sync user %s: %v", userID, err)
- }
-
- sub, err := h.paymentService.GetCurrentSubscription(ctx, userID)
- if err != nil {
- log.Printf("⚠️ Failed to get subscription for user %s: %v", userID, err)
- return c.Status(fiber.StatusInternalServerError).JSON(fiber.Map{
- "error": "Failed to get subscription",
- })
- }
-
- // Get user data for promo detection and welcome popup status
- isPromoUser := false
- hasSeenWelcomePopup := false
-
- if user != nil {
- hasSeenWelcomePopup = user.HasSeenWelcomePopup
- // Promo user = PRO tier + has expiration + no Dodo subscription
- isPromoUser = user.SubscriptionTier == "pro" &&
- user.SubscriptionExpiresAt != nil &&
- user.DodoSubscriptionID == ""
- }
-
- // Build complete subscription response
- response := fiber.Map{
- "id": sub.ID.Hex(),
- "user_id": sub.UserID,
- "tier": sub.Tier,
- "status": sub.Status,
- "current_period_start": sub.CurrentPeriodStart,
- "current_period_end": sub.CurrentPeriodEnd,
- "cancel_at_period_end": sub.CancelAtPeriodEnd,
- "is_promo_user": isPromoUser,
- "has_seen_welcome_popup": hasSeenWelcomePopup,
- "created_at": sub.CreatedAt,
- "updated_at": sub.UpdatedAt,
- }
-
- // Add optional Dodo fields if present
- if sub.DodoSubscriptionID != "" {
- response["dodo_subscription_id"] = sub.DodoSubscriptionID
- }
- if sub.DodoCustomerID != "" {
- response["dodo_customer_id"] = sub.DodoCustomerID
- }
-
- // Add subscription expiration (for promo users)
- if user != nil && user.SubscriptionExpiresAt != nil {
- response["subscription_expires_at"] = user.SubscriptionExpiresAt.Format("2006-01-02T15:04:05Z07:00")
- }
-
- // Add scheduled change info if exists
- if sub.HasScheduledChange() {
- response["scheduled_tier"] = sub.ScheduledTier
- response["scheduled_change_at"] = sub.ScheduledChangeAt
- }
-
- // Add cancelled_at if present
- if sub.CancelledAt != nil && !sub.CancelledAt.IsZero() {
- response["cancelled_at"] = sub.CancelledAt
- }
-
- return c.JSON(response)
-}
-
-// CreateCheckoutRequest represents a checkout creation request
-type CreateCheckoutRequest struct {
- PlanID string `json:"plan_id" validate:"required"`
-}
-
-// CreateCheckout creates a checkout session for a subscription
-// POST /api/subscriptions/checkout
-func (h *SubscriptionHandler) CreateCheckout(c *fiber.Ctx) error {
- userID, ok := c.Locals("user_id").(string)
- if !ok || userID == "" {
- return c.Status(fiber.StatusUnauthorized).JSON(fiber.Map{
- "error": "Authentication required",
- })
- }
-
- // Get email from auth context for syncing new users
- userEmail, _ := c.Locals("user_email").(string)
-
- var req CreateCheckoutRequest
- if err := c.BodyParser(&req); err != nil {
- return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{
- "error": "Invalid request body",
- })
- }
-
- if req.PlanID == "" {
- return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{
- "error": "plan_id is required",
- })
- }
-
- ctx := context.Background()
- checkout, err := h.paymentService.CreateCheckoutSession(ctx, userID, userEmail, req.PlanID)
- if err != nil {
- log.Printf("⚠️ Failed to create checkout for user %s: %v", userID, err)
- return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{
- "error": err.Error(),
- })
- }
-
- return c.JSON(checkout)
-}
-
-// ChangePlanRequest represents a plan change request
-type ChangePlanRequest struct {
- PlanID string `json:"plan_id" validate:"required"`
-}
-
-// ChangePlan changes the user's subscription plan
-// POST /api/subscriptions/change-plan
-func (h *SubscriptionHandler) ChangePlan(c *fiber.Ctx) error {
- userID, ok := c.Locals("user_id").(string)
- if !ok || userID == "" {
- return c.Status(fiber.StatusUnauthorized).JSON(fiber.Map{
- "error": "Authentication required",
- })
- }
-
- var req ChangePlanRequest
- if err := c.BodyParser(&req); err != nil {
- return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{
- "error": "Invalid request body",
- })
- }
-
- if req.PlanID == "" {
- return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{
- "error": "plan_id is required",
- })
- }
-
- ctx := context.Background()
- result, err := h.paymentService.ChangePlan(ctx, userID, req.PlanID)
- if err != nil {
- log.Printf("⚠️ Failed to change plan for user %s: %v", userID, err)
- return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{
- "error": err.Error(),
- })
- }
-
- return c.JSON(result)
-}
-
-// PreviewPlanChange previews a plan change
-// GET /api/subscriptions/change-plan/preview?plan_id=pro
-func (h *SubscriptionHandler) PreviewPlanChange(c *fiber.Ctx) error {
- userID, ok := c.Locals("user_id").(string)
- if !ok || userID == "" {
- return c.Status(fiber.StatusUnauthorized).JSON(fiber.Map{
- "error": "Authentication required",
- })
- }
-
- planID := c.Query("plan_id")
- if planID == "" {
- return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{
- "error": "plan_id query parameter is required",
- })
- }
-
- ctx := context.Background()
- preview, err := h.paymentService.PreviewPlanChange(ctx, userID, planID)
- if err != nil {
- log.Printf("⚠️ Failed to preview plan change for user %s: %v", userID, err)
- return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{
- "error": err.Error(),
- })
- }
-
- return c.JSON(preview)
-}
-
-// Cancel cancels the user's subscription
-// POST /api/subscriptions/cancel
-func (h *SubscriptionHandler) Cancel(c *fiber.Ctx) error {
- userID, ok := c.Locals("user_id").(string)
- if !ok || userID == "" {
- return c.Status(fiber.StatusUnauthorized).JSON(fiber.Map{
- "error": "Authentication required",
- })
- }
-
- ctx := context.Background()
- err := h.paymentService.CancelSubscription(ctx, userID)
- if err != nil {
- log.Printf("⚠️ Failed to cancel subscription for user %s: %v", userID, err)
- return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{
- "error": err.Error(),
- })
- }
-
- // Get updated subscription to return cancel date
- sub, _ := h.paymentService.GetCurrentSubscription(ctx, userID)
- return c.JSON(fiber.Map{
- "status": "pending_cancel",
- "cancel_at": sub.CurrentPeriodEnd,
- "message": "Your subscription will be cancelled at the end of the billing period. You'll retain access until then.",
- })
-}
-
-// Reactivate reactivates a cancelled subscription
-// POST /api/subscriptions/reactivate
-func (h *SubscriptionHandler) Reactivate(c *fiber.Ctx) error {
- userID, ok := c.Locals("user_id").(string)
- if !ok || userID == "" {
- return c.Status(fiber.StatusUnauthorized).JSON(fiber.Map{
- "error": "Authentication required",
- })
- }
-
- ctx := context.Background()
- err := h.paymentService.ReactivateSubscription(ctx, userID)
- if err != nil {
- log.Printf("⚠️ Failed to reactivate subscription for user %s: %v", userID, err)
- return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{
- "error": err.Error(),
- })
- }
-
- return c.JSON(fiber.Map{
- "status": "active",
- "message": "Subscription reactivated successfully",
- })
-}
-
-// GetPortalURL returns the DodoPayments customer portal URL
-// GET /api/subscriptions/portal
-func (h *SubscriptionHandler) GetPortalURL(c *fiber.Ctx) error {
- userID, ok := c.Locals("user_id").(string)
- if !ok || userID == "" {
- return c.Status(fiber.StatusUnauthorized).JSON(fiber.Map{
- "error": "Authentication required",
- })
- }
-
- ctx := context.Background()
- url, err := h.paymentService.GetCustomerPortalURL(ctx, userID)
- if err != nil {
- log.Printf("⚠️ Failed to get portal URL for user %s: %v", userID, err)
- return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{
- "error": err.Error(),
- })
- }
-
- return c.JSON(fiber.Map{
- "portal_url": url,
- })
-}
-
-// ListInvoices returns invoice history (placeholder - requires DodoPayments API)
-// GET /api/subscriptions/invoices
-func (h *SubscriptionHandler) ListInvoices(c *fiber.Ctx) error {
- userID, ok := c.Locals("user_id").(string)
- if !ok || userID == "" {
- return c.Status(fiber.StatusUnauthorized).JSON(fiber.Map{
- "error": "Authentication required",
- })
- }
-
- // TODO: Implement invoice listing when DodoPayments API is available
- return c.JSON(fiber.Map{
- "invoices": []interface{}{},
- "message": "Invoice history coming soon",
- })
-}
-
-// SyncSubscription manually syncs subscription data from DodoPayments
-// POST /api/subscriptions/sync
-func (h *SubscriptionHandler) SyncSubscription(c *fiber.Ctx) error {
- userID, ok := c.Locals("user_id").(string)
- if !ok || userID == "" {
- return c.Status(fiber.StatusUnauthorized).JSON(fiber.Map{
- "error": "Authentication required",
- })
- }
-
- result, err := h.paymentService.SyncSubscriptionFromDodo(c.Context(), userID)
- if err != nil {
- return c.Status(fiber.StatusInternalServerError).JSON(fiber.Map{
- "error": err.Error(),
- })
- }
-
- return c.JSON(result)
-}
-
-// GetUsageStats returns current usage statistics for the user
-// GET /api/subscriptions/usage
-func (h *SubscriptionHandler) GetUsageStats(c *fiber.Ctx) error {
- userID, ok := c.Locals("user_id").(string)
- if !ok || userID == "" {
- return c.Status(fiber.StatusUnauthorized).JSON(fiber.Map{
- "error": "Authentication required",
- })
- }
-
- ctx := c.Context()
- stats, err := h.paymentService.GetUsageStats(ctx, userID)
- if err != nil {
- log.Printf("⚠️ Failed to get usage stats for user %s: %v", userID, err)
- return c.Status(fiber.StatusInternalServerError).JSON(fiber.Map{
- "error": "Failed to get usage statistics",
- })
- }
-
- return c.JSON(stats)
-}
diff --git a/backend/internal/handlers/subscription_handler_test.go b/backend/internal/handlers/subscription_handler_test.go
deleted file mode 100644
index 2d472e6c..00000000
--- a/backend/internal/handlers/subscription_handler_test.go
+++ /dev/null
@@ -1,297 +0,0 @@
-package handlers
-
-import (
- "bytes"
- "claraverse/internal/models"
- "claraverse/internal/services"
- "encoding/json"
- "io"
- "net/http/httptest"
- "testing"
-
- "github.com/gofiber/fiber/v2"
-)
-
-func setupSubscriptionTestApp(t *testing.T) (*fiber.App, *services.PaymentService) {
- app := fiber.New()
- paymentService := services.NewPaymentService("test", "secret", "biz", nil, nil, nil)
- return app, paymentService
-}
-
-func mockSubscriptionAuthMiddleware(userID string) fiber.Handler {
- return func(c *fiber.Ctx) error {
- c.Locals("user_id", userID)
- c.Locals("user_email", "test@example.com")
- return c.Next()
- }
-}
-
-func TestSubscriptionHandler_ListPlans(t *testing.T) {
- app, paymentService := setupSubscriptionTestApp(t)
- handler := NewSubscriptionHandler(paymentService)
-
- app.Get("/api/subscriptions/plans", handler.ListPlans)
-
- req := httptest.NewRequest("GET", "/api/subscriptions/plans", nil)
- resp, err := app.Test(req)
- if err != nil {
- t.Fatalf("Request failed: %v", err)
- }
- defer resp.Body.Close()
-
- if resp.StatusCode != fiber.StatusOK {
- t.Errorf("Expected 200, got %d", resp.StatusCode)
- }
-
- body, _ := io.ReadAll(resp.Body)
- var result struct {
- Plans []models.Plan `json:"plans"`
- }
- json.Unmarshal(body, &result)
-
- if len(result.Plans) < 4 {
- t.Errorf("Expected at least 4 plans, got %d", len(result.Plans))
- }
-}
-
-func TestSubscriptionHandler_GetCurrent_Unauthenticated(t *testing.T) {
- app, paymentService := setupSubscriptionTestApp(t)
- handler := NewSubscriptionHandler(paymentService)
-
- app.Get("/api/subscriptions/current", handler.GetCurrent)
-
- req := httptest.NewRequest("GET", "/api/subscriptions/current", nil)
- resp, err := app.Test(req)
- if err != nil {
- t.Fatalf("Request failed: %v", err)
- }
-
- if resp.StatusCode != fiber.StatusUnauthorized {
- t.Errorf("Expected 401, got %d", resp.StatusCode)
- }
-}
-
-func TestSubscriptionHandler_GetCurrent_Authenticated(t *testing.T) {
- app, paymentService := setupSubscriptionTestApp(t)
- handler := NewSubscriptionHandler(paymentService)
-
- app.Use(mockAuthMiddleware("user-123"))
- app.Get("/api/subscriptions/current", handler.GetCurrent)
-
- req := httptest.NewRequest("GET", "/api/subscriptions/current", nil)
- resp, err := app.Test(req)
- if err != nil {
- t.Fatalf("Request failed: %v", err)
- }
-
- // Without MongoDB, expect default free tier response
- if resp.StatusCode != fiber.StatusOK {
- t.Errorf("Expected 200, got %d", resp.StatusCode)
- }
-
- body, _ := io.ReadAll(resp.Body)
- var result struct {
- Tier string `json:"tier"`
- Status string `json:"status"`
- }
- json.Unmarshal(body, &result)
-
- if result.Tier != models.TierFree {
- t.Errorf("Expected free tier, got %s", result.Tier)
- }
-}
-
-func TestSubscriptionHandler_CreateCheckout_InvalidPlan(t *testing.T) {
- app, paymentService := setupSubscriptionTestApp(t)
- handler := NewSubscriptionHandler(paymentService)
-
- app.Use(mockAuthMiddleware("user-123"))
- app.Post("/api/subscriptions/checkout", handler.CreateCheckout)
-
- reqBody := bytes.NewBuffer([]byte(`{"plan_id": "invalid_plan"}`))
- req := httptest.NewRequest("POST", "/api/subscriptions/checkout", reqBody)
- req.Header.Set("Content-Type", "application/json")
-
- resp, err := app.Test(req)
- if err != nil {
- t.Fatalf("Request failed: %v", err)
- }
-
- if resp.StatusCode != fiber.StatusBadRequest {
- t.Errorf("Expected 400 for invalid plan, got %d", resp.StatusCode)
- }
-}
-
-func TestSubscriptionHandler_CreateCheckout_FreePlan(t *testing.T) {
- app, paymentService := setupSubscriptionTestApp(t)
- handler := NewSubscriptionHandler(paymentService)
-
- app.Use(mockAuthMiddleware("user-123"))
- app.Post("/api/subscriptions/checkout", handler.CreateCheckout)
-
- reqBody := bytes.NewBuffer([]byte(`{"plan_id": "free"}`))
- req := httptest.NewRequest("POST", "/api/subscriptions/checkout", reqBody)
- req.Header.Set("Content-Type", "application/json")
-
- resp, err := app.Test(req)
- if err != nil {
- t.Fatalf("Request failed: %v", err)
- }
-
- // Should reject - can't checkout for free plan
- if resp.StatusCode != fiber.StatusBadRequest {
- t.Errorf("Expected 400 for free plan checkout, got %d", resp.StatusCode)
- }
-}
-
-func TestSubscriptionHandler_CreateCheckout_EnterprisePlan(t *testing.T) {
- app, paymentService := setupSubscriptionTestApp(t)
- handler := NewSubscriptionHandler(paymentService)
-
- app.Use(mockAuthMiddleware("user-123"))
- app.Post("/api/subscriptions/checkout", handler.CreateCheckout)
-
- reqBody := bytes.NewBuffer([]byte(`{"plan_id": "enterprise"}`))
- req := httptest.NewRequest("POST", "/api/subscriptions/checkout", reqBody)
- req.Header.Set("Content-Type", "application/json")
-
- resp, err := app.Test(req)
- if err != nil {
- t.Fatalf("Request failed: %v", err)
- }
-
- // Should reject - enterprise requires contact sales
- if resp.StatusCode != fiber.StatusBadRequest {
- t.Errorf("Expected 400 for enterprise checkout, got %d", resp.StatusCode)
- }
-}
-
-func TestSubscriptionHandler_CreateCheckout_MissingPlanID(t *testing.T) {
- app, paymentService := setupSubscriptionTestApp(t)
- handler := NewSubscriptionHandler(paymentService)
-
- app.Use(mockAuthMiddleware("user-123"))
- app.Post("/api/subscriptions/checkout", handler.CreateCheckout)
-
- reqBody := bytes.NewBuffer([]byte(`{}`))
- req := httptest.NewRequest("POST", "/api/subscriptions/checkout", reqBody)
- req.Header.Set("Content-Type", "application/json")
-
- resp, err := app.Test(req)
- if err != nil {
- t.Fatalf("Request failed: %v", err)
- }
-
- if resp.StatusCode != fiber.StatusBadRequest {
- t.Errorf("Expected 400 for missing plan_id, got %d", resp.StatusCode)
- }
-}
-
-func TestSubscriptionHandler_PreviewPlanChange(t *testing.T) {
- app, paymentService := setupSubscriptionTestApp(t)
- handler := NewSubscriptionHandler(paymentService)
-
- app.Use(mockAuthMiddleware("user-123"))
- app.Get("/api/subscriptions/change-plan/preview", handler.PreviewPlanChange)
-
- tests := []struct {
- name string
- planID string
- expectCode int
- }{
- {"valid upgrade", "pro", fiber.StatusOK},
- {"invalid plan", "invalid", fiber.StatusBadRequest},
- {"missing plan", "", fiber.StatusBadRequest},
- }
-
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- url := "/api/subscriptions/change-plan/preview"
- if tt.planID != "" {
- url += "?plan_id=" + tt.planID
- }
-
- req := httptest.NewRequest("GET", url, nil)
- resp, _ := app.Test(req)
-
- if resp.StatusCode != tt.expectCode {
- t.Errorf("Expected %d, got %d", tt.expectCode, resp.StatusCode)
- }
- })
- }
-}
-
-func TestSubscriptionHandler_Cancel(t *testing.T) {
- app, paymentService := setupSubscriptionTestApp(t)
- handler := NewSubscriptionHandler(paymentService)
-
- app.Use(mockAuthMiddleware("user-123"))
- app.Post("/api/subscriptions/cancel", handler.Cancel)
-
- req := httptest.NewRequest("POST", "/api/subscriptions/cancel", nil)
- resp, err := app.Test(req)
- if err != nil {
- t.Fatalf("Request failed: %v", err)
- }
-
- // Without active subscription, should handle gracefully
- // Could be 200 (already free) or 400 (nothing to cancel)
- if resp.StatusCode != fiber.StatusOK && resp.StatusCode != fiber.StatusBadRequest {
- t.Errorf("Unexpected status: %d", resp.StatusCode)
- }
-}
-
-func TestSubscriptionHandler_Reactivate_NoActiveCancellation(t *testing.T) {
- app, paymentService := setupSubscriptionTestApp(t)
- handler := NewSubscriptionHandler(paymentService)
-
- app.Use(mockAuthMiddleware("user-123"))
- app.Post("/api/subscriptions/reactivate", handler.Reactivate)
-
- req := httptest.NewRequest("POST", "/api/subscriptions/reactivate", nil)
- resp, err := app.Test(req)
- if err != nil {
- t.Fatalf("Request failed: %v", err)
- }
-
- // Without pending cancellation, should fail
- if resp.StatusCode != fiber.StatusBadRequest {
- t.Errorf("Expected 400, got %d", resp.StatusCode)
- }
-}
-
-func TestSubscriptionHandler_GetPortalURL_Unauthenticated(t *testing.T) {
- app, paymentService := setupSubscriptionTestApp(t)
- handler := NewSubscriptionHandler(paymentService)
-
- app.Get("/api/subscriptions/portal", handler.GetPortalURL)
-
- req := httptest.NewRequest("GET", "/api/subscriptions/portal", nil)
- resp, err := app.Test(req)
- if err != nil {
- t.Fatalf("Request failed: %v", err)
- }
-
- if resp.StatusCode != fiber.StatusUnauthorized {
- t.Errorf("Expected 401, got %d", resp.StatusCode)
- }
-}
-
-func TestSubscriptionHandler_ListInvoices(t *testing.T) {
- app, paymentService := setupSubscriptionTestApp(t)
- handler := NewSubscriptionHandler(paymentService)
-
- app.Use(mockAuthMiddleware("user-123"))
- app.Get("/api/subscriptions/invoices", handler.ListInvoices)
-
- req := httptest.NewRequest("GET", "/api/subscriptions/invoices", nil)
- resp, err := app.Test(req)
- if err != nil {
- t.Fatalf("Request failed: %v", err)
- }
-
- if resp.StatusCode != fiber.StatusOK {
- t.Errorf("Expected 200, got %d", resp.StatusCode)
- }
-}
-
diff --git a/backend/internal/handlers/tools.go b/backend/internal/handlers/tools.go
deleted file mode 100644
index 85d95649..00000000
--- a/backend/internal/handlers/tools.go
+++ /dev/null
@@ -1,381 +0,0 @@
-package handlers
-
-import (
- "claraverse/internal/services"
- "claraverse/internal/tools"
- "sort"
- "strings"
-
- "github.com/gofiber/fiber/v2"
-)
-
-// ToolsHandler handles tool-related requests
-type ToolsHandler struct {
- registry *tools.Registry
- toolService *services.ToolService
-}
-
-// NewToolsHandler creates a new tools handler
-func NewToolsHandler(registry *tools.Registry, toolService *services.ToolService) *ToolsHandler {
- return &ToolsHandler{
- registry: registry,
- toolService: toolService,
- }
-}
-
-// ToolResponse represents a tool in the API response
-type ToolResponse struct {
- Name string `json:"name"`
- DisplayName string `json:"display_name"`
- Description string `json:"description"`
- Icon string `json:"icon"`
- Category string `json:"category"`
- Keywords []string `json:"keywords"`
- Source string `json:"source"`
-}
-
-// CategoryResponse represents a category with its tools
-type CategoryResponse struct {
- Name string `json:"name"`
- Count int `json:"count"`
- Tools []ToolResponse `json:"tools"`
-}
-
-// ListTools returns all tools available to the authenticated user, grouped by category
-func (h *ToolsHandler) ListTools(c *fiber.Ctx) error {
- // Extract user ID from auth middleware
- userID, ok := c.Locals("user_id").(string)
- if !ok || userID == "" {
- return c.Status(fiber.StatusUnauthorized).JSON(fiber.Map{
- "error": "User not authenticated",
- })
- }
-
- // Get all tools for the user (built-in + MCP)
- toolsList := h.registry.GetUserTools(userID)
-
- // Group tools by category
- categoryMap := make(map[string][]ToolResponse)
-
- for _, toolDef := range toolsList {
- function, ok := toolDef["function"].(map[string]interface{})
- if !ok {
- continue
- }
-
- name, _ := function["name"].(string)
- description, _ := function["description"].(string)
-
- // Get the actual tool to extract metadata
- tool, exists := h.registry.GetUserTool(userID, name)
- if !exists {
- continue
- }
-
- toolResponse := ToolResponse{
- Name: tool.Name,
- DisplayName: tool.DisplayName,
- Description: description,
- Icon: tool.Icon,
- Category: tool.Category,
- Keywords: tool.Keywords,
- Source: string(tool.Source),
- }
-
- // Group by category (default to "other" if no category)
- category := tool.Category
- if category == "" {
- category = "other"
- }
-
- categoryMap[category] = append(categoryMap[category], toolResponse)
- }
-
- // Convert map to array of CategoryResponse
- categories := make([]CategoryResponse, 0, len(categoryMap))
- for categoryName, categoryTools := range categoryMap {
- categories = append(categories, CategoryResponse{
- Name: categoryName,
- Count: len(categoryTools),
- Tools: categoryTools,
- })
- }
-
- // Sort categories alphabetically
- sort.Slice(categories, func(i, j int) bool {
- return categories[i].Name < categories[j].Name
- })
-
- return c.JSON(fiber.Map{
- "categories": categories,
- "total": h.registry.CountUserTools(userID),
- })
-}
-
-// AvailableToolResponse represents a tool with credential metadata
-type AvailableToolResponse struct {
- Name string `json:"name"`
- DisplayName string `json:"display_name"`
- Description string `json:"description"`
- Icon string `json:"icon"`
- Category string `json:"category"`
- Keywords []string `json:"keywords"`
- Source string `json:"source"`
- RequiresCredential bool `json:"requires_credential"`
- IntegrationType string `json:"integration_type,omitempty"`
-}
-
-// GetAvailableTools returns tools filtered by user's credentials
-// Only tools that don't require credentials OR tools where user has configured credentials are returned
-func (h *ToolsHandler) GetAvailableTools(c *fiber.Ctx) error {
- // Extract user ID from auth middleware
- userID, ok := c.Locals("user_id").(string)
- if !ok || userID == "" {
- return c.Status(fiber.StatusUnauthorized).JSON(fiber.Map{
- "error": "User not authenticated",
- })
- }
-
- // If no tool service available, fall back to all tools
- if h.toolService == nil {
- return h.ListTools(c)
- }
-
- // Get filtered tools from tool service
- filteredTools := h.toolService.GetAvailableTools(c.Context(), userID)
-
- // Build response with metadata
- toolResponses := make([]AvailableToolResponse, 0, len(filteredTools))
- categoryMap := make(map[string][]AvailableToolResponse)
-
- for _, toolDef := range filteredTools {
- function, ok := toolDef["function"].(map[string]interface{})
- if !ok {
- continue
- }
-
- name, _ := function["name"].(string)
- description, _ := function["description"].(string)
-
- // Get the actual tool to extract metadata
- tool, exists := h.registry.GetUserTool(userID, name)
- if !exists {
- continue
- }
-
- // Check if tool requires credentials
- integrationType := tools.GetIntegrationTypeForTool(name)
- requiresCredential := integrationType != ""
-
- toolResponse := AvailableToolResponse{
- Name: tool.Name,
- DisplayName: tool.DisplayName,
- Description: description,
- Icon: tool.Icon,
- Category: tool.Category,
- Keywords: tool.Keywords,
- Source: string(tool.Source),
- RequiresCredential: requiresCredential,
- IntegrationType: integrationType,
- }
-
- toolResponses = append(toolResponses, toolResponse)
-
- // Group by category
- category := tool.Category
- if category == "" {
- category = "other"
- }
- categoryMap[category] = append(categoryMap[category], toolResponse)
- }
-
- // Convert to category response format
- categories := make([]struct {
- Name string `json:"name"`
- Count int `json:"count"`
- Tools []AvailableToolResponse `json:"tools"`
- }, 0, len(categoryMap))
-
- for categoryName, categoryTools := range categoryMap {
- categories = append(categories, struct {
- Name string `json:"name"`
- Count int `json:"count"`
- Tools []AvailableToolResponse `json:"tools"`
- }{
- Name: categoryName,
- Count: len(categoryTools),
- Tools: categoryTools,
- })
- }
-
- // Sort categories alphabetically
- sort.Slice(categories, func(i, j int) bool {
- return categories[i].Name < categories[j].Name
- })
-
- // Get total count for comparison
- allToolsCount := h.registry.CountUserTools(userID)
- filteredCount := allToolsCount - len(filteredTools)
-
- return c.JSON(fiber.Map{
- "categories": categories,
- "total": len(filteredTools),
- "filtered_count": filteredCount, // Number of tools filtered out due to missing credentials
- })
-}
-
-// RecommendToolsRequest represents the request body for tool recommendations
-type RecommendToolsRequest struct {
- BlockName string `json:"block_name"`
- BlockDescription string `json:"block_description"`
- BlockType string `json:"block_type"`
-}
-
-// ToolRecommendation represents a recommended tool with a score
-type ToolRecommendation struct {
- Name string `json:"name"`
- DisplayName string `json:"display_name"`
- Description string `json:"description"`
- Icon string `json:"icon"`
- Category string `json:"category"`
- Keywords []string `json:"keywords"`
- Source string `json:"source"`
- Score int `json:"score"`
- Reason string `json:"reason"`
-}
-
-// RecommendTools returns scored and ranked tool recommendations based on block context
-func (h *ToolsHandler) RecommendTools(c *fiber.Ctx) error {
- // Extract user ID from auth middleware
- userID, ok := c.Locals("user_id").(string)
- if !ok || userID == "" {
- return c.Status(fiber.StatusUnauthorized).JSON(fiber.Map{
- "error": "User not authenticated",
- })
- }
-
- // Parse request body
- var req RecommendToolsRequest
- if err := c.BodyParser(&req); err != nil {
- return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{
- "error": "Invalid request body",
- })
- }
-
- // Tokenize block context (name + description)
- context := strings.ToLower(req.BlockName + " " + req.BlockDescription)
- contextTokens := tokenize(context)
-
- // Get all tools for the user
- toolsList := h.registry.GetUserTools(userID)
-
- // Score each tool based on keyword matching
- recommendations := []ToolRecommendation{}
-
- for _, toolDef := range toolsList {
- function, ok := toolDef["function"].(map[string]interface{})
- if !ok {
- continue
- }
-
- name, _ := function["name"].(string)
-
- // Get the actual tool to extract metadata
- tool, exists := h.registry.GetUserTool(userID, name)
- if !exists {
- continue
- }
-
- // Calculate match score
- score, matchedKeywords := calculateMatchScore(contextTokens, tool.Keywords)
-
- // Only include tools with a score > 0
- if score > 0 {
- reason := "Matches: " + strings.Join(matchedKeywords, ", ")
-
- recommendations = append(recommendations, ToolRecommendation{
- Name: tool.Name,
- DisplayName: tool.DisplayName,
- Description: tool.Description,
- Icon: tool.Icon,
- Category: tool.Category,
- Keywords: tool.Keywords,
- Source: string(tool.Source),
- Score: score,
- Reason: reason,
- })
- }
- }
-
- // Sort recommendations by score (descending)
- sort.Slice(recommendations, func(i, j int) bool {
- return recommendations[i].Score > recommendations[j].Score
- })
-
- // Limit to top 10 recommendations
- if len(recommendations) > 10 {
- recommendations = recommendations[:10]
- }
-
- return c.JSON(fiber.Map{
- "recommendations": recommendations,
- "count": len(recommendations),
- })
-}
-
-// tokenize splits a string into lowercase tokens
-func tokenize(text string) []string {
- // Replace common separators with spaces
- text = strings.ReplaceAll(text, "-", " ")
- text = strings.ReplaceAll(text, "_", " ")
- text = strings.ReplaceAll(text, "/", " ")
-
- // Split by whitespace
- tokens := strings.Fields(text)
-
- // Deduplicate tokens
- tokenSet := make(map[string]bool)
- uniqueTokens := []string{}
- for _, token := range tokens {
- token = strings.ToLower(token)
- if !tokenSet[token] && token != "" {
- tokenSet[token] = true
- uniqueTokens = append(uniqueTokens, token)
- }
- }
-
- return uniqueTokens
-}
-
-// calculateMatchScore calculates how well a tool matches the context tokens
-func calculateMatchScore(contextTokens []string, keywords []string) (int, []string) {
- score := 0
- matchedKeywords := []string{}
-
- // Normalize keywords to lowercase
- normalizedKeywords := make([]string, len(keywords))
- for i, keyword := range keywords {
- normalizedKeywords[i] = strings.ToLower(keyword)
- }
-
- // Check each context token against keywords
- for _, token := range contextTokens {
- for _, keyword := range normalizedKeywords {
- // Exact match
- if token == keyword {
- score += 10
- matchedKeywords = append(matchedKeywords, keyword)
- break
- }
-
- // Partial match (substring)
- if strings.Contains(keyword, token) || strings.Contains(token, keyword) {
- score += 5
- matchedKeywords = append(matchedKeywords, keyword)
- break
- }
- }
- }
-
- return score, matchedKeywords
-}
diff --git a/backend/internal/handlers/trigger.go b/backend/internal/handlers/trigger.go
deleted file mode 100644
index 1e651be7..00000000
--- a/backend/internal/handlers/trigger.go
+++ /dev/null
@@ -1,333 +0,0 @@
-package handlers
-
-import (
- "claraverse/internal/execution"
- "claraverse/internal/models"
- "claraverse/internal/services"
- "context"
- "log"
-
- "github.com/gofiber/fiber/v2"
- "go.mongodb.org/mongo-driver/bson/primitive"
-)
-
-// TriggerHandler handles agent trigger endpoints (API key authenticated)
-type TriggerHandler struct {
- agentService *services.AgentService
- executionService *services.ExecutionService
- workflowEngine *execution.WorkflowEngine
-}
-
-// NewTriggerHandler creates a new trigger handler
-func NewTriggerHandler(
- agentService *services.AgentService,
- executionService *services.ExecutionService,
- workflowEngine *execution.WorkflowEngine,
-) *TriggerHandler {
- return &TriggerHandler{
- agentService: agentService,
- executionService: executionService,
- workflowEngine: workflowEngine,
- }
-}
-
-// TriggerAgent executes an agent via API key
-// POST /api/trigger/:agentId
-func (h *TriggerHandler) TriggerAgent(c *fiber.Ctx) error {
- agentID := c.Params("agentId")
- userID := c.Locals("user_id").(string)
-
- // Parse request body
- var req models.TriggerAgentRequest
- if err := c.BodyParser(&req); err != nil && err.Error() != "Unprocessable Entity" {
- return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{
- "error": "Invalid request body",
- })
- }
-
- // Get the agent
- agent, err := h.agentService.GetAgentByID(agentID)
- if err != nil {
- log.Printf("❌ [TRIGGER] Agent not found: %s", agentID)
- return c.Status(fiber.StatusNotFound).JSON(fiber.Map{
- "error": "Agent not found",
- })
- }
-
- // Verify the agent belongs to the user (API key owner)
- if agent.UserID != userID {
- log.Printf("🚫 [TRIGGER] User %s attempted to trigger agent %s (owned by %s)", userID, agentID, agent.UserID)
- return c.Status(fiber.StatusForbidden).JSON(fiber.Map{
- "error": "You do not have permission to trigger this agent",
- })
- }
-
- // Check if agent has a workflow
- if agent.Workflow == nil || len(agent.Workflow.Blocks) == 0 {
- return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{
- "error": "Agent has no workflow configured",
- })
- }
-
- // Get API key ID from context (for tracking)
- var apiKeyID primitive.ObjectID
- if apiKey, ok := c.Locals("api_key").(*models.APIKey); ok {
- apiKeyID = apiKey.ID
- }
-
- // Create execution record
- execReq := &services.CreateExecutionRequest{
- AgentID: agentID,
- UserID: userID,
- WorkflowVersion: agent.Workflow.Version,
- TriggerType: "api",
- APIKeyID: apiKeyID,
- Input: req.Input,
- }
-
- execRecord, err := h.executionService.Create(c.Context(), execReq)
- if err != nil {
- log.Printf("❌ [TRIGGER] Failed to create execution record: %v", err)
- return c.Status(fiber.StatusInternalServerError).JSON(fiber.Map{
- "error": "Failed to create execution",
- })
- }
-
- // Update execution status to running
- if err := h.executionService.UpdateStatus(c.Context(), execRecord.ID, "running"); err != nil {
- log.Printf("⚠️ [TRIGGER] Failed to update status: %v", err)
- }
-
- // Build execution options - block checker is DISABLED for API triggers
- // Block checker should only run during platform testing (WebSocket), not production API calls
- execOpts := &ExecuteWorkflowOptions{
- AgentDescription: agent.Description,
- EnableBlockChecker: false, // Disabled for API triggers
- CheckerModelID: req.CheckerModelID,
- }
-
- // Execute workflow asynchronously (pass userID for credential resolution)
- go h.executeWorkflow(execRecord.ID, agent.Workflow, req.Input, userID, execOpts)
-
- log.Printf("🚀 [TRIGGER] Triggered agent %s via API (execution: %s)", agentID, execRecord.ID.Hex())
-
- return c.Status(fiber.StatusAccepted).JSON(models.TriggerAgentResponse{
- ExecutionID: execRecord.ID.Hex(),
- Status: "running",
- Message: "Agent execution started",
- })
-}
-
-// ExecuteWorkflowOptions contains options for executing a workflow
-type ExecuteWorkflowOptions struct {
- AgentDescription string
- EnableBlockChecker bool
- CheckerModelID string
-}
-
-// executeWorkflow runs the workflow and updates the execution record
-func (h *TriggerHandler) executeWorkflow(executionID primitive.ObjectID, workflow *models.Workflow, input map[string]interface{}, userID string, opts *ExecuteWorkflowOptions) {
- ctx := context.Background()
-
- // Create a channel for status updates (we'll drain it since API triggers don't need real-time)
- statusChan := make(chan models.ExecutionUpdate, 100)
- go func() {
- for range statusChan {
- // Drain channel - future: could publish to Redis for status polling
- }
- }()
-
- // Transform input to properly wrap file references for Start blocks
- transformedInput := h.transformInputForWorkflow(workflow, input)
-
- // Inject user context for credential resolution and tool execution
- if transformedInput == nil {
- transformedInput = make(map[string]interface{})
- }
- transformedInput["__user_id__"] = userID
-
- // Build execution options - block checker DISABLED for API triggers
- // Block checker should only run during platform testing (WebSocket), not production API calls
- execOptions := &execution.ExecutionOptions{
- EnableBlockChecker: false, // Disabled for API triggers
- }
- if opts != nil {
- execOptions.WorkflowGoal = opts.AgentDescription
- execOptions.CheckerModelID = opts.CheckerModelID
- }
- log.Printf("🔍 [TRIGGER] Block checker disabled (API trigger - validation only runs during platform testing)")
-
- // Execute the workflow
- result, err := h.workflowEngine.ExecuteWithOptions(ctx, workflow, transformedInput, statusChan, execOptions)
- close(statusChan)
-
- // Update execution record
- completeReq := &services.ExecutionCompleteRequest{
- Status: "completed",
- }
-
- if err != nil {
- completeReq.Status = "failed"
- completeReq.Error = err.Error()
- log.Printf("❌ [TRIGGER] Execution %s failed: %v", executionID.Hex(), err)
- } else {
- completeReq.Status = result.Status
- completeReq.Output = result.Output
- completeReq.BlockStates = result.BlockStates
- if result.Error != "" {
- completeReq.Error = result.Error
- }
- log.Printf("✅ [TRIGGER] Execution %s completed with status: %s", executionID.Hex(), result.Status)
- }
-
- if err := h.executionService.Complete(ctx, executionID, completeReq); err != nil {
- log.Printf("⚠️ [TRIGGER] Failed to complete execution record: %v", err)
- }
-}
-
-// GetExecutionStatus gets the status of an execution
-// GET /api/trigger/status/:executionId
-func (h *TriggerHandler) GetExecutionStatus(c *fiber.Ctx) error {
- executionIDStr := c.Params("executionId")
- userID := c.Locals("user_id").(string)
-
- executionID, err := primitive.ObjectIDFromHex(executionIDStr)
- if err != nil {
- return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{
- "error": "Invalid execution ID",
- })
- }
-
- execution, err := h.executionService.GetByIDAndUser(c.Context(), executionID, userID)
- if err != nil {
- if err.Error() == "execution not found" {
- return c.Status(fiber.StatusNotFound).JSON(fiber.Map{
- "error": "Execution not found",
- })
- }
- log.Printf("❌ [TRIGGER] Failed to get execution: %v", err)
- return c.Status(fiber.StatusInternalServerError).JSON(fiber.Map{
- "error": "Failed to get execution status",
- })
- }
-
- return c.JSON(execution)
-}
-
-// transformInputForWorkflow transforms API input to match workflow expectations
-// This handles the case where file reference fields (file_id, filename, mime_type)
-// are passed directly in input but the Start block expects them nested under a variable name
-func (h *TriggerHandler) transformInputForWorkflow(workflow *models.Workflow, input map[string]interface{}) map[string]interface{} {
- if workflow == nil || input == nil {
- return input
- }
-
- // Find the Start block (variable block with operation "read")
- var startBlockVariableName string
- var startBlockInputType string
-
- for _, block := range workflow.Blocks {
- if block.Type == "variable" {
- config := block.Config
-
- operation, _ := config["operation"].(string)
- if operation != "read" {
- continue
- }
-
- // Found a Start block - get its variable name and input type
- if varName, ok := config["variableName"].(string); ok {
- startBlockVariableName = varName
- }
- if inputType, ok := config["inputType"].(string); ok {
- startBlockInputType = inputType
- }
- break
- }
- }
-
- // If no Start block found, return input as-is
- if startBlockVariableName == "" {
- return input
- }
-
- log.Printf("🔧 [TRIGGER] Found Start block: variableName=%s, inputType=%s", startBlockVariableName, startBlockInputType)
-
- // Check if input contains file reference fields at top level
- _, hasFileID := input["file_id"]
- _, hasFilename := input["filename"]
- _, hasMimeType := input["mime_type"]
-
- isFileReferenceInput := hasFileID && (hasFilename || hasMimeType)
-
- // If this is a file input type and we have file reference fields at top level,
- // wrap them under the Start block's variable name
- if startBlockInputType == "file" && isFileReferenceInput {
- log.Printf("🔧 [TRIGGER] Wrapping file reference fields under variable '%s'", startBlockVariableName)
-
- fileRef := map[string]interface{}{
- "file_id": input["file_id"],
- }
- if hasFilename {
- fileRef["filename"] = input["filename"]
- }
- if hasMimeType {
- fileRef["mime_type"] = input["mime_type"]
- }
-
- // Create new input with file reference wrapped
- newInput := make(map[string]interface{})
-
- // Copy non-file-reference fields
- for k, v := range input {
- if k != "file_id" && k != "filename" && k != "mime_type" {
- newInput[k] = v
- }
- }
-
- // Add wrapped file reference
- newInput[startBlockVariableName] = fileRef
-
- log.Printf("✅ [TRIGGER] Transformed input: %+v", newInput)
- return newInput
- }
-
- // For text inputs, check if the variable name doesn't exist but we have a single text value
- if startBlockInputType == "text" || startBlockInputType == "" {
- // If the variable already exists in input, no transformation needed
- if _, exists := input[startBlockVariableName]; exists {
- return input
- }
-
- // If input has a single "text", "value", or "message" field, map it to the variable name
- if text, ok := input["text"].(string); ok {
- newInput := make(map[string]interface{})
- for k, v := range input {
- newInput[k] = v
- }
- newInput[startBlockVariableName] = text
- log.Printf("🔧 [TRIGGER] Mapped 'text' field to variable '%s'", startBlockVariableName)
- return newInput
- }
- if value, ok := input["value"].(string); ok {
- newInput := make(map[string]interface{})
- for k, v := range input {
- newInput[k] = v
- }
- newInput[startBlockVariableName] = value
- log.Printf("🔧 [TRIGGER] Mapped 'value' field to variable '%s'", startBlockVariableName)
- return newInput
- }
- if message, ok := input["message"].(string); ok {
- newInput := make(map[string]interface{})
- for k, v := range input {
- newInput[k] = v
- }
- newInput[startBlockVariableName] = message
- log.Printf("🔧 [TRIGGER] Mapped 'message' field to variable '%s'", startBlockVariableName)
- return newInput
- }
- }
-
- return input
-}
diff --git a/backend/internal/handlers/upload.go b/backend/internal/handlers/upload.go
deleted file mode 100644
index 8e6b997c..00000000
--- a/backend/internal/handlers/upload.go
+++ /dev/null
@@ -1,918 +0,0 @@
-package handlers
-
-import (
- "bytes"
- "claraverse/internal/filecache"
- "claraverse/internal/security"
- "claraverse/internal/services"
- "claraverse/internal/utils"
- "encoding/csv"
- "fmt"
- "io"
- "log"
- "mime/multipart"
- "net/http"
- "os"
- "path/filepath"
- "strings"
- "time"
-
- "github.com/gofiber/fiber/v2"
- "github.com/google/uuid"
-)
-
-// UploadHandler handles file upload requests
-type UploadHandler struct {
- uploadDir string
- maxImageSize int64
- maxPDFSize int64
- maxDocSize int64 // For DOCX and PPTX
- allowedTypes map[string]bool
- fileCache *filecache.Service
- usageLimiter *services.UsageLimiterService
-}
-
-// NewUploadHandler creates a new upload handler
-func NewUploadHandler(uploadDir string, usageLimiter *services.UsageLimiterService) *UploadHandler {
- // Ensure upload directory exists with restricted permissions
- if err := os.MkdirAll(uploadDir, 0700); err != nil {
- log.Printf("⚠️ Warning: Could not create upload directory: %v", err)
- }
-
- return &UploadHandler{
- uploadDir: uploadDir,
- maxImageSize: 20 * 1024 * 1024, // 20MB for images
- maxPDFSize: 10 * 1024 * 1024, // 10MB for PDFs
- maxDocSize: 10 * 1024 * 1024, // 10MB for DOCX/PPTX
- usageLimiter: usageLimiter,
- allowedTypes: map[string]bool{
- "image/jpeg": true,
- "image/jpg": true,
- "image/png": true,
- "image/webp": true,
- "image/gif": true,
- "application/pdf": true,
- "text/csv": true,
- "application/vnd.ms-excel": true, // .xls
- "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet": true, // .xlsx
- "application/json": true,
- "text/plain": true,
- // Office documents
- "application/vnd.openxmlformats-officedocument.wordprocessingml.document": true, // .docx
- "application/vnd.openxmlformats-officedocument.presentationml.presentation": true, // .pptx
- // Audio files (for Whisper transcription)
- "audio/mpeg": true, // .mp3
- "audio/mp3": true, // .mp3 alternate
- "audio/wav": true, // .wav
- "audio/x-wav": true, // .wav alternate
- "audio/wave": true, // .wav alternate
- "audio/mp4": true, // .m4a
- "audio/x-m4a": true, // .m4a alternate
- "audio/webm": true, // .webm
- "audio/ogg": true, // .ogg
- "audio/flac": true, // .flac
- },
- fileCache: filecache.GetService(),
- }
-}
-
-// UploadResponse represents the upload API response
-type UploadResponse struct {
- FileID string `json:"file_id"`
- Filename string `json:"filename"`
- MimeType string `json:"mime_type"`
- Size int64 `json:"size"`
- Hash string `json:"hash,omitempty"`
- PageCount int `json:"page_count,omitempty"`
- WordCount int `json:"word_count,omitempty"`
- Preview string `json:"preview,omitempty"`
- ConversationID string `json:"conversation_id,omitempty"`
- URL string `json:"url,omitempty"` // Deprecated for PDFs - use file_id
- DataPreview *CSVPreview `json:"data_preview,omitempty"`
-}
-
-// CSVPreview represents a preview of CSV/tabular data
-type CSVPreview struct {
- Headers []string `json:"headers"`
- Rows [][]string `json:"rows"`
- RowCount int `json:"row_count"` // Total rows in file
- ColCount int `json:"col_count"` // Total columns
-}
-
-// Upload handles file upload requests
-func (h *UploadHandler) Upload(c *fiber.Ctx) error {
- // Check authentication
- userID, ok := c.Locals("user_id").(string)
- if !ok || userID == "" || userID == "anonymous" {
- return c.Status(fiber.StatusUnauthorized).JSON(fiber.Map{
- "error": "Authentication required for file uploads",
- })
- }
-
- // Check file upload limit
- if h.usageLimiter != nil {
- ctx := c.Context()
- if err := h.usageLimiter.CheckFileUploadLimit(ctx, userID); err != nil {
- if limitErr, ok := err.(*services.LimitExceededError); ok {
- log.Printf("⚠️ [LIMIT] File upload limit exceeded for user %s: %s", userID, limitErr.Message)
- return c.Status(fiber.StatusTooManyRequests).JSON(fiber.Map{
- "error": limitErr.Message,
- "error_code": limitErr.ErrorCode,
- "limit": limitErr.Limit,
- "used": limitErr.Used,
- "reset_at": limitErr.ResetAt,
- "upgrade_to": limitErr.UpgradeTo,
- })
- }
- }
-
- // Increment file upload count (check passed) - defer to ensure it runs even if upload fails
- defer func() {
- if err := h.usageLimiter.IncrementFileUploadCount(c.Context(), userID); err != nil {
- log.Printf("⚠️ [LIMIT] Failed to increment file upload count for user %s: %v", userID, err)
- }
- }()
- }
-
- // Get conversation_id from form or create new one
- conversationID := c.FormValue("conversation_id")
- if conversationID == "" {
- conversationID = uuid.New().String()
- }
-
- // Get uploaded file
- fileHeader, err := c.FormFile("file")
- if err != nil {
- log.Printf("❌ [UPLOAD] Failed to parse file: %v", err)
- return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{
- "error": "No file provided or invalid file",
- })
- }
-
- // Open uploaded file
- file, err := fileHeader.Open()
- if err != nil {
- log.Printf("❌ [UPLOAD] Failed to open file: %v", err)
- return c.Status(fiber.StatusInternalServerError).JSON(fiber.Map{
- "error": "Failed to process file",
- })
- }
- defer file.Close()
-
- // Read file data into memory
- fileData, err := io.ReadAll(file)
- if err != nil {
- log.Printf("❌ [UPLOAD] Failed to read file: %v", err)
- return c.Status(fiber.StatusInternalServerError).JSON(fiber.Map{
- "error": "Failed to read file",
- })
- }
-
- // Detect content type
- detectedMimeType := h.detectContentTypeFromData(fileData, fileHeader)
-
- // Strip charset and other parameters from MIME type (e.g., "text/plain; charset=utf-8" -> "text/plain")
- mimeType := strings.Split(detectedMimeType, ";")[0]
- mimeType = strings.TrimSpace(mimeType)
-
- // Validate content type
- if !h.allowedTypes[mimeType] {
- log.Printf("⚠️ [UPLOAD] Disallowed file type: %s (detected as: %s)", mimeType, detectedMimeType)
- return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{
- "error": fmt.Sprintf("File type not allowed: %s. Allowed types: PNG, JPG, WebP, GIF, PDF, DOCX, PPTX, CSV, Excel, JSON, MP3, WAV, M4A, OGG, FLAC", mimeType),
- })
- }
-
- // Validate file size based on type
- maxSize := h.maxImageSize
- if mimeType == "application/pdf" {
- maxSize = h.maxPDFSize
- } else if mimeType == "application/vnd.openxmlformats-officedocument.wordprocessingml.document" ||
- mimeType == "application/vnd.openxmlformats-officedocument.presentationml.presentation" {
- // DOCX, PPTX files: 10MB limit
- maxSize = h.maxDocSize
- } else if strings.HasPrefix(mimeType, "text/") || strings.Contains(mimeType, "json") || strings.Contains(mimeType, "spreadsheet") || strings.Contains(mimeType, "excel") {
- // CSV, JSON, Excel files: 100MB limit
- maxSize = 100 * 1024 * 1024
- } else if strings.HasPrefix(mimeType, "audio/") {
- // Audio files: 25MB limit (OpenAI Whisper limit)
- maxSize = 25 * 1024 * 1024
- }
-
- if fileHeader.Size > maxSize {
- log.Printf("⚠️ [UPLOAD] File too large: %d bytes (max %d)", fileHeader.Size, maxSize)
- return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{
- "error": fmt.Sprintf("File too large. Maximum size is %d MB", maxSize/(1024*1024)),
- })
- }
-
- // Calculate file hash (before encryption)
- fileHash := security.CalculateDataHash(fileData)
-
- // Generate unique file ID
- fileID := uuid.New().String()
-
- // Handle PDF files with secure processing
- if mimeType == "application/pdf" {
- return h.handlePDFUpload(c, fileID, userID, conversationID, fileHeader, fileData, fileHash)
- }
-
- // Handle DOCX files with secure processing
- if mimeType == "application/vnd.openxmlformats-officedocument.wordprocessingml.document" {
- return h.handleDOCXUpload(c, fileID, userID, conversationID, fileHeader, fileData, fileHash)
- }
-
- // Handle PPTX files with secure processing
- if mimeType == "application/vnd.openxmlformats-officedocument.presentationml.presentation" {
- return h.handlePPTXUpload(c, fileID, userID, conversationID, fileHeader, fileData, fileHash)
- }
-
- // Handle CSV/Excel/JSON files for E2B tools
- if strings.HasPrefix(mimeType, "text/csv") || strings.Contains(mimeType, "spreadsheet") || strings.Contains(mimeType, "excel") || mimeType == "application/json" || mimeType == "text/plain" {
- return h.handleDataFileUpload(c, fileID, userID, fileHeader, fileData, mimeType, fileHash)
- }
-
- // Handle audio files (for Whisper transcription)
- if strings.HasPrefix(mimeType, "audio/") {
- return h.handleAudioUpload(c, fileID, userID, fileHeader, fileData, mimeType, fileHash)
- }
-
- // Handle image files (existing flow)
- return h.handleImageUpload(c, fileID, userID, fileHeader, fileData, mimeType, fileHash)
-}
-
-// handlePDFUpload processes PDF files with maximum security
-func (h *UploadHandler) handlePDFUpload(c *fiber.Ctx, fileID, userID, conversationID string, fileHeader *multipart.FileHeader, fileData []byte, fileHash *security.Hash) error {
- log.Printf("📄 [UPLOAD] Processing PDF: %s (user: %s, size: %d bytes)", fileHeader.Filename, userID, len(fileData))
-
- // Validate PDF structure
- if err := utils.ValidatePDF(fileData); err != nil {
- log.Printf("❌ [UPLOAD] Invalid PDF: %v", err)
- return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{
- "error": "Invalid or corrupted PDF file",
- })
- }
-
- // Create temporary encrypted file
- tempDir := os.TempDir()
- tempEncryptedPath := filepath.Join(tempDir, fileID+".encrypted")
-
- // Write encrypted file temporarily
- encKey, err := security.GenerateKey()
- if err != nil {
- log.Printf("❌ [UPLOAD] Failed to generate encryption key: %v", err)
- return c.Status(fiber.StatusInternalServerError).JSON(fiber.Map{
- "error": "Failed to process file",
- })
- }
-
- encryptedData, err := security.EncryptData(fileData, encKey)
- if err != nil {
- log.Printf("❌ [UPLOAD] Failed to encrypt file: %v", err)
- return c.Status(fiber.StatusInternalServerError).JSON(fiber.Map{
- "error": "Failed to process file",
- })
- }
-
- if err := os.WriteFile(tempEncryptedPath, encryptedData, 0600); err != nil {
- log.Printf("❌ [UPLOAD] Failed to write temp file: %v", err)
- return c.Status(fiber.StatusInternalServerError).JSON(fiber.Map{
- "error": "Failed to process file",
- })
- }
-
- // Extract text from PDF (in memory)
- metadata, err := utils.ExtractPDFText(fileData)
- if err != nil {
- // Clean up temp file before returning
- security.SecureDeleteFile(tempEncryptedPath)
- log.Printf("❌ [UPLOAD] Failed to extract PDF text: %v", err)
- return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{
- "error": "Failed to extract text from PDF. File may be corrupted or scanned.",
- })
- }
-
- // Delete encrypted file immediately (max 3 seconds on disk)
- if err := security.SecureDeleteFile(tempEncryptedPath); err != nil {
- log.Printf("⚠️ [UPLOAD] Failed to securely delete temp file: %v", err)
- // Continue anyway - file is encrypted
- }
-
- log.Printf("🗑️ [UPLOAD] Encrypted temp file deleted (file was on disk < 3 seconds)")
-
- // Store in memory cache only
- cachedFile := &filecache.CachedFile{
- FileID: fileID,
- UserID: userID,
- ConversationID: conversationID,
- ExtractedText: security.NewSecureString(metadata.Text),
- FileHash: *fileHash,
- Filename: fileHeader.Filename,
- MimeType: "application/pdf",
- Size: fileHeader.Size,
- PageCount: metadata.PageCount,
- WordCount: metadata.WordCount,
- UploadedAt: time.Now(),
- }
-
- h.fileCache.Store(cachedFile)
-
- // Generate preview
- preview := utils.GetPDFPreview(metadata.Text, 200)
-
- log.Printf("✅ [UPLOAD] PDF uploaded successfully: %s (pages: %d, words: %d)", fileHeader.Filename, metadata.PageCount, metadata.WordCount)
-
- return c.Status(fiber.StatusCreated).JSON(UploadResponse{
- FileID: fileID,
- Filename: fileHeader.Filename,
- MimeType: "application/pdf",
- Size: fileHeader.Size,
- Hash: fileHash.String(),
- PageCount: metadata.PageCount,
- WordCount: metadata.WordCount,
- Preview: preview,
- ConversationID: conversationID,
- })
-}
-
-// handleDOCXUpload processes DOCX files with secure text extraction
-func (h *UploadHandler) handleDOCXUpload(c *fiber.Ctx, fileID, userID, conversationID string, fileHeader *multipart.FileHeader, fileData []byte, fileHash *security.Hash) error {
- log.Printf("📄 [UPLOAD] Processing DOCX: %s (user: %s, size: %d bytes)", fileHeader.Filename, userID, len(fileData))
-
- // Validate DOCX structure
- if err := utils.ValidateDOCX(fileData); err != nil {
- log.Printf("❌ [UPLOAD] Invalid DOCX: %v", err)
- return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{
- "error": "Invalid or corrupted DOCX file",
- })
- }
-
- // Extract text from DOCX (in memory)
- metadata, err := utils.ExtractDOCXText(fileData)
- if err != nil {
- log.Printf("❌ [UPLOAD] Failed to extract DOCX text: %v", err)
- return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{
- "error": "Failed to extract text from DOCX. File may be corrupted.",
- })
- }
-
- // Store in memory cache only
- cachedFile := &filecache.CachedFile{
- FileID: fileID,
- UserID: userID,
- ConversationID: conversationID,
- ExtractedText: security.NewSecureString(metadata.Text),
- FileHash: *fileHash,
- Filename: fileHeader.Filename,
- MimeType: "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
- Size: fileHeader.Size,
- PageCount: metadata.PageCount,
- WordCount: metadata.WordCount,
- UploadedAt: time.Now(),
- }
-
- h.fileCache.Store(cachedFile)
-
- // Generate preview
- preview := utils.GetDOCXPreview(metadata.Text, 200)
-
- log.Printf("✅ [UPLOAD] DOCX uploaded successfully: %s (pages: %d, words: %d)", fileHeader.Filename, metadata.PageCount, metadata.WordCount)
-
- return c.Status(fiber.StatusCreated).JSON(UploadResponse{
- FileID: fileID,
- Filename: fileHeader.Filename,
- MimeType: "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
- Size: fileHeader.Size,
- Hash: fileHash.String(),
- PageCount: metadata.PageCount,
- WordCount: metadata.WordCount,
- Preview: preview,
- ConversationID: conversationID,
- })
-}
-
-// handlePPTXUpload processes PPTX files with secure text extraction
-func (h *UploadHandler) handlePPTXUpload(c *fiber.Ctx, fileID, userID, conversationID string, fileHeader *multipart.FileHeader, fileData []byte, fileHash *security.Hash) error {
- log.Printf("📊 [UPLOAD] Processing PPTX: %s (user: %s, size: %d bytes)", fileHeader.Filename, userID, len(fileData))
-
- // Validate PPTX structure
- if err := utils.ValidatePPTX(fileData); err != nil {
- log.Printf("❌ [UPLOAD] Invalid PPTX: %v", err)
- return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{
- "error": "Invalid or corrupted PPTX file",
- })
- }
-
- // Extract text from PPTX (in memory)
- metadata, err := utils.ExtractPPTXText(fileData)
- if err != nil {
- log.Printf("❌ [UPLOAD] Failed to extract PPTX text: %v", err)
- return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{
- "error": "Failed to extract text from PPTX. File may be corrupted.",
- })
- }
-
- // Store in memory cache only
- cachedFile := &filecache.CachedFile{
- FileID: fileID,
- UserID: userID,
- ConversationID: conversationID,
- ExtractedText: security.NewSecureString(metadata.Text),
- FileHash: *fileHash,
- Filename: fileHeader.Filename,
- MimeType: "application/vnd.openxmlformats-officedocument.presentationml.presentation",
- Size: fileHeader.Size,
- PageCount: metadata.SlideCount, // Use SlideCount as PageCount
- WordCount: metadata.WordCount,
- UploadedAt: time.Now(),
- }
-
- h.fileCache.Store(cachedFile)
-
- // Generate preview
- preview := utils.GetPPTXPreview(metadata.Text, 200)
-
- log.Printf("✅ [UPLOAD] PPTX uploaded successfully: %s (slides: %d, words: %d)", fileHeader.Filename, metadata.SlideCount, metadata.WordCount)
-
- return c.Status(fiber.StatusCreated).JSON(UploadResponse{
- FileID: fileID,
- Filename: fileHeader.Filename,
- MimeType: "application/vnd.openxmlformats-officedocument.presentationml.presentation",
- Size: fileHeader.Size,
- Hash: fileHash.String(),
- PageCount: metadata.SlideCount,
- WordCount: metadata.WordCount,
- Preview: preview,
- ConversationID: conversationID,
- })
-}
-
-// handleImageUpload processes image files (existing flow, now with hash)
-func (h *UploadHandler) handleImageUpload(c *fiber.Ctx, fileID, userID string, fileHeader *multipart.FileHeader, fileData []byte, mimeType string, fileHash *security.Hash) error {
- // Get conversation_id from form (may be empty)
- conversationID := c.FormValue("conversation_id")
-
- // Generate filename with extension
- ext := filepath.Ext(fileHeader.Filename)
- if ext == "" {
- ext = h.getExtensionFromMimeType(mimeType)
- }
- savedFilename := fileID + ext
- filePath := filepath.Join(h.uploadDir, savedFilename)
-
- // Save image to disk with restricted permissions (owner read/write only for security)
- if err := os.WriteFile(filePath, fileData, 0600); err != nil {
- log.Printf("❌ [UPLOAD] Failed to save file: %v", err)
- return c.Status(fiber.StatusInternalServerError).JSON(fiber.Map{
- "error": "Failed to save file",
- })
- }
-
- // Register image in cache for auto-deletion
- cachedFile := &filecache.CachedFile{
- FileID: fileID,
- UserID: userID,
- ConversationID: conversationID,
- FileHash: *fileHash,
- Filename: fileHeader.Filename,
- MimeType: mimeType,
- Size: fileHeader.Size,
- FilePath: filePath, // Track disk location
- UploadedAt: time.Now(),
- }
- h.fileCache.Store(cachedFile)
-
- log.Printf("✅ [UPLOAD] Image uploaded successfully: %s (user: %s, size: %d bytes)", savedFilename, userID, fileHeader.Size)
-
- // Build file URL
- fileURL := fmt.Sprintf("/uploads/%s", savedFilename)
-
- return c.Status(fiber.StatusCreated).JSON(UploadResponse{
- FileID: fileID,
- URL: fileURL,
- MimeType: mimeType,
- Size: fileHeader.Size,
- Filename: fileHeader.Filename,
- Hash: fileHash.String(),
- })
-}
-
-// handleDataFileUpload processes CSV/Excel/JSON files for E2B tools
-func (h *UploadHandler) handleDataFileUpload(c *fiber.Ctx, fileID, userID string, fileHeader *multipart.FileHeader, fileData []byte, mimeType string, fileHash *security.Hash) error {
- // Get conversation_id from form (may be empty)
- conversationID := c.FormValue("conversation_id")
-
- // Generate filename with extension
- ext := filepath.Ext(fileHeader.Filename)
- if ext == "" {
- ext = h.getExtensionFromMimeType(mimeType)
- }
- savedFilename := fileID + ext
- filePath := filepath.Join(h.uploadDir, savedFilename)
-
- // Save file to disk with restricted permissions
- if err := os.WriteFile(filePath, fileData, 0600); err != nil {
- log.Printf("❌ [UPLOAD] Failed to save data file: %v", err)
- return c.Status(fiber.StatusInternalServerError).JSON(fiber.Map{
- "error": "Failed to save file",
- })
- }
-
- // Register file in cache for auto-deletion
- cachedFile := &filecache.CachedFile{
- FileID: fileID,
- UserID: userID,
- ConversationID: conversationID,
- FileHash: *fileHash,
- Filename: fileHeader.Filename,
- MimeType: mimeType,
- Size: fileHeader.Size,
- FilePath: filePath,
- UploadedAt: time.Now(),
- }
- h.fileCache.Store(cachedFile)
-
- // Parse CSV preview if it's a CSV file
- var dataPreview *CSVPreview
- if mimeType == "text/csv" || strings.HasSuffix(strings.ToLower(fileHeader.Filename), ".csv") {
- dataPreview = h.parseCSVPreview(fileData)
- }
-
- log.Printf("✅ [UPLOAD] Data file uploaded successfully: %s (user: %s, size: %d bytes, type: %s)", savedFilename, userID, fileHeader.Size, mimeType)
-
- // Build file URL
- fileURL := fmt.Sprintf("/uploads/%s", savedFilename)
-
- return c.Status(fiber.StatusCreated).JSON(UploadResponse{
- FileID: fileID,
- URL: fileURL,
- MimeType: mimeType,
- Size: fileHeader.Size,
- Filename: fileHeader.Filename,
- Hash: fileHash.String(),
- DataPreview: dataPreview,
- })
-}
-
-// handleAudioUpload processes audio files for Whisper transcription
-func (h *UploadHandler) handleAudioUpload(c *fiber.Ctx, fileID, userID string, fileHeader *multipart.FileHeader, fileData []byte, mimeType string, fileHash *security.Hash) error {
- // Get conversation_id from form (may be empty)
- conversationID := c.FormValue("conversation_id")
-
- log.Printf("🎵 [UPLOAD] Processing audio: %s (user: %s, size: %d bytes, type: %s)", fileHeader.Filename, userID, len(fileData), mimeType)
-
- // Generate filename with extension
- ext := filepath.Ext(fileHeader.Filename)
- if ext == "" {
- ext = h.getExtensionFromMimeType(mimeType)
- }
- savedFilename := fileID + ext
- filePath := filepath.Join(h.uploadDir, savedFilename)
-
- // Save audio to disk with restricted permissions
- if err := os.WriteFile(filePath, fileData, 0600); err != nil {
- log.Printf("❌ [UPLOAD] Failed to save audio file: %v", err)
- return c.Status(fiber.StatusInternalServerError).JSON(fiber.Map{
- "error": "Failed to save file",
- })
- }
-
- // Register file in cache for auto-deletion
- cachedFile := &filecache.CachedFile{
- FileID: fileID,
- UserID: userID,
- ConversationID: conversationID,
- FileHash: *fileHash,
- Filename: fileHeader.Filename,
- MimeType: mimeType,
- Size: fileHeader.Size,
- FilePath: filePath,
- UploadedAt: time.Now(),
- }
- h.fileCache.Store(cachedFile)
-
- log.Printf("✅ [UPLOAD] Audio file uploaded successfully: %s (user: %s, size: %d bytes)", savedFilename, userID, fileHeader.Size)
-
- // Build file URL
- fileURL := fmt.Sprintf("/uploads/%s", savedFilename)
-
- return c.Status(fiber.StatusCreated).JSON(UploadResponse{
- FileID: fileID,
- URL: fileURL,
- MimeType: mimeType,
- Size: fileHeader.Size,
- Filename: fileHeader.Filename,
- Hash: fileHash.String(),
- })
-}
-
-// parseCSVPreview extracts headers and first rows from CSV data
-func (h *UploadHandler) parseCSVPreview(data []byte) *CSVPreview {
- reader := csv.NewReader(bytes.NewReader(data))
- reader.LazyQuotes = true
- reader.TrimLeadingSpace = true
-
- // Read all records to get total count
- allRecords, err := reader.ReadAll()
- if err != nil || len(allRecords) == 0 {
- log.Printf("⚠️ [UPLOAD] Failed to parse CSV preview: %v", err)
- return nil
- }
-
- // First row is headers
- headers := allRecords[0]
- totalRows := len(allRecords) - 1 // Exclude header row
-
- // Get first 5 rows for preview (excluding header)
- maxPreviewRows := 5
- if totalRows < maxPreviewRows {
- maxPreviewRows = totalRows
- }
-
- previewRows := make([][]string, maxPreviewRows)
- for i := 0; i < maxPreviewRows; i++ {
- previewRows[i] = allRecords[i+1] // +1 to skip header
- }
-
- return &CSVPreview{
- Headers: headers,
- Rows: previewRows,
- RowCount: totalRows,
- ColCount: len(headers),
- }
-}
-
-// detectContentTypeFromData detects MIME type from byte data
-func (h *UploadHandler) detectContentTypeFromData(data []byte, header *multipart.FileHeader) string {
- // Check PDF magic bytes first
- if bytes.HasPrefix(data, []byte("%PDF-")) {
- return "application/pdf"
- }
-
- // Check for ZIP-based Office formats (DOCX, PPTX, XLSX start with PK)
- // These are ZIP files, so we need to check extension
- if bytes.HasPrefix(data, []byte("PK")) {
- ext := strings.ToLower(filepath.Ext(header.Filename))
- switch ext {
- case ".docx":
- return "application/vnd.openxmlformats-officedocument.wordprocessingml.document"
- case ".pptx":
- return "application/vnd.openxmlformats-officedocument.presentationml.presentation"
- case ".xlsx":
- return "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"
- }
- }
-
- // Use http.DetectContentType for other types
- mimeType := http.DetectContentType(data)
-
- // Handle fallback for octet-stream or application/zip
- if mimeType == "application/octet-stream" || mimeType == "application/zip" {
- ext := strings.ToLower(filepath.Ext(header.Filename))
- switch ext {
- case ".pdf":
- return "application/pdf"
- case ".jpg", ".jpeg":
- return "image/jpeg"
- case ".png":
- return "image/png"
- case ".gif":
- return "image/gif"
- case ".webp":
- return "image/webp"
- case ".docx":
- return "application/vnd.openxmlformats-officedocument.wordprocessingml.document"
- case ".pptx":
- return "application/vnd.openxmlformats-officedocument.presentationml.presentation"
- case ".xlsx":
- return "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"
- // Audio formats
- case ".mp3":
- return "audio/mpeg"
- case ".wav":
- return "audio/wav"
- case ".m4a":
- return "audio/mp4"
- case ".webm":
- return "audio/webm"
- case ".ogg":
- return "audio/ogg"
- case ".flac":
- return "audio/flac"
- }
- }
-
- return mimeType
-}
-
-// detectContentType detects the MIME type of the uploaded file
-func (h *UploadHandler) detectContentType(file multipart.File, header *multipart.FileHeader) (string, error) {
- // Read first 512 bytes for content type detection
- buffer := make([]byte, 512)
- n, err := file.Read(buffer)
- if err != nil && err != io.EOF {
- return "", err
- }
-
- // Detect content type
- mimeType := http.DetectContentType(buffer[:n])
-
- // Handle some edge cases where DetectContentType returns generic types
- if mimeType == "application/octet-stream" {
- // Fall back to extension-based detection
- ext := strings.ToLower(filepath.Ext(header.Filename))
- switch ext {
- case ".jpg", ".jpeg":
- mimeType = "image/jpeg"
- case ".png":
- mimeType = "image/png"
- case ".gif":
- mimeType = "image/gif"
- case ".webp":
- mimeType = "image/webp"
- }
- }
-
- return mimeType, nil
-}
-
-// getExtensionFromMimeType returns file extension for a given MIME type
-func (h *UploadHandler) getExtensionFromMimeType(mimeType string) string {
- switch mimeType {
- case "application/pdf":
- return ".pdf"
- case "image/jpeg", "image/jpg":
- return ".jpg"
- case "image/png":
- return ".png"
- case "image/gif":
- return ".gif"
- case "image/webp":
- return ".webp"
- case "text/csv":
- return ".csv"
- case "application/json":
- return ".json"
- case "text/plain":
- return ".txt"
- case "application/vnd.ms-excel":
- return ".xls"
- case "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet":
- return ".xlsx"
- case "application/vnd.openxmlformats-officedocument.wordprocessingml.document":
- return ".docx"
- case "application/vnd.openxmlformats-officedocument.presentationml.presentation":
- return ".pptx"
- // Audio formats
- case "audio/mpeg", "audio/mp3":
- return ".mp3"
- case "audio/wav", "audio/x-wav", "audio/wave":
- return ".wav"
- case "audio/mp4", "audio/x-m4a":
- return ".m4a"
- case "audio/webm":
- return ".webm"
- case "audio/ogg":
- return ".ogg"
- case "audio/flac":
- return ".flac"
- default:
- return ".bin"
- }
-}
-
-// saveFile saves the uploaded file to disk
-func (h *UploadHandler) saveFile(src multipart.File, dst string) error {
- // Create destination file
- out, err := os.Create(dst)
- if err != nil {
- return err
- }
- defer out.Close()
-
- // Copy file contents
- _, err = io.Copy(out, src)
- return err
-}
-
-// CheckFileStatus checks if a file is still available (not expired)
-// This is used by the frontend to validate file references before workflow execution
-func (h *UploadHandler) CheckFileStatus(c *fiber.Ctx) error {
- // Get file ID from URL params
- fileID := c.Params("id")
- if fileID == "" {
- return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{
- "error": "File ID required",
- "available": false,
- "expired": true,
- })
- }
-
- // Check if user is authenticated (optional - for ownership validation)
- userID, _ := c.Locals("user_id").(string)
-
- // Check file cache
- if h.fileCache == nil {
- return c.Status(fiber.StatusInternalServerError).JSON(fiber.Map{
- "error": "File cache service unavailable",
- "available": false,
- "expired": false,
- })
- }
-
- // Try to get file from cache
- var file *filecache.CachedFile
- var err error
-
- if userID != "" && userID != "anonymous" {
- // If authenticated, verify ownership
- file, err = h.fileCache.GetByUser(fileID, userID)
- } else {
- // If not authenticated, just check existence
- var found bool
- file, found = h.fileCache.Get(fileID)
- if !found {
- err = fmt.Errorf("file not found or expired")
- }
- }
-
- if err != nil || file == nil {
- // File not found or expired
- log.Printf("⚠️ [UPLOAD] File status check - file not available: %s (user: %s)", fileID, userID)
- return c.JSON(fiber.Map{
- "file_id": fileID,
- "available": false,
- "expired": true,
- "error": "File not found or has expired. Files are only available for 30 minutes after upload.",
- })
- }
-
- // File is available
- log.Printf("✅ [UPLOAD] File status check - file available: %s (user: %s)", fileID, userID)
- return c.JSON(fiber.Map{
- "file_id": fileID,
- "available": true,
- "expired": false,
- "filename": file.Filename,
- "mime_type": file.MimeType,
- "size": file.Size,
- })
-}
-
-// Delete removes an uploaded file
-func (h *UploadHandler) Delete(c *fiber.Ctx) error {
- // Check authentication
- userID, ok := c.Locals("user_id").(string)
- if !ok || userID == "" || userID == "anonymous" {
- return c.Status(fiber.StatusUnauthorized).JSON(fiber.Map{
- "error": "Authentication required",
- })
- }
-
- // Get file ID from URL params
- fileID := c.Params("id")
- if fileID == "" {
- return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{
- "error": "File ID required",
- })
- }
-
- // SECURITY: Validate fileID to prevent path traversal attacks
- if err := security.ValidateFileID(fileID); err != nil {
- log.Printf("⚠️ [UPLOAD] Invalid file ID in delete request: %v", err)
- return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{
- "error": "Invalid file ID format",
- })
- }
-
- // Find file by ID (try all extensions)
- var filePath string
- extensions := []string{".jpg", ".jpeg", ".png", ".gif", ".webp"}
- for _, ext := range extensions {
- testPath := filepath.Join(h.uploadDir, fileID+ext)
- if _, err := os.Stat(testPath); err == nil {
- filePath = testPath
- break
- }
- }
-
- if filePath == "" {
- return c.Status(fiber.StatusNotFound).JSON(fiber.Map{
- "error": "File not found",
- })
- }
-
- // Delete file
- if err := os.Remove(filePath); err != nil {
- log.Printf("❌ [UPLOAD] Failed to delete file %s: %v", filePath, err)
- return c.Status(fiber.StatusInternalServerError).JSON(fiber.Map{
- "error": "Failed to delete file",
- })
- }
-
- log.Printf("🗑️ [UPLOAD] File deleted: %s (user: %s)", filePath, userID)
-
- return c.JSON(fiber.Map{
- "message": "File deleted successfully",
- })
-}
diff --git a/backend/internal/handlers/user.go b/backend/internal/handlers/user.go
deleted file mode 100644
index f6b52024..00000000
--- a/backend/internal/handlers/user.go
+++ /dev/null
@@ -1,422 +0,0 @@
-package handlers
-
-import (
- "claraverse/internal/filecache"
- "claraverse/internal/models"
- "claraverse/internal/services"
- "context"
- "log"
- "strings"
- "time"
-
- "github.com/gofiber/fiber/v2"
-)
-
-// UserHandler handles user data and GDPR compliance endpoints
-type UserHandler struct {
- chatService *services.ChatService
- userService *services.UserService
- agentService *services.AgentService
- executionService *services.ExecutionService
- apiKeyService *services.APIKeyService
- credentialService *services.CredentialService
- chatSyncService *services.ChatSyncService
- schedulerService *services.SchedulerService
- builderConvService *services.BuilderConversationService
-}
-
-// NewUserHandler creates a new user handler
-func NewUserHandler(chatService *services.ChatService, userService *services.UserService) *UserHandler {
- return &UserHandler{
- chatService: chatService,
- userService: userService,
- }
-}
-
-// SetGDPRServices sets optional services needed for complete GDPR deletion
-func (h *UserHandler) SetGDPRServices(
- agentService *services.AgentService,
- executionService *services.ExecutionService,
- apiKeyService *services.APIKeyService,
- credentialService *services.CredentialService,
- chatSyncService *services.ChatSyncService,
- schedulerService *services.SchedulerService,
- builderConvService *services.BuilderConversationService,
-) {
- h.agentService = agentService
- h.executionService = executionService
- h.apiKeyService = apiKeyService
- h.credentialService = credentialService
- h.chatSyncService = chatSyncService
- h.schedulerService = schedulerService
- h.builderConvService = builderConvService
-}
-
-// GetPreferences returns user preferences
-// GET /api/user/preferences
-func (h *UserHandler) GetPreferences(c *fiber.Ctx) error {
- userID, ok := c.Locals("user_id").(string)
- if !ok || userID == "" {
- return c.Status(fiber.StatusUnauthorized).JSON(fiber.Map{
- "error": "Authentication required",
- })
- }
-
- if h.userService == nil {
- // Fallback if MongoDB not configured
- return c.JSON(models.UserPreferences{
- StoreBuilderChatHistory: true,
- DefaultModelID: "",
- })
- }
-
- prefs, err := h.userService.GetPreferences(c.Context(), userID)
- if err != nil {
- log.Printf("⚠️ Failed to get preferences for user %s: %v", userID, err)
- // Return defaults on error
- return c.JSON(models.UserPreferences{
- StoreBuilderChatHistory: true,
- DefaultModelID: "",
- })
- }
-
- return c.JSON(prefs)
-}
-
-// UpdatePreferences updates user preferences
-// PUT /api/user/preferences
-func (h *UserHandler) UpdatePreferences(c *fiber.Ctx) error {
- userID, ok := c.Locals("user_id").(string)
- if !ok || userID == "" {
- return c.Status(fiber.StatusUnauthorized).JSON(fiber.Map{
- "error": "Authentication required",
- })
- }
-
- if h.userService == nil {
- return c.Status(fiber.StatusServiceUnavailable).JSON(fiber.Map{
- "error": "User service not available",
- })
- }
-
- var req models.UpdateUserPreferencesRequest
- if err := c.BodyParser(&req); err != nil {
- return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{
- "error": "Invalid request body",
- })
- }
-
- prefs, err := h.userService.UpdatePreferences(c.Context(), userID, &req)
- if err != nil {
- log.Printf("❌ Failed to update preferences for user %s: %v", userID, err)
- return c.Status(fiber.StatusInternalServerError).JSON(fiber.Map{
- "error": "Failed to update preferences",
- })
- }
-
- log.Printf("✅ Updated preferences for user %s", userID)
- return c.JSON(prefs)
-}
-
-// MarkWelcomePopupSeen marks the welcome popup as seen
-// POST /api/user/welcome-popup-seen
-func (h *UserHandler) MarkWelcomePopupSeen(c *fiber.Ctx) error {
- userID, ok := c.Locals("user_id").(string)
- if !ok || userID == "" {
- return c.Status(fiber.StatusUnauthorized).JSON(fiber.Map{
- "error": "Authentication required",
- })
- }
-
- if h.userService == nil {
- return c.Status(fiber.StatusServiceUnavailable).JSON(fiber.Map{
- "error": "User service not available",
- })
- }
-
- err := h.userService.MarkWelcomePopupSeen(c.Context(), userID)
- if err != nil {
- log.Printf("❌ Failed to mark welcome popup seen for user %s: %v", userID, err)
- return c.Status(fiber.StatusInternalServerError).JSON(fiber.Map{
- "error": "Failed to update popup status",
- })
- }
-
- log.Printf("✅ Welcome popup marked as seen for user %s", userID)
- return c.JSON(fiber.Map{
- "success": true,
- "message": "Welcome popup marked as seen",
- })
-}
-
-// ExportData exports all user data (GDPR Article 15 & 20 - Right to Access and Portability)
-// GET /api/user/data
-func (h *UserHandler) ExportData(c *fiber.Ctx) error {
- userID, ok := c.Locals("user_id").(string)
- if !ok || userID == "" {
- return c.Status(fiber.StatusUnauthorized).JSON(fiber.Map{
- "error": "Authentication required",
- })
- }
-
- userEmail, _ := c.Locals("user_email").(string)
-
- log.Printf("📦 [GDPR] Data export requested by user: %s", userID)
-
- // Get all conversations for this user
- conversations, err := h.chatService.GetAllConversationsByUser(userID)
- if err != nil {
- log.Printf("❌ Failed to export conversations: %v", err)
- return c.Status(fiber.StatusInternalServerError).JSON(fiber.Map{
- "error": "Failed to export data",
- })
- }
-
- // Get file metadata (uploaded files)
- fileCache := filecache.GetService()
- fileMetadata := fileCache.GetAllFilesByUser(userID)
-
- // Compile all user data
- exportData := fiber.Map{
- "user_id": userID,
- "user_email": userEmail,
- "export_date": time.Now().Format(time.RFC3339),
- "conversations": conversations,
- "uploaded_files": fileMetadata,
- "data_categories": []string{
- "conversations",
- "messages",
- "uploaded_files",
- "user_profile",
- },
- "privacy_notice": "This export contains all personal data we have stored for your account.",
- }
-
- log.Printf("✅ [GDPR] Data exported for user %s: %d conversations, %d files",
- userID, len(conversations), len(fileMetadata))
-
- return c.JSON(exportData)
-}
-
-// DeleteAccountRequest is the request body for account deletion
-type DeleteAccountRequest struct {
- Confirmation string `json:"confirmation"`
-}
-
-// DeleteAccount deletes all user data (GDPR Article 17 - Right to Erasure)
-// DELETE /api/user/account
-// Requires confirmation phrase: "delete my account"
-func (h *UserHandler) DeleteAccount(c *fiber.Ctx) error {
- userID, ok := c.Locals("user_id").(string)
- if !ok || userID == "" {
- return c.Status(fiber.StatusUnauthorized).JSON(fiber.Map{
- "error": "Authentication required",
- })
- }
-
- // Parse and validate confirmation phrase
- var req DeleteAccountRequest
- if err := c.BodyParser(&req); err != nil {
- return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{
- "error": "Invalid request body",
- })
- }
-
- // Validate confirmation phrase (case-insensitive)
- if strings.TrimSpace(strings.ToLower(req.Confirmation)) != "delete my account" {
- return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{
- "error": "Invalid confirmation phrase",
- "required": "delete my account",
- })
- }
-
- userEmail, _ := c.Locals("user_email").(string)
- ctx := context.Background()
-
- log.Printf("🗑️ [GDPR] Account deletion CONFIRMED by user: %s (%s)", userID, userEmail)
-
- // Track deletion results
- deletionResults := fiber.Map{}
-
- // 1. Delete schedules first (they reference agents)
- if h.schedulerService != nil {
- count, err := h.schedulerService.DeleteAllByUser(ctx, userID)
- if err != nil {
- log.Printf("⚠️ [GDPR] Failed to delete schedules: %v", err)
- } else {
- deletionResults["schedules"] = count
- }
- }
-
- // 2. Delete executions (they reference agents)
- if h.executionService != nil {
- count, err := h.executionService.DeleteAllByUser(ctx, userID)
- if err != nil {
- log.Printf("⚠️ [GDPR] Failed to delete executions: %v", err)
- } else {
- deletionResults["executions"] = count
- }
- }
-
- // 3. Delete agents, workflows, and workflow versions
- if h.agentService != nil {
- count, err := h.agentService.DeleteAllByUser(ctx, userID)
- if err != nil {
- log.Printf("⚠️ [GDPR] Failed to delete agents: %v", err)
- } else {
- deletionResults["agents"] = count
- }
- }
-
- // 4. Delete API keys
- if h.apiKeyService != nil {
- count, err := h.apiKeyService.DeleteAllByUser(ctx, userID)
- if err != nil {
- log.Printf("⚠️ [GDPR] Failed to delete API keys: %v", err)
- } else {
- deletionResults["api_keys"] = count
- }
- }
-
- // 5. Delete credentials
- if h.credentialService != nil {
- count, err := h.credentialService.DeleteAllByUser(ctx, userID)
- if err != nil {
- log.Printf("⚠️ [GDPR] Failed to delete credentials: %v", err)
- } else {
- deletionResults["credentials"] = count
- }
- }
-
- // 6. Delete cloud-synced chats
- if h.chatSyncService != nil {
- count, err := h.chatSyncService.DeleteAllUserChats(ctx, userID)
- if err != nil {
- log.Printf("⚠️ [GDPR] Failed to delete synced chats: %v", err)
- } else {
- deletionResults["synced_chats"] = count
- }
- }
-
- // 7. Delete builder conversations
- if h.builderConvService != nil {
- err := h.builderConvService.DeleteConversationsByUser(ctx, userID)
- if err != nil {
- log.Printf("⚠️ [GDPR] Failed to delete builder conversations: %v", err)
- } else {
- deletionResults["builder_conversations"] = "deleted"
- }
- }
-
- // 8. Delete SQL conversations
- if err := h.chatService.DeleteAllConversationsByUser(userID); err != nil {
- log.Printf("⚠️ [GDPR] Failed to delete conversations: %v", err)
- } else {
- deletionResults["conversations"] = "deleted"
- }
-
- // 9. Delete all uploaded files
- fileCache := filecache.GetService()
- deletedFiles, err := fileCache.DeleteAllFilesByUser(userID)
- if err != nil {
- log.Printf("⚠️ [GDPR] Failed to delete some files: %v", err)
- }
- deletionResults["files"] = deletedFiles
-
- // 10. Delete user record (last)
- if h.userService != nil {
- err := h.userService.DeleteUser(ctx, userID)
- if err != nil {
- log.Printf("⚠️ [GDPR] Failed to delete user record: %v", err)
- } else {
- deletionResults["user_record"] = "deleted"
- }
- }
-
- log.Printf("✅ [GDPR] Account deletion completed for user %s", userID)
-
- return c.JSON(fiber.Map{
- "message": "Account and all associated data deleted successfully",
- "user_id": userID,
- "deletion_timestamp": time.Now().Format(time.RFC3339),
- "deleted": deletionResults,
- "retention_note": "Audit logs may be retained for up to 90 days for security purposes.",
- })
-}
-
-// GetPrivacyPolicy returns privacy policy information (GDPR Article 13 - Transparency)
-// GET /api/privacy-policy
-func (h *UserHandler) GetPrivacyPolicy(c *fiber.Ctx) error {
- policy := fiber.Map{
- "service_name": "ClaraVerse",
- "last_updated": "2025-11-18",
-
- "data_collected": []string{
- "User ID (from authentication provider)",
- "Email address (from authentication provider)",
- "Chat messages and conversation history",
- "Uploaded files (images, PDFs, CSV, Excel, JSON, text files)",
- "Usage metadata (timestamps, model selections)",
- },
-
- "legal_basis": "Legitimate interest in providing the service (GDPR Article 6(1)(f))",
-
- "data_retention": fiber.Map{
- "conversations": "30 minutes (automatic deletion)",
- "uploaded_files": "Linked to conversation lifetime (30 minutes) - includes images, PDFs, CSV, Excel, JSON, and all data files",
- "audit_logs": "90 days (security and compliance purposes)",
- },
-
- "third_parties": []fiber.Map{
- {
- "name": "Supabase",
- "purpose": "Authentication and user management",
- "data": []string{"user_id", "email", "authentication tokens"},
- },
- {
- "name": "AI Model Providers",
- "purpose": "Processing chat messages and generating responses",
- "data": []string{"chat messages", "uploaded file content"},
- "note": "Varies based on selected model provider (OpenAI, Anthropic, etc.)",
- },
- },
-
- "user_rights": []fiber.Map{
- {
- "right": "Right to Access (Art. 15)",
- "description": "Download all your personal data",
- "endpoint": "GET /api/user/data",
- },
- {
- "right": "Right to Erasure (Art. 17)",
- "description": "Delete all your personal data",
- "endpoint": "DELETE /api/user/account",
- },
- {
- "right": "Right to Data Portability (Art. 20)",
- "description": "Export your data in machine-readable format (JSON)",
- "endpoint": "GET /api/user/data",
- },
- },
-
- "security_measures": []string{
- "AES-256-GCM encryption for sensitive file content",
- "JWT-based authentication",
- "HTTPS encryption in transit (production)",
- "Automatic data expiration (30 minutes)",
- "Rate limiting and DDoS protection",
- },
-
- "contact": fiber.Map{
- "data_controller": "ClaraVerse Team",
- "email": "privacy@claraverse.com",
- "note": "For privacy inquiries, data requests, or to exercise your rights",
- },
-
- "cookie_policy": "This service uses minimal cookies for authentication purposes only.",
-
- "changes_to_policy": "We will notify users of significant changes via email or in-app notification.",
- }
-
- return c.JSON(policy)
-}
diff --git a/backend/internal/handlers/user_preferences.go b/backend/internal/handlers/user_preferences.go
deleted file mode 100644
index f54469c5..00000000
--- a/backend/internal/handlers/user_preferences.go
+++ /dev/null
@@ -1,89 +0,0 @@
-package handlers
-
-import (
- "claraverse/internal/models"
- "claraverse/internal/services"
-
- "github.com/gofiber/fiber/v2"
-)
-
-// UserPreferencesHandler handles user preferences HTTP requests
-type UserPreferencesHandler struct {
- userService *services.UserService
-}
-
-// NewUserPreferencesHandler creates a new UserPreferencesHandler
-func NewUserPreferencesHandler(userService *services.UserService) *UserPreferencesHandler {
- return &UserPreferencesHandler{userService: userService}
-}
-
-// Get retrieves user preferences
-// GET /api/preferences
-func (h *UserPreferencesHandler) Get(c *fiber.Ctx) error {
- userID, ok := c.Locals("user_id").(string)
- if !ok || userID == "" {
- return c.Status(fiber.StatusUnauthorized).JSON(fiber.Map{
- "error": "Authentication required",
- })
- }
-
- // Get email from context (set by auth middleware)
- email, _ := c.Locals("user_email").(string)
-
- // Try to sync user first (creates if not exists)
- _, err := h.userService.SyncUserFromSupabase(c.Context(), userID, email)
- if err != nil {
- // Log but don't fail - try to get preferences anyway
- println("Warning: Failed to sync user:", err.Error())
- }
-
- prefs, err := h.userService.GetPreferences(c.Context(), userID)
- if err != nil {
- // Return default preferences if user not found
- return c.JSON(models.UserPreferences{
- StoreBuilderChatHistory: true,
- ChatPrivacyMode: "",
- Theme: "dark",
- FontSize: "medium",
- })
- }
-
- return c.JSON(prefs)
-}
-
-// Update updates user preferences
-// PUT /api/preferences
-func (h *UserPreferencesHandler) Update(c *fiber.Ctx) error {
- userID, ok := c.Locals("user_id").(string)
- if !ok || userID == "" {
- return c.Status(fiber.StatusUnauthorized).JSON(fiber.Map{
- "error": "Authentication required",
- })
- }
-
- var req models.UpdateUserPreferencesRequest
- if err := c.BodyParser(&req); err != nil {
- return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{
- "error": "Invalid request body",
- })
- }
-
- // Validate chat privacy mode if provided
- if req.ChatPrivacyMode != nil {
- mode := *req.ChatPrivacyMode
- if mode != models.ChatPrivacyModeLocal && mode != models.ChatPrivacyModeCloud && mode != "" {
- return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{
- "error": "Invalid chat_privacy_mode. Must be 'local' or 'cloud'",
- })
- }
- }
-
- prefs, err := h.userService.UpdatePreferences(c.Context(), userID, &req)
- if err != nil {
- return c.Status(fiber.StatusInternalServerError).JSON(fiber.Map{
- "error": "Failed to update preferences",
- })
- }
-
- return c.JSON(prefs)
-}
diff --git a/backend/internal/handlers/webhook.go b/backend/internal/handlers/webhook.go
deleted file mode 100644
index caa80451..00000000
--- a/backend/internal/handlers/webhook.go
+++ /dev/null
@@ -1,93 +0,0 @@
-package handlers
-
-import (
- "claraverse/internal/services"
- "context"
- "log"
- "net/http"
- "strings"
-
- "github.com/gofiber/fiber/v2"
-)
-
-// WebhookHandler handles DodoPayments webhooks
-type WebhookHandler struct {
- paymentService *services.PaymentService
-}
-
-// NewWebhookHandler creates a new webhook handler
-func NewWebhookHandler(paymentService *services.PaymentService) *WebhookHandler {
- return &WebhookHandler{
- paymentService: paymentService,
- }
-}
-
-// HandleDodoWebhook handles incoming webhooks from DodoPayments
-// POST /api/webhooks/dodo
-// DodoPayments uses Standard Webhooks format with headers:
-// - webhook-id: unique message ID
-// - webhook-signature: v1,
-// - webhook-timestamp: unix timestamp
-func (h *WebhookHandler) HandleDodoWebhook(c *fiber.Ctx) error {
- // Get payload
- payload := c.Body()
- if len(payload) == 0 {
- log.Printf("❌ Webhook missing payload")
- return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{
- "error": "Missing payload",
- })
- }
-
- // Convert Fiber headers to http.Header for SDK
- headers := make(http.Header)
- c.Request().Header.VisitAll(func(key, value []byte) {
- headers.Add(string(key), string(value))
- })
-
- // Verify and parse webhook using SDK
- event, err := h.paymentService.VerifyAndParseWebhook(payload, headers)
- if err != nil {
- log.Printf("❌ Webhook verification failed: %v", err)
-
- // Distinguish between parse errors (400) and auth errors (401)
- if strings.Contains(err.Error(), "parse") || strings.Contains(err.Error(), "invalid character") {
- return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{
- "error": "Invalid payload format: " + err.Error(),
- })
- }
- return c.Status(fiber.StatusUnauthorized).JSON(fiber.Map{
- "error": "Invalid webhook: " + err.Error(),
- })
- }
-
- // Handle event
- ctx := context.Background()
- if err := h.paymentService.HandleWebhookEvent(ctx, event); err != nil {
- log.Printf("❌ Webhook processing error: %v", err)
-
- // Return 200 for idempotency errors (duplicate events) - no retry needed
- if strings.Contains(err.Error(), "already processed") ||
- strings.Contains(err.Error(), "duplicate") {
- log.Printf("⚠️ Duplicate webhook event %s (ID: %s) - already processed", event.Type, event.ID)
- return c.Status(fiber.StatusOK).JSON(fiber.Map{
- "received": true,
- "message": "Event already processed (idempotent)",
- })
- }
-
- // Return 500 for actual processing failures to allow DodoPayments to retry
- return c.Status(fiber.StatusInternalServerError).JSON(fiber.Map{
- "error": "Failed to process webhook",
- "event_id": event.ID,
- "type": event.Type,
- })
- }
-
- log.Printf("✅ Webhook event processed: %s (ID: %s)", event.Type, event.ID)
- return c.Status(fiber.StatusOK).JSON(fiber.Map{
- "received": true,
- "event_id": event.ID,
- "type": event.Type,
- })
-}
-
diff --git a/backend/internal/handlers/webhook_test.go b/backend/internal/handlers/webhook_test.go
deleted file mode 100644
index 16066b0b..00000000
--- a/backend/internal/handlers/webhook_test.go
+++ /dev/null
@@ -1,166 +0,0 @@
-package handlers
-
-import (
- "bytes"
- "claraverse/internal/services"
- "crypto/hmac"
- "crypto/sha256"
- "encoding/hex"
- "net/http/httptest"
- "testing"
-
- "github.com/gofiber/fiber/v2"
-)
-
-func TestWebhookHandler_InvalidSignature(t *testing.T) {
- app := fiber.New()
- paymentService := services.NewPaymentService("", "secret123", "", nil, nil, nil)
- handler := NewWebhookHandler(paymentService)
-
- app.Post("/api/webhooks/dodo", handler.HandleDodoWebhook)
-
- payload := []byte(`{"type":"subscription.active"}`)
- req := httptest.NewRequest("POST", "/api/webhooks/dodo", bytes.NewBuffer(payload))
- req.Header.Set("Content-Type", "application/json")
- req.Header.Set("Webhook-Signature", "invalid_signature")
-
- resp, _ := app.Test(req)
-
- if resp.StatusCode != fiber.StatusUnauthorized {
- t.Errorf("Expected 401, got %d", resp.StatusCode)
- }
-}
-
-func TestWebhookHandler_ValidSignature(t *testing.T) {
- secret := "webhook_secret_123"
- app := fiber.New()
- paymentService := services.NewPaymentService("", secret, "", nil, nil, nil)
- handler := NewWebhookHandler(paymentService)
-
- app.Post("/api/webhooks/dodo", handler.HandleDodoWebhook)
-
- payload := []byte(`{"type":"subscription.active","data":{"subscription_id":"sub_123"}}`)
-
- mac := hmac.New(sha256.New, []byte(secret))
- mac.Write(payload)
- signature := hex.EncodeToString(mac.Sum(nil))
-
- req := httptest.NewRequest("POST", "/api/webhooks/dodo", bytes.NewBuffer(payload))
- req.Header.Set("Content-Type", "application/json")
- req.Header.Set("Webhook-Signature", signature)
-
- resp, _ := app.Test(req)
-
- if resp.StatusCode != fiber.StatusOK {
- t.Errorf("Expected 200, got %d", resp.StatusCode)
- }
-}
-
-func TestWebhookHandler_MissingSignature(t *testing.T) {
- app := fiber.New()
- paymentService := services.NewPaymentService("", "secret", "", nil, nil, nil)
- handler := NewWebhookHandler(paymentService)
-
- app.Post("/api/webhooks/dodo", handler.HandleDodoWebhook)
-
- payload := []byte(`{"type":"subscription.active"}`)
- req := httptest.NewRequest("POST", "/api/webhooks/dodo", bytes.NewBuffer(payload))
- req.Header.Set("Content-Type", "application/json")
- // No signature header
-
- resp, _ := app.Test(req)
-
- if resp.StatusCode != fiber.StatusUnauthorized {
- t.Errorf("Expected 401, got %d", resp.StatusCode)
- }
-}
-
-func TestWebhookHandler_InvalidJSON(t *testing.T) {
- secret := "webhook_secret"
- app := fiber.New()
- paymentService := services.NewPaymentService("", secret, "", nil, nil, nil)
- handler := NewWebhookHandler(paymentService)
-
- app.Post("/api/webhooks/dodo", handler.HandleDodoWebhook)
-
- payload := []byte(`{invalid json}`)
-
- mac := hmac.New(sha256.New, []byte(secret))
- mac.Write(payload)
- signature := hex.EncodeToString(mac.Sum(nil))
-
- req := httptest.NewRequest("POST", "/api/webhooks/dodo", bytes.NewBuffer(payload))
- req.Header.Set("Content-Type", "application/json")
- req.Header.Set("Webhook-Signature", signature)
-
- resp, _ := app.Test(req)
-
- if resp.StatusCode != fiber.StatusBadRequest {
- t.Errorf("Expected 400, got %d", resp.StatusCode)
- }
-}
-
-func TestWebhookHandler_AllEventTypes(t *testing.T) {
- secret := "webhook_secret"
- app := fiber.New()
- paymentService := services.NewPaymentService("", secret, "", nil, nil, nil)
- handler := NewWebhookHandler(paymentService)
-
- app.Post("/api/webhooks/dodo", handler.HandleDodoWebhook)
-
- eventTypes := []string{
- "subscription.active",
- "subscription.updated",
- "subscription.on_hold",
- "subscription.renewed",
- "subscription.cancelled",
- "payment.succeeded",
- "payment.failed",
- }
-
- for _, eventType := range eventTypes {
- t.Run(eventType, func(t *testing.T) {
- payload := []byte(`{"type":"` + eventType + `","data":{},"id":"evt_123"}`)
-
- mac := hmac.New(sha256.New, []byte(secret))
- mac.Write(payload)
- signature := hex.EncodeToString(mac.Sum(nil))
-
- req := httptest.NewRequest("POST", "/api/webhooks/dodo", bytes.NewBuffer(payload))
- req.Header.Set("Content-Type", "application/json")
- req.Header.Set("Webhook-Signature", signature)
-
- resp, _ := app.Test(req)
-
- if resp.StatusCode != fiber.StatusOK {
- t.Errorf("Event %s: expected 200, got %d", eventType, resp.StatusCode)
- }
- })
- }
-}
-
-func TestWebhookHandler_AlternativeSignatureHeader(t *testing.T) {
- secret := "webhook_secret"
- app := fiber.New()
- paymentService := services.NewPaymentService("", secret, "", nil, nil, nil)
- handler := NewWebhookHandler(paymentService)
-
- app.Post("/api/webhooks/dodo", handler.HandleDodoWebhook)
-
- payload := []byte(`{"type":"subscription.active","data":{},"id":"evt_123"}`)
-
- mac := hmac.New(sha256.New, []byte(secret))
- mac.Write(payload)
- signature := hex.EncodeToString(mac.Sum(nil))
-
- req := httptest.NewRequest("POST", "/api/webhooks/dodo", bytes.NewBuffer(payload))
- req.Header.Set("Content-Type", "application/json")
- req.Header.Set("Dodo-Signature", signature) // Alternative header name
-
- resp, _ := app.Test(req)
-
- if resp.StatusCode != fiber.StatusOK {
- t.Errorf("Expected 200 with Dodo-Signature header, got %d", resp.StatusCode)
- }
-}
-
diff --git a/backend/internal/handlers/websocket.go b/backend/internal/handlers/websocket.go
deleted file mode 100644
index 03dff6ae..00000000
--- a/backend/internal/handlers/websocket.go
+++ /dev/null
@@ -1,881 +0,0 @@
-package handlers
-
-import (
- "claraverse/internal/filecache"
- "claraverse/internal/models"
- "claraverse/internal/services"
- "claraverse/internal/utils"
- "context"
- "encoding/json"
- "fmt"
- "log"
- "os"
- "path/filepath"
- "strings"
- "sync"
- "time"
-
- "github.com/gofiber/contrib/websocket"
- "github.com/google/uuid"
-)
-
-// PromptResponse stores a user's response to an interactive prompt
-type PromptResponse struct {
- PromptID string
- UserID string
- Answers map[string]models.InteractiveAnswer
- Skipped bool
- ReceivedAt time.Time
-}
-
-// PromptResponseCache stores prompt responses waiting to be processed
-type PromptResponseCache struct {
- responses map[string]*PromptResponse // promptID -> response
- mutex sync.RWMutex
-}
-
-// WebSocketHandler handles WebSocket connections
-type WebSocketHandler struct {
- connManager *services.ConnectionManager
- chatService *services.ChatService
- analyticsService *services.AnalyticsService // Optional: minimal usage tracking
- usageLimiter *services.UsageLimiterService
- promptCache *PromptResponseCache // Cache for interactive prompt responses
-}
-
-// NewWebSocketHandler creates a new WebSocket handler
-func NewWebSocketHandler(connManager *services.ConnectionManager, chatService *services.ChatService, analyticsService *services.AnalyticsService, usageLimiter *services.UsageLimiterService) *WebSocketHandler {
- return &WebSocketHandler{
- connManager: connManager,
- chatService: chatService,
- analyticsService: analyticsService,
- usageLimiter: usageLimiter,
- promptCache: &PromptResponseCache{
- responses: make(map[string]*PromptResponse),
- },
- }
-}
-
-// Handle handles a new WebSocket connection
-func (h *WebSocketHandler) Handle(c *websocket.Conn) {
- connID := uuid.New().String()
- userID := c.Locals("user_id").(string)
-
- // Create a done channel to signal goroutines to stop
- done := make(chan struct{})
-
- userConn := &models.UserConnection{
- ConnID: connID,
- UserID: userID,
- Conn: c,
- ConversationID: "",
- Messages: make([]map[string]interface{}, 0),
- MessageCount: 0,
- CreatedAt: time.Now(),
- WriteChan: make(chan models.ServerMessage, 100),
- StopChan: make(chan bool, 1),
- // Create a waiter function that tools can use to wait for prompt responses
- PromptWaiter: func(promptID string, timeout time.Duration) (map[string]models.InteractiveAnswer, bool, error) {
- response, err := h.WaitForPromptResponse(promptID, timeout)
- if err != nil {
- return nil, false, err
- }
- return response.Answers, response.Skipped, nil
- },
- }
-
- h.connManager.Add(userConn)
- defer func() {
- close(done) // Signal all goroutines to stop
- h.connManager.Remove(connID)
-
- // Track session end (minimal analytics)
- if h.analyticsService != nil && userConn.ConversationID != "" {
- ctx := context.Background()
- h.analyticsService.TrackChatSessionEnd(ctx, connID, userConn.MessageCount)
- }
- }()
-
- // Configure WebSocket timeouts for long-running operations
- // Set read deadline to 6 minutes (allows for 5 min tool execution + buffer)
- c.SetReadDeadline(time.Now().Add(360 * time.Second))
-
- // Set up ping/pong handlers to keep connection alive during long tool executions
- c.SetPongHandler(func(appData string) error {
- // Reset read deadline on pong received
- c.SetReadDeadline(time.Now().Add(360 * time.Second))
- return nil
- })
-
- // Start ping goroutine to keep connection alive
- go h.pingLoop(userConn, done)
-
- // Start write goroutine
- go h.writeLoop(userConn)
-
- // Send connected message (no conversation_id - that comes from client)
- userConn.WriteChan <- models.ServerMessage{
- Type: "connected",
- Content: "WebSocket connected. Ready to receive messages.",
- }
-
- // Read loop
- h.readLoop(userConn)
-}
-
-// pingLoop sends periodic pings to keep the WebSocket connection alive
-// This is crucial for long-running tool executions (e.g., Python runner up to 5 min)
-func (h *WebSocketHandler) pingLoop(userConn *models.UserConnection, done <-chan struct{}) {
- ticker := time.NewTicker(30 * time.Second) // Send ping every 30 seconds
- defer ticker.Stop()
-
- for {
- select {
- case <-done:
- return
- case <-ticker.C:
- userConn.Mutex.Lock()
- if err := userConn.Conn.WriteControl(websocket.PingMessage, []byte{}, time.Now().Add(10*time.Second)); err != nil {
- log.Printf("⚠️ Ping failed for %s: %v", userConn.ConnID, err)
- userConn.Mutex.Unlock()
- return
- }
- userConn.Mutex.Unlock()
- }
- }
-}
-
-// readLoop handles incoming messages from the client
-func (h *WebSocketHandler) readLoop(userConn *models.UserConnection) {
- defer func() {
- if r := recover(); r != nil {
- log.Printf("❌ Panic in readLoop: %v", r)
- }
- }()
-
- for {
- _, msg, err := userConn.Conn.ReadMessage()
- if err != nil {
- log.Printf("❌ WebSocket read error for %s: %v", userConn.ConnID, err)
- break
- }
-
- // Reset read deadline after successful read
- userConn.Conn.SetReadDeadline(time.Now().Add(360 * time.Second))
-
- var clientMsg models.ClientMessage
- if err := json.Unmarshal(msg, &clientMsg); err != nil {
- log.Printf("⚠️ Invalid message format from %s: %v", userConn.ConnID, err)
- userConn.WriteChan <- models.ServerMessage{
- Type: "error",
- ErrorCode: "invalid_format",
- ErrorMessage: "Invalid message format",
- }
- continue
- }
-
- switch clientMsg.Type {
- case "ping":
- // Respond to client heartbeat immediately
- userConn.WriteChan <- models.ServerMessage{
- Type: "pong",
- }
- case "chat_message":
- h.handleChatMessage(userConn, clientMsg)
- case "new_conversation":
- h.handleNewConversation(userConn, clientMsg)
- case "stop_generation":
- h.handleStopGeneration(userConn)
- case "resume_stream":
- h.handleResumeStream(userConn, clientMsg)
- case "interactive_prompt_response":
- h.handleInteractivePromptResponse(userConn, clientMsg)
- default:
- log.Printf("⚠️ Unknown message type: %s", clientMsg.Type)
- }
- }
-}
-
-// handleChatMessage handles a chat message from the client
-func (h *WebSocketHandler) handleChatMessage(userConn *models.UserConnection, clientMsg models.ClientMessage) {
- // Update conversation ID if provided
- if clientMsg.ConversationID != "" {
- userConn.ConversationID = clientMsg.ConversationID
- }
-
- // Update model ID if provided (platform model selection)
- if clientMsg.ModelID != "" {
- userConn.ModelID = clientMsg.ModelID
- log.Printf("🎯 Model selected for %s: %s", userConn.ConnID, clientMsg.ModelID)
- }
-
- // Update custom config if provided (BYOK)
- if clientMsg.CustomConfig != nil {
- userConn.CustomConfig = clientMsg.CustomConfig
- log.Printf("🔑 BYOK config updated for %s: Model=%s",
- userConn.ConnID, clientMsg.CustomConfig.Model)
- }
-
- // Update system instructions if provided (per-request override)
- if clientMsg.SystemInstructions != "" {
- userConn.SystemInstructions = clientMsg.SystemInstructions
- log.Printf("📝 System instructions updated for %s (length: %d chars)",
- userConn.ConnID, len(clientMsg.SystemInstructions))
- }
-
- // Update disable tools flag (e.g., for agent builder)
- userConn.DisableTools = clientMsg.DisableTools
- if userConn.DisableTools {
- log.Printf("🔒 Tools disabled for %s (agent builder mode)", userConn.ConnID)
- }
-
- // Priority-based history handling: prefer backend cache, fall back to client history
- userConn.Mutex.Lock()
-
- // Step 1: Try to get messages from backend cache first
- var cachedMessages []map[string]interface{}
- if userConn.ConversationID != "" {
- cachedMessages = h.chatService.GetConversationMessages(userConn.ConversationID)
- }
-
- if len(cachedMessages) > 0 {
- // ✅ Cache HIT - backend has valid cache, use it (ignore client history)
- userConn.Messages = cachedMessages
-
- // Count assistant messages from cache
- assistantCount := 0
- for _, msg := range cachedMessages {
- if role, ok := msg["role"].(string); ok && role == "assistant" {
- assistantCount++
- }
- }
- userConn.MessageCount = assistantCount
-
- log.Printf("✅ [CACHE-HIT] Using backend cache for %s: %d messages (%d assistant)",
- userConn.ConversationID, len(cachedMessages), assistantCount)
-
- } else if len(clientMsg.History) > 0 {
- // ❌ Cache MISS - no backend cache, use client history and repopulate
- userConn.Messages = clientMsg.History
-
- // Count assistant messages from client history
- assistantCount := 0
- for _, msg := range clientMsg.History {
- if role, ok := msg["role"].(string); ok && role == "assistant" {
- assistantCount++
- }
- }
- userConn.MessageCount = assistantCount
-
- log.Printf("♻️ [CACHE-MISS] Recreating from client history for %s: %d messages (%d assistant)",
- userConn.ConversationID, len(clientMsg.History), assistantCount)
-
- // Repopulate backend cache from client history
- if userConn.ConversationID != "" {
- h.chatService.SetConversationMessages(userConn.ConversationID, clientMsg.History)
- }
-
- } else {
- // 🆕 New conversation - no cache, no history
- userConn.Messages = make([]map[string]interface{}, 0)
- userConn.MessageCount = 0
-
- log.Printf("🆕 [NEW-CONVERSATION] Starting fresh for %s", userConn.ConversationID)
-
- // Create conversation in database with ownership tracking
- if userConn.ConversationID != "" {
- if err := h.chatService.CreateConversation(userConn.ConversationID, userConn.UserID, "New Conversation"); err != nil {
- log.Printf("⚠️ Failed to create conversation in database: %v", err)
- // Continue anyway - conversation will work from cache
- }
- }
- }
-
- // 🔍 DIAGNOSTIC: Log final state after history handling
- log.Printf("🔍 [DIAGNOSTIC] After history handling - userConn.Messages count: %d, conversationID: %s",
- len(userConn.Messages), userConn.ConversationID)
- if len(userConn.Messages) > 0 {
- firstMsg := userConn.Messages[0]
- lastMsg := userConn.Messages[len(userConn.Messages)-1]
- log.Printf("🔍 [DIAGNOSTIC] First message role: %v, Last message role: %v",
- firstMsg["role"], lastMsg["role"])
- }
-
- userConn.Mutex.Unlock()
-
- // Add user message to conversation
- userConn.Mutex.Lock()
-
- // Build message content based on whether there are attachments
- var messageContent interface{}
- var documentContext strings.Builder
- var dataFileContext strings.Builder
- var expiredFiles []string // Track expired files
-
- if len(clientMsg.Attachments) > 0 {
- // Get file cache service
- fileCache := filecache.GetService()
-
- // Process document attachments first (PDF, DOCX, PPTX)
- for _, att := range clientMsg.Attachments {
- // Check for document files (PDF, DOCX, PPTX)
- isDocument := att.MimeType == "application/pdf" ||
- att.MimeType == "application/vnd.openxmlformats-officedocument.wordprocessingml.document" ||
- att.MimeType == "application/vnd.openxmlformats-officedocument.presentationml.presentation"
-
- if isDocument && att.FileID != "" {
- // Fetch document text from cache
- cachedFile, err := fileCache.GetByUserAndConversation(
- att.FileID,
- userConn.UserID,
- userConn.ConversationID,
- )
-
- if err != nil {
- log.Printf("⚠️ Failed to fetch document file %s: %v", att.FileID, err)
- // Track expired file instead of returning error
- expiredFiles = append(expiredFiles, att.Filename)
- continue
- }
-
- // Build document context
- documentContext.WriteString(fmt.Sprintf("\n\n[Document: %s]\n", att.Filename))
- documentContext.WriteString(fmt.Sprintf("Pages: %d | Words: %d\n\n", cachedFile.PageCount, cachedFile.WordCount))
- documentContext.WriteString(cachedFile.ExtractedText.String())
- documentContext.WriteString("\n---\n")
-
- log.Printf("📄 Injected document context: %s (%d words) for %s", att.Filename, cachedFile.WordCount, userConn.ConnID)
- }
- }
-
- // Process CSV/Excel/JSON/Text data files (for context)
- for _, att := range clientMsg.Attachments {
- // Check if it's a data file (CSV, Excel, JSON, Text)
- isDataFile := att.MimeType == "text/csv" ||
- att.MimeType == "text/plain" ||
- att.MimeType == "application/json" ||
- att.MimeType == "application/vnd.ms-excel" ||
- att.MimeType == "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"
-
- if isDataFile && att.FileID != "" {
- // Fetch data file from cache
- cachedFile, err := fileCache.GetByUserAndConversation(
- att.FileID,
- userConn.UserID,
- userConn.ConversationID,
- )
-
- if err != nil {
- log.Printf("⚠️ Failed to fetch data file %s: %v", att.FileID, err)
- // Track expired file instead of returning error
- expiredFiles = append(expiredFiles, att.Filename)
- continue
- }
-
- // Read file content to get preview (first 10 lines for CSV/text)
- uploadDir := os.Getenv("UPLOAD_DIR")
- if uploadDir == "" {
- uploadDir = "./uploads"
- }
-
- fileContent, err := os.ReadFile(cachedFile.FilePath)
- if err != nil {
- log.Printf("⚠️ Failed to read data file %s: %v", att.FileID, err)
- continue
- }
-
- // Get first 10 lines as preview
- lines := strings.Split(string(fileContent), "\n")
- previewLines := 10
- if len(lines) < previewLines {
- previewLines = len(lines)
- }
- preview := strings.Join(lines[:previewLines], "\n")
-
- // Build data file context
- dataFileContext.WriteString(fmt.Sprintf("\n\n[Data File: %s]\n", att.Filename))
- dataFileContext.WriteString(fmt.Sprintf("File ID: %s\n", att.FileID))
- dataFileContext.WriteString(fmt.Sprintf("Type: %s | Size: %d bytes\n", att.MimeType, cachedFile.Size))
- dataFileContext.WriteString(fmt.Sprintf("\nPreview (first %d lines):\n", previewLines))
- dataFileContext.WriteString("```\n")
- dataFileContext.WriteString(preview)
- dataFileContext.WriteString("\n```\n")
- dataFileContext.WriteString("---\n")
-
- log.Printf("📊 Injected data file context: %s (file_id: %s) for %s", att.Filename, att.FileID, userConn.ConnID)
- }
- }
-
- // Check if we have ACTUAL images (for vision models)
- // CSV/Excel/JSON/Text files should NOT be treated as images
- hasImages := false
- imageRegistry := services.GetImageRegistryService()
- for _, att := range clientMsg.Attachments {
- // Only count as image if Type is "image" AND MimeType starts with "image/"
- isActualImage := att.Type == "image" && strings.HasPrefix(att.MimeType, "image/")
- log.Printf("📎 [ATTACHMENT] Type=%s, MimeType=%s, IsActualImage=%v, Filename=%s",
- att.Type, att.MimeType, isActualImage, att.Filename)
- if isActualImage {
- hasImages = true
-
- // Register image in the image registry for LLM referencing
- if att.FileID != "" && clientMsg.ConversationID != "" {
- handle := imageRegistry.RegisterUploadedImage(
- clientMsg.ConversationID,
- att.FileID,
- att.Filename,
- 0, 0, // Width/height not available here, could be extracted from image if needed
- )
- log.Printf("📸 [IMAGE-REGISTRY] Registered uploaded image as %s (file_id: %s)", handle, att.FileID)
- }
- }
- }
-
- if hasImages {
- // Vision model format - array of content parts
- contentParts := []map[string]interface{}{}
-
- // Build combined text content with PDF and data file contexts
- textContent := clientMsg.Content
- if documentContext.Len() > 0 {
- textContent = documentContext.String() + "\n" + textContent
- }
- if dataFileContext.Len() > 0 {
- textContent = dataFileContext.String() + "\n" + textContent
- }
- // Add user query label
- if documentContext.Len() > 0 || dataFileContext.Len() > 0 {
- textContent = textContent + "\n\nUser query: " + clientMsg.Content
- }
-
- contentParts = append(contentParts, map[string]interface{}{
- "type": "text",
- "text": textContent,
- })
-
- // Add image attachments (only actual images, not CSV/data files)
- imageUtils := utils.NewImageUtils()
- for _, att := range clientMsg.Attachments {
- // Only process actual images (not CSV/Excel/JSON disguised as images)
- isActualImage := att.Type == "image" && strings.HasPrefix(att.MimeType, "image/")
- if isActualImage {
- var imageURL string
-
- // If URL is relative (starts with /uploads/), convert to base64
- if strings.HasPrefix(att.URL, "/uploads/") {
- // Extract filename and build local path
- filename := filepath.Base(att.URL)
- localPath := filepath.Join("./uploads", filename)
-
- // Convert to base64 data URL
- base64URL, err := imageUtils.EncodeToBase64(localPath)
- if err != nil {
- log.Printf("⚠️ Failed to encode image to base64: %v", err)
- // Fall back to original URL
- imageURL = att.URL
- } else {
- imageURL = base64URL
- log.Printf("🔄 Converted local image to base64 (size: %d bytes)", att.Size)
- }
- } else {
- // Already a full URL (http:// or https://)
- imageURL = att.URL
- }
-
- contentParts = append(contentParts, map[string]interface{}{
- "type": "image_url",
- "image_url": map[string]interface{}{
- "url": imageURL,
- },
- })
- }
- }
-
- messageContent = contentParts
- log.Printf("🖼️ Chat message from %s with %d attachment(s)", userConn.ConnID, len(clientMsg.Attachments))
- } else if documentContext.Len() > 0 || dataFileContext.Len() > 0 {
- // Document/Data file message (no images)
- var combinedContext strings.Builder
- if documentContext.Len() > 0 {
- combinedContext.WriteString(documentContext.String())
- }
- if dataFileContext.Len() > 0 {
- combinedContext.WriteString(dataFileContext.String())
- }
- combinedContext.WriteString("\n\nUser query: ")
- combinedContext.WriteString(clientMsg.Content)
- messageContent = combinedContext.String()
- } else {
- // No usable attachments
- messageContent = clientMsg.Content
- }
- } else {
- // Text-only message
- messageContent = clientMsg.Content
- }
-
- userConn.Mutex.Unlock()
-
- // Check message limit before processing
- if h.usageLimiter != nil {
- ctx := context.Background()
- if err := h.usageLimiter.CheckMessageLimit(ctx, userConn.UserID); err != nil {
- if limitErr, ok := err.(*services.LimitExceededError); ok {
- userConn.WriteChan <- models.ServerMessage{
- Type: "limit_exceeded",
- ErrorCode: limitErr.ErrorCode,
- ErrorMessage: limitErr.Message,
- Arguments: map[string]interface{}{
- "limit": limitErr.Limit,
- "used": limitErr.Used,
- "reset_at": limitErr.ResetAt,
- "upgrade_to": limitErr.UpgradeTo,
- },
- }
- log.Printf("⚠️ [LIMIT] Message limit exceeded for user %s: %s", userConn.UserID, limitErr.Message)
- return
- }
- }
-
- // Increment message count (check passed)
- go func() {
- if err := h.usageLimiter.IncrementMessageCount(context.Background(), userConn.UserID); err != nil {
- log.Printf("⚠️ [LIMIT] Failed to increment message count for user %s: %v", userConn.UserID, err)
- }
- }()
- }
-
- // Add user message to conversation cache via ChatService
- h.chatService.AddUserMessage(userConn.ConversationID, messageContent)
-
- log.Printf("💬 Chat message from %s (user: %s, length: %d chars)",
- userConn.ConnID, userConn.UserID, len(clientMsg.Content))
-
- // Send warning if any files have expired
- if len(expiredFiles) > 0 {
- warningMsg := fmt.Sprintf("⚠ Warning: %d file(s) expired and unavailable: %s",
- len(expiredFiles), strings.Join(expiredFiles, ", "))
- log.Printf("⚠️ [FILE-EXPIRED] %s", warningMsg)
-
- userConn.WriteChan <- models.ServerMessage{
- Type: "files_expired",
- ErrorCode: "files_expired",
- ErrorMessage: warningMsg,
- Content: strings.Join(expiredFiles, ", "), // File names as comma-separated string
- }
- }
-
- // Stream response
- go func() {
- if err := h.chatService.StreamChatCompletion(userConn); err != nil {
- log.Printf("❌ Chat completion error: %v", err)
- userConn.WriteChan <- models.ServerMessage{
- Type: "error",
- ErrorCode: "chat_error",
- ErrorMessage: err.Error(),
- }
- }
- }()
-}
-
-// handleNewConversation handles starting a new conversation (clears history)
-func (h *WebSocketHandler) handleNewConversation(userConn *models.UserConnection, clientMsg models.ClientMessage) {
- userConn.Mutex.Lock()
-
- // Clear all conversation history
- userConn.Messages = make([]map[string]interface{}, 0)
- userConn.MessageCount = 0 // Reset message counter for new conversation
-
- // Update conversation ID
- if clientMsg.ConversationID != "" {
- userConn.ConversationID = clientMsg.ConversationID
- } else {
- userConn.ConversationID = uuid.New().String()
- }
-
- // Update model if provided
- if clientMsg.ModelID != "" {
- userConn.ModelID = clientMsg.ModelID
- }
-
- // Update system instructions if provided
- if clientMsg.SystemInstructions != "" {
- userConn.SystemInstructions = clientMsg.SystemInstructions
- }
-
- // Update custom config if provided
- if clientMsg.CustomConfig != nil {
- userConn.CustomConfig = clientMsg.CustomConfig
- }
-
- userConn.Mutex.Unlock()
-
- // Clear conversation cache
- h.chatService.ClearConversation(userConn.ConversationID)
-
- // Create conversation in database with ownership tracking
- if err := h.chatService.CreateConversation(userConn.ConversationID, userConn.UserID, "New Conversation"); err != nil {
- log.Printf("⚠️ Failed to create conversation in database: %v", err)
- // Continue anyway - conversation will work from cache
- }
-
- // Track chat session start (minimal analytics)
- if h.analyticsService != nil {
- ctx := context.Background()
- h.analyticsService.TrackChatSessionStart(ctx, userConn.ConnID, userConn.UserID, userConn.ConversationID)
-
- // Update model info if available
- if userConn.ModelID != "" {
- h.analyticsService.UpdateChatSessionModel(ctx, userConn.ConnID, userConn.ModelID, userConn.DisableTools)
- }
- }
-
- log.Printf("🆕 New conversation started for %s: conversation_id=%s, model=%s",
- userConn.ConnID, userConn.ConversationID, userConn.ModelID)
-
- // Send acknowledgment
- userConn.WriteChan <- models.ServerMessage{
- Type: "conversation_reset",
- ConversationID: userConn.ConversationID,
- Content: "New conversation started",
- }
-}
-
-// handleStopGeneration handles a stop generation request
-func (h *WebSocketHandler) handleStopGeneration(userConn *models.UserConnection) {
- select {
- case userConn.StopChan <- true:
- log.Printf("⏹️ Stop signal sent for %s", userConn.ConnID)
- default:
- log.Printf("⚠️ Stop channel full or closed for %s", userConn.ConnID)
- }
-}
-
-// handleResumeStream handles a request to resume a disconnected stream
-func (h *WebSocketHandler) handleResumeStream(userConn *models.UserConnection, clientMsg models.ClientMessage) {
- conversationID := clientMsg.ConversationID
- if conversationID == "" {
- log.Printf("⚠️ Resume stream request with empty conversation ID from %s", userConn.ConnID)
- userConn.WriteChan <- models.ServerMessage{
- Type: "error",
- ErrorCode: "missing_conversation_id",
- ErrorMessage: "Conversation ID is required for resume",
- }
- return
- }
-
- log.Printf("🔄 [RESUME] Resume stream request for conversation %s from %s", conversationID, userConn.ConnID)
-
- // Get the stream buffer
- streamBuffer := h.chatService.GetStreamBuffer()
- bufferData, err := streamBuffer.GetBufferData(conversationID)
-
- if err != nil {
- // Buffer not found or rate limited
- log.Printf("⚠️ [RESUME] Buffer not available for %s: %v", conversationID, err)
- userConn.WriteChan <- models.ServerMessage{
- Type: "stream_missed",
- ConversationID: conversationID,
- Reason: "expired",
- }
- return
- }
-
- // Validate user owns this buffer
- if bufferData.UserID != userConn.UserID {
- log.Printf("⚠️ [RESUME] User %s attempted to resume buffer owned by %s", userConn.UserID, bufferData.UserID)
- userConn.WriteChan <- models.ServerMessage{
- Type: "stream_missed",
- ConversationID: conversationID,
- Reason: "not_found",
- }
- return
- }
-
- log.Printf("📦 [RESUME] Sending %d buffered chunks (%d bytes), %d pending messages for conversation %s (complete: %v)",
- bufferData.ChunkCount, len(bufferData.CombinedChunks), len(bufferData.PendingMessages), conversationID, bufferData.IsComplete)
-
- // First, replay any pending messages (tool results with artifacts, etc.)
- // These are critical messages that might have been missed during disconnect
- for _, pendingMsg := range bufferData.PendingMessages {
- // Skip already delivered messages (prevents duplicates on rapid reconnects)
- if pendingMsg.Delivered {
- continue
- }
-
- log.Printf("📦 [RESUME] Replaying pending message type=%s tool=%s for conversation %s",
- pendingMsg.Type, pendingMsg.ToolName, conversationID)
-
- // Convert BufferedMessage to ServerMessage with all fields
- serverMsg := models.ServerMessage{
- Type: pendingMsg.Type,
- ToolName: pendingMsg.ToolName,
- ToolDisplayName: pendingMsg.ToolDisplayName,
- ToolIcon: pendingMsg.ToolIcon,
- ToolDescription: pendingMsg.ToolDescription,
- Status: pendingMsg.Status,
- Result: pendingMsg.Result,
- }
-
- // Handle plots (for image artifacts)
- if pendingMsg.Plots != nil {
- if plots, ok := pendingMsg.Plots.([]models.PlotData); ok {
- serverMsg.Plots = plots
- } else {
- log.Printf("⚠️ [RESUME] Failed to cast plots for %s - type: %T", pendingMsg.ToolName, pendingMsg.Plots)
- }
- }
-
- userConn.WriteChan <- serverMsg
- }
-
- // Mark pending messages as delivered (prevents duplicates on next resume)
- streamBuffer.MarkMessagesDelivered(conversationID)
-
- // Send the resume message with all buffered text content
- if len(bufferData.CombinedChunks) > 0 {
- userConn.WriteChan <- models.ServerMessage{
- Type: "stream_resume",
- ConversationID: conversationID,
- Content: bufferData.CombinedChunks,
- IsComplete: bufferData.IsComplete,
- }
- }
-
- // If the stream is complete, also send stream_end
- if bufferData.IsComplete {
- userConn.WriteChan <- models.ServerMessage{
- Type: "stream_end",
- ConversationID: conversationID,
- }
- // Clear the buffer since it's complete and delivered
- streamBuffer.ClearBuffer(conversationID)
- log.Printf("📦 [RESUME] Stream complete, buffer cleared for conversation %s", conversationID)
- } else {
- // Stream still in progress - update connection ID so new chunks go to this connection
- // Note: The stream buffer continues to collect chunks from the ongoing generation
- log.Printf("📦 [RESUME] Stream still in progress for conversation %s", conversationID)
- }
-}
-
-// handleInteractivePromptResponse handles a user's response to an interactive prompt
-func (h *WebSocketHandler) handleInteractivePromptResponse(userConn *models.UserConnection, clientMsg models.ClientMessage) {
- promptID := clientMsg.PromptID
- if promptID == "" {
- log.Printf("⚠️ Interactive prompt response with empty prompt ID from %s", userConn.ConnID)
- userConn.WriteChan <- models.ServerMessage{
- Type: "error",
- ErrorCode: "missing_prompt_id",
- ErrorMessage: "Prompt ID is required",
- }
- return
- }
-
- if clientMsg.Skipped {
- log.Printf("📋 [PROMPT] User %s skipped prompt %s", userConn.UserID, promptID)
- } else {
- log.Printf("📋 [PROMPT] User %s answered prompt %s with %d answers", userConn.UserID, promptID, len(clientMsg.Answers))
-
- // Log each answer for debugging
- for questionID, answer := range clientMsg.Answers {
- log.Printf(" Question %s: %v (is_other: %v)", questionID, answer.Value, answer.IsOther)
- }
- }
-
- // Store the response in cache for the waiting tool execution
- h.promptCache.mutex.Lock()
- h.promptCache.responses[promptID] = &PromptResponse{
- PromptID: promptID,
- UserID: userConn.UserID,
- Answers: clientMsg.Answers,
- Skipped: clientMsg.Skipped,
- ReceivedAt: time.Now(),
- }
- h.promptCache.mutex.Unlock()
-
- log.Printf("✅ [PROMPT] Prompt %s response stored in cache (waiting tool will receive it)", promptID)
-}
-
-// SendInteractivePrompt sends an interactive prompt to the client
-// This can be called from anywhere (e.g., during tool execution) to ask the user questions
-func (h *WebSocketHandler) SendInteractivePrompt(userConn *models.UserConnection, prompt models.ServerMessage) bool {
- if prompt.Type != "interactive_prompt" {
- log.Printf("⚠️ SendInteractivePrompt called with invalid type: %s", prompt.Type)
- return false
- }
-
- if prompt.PromptID == "" {
- log.Printf("⚠️ SendInteractivePrompt called with empty PromptID")
- return false
- }
-
- if len(prompt.Questions) == 0 {
- log.Printf("⚠️ SendInteractivePrompt called with no questions")
- return false
- }
-
- // Set conversation ID
- prompt.ConversationID = userConn.ConversationID
-
- // Send the prompt
- success := userConn.SafeSend(prompt)
- if success {
- log.Printf("📋 [PROMPT] Sent interactive prompt %s with %d questions to user %s",
- prompt.PromptID, len(prompt.Questions), userConn.UserID)
- } else {
- log.Printf("❌ [PROMPT] Failed to send interactive prompt %s to user %s",
- prompt.PromptID, userConn.UserID)
- }
-
- return success
-}
-
-// WaitForPromptResponse waits for a user to respond to an interactive prompt
-// Blocks until response is received or timeout occurs (default 5 minutes)
-func (h *WebSocketHandler) WaitForPromptResponse(promptID string, timeout time.Duration) (*PromptResponse, error) {
- deadline := time.Now().Add(timeout)
- pollInterval := 100 * time.Millisecond
-
- log.Printf("⏳ [PROMPT] Waiting for response to prompt %s (timeout: %v)", promptID, timeout)
-
- for time.Now().Before(deadline) {
- // Check if response exists
- h.promptCache.mutex.RLock()
- response, exists := h.promptCache.responses[promptID]
- h.promptCache.mutex.RUnlock()
-
- if exists {
- // Remove from cache
- h.promptCache.mutex.Lock()
- delete(h.promptCache.responses, promptID)
- h.promptCache.mutex.Unlock()
-
- log.Printf("✅ [PROMPT] Received response for prompt %s after %.2f seconds",
- promptID, time.Since(response.ReceivedAt.Add(-time.Since(response.ReceivedAt))).Seconds())
- return response, nil
- }
-
- // Sleep before next poll
- time.Sleep(pollInterval)
- }
-
- // Timeout
- log.Printf("⏱️ [PROMPT] Timeout waiting for response to prompt %s", promptID)
- return nil, fmt.Errorf("timeout waiting for user response")
-}
-
-// writeLoop handles outgoing messages to the client
-func (h *WebSocketHandler) writeLoop(userConn *models.UserConnection) {
- defer func() {
- if r := recover(); r != nil {
- log.Printf("❌ Panic in writeLoop: %v", r)
- }
- }()
-
- for msg := range userConn.WriteChan {
- if err := userConn.Conn.WriteJSON(msg); err != nil {
- log.Printf("❌ WebSocket write error for %s: %v", userConn.ConnID, err)
- return
- }
- }
-}
diff --git a/backend/internal/handlers/workflow_websocket.go b/backend/internal/handlers/workflow_websocket.go
deleted file mode 100644
index 5546181a..00000000
--- a/backend/internal/handlers/workflow_websocket.go
+++ /dev/null
@@ -1,315 +0,0 @@
-package handlers
-
-import (
- "claraverse/internal/execution"
- "claraverse/internal/middleware"
- "claraverse/internal/models"
- "claraverse/internal/services"
- "context"
- "encoding/json"
- "log"
- "time"
-
- "github.com/gofiber/contrib/websocket"
- "github.com/google/uuid"
- "go.mongodb.org/mongo-driver/bson/primitive"
-)
-
-// WorkflowWebSocketHandler handles WebSocket connections for workflow execution
-type WorkflowWebSocketHandler struct {
- agentService *services.AgentService
- executionService *services.ExecutionService
- workflowEngine *execution.WorkflowEngine
- executionLimiter *middleware.ExecutionLimiter
-}
-
-// NewWorkflowWebSocketHandler creates a new workflow WebSocket handler
-func NewWorkflowWebSocketHandler(
- agentService *services.AgentService,
- workflowEngine *execution.WorkflowEngine,
- executionLimiter *middleware.ExecutionLimiter,
-) *WorkflowWebSocketHandler {
- return &WorkflowWebSocketHandler{
- agentService: agentService,
- workflowEngine: workflowEngine,
- executionLimiter: executionLimiter,
- }
-}
-
-// SetExecutionService sets the execution service (optional, for MongoDB execution tracking)
-func (h *WorkflowWebSocketHandler) SetExecutionService(svc *services.ExecutionService) {
- h.executionService = svc
-}
-
-// WorkflowClientMessage represents a message from the client
-type WorkflowClientMessage struct {
- Type string `json:"type"` // execute_workflow, cancel_execution
- AgentID string `json:"agent_id,omitempty"`
- Input map[string]any `json:"input,omitempty"`
-
- // EnableBlockChecker enables block completion validation (optional)
- // When true, each block is checked to ensure it accomplished its job
- EnableBlockChecker bool `json:"enable_block_checker,omitempty"`
-
- // CheckerModelID is the model to use for block checking (optional)
- // Defaults to gpt-4o-mini for fast, cheap validation
- CheckerModelID string `json:"checker_model_id,omitempty"`
-}
-
-// WorkflowServerMessage represents a message to send to the client
-type WorkflowServerMessage struct {
- Type string `json:"type"` // connected, execution_started, execution_update, execution_complete, error
- ExecutionID string `json:"execution_id,omitempty"`
- BlockID string `json:"block_id,omitempty"`
- Status string `json:"status,omitempty"`
- Inputs map[string]any `json:"inputs,omitempty"`
- Output map[string]any `json:"output,omitempty"`
- FinalOutput map[string]any `json:"final_output,omitempty"`
- Duration int64 `json:"duration_ms,omitempty"`
- Error string `json:"error,omitempty"`
-
- // APIResponse is the standardized, clean response for API consumers
- // This provides a well-structured output with result, artifacts, files, etc.
- APIResponse *models.ExecutionAPIResponse `json:"api_response,omitempty"`
-}
-
-// Handle handles a new WebSocket connection for workflow execution
-func (h *WorkflowWebSocketHandler) Handle(c *websocket.Conn) {
- userID := c.Locals("user_id").(string)
- connID := uuid.New().String()
-
- log.Printf("🔌 [WORKFLOW-WS] New connection: connID=%s, userID=%s", connID, userID)
-
- // Send connected message
- if err := c.WriteJSON(WorkflowServerMessage{
- Type: "connected",
- }); err != nil {
- log.Printf("❌ [WORKFLOW-WS] Failed to send connected message: %v", err)
- return
- }
-
- // Context for cancellation
- ctx, cancel := context.WithCancel(context.Background())
- defer cancel()
-
- // Read loop
- for {
- _, msg, err := c.ReadMessage()
- if err != nil {
- log.Printf("❌ [WORKFLOW-WS] Read error for %s: %v", connID, err)
- break
- }
-
- var clientMsg WorkflowClientMessage
- if err := json.Unmarshal(msg, &clientMsg); err != nil {
- log.Printf("⚠️ [WORKFLOW-WS] Invalid message format from %s: %v", connID, err)
- c.WriteJSON(WorkflowServerMessage{
- Type: "error",
- Error: "Invalid message format",
- })
- continue
- }
-
- switch clientMsg.Type {
- case "execute_workflow":
- h.handleExecuteWorkflow(ctx, c, userID, clientMsg)
- case "cancel_execution":
- cancel()
- ctx, cancel = context.WithCancel(context.Background())
- default:
- log.Printf("⚠️ [WORKFLOW-WS] Unknown message type: %s", clientMsg.Type)
- }
- }
-}
-
-// handleExecuteWorkflow handles a workflow execution request
-func (h *WorkflowWebSocketHandler) handleExecuteWorkflow(
- ctx context.Context,
- c *websocket.Conn,
- userID string,
- msg WorkflowClientMessage,
-) {
- startTime := time.Now()
-
- log.Printf("🔍 [WORKFLOW-WS] Received execute request: AgentID=%s, Input=%+v", msg.AgentID, msg.Input)
-
- // Check daily execution limit
- if h.executionLimiter != nil {
- remaining, err := h.executionLimiter.GetRemainingExecutions(userID)
- if err != nil {
- log.Printf("⚠️ [WORKFLOW-WS] Failed to check execution limit: %v", err)
- // Continue on error, don't block execution
- } else if remaining == 0 {
- log.Printf("⚠️ [WORKFLOW-WS] User %s exceeded daily execution limit", userID)
- c.WriteJSON(WorkflowServerMessage{
- Type: "error",
- Error: "Daily execution limit exceeded. Please upgrade your plan or wait until tomorrow.",
- })
- return
- } else if remaining > 0 {
- log.Printf("✅ [WORKFLOW-WS] User %s has %d executions remaining today", userID, remaining)
- }
- }
-
- // Get agent and workflow
- agent, err := h.agentService.GetAgent(msg.AgentID, userID)
- if err != nil {
- log.Printf("❌ [WORKFLOW-WS] Agent not found: %s", msg.AgentID)
- c.WriteJSON(WorkflowServerMessage{
- Type: "error",
- Error: "Agent not found: " + err.Error(),
- })
- return
- }
-
- if agent.Workflow == nil {
- log.Printf("❌ [WORKFLOW-WS] No workflow for agent: %s", msg.AgentID)
- c.WriteJSON(WorkflowServerMessage{
- Type: "error",
- Error: "Agent has no workflow defined",
- })
- return
- }
-
- // Create execution record using ExecutionService (MongoDB) if available
- var execID string
- var execObjectID primitive.ObjectID
-
- if h.executionService != nil {
- execRecord, err := h.executionService.Create(ctx, &services.CreateExecutionRequest{
- AgentID: msg.AgentID,
- UserID: userID,
- WorkflowVersion: agent.Workflow.Version,
- TriggerType: "manual",
- Input: msg.Input,
- })
- if err != nil {
- log.Printf("❌ [WORKFLOW-WS] Failed to create execution: %v", err)
- c.WriteJSON(WorkflowServerMessage{
- Type: "error",
- Error: "Failed to create execution: " + err.Error(),
- })
- return
- }
- execID = execRecord.ID.Hex()
- execObjectID = execRecord.ID
- } else {
- // Fallback: generate a local ID if ExecutionService is not available
- execID = uuid.New().String()
- log.Printf("⚠️ [WORKFLOW-WS] ExecutionService not available, using local ID: %s", execID)
- }
-
- log.Printf("🚀 [WORKFLOW-WS] Starting execution %s for agent %s", execID, msg.AgentID)
-
- // Send execution started message
- c.WriteJSON(WorkflowServerMessage{
- Type: "execution_started",
- ExecutionID: execID,
- })
-
- // Increment execution counter for today
- if h.executionLimiter != nil {
- if err := h.executionLimiter.IncrementCount(userID); err != nil {
- log.Printf("⚠️ [WORKFLOW-WS] Failed to increment execution count: %v", err)
- // Don't fail the execution if counter increment fails
- }
- }
-
- // Create status channel
- statusChan := make(chan models.ExecutionUpdate, 100)
-
- // Start goroutine to forward status updates to WebSocket
- go func() {
- for update := range statusChan {
- update.ExecutionID = execID
- c.WriteJSON(WorkflowServerMessage{
- Type: "execution_update",
- ExecutionID: execID,
- BlockID: update.BlockID,
- Status: update.Status,
- Inputs: update.Inputs,
- Output: update.Output,
- Error: update.Error,
- })
- }
- }()
-
- // Inject user context into input for credential resolution and tool execution
- if msg.Input == nil {
- msg.Input = make(map[string]interface{})
- }
- msg.Input["__user_id__"] = userID
-
- // Build execution options - block checker is controlled by client request
- // When enabled, it validates that each block actually accomplished its job
- execOptions := &execution.ExecutionOptions{
- WorkflowGoal: agent.Description, // Use agent description as workflow goal
- EnableBlockChecker: msg.EnableBlockChecker, // Controlled by frontend toggle
- CheckerModelID: msg.CheckerModelID,
- }
- if msg.EnableBlockChecker {
- log.Printf("🔍 [WORKFLOW-WS] Block checker ENABLED (model: %s)", execOptions.CheckerModelID)
- } else {
- log.Printf("🔍 [WORKFLOW-WS] Block checker DISABLED")
- }
-
- // Execute workflow
- log.Printf("🔍 [WORKFLOW-WS] Executing with input: %+v", msg.Input)
- result, err := h.workflowEngine.ExecuteWithOptions(ctx, agent.Workflow, msg.Input, statusChan, execOptions)
- close(statusChan)
-
- duration := time.Since(startTime).Milliseconds()
-
- if err != nil {
- log.Printf("❌ [WORKFLOW-WS] Execution failed: %v", err)
-
- // Update execution status using ExecutionService if available
- if h.executionService != nil {
- h.executionService.Complete(ctx, execObjectID, &services.ExecutionCompleteRequest{
- Status: "failed",
- Error: err.Error(),
- })
- }
-
- c.WriteJSON(WorkflowServerMessage{
- Type: "execution_complete",
- ExecutionID: execID,
- Status: "failed",
- Duration: duration,
- Error: err.Error(),
- })
- return
- }
-
- // Build the standardized API response
- apiResponse := h.workflowEngine.BuildAPIResponse(result, agent.Workflow, execID, duration)
- apiResponse.Metadata.AgentID = msg.AgentID
-
- // Update execution status in database using ExecutionService if available
- if h.executionService != nil {
- h.executionService.Complete(ctx, execObjectID, &services.ExecutionCompleteRequest{
- Status: result.Status,
- Output: result.Output,
- BlockStates: result.BlockStates,
- Error: result.Error,
- // Store clean API response fields
- Result: apiResponse.Result,
- Artifacts: apiResponse.Artifacts,
- Files: apiResponse.Files,
- })
- }
-
- log.Printf("✅ [WORKFLOW-WS] Execution %s completed: status=%s, duration=%dms, result=%d chars",
- execID, result.Status, duration, len(apiResponse.Result))
-
- // Send completion message with both legacy and new API response format
- c.WriteJSON(WorkflowServerMessage{
- Type: "execution_complete",
- ExecutionID: execID,
- Status: result.Status,
- FinalOutput: result.Output, // Legacy format (backward compat)
- Duration: duration,
- Error: result.Error,
- APIResponse: apiResponse, // New standardized format
- })
-}
diff --git a/backend/internal/jobs/grace_period_checker.go b/backend/internal/jobs/grace_period_checker.go
deleted file mode 100644
index 9993a442..00000000
--- a/backend/internal/jobs/grace_period_checker.go
+++ /dev/null
@@ -1,174 +0,0 @@
-package jobs
-
-import (
- "claraverse/internal/database"
- "claraverse/internal/models"
- "claraverse/internal/services"
- "context"
- "log"
- "time"
-
- "go.mongodb.org/mongo-driver/bson"
- "go.mongodb.org/mongo-driver/mongo"
-)
-
-// GracePeriodChecker handles expiration of grace periods for ON_HOLD subscriptions
-type GracePeriodChecker struct {
- mongoDB *database.MongoDB
- userService *services.UserService
- tierService *services.TierService
- gracePeriodDays int
- subscriptions *mongo.Collection
-}
-
-// NewGracePeriodChecker creates a new grace period checker
-func NewGracePeriodChecker(
- mongoDB *database.MongoDB,
- userService *services.UserService,
- tierService *services.TierService,
- gracePeriodDays int,
-) *GracePeriodChecker {
- if gracePeriodDays == 0 {
- gracePeriodDays = 7 // Default: 7 day grace period
- }
-
- var subscriptions *mongo.Collection
- if mongoDB != nil {
- subscriptions = mongoDB.Database().Collection("subscriptions")
- }
-
- return &GracePeriodChecker{
- mongoDB: mongoDB,
- userService: userService,
- tierService: tierService,
- gracePeriodDays: gracePeriodDays,
- subscriptions: subscriptions,
- }
-}
-
-// Run checks for expired grace periods and downgrades subscriptions
-func (g *GracePeriodChecker) Run(ctx context.Context) error {
- if g.mongoDB == nil || g.userService == nil || g.tierService == nil {
- log.Println("⚠️ [GRACE-PERIOD] Grace period checker disabled (requires MongoDB, UserService, TierService)")
- return nil
- }
-
- log.Println("⏰ [GRACE-PERIOD] Checking for expired grace periods...")
- startTime := time.Now()
-
- // Calculate cutoff date
- cutoffDate := time.Now().UTC().AddDate(0, 0, -g.gracePeriodDays)
-
- // Find subscriptions that are ON_HOLD and past grace period
- filter := bson.M{
- "status": models.SubStatusOnHold,
- "updatedAt": bson.M{
- "$lt": cutoffDate,
- },
- }
-
- cursor, err := g.subscriptions.Find(ctx, filter)
- if err != nil {
- log.Printf("❌ [GRACE-PERIOD] Failed to query subscriptions: %v", err)
- return err
- }
- defer cursor.Close(ctx)
-
- expiredCount := 0
- for cursor.Next(ctx) {
- var sub models.Subscription
- if err := cursor.Decode(&sub); err != nil {
- log.Printf("⚠️ [GRACE-PERIOD] Failed to decode subscription: %v", err)
- continue
- }
-
- if err := g.expireSubscription(ctx, &sub); err != nil {
- log.Printf("⚠️ [GRACE-PERIOD] Failed to expire subscription %s: %v", sub.ID.Hex(), err)
- continue
- }
-
- expiredCount++
- log.Printf("✅ [GRACE-PERIOD] Expired subscription %s for user %s (on hold for %d days)",
- sub.ID.Hex(), sub.UserID, g.gracePeriodDays)
- }
-
- duration := time.Since(startTime)
- log.Printf("✅ [GRACE-PERIOD] Check complete: expired %d subscriptions in %v", expiredCount, duration)
-
- return nil
-}
-
-// expireSubscription downgrades a subscription after grace period expires
-func (g *GracePeriodChecker) expireSubscription(ctx context.Context, sub *models.Subscription) error {
- // Update subscription to cancelled
- now := time.Now()
- update := bson.M{
- "$set": bson.M{
- "tier": models.TierFree,
- "status": models.SubStatusCancelled,
- "cancelledAt": now,
- "updatedAt": now,
- },
- }
-
- _, err := g.subscriptions.UpdateOne(ctx, bson.M{"_id": sub.ID}, update)
- if err != nil {
- return err
- }
-
- // Update user tier to free
- if g.userService != nil {
- err = g.userService.UpdateSubscriptionWithStatus(
- ctx,
- sub.UserID,
- models.TierFree,
- models.SubStatusCancelled,
- nil,
- )
- if err != nil {
- log.Printf("⚠️ [GRACE-PERIOD] Failed to update user tier: %v", err)
- // Don't fail the job if user update fails
- }
- }
-
- // Invalidate tier cache
- if g.tierService != nil {
- g.tierService.InvalidateCache(sub.UserID)
- }
-
- return nil
-}
-
-// GetNextRunTime returns when the job should run next (hourly)
-func (g *GracePeriodChecker) GetNextRunTime() time.Time {
- return time.Now().UTC().Add(1 * time.Hour)
-}
-
-// GetExpiredSubscriptions returns subscriptions past grace period (for monitoring)
-func (g *GracePeriodChecker) GetExpiredSubscriptions(ctx context.Context) ([]models.Subscription, error) {
- if g.mongoDB == nil {
- return nil, nil
- }
-
- cutoffDate := time.Now().UTC().AddDate(0, 0, -g.gracePeriodDays)
-
- filter := bson.M{
- "status": models.SubStatusOnHold,
- "updatedAt": bson.M{
- "$lt": cutoffDate,
- },
- }
-
- cursor, err := g.subscriptions.Find(ctx, filter)
- if err != nil {
- return nil, err
- }
- defer cursor.Close(ctx)
-
- var subscriptions []models.Subscription
- if err := cursor.All(ctx, &subscriptions); err != nil {
- return nil, err
- }
-
- return subscriptions, nil
-}
diff --git a/backend/internal/jobs/promo_expiration_checker.go b/backend/internal/jobs/promo_expiration_checker.go
deleted file mode 100644
index 314649e1..00000000
--- a/backend/internal/jobs/promo_expiration_checker.go
+++ /dev/null
@@ -1,122 +0,0 @@
-package jobs
-
-import (
- "claraverse/internal/database"
- "claraverse/internal/models"
- "claraverse/internal/services"
- "context"
- "log"
- "time"
-
- "go.mongodb.org/mongo-driver/bson"
-)
-
-// PromoExpirationChecker handles expiration of promotional pro subscriptions
-type PromoExpirationChecker struct {
- mongoDB *database.MongoDB
- userService *services.UserService
- tierService *services.TierService
-}
-
-// NewPromoExpirationChecker creates a new promo expiration checker
-func NewPromoExpirationChecker(
- mongoDB *database.MongoDB,
- userService *services.UserService,
- tierService *services.TierService,
-) *PromoExpirationChecker {
- return &PromoExpirationChecker{
- mongoDB: mongoDB,
- userService: userService,
- tierService: tierService,
- }
-}
-
-// Run checks for expired promotional subscriptions and downgrades users
-func (p *PromoExpirationChecker) Run(ctx context.Context) error {
- if p.mongoDB == nil || p.userService == nil || p.tierService == nil {
- log.Println("⚠️ [PROMO-EXPIRATION] Promo expiration checker disabled (requires MongoDB, UserService, TierService)")
- return nil
- }
-
- log.Println("⏰ [PROMO-EXPIRATION] Checking for expired promotional subscriptions...")
- startTime := time.Now()
-
- // Find users collection
- collection := p.mongoDB.Database().Collection("users")
-
- // Find promo users with expired subscriptions
- // Criteria:
- // - subscriptionTier = "pro"
- // - subscriptionExpiresAt < now
- // - dodoSubscriptionId is empty (not a paid subscriber)
- filter := bson.M{
- "subscriptionTier": models.TierPro,
- "subscriptionExpiresAt": bson.M{
- "$lt": time.Now().UTC(),
- },
- "$or": []bson.M{
- {"dodoSubscriptionId": ""},
- {"dodoSubscriptionId": bson.M{"$exists": false}},
- },
- }
-
- cursor, err := collection.Find(ctx, filter)
- if err != nil {
- log.Printf("❌ [PROMO-EXPIRATION] Failed to query users: %v", err)
- return err
- }
- defer cursor.Close(ctx)
-
- expiredCount := 0
- for cursor.Next(ctx) {
- var user models.User
- if err := cursor.Decode(&user); err != nil {
- log.Printf("⚠️ [PROMO-EXPIRATION] Failed to decode user: %v", err)
- continue
- }
-
- if err := p.expirePromoSubscription(ctx, &user); err != nil {
- log.Printf("⚠️ [PROMO-EXPIRATION] Failed to expire promo for user %s: %v", user.SupabaseUserID, err)
- continue
- }
-
- expiredCount++
- log.Printf("✅ [PROMO-EXPIRATION] Expired promo subscription for user %s (promo ended %v ago)",
- user.SupabaseUserID, time.Since(*user.SubscriptionExpiresAt).Round(time.Hour))
- }
-
- duration := time.Since(startTime)
- log.Printf("✅ [PROMO-EXPIRATION] Check complete: expired %d promotional subscriptions in %v", expiredCount, duration)
-
- return nil
-}
-
-// expirePromoSubscription downgrades a user from promo pro to free tier
-func (p *PromoExpirationChecker) expirePromoSubscription(ctx context.Context, user *models.User) error {
- // Update user to free tier with cancelled status
- collection := p.mongoDB.Database().Collection("users")
-
- update := bson.M{
- "$set": bson.M{
- "subscriptionTier": models.TierFree,
- "subscriptionStatus": models.SubStatusCancelled,
- },
- }
-
- _, err := collection.UpdateOne(ctx, bson.M{"_id": user.ID}, update)
- if err != nil {
- return err
- }
-
- // Invalidate tier cache so user immediately sees free tier on next request
- if p.tierService != nil {
- p.tierService.InvalidateCache(user.SupabaseUserID)
- }
-
- return nil
-}
-
-// GetNextRunTime returns when the job should run next (hourly)
-func (p *PromoExpirationChecker) GetNextRunTime() time.Time {
- return time.Now().UTC().Add(1 * time.Hour)
-}
diff --git a/backend/internal/jobs/retention_cleanup.go b/backend/internal/jobs/retention_cleanup.go
deleted file mode 100644
index 144909b4..00000000
--- a/backend/internal/jobs/retention_cleanup.go
+++ /dev/null
@@ -1,193 +0,0 @@
-package jobs
-
-import (
- "claraverse/internal/database"
- "claraverse/internal/services"
- "context"
- "log"
- "time"
-
- "go.mongodb.org/mongo-driver/bson"
- "go.mongodb.org/mongo-driver/bson/primitive"
-)
-
-// RetentionCleanupJob handles deletion of old execution data based on tier retention limits
-type RetentionCleanupJob struct {
- mongoDB *database.MongoDB
- tierService *services.TierService
- executions interface{} // Will be *mongo.Collection
-}
-
-// NewRetentionCleanupJob creates a new retention cleanup job
-func NewRetentionCleanupJob(mongoDB *database.MongoDB, tierService *services.TierService) *RetentionCleanupJob {
- var executions interface{}
- if mongoDB != nil {
- executions = mongoDB.Database().Collection("executions")
- }
-
- return &RetentionCleanupJob{
- mongoDB: mongoDB,
- tierService: tierService,
- executions: executions,
- }
-}
-
-// Run executes the retention cleanup for all users
-func (j *RetentionCleanupJob) Run(ctx context.Context) error {
- if j.mongoDB == nil || j.tierService == nil {
- log.Println("⚠️ [RETENTION] Retention cleanup disabled (requires MongoDB and TierService)")
- return nil
- }
-
- log.Println("🗑️ [RETENTION] Starting execution retention cleanup...")
- startTime := time.Now()
-
- // Get all unique user IDs from executions collection
- userIDs, err := j.getUniqueUserIDs(ctx)
- if err != nil {
- log.Printf("❌ [RETENTION] Failed to get user IDs: %v", err)
- return err
- }
-
- log.Printf("🔍 [RETENTION] Found %d users with executions", len(userIDs))
-
- totalDeleted := 0
- for _, userID := range userIDs {
- deleted, err := j.cleanupUserExecutions(ctx, userID)
- if err != nil {
- log.Printf("⚠️ [RETENTION] Failed to cleanup executions for user %s: %v", userID, err)
- continue
- }
- if deleted > 0 {
- totalDeleted += deleted
- log.Printf("✅ [RETENTION] Deleted %d old executions for user %s", deleted, userID)
- }
- }
-
- duration := time.Since(startTime)
- log.Printf("✅ [RETENTION] Cleanup complete: deleted %d executions in %v", totalDeleted, duration)
-
- return nil
-}
-
-// getUniqueUserIDs returns all unique user IDs that have executions
-func (j *RetentionCleanupJob) getUniqueUserIDs(ctx context.Context) ([]string, error) {
- collection := j.executions.(interface {
- Distinct(context.Context, string, interface{}) ([]interface{}, error)
- })
-
- results, err := collection.Distinct(ctx, "userId", bson.M{})
- if err != nil {
- return nil, err
- }
-
- userIDs := make([]string, 0, len(results))
- for _, result := range results {
- if userID, ok := result.(string); ok {
- userIDs = append(userIDs, userID)
- }
- }
-
- return userIDs, nil
-}
-
-// cleanupUserExecutions deletes old executions for a specific user based on their tier retention
-func (j *RetentionCleanupJob) cleanupUserExecutions(ctx context.Context, userID string) (int, error) {
- // Get user's tier limits
- limits := j.tierService.GetLimits(ctx, userID)
-
- // Calculate cutoff date
- cutoffDate := time.Now().UTC().AddDate(0, 0, -limits.RetentionDays)
-
- // Delete executions older than retention period
- collection := j.executions.(interface {
- DeleteMany(context.Context, interface{}) (interface{ DeletedCount() int64 }, error)
- })
-
- filter := bson.M{
- "userId": userID,
- "createdAt": bson.M{
- "$lt": cutoffDate,
- },
- }
-
- result, err := collection.DeleteMany(ctx, filter)
- if err != nil {
- return 0, err
- }
-
- return int(result.DeletedCount()), nil
-}
-
-// GetNextRunTime returns when the job should run next (daily at 2 AM UTC)
-func (j *RetentionCleanupJob) GetNextRunTime() time.Time {
- now := time.Now().UTC()
-
- // Schedule for 2 AM UTC
- nextRun := time.Date(now.Year(), now.Month(), now.Day(), 2, 0, 0, 0, time.UTC)
-
- // If we've passed 2 AM today, schedule for tomorrow
- if now.After(nextRun) {
- nextRun = nextRun.Add(24 * time.Hour)
- }
-
- return nextRun
-}
-
-// GetStats returns statistics about execution retention
-func (j *RetentionCleanupJob) GetStats(ctx context.Context, userID string) (*RetentionStats, error) {
- if j.mongoDB == nil || j.tierService == nil {
- return nil, nil
- }
-
- limits := j.tierService.GetLimits(ctx, userID)
- cutoffDate := time.Now().UTC().AddDate(0, 0, -limits.RetentionDays)
-
- collection := j.executions.(interface {
- CountDocuments(context.Context, interface{}) (int64, error)
- })
-
- // Count total executions
- total, err := collection.CountDocuments(ctx, bson.M{"userId": userID})
- if err != nil {
- return nil, err
- }
-
- // Count old executions (will be deleted)
- old, err := collection.CountDocuments(ctx, bson.M{
- "userId": userID,
- "createdAt": bson.M{
- "$lt": cutoffDate,
- },
- })
- if err != nil {
- return nil, err
- }
-
- return &RetentionStats{
- TotalExecutions: int(total),
- RetainedExecutions: int(total - old),
- DeletableExecutions: int(old),
- RetentionDays: limits.RetentionDays,
- CutoffDate: cutoffDate,
- }, nil
-}
-
-// RetentionStats provides statistics about execution retention
-type RetentionStats struct {
- TotalExecutions int `json:"total_executions"`
- RetainedExecutions int `json:"retained_executions"`
- DeletableExecutions int `json:"deletable_executions"`
- RetentionDays int `json:"retention_days"`
- CutoffDate time.Time `json:"cutoff_date"`
-}
-
-// ExecutionRetention model for tracking deletion events (audit log)
-type ExecutionRetention struct {
- ID primitive.ObjectID `bson:"_id,omitempty" json:"id"`
- UserID string `bson:"userId" json:"user_id"`
- DeletedAt time.Time `bson:"deletedAt" json:"deleted_at"`
- Count int `bson:"count" json:"count"`
- RetentionDays int `bson:"retentionDays" json:"retention_days"`
- CutoffDate time.Time `bson:"cutoffDate" json:"cutoff_date"`
-}
diff --git a/backend/internal/jobs/scheduler.go b/backend/internal/jobs/scheduler.go
deleted file mode 100644
index c23f6bdc..00000000
--- a/backend/internal/jobs/scheduler.go
+++ /dev/null
@@ -1,171 +0,0 @@
-package jobs
-
-import (
- "context"
- "log"
- "sync"
- "time"
-)
-
-// Job interface that all scheduled jobs must implement
-type Job interface {
- Run(ctx context.Context) error
- GetNextRunTime() time.Time
-}
-
-// JobScheduler manages and runs scheduled jobs
-type JobScheduler struct {
- jobs map[string]Job
- timers map[string]*time.Timer
- ctx context.Context
- cancel context.CancelFunc
- wg sync.WaitGroup
- mu sync.Mutex
- running bool
-}
-
-// NewJobScheduler creates a new job scheduler
-func NewJobScheduler() *JobScheduler {
- ctx, cancel := context.WithCancel(context.Background())
- return &JobScheduler{
- jobs: make(map[string]Job),
- timers: make(map[string]*time.Timer),
- ctx: ctx,
- cancel: cancel,
- }
-}
-
-// Register adds a job to the scheduler
-func (s *JobScheduler) Register(name string, job Job) {
- s.mu.Lock()
- defer s.mu.Unlock()
-
- s.jobs[name] = job
- log.Printf("✅ [SCHEDULER] Registered job: %s", name)
-}
-
-// Start begins running all registered jobs
-func (s *JobScheduler) Start() error {
- s.mu.Lock()
- defer s.mu.Unlock()
-
- if s.running {
- return nil
- }
-
- s.running = true
- log.Printf("🚀 [SCHEDULER] Starting job scheduler with %d jobs", len(s.jobs))
-
- // Schedule all jobs
- for name, job := range s.jobs {
- s.scheduleJob(name, job)
- }
-
- return nil
-}
-
-// scheduleJob schedules a single job
-func (s *JobScheduler) scheduleJob(name string, job Job) {
- nextRun := job.GetNextRunTime()
- duration := time.Until(nextRun)
-
- log.Printf("⏰ [SCHEDULER] Job '%s' scheduled to run at %s (in %v)",
- name, nextRun.Format(time.RFC3339), duration)
-
- timer := time.AfterFunc(duration, func() {
- s.runJob(name, job)
- })
-
- s.timers[name] = timer
-}
-
-// runJob executes a job and reschedules it
-func (s *JobScheduler) runJob(name string, job Job) {
- s.wg.Add(1)
- defer s.wg.Done()
-
- log.Printf("▶️ [SCHEDULER] Running job: %s", name)
- startTime := time.Now()
-
- // Run the job
- if err := job.Run(s.ctx); err != nil {
- log.Printf("❌ [SCHEDULER] Job '%s' failed: %v", name, err)
- }
-
- duration := time.Since(startTime)
- log.Printf("✅ [SCHEDULER] Job '%s' completed in %v", name, duration)
-
- // Reschedule the job
- s.mu.Lock()
- defer s.mu.Unlock()
-
- if s.running {
- s.scheduleJob(name, job)
- }
-}
-
-// Stop gracefully stops all jobs
-func (s *JobScheduler) Stop() {
- s.mu.Lock()
- if !s.running {
- s.mu.Unlock()
- return
- }
-
- log.Println("🛑 [SCHEDULER] Stopping job scheduler...")
- s.running = false
-
- // Stop all timers
- for name, timer := range s.timers {
- timer.Stop()
- log.Printf("⏹️ [SCHEDULER] Stopped job: %s", name)
- }
- s.timers = make(map[string]*time.Timer)
-
- s.mu.Unlock()
-
- // Cancel context and wait for running jobs
- s.cancel()
- s.wg.Wait()
-
- log.Println("✅ [SCHEDULER] Job scheduler stopped")
-}
-
-// RunNow immediately runs a specific job (useful for testing)
-func (s *JobScheduler) RunNow(name string) error {
- s.mu.Lock()
- job, exists := s.jobs[name]
- s.mu.Unlock()
-
- if !exists {
- log.Printf("⚠️ [SCHEDULER] Job '%s' not found", name)
- return nil
- }
-
- log.Printf("🚀 [SCHEDULER] Running job '%s' immediately", name)
- return job.Run(s.ctx)
-}
-
-// GetStatus returns the status of all jobs
-func (s *JobScheduler) GetStatus() map[string]JobStatus {
- s.mu.Lock()
- defer s.mu.Unlock()
-
- status := make(map[string]JobStatus)
- for name, job := range s.jobs {
- status[name] = JobStatus{
- Name: name,
- NextRunTime: job.GetNextRunTime(),
- Registered: true,
- }
- }
-
- return status
-}
-
-// JobStatus represents the status of a job
-type JobStatus struct {
- Name string `json:"name"`
- NextRunTime time.Time `json:"next_run_time"`
- Registered bool `json:"registered"`
-}
diff --git a/backend/internal/middleware/admin.go b/backend/internal/middleware/admin.go
deleted file mode 100644
index 93cdd16b..00000000
--- a/backend/internal/middleware/admin.go
+++ /dev/null
@@ -1,60 +0,0 @@
-package middleware
-
-import (
- "claraverse/internal/config"
- "log"
-
- "github.com/gofiber/fiber/v2"
-)
-
-// AdminMiddleware checks if the authenticated user is a superadmin
-func AdminMiddleware(cfg *config.Config) fiber.Handler {
- return func(c *fiber.Ctx) error {
- userID, ok := c.Locals("user_id").(string)
- if !ok || userID == "" {
- return c.Status(fiber.StatusUnauthorized).JSON(fiber.Map{
- "error": "Authentication required",
- })
- }
-
- // Check role from context (set by JWT middleware)
- role, hasRole := c.Locals("user_role").(string)
-
- // First check role field (preferred method)
- if hasRole && role == "admin" {
- c.Locals("is_superadmin", true)
- log.Printf("✅ Admin access granted to user %s (role: %s)", userID, role)
- return c.Next()
- }
-
- // Fallback: Check if user is in superadmin list (legacy support)
- isSuperadmin := false
- for _, adminID := range cfg.SuperadminUserIDs {
- if adminID == userID {
- isSuperadmin = true
- break
- }
- }
-
- if !isSuperadmin {
- log.Printf("🚫 Non-admin user %s attempted to access admin endpoint (role: %s)", userID, role)
- return c.Status(fiber.StatusForbidden).JSON(fiber.Map{
- "error": "Admin access required",
- })
- }
-
- // Store admin flag for handlers to use
- c.Locals("is_superadmin", true)
- return c.Next()
- }
-}
-
-// IsSuperadmin is a helper function to check if a user ID is a superadmin
-func IsSuperadmin(userID string, cfg *config.Config) bool {
- for _, adminID := range cfg.SuperadminUserIDs {
- if adminID == userID {
- return true
- }
- }
- return false
-}
diff --git a/backend/internal/middleware/apikey.go b/backend/internal/middleware/apikey.go
deleted file mode 100644
index f6b6d413..00000000
--- a/backend/internal/middleware/apikey.go
+++ /dev/null
@@ -1,235 +0,0 @@
-package middleware
-
-import (
- "claraverse/internal/models"
- "claraverse/internal/services"
- "log"
- "strconv"
-
- "github.com/gofiber/fiber/v2"
-)
-
-// APIKeyMiddleware validates API keys for programmatic access
-// This middleware checks the X-API-Key header and validates the key
-func APIKeyMiddleware(apiKeyService *services.APIKeyService) fiber.Handler {
- return func(c *fiber.Ctx) error {
- // Get API key from header
- apiKey := c.Get("X-API-Key")
- if apiKey == "" {
- return c.Status(fiber.StatusUnauthorized).JSON(fiber.Map{
- "error": "Missing API key. Include X-API-Key header.",
- })
- }
-
- // Validate the key
- key, err := apiKeyService.ValidateKey(c.Context(), apiKey)
- if err != nil {
- log.Printf("❌ [APIKEY-AUTH] Invalid key attempt: %v", err)
- return c.Status(fiber.StatusUnauthorized).JSON(fiber.Map{
- "error": "Invalid or expired API key",
- })
- }
-
- // Check if revoked
- if key.IsRevoked() {
- return c.Status(fiber.StatusUnauthorized).JSON(fiber.Map{
- "error": "API key has been revoked",
- })
- }
-
- // Store key info in context for handlers
- c.Locals("api_key", key)
- c.Locals("user_id", key.UserID)
- c.Locals("auth_type", "api_key")
-
- log.Printf("🔑 [APIKEY-AUTH] Authenticated via API key %s (user: %s)", key.KeyPrefix, key.UserID)
-
- return c.Next()
- }
-}
-
-// APIKeyOrJWTMiddleware allows authentication via either API key or JWT
-// Checks API key first, falls back to JWT
-func APIKeyOrJWTMiddleware(apiKeyService *services.APIKeyService, jwtMiddleware fiber.Handler) fiber.Handler {
- return func(c *fiber.Ctx) error {
- // Check for API key first
- apiKey := c.Get("X-API-Key")
- if apiKey != "" {
- // Validate the key
- key, err := apiKeyService.ValidateKey(c.Context(), apiKey)
- if err != nil {
- log.Printf("❌ [APIKEY-AUTH] Invalid key: %v", err)
- return c.Status(fiber.StatusUnauthorized).JSON(fiber.Map{
- "error": "Invalid or expired API key",
- })
- }
-
- if key.IsRevoked() {
- return c.Status(fiber.StatusUnauthorized).JSON(fiber.Map{
- "error": "API key has been revoked",
- })
- }
-
- // Authenticated via API key
- c.Locals("api_key", key)
- c.Locals("user_id", key.UserID)
- c.Locals("auth_type", "api_key")
-
- log.Printf("🔑 [APIKEY-AUTH] Authenticated via API key %s", key.KeyPrefix)
- return c.Next()
- }
-
- // Fall back to JWT middleware
- return jwtMiddleware(c)
- }
-}
-
-// RequireScope middleware checks if the authenticated API key has a specific scope
-func RequireScope(scope string) fiber.Handler {
- return func(c *fiber.Ctx) error {
- // Check if authenticated via API key
- authType, _ := c.Locals("auth_type").(string)
- if authType != "api_key" {
- // JWT auth - allow through (JWT has full access)
- return c.Next()
- }
-
- // Get API key from context
- key, ok := c.Locals("api_key").(*models.APIKey)
- if !ok {
- // Fallback - allow through
- return c.Next()
- }
-
- // Check if key has required scope
- if !key.HasScope(scope) {
- log.Printf("🚫 [APIKEY-AUTH] Scope denied: %s (has: %v)", scope, key.Scopes)
- return c.Status(fiber.StatusForbidden).JSON(fiber.Map{
- "error": "API key does not have required permission: " + scope,
- })
- }
-
- return c.Next()
- }
-}
-
-// RequireExecuteScope middleware checks if the API key can execute a specific agent
-func RequireExecuteScope(agentIDParam string) fiber.Handler {
- return func(c *fiber.Ctx) error {
- // Check if authenticated via API key
- authType, _ := c.Locals("auth_type").(string)
- if authType != "api_key" {
- // JWT auth - allow through
- return c.Next()
- }
-
- // Get agent ID from params
- agentID := c.Params(agentIDParam)
- if agentID == "" {
- return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{
- "error": "Missing agent ID",
- })
- }
-
- // Get API key from context
- key, ok := c.Locals("api_key").(*models.APIKey)
- if !ok {
- return c.Next()
- }
-
- // Check if key can execute this agent
- if !key.HasExecuteScope(agentID) {
- log.Printf("🚫 [APIKEY-AUTH] Execute denied for agent %s (has: %v)", agentID, key.Scopes)
- return c.Status(fiber.StatusForbidden).JSON(fiber.Map{
- "error": "API key cannot execute this agent",
- })
- }
-
- return c.Next()
- }
-}
-
-// RateLimitByAPIKey applies rate limiting based on API key limits
-func RateLimitByAPIKey(redisService *services.RedisService) fiber.Handler {
- return func(c *fiber.Ctx) error {
- // Only apply to API key auth
- authType, _ := c.Locals("auth_type").(string)
- if authType != "api_key" {
- return c.Next()
- }
-
- // Get API key from context
- key, ok := c.Locals("api_key").(*models.APIKey)
- if !ok || redisService == nil {
- return c.Next()
- }
-
- // Get rate limits
- var perMinute, perHour int64 = 60, 1000 // Defaults
- if key.RateLimit != nil {
- perMinute = key.RateLimit.RequestsPerMinute
- perHour = key.RateLimit.RequestsPerHour
- }
-
- // Check rate limits using Redis
- keyPrefix := key.KeyPrefix
- ctx := c.Context()
-
- // Check per-minute limit
- minuteKey := "ratelimit:minute:" + keyPrefix
- count, err := redisService.Incr(ctx, minuteKey)
- if err != nil {
- log.Printf("⚠️ [RATE-LIMIT] Redis error: %v", err)
- return c.Next() // Allow on error
- }
-
- if count == 1 {
- // First request, set expiry
- redisService.Expire(ctx, minuteKey, 60)
- }
-
- if count > perMinute {
- return c.Status(fiber.StatusTooManyRequests).JSON(fiber.Map{
- "error": "Rate limit exceeded (per minute)",
- "retry_after": "60 seconds",
- })
- }
-
- // Check per-hour limit
- hourKey := "ratelimit:hour:" + keyPrefix
- hourCount, err := redisService.Incr(ctx, hourKey)
- if err != nil {
- return c.Next()
- }
-
- if hourCount == 1 {
- redisService.Expire(ctx, hourKey, 3600)
- }
-
- if hourCount > perHour {
- return c.Status(fiber.StatusTooManyRequests).JSON(fiber.Map{
- "error": "Rate limit exceeded (per hour)",
- "retry_after": "3600 seconds",
- })
- }
-
- // Add rate limit headers
- c.Set("X-RateLimit-Limit-Minute", formatInt64(perMinute))
- c.Set("X-RateLimit-Remaining-Minute", formatInt64(max(0, perMinute-count)))
- c.Set("X-RateLimit-Limit-Hour", formatInt64(perHour))
- c.Set("X-RateLimit-Remaining-Hour", formatInt64(max(0, perHour-hourCount)))
-
- return c.Next()
- }
-}
-
-func formatInt64(n int64) string {
- return strconv.FormatInt(n, 10)
-}
-
-func max(a, b int64) int64 {
- if a > b {
- return a
- }
- return b
-}
diff --git a/backend/internal/middleware/auth.go b/backend/internal/middleware/auth.go
deleted file mode 100644
index 10bfa528..00000000
--- a/backend/internal/middleware/auth.go
+++ /dev/null
@@ -1,151 +0,0 @@
-package middleware
-
-import (
- "claraverse/pkg/auth"
- "log"
- "os"
-
- "github.com/gofiber/fiber/v2"
-)
-
-// AuthMiddleware verifies Supabase JWT tokens
-// Supports both Authorization header and query parameter (for WebSocket connections)
-func AuthMiddleware(supabaseAuth *auth.SupabaseAuth) fiber.Handler {
- return func(c *fiber.Ctx) error {
- // SECURITY: DEV_API_KEY bypass has been removed for security reasons.
- // Use proper Supabase authentication or separate development/staging environments.
-
- // Skip auth if Supabase is not configured (development mode ONLY)
- if supabaseAuth.URL == "" {
- environment := os.Getenv("ENVIRONMENT")
-
- // CRITICAL: Never allow auth bypass in production
- if environment == "production" {
- log.Fatal("❌ CRITICAL SECURITY ERROR: Supabase not configured in production environment. Authentication is required.")
- }
-
- // Only allow bypass in development/testing
- if environment != "development" && environment != "testing" && environment != "" {
- return c.Status(fiber.StatusServiceUnavailable).JSON(fiber.Map{
- "error": "Authentication service unavailable",
- })
- }
-
- log.Println("⚠️ Auth skipped: Supabase not configured (development mode)")
- c.Locals("user_id", "dev-user")
- c.Locals("user_email", "dev@localhost")
- c.Locals("user_role", "authenticated")
- return c.Next()
- }
-
- // Try to extract token from multiple sources
- var token string
-
- // 1. Try Authorization header first
- authHeader := c.Get("Authorization")
- if authHeader != "" {
- extractedToken, err := auth.ExtractToken(authHeader)
- if err == nil {
- token = extractedToken
- }
- }
-
- // 2. Try query parameter (for WebSocket connections)
- if token == "" {
- token = c.Query("token")
- }
-
- // No token found
- if token == "" {
- return c.Status(fiber.StatusUnauthorized).JSON(fiber.Map{
- "error": "Missing or invalid authorization token",
- })
- }
-
- // Verify token with Supabase
- user, err := supabaseAuth.VerifyToken(token)
- if err != nil {
- log.Printf("❌ Auth failed: %v", err)
- return c.Status(fiber.StatusUnauthorized).JSON(fiber.Map{
- "error": "Invalid or expired token",
- })
- }
-
- // Store user info in context
- c.Locals("user_id", user.ID)
- c.Locals("user_email", user.Email)
- c.Locals("user_role", user.Role)
-
- log.Printf("✅ Authenticated user: %s (%s)", user.Email, user.ID)
- return c.Next()
- }
-}
-
-// OptionalAuthMiddleware makes authentication optional
-// Supports both Authorization header and query parameter (for WebSocket)
-func OptionalAuthMiddleware(supabaseAuth *auth.SupabaseAuth) fiber.Handler {
- return func(c *fiber.Ctx) error {
- // Try to extract token from multiple sources
- var token string
-
- // 1. Try Authorization header first
- authHeader := c.Get("Authorization")
- if authHeader != "" {
- extractedToken, err := auth.ExtractToken(authHeader)
- if err == nil {
- token = extractedToken
- }
- }
-
- // 2. Try query parameter (for WebSocket connections)
- if token == "" {
- token = c.Query("token")
- }
-
- // If no token found, proceed as anonymous
- if token == "" {
- c.Locals("user_id", "anonymous")
- log.Println("🔓 Anonymous connection")
- return c.Next()
- }
-
- // Skip validation if Supabase is not configured (development mode ONLY)
- if supabaseAuth == nil || supabaseAuth.URL == "" {
- environment := os.Getenv("ENVIRONMENT")
-
- // CRITICAL: Never allow auth bypass in production
- if environment == "production" {
- log.Fatal("❌ CRITICAL SECURITY ERROR: Supabase not configured in production environment. Authentication is required.")
- }
-
- // Only allow in development/testing
- if environment != "development" && environment != "testing" && environment != "" {
- c.Locals("user_id", "anonymous")
- log.Println("⚠️ Supabase unavailable, proceeding as anonymous")
- return c.Next()
- }
-
- c.Locals("user_id", "dev-user-" + token[:min(8, len(token))])
- c.Locals("user_email", "dev@localhost")
- c.Locals("user_role", "authenticated")
- log.Println("⚠️ Auth skipped: Supabase not configured (dev mode)")
- return c.Next()
- }
-
- // Verify token with Supabase
- user, err := supabaseAuth.VerifyToken(token)
- if err != nil {
- log.Printf("⚠️ Token validation failed: %v (continuing as anonymous)", err)
- c.Locals("user_id", "anonymous")
- return c.Next()
- }
-
- // Store authenticated user info
- c.Locals("user_id", user.ID)
- c.Locals("user_email", user.Email)
- c.Locals("user_role", user.Role)
-
- log.Printf("✅ Authenticated user: %s (%s)", user.Email, user.ID)
- return c.Next()
- }
-}
diff --git a/backend/internal/middleware/auth_local.go b/backend/internal/middleware/auth_local.go
deleted file mode 100644
index 737b5508..00000000
--- a/backend/internal/middleware/auth_local.go
+++ /dev/null
@@ -1,152 +0,0 @@
-package middleware
-
-import (
- "claraverse/pkg/auth"
- "log"
- "os"
-
- "github.com/gofiber/fiber/v2"
-)
-
-// LocalAuthMiddleware verifies local JWT tokens
-// Supports both Authorization header and query parameter (for WebSocket connections)
-func LocalAuthMiddleware(jwtAuth *auth.LocalJWTAuth) fiber.Handler {
- return func(c *fiber.Ctx) error {
- // Skip auth if JWT secret is not configured (development mode ONLY)
- environment := os.Getenv("ENVIRONMENT")
-
- if jwtAuth == nil {
- // CRITICAL: Never allow auth bypass in production
- if environment == "production" {
- log.Fatal("❌ CRITICAL SECURITY ERROR: JWT auth not configured in production environment. Authentication is required.")
- }
-
- // Only allow bypass in development/testing
- if environment != "development" && environment != "testing" && environment != "" {
- return c.Status(fiber.StatusServiceUnavailable).JSON(fiber.Map{
- "error": "Authentication service unavailable",
- })
- }
-
- log.Println("⚠️ Auth skipped: JWT not configured (development mode)")
- c.Locals("user_id", "dev-user")
- c.Locals("user_email", "dev@localhost")
- c.Locals("user_role", "user")
- return c.Next()
- }
-
- // Try to extract token from multiple sources
- var token string
-
- // 1. Try Authorization header first
- authHeader := c.Get("Authorization")
- if authHeader != "" {
- extractedToken, err := auth.ExtractToken(authHeader)
- if err == nil {
- token = extractedToken
- }
- }
-
- // 2. Try query parameter (for WebSocket connections)
- if token == "" {
- token = c.Query("token")
- }
-
- // No token found
- if token == "" {
- return c.Status(fiber.StatusUnauthorized).JSON(fiber.Map{
- "error": "Missing or invalid authorization token",
- })
- }
-
- // Verify JWT token
- user, err := jwtAuth.VerifyAccessToken(token)
- if err != nil {
- log.Printf("❌ Auth failed: %v", err)
- return c.Status(fiber.StatusUnauthorized).JSON(fiber.Map{
- "error": "Invalid or expired token",
- })
- }
-
- // Store user info in context
- c.Locals("user_id", user.ID)
- c.Locals("user_email", user.Email)
- c.Locals("user_role", user.Role)
-
- log.Printf("✅ Authenticated user: %s (%s)", user.Email, user.ID)
- return c.Next()
- }
-}
-
-// OptionalLocalAuthMiddleware makes authentication optional
-// Supports both Authorization header and query parameter (for WebSocket)
-func OptionalLocalAuthMiddleware(jwtAuth *auth.LocalJWTAuth) fiber.Handler {
- return func(c *fiber.Ctx) error {
- // Try to extract token from multiple sources
- var token string
-
- // 1. Try Authorization header first
- authHeader := c.Get("Authorization")
- if authHeader != "" {
- extractedToken, err := auth.ExtractToken(authHeader)
- if err == nil {
- token = extractedToken
- }
- }
-
- // 2. Try query parameter (for WebSocket connections)
- if token == "" {
- token = c.Query("token")
- }
-
- // If no token found, proceed as anonymous
- if token == "" {
- c.Locals("user_id", "anonymous")
- log.Println("🔓 Anonymous connection")
- return c.Next()
- }
-
- // Skip validation if JWT auth is not configured (development mode ONLY)
- environment := os.Getenv("ENVIRONMENT")
-
- if jwtAuth == nil {
- // CRITICAL: Never allow auth bypass in production
- if environment == "production" {
- log.Fatal("❌ CRITICAL SECURITY ERROR: JWT auth not configured in production environment")
- }
-
- // Only allow in development/testing
- if environment != "development" && environment != "testing" && environment != "" {
- c.Locals("user_id", "anonymous")
- log.Println("⚠️ JWT unavailable, proceeding as anonymous")
- return c.Next()
- }
-
- c.Locals("user_id", "dev-user")
- c.Locals("user_email", "dev@localhost")
- c.Locals("user_role", "user")
- log.Println("⚠️ Auth skipped: JWT not configured (dev mode)")
- return c.Next()
- }
-
- // Verify JWT token
- user, err := jwtAuth.VerifyAccessToken(token)
- if err != nil {
- log.Printf("⚠️ Token validation failed: %v (continuing as anonymous)", err)
- c.Locals("user_id", "anonymous")
- return c.Next()
- }
-
- // Store authenticated user info
- c.Locals("user_id", user.ID)
- c.Locals("user_email", user.Email)
- c.Locals("user_role", user.Role)
-
- log.Printf("✅ Authenticated user: %s (%s)", user.Email, user.ID)
- return c.Next()
- }
-}
-
-// RateLimitedAuthMiddleware combines rate limiting with authentication
-// Rate limit: 5 attempts per 15 minutes per IP
-// Note: This function is currently unused. Apply rate limiting separately in routes if needed.
diff --git a/backend/internal/middleware/execution_limiter.go b/backend/internal/middleware/execution_limiter.go
deleted file mode 100644
index 14b1e004..00000000
--- a/backend/internal/middleware/execution_limiter.go
+++ /dev/null
@@ -1,149 +0,0 @@
-package middleware
-
-import (
- "claraverse/internal/services"
- "context"
- "fmt"
- "log"
- "time"
-
- "github.com/gofiber/fiber/v2"
- "github.com/redis/go-redis/v9"
-)
-
-// ExecutionLimiter middleware checks daily execution limits based on user tier
-type ExecutionLimiter struct {
- tierService *services.TierService
- redis *redis.Client
-}
-
-// NewExecutionLimiter creates a new execution limiter middleware
-func NewExecutionLimiter(tierService *services.TierService, redisClient *redis.Client) *ExecutionLimiter {
- return &ExecutionLimiter{
- tierService: tierService,
- redis: redisClient,
- }
-}
-
-// CheckLimit verifies if user can execute another workflow today
-func (el *ExecutionLimiter) CheckLimit(c *fiber.Ctx) error {
- userID := c.Locals("user_id")
- if userID == nil {
- return c.Status(fiber.StatusUnauthorized).JSON(fiber.Map{
- "error": "Unauthorized",
- })
- }
-
- userIDStr, ok := userID.(string)
- if !ok {
- return c.Status(fiber.StatusInternalServerError).JSON(fiber.Map{
- "error": "Invalid user ID",
- })
- }
-
- ctx := context.Background()
-
- // Get user's tier limits
- limits := el.tierService.GetLimits(ctx, userIDStr)
-
- // If unlimited executions, skip check
- if limits.MaxExecutionsPerDay == -1 {
- return c.Next()
- }
-
- // Get today's execution count from Redis
- today := time.Now().UTC().Format("2006-01-02")
- key := fmt.Sprintf("executions:%s:%s", userIDStr, today)
-
- // Get current count
- count, err := el.redis.Get(ctx, key).Int64()
- if err != nil && err != redis.Nil {
- log.Printf("⚠️ Failed to get execution count from Redis: %v", err)
- // On Redis error, allow execution but log warning
- return c.Next()
- }
-
- // Check if limit exceeded
- if count >= limits.MaxExecutionsPerDay {
- return c.Status(fiber.StatusTooManyRequests).JSON(fiber.Map{
- "error": "Daily execution limit exceeded",
- "limit": limits.MaxExecutionsPerDay,
- "used": count,
- "reset_at": getNextMidnightUTC(),
- })
- }
-
- // Store current count in context for post-execution increment
- c.Locals("execution_count_key", key)
-
- return c.Next()
-}
-
-// IncrementCount increments the execution counter after successful execution start
-func (el *ExecutionLimiter) IncrementCount(userID string) error {
- if el.redis == nil {
- return nil // Redis not available, skip increment
- }
-
- ctx := context.Background()
- today := time.Now().UTC().Format("2006-01-02")
- key := fmt.Sprintf("executions:%s:%s", userID, today)
-
- // Increment counter
- pipe := el.redis.Pipeline()
- pipe.Incr(ctx, key)
-
- // Set expiry to end of day + 1 day (to allow historical querying)
- midnight := getNextMidnightUTC()
- expiryDuration := time.Until(midnight) + 24*time.Hour
- pipe.Expire(ctx, key, expiryDuration)
-
- _, err := pipe.Exec(ctx)
- if err != nil {
- log.Printf("⚠️ Failed to increment execution count: %v", err)
- return err
- }
-
- log.Printf("✅ Incremented execution count for user %s (key: %s)", userID, key)
- return nil
-}
-
-// GetRemainingExecutions returns how many executions user has left today
-func (el *ExecutionLimiter) GetRemainingExecutions(userID string) (int64, error) {
- if el.redis == nil {
- return -1, nil // Redis not available, return unlimited
- }
-
- ctx := context.Background()
-
- // Get user's tier limits
- limits := el.tierService.GetLimits(ctx, userID)
- if limits.MaxExecutionsPerDay == -1 {
- return -1, nil // Unlimited
- }
-
- // Get today's count
- today := time.Now().UTC().Format("2006-01-02")
- key := fmt.Sprintf("executions:%s:%s", userID, today)
-
- count, err := el.redis.Get(ctx, key).Int64()
- if err == redis.Nil {
- return limits.MaxExecutionsPerDay, nil // No executions today
- }
- if err != nil {
- return -1, err
- }
-
- remaining := limits.MaxExecutionsPerDay - count
- if remaining < 0 {
- return 0, nil
- }
-
- return remaining, nil
-}
-
-// getNextMidnightUTC returns the next midnight UTC
-func getNextMidnightUTC() time.Time {
- now := time.Now().UTC()
- return time.Date(now.Year(), now.Month(), now.Day()+1, 0, 0, 0, 0, time.UTC)
-}
diff --git a/backend/internal/middleware/ratelimit.go b/backend/internal/middleware/ratelimit.go
deleted file mode 100644
index 7b881543..00000000
--- a/backend/internal/middleware/ratelimit.go
+++ /dev/null
@@ -1,245 +0,0 @@
-package middleware
-
-import (
- "log"
- "os"
- "strconv"
- "time"
-
- "github.com/gofiber/fiber/v2"
- "github.com/gofiber/fiber/v2/middleware/limiter"
-)
-
-// RateLimitConfig holds rate limiting settings
-type RateLimitConfig struct {
- // Global limits (per IP)
- GlobalAPIMax int // Max requests per minute for all API endpoints
- GlobalAPIExpiration time.Duration // Expiration window
-
- // Public endpoint limits (per IP) - read-only, cacheable
- PublicReadMax int
- PublicReadExpiration time.Duration
-
- // Authenticated endpoint limits (per user ID)
- AuthenticatedMax int
- AuthenticatedExpiration time.Duration
-
- // Heavy operation limits
- TranscribeMax int
- TranscribeExpiration time.Duration
-
- // WebSocket/Connection limits (per IP)
- WebSocketMax int
- WebSocketExpiration time.Duration
-
- // Image proxy limits (per IP) - can be abused for bandwidth
- ImageProxyMax int
- ImageProxyExpiration time.Duration
-}
-
-// DefaultRateLimitConfig returns production-safe defaults
-// These are designed to prevent abuse while avoiding false positives
-func DefaultRateLimitConfig() *RateLimitConfig {
- return &RateLimitConfig{
- // Global: 200/min = ~3.3 req/sec - very generous for normal use
- GlobalAPIMax: 200,
- GlobalAPIExpiration: 1 * time.Minute,
-
- // Public read endpoints: 120/min = 2 req/sec
- PublicReadMax: 120,
- PublicReadExpiration: 1 * time.Minute,
-
- // Authenticated operations: 60/min = 1 req/sec average
- AuthenticatedMax: 60,
- AuthenticatedExpiration: 1 * time.Minute,
-
- // Transcription: 10/min (expensive GPU operation)
- TranscribeMax: 10,
- TranscribeExpiration: 1 * time.Minute,
-
- // WebSocket: 20 connections/min in production
- WebSocketMax: 20,
- WebSocketExpiration: 1 * time.Minute,
-
- // Image proxy: 60/min (bandwidth protection)
- ImageProxyMax: 60,
- ImageProxyExpiration: 1 * time.Minute,
- }
-}
-
-// LoadRateLimitConfig loads config from environment variables with defaults
-func LoadRateLimitConfig() *RateLimitConfig {
- config := DefaultRateLimitConfig()
-
- // Allow environment overrides for tuning
- if v := os.Getenv("RATE_LIMIT_GLOBAL_API"); v != "" {
- if n, err := strconv.Atoi(v); err == nil && n > 0 {
- config.GlobalAPIMax = n
- }
- }
-
- if v := os.Getenv("RATE_LIMIT_PUBLIC_READ"); v != "" {
- if n, err := strconv.Atoi(v); err == nil && n > 0 {
- config.PublicReadMax = n
- }
- }
-
- if v := os.Getenv("RATE_LIMIT_AUTHENTICATED"); v != "" {
- if n, err := strconv.Atoi(v); err == nil && n > 0 {
- config.AuthenticatedMax = n
- }
- }
-
- if v := os.Getenv("RATE_LIMIT_WEBSOCKET"); v != "" {
- if n, err := strconv.Atoi(v); err == nil && n > 0 {
- config.WebSocketMax = n
- }
- }
-
- if v := os.Getenv("RATE_LIMIT_IMAGE_PROXY"); v != "" {
- if n, err := strconv.Atoi(v); err == nil && n > 0 {
- config.ImageProxyMax = n
- }
- }
-
- // Development mode: more lenient limits
- if os.Getenv("ENVIRONMENT") == "development" {
- config.GlobalAPIMax = 1000 // Very high for dev
- config.WebSocketMax = 100 // Keep high for dev
- config.ImageProxyMax = 200 // More lenient
- log.Println("⚠️ [RATE-LIMIT] Development mode: using relaxed rate limits")
- }
-
- return config
-}
-
-// GlobalAPIRateLimiter creates a rate limiter for all API requests
-// This is the first line of defense against DDoS
-func GlobalAPIRateLimiter(config *RateLimitConfig) fiber.Handler {
- return limiter.New(limiter.Config{
- Max: config.GlobalAPIMax,
- Expiration: config.GlobalAPIExpiration,
- KeyGenerator: func(c *fiber.Ctx) string {
- return "global:" + c.IP()
- },
- LimitReached: func(c *fiber.Ctx) error {
- log.Printf("🚫 [RATE-LIMIT] Global limit reached for IP: %s", c.IP())
- return c.Status(fiber.StatusTooManyRequests).JSON(fiber.Map{
- "error": "Too many requests. Please slow down.",
- "retry_after": int(config.GlobalAPIExpiration.Seconds()),
- })
- },
- SkipFailedRequests: false,
- SkipSuccessfulRequests: false,
- })
-}
-
-// PublicReadRateLimiter for public read-only endpoints
-func PublicReadRateLimiter(config *RateLimitConfig) fiber.Handler {
- return limiter.New(limiter.Config{
- Max: config.PublicReadMax,
- Expiration: config.PublicReadExpiration,
- KeyGenerator: func(c *fiber.Ctx) string {
- return "public:" + c.IP()
- },
- LimitReached: func(c *fiber.Ctx) error {
- log.Printf("⚠️ [RATE-LIMIT] Public endpoint limit reached for IP: %s on %s", c.IP(), c.Path())
- return c.Status(fiber.StatusTooManyRequests).JSON(fiber.Map{
- "error": "Too many requests to this endpoint.",
- "retry_after": int(config.PublicReadExpiration.Seconds()),
- })
- },
- })
-}
-
-// AuthenticatedRateLimiter for authenticated endpoints (uses user ID)
-func AuthenticatedRateLimiter(config *RateLimitConfig) fiber.Handler {
- return limiter.New(limiter.Config{
- Max: config.AuthenticatedMax,
- Expiration: config.AuthenticatedExpiration,
- KeyGenerator: func(c *fiber.Ctx) string {
- // Use user ID if available, fall back to IP
- if userID, ok := c.Locals("user_id").(string); ok && userID != "" && userID != "anonymous" {
- return "auth:" + userID
- }
- return "auth-ip:" + c.IP()
- },
- LimitReached: func(c *fiber.Ctx) error {
- userID, _ := c.Locals("user_id").(string)
- log.Printf("⚠️ [RATE-LIMIT] Auth endpoint limit reached for user: %s on %s", userID, c.Path())
- return c.Status(fiber.StatusTooManyRequests).JSON(fiber.Map{
- "error": "Too many requests. Please wait before trying again.",
- "retry_after": int(config.AuthenticatedExpiration.Seconds()),
- })
- },
- })
-}
-
-// TranscribeRateLimiter for expensive audio transcription
-func TranscribeRateLimiter(config *RateLimitConfig) fiber.Handler {
- return limiter.New(limiter.Config{
- Max: config.TranscribeMax,
- Expiration: config.TranscribeExpiration,
- KeyGenerator: func(c *fiber.Ctx) string {
- if userID, ok := c.Locals("user_id").(string); ok && userID != "" && userID != "anonymous" {
- return "transcribe:" + userID
- }
- return "transcribe-ip:" + c.IP()
- },
- LimitReached: func(c *fiber.Ctx) error {
- log.Printf("⚠️ [RATE-LIMIT] Transcription limit reached for: %v", c.Locals("user_id"))
- return c.Status(fiber.StatusTooManyRequests).JSON(fiber.Map{
- "error": "Transcription rate limit reached. Please wait before transcribing more audio.",
- "retry_after": int(config.TranscribeExpiration.Seconds()),
- })
- },
- })
-}
-
-// WebSocketRateLimiter for WebSocket connection attempts
-func WebSocketRateLimiter(config *RateLimitConfig) fiber.Handler {
- return limiter.New(limiter.Config{
- Max: config.WebSocketMax,
- Expiration: config.WebSocketExpiration,
- KeyGenerator: func(c *fiber.Ctx) string {
- return "ws:" + c.IP()
- },
- LimitReached: func(c *fiber.Ctx) error {
- log.Printf("🚫 [RATE-LIMIT] WebSocket connection limit reached for IP: %s", c.IP())
- return c.Status(fiber.StatusTooManyRequests).JSON(fiber.Map{
- "error": "Too many connection attempts. Please wait before reconnecting.",
- "retry_after": int(config.WebSocketExpiration.Seconds()),
- })
- },
- })
-}
-
-// ImageProxyRateLimiter for image proxy requests (bandwidth protection)
-func ImageProxyRateLimiter(config *RateLimitConfig) fiber.Handler {
- return limiter.New(limiter.Config{
- Max: config.ImageProxyMax,
- Expiration: config.ImageProxyExpiration,
- KeyGenerator: func(c *fiber.Ctx) string {
- return "imgproxy:" + c.IP()
- },
- LimitReached: func(c *fiber.Ctx) error {
- log.Printf("⚠️ [RATE-LIMIT] Image proxy limit reached for IP: %s", c.IP())
- return c.Status(fiber.StatusTooManyRequests).JSON(fiber.Map{
- "error": "Too many image requests. Please wait.",
- "retry_after": int(config.ImageProxyExpiration.Seconds()),
- })
- },
- })
-}
-
-// SlowdownMiddleware adds progressive delays for rapid requests
-// This discourages automated attacks without hard blocking
-func SlowdownMiddleware(threshold int, delay time.Duration) fiber.Handler {
- // Use a simple in-memory counter (for single-instance deployments)
- // For multi-instance, use Redis-backed rate limiting
- return func(c *fiber.Ctx) error {
- // This is a placeholder - the limiter middleware handles the actual limiting
- // This could be enhanced to add progressive delays before hard blocking
- return c.Next()
- }
-}
diff --git a/backend/internal/models/agent.go b/backend/internal/models/agent.go
deleted file mode 100644
index fbd2d61f..00000000
--- a/backend/internal/models/agent.go
+++ /dev/null
@@ -1,380 +0,0 @@
-package models
-
-import "time"
-
-// Agent represents a workflow automation agent
-type Agent struct {
- ID string `json:"id"`
- UserID string `json:"user_id"`
- Name string `json:"name"`
- Description string `json:"description,omitempty"`
- Status string `json:"status"` // draft, active, deployed
- Workflow *Workflow `json:"workflow,omitempty"`
- CreatedAt time.Time `json:"created_at"`
- UpdatedAt time.Time `json:"updated_at"`
-}
-
-// Workflow represents a DAG of blocks for an agent
-type Workflow struct {
- ID string `json:"id"`
- AgentID string `json:"agent_id"`
- Blocks []Block `json:"blocks"`
- Connections []Connection `json:"connections"`
- Variables []Variable `json:"variables"`
- Version int `json:"version"`
- CreatedAt time.Time `json:"created_at,omitempty"`
- UpdatedAt time.Time `json:"updated_at,omitempty"`
-}
-
-// Block represents a single node in the workflow DAG
-type Block struct {
- ID string `json:"id"`
- NormalizedID string `json:"normalizedId"` // Normalized name for variable interpolation (e.g., "search-latest-news")
- Type string `json:"type"` // llm_inference, tool_execution, webhook, variable, python_tool
- Name string `json:"name"`
- Description string `json:"description,omitempty"`
- Config map[string]any `json:"config"`
- Position Position `json:"position"`
- Timeout int `json:"timeout"` // seconds, default 30
-}
-
-// Position represents x,y coordinates for canvas layout
-type Position struct {
- X float64 `json:"x"`
- Y float64 `json:"y"`
-}
-
-// Connection represents an edge between two blocks
-type Connection struct {
- ID string `json:"id"`
- SourceBlockID string `json:"sourceBlockId"`
- SourceOutput string `json:"sourceOutput,omitempty"`
- TargetBlockID string `json:"targetBlockId"`
- TargetInput string `json:"targetInput,omitempty"`
-}
-
-// Variable represents a workflow-level variable
-type Variable struct {
- Name string `json:"name"`
- Type string `json:"type"` // string, number, boolean, array, object
- DefaultValue any `json:"defaultValue,omitempty"`
-}
-
-// Execution represents a single workflow execution run
-type Execution struct {
- ID string `json:"id"`
- AgentID string `json:"agent_id"`
- WorkflowVersion int `json:"workflow_version"`
- Status string `json:"status"` // pending, running, completed, failed, partial_failure
- Input map[string]any `json:"input,omitempty"`
- Output map[string]any `json:"output,omitempty"`
- BlockStates map[string]*BlockState `json:"block_states,omitempty"`
- StartedAt *time.Time `json:"started_at,omitempty"`
- CompletedAt *time.Time `json:"completed_at,omitempty"`
-}
-
-// BlockState represents the execution state of a single block
-type BlockState struct {
- Status string `json:"status"` // pending, running, completed, failed, skipped
- Inputs map[string]any `json:"inputs,omitempty"`
- Outputs map[string]any `json:"outputs,omitempty"`
- Error string `json:"error,omitempty"`
- StartedAt *time.Time `json:"started_at,omitempty"`
- CompletedAt *time.Time `json:"completed_at,omitempty"`
-
- // Retry tracking
- RetryCount int `json:"retry_count,omitempty"` // Number of retries attempted
- RetryHistory []RetryAttempt `json:"retry_history,omitempty"` // Detailed retry history
-}
-
-// RetryAttempt records a single retry attempt for debugging and monitoring
-type RetryAttempt struct {
- Attempt int `json:"attempt"` // 0-indexed attempt number
- Error string `json:"error"` // Error message from this attempt
- ErrorType string `json:"error_type"` // "timeout", "rate_limit", "server_error", etc.
- Timestamp time.Time `json:"timestamp"` // When the attempt occurred
- Duration int64 `json:"duration_ms"` // How long the attempt took
-}
-
-// ExecutionUpdate is sent via WebSocket to stream execution progress
-type ExecutionUpdate struct {
- Type string `json:"type"` // execution_update
- ExecutionID string `json:"execution_id"`
- BlockID string `json:"block_id"`
- Status string `json:"status"`
- Inputs map[string]any `json:"inputs,omitempty"` // Available inputs for debugging
- Output map[string]any `json:"output,omitempty"`
- Error string `json:"error,omitempty"`
-}
-
-// ExecutionComplete is sent when workflow execution finishes
-type ExecutionComplete struct {
- Type string `json:"type"` // execution_complete
- ExecutionID string `json:"execution_id"`
- Status string `json:"status"` // completed, failed, partial_failure
- FinalOutput map[string]any `json:"final_output,omitempty"`
- Duration int64 `json:"duration_ms"`
-}
-
-// ============================================================================
-// Standardized API Response Types
-// Clean, well-structured output for API consumers
-// ============================================================================
-
-// ExecutionAPIResponse is the standardized response for workflow execution
-// This provides a clean, predictable structure for API consumers
-type ExecutionAPIResponse struct {
- // Status of the execution: completed, failed, partial
- Status string `json:"status"`
-
- // Result contains the primary output from the workflow
- // This is the "answer" - extracted from the final block's response
- Result string `json:"result"`
-
- // Data contains the structured JSON data from the final block (if it was a structured output block)
- // This is populated when the terminal block has outputFormat="json" and valid parsed data
- Data any `json:"data,omitempty"`
-
- // Artifacts contains all generated charts, images, visualizations
- // Each artifact has type, format, and base64 data
- Artifacts []APIArtifact `json:"artifacts,omitempty"`
-
- // Files contains all generated files with download URLs
- Files []APIFile `json:"files,omitempty"`
-
- // Blocks contains detailed output from each block (for debugging/advanced use)
- Blocks map[string]APIBlockOutput `json:"blocks,omitempty"`
-
- // Metadata contains execution statistics
- Metadata ExecutionMetadata `json:"metadata"`
-
- // Error contains error message if status is failed
- Error string `json:"error,omitempty"`
-}
-
-// APIArtifact represents a generated artifact (chart, image, etc.)
-type APIArtifact struct {
- Type string `json:"type"` // "chart", "image", "plot"
- Format string `json:"format"` // "png", "jpeg", "svg"
- Data string `json:"data"` // Base64 encoded data
- Title string `json:"title,omitempty"` // Description/title
- SourceBlock string `json:"source_block,omitempty"` // Which block generated this
-}
-
-// APIFile represents a generated file
-type APIFile struct {
- FileID string `json:"file_id"`
- Filename string `json:"filename"`
- DownloadURL string `json:"download_url"`
- MimeType string `json:"mime_type,omitempty"`
- Size int64 `json:"size,omitempty"`
- SourceBlock string `json:"source_block,omitempty"`
-}
-
-// APIBlockOutput is a clean representation of a block's output
-type APIBlockOutput struct {
- Name string `json:"name"`
- Type string `json:"type"`
- Status string `json:"status"`
- Response string `json:"response,omitempty"` // Primary text output
- Data map[string]any `json:"data,omitempty"` // Structured data (if JSON output)
- Error string `json:"error,omitempty"`
- DurationMs int64 `json:"duration_ms,omitempty"`
-}
-
-// ExecutionMetadata contains execution statistics
-type ExecutionMetadata struct {
- ExecutionID string `json:"execution_id"`
- AgentID string `json:"agent_id,omitempty"`
- WorkflowVersion int `json:"workflow_version,omitempty"`
- DurationMs int64 `json:"duration_ms"`
- TotalTokens int `json:"total_tokens,omitempty"`
- BlocksExecuted int `json:"blocks_executed"`
- BlocksFailed int `json:"blocks_failed"`
-}
-
-// ExecuteWorkflowRequest is received from the client to start execution
-type ExecuteWorkflowRequest struct {
- Type string `json:"type"` // execute_workflow
- AgentID string `json:"agent_id"`
- Input map[string]any `json:"input,omitempty"`
-}
-
-// CreateAgentRequest is the request body for creating an agent
-type CreateAgentRequest struct {
- Name string `json:"name"`
- Description string `json:"description,omitempty"`
-}
-
-// UpdateAgentRequest is the request body for updating an agent
-type UpdateAgentRequest struct {
- Name string `json:"name,omitempty"`
- Description string `json:"description,omitempty"`
- Status string `json:"status,omitempty"`
-}
-
-// SaveWorkflowRequest is the request body for saving a workflow
-type SaveWorkflowRequest struct {
- Blocks []Block `json:"blocks"`
- Connections []Connection `json:"connections"`
- Variables []Variable `json:"variables,omitempty"`
- CreateVersion bool `json:"createVersion,omitempty"` // Only create version snapshot if true
- VersionDescription string `json:"versionDescription,omitempty"` // Description for the version (if created)
-}
-
-// ============================================================================
-// Agent-Per-Block Architecture Types (Sprint 4)
-// Each LLM block can act as a mini-agent with tool access and structured output
-// ============================================================================
-
-// AgentBlockConfig defines the configuration for an LLM block with agent capabilities
-type AgentBlockConfig struct {
- // Model Configuration
- Model string `json:"model,omitempty"` // Default: "sonnet-4.5" (resolves to glm-4.6)
- Temperature float64 `json:"temperature,omitempty"` // Default: 0.7
-
- // Prompts
- SystemPrompt string `json:"systemPrompt,omitempty"`
- UserPrompt string `json:"userPrompt,omitempty"` // Supports {{variable}} interpolation
-
- // Tool Configuration
- EnabledTools []string `json:"enabledTools,omitempty"` // e.g., ["search_web", "calculate_math"]
- MaxToolCalls int `json:"maxToolCalls,omitempty"` // Default: 15
- Credentials []string `json:"credentials,omitempty"` // Credential IDs for tool authentication
-
- // Execution Mode Configuration (for deterministic block execution)
- RequireToolUsage bool `json:"requireToolUsage,omitempty"` // Default: true when tools exist - forces tool calls
- MaxRetries int `json:"maxRetries,omitempty"` // Default: 2 - retry attempts if tool not called
- RequiredTools []string `json:"requiredTools,omitempty"` // Specific tools that MUST be called
-
- // Retry Policy for LLM API calls (transient error handling)
- RetryPolicy *RetryPolicy `json:"retryPolicy,omitempty"` // Optional retry configuration for API failures
-
- // Output Configuration
- OutputSchema *JSONSchema `json:"outputSchema,omitempty"` // JSON schema for validation
- StrictOutput bool `json:"strictOutput,omitempty"` // Require exact schema match
-}
-
-// RetryPolicy defines retry behavior for block execution (LLM API calls)
-type RetryPolicy struct {
- // MaxRetries is the maximum number of retry attempts (default: 1)
- MaxRetries int `json:"maxRetries,omitempty"`
-
- // InitialDelay is the initial delay before first retry in milliseconds (default: 1000)
- InitialDelay int `json:"initialDelay,omitempty"`
-
- // MaxDelay is the maximum delay between retries in milliseconds (default: 30000)
- MaxDelay int `json:"maxDelay,omitempty"`
-
- // BackoffMultiplier is the exponential backoff multiplier (default: 2.0)
- BackoffMultiplier float64 `json:"backoffMultiplier,omitempty"`
-
- // RetryOn specifies which error types to retry (default: ["timeout", "rate_limit", "server_error"])
- RetryOn []string `json:"retryOn,omitempty"`
-
- // JitterPercent adds randomness to delay to prevent thundering herd (0-100, default: 20)
- JitterPercent int `json:"jitterPercent,omitempty"`
-}
-
-// DefaultRetryPolicy returns sensible production defaults for retry behavior
-func DefaultRetryPolicy() *RetryPolicy {
- return &RetryPolicy{
- MaxRetries: 1,
- InitialDelay: 1000, // 1 second
- MaxDelay: 30000, // 30 seconds
- BackoffMultiplier: 2.0,
- RetryOn: []string{"timeout", "rate_limit", "server_error"},
- JitterPercent: 20,
- }
-}
-
-// JSONSchema represents a JSON Schema for output validation
-type JSONSchema struct {
- Type string `json:"type"`
- Properties map[string]*JSONSchema `json:"properties,omitempty"`
- Items *JSONSchema `json:"items,omitempty"`
- Required []string `json:"required,omitempty"`
- AdditionalProperties *bool `json:"additionalProperties,omitempty"`
- Description string `json:"description,omitempty"`
- Enum []string `json:"enum,omitempty"`
- Default any `json:"default,omitempty"`
-}
-
-// AgentBlockResult represents the result of an agent block execution
-type AgentBlockResult struct {
- // The validated output (matches OutputSchema if provided)
- Output map[string]any `json:"output"`
-
- // Raw LLM response (before parsing)
- RawResponse string `json:"rawResponse,omitempty"`
-
- // Model used for execution
- Model string `json:"model"`
-
- // Token usage
- Tokens TokenUsage `json:"tokens"`
-
- // Tool calls made during execution
- ToolCalls []ToolCallRecord `json:"toolCalls,omitempty"`
-
- // Number of iterations in the agent loop
- Iterations int `json:"iterations"`
-}
-
-// ToolCallRecord records a tool call made during execution
-type ToolCallRecord struct {
- Name string `json:"name"`
- Arguments map[string]any `json:"arguments"`
- Result string `json:"result"`
- Error string `json:"error,omitempty"`
- Duration int64 `json:"durationMs"`
-}
-
-// ============================================================================
-// Pagination Types
-// ============================================================================
-
-// AgentListItem is a lightweight agent representation for list views
-type AgentListItem struct {
- ID string `json:"id"`
- Name string `json:"name"`
- Description string `json:"description,omitempty"`
- Status string `json:"status"`
- HasWorkflow bool `json:"has_workflow"`
- BlockCount int `json:"block_count"`
- CreatedAt time.Time `json:"created_at"`
- UpdatedAt time.Time `json:"updated_at"`
-}
-
-// PaginatedAgentsResponse is the response for paginated agent list
-type PaginatedAgentsResponse struct {
- Agents []AgentListItem `json:"agents"`
- Total int `json:"total"`
- Limit int `json:"limit"`
- Offset int `json:"offset"`
- HasMore bool `json:"has_more"`
-}
-
-// RecentAgentsResponse is the response for recent agents (landing page)
-type RecentAgentsResponse struct {
- Agents []AgentListItem `json:"agents"`
-}
-
-// ============================================================================
-// Sync Types (for first-message persistence)
-// ============================================================================
-
-// SyncAgentRequest is the request body for syncing a local agent to backend
-type SyncAgentRequest struct {
- Name string `json:"name"`
- Description string `json:"description,omitempty"`
- Workflow SaveWorkflowRequest `json:"workflow"`
- ModelID string `json:"model_id,omitempty"`
-}
-
-// SyncAgentResponse is the response after syncing an agent
-type SyncAgentResponse struct {
- Agent *Agent `json:"agent"`
- Workflow *Workflow `json:"workflow"`
- ConversationID string `json:"conversation_id"`
-}
diff --git a/backend/internal/models/apikey.go b/backend/internal/models/apikey.go
deleted file mode 100644
index 9bcf25fd..00000000
--- a/backend/internal/models/apikey.go
+++ /dev/null
@@ -1,195 +0,0 @@
-package models
-
-import (
- "time"
-
- "go.mongodb.org/mongo-driver/bson/primitive"
-)
-
-// APIKey represents an API key for programmatic access
-type APIKey struct {
- ID primitive.ObjectID `bson:"_id,omitempty" json:"id"`
- UserID string `bson:"userId" json:"userId"`
-
- // Key info (hash stored, never plain text)
- KeyPrefix string `bson:"keyPrefix" json:"keyPrefix"` // First 8 chars for display (e.g., "clv_a1b2")
- KeyHash string `bson:"keyHash" json:"-"` // bcrypt hash, never exposed in JSON
- PlainKey string `bson:"plainKey,omitempty" json:"key"` // TEMPORARY: Plain key for early platform phase
-
- // Metadata
- Name string `bson:"name" json:"name"`
- Description string `bson:"description,omitempty" json:"description,omitempty"`
-
- // Permissions
- Scopes []string `bson:"scopes" json:"scopes"` // e.g., ["execute:*"], ["execute:agent-123", "read:executions"]
-
- // Rate limits (tier-based defaults can be overridden)
- RateLimit *APIKeyRateLimit `bson:"rateLimit,omitempty" json:"rateLimit,omitempty"`
-
- // Status
- LastUsedAt *time.Time `bson:"lastUsedAt,omitempty" json:"lastUsedAt,omitempty"`
- RevokedAt *time.Time `bson:"revokedAt,omitempty" json:"revokedAt,omitempty"` // Soft delete
- ExpiresAt *time.Time `bson:"expiresAt,omitempty" json:"expiresAt,omitempty"` // Optional expiration
-
- CreatedAt time.Time `bson:"createdAt" json:"createdAt"`
- UpdatedAt time.Time `bson:"updatedAt" json:"updatedAt"`
-}
-
-// APIKeyRateLimit defines custom rate limits for an API key
-type APIKeyRateLimit struct {
- RequestsPerMinute int64 `bson:"requestsPerMinute" json:"requestsPerMinute"`
- RequestsPerHour int64 `bson:"requestsPerHour" json:"requestsPerHour"`
-}
-
-// IsRevoked returns true if the API key has been revoked
-func (k *APIKey) IsRevoked() bool {
- return k.RevokedAt != nil
-}
-
-// IsExpired returns true if the API key has expired
-func (k *APIKey) IsExpired() bool {
- if k.ExpiresAt == nil {
- return false
- }
- return time.Now().After(*k.ExpiresAt)
-}
-
-// IsValid returns true if the API key is not revoked and not expired
-func (k *APIKey) IsValid() bool {
- return !k.IsRevoked() && !k.IsExpired()
-}
-
-// HasScope checks if the API key has a specific scope
-func (k *APIKey) HasScope(scope string) bool {
- for _, s := range k.Scopes {
- if s == scope || s == "*" {
- return true
- }
- // Check wildcard patterns like "execute:*"
- if matchWildcardScope(s, scope) {
- return true
- }
- }
- return false
-}
-
-// HasExecuteScope checks if the API key can execute a specific agent
-func (k *APIKey) HasExecuteScope(agentID string) bool {
- // Check for universal execute permission
- if k.HasScope("execute:*") {
- return true
- }
- // Check for specific agent permission
- return k.HasScope("execute:" + agentID)
-}
-
-// HasReadExecutionsScope checks if the API key can read executions
-func (k *APIKey) HasReadExecutionsScope() bool {
- return k.HasScope("read:executions") || k.HasScope("read:*") || k.HasScope("*")
-}
-
-// matchWildcardScope checks if a wildcard scope matches a target scope
-// e.g., "execute:*" matches "execute:agent-123"
-func matchWildcardScope(pattern, target string) bool {
- if len(pattern) < 2 || pattern[len(pattern)-1] != '*' {
- return false
- }
- prefix := pattern[:len(pattern)-1] // Remove the '*'
- return len(target) >= len(prefix) && target[:len(prefix)] == prefix
-}
-
-// CreateAPIKeyRequest is the request body for creating an API key
-type CreateAPIKeyRequest struct {
- Name string `json:"name"`
- Description string `json:"description,omitempty"`
- Scopes []string `json:"scopes"` // Required: what the key can do
- RateLimit *APIKeyRateLimit `json:"rateLimit,omitempty"` // Optional: custom rate limits
- ExpiresIn int `json:"expiresIn,omitempty"` // Optional: expiration in days
-}
-
-// CreateAPIKeyResponse is returned after creating an API key
-// This is the ONLY time the full key is returned
-type CreateAPIKeyResponse struct {
- ID string `json:"id"`
- Key string `json:"key"` // Full API key (ONLY shown once)
- KeyPrefix string `json:"keyPrefix"` // Display prefix
- Name string `json:"name"`
- Scopes []string `json:"scopes"`
- ExpiresAt *time.Time `json:"expiresAt,omitempty"`
- CreatedAt time.Time `json:"createdAt"`
-}
-
-// APIKeyListItem is a safe representation of an API key for listing
-// Never includes the key hash
-type APIKeyListItem struct {
- ID string `json:"id"`
- KeyPrefix string `json:"keyPrefix"`
- Key string `json:"key,omitempty"` // TEMPORARY: Plain key for early platform phase
- Name string `json:"name"`
- Description string `json:"description,omitempty"`
- Scopes []string `json:"scopes"`
- LastUsedAt *time.Time `json:"lastUsedAt,omitempty"`
- ExpiresAt *time.Time `json:"expiresAt,omitempty"`
- IsRevoked bool `json:"isRevoked"`
- CreatedAt time.Time `json:"createdAt"`
-}
-
-// ToListItem converts an APIKey to a safe list representation
-func (k *APIKey) ToListItem() *APIKeyListItem {
- return &APIKeyListItem{
- ID: k.ID.Hex(),
- KeyPrefix: k.KeyPrefix,
- Key: k.PlainKey, // TEMPORARY: Include plain key for early platform phase
- Name: k.Name,
- Description: k.Description,
- Scopes: k.Scopes,
- LastUsedAt: k.LastUsedAt,
- ExpiresAt: k.ExpiresAt,
- IsRevoked: k.IsRevoked(),
- CreatedAt: k.CreatedAt,
- }
-}
-
-// TriggerAgentRequest is the request body for triggering an agent via API
-type TriggerAgentRequest struct {
- Input map[string]interface{} `json:"input,omitempty"`
-
- // EnableBlockChecker enables block completion validation (optional)
- // When true, each block is checked to ensure it accomplished its job
- EnableBlockChecker bool `json:"enable_block_checker,omitempty"`
-
- // CheckerModelID is the model to use for block checking (optional)
- // Defaults to gpt-4o-mini for fast, cheap validation
- CheckerModelID string `json:"checker_model_id,omitempty"`
-}
-
-// TriggerAgentResponse is returned after triggering an agent
-type TriggerAgentResponse struct {
- ExecutionID string `json:"executionId"`
- Status string `json:"status"` // "queued" or "running"
- Message string `json:"message"`
-}
-
-// ValidScopes lists all valid API key scopes
-var ValidScopes = []string{
- "execute:*", // Execute any agent
- "upload", // Upload files for workflow inputs
- "read:executions", // Read execution history
- "read:*", // Read all resources
- "*", // Full access (admin)
-}
-
-// IsValidScope checks if a scope is valid
-func IsValidScope(scope string) bool {
- // Check exact match
- for _, valid := range ValidScopes {
- if scope == valid {
- return true
- }
- }
- // Check agent-specific execute scope (execute:agent-xxx)
- if len(scope) > 8 && scope[:8] == "execute:" {
- return true
- }
- return false
-}
diff --git a/backend/internal/models/chat.go b/backend/internal/models/chat.go
deleted file mode 100644
index 41f7a52e..00000000
--- a/backend/internal/models/chat.go
+++ /dev/null
@@ -1,178 +0,0 @@
-package models
-
-import (
- "time"
-
- "go.mongodb.org/mongo-driver/bson/primitive"
-)
-
-// EncryptedChat represents a chat stored in MongoDB with encrypted messages
-type EncryptedChat struct {
- ID primitive.ObjectID `bson:"_id,omitempty" json:"id"`
- UserID string `bson:"userId" json:"user_id"`
- ChatID string `bson:"chatId" json:"chat_id"` // Frontend-generated UUID
- Title string `bson:"title" json:"title"` // Plaintext title for listing
- EncryptedMessages string `bson:"encryptedMessages" json:"-"` // AES-256-GCM encrypted JSON array of messages
- IsStarred bool `bson:"isStarred" json:"is_starred"`
- Model string `bson:"model,omitempty" json:"model,omitempty"` // Selected model for this chat
- Version int64 `bson:"version" json:"version"` // Optimistic locking
- CreatedAt time.Time `bson:"createdAt" json:"created_at"`
- UpdatedAt time.Time `bson:"updatedAt" json:"updated_at"`
-}
-
-// ChatMessage represents a single message in a chat (unencrypted form)
-type ChatMessage struct {
- ID string `json:"id"`
- Role string `json:"role"` // "user", "assistant", "system"
- Content string `json:"content"`
- Timestamp int64 `json:"timestamp"` // Unix milliseconds
- IsStreaming bool `json:"isStreaming,omitempty"`
- Status string `json:"status,omitempty"` // "sending", "sent", "error"
- Error string `json:"error,omitempty"`
- Attachments []ChatAttachment `json:"attachments,omitempty"`
- ToolCalls []ToolCall `json:"toolCalls,omitempty"`
- Reasoning string `json:"reasoning,omitempty"` // Thinking/reasoning process
- Artifacts []Artifact `json:"artifacts,omitempty"`
- AgentId string `json:"agentId,omitempty"`
- AgentName string `json:"agentName,omitempty"`
- AgentAvatar string `json:"agentAvatar,omitempty"`
-
- // Response versioning fields
- VersionGroupId string `json:"versionGroupId,omitempty"` // Groups all versions of same response
- VersionNumber int `json:"versionNumber,omitempty"` // 1, 2, 3... within the group
- IsHidden bool `json:"isHidden,omitempty"` // Hidden versions (not current)
- RetryType string `json:"retryType,omitempty"` // Type of retry: regenerate, add_details, etc.
-}
-
-// ToolCall represents a tool invocation in a message
-type ToolCall struct {
- ID string `json:"id"`
- Name string `json:"name"`
- DisplayName string `json:"displayName,omitempty"`
- Icon string `json:"icon,omitempty"`
- Status string `json:"status"` // "executing", "completed"
- Query string `json:"query,omitempty"`
- Result string `json:"result,omitempty"`
- Plots []PlotData `json:"plots,omitempty"`
- Timestamp int64 `json:"timestamp"`
- IsExpanded bool `json:"isExpanded,omitempty"`
-}
-
-// Artifact represents renderable content (HTML, SVG, Mermaid)
-type Artifact struct {
- ID string `json:"id"`
- Type string `json:"type"` // "html", "svg", "mermaid", "image"
- Title string `json:"title"`
- Content string `json:"content"`
- Images []ArtifactImage `json:"images,omitempty"`
- Metadata map[string]interface{} `json:"metadata,omitempty"`
-}
-
-// ArtifactImage represents an image in an artifact
-type ArtifactImage struct {
- Data string `json:"data"` // Base64-encoded
- Format string `json:"format"` // png, jpg, svg
- Caption string `json:"caption,omitempty"`
-}
-
-// ChatAttachment represents a file attached to a message
-type ChatAttachment struct {
- FileID string `json:"file_id"`
- Type string `json:"type"` // Attachment type: "image", "document", "data"
- URL string `json:"url"`
- MimeType string `json:"mime_type"`
- Size int64 `json:"size"`
- Filename string `json:"filename,omitempty"`
- Expired bool `json:"expired,omitempty"`
- // Document-specific fields
- PageCount int `json:"page_count,omitempty"`
- WordCount int `json:"word_count,omitempty"`
- Preview string `json:"preview,omitempty"` // Text preview or thumbnail
- // Data file-specific fields
- DataPreview *DataPreview `json:"data_preview,omitempty"`
-}
-
-// DataPreview represents a preview of CSV/tabular data
-type DataPreview struct {
- Headers []string `json:"headers"`
- Rows [][]string `json:"rows"`
- RowCount int `json:"row_count"` // Total rows in file
- ColCount int `json:"col_count"` // Total columns
-}
-
-// ChatResponse is the decrypted chat returned to the frontend
-type ChatResponse struct {
- ID string `json:"id"`
- Title string `json:"title"`
- Messages []ChatMessage `json:"messages"`
- IsStarred bool `json:"is_starred"`
- Model string `json:"model,omitempty"`
- Version int64 `json:"version"`
- CreatedAt time.Time `json:"created_at"`
- UpdatedAt time.Time `json:"updated_at"`
-}
-
-// ChatListItem is a summary of a chat for listing (no messages)
-type ChatListItem struct {
- ID string `json:"id"`
- Title string `json:"title"`
- IsStarred bool `json:"is_starred"`
- Model string `json:"model,omitempty"`
- MessageCount int `json:"message_count"`
- Version int64 `json:"version"`
- CreatedAt time.Time `json:"created_at"`
- UpdatedAt time.Time `json:"updated_at"`
-}
-
-// CreateChatRequest is the request body for creating/updating a chat
-type CreateChatRequest struct {
- ID string `json:"id"` // Frontend-generated UUID
- Title string `json:"title"`
- Messages []ChatMessage `json:"messages"`
- IsStarred bool `json:"is_starred"`
- Model string `json:"model,omitempty"`
- Version int64 `json:"version,omitempty"` // For optimistic locking on updates
-}
-
-// UpdateChatRequest is the request body for partial chat updates
-type UpdateChatRequest struct {
- Title *string `json:"title,omitempty"`
- IsStarred *bool `json:"is_starred,omitempty"`
- Model *string `json:"model,omitempty"`
- Version int64 `json:"version"` // Required for optimistic locking
-}
-
-// ChatAddMessageRequest is the request body for adding a single message to a synced chat
-type ChatAddMessageRequest struct {
- Message ChatMessage `json:"message"`
- Version int64 `json:"version"` // For optimistic locking
-}
-
-// BulkSyncRequest is the request body for uploading multiple chats
-type BulkSyncRequest struct {
- Chats []CreateChatRequest `json:"chats"`
-}
-
-// BulkSyncResponse is the response for bulk sync operation
-type BulkSyncResponse struct {
- Synced int `json:"synced"`
- Failed int `json:"failed"`
- Errors []string `json:"errors,omitempty"`
- ChatIDs []string `json:"chat_ids"` // IDs of successfully synced chats
-}
-
-// SyncAllResponse is the response for fetching all chats for initial sync
-type SyncAllResponse struct {
- Chats []ChatResponse `json:"chats"`
- TotalCount int `json:"total_count"`
- SyncedAt time.Time `json:"synced_at"`
-}
-
-// ChatListResponse is the paginated response for listing chats
-type ChatListResponse struct {
- Chats []ChatListItem `json:"chats"`
- TotalCount int64 `json:"total_count"`
- Page int `json:"page"`
- PageSize int `json:"page_size"`
- HasMore bool `json:"has_more"`
-}
diff --git a/backend/internal/models/config.go b/backend/internal/models/config.go
deleted file mode 100644
index 7925935e..00000000
--- a/backend/internal/models/config.go
+++ /dev/null
@@ -1,8 +0,0 @@
-package models
-
-// Config represents API configuration (legacy, kept for backward compatibility)
-type Config struct {
- BaseURL string `json:"base_url"`
- APIKey string `json:"api_key"`
- Model string `json:"model"`
-}
diff --git a/backend/internal/models/conversation.go b/backend/internal/models/conversation.go
deleted file mode 100644
index 8570d5b0..00000000
--- a/backend/internal/models/conversation.go
+++ /dev/null
@@ -1,88 +0,0 @@
-package models
-
-import (
- "time"
-
- "go.mongodb.org/mongo-driver/bson/primitive"
-)
-
-// BuilderConversation represents a chat history for building an agent
-type BuilderConversation struct {
- ID primitive.ObjectID `bson:"_id,omitempty" json:"id"`
- AgentID string `bson:"agentId" json:"agent_id"` // String ID (timestamp-based from SQLite)
- UserID string `bson:"userId" json:"user_id"` // Supabase user ID
- Messages []BuilderMessage `bson:"messages" json:"messages"`
- ModelID string `bson:"modelId" json:"model_id"`
- CreatedAt time.Time `bson:"createdAt" json:"created_at"`
- UpdatedAt time.Time `bson:"updatedAt" json:"updated_at"`
- ExpiresAt *time.Time `bson:"expiresAt,omitempty" json:"expires_at,omitempty"` // TTL for auto-deletion if user opts out
-}
-
-// BuilderMessage represents a single message in the builder conversation
-type BuilderMessage struct {
- ID string `bson:"id" json:"id"`
- Role string `bson:"role" json:"role"` // "user" or "assistant"
- Content string `bson:"content" json:"content"`
- Timestamp time.Time `bson:"timestamp" json:"timestamp"`
- WorkflowSnapshot *WorkflowSnapshot `bson:"workflowSnapshot,omitempty" json:"workflow_snapshot,omitempty"`
-}
-
-// WorkflowSnapshot captures the state of the workflow at a message point
-type WorkflowSnapshot struct {
- Version int `bson:"version" json:"version"`
- Action string `bson:"action,omitempty" json:"action,omitempty"` // "create" or "modify" or null
- Explanation string `bson:"explanation,omitempty" json:"explanation,omitempty"`
-}
-
-// EncryptedBuilderConversation stores encrypted conversation data
-// The Messages field is encrypted as a JSON blob
-type EncryptedBuilderConversation struct {
- ID primitive.ObjectID `bson:"_id,omitempty" json:"id"`
- AgentID string `bson:"agentId" json:"agent_id"` // String ID (timestamp-based from SQLite)
- UserID string `bson:"userId" json:"user_id"` // Supabase user ID
- EncryptedMessages string `bson:"encryptedMessages" json:"-"` // Base64-encoded encrypted JSON
- ModelID string `bson:"modelId" json:"model_id"`
- MessageCount int `bson:"messageCount" json:"message_count"` // For display without decryption
- CreatedAt time.Time `bson:"createdAt" json:"created_at"`
- UpdatedAt time.Time `bson:"updatedAt" json:"updated_at"`
- ExpiresAt *time.Time `bson:"expiresAt,omitempty" json:"expires_at,omitempty"`
-}
-
-// ConversationListItem is a summary for listing conversations
-type ConversationListItem struct {
- ID string `json:"id"`
- AgentID string `json:"agent_id"`
- ModelID string `json:"model_id"`
- MessageCount int `json:"message_count"`
- CreatedAt time.Time `json:"created_at"`
- UpdatedAt time.Time `json:"updated_at"`
-}
-
-// AddMessageRequest is the request body for adding a message to a conversation
-type AddMessageRequest struct {
- Role string `json:"role"`
- Content string `json:"content"`
- WorkflowSnapshot *WorkflowSnapshot `json:"workflow_snapshot,omitempty"`
-}
-
-// ConversationResponse is the full conversation response
-type ConversationResponse struct {
- ID string `json:"id"`
- AgentID string `json:"agent_id"`
- ModelID string `json:"model_id"`
- Messages []BuilderMessage `json:"messages"`
- CreatedAt time.Time `json:"created_at"`
- UpdatedAt time.Time `json:"updated_at"`
-}
-
-// ToListItem converts an EncryptedBuilderConversation to ConversationListItem
-func (c *EncryptedBuilderConversation) ToListItem() ConversationListItem {
- return ConversationListItem{
- ID: c.ID.Hex(),
- AgentID: c.AgentID, // AgentID is already a string
- ModelID: c.ModelID,
- MessageCount: c.MessageCount,
- CreatedAt: c.CreatedAt,
- UpdatedAt: c.UpdatedAt,
- }
-}
diff --git a/backend/internal/models/credential.go b/backend/internal/models/credential.go
deleted file mode 100644
index 510ecf5c..00000000
--- a/backend/internal/models/credential.go
+++ /dev/null
@@ -1,153 +0,0 @@
-package models
-
-import (
- "time"
-
- "go.mongodb.org/mongo-driver/bson/primitive"
-)
-
-// Credential represents an encrypted credential for external integrations
-type Credential struct {
- ID primitive.ObjectID `bson:"_id,omitempty" json:"id"`
- UserID string `bson:"userId" json:"userId"`
- Name string `bson:"name" json:"name"`
- IntegrationType string `bson:"integrationType" json:"integrationType"`
-
- // Encrypted data - NEVER exposed to frontend or LLM
- EncryptedData string `bson:"encryptedData" json:"-"` // json:"-" ensures it's never serialized
-
- // Metadata (safe to expose)
- Metadata CredentialMetadata `bson:"metadata" json:"metadata"`
-
- CreatedAt time.Time `bson:"createdAt" json:"createdAt"`
- UpdatedAt time.Time `bson:"updatedAt" json:"updatedAt"`
-}
-
-// CredentialMetadata contains non-sensitive information about a credential
-type CredentialMetadata struct {
- MaskedPreview string `bson:"maskedPreview" json:"maskedPreview"` // e.g., "https://discord...xxx"
- Icon string `bson:"icon,omitempty" json:"icon,omitempty"`
- LastUsedAt *time.Time `bson:"lastUsedAt,omitempty" json:"lastUsedAt,omitempty"`
- UsageCount int64 `bson:"usageCount" json:"usageCount"`
- LastTestAt *time.Time `bson:"lastTestAt,omitempty" json:"lastTestAt,omitempty"`
- TestStatus string `bson:"testStatus,omitempty" json:"testStatus,omitempty"` // "success", "failed", "pending"
-}
-
-// CredentialListItem is a safe representation for listing credentials
-// Never includes the encrypted data
-type CredentialListItem struct {
- ID string `json:"id"`
- UserID string `json:"userId"`
- Name string `json:"name"`
- IntegrationType string `json:"integrationType"`
- Metadata CredentialMetadata `json:"metadata"`
- CreatedAt time.Time `json:"createdAt"`
- UpdatedAt time.Time `json:"updatedAt"`
-}
-
-// ToListItem converts a Credential to a safe list representation
-func (c *Credential) ToListItem() *CredentialListItem {
- return &CredentialListItem{
- ID: c.ID.Hex(),
- UserID: c.UserID,
- Name: c.Name,
- IntegrationType: c.IntegrationType,
- Metadata: c.Metadata,
- CreatedAt: c.CreatedAt,
- UpdatedAt: c.UpdatedAt,
- }
-}
-
-// DecryptedCredential is used internally by tools to access credential data
-// This should NEVER be returned to the frontend or exposed to the LLM
-type DecryptedCredential struct {
- ID string `json:"id"`
- Name string `json:"name"`
- IntegrationType string `json:"integrationType"`
- Data map[string]interface{} `json:"data"` // The actual credential values
-}
-
-// CreateCredentialRequest is the request body for creating a credential
-type CreateCredentialRequest struct {
- Name string `json:"name" validate:"required,min=1,max=100"`
- IntegrationType string `json:"integrationType" validate:"required"`
- Data map[string]interface{} `json:"data" validate:"required"` // Will be encrypted
-}
-
-// UpdateCredentialRequest is the request body for updating a credential
-type UpdateCredentialRequest struct {
- Name string `json:"name,omitempty"`
- Data map[string]interface{} `json:"data,omitempty"` // Will be re-encrypted if provided
-}
-
-// TestCredentialResponse is returned after testing a credential
-type TestCredentialResponse struct {
- Success bool `json:"success"`
- Message string `json:"message"`
- Details string `json:"details,omitempty"` // Additional info (sanitized)
-}
-
-// CredentialReference is used in block configs to reference credentials
-// The LLM only sees the Name, tools use the ID to fetch the actual credential
-type CredentialReference struct {
- ID string `json:"id"`
- Name string `json:"name"`
- IntegrationType string `json:"integrationType"`
-}
-
-// Integration represents a supported external integration type
-type Integration struct {
- ID string `json:"id"` // e.g., "discord", "notion"
- Name string `json:"name"` // e.g., "Discord", "Notion"
- Description string `json:"description"` // Short description
- Icon string `json:"icon"` // Icon identifier (lucide or custom)
- Category string `json:"category"` // e.g., "communication", "productivity"
- Fields []IntegrationField `json:"fields"` // Required/optional fields
- Tools []string `json:"tools"` // Which tools use this integration
- DocsURL string `json:"docsUrl,omitempty"`
- ComingSoon bool `json:"comingSoon,omitempty"` // If true, integration is not yet available
-}
-
-// IntegrationField defines a field required for an integration
-type IntegrationField struct {
- Key string `json:"key"` // e.g., "webhook_url", "api_key"
- Label string `json:"label"` // e.g., "Webhook URL", "API Key"
- Type string `json:"type"` // "api_key", "webhook_url", "token", "text", "select", "json"
- Required bool `json:"required"` // Is this field required?
- Placeholder string `json:"placeholder,omitempty"` // Placeholder text
- HelpText string `json:"helpText,omitempty"` // Help text for the user
- Options []string `json:"options,omitempty"` // For select type
- Default string `json:"default,omitempty"` // Default value
- Sensitive bool `json:"sensitive"` // Should this be masked in UI?
-}
-
-// IntegrationCategory represents a category of integrations
-type IntegrationCategory struct {
- ID string `json:"id"`
- Name string `json:"name"`
- Icon string `json:"icon"`
- Integrations []Integration `json:"integrations"`
-}
-
-// GetIntegrationsResponse is the response for listing available integrations
-type GetIntegrationsResponse struct {
- Categories []IntegrationCategory `json:"categories"`
-}
-
-// GetCredentialsResponse is the response for listing user credentials
-type GetCredentialsResponse struct {
- Credentials []*CredentialListItem `json:"credentials"`
- Total int `json:"total"`
-}
-
-// CredentialsByIntegration groups credentials by integration type
-type CredentialsByIntegration struct {
- IntegrationType string `json:"integrationType"`
- Integration Integration `json:"integration"`
- Credentials []*CredentialListItem `json:"credentials"`
-}
-
-// GetCredentialsByIntegrationResponse groups credentials by integration
-type GetCredentialsByIntegrationResponse struct {
- Integrations []CredentialsByIntegration `json:"integrations"`
-}
diff --git a/backend/internal/models/integration_registry.go b/backend/internal/models/integration_registry.go
deleted file mode 100644
index 4b8af1b0..00000000
--- a/backend/internal/models/integration_registry.go
+++ /dev/null
@@ -1,1323 +0,0 @@
-package models
-
-// IntegrationRegistry contains all supported integrations
-// This is the source of truth for what integrations are available
-var IntegrationRegistry = map[string]Integration{
- // ============================================
- // COMMUNICATION
- // ============================================
- "discord": {
- ID: "discord",
- Name: "Discord",
- Description: "Send messages to Discord channels via webhooks",
- Icon: "discord",
- Category: "communication",
- Fields: []IntegrationField{
- {
- Key: "webhook_url",
- Label: "Webhook URL",
- Type: "webhook_url",
- Required: true,
- Placeholder: "https://discord.com/api/webhooks/...",
- HelpText: "Create a webhook in Discord: Server Settings → Integrations → Webhooks",
- Sensitive: true,
- },
- },
- Tools: []string{"send_discord_message"},
- DocsURL: "https://support.discord.com/hc/en-us/articles/228383668-Intro-to-Webhooks",
- },
-
- "slack": {
- ID: "slack",
- Name: "Slack",
- Description: "Send messages to Slack channels via webhooks",
- Icon: "slack",
- Category: "communication",
- Fields: []IntegrationField{
- {
- Key: "webhook_url",
- Label: "Webhook URL",
- Type: "webhook_url",
- Required: true,
- Placeholder: "https://hooks.slack.com/services/...",
- HelpText: "Create an Incoming Webhook in your Slack App settings",
- Sensitive: true,
- },
- },
- Tools: []string{"send_slack_message"},
- DocsURL: "https://api.slack.com/messaging/webhooks",
- },
-
- "telegram": {
- ID: "telegram",
- Name: "Telegram",
- Description: "Send messages via Telegram Bot API",
- Icon: "telegram",
- Category: "communication",
- Fields: []IntegrationField{
- {
- Key: "bot_token",
- Label: "Bot Token",
- Type: "api_key",
- Required: true,
- Placeholder: "123456789:ABCdefGHIjklMNOpqrsTUVwxyz",
- HelpText: "Get your bot token from @BotFather on Telegram",
- Sensitive: true,
- },
- {
- Key: "chat_id",
- Label: "Chat ID",
- Type: "text",
- Required: true,
- Placeholder: "-1001234567890",
- HelpText: "The chat ID where messages will be sent (use @userinfobot to find it)",
- Sensitive: false,
- },
- },
- Tools: []string{"send_telegram_message"},
- DocsURL: "https://core.telegram.org/bots/api",
- },
-
- "teams": {
- ID: "teams",
- Name: "Microsoft Teams",
- Description: "Send messages to Microsoft Teams channels",
- Icon: "microsoft",
- Category: "communication",
- Fields: []IntegrationField{
- {
- Key: "webhook_url",
- Label: "Webhook URL",
- Type: "webhook_url",
- Required: true,
- Placeholder: "https://outlook.office.com/webhook/...",
- HelpText: "Create an Incoming Webhook connector in your Teams channel",
- Sensitive: true,
- },
- },
- Tools: []string{"send_teams_message"},
- DocsURL: "https://learn.microsoft.com/en-us/microsoftteams/platform/webhooks-and-connectors/how-to/add-incoming-webhook",
- },
-
- "google_chat": {
- ID: "google_chat",
- Name: "Google Chat",
- Description: "Send messages to Google Chat spaces via webhooks",
- Icon: "google",
- Category: "communication",
- Fields: []IntegrationField{
- {
- Key: "webhook_url",
- Label: "Webhook URL",
- Type: "webhook_url",
- Required: true,
- Placeholder: "https://chat.googleapis.com/v1/spaces/.../messages?key=...",
- HelpText: "Create a webhook in Google Chat: Space settings → Integrations → Webhooks",
- Sensitive: true,
- },
- },
- Tools: []string{"send_google_chat_message"},
- DocsURL: "https://developers.google.com/chat/how-tos/webhooks",
- },
-
- "zoom": {
- ID: "zoom",
- Name: "Zoom",
- Description: "Create and manage Zoom meetings, handle registrations, and schedule video conferences",
- Icon: "zoom",
- Category: "communication",
- Fields: []IntegrationField{
- {
- Key: "account_id",
- Label: "Account ID",
- Type: "text",
- Required: true,
- Placeholder: "Your Zoom Account ID",
- HelpText: "Find in Zoom Marketplace: Your app → App Credentials → Account ID",
- Sensitive: false,
- },
- {
- Key: "client_id",
- Label: "Client ID",
- Type: "api_key",
- Required: true,
- Placeholder: "Your Zoom Client ID",
- HelpText: "Find in Zoom Marketplace: Your app → App Credentials → Client ID",
- Sensitive: true,
- },
- {
- Key: "client_secret",
- Label: "Client Secret",
- Type: "api_key",
- Required: true,
- Placeholder: "Your Zoom Client Secret",
- HelpText: "Find in Zoom Marketplace: Your app → App Credentials → Client Secret",
- Sensitive: true,
- },
- },
- Tools: []string{"zoom_meeting"},
- DocsURL: "https://developers.zoom.us/docs/internal-apps/s2s-oauth/",
- },
-
- "twilio": {
- ID: "twilio",
- Name: "Twilio",
- Description: "Send SMS, MMS, and WhatsApp messages via Twilio API",
- Icon: "twilio",
- Category: "communication",
- Fields: []IntegrationField{
- {
- Key: "account_sid",
- Label: "Account SID",
- Type: "text",
- Required: true,
- Placeholder: "ACxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx",
- HelpText: "Find in Twilio Console: Account → Account SID",
- Sensitive: false,
- },
- {
- Key: "auth_token",
- Label: "Auth Token",
- Type: "api_key",
- Required: true,
- Placeholder: "Your Twilio Auth Token",
- HelpText: "Find in Twilio Console: Account → Auth Token",
- Sensitive: true,
- },
- {
- Key: "from_number",
- Label: "Default From Number",
- Type: "text",
- Required: false,
- Placeholder: "+1234567890",
- HelpText: "Default phone number to send messages from (must be a Twilio number)",
- Sensitive: false,
- },
- },
- Tools: []string{"twilio_send_sms", "twilio_send_whatsapp"},
- DocsURL: "https://www.twilio.com/docs/sms/api",
- },
-
- "referralmonk": {
- ID: "referralmonk",
- Name: "ReferralMonk",
- Description: "Send WhatsApp messages via ReferralMonk with template support for campaigns and nurture flows",
- Icon: "message-square",
- Category: "communication",
- Fields: []IntegrationField{
- {
- Key: "api_token",
- Label: "API Token",
- Type: "api_key",
- Required: true,
- Placeholder: "Your ReferralMonk API Token",
- HelpText: "Get your API credentials from ReferralMonk dashboard (AhaGuru instance)",
- Sensitive: true,
- },
- {
- Key: "api_secret",
- Label: "API Secret",
- Type: "api_key",
- Required: true,
- Placeholder: "Your ReferralMonk API Secret",
- HelpText: "Your API secret key from ReferralMonk dashboard",
- Sensitive: true,
- },
- },
- Tools: []string{"referralmonk_whatsapp"},
- DocsURL: "https://ahaguru.referralmonk.com/",
- },
-
- // ============================================
- // PRODUCTIVITY
- // ============================================
- "notion": {
- ID: "notion",
- Name: "Notion",
- Description: "Read and write to Notion databases and pages",
- Icon: "notion",
- Category: "productivity",
- Fields: []IntegrationField{
- {
- Key: "api_key",
- Label: "Integration Token",
- Type: "api_key",
- Required: true,
- Placeholder: "secret_...",
- HelpText: "Create an integration at notion.so/my-integrations and share pages with it",
- Sensitive: true,
- },
- },
- Tools: []string{"notion_search", "notion_query_database", "notion_create_page", "notion_update_page"},
- DocsURL: "https://developers.notion.com/docs/getting-started",
- },
-
- "airtable": {
- ID: "airtable",
- Name: "Airtable",
- Description: "Read and write to Airtable bases",
- Icon: "airtable",
- Category: "productivity",
- Fields: []IntegrationField{
- {
- Key: "api_key",
- Label: "Personal Access Token",
- Type: "api_key",
- Required: true,
- Placeholder: "pat...",
- HelpText: "Create a Personal Access Token in your Airtable account settings",
- Sensitive: true,
- },
- {
- Key: "base_id",
- Label: "Base ID",
- Type: "text",
- Required: false,
- Placeholder: "appXXXXXXXXXXXXXX",
- HelpText: "Optional: Default base ID (can be overridden per request)",
- Sensitive: false,
- },
- },
- Tools: []string{"airtable_list", "airtable_read", "airtable_create", "airtable_update"},
- DocsURL: "https://airtable.com/developers/web/api/introduction",
- },
-
- "trello": {
- ID: "trello",
- Name: "Trello",
- Description: "Manage Trello boards, lists, and cards",
- Icon: "trello",
- Category: "productivity",
- Fields: []IntegrationField{
- {
- Key: "api_key",
- Label: "API Key",
- Type: "api_key",
- Required: true,
- Placeholder: "Your Trello API key",
- HelpText: "Get your API key from trello.com/app-key",
- Sensitive: true,
- },
- {
- Key: "token",
- Label: "Token",
- Type: "token",
- Required: true,
- Placeholder: "Your Trello token",
- HelpText: "Generate a token using your API key",
- Sensitive: true,
- },
- },
- Tools: []string{"trello_boards", "trello_lists", "trello_cards", "trello_create_card"},
- DocsURL: "https://developer.atlassian.com/cloud/trello/rest/",
- },
-
- "clickup": {
- ID: "clickup",
- Name: "ClickUp",
- Description: "Manage ClickUp tasks, lists, and spaces",
- Icon: "clickup",
- Category: "productivity",
- Fields: []IntegrationField{
- {
- Key: "api_key",
- Label: "API Key",
- Type: "api_key",
- Required: true,
- Placeholder: "pk_...",
- HelpText: "Get your API key from ClickUp: Settings → Apps → API Token",
- Sensitive: true,
- },
- },
- Tools: []string{"clickup_tasks", "clickup_create_task", "clickup_update_task"},
- DocsURL: "https://clickup.com/api",
- },
-
- "calendly": {
- ID: "calendly",
- Name: "Calendly",
- Description: "Manage Calendly events, scheduling links, and invitees",
- Icon: "calendly",
- Category: "productivity",
- Fields: []IntegrationField{
- {
- Key: "api_key",
- Label: "Personal Access Token",
- Type: "api_key",
- Required: true,
- Placeholder: "eyJraW...",
- HelpText: "Get your token from Calendly: Integrations → API & Webhooks → Personal Access Tokens",
- Sensitive: true,
- },
- },
- Tools: []string{"calendly_events", "calendly_event_types", "calendly_invitees"},
- DocsURL: "https://developer.calendly.com/api-docs",
- },
-
- "composio_googlesheets": {
- ID: "composio_googlesheets",
- Name: "Google Sheets",
- Description: "Complete Google Sheets integration via Composio OAuth - no GCP setup required. Create, read, write, search, and manage spreadsheets.",
- Icon: "file-spreadsheet",
- Category: "productivity",
- Fields: []IntegrationField{
- {
- Key: "composio_entity_id",
- Label: "Entity ID",
- Type: "text",
- Required: true,
- Placeholder: "Automatically filled after OAuth",
- HelpText: "Connect your Google account via Composio OAuth (managed by ClaraVerse)",
- Sensitive: false,
- },
- },
- Tools: []string{
- "googlesheets_read",
- "googlesheets_write",
- "googlesheets_append",
- "googlesheets_create",
- "googlesheets_get_info",
- "googlesheets_list_sheets",
- "googlesheets_search",
- "googlesheets_clear",
- "googlesheets_add_sheet",
- "googlesheets_delete_sheet",
- "googlesheets_find_replace",
- "googlesheets_upsert_rows",
- },
- DocsURL: "https://docs.composio.dev/toolkits/googlesheets",
- },
-
- "composio_gmail": {
- ID: "composio_gmail",
- Name: "Gmail",
- Description: "Complete Gmail integration via Composio OAuth - no GCP setup required. Send, fetch, reply, manage drafts, and organize emails.",
- Icon: "mail",
- Category: "communication",
- Fields: []IntegrationField{
- {
- Key: "composio_entity_id",
- Label: "Entity ID",
- Type: "text",
- Required: true,
- Placeholder: "Automatically filled after OAuth",
- HelpText: "Connect your Gmail account via Composio OAuth (managed by ClaraVerse)",
- Sensitive: false,
- },
- },
- Tools: []string{
- "gmail_send_email",
- "gmail_fetch_emails",
- "gmail_get_message",
- "gmail_reply_to_thread",
- "gmail_create_draft",
- "gmail_send_draft",
- "gmail_list_drafts",
- "gmail_add_label",
- "gmail_list_labels",
- "gmail_move_to_trash",
- },
- DocsURL: "https://docs.composio.dev/toolkits/gmail",
- },
-
- // ============================================
- // DEVELOPMENT
- // ============================================
- "github": {
- ID: "github",
- Name: "GitHub",
- Description: "Access GitHub repositories, issues, and pull requests",
- Icon: "github",
- Category: "development",
- Fields: []IntegrationField{
- {
- Key: "personal_access_token",
- Label: "Personal Access Token",
- Type: "api_key",
- Required: true,
- Placeholder: "ghp_...",
- HelpText: "Create a PAT at github.com/settings/tokens with required scopes",
- Sensitive: true,
- },
- },
- Tools: []string{"github_create_issue", "github_list_issues", "github_get_repo", "github_add_comment"},
- DocsURL: "https://docs.github.com/en/rest",
- },
-
- "gitlab": {
- ID: "gitlab",
- Name: "GitLab",
- Description: "Access GitLab projects, issues, and merge requests",
- Icon: "gitlab",
- Category: "development",
- Fields: []IntegrationField{
- {
- Key: "personal_access_token",
- Label: "Personal Access Token",
- Type: "api_key",
- Required: true,
- Placeholder: "glpat-...",
- HelpText: "Create a PAT in GitLab: Settings → Access Tokens",
- Sensitive: true,
- },
- {
- Key: "base_url",
- Label: "GitLab URL",
- Type: "text",
- Required: false,
- Placeholder: "https://gitlab.com",
- HelpText: "Leave empty for gitlab.com, or enter your self-hosted URL",
- Default: "https://gitlab.com",
- Sensitive: false,
- },
- },
- Tools: []string{"gitlab_projects", "gitlab_issues", "gitlab_mrs"},
- DocsURL: "https://docs.gitlab.com/ee/api/",
- },
-
- "linear": {
- ID: "linear",
- Name: "Linear",
- Description: "Manage Linear issues and projects",
- Icon: "linear",
- Category: "development",
- Fields: []IntegrationField{
- {
- Key: "api_key",
- Label: "API Key",
- Type: "api_key",
- Required: true,
- Placeholder: "lin_api_...",
- HelpText: "Create an API key in Linear: Settings → API → Personal API keys",
- Sensitive: true,
- },
- },
- Tools: []string{"linear_issues", "linear_create_issue", "linear_update_issue"},
- DocsURL: "https://developers.linear.app/docs/graphql/working-with-the-graphql-api",
- },
-
- "jira": {
- ID: "jira",
- Name: "Jira",
- Description: "Manage Jira issues and projects",
- Icon: "jira",
- Category: "development",
- Fields: []IntegrationField{
- {
- Key: "email",
- Label: "Email",
- Type: "text",
- Required: true,
- Placeholder: "your@email.com",
- HelpText: "Your Atlassian account email",
- Sensitive: false,
- },
- {
- Key: "api_token",
- Label: "API Token",
- Type: "api_key",
- Required: true,
- Placeholder: "Your Jira API token",
- HelpText: "Create an API token at id.atlassian.com/manage-profile/security/api-tokens",
- Sensitive: true,
- },
- {
- Key: "domain",
- Label: "Jira Domain",
- Type: "text",
- Required: true,
- Placeholder: "your-company.atlassian.net",
- HelpText: "Your Jira Cloud domain (without https://)",
- Sensitive: false,
- },
- },
- Tools: []string{"jira_issues", "jira_create_issue", "jira_update_issue"},
- DocsURL: "https://developer.atlassian.com/cloud/jira/platform/rest/v3/intro/",
- },
-
- // ============================================
- // CRM / SALES
- // ============================================
- "hubspot": {
- ID: "hubspot",
- Name: "HubSpot",
- Description: "Access HubSpot CRM contacts, deals, and companies",
- Icon: "hubspot",
- Category: "crm",
- Fields: []IntegrationField{
- {
- Key: "access_token",
- Label: "Private App Access Token",
- Type: "api_key",
- Required: true,
- Placeholder: "pat-...",
- HelpText: "Create a Private App in HubSpot: Settings → Integrations → Private Apps",
- Sensitive: true,
- },
- },
- Tools: []string{"hubspot_contacts", "hubspot_deals", "hubspot_companies"},
- DocsURL: "https://developers.hubspot.com/docs/api/overview",
- },
-
- "leadsquared": {
- ID: "leadsquared",
- Name: "LeadSquared",
- Description: "Access LeadSquared CRM leads and activities",
- Icon: "leadsquared",
- Category: "crm",
- Fields: []IntegrationField{
- {
- Key: "access_key",
- Label: "Access Key",
- Type: "api_key",
- Required: true,
- Placeholder: "Your LeadSquared Access Key",
- HelpText: "Find in LeadSquared: Settings → API & Webhooks → Access Keys",
- Sensitive: true,
- },
- {
- Key: "secret_key",
- Label: "Secret Key",
- Type: "api_key",
- Required: true,
- Placeholder: "Your LeadSquared Secret Key",
- HelpText: "Find alongside your Access Key",
- Sensitive: true,
- },
- {
- Key: "host",
- Label: "API Host",
- Type: "text",
- Required: true,
- Placeholder: "api.leadsquared.com",
- HelpText: "Your LeadSquared API host (varies by region)",
- Default: "api.leadsquared.com",
- Sensitive: false,
- },
- },
- Tools: []string{"leadsquared_leads", "leadsquared_create_lead", "leadsquared_activities"},
- DocsURL: "https://apidocs.leadsquared.com/",
- },
-
- // ============================================
- // MARKETING / EMAIL
- // ============================================
- "sendgrid": {
- ID: "sendgrid",
- Name: "SendGrid",
- Description: "Send emails via SendGrid API. Supports HTML/text emails, multiple recipients, CC/BCC, and attachments.",
- Icon: "sendgrid",
- Category: "marketing",
- Fields: []IntegrationField{
- {
- Key: "api_key",
- Label: "API Key",
- Type: "api_key",
- Required: true,
- Placeholder: "SG...",
- HelpText: "Create an API key in SendGrid: Settings → API Keys",
- Sensitive: true,
- },
- {
- Key: "from_email",
- Label: "Default From Email",
- Type: "text",
- Required: false,
- Placeholder: "noreply@yourdomain.com",
- HelpText: "Default sender email (must be verified in SendGrid)",
- Sensitive: false,
- },
- },
- Tools: []string{"send_email"},
- DocsURL: "https://docs.sendgrid.com/api-reference/mail-send/mail-send",
- },
-
- "brevo": {
- ID: "brevo",
- Name: "Brevo",
- Description: "Send transactional and marketing emails via Brevo (formerly SendInBlue). Supports templates, tracking, and automation.",
- Icon: "brevo",
- Category: "marketing",
- Fields: []IntegrationField{
- {
- Key: "api_key",
- Label: "API Key",
- Type: "api_key",
- Required: true,
- Placeholder: "xkeysib-...",
- HelpText: "Create an API key in Brevo: Settings → SMTP & API → API Keys",
- Sensitive: true,
- },
- {
- Key: "from_email",
- Label: "Default From Email",
- Type: "text",
- Required: false,
- Placeholder: "noreply@yourdomain.com",
- HelpText: "Default sender email (must be verified in Brevo)",
- Sensitive: false,
- },
- {
- Key: "from_name",
- Label: "Default From Name",
- Type: "text",
- Required: false,
- Placeholder: "My Company",
- HelpText: "Default sender display name",
- Sensitive: false,
- },
- },
- Tools: []string{"send_brevo_email"},
- DocsURL: "https://developers.brevo.com/docs/send-a-transactional-email",
- },
-
- "mailchimp": {
- ID: "mailchimp",
- Name: "Mailchimp",
- Description: "Manage Mailchimp audiences and campaigns",
- Icon: "mailchimp",
- Category: "marketing",
- Fields: []IntegrationField{
- {
- Key: "api_key",
- Label: "API Key",
- Type: "api_key",
- Required: true,
- Placeholder: "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx-usX",
- HelpText: "Create an API key in Mailchimp: Account → Extras → API keys",
- Sensitive: true,
- },
- },
- Tools: []string{"mailchimp_lists", "mailchimp_add_subscriber"},
- DocsURL: "https://mailchimp.com/developer/marketing/api/",
- },
-
- // ============================================
- // ANALYTICS
- // ============================================
- "mixpanel": {
- ID: "mixpanel",
- Name: "Mixpanel",
- Description: "Track events and analyze user behavior with Mixpanel",
- Icon: "mixpanel",
- Category: "analytics",
- Fields: []IntegrationField{
- {
- Key: "project_token",
- Label: "Project Token",
- Type: "api_key",
- Required: true,
- Placeholder: "Your Mixpanel Project Token",
- HelpText: "Find in Mixpanel: Settings → Project Settings → Project Token",
- Sensitive: true,
- },
- {
- Key: "api_secret",
- Label: "API Secret",
- Type: "api_key",
- Required: false,
- Placeholder: "Your Mixpanel API Secret",
- HelpText: "Required for data export. Find in Project Settings → API Secret",
- Sensitive: true,
- },
- },
- Tools: []string{"mixpanel_track", "mixpanel_user_profile"},
- DocsURL: "https://developer.mixpanel.com/reference/overview",
- },
-
- "posthog": {
- ID: "posthog",
- Name: "PostHog",
- Description: "Track events and analyze product usage with PostHog",
- Icon: "posthog",
- Category: "analytics",
- Fields: []IntegrationField{
- {
- Key: "api_key",
- Label: "Project API Key",
- Type: "api_key",
- Required: true,
- Placeholder: "phc_...",
- HelpText: "Find in PostHog: Settings → Project → Project API Key",
- Sensitive: true,
- },
- {
- Key: "host",
- Label: "PostHog Host",
- Type: "text",
- Required: false,
- Placeholder: "https://app.posthog.com",
- HelpText: "Leave empty for PostHog Cloud, or enter your self-hosted URL",
- Default: "https://app.posthog.com",
- Sensitive: false,
- },
- {
- Key: "personal_api_key",
- Label: "Personal API Key",
- Type: "api_key",
- Required: false,
- Placeholder: "phx_...",
- HelpText: "Required for querying data. Create at Settings → Personal API Keys",
- Sensitive: true,
- },
- },
- Tools: []string{"posthog_capture", "posthog_identify", "posthog_query"},
- DocsURL: "https://posthog.com/docs/api",
- },
-
- // ============================================
- // E-COMMERCE
- // ============================================
- "shopify": {
- ID: "shopify",
- Name: "Shopify",
- Description: "Manage Shopify products, orders, and customers",
- Icon: "shopify",
- Category: "ecommerce",
- Fields: []IntegrationField{
- {
- Key: "store_url",
- Label: "Store URL",
- Type: "text",
- Required: true,
- Placeholder: "your-store.myshopify.com",
- HelpText: "Your Shopify store URL (without https://)",
- Sensitive: false,
- },
- {
- Key: "access_token",
- Label: "Admin API Access Token",
- Type: "api_key",
- Required: true,
- Placeholder: "shpat_...",
- HelpText: "Create in Shopify Admin: Settings → Apps → Develop apps → Create an app",
- Sensitive: true,
- },
- },
- Tools: []string{"shopify_products", "shopify_orders", "shopify_customers"},
- DocsURL: "https://shopify.dev/docs/api/admin-rest",
- },
-
- // ============================================
- // DEPLOYMENT
- // ============================================
- "netlify": {
- ID: "netlify",
- Name: "Netlify",
- Description: "Manage Netlify sites, deploys, and DNS settings",
- Icon: "netlify",
- Category: "deployment",
- Fields: []IntegrationField{
- {
- Key: "access_token",
- Label: "Personal Access Token",
- Type: "api_key",
- Required: true,
- Placeholder: "Your Netlify Personal Access Token",
- HelpText: "Create at app.netlify.com/user/applications#personal-access-tokens",
- Sensitive: true,
- },
- },
- Tools: []string{"netlify_sites", "netlify_deploys", "netlify_trigger_build"},
- DocsURL: "https://docs.netlify.com/api/get-started/",
- },
-
- // ============================================
- // STORAGE
- // ============================================
- "aws_s3": {
- ID: "aws_s3",
- Name: "AWS S3",
- Description: "Access AWS S3 buckets for file storage",
- Icon: "aws",
- Category: "storage",
- Fields: []IntegrationField{
- {
- Key: "access_key_id",
- Label: "Access Key ID",
- Type: "api_key",
- Required: true,
- Placeholder: "AKIA...",
- HelpText: "Your AWS Access Key ID",
- Sensitive: true,
- },
- {
- Key: "secret_access_key",
- Label: "Secret Access Key",
- Type: "api_key",
- Required: true,
- Placeholder: "Your AWS Secret Access Key",
- HelpText: "Your AWS Secret Access Key",
- Sensitive: true,
- },
- {
- Key: "region",
- Label: "Region",
- Type: "text",
- Required: true,
- Placeholder: "us-east-1",
- HelpText: "AWS region for your S3 bucket",
- Default: "us-east-1",
- Sensitive: false,
- },
- {
- Key: "bucket",
- Label: "Default Bucket",
- Type: "text",
- Required: false,
- Placeholder: "my-bucket",
- HelpText: "Optional: Default S3 bucket name",
- Sensitive: false,
- },
- },
- Tools: []string{"s3_list", "s3_upload", "s3_download", "s3_delete"},
- DocsURL: "https://docs.aws.amazon.com/s3/",
- },
-
- // ============================================
- // SOCIAL MEDIA
- // ============================================
- "x_twitter": {
- ID: "x_twitter",
- Name: "X (Twitter)",
- Description: "Access X (Twitter) API v2 to post tweets, search posts, manage users, and interact with the platform programmatically.",
- Icon: "twitter",
- Category: "social",
- Fields: []IntegrationField{
- {
- Key: "bearer_token",
- Label: "Bearer Token",
- Type: "api_key",
- Required: true,
- Placeholder: "AAAAAAAAAAAAAAAAAAAAAA...",
- HelpText: "Get your Bearer Token from developer.x.com portal",
- Sensitive: true,
- },
- {
- Key: "api_key",
- Label: "API Key (Consumer Key)",
- Type: "api_key",
- Required: false,
- Placeholder: "Your API Key",
- HelpText: "Required for posting tweets (OAuth 1.0a)",
- Sensitive: true,
- },
- {
- Key: "api_secret",
- Label: "API Secret (Consumer Secret)",
- Type: "api_key",
- Required: false,
- Placeholder: "Your API Secret",
- HelpText: "Required for posting tweets (OAuth 1.0a)",
- Sensitive: true,
- },
- {
- Key: "access_token",
- Label: "Access Token",
- Type: "api_key",
- Required: false,
- Placeholder: "Your Access Token",
- HelpText: "Required for posting tweets on behalf of a user",
- Sensitive: true,
- },
- {
- Key: "access_token_secret",
- Label: "Access Token Secret",
- Type: "api_key",
- Required: false,
- Placeholder: "Your Access Token Secret",
- HelpText: "Required for posting tweets on behalf of a user",
- Sensitive: true,
- },
- },
- Tools: []string{"x_search_posts", "x_post_tweet", "x_get_user", "x_get_user_posts"},
- DocsURL: "https://docs.x.com/x-api/getting-started/about-x-api",
- },
-
- // ============================================
- // CUSTOM
- // ============================================
- "custom_webhook": {
- ID: "custom_webhook",
- Name: "Custom Webhook",
- Description: "Send data to any HTTP endpoint",
- Icon: "webhook",
- Category: "custom",
- Fields: []IntegrationField{
- {
- Key: "url",
- Label: "Webhook URL",
- Type: "webhook_url",
- Required: true,
- Placeholder: "https://your-endpoint.com/webhook",
- HelpText: "The URL to send webhook requests to",
- Sensitive: true,
- },
- {
- Key: "method",
- Label: "HTTP Method",
- Type: "select",
- Required: true,
- Options: []string{"POST", "PUT", "PATCH"},
- Default: "POST",
- HelpText: "HTTP method for the webhook request",
- Sensitive: false,
- },
- {
- Key: "auth_type",
- Label: "Authentication Type",
- Type: "select",
- Required: false,
- Options: []string{"none", "bearer", "basic", "api_key"},
- Default: "none",
- HelpText: "Type of authentication to use",
- Sensitive: false,
- },
- {
- Key: "auth_value",
- Label: "Auth Token/Key",
- Type: "api_key",
- Required: false,
- Placeholder: "Your authentication token or key",
- HelpText: "The authentication value (token, API key, or user:pass for basic)",
- Sensitive: true,
- },
- {
- Key: "headers",
- Label: "Custom Headers (JSON)",
- Type: "json",
- Required: false,
- Placeholder: `{"X-Custom-Header": "value"}`,
- HelpText: "Additional headers as JSON object",
- Sensitive: false,
- },
- },
- Tools: []string{"send_webhook"},
- DocsURL: "",
- },
-
- "rest_api": {
- ID: "rest_api",
- Name: "REST API",
- Description: "Connect to any REST API endpoint",
- Icon: "api",
- Category: "custom",
- Fields: []IntegrationField{
- {
- Key: "base_url",
- Label: "Base URL",
- Type: "text",
- Required: true,
- Placeholder: "https://api.example.com/v1",
- HelpText: "Base URL for the API (endpoints will be appended)",
- Sensitive: false,
- },
- {
- Key: "auth_type",
- Label: "Authentication Type",
- Type: "select",
- Required: false,
- Options: []string{"none", "bearer", "basic", "api_key_header", "api_key_query"},
- Default: "none",
- HelpText: "Type of authentication to use",
- Sensitive: false,
- },
- {
- Key: "auth_value",
- Label: "Auth Token/Key",
- Type: "api_key",
- Required: false,
- Placeholder: "Your authentication token or key",
- HelpText: "The authentication value",
- Sensitive: true,
- },
- {
- Key: "auth_header_name",
- Label: "API Key Header Name",
- Type: "text",
- Required: false,
- Placeholder: "X-API-Key",
- HelpText: "Header name for API key authentication",
- Default: "X-API-Key",
- Sensitive: false,
- },
- {
- Key: "headers",
- Label: "Default Headers (JSON)",
- Type: "json",
- Required: false,
- Placeholder: `{"Accept": "application/json"}`,
- HelpText: "Default headers to include in all requests",
- Sensitive: false,
- },
- },
- Tools: []string{"api_request"},
- DocsURL: "",
- },
-
- // ============================================
- // DATABASE
- // ============================================
- "mongodb": {
- ID: "mongodb",
- Name: "MongoDB",
- Description: "Query and write to MongoDB databases. Supports find, insert, update operations (no delete for safety).",
- Icon: "database",
- Category: "database",
- Fields: []IntegrationField{
- {
- Key: "connection_string",
- Label: "Connection String",
- Type: "api_key",
- Required: true,
- Placeholder: "mongodb+srv://user:password@cluster.mongodb.net",
- HelpText: "MongoDB connection URI (SRV or standard format)",
- Sensitive: true,
- },
- {
- Key: "database",
- Label: "Database Name",
- Type: "text",
- Required: true,
- Placeholder: "myDatabase",
- HelpText: "The database to connect to",
- Sensitive: false,
- },
- },
- Tools: []string{"mongodb_query", "mongodb_write"},
- DocsURL: "https://www.mongodb.com/docs/drivers/go/current/",
- },
-
- "redis": {
- ID: "redis",
- Name: "Redis",
- Description: "Read and write to Redis key-value store. Supports strings, hashes, lists, sets, and sorted sets (no delete for safety).",
- Icon: "database",
- Category: "database",
- Fields: []IntegrationField{
- {
- Key: "host",
- Label: "Host",
- Type: "text",
- Required: true,
- Placeholder: "localhost",
- HelpText: "Redis server hostname or IP",
- Default: "localhost",
- Sensitive: false,
- },
- {
- Key: "port",
- Label: "Port",
- Type: "text",
- Required: false,
- Placeholder: "6379",
- HelpText: "Redis server port (default: 6379)",
- Default: "6379",
- Sensitive: false,
- },
- {
- Key: "password",
- Label: "Password",
- Type: "api_key",
- Required: false,
- Placeholder: "Your Redis password",
- HelpText: "Redis authentication password (leave empty if not required)",
- Sensitive: true,
- },
- {
- Key: "database",
- Label: "Database Number",
- Type: "text",
- Required: false,
- Placeholder: "0",
- HelpText: "Redis database number (default: 0)",
- Default: "0",
- Sensitive: false,
- },
- },
- Tools: []string{"redis_read", "redis_write"},
- DocsURL: "https://redis.io/docs/",
- },
-}
-
-// IntegrationCategories defines the categories and their order
-var IntegrationCategories = []IntegrationCategory{
- {
- ID: "communication",
- Name: "Communication",
- Icon: "message-square",
- },
- {
- ID: "productivity",
- Name: "Productivity",
- Icon: "layout-grid",
- },
- {
- ID: "development",
- Name: "Development",
- Icon: "code",
- },
- {
- ID: "crm",
- Name: "CRM / Sales",
- Icon: "users",
- },
- {
- ID: "marketing",
- Name: "Marketing / Email",
- Icon: "mail",
- },
- {
- ID: "analytics",
- Name: "Analytics",
- Icon: "bar-chart-2",
- },
- {
- ID: "ecommerce",
- Name: "E-Commerce",
- Icon: "shopping-cart",
- },
- {
- ID: "deployment",
- Name: "Deployment",
- Icon: "rocket",
- },
- {
- ID: "storage",
- Name: "Storage",
- Icon: "hard-drive",
- },
- {
- ID: "database",
- Name: "Database",
- Icon: "database",
- },
- {
- ID: "social",
- Name: "Social Media",
- Icon: "share-2",
- },
- {
- ID: "custom",
- Name: "Custom",
- Icon: "settings",
- },
-}
-
-// GetIntegration returns an integration by ID
-func GetIntegration(id string) (Integration, bool) {
- integration, exists := IntegrationRegistry[id]
- return integration, exists
-}
-
-// GetIntegrationsByCategory returns all integrations grouped by category
-func GetIntegrationsByCategory() []IntegrationCategory {
- result := make([]IntegrationCategory, len(IntegrationCategories))
-
- for i, category := range IntegrationCategories {
- result[i] = IntegrationCategory{
- ID: category.ID,
- Name: category.Name,
- Icon: category.Icon,
- Integrations: []Integration{},
- }
-
- for _, integration := range IntegrationRegistry {
- if integration.Category == category.ID {
- result[i].Integrations = append(result[i].Integrations, integration)
- }
- }
- }
-
- return result
-}
-
-// ValidateCredentialData validates that the provided data matches the integration schema
-func ValidateCredentialData(integrationType string, data map[string]interface{}) error {
- integration, exists := IntegrationRegistry[integrationType]
- if !exists {
- return &CredentialValidationError{Field: "integrationType", Message: "unknown integration type"}
- }
-
- for _, field := range integration.Fields {
- value, hasValue := data[field.Key]
- if field.Required && (!hasValue || value == nil || value == "") {
- return &CredentialValidationError{Field: field.Key, Message: "required field is missing"}
- }
- }
-
- return nil
-}
-
-// CredentialValidationError represents a credential validation error
-type CredentialValidationError struct {
- Field string `json:"field"`
- Message string `json:"message"`
-}
-
-func (e *CredentialValidationError) Error() string {
- return e.Field + ": " + e.Message
-}
-
-// MaskCredentialValue masks a sensitive value for display
-// e.g., "sk-1234567890abcdef" -> "sk-...cdef"
-func MaskCredentialValue(value string, fieldType string) string {
- if value == "" {
- return ""
- }
-
- switch fieldType {
- case "webhook_url":
- // For URLs, show domain and mask the rest
- if len(value) > 30 {
- return value[:20] + "..." + value[len(value)-8:]
- }
- return value
-
- case "api_key", "token":
- // For API keys, show prefix and last few chars
- if len(value) > 12 {
- return value[:6] + "..." + value[len(value)-4:]
- }
- if len(value) > 6 {
- return value[:3] + "..." + value[len(value)-2:]
- }
- return "***"
-
- case "json":
- return "[JSON data]"
-
- default:
- // For other sensitive data, basic masking
- if len(value) > 8 {
- return value[:4] + "..." + value[len(value)-4:]
- }
- return "***"
- }
-}
-
-// GenerateMaskedPreview generates a masked preview for a credential
-func GenerateMaskedPreview(integrationType string, data map[string]interface{}) string {
- integration, exists := IntegrationRegistry[integrationType]
- if !exists {
- return ""
- }
-
- // Find the primary field (first required sensitive field)
- for _, field := range integration.Fields {
- if field.Required && field.Sensitive {
- if value, ok := data[field.Key].(string); ok {
- return MaskCredentialValue(value, field.Type)
- }
- }
- }
-
- // Fallback: mask first field
- for _, field := range integration.Fields {
- if value, ok := data[field.Key].(string); ok && value != "" {
- return MaskCredentialValue(value, field.Type)
- }
- }
-
- return ""
-}
diff --git a/backend/internal/models/mcp.go b/backend/internal/models/mcp.go
deleted file mode 100644
index 61cf7e7c..00000000
--- a/backend/internal/models/mcp.go
+++ /dev/null
@@ -1,69 +0,0 @@
-package models
-
-import "time"
-
-// MCPConnection represents an active MCP client connection
-type MCPConnection struct {
- ID string `json:"id"`
- UserID string `json:"user_id"`
- ClientID string `json:"client_id"`
- ClientVersion string `json:"client_version"`
- Platform string `json:"platform"`
- ConnectedAt time.Time `json:"connected_at"`
- LastHeartbeat time.Time `json:"last_heartbeat"`
- IsActive bool `json:"is_active"`
- Tools []MCPTool `json:"tools"`
- WriteChan chan MCPServerMessage `json:"-"`
- StopChan chan bool `json:"-"`
- PendingResults map[string]chan MCPToolResult `json:"-"` // call_id -> result channel
-}
-
-// MCPTool represents a tool registered by an MCP client
-type MCPTool struct {
- Name string `json:"name"`
- Description string `json:"description"`
- Parameters map[string]interface{} `json:"parameters"` // JSON Schema
- Source string `json:"source"` // "mcp_local"
- UserID string `json:"user_id"`
-}
-
-// MCPClientMessage represents messages from MCP client to backend
-type MCPClientMessage struct {
- Type string `json:"type"` // "register_tools", "tool_result", "heartbeat", "disconnect"
- Payload map[string]interface{} `json:"payload"`
-}
-
-// MCPServerMessage represents messages from backend to MCP client
-type MCPServerMessage struct {
- Type string `json:"type"` // "tool_call", "ack", "error"
- Payload map[string]interface{} `json:"payload"`
-}
-
-// MCPToolRegistration represents the registration payload from client
-type MCPToolRegistration struct {
- ClientID string `json:"client_id"`
- ClientVersion string `json:"client_version"`
- Platform string `json:"platform"`
- Tools []MCPTool `json:"tools"`
-}
-
-// MCPToolCall represents a tool execution request to client
-type MCPToolCall struct {
- CallID string `json:"call_id"`
- ToolName string `json:"tool_name"`
- Arguments map[string]interface{} `json:"arguments"`
- Timeout int `json:"timeout"` // seconds
-}
-
-// MCPToolResult represents a tool execution result from client
-type MCPToolResult struct {
- CallID string `json:"call_id"`
- Success bool `json:"success"`
- Result string `json:"result"`
- Error string `json:"error,omitempty"`
-}
-
-// MCPHeartbeat represents a heartbeat message
-type MCPHeartbeat struct {
- Timestamp time.Time `json:"timestamp"`
-}
diff --git a/backend/internal/models/memory.go b/backend/internal/models/memory.go
deleted file mode 100644
index b5cb2a14..00000000
--- a/backend/internal/models/memory.go
+++ /dev/null
@@ -1,114 +0,0 @@
-package models
-
-import (
- "time"
-
- "go.mongodb.org/mongo-driver/bson/primitive"
-)
-
-// Memory represents a single memory extracted from conversations
-type Memory struct {
- ID primitive.ObjectID `bson:"_id,omitempty" json:"id"`
- UserID string `bson:"userId" json:"user_id"`
- ConversationID string `bson:"conversationId,omitempty" json:"conversation_id,omitempty"` // Source conversation (optional)
-
- // Memory Content (encrypted)
- EncryptedContent string `bson:"encryptedContent" json:"-"` // AES-256-GCM encrypted memory text
- ContentHash string `bson:"contentHash" json:"content_hash"` // SHA-256 hash for deduplication
-
- // Metadata (plaintext for querying)
- Category string `bson:"category" json:"category"` // "personal_info", "preferences", "context", "fact", "instruction"
- Tags []string `bson:"tags,omitempty" json:"tags,omitempty"` // Searchable tags (e.g., "coding", "music", "work")
-
- // PageRank-like Scoring
- Score float64 `bson:"score" json:"score"` // Current relevance score (0.0-1.0)
- AccessCount int64 `bson:"accessCount" json:"access_count"` // How many times memory was selected/used
- LastAccessedAt *time.Time `bson:"lastAccessedAt,omitempty" json:"last_accessed_at,omitempty"`
-
- // Decay & Archival
- IsArchived bool `bson:"isArchived" json:"is_archived"` // Decayed below threshold
- ArchivedAt *time.Time `bson:"archivedAt,omitempty" json:"archived_at,omitempty"`
-
- // Engagement Metrics (for PageRank calculation)
- SourceEngagement float64 `bson:"sourceEngagement" json:"source_engagement"` // Engagement score of conversation it came from
-
- // Timestamps
- CreatedAt time.Time `bson:"createdAt" json:"created_at"`
- UpdatedAt time.Time `bson:"updatedAt" json:"updated_at"`
-
- // Version (for deduplication/updates)
- Version int64 `bson:"version" json:"version"` // Incremented on updates
-}
-
-// DecryptedMemory represents a memory with decrypted content (for internal use only)
-type DecryptedMemory struct {
- Memory
- DecryptedContent string `json:"content"` // Decrypted content
-}
-
-// MemoryExtractionJob represents a pending extraction job
-type MemoryExtractionJob struct {
- ID primitive.ObjectID `bson:"_id,omitempty" json:"id"`
- UserID string `bson:"userId" json:"user_id"`
- ConversationID string `bson:"conversationId" json:"conversation_id"`
- MessageCount int `bson:"messageCount" json:"message_count"` // Number of messages to process
- EncryptedMessages string `bson:"encryptedMessages" json:"-"` // Encrypted message batch
-
- Status string `bson:"status" json:"status"` // "pending", "processing", "completed", "failed"
- AttemptCount int `bson:"attemptCount" json:"attempt_count"` // For retry logic
- ErrorMessage string `bson:"errorMessage,omitempty" json:"error_message,omitempty"`
-
- CreatedAt time.Time `bson:"createdAt" json:"created_at"`
- ProcessedAt *time.Time `bson:"processedAt,omitempty" json:"processed_at,omitempty"`
-}
-
-// ConversationEngagement tracks engagement for PageRank calculation
-type ConversationEngagement struct {
- ID primitive.ObjectID `bson:"_id,omitempty" json:"id"`
- UserID string `bson:"userId" json:"user_id"`
- ConversationID string `bson:"conversationId" json:"conversation_id"`
-
- MessageCount int `bson:"messageCount" json:"message_count"` // Total messages in conversation
- UserMessageCount int `bson:"userMessageCount" json:"user_message_count"` // User's message count
- AvgResponseLength int `bson:"avgResponseLength" json:"avg_response_length"` // Average user response length
-
- EngagementScore float64 `bson:"engagementScore" json:"engagement_score"` // Calculated engagement (0.0-1.0)
-
- CreatedAt time.Time `bson:"createdAt" json:"created_at"`
- UpdatedAt time.Time `bson:"updatedAt" json:"updated_at"`
-}
-
-// MemoryCategory constants
-const (
- MemoryCategoryPersonalInfo = "personal_info"
- MemoryCategoryPreferences = "preferences"
- MemoryCategoryContext = "context"
- MemoryCategoryFact = "fact"
- MemoryCategoryInstruction = "instruction"
-)
-
-// MemoryExtractionJobStatus constants
-const (
- JobStatusPending = "pending"
- JobStatusProcessing = "processing"
- JobStatusCompleted = "completed"
- JobStatusFailed = "failed"
-)
-
-// Memory archive threshold (memories with score below this are archived)
-const MemoryArchiveThreshold = 0.15
-
-// ExtractedMemoryFromLLM represents the structured output from the extraction LLM
-type ExtractedMemoryFromLLM struct {
- Memories []struct {
- Content string `json:"content"`
- Category string `json:"category"`
- Tags []string `json:"tags"`
- } `json:"memories"`
-}
-
-// SelectedMemoriesFromLLM represents the structured output from the selection LLM
-type SelectedMemoriesFromLLM struct {
- SelectedMemoryIDs []string `json:"selected_memory_ids"`
- Reasoning string `json:"reasoning"`
-}
diff --git a/backend/internal/models/model.go b/backend/internal/models/model.go
deleted file mode 100644
index 3e098d85..00000000
--- a/backend/internal/models/model.go
+++ /dev/null
@@ -1,43 +0,0 @@
-package models
-
-import "time"
-
-// Model represents an LLM model from a provider
-type Model struct {
- ID string `json:"id"`
- ProviderID int `json:"provider_id"`
- ProviderName string `json:"provider_name,omitempty"`
- ProviderFavicon string `json:"provider_favicon,omitempty"`
- Name string `json:"name"`
- DisplayName string `json:"display_name,omitempty"`
- Description string `json:"description,omitempty"`
- ContextLength int `json:"context_length,omitempty"`
- SupportsTools bool `json:"supports_tools"`
- SupportsStreaming bool `json:"supports_streaming"`
- SupportsVision bool `json:"supports_vision"`
- SmartToolRouter bool `json:"smart_tool_router"` // If true, model can be used as tool predictor
- AgentsEnabled bool `json:"agents_enabled"` // If true, model is available in agent builder
- IsVisible bool `json:"is_visible"`
- SystemPrompt string `json:"system_prompt,omitempty"`
- FetchedAt time.Time `json:"fetched_at"`
-}
-
-// ModelFilter represents a filter rule for showing/hiding models
-type ModelFilter struct {
- ID int `json:"id"`
- ProviderID int `json:"provider_id"`
- ModelPattern string `json:"model_pattern"`
- Action string `json:"action"` // "include" or "exclude"
- Priority int `json:"priority"`
-}
-
-// OpenAIModelsResponse represents the response from OpenAI-compatible /v1/models endpoint
-type OpenAIModelsResponse struct {
- Object string `json:"object"`
- Data []struct {
- ID string `json:"id"`
- Object string `json:"object"`
- Created int64 `json:"created"`
- OwnedBy string `json:"owned_by"`
- } `json:"data"`
-}
diff --git a/backend/internal/models/provider.go b/backend/internal/models/provider.go
deleted file mode 100644
index 783e95b0..00000000
--- a/backend/internal/models/provider.go
+++ /dev/null
@@ -1,100 +0,0 @@
-package models
-
-import "time"
-
-// Provider represents an AI API provider (OpenAI, Anthropic, etc.)
-type Provider struct {
- ID int `json:"id"`
- Name string `json:"name"`
- BaseURL string `json:"base_url"`
- APIKey string `json:"api_key,omitempty"` // Omit from responses for security
- Enabled bool `json:"enabled"`
- AudioOnly bool `json:"audio_only,omitempty"` // If true, provider is only used for audio transcription (not shown in model list)
- ImageOnly bool `json:"image_only,omitempty"` // If true, provider is only used for image generation (not shown in model list)
- ImageEditOnly bool `json:"image_edit_only,omitempty"` // If true, provider is only used for image editing (not shown in model list)
- Secure bool `json:"secure,omitempty"` // If true, provider doesn't store user data
- DefaultModel string `json:"default_model,omitempty"` // Default model for image generation
- SystemPrompt string `json:"system_prompt,omitempty"`
- Favicon string `json:"favicon,omitempty"` // Optional favicon URL for the provider
- CreatedAt time.Time `json:"created_at"`
- UpdatedAt time.Time `json:"updated_at"`
-}
-
-// ModelAlias represents a model alias with display name and description
-type ModelAlias struct {
- ActualModel string `json:"actual_model"` // The actual model name to use with the provider API
- DisplayName string `json:"display_name"` // Human-readable name shown in the UI
- Description string `json:"description,omitempty"` // Optional description for the model
- SupportsVision *bool `json:"supports_vision,omitempty"` // Optional override for vision/image support
- Agents *bool `json:"agents,omitempty"` // If true, model is available for agent builder. If false/nil, hidden from agents
- SmartToolRouter *bool `json:"smart_tool_router,omitempty"` // If true, model can be used as tool predictor for chat
- FreeTier *bool `json:"free_tier,omitempty"` // If true, model is available for anonymous/free tier users
- StructuredOutputSupport string `json:"structured_output_support,omitempty"` // Structured output quality: "excellent", "good", "poor", "unknown"
- StructuredOutputCompliance *int `json:"structured_output_compliance,omitempty"` // Compliance percentage (0-100)
- StructuredOutputWarning string `json:"structured_output_warning,omitempty"` // Warning message about structured output
- StructuredOutputSpeedMs *int `json:"structured_output_speed_ms,omitempty"` // Average response time in milliseconds
- StructuredOutputBadge string `json:"structured_output_badge,omitempty"` // Badge label (e.g., "FASTEST")
- MemoryExtractor *bool `json:"memory_extractor,omitempty"` // If true, model can extract memories from conversations
- MemorySelector *bool `json:"memory_selector,omitempty"` // If true, model can select relevant memories for context
-}
-
-// ModelAliasView represents a model alias from the database (includes DB metadata)
-type ModelAliasView struct {
- ID int `json:"id"`
- AliasName string `json:"alias_name"`
- ModelID string `json:"model_id"`
- ProviderID int `json:"provider_id"`
- DisplayName string `json:"display_name"`
- Description *string `json:"description,omitempty"`
- SupportsVision *bool `json:"supports_vision,omitempty"`
- AgentsEnabled *bool `json:"agents_enabled,omitempty"`
- SmartToolRouter *bool `json:"smart_tool_router,omitempty"`
- FreeTier *bool `json:"free_tier,omitempty"`
- StructuredOutputSupport *string `json:"structured_output_support,omitempty"`
- StructuredOutputCompliance *int `json:"structured_output_compliance,omitempty"`
- StructuredOutputWarning *string `json:"structured_output_warning,omitempty"`
- StructuredOutputSpeedMs *int `json:"structured_output_speed_ms,omitempty"`
- StructuredOutputBadge *string `json:"structured_output_badge,omitempty"`
- MemoryExtractor *bool `json:"memory_extractor,omitempty"`
- MemorySelector *bool `json:"memory_selector,omitempty"`
- CreatedAt time.Time `json:"created_at"`
- UpdatedAt time.Time `json:"updated_at"`
-}
-
-// RecommendedModels represents recommended model tiers
-type RecommendedModels struct {
- Top string `json:"top"` // Best/most capable model
- Medium string `json:"medium"` // Balanced model
- Fastest string `json:"fastest"` // Fastest/cheapest model
- New string `json:"new"` // Newly added model
-}
-
-// ProvidersConfig represents the providers.json file structure
-type ProvidersConfig struct {
- Providers []ProviderConfig `json:"providers"`
-}
-
-// ProviderConfig represents a provider configuration from JSON
-type ProviderConfig struct {
- Name string `json:"name"`
- BaseURL string `json:"base_url"`
- APIKey string `json:"api_key"`
- Enabled bool `json:"enabled"`
- Secure bool `json:"secure,omitempty"` // Indicates provider doesn't store user data
- AudioOnly bool `json:"audio_only,omitempty"` // If true, provider is only used for audio transcription (not shown in model list)
- ImageOnly bool `json:"image_only,omitempty"` // If true, provider is only used for image generation (not shown in model list)
- ImageEditOnly bool `json:"image_edit_only,omitempty"` // If true, provider is only used for image editing (not shown in model list)
- DefaultModel string `json:"default_model,omitempty"` // Default model for image generation
- SystemPrompt string `json:"system_prompt,omitempty"`
- Favicon string `json:"favicon,omitempty"` // Optional favicon URL
- Filters []FilterConfig `json:"filters"`
- ModelAliases map[string]ModelAlias `json:"model_aliases,omitempty"` // Maps frontend model names to actual model names with descriptions
- RecommendedModels *RecommendedModels `json:"recommended_models,omitempty"` // Recommended model tiers
-}
-
-// FilterConfig represents a filter configuration from JSON
-type FilterConfig struct {
- Pattern string `json:"pattern"`
- Action string `json:"action"` // "include" or "exclude"
- Priority int `json:"priority"` // Higher priority = applied first
-}
diff --git a/backend/internal/models/schedule.go b/backend/internal/models/schedule.go
deleted file mode 100644
index 9467d8af..00000000
--- a/backend/internal/models/schedule.go
+++ /dev/null
@@ -1,171 +0,0 @@
-package models
-
-import (
- "time"
-
- "go.mongodb.org/mongo-driver/bson/primitive"
-)
-
-// Schedule represents a cron-based schedule for agent execution
-type Schedule struct {
- ID primitive.ObjectID `bson:"_id,omitempty" json:"id"`
- AgentID string `bson:"agentId" json:"agentId"`
- UserID string `bson:"userId" json:"userId"`
- CronExpression string `bson:"cronExpression" json:"cronExpression"`
- Timezone string `bson:"timezone" json:"timezone"`
- Enabled bool `bson:"enabled" json:"enabled"`
- InputTemplate map[string]any `bson:"inputTemplate,omitempty" json:"inputTemplate,omitempty"`
-
- // Tracking
- NextRunAt *time.Time `bson:"nextRunAt,omitempty" json:"nextRunAt,omitempty"`
- LastRunAt *time.Time `bson:"lastRunAt,omitempty" json:"lastRunAt,omitempty"`
-
- // Statistics
- TotalRuns int64 `bson:"totalRuns" json:"totalRuns"`
- SuccessfulRuns int64 `bson:"successfulRuns" json:"successfulRuns"`
- FailedRuns int64 `bson:"failedRuns" json:"failedRuns"`
-
- // Timestamps
- CreatedAt time.Time `bson:"createdAt" json:"createdAt"`
- UpdatedAt time.Time `bson:"updatedAt" json:"updatedAt"`
-}
-
-// CreateScheduleRequest represents a request to create a schedule
-type CreateScheduleRequest struct {
- CronExpression string `json:"cronExpression" validate:"required"`
- Timezone string `json:"timezone" validate:"required"`
- InputTemplate map[string]any `json:"inputTemplate,omitempty"`
- Enabled *bool `json:"enabled,omitempty"` // Defaults to true
-}
-
-// UpdateScheduleRequest represents a request to update a schedule
-type UpdateScheduleRequest struct {
- CronExpression *string `json:"cronExpression,omitempty"`
- Timezone *string `json:"timezone,omitempty"`
- InputTemplate map[string]any `json:"inputTemplate,omitempty"`
- Enabled *bool `json:"enabled,omitempty"`
-}
-
-// ScheduleResponse represents the API response for a schedule
-type ScheduleResponse struct {
- ID string `json:"id"`
- AgentID string `json:"agentId"`
- CronExpression string `json:"cronExpression"`
- Timezone string `json:"timezone"`
- Enabled bool `json:"enabled"`
- InputTemplate map[string]any `json:"inputTemplate,omitempty"`
- NextRunAt *time.Time `json:"nextRunAt,omitempty"`
- LastRunAt *time.Time `json:"lastRunAt,omitempty"`
- TotalRuns int64 `json:"totalRuns"`
- SuccessfulRuns int64 `json:"successfulRuns"`
- FailedRuns int64 `json:"failedRuns"`
- CreatedAt time.Time `json:"createdAt"`
- UpdatedAt time.Time `json:"updatedAt"`
-}
-
-// ToResponse converts a Schedule to ScheduleResponse
-func (s *Schedule) ToResponse() *ScheduleResponse {
- return &ScheduleResponse{
- ID: s.ID.Hex(),
- AgentID: s.AgentID,
- CronExpression: s.CronExpression,
- Timezone: s.Timezone,
- Enabled: s.Enabled,
- InputTemplate: s.InputTemplate,
- NextRunAt: s.NextRunAt,
- LastRunAt: s.LastRunAt,
- TotalRuns: s.TotalRuns,
- SuccessfulRuns: s.SuccessfulRuns,
- FailedRuns: s.FailedRuns,
- CreatedAt: s.CreatedAt,
- UpdatedAt: s.UpdatedAt,
- }
-}
-
-// TierLimits defines rate limits and quotas per subscription tier
-type TierLimits struct {
- MaxSchedules int `json:"maxSchedules"`
- MaxAPIKeys int `json:"maxApiKeys"`
- RequestsPerMinute int64 `json:"requestsPerMinute"`
- RequestsPerHour int64 `json:"requestsPerHour"`
- RetentionDays int `json:"retentionDays"`
- MaxExecutionsPerDay int64 `json:"maxExecutionsPerDay"`
-
- // Usage limits
- MaxMessagesPerMonth int64 `json:"maxMessagesPerMonth"` // Monthly message count limit
- MaxFileUploadsPerDay int64 `json:"maxFileUploadsPerDay"` // Daily file upload limit
- MaxImageGensPerDay int64 `json:"maxImageGensPerDay"` // Daily image generation limit
- MaxMemoryExtractionsPerDay int64 `json:"maxMemoryExtractionsPerDay"` // Daily memory extraction limit
-}
-
-// DefaultTierLimits provides tier configurations
-var DefaultTierLimits = map[string]TierLimits{
- "free": {
- MaxSchedules: 5,
- MaxAPIKeys: 3,
- RequestsPerMinute: 60,
- RequestsPerHour: 1000,
- RetentionDays: 30,
- MaxExecutionsPerDay: 100,
- MaxMessagesPerMonth: 300,
- MaxFileUploadsPerDay: 10,
- MaxImageGensPerDay: 10,
- MaxMemoryExtractionsPerDay: 15, // ~15 extractions/day for free tier
- },
- "pro": {
- MaxSchedules: 50,
- MaxAPIKeys: 50,
- RequestsPerMinute: 300,
- RequestsPerHour: 5000,
- RetentionDays: 30,
- MaxExecutionsPerDay: 1000,
- MaxMessagesPerMonth: 10000,
- MaxFileUploadsPerDay: 50,
- MaxImageGensPerDay: 50,
- MaxMemoryExtractionsPerDay: 100, // ~100 extractions/day for pro
- },
- "max": {
- MaxSchedules: 100,
- MaxAPIKeys: 100,
- RequestsPerMinute: 500,
- RequestsPerHour: 10000,
- RetentionDays: 30,
- MaxExecutionsPerDay: 2000,
- MaxMessagesPerMonth: -1, // unlimited
- MaxFileUploadsPerDay: -1, // unlimited
- MaxImageGensPerDay: -1, // unlimited
- MaxMemoryExtractionsPerDay: -1, // unlimited
- },
- "enterprise": {
- MaxSchedules: -1, // unlimited
- MaxAPIKeys: -1,
- RequestsPerMinute: 1000,
- RequestsPerHour: -1, // unlimited
- RetentionDays: 365,
- MaxExecutionsPerDay: -1, // unlimited
- MaxMessagesPerMonth: -1, // unlimited
- MaxFileUploadsPerDay: -1, // unlimited
- MaxImageGensPerDay: -1, // unlimited
- MaxMemoryExtractionsPerDay: -1, // unlimited
- },
- "legacy_unlimited": {
- MaxSchedules: -1, // unlimited
- MaxAPIKeys: -1, // unlimited
- RequestsPerMinute: -1, // unlimited
- RequestsPerHour: -1, // unlimited
- RetentionDays: 365,
- MaxExecutionsPerDay: -1, // unlimited
- MaxMessagesPerMonth: -1, // unlimited
- MaxFileUploadsPerDay: -1, // unlimited
- MaxImageGensPerDay: -1, // unlimited
- MaxMemoryExtractionsPerDay: -1, // unlimited
- },
-}
-
-// GetTierLimits returns the limits for a given tier
-func GetTierLimits(tier string) TierLimits {
- if limits, ok := DefaultTierLimits[tier]; ok {
- return limits
- }
- return DefaultTierLimits["free"]
-}
diff --git a/backend/internal/models/schedule_test.go b/backend/internal/models/schedule_test.go
deleted file mode 100644
index 6e6f7f2d..00000000
--- a/backend/internal/models/schedule_test.go
+++ /dev/null
@@ -1,140 +0,0 @@
-package models
-
-import (
- "testing"
- "time"
-
- "go.mongodb.org/mongo-driver/bson/primitive"
-)
-
-func TestScheduleToResponse(t *testing.T) {
- now := time.Now()
- nextRun := now.Add(1 * time.Hour)
- lastRun := now.Add(-1 * time.Hour)
-
- schedule := &Schedule{
- ID: primitive.NewObjectID(),
- AgentID: "agent-123",
- UserID: "user-456",
- CronExpression: "0 9 * * *",
- Timezone: "America/New_York",
- Enabled: true,
- InputTemplate: map[string]interface{}{"topic": "AI news"},
- NextRunAt: &nextRun,
- LastRunAt: &lastRun,
- TotalRuns: 10,
- SuccessfulRuns: 8,
- FailedRuns: 2,
- CreatedAt: now,
- UpdatedAt: now,
- }
-
- resp := schedule.ToResponse()
-
- if resp.ID != schedule.ID.Hex() {
- t.Errorf("Expected ID %s, got %s", schedule.ID.Hex(), resp.ID)
- }
-
- if resp.AgentID != schedule.AgentID {
- t.Errorf("Expected AgentID %s, got %s", schedule.AgentID, resp.AgentID)
- }
-
- if resp.CronExpression != schedule.CronExpression {
- t.Errorf("Expected CronExpression %s, got %s", schedule.CronExpression, resp.CronExpression)
- }
-
- if resp.Timezone != schedule.Timezone {
- t.Errorf("Expected Timezone %s, got %s", schedule.Timezone, resp.Timezone)
- }
-
- if resp.Enabled != schedule.Enabled {
- t.Errorf("Expected Enabled %v, got %v", schedule.Enabled, resp.Enabled)
- }
-
- if resp.TotalRuns != schedule.TotalRuns {
- t.Errorf("Expected TotalRuns %d, got %d", schedule.TotalRuns, resp.TotalRuns)
- }
-}
-
-func TestGetTierLimits(t *testing.T) {
- tests := []struct {
- tier string
- maxSchedules int
- maxAPIKeys int
- }{
- {"free", 5, 3},
- {"pro", 50, 50},
- {"max", 100, 100},
- {"enterprise", -1, -1}, // unlimited
- {"legacy_unlimited", -1, -1}, // unlimited
- {"unknown", 5, 3}, // defaults to free
- }
-
- for _, tt := range tests {
- t.Run(tt.tier, func(t *testing.T) {
- limits := GetTierLimits(tt.tier)
-
- if limits.MaxSchedules != tt.maxSchedules {
- t.Errorf("Expected MaxSchedules %d for tier %s, got %d", tt.maxSchedules, tt.tier, limits.MaxSchedules)
- }
-
- if limits.MaxAPIKeys != tt.maxAPIKeys {
- t.Errorf("Expected MaxAPIKeys %d for tier %s, got %d", tt.maxAPIKeys, tt.tier, limits.MaxAPIKeys)
- }
- })
- }
-}
-
-func TestDefaultTierLimits(t *testing.T) {
- // Verify all expected tiers exist
- expectedTiers := []string{"free", "pro", "max", "enterprise", "legacy_unlimited"}
-
- for _, tier := range expectedTiers {
- if _, ok := DefaultTierLimits[tier]; !ok {
- t.Errorf("Expected tier %s in DefaultTierLimits", tier)
- }
- }
-
- // Verify free tier has reasonable defaults
- freeLimits := DefaultTierLimits["free"]
- if freeLimits.MaxSchedules <= 0 {
- t.Error("Free tier should have positive MaxSchedules limit")
- }
- if freeLimits.RequestsPerMinute <= 0 {
- t.Error("Free tier should have positive RequestsPerMinute limit")
- }
- if freeLimits.RetentionDays <= 0 {
- t.Error("Free tier should have positive RetentionDays")
- }
-
- // Verify enterprise tier has unlimited (-1) for schedules and API keys
- enterpriseLimits := DefaultTierLimits["enterprise"]
- if enterpriseLimits.MaxSchedules != -1 {
- t.Error("Enterprise tier should have unlimited MaxSchedules (-1)")
- }
- if enterpriseLimits.MaxAPIKeys != -1 {
- t.Error("Enterprise tier should have unlimited MaxAPIKeys (-1)")
- }
-}
-
-func TestCreateScheduleRequest(t *testing.T) {
- enabled := true
- req := CreateScheduleRequest{
- CronExpression: "0 9 * * *",
- Timezone: "UTC",
- InputTemplate: map[string]interface{}{"key": "value"},
- Enabled: &enabled,
- }
-
- if req.CronExpression != "0 9 * * *" {
- t.Errorf("Expected CronExpression '0 9 * * *', got %s", req.CronExpression)
- }
-
- if req.Timezone != "UTC" {
- t.Errorf("Expected Timezone 'UTC', got %s", req.Timezone)
- }
-
- if req.Enabled == nil || *req.Enabled != true {
- t.Error("Expected Enabled to be true")
- }
-}
diff --git a/backend/internal/models/subscription.go b/backend/internal/models/subscription.go
deleted file mode 100644
index f9886f84..00000000
--- a/backend/internal/models/subscription.go
+++ /dev/null
@@ -1,195 +0,0 @@
-package models
-
-import (
- "time"
-
- "go.mongodb.org/mongo-driver/bson/primitive"
-)
-
-// Subscription status constants
-const (
- SubStatusActive = "active"
- SubStatusOnHold = "on_hold" // Payment failed, grace period
- SubStatusPendingCancel = "pending_cancel" // Will cancel at period end
- SubStatusCancelled = "cancelled"
- SubStatusPaused = "paused"
-)
-
-// Subscription tiers
-const (
- TierFree = "free"
- TierPro = "pro"
- TierMax = "max"
- TierEnterprise = "enterprise"
- TierLegacyUnlimited = "legacy_unlimited" // For grandfathered users
-)
-
-// Plan represents a subscription plan with pricing
-type Plan struct {
- ID string `json:"id"`
- Name string `json:"name"`
- Tier string `json:"tier"`
- PriceMonthly int64 `json:"price_monthly"` // cents
- DodoProductID string `json:"dodo_product_id"`
- Features []string `json:"features"`
- Limits TierLimits `json:"limits"`
- ContactSales bool `json:"contact_sales"` // true for enterprise
-}
-
-// Subscription tracks a user's subscription state
-type Subscription struct {
- ID primitive.ObjectID `bson:"_id,omitempty" json:"id"`
- UserID string `bson:"userId" json:"user_id"`
- DodoSubscriptionID string `bson:"dodoSubscriptionId,omitempty" json:"dodo_subscription_id,omitempty"`
- DodoCustomerID string `bson:"dodoCustomerId,omitempty" json:"dodo_customer_id,omitempty"`
-
- // Current state
- Tier string `bson:"tier" json:"tier"`
- Status string `bson:"status" json:"status"`
-
- // Billing info
- CurrentPeriodStart time.Time `bson:"currentPeriodStart,omitempty" json:"current_period_start,omitempty"`
- CurrentPeriodEnd time.Time `bson:"currentPeriodEnd,omitempty" json:"current_period_end,omitempty"`
-
- // Scheduled changes (for downgrades/cancellations)
- ScheduledTier string `bson:"scheduledTier,omitempty" json:"scheduled_tier,omitempty"`
- ScheduledChangeAt *time.Time `bson:"scheduledChangeAt,omitempty" json:"scheduled_change_at,omitempty"`
- CancelAtPeriodEnd bool `bson:"cancelAtPeriodEnd" json:"cancel_at_period_end"`
-
- // Timestamps
- CreatedAt time.Time `bson:"createdAt" json:"created_at"`
- UpdatedAt time.Time `bson:"updatedAt" json:"updated_at"`
- CancelledAt *time.Time `bson:"cancelledAt,omitempty" json:"cancelled_at,omitempty"`
-}
-
-// IsActive returns true if subscription is currently active (user has access)
-func (s *Subscription) IsActive() bool {
- switch s.Status {
- case SubStatusActive, SubStatusOnHold, SubStatusPendingCancel:
- return true
- default:
- return false
- }
-}
-
-// IsExpired returns true if subscription period has ended
-func (s *Subscription) IsExpired() bool {
- return !s.CurrentPeriodEnd.IsZero() && s.CurrentPeriodEnd.Before(time.Now())
-}
-
-// HasScheduledChange returns true if there's a scheduled tier change
-func (s *Subscription) HasScheduledChange() bool {
- return s.ScheduledTier != "" && s.ScheduledChangeAt != nil
-}
-
-// SubscriptionEvent for audit logging
-type SubscriptionEvent struct {
- ID primitive.ObjectID `bson:"_id,omitempty" json:"id"`
- UserID string `bson:"userId" json:"user_id"`
- SubscriptionID string `bson:"subscriptionId" json:"subscription_id"`
- EventType string `bson:"eventType" json:"event_type"`
- FromTier string `bson:"fromTier,omitempty" json:"from_tier,omitempty"`
- ToTier string `bson:"toTier,omitempty" json:"to_tier,omitempty"`
- DodoEventID string `bson:"dodoEventId,omitempty" json:"dodo_event_id,omitempty"`
- Metadata map[string]any `bson:"metadata,omitempty" json:"metadata,omitempty"`
- CreatedAt time.Time `bson:"createdAt" json:"created_at"`
-}
-
-// TierOrder defines the order of tiers for comparison
-var TierOrder = map[string]int{
- TierFree: 0,
- TierPro: 1,
- TierMax: 2,
- TierEnterprise: 3,
- TierLegacyUnlimited: 4, // Highest tier
-}
-
-// CompareTiers compares two tiers and returns:
-// -1 if fromTier < toTier (upgrade)
-// 0 if fromTier == toTier (same)
-// 1 if fromTier > toTier (downgrade)
-func CompareTiers(fromTier, toTier string) int {
- fromOrder, fromOk := TierOrder[fromTier]
- toOrder, toOk := TierOrder[toTier]
-
- if !fromOk || !toOk {
- // Unknown tier, treat as same
- return 0
- }
-
- if fromOrder < toOrder {
- return -1
- } else if fromOrder > toOrder {
- return 1
- }
- return 0
-}
-
-// AvailablePlans returns all available subscription plans
-var AvailablePlans = []Plan{
- {
- ID: "free",
- Name: "Free",
- Tier: TierFree,
- PriceMonthly: 0,
- DodoProductID: "",
- Features: []string{"Basic features", "Limited usage"},
- Limits: GetTierLimits(TierFree),
- ContactSales: false,
- },
- {
- ID: "pro",
- Name: "Pro",
- Tier: TierPro,
- PriceMonthly: 1499, // $14.99 in cents - configure DodoProductID when ready
- DodoProductID: "pdt_0NVGlqj3fgVkEeAmygtuj", // Set when creating product in DodoPayments
- Features: []string{"Advanced features", "Higher limits", "Priority support"},
- Limits: GetTierLimits(TierPro),
- ContactSales: false,
- },
- {
- ID: "max",
- Name: "Max",
- Tier: TierMax,
- PriceMonthly: 3999, // $39.99 in cents - configure DodoProductID when ready
- DodoProductID: "pdt_0NVGm0KQk5F4a8NVoaQst", // Set when creating product in DodoPayments
- Features: []string{"All Pro features", "Maximum limits", "Premium support"},
- Limits: GetTierLimits(TierMax),
- ContactSales: false,
- },
- {
- ID: "enterprise",
- Name: "Enterprise",
- Tier: TierEnterprise,
- PriceMonthly: 0,
- DodoProductID: "",
- Features: []string{"Custom features", "Unlimited usage", "Dedicated support"},
- Limits: GetTierLimits(TierEnterprise),
- ContactSales: true,
- },
-}
-
-// GetPlanByID returns a plan by its ID
-func GetPlanByID(planID string) *Plan {
- for i := range AvailablePlans {
- if AvailablePlans[i].ID == planID {
- return &AvailablePlans[i]
- }
- }
- return nil
-}
-
-// GetPlanByTier returns a plan by its tier
-func GetPlanByTier(tier string) *Plan {
- for i := range AvailablePlans {
- if AvailablePlans[i].Tier == tier {
- return &AvailablePlans[i]
- }
- }
- return nil
-}
-
-// GetAvailablePlans returns all available plans
-func GetAvailablePlans() []Plan {
- return AvailablePlans
-}
diff --git a/backend/internal/models/subscription_test.go b/backend/internal/models/subscription_test.go
deleted file mode 100644
index 9bf29ed7..00000000
--- a/backend/internal/models/subscription_test.go
+++ /dev/null
@@ -1,168 +0,0 @@
-package models
-
-import (
- "testing"
- "time"
-)
-
-func TestPlanComparison(t *testing.T) {
- tests := []struct {
- name string
- fromTier string
- toTier string
- expected int // -1 = upgrade, 0 = same, 1 = downgrade
- }{
- {"free to pro is upgrade", TierFree, TierPro, -1},
- {"free to max is upgrade", TierFree, TierMax, -1},
- {"pro to max is upgrade", TierPro, TierMax, -1},
- {"max to pro is downgrade", TierMax, TierPro, 1},
- {"pro to free is downgrade", TierPro, TierFree, 1},
- {"max to free is downgrade", TierMax, TierFree, 1},
- {"same tier", TierPro, TierPro, 0},
- }
-
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- result := CompareTiers(tt.fromTier, tt.toTier)
- if result != tt.expected {
- t.Errorf("CompareTiers(%s, %s) = %d, want %d",
- tt.fromTier, tt.toTier, result, tt.expected)
- }
- })
- }
-}
-
-func TestSubscriptionStatus_IsActive(t *testing.T) {
- tests := []struct {
- status string
- expected bool
- }{
- {SubStatusActive, true},
- {SubStatusOnHold, true}, // Still active during grace
- {SubStatusPendingCancel, true}, // Active until period ends
- {SubStatusCancelled, false},
- {SubStatusPaused, false},
- }
-
- for _, tt := range tests {
- t.Run(tt.status, func(t *testing.T) {
- sub := &Subscription{Status: tt.status}
- if sub.IsActive() != tt.expected {
- t.Errorf("IsActive() for status %s = %v, want %v",
- tt.status, sub.IsActive(), tt.expected)
- }
- })
- }
-}
-
-func TestSubscription_IsExpired(t *testing.T) {
- now := time.Now()
-
- tests := []struct {
- name string
- periodEnd time.Time
- expected bool
- }{
- {"future date", now.Add(24 * time.Hour), false},
- {"past date", now.Add(-24 * time.Hour), true},
- {"just now", now, true},
- }
-
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- sub := &Subscription{CurrentPeriodEnd: tt.periodEnd}
- if sub.IsExpired() != tt.expected {
- t.Errorf("IsExpired() = %v, want %v", sub.IsExpired(), tt.expected)
- }
- })
- }
-}
-
-func TestSubscription_HasScheduledChange(t *testing.T) {
- future := time.Now().Add(24 * time.Hour)
-
- tests := []struct {
- name string
- scheduledTier string
- scheduledAt *time.Time
- expected bool
- }{
- {"no scheduled change", "", nil, false},
- {"has scheduled downgrade", TierPro, &future, true},
- {"empty tier with date", "", &future, false},
- }
-
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- sub := &Subscription{
- ScheduledTier: tt.scheduledTier,
- ScheduledChangeAt: tt.scheduledAt,
- }
- if sub.HasScheduledChange() != tt.expected {
- t.Errorf("HasScheduledChange() = %v, want %v",
- sub.HasScheduledChange(), tt.expected)
- }
- })
- }
-}
-
-func TestPlan_GetByID(t *testing.T) {
- plans := GetAvailablePlans()
-
- // Test finding each plan
- for _, plan := range plans {
- found := GetPlanByID(plan.ID)
- if found == nil {
- t.Errorf("GetPlanByID(%s) returned nil", plan.ID)
- }
- if found != nil && found.ID != plan.ID {
- t.Errorf("GetPlanByID(%s) returned plan with ID %s", plan.ID, found.ID)
- }
- }
-
- // Test non-existent plan
- if GetPlanByID("nonexistent") != nil {
- t.Error("Expected nil for non-existent plan")
- }
-}
-
-func TestTierLimits_Max(t *testing.T) {
- limits := GetTierLimits(TierMax)
-
- if limits.MaxSchedules != 100 {
- t.Errorf("Expected MaxSchedules 100, got %d", limits.MaxSchedules)
- }
- if limits.MaxAPIKeys != 100 {
- t.Errorf("Expected MaxAPIKeys 100, got %d", limits.MaxAPIKeys)
- }
-}
-
-func TestGetPlanByTier(t *testing.T) {
- tests := []struct {
- tier string
- expected string
- }{
- {TierFree, "free"},
- {TierPro, "pro"},
- {TierMax, "max"},
- {TierEnterprise, "enterprise"},
- {"invalid", ""},
- }
-
- for _, tt := range tests {
- t.Run(tt.tier, func(t *testing.T) {
- plan := GetPlanByTier(tt.tier)
- if tt.expected == "" {
- if plan != nil {
- t.Errorf("Expected nil for invalid tier, got %v", plan)
- }
- } else {
- if plan == nil {
- t.Errorf("Expected plan for tier %s, got nil", tt.tier)
- } else if plan.ID != tt.expected {
- t.Errorf("Expected plan ID %s, got %s", tt.expected, plan.ID)
- }
- }
- })
- }
-}
diff --git a/backend/internal/models/team.go b/backend/internal/models/team.go
deleted file mode 100644
index 61b72a78..00000000
--- a/backend/internal/models/team.go
+++ /dev/null
@@ -1,169 +0,0 @@
-package models
-
-import (
- "time"
-
- "go.mongodb.org/mongo-driver/bson/primitive"
-)
-
-// Team represents a team for collaborative agent access
-// NOTE: This is schema-only for now - team logic not yet implemented
-type Team struct {
- ID primitive.ObjectID `bson:"_id,omitempty" json:"id"`
- Name string `bson:"name" json:"name"`
- OwnerID string `bson:"ownerId" json:"ownerId"` // Supabase user ID
-
- // Team members
- Members []TeamMember `bson:"members" json:"members"`
-
- // Team settings
- Settings TeamSettings `bson:"settings" json:"settings"`
-
- CreatedAt time.Time `bson:"createdAt" json:"createdAt"`
- UpdatedAt time.Time `bson:"updatedAt" json:"updatedAt"`
-}
-
-// TeamMember represents a member of a team
-type TeamMember struct {
- UserID string `bson:"userId" json:"userId"`
- Role string `bson:"role" json:"role"` // owner, admin, editor, viewer
- AddedAt time.Time `bson:"addedAt" json:"addedAt"`
- AddedBy string `bson:"addedBy" json:"addedBy"`
-}
-
-// TeamSettings contains team configuration
-type TeamSettings struct {
- DefaultAgentVisibility string `bson:"defaultAgentVisibility" json:"defaultAgentVisibility"` // private, team
-}
-
-// TeamRole constants
-const (
- TeamRoleOwner = "owner"
- TeamRoleAdmin = "admin"
- TeamRoleEditor = "editor"
- TeamRoleViewer = "viewer"
-)
-
-// IsOwner checks if a user is the team owner
-func (t *Team) IsOwner(userID string) bool {
- return t.OwnerID == userID
-}
-
-// GetMemberRole returns the role of a member (empty if not a member)
-func (t *Team) GetMemberRole(userID string) string {
- if t.OwnerID == userID {
- return TeamRoleOwner
- }
- for _, m := range t.Members {
- if m.UserID == userID {
- return m.Role
- }
- }
- return ""
-}
-
-// HasMember checks if a user is a team member
-func (t *Team) HasMember(userID string) bool {
- return t.GetMemberRole(userID) != ""
-}
-
-// CanManageTeam checks if a user can manage team settings
-func (t *Team) CanManageTeam(userID string) bool {
- role := t.GetMemberRole(userID)
- return role == TeamRoleOwner || role == TeamRoleAdmin
-}
-
-// CanEditAgents checks if a user can edit team agents
-func (t *Team) CanEditAgents(userID string) bool {
- role := t.GetMemberRole(userID)
- return role == TeamRoleOwner || role == TeamRoleAdmin || role == TeamRoleEditor
-}
-
-// CanViewAgents checks if a user can view team agents
-func (t *Team) CanViewAgents(userID string) bool {
- return t.HasMember(userID)
-}
-
-// AgentPermission represents granular agent access permissions
-// NOTE: This is schema-only for now - permission logic not yet implemented
-type AgentPermission struct {
- ID primitive.ObjectID `bson:"_id,omitempty" json:"id"`
- AgentID string `bson:"agentId" json:"agentId"`
-
- // Who has access (one of these should be set)
- TeamID primitive.ObjectID `bson:"teamId,omitempty" json:"teamId,omitempty"` // If shared with team
- UserID string `bson:"userId,omitempty" json:"userId,omitempty"` // If shared with individual
-
- // What access level
- Permission string `bson:"permission" json:"permission"` // view, execute, edit, admin
-
- GrantedBy string `bson:"grantedBy" json:"grantedBy"`
- GrantedAt time.Time `bson:"grantedAt" json:"grantedAt"`
-}
-
-// Permission level constants
-const (
- PermissionView = "view" // Can view agent and executions
- PermissionExecute = "execute" // Can execute agent
- PermissionEdit = "edit" // Can edit agent workflow
- PermissionAdmin = "admin" // Full control including sharing
-)
-
-// CanView checks if the permission allows viewing
-func (p *AgentPermission) CanView() bool {
- return p.Permission == PermissionView ||
- p.Permission == PermissionExecute ||
- p.Permission == PermissionEdit ||
- p.Permission == PermissionAdmin
-}
-
-// CanExecute checks if the permission allows execution
-func (p *AgentPermission) CanExecute() bool {
- return p.Permission == PermissionExecute ||
- p.Permission == PermissionEdit ||
- p.Permission == PermissionAdmin
-}
-
-// CanEdit checks if the permission allows editing
-func (p *AgentPermission) CanEdit() bool {
- return p.Permission == PermissionEdit ||
- p.Permission == PermissionAdmin
-}
-
-// CanAdmin checks if the permission allows admin access
-func (p *AgentPermission) CanAdmin() bool {
- return p.Permission == PermissionAdmin
-}
-
-// AgentVisibility constants for agent model extension
-const (
- VisibilityPrivate = "private" // Only owner
- VisibilityTeam = "team" // Team members
- VisibilityPublic = "public" // Anyone (future)
-)
-
-// CreateTeamRequest is the request body for creating a team
-type CreateTeamRequest struct {
- Name string `json:"name"`
-}
-
-// InviteMemberRequest is the request body for inviting a team member
-type InviteMemberRequest struct {
- Email string `json:"email"` // User email to invite
- Role string `json:"role"` // Role to assign
-}
-
-// ShareAgentRequest is the request body for sharing an agent
-type ShareAgentRequest struct {
- TeamID string `json:"teamId,omitempty"` // Share with team
- UserID string `json:"userId,omitempty"` // Share with user
- Permission string `json:"permission"` // Permission level
-}
-
-// TeamListItem is a lightweight team representation
-type TeamListItem struct {
- ID string `json:"id"`
- Name string `json:"name"`
- Role string `json:"role"` // Current user's role
- MemberCount int `json:"memberCount"`
-}
diff --git a/backend/internal/models/user.go b/backend/internal/models/user.go
deleted file mode 100644
index b3ab99a8..00000000
--- a/backend/internal/models/user.go
+++ /dev/null
@@ -1,149 +0,0 @@
-package models
-
-import (
- "time"
-
- "go.mongodb.org/mongo-driver/bson/primitive"
-)
-
-// User represents a user with local JWT authentication
-type User struct {
- ID primitive.ObjectID `bson:"_id,omitempty" json:"id"`
- SupabaseUserID string `bson:"supabaseUserId,omitempty" json:"supabase_user_id,omitempty"` // Legacy field (optional)
- Email string `bson:"email" json:"email"`
- CreatedAt time.Time `bson:"createdAt" json:"created_at"`
- LastLoginAt time.Time `bson:"lastLoginAt" json:"last_login_at"`
-
- // Local authentication fields (v2.0)
- PasswordHash string `bson:"passwordHash,omitempty" json:"-"` // Argon2id hash (never expose in API)
- EmailVerified bool `bson:"emailVerified" json:"email_verified"`
- RefreshTokenVersion int `bson:"refreshTokenVersion" json:"-"` // For token revocation (never expose)
- Role string `bson:"role,omitempty" json:"role,omitempty"` // user, admin
-
- // Subscription fields
- SubscriptionTier string `bson:"subscriptionTier,omitempty" json:"subscription_tier,omitempty"`
- SubscriptionStatus string `bson:"subscriptionStatus,omitempty" json:"subscription_status,omitempty"`
- SubscriptionExpiresAt *time.Time `bson:"subscriptionExpiresAt,omitempty" json:"subscription_expires_at,omitempty"`
-
- // Migration audit trail
- MigratedToLegacyAt *time.Time `bson:"migratedToLegacyAt,omitempty" json:"migrated_to_legacy_at,omitempty"`
-
- // Manual tier override (set by superadmin)
- TierOverride *string `bson:"tierOverride,omitempty" json:"tier_override,omitempty"` // Override tier (if set)
-
- // Granular limit overrides (set by superadmin) - per-feature overrides
- LimitOverrides *TierLimits `bson:"limitOverrides,omitempty" json:"limit_overrides,omitempty"` // Custom limits per user
-
- // Audit trail for overrides
- OverrideSetBy string `bson:"overrideSetBy,omitempty" json:"override_set_by,omitempty"` // Admin user ID
- OverrideSetAt *time.Time `bson:"overrideSetAt,omitempty" json:"override_set_at,omitempty"` // When override was set
- OverrideReason string `bson:"overrideReason,omitempty" json:"override_reason,omitempty"` // Why override was set
-
- // DodoPayments integration
- DodoCustomerID string `bson:"dodoCustomerId,omitempty" json:"dodo_customer_id,omitempty"`
- DodoSubscriptionID string `bson:"dodoSubscriptionId,omitempty" json:"-"` // Don't expose in API
-
- // User preferences
- Preferences UserPreferences `bson:"preferences" json:"preferences"`
-
- // Onboarding state
- HasSeenWelcomePopup bool `bson:"hasSeenWelcomePopup" json:"has_seen_welcome_popup"`
-}
-
-// ChatPrivacyMode represents how chats are stored
-type ChatPrivacyMode string
-
-const (
- ChatPrivacyModeLocal ChatPrivacyMode = "local"
- ChatPrivacyModeCloud ChatPrivacyMode = "cloud"
-)
-
-// UserPreferences holds user-specific settings
-type UserPreferences struct {
- StoreBuilderChatHistory bool `bson:"storeBuilderChatHistory" json:"store_builder_chat_history"`
- DefaultModelID string `bson:"defaultModelId,omitempty" json:"default_model_id,omitempty"`
- ToolPredictorModelID string `bson:"toolPredictorModelId,omitempty" json:"tool_predictor_model_id,omitempty"`
- ChatPrivacyMode ChatPrivacyMode `bson:"chatPrivacyMode,omitempty" json:"chat_privacy_mode,omitempty"`
- Theme string `bson:"theme,omitempty" json:"theme,omitempty"`
- FontSize string `bson:"fontSize,omitempty" json:"font_size,omitempty"`
-
- // Memory system preferences
- MemoryEnabled bool `bson:"memoryEnabled" json:"memory_enabled"` // Default: false (opt-in)
- MemoryExtractionThreshold int `bson:"memoryExtractionThreshold,omitempty" json:"memory_extraction_threshold,omitempty"` // Default: 2 messages (for quick testing, range: 2-50)
- MemoryMaxInjection int `bson:"memoryMaxInjection,omitempty" json:"memory_max_injection,omitempty"` // Default: 5 memories
- MemoryExtractorModelID string `bson:"memoryExtractorModelId,omitempty" json:"memory_extractor_model_id,omitempty"`
- MemorySelectorModelID string `bson:"memorySelectorModelId,omitempty" json:"memory_selector_model_id,omitempty"`
-}
-
-// UpdateUserPreferencesRequest is the request body for updating preferences
-type UpdateUserPreferencesRequest struct {
- StoreBuilderChatHistory *bool `json:"store_builder_chat_history,omitempty"`
- DefaultModelID *string `json:"default_model_id,omitempty"`
- ToolPredictorModelID *string `json:"tool_predictor_model_id,omitempty"`
- ChatPrivacyMode *ChatPrivacyMode `json:"chat_privacy_mode,omitempty"`
- Theme *string `json:"theme,omitempty"`
- FontSize *string `json:"font_size,omitempty"`
-
- // Memory system preferences
- MemoryEnabled *bool `json:"memory_enabled,omitempty"`
- MemoryExtractionThreshold *int `json:"memory_extraction_threshold,omitempty"`
- MemoryMaxInjection *int `json:"memory_max_injection,omitempty"`
- MemoryExtractorModelID *string `json:"memory_extractor_model_id,omitempty"`
- MemorySelectorModelID *string `json:"memory_selector_model_id,omitempty"`
-}
-
-// UserResponse is the API response for user data
-type UserResponse struct {
- ID string `json:"id"`
- Email string `json:"email"`
- Role string `json:"role"`
- CreatedAt time.Time `json:"created_at"`
- LastLoginAt time.Time `json:"last_login_at"`
- SubscriptionTier string `json:"subscription_tier,omitempty"`
- SubscriptionStatus string `json:"subscription_status,omitempty"`
- SubscriptionExpiresAt *time.Time `json:"subscription_expires_at,omitempty"`
- Preferences UserPreferences `json:"preferences"`
- HasSeenWelcomePopup bool `json:"has_seen_welcome_popup"`
-}
-
-// ToResponse converts User to UserResponse
-func (u *User) ToResponse() UserResponse {
- return UserResponse{
- ID: u.ID.Hex(),
- Email: u.Email,
- Role: u.Role,
- CreatedAt: u.CreatedAt,
- LastLoginAt: u.LastLoginAt,
- SubscriptionTier: u.SubscriptionTier,
- SubscriptionStatus: u.SubscriptionStatus,
- SubscriptionExpiresAt: u.SubscriptionExpiresAt,
- Preferences: u.Preferences,
- HasSeenWelcomePopup: u.HasSeenWelcomePopup,
- }
-}
-
-// SetLimitOverridesRequest for admin to set granular limit overrides
-type SetLimitOverridesRequest struct {
- // Option 1: Set entire tier (simple)
- Tier *string `json:"tier,omitempty"` // Set to a tier name
-
- // Option 2: Set custom limits (granular)
- Limits *TierLimits `json:"limits,omitempty"` // Custom limits
-
- // Metadata
- Reason string `json:"reason"` // Why override is being set
-}
-
-// AdminUserResponse includes override information
-type AdminUserResponse struct {
- UserResponse // Embed normal user response
- EffectiveTier string `json:"effective_tier"` // Tier being used
- EffectiveLimits TierLimits `json:"effective_limits"` // Actual limits after overrides
- HasTierOverride bool `json:"has_tier_override"` // Whether tier is overridden
- HasLimitOverrides bool `json:"has_limit_overrides"` // Whether limits are overridden
- TierOverride *string `json:"tier_override,omitempty"` //
- LimitOverrides *TierLimits `json:"limit_overrides,omitempty"` //
- OverrideSetBy string `json:"override_set_by,omitempty"` //
- OverrideSetAt *time.Time `json:"override_set_at,omitempty"` //
- OverrideReason string `json:"override_reason,omitempty"` //
-}
diff --git a/backend/internal/models/websocket.go b/backend/internal/models/websocket.go
deleted file mode 100644
index 3b6956e9..00000000
--- a/backend/internal/models/websocket.go
+++ /dev/null
@@ -1,183 +0,0 @@
-package models
-
-import (
- "sync"
- "time"
-
- "github.com/gofiber/contrib/websocket"
-)
-
-// ClientMessage represents a message from the client
-type ClientMessage struct {
- Type string `json:"type"` // "chat_message", "new_conversation", "stop_generation", "resume_stream", or "interactive_prompt_response"
- ConversationID string `json:"conversation_id"`
- Content string `json:"content,omitempty"`
- History []map[string]interface{} `json:"history,omitempty"` // Optional: Client provides conversation history
- ModelID string `json:"model_id,omitempty"` // Option 1: Select from platform models
- CustomConfig *CustomAPIConfig `json:"custom_config,omitempty"` // Option 2: Bring your own API key
- SystemInstructions string `json:"system_instructions,omitempty"` // Optional: Custom system prompt override
- Attachments []MessageAttachment `json:"attachments,omitempty"` // File attachments (images, documents)
- DisableTools bool `json:"disable_tools,omitempty"` // Disable tools for this message (e.g., agent builder)
-
- // Interactive prompt response fields
- PromptID string `json:"prompt_id,omitempty"` // ID of the prompt being responded to
- Answers map[string]InteractiveAnswer `json:"answers,omitempty"` // Map of question_id -> answer
- Skipped bool `json:"skipped,omitempty"` // True if user skipped/cancelled
-}
-
-// MessageAttachment represents a file attachment in a message
-type MessageAttachment struct {
- Type string `json:"type"` // "image", "document", "audio"
- FileID string `json:"file_id"` // UUID from upload endpoint
- URL string `json:"url"` // File URL (e.g., "/uploads/uuid.jpg")
- MimeType string `json:"mime_type"` // MIME type (e.g., "image/jpeg")
- Size int64 `json:"size"` // File size in bytes
- Filename string `json:"filename"` // Original filename
-}
-
-// CustomAPIConfig allows users to bring their own API keys (BYOK)
-type CustomAPIConfig struct {
- BaseURL string `json:"base_url,omitempty"`
- APIKey string `json:"api_key,omitempty"`
- Model string `json:"model,omitempty"`
-}
-
-// InteractiveQuestion represents a question in an interactive prompt
-type InteractiveQuestion struct {
- ID string `json:"id"` // Unique question ID
- Type string `json:"type"` // "text", "select", "multi-select", "number", "checkbox"
- Label string `json:"label"` // Question text
- Placeholder string `json:"placeholder,omitempty"` // Placeholder for text inputs
- Required bool `json:"required,omitempty"` // Whether answer is required
- Options []string `json:"options,omitempty"` // Options for select/multi-select
- AllowOther bool `json:"allow_other,omitempty"` // Enable "Other" option
- DefaultValue interface{} `json:"default_value,omitempty"` // Default value
- Validation *QuestionValidation `json:"validation,omitempty"` // Validation rules
-}
-
-// QuestionValidation represents validation rules for a question
-type QuestionValidation struct {
- Min *float64 `json:"min,omitempty"` // Minimum value for number type
- Max *float64 `json:"max,omitempty"` // Maximum value for number type
- Pattern string `json:"pattern,omitempty"` // Regex pattern for text type
- MinLength *int `json:"min_length,omitempty"` // Minimum length for text
- MaxLength *int `json:"max_length,omitempty"` // Maximum length for text
-}
-
-// InteractiveAnswer represents a user's answer to a question
-type InteractiveAnswer struct {
- QuestionID string `json:"question_id"` // ID of the question
- Value interface{} `json:"value"` // Answer value (string, number, bool, or []string)
- IsOther bool `json:"is_other,omitempty"` // True if "Other" option selected
-}
-
-// ServerMessage represents a message sent to the client
-type ServerMessage struct {
- Type string `json:"type"` // "stream_chunk", "reasoning_chunk", "tool_call", "tool_result", "stream_end", "stream_resume", "stream_missed", "conversation_reset", "conversation_title", "context_optimizing", "interactive_prompt", "prompt_timeout", "prompt_validation_error", "error"
- Content string `json:"content,omitempty"`
- Title string `json:"title,omitempty"` // Auto-generated conversation title OR interactive prompt title
- ToolName string `json:"tool_name,omitempty"`
- ToolDisplayName string `json:"tool_display_name,omitempty"` // User-friendly tool name (e.g., "Search Web")
- ToolIcon string `json:"tool_icon,omitempty"` // Lucide React icon name (e.g., "Calculator", "Search", "Clock")
- ToolDescription string `json:"tool_description,omitempty"` // Human-readable tool description
- Status string `json:"status,omitempty"` // "executing", "completed", "started"
- Arguments map[string]interface{} `json:"arguments,omitempty"`
- Result string `json:"result,omitempty"`
- Plots []PlotData `json:"plots,omitempty"` // Visualization plots from E2B tools
- ConversationID string `json:"conversation_id,omitempty"`
- Tokens *TokenUsage `json:"tokens,omitempty"`
- ErrorCode string `json:"code,omitempty"`
- ErrorMessage string `json:"message,omitempty"`
- IsComplete bool `json:"is_complete,omitempty"` // For stream_resume: whether generation completed
- Reason string `json:"reason,omitempty"` // For stream_missed: "expired" or "not_found"
- Progress int `json:"progress,omitempty"` // For context_optimizing: progress percentage (0-100)
-
- // Interactive prompt fields
- PromptID string `json:"prompt_id,omitempty"` // Unique prompt ID
- Description string `json:"description,omitempty"` // Optional prompt description
- Questions []InteractiveQuestion `json:"questions,omitempty"` // Array of questions
- AllowSkip *bool `json:"allow_skip,omitempty"` // Whether user can skip (pointer to distinguish false from unset)
- Errors map[string]string `json:"errors,omitempty"` // Validation errors (question_id -> error message)
-}
-
-// PlotData represents a visualization plot (chart/graph) from E2B tools
-type PlotData struct {
- Format string `json:"format"` // "png", "jpg", "svg"
- Data string `json:"data"` // Base64-encoded image data
-}
-
-// TokenUsage represents token consumption statistics
-type TokenUsage struct {
- Input int `json:"input"`
- Output int `json:"output"`
-}
-
-// PromptWaiterFunc is a function type that waits for a prompt response
-// Returns (answers, skipped, error)
-type PromptWaiterFunc func(promptID string, timeout time.Duration) (map[string]InteractiveAnswer, bool, error)
-
-// UserConnection represents a single WebSocket connection
-type UserConnection struct {
- ConnID string
- UserID string // User ID from authentication
- Conn *websocket.Conn
- ConversationID string
- Messages []map[string]interface{}
- MessageCount int // Track number of messages for title generation
- ModelID string // Selected model ID from platform
- CustomConfig *CustomAPIConfig // OR user's custom API configuration (BYOK)
- SystemInstructions string // Optional: User-provided system prompt override
- DisableTools bool // Disable tools for this connection (e.g., agent builder)
- CreatedAt time.Time
- WriteChan chan ServerMessage
- StopChan chan bool
- Mutex sync.Mutex
- closed bool // Track if connection is closed
- PromptWaiter PromptWaiterFunc // Function to wait for interactive prompt responses
-}
-
-// SafeSend sends a message to WriteChan safely, returning false if the channel is closed
-func (uc *UserConnection) SafeSend(msg ServerMessage) bool {
- uc.Mutex.Lock()
- if uc.closed {
- uc.Mutex.Unlock()
- return false
- }
- uc.Mutex.Unlock()
-
- // Use defer/recover to handle panic from send on closed channel
- defer func() {
- if r := recover(); r != nil {
- // Channel was closed, mark connection as closed
- uc.Mutex.Lock()
- uc.closed = true
- uc.Mutex.Unlock()
- }
- }()
-
- uc.WriteChan <- msg
- return true
-}
-
-// MarkClosed marks the connection as closed
-func (uc *UserConnection) MarkClosed() {
- uc.Mutex.Lock()
- uc.closed = true
- uc.Mutex.Unlock()
-}
-
-// IsClosed returns true if the connection has been marked as closed
-func (uc *UserConnection) IsClosed() bool {
- uc.Mutex.Lock()
- defer uc.Mutex.Unlock()
- return uc.closed
-}
-
-// ChatRequest represents a request to OpenAI-compatible chat completion API
-type ChatRequest struct {
- Model string `json:"model"`
- Messages []map[string]interface{} `json:"messages"`
- Tools []map[string]interface{} `json:"tools,omitempty"`
- Stream bool `json:"stream"`
- Temperature float64 `json:"temperature,omitempty"`
-}
diff --git a/backend/internal/models/workflow.go b/backend/internal/models/workflow.go
deleted file mode 100644
index c896c70b..00000000
--- a/backend/internal/models/workflow.go
+++ /dev/null
@@ -1,139 +0,0 @@
-package models
-
-// ConversationMessage represents a message in the conversation history
-type ConversationMessage struct {
- Role string `json:"role"` // "user" or "assistant"
- Content string `json:"content"` // Message content
-}
-
-// WorkflowGenerateRequest represents a request to generate or modify a workflow
-type WorkflowGenerateRequest struct {
- AgentID string `json:"agent_id"`
- UserMessage string `json:"user_message"`
- CurrentWorkflow *Workflow `json:"current_workflow,omitempty"` // For modifications
- ModelID string `json:"model_id,omitempty"` // Optional model override
- ConversationID string `json:"conversation_id,omitempty"` // For conversation persistence
- ConversationHistory []ConversationMessage `json:"conversation_history,omitempty"` // Recent conversation context for better tool selection
-}
-
-// WorkflowGenerateResponse represents the structured output from workflow generation
-type WorkflowGenerateResponse struct {
- Success bool `json:"success"`
- Workflow *Workflow `json:"workflow,omitempty"`
- Explanation string `json:"explanation"`
- Action string `json:"action"` // "create" or "modify"
- Error string `json:"error,omitempty"`
- Version int `json:"version"`
- Errors []ValidationError `json:"errors,omitempty"`
- SuggestedName string `json:"suggested_name,omitempty"` // AI-generated agent name suggestion
- SuggestedDescription string `json:"suggested_description,omitempty"` // AI-generated agent description
-}
-
-// ValidationError represents a workflow validation error
-type ValidationError struct {
- Type string `json:"type"` // "schema", "cycle", "type_mismatch", "missing_input"
- Message string `json:"message"`
- BlockID string `json:"blockId,omitempty"`
- ConnectionID string `json:"connectionId,omitempty"`
-}
-
-// WorkflowJSONSchema returns the JSON schema for structured output
-func WorkflowJSONSchema() map[string]interface{} {
- return map[string]interface{}{
- "type": "object",
- "properties": map[string]interface{}{
- "blocks": map[string]interface{}{
- "type": "array",
- "items": map[string]interface{}{
- "type": "object",
- "properties": map[string]interface{}{
- "id": map[string]interface{}{
- "type": "string",
- "description": "Unique block ID in kebab-case format matching the block name",
- },
- "type": map[string]interface{}{
- "type": "string",
- "enum": []string{"llm_inference", "variable", "code_block"},
- "description": "Block type - llm_inference for AI agents, variable for inputs, code_block for direct tool execution",
- },
- "name": map[string]interface{}{
- "type": "string",
- "description": "Human-readable block name",
- },
- "description": map[string]interface{}{
- "type": "string",
- "description": "What this block does",
- },
- "config": map[string]interface{}{
- "type": "object",
- "description": "Block-specific configuration",
- },
- "position": map[string]interface{}{
- "type": "object",
- "properties": map[string]interface{}{
- "x": map[string]interface{}{"type": "integer"},
- "y": map[string]interface{}{"type": "integer"},
- },
- "required": []string{"x", "y"},
- },
- "timeout": map[string]interface{}{
- "type": "integer",
- "description": "Timeout in seconds (default 30, max 120 for LLM blocks)",
- },
- },
- "required": []string{"id", "type", "name", "description", "config", "position", "timeout"},
- },
- },
- "connections": map[string]interface{}{
- "type": "array",
- "items": map[string]interface{}{
- "type": "object",
- "properties": map[string]interface{}{
- "id": map[string]interface{}{
- "type": "string",
- "description": "Unique connection ID",
- },
- "sourceBlockId": map[string]interface{}{
- "type": "string",
- "description": "ID of the source block",
- },
- "sourceOutput": map[string]interface{}{
- "type": "string",
- "description": "Output port name (usually 'output')",
- },
- "targetBlockId": map[string]interface{}{
- "type": "string",
- "description": "ID of the target block",
- },
- "targetInput": map[string]interface{}{
- "type": "string",
- "description": "Input port name (usually 'input')",
- },
- },
- "required": []string{"id", "sourceBlockId", "sourceOutput", "targetBlockId", "targetInput"},
- },
- },
- "variables": map[string]interface{}{
- "type": "array",
- "items": map[string]interface{}{"type": "object"},
- },
- "explanation": map[string]interface{}{
- "type": "string",
- "description": "Brief explanation of the workflow or changes made",
- },
- },
- "required": []string{"blocks", "connections", "variables", "explanation"},
- }
-}
-
-// WorkflowExecuteResult contains the result of a workflow execution
-type WorkflowExecuteResult struct {
- Status string
- Output map[string]interface{}
- BlockStates map[string]*BlockState
- Error string
-}
-
-// WorkflowExecuteFunc is a function type for executing workflows
-// This allows scheduler to call workflow engine without import cycle
-type WorkflowExecuteFunc func(workflow *Workflow, inputs map[string]interface{}) (*WorkflowExecuteResult, error)
diff --git a/backend/internal/preflight/checks.go b/backend/internal/preflight/checks.go
deleted file mode 100644
index f6934ce4..00000000
--- a/backend/internal/preflight/checks.go
+++ /dev/null
@@ -1,204 +0,0 @@
-package preflight
-
-import (
- "claraverse/internal/database"
- "fmt"
- "log"
- "os"
-)
-
-// CheckResult represents the result of a preflight check
-type CheckResult struct {
- Name string
- Status string // "pass", "fail", "warning"
- Message string
- Error error
-}
-
-// Checker performs pre-flight checks before server starts
-type Checker struct {
- db *database.DB
- requiredEnvars []string
-}
-
-// NewChecker creates a new preflight checker
-func NewChecker(db *database.DB) *Checker {
- return &Checker{
- db: db,
- requiredEnvars: []string{
- // Optional: Add required environment variables here
- },
- }
-}
-
-// RunAll runs all preflight checks and returns results
-func (c *Checker) RunAll() []CheckResult {
- log.Println("🔍 Running pre-flight checks...")
-
- results := []CheckResult{
- c.checkDatabaseConnection(),
- c.checkDatabaseSchema(),
- c.checkEnvironmentVariables(),
- }
-
- // Print summary
- passed := 0
- failed := 0
- warnings := 0
-
- for _, result := range results {
- switch result.Status {
- case "pass":
- log.Printf(" ✅ %s: %s", result.Name, result.Message)
- passed++
- case "fail":
- log.Printf(" ❌ %s: %s", result.Name, result.Message)
- if result.Error != nil {
- log.Printf(" Error: %v", result.Error)
- }
- failed++
- case "warning":
- log.Printf(" ⚠️ %s: %s", result.Name, result.Message)
- warnings++
- }
- }
-
- log.Printf("\n📊 Pre-flight summary: %d passed, %d failed, %d warnings\n", passed, failed, warnings)
-
- return results
-}
-
-// HasFailures returns true if any check failed
-func HasFailures(results []CheckResult) bool {
- for _, result := range results {
- if result.Status == "fail" {
- return true
- }
- }
- return false
-}
-
-// checkDatabaseConnection verifies database connectivity
-func (c *Checker) checkDatabaseConnection() CheckResult {
- if err := c.db.Ping(); err != nil {
- return CheckResult{
- Name: "Database Connection",
- Status: "fail",
- Message: "Cannot connect to database",
- Error: err,
- }
- }
-
- return CheckResult{
- Name: "Database Connection",
- Status: "pass",
- Message: "Database connection successful",
- }
-}
-
-// checkDatabaseSchema verifies all required tables exist
-func (c *Checker) checkDatabaseSchema() CheckResult {
- requiredTables := []string{
- "providers",
- "models",
- "provider_model_filters",
- "model_capabilities",
- "model_refresh_log",
- }
-
- for _, table := range requiredTables {
- var count int
- // MySQL-compatible query using INFORMATION_SCHEMA
- query := "SELECT COUNT(*) FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = ?"
- err := c.db.QueryRow(query, table).Scan(&count)
- if err != nil || count == 0 {
- return CheckResult{
- Name: "Database Schema",
- Status: "fail",
- Message: fmt.Sprintf("Required table '%s' not found", table),
- Error: err,
- }
- }
- }
-
- return CheckResult{
- Name: "Database Schema",
- Status: "pass",
- Message: fmt.Sprintf("All %d required tables exist", len(requiredTables)),
- }
-}
-
-
-// checkEnvironmentVariables verifies required environment variables are set
-func (c *Checker) checkEnvironmentVariables() CheckResult {
- missing := []string{}
-
- for _, envar := range c.requiredEnvars {
- if os.Getenv(envar) == "" {
- missing = append(missing, envar)
- }
- }
-
- if len(missing) > 0 {
- return CheckResult{
- Name: "Environment Variables",
- Status: "warning",
- Message: fmt.Sprintf("Missing environment variables: %v", missing),
- }
- }
-
- // Check optional but recommended variables
- supabaseURL := os.Getenv("SUPABASE_URL")
- supabaseKey := os.Getenv("SUPABASE_KEY")
-
- if supabaseURL == "" || supabaseKey == "" {
- return CheckResult{
- Name: "Environment Variables",
- Status: "warning",
- Message: "Supabase authentication not configured (running in development mode)",
- }
- }
-
- return CheckResult{
- Name: "Environment Variables",
- Status: "pass",
- Message: "All environment variables configured",
- }
-}
-
-// checkProviderConnectivity tests if we can reach provider APIs (optional, can be slow)
-func (c *Checker) checkProviderConnectivity() CheckResult {
- // This is an optional check that could be added
- // It would test actual connectivity to provider APIs
- // For now, we'll skip it to keep startup fast
-
- return CheckResult{
- Name: "Provider Connectivity",
- Status: "pass",
- Message: "Skipped (optional check)",
- }
-}
-
-// QuickCheck runs minimal checks for fast startup
-func (c *Checker) QuickCheck() []CheckResult {
- log.Println("⚡ Running quick pre-flight checks...")
-
- results := []CheckResult{
- c.checkDatabaseConnection(),
- }
-
- passed := 0
- failed := 0
-
- for _, result := range results {
- if result.Status == "pass" {
- log.Printf(" ✅ %s", result.Name)
- passed++
- } else if result.Status == "fail" {
- log.Printf(" ❌ %s: %s", result.Name, result.Message)
- failed++
- }
- }
-
- return results
-}
diff --git a/backend/internal/preflight/checks_test.go b/backend/internal/preflight/checks_test.go
deleted file mode 100644
index 30a506bf..00000000
--- a/backend/internal/preflight/checks_test.go
+++ /dev/null
@@ -1,204 +0,0 @@
-package preflight
-
-import (
- "claraverse/internal/database"
- "os"
- "testing"
-)
-
-func setupPreflightTest(t *testing.T) (*database.DB, func()) {
- t.Skip("SQLite tests are deprecated - preflight tests require MySQL DSN via DATABASE_URL")
- tmpDB := "test_preflight.db"
-
- db, err := database.New(tmpDB)
- if err != nil {
- t.Fatalf("Failed to create test database: %v", err)
- }
-
- if err := db.Initialize(); err != nil {
- t.Fatalf("Failed to initialize test database: %v", err)
- }
-
- cleanup := func() {
- db.Close()
- os.Remove(tmpDB)
- }
-
- return db, cleanup
-}
-
-func TestNewChecker(t *testing.T) {
- db, cleanup := setupPreflightTest(t)
- defer cleanup()
-
- checker := NewChecker(db)
- if checker == nil {
- t.Fatal("Expected non-nil checker")
- }
-
- if checker.db != db {
- t.Error("Checker database not set correctly")
- }
-}
-
-func TestCheckDatabaseConnection_Success(t *testing.T) {
- db, cleanup := setupPreflightTest(t)
- defer cleanup()
-
- checker := NewChecker(db)
- result := checker.checkDatabaseConnection()
-
- if result.Status != "pass" {
- t.Errorf("Expected status 'pass', got '%s'", result.Status)
- }
-
- if result.Name != "Database Connection" {
- t.Errorf("Expected name 'Database Connection', got '%s'", result.Name)
- }
-}
-
-func TestCheckDatabaseConnection_Failure(t *testing.T) {
- db, cleanup := setupPreflightTest(t)
- cleanup() // Close database immediately to simulate failure
-
- checker := NewChecker(db)
- result := checker.checkDatabaseConnection()
-
- if result.Status != "fail" {
- t.Errorf("Expected status 'fail', got '%s'", result.Status)
- }
-
- if result.Error == nil {
- t.Error("Expected error to be set")
- }
-}
-
-func TestCheckDatabaseSchema_Success(t *testing.T) {
- db, cleanup := setupPreflightTest(t)
- defer cleanup()
-
- checker := NewChecker(db)
- result := checker.checkDatabaseSchema()
-
- if result.Status != "pass" {
- t.Errorf("Expected status 'pass', got '%s': %s", result.Status, result.Message)
- }
-}
-
-func TestCheckDatabaseSchema_MissingTable(t *testing.T) {
- t.Skip("SQLite tests are deprecated - preflight tests require MySQL DSN via DATABASE_URL")
-}
-
-func TestCheckProvidersFile_Exists(t *testing.T) {
- t.Skip("Provider file checks have been removed from preflight - providers are now managed via database")
-}
-
-func TestCheckProvidersFile_Missing(t *testing.T) {
- t.Skip("Provider file checks have been removed from preflight - providers are now managed via database")
-}
-
-func TestCheckProvidersJSON_Valid(t *testing.T) {
- t.Skip("Provider file checks have been removed from preflight - providers are now managed via database")
-}
-
-func TestCheckProvidersJSON_InvalidJSON(t *testing.T) {
- t.Skip("Provider file checks have been removed from preflight - providers are now managed via database")
-}
-
-func TestCheckProvidersJSON_EmptyProviders(t *testing.T) {
- t.Skip("Provider file checks have been removed from preflight - providers are now managed via database")
-}
-
-func TestCheckProvidersJSON_MissingName(t *testing.T) {
- t.Skip("Provider file checks have been removed from preflight - providers are now managed via database")
-}
-
-func TestCheckProvidersJSON_MissingBaseURL(t *testing.T) {
- t.Skip("Provider file checks have been removed from preflight - providers are now managed via database")
-}
-
-func TestCheckProvidersJSON_MissingAPIKey(t *testing.T) {
- t.Skip("Provider file checks have been removed from preflight - providers are now managed via database")
-}
-
-func TestCheckEnvironmentVariables(t *testing.T) {
- db, cleanup := setupPreflightTest(t)
- defer cleanup()
-
- checker := NewChecker(db)
- result := checker.checkEnvironmentVariables()
-
- // Should pass or warn, but not fail
- if result.Status == "fail" {
- t.Errorf("Expected status 'pass' or 'warning', got 'fail': %s", result.Message)
- }
-}
-
-func TestRunAll(t *testing.T) {
- db, cleanup := setupPreflightTest(t)
- defer cleanup()
-
- checker := NewChecker(db)
- results := checker.RunAll()
-
- if len(results) == 0 {
- t.Error("Expected results, got empty slice")
- }
-
- // Verify all expected checks ran
- expectedChecks := map[string]bool{
- "Database Connection": false,
- "Database Schema": false,
- "Environment Variables": false,
- }
-
- for _, result := range results {
- if _, exists := expectedChecks[result.Name]; exists {
- expectedChecks[result.Name] = true
- }
- }
-
- for checkName, ran := range expectedChecks {
- if !ran {
- t.Errorf("Expected check '%s' to run", checkName)
- }
- }
-}
-
-func TestHasFailures(t *testing.T) {
- // Test with no failures
- results := []CheckResult{
- {Status: "pass"},
- {Status: "pass"},
- {Status: "warning"},
- }
-
- if HasFailures(results) {
- t.Error("Expected no failures")
- }
-
- // Test with failures
- results = append(results, CheckResult{Status: "fail"})
-
- if !HasFailures(results) {
- t.Error("Expected failures to be detected")
- }
-}
-
-func TestQuickCheck(t *testing.T) {
- db, cleanup := setupPreflightTest(t)
- defer cleanup()
-
- checker := NewChecker(db)
- results := checker.QuickCheck()
-
- if len(results) == 0 {
- t.Error("Expected results from quick check")
- }
-
- // Quick check should run fewer checks than full check
- fullResults := checker.RunAll()
- if len(results) >= len(fullResults) {
- t.Error("Expected quick check to run fewer checks than full check")
- }
-}
diff --git a/backend/internal/presentation/service.go b/backend/internal/presentation/service.go
deleted file mode 100644
index b4344aae..00000000
--- a/backend/internal/presentation/service.go
+++ /dev/null
@@ -1,317 +0,0 @@
-package presentation
-
-import (
- "bytes"
- "context"
- "fmt"
- "html"
- "log"
- "os"
- "path/filepath"
- "strings"
- "sync"
- "time"
-
- "github.com/chromedp/cdproto/page"
- "github.com/chromedp/chromedp"
- "github.com/google/uuid"
-)
-
-// Slide represents a single slide/page in the presentation
-// Each slide contains a complete standalone HTML document
-type Slide struct {
- HTML string `json:"html"` // Complete HTML document (must include , , and tags)
-}
-
-// PresentationConfig holds the presentation configuration
-type PresentationConfig struct {
- Title string `json:"title"`
- Slides []Slide `json:"slides"` // Array of slides, each with custom HTML content
-}
-
-// GeneratedPresentation represents a generated presentation file
-type GeneratedPresentation struct {
- PresentationID string
- UserID string
- ConversationID string
- Filename string
- FilePath string
- Size int64
- ContentType string
- CreatedAt time.Time
-}
-
-// Service handles presentation generation
-type Service struct {
- outputDir string
- presentations map[string]*GeneratedPresentation
- mu sync.RWMutex
-}
-
-var (
- serviceInstance *Service
- serviceOnce sync.Once
-)
-
-// GetService returns the singleton presentation service
-func GetService() *Service {
- serviceOnce.Do(func() {
- outputDir := "./generated"
- if err := os.MkdirAll(outputDir, 0700); err != nil {
- log.Printf("⚠️ Warning: Could not create generated directory: %v", err)
- }
- serviceInstance = &Service{
- outputDir: outputDir,
- presentations: make(map[string]*GeneratedPresentation),
- }
- })
- return serviceInstance
-}
-
-// GeneratePresentation creates a PDF presentation with custom HTML pages
-// Each page is rendered in 16:9 landscape format with complete creative freedom
-func (s *Service) GeneratePresentation(config PresentationConfig, userID, conversationID string) (*GeneratedPresentation, error) {
- // Validate
- if config.Title == "" {
- config.Title = "Presentation"
- }
- if len(config.Slides) == 0 {
- return nil, fmt.Errorf("presentation must have at least one slide")
- }
-
- // Generate multi-page HTML with CSS page breaks
- htmlContent := s.generateMultiPageHTML(config)
-
- // Generate unique ID and filename
- presentationID := uuid.New().String()
- safeFilename := sanitizeFilename(config.Title) + ".pdf"
- filePath := filepath.Join(s.outputDir, presentationID+".pdf")
-
- // Convert to PDF using chromedp (16:9 landscape)
- if err := s.generatePDFLandscape(htmlContent, filePath); err != nil {
- return nil, fmt.Errorf("failed to generate PDF: %w", err)
- }
-
- // Get file size
- fileInfo, err := os.Stat(filePath)
- if err != nil {
- return nil, fmt.Errorf("failed to get file info: %w", err)
- }
-
- // Create record
- pres := &GeneratedPresentation{
- PresentationID: presentationID,
- UserID: userID,
- ConversationID: conversationID,
- Filename: safeFilename,
- FilePath: filePath,
- Size: fileInfo.Size(),
- ContentType: "application/pdf",
- CreatedAt: time.Now(),
- }
-
- // Store
- s.mu.Lock()
- s.presentations[presentationID] = pres
- s.mu.Unlock()
-
- log.Printf("🎯 [PRESENTATION-SERVICE] Generated PDF presentation: %s (%d bytes, %d pages)", safeFilename, fileInfo.Size(), len(config.Slides))
-
- return pres, nil
-}
-
-// generateMultiPageHTML creates HTML document with each slide as a separate page
-// Extracts content from each slide's HTML and renders directly (no iframes)
-func (s *Service) generateMultiPageHTML(config PresentationConfig) string {
- var pagesHTML bytes.Buffer
- var allStyles bytes.Buffer
-
- for i, slide := range config.Slides {
- slideHTML := slide.HTML
-
- // Extract styles from (everything between )
- styleStart := strings.Index(strings.ToLower(slideHTML), "")
- var styles string
- if styleStart != -1 && styleEnd != -1 {
- // Find the end of the opening
-
-
-%s
-
-`, html.EscapeString(config.Title), allStyles.String(), pagesHTML.String())
-}
-
-// generatePDFLandscape converts HTML to PDF using chromedp with 16:9 landscape format
-func (s *Service) generatePDFLandscape(htmlContent, outputPath string) error {
- // Create allocator options for headless Chrome
- opts := append(chromedp.DefaultExecAllocatorOptions[:],
- chromedp.ExecPath("/usr/bin/chromium-browser"),
- chromedp.NoSandbox,
- chromedp.DisableGPU,
- chromedp.Flag("disable-dev-shm-usage", true),
- chromedp.Flag("no-first-run", true),
- chromedp.Flag("no-default-browser-check", true),
- )
-
- // Create allocator context
- allocCtx, allocCancel := chromedp.NewExecAllocator(context.Background(), opts...)
- defer allocCancel()
-
- // Create context
- ctx, cancel := chromedp.NewContext(allocCtx)
- defer cancel()
-
- // Set timeout
- ctx, cancel = context.WithTimeout(ctx, 60*time.Second)
- defer cancel()
-
- // Generate PDF in 16:9 landscape format
- var pdfBuffer []byte
- if err := chromedp.Run(ctx,
- chromedp.Navigate("about:blank"),
- chromedp.ActionFunc(func(ctx context.Context) error {
- frameTree, err := page.GetFrameTree().Do(ctx)
- if err != nil {
- return err
- }
- return page.SetDocumentContent(frameTree.Frame.ID, htmlContent).Do(ctx)
- }),
- chromedp.Sleep(3*time.Second), // Wait for fonts, images, and custom styles to load
- chromedp.ActionFunc(func(ctx context.Context) error {
- var err error
- // Use PreferCSSPageSize to ensure Chrome respects CSS page-break properties
- pdfBuffer, _, err = page.PrintToPDF().
- WithPrintBackground(true).
- WithDisplayHeaderFooter(false).
- WithMarginTop(0).
- WithMarginBottom(0).
- WithMarginLeft(0).
- WithMarginRight(0).
- WithPaperWidth(10.67). // 16:9 landscape width
- WithPaperHeight(6). // 16:9 landscape height
- WithPreferCSSPageSize(true). // CRITICAL: Enables CSS page-break properties
- Do(ctx)
- return err
- }),
- ); err != nil {
- return err
- }
-
- // Write PDF to file
- if err := os.WriteFile(outputPath, pdfBuffer, 0600); err != nil {
- return err
- }
-
- return nil
-}
-
-// sanitizeFilename removes invalid characters from filename
-func sanitizeFilename(name string) string {
- // Replace invalid characters with underscore
- invalid := []string{"/", "\\", ":", "*", "?", "\"", "<", ">", "|"}
- result := name
- for _, char := range invalid {
- result = strings.ReplaceAll(result, char, "_")
- }
- // Limit length
- if len(result) > 50 {
- result = result[:50]
- }
- if result == "" {
- result = "presentation"
- }
- return result
-}
-
-// GetPresentation retrieves a presentation by ID
-func (s *Service) GetPresentation(presentationID string) (*GeneratedPresentation, bool) {
- s.mu.RLock()
- defer s.mu.RUnlock()
- pres, exists := s.presentations[presentationID]
- return pres, exists
-}
-
diff --git a/backend/internal/securefile/service.go b/backend/internal/securefile/service.go
deleted file mode 100644
index ca6f8d70..00000000
--- a/backend/internal/securefile/service.go
+++ /dev/null
@@ -1,374 +0,0 @@
-package securefile
-
-import (
- "crypto/rand"
- "crypto/sha256"
- "encoding/hex"
- "fmt"
- "log"
- "os"
- "path/filepath"
- "sync"
- "time"
-
- "github.com/google/uuid"
- "github.com/patrickmn/go-cache"
-)
-
-// File represents a file stored with access code protection
-type File struct {
- ID string `json:"id"`
- UserID string `json:"user_id"`
- Filename string `json:"filename"`
- MimeType string `json:"mime_type"`
- Size int64 `json:"size"`
- FilePath string `json:"-"` // Not exposed in JSON
- AccessCodeHash string `json:"-"` // SHA256 hash, not exposed
- CreatedAt time.Time `json:"created_at"`
- ExpiresAt time.Time `json:"expires_at"`
-}
-
-// Result is returned when creating a file
-type Result struct {
- ID string `json:"id"`
- Filename string `json:"filename"`
- DownloadURL string `json:"download_url"`
- AccessCode string `json:"access_code"` // Only returned once at creation
- Size int64 `json:"size"`
- MimeType string `json:"mime_type"`
- ExpiresAt time.Time `json:"expires_at"`
-}
-
-// Service manages files with access code protection
-type Service struct {
- cache *cache.Cache
- storageDir string
- mu sync.RWMutex
-}
-
-var (
- instance *Service
- once sync.Once
-)
-
-// GetService returns the singleton secure file service
-func GetService() *Service {
- once.Do(func() {
- instance = NewService("./secure_files")
- })
- return instance
-}
-
-// NewService creates a new secure file service
-func NewService(storageDir string) *Service {
- // Create storage directory if it doesn't exist
- if err := os.MkdirAll(storageDir, 0700); err != nil {
- log.Printf("⚠️ [SECURE-FILE] Failed to create storage directory: %v", err)
- }
-
- // 30-day default expiration, cleanup every hour
- c := cache.New(30*24*time.Hour, 1*time.Hour)
-
- // Set eviction handler to delete files when they expire
- c.OnEvicted(func(key string, value interface{}) {
- if file, ok := value.(*File); ok {
- log.Printf("🗑️ [SECURE-FILE] Expiring file %s (%s)", file.ID, file.Filename)
- if file.FilePath != "" {
- if err := os.Remove(file.FilePath); err != nil && !os.IsNotExist(err) {
- log.Printf("⚠️ [SECURE-FILE] Failed to delete expired file: %v", err)
- }
- }
- }
- })
-
- svc := &Service{
- cache: c,
- storageDir: storageDir,
- }
-
- // Run startup cleanup
- go svc.cleanupExpiredFiles()
-
- return svc
-}
-
-// generateAccessCode generates a cryptographically secure access code
-func generateAccessCode() (string, error) {
- bytes := make([]byte, 16) // 16 bytes = 32 hex characters
- if _, err := rand.Read(bytes); err != nil {
- return "", err
- }
- return hex.EncodeToString(bytes), nil
-}
-
-// hashAccessCode creates a SHA256 hash of the access code
-func hashAccessCode(code string) string {
- hash := sha256.Sum256([]byte(code))
- return hex.EncodeToString(hash[:])
-}
-
-// CreateFile stores content as a secure file with access code
-func (s *Service) CreateFile(userID string, content []byte, filename, mimeType string) (*Result, error) {
- s.mu.Lock()
- defer s.mu.Unlock()
-
- // Generate unique ID
- fileID := uuid.New().String()
-
- // Generate access code
- accessCode, err := generateAccessCode()
- if err != nil {
- return nil, fmt.Errorf("failed to generate access code: %w", err)
- }
-
- // Hash the access code for storage
- accessCodeHash := hashAccessCode(accessCode)
-
- // Determine file extension from filename or mime type
- ext := filepath.Ext(filename)
- if ext == "" {
- ext = getExtensionFromMimeType(mimeType)
- }
-
- // Create file path
- storedFilename := fmt.Sprintf("%s%s", fileID, ext)
- filePath := filepath.Join(s.storageDir, storedFilename)
-
- // Write file to disk
- if err := os.WriteFile(filePath, content, 0600); err != nil {
- return nil, fmt.Errorf("failed to write file: %w", err)
- }
-
- // Set expiration (30 days)
- now := time.Now()
- expiresAt := now.Add(30 * 24 * time.Hour)
-
- // Create secure file record
- secureFile := &File{
- ID: fileID,
- UserID: userID,
- Filename: filename,
- MimeType: mimeType,
- Size: int64(len(content)),
- FilePath: filePath,
- AccessCodeHash: accessCodeHash,
- CreatedAt: now,
- ExpiresAt: expiresAt,
- }
-
- // Store in cache
- s.cache.Set(fileID, secureFile, 30*24*time.Hour)
-
- log.Printf("✅ [SECURE-FILE] Created file %s (%s) for user %s, expires %s",
- fileID, filename, userID, expiresAt.Format(time.RFC3339))
-
- // Build download URL with full backend URL for LLM tools
- // LLM agents pass this URL to other tools (Discord, SendGrid, etc.) which need full URLs
- backendURL := os.Getenv("BACKEND_URL")
- if backendURL == "" {
- backendURL = "http://localhost:3001" // Default fallback for development
- }
- downloadURL := fmt.Sprintf("%s/api/files/%s?code=%s", backendURL, fileID, accessCode)
-
- return &Result{
- ID: fileID,
- Filename: filename,
- DownloadURL: downloadURL,
- AccessCode: accessCode, // Only returned once
- Size: int64(len(content)),
- MimeType: mimeType,
- ExpiresAt: expiresAt,
- }, nil
-}
-
-// GetFile retrieves a file if the access code is valid
-func (s *Service) GetFile(fileID, accessCode string) (*File, []byte, error) {
- s.mu.RLock()
- defer s.mu.RUnlock()
-
- // Get file from cache
- value, found := s.cache.Get(fileID)
- if !found {
- return nil, nil, fmt.Errorf("file not found or expired")
- }
-
- file, ok := value.(*File)
- if !ok {
- return nil, nil, fmt.Errorf("invalid file data")
- }
-
- // Verify access code
- providedHash := hashAccessCode(accessCode)
- if providedHash != file.AccessCodeHash {
- log.Printf("🚫 [SECURE-FILE] Invalid access code for file %s", fileID)
- return nil, nil, fmt.Errorf("invalid access code")
- }
-
- // Read file content
- content, err := os.ReadFile(file.FilePath)
- if err != nil {
- log.Printf("❌ [SECURE-FILE] Failed to read file %s: %v", fileID, err)
- return nil, nil, fmt.Errorf("failed to read file")
- }
-
- log.Printf("✅ [SECURE-FILE] File %s accessed successfully", fileID)
- return file, content, nil
-}
-
-// GetFileInfo returns file metadata without content (requires access code)
-func (s *Service) GetFileInfo(fileID, accessCode string) (*File, error) {
- s.mu.RLock()
- defer s.mu.RUnlock()
-
- value, found := s.cache.Get(fileID)
- if !found {
- return nil, fmt.Errorf("file not found or expired")
- }
-
- file, ok := value.(*File)
- if !ok {
- return nil, fmt.Errorf("invalid file data")
- }
-
- // Verify access code
- providedHash := hashAccessCode(accessCode)
- if providedHash != file.AccessCodeHash {
- return nil, fmt.Errorf("invalid access code")
- }
-
- return file, nil
-}
-
-// DeleteFile removes a file (requires ownership)
-func (s *Service) DeleteFile(fileID, userID string) error {
- s.mu.Lock()
- defer s.mu.Unlock()
-
- value, found := s.cache.Get(fileID)
- if !found {
- return fmt.Errorf("file not found")
- }
-
- file, ok := value.(*File)
- if !ok {
- return fmt.Errorf("invalid file data")
- }
-
- // Verify ownership
- if file.UserID != userID {
- return fmt.Errorf("access denied")
- }
-
- // Delete from disk
- if file.FilePath != "" {
- if err := os.Remove(file.FilePath); err != nil && !os.IsNotExist(err) {
- log.Printf("⚠️ [SECURE-FILE] Failed to delete file from disk: %v", err)
- }
- }
-
- // Delete from cache
- s.cache.Delete(fileID)
-
- log.Printf("✅ [SECURE-FILE] Deleted file %s", fileID)
- return nil
-}
-
-// ListUserFiles returns all files for a user
-func (s *Service) ListUserFiles(userID string) []*File {
- s.mu.RLock()
- defer s.mu.RUnlock()
-
- var files []*File
- for _, item := range s.cache.Items() {
- if file, ok := item.Object.(*File); ok {
- if file.UserID == userID {
- files = append(files, file)
- }
- }
- }
- return files
-}
-
-// cleanupExpiredFiles removes files that have expired
-func (s *Service) cleanupExpiredFiles() {
- // Scan storage directory for orphaned files
- entries, err := os.ReadDir(s.storageDir)
- if err != nil {
- log.Printf("⚠️ [SECURE-FILE] Failed to read storage directory: %v", err)
- return
- }
-
- now := time.Now()
- orphanedCount := 0
-
- for _, entry := range entries {
- if entry.IsDir() {
- continue
- }
-
- filePath := filepath.Join(s.storageDir, entry.Name())
- info, err := entry.Info()
- if err != nil {
- continue
- }
-
- // Delete files older than 31 days (1 day buffer)
- if now.Sub(info.ModTime()) > 31*24*time.Hour {
- log.Printf("🗑️ [SECURE-FILE] Deleting orphaned/expired file: %s", entry.Name())
- if err := os.Remove(filePath); err != nil {
- log.Printf("⚠️ [SECURE-FILE] Failed to delete: %v", err)
- } else {
- orphanedCount++
- }
- }
- }
-
- if orphanedCount > 0 {
- log.Printf("✅ [SECURE-FILE] Cleaned up %d orphaned files", orphanedCount)
- }
-}
-
-// GetStats returns service statistics
-func (s *Service) GetStats() map[string]interface{} {
- s.mu.RLock()
- defer s.mu.RUnlock()
-
- items := s.cache.Items()
- totalSize := int64(0)
-
- for _, item := range items {
- if file, ok := item.Object.(*File); ok {
- totalSize += file.Size
- }
- }
-
- return map[string]interface{}{
- "total_files": len(items),
- "total_size": totalSize,
- "storage_dir": s.storageDir,
- }
-}
-
-// getExtensionFromMimeType returns a file extension for common mime types
-func getExtensionFromMimeType(mimeType string) string {
- extensions := map[string]string{
- "application/pdf": ".pdf",
- "application/json": ".json",
- "text/plain": ".txt",
- "text/csv": ".csv",
- "text/html": ".html",
- "text/css": ".css",
- "text/javascript": ".js",
- "application/xml": ".xml",
- "text/xml": ".xml",
- "text/yaml": ".yaml",
- "application/x-yaml": ".yaml",
- "text/markdown": ".md",
- "application/octet-stream": ".bin",
- }
-
- if ext, ok := extensions[mimeType]; ok {
- return ext
- }
- return ".bin"
-}
diff --git a/backend/internal/securefile/service_test.go b/backend/internal/securefile/service_test.go
deleted file mode 100644
index 46d5f429..00000000
--- a/backend/internal/securefile/service_test.go
+++ /dev/null
@@ -1,452 +0,0 @@
-package securefile
-
-import (
- "os"
- "path/filepath"
- "testing"
- "time"
-)
-
-// TestNewService tests service creation
-func TestNewService(t *testing.T) {
- tempDir := t.TempDir()
- svc := NewService(tempDir)
-
- if svc == nil {
- t.Fatal("NewService should return non-nil service")
- }
-
- if svc.storageDir != tempDir {
- t.Errorf("Expected storage dir %s, got %s", tempDir, svc.storageDir)
- }
-}
-
-// TestCreateFile tests file creation with access code
-func TestCreateFile(t *testing.T) {
- tempDir := t.TempDir()
- svc := NewService(tempDir)
-
- content := []byte("test file content")
- userID := "user-123"
- filename := "test.txt"
- mimeType := "text/plain"
-
- result, err := svc.CreateFile(userID, content, filename, mimeType)
- if err != nil {
- t.Fatalf("CreateFile failed: %v", err)
- }
-
- // Verify result fields
- if result.ID == "" {
- t.Error("File ID should not be empty")
- }
- if result.Filename != filename {
- t.Errorf("Expected filename %s, got %s", filename, result.Filename)
- }
- if result.AccessCode == "" {
- t.Error("Access code should not be empty")
- }
- if len(result.AccessCode) != 32 {
- t.Errorf("Access code should be 32 chars, got %d", len(result.AccessCode))
- }
- if result.Size != int64(len(content)) {
- t.Errorf("Expected size %d, got %d", len(content), result.Size)
- }
- if result.MimeType != mimeType {
- t.Errorf("Expected mime type %s, got %s", mimeType, result.MimeType)
- }
- if result.ExpiresAt.Before(time.Now().Add(29 * 24 * time.Hour)) {
- t.Error("Expiration should be ~30 days from now")
- }
-
- // Verify download URL format (should contain the API path and code)
- expectedURLSuffix := "/api/files/" + result.ID + "?code=" + result.AccessCode
- if len(result.DownloadURL) < len(expectedURLSuffix) {
- t.Errorf("Download URL too short: %s", result.DownloadURL)
- }
- // Check that URL ends with the expected path (ignoring the host prefix)
- if result.DownloadURL[len(result.DownloadURL)-len(expectedURLSuffix):] != expectedURLSuffix {
- t.Errorf("Download URL format incorrect: %s (expected to end with %s)", result.DownloadURL, expectedURLSuffix)
- }
-}
-
-// TestGetFile tests retrieving file with valid access code
-func TestGetFile(t *testing.T) {
- tempDir := t.TempDir()
- svc := NewService(tempDir)
-
- content := []byte("secret content")
- userID := "user-456"
- filename := "secret.txt"
-
- result, err := svc.CreateFile(userID, content, filename, "text/plain")
- if err != nil {
- t.Fatalf("CreateFile failed: %v", err)
- }
-
- // Retrieve with valid access code
- file, retrievedContent, err := svc.GetFile(result.ID, result.AccessCode)
- if err != nil {
- t.Fatalf("GetFile failed: %v", err)
- }
-
- if file.ID != result.ID {
- t.Errorf("Expected file ID %s, got %s", result.ID, file.ID)
- }
- if file.Filename != filename {
- t.Errorf("Expected filename %s, got %s", filename, file.Filename)
- }
- if string(retrievedContent) != string(content) {
- t.Errorf("Content mismatch: expected %s, got %s", content, retrievedContent)
- }
-}
-
-// TestGetFileInvalidAccessCode tests access code validation
-func TestGetFileInvalidAccessCode(t *testing.T) {
- tempDir := t.TempDir()
- svc := NewService(tempDir)
-
- content := []byte("protected content")
- result, _ := svc.CreateFile("user-789", content, "protected.txt", "text/plain")
-
- // Try with wrong access code
- _, _, err := svc.GetFile(result.ID, "wrong-access-code")
- if err == nil {
- t.Error("Expected error for invalid access code")
- }
- if err.Error() != "invalid access code" {
- t.Errorf("Expected 'invalid access code' error, got: %v", err)
- }
-}
-
-// TestGetFileNotFound tests file not found error
-func TestGetFileNotFound(t *testing.T) {
- tempDir := t.TempDir()
- svc := NewService(tempDir)
-
- _, _, err := svc.GetFile("non-existent-id", "some-code")
- if err == nil {
- t.Error("Expected error for non-existent file")
- }
- if err.Error() != "file not found or expired" {
- t.Errorf("Expected 'file not found or expired' error, got: %v", err)
- }
-}
-
-// TestGetFileInfo tests retrieving file metadata
-func TestGetFileInfo(t *testing.T) {
- tempDir := t.TempDir()
- svc := NewService(tempDir)
-
- content := []byte("info test content")
- filename := "info-test.json"
- mimeType := "application/json"
-
- result, _ := svc.CreateFile("user-info", content, filename, mimeType)
-
- // Get info with valid access code
- file, err := svc.GetFileInfo(result.ID, result.AccessCode)
- if err != nil {
- t.Fatalf("GetFileInfo failed: %v", err)
- }
-
- if file.Filename != filename {
- t.Errorf("Expected filename %s, got %s", filename, file.Filename)
- }
- if file.MimeType != mimeType {
- t.Errorf("Expected mime type %s, got %s", mimeType, file.MimeType)
- }
- if file.Size != int64(len(content)) {
- t.Errorf("Expected size %d, got %d", len(content), file.Size)
- }
-}
-
-// TestGetFileInfoInvalidCode tests GetFileInfo with invalid access code
-func TestGetFileInfoInvalidCode(t *testing.T) {
- tempDir := t.TempDir()
- svc := NewService(tempDir)
-
- result, _ := svc.CreateFile("user-x", []byte("data"), "file.txt", "text/plain")
-
- _, err := svc.GetFileInfo(result.ID, "bad-code")
- if err == nil {
- t.Error("Expected error for invalid access code")
- }
-}
-
-// TestDeleteFile tests file deletion with ownership check
-func TestDeleteFile(t *testing.T) {
- tempDir := t.TempDir()
- svc := NewService(tempDir)
-
- userID := "owner-user"
- content := []byte("deletable content")
-
- result, _ := svc.CreateFile(userID, content, "delete-me.txt", "text/plain")
-
- // Verify file exists on disk
- files, _ := os.ReadDir(tempDir)
- initialCount := len(files)
-
- // Delete with correct owner
- err := svc.DeleteFile(result.ID, userID)
- if err != nil {
- t.Fatalf("DeleteFile failed: %v", err)
- }
-
- // Verify file removed from disk
- files, _ = os.ReadDir(tempDir)
- if len(files) != initialCount-1 {
- t.Error("File should be deleted from disk")
- }
-
- // Verify file removed from cache
- _, _, err = svc.GetFile(result.ID, result.AccessCode)
- if err == nil {
- t.Error("File should not be accessible after deletion")
- }
-}
-
-// TestDeleteFileWrongOwner tests deletion by non-owner
-func TestDeleteFileWrongOwner(t *testing.T) {
- tempDir := t.TempDir()
- svc := NewService(tempDir)
-
- result, _ := svc.CreateFile("owner-a", []byte("data"), "owned.txt", "text/plain")
-
- // Try to delete with wrong owner
- err := svc.DeleteFile(result.ID, "owner-b")
- if err == nil {
- t.Error("Expected error for non-owner deletion")
- }
- if err.Error() != "access denied" {
- t.Errorf("Expected 'access denied' error, got: %v", err)
- }
-}
-
-// TestDeleteFileNotFound tests deleting non-existent file
-func TestDeleteFileNotFound(t *testing.T) {
- tempDir := t.TempDir()
- svc := NewService(tempDir)
-
- err := svc.DeleteFile("non-existent", "user-1")
- if err == nil {
- t.Error("Expected error for non-existent file")
- }
-}
-
-// TestListUserFiles tests listing files for a user
-func TestListUserFiles(t *testing.T) {
- tempDir := t.TempDir()
- svc := NewService(tempDir)
-
- userA := "user-a"
- userB := "user-b"
-
- // Create files for user A
- svc.CreateFile(userA, []byte("file1"), "file1.txt", "text/plain")
- svc.CreateFile(userA, []byte("file2"), "file2.txt", "text/plain")
-
- // Create file for user B
- svc.CreateFile(userB, []byte("file3"), "file3.txt", "text/plain")
-
- // List user A's files
- filesA := svc.ListUserFiles(userA)
- if len(filesA) != 2 {
- t.Errorf("Expected 2 files for user A, got %d", len(filesA))
- }
-
- // List user B's files
- filesB := svc.ListUserFiles(userB)
- if len(filesB) != 1 {
- t.Errorf("Expected 1 file for user B, got %d", len(filesB))
- }
-
- // List non-existent user's files
- filesC := svc.ListUserFiles("user-c")
- if len(filesC) != 0 {
- t.Errorf("Expected 0 files for user C, got %d", len(filesC))
- }
-}
-
-// TestGetStats tests service statistics
-func TestGetStats(t *testing.T) {
- tempDir := t.TempDir()
- svc := NewService(tempDir)
-
- // Create some files
- svc.CreateFile("user-1", []byte("content 1"), "file1.txt", "text/plain")
- svc.CreateFile("user-2", []byte("content 2 longer"), "file2.txt", "text/plain")
-
- stats := svc.GetStats()
-
- totalFiles, ok := stats["total_files"].(int)
- if !ok || totalFiles != 2 {
- t.Errorf("Expected 2 total files, got %v", stats["total_files"])
- }
-
- totalSize, ok := stats["total_size"].(int64)
- if !ok || totalSize != int64(len("content 1")+len("content 2 longer")) {
- t.Errorf("Expected total size %d, got %v", len("content 1")+len("content 2 longer"), stats["total_size"])
- }
-
- storageDir, ok := stats["storage_dir"].(string)
- if !ok || storageDir != tempDir {
- t.Errorf("Expected storage dir %s, got %v", tempDir, stats["storage_dir"])
- }
-}
-
-// TestAccessCodeGeneration tests access code is cryptographically random
-func TestAccessCodeGeneration(t *testing.T) {
- codes := make(map[string]bool)
-
- // Generate many codes and check uniqueness
- for i := 0; i < 100; i++ {
- code, err := generateAccessCode()
- if err != nil {
- t.Fatalf("generateAccessCode failed: %v", err)
- }
-
- if len(code) != 32 {
- t.Errorf("Access code length should be 32, got %d", len(code))
- }
-
- if codes[code] {
- t.Error("Duplicate access code generated")
- }
- codes[code] = true
- }
-}
-
-// TestAccessCodeHashing tests SHA256 hashing
-func TestAccessCodeHashing(t *testing.T) {
- code := "test-access-code-12345678"
- hash := hashAccessCode(code)
-
- // SHA256 produces 64-char hex string
- if len(hash) != 64 {
- t.Errorf("Hash length should be 64, got %d", len(hash))
- }
-
- // Same input should produce same hash
- hash2 := hashAccessCode(code)
- if hash != hash2 {
- t.Error("Same input should produce same hash")
- }
-
- // Different input should produce different hash
- hash3 := hashAccessCode("different-code")
- if hash == hash3 {
- t.Error("Different inputs should produce different hashes")
- }
-}
-
-// TestMimeTypeExtensionMapping tests extension detection from MIME type
-func TestMimeTypeExtensionMapping(t *testing.T) {
- testCases := []struct {
- mimeType string
- expected string
- }{
- {"application/pdf", ".pdf"},
- {"application/json", ".json"},
- {"text/plain", ".txt"},
- {"text/csv", ".csv"},
- {"text/html", ".html"},
- {"unknown/type", ".bin"},
- }
-
- for _, tc := range testCases {
- result := getExtensionFromMimeType(tc.mimeType)
- if result != tc.expected {
- t.Errorf("getExtensionFromMimeType(%s) = %s, expected %s", tc.mimeType, result, tc.expected)
- }
- }
-}
-
-// TestFileExtensionFromFilename tests extension detection from filename
-func TestFileExtensionFromFilename(t *testing.T) {
- tempDir := t.TempDir()
- svc := NewService(tempDir)
-
- // Create file with extension in filename
- result, _ := svc.CreateFile("user", []byte("data"), "document.pdf", "application/pdf")
-
- // Verify file was created with correct extension
- files, _ := filepath.Glob(filepath.Join(tempDir, result.ID+"*"))
- if len(files) != 1 {
- t.Fatal("Expected exactly one file")
- }
-
- if filepath.Ext(files[0]) != ".pdf" {
- t.Errorf("Expected .pdf extension, got %s", filepath.Ext(files[0]))
- }
-}
-
-// TestConcurrentAccess tests thread safety
-func TestConcurrentAccess(t *testing.T) {
- tempDir := t.TempDir()
- svc := NewService(tempDir)
-
- done := make(chan bool, 10)
-
- // Create files concurrently
- for i := 0; i < 10; i++ {
- go func(idx int) {
- _, err := svc.CreateFile("concurrent-user", []byte("data"), "concurrent.txt", "text/plain")
- if err != nil {
- t.Errorf("Concurrent CreateFile failed: %v", err)
- }
- done <- true
- }(i)
- }
-
- // Wait for all goroutines
- for i := 0; i < 10; i++ {
- <-done
- }
-
- // Verify all files created
- files := svc.ListUserFiles("concurrent-user")
- if len(files) != 10 {
- t.Errorf("Expected 10 files, got %d", len(files))
- }
-}
-
-// Benchmark tests
-func BenchmarkCreateFile(b *testing.B) {
- tempDir := b.TempDir()
- svc := NewService(tempDir)
- content := []byte("benchmark content")
-
- b.ResetTimer()
- for i := 0; i < b.N; i++ {
- svc.CreateFile("bench-user", content, "bench.txt", "text/plain")
- }
-}
-
-func BenchmarkGetFile(b *testing.B) {
- tempDir := b.TempDir()
- svc := NewService(tempDir)
- content := []byte("benchmark content")
-
- result, _ := svc.CreateFile("bench-user", content, "bench.txt", "text/plain")
-
- b.ResetTimer()
- for i := 0; i < b.N; i++ {
- svc.GetFile(result.ID, result.AccessCode)
- }
-}
-
-func BenchmarkAccessCodeGeneration(b *testing.B) {
- for i := 0; i < b.N; i++ {
- generateAccessCode()
- }
-}
-
-func BenchmarkAccessCodeHashing(b *testing.B) {
- code := "test-access-code-12345678"
- b.ResetTimer()
- for i := 0; i < b.N; i++ {
- hashAccessCode(code)
- }
-}
diff --git a/backend/internal/security/crypto.go b/backend/internal/security/crypto.go
deleted file mode 100644
index 7e934b2f..00000000
--- a/backend/internal/security/crypto.go
+++ /dev/null
@@ -1,184 +0,0 @@
-package security
-
-import (
- "crypto/aes"
- "crypto/cipher"
- "crypto/rand"
- "fmt"
- "io"
- "os"
-)
-
-// EncryptionKey represents a 32-byte AES-256 key
-type EncryptionKey [32]byte
-
-// GenerateKey creates a cryptographically secure random encryption key
-func GenerateKey() (*EncryptionKey, error) {
- var key EncryptionKey
- if _, err := io.ReadFull(rand.Reader, key[:]); err != nil {
- return nil, fmt.Errorf("failed to generate encryption key: %w", err)
- }
- return &key, nil
-}
-
-// EncryptFile encrypts a file using AES-256-GCM and saves it to destPath
-// Returns the encryption key used (needed for decryption)
-func EncryptFile(srcPath, destPath string) (*EncryptionKey, error) {
- // Generate encryption key
- key, err := GenerateKey()
- if err != nil {
- return nil, err
- }
-
- // Read source file
- plaintext, err := os.ReadFile(srcPath)
- if err != nil {
- return nil, fmt.Errorf("failed to read source file: %w", err)
- }
-
- // Encrypt data
- ciphertext, err := EncryptData(plaintext, key)
- if err != nil {
- return nil, err
- }
-
- // Write encrypted file
- if err := os.WriteFile(destPath, ciphertext, 0600); err != nil {
- return nil, fmt.Errorf("failed to write encrypted file: %w", err)
- }
-
- return key, nil
-}
-
-// DecryptFile decrypts a file and returns the plaintext data
-// File is NOT written to disk - returned in memory only
-func DecryptFile(srcPath string, key *EncryptionKey) ([]byte, error) {
- // Read encrypted file
- ciphertext, err := os.ReadFile(srcPath)
- if err != nil {
- return nil, fmt.Errorf("failed to read encrypted file: %w", err)
- }
-
- // Decrypt data
- plaintext, err := DecryptData(ciphertext, key)
- if err != nil {
- return nil, err
- }
-
- return plaintext, nil
-}
-
-// EncryptData encrypts data using AES-256-GCM
-func EncryptData(plaintext []byte, key *EncryptionKey) ([]byte, error) {
- // Create cipher block
- block, err := aes.NewCipher(key[:])
- if err != nil {
- return nil, fmt.Errorf("failed to create cipher: %w", err)
- }
-
- // Create GCM mode
- gcm, err := cipher.NewGCM(block)
- if err != nil {
- return nil, fmt.Errorf("failed to create GCM: %w", err)
- }
-
- // Generate nonce
- nonce := make([]byte, gcm.NonceSize())
- if _, err := io.ReadFull(rand.Reader, nonce); err != nil {
- return nil, fmt.Errorf("failed to generate nonce: %w", err)
- }
-
- // Encrypt and authenticate
- // Format: [nonce][ciphertext+tag]
- ciphertext := gcm.Seal(nonce, nonce, plaintext, nil)
-
- return ciphertext, nil
-}
-
-// DecryptData decrypts data using AES-256-GCM
-func DecryptData(ciphertext []byte, key *EncryptionKey) ([]byte, error) {
- // Create cipher block
- block, err := aes.NewCipher(key[:])
- if err != nil {
- return nil, fmt.Errorf("failed to create cipher: %w", err)
- }
-
- // Create GCM mode
- gcm, err := cipher.NewGCM(block)
- if err != nil {
- return nil, fmt.Errorf("failed to create GCM: %w", err)
- }
-
- // Extract nonce
- nonceSize := gcm.NonceSize()
- if len(ciphertext) < nonceSize {
- return nil, fmt.Errorf("ciphertext too short")
- }
-
- nonce, ciphertext := ciphertext[:nonceSize], ciphertext[nonceSize:]
-
- // Decrypt and verify authentication tag
- plaintext, err := gcm.Open(nil, nonce, ciphertext, nil)
- if err != nil {
- return nil, fmt.Errorf("decryption failed (tampered or wrong key): %w", err)
- }
-
- return plaintext, nil
-}
-
-// SecureDeleteFile deletes a file and attempts to overwrite its contents first
-// Note: This is best-effort on modern filesystems with journaling/SSDs
-func SecureDeleteFile(path string) error {
- // Get file info
- info, err := os.Stat(path)
- if err != nil {
- return err
- }
-
- // Open file for writing
- file, err := os.OpenFile(path, os.O_WRONLY, 0600)
- if err != nil {
- return err
- }
-
- // Overwrite with zeros
- zeros := make([]byte, info.Size())
- if _, err := file.Write(zeros); err != nil {
- file.Close()
- return err
- }
-
- // Sync to disk
- if err := file.Sync(); err != nil {
- file.Close()
- return err
- }
-
- // Overwrite with random data
- random := make([]byte, info.Size())
- if _, err := io.ReadFull(rand.Reader, random); err != nil {
- file.Close()
- return err
- }
-
- if _, err := file.Seek(0, 0); err != nil {
- file.Close()
- return err
- }
-
- if _, err := file.Write(random); err != nil {
- file.Close()
- return err
- }
-
- // Final sync
- if err := file.Sync(); err != nil {
- file.Close()
- return err
- }
-
- file.Close()
-
- // Delete file
- return os.Remove(path)
-}
diff --git a/backend/internal/security/hash.go b/backend/internal/security/hash.go
deleted file mode 100644
index 0ba708fb..00000000
--- a/backend/internal/security/hash.go
+++ /dev/null
@@ -1,88 +0,0 @@
-package security
-
-import (
- "crypto/sha256"
- "crypto/subtle"
- "encoding/hex"
- "fmt"
- "io"
- "os"
-)
-
-// Hash represents a SHA-256 hash (32 bytes)
-type Hash [32]byte
-
-// CalculateFileHash computes the SHA-256 hash of a file
-func CalculateFileHash(path string) (*Hash, error) {
- file, err := os.Open(path)
- if err != nil {
- return nil, fmt.Errorf("failed to open file: %w", err)
- }
- defer file.Close()
-
- hash := sha256.New()
- if _, err := io.Copy(hash, file); err != nil {
- return nil, fmt.Errorf("failed to read file: %w", err)
- }
-
- var result Hash
- copy(result[:], hash.Sum(nil))
- return &result, nil
-}
-
-// CalculateDataHash computes the SHA-256 hash of byte data
-func CalculateDataHash(data []byte) *Hash {
- hashArray := sha256.Sum256(data)
- hash := Hash(hashArray)
- return &hash
-}
-
-// String returns the hash as a hex string
-func (h *Hash) String() string {
- return hex.EncodeToString(h[:])
-}
-
-// Bytes returns the hash as a byte slice
-func (h *Hash) Bytes() []byte {
- return h[:]
-}
-
-// Equal compares two hashes using constant-time comparison
-// This prevents timing attacks
-func (h *Hash) Equal(other *Hash) bool {
- if other == nil {
- return false
- }
- return subtle.ConstantTimeCompare(h[:], other[:]) == 1
-}
-
-// FromHexString creates a Hash from a hex string
-func FromHexString(s string) (*Hash, error) {
- bytes, err := hex.DecodeString(s)
- if err != nil {
- return nil, fmt.Errorf("invalid hex string: %w", err)
- }
-
- if len(bytes) != 32 {
- return nil, fmt.Errorf("invalid hash length: expected 32 bytes, got %d", len(bytes))
- }
-
- var hash Hash
- copy(hash[:], bytes)
- return &hash, nil
-}
-
-// Verify checks if the given data matches the hash
-func (h *Hash) Verify(data []byte) bool {
- computed := CalculateDataHash(data)
- return h.Equal(computed)
-}
-
-// VerifyFile checks if the given file matches the hash
-func (h *Hash) VerifyFile(path string) (bool, error) {
- computed, err := CalculateFileHash(path)
- if err != nil {
- return false, err
- }
- return h.Equal(computed), nil
-}
diff --git a/backend/internal/security/memory.go b/backend/internal/security/memory.go
deleted file mode 100644
index 040c6427..00000000
--- a/backend/internal/security/memory.go
+++ /dev/null
@@ -1,183 +0,0 @@
-package security
-
-import (
- "crypto/rand"
- "runtime"
- "sync"
-)
-
-// SecureString holds sensitive string data that can be securely wiped
-type SecureString struct {
- data []byte
- mu sync.Mutex
-}
-
-// NewSecureString creates a new SecureString from a regular string
-func NewSecureString(s string) *SecureString {
- ss := &SecureString{
- data: []byte(s),
- }
- // Set finalizer to wipe memory when garbage collected
- runtime.SetFinalizer(ss, func(s *SecureString) {
- s.Wipe()
- })
- return ss
-}
-
-// String returns the string value (use sparingly)
-func (s *SecureString) String() string {
- s.mu.Lock()
- defer s.mu.Unlock()
- if s.data == nil {
- return ""
- }
- return string(s.data)
-}
-
-// Bytes returns the byte slice (use sparingly)
-func (s *SecureString) Bytes() []byte {
- s.mu.Lock()
- defer s.mu.Unlock()
- if s.data == nil {
- return nil
- }
- // Return a copy to prevent external modification
- result := make([]byte, len(s.data))
- copy(result, s.data)
- return result
-}
-
-// Len returns the length of the string
-func (s *SecureString) Len() int {
- s.mu.Lock()
- defer s.mu.Unlock()
- if s.data == nil {
- return 0
- }
- return len(s.data)
-}
-
-// IsEmpty returns true if the string is empty or wiped
-func (s *SecureString) IsEmpty() bool {
- s.mu.Lock()
- defer s.mu.Unlock()
- return s.data == nil || len(s.data) == 0
-}
-
-// Wipe securely erases the string data from memory
-func (s *SecureString) Wipe() {
- s.mu.Lock()
- defer s.mu.Unlock()
-
- if s.data == nil {
- return
- }
-
- // Three-pass wipe: zeros, random, zeros
- // Pass 1: Overwrite with zeros
- for i := range s.data {
- s.data[i] = 0
- }
-
- // Pass 2: Overwrite with random data
- if len(s.data) > 0 {
- random := make([]byte, len(s.data))
- rand.Read(random)
- copy(s.data, random)
- // Wipe the random buffer too
- for i := range random {
- random[i] = 0
- }
- }
-
- // Pass 3: Overwrite with zeros again
- for i := range s.data {
- s.data[i] = 0
- }
-
- // Clear the slice
- s.data = nil
-}
-
-// WipeBytes securely wipes a byte slice
-func WipeBytes(data []byte) {
- if data == nil {
- return
- }
-
- // Pass 1: Zeros
- for i := range data {
- data[i] = 0
- }
-
- // Pass 2: Random
- if len(data) > 0 {
- random := make([]byte, len(data))
- rand.Read(random)
- copy(data, random)
- // Wipe random buffer
- for i := range random {
- random[i] = 0
- }
- }
-
- // Pass 3: Zeros
- for i := range data {
- data[i] = 0
- }
-}
-
-// WipeString securely wipes a string by converting to byte slice
-// Note: Strings in Go are immutable, so this creates a copy
-// Use SecureString for better security
-func WipeString(s *string) {
- if s == nil || *s == "" {
- return
- }
- data := []byte(*s)
- WipeBytes(data)
- *s = ""
-}
-
-// SecureBuffer is a buffer that automatically wipes its contents
-type SecureBuffer struct {
- data []byte
- mu sync.Mutex
-}
-
-// NewSecureBuffer creates a new secure buffer with given size
-func NewSecureBuffer(size int) *SecureBuffer {
- sb := &SecureBuffer{
- data: make([]byte, size),
- }
- runtime.SetFinalizer(sb, func(b *SecureBuffer) {
- b.Wipe()
- })
- return sb
-}
-
-// Write writes data to the buffer
-func (b *SecureBuffer) Write(data []byte) {
- b.mu.Lock()
- defer b.mu.Unlock()
- if len(data) <= len(b.data) {
- copy(b.data, data)
- }
-}
-
-// Read reads data from the buffer
-func (b *SecureBuffer) Read() []byte {
- b.mu.Lock()
- defer b.mu.Unlock()
- result := make([]byte, len(b.data))
- copy(result, b.data)
- return result
-}
-
-// Wipe securely erases the buffer
-func (b *SecureBuffer) Wipe() {
- b.mu.Lock()
- defer b.mu.Unlock()
- WipeBytes(b.data)
- b.data = nil
-}
diff --git a/backend/internal/security/oauth_state.go b/backend/internal/security/oauth_state.go
deleted file mode 100644
index 10054b53..00000000
--- a/backend/internal/security/oauth_state.go
+++ /dev/null
@@ -1,106 +0,0 @@
-package security
-
-import (
- "crypto/rand"
- "encoding/hex"
- "errors"
- "sync"
- "time"
-)
-
-// OAuthState represents a stored OAuth state token
-type OAuthState struct {
- UserID string
- Service string // "gmail" or "googlesheets"
- ExpiresAt time.Time
-}
-
-// OAuthStateStore manages OAuth state tokens with expiration
-type OAuthStateStore struct {
- states map[string]*OAuthState
- mutex sync.RWMutex
-}
-
-// NewOAuthStateStore creates a new OAuth state store
-func NewOAuthStateStore() *OAuthStateStore {
- store := &OAuthStateStore{
- states: make(map[string]*OAuthState),
- }
-
- // Start cleanup goroutine to remove expired states every minute
- go store.cleanupExpired()
-
- return store
-}
-
-// GenerateState generates a cryptographically secure random state token
-func (s *OAuthStateStore) GenerateState(userID, service string) (string, error) {
- // Generate 32 random bytes (256 bits)
- randomBytes := make([]byte, 32)
- if _, err := rand.Read(randomBytes); err != nil {
- return "", err
- }
-
- stateToken := hex.EncodeToString(randomBytes)
-
- s.mutex.Lock()
- defer s.mutex.Unlock()
-
- // Store state with 10-minute expiration
- s.states[stateToken] = &OAuthState{
- UserID: userID,
- Service: service,
- ExpiresAt: time.Now().Add(10 * time.Minute),
- }
-
- return stateToken, nil
-}
-
-// ValidateState validates a state token and returns the associated user ID
-// The state token is consumed (one-time use) after validation
-func (s *OAuthStateStore) ValidateState(stateToken string) (string, string, error) {
- s.mutex.Lock()
- defer s.mutex.Unlock()
-
- state, exists := s.states[stateToken]
- if !exists {
- return "", "", errors.New("invalid or expired state token")
- }
-
- // Check expiration
- if time.Now().After(state.ExpiresAt) {
- delete(s.states, stateToken)
- return "", "", errors.New("state token expired")
- }
-
- // Delete state token (one-time use for CSRF protection)
- userID := state.UserID
- service := state.Service
- delete(s.states, stateToken)
-
- return userID, service, nil
-}
-
-// cleanupExpired removes expired state tokens every minute
-func (s *OAuthStateStore) cleanupExpired() {
- ticker := time.NewTicker(1 * time.Minute)
- defer ticker.Stop()
-
- for range ticker.C {
- s.mutex.Lock()
- now := time.Now()
- for token, state := range s.states {
- if now.After(state.ExpiresAt) {
- delete(s.states, token)
- }
- }
- s.mutex.Unlock()
- }
-}
-
-// Count returns the number of active state tokens (for monitoring)
-func (s *OAuthStateStore) Count() int {
- s.mutex.RLock()
- defer s.mutex.RUnlock()
- return len(s.states)
-}
diff --git a/backend/internal/security/path_validation.go b/backend/internal/security/path_validation.go
deleted file mode 100644
index a033b962..00000000
--- a/backend/internal/security/path_validation.go
+++ /dev/null
@@ -1,43 +0,0 @@
-package security
-
-import (
- "fmt"
- "regexp"
- "strings"
-)
-
-// ValidateFileID validates that a file ID is a valid UUID and contains no path traversal sequences.
-// This prevents path traversal attacks like "../../../etc/passwd" or absolute paths.
-//
-// Returns an error if the fileID:
-// - Is empty
-// - Contains path traversal sequences (.., /, \)
-// - Is not a valid UUID format (xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx)
-//
-// This function should be called before using any user-provided file ID in file system operations.
-func ValidateFileID(fileID string) error {
- if fileID == "" {
- return fmt.Errorf("file_id cannot be empty")
- }
-
- // Check for path traversal sequences
- if strings.Contains(fileID, "..") {
- return fmt.Errorf("invalid file_id: path traversal attempt detected (..)")
- }
- if strings.Contains(fileID, "/") {
- return fmt.Errorf("invalid file_id: path traversal attempt detected (/)")
- }
- if strings.Contains(fileID, "\\") {
- return fmt.Errorf("invalid file_id: path traversal attempt detected (\\)")
- }
-
- // Validate UUID format (standard UUID format: xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx)
- // UUIDs are always 36 characters with hyphens at positions 8, 13, 18, 23
- // This regex matches UUID v1, v4, and other valid UUID formats
- uuidPattern := regexp.MustCompile(`^[a-fA-F0-9]{8}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{12}$`)
- if !uuidPattern.MatchString(fileID) {
- return fmt.Errorf("invalid file_id format: expected UUID (got %q)", fileID)
- }
-
- return nil
-}
diff --git a/backend/internal/security/ssrf.go b/backend/internal/security/ssrf.go
deleted file mode 100644
index 97095f38..00000000
--- a/backend/internal/security/ssrf.go
+++ /dev/null
@@ -1,158 +0,0 @@
-package security
-
-import (
- "fmt"
- "net"
- "net/url"
- "strings"
-)
-
-// privateIPRanges contains CIDR ranges for private/internal networks
-var privateIPRanges = []string{
- "127.0.0.0/8", // IPv4 loopback
- "10.0.0.0/8", // RFC1918 private
- "172.16.0.0/12", // RFC1918 private
- "192.168.0.0/16", // RFC1918 private
- "169.254.0.0/16", // Link-local
- "::1/128", // IPv6 loopback
- "fc00::/7", // IPv6 unique local
- "fe80::/10", // IPv6 link-local
- "0.0.0.0/8", // "This" network
-}
-
-// blockedHostnames contains hostnames that should never be accessed
-var blockedHostnames = []string{
- "localhost",
- "localhost.localdomain",
- "ip6-localhost",
- "ip6-loopback",
- "metadata.google.internal", // GCP metadata
- "169.254.169.254", // AWS/GCP/Azure metadata endpoint
- "metadata.google.internal.", // GCP metadata with trailing dot
- "kubernetes.default.svc", // Kubernetes
- "kubernetes.default", // Kubernetes
-}
-
-var parsedCIDRs []*net.IPNet
-
-func init() {
- // Pre-parse CIDR ranges for efficiency
- for _, cidr := range privateIPRanges {
- _, network, err := net.ParseCIDR(cidr)
- if err == nil {
- parsedCIDRs = append(parsedCIDRs, network)
- }
- }
-}
-
-// IsPrivateIP checks if an IP address is in a private/internal range
-func IsPrivateIP(ip net.IP) bool {
- if ip == nil {
- return true // Treat nil as blocked for safety
- }
-
- for _, network := range parsedCIDRs {
- if network.Contains(ip) {
- return true
- }
- }
- return false
-}
-
-// IsBlockedHostname checks if a hostname is in the blocklist
-func IsBlockedHostname(hostname string) bool {
- hostname = strings.ToLower(strings.TrimSuffix(hostname, "."))
-
- for _, blocked := range blockedHostnames {
- if hostname == blocked {
- return true
- }
- // Also check if it ends with the blocked hostname (subdomain matching)
- if strings.HasSuffix(hostname, "."+blocked) {
- return true
- }
- }
- return false
-}
-
-// ValidateURLForSSRF validates a URL to prevent SSRF attacks
-// Returns an error if the URL points to a private/internal resource
-func ValidateURLForSSRF(rawURL string) error {
- parsedURL, err := url.Parse(rawURL)
- if err != nil {
- return fmt.Errorf("invalid URL format: %w", err)
- }
-
- // Only allow http/https schemes
- if parsedURL.Scheme != "http" && parsedURL.Scheme != "https" {
- return fmt.Errorf("only http and https schemes are allowed")
- }
-
- hostname := parsedURL.Hostname()
- if hostname == "" {
- return fmt.Errorf("URL must have a hostname")
- }
-
- // Check against blocked hostnames
- if IsBlockedHostname(hostname) {
- return fmt.Errorf("access to internal hostname '%s' is not allowed", hostname)
- }
-
- // Try to parse as IP address first
- ip := net.ParseIP(hostname)
- if ip != nil {
- if IsPrivateIP(ip) {
- return fmt.Errorf("access to private IP address '%s' is not allowed", hostname)
- }
- return nil
- }
-
- // Resolve hostname to IP addresses
- ips, err := net.LookupIP(hostname)
- if err != nil {
- // DNS resolution failed - allow the request to proceed
- // The actual HTTP request will fail if the host is unreachable
- return nil
- }
-
- // Check all resolved IPs
- for _, resolvedIP := range ips {
- if IsPrivateIP(resolvedIP) {
- return fmt.Errorf("hostname '%s' resolves to private IP address '%s'", hostname, resolvedIP.String())
- }
- }
-
- return nil
-}
-
-// ValidateURLForSSRFQuick performs a quick validation without DNS resolution
-// Use this when DNS resolution overhead is unacceptable
-func ValidateURLForSSRFQuick(rawURL string) error {
- parsedURL, err := url.Parse(rawURL)
- if err != nil {
- return fmt.Errorf("invalid URL format: %w", err)
- }
-
- // Only allow http/https schemes
- if parsedURL.Scheme != "http" && parsedURL.Scheme != "https" {
- return fmt.Errorf("only http and https schemes are allowed")
- }
-
- hostname := parsedURL.Hostname()
- if hostname == "" {
- return fmt.Errorf("URL must have a hostname")
- }
-
- // Check against blocked hostnames
- if IsBlockedHostname(hostname) {
- return fmt.Errorf("access to internal hostname '%s' is not allowed", hostname)
- }
-
- // Check if hostname is an IP address
- ip := net.ParseIP(hostname)
- if ip != nil && IsPrivateIP(ip) {
- return fmt.Errorf("access to private IP address '%s' is not allowed", hostname)
- }
-
- return nil
-}
diff --git a/backend/internal/services/agent_service.go b/backend/internal/services/agent_service.go
deleted file mode 100644
index 918a08e8..00000000
--- a/backend/internal/services/agent_service.go
+++ /dev/null
@@ -1,819 +0,0 @@
-package services
-
-import (
- "claraverse/internal/database"
- "claraverse/internal/models"
- "context"
- "fmt"
- "log"
- "time"
-
- "github.com/google/uuid"
- "go.mongodb.org/mongo-driver/bson"
- "go.mongodb.org/mongo-driver/bson/primitive"
- "go.mongodb.org/mongo-driver/mongo"
- "go.mongodb.org/mongo-driver/mongo/options"
-)
-
-// ============================================================================
-// MongoDB Records
-// ============================================================================
-
-// AgentRecord is the MongoDB representation of an agent
-type AgentRecord struct {
- ID primitive.ObjectID `bson:"_id,omitempty" json:"_id,omitempty"`
- AgentID string `bson:"agentId" json:"agentId"` // String ID for API compatibility
- UserID string `bson:"userId" json:"userId"`
- Name string `bson:"name" json:"name"`
- Description string `bson:"description,omitempty" json:"description,omitempty"`
- Status string `bson:"status" json:"status"`
- CreatedAt time.Time `bson:"createdAt" json:"createdAt"`
- UpdatedAt time.Time `bson:"updatedAt" json:"updatedAt"`
-}
-
-// ToModel converts AgentRecord to models.Agent
-func (r *AgentRecord) ToModel() *models.Agent {
- return &models.Agent{
- ID: r.AgentID,
- UserID: r.UserID,
- Name: r.Name,
- Description: r.Description,
- Status: r.Status,
- CreatedAt: r.CreatedAt,
- UpdatedAt: r.UpdatedAt,
- }
-}
-
-// WorkflowRecord is the MongoDB representation of a workflow
-type WorkflowRecord struct {
- ID primitive.ObjectID `bson:"_id,omitempty" json:"_id,omitempty"`
- WorkflowID string `bson:"workflowId" json:"workflowId"` // String ID for API compatibility
- AgentID string `bson:"agentId" json:"agentId"`
- Blocks []models.Block `bson:"blocks" json:"blocks"`
- Connections []models.Connection `bson:"connections" json:"connections"`
- Variables []models.Variable `bson:"variables" json:"variables"`
- Version int `bson:"version" json:"version"`
- CreatedAt time.Time `bson:"createdAt" json:"createdAt"`
- UpdatedAt time.Time `bson:"updatedAt" json:"updatedAt"`
-}
-
-// ToModel converts WorkflowRecord to models.Workflow
-func (r *WorkflowRecord) ToModel() *models.Workflow {
- return &models.Workflow{
- ID: r.WorkflowID,
- AgentID: r.AgentID,
- Blocks: r.Blocks,
- Connections: r.Connections,
- Variables: r.Variables,
- Version: r.Version,
- CreatedAt: r.CreatedAt,
- UpdatedAt: r.UpdatedAt,
- }
-}
-
-// WorkflowVersionRecord stores historical workflow versions
-type WorkflowVersionRecord struct {
- ID primitive.ObjectID `bson:"_id,omitempty" json:"_id,omitempty"`
- AgentID string `bson:"agentId" json:"agentId"`
- Version int `bson:"version" json:"version"`
- Blocks []models.Block `bson:"blocks" json:"blocks"`
- Connections []models.Connection `bson:"connections" json:"connections"`
- Variables []models.Variable `bson:"variables" json:"variables"`
- Description string `bson:"description,omitempty" json:"description,omitempty"`
- CreatedAt time.Time `bson:"createdAt" json:"createdAt"`
-}
-
-// WorkflowVersionResponse is the API response for workflow versions
-type WorkflowVersionResponse struct {
- Version int `json:"version"`
- Description string `json:"description,omitempty"`
- BlockCount int `json:"blockCount"`
- CreatedAt time.Time `json:"createdAt"`
-}
-
-// ============================================================================
-// AgentService
-// ============================================================================
-
-// AgentService handles agent and workflow operations using MongoDB
-type AgentService struct {
- mongoDB *database.MongoDB
-}
-
-// NewAgentService creates a new agent service
-func NewAgentService(mongoDB *database.MongoDB) *AgentService {
- return &AgentService{mongoDB: mongoDB}
-}
-
-// Collection helpers
-func (s *AgentService) agentsCollection() *mongo.Collection {
- return s.mongoDB.Database().Collection("agents")
-}
-
-func (s *AgentService) workflowsCollection() *mongo.Collection {
- return s.mongoDB.Database().Collection("workflows")
-}
-
-func (s *AgentService) workflowVersionsCollection() *mongo.Collection {
- return s.mongoDB.Database().Collection("workflow_versions")
-}
-
-// ============================================================================
-// Agent CRUD Operations
-// ============================================================================
-
-// CreateAgent creates a new agent for a user with auto-generated ID
-func (s *AgentService) CreateAgent(userID, name, description string) (*models.Agent, error) {
- id := uuid.New().String()
- return s.CreateAgentWithID(id, userID, name, description)
-}
-
-// CreateAgentWithID creates a new agent with a specific ID (for frontend-generated IDs)
-func (s *AgentService) CreateAgentWithID(id, userID, name, description string) (*models.Agent, error) {
- ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
- defer cancel()
-
- now := time.Now()
-
- record := &AgentRecord{
- AgentID: id,
- UserID: userID,
- Name: name,
- Description: description,
- Status: "draft",
- CreatedAt: now,
- UpdatedAt: now,
- }
-
- _, err := s.agentsCollection().InsertOne(ctx, record)
- if err != nil {
- return nil, fmt.Errorf("failed to create agent: %w", err)
- }
-
- log.Printf("📝 [AGENT] Created agent %s for user %s", id, userID)
- return record.ToModel(), nil
-}
-
-// GetAgent retrieves an agent by ID for a specific user
-func (s *AgentService) GetAgent(agentID, userID string) (*models.Agent, error) {
- ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
- defer cancel()
-
- var record AgentRecord
- err := s.agentsCollection().FindOne(ctx, bson.M{
- "agentId": agentID,
- "userId": userID,
- }).Decode(&record)
-
- if err == mongo.ErrNoDocuments {
- return nil, fmt.Errorf("agent not found")
- }
- if err != nil {
- return nil, fmt.Errorf("failed to get agent: %w", err)
- }
-
- agent := record.ToModel()
-
- // Also load the workflow if it exists
- workflow, err := s.GetWorkflow(agentID)
- if err == nil {
- agent.Workflow = workflow
- }
-
- return agent, nil
-}
-
-// GetAgentByID retrieves an agent by ID only (for internal/scheduled use)
-// WARNING: This bypasses user ownership check - use only for scheduled jobs
-func (s *AgentService) GetAgentByID(agentID string) (*models.Agent, error) {
- ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
- defer cancel()
-
- var record AgentRecord
- err := s.agentsCollection().FindOne(ctx, bson.M{
- "agentId": agentID,
- }).Decode(&record)
-
- if err == mongo.ErrNoDocuments {
- return nil, fmt.Errorf("agent not found")
- }
- if err != nil {
- return nil, fmt.Errorf("failed to get agent: %w", err)
- }
-
- agent := record.ToModel()
-
- // Also load the workflow if it exists
- workflow, err := s.GetWorkflow(agentID)
- if err == nil {
- agent.Workflow = workflow
- }
-
- return agent, nil
-}
-
-// ListAgents returns all agents for a user
-func (s *AgentService) ListAgents(userID string) ([]*models.Agent, error) {
- ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
- defer cancel()
-
- cursor, err := s.agentsCollection().Find(ctx, bson.M{"userId": userID},
- options.Find().SetSort(bson.D{{Key: "updatedAt", Value: -1}}))
- if err != nil {
- return nil, fmt.Errorf("failed to list agents: %w", err)
- }
- defer cursor.Close(ctx)
-
- var records []AgentRecord
- if err := cursor.All(ctx, &records); err != nil {
- return nil, fmt.Errorf("failed to decode agents: %w", err)
- }
-
- agents := make([]*models.Agent, len(records))
- for i, record := range records {
- agents[i] = record.ToModel()
- }
-
- return agents, nil
-}
-
-// UpdateAgent updates an agent's metadata
-func (s *AgentService) UpdateAgent(agentID, userID string, req *models.UpdateAgentRequest) (*models.Agent, error) {
- ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
- defer cancel()
-
- // First check if agent exists and belongs to user
- agent, err := s.GetAgent(agentID, userID)
- if err != nil {
- return nil, err
- }
-
- // Build update document
- updateFields := bson.M{"updatedAt": time.Now()}
- if req.Name != "" {
- updateFields["name"] = req.Name
- agent.Name = req.Name
- }
- if req.Description != "" {
- updateFields["description"] = req.Description
- agent.Description = req.Description
- }
- if req.Status != "" {
- updateFields["status"] = req.Status
- agent.Status = req.Status
- }
-
- _, err = s.agentsCollection().UpdateOne(ctx,
- bson.M{"agentId": agentID, "userId": userID},
- bson.M{"$set": updateFields})
- if err != nil {
- return nil, fmt.Errorf("failed to update agent: %w", err)
- }
-
- agent.UpdatedAt = time.Now()
- return agent, nil
-}
-
-// DeleteAgent deletes an agent and its workflow/versions
-func (s *AgentService) DeleteAgent(agentID, userID string) error {
- ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
- defer cancel()
-
- // Delete agent
- result, err := s.agentsCollection().DeleteOne(ctx, bson.M{
- "agentId": agentID,
- "userId": userID,
- })
- if err != nil {
- return fmt.Errorf("failed to delete agent: %w", err)
- }
- if result.DeletedCount == 0 {
- return fmt.Errorf("agent not found")
- }
-
- // Cascade delete workflow
- s.workflowsCollection().DeleteOne(ctx, bson.M{"agentId": agentID})
-
- // Cascade delete workflow versions
- s.workflowVersionsCollection().DeleteMany(ctx, bson.M{"agentId": agentID})
-
- log.Printf("🗑️ [AGENT] Deleted agent %s and associated workflows", agentID)
- return nil
-}
-
-// DeleteAllByUser deletes all agents, workflows, and workflow versions for a user (GDPR compliance)
-func (s *AgentService) DeleteAllByUser(ctx context.Context, userID string) (int64, error) {
- if userID == "" {
- return 0, fmt.Errorf("user ID is required")
- }
-
- // Get all agent IDs for this user first (for cascade deletion)
- cursor, err := s.agentsCollection().Find(ctx, bson.M{"userId": userID})
- if err != nil {
- return 0, fmt.Errorf("failed to find user agents: %w", err)
- }
- defer cursor.Close(ctx)
-
- var agentIDs []string
- for cursor.Next(ctx) {
- var agent struct {
- AgentID string `bson:"agentId"`
- }
- if err := cursor.Decode(&agent); err == nil {
- agentIDs = append(agentIDs, agent.AgentID)
- }
- }
-
- // Delete all workflow versions for these agents
- if len(agentIDs) > 0 {
- _, err := s.workflowVersionsCollection().DeleteMany(ctx, bson.M{
- "agentId": bson.M{"$in": agentIDs},
- })
- if err != nil {
- log.Printf("⚠️ [GDPR] Failed to delete workflow versions: %v", err)
- }
-
- // Delete all workflows for these agents
- _, err = s.workflowsCollection().DeleteMany(ctx, bson.M{
- "agentId": bson.M{"$in": agentIDs},
- })
- if err != nil {
- log.Printf("⚠️ [GDPR] Failed to delete workflows: %v", err)
- }
- }
-
- // Delete all agents for this user
- result, err := s.agentsCollection().DeleteMany(ctx, bson.M{"userId": userID})
- if err != nil {
- return 0, fmt.Errorf("failed to delete agents: %w", err)
- }
-
- log.Printf("🗑️ [GDPR] Deleted %d agents and associated data for user %s", result.DeletedCount, userID)
- return result.DeletedCount, nil
-}
-
-// ============================================================================
-// Workflow Operations
-// ============================================================================
-
-// SaveWorkflow creates or updates a workflow for an agent
-func (s *AgentService) SaveWorkflow(agentID, userID string, req *models.SaveWorkflowRequest) (*models.Workflow, error) {
- return s.SaveWorkflowWithDescription(agentID, userID, req, "")
-}
-
-// SaveWorkflowWithDescription creates or updates a workflow with a version description
-func (s *AgentService) SaveWorkflowWithDescription(agentID, userID string, req *models.SaveWorkflowRequest, description string) (*models.Workflow, error) {
- ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
- defer cancel()
-
- // Verify agent exists and belongs to user
- _, err := s.GetAgent(agentID, userID)
- if err != nil {
- return nil, err
- }
-
- now := time.Now()
-
- // Check if workflow exists
- var existingWorkflow WorkflowRecord
- err = s.workflowsCollection().FindOne(ctx, bson.M{"agentId": agentID}).Decode(&existingWorkflow)
-
- var workflow *models.Workflow
- var newVersion int
-
- if err == mongo.ErrNoDocuments {
- // Create new workflow
- workflowID := uuid.New().String()
- newVersion = 1
-
- record := &WorkflowRecord{
- WorkflowID: workflowID,
- AgentID: agentID,
- Blocks: req.Blocks,
- Connections: req.Connections,
- Variables: req.Variables,
- Version: newVersion,
- CreatedAt: now,
- UpdatedAt: now,
- }
-
- _, err = s.workflowsCollection().InsertOne(ctx, record)
- if err != nil {
- return nil, fmt.Errorf("failed to create workflow: %w", err)
- }
-
- workflow = record.ToModel()
- } else if err != nil {
- return nil, fmt.Errorf("failed to check existing workflow: %w", err)
- } else {
- // Update existing workflow
- newVersion = existingWorkflow.Version + 1
-
- _, err = s.workflowsCollection().UpdateOne(ctx,
- bson.M{"agentId": agentID},
- bson.M{"$set": bson.M{
- "blocks": req.Blocks,
- "connections": req.Connections,
- "variables": req.Variables,
- "version": newVersion,
- "updatedAt": now,
- }})
- if err != nil {
- return nil, fmt.Errorf("failed to update workflow: %w", err)
- }
-
- workflow = &models.Workflow{
- ID: existingWorkflow.WorkflowID,
- AgentID: agentID,
- Blocks: req.Blocks,
- Connections: req.Connections,
- Variables: req.Variables,
- Version: newVersion,
- CreatedAt: existingWorkflow.CreatedAt,
- UpdatedAt: now,
- }
- }
-
- // Only save workflow version snapshot when explicitly requested (e.g., when AI generates/modifies workflow)
- if req.CreateVersion {
- versionDescription := description
- if req.VersionDescription != "" {
- versionDescription = req.VersionDescription
- }
-
- versionRecord := &WorkflowVersionRecord{
- AgentID: agentID,
- Version: newVersion,
- Blocks: req.Blocks,
- Connections: req.Connections,
- Variables: req.Variables,
- Description: versionDescription,
- CreatedAt: now,
- }
-
- _, err = s.workflowVersionsCollection().InsertOne(ctx, versionRecord)
- if err != nil {
- log.Printf("⚠️ [WORKFLOW] Failed to save version snapshot: %v", err)
- // Don't fail the whole operation for version snapshot failure
- } else {
- log.Printf("📸 [WORKFLOW] Saved version %d snapshot for agent %s", newVersion, agentID)
- }
- }
-
- // Update agent's updated_at
- s.agentsCollection().UpdateOne(ctx,
- bson.M{"agentId": agentID},
- bson.M{"$set": bson.M{"updatedAt": now}})
-
- return workflow, nil
-}
-
-// GetWorkflow retrieves a workflow for an agent
-func (s *AgentService) GetWorkflow(agentID string) (*models.Workflow, error) {
- ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
- defer cancel()
-
- var record WorkflowRecord
- err := s.workflowsCollection().FindOne(ctx, bson.M{"agentId": agentID}).Decode(&record)
-
- if err == mongo.ErrNoDocuments {
- return nil, fmt.Errorf("workflow not found")
- }
- if err != nil {
- return nil, fmt.Errorf("failed to get workflow: %w", err)
- }
-
- return record.ToModel(), nil
-}
-
-// ============================================================================
-// Workflow Version History
-// ============================================================================
-
-// ListWorkflowVersions returns all versions for an agent's workflow
-func (s *AgentService) ListWorkflowVersions(agentID, userID string) ([]WorkflowVersionResponse, error) {
- ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
- defer cancel()
-
- // Verify agent belongs to user
- _, err := s.GetAgent(agentID, userID)
- if err != nil {
- return nil, err
- }
-
- cursor, err := s.workflowVersionsCollection().Find(ctx,
- bson.M{"agentId": agentID},
- options.Find().SetSort(bson.D{{Key: "version", Value: -1}}))
- if err != nil {
- return nil, fmt.Errorf("failed to list workflow versions: %w", err)
- }
- defer cursor.Close(ctx)
-
- var records []WorkflowVersionRecord
- if err := cursor.All(ctx, &records); err != nil {
- return nil, fmt.Errorf("failed to decode versions: %w", err)
- }
-
- versions := make([]WorkflowVersionResponse, len(records))
- for i, record := range records {
- versions[i] = WorkflowVersionResponse{
- Version: record.Version,
- Description: record.Description,
- BlockCount: len(record.Blocks),
- CreatedAt: record.CreatedAt,
- }
- }
-
- return versions, nil
-}
-
-// GetWorkflowVersion retrieves a specific workflow version
-func (s *AgentService) GetWorkflowVersion(agentID, userID string, version int) (*models.Workflow, error) {
- ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
- defer cancel()
-
- // Verify agent belongs to user
- _, err := s.GetAgent(agentID, userID)
- if err != nil {
- return nil, err
- }
-
- var record WorkflowVersionRecord
- err = s.workflowVersionsCollection().FindOne(ctx, bson.M{
- "agentId": agentID,
- "version": version,
- }).Decode(&record)
-
- if err == mongo.ErrNoDocuments {
- return nil, fmt.Errorf("workflow version not found")
- }
- if err != nil {
- return nil, fmt.Errorf("failed to get workflow version: %w", err)
- }
-
- return &models.Workflow{
- AgentID: record.AgentID,
- Blocks: record.Blocks,
- Connections: record.Connections,
- Variables: record.Variables,
- Version: record.Version,
- CreatedAt: record.CreatedAt,
- }, nil
-}
-
-// RestoreWorkflowVersion restores a workflow to a previous version
-func (s *AgentService) RestoreWorkflowVersion(agentID, userID string, version int) (*models.Workflow, error) {
- // Get the version to restore
- versionWorkflow, err := s.GetWorkflowVersion(agentID, userID, version)
- if err != nil {
- return nil, err
- }
-
- // Save as new version with description - restoring always creates a version snapshot
- req := &models.SaveWorkflowRequest{
- Blocks: versionWorkflow.Blocks,
- Connections: versionWorkflow.Connections,
- Variables: versionWorkflow.Variables,
- CreateVersion: true, // Always create version when restoring
- VersionDescription: fmt.Sprintf("Restored from version %d", version),
- }
-
- return s.SaveWorkflowWithDescription(agentID, userID, req, "")
-}
-
-// ============================================================================
-// Pagination Methods
-// ============================================================================
-
-// ListAgentsPaginated returns agents with pagination support
-func (s *AgentService) ListAgentsPaginated(userID string, limit, offset int) (*models.PaginatedAgentsResponse, error) {
- ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
- defer cancel()
-
- if limit <= 0 {
- limit = 20
- }
- if limit > 100 {
- limit = 100
- }
- if offset < 0 {
- offset = 0
- }
-
- // Get total count
- total, err := s.agentsCollection().CountDocuments(ctx, bson.M{"userId": userID})
- if err != nil {
- return nil, fmt.Errorf("failed to count agents: %w", err)
- }
-
- // Get agents with pagination
- cursor, err := s.agentsCollection().Find(ctx,
- bson.M{"userId": userID},
- options.Find().
- SetSort(bson.D{{Key: "updatedAt", Value: -1}}).
- SetSkip(int64(offset)).
- SetLimit(int64(limit)))
- if err != nil {
- return nil, fmt.Errorf("failed to list agents: %w", err)
- }
- defer cursor.Close(ctx)
-
- var records []AgentRecord
- if err := cursor.All(ctx, &records); err != nil {
- return nil, fmt.Errorf("failed to decode agents: %w", err)
- }
-
- // Build list items with workflow info
- agents := make([]models.AgentListItem, len(records))
- for i, record := range records {
- item := models.AgentListItem{
- ID: record.AgentID,
- Name: record.Name,
- Description: record.Description,
- Status: record.Status,
- CreatedAt: record.CreatedAt,
- UpdatedAt: record.UpdatedAt,
- }
-
- // Get workflow info
- workflow, err := s.GetWorkflow(record.AgentID)
- if err == nil {
- item.HasWorkflow = true
- item.BlockCount = len(workflow.Blocks)
- }
-
- agents[i] = item
- }
-
- return &models.PaginatedAgentsResponse{
- Agents: agents,
- Total: int(total),
- Limit: limit,
- Offset: offset,
- HasMore: offset+len(agents) < int(total),
- }, nil
-}
-
-// GetRecentAgents returns the 10 most recently updated agents for the landing page
-func (s *AgentService) GetRecentAgents(userID string) (*models.RecentAgentsResponse, error) {
- ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
- defer cancel()
-
- cursor, err := s.agentsCollection().Find(ctx,
- bson.M{"userId": userID},
- options.Find().
- SetSort(bson.D{{Key: "updatedAt", Value: -1}}).
- SetLimit(10))
- if err != nil {
- return nil, fmt.Errorf("failed to get recent agents: %w", err)
- }
- defer cursor.Close(ctx)
-
- var records []AgentRecord
- if err := cursor.All(ctx, &records); err != nil {
- return nil, fmt.Errorf("failed to decode agents: %w", err)
- }
-
- agents := make([]models.AgentListItem, len(records))
- for i, record := range records {
- item := models.AgentListItem{
- ID: record.AgentID,
- Name: record.Name,
- Description: record.Description,
- Status: record.Status,
- CreatedAt: record.CreatedAt,
- UpdatedAt: record.UpdatedAt,
- }
-
- // Get workflow info
- workflow, err := s.GetWorkflow(record.AgentID)
- if err == nil {
- item.HasWorkflow = true
- item.BlockCount = len(workflow.Blocks)
- }
-
- agents[i] = item
- }
-
- return &models.RecentAgentsResponse{
- Agents: agents,
- }, nil
-}
-
-// ============================================================================
-// Sync Method (for first-message persistence)
-// ============================================================================
-
-// SyncAgent creates or updates an agent with its workflow in a single operation
-// This is called when a user sends their first message to persist the local agent
-func (s *AgentService) SyncAgent(agentID, userID string, req *models.SyncAgentRequest) (*models.Agent, *models.Workflow, error) {
- ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
- defer cancel()
-
- now := time.Now()
-
- // Check if agent already exists
- existingAgent, err := s.GetAgent(agentID, userID)
- if err != nil && err.Error() != "agent not found" {
- return nil, nil, fmt.Errorf("failed to check existing agent: %w", err)
- }
-
- var agent *models.Agent
-
- if existingAgent != nil {
- // Update existing agent
- agent, err = s.UpdateAgent(agentID, userID, &models.UpdateAgentRequest{
- Name: req.Name,
- Description: req.Description,
- })
- if err != nil {
- return nil, nil, fmt.Errorf("failed to update agent: %w", err)
- }
- } else {
- // Create new agent with the provided ID
- record := &AgentRecord{
- AgentID: agentID,
- UserID: userID,
- Name: req.Name,
- Description: req.Description,
- Status: "draft",
- CreatedAt: now,
- UpdatedAt: now,
- }
-
- _, err = s.agentsCollection().InsertOne(ctx, record)
- if err != nil {
- return nil, nil, fmt.Errorf("failed to create agent: %w", err)
- }
-
- agent = record.ToModel()
- }
-
- // Save the workflow
- workflow, err := s.SaveWorkflow(agentID, userID, &req.Workflow)
- if err != nil {
- return nil, nil, fmt.Errorf("failed to save workflow: %w", err)
- }
-
- return agent, workflow, nil
-}
-
-// ============================================================================
-// Index Initialization
-// ============================================================================
-
-// EnsureIndexes creates indexes for agents and workflows collections
-func (s *AgentService) EnsureIndexes(ctx context.Context) error {
- // Agents collection indexes
- agentIndexes := []mongo.IndexModel{
- {
- Keys: bson.D{{Key: "agentId", Value: 1}},
- Options: options.Index().SetUnique(true),
- },
- {
- Keys: bson.D{
- {Key: "userId", Value: 1},
- {Key: "updatedAt", Value: -1},
- },
- },
- {
- Keys: bson.D{{Key: "status", Value: 1}},
- },
- }
-
- _, err := s.agentsCollection().Indexes().CreateMany(ctx, agentIndexes)
- if err != nil {
- return fmt.Errorf("failed to create agent indexes: %w", err)
- }
-
- // Workflows collection indexes
- workflowIndexes := []mongo.IndexModel{
- {
- Keys: bson.D{{Key: "agentId", Value: 1}},
- Options: options.Index().SetUnique(true),
- },
- }
-
- _, err = s.workflowsCollection().Indexes().CreateMany(ctx, workflowIndexes)
- if err != nil {
- return fmt.Errorf("failed to create workflow indexes: %w", err)
- }
-
- // Workflow versions collection indexes
- versionIndexes := []mongo.IndexModel{
- {
- Keys: bson.D{
- {Key: "agentId", Value: 1},
- {Key: "version", Value: -1},
- },
- },
- }
-
- _, err = s.workflowVersionsCollection().Indexes().CreateMany(ctx, versionIndexes)
- if err != nil {
- return fmt.Errorf("failed to create workflow version indexes: %w", err)
- }
-
- log.Println("✅ [AGENT] Ensured indexes for agents, workflows, and workflow_versions collections")
- return nil
-}
diff --git a/backend/internal/services/analytics_service.go b/backend/internal/services/analytics_service.go
deleted file mode 100644
index b12e1e02..00000000
--- a/backend/internal/services/analytics_service.go
+++ /dev/null
@@ -1,993 +0,0 @@
-package services
-
-import (
- "context"
- "encoding/json"
- "log"
- "os"
- "path/filepath"
- "time"
-
- "claraverse/internal/database"
- "claraverse/internal/models"
- "go.mongodb.org/mongo-driver/bson"
- "go.mongodb.org/mongo-driver/bson/primitive"
- "go.mongodb.org/mongo-driver/mongo"
-)
-
-// AnalyticsService handles minimal usage tracking (non-invasive)
-type AnalyticsService struct {
- mongoDB *database.MongoDB
-}
-
-// NewAnalyticsService creates a new analytics service
-func NewAnalyticsService(mongoDB *database.MongoDB) *AnalyticsService {
- return &AnalyticsService{
- mongoDB: mongoDB,
- }
-}
-
-// ChatSessionAnalytics stores minimal chat session data
-type ChatSessionAnalytics struct {
- ID primitive.ObjectID `bson:"_id,omitempty" json:"id"`
- UserID string `bson:"userId" json:"userId"`
- ConversationID string `bson:"conversationId" json:"conversationId"`
- SessionID string `bson:"sessionId" json:"sessionId"` // WebSocket connection ID
-
- // Minimal metrics
- MessageCount int `bson:"messageCount" json:"messageCount"`
- StartedAt time.Time `bson:"startedAt" json:"startedAt"`
- EndedAt *time.Time `bson:"endedAt,omitempty" json:"endedAt,omitempty"`
- DurationMs int64 `bson:"durationMs,omitempty" json:"durationMs,omitempty"`
-
- // Optional context (if available)
- ModelID string `bson:"modelId,omitempty" json:"modelId,omitempty"`
- DisabledTools bool `bson:"disabledTools,omitempty" json:"disabledTools,omitempty"`
-}
-
-// AgentUsageAnalytics stores minimal agent execution context
-type AgentUsageAnalytics struct {
- ID primitive.ObjectID `bson:"_id,omitempty" json:"id"`
- UserID string `bson:"userId" json:"userId"`
- AgentID string `bson:"agentId" json:"agentId"`
- ExecutionID primitive.ObjectID `bson:"executionId" json:"executionId"`
-
- TriggerType string `bson:"triggerType" json:"triggerType"` // chat, api, scheduled
- ExecutedAt time.Time `bson:"executedAt" json:"executedAt"`
-}
-
-// TrackChatSessionStart records the start of a chat session
-func (s *AnalyticsService) TrackChatSessionStart(ctx context.Context, sessionID, userID, conversationID string) error {
- if s.mongoDB == nil {
- return nil // Analytics disabled
- }
-
- session := &ChatSessionAnalytics{
- UserID: userID,
- ConversationID: conversationID,
- SessionID: sessionID,
- MessageCount: 0,
- StartedAt: time.Now(),
- }
-
- _, err := s.collection("chat_sessions").InsertOne(ctx, session)
- if err != nil {
- log.Printf("⚠️ [ANALYTICS] Failed to track session start: %v", err)
- return err
- }
-
- return nil
-}
-
-// TrackChatSessionEnd records the end of a chat session
-func (s *AnalyticsService) TrackChatSessionEnd(ctx context.Context, sessionID string, messageCount int) error {
- if s.mongoDB == nil {
- return nil
- }
-
- now := time.Now()
-
- // Find the session
- var session ChatSessionAnalytics
- err := s.collection("chat_sessions").FindOne(ctx, bson.M{"sessionId": sessionID}).Decode(&session)
- if err != nil {
- if err == mongo.ErrNoDocuments {
- // Session wasn't tracked (maybe analytics added after session started)
- return nil
- }
- return err
- }
-
- durationMs := now.Sub(session.StartedAt).Milliseconds()
-
- // Update the session
- _, err = s.collection("chat_sessions").UpdateOne(
- ctx,
- bson.M{"sessionId": sessionID},
- bson.M{
- "$set": bson.M{
- "endedAt": now,
- "messageCount": messageCount,
- "durationMs": durationMs,
- },
- },
- )
-
- if err != nil {
- log.Printf("⚠️ [ANALYTICS] Failed to track session end: %v", err)
- }
-
- return err
-}
-
-// UpdateChatSessionModel updates the model used in a session
-func (s *AnalyticsService) UpdateChatSessionModel(ctx context.Context, sessionID, modelID string, disabledTools bool) error {
- if s.mongoDB == nil {
- return nil
- }
-
- _, err := s.collection("chat_sessions").UpdateOne(
- ctx,
- bson.M{"sessionId": sessionID},
- bson.M{
- "$set": bson.M{
- "modelId": modelID,
- "disabledTools": disabledTools,
- },
- },
- )
-
- return err
-}
-
-// TrackAgentUsage records when an agent is used
-func (s *AnalyticsService) TrackAgentUsage(ctx context.Context, userID, agentID string, executionID primitive.ObjectID, triggerType string) error {
- if s.mongoDB == nil {
- return nil
- }
-
- usage := &AgentUsageAnalytics{
- UserID: userID,
- AgentID: agentID,
- ExecutionID: executionID,
- TriggerType: triggerType,
- ExecutedAt: time.Now(),
- }
-
- _, err := s.collection("agent_usage").InsertOne(ctx, usage)
- if err != nil {
- log.Printf("⚠️ [ANALYTICS] Failed to track agent usage: %v", err)
- }
-
- return err
-}
-
-// collection returns a MongoDB collection
-func (s *AnalyticsService) collection(name string) *mongo.Collection {
- return s.mongoDB.Database().Collection(name)
-}
-
-// loadProvidersConfig reads and parses the providers.json file
-func (s *AnalyticsService) loadProvidersConfig() (*models.ProvidersConfig, error) {
- // Get the path to providers.json (relative to backend root)
- providersPath := filepath.Join("providers.json")
-
- // Read the file
- data, err := os.ReadFile(providersPath)
- if err != nil {
- log.Printf("⚠️ [ANALYTICS] Failed to read providers.json: %v", err)
- return nil, err
- }
-
- // Parse JSON
- var config models.ProvidersConfig
- if err := json.Unmarshal(data, &config); err != nil {
- log.Printf("⚠️ [ANALYTICS] Failed to parse providers.json: %v", err)
- return nil, err
- }
-
- return &config, nil
-}
-
-// GetOverviewStats returns system overview statistics
-func (s *AnalyticsService) GetOverviewStats(ctx context.Context) (map[string]interface{}, error) {
- if s.mongoDB == nil {
- return map[string]interface{}{
- "total_users": 0,
- "active_chats": 0,
- "total_messages": 0,
- "api_calls_today": 0,
- "active_providers": 0,
- "total_models": 0,
- "total_agents": 0,
- "agent_executions": 0,
- "agents_run_today": 0,
- }, nil
- }
-
- // Count total chat sessions (approximation of active chats)
- activeChats, _ := s.collection("chat_sessions").CountDocuments(ctx, bson.M{"endedAt": bson.M{"$exists": false}})
-
- // Count total messages from all sessions
- pipeline := mongo.Pipeline{
- {{Key: "$group", Value: bson.M{"_id": nil, "totalMessages": bson.M{"$sum": "$messageCount"}}}},
- }
- cursor, err := s.collection("chat_sessions").Aggregate(ctx, pipeline)
- var totalMessages int64
- if err == nil {
- var results []bson.M
- if err := cursor.All(ctx, &results); err == nil && len(results) > 0 {
- if count, ok := results[0]["totalMessages"].(int32); ok {
- totalMessages = int64(count)
- } else if count, ok := results[0]["totalMessages"].(int64); ok {
- totalMessages = count
- }
- }
- }
-
- // Count unique users
- uniqueUsers, _ := s.collection("chat_sessions").Distinct(ctx, "userId", bson.M{})
-
- // Count API calls today
- today := time.Now().Truncate(24 * time.Hour)
- apiCallsToday, _ := s.collection("chat_sessions").CountDocuments(ctx, bson.M{"startedAt": bson.M{"$gte": today}})
-
- // Count total models from providers.json
- totalModels := 0
- providersConfig, err := s.loadProvidersConfig()
- if err == nil {
- for _, provider := range providersConfig.Providers {
- // Only count enabled providers
- if provider.Enabled {
- // Count models from ModelAliases map
- totalModels += len(provider.ModelAliases)
- }
- }
- }
-
- // Count active providers (providers that have been used in the last 30 days)
- activeProviders := 0
- thirtyDaysAgo := time.Now().Add(-30 * 24 * time.Hour)
-
- // Get distinct model IDs used in the last 30 days
- usedModels, err := s.collection("chat_sessions").Distinct(ctx, "modelId", bson.M{
- "modelId": bson.M{"$exists": true, "$ne": ""},
- "startedAt": bson.M{"$gte": thirtyDaysAgo},
- })
-
- if err == nil && providersConfig != nil {
- // Create a set of used model IDs for faster lookup
- usedModelSet := make(map[string]bool)
- for _, modelID := range usedModels {
- if modelStr, ok := modelID.(string); ok {
- usedModelSet[modelStr] = true
- }
- }
-
- // Check each provider to see if any of their models were used
- for _, provider := range providersConfig.Providers {
- if !provider.Enabled {
- continue
- }
-
- // Check if any model from this provider was used
- for modelAlias := range provider.ModelAliases {
- if usedModelSet[modelAlias] {
- activeProviders++
- break // Count provider only once
- }
- }
- }
- }
-
- // Count agent metrics
- totalAgentExecutions, _ := s.collection("agent_usage").CountDocuments(ctx, bson.M{})
- agentsRunToday, _ := s.collection("agent_usage").CountDocuments(ctx, bson.M{"executedAt": bson.M{"$gte": today}})
-
- // Count unique agents
- uniqueAgents, _ := s.collection("agent_usage").Distinct(ctx, "agentId", bson.M{})
- totalAgents := len(uniqueAgents)
-
- return map[string]interface{}{
- "total_users": len(uniqueUsers),
- "active_chats": activeChats,
- "total_messages": totalMessages,
- "api_calls_today": apiCallsToday,
- "active_providers": activeProviders,
- "total_models": totalModels,
- "total_agents": totalAgents,
- "agent_executions": totalAgentExecutions,
- "agents_run_today": agentsRunToday,
- }, nil
-}
-
-// GetProviderAnalytics returns usage analytics per provider
-func (s *AnalyticsService) GetProviderAnalytics(ctx context.Context) ([]map[string]interface{}, error) {
- if s.mongoDB == nil {
- return []map[string]interface{}{}, nil
- }
-
- // Group by model ID and aggregate usage
- pipeline := mongo.Pipeline{
- {{Key: "$match", Value: bson.M{"modelId": bson.M{"$exists": true, "$ne": ""}}}},
- {{Key: "$group", Value: bson.M{
- "_id": "$modelId",
- "total_requests": bson.M{"$sum": 1},
- "last_used_at": bson.M{"$max": "$startedAt"},
- }}},
- {{Key: "$sort", Value: bson.M{"total_requests": -1}}},
- }
-
- cursor, err := s.collection("chat_sessions").Aggregate(ctx, pipeline)
- if err != nil {
- return []map[string]interface{}{}, err
- }
-
- var results []bson.M
- if err := cursor.All(ctx, &results); err != nil {
- return []map[string]interface{}{}, err
- }
-
- analytics := make([]map[string]interface{}, 0, len(results))
- for _, result := range results {
- analytics = append(analytics, map[string]interface{}{
- "provider_id": result["_id"],
- "provider_name": result["_id"], // TODO: Resolve from providers.json
- "total_requests": result["total_requests"],
- "total_tokens": 0, // TODO: Track tokens
- "estimated_cost": nil,
- "active_models": []string{},
- "last_used_at": result["last_used_at"],
- })
- }
-
- return analytics, nil
-}
-
-// GetChatAnalytics returns chat usage statistics
-func (s *AnalyticsService) GetChatAnalytics(ctx context.Context) (map[string]interface{}, error) {
- if s.mongoDB == nil {
- return map[string]interface{}{
- "total_chats": 0,
- "active_chats": 0,
- "total_messages": 0,
- "avg_messages_per_chat": 0.0,
- "chats_created_today": 0,
- "messages_sent_today": 0,
- "time_series": []map[string]interface{}{},
- }, nil
- }
-
- totalChats, _ := s.collection("chat_sessions").CountDocuments(ctx, bson.M{})
- activeChats, _ := s.collection("chat_sessions").CountDocuments(ctx, bson.M{"endedAt": bson.M{"$exists": false}})
-
- // Total messages
- pipeline := mongo.Pipeline{
- {{Key: "$group", Value: bson.M{"_id": nil, "totalMessages": bson.M{"$sum": "$messageCount"}}}},
- }
- cursor, _ := s.collection("chat_sessions").Aggregate(ctx, pipeline)
- var totalMessages int64
- var results []bson.M
- if cursor.All(ctx, &results) == nil && len(results) > 0 {
- if count, ok := results[0]["totalMessages"].(int32); ok {
- totalMessages = int64(count)
- } else if count, ok := results[0]["totalMessages"].(int64); ok {
- totalMessages = count
- }
- }
-
- avgMessages := 0.0
- if totalChats > 0 {
- avgMessages = float64(totalMessages) / float64(totalChats)
- }
-
- // Today's stats
- today := time.Now().Truncate(24 * time.Hour)
- chatsToday, _ := s.collection("chat_sessions").CountDocuments(ctx, bson.M{"startedAt": bson.M{"$gte": today}})
-
- // Get time series data for the last 30 days
- timeSeries, _ := s.getTimeSeriesData(ctx, 30)
-
- return map[string]interface{}{
- "total_chats": totalChats,
- "active_chats": activeChats,
- "total_messages": totalMessages,
- "avg_messages_per_chat": avgMessages,
- "chats_created_today": chatsToday,
- "messages_sent_today": 0, // TODO: Track per-day messages
- "time_series": timeSeries,
- }, nil
-}
-
-// getTimeSeriesData returns daily statistics for the specified number of days
-func (s *AnalyticsService) getTimeSeriesData(ctx context.Context, days int) ([]map[string]interface{}, error) {
- if s.mongoDB == nil {
- return []map[string]interface{}{}, nil
- }
-
- // Calculate start date
- startDate := time.Now().Add(-time.Duration(days) * 24 * time.Hour).Truncate(24 * time.Hour)
-
- // Aggregate chats and messages by day
- pipeline := mongo.Pipeline{
- {{Key: "$match", Value: bson.M{"startedAt": bson.M{"$gte": startDate}}}},
- {{Key: "$group", Value: bson.M{
- "_id": bson.M{
- "$dateToString": bson.M{
- "format": "%Y-%m-%d",
- "date": "$startedAt",
- },
- },
- "chat_count": bson.M{"$sum": 1},
- "message_count": bson.M{"$sum": "$messageCount"},
- "unique_users": bson.M{"$addToSet": "$userId"},
- }}},
- {{Key: "$sort", Value: bson.M{"_id": 1}}},
- }
-
- cursor, err := s.collection("chat_sessions").Aggregate(ctx, pipeline)
- if err != nil {
- log.Printf("⚠️ [ANALYTICS] Failed to aggregate time series: %v", err)
- return []map[string]interface{}{}, err
- }
-
- var results []bson.M
- if err := cursor.All(ctx, &results); err != nil {
- return []map[string]interface{}{}, err
- }
-
- // Aggregate agent executions by day
- agentPipeline := mongo.Pipeline{
- {{Key: "$match", Value: bson.M{"executedAt": bson.M{"$gte": startDate}}}},
- {{Key: "$group", Value: bson.M{
- "_id": bson.M{
- "$dateToString": bson.M{
- "format": "%Y-%m-%d",
- "date": "$executedAt",
- },
- },
- "agent_count": bson.M{"$sum": 1},
- }}},
- {{Key: "$sort", Value: bson.M{"_id": 1}}},
- }
-
- agentCursor, err := s.collection("agent_usage").Aggregate(ctx, agentPipeline)
- var agentResults []bson.M
- if err == nil {
- agentCursor.All(ctx, &agentResults)
- }
-
- // Create a map of agent counts by date for easy lookup
- agentCountByDate := make(map[string]int64)
- for _, result := range agentResults {
- date, _ := result["_id"].(string)
- count := int64(0)
- if c, ok := result["agent_count"].(int32); ok {
- count = int64(c)
- } else if c, ok := result["agent_count"].(int64); ok {
- count = c
- }
- agentCountByDate[date] = count
- }
-
- // Convert to response format with agent data
- timeSeries := make([]map[string]interface{}, 0, len(results))
- for _, result := range results {
- date, _ := result["_id"].(string)
- chatCount := int64(0)
- messageCount := int64(0)
- uniqueUsers := 0
-
- if count, ok := result["chat_count"].(int32); ok {
- chatCount = int64(count)
- } else if count, ok := result["chat_count"].(int64); ok {
- chatCount = count
- }
-
- if count, ok := result["message_count"].(int32); ok {
- messageCount = int64(count)
- } else if count, ok := result["message_count"].(int64); ok {
- messageCount = count
- }
-
- if users, ok := result["unique_users"].(primitive.A); ok {
- uniqueUsers = len(users)
- }
-
- // Get agent count for this date
- agentCount := agentCountByDate[date]
-
- timeSeries = append(timeSeries, map[string]interface{}{
- "date": date,
- "chat_count": chatCount,
- "message_count": messageCount,
- "user_count": uniqueUsers,
- "agent_count": agentCount,
- })
- }
-
- // Fill in missing dates with zeros
- filledSeries := s.fillMissingDates(timeSeries, startDate, days)
-
- return filledSeries, nil
-}
-
-// fillMissingDates ensures all dates in the range have entries (with zeros if no data)
-func (s *AnalyticsService) fillMissingDates(data []map[string]interface{}, startDate time.Time, days int) []map[string]interface{} {
- // Create a map of existing dates
- dataMap := make(map[string]map[string]interface{})
- for _, entry := range data {
- if date, ok := entry["date"].(string); ok {
- dataMap[date] = entry
- }
- }
-
- // Fill all dates
- result := make([]map[string]interface{}, 0, days)
- for i := 0; i < days; i++ {
- date := startDate.Add(time.Duration(i) * 24 * time.Hour)
- dateStr := date.Format("2006-01-02")
-
- if entry, exists := dataMap[dateStr]; exists {
- result = append(result, entry)
- } else {
- result = append(result, map[string]interface{}{
- "date": dateStr,
- "chat_count": 0,
- "message_count": 0,
- "user_count": 0,
- "agent_count": 0,
- })
- }
- }
-
- return result
-}
-
-// EnsureIndexes creates indexes for analytics collections
-func (s *AnalyticsService) EnsureIndexes(ctx context.Context) error {
- if s.mongoDB == nil {
- return nil
- }
-
- // Chat sessions indexes
- _, err := s.collection("chat_sessions").Indexes().CreateMany(ctx, []mongo.IndexModel{
- {Keys: bson.D{{Key: "userId", Value: 1}, {Key: "startedAt", Value: -1}}},
- {Keys: bson.D{{Key: "sessionId", Value: 1}}},
- {Keys: bson.D{{Key: "startedAt", Value: -1}}},
- })
- if err != nil {
- return err
- }
-
- // Agent usage indexes
- _, err = s.collection("agent_usage").Indexes().CreateMany(ctx, []mongo.IndexModel{
- {Keys: bson.D{{Key: "userId", Value: 1}, {Key: "executedAt", Value: -1}}},
- {Keys: bson.D{{Key: "agentId", Value: 1}, {Key: "executedAt", Value: -1}}},
- {Keys: bson.D{{Key: "executedAt", Value: -1}}},
- })
-
- log.Println("✅ [ANALYTICS] Indexes created")
- return err
-}
-
-// MigrateChatSessionTimestamps fixes existing chat sessions that don't have proper startedAt timestamps
-// Uses the MongoDB ObjectID creation time as the startedAt value
-func (s *AnalyticsService) MigrateChatSessionTimestamps(ctx context.Context) (int, error) {
- if s.mongoDB == nil {
- return 0, nil
- }
-
- log.Println("🔄 [ANALYTICS MIGRATION] Starting chat session timestamp migration...")
-
- // Find all sessions without startedAt or with zero time
- zeroTime := time.Time{}
- cursor, err := s.collection("chat_sessions").Find(ctx, bson.M{
- "$or": []bson.M{
- {"startedAt": bson.M{"$exists": false}},
- {"startedAt": zeroTime},
- {"startedAt": nil},
- },
- })
- if err != nil {
- log.Printf("❌ [ANALYTICS MIGRATION] Failed to query sessions: %v", err)
- return 0, err
- }
- defer cursor.Close(ctx)
-
- updatedCount := 0
- for cursor.Next(ctx) {
- var session ChatSessionAnalytics
- if err := cursor.Decode(&session); err != nil {
- log.Printf("⚠️ [ANALYTICS MIGRATION] Failed to decode session: %v", err)
- continue
- }
-
- // Extract timestamp from MongoDB ObjectID
- // ObjectID first 4 bytes are Unix timestamp
- createdAt := session.ID.Timestamp()
-
- // Update the session with the extracted timestamp
- _, err := s.collection("chat_sessions").UpdateOne(
- ctx,
- bson.M{"_id": session.ID},
- bson.M{"$set": bson.M{"startedAt": createdAt}},
- )
- if err != nil {
- log.Printf("⚠️ [ANALYTICS MIGRATION] Failed to update session %s: %v", session.ID.Hex(), err)
- continue
- }
-
- updatedCount++
- }
-
- if err := cursor.Err(); err != nil {
- log.Printf("❌ [ANALYTICS MIGRATION] Cursor error: %v", err)
- return updatedCount, err
- }
-
- log.Printf("✅ [ANALYTICS MIGRATION] Successfully migrated %d chat sessions with proper timestamps", updatedCount)
- return updatedCount, nil
-}
-
-
-// GetAgentAnalytics returns comprehensive agent activity analytics
-func (s *AnalyticsService) GetAgentAnalytics(ctx context.Context) (map[string]interface{}, error) {
- if s.mongoDB == nil {
- return map[string]interface{}{
- "total_agents": 0,
- "deployed_agents": 0,
- "total_executions": 0,
- "active_schedules": 0,
- "executions_today": 0,
- "time_series": []map[string]interface{}{},
- }, nil
- }
-
- // Count total agents
- totalAgents, _ := s.collection("agents").CountDocuments(ctx, bson.M{})
-
- // Count deployed agents (status = "deployed")
- deployedAgents, _ := s.collection("agents").CountDocuments(ctx, bson.M{"status": "deployed"})
-
- // Count total agent executions
- totalExecutions, _ := s.collection("agent_usage").CountDocuments(ctx, bson.M{})
-
- // Count active schedules
- activeSchedules, _ := s.collection("schedules").CountDocuments(ctx, bson.M{"enabled": true})
-
- // Count executions today
- today := time.Now().Truncate(24 * time.Hour)
- executionsToday, _ := s.collection("agent_usage").CountDocuments(ctx, bson.M{"executedAt": bson.M{"$gte": today}})
-
- // Get time series data for the last 30 days
- timeSeries, _ := s.getAgentTimeSeriesData(ctx, 30)
-
- return map[string]interface{}{
- "total_agents": totalAgents,
- "deployed_agents": deployedAgents,
- "total_executions": totalExecutions,
- "active_schedules": activeSchedules,
- "executions_today": executionsToday,
- "time_series": timeSeries,
- }, nil
-}
-
-// getAgentTimeSeriesData returns daily agent activity statistics
-func (s *AnalyticsService) getAgentTimeSeriesData(ctx context.Context, days int) ([]map[string]interface{}, error) {
- if s.mongoDB == nil {
- return []map[string]interface{}{}, nil
- }
-
- startDate := time.Now().Add(-time.Duration(days) * 24 * time.Hour).Truncate(24 * time.Hour)
-
- // Aggregate agents created by day
- agentsCreatedPipeline := mongo.Pipeline{
- {{Key: "$match", Value: bson.M{"createdAt": bson.M{"$gte": startDate}}}},
- {{Key: "$group", Value: bson.M{
- "_id": bson.M{
- "$dateToString": bson.M{
- "format": "%Y-%m-%d",
- "date": "$createdAt",
- },
- },
- "agents_created": bson.M{"$sum": 1},
- }}},
- {{Key: "$sort", Value: bson.M{"_id": 1}}},
- }
-
- agentsCreatedCursor, err := s.collection("agents").Aggregate(ctx, agentsCreatedPipeline)
- var agentsCreatedResults []bson.M
- if err == nil {
- agentsCreatedCursor.All(ctx, &agentsCreatedResults)
- }
-
- // Aggregate agents deployed by day (when updatedAt changed to deployed status)
- agentsDeployedPipeline := mongo.Pipeline{
- {{Key: "$match", Value: bson.M{
- "status": "deployed",
- "updatedAt": bson.M{"$gte": startDate},
- }}},
- {{Key: "$group", Value: bson.M{
- "_id": bson.M{
- "$dateToString": bson.M{
- "format": "%Y-%m-%d",
- "date": "$updatedAt",
- },
- },
- "agents_deployed": bson.M{"$sum": 1},
- }}},
- {{Key: "$sort", Value: bson.M{"_id": 1}}},
- }
-
- agentsDeployedCursor, err := s.collection("agents").Aggregate(ctx, agentsDeployedPipeline)
- var agentsDeployedResults []bson.M
- if err == nil {
- agentsDeployedCursor.All(ctx, &agentsDeployedResults)
- }
-
- // Aggregate agent executions by day
- agentRunsPipeline := mongo.Pipeline{
- {{Key: "$match", Value: bson.M{"executedAt": bson.M{"$gte": startDate}}}},
- {{Key: "$group", Value: bson.M{
- "_id": bson.M{
- "$dateToString": bson.M{
- "format": "%Y-%m-%d",
- "date": "$executedAt",
- },
- },
- "agent_runs": bson.M{"$sum": 1},
- }}},
- {{Key: "$sort", Value: bson.M{"_id": 1}}},
- }
-
- agentRunsCursor, err := s.collection("agent_usage").Aggregate(ctx, agentRunsPipeline)
- var agentRunsResults []bson.M
- if err == nil {
- agentRunsCursor.All(ctx, &agentRunsResults)
- }
-
- // Aggregate schedules created by day
- schedulesCreatedPipeline := mongo.Pipeline{
- {{Key: "$match", Value: bson.M{"createdAt": bson.M{"$gte": startDate}}}},
- {{Key: "$group", Value: bson.M{
- "_id": bson.M{
- "$dateToString": bson.M{
- "format": "%Y-%m-%d",
- "date": "$createdAt",
- },
- },
- "schedules_created": bson.M{"$sum": 1},
- }}},
- {{Key: "$sort", Value: bson.M{"_id": 1}}},
- }
-
- schedulesCreatedCursor, err := s.collection("schedules").Aggregate(ctx, schedulesCreatedPipeline)
- var schedulesCreatedResults []bson.M
- if err == nil {
- schedulesCreatedCursor.All(ctx, &schedulesCreatedResults)
- }
-
- // Create maps for easy lookup
- agentsCreatedByDate := make(map[string]int64)
- agentsDeployedByDate := make(map[string]int64)
- agentRunsByDate := make(map[string]int64)
- schedulesCreatedByDate := make(map[string]int64)
-
- for _, result := range agentsCreatedResults {
- date, _ := result["_id"].(string)
- count := extractInt64(result, "agents_created")
- agentsCreatedByDate[date] = count
- }
-
- for _, result := range agentsDeployedResults {
- date, _ := result["_id"].(string)
- count := extractInt64(result, "agents_deployed")
- agentsDeployedByDate[date] = count
- }
-
- for _, result := range agentRunsResults {
- date, _ := result["_id"].(string)
- count := extractInt64(result, "agent_runs")
- agentRunsByDate[date] = count
- }
-
- for _, result := range schedulesCreatedResults {
- date, _ := result["_id"].(string)
- count := extractInt64(result, "schedules_created")
- schedulesCreatedByDate[date] = count
- }
-
- // Fill all dates with data
- timeSeries := make([]map[string]interface{}, 0, days)
- for i := 0; i < days; i++ {
- date := startDate.Add(time.Duration(i) * 24 * time.Hour)
- dateStr := date.Format("2006-01-02")
-
- timeSeries = append(timeSeries, map[string]interface{}{
- "date": dateStr,
- "agents_created": agentsCreatedByDate[dateStr],
- "agents_deployed": agentsDeployedByDate[dateStr],
- "agent_runs": agentRunsByDate[dateStr],
- "schedules_created": schedulesCreatedByDate[dateStr],
- })
- }
-
- return timeSeries, nil
-}
-
-// extractInt64 safely extracts an int64 value from a bson.M result
-func extractInt64(result bson.M, key string) int64 {
- if count, ok := result[key].(int32); ok {
- return int64(count)
- } else if count, ok := result[key].(int64); ok {
- return count
- }
- return 0
-}
-
-// UserListItemGDPR represents a GDPR-compliant user list item
-type UserListItemGDPR struct {
- UserID string `json:"user_id"`
- EmailDomain string `json:"email_domain,omitempty"`
- Tier string `json:"tier"`
- CreatedAt time.Time `json:"created_at"`
- LastActive *time.Time `json:"last_active,omitempty"`
- TotalChats int64 `json:"total_chats"`
- TotalMessages int64 `json:"total_messages"`
- TotalAgentRuns int64 `json:"total_agent_runs"`
- HasOverrides bool `json:"has_overrides"`
-}
-
-// GetUserListGDPR returns a GDPR-compliant paginated user list
-// Only includes aggregated analytics, no PII except anonymized user IDs and email domains
-func (s *AnalyticsService) GetUserListGDPR(ctx context.Context, page, pageSize int, tierFilter, searchFilter string) ([]UserListItemGDPR, int64, error) {
- if s.mongoDB == nil {
- return []UserListItemGDPR{}, 0, nil
- }
-
- // Get unique users from chat sessions with their activity
- pipeline := mongo.Pipeline{
- {{Key: "$group", Value: bson.M{
- "_id": "$userId",
- "total_chats": bson.M{"$sum": 1},
- "total_messages": bson.M{"$sum": "$messageCount"},
- "last_active": bson.M{"$max": "$startedAt"},
- "first_seen": bson.M{"$min": "$startedAt"},
- }}},
- }
-
- cursor, err := s.collection("chat_sessions").Aggregate(ctx, pipeline)
- if err != nil {
- return nil, 0, err
- }
-
- var sessionStats []bson.M
- if err := cursor.All(ctx, &sessionStats); err != nil {
- return nil, 0, err
- }
-
- // Get agent usage counts per user
- agentPipeline := mongo.Pipeline{
- {{Key: "$group", Value: bson.M{
- "_id": "$userId",
- "total_agent_runs": bson.M{"$sum": 1},
- }}},
- }
-
- agentCursor, err := s.collection("agent_usage").Aggregate(ctx, agentPipeline)
- var agentStats []bson.M
- agentCountByUser := make(map[string]int64)
- if err == nil {
- agentCursor.All(ctx, &agentStats)
- for _, stat := range agentStats {
- userID, _ := stat["_id"].(string)
- count := extractInt64(stat, "total_agent_runs")
- agentCountByUser[userID] = count
- }
- }
-
- // Build user list
- users := make([]UserListItemGDPR, 0, len(sessionStats))
- for _, stat := range sessionStats {
- userID, ok := stat["_id"].(string)
- if !ok || userID == "" {
- continue
- }
-
- // Extract email domain from user ID if it's an email format
- emailDomain := extractEmailDomain(userID)
-
- totalChats := extractInt64(stat, "total_chats")
- totalMessages := extractInt64(stat, "total_messages")
- totalAgentRuns := agentCountByUser[userID]
-
- var lastActive *time.Time
- if lastActiveVal, ok := stat["last_active"].(primitive.DateTime); ok {
- t := lastActiveVal.Time()
- lastActive = &t
- }
-
- var createdAt time.Time
- if firstSeenVal, ok := stat["first_seen"].(primitive.DateTime); ok {
- createdAt = firstSeenVal.Time()
- }
-
- user := UserListItemGDPR{
- UserID: anonymizeUserID(userID), // Anonymize user ID
- EmailDomain: emailDomain,
- Tier: "free", // Default, would come from user service in production
- CreatedAt: createdAt,
- LastActive: lastActive,
- TotalChats: totalChats,
- TotalMessages: totalMessages,
- TotalAgentRuns: totalAgentRuns,
- HasOverrides: false, // Would check user service for overrides
- }
-
- users = append(users, user)
- }
-
- // Sort by last active (most recent first)
- // Note: For production, this should be done in the database query
- // Simple in-memory sort for now
- totalCount := int64(len(users))
-
- // Pagination
- start := (page - 1) * pageSize
- end := start + pageSize
- if start >= len(users) {
- return []UserListItemGDPR{}, totalCount, nil
- }
- if end > len(users) {
- end = len(users)
- }
-
- return users[start:end], totalCount, nil
-}
-
-// extractEmailDomain extracts the domain from an email address
-func extractEmailDomain(email string) string {
- parts := splitString(email, "@")
- if len(parts) == 2 {
- return "@" + parts[1]
- }
- return ""
-}
-
-// splitString is a simple string split helper
-func splitString(s, sep string) []string {
- result := []string{}
- current := ""
- sepLen := len(sep)
-
- for i := 0; i < len(s); i++ {
- if i+sepLen <= len(s) && s[i:i+sepLen] == sep {
- result = append(result, current)
- current = ""
- i += sepLen - 1
- } else {
- current += string(s[i])
- }
- }
- result = append(result, current)
- return result
-}
-
-// anonymizeUserID creates a privacy-safe representation of a user ID
-func anonymizeUserID(userID string) string {
- // For emails, show first 3 chars + *** + domain
- parts := splitString(userID, "@")
- if len(parts) == 2 {
- prefix := "***"
- if len(parts[0]) > 3 {
- prefix = parts[0][:3] + "***"
- }
- return prefix + "@" + parts[1]
- }
- // For non-email IDs, just show first 8 chars
- if len(userID) > 8 {
- return userID[:8] + "..."
- }
- return userID
-}
diff --git a/backend/internal/services/apikey_service.go b/backend/internal/services/apikey_service.go
deleted file mode 100644
index 77e349dc..00000000
--- a/backend/internal/services/apikey_service.go
+++ /dev/null
@@ -1,349 +0,0 @@
-package services
-
-import (
- "claraverse/internal/database"
- "claraverse/internal/models"
- "context"
- "crypto/rand"
- "encoding/hex"
- "fmt"
- "log"
- "time"
-
- "go.mongodb.org/mongo-driver/bson"
- "go.mongodb.org/mongo-driver/bson/primitive"
- "go.mongodb.org/mongo-driver/mongo"
- "go.mongodb.org/mongo-driver/mongo/options"
- "golang.org/x/crypto/bcrypt"
-)
-
-const (
- // APIKeyPrefix is the prefix for all API keys
- APIKeyPrefix = "clv_"
- // APIKeyLength is the length of the random part of the key (32 bytes = 64 hex chars)
- APIKeyLength = 32
- // APIKeyPrefixLength is how many chars to show as prefix (including "clv_")
- APIKeyPrefixLength = 12
-)
-
-// APIKeyService manages API keys
-type APIKeyService struct {
- mongoDB *database.MongoDB
- tierService *TierService
-}
-
-// NewAPIKeyService creates a new API key service
-func NewAPIKeyService(mongoDB *database.MongoDB, tierService *TierService) *APIKeyService {
- return &APIKeyService{
- mongoDB: mongoDB,
- tierService: tierService,
- }
-}
-
-// collection returns the api_keys collection
-func (s *APIKeyService) collection() *mongo.Collection {
- return s.mongoDB.Database().Collection("api_keys")
-}
-
-// GenerateKey generates a new API key
-func (s *APIKeyService) GenerateKey() (string, error) {
- bytes := make([]byte, APIKeyLength)
- if _, err := rand.Read(bytes); err != nil {
- return "", fmt.Errorf("failed to generate random bytes: %w", err)
- }
- return APIKeyPrefix + hex.EncodeToString(bytes), nil
-}
-
-// HashKey hashes an API key for storage
-func (s *APIKeyService) HashKey(key string) (string, error) {
- hash, err := bcrypt.GenerateFromPassword([]byte(key), bcrypt.DefaultCost)
- if err != nil {
- return "", fmt.Errorf("failed to hash key: %w", err)
- }
- return string(hash), nil
-}
-
-// VerifyKey verifies an API key against a hash
-func (s *APIKeyService) VerifyKey(key, hash string) bool {
- err := bcrypt.CompareHashAndPassword([]byte(hash), []byte(key))
- return err == nil
-}
-
-// Create creates a new API key
-func (s *APIKeyService) Create(ctx context.Context, userID string, req *models.CreateAPIKeyRequest) (*models.CreateAPIKeyResponse, error) {
- // Check API key limit
- if s.tierService != nil {
- count, err := s.CountByUser(ctx, userID)
- if err != nil {
- return nil, err
- }
- if !s.tierService.CheckAPIKeyLimit(ctx, userID, count) {
- limits := s.tierService.GetLimits(ctx, userID)
- return nil, fmt.Errorf("API key limit reached (%d/%d)", count, limits.MaxAPIKeys)
- }
- }
-
- // Validate scopes
- for _, scope := range req.Scopes {
- if !models.IsValidScope(scope) {
- return nil, fmt.Errorf("invalid scope: %s", scope)
- }
- }
-
- // Generate key
- key, err := s.GenerateKey()
- if err != nil {
- return nil, err
- }
-
- // Hash key for storage
- hash, err := s.HashKey(key)
- if err != nil {
- return nil, err
- }
-
- // Calculate expiration if specified
- var expiresAt *time.Time
- if req.ExpiresIn > 0 {
- exp := time.Now().Add(time.Duration(req.ExpiresIn) * 24 * time.Hour)
- expiresAt = &exp
- }
-
- now := time.Now()
- apiKey := &models.APIKey{
- UserID: userID,
- KeyPrefix: key[:APIKeyPrefixLength],
- KeyHash: hash,
- PlainKey: key, // TEMPORARY: Store plain key for early platform phase
- Name: req.Name,
- Description: req.Description,
- Scopes: req.Scopes,
- RateLimit: req.RateLimit,
- ExpiresAt: expiresAt,
- CreatedAt: now,
- UpdatedAt: now,
- }
-
- result, err := s.collection().InsertOne(ctx, apiKey)
- if err != nil {
- return nil, fmt.Errorf("failed to create API key: %w", err)
- }
-
- apiKey.ID = result.InsertedID.(primitive.ObjectID)
-
- log.Printf("🔑 [APIKEY] Created API key %s for user %s (prefix: %s)",
- apiKey.ID.Hex(), userID, apiKey.KeyPrefix)
-
- return &models.CreateAPIKeyResponse{
- ID: apiKey.ID.Hex(),
- Key: key, // Full key - only returned once!
- KeyPrefix: apiKey.KeyPrefix,
- Name: apiKey.Name,
- Scopes: apiKey.Scopes,
- ExpiresAt: expiresAt,
- CreatedAt: now,
- }, nil
-}
-
-// ValidateKey validates an API key and returns the key record
-func (s *APIKeyService) ValidateKey(ctx context.Context, key string) (*models.APIKey, error) {
- if len(key) < APIKeyPrefixLength {
- return nil, fmt.Errorf("invalid API key format")
- }
-
- // Extract prefix for lookup
- prefix := key[:APIKeyPrefixLength]
-
- // Find by prefix (there could be multiple with same prefix, but unlikely)
- cursor, err := s.collection().Find(ctx, bson.M{
- "keyPrefix": prefix,
- "revokedAt": bson.M{"$exists": false}, // Not revoked
- })
- if err != nil {
- return nil, fmt.Errorf("failed to lookup API key: %w", err)
- }
- defer cursor.Close(ctx)
-
- // Check each matching key (usually just one)
- for cursor.Next(ctx) {
- var apiKey models.APIKey
- if err := cursor.Decode(&apiKey); err != nil {
- continue
- }
-
- // Verify the hash
- if s.VerifyKey(key, apiKey.KeyHash) {
- // Check expiration
- if apiKey.IsExpired() {
- return nil, fmt.Errorf("API key has expired")
- }
-
- // Update last used
- go s.updateLastUsed(context.Background(), apiKey.ID)
-
- return &apiKey, nil
- }
- }
-
- return nil, fmt.Errorf("invalid API key")
-}
-
-// updateLastUsed updates the last used timestamp
-func (s *APIKeyService) updateLastUsed(ctx context.Context, keyID primitive.ObjectID) {
- _, err := s.collection().UpdateByID(ctx, keyID, bson.M{
- "$set": bson.M{
- "lastUsedAt": time.Now(),
- },
- })
- if err != nil {
- log.Printf("⚠️ [APIKEY] Failed to update last used: %v", err)
- }
-}
-
-// ListByUser returns all API keys for a user (without hashes)
-func (s *APIKeyService) ListByUser(ctx context.Context, userID string) ([]*models.APIKeyListItem, error) {
- cursor, err := s.collection().Find(ctx, bson.M{
- "userId": userID,
- }, options.Find().SetSort(bson.D{{Key: "createdAt", Value: -1}}))
- if err != nil {
- return nil, fmt.Errorf("failed to list API keys: %w", err)
- }
- defer cursor.Close(ctx)
-
- var keys []*models.APIKeyListItem
- for cursor.Next(ctx) {
- var key models.APIKey
- if err := cursor.Decode(&key); err != nil {
- continue
- }
- keys = append(keys, key.ToListItem())
- }
-
- return keys, nil
-}
-
-// GetByID retrieves an API key by ID
-func (s *APIKeyService) GetByID(ctx context.Context, keyID primitive.ObjectID) (*models.APIKey, error) {
- var key models.APIKey
- err := s.collection().FindOne(ctx, bson.M{"_id": keyID}).Decode(&key)
- if err != nil {
- if err == mongo.ErrNoDocuments {
- return nil, fmt.Errorf("API key not found")
- }
- return nil, fmt.Errorf("failed to get API key: %w", err)
- }
- return &key, nil
-}
-
-// GetByIDAndUser retrieves an API key by ID ensuring user ownership
-func (s *APIKeyService) GetByIDAndUser(ctx context.Context, keyID primitive.ObjectID, userID string) (*models.APIKey, error) {
- var key models.APIKey
- err := s.collection().FindOne(ctx, bson.M{
- "_id": keyID,
- "userId": userID,
- }).Decode(&key)
- if err != nil {
- if err == mongo.ErrNoDocuments {
- return nil, fmt.Errorf("API key not found")
- }
- return nil, fmt.Errorf("failed to get API key: %w", err)
- }
- return &key, nil
-}
-
-// Revoke revokes an API key (soft delete)
-func (s *APIKeyService) Revoke(ctx context.Context, keyID primitive.ObjectID, userID string) error {
- result, err := s.collection().UpdateOne(ctx, bson.M{
- "_id": keyID,
- "userId": userID,
- }, bson.M{
- "$set": bson.M{
- "revokedAt": time.Now(),
- "updatedAt": time.Now(),
- },
- })
- if err != nil {
- return fmt.Errorf("failed to revoke API key: %w", err)
- }
-
- if result.MatchedCount == 0 {
- return fmt.Errorf("API key not found")
- }
-
- log.Printf("🔒 [APIKEY] Revoked API key %s for user %s", keyID.Hex(), userID)
- return nil
-}
-
-// Delete permanently deletes an API key
-func (s *APIKeyService) Delete(ctx context.Context, keyID primitive.ObjectID, userID string) error {
- result, err := s.collection().DeleteOne(ctx, bson.M{
- "_id": keyID,
- "userId": userID,
- })
- if err != nil {
- return fmt.Errorf("failed to delete API key: %w", err)
- }
-
- if result.DeletedCount == 0 {
- return fmt.Errorf("API key not found")
- }
-
- log.Printf("🗑️ [APIKEY] Deleted API key %s for user %s", keyID.Hex(), userID)
- return nil
-}
-
-// DeleteAllByUser deletes all API keys for a user (GDPR compliance)
-func (s *APIKeyService) DeleteAllByUser(ctx context.Context, userID string) (int64, error) {
- if userID == "" {
- return 0, fmt.Errorf("user ID is required")
- }
-
- result, err := s.collection().DeleteMany(ctx, bson.M{"userId": userID})
- if err != nil {
- return 0, fmt.Errorf("failed to delete user API keys: %w", err)
- }
-
- log.Printf("🗑️ [GDPR] Deleted %d API keys for user %s", result.DeletedCount, userID)
- return result.DeletedCount, nil
-}
-
-// CountByUser counts API keys for a user (non-revoked)
-func (s *APIKeyService) CountByUser(ctx context.Context, userID string) (int64, error) {
- count, err := s.collection().CountDocuments(ctx, bson.M{
- "userId": userID,
- "revokedAt": bson.M{"$exists": false},
- })
- if err != nil {
- return 0, fmt.Errorf("failed to count API keys: %w", err)
- }
- return count, nil
-}
-
-// EnsureIndexes creates the necessary indexes for the api_keys collection
-func (s *APIKeyService) EnsureIndexes(ctx context.Context) error {
- indexes := []mongo.IndexModel{
- // User ID for listing
- {
- Keys: bson.D{{Key: "userId", Value: 1}},
- },
- // Key prefix for lookup during validation
- {
- Keys: bson.D{{Key: "keyPrefix", Value: 1}},
- },
- // Compound index for revoked check
- {
- Keys: bson.D{
- {Key: "keyPrefix", Value: 1},
- {Key: "revokedAt", Value: 1},
- },
- },
- }
-
- _, err := s.collection().Indexes().CreateMany(ctx, indexes)
- if err != nil {
- return fmt.Errorf("failed to create API key indexes: %w", err)
- }
-
- log.Println("✅ [APIKEY] Ensured indexes for api_keys collection")
- return nil
-}
diff --git a/backend/internal/services/apikey_service_test.go b/backend/internal/services/apikey_service_test.go
deleted file mode 100644
index b443c0c4..00000000
--- a/backend/internal/services/apikey_service_test.go
+++ /dev/null
@@ -1,258 +0,0 @@
-package services
-
-import (
- "claraverse/internal/models"
- "context"
- "strings"
- "testing"
-)
-
-func TestNewAPIKeyService(t *testing.T) {
- // Test creation without MongoDB (nil)
- service := NewAPIKeyService(nil, nil)
- if service == nil {
- t.Fatal("Expected non-nil API key service")
- }
-}
-
-func TestAPIKeyService_GenerateKey(t *testing.T) {
- service := NewAPIKeyService(nil, nil)
-
- key, err := service.GenerateKey()
- if err != nil {
- t.Fatalf("Failed to generate key: %v", err)
- }
-
- // Check prefix
- if !strings.HasPrefix(key, APIKeyPrefix) {
- t.Errorf("Expected key to start with '%s', got '%s'", APIKeyPrefix, key[:len(APIKeyPrefix)])
- }
-
- // Check length (prefix + 64 hex chars)
- expectedLen := len(APIKeyPrefix) + APIKeyLength*2
- if len(key) != expectedLen {
- t.Errorf("Expected key length %d, got %d", expectedLen, len(key))
- }
-
- // Generate another key - should be different
- key2, err := service.GenerateKey()
- if err != nil {
- t.Fatalf("Failed to generate second key: %v", err)
- }
-
- if key == key2 {
- t.Error("Generated keys should be unique")
- }
-}
-
-func TestAPIKeyService_HashAndVerify(t *testing.T) {
- service := NewAPIKeyService(nil, nil)
-
- key, _ := service.GenerateKey()
-
- // Hash the key
- hash, err := service.HashKey(key)
- if err != nil {
- t.Fatalf("Failed to hash key: %v", err)
- }
-
- // Hash should not be empty
- if hash == "" {
- t.Error("Hash should not be empty")
- }
-
- // Hash should not equal the key
- if hash == key {
- t.Error("Hash should not equal the original key")
- }
-
- // Verify correct key
- if !service.VerifyKey(key, hash) {
- t.Error("VerifyKey should return true for correct key")
- }
-
- // Verify wrong key
- wrongKey := key + "x"
- if service.VerifyKey(wrongKey, hash) {
- t.Error("VerifyKey should return false for wrong key")
- }
-}
-
-func TestAPIKeyModel_Scopes(t *testing.T) {
- tests := []struct {
- name string
- scopes []string
- check string
- expected bool
- }{
- {
- name: "exact match",
- scopes: []string{"execute:*", "read:executions"},
- check: "execute:*",
- expected: true,
- },
- {
- name: "wildcard execute",
- scopes: []string{"execute:*"},
- check: "execute:agent-123",
- expected: true,
- },
- {
- name: "specific agent",
- scopes: []string{"execute:agent-123"},
- check: "execute:agent-123",
- expected: true,
- },
- {
- name: "wrong agent",
- scopes: []string{"execute:agent-123"},
- check: "execute:agent-456",
- expected: false,
- },
- {
- name: "full access",
- scopes: []string{"*"},
- check: "execute:agent-123",
- expected: true,
- },
- {
- name: "no match",
- scopes: []string{"read:executions"},
- check: "execute:*",
- expected: false,
- },
- }
-
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- key := &models.APIKey{Scopes: tt.scopes}
- result := key.HasScope(tt.check)
- if result != tt.expected {
- t.Errorf("HasScope(%s) = %v, expected %v", tt.check, result, tt.expected)
- }
- })
- }
-}
-
-func TestAPIKeyModel_HasExecuteScope(t *testing.T) {
- tests := []struct {
- name string
- scopes []string
- agentID string
- expected bool
- }{
- {
- name: "wildcard execute",
- scopes: []string{"execute:*"},
- agentID: "agent-123",
- expected: true,
- },
- {
- name: "specific agent match",
- scopes: []string{"execute:agent-123"},
- agentID: "agent-123",
- expected: true,
- },
- {
- name: "specific agent no match",
- scopes: []string{"execute:agent-456"},
- agentID: "agent-123",
- expected: false,
- },
- {
- name: "no execute scope",
- scopes: []string{"read:executions"},
- agentID: "agent-123",
- expected: false,
- },
- }
-
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- key := &models.APIKey{Scopes: tt.scopes}
- result := key.HasExecuteScope(tt.agentID)
- if result != tt.expected {
- t.Errorf("HasExecuteScope(%s) = %v, expected %v", tt.agentID, result, tt.expected)
- }
- })
- }
-}
-
-func TestAPIKeyModel_IsValid(t *testing.T) {
- key := &models.APIKey{}
-
- // Should be valid by default (no revocation, no expiration)
- if !key.IsValid() {
- t.Error("New key should be valid")
- }
-
- if key.IsRevoked() {
- t.Error("New key should not be revoked")
- }
-
- if key.IsExpired() {
- t.Error("New key should not be expired")
- }
-}
-
-func TestAPIKeyListItem_Conversion(t *testing.T) {
- key := &models.APIKey{
- KeyPrefix: "clv_test1234",
- Name: "Test Key",
- Description: "A test key",
- Scopes: []string{"execute:*"},
- }
-
- item := key.ToListItem()
-
- if item.KeyPrefix != key.KeyPrefix {
- t.Errorf("KeyPrefix mismatch: got %s, want %s", item.KeyPrefix, key.KeyPrefix)
- }
- if item.Name != key.Name {
- t.Errorf("Name mismatch: got %s, want %s", item.Name, key.Name)
- }
- if item.Description != key.Description {
- t.Errorf("Description mismatch: got %s, want %s", item.Description, key.Description)
- }
- if len(item.Scopes) != len(key.Scopes) {
- t.Errorf("Scopes length mismatch: got %d, want %d", len(item.Scopes), len(key.Scopes))
- }
-}
-
-func TestIsValidScope(t *testing.T) {
- tests := []struct {
- scope string
- expected bool
- }{
- {"execute:*", true},
- {"read:executions", true},
- {"read:*", true},
- {"*", true},
- {"execute:agent-123", true},
- {"invalid", false},
- {"write:*", false},
- {"delete:*", false},
- }
-
- for _, tt := range tests {
- t.Run(tt.scope, func(t *testing.T) {
- result := models.IsValidScope(tt.scope)
- if result != tt.expected {
- t.Errorf("IsValidScope(%s) = %v, expected %v", tt.scope, result, tt.expected)
- }
- })
- }
-}
-
-func TestAPIKeyService_Integration(t *testing.T) {
- if testing.Short() {
- t.Skip("Skipping integration test in short mode")
- }
-
- // This test would require MongoDB
- _ = context.Background()
- service := NewAPIKeyService(nil, nil)
- if service == nil {
- t.Fatal("Expected non-nil service")
- }
-}
diff --git a/backend/internal/services/audio_init.go b/backend/internal/services/audio_init.go
deleted file mode 100644
index 78acca4e..00000000
--- a/backend/internal/services/audio_init.go
+++ /dev/null
@@ -1,84 +0,0 @@
-package services
-
-import (
- "claraverse/internal/audio"
- "fmt"
- "log"
- "sync"
-)
-
-var audioInitOnce sync.Once
-
-// InitAudioService initializes the audio package with provider access
-// Priority: Groq (cheaper) -> OpenAI (fallback)
-func InitAudioService() {
- if visionProviderSvc == nil {
- log.Println("⚠️ [AUDIO-INIT] Provider service not set, audio service disabled")
- return
- }
-
- audioInitOnce.Do(func() {
- // Groq provider getter callback (primary - much cheaper)
- groqProviderGetter := func() (*audio.Provider, error) {
- // Try to get Groq provider by name (try both cases)
- provider, err := visionProviderSvc.GetByName("Groq")
- if err != nil || provider == nil {
- // Fallback to lowercase
- provider, err = visionProviderSvc.GetByName("groq")
- }
- if err != nil {
- return nil, fmt.Errorf("Groq provider not found: %w", err)
- }
- if provider == nil {
- return nil, fmt.Errorf("Groq provider not configured")
- }
- if !provider.Enabled {
- return nil, fmt.Errorf("Groq provider is disabled")
- }
- if provider.APIKey == "" {
- return nil, fmt.Errorf("Groq API key not configured")
- }
-
- return &audio.Provider{
- ID: provider.ID,
- Name: provider.Name,
- BaseURL: provider.BaseURL,
- APIKey: provider.APIKey,
- Enabled: provider.Enabled,
- }, nil
- }
-
- // OpenAI provider getter callback (fallback)
- openaiProviderGetter := func() (*audio.Provider, error) {
- // Try to get OpenAI provider by name (try both cases)
- provider, err := visionProviderSvc.GetByName("OpenAI")
- if err != nil || provider == nil {
- // Fallback to lowercase
- provider, err = visionProviderSvc.GetByName("openai")
- }
- if err != nil {
- return nil, fmt.Errorf("OpenAI provider not found: %w", err)
- }
- if provider == nil {
- return nil, fmt.Errorf("OpenAI provider not configured")
- }
- if !provider.Enabled {
- return nil, fmt.Errorf("OpenAI provider is disabled")
- }
- if provider.APIKey == "" {
- return nil, fmt.Errorf("OpenAI API key not configured")
- }
-
- return &audio.Provider{
- ID: provider.ID,
- Name: provider.Name,
- BaseURL: provider.BaseURL,
- APIKey: provider.APIKey,
- Enabled: provider.Enabled,
- }, nil
- }
-
- audio.InitService(groqProviderGetter, openaiProviderGetter)
- log.Printf("✅ [AUDIO-INIT] Audio service initialized (Groq primary, OpenAI fallback)")
- })
-}
diff --git a/backend/internal/services/builder_conversation_service.go b/backend/internal/services/builder_conversation_service.go
deleted file mode 100644
index 087fbaa7..00000000
--- a/backend/internal/services/builder_conversation_service.go
+++ /dev/null
@@ -1,250 +0,0 @@
-package services
-
-import (
- "claraverse/internal/crypto"
- "claraverse/internal/database"
- "claraverse/internal/models"
- "context"
- "encoding/json"
- "fmt"
- "log"
- "time"
-
- "go.mongodb.org/mongo-driver/bson"
- "go.mongodb.org/mongo-driver/bson/primitive"
- "go.mongodb.org/mongo-driver/mongo"
- "go.mongodb.org/mongo-driver/mongo/options"
-)
-
-// BuilderConversationService handles builder conversation operations with MongoDB
-type BuilderConversationService struct {
- db *database.MongoDB
- collection *mongo.Collection
- encryption *crypto.EncryptionService
-}
-
-// NewBuilderConversationService creates a new builder conversation service
-func NewBuilderConversationService(db *database.MongoDB, encryption *crypto.EncryptionService) *BuilderConversationService {
- return &BuilderConversationService{
- db: db,
- collection: db.Collection(database.CollectionBuilderConversations),
- encryption: encryption,
- }
-}
-
-// CreateConversation creates a new builder conversation for an agent
-func (s *BuilderConversationService) CreateConversation(ctx context.Context, agentID, userID, modelID string) (*models.ConversationResponse, error) {
- now := time.Now()
-
- // Create encrypted conversation with string-based IDs
- // AgentID is a timestamp-based string (e.g., "1765018813035-yplenlye1")
- // UserID is a Supabase UUID string
- conv := &models.EncryptedBuilderConversation{
- AgentID: agentID,
- UserID: userID,
- EncryptedMessages: "", // Empty at creation
- ModelID: modelID,
- MessageCount: 0,
- CreatedAt: now,
- UpdatedAt: now,
- }
-
- result, err := s.collection.InsertOne(ctx, conv)
- if err != nil {
- return nil, fmt.Errorf("failed to create conversation: %w", err)
- }
-
- conv.ID = result.InsertedID.(primitive.ObjectID)
-
- return &models.ConversationResponse{
- ID: conv.ID.Hex(),
- AgentID: agentID,
- ModelID: modelID,
- Messages: []models.BuilderMessage{},
- CreatedAt: now,
- UpdatedAt: now,
- }, nil
-}
-
-// GetConversation retrieves a conversation by ID and decrypts messages
-func (s *BuilderConversationService) GetConversation(ctx context.Context, conversationID, userID string) (*models.ConversationResponse, error) {
- convOID, err := primitive.ObjectIDFromHex(conversationID)
- if err != nil {
- return nil, fmt.Errorf("invalid conversation ID: %w", err)
- }
-
- var encrypted models.EncryptedBuilderConversation
- err = s.collection.FindOne(ctx, bson.M{"_id": convOID}).Decode(&encrypted)
- if err == mongo.ErrNoDocuments {
- return nil, fmt.Errorf("conversation not found")
- }
- if err != nil {
- return nil, fmt.Errorf("failed to get conversation: %w", err)
- }
-
- // Decrypt messages
- var messages []models.BuilderMessage
- if encrypted.EncryptedMessages != "" {
- decrypted, err := s.encryption.Decrypt(userID, encrypted.EncryptedMessages)
- if err != nil {
- log.Printf("⚠️ Failed to decrypt conversation %s: %v", conversationID, err)
- // Return empty messages on decryption failure
- messages = []models.BuilderMessage{}
- } else {
- if err := json.Unmarshal(decrypted, &messages); err != nil {
- log.Printf("⚠️ Failed to unmarshal decrypted messages: %v", err)
- messages = []models.BuilderMessage{}
- }
- }
- }
-
- return &models.ConversationResponse{
- ID: conversationID,
- AgentID: encrypted.AgentID, // AgentID is already a string
- ModelID: encrypted.ModelID,
- Messages: messages,
- CreatedAt: encrypted.CreatedAt,
- UpdatedAt: encrypted.UpdatedAt,
- }, nil
-}
-
-// GetConversationsByAgent retrieves all conversations for an agent
-func (s *BuilderConversationService) GetConversationsByAgent(ctx context.Context, agentID, userID string) ([]models.ConversationListItem, error) {
- opts := options.Find().
- SetSort(bson.M{"updatedAt": -1}).
- SetLimit(50)
-
- // AgentID is a string (timestamp-based), not an ObjectID
- cursor, err := s.collection.Find(ctx, bson.M{"agentId": agentID}, opts)
- if err != nil {
- return nil, fmt.Errorf("failed to list conversations: %w", err)
- }
- defer cursor.Close(ctx)
-
- var conversations []models.ConversationListItem
- for cursor.Next(ctx) {
- var encrypted models.EncryptedBuilderConversation
- if err := cursor.Decode(&encrypted); err != nil {
- log.Printf("⚠️ Failed to decode conversation: %v", err)
- continue
- }
- conversations = append(conversations, encrypted.ToListItem())
- }
-
- return conversations, nil
-}
-
-// AddMessage adds a message to a conversation
-func (s *BuilderConversationService) AddMessage(ctx context.Context, conversationID, userID string, req *models.AddMessageRequest) (*models.BuilderMessage, error) {
- // Get existing conversation
- conv, err := s.GetConversation(ctx, conversationID, userID)
- if err != nil {
- return nil, err
- }
-
- // Create new message
- message := models.BuilderMessage{
- ID: fmt.Sprintf("msg-%d", time.Now().UnixNano()),
- Role: req.Role,
- Content: req.Content,
- Timestamp: time.Now(),
- WorkflowSnapshot: req.WorkflowSnapshot,
- }
-
- // Add message to list
- messages := append(conv.Messages, message)
-
- // Serialize and encrypt messages
- messagesJSON, err := json.Marshal(messages)
- if err != nil {
- return nil, fmt.Errorf("failed to serialize messages: %w", err)
- }
-
- encryptedMessages, err := s.encryption.Encrypt(userID, messagesJSON)
- if err != nil {
- return nil, fmt.Errorf("failed to encrypt messages: %w", err)
- }
-
- // Update conversation
- convOID, _ := primitive.ObjectIDFromHex(conversationID)
- _, err = s.collection.UpdateOne(ctx,
- bson.M{"_id": convOID},
- bson.M{
- "$set": bson.M{
- "encryptedMessages": encryptedMessages,
- "messageCount": len(messages),
- "updatedAt": time.Now(),
- },
- },
- )
- if err != nil {
- return nil, fmt.Errorf("failed to update conversation: %w", err)
- }
-
- return &message, nil
-}
-
-// DeleteConversation deletes a conversation
-func (s *BuilderConversationService) DeleteConversation(ctx context.Context, conversationID, userID string) error {
- convOID, err := primitive.ObjectIDFromHex(conversationID)
- if err != nil {
- return fmt.Errorf("invalid conversation ID: %w", err)
- }
-
- result, err := s.collection.DeleteOne(ctx, bson.M{"_id": convOID})
- if err != nil {
- return fmt.Errorf("failed to delete conversation: %w", err)
- }
-
- if result.DeletedCount == 0 {
- return fmt.Errorf("conversation not found")
- }
-
- log.Printf("✅ Deleted builder conversation %s for user %s", conversationID, userID)
- return nil
-}
-
-// DeleteConversationsByAgent deletes all conversations for an agent
-func (s *BuilderConversationService) DeleteConversationsByAgent(ctx context.Context, agentID string) error {
- // AgentID is a string (timestamp-based), not an ObjectID
- result, err := s.collection.DeleteMany(ctx, bson.M{"agentId": agentID})
- if err != nil {
- return fmt.Errorf("failed to delete conversations: %w", err)
- }
-
- log.Printf("✅ Deleted %d builder conversations for agent %s", result.DeletedCount, agentID)
- return nil
-}
-
-// DeleteConversationsByUser deletes all conversations for a user (GDPR)
-func (s *BuilderConversationService) DeleteConversationsByUser(ctx context.Context, userID string) error {
- // UserID is a Supabase UUID string, not an ObjectID
- result, err := s.collection.DeleteMany(ctx, bson.M{"userId": userID})
- if err != nil {
- return fmt.Errorf("failed to delete user conversations: %w", err)
- }
-
- log.Printf("✅ [GDPR] Deleted %d builder conversations for user %s", result.DeletedCount, userID)
- return nil
-}
-
-// GetOrCreateConversation gets the most recent conversation for an agent, or creates one if none exists
-func (s *BuilderConversationService) GetOrCreateConversation(ctx context.Context, agentID, userID, modelID string) (*models.ConversationResponse, error) {
- // Try to find existing conversation
- // AgentID is a string (timestamp-based), not an ObjectID
- opts := options.FindOne().SetSort(bson.M{"updatedAt": -1})
-
- var encrypted models.EncryptedBuilderConversation
- err := s.collection.FindOne(ctx, bson.M{"agentId": agentID}, opts).Decode(&encrypted)
-
- if err == mongo.ErrNoDocuments {
- // Create new conversation
- return s.CreateConversation(ctx, agentID, userID, modelID)
- }
- if err != nil {
- return nil, fmt.Errorf("failed to find conversation: %w", err)
- }
-
- // Return existing conversation
- return s.GetConversation(ctx, encrypted.ID.Hex(), userID)
-}
diff --git a/backend/internal/services/chat_service.go b/backend/internal/services/chat_service.go
deleted file mode 100644
index d7e2966a..00000000
--- a/backend/internal/services/chat_service.go
+++ /dev/null
@@ -1,3043 +0,0 @@
-package services
-
-import (
- "bufio"
- "bytes"
- "context"
- "encoding/json"
- "fmt"
- "io"
- "log"
- "net/http"
- "strings"
- "time"
-
- "claraverse/internal/database"
- "claraverse/internal/models"
- "claraverse/internal/tools"
-
- cache "github.com/patrickmn/go-cache"
-)
-
-// truncateToolCallID ensures tool call IDs are max 40 chars (OpenAI API constraint)
-// OpenAI may send IDs > 40 chars, but rejects them when sent back
-func truncateToolCallID(id string) string {
- if len(id) <= 40 {
- return id
- }
- // Keep prefix (like "call_") and truncate to 40 chars
- return id[:40]
-}
-
-// ChatService handles chat operations
-type ChatService struct {
- db *database.DB
- providerService *ProviderService
- conversationCache *cache.Cache // TTL cache for conversation history (30 min)
- summaryCache *cache.Cache // Cache for AI-generated context summaries
- toolRegistry *tools.Registry
- toolService *ToolService // Tool service for credential-filtered tools
- mcpBridge *MCPBridgeService // MCP bridge for local client tools
- modelAliases map[int]map[string]string // Provider ID -> (Frontend Model -> Actual Model) mapping
- streamBuffer *StreamBufferService // Buffer for resumable streaming
- usageLimiter *UsageLimiterService // Usage limiter for tier-based limits
- toolPredictorService *ToolPredictorService // Tool predictor for dynamic tool selection
- memoryExtractionService *MemoryExtractionService // Memory extraction service for extracting memories from chats
- memorySelectionService *MemorySelectionService // Memory selection service for selecting relevant memories
- userService *UserService // User service for getting user preferences
-}
-
-// ContextSummary stores AI-generated summary of older messages
-type ContextSummary struct {
- Summary string // AI-generated summary text
- SummarizedCount int // Number of messages that were summarized
- LastMessageIndex int // Index of the last message that was summarized
- CreatedAt time.Time // When this summary was created
-}
-
-// ImageRegistryAdapter wraps ImageRegistryService to implement tools.ImageRegistryInterface
-// This adapter is needed to avoid import cycles between services and tools packages
-type ImageRegistryAdapter struct {
- registry *ImageRegistryService
-}
-
-// GetByHandle implements tools.ImageRegistryInterface
-func (a *ImageRegistryAdapter) GetByHandle(conversationID, handle string) *tools.ImageRegistryEntry {
- entry := a.registry.GetByHandle(conversationID, handle)
- if entry == nil {
- return nil
- }
- return &tools.ImageRegistryEntry{
- Handle: entry.Handle,
- FileID: entry.FileID,
- Filename: entry.Filename,
- Source: entry.Source,
- }
-}
-
-// ListHandles implements tools.ImageRegistryInterface
-func (a *ImageRegistryAdapter) ListHandles(conversationID string) []string {
- return a.registry.ListHandles(conversationID)
-}
-
-// RegisterGeneratedImage implements tools.ImageRegistryInterface
-func (a *ImageRegistryAdapter) RegisterGeneratedImage(conversationID, fileID, prompt string) string {
- return a.registry.RegisterGeneratedImage(conversationID, fileID, prompt)
-}
-
-// RegisterEditedImage implements tools.ImageRegistryInterface
-func (a *ImageRegistryAdapter) RegisterEditedImage(conversationID, fileID, sourceHandle, prompt string) string {
- return a.registry.RegisterEditedImage(conversationID, fileID, sourceHandle, prompt)
-}
-
-// NewChatService creates a new chat service
-func NewChatService(
- db *database.DB,
- providerService *ProviderService,
- mcpBridge *MCPBridgeService,
- toolService *ToolService,
-) *ChatService {
- // Create conversation cache with eviction hook for file cleanup
- conversationCache := cache.New(30*time.Minute, 10*time.Minute)
-
- // Create summary cache with longer TTL (1 hour) - summaries are expensive to regenerate
- summaryCache := cache.New(1*time.Hour, 15*time.Minute)
-
- // Set up eviction handler to cleanup associated files
- conversationCache.OnEvicted(func(key string, value interface{}) {
- conversationID := key
- log.Printf("🗑️ [CACHE-EVICT] Conversation %s expired, cleaning up associated files...", conversationID)
-
- // Get file cache service
- fileCache := GetFileCacheService()
-
- // Delete all files for this conversation
- fileCache.DeleteConversationFiles(conversationID)
-
- // Also clean up the summary cache
- summaryCache.Delete(conversationID)
-
- log.Printf("✅ [CACHE-EVICT] Cleanup completed for conversation %s", conversationID)
- })
-
- return &ChatService{
- db: db,
- providerService: providerService,
- conversationCache: conversationCache,
- summaryCache: summaryCache,
- toolRegistry: tools.GetRegistry(),
- toolService: toolService,
- mcpBridge: mcpBridge,
- modelAliases: make(map[int]map[string]string),
- streamBuffer: NewStreamBufferService(),
- }
-}
-
-// SetToolService sets the tool service for credential-filtered tools
-// This allows setting the tool service after initialization when there are circular dependencies
-func (s *ChatService) SetToolService(toolService *ToolService) {
- s.toolService = toolService
- log.Println("✅ [CHAT-SERVICE] Tool service set for credential-filtered tools")
-}
-
-// SetUsageLimiter sets the usage limiter for tier-based usage limits
-func (s *ChatService) SetUsageLimiter(usageLimiter *UsageLimiterService) {
- s.usageLimiter = usageLimiter
- log.Println("✅ [CHAT-SERVICE] Usage limiter set for tier-based limits")
-}
-
-// SetToolPredictorService sets the tool predictor service for dynamic tool selection
-func (s *ChatService) SetToolPredictorService(predictor *ToolPredictorService) {
- s.toolPredictorService = predictor
- log.Println("✅ [CHAT-SERVICE] Tool predictor service set for smart tool routing")
-}
-
-// SetMemoryExtractionService sets the memory extraction service for extracting memories from chats
-func (s *ChatService) SetMemoryExtractionService(memoryExtraction *MemoryExtractionService) {
- s.memoryExtractionService = memoryExtraction
- log.Println("✅ [CHAT-SERVICE] Memory extraction service set for conversation memory extraction")
-}
-
-// SetMemorySelectionService sets the memory selection service for selecting relevant memories
-func (s *ChatService) SetMemorySelectionService(memorySelection *MemorySelectionService) {
- s.memorySelectionService = memorySelection
- log.Println("✅ [CHAT-SERVICE] Memory selection service set for memory injection")
-}
-
-// SetUserService sets the user service for getting user preferences
-func (s *ChatService) SetUserService(userService *UserService) {
- s.userService = userService
- log.Println("✅ [CHAT-SERVICE] User service set for preference checking")
-}
-
-// GetStreamBuffer returns the stream buffer service for resume handling
-func (s *ChatService) GetStreamBuffer() *StreamBufferService {
- return s.streamBuffer
-}
-
-// SetModelAliases sets model aliases for a provider
-func (s *ChatService) SetModelAliases(providerID int, aliases map[string]models.ModelAlias) {
- if aliases != nil && len(aliases) > 0 {
- // Convert ModelAlias to string map for internal storage
- stringAliases := make(map[string]string)
- for frontend, alias := range aliases {
- stringAliases[frontend] = alias.ActualModel
- }
- s.modelAliases[providerID] = stringAliases
-
- log.Printf("🔄 [MODEL-ALIAS] Loaded %d model aliases for provider %d", len(aliases), providerID)
- for frontend, alias := range aliases {
- if alias.Description != "" {
- log.Printf(" %s -> %s (%s)", frontend, alias.ActualModel, alias.Description)
- } else {
- log.Printf(" %s -> %s", frontend, alias.ActualModel)
- }
- }
- }
-}
-
-// resolveModelName resolves a frontend model name to the actual model name using aliases
-func (s *ChatService) resolveModelName(providerID int, modelName string) string {
- if aliases, exists := s.modelAliases[providerID]; exists {
- if actualModel, found := aliases[modelName]; found {
- log.Printf("🔄 [MODEL-ALIAS] Resolved '%s' -> '%s' for provider %d", modelName, actualModel, providerID)
- return actualModel
- }
- }
- // No alias found, return original model name
- return modelName
-}
-
-// resolveModelAlias searches all providers for a model alias and returns the provider ID and actual model name
-// Returns (providerID, actualModelName, found)
-func (s *ChatService) resolveModelAlias(aliasName string) (int, string, bool) {
- for providerID, aliases := range s.modelAliases {
- if actualModel, found := aliases[aliasName]; found {
- log.Printf("🔄 [MODEL-ALIAS] Resolved alias '%s' -> provider=%d, model='%s'", aliasName, providerID, actualModel)
- return providerID, actualModel, true
- }
- }
- return 0, "", false
-}
-
-// ResolveModelAlias is the public version that returns provider and actual model name
-// Returns (provider, actualModelName, found)
-func (s *ChatService) ResolveModelAlias(aliasName string) (*models.Provider, string, bool) {
- providerID, actualModel, found := s.resolveModelAlias(aliasName)
- if !found {
- return nil, "", false
- }
-
- provider, err := s.providerService.GetByID(providerID)
- if err != nil {
- log.Printf("⚠️ [MODEL-ALIAS] Found alias but provider %d not found: %v", providerID, err)
- return nil, "", false
- }
-
- return provider, actualModel, true
-}
-
-// GetDefaultProvider returns the first available enabled provider (for fallback)
-func (s *ChatService) GetDefaultProvider() (*models.Provider, error) {
- providers, err := s.providerService.GetAll()
- if err != nil {
- return nil, fmt.Errorf("failed to get providers: %w", err)
- }
-
- if len(providers) == 0 {
- return nil, fmt.Errorf("no providers configured")
- }
-
- // Return first enabled provider
- return &providers[0], nil
-}
-
-// GetDefaultProviderWithModel returns the first available provider and a default model from it
-func (s *ChatService) GetDefaultProviderWithModel() (*models.Provider, string, error) {
- provider, err := s.GetDefaultProvider()
- if err != nil {
- return nil, "", err
- }
-
- // Query for the first visible model from this provider
- var modelID string
- err = s.db.QueryRow(`
- SELECT id FROM models
- WHERE provider_id = ? AND is_visible = 1
- ORDER BY name
- LIMIT 1
- `, provider.ID).Scan(&modelID)
-
- if err != nil {
- // No models found, try without visibility filter
- err = s.db.QueryRow(`
- SELECT id FROM models
- WHERE provider_id = ?
- ORDER BY name
- LIMIT 1
- `, provider.ID).Scan(&modelID)
-
- if err != nil {
- return nil, "", fmt.Errorf("no models found for default provider %s: %w", provider.Name, err)
- }
- }
-
- log.Printf("🔧 [DEFAULT] Using provider '%s' with model '%s'", provider.Name, modelID)
- return provider, modelID, nil
-}
-
-// GetTextProviderWithModel returns a text-capable provider and model for internal use (metadata generation, etc.)
-// It tries model aliases first, then falls back to finding any enabled text provider
-// This filters out audio-only and image-only providers
-func (s *ChatService) GetTextProviderWithModel() (*models.Provider, string, error) {
- // Strategy 1: Try to use model aliases from config (these are known good text models)
- configService := GetConfigService()
- allAliases := configService.GetAllModelAliases()
-
- // Get image-only provider names to filter them out
- imageProviderService := GetImageProviderService()
- imageProviders := imageProviderService.GetAllProviders()
- imageProviderNames := make(map[string]bool)
- for _, ip := range imageProviders {
- imageProviderNames[ip.Name] = true
- }
-
- // Strategy 1.5: Query database for smallest/fastest available model (prefer smaller models for metadata)
- // Try to get models with lower context length (usually faster/cheaper)
- log.Printf("📋 [TEXT-PROVIDER] Querying database for optimal text model...")
- var modelID string
- var modelProviderID int
- err := s.db.QueryRow(`
- SELECT m.id, m.provider_id
- FROM models m
- JOIN providers p ON m.provider_id = p.id
- WHERE m.is_visible = 1
- AND p.enabled = 1
- AND (p.audio_only = 0 OR p.audio_only IS NULL)
- ORDER BY m.context_length ASC, m.id ASC
- LIMIT 1
- `).Scan(&modelID, &modelProviderID)
-
- if err == nil {
- provider, err := s.providerService.GetByID(modelProviderID)
- if err == nil && !provider.AudioOnly && !imageProviderNames[provider.Name] {
- log.Printf("📋 [TEXT-PROVIDER] Found optimal model from database: %s (provider: %s)", modelID, provider.Name)
- return provider, modelID, nil
- }
- }
-
- // Strategy 2: Try any available model alias
- for providerID, aliases := range allAliases {
- for aliasName, aliasInfo := range aliases {
- provider, err := s.providerService.GetByID(providerID)
- if err != nil || !provider.Enabled || provider.AudioOnly {
- continue
- }
- if imageProviderNames[provider.Name] {
- continue
- }
-
- log.Printf("📋 [TEXT-PROVIDER] Found via any alias: %s -> %s (provider: %s)",
- aliasName, aliasInfo.ActualModel, provider.Name)
- return provider, aliasInfo.ActualModel, nil
- }
- }
-
- // Strategy 3: Query database for any text-capable provider with models
- log.Printf("📋 [TEXT-PROVIDER] No aliases found, querying database for text provider...")
-
- var providerID int
- var providerName, baseURL, apiKey string
- var systemPrompt, favicon *string
-
- // Find first enabled text provider (not audio_only) that has models
- err = s.db.QueryRow(`
- SELECT p.id, p.name, p.base_url, p.api_key, p.system_prompt, p.favicon
- FROM providers p
- WHERE p.enabled = 1 AND (p.audio_only = 0 OR p.audio_only IS NULL)
- AND EXISTS (SELECT 1 FROM models m WHERE m.provider_id = p.id)
- ORDER BY p.id ASC
- LIMIT 1
- `).Scan(&providerID, &providerName, &baseURL, &apiKey, &systemPrompt, &favicon)
-
- if err != nil {
- return nil, "", fmt.Errorf("no text-capable provider found: %w", err)
- }
-
- // Check if this provider is an image-only provider
- if imageProviderNames[providerName] {
- // Try to find the next one that's not image-only
- rows, err := s.db.Query(`
- SELECT p.id, p.name, p.base_url, p.api_key, p.system_prompt, p.favicon
- FROM providers p
- WHERE p.enabled = 1 AND (p.audio_only = 0 OR p.audio_only IS NULL)
- AND EXISTS (SELECT 1 FROM models m WHERE m.provider_id = p.id)
- ORDER BY p.id ASC
- `)
- if err != nil {
- return nil, "", fmt.Errorf("failed to query providers: %w", err)
- }
- defer rows.Close()
-
- found := false
- for rows.Next() {
- if err := rows.Scan(&providerID, &providerName, &baseURL, &apiKey, &systemPrompt, &favicon); err != nil {
- continue
- }
- if !imageProviderNames[providerName] {
- found = true
- break
- }
- }
-
- if !found {
- return nil, "", fmt.Errorf("no text-capable provider found (all are image-only)")
- }
- }
-
- // Get first model from this provider
- modelID = "" // Reset modelID for this provider
- err = s.db.QueryRow(`
- SELECT id FROM models
- WHERE provider_id = ? AND is_visible = 1
- ORDER BY name
- LIMIT 1
- `, providerID).Scan(&modelID)
-
- if err != nil {
- // Try without visibility filter
- err = s.db.QueryRow(`
- SELECT id FROM models
- WHERE provider_id = ?
- ORDER BY name
- LIMIT 1
- `, providerID).Scan(&modelID)
-
- if err != nil {
- return nil, "", fmt.Errorf("no models found for provider %s: %w", providerName, err)
- }
- }
-
- provider := &models.Provider{
- ID: providerID,
- Name: providerName,
- BaseURL: baseURL,
- APIKey: apiKey,
- Enabled: true,
- }
- if systemPrompt != nil {
- provider.SystemPrompt = *systemPrompt
- }
- if favicon != nil {
- provider.Favicon = *favicon
- }
-
- log.Printf("📋 [TEXT-PROVIDER] Found via database: provider=%s, model=%s", providerName, modelID)
- return provider, modelID, nil
-}
-
-// getConversationMessages retrieves messages from cache
-func (s *ChatService) getConversationMessages(conversationID string) []map[string]interface{} {
- if cached, found := s.conversationCache.Get(conversationID); found {
- if messages, ok := cached.([]map[string]interface{}); ok {
- log.Printf("📖 [CACHE] Retrieved %d messages for conversation %s", len(messages), conversationID)
- return messages
- }
- log.Printf("⚠️ [CACHE] Invalid cache data type for conversation %s", conversationID)
- }
- log.Printf("📖 [CACHE] No cached messages for conversation %s (starting fresh)", conversationID)
- return []map[string]interface{}{}
-}
-
-// GetConversationMessages retrieves messages from cache (public)
-func (s *ChatService) GetConversationMessages(conversationID string) []map[string]interface{} {
- return s.getConversationMessages(conversationID)
-}
-
-// setConversationMessages stores messages in cache with TTL
-func (s *ChatService) setConversationMessages(conversationID string, messages []map[string]interface{}) {
- s.conversationCache.Set(conversationID, messages, cache.DefaultExpiration)
- log.Printf("💾 [CACHE] Stored %d messages for conversation %s", len(messages), conversationID)
-}
-
-// Context Window Management Constants
-const (
- // Maximum tokens to send to the model (conservative limit for safety)
- // Most models support 128K+, but we use 80K to leave room for response
- MaxContextTokens = 80000
-
- // Threshold to trigger summarization (70% of max)
- SummarizationThreshold = 56000
-
- // Approximate tokens per character (conservative estimate)
- TokensPerChar = 0.25
-
- // Number of recent messages to always keep verbatim (higher = more context preserved)
- RecentMessagesToKeep = 20
-
- // Maximum characters for a single message before truncation
- MaxMessageChars = 50000
-
- // Minimum messages before summarization kicks in
- MinMessagesForSummary = 15
-)
-
-// estimateTokens provides a rough token count for a string
-// Uses the conservative estimate of ~4 chars per token
-func estimateTokens(s string) int {
- return int(float64(len(s)) * TokensPerChar)
-}
-
-// estimateMessagesTokens calculates approximate token count for messages
-func estimateMessagesTokens(messages []map[string]interface{}) int {
- total := 0
- for _, msg := range messages {
- if content, ok := msg["content"].(string); ok {
- total += estimateTokens(content)
- }
- // Account for role and structure overhead
- total += 10
- }
- return total
-}
-
-// getContextSummary retrieves a cached context summary for a conversation
-func (s *ChatService) getContextSummary(conversationID string) *ContextSummary {
- if cached, found := s.summaryCache.Get(conversationID); found {
- if summary, ok := cached.(*ContextSummary); ok {
- return summary
- }
- }
- return nil
-}
-
-// setContextSummary stores a context summary in cache
-func (s *ChatService) setContextSummary(conversationID string, summary *ContextSummary) {
- s.summaryCache.Set(conversationID, summary, cache.DefaultExpiration)
- log.Printf("💾 [SUMMARY] Stored context summary for %s (%d messages summarized)", conversationID, summary.SummarizedCount)
-}
-
-// generateContextSummary uses AI to create a summary of older messages
-// This runs asynchronously to not block the main conversation
-func (s *ChatService) generateContextSummary(conversationID string, messages []map[string]interface{}, config *models.Config) string {
- // Build the content to summarize
- var contentToSummarize strings.Builder
- for i, msg := range messages {
- role, _ := msg["role"].(string)
- content, _ := msg["content"].(string)
- if role == "system" {
- continue // Skip system messages
- }
- // Truncate very long messages for the summary (keep more context for technical conversations)
- if len(content) > 8000 {
- content = content[:4000] + "\n\n[... middle content truncated for summary ...]\n\n" + content[len(content)-2000:]
- }
- contentToSummarize.WriteString(fmt.Sprintf("[%s #%d]: %s\n\n", role, i+1, content))
- }
-
- // Create summarization prompt - optimized for technical conversations
- summaryPrompt := []map[string]interface{}{
- {
- "role": "system",
- "content": `You are a technical conversation summarizer. Your job is to create a detailed context summary that preserves ALL important information needed to continue the conversation seamlessly.
-
-CRITICAL - You MUST preserve:
-1. **FILE PATHS & CODE** - Every file path, function name, class name, variable name mentioned
-2. **TECHNICAL DECISIONS** - Architecture choices, implementation approaches, why certain solutions were chosen/rejected
-3. **BUGS & FIXES** - What was broken, what fixed it, error messages encountered
-4. **CONFIGURATION** - Settings, thresholds, environment variables, API endpoints discussed
-5. **CURRENT STATE** - What has been implemented, what's pending, what's blocked
-6. **USER PREFERENCES** - Coding style, frameworks preferred, constraints mentioned
-7. **SPECIFIC VALUES** - Numbers, dates, versions, exact strings that were important
-
-FORMAT YOUR SUMMARY AS:
-## Project Context
-[What is being built/modified]
-
-## Files Modified/Discussed
-- path/to/file.ext - what was done
-- path/to/another.ext - what was changed
-
-## Key Technical Details
-[Specific implementations, code patterns, configurations]
-
-## Current Status
-[What's done, what's in progress, what's next]
-
-## Important Decisions Made
-[Why certain approaches were chosen]
-
-## Open Issues/Blockers
-[Any unresolved problems]
-
-Be THOROUGH - it's better to include too much detail than to lose critical context. Max 1500 words.`,
- },
- {
- "role": "user",
- "content": fmt.Sprintf("Create a detailed technical summary of this conversation that preserves all context needed to continue:\n\n%s", contentToSummarize.String()),
- },
- }
-
- // Make a non-streaming request for summary
- chatReq := models.ChatRequest{
- Model: config.Model,
- Messages: summaryPrompt,
- Stream: false,
- Temperature: 0.3, // Low temperature for consistency
- }
-
- reqBody, err := json.Marshal(chatReq)
- if err != nil {
- log.Printf("❌ [SUMMARY] Failed to marshal request: %v", err)
- return ""
- }
-
- req, err := http.NewRequest("POST", config.BaseURL+"/chat/completions", bytes.NewBuffer(reqBody))
- if err != nil {
- log.Printf("❌ [SUMMARY] Failed to create request: %v", err)
- return ""
- }
-
- req.Header.Set("Content-Type", "application/json")
- req.Header.Set("Authorization", "Bearer "+config.APIKey)
-
- client := &http.Client{Timeout: 60 * time.Second}
- resp, err := client.Do(req)
- if err != nil {
- log.Printf("❌ [SUMMARY] Request failed: %v", err)
- return ""
- }
- defer resp.Body.Close()
-
- if resp.StatusCode != http.StatusOK {
- body, _ := io.ReadAll(resp.Body)
- log.Printf("❌ [SUMMARY] API error (status %d): %s", resp.StatusCode, string(body))
- return ""
- }
-
- var result struct {
- Choices []struct {
- Message struct {
- Content string `json:"content"`
- } `json:"message"`
- } `json:"choices"`
- }
-
- if err := json.NewDecoder(resp.Body).Decode(&result); err != nil {
- log.Printf("❌ [SUMMARY] Failed to decode response: %v", err)
- return ""
- }
-
- if len(result.Choices) == 0 {
- log.Printf("⚠️ [SUMMARY] No choices in response")
- return ""
- }
-
- summary := strings.TrimSpace(result.Choices[0].Message.Content)
- log.Printf("✅ [SUMMARY] Generated summary for %s (%d chars)", conversationID, len(summary))
- return summary
-}
-
-// optimizeContextWindow manages context to prevent exceeding token limits
-// Uses AI-powered summarization for older messages, preserving context
-func (s *ChatService) optimizeContextWindow(messages []map[string]interface{}, conversationID string, config *models.Config, writeChan chan models.ServerMessage) []map[string]interface{} {
- totalTokens := estimateMessagesTokens(messages)
-
- // If within limits, return as-is
- if totalTokens <= SummarizationThreshold {
- return messages
- }
-
- log.Printf("📊 [CONTEXT] Context optimization needed: %d tokens exceeds %d threshold", totalTokens, SummarizationThreshold)
-
- // Notify client that context optimization is starting
- if writeChan != nil {
- select {
- case writeChan <- models.ServerMessage{
- Type: "context_optimizing",
- Status: "started",
- Progress: 0,
- Content: "Compacting conversation to keep chatting...",
- }:
- default:
- log.Printf("⚠️ [CONTEXT] WriteChan unavailable for optimization status")
- }
- }
-
- // Strategy 1: Truncate very long individual messages
- for i := range messages {
- if content, ok := messages[i]["content"].(string); ok {
- if len(content) > MaxMessageChars {
- keepFirst := MaxMessageChars / 2
- keepLast := MaxMessageChars / 4
- truncated := content[:keepFirst] + "\n\n[... content truncated ...]\n\n" + content[len(content)-keepLast:]
- messages[i]["content"] = truncated
- log.Printf("✂️ [CONTEXT] Truncated message %d from %d to %d chars", i, len(content), len(truncated))
- }
- }
- }
-
- // Recalculate after truncation
- totalTokens = estimateMessagesTokens(messages)
- if totalTokens <= SummarizationThreshold {
- // Truncation was sufficient - notify completion before returning
- if writeChan != nil {
- select {
- case writeChan <- models.ServerMessage{
- Type: "context_optimizing",
- Status: "completed",
- Progress: 100,
- Content: "Context optimized via truncation",
- }:
- default:
- }
- }
- return messages
- }
-
- // Strategy 2: Use AI summary for older messages
- // Separate system message from conversation
- var systemMsg map[string]interface{}
- nonSystemMessages := make([]map[string]interface{}, 0)
-
- for _, msg := range messages {
- if role, ok := msg["role"].(string); ok && role == "system" {
- systemMsg = msg
- } else {
- nonSystemMessages = append(nonSystemMessages, msg)
- }
- }
-
- // Need enough messages to summarize
- if len(nonSystemMessages) < MinMessagesForSummary {
- // Not enough messages for summarization - notify completion
- if writeChan != nil {
- select {
- case writeChan <- models.ServerMessage{
- Type: "context_optimizing",
- Status: "completed",
- Progress: 100,
- Content: "Context optimization complete",
- }:
- default:
- }
- }
- return messages
- }
-
- // Calculate how many messages to keep vs summarize
- recentCount := RecentMessagesToKeep
- if recentCount > len(nonSystemMessages) {
- recentCount = len(nonSystemMessages)
- }
-
- oldMessages := nonSystemMessages[:len(nonSystemMessages)-recentCount]
- recentMessages := nonSystemMessages[len(nonSystemMessages)-recentCount:]
-
- // Check if we have a valid cached summary
- existingSummary := s.getContextSummary(conversationID)
- var summaryText string
-
- if existingSummary != nil && existingSummary.SummarizedCount >= len(oldMessages)-2 {
- // Use existing summary if it covers most of the old messages
- summaryText = existingSummary.Summary
- log.Printf("📖 [CONTEXT] Using cached summary for %s (covers %d messages)", conversationID, existingSummary.SummarizedCount)
-
- // Quick progress update for cached summary
- if writeChan != nil {
- select {
- case writeChan <- models.ServerMessage{
- Type: "context_optimizing",
- Status: "completed",
- Progress: 100,
- Content: "Using cached summary...",
- }:
- default:
- }
- }
- } else if config != nil {
- // Generate new AI summary - send progress update
- log.Printf("🤖 [CONTEXT] Generating AI summary for %d messages in %s", len(oldMessages), conversationID)
-
- if writeChan != nil {
- select {
- case writeChan <- models.ServerMessage{
- Type: "context_optimizing",
- Status: "summarizing",
- Progress: 30,
- Content: "Summarizing older messages...",
- }:
- default:
- }
- }
-
- summaryText = s.generateContextSummary(conversationID, oldMessages, config)
-
- if summaryText != "" {
- // Cache the summary
- s.setContextSummary(conversationID, &ContextSummary{
- Summary: summaryText,
- SummarizedCount: len(oldMessages),
- LastMessageIndex: len(nonSystemMessages) - recentCount - 1,
- CreatedAt: time.Now(),
- })
-
- // Summary complete
- if writeChan != nil {
- select {
- case writeChan <- models.ServerMessage{
- Type: "context_optimizing",
- Status: "completed",
- Progress: 100,
- Content: "Context optimized successfully",
- }:
- default:
- }
- }
- } else {
- // AI summary failed - still notify completion so modal closes
- if writeChan != nil {
- select {
- case writeChan <- models.ServerMessage{
- Type: "context_optimizing",
- Status: "completed",
- Progress: 100,
- Content: "Context trimmed (summary unavailable)",
- }:
- default:
- }
- }
- }
- } else {
- // No cached summary and no config - just notify completion
- if writeChan != nil {
- select {
- case writeChan <- models.ServerMessage{
- Type: "context_optimizing",
- Status: "completed",
- Progress: 100,
- Content: "Context trimmed",
- }:
- default:
- }
- }
- }
-
- // Build optimized context
- result := make([]map[string]interface{}, 0)
-
- // Add system message first
- if systemMsg != nil {
- result = append(result, systemMsg)
- }
-
- // Add summary as a system context message
- if summaryText != "" {
- summaryMsg := map[string]interface{}{
- "role": "system",
- "content": fmt.Sprintf(`[Conversation Context Summary - %d earlier messages]
-%s
-
-[End of summary - continuing with recent messages]`, len(oldMessages), summaryText),
- }
- result = append(result, summaryMsg)
- } else {
- // Fallback: just note that context was trimmed
- summaryMsg := map[string]interface{}{
- "role": "system",
- "content": fmt.Sprintf("[Note: %d earlier messages were condensed. Recent conversation continues below.]", len(oldMessages)),
- }
- result = append(result, summaryMsg)
- }
-
- // Add recent messages
- result = append(result, recentMessages...)
-
- newTokens := estimateMessagesTokens(result)
- log.Printf("📉 [CONTEXT] Reduced from %d to %d tokens (kept %d messages + summary)", totalTokens, newTokens, len(recentMessages))
-
- return result
-}
-
-// optimizeContextAfterStream runs context optimization AFTER streaming completes
-// This is called asynchronously so it doesn't block the user experience
-func (s *ChatService) optimizeContextAfterStream(userConn *models.UserConnection) {
- // Recover from panics (user may disconnect)
- defer func() {
- if r := recover(); r != nil {
- log.Printf("⚠️ [CONTEXT] Recovered from panic during post-stream optimization: %v", r)
- }
- }()
-
- // Get current messages from cache
- messages := s.getConversationMessages(userConn.ConversationID)
- totalTokens := estimateMessagesTokens(messages)
-
- // Check if optimization is needed
- if totalTokens <= SummarizationThreshold {
- log.Printf("📊 [CONTEXT] Post-stream check: %d tokens, no optimization needed (threshold: %d)",
- totalTokens, SummarizationThreshold)
- return
- }
-
- log.Printf("📊 [CONTEXT] Post-stream optimization starting: %d tokens exceeds %d threshold",
- totalTokens, SummarizationThreshold)
-
- // Get config for summarization API call
- config, err := s.GetEffectiveConfig(userConn, userConn.ModelID)
- if err != nil {
- log.Printf("❌ [CONTEXT] Failed to get config for optimization: %v", err)
- return
- }
-
- // Run the optimization (this will send UI notifications via WriteChan)
- optimizedMessages := s.optimizeContextWindow(messages, userConn.ConversationID, config, userConn.WriteChan)
-
- // Save optimized messages back to cache
- s.setConversationMessages(userConn.ConversationID, optimizedMessages)
-
- log.Printf("✅ [CONTEXT] Post-stream optimization complete for %s", userConn.ConversationID)
-}
-
-// checkAndTriggerMemoryExtraction checks if memory extraction threshold is reached
-// This is called asynchronously after each assistant message
-func (s *ChatService) checkAndTriggerMemoryExtraction(userConn *models.UserConnection) {
- // Recover from panics
- defer func() {
- if r := recover(); r != nil {
- log.Printf("⚠️ [MEMORY] Recovered from panic during memory extraction check: %v", r)
- }
- }()
-
- // Get user preferences to check if memory is enabled and get threshold
- ctx := context.Background()
- user, err := s.userService.GetUserBySupabaseID(ctx, userConn.UserID)
- if err != nil {
- log.Printf("⚠️ [MEMORY] Failed to get user preferences: %v", err)
- return
- }
-
- // Check if memory system is enabled for this user
- if !user.Preferences.MemoryEnabled {
- return // Memory system disabled, skip extraction
- }
-
- // Get user's configured threshold (default to 20 if not set)
- threshold := user.Preferences.MemoryExtractionThreshold
- if threshold <= 0 {
- threshold = 20 // Default to 20 messages (conservative)
- }
-
- // Get current messages from cache
- messages := s.getConversationMessages(userConn.ConversationID)
- messageCount := len(messages)
-
- // Check if threshold reached (message count is multiple of threshold)
- if messageCount > 0 && messageCount%threshold == 0 {
- log.Printf("🧠 [MEMORY] Threshold reached (%d messages), enqueuing extraction job for conversation %s",
- messageCount, userConn.ConversationID)
-
- // Get recent messages (last 'threshold' messages for extraction)
- startIndex := messageCount - threshold
- if startIndex < 0 {
- startIndex = 0
- }
- recentMessages := messages[startIndex:]
-
- // Enqueue extraction job (non-blocking)
- ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
- defer cancel()
-
- err := s.memoryExtractionService.EnqueueExtraction(
- ctx,
- userConn.UserID,
- userConn.ConversationID,
- recentMessages,
- )
- if err != nil {
- log.Printf("⚠️ [MEMORY] Failed to enqueue extraction job: %v", err)
- } else {
- log.Printf("✅ [MEMORY] Extraction job enqueued successfully")
- }
- }
-}
-
-// SetConversationMessages stores messages in cache with TTL (public)
-func (s *ChatService) SetConversationMessages(conversationID string, messages []map[string]interface{}) {
- s.setConversationMessages(conversationID, messages)
-}
-
-// clearConversation removes all messages for a conversation (internal)
-func (s *ChatService) clearConversation(conversationID string) {
- s.conversationCache.Delete(conversationID)
- log.Printf("🗑️ [CACHE] Cleared conversation %s", conversationID)
-}
-
-// ClearConversation removes all messages for a conversation (public)
-func (s *ChatService) ClearConversation(conversationID string) {
- s.clearConversation(conversationID)
-}
-
-// CreateConversation creates a new conversation in the database with ownership tracking
-func (s *ChatService) CreateConversation(conversationID, userID, title string) error {
- _, err := s.db.Exec(`
- INSERT INTO conversations (id, user_id, title, expires_at)
- VALUES (?, ?, ?, DATE_ADD(NOW(), INTERVAL 30 MINUTE))
- ON DUPLICATE KEY UPDATE
- last_activity_at = NOW(),
- expires_at = DATE_ADD(NOW(), INTERVAL 30 MINUTE)
- `, conversationID, userID, title)
-
- if err != nil {
- return fmt.Errorf("failed to create conversation: %w", err)
- }
-
- log.Printf("📝 [OWNERSHIP] Created conversation %s for user %s", conversationID, userID)
- return nil
-}
-
-// IsConversationOwner checks if a user owns a specific conversation
-func (s *ChatService) IsConversationOwner(conversationID, userID string) bool {
- var ownerID string
- err := s.db.QueryRow("SELECT user_id FROM conversations WHERE id = ?", conversationID).Scan(&ownerID)
-
- if err != nil {
- // Conversation doesn't exist in database - allow access (for backward compatibility with cache-only conversations)
- log.Printf("⚠️ [OWNERSHIP] Conversation %s not in database, allowing access", conversationID)
- return true
- }
-
- isOwner := ownerID == userID
- if !isOwner {
- log.Printf("🚫 [OWNERSHIP] User %s denied access to conversation %s (owned by %s)", userID, conversationID, ownerID)
- }
-
- return isOwner
-}
-
-// UpdateConversationActivity updates the last activity timestamp and extends expiration
-func (s *ChatService) UpdateConversationActivity(conversationID string) error {
- _, err := s.db.Exec(`
- UPDATE conversations
- SET last_activity_at = NOW(),
- expires_at = DATE_ADD(NOW(), INTERVAL 30 MINUTE),
- updated_at = NOW()
- WHERE id = ?
- `, conversationID)
-
- if err != nil {
- return fmt.Errorf("failed to update conversation activity: %w", err)
- }
-
- return nil
-}
-
-// DeleteAllConversationsByUser deletes all conversations for a specific user (for GDPR compliance)
-func (s *ChatService) DeleteAllConversationsByUser(userID string) error {
- result, err := s.db.Exec("DELETE FROM conversations WHERE user_id = ?", userID)
- if err != nil {
- return fmt.Errorf("failed to delete user conversations: %w", err)
- }
-
- rowsAffected, _ := result.RowsAffected()
- log.Printf("🗑️ [GDPR] Deleted %d conversations for user %s", rowsAffected, userID)
-
- return nil
-}
-
-// GetAllConversationsByUser retrieves all conversations for a user (for GDPR data export)
-func (s *ChatService) GetAllConversationsByUser(userID string) ([]map[string]interface{}, error) {
- rows, err := s.db.Query(`
- SELECT id, title, created_at, updated_at, last_activity_at, expires_at
- FROM conversations
- WHERE user_id = ?
- ORDER BY created_at DESC
- `, userID)
-
- if err != nil {
- return nil, fmt.Errorf("failed to query conversations: %w", err)
- }
- defer rows.Close()
-
- conversations := make([]map[string]interface{}, 0)
-
- for rows.Next() {
- var id, title, createdAt, updatedAt, lastActivityAt, expiresAt string
-
- if err := rows.Scan(&id, &title, &createdAt, &updatedAt, &lastActivityAt, &expiresAt); err != nil {
- log.Printf("⚠️ Failed to scan conversation: %v", err)
- continue
- }
-
- // Get messages from cache if available
- messages := s.getConversationMessages(id)
-
- conversation := map[string]interface{}{
- "id": id,
- "title": title,
- "created_at": createdAt,
- "updated_at": updatedAt,
- "last_activity_at": lastActivityAt,
- "expires_at": expiresAt,
- "message_count": len(messages),
- "messages": messages,
- }
-
- conversations = append(conversations, conversation)
- }
-
- return conversations, nil
-}
-
-// ConversationStatus holds status information about a conversation
-type ConversationStatus struct {
- Exists bool `json:"exists"`
- HasFiles bool `json:"hasFiles"`
- ExpiresIn int64 `json:"expiresIn"` // seconds until expiration, -1 if expired
-}
-
-// GetConversationStatus checks if a conversation exists and when it expires
-func (s *ChatService) GetConversationStatus(conversationID string) *ConversationStatus {
- status := &ConversationStatus{
- Exists: false,
- HasFiles: false,
- ExpiresIn: -1,
- }
-
- // Check if conversation exists in cache
- if _, expiration, found := s.conversationCache.GetWithExpiration(conversationID); found {
- status.Exists = true
-
- // Calculate time until expiration
- if !expiration.IsZero() {
- timeUntilExpiration := time.Until(expiration)
- status.ExpiresIn = int64(timeUntilExpiration.Seconds())
- }
-
- // Check if conversation has files
- fileCache := GetFileCacheService()
- fileIDs := fileCache.GetConversationFiles(conversationID)
- status.HasFiles = len(fileIDs) > 0
-
- log.Printf("📊 [STATUS] Conversation %s: exists=%v, hasFiles=%v, expiresIn=%ds",
- conversationID, status.Exists, status.HasFiles, status.ExpiresIn)
- } else {
- log.Printf("📊 [STATUS] Conversation %s: not found in cache", conversationID)
- }
-
- return status
-}
-
-// AddUserMessage adds a user message to the conversation cache
-func (s *ChatService) AddUserMessage(conversationID string, content interface{}) {
- messages := s.getConversationMessages(conversationID)
-
- // 🔍 DIAGNOSTIC: Log messages retrieved before adding new one
- log.Printf("🔍 [ADD-USER] Retrieved %d messages from cache for conversation %s",
- len(messages), conversationID)
-
- messages = append(messages, map[string]interface{}{
- "role": "user",
- "content": content,
- })
-
- // 🔍 DIAGNOSTIC: Log messages after adding new user message
- log.Printf("🔍 [ADD-USER] After append: %d messages (added 1 user message)", len(messages))
-
- s.setConversationMessages(conversationID, messages)
-}
-
-// hasImageAttachments checks if messages contain any image attachments
-func (s *ChatService) hasImageAttachments(messages []map[string]interface{}) bool {
- for _, msg := range messages {
- content := msg["content"]
- if content == nil {
- continue
- }
-
- // Try []interface{} first (generic slice)
- if contentArr, ok := content.([]interface{}); ok {
- for _, part := range contentArr {
- if partMap, ok := part.(map[string]interface{}); ok {
- if partType, ok := partMap["type"].(string); ok && partType == "image_url" {
- log.Printf("🖼️ [VISION-CHECK] Found image_url in []interface{} content")
- return true
- }
- }
- }
- }
-
- // Try []map[string]interface{} (typed slice - this is what websocket handler creates)
- if contentArr, ok := content.([]map[string]interface{}); ok {
- for _, part := range contentArr {
- if partType, ok := part["type"].(string); ok && partType == "image_url" {
- log.Printf("🖼️ [VISION-CHECK] Found image_url in []map[string]interface{} content")
- return true
- }
- }
- }
- }
- return false
-}
-
-// modelSupportsVision checks if a model supports vision/image inputs
-func (s *ChatService) modelSupportsVision(modelID string) bool {
- // First check if it's an alias and get the actual model info
- configService := GetConfigService()
- var actualModelName string
-
- for providerID, aliases := range s.modelAliases {
- for aliasKey, aliasValue := range aliases {
- if aliasKey == modelID {
- // Found as alias - check the alias config for vision support
- aliasInfo := configService.GetModelAlias(providerID, aliasKey)
- if aliasInfo != nil && aliasInfo.SupportsVision != nil {
- log.Printf("📊 [VISION CHECK] Alias '%s' has explicit vision support: %v", modelID, *aliasInfo.SupportsVision)
- return *aliasInfo.SupportsVision
- }
- // If not explicitly set in alias, use actual model name for DB lookup
- actualModelName = aliasValue
- log.Printf("📊 [VISION CHECK] Alias '%s' -> actual model '%s' (no explicit vision setting)", modelID, actualModelName)
- break
- }
- }
- if actualModelName != "" {
- break
- }
- }
-
- // Use actual model name if found via alias, otherwise use the provided modelID
- queryModelName := modelID
- if actualModelName != "" {
- queryModelName = actualModelName
- }
-
- // Check database for model's vision support
- var supportsVision int
- err := s.db.QueryRow("SELECT supports_vision FROM models WHERE id = ? OR name = ?", queryModelName, queryModelName).Scan(&supportsVision)
- if err != nil {
- // Model not found - assume it doesn't support vision (safer approach)
- log.Printf("📊 [VISION CHECK] Model '%s' not found in database - assuming no vision support", queryModelName)
- return false
- }
-
- result := supportsVision == 1
- log.Printf("📊 [VISION CHECK] Model '%s' supports_vision=%v", queryModelName, result)
- return result
-}
-
-// findVisionCapableModel finds a vision-capable model to use as fallback
-// Returns (providerID, modelName, found)
-func (s *ChatService) findVisionCapableModel() (int, string, bool) {
- // First, check aliases for vision-capable models (preferred)
- configService := GetConfigService()
- allAliases := configService.GetAllModelAliases()
-
- for providerID, aliases := range allAliases {
- for aliasKey, aliasInfo := range aliases {
- if aliasInfo.SupportsVision != nil && *aliasInfo.SupportsVision {
- log.Printf("🔍 [VISION FALLBACK] Found vision-capable alias: %s (provider %d)", aliasKey, providerID)
- return providerID, aliasKey, true
- }
- }
- }
-
- // Query database for any vision-capable model
- var providerID int
- var modelName string
- err := s.db.QueryRow(`
- SELECT m.provider_id, m.name
- FROM models m
- JOIN providers p ON m.provider_id = p.id
- WHERE m.supports_vision = 1 AND m.is_visible = 1 AND p.enabled = 1
- ORDER BY m.provider_id ASC
- LIMIT 1
- `).Scan(&providerID, &modelName)
-
- if err != nil {
- log.Printf("⚠️ [VISION FALLBACK] No vision-capable model found in database")
- return 0, "", false
- }
-
- log.Printf("🔍 [VISION FALLBACK] Found vision-capable model: %s (provider %d)", modelName, providerID)
- return providerID, modelName, true
-}
-
-// modelSupportsTools checks if a model supports tools (returns true if unknown - optimistic approach)
-func (s *ChatService) modelSupportsTools(modelID string) bool {
- log.Printf("🔍 [REQUEST] Checking if model '%s' supports tools...", modelID)
- log.Printf("🔍 [DB CHECK] Querying database for model: '%s'", modelID)
-
- var supportsTools int
- err := s.db.QueryRow("SELECT supports_tools FROM model_capabilities WHERE model_id = ?", modelID).Scan(&supportsTools)
-
- if err != nil {
- // Model not in database or error, assume it supports tools (optimistic)
- log.Printf("📊 [DB CHECK] Model '%s' NOT FOUND in database - assuming tools supported (optimistic)", modelID)
- return true
- }
-
- result := supportsTools == 1
- log.Printf("📊 [DB CHECK] Model '%s' found in database: supports_tools=%d (returning %v)", modelID, supportsTools, result)
- return result
-}
-
-// markModelNoToolSupport marks a model as not supporting tools
-func (s *ChatService) markModelNoToolSupport(modelID string) error {
- log.Printf("💾 [DB WRITE] Attempting to mark model '%s' as NOT supporting tools", modelID)
-
- result, err := s.db.Exec(
- "REPLACE INTO model_capabilities (model_id, supports_tools) VALUES (?, 0)",
- modelID,
- )
-
- if err != nil {
- log.Printf("❌ [DB WRITE] Failed to mark model as no tool support: %v", err)
- return fmt.Errorf("failed to mark model as no tool support: %v", err)
- }
-
- rowsAffected, _ := result.RowsAffected()
- log.Printf("✅ [DB WRITE] Successfully marked model '%s' as NOT supporting tools (rows affected: %d)", modelID, rowsAffected)
- return nil
-}
-
-// getFreeTierConfig returns the configuration for the free tier model
-// This is used when anonymous users try to access restricted models
-func (s *ChatService) getFreeTierConfig(connID string) (*models.Config, error) {
- // Query for a free tier model
- var freeTierModelID string
- var freeTierModelName string
- var freeTierProviderID int
-
- err := s.db.QueryRow(`
- SELECT id, name, provider_id
- FROM models
- WHERE free_tier = 1 AND is_visible = 1
- LIMIT 1
- `).Scan(&freeTierModelID, &freeTierModelName, &freeTierProviderID)
-
- if err != nil {
- log.Printf("❌ [AUTH] No free tier model configured! Anonymous users cannot use the system.")
- return nil, fmt.Errorf("no free tier model available for anonymous users")
- }
-
- provider, err := s.providerService.GetByID(freeTierProviderID)
- if err != nil || !provider.Enabled {
- log.Printf("❌ [AUTH] Free tier model provider is disabled or not found")
- return nil, fmt.Errorf("free tier provider unavailable")
- }
-
- log.Printf("🔒 [AUTH] Restricting connection %s to free tier model: %s", connID, freeTierModelName)
- return &models.Config{
- BaseURL: provider.BaseURL,
- APIKey: provider.APIKey,
- Model: freeTierModelName,
- }, nil
-}
-
-// GetEffectiveConfig returns the appropriate configuration based on user's selection
-func (s *ChatService) GetEffectiveConfig(userConn *models.UserConnection, modelID string) (*models.Config, error) {
- // Priority 1: User provided their own API key (BYOK - Bring Your Own Key)
- if userConn.CustomConfig != nil {
- if userConn.CustomConfig.BaseURL != "" &&
- userConn.CustomConfig.APIKey != "" &&
- userConn.CustomConfig.Model != "" {
- log.Printf("🔑 [CONFIG] Using BYOK for user %s: model=%s", userConn.ConnID, userConn.CustomConfig.Model)
- return &models.Config{
- BaseURL: userConn.CustomConfig.BaseURL,
- APIKey: userConn.CustomConfig.APIKey,
- Model: userConn.CustomConfig.Model,
- }, nil
- }
-
- // Partial custom config - fall through to use platform providers if incomplete
- log.Printf("⚠️ [CONFIG] Incomplete custom config for user %s, falling back to platform providers", userConn.ConnID)
- }
-
- // Priority 2: User selected a model from platform (uses platform API keys)
- if modelID != "" {
- var providerID int
- var modelName string
- var foundModel bool
-
- // First, check if modelID is a model alias (e.g., "haiku-4.5" -> "glm-4.5-air")
- if aliasProviderID, actualModel, found := s.resolveModelAlias(modelID); found {
- // It's an alias - get the provider directly
- provider, err := s.providerService.GetByID(aliasProviderID)
- if err == nil && provider.Enabled {
- // Check if anonymous user is trying to use non-free-tier model
- if userConn.UserID == "anonymous" {
- // Check if this model is free tier
- var isFreeTier int
- err := s.db.QueryRow(
- "SELECT COALESCE(free_tier, 0) FROM models WHERE id = ?",
- modelID,
- ).Scan(&isFreeTier)
-
- if err != nil || isFreeTier == 0 {
- // Not free tier - redirect to free tier model
- log.Printf("⚠️ [AUTH] Anonymous user %s attempted to use restricted model %s (alias: %s), forcing free tier",
- userConn.ConnID, actualModel, modelID)
- return s.getFreeTierConfig(userConn.ConnID)
- }
- }
-
- log.Printf("🏢 [CONFIG] Using aliased model for user %s: alias=%s, actual_model=%s, provider=%s",
- userConn.ConnID, modelID, actualModel, provider.Name)
-
- return &models.Config{
- BaseURL: provider.BaseURL,
- APIKey: provider.APIKey,
- Model: actualModel,
- }, nil
- }
- }
-
- // Not an alias, try to find in database by model ID
- err := s.db.QueryRow(
- "SELECT provider_id, name FROM models WHERE id = ? AND is_visible = 1",
- modelID,
- ).Scan(&providerID, &modelName)
-
- if err == nil {
- foundModel = true
- }
-
- if foundModel {
- // Check if anonymous user is trying to use non-free-tier model
- if userConn.UserID == "anonymous" {
- var isFreeTier int
- err := s.db.QueryRow(
- "SELECT COALESCE(free_tier, 0) FROM models WHERE id = ? AND is_visible = 1",
- modelID,
- ).Scan(&isFreeTier)
-
- if err != nil || isFreeTier == 0 {
- // Not free tier - redirect to free tier model
- log.Printf("⚠️ [AUTH] Anonymous user %s attempted to use restricted model %s, forcing free tier",
- userConn.ConnID, modelName)
- return s.getFreeTierConfig(userConn.ConnID)
- }
- }
-
- provider, err := s.providerService.GetByID(providerID)
- if err == nil && provider.Enabled {
- // Resolve model name using aliases (if configured)
- actualModelName := s.resolveModelName(providerID, modelName)
-
- if actualModelName != modelName {
- log.Printf("🏢 [CONFIG] Using platform model for user %s: frontend_model=%s, actual_model=%s, provider=%s",
- userConn.ConnID, modelName, actualModelName, provider.Name)
- } else {
- log.Printf("🏢 [CONFIG] Using platform model for user %s: model=%s, provider=%s",
- userConn.ConnID, modelName, provider.Name)
- }
-
- return &models.Config{
- BaseURL: provider.BaseURL,
- APIKey: provider.APIKey,
- Model: actualModelName, // Use resolved model name
- }, nil
- }
- }
- }
-
- // Priority 3: Fallback to first enabled provider with visible models
- log.Printf("⚙️ [CONFIG] No model selected, using fallback for user %s", userConn.ConnID)
-
- // Get first enabled provider
- var providerID int
- var providerName, baseURL, apiKey string
- err := s.db.QueryRow(`
- SELECT id, name, base_url, api_key
- FROM providers
- WHERE enabled = 1
- ORDER BY id ASC
- LIMIT 1
- `).Scan(&providerID, &providerName, &baseURL, &apiKey)
-
- if err != nil {
- return nil, fmt.Errorf("no enabled providers found: %w", err)
- }
-
- // Get first visible model from this provider
- var modelName string
- err = s.db.QueryRow(`
- SELECT name
- FROM models
- WHERE provider_id = ? AND is_visible = 1
- ORDER BY id ASC
- LIMIT 1
- `, providerID).Scan(&modelName)
-
- if err != nil {
- return nil, fmt.Errorf("no visible models found for provider %s: %w", providerName, err)
- }
-
- log.Printf("🔄 [CONFIG] Fallback using provider=%s, model=%s for user %s", providerName, modelName, userConn.ConnID)
-
- return &models.Config{
- BaseURL: baseURL,
- APIKey: apiKey,
- Model: modelName,
- }, nil
-}
-
-// StreamChatCompletion streams chat completion responses
-func (s *ChatService) StreamChatCompletion(userConn *models.UserConnection) error {
- config, err := s.GetEffectiveConfig(userConn, userConn.ModelID)
- if err != nil {
- return fmt.Errorf("failed to get config: %w", err)
- }
-
- // Get messages from cache instead of userConn.Messages
- messages := s.getConversationMessages(userConn.ConversationID)
-
- // 🖼️ Auto-switch to vision model if images are present but current model doesn't support vision
- if s.hasImageAttachments(messages) && !s.modelSupportsVision(userConn.ModelID) {
- log.Printf("🖼️ [VISION] Images detected but model '%s' doesn't support vision - finding fallback", userConn.ModelID)
-
- if fallbackProviderID, fallbackModel, found := s.findVisionCapableModel(); found {
- // Get the provider config for the fallback model
- provider, err := s.providerService.GetByID(fallbackProviderID)
- if err == nil && provider.Enabled {
- // Check if fallback is an alias
- if aliasProviderID, actualModel, isAlias := s.resolveModelAlias(fallbackModel); isAlias {
- aliasProvider, err := s.providerService.GetByID(aliasProviderID)
- if err == nil && aliasProvider.Enabled {
- config = &models.Config{
- BaseURL: aliasProvider.BaseURL,
- APIKey: aliasProvider.APIKey,
- Model: actualModel,
- }
- log.Printf("🖼️ [VISION] Silently switched to vision model: %s (alias for %s)", fallbackModel, actualModel)
- }
- } else {
- config = &models.Config{
- BaseURL: provider.BaseURL,
- APIKey: provider.APIKey,
- Model: fallbackModel,
- }
- log.Printf("🖼️ [VISION] Silently switched to vision model: %s", fallbackModel)
- }
- }
- } else {
- log.Printf("⚠️ [VISION] No vision-capable model available - proceeding with current model (may fail)")
- }
- }
-
- // 🔍 DIAGNOSTIC: Log messages retrieved from cache for streaming
- log.Printf("🔍 [STREAM] Retrieved %d messages from cache for conversation %s",
- len(messages), userConn.ConversationID)
- if len(messages) > 0 {
- // Count message types
- systemCount, userCount, assistantCount := 0, 0, 0
- for _, msg := range messages {
- if role, ok := msg["role"].(string); ok {
- switch role {
- case "system":
- systemCount++
- case "user":
- userCount++
- case "assistant":
- assistantCount++
- }
- }
- }
- log.Printf("🔍 [STREAM] Message breakdown BEFORE system prompt: system=%d, user=%d, assistant=%d",
- systemCount, userCount, assistantCount)
- }
-
- // ═══════════════════════════════════════════════════════════════════════════
- // TOOL SELECTION - Must happen BEFORE system prompt to determine ask_user inclusion
- // ═══════════════════════════════════════════════════════════════════════════
- var tools []map[string]interface{}
- if userConn.DisableTools {
- log.Printf("🔒 [REQUEST] TOOLS DISABLED by client (agent builder mode)")
- } else if s.modelSupportsTools(config.Model) {
- // Get credential-filtered tools for user (only tools they have credentials for)
- credentialFilteredTools := []map[string]interface{}{}
- if s.toolService != nil {
- credentialFilteredTools = s.toolService.GetAvailableTools(context.Background(), userConn.UserID)
- } else {
- // Fallback: Get ALL user tools (built-in + MCP tools) without filtering
- credentialFilteredTools = s.toolRegistry.GetUserTools(userConn.UserID)
- }
-
- log.Printf("📦 [REQUEST] Credential-filtered tools: %d", len(credentialFilteredTools))
-
- // Use tool predictor to select subset of tools if available
- if s.toolPredictorService != nil && len(credentialFilteredTools) > 0 {
- // Extract user message from messages array (last user message)
- userMessage := extractLastUserMessage(messages)
-
- log.Printf("🤖 [TOOL-PREDICTOR] Starting tool prediction with conversation history (%d messages)...", len(messages))
- predictedTools, err := s.toolPredictorService.PredictTools(
- context.Background(),
- userConn.UserID,
- userMessage,
- credentialFilteredTools,
- messages, // Pass full conversation history for context-aware tool selection
- )
-
- if err != nil {
- log.Printf("⚠️ [TOOL-PREDICTOR] Prediction failed: %v, falling back to all tools", err)
- tools = credentialFilteredTools // Graceful fallback
- } else {
- log.Printf("✅ [TOOL-PREDICTOR] Using predicted tools: %d selected", len(predictedTools))
- tools = predictedTools
- }
- } else {
- if s.toolPredictorService == nil {
- log.Printf("📦 [REQUEST] Tool predictor not initialized, using all filtered tools")
- }
- tools = credentialFilteredTools
- }
-
- // Log MCP connection status
- if s.mcpBridge != nil && s.mcpBridge.IsUserConnected(userConn.UserID) {
- builtinCount := s.toolRegistry.Count()
- mcpCount := s.toolRegistry.CountUserTools(userConn.UserID) - builtinCount
- log.Printf("📦 [REQUEST] INCLUDING TOOLS for model: %s (built-in: %d, MCP: %d, selected: %d)",
- config.Model, builtinCount, mcpCount, len(tools))
- } else {
- log.Printf("📦 [REQUEST] INCLUDING TOOLS for model: %s (selected tools: %d)", config.Model, len(tools))
- }
- } else {
- log.Printf("🚫 [REQUEST] EXCLUDING TOOLS for model: %s (marked as incompatible)", config.Model)
- }
-
- // Get system prompt - include ask_user instructions only if tools are available
- // This prevents models like Gemini from failing with MALFORMED_FUNCTION_CALL
- includeAskUser := len(tools) > 0
- systemPrompt := s.GetSystemPrompt(userConn, includeAskUser)
-
- // Inject available images context if there are images in this conversation
- imageRegistry := GetImageRegistryService()
- if imageRegistry.HasImages(userConn.ConversationID) {
- imageContext := imageRegistry.BuildSystemContext(userConn.ConversationID)
- if imageContext != "" {
- systemPrompt = systemPrompt + "\n\n" + imageContext
- log.Printf("📸 [SYSTEM] Injected image context for conversation %s", userConn.ConversationID)
- }
- }
-
- messages = s.buildMessagesWithSystemPrompt(systemPrompt, messages)
-
- // Note: Context optimization now happens AFTER streaming ends (in processStream)
- // This prevents blocking the response while the user waits
-
- // Prepare chat request
- chatReq := models.ChatRequest{
- Model: config.Model,
- Messages: messages,
- Stream: true,
- Temperature: 0.7,
- }
-
- // Only include tools if non-empty (some APIs reject empty tools array)
- if len(tools) > 0 {
- chatReq.Tools = tools
- }
-
- reqBody, err := json.Marshal(chatReq)
- if err != nil {
- return fmt.Errorf("failed to marshal request: %w", err)
- }
-
- // 🔍 DIAGNOSTIC: Log exactly what's being sent to LLM
- log.Printf("🔍 [LLM-REQUEST] Sending to LLM - Model: %s, Messages: %d, Tools: %d",
- chatReq.Model, len(chatReq.Messages), len(chatReq.Tools))
- log.Printf("🔍 [LLM-REQUEST] Request body size: %d bytes", len(reqBody))
-
- // 📋 Print the FULL JSON payload being sent to LLM
- prettyJSON, _ := json.MarshalIndent(chatReq, "", " ")
- log.Printf("📋 [LLM-REQUEST] FULL JSON PAYLOAD:\n%s", string(prettyJSON))
-
- // Log all messages with FULL content for the user message
- if len(chatReq.Messages) > 0 {
- log.Printf("🔍 [LLM-REQUEST] === ALL MESSAGES BEING SENT TO LLM ===")
- for i, msg := range chatReq.Messages {
- role, _ := msg["role"].(string)
- contentStr := ""
-
- // Handle different content types
- if content, ok := msg["content"].(string); ok {
- contentStr = content
- } else if contentArr, ok := msg["content"].([]interface{}); ok {
- // Multi-part content (vision models)
- for j, part := range contentArr {
- if partMap, ok := part.(map[string]interface{}); ok {
- partType, _ := partMap["type"].(string)
- if partType == "text" {
- if text, ok := partMap["text"].(string); ok {
- contentStr += fmt.Sprintf("[Part %d - text]: %s\n", j, text)
- }
- } else if partType == "image_url" {
- contentStr += fmt.Sprintf("[Part %d - image_url]: \n", j)
- }
- }
- }
- }
-
- toolCallID, _ := msg["tool_call_id"].(string)
- toolName, _ := msg["name"].(string)
-
- log.Printf("━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━")
- if role == "tool" {
- log.Printf("📨 [MSG %d] role=%s, tool_call_id=%s, name=%s", i, role, toolCallID, toolName)
- // Truncate tool responses for readability
- if len(contentStr) > 500 {
- log.Printf(" content (truncated): %s...", contentStr[:500])
- } else {
- log.Printf(" content: %s", contentStr)
- }
- } else if role == "user" {
- // Show FULL user message content (includes injected CSV context)
- log.Printf("👤 [MSG %d] role=%s", i, role)
- log.Printf(" FULL CONTENT:\n%s", contentStr)
- } else if role == "system" {
- log.Printf("⚙️ [MSG %d] role=%s", i, role)
- if len(contentStr) > 200 {
- log.Printf(" content (truncated): %s...", contentStr[:200])
- } else {
- log.Printf(" content: %s", contentStr)
- }
- } else if role == "assistant" {
- log.Printf("🤖 [MSG %d] role=%s", i, role)
- if len(contentStr) > 300 {
- log.Printf(" content (truncated): %s...", contentStr[:300])
- } else {
- log.Printf(" content: %s", contentStr)
- }
- // Log tool calls if present
- if toolCalls, ok := msg["tool_calls"].([]interface{}); ok && len(toolCalls) > 0 {
- log.Printf(" tool_calls: %d calls", len(toolCalls))
- for _, tc := range toolCalls {
- if tcMap, ok := tc.(map[string]interface{}); ok {
- if fn, ok := tcMap["function"].(map[string]interface{}); ok {
- fnName, _ := fn["name"].(string)
- fnArgs, _ := fn["arguments"].(string)
- log.Printf(" - %s(%s)", fnName, fnArgs)
- }
- }
- }
- }
- } else {
- log.Printf("❓ [MSG %d] role=%s, content=%s", i, role, contentStr)
- }
- }
- log.Printf("━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━")
- log.Printf("🔍 [LLM-REQUEST] === END OF MESSAGES ===")
- }
-
- // Create HTTP request
- req, err := http.NewRequest("POST", config.BaseURL+"/chat/completions", bytes.NewBuffer(reqBody))
- if err != nil {
- return fmt.Errorf("failed to create request: %w", err)
- }
-
- req.Header.Set("Content-Type", "application/json")
- req.Header.Set("Authorization", "Bearer "+config.APIKey)
-
- // Send request
- client := &http.Client{Timeout: 120 * time.Second}
- resp, err := client.Do(req)
- if err != nil {
- return fmt.Errorf("request failed: %w", err)
- }
- defer resp.Body.Close()
-
- if resp.StatusCode != http.StatusOK {
- body, _ := io.ReadAll(resp.Body)
- errorMsg := string(body)
-
- log.Printf("⚠️ [API ERROR] API Error for %s: %s", userConn.ConnID, errorMsg)
-
- // Check if error is due to tool incompatibility
- if len(tools) > 0 && s.detectToolIncompatibility(errorMsg) {
- log.Printf("🔍 [ERROR DETECTION] Tool incompatibility detected for model: %s", config.Model)
-
- // Mark model as not supporting tools
- if err := s.markModelNoToolSupport(config.Model); err != nil {
- log.Printf("⚠️ [ERROR DETECTION] Failed to mark model: %v", err)
- }
-
- // Add assistant error message to maintain alternation
- messages := s.getConversationMessages(userConn.ConversationID)
- errorMsgText := "I encountered an error. This model doesn't support tool calling. Tools have been disabled for future requests."
- messages = append(messages, map[string]interface{}{
- "role": "assistant",
- "content": errorMsgText,
- })
- s.setConversationMessages(userConn.ConversationID, messages)
- log.Printf("✅ [ERROR DETECTION] Added assistant error message to cache to maintain alternation")
-
- // Inform user about the error
- userConn.WriteChan <- models.ServerMessage{
- Type: "error",
- ErrorCode: "model_tool_incompatible",
- ErrorMessage: fmt.Sprintf("Model '%s' doesn't support tool calling. Tools will be automatically disabled for this model on the next message.", config.Model),
- }
-
- // Retry WITHOUT tools
- log.Printf("🔄 [ERROR DETECTION] Retrying request WITHOUT tools for model: %s", config.Model)
- return s.StreamChatCompletion(userConn)
- }
-
- return fmt.Errorf("API error (status %d): %s", resp.StatusCode, errorMsg)
- }
-
- // Process SSE stream
- return s.processStream(resp.Body, userConn)
-}
-
-// detectToolIncompatibility checks if an error message indicates tool incompatibility
-func (s *ChatService) detectToolIncompatibility(errorMsg string) bool {
- errorLower := strings.ToLower(errorMsg)
-
- // Common error patterns for tool incompatibility
- patterns := []string{
- "roles must alternate",
- "tool",
- "not supported",
- "function calling",
- "unsupported",
- }
-
- // Check if error contains patterns related to tools
- hasToolKeyword := false
- hasErrorKeyword := false
-
- for _, pattern := range patterns {
- if strings.Contains(errorLower, pattern) {
- if pattern == "tool" || pattern == "function calling" {
- hasToolKeyword = true
- } else {
- hasErrorKeyword = true
- }
- }
- }
-
- // Must have both a tool-related keyword AND an error keyword
- result := hasToolKeyword && hasErrorKeyword
-
- if result {
- log.Printf("🔍 [ERROR DETECTION] Tool incompatibility pattern detected in error: %s", errorMsg)
- }
-
- return result || strings.Contains(errorLower, "roles must alternate")
-}
-
-// ToolCallAccumulator accumulates streaming tool call data
-type ToolCallAccumulator struct {
- ID string
- Type string
- Name string
- Arguments strings.Builder
-}
-
-// safeSendChunk sends a chunk to the client with graceful error handling
-// This prevents panics if the channel is closed (client disconnected)
-func (s *ChatService) safeSendChunk(userConn *models.UserConnection, content string) {
- defer func() {
- if r := recover(); r != nil {
- log.Printf("⚠️ [STREAM] Recovered from WriteChan panic for %s: %v (chunk buffered)", userConn.ConnID, r)
- // Chunk is already buffered in streamBuffer, so no data loss
- }
- }()
-
- select {
- case userConn.WriteChan <- models.ServerMessage{
- Type: "stream_chunk",
- Content: content,
- }:
- // Successfully sent
- case <-time.After(100 * time.Millisecond):
- // Channel backpressure detected - client rendering slower than generation
- bufferLen := len(userConn.WriteChan)
- log.Printf("⚠️ [STREAM] WriteChan backpressure for %s (buffer: %d/100), chunk buffered for resume",
- userConn.ConnID, bufferLen)
-
- // Chunk is already buffered in streamBuffer via AppendChunk before this call
- // If backpressure persists, client may need to reconnect and resume
- }
-}
-
-// processStream processes the SSE stream from the AI provider
-func (s *ChatService) processStream(reader io.Reader, userConn *models.UserConnection) error {
- scanner := bufio.NewScanner(reader)
-
- // Increase buffer to 1MB for large SSE chunks (default is 64KB)
- // Prevents "bufio.Scanner: token too long" errors with large tool call arguments
- const maxCapacity = 1024 * 1024 // 1MB
- buf := make([]byte, maxCapacity)
- scanner.Buffer(buf, maxCapacity)
-
- var fullContent strings.Builder
-
- // Create stream buffer for this conversation (for resume capability)
- s.streamBuffer.CreateBuffer(userConn.ConversationID, userConn.UserID, userConn.ConnID)
- log.Printf("📦 [STREAM] Buffer created for conversation %s", userConn.ConversationID)
-
- // Track tool calls by index to accumulate streaming arguments
- toolCallsMap := make(map[int]*ToolCallAccumulator)
- var finishReason string
-
- for scanner.Scan() {
- select {
- case <-userConn.StopChan:
- log.Printf("⏹️ Generation stopped for %s", userConn.ConnID)
- // Clear buffer on stop - user explicitly cancelled
- s.streamBuffer.ClearBuffer(userConn.ConversationID)
- return nil
- default:
- }
-
- line := scanner.Text()
- if !strings.HasPrefix(line, "data: ") {
- continue
- }
-
- data := strings.TrimPrefix(line, "data: ")
- if data == "[DONE]" {
- break
- }
-
- var chunk map[string]interface{}
- if err := json.Unmarshal([]byte(data), &chunk); err != nil {
- continue
- }
-
- choices, ok := chunk["choices"].([]interface{})
- if !ok || len(choices) == 0 {
- continue
- }
-
- choice := choices[0].(map[string]interface{})
- delta, ok := choice["delta"].(map[string]interface{})
- if !ok {
- continue
- }
-
- // Check for finish reason
- if reason, ok := choice["finish_reason"].(string); ok && reason != "" {
- finishReason = reason
- }
-
- // Handle reasoning/thinking content (o1/o3 models)
- if reasoningContent, ok := delta["reasoning_content"].(string); ok {
- userConn.WriteChan <- models.ServerMessage{
- Type: "reasoning_chunk",
- Content: reasoningContent,
- }
- }
-
- // Handle content chunks
- if content, ok := delta["content"].(string); ok {
- fullContent.WriteString(content)
-
- // Buffer chunk for potential resume (always buffer, even if send succeeds)
- s.streamBuffer.AppendChunk(userConn.ConversationID, content)
-
- // Send to client with graceful handling for closed channel
- s.safeSendChunk(userConn, content)
- }
-
- // Handle tool calls - ACCUMULATE, don't execute yet!
- if toolCallsData, ok := delta["tool_calls"].([]interface{}); ok {
- for _, tc := range toolCallsData {
- toolCallChunk := tc.(map[string]interface{})
-
- // Get tool call index
- var index int
- if idx, ok := toolCallChunk["index"].(float64); ok {
- index = int(idx)
- }
-
- // Initialize accumulator if needed
- if _, exists := toolCallsMap[index]; !exists {
- toolCallsMap[index] = &ToolCallAccumulator{}
- }
-
- acc := toolCallsMap[index]
-
- // Accumulate fields
- if id, ok := toolCallChunk["id"].(string); ok {
- acc.ID = truncateToolCallID(id) // Truncate to 40 chars (OpenAI constraint)
- }
- if typ, ok := toolCallChunk["type"].(string); ok {
- acc.Type = typ
- }
- if function, ok := toolCallChunk["function"].(map[string]interface{}); ok {
- if name, ok := function["name"].(string); ok {
- acc.Name = name
- log.Printf("🔧 [TOOL] Starting to accumulate tool call: %s (index %d)", name, index)
- }
- // ✅ ACCUMULATE arguments, don't parse yet!
- if args, ok := function["arguments"].(string); ok {
- acc.Arguments.WriteString(args)
- }
- }
- }
- }
- }
-
- // Execute tools ONLY after streaming completes with tool_calls finish reason
- if finishReason == "tool_calls" {
- log.Printf("🔧 [TOOL] Streaming complete, executing %d tool call(s)", len(toolCallsMap))
-
- // Get messages from cache
- messages := s.getConversationMessages(userConn.ConversationID)
-
- // Build tool call messages for conversation history
- var toolCallMessages []map[string]interface{}
- var toolResults []map[string]interface{}
-
- // Execute all tools and collect results
- for index, acc := range toolCallsMap {
- if acc.Name != "" && acc.Arguments.Len() > 0 {
- argsStr := acc.Arguments.String()
- log.Printf("🔧 [TOOL] Executing tool %s (index %d, args length: %d bytes)", acc.Name, index, len(argsStr))
-
- // Add to tool call messages
- toolCallMessages = append(toolCallMessages, map[string]interface{}{
- "id": acc.ID,
- "type": acc.Type,
- "function": map[string]interface{}{
- "name": acc.Name,
- "arguments": argsStr,
- },
- })
-
- // Execute tool and get result
- result := s.executeToolSyncWithResult(acc.ID, acc.Name, argsStr, userConn)
- toolResults = append(toolResults, map[string]interface{}{
- "role": "tool",
- "tool_call_id": acc.ID,
- "name": acc.Name,
- "content": result,
- })
- }
- }
-
- // Only add assistant message if we have actual tool calls
- if len(toolCallMessages) > 0 {
- assistantMsg := map[string]interface{}{
- "role": "assistant",
- "tool_calls": toolCallMessages,
- }
- // Only include content if it's not empty
- if fullContent.Len() > 0 {
- assistantMsg["content"] = fullContent.String()
- }
- messages = append(messages, assistantMsg)
-
- // Add all tool results
- for _, toolResult := range toolResults {
- messages = append(messages, toolResult)
- }
-
- // Save updated messages to cache
- s.setConversationMessages(userConn.ConversationID, messages)
-
- // Clear buffer for tool calls - a new stream will start
- s.streamBuffer.ClearBuffer(userConn.ConversationID)
-
- // After ALL tools complete, continue conversation ONCE
- log.Printf("🔄 [TOOL] All tools executed, continuing conversation with %d tool result(s)", len(toolCallMessages))
- go s.StreamChatCompletion(userConn)
- } else {
- // No valid tool calls - treat as error
- log.Printf("⚠️ [STREAM] Tool calls detected but none were valid")
- userConn.WriteChan <- models.ServerMessage{
- Type: "error",
- ErrorCode: "invalid_tool_calls",
- ErrorMessage: "The model attempted to call tools but the calls were invalid. Please try again.",
- }
- }
- } else {
- // Regular message without tool calls
- content := fullContent.String()
-
- // Only add assistant message if there's actual content
- if content != "" {
- // Get messages from cache and add assistant response
- messages := s.getConversationMessages(userConn.ConversationID)
- messages = append(messages, map[string]interface{}{
- "role": "assistant",
- "content": content,
- })
- s.setConversationMessages(userConn.ConversationID, messages)
-
- // Mark stream buffer as complete before sending stream_end
- s.streamBuffer.MarkComplete(userConn.ConversationID, content)
- log.Printf("📦 [STREAM] Buffer marked complete for conversation %s", userConn.ConversationID)
-
- // Increment message counter
- userConn.Mutex.Lock()
- userConn.MessageCount++
- currentCount := userConn.MessageCount
- userConn.Mutex.Unlock()
-
- // Send completion message
- userConn.WriteChan <- models.ServerMessage{
- Type: "stream_end",
- ConversationID: userConn.ConversationID,
- }
-
- // Generate title after first user-assistant exchange (2 messages: user + assistant)
- log.Printf("🔍 [TITLE] MessageCount=%d for conversation %s", currentCount, userConn.ConversationID)
- if currentCount == 1 {
- log.Printf("🎯 [TITLE] Triggering title generation for %s", userConn.ConversationID)
- go s.generateConversationTitle(userConn, content)
- } else {
- log.Printf("⏭️ [TITLE] Skipping title generation (MessageCount=%d, need 1)", currentCount)
- }
-
- // 🗜️ Context optimization - runs AFTER streaming ends (non-blocking)
- // This compacts conversation history for the NEXT message
- go s.optimizeContextAfterStream(userConn)
-
- // 🧠 Memory extraction - check if threshold reached (non-blocking)
- if s.memoryExtractionService != nil {
- go s.checkAndTriggerMemoryExtraction(userConn)
- }
- } else {
- // Empty response - log warning and send error to client
- log.Printf("⚠️ [STREAM] Received empty response from API for %s", userConn.ConnID)
- userConn.WriteChan <- models.ServerMessage{
- Type: "error",
- ErrorCode: "empty_response",
- ErrorMessage: "The model returned an empty response. Please try again.",
- }
- }
- }
-
- // Check for scanner errors (e.g., buffer overflow, I/O errors)
- if err := scanner.Err(); err != nil {
- log.Printf("❌ [STREAM] Scanner error for %s: %v", userConn.ConnID, err)
- userConn.WriteChan <- models.ServerMessage{
- Type: "error",
- ErrorCode: "stream_error",
- ErrorMessage: "An error occurred while processing the stream. Please try again.",
- }
- return fmt.Errorf("stream scanner error: %w", err)
- }
-
- return nil
-}
-
-// executeToolSyncWithResult executes a tool call synchronously and returns the result
-func (s *ChatService) executeToolSyncWithResult(toolCallID, toolName, argsJSON string, userConn *models.UserConnection) string {
- // Get tool metadata from registry
- toolDisplayName := toolName
- toolIcon := ""
- toolDescription := ""
- if tool, exists := s.toolRegistry.Get(toolName); exists {
- toolDisplayName = tool.DisplayName
- toolIcon = tool.Icon
- toolDescription = tool.Description
- }
-
- // Parse complete JSON arguments
- var args map[string]interface{}
- if err := json.Unmarshal([]byte(argsJSON), &args); err != nil {
- log.Printf("❌ Failed to parse tool arguments for %s: %v (length: %d bytes)", toolName, err, len(argsJSON))
-
- // Send error to client
- errorMsg := fmt.Sprintf("Failed to parse arguments: %v", err)
- userConn.WriteChan <- models.ServerMessage{
- Type: "tool_result",
- ToolName: toolName,
- ToolDisplayName: toolDisplayName,
- ToolIcon: toolIcon,
- ToolDescription: toolDescription,
- Status: "failed",
- Result: errorMsg,
- }
-
- return fmt.Sprintf("Error: %v", err)
- }
-
- log.Printf("✅ [TOOL] Successfully parsed arguments for %s: %+v", toolName, args)
-
- // Inject user context into args (internal use only, not exposed to AI)
- // This allows tools to access authenticated user info without breaking the tool interface
- args["__user_id__"] = userConn.UserID
- args["__conversation_id__"] = userConn.ConversationID
-
- // Auto-inject credentials for tools that require them
- if s.toolService != nil {
- // Inject credential resolver for secure credential access
- resolver := s.toolService.CreateCredentialResolver(userConn.UserID)
- if resolver != nil {
- args[tools.CredentialResolverKey] = resolver
- }
-
- // Auto-inject credential_id for tools that need it
- credentialID := s.toolService.GetCredentialForTool(context.Background(), userConn.UserID, toolName)
- if credentialID != "" {
- args["credential_id"] = credentialID
- log.Printf("🔐 [CHAT] Auto-injected credential_id=%s for tool=%s", credentialID, toolName)
- }
- }
-
- // Inject user connection and waiter for ask_user tool (interactive prompts)
- if toolName == "ask_user" {
- args[tools.UserConnectionKey] = userConn
- args[tools.PromptWaiterKey] = userConn.PromptWaiter
- log.Printf("🔌 [CHAT] Injected user connection and prompt waiter for ask_user tool")
- }
-
- // Inject image provider config and registry for generate_image tool
- if toolName == "generate_image" {
- imageProviderService := GetImageProviderService()
- provider := imageProviderService.GetProvider()
- if provider != nil {
- args[tools.ImageProviderConfigKey] = &tools.ImageProviderConfig{
- Name: provider.Name,
- BaseURL: provider.BaseURL,
- APIKey: provider.APIKey,
- DefaultModel: provider.DefaultModel,
- }
- log.Printf("🎨 [CHAT] Injected image provider config for generate_image tool (provider: %s)", provider.Name)
- }
- // Inject image registry for registering generated images
- imageRegistry := GetImageRegistryService()
- args[tools.ImageRegistryKey] = &ImageRegistryAdapter{registry: imageRegistry}
-
- // Inject usage limiter for tier-based image generation limits
- if s.usageLimiter != nil {
- args[tools.UsageLimiterKey] = s.usageLimiter
- }
- }
-
- // Inject image edit config and registry for edit_image tool
- if toolName == "edit_image" {
- // Inject image registry adapter for handle lookup (adapter implements tools.ImageRegistryInterface)
- imageRegistry := GetImageRegistryService()
- args[tools.ImageRegistryKey] = &ImageRegistryAdapter{registry: imageRegistry}
-
- // Inject image edit provider config from dedicated edit provider
- imageEditProviderService := GetImageEditProviderService()
- editProvider := imageEditProviderService.GetProvider()
- if editProvider != nil {
- args[tools.ImageEditConfigKey] = &tools.ImageEditConfig{
- BaseURL: editProvider.BaseURL,
- APIKey: editProvider.APIKey,
- }
- log.Printf("🖌️ [CHAT] Injected image edit config for edit_image tool (provider: %s)", editProvider.Name)
- } else {
- log.Printf("⚠️ [CHAT] No image edit provider configured - edit_image tool will fail")
- }
- }
-
- // Inject image registry for describe_image tool (allows using image_id handles)
- if toolName == "describe_image" {
- imageRegistry := GetImageRegistryService()
- args[tools.ImageRegistryKey] = &ImageRegistryAdapter{registry: imageRegistry}
- log.Printf("🖼️ [CHAT] Injected image registry for describe_image tool")
- }
-
- // Notify client that tool is executing (send original args without internal params)
- displayArgs := make(map[string]interface{})
- for k, v := range args {
- // Filter out internal/sensitive params
- if k != "__user_id__" && k != "__conversation_id__" && k != tools.CredentialResolverKey && k != "credential_id" && k != tools.ImageProviderConfigKey && k != tools.ImageEditConfigKey && k != tools.ImageRegistryKey && k != tools.UsageLimiterKey && k != tools.UserConnectionKey && k != tools.PromptWaiterKey {
- displayArgs[k] = v
- }
- }
-
- // Use SafeSend to prevent panic if connection was closed
- if !userConn.SafeSend(models.ServerMessage{
- Type: "tool_call",
- ToolName: toolName,
- ToolDisplayName: toolDisplayName,
- ToolIcon: toolIcon,
- ToolDescription: toolDescription,
- Status: "executing",
- Arguments: displayArgs,
- }) {
- log.Printf("⚠️ [TOOL] Connection closed before tool execution for %s", toolName)
- return ""
- }
-
- // Execute tool (with injected user context)
- // Check if this is a built-in tool or MCP tool
- tool, exists := s.toolRegistry.GetUserTool(userConn.UserID, toolName)
- var result string
- var err error
-
- if exists && tool.Source == tools.ToolSourceMCPLocal {
- // MCP tool - route to local client
- log.Printf("🔌 [MCP] Routing tool %s to local MCP client for user %s", toolName, userConn.UserID)
-
- if s.mcpBridge == nil || !s.mcpBridge.IsUserConnected(userConn.UserID) {
- errorMsg := "MCP client not connected. Please start your local MCP client."
- log.Printf("❌ [MCP] No client connected for user %s", userConn.UserID)
- userConn.SafeSend(models.ServerMessage{
- Type: "tool_result",
- ToolName: toolName,
- ToolDisplayName: toolDisplayName,
- ToolIcon: toolIcon,
- ToolDescription: toolDescription,
- Status: "failed",
- Result: errorMsg,
- })
- return errorMsg
- }
-
- // Execute on MCP client with 30 second timeout
- startTime := time.Now()
- result, err = s.mcpBridge.ExecuteToolOnClient(userConn.UserID, toolName, args, 30*time.Second)
- executionTime := int(time.Since(startTime).Milliseconds())
-
- // Log execution for audit
- s.mcpBridge.LogToolExecution(userConn.UserID, toolName, userConn.ConversationID, executionTime, err == nil, "")
-
- if err != nil {
- log.Printf("❌ [MCP] Tool execution failed for %s: %v", toolName, err)
- errorMsg := fmt.Sprintf("Error: %v", err)
- userConn.SafeSend(models.ServerMessage{
- Type: "tool_result",
- ToolName: toolName,
- ToolDisplayName: toolDisplayName,
- ToolIcon: toolIcon,
- ToolDescription: toolDescription,
- Status: "failed",
- Result: errorMsg,
- })
- return errorMsg
- }
- } else {
- // Built-in tool - execute locally
- result, err = s.toolRegistry.Execute(toolName, args)
- if err != nil {
- log.Printf("❌ Tool execution failed for %s: %v", toolName, err)
- errorMsg := fmt.Sprintf("Error: %v", err)
- userConn.SafeSend(models.ServerMessage{
- Type: "tool_result",
- ToolName: toolName,
- ToolDisplayName: toolDisplayName,
- ToolIcon: toolIcon,
- ToolDescription: toolDescription,
- Status: "failed",
- Result: errorMsg,
- })
-
- return errorMsg
- }
- }
-
- log.Printf("✅ [TOOL] Tool %s executed successfully, result length: %d", toolName, len(result))
-
- // Try to parse result as JSON to extract plots and files (for E2B tools)
- // We strip base64 data from the LLM result to avoid sending huge payloads
- var resultData map[string]interface{}
- var plots []models.PlotData
- llmResult := result // Default: send full result to LLM
- needsLLMSummary := false
-
- if err := json.Unmarshal([]byte(result), &resultData); err == nil {
- // Check for plots - extract for frontend, strip from LLM
- if plotsRaw, hasPlots := resultData["plots"]; hasPlots {
- if plotsArray, ok := plotsRaw.([]interface{}); ok && len(plotsArray) > 0 {
- // Extract plots for frontend
- for _, p := range plotsArray {
- if plotMap, ok := p.(map[string]interface{}); ok {
- format, _ := plotMap["format"].(string)
- data, _ := plotMap["data"].(string)
- if format != "" && data != "" {
- plots = append(plots, models.PlotData{
- Format: format,
- Data: data,
- })
- }
- }
- }
- needsLLMSummary = true
- log.Printf("📊 [TOOL] Extracted %d plot(s) from %s result", len(plots), toolName)
- }
- }
-
- // Check for files - strip base64 data from LLM result
- if filesRaw, hasFiles := resultData["files"]; hasFiles {
- if filesArray, ok := filesRaw.([]interface{}); ok && len(filesArray) > 0 {
- needsLLMSummary = true
- log.Printf("📁 [TOOL] Detected %d file(s) in %s result, stripping base64 from LLM", len(filesArray), toolName)
- }
- }
-
- // Create LLM-friendly summary (without base64 image/file data)
- if needsLLMSummary {
- llmSummary := map[string]interface{}{
- "success": resultData["success"],
- "stdout": resultData["stdout"],
- "stderr": resultData["stderr"],
- }
-
- // Add plot count if plots exist
- if len(plots) > 0 {
- llmSummary["plot_count"] = len(plots)
- llmSummary["plots_generated"] = fmt.Sprintf("%d visualization(s) generated and shown to user", len(plots))
- }
-
- // Add file info without base64 data
- if filesRaw, hasFiles := resultData["files"]; hasFiles {
- if filesArray, ok := filesRaw.([]interface{}); ok && len(filesArray) > 0 {
- var fileNames []string
- for _, f := range filesArray {
- if fileMap, ok := f.(map[string]interface{}); ok {
- if filename, ok := fileMap["filename"].(string); ok {
- fileNames = append(fileNames, filename)
- }
- }
- }
- llmSummary["file_count"] = len(filesArray)
- llmSummary["files_generated"] = fileNames
- llmSummary["files_message"] = fmt.Sprintf("%d file(s) generated and available for user download", len(filesArray))
- }
- }
-
- // Preserve other useful fields
- if analysis, ok := resultData["analysis"]; ok {
- llmSummary["analysis"] = analysis
- }
- if filename, ok := resultData["filename"]; ok {
- llmSummary["filename"] = filename
- }
- if execTime, ok := resultData["execution_time"]; ok {
- llmSummary["execution_time"] = execTime
- }
- if installOutput, ok := resultData["install_output"]; ok {
- llmSummary["install_output"] = installOutput
- }
-
- llmResultBytes, _ := json.Marshal(llmSummary)
- llmResult = string(llmResultBytes)
- }
- }
-
- // Send result to client (with plots for frontend visualization)
- // Use SafeSend to prevent panic if connection was closed during long tool execution
- toolResultMsg := models.ServerMessage{
- Type: "tool_result",
- ToolName: toolName,
- ToolDisplayName: toolDisplayName,
- ToolIcon: toolIcon,
- ToolDescription: toolDescription,
- Status: "completed",
- Result: result, // Full result for frontend
- Plots: plots, // Extracted plots for rendering
- }
-
- // Try to send the tool result
- if !userConn.SafeSend(toolResultMsg) {
- log.Printf("⚠️ [TOOL] Connection closed, could not send tool result for %s", toolName)
-
- // Buffer tool results with artifacts (images, etc.) for reconnection recovery
- // Only buffer if send failed - this ensures users don't lose generated images
- if len(plots) > 0 && userConn.ConversationID != "" {
- s.streamBuffer.AppendMessage(userConn.ConversationID, BufferedMessage{
- Type: "tool_result",
- ToolName: toolName,
- ToolDisplayName: toolDisplayName,
- ToolIcon: toolIcon,
- ToolDescription: toolDescription,
- Status: "completed",
- Result: result,
- Plots: plots,
- })
- log.Printf("📦 [TOOL] Buffered tool result for %s for reconnection recovery", toolName)
- }
- return llmResult
- }
-
- log.Printf("✅ [TOOL] Tool result for %s ready (plots: %d)", toolName, len(plots))
- // Return LLM-friendly result (without heavy image data)
- return llmResult
-}
-
-// getMarkdownFormattingGuidelines returns formatting rules appended to all system prompts
-// getAskUserInstructions returns intelligent guidance for ask_user tool usage
-// Balanced approach: use when it adds value, skip when it interrupts natural flow
-func getAskUserInstructions() string {
- return `
-
-## 🎯 Interactive Tool - ask_user
-
-You have an **ask_user** tool that creates interactive modal dialogs. Use it intelligently for gathering structured input.
-
-**When to USE ask_user (high value scenarios):**
-
-1. **Planning complex tasks** - Gathering requirements before implementation
- - Example: "Create a website" → ask: style, colors, features, pages
- - Example: "Build a game" → ask: language, library, controls, difficulty
-
-2. **User explicitly requests questions**
- - User: "Ask me questions to understand what I need"
- - User: "Help me figure out what I want"
- - User: "Guide me through this"
-
-3. **Important decisions with multiple valid options**
- - Technical choices: "Which framework? React/Vue/Angular"
- - Approach selection: "Approach A (fast) or B (thorough)?"
- - Confirmation for destructive actions: "Delete all files?"
-
-4. **Missing critical information for task execution**
- - Need specific values: project name, API key, configuration
- - Need preferences that significantly impact output: code style, documentation level
-
-**When NOT to use ask_user (let conversation flow naturally):**
-
-1. **Casual conversation** - Just chat normally
-2. **Emotional support** - Be empathetic in text, don't interrupt with modals
-3. **Simple clarifications** - Ask in text: "Did you mean X or Y?"
-4. **Follow-up questions in dialogue** - Natural back-and-forth
-5. **Rhetorical questions** - Part of your explanation style
-
-**Smart Usage Examples:**
-
-✅ GOOD:
-- User: "Create a landing page" → ask_user: Design style? Color scheme? Sections?
-- User: "I need help planning my app" → ask_user: Features? Users? Platform?
-- User: "Build me a calculator" → ask_user: Basic or scientific? UI style?
-
-❌ NOT NEEDED:
-- User: "I'm feeling lost" → Just respond with empathy, don't open modal
-- User: "Tell me about React" → Just explain, don't ask questions
-- Natural conversation → Keep it flowing, don't interrupt
-
-**Guideline:** Use ask_user when it **helps you gather structured input for better results**. Skip it when it would **interrupt natural conversation flow**.
-`
-}
-
-func getMarkdownFormattingGuidelines() string {
- return `
-
-## Response Style (CRITICAL)
-- **Answer first**: Lead with the direct answer or solution. Context and explanation come after.
-- **No filler phrases**: Never start with "Great question!", "Certainly!", "Of course!", "I'd be happy to", "Absolutely!", or similar. Just answer.
-- **Be concise**: Give complete answers without unnecessary padding. Every sentence should add value.
-- **No excessive caveats**: Don't lead with disclaimers or hedging. If caveats are needed, put them at the end.
-- **Use structure for complex answers**: Use headers and lists for multi-part responses, but avoid over-formatting simple answers.
-- **Match response length to question complexity**: Simple questions get short answers. Complex questions get thorough answers.
-
-## Markdown Formatting
-- **Tables**: Use standard syntax with ` + "`|`" + ` separators and ` + "`---`" + ` header dividers
-- **Lists**: Use ` + "`-`" + ` for unordered lists, ` + "`1.`" + ` for ordered lists (not ` + "`1)`" + `)
-- **Headers**: Always include a space after ` + "`#`" + ` symbols (` + "`## Title`" + ` not ` + "`##Title`" + `)
-- **Code blocks**: Always specify language after ` + "` + \"```\" + `" + ` (e.g., ` + "` + \"```python\" + `" + `, ` + "` + \"```json\" + `" + `)
-- **Links**: Use ` + "`[text](url)`" + ` with no space between ` + "`]`" + ` and ` + "`(`" + `
-- **Avoid**: Citation-style ` + "`[1]`" + ` references, decorative unicode lines, non-standard bullets, emojis (unless user requests them)`
-}
-
-// buildTemporalContext builds context string with current date/time and user name
-// This provides the model with temporal awareness and personalization
-func (s *ChatService) buildTemporalContext(userID string) string {
- now := time.Now()
-
- // Format date and time
- currentDate := now.Format("Monday, January 2, 2006")
- currentTime := now.Format("3:04 PM MST")
-
- // Try to get user's name from database (if MongoDB is available)
- userName := "User" // Default fallback
-
- // Check if we have MongoDB access to get user name
- // Note: ChatService doesn't have direct MongoDB access, but we can try via the database
- // For now, use a simple approach - just use UserID as identifier
- // TODO: Could enhance this with UserService integration if needed
- if userID != "" {
- userName = userID // Use user ID as fallback
- }
-
- // Build temporal context
- context := fmt.Sprintf(`# Current Context
-- **User**: %s
-- **Date**: %s
-- **Time**: %s
-
-`, userName, currentDate, currentTime)
-
- return context
-}
-
-// buildMemoryContext selects and formats relevant memories for injection
-func (s *ChatService) buildMemoryContext(userConn *models.UserConnection) string {
- // Check if memory selection service is available
- if s.memorySelectionService == nil {
- return ""
- }
-
- // Get recent messages from cache for context
- messages := s.getConversationMessages(userConn.ConversationID)
- if len(messages) == 0 {
- return "" // No conversation history yet
- }
-
- // Limit to last 10 messages for context
- recentMessages := messages
- if len(messages) > 10 {
- recentMessages = messages[len(messages)-10:]
- }
-
- // TODO: Get max memories from user preferences (default: 5)
- maxMemories := 5
-
- // Select relevant memories
- ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
- defer cancel()
-
- selectedMemories, err := s.memorySelectionService.SelectRelevantMemories(
- ctx,
- userConn.UserID,
- recentMessages,
- maxMemories,
- )
-
- if err != nil {
- log.Printf("⚠️ [MEMORY] Failed to select memories: %v", err)
- return ""
- }
-
- if len(selectedMemories) == 0 {
- return "" // No relevant memories
- }
-
- // Build memory context string
- var builder strings.Builder
- builder.WriteString("\n\n## Relevant Context from Previous Conversations\n\n")
- builder.WriteString("The following information was extracted from your past interactions with this user:\n\n")
-
- for i, mem := range selectedMemories {
- builder.WriteString(fmt.Sprintf("%d. %s\n", i+1, mem.DecryptedContent))
- }
-
- builder.WriteString("\nUse this context to personalize responses and avoid asking for information the user has already provided.\n")
-
- log.Printf("🧠 [MEMORY] Injected %d memories into system prompt for user %s", len(selectedMemories), userConn.UserID)
-
- return builder.String()
-}
-
-// GetSystemPrompt returns the appropriate system prompt based on priority hierarchy
-// includeAskUser: whether to include ask_user tool instructions (should be true if tools are available)
-func (s *ChatService) GetSystemPrompt(userConn *models.UserConnection, includeAskUser bool) string {
- formattingGuidelines := getMarkdownFormattingGuidelines()
-
- // Only include ask_user instructions if tools will be available in the request
- // Otherwise models like Gemini will fail with MALFORMED_FUNCTION_CALL when trying to use a tool that doesn't exist
- var appendix string
- if includeAskUser {
- appendix = getAskUserInstructions() + formattingGuidelines
- } else {
- appendix = formattingGuidelines
- log.Printf("📝 [SYSTEM] Skipping ask_user instructions (no tools selected)")
- }
-
- // Build temporal context (user name, date, time) - prepended to all prompts
- temporalContext := s.buildTemporalContext(userConn.UserID)
-
- // 🧠 Build memory context (injected memories from user's memory bank)
- memoryContext := s.buildMemoryContext(userConn)
-
- // Priority 1: User-provided system instructions (per-request override)
- if userConn.SystemInstructions != "" {
- log.Printf("🎯 [SYSTEM] Using user-provided system instructions for %s", userConn.ConnID)
- log.Printf("✅ [SYSTEM] Appending MANDATORY ask_user instructions")
- return temporalContext + userConn.SystemInstructions + memoryContext + appendix
- }
-
- // Priority 2: Model-specific system prompt (from database)
- if userConn.ModelID != "" {
- var modelSystemPrompt string
- err := s.db.QueryRow(`
- SELECT system_prompt FROM models WHERE id = ? AND system_prompt IS NOT NULL AND system_prompt != ''
- `, userConn.ModelID).Scan(&modelSystemPrompt)
-
- if err == nil && modelSystemPrompt != "" {
- log.Printf("📋 [SYSTEM] Using model-specific system prompt for %s (model: %s)", userConn.ConnID, userConn.ModelID)
- log.Printf("✅ [SYSTEM] Appending MANDATORY ask_user instructions to database prompt")
- return temporalContext + modelSystemPrompt + memoryContext + appendix
- }
- }
-
- // Priority 3: Provider default system prompt (from providers table)
- if userConn.ModelID != "" {
- var providerSystemPrompt string
- err := s.db.QueryRow(`
- SELECT p.system_prompt
- FROM providers p
- JOIN models m ON m.provider_id = p.id
- WHERE m.id = ? AND p.system_prompt IS NOT NULL AND p.system_prompt != ''
- `, userConn.ModelID).Scan(&providerSystemPrompt)
-
- if err == nil && providerSystemPrompt != "" {
- log.Printf("🏢 [SYSTEM] Using provider default system prompt for %s", userConn.ConnID)
- log.Printf("✅ [SYSTEM] Appending MANDATORY ask_user instructions to provider prompt")
- return temporalContext + providerSystemPrompt + memoryContext + appendix
- }
- }
-
- // Priority 4: Global fallback prompt (already has ask_user instructions built-in)
- log.Printf("🌐 [SYSTEM] Using global fallback system prompt for %s", userConn.ConnID)
- defaultPrompt := getDefaultSystemPrompt()
-
- // Verify ask_user instructions are present
- if strings.Contains(defaultPrompt, "ask_user") {
- log.Printf("✅ [SYSTEM] ask_user tool instructions included in system prompt")
- } else {
- log.Printf("⚠️ [SYSTEM] WARNING: ask_user instructions NOT found in system prompt!")
- }
-
- return temporalContext + defaultPrompt + memoryContext
-}
-
-// getDefaultSystemPrompt returns the ClaraVerse-specific system prompt
-// Tailored to the platform's actual capabilities and tools
-func getDefaultSystemPrompt() string {
- return `You are ClaraVerse AI, an intelligent and helpful assistant with access to powerful tools.
-
-## Your Capabilities
-
-### Interactive Prompts
-- **ask_user** - Create interactive modal dialogs to gather structured input when planning tasks or making important decisions. Use this intelligently for complex workflows, not casual conversation
-
-### Research & Information
-- **search_web** - Search the internet for current information
-- **search_images** - Find images on any topic
-- **scrape_web** - Extract content from specific web pages
-- **get_current_time** - Get current time in any timezone
-
-### File Processing (when user uploads files)
-- **describe_image** - Analyze and describe images in detail
-- **read_document** - Extract text from PDF, DOCX, PPTX files
-- **read_data_file** - Parse CSV, JSON, Excel files
-- **transcribe_audio** - Convert speech to text (MP3, WAV, M4A, etc.)
-
-### Data Analysis & Code
-- **analyze_data** - Statistical analysis with automatic visualizations (charts, graphs)
-- **run_python** - Execute Python code with package support
-- **train_model** - Build ML models (classification, regression, clustering)
-
-### Content Generation
-- **generate_image** - Create AI-generated images from descriptions
-- **create_presentation** - Build Reveal.js slideshows
-
-### Integrations (when user has credentials configured)
-- **GitHub** - Create issues, list repos, add comments
-- **Notion** - Search, query databases, create/update pages
-- **Discord/Slack/Telegram** - Send messages to channels
-- **Custom webhooks** - Send data to any endpoint
-
-## How to Use Tools (In Priority Order)
-
-1. **ask_user FIRST - ALWAYS**
- - ANY question you want to ask → Use ask_user (no exceptions)
- - User says ANYTHING that could go multiple ways → Use ask_user to clarify
- - Examples:
- * User: "Create a website" → ask_user: "What style? Modern/Classic/Minimal?" + "What colors?" + "How many pages?"
- * User: "I'm not figuring out myself" → ask_user: "What did you enjoy as a kid?" + "What makes you lose track of time?"
- * User: "Help me code" → ask_user: "What language?" + "What are you building?"
- * User: "I'm sad" → ask_user: "What's going on?" + "How can I help?"
-
- **Key point**: It doesn't matter if it's technical, emotional, casual, or formal → If you're asking a question → Use ask_user
-
-2. **Other tools SECOND**
- - search_web - For current events, prices, facts
- - analyze_data - When user uploads files
- - generate_image - For illustrations
- - etc.
-
-## Response Guidelines
-
-### Format Rules
-- **Start with the answer** - No preambles like "Great question!"
-- **Match complexity** - Simple question = brief answer, Complex = structured with headers
-- **Use markdown** - Headers, lists, code blocks, tables where appropriate
-- **Code inline** - Add comments in code, not paragraphs explaining it
-
-### Markdown Formatting (IMPORTANT)
-- **Tables**: Use standard syntax with ` + "`|`" + ` separators and ` + "`---`" + ` header dividers
-- **Lists**: Use ` + "`-`" + ` for unordered lists, ` + "`1.`" + ` for ordered lists (not ` + "`1)`" + `)
-- **Headers**: Always include a space after ` + "`#`" + ` symbols (` + "`## Title`" + ` not ` + "`##Title`" + `)
-- **Code blocks**: Always specify language after ` + "` + \"```\" + `" + ` (e.g., ` + "` + \"```python\" + `" + `, ` + "` + \"```json\" + `" + `)
-- **Links**: Use ` + "`[text](url)`" + ` with no space between ` + "`]`" + ` and ` + "`(`" + `
-- **Avoid**: Citation-style ` + "`[1]`" + ` references, decorative unicode lines, non-standard bullets
-
-### Source Citations (CRITICAL)
-When using search_web or search_images, ALWAYS cite sources:
-
-**Inline:** "The iPhone 16 starts at $799 ([Apple](https://apple.com/iphone))."
-
-**End of response:**
----
-**Sources:**
-- [Source Title](url)
-
-### Tool Usage Rules
-
-**🚨 MANDATORY: ask_user**
-
-Use ask_user for EVERY SINGLE QUESTION. No thinking. No judgment. Just do it.
-
-- Typing "Would you..." → STOP → ask_user
-- Typing "Should I..." → STOP → ask_user
-- Typing "Do you want..." → STOP → ask_user
-- Typing "What do you think about..." → STOP → ask_user
-- Typing ANY sentence with a question mark → STOP → ask_user
-
-**Examples that MUST use ask_user:**
-- "What language do you prefer?" → ask_user
-- "How are you feeling?" → ask_user
-- "Want me to continue?" → ask_user
-- "Which approach?" → ask_user
-- "Tell me more?" → ask_user
-- "What makes you happy?" → ask_user
-- "Should I proceed?" → ask_user
-
-**The ONLY time you don't use ask_user:**
-- When you're making a statement (no question mark)
-- When you're explaining a concept they asked about
-- When you're showing results/code/answers
-
-**Other Tools (use when appropriate):**
-- search_web - Current events, prices, facts
-- analyze_data - Uploaded files
-- generate_image - Create images
-
-## Artifacts
-
-You can create interactive artifacts that render in the UI:
-
-1. **HTML/CSS/JS** - Use html code blocks for interactive web content
-2. **SVG** - Use svg code blocks for vector graphics
-3. **Mermaid diagrams** - Use mermaid code blocks for flowcharts, sequence diagrams, etc.
-
-Example Mermaid diagram:
-` + "```" + `mermaid
-graph LR
- A[Start] --> B[Process]
- B --> C[End]
-` + "```" + `
-
-## Tone & Style
-
-- **Interactive and Conversational** - Your defining trait. Engage users in dialogue via ask_user
-- Professional but approachable
-- No emojis unless user uses them first
-- Direct and efficient (but not at the expense of being thorough with ask_user)
-- Technical when appropriate, simple when not
-
-## Your Interactive Character
-
-You are designed to be a **collaborative partner**, not just a command executor. Embrace dialogue:
-
-✅ **DO THIS:**
-- Ask questions before assuming
-- Offer choices instead of making decisions for users
-- Confirm understanding before executing
-- Gather requirements through conversation
-- Use ask_user early and often
-- Treat every request as a conversation starter
-
-❌ **AVOID THIS:**
-- Guessing at user preferences
-- Making assumptions about unstated requirements
-- Jumping straight to implementation without clarifying
-- Asking questions in your text response when ask_user exists
-- Being passive - actively engage the user
-
-**Remember:** Users chose ClaraVerse because they want an interactive AI that collaborates with them, not one that makes assumptions.
-
-## Never Do
-
-- Hallucinate URLs - only use URLs from actual search results
-- Skip citations when using search tools
-- Add unnecessary disclaimers
-- Over-explain simple things
-- Repeat the user's question back to them
-
-**🚨 CRITICALLY IMPORTANT - NEVER DO THESE:**
-- **NEVER ask questions in your text response** - Use ask_user instead
-- **NEVER rationalize why a question "doesn't need ask_user"** - Just use it
-- **NEVER think "this is too casual for a modal"** - Wrong thinking, use ask_user
-- **NEVER think "this is emotional support so I shouldn't use tools"** - Wrong thinking, use ask_user
-- **NEVER think "I'll just ask in text this time"** - Wrong, use ask_user
-- **NEVER write "You're right to notice that. The ask_user tool I have is designed for..."** - This is you rationalizing. Stop. Use the tool.
-- **NEVER explain why you're not using ask_user** - You should be using it`
-}
-
-// buildMessagesWithSystemPrompt ensures system prompt is the first message
-func (s *ChatService) buildMessagesWithSystemPrompt(systemPrompt string, messages []map[string]interface{}) []map[string]interface{} {
- // Check if first message is already a system message
- if len(messages) > 0 {
- if role, ok := messages[0]["role"].(string); ok && role == "system" {
- // Update existing system message
- messages[0]["content"] = systemPrompt
- return messages
- }
- }
-
- // Prepend system message
- systemMessage := map[string]interface{}{
- "role": "system",
- "content": systemPrompt,
- }
-
- return append([]map[string]interface{}{systemMessage}, messages...)
-}
-
-// generateConversationTitle generates a short title from the conversation
-func (s *ChatService) generateConversationTitle(userConn *models.UserConnection, assistantResponse string) {
- // Recover from panics (e.g., send on closed channel if user disconnects)
- defer func() {
- if r := recover(); r != nil {
- log.Printf("⚠️ [TITLE] Recovered from panic (user likely disconnected): %v", r)
- }
- }()
-
- // Get the first user message from cache
- messages := s.getConversationMessages(userConn.ConversationID)
- var firstUserMessage string
- for _, msg := range messages {
- if role, ok := msg["role"].(string); ok && role == "user" {
- if content, ok := msg["content"].(string); ok {
- firstUserMessage = content
- break
- }
- }
- }
-
- if firstUserMessage == "" {
- log.Printf("⚠️ [TITLE] No user message found for title generation")
- return
- }
-
- config, err := s.GetEffectiveConfig(userConn, userConn.ModelID)
- if err != nil {
- log.Printf("❌ [TITLE] Failed to get config: %v", err)
- return
- }
-
- // Create a simple prompt for title generation
- titlePrompt := []map[string]interface{}{
- {
- "role": "system",
- "content": "Generate a short, descriptive title (4-5 words maximum) for this conversation. Respond with only the title, no quotes or punctuation.",
- },
- {
- "role": "user",
- "content": fmt.Sprintf("First message: %s\n\nAssistant response: %s", firstUserMessage, assistantResponse),
- },
- }
-
- // Make a non-streaming request for title
- chatReq := models.ChatRequest{
- Model: config.Model,
- Messages: titlePrompt,
- Stream: false,
- Temperature: 0.7,
- }
-
- reqBody, err := json.Marshal(chatReq)
- if err != nil {
- log.Printf("❌ [TITLE] Failed to marshal request: %v", err)
- return
- }
-
- req, err := http.NewRequest("POST", config.BaseURL+"/chat/completions", bytes.NewBuffer(reqBody))
- if err != nil {
- log.Printf("❌ [TITLE] Failed to create request: %v", err)
- return
- }
-
- req.Header.Set("Content-Type", "application/json")
- req.Header.Set("Authorization", "Bearer "+config.APIKey)
-
- client := &http.Client{Timeout: 30 * time.Second}
- resp, err := client.Do(req)
- if err != nil {
- log.Printf("❌ [TITLE] Request failed: %v", err)
- return
- }
- defer resp.Body.Close()
-
- if resp.StatusCode != http.StatusOK {
- body, _ := io.ReadAll(resp.Body)
- log.Printf("❌ [TITLE] API error (status %d): %s", resp.StatusCode, string(body))
- return
- }
-
- var result struct {
- Choices []struct {
- Message struct {
- Content string `json:"content"`
- } `json:"message"`
- } `json:"choices"`
- }
-
- if err := json.NewDecoder(resp.Body).Decode(&result); err != nil {
- log.Printf("❌ [TITLE] Failed to decode response: %v", err)
- return
- }
-
- if len(result.Choices) == 0 {
- log.Printf("⚠️ [TITLE] No choices in response")
- return
- }
-
- title := strings.TrimSpace(result.Choices[0].Message.Content)
- title = strings.Trim(title, `"'`) // Remove quotes if present
-
- // Limit to 5 words
- words := strings.Fields(title)
- if len(words) > 5 {
- words = words[:5]
- title = strings.Join(words, " ")
- }
-
- log.Printf("📝 [TITLE] Generated title for %s (length: %d chars)", userConn.ConversationID, len(title))
-
- // Send title to client (safe send - channel may be closed if user disconnected)
- select {
- case userConn.WriteChan <- models.ServerMessage{
- Type: "conversation_title",
- ConversationID: userConn.ConversationID,
- Title: title,
- }:
- log.Printf("✅ [TITLE] Sent title to client for %s", userConn.ConversationID)
- default:
- log.Printf("⚠️ [TITLE] Channel closed or full, skipping title send for %s", userConn.ConversationID)
- }
-}
-
-// extractLastUserMessage extracts the last user message content from messages array
-// Handles both string content and array content (for vision messages)
-func extractLastUserMessage(messages []map[string]interface{}) string {
- for i := len(messages) - 1; i >= 0; i-- {
- msg := messages[i]
- role, _ := msg["role"].(string)
- if role == "user" {
- // Handle string content
- if content, ok := msg["content"].(string); ok {
- return content
- }
- // Handle array content (vision messages)
- if contentArr, ok := msg["content"].([]interface{}); ok {
- for _, part := range contentArr {
- if partMap, ok := part.(map[string]interface{}); ok {
- if partType, _ := partMap["type"].(string); partType == "text" {
- if text, ok := partMap["text"].(string); ok {
- return text
- }
- }
- }
- }
- }
- }
- }
- return ""
-}
diff --git a/backend/internal/services/chat_sync_service.go b/backend/internal/services/chat_sync_service.go
deleted file mode 100644
index 612ebcaa..00000000
--- a/backend/internal/services/chat_sync_service.go
+++ /dev/null
@@ -1,581 +0,0 @@
-package services
-
-import (
- "bytes"
- "claraverse/internal/crypto"
- "claraverse/internal/database"
- "claraverse/internal/models"
- "compress/gzip"
- "context"
- "encoding/base64"
- "encoding/json"
- "fmt"
- "io"
- "log"
- "strings"
- "time"
-
- "go.mongodb.org/mongo-driver/bson"
- "go.mongodb.org/mongo-driver/mongo"
- "go.mongodb.org/mongo-driver/mongo/options"
-)
-
-// ChatSyncService handles cloud sync operations for chats with encryption
-type ChatSyncService struct {
- db *database.MongoDB
- collection *mongo.Collection
- encryptionService *crypto.EncryptionService
-}
-
-// NewChatSyncService creates a new chat sync service
-func NewChatSyncService(db *database.MongoDB, encryptionService *crypto.EncryptionService) *ChatSyncService {
- return &ChatSyncService{
- db: db,
- collection: db.Collection(database.CollectionChats),
- encryptionService: encryptionService,
- }
-}
-
-// CreateOrUpdateChat creates a new chat or updates an existing one
-// Uses atomic upsert to prevent race conditions when multiple syncs arrive simultaneously
-func (s *ChatSyncService) CreateOrUpdateChat(ctx context.Context, userID string, req *models.CreateChatRequest) (*models.ChatResponse, error) {
- if userID == "" {
- return nil, fmt.Errorf("user ID is required")
- }
- if req.ID == "" {
- return nil, fmt.Errorf("chat ID is required")
- }
-
- // Encrypt messages
- messagesJSON, err := json.Marshal(req.Messages)
- if err != nil {
- return nil, fmt.Errorf("failed to serialize messages: %w", err)
- }
-
- encryptedMessages, err := s.encryptionService.Encrypt(userID, messagesJSON)
- if err != nil {
- return nil, fmt.Errorf("failed to encrypt messages: %w", err)
- }
-
- // Compress encrypted messages to reduce storage size (helps avoid MongoDB 16MB limit)
- compressedMessages, err := s.compressData(encryptedMessages)
- if err != nil {
- return nil, fmt.Errorf("failed to compress messages: %w", err)
- }
-
- now := time.Now()
-
- filter := bson.M{
- "userId": userID,
- "chatId": req.ID,
- }
-
- // Use atomic upsert to handle race conditions
- // $setOnInsert only applies when creating a new document
- // $set applies to both insert and update
- // Note: Cannot use $setOnInsert and $inc on the same field (version),
- // so we set version to 1 on insert via $setOnInsert, and increment for updates via $inc
- update := bson.M{
- "$set": bson.M{
- "title": req.Title,
- "encryptedMessages": compressedMessages,
- "isStarred": req.IsStarred,
- "model": req.Model,
- "updatedAt": now,
- },
- "$setOnInsert": bson.M{
- "userId": userID,
- "chatId": req.ID,
- "createdAt": now,
- },
- "$inc": bson.M{
- "version": 1,
- },
- }
-
- opts := options.FindOneAndUpdate().
- SetUpsert(true).
- SetReturnDocument(options.After)
-
- var resultChat models.EncryptedChat
- err = s.collection.FindOneAndUpdate(ctx, filter, update, opts).Decode(&resultChat)
- if err != nil {
- return nil, fmt.Errorf("failed to upsert chat: %w", err)
- }
-
- return &models.ChatResponse{
- ID: req.ID,
- Title: resultChat.Title,
- Messages: req.Messages,
- IsStarred: resultChat.IsStarred,
- Model: resultChat.Model,
- Version: resultChat.Version,
- CreatedAt: resultChat.CreatedAt,
- UpdatedAt: resultChat.UpdatedAt,
- }, nil
-}
-
-// GetChat retrieves and decrypts a single chat
-func (s *ChatSyncService) GetChat(ctx context.Context, userID, chatID string) (*models.ChatResponse, error) {
- if userID == "" || chatID == "" {
- return nil, fmt.Errorf("user ID and chat ID are required")
- }
-
- filter := bson.M{
- "userId": userID,
- "chatId": chatID,
- }
-
- var chat models.EncryptedChat
- err := s.collection.FindOne(ctx, filter).Decode(&chat)
- if err == mongo.ErrNoDocuments {
- return nil, fmt.Errorf("chat not found")
- }
- if err != nil {
- return nil, fmt.Errorf("failed to get chat: %w", err)
- }
-
- // Decrypt messages
- messages, err := s.decryptMessages(userID, chat.EncryptedMessages)
- if err != nil {
- return nil, fmt.Errorf("failed to decrypt messages: %w", err)
- }
-
- return &models.ChatResponse{
- ID: chat.ChatID,
- Title: chat.Title,
- Messages: messages,
- IsStarred: chat.IsStarred,
- Model: chat.Model,
- Version: chat.Version,
- CreatedAt: chat.CreatedAt,
- UpdatedAt: chat.UpdatedAt,
- }, nil
-}
-
-// ListChats returns a paginated list of chats (metadata only, no messages)
-func (s *ChatSyncService) ListChats(ctx context.Context, userID string, page, pageSize int, starredOnly bool) (*models.ChatListResponse, error) {
- if userID == "" {
- return nil, fmt.Errorf("user ID is required")
- }
-
- if page < 1 {
- page = 1
- }
- if pageSize < 1 || pageSize > 100 {
- pageSize = 20
- }
-
- filter := bson.M{"userId": userID}
- if starredOnly {
- filter["isStarred"] = true
- }
-
- // Get total count
- totalCount, err := s.collection.CountDocuments(ctx, filter)
- if err != nil {
- return nil, fmt.Errorf("failed to count chats: %w", err)
- }
-
- // Find chats with pagination
- skip := int64((page - 1) * pageSize)
- opts := options.Find().
- SetSort(bson.D{{Key: "updatedAt", Value: -1}}).
- SetSkip(skip).
- SetLimit(int64(pageSize)).
- SetProjection(bson.M{
- "_id": 1,
- "chatId": 1,
- "title": 1,
- "isStarred": 1,
- "model": 1,
- "version": 1,
- "createdAt": 1,
- "updatedAt": 1,
- "encryptedMessages": 1, // Need this to count messages
- })
-
- cursor, err := s.collection.Find(ctx, filter, opts)
- if err != nil {
- return nil, fmt.Errorf("failed to list chats: %w", err)
- }
- defer cursor.Close(ctx)
-
- var chats []models.ChatListItem
- for cursor.Next(ctx) {
- var encChat models.EncryptedChat
- if err := cursor.Decode(&encChat); err != nil {
- log.Printf("⚠️ Failed to decode chat: %v", err)
- continue
- }
-
- // Count messages (decrypt to get count)
- messageCount := 0
- if encChat.EncryptedMessages != "" {
- messages, err := s.decryptMessages(userID, encChat.EncryptedMessages)
- if err == nil {
- messageCount = len(messages)
- }
- }
-
- chats = append(chats, models.ChatListItem{
- ID: encChat.ChatID,
- Title: encChat.Title,
- IsStarred: encChat.IsStarred,
- Model: encChat.Model,
- MessageCount: messageCount,
- Version: encChat.Version,
- CreatedAt: encChat.CreatedAt,
- UpdatedAt: encChat.UpdatedAt,
- })
- }
-
- return &models.ChatListResponse{
- Chats: chats,
- TotalCount: totalCount,
- Page: page,
- PageSize: pageSize,
- HasMore: int64(page*pageSize) < totalCount,
- }, nil
-}
-
-// UpdateChat performs a partial update on a chat
-func (s *ChatSyncService) UpdateChat(ctx context.Context, userID, chatID string, req *models.UpdateChatRequest) (*models.ChatListItem, error) {
- if userID == "" || chatID == "" {
- return nil, fmt.Errorf("user ID and chat ID are required")
- }
-
- filter := bson.M{
- "userId": userID,
- "chatId": chatID,
- "version": req.Version, // Optimistic locking
- }
-
- updateFields := bson.M{
- "updatedAt": time.Now(),
- }
-
- if req.Title != nil {
- updateFields["title"] = *req.Title
- }
- if req.IsStarred != nil {
- updateFields["isStarred"] = *req.IsStarred
- }
- if req.Model != nil {
- updateFields["model"] = *req.Model
- }
-
- update := bson.M{
- "$set": updateFields,
- "$inc": bson.M{"version": 1},
- }
-
- opts := options.FindOneAndUpdate().SetReturnDocument(options.After)
- var updatedChat models.EncryptedChat
- err := s.collection.FindOneAndUpdate(ctx, filter, update, opts).Decode(&updatedChat)
- if err == mongo.ErrNoDocuments {
- return nil, fmt.Errorf("chat not found or version conflict")
- }
- if err != nil {
- return nil, fmt.Errorf("failed to update chat: %w", err)
- }
-
- // Count messages
- messageCount := 0
- if updatedChat.EncryptedMessages != "" {
- messages, err := s.decryptMessages(userID, updatedChat.EncryptedMessages)
- if err == nil {
- messageCount = len(messages)
- }
- }
-
- return &models.ChatListItem{
- ID: updatedChat.ChatID,
- Title: updatedChat.Title,
- IsStarred: updatedChat.IsStarred,
- Model: updatedChat.Model,
- MessageCount: messageCount,
- Version: updatedChat.Version,
- CreatedAt: updatedChat.CreatedAt,
- UpdatedAt: updatedChat.UpdatedAt,
- }, nil
-}
-
-// DeleteChat removes a chat
-func (s *ChatSyncService) DeleteChat(ctx context.Context, userID, chatID string) error {
- if userID == "" || chatID == "" {
- return fmt.Errorf("user ID and chat ID are required")
- }
-
- filter := bson.M{
- "userId": userID,
- "chatId": chatID,
- }
-
- result, err := s.collection.DeleteOne(ctx, filter)
- if err != nil {
- return fmt.Errorf("failed to delete chat: %w", err)
- }
-
- if result.DeletedCount == 0 {
- return fmt.Errorf("chat not found")
- }
-
- return nil
-}
-
-// BulkSync uploads multiple chats at once
-func (s *ChatSyncService) BulkSync(ctx context.Context, userID string, req *models.BulkSyncRequest) (*models.BulkSyncResponse, error) {
- if userID == "" {
- return nil, fmt.Errorf("user ID is required")
- }
-
- response := &models.BulkSyncResponse{
- ChatIDs: make([]string, 0),
- }
-
- for _, chatReq := range req.Chats {
- _, err := s.CreateOrUpdateChat(ctx, userID, &chatReq)
- if err != nil {
- response.Failed++
- response.Errors = append(response.Errors, fmt.Sprintf("chat %s: %v", chatReq.ID, err))
- log.Printf("⚠️ Failed to sync chat %s: %v", chatReq.ID, err)
- } else {
- response.Synced++
- response.ChatIDs = append(response.ChatIDs, chatReq.ID)
- }
- }
-
- return response, nil
-}
-
-// GetAllChats returns all chats for initial sync (with decrypted messages)
-func (s *ChatSyncService) GetAllChats(ctx context.Context, userID string) (*models.SyncAllResponse, error) {
- if userID == "" {
- return nil, fmt.Errorf("user ID is required")
- }
-
- filter := bson.M{"userId": userID}
- opts := options.Find().SetSort(bson.D{{Key: "updatedAt", Value: -1}})
-
- cursor, err := s.collection.Find(ctx, filter, opts)
- if err != nil {
- return nil, fmt.Errorf("failed to get chats: %w", err)
- }
- defer cursor.Close(ctx)
-
- chats := make([]models.ChatResponse, 0) // Initialize empty slice to avoid null in JSON
- for cursor.Next(ctx) {
- var encChat models.EncryptedChat
- if err := cursor.Decode(&encChat); err != nil {
- log.Printf("⚠️ Failed to decode chat: %v", err)
- continue
- }
-
- // Decrypt messages
- messages, err := s.decryptMessages(userID, encChat.EncryptedMessages)
- if err != nil {
- log.Printf("⚠️ Failed to decrypt messages for chat %s: %v", encChat.ChatID, err)
- continue
- }
-
- chats = append(chats, models.ChatResponse{
- ID: encChat.ChatID,
- Title: encChat.Title,
- Messages: messages,
- IsStarred: encChat.IsStarred,
- Model: encChat.Model,
- Version: encChat.Version,
- CreatedAt: encChat.CreatedAt,
- UpdatedAt: encChat.UpdatedAt,
- })
- }
-
- return &models.SyncAllResponse{
- Chats: chats,
- TotalCount: len(chats),
- SyncedAt: time.Now(),
- }, nil
-}
-
-// AddMessage adds a single message to a chat with optimistic locking
-func (s *ChatSyncService) AddMessage(ctx context.Context, userID, chatID string, req *models.ChatAddMessageRequest) (*models.ChatResponse, error) {
- if userID == "" || chatID == "" {
- return nil, fmt.Errorf("user ID and chat ID are required")
- }
-
- // Get current chat
- filter := bson.M{
- "userId": userID,
- "chatId": chatID,
- "version": req.Version, // Optimistic locking
- }
-
- var chat models.EncryptedChat
- err := s.collection.FindOne(ctx, filter).Decode(&chat)
- if err == mongo.ErrNoDocuments {
- return nil, fmt.Errorf("chat not found or version conflict")
- }
- if err != nil {
- return nil, fmt.Errorf("failed to get chat: %w", err)
- }
-
- // Decrypt existing messages
- messages, err := s.decryptMessages(userID, chat.EncryptedMessages)
- if err != nil {
- return nil, fmt.Errorf("failed to decrypt messages: %w", err)
- }
-
- // Add new message
- messages = append(messages, req.Message)
-
- // Re-encrypt messages
- messagesJSON, err := json.Marshal(messages)
- if err != nil {
- return nil, fmt.Errorf("failed to serialize messages: %w", err)
- }
-
- encryptedMessages, err := s.encryptionService.Encrypt(userID, messagesJSON)
- if err != nil {
- return nil, fmt.Errorf("failed to encrypt messages: %w", err)
- }
-
- // Compress encrypted messages to reduce storage size
- compressedMessages, err := s.compressData(encryptedMessages)
- if err != nil {
- return nil, fmt.Errorf("failed to compress messages: %w", err)
- }
-
- // Update chat
- now := time.Now()
- update := bson.M{
- "$set": bson.M{
- "encryptedMessages": compressedMessages,
- "updatedAt": now,
- },
- "$inc": bson.M{"version": 1},
- }
-
- opts := options.FindOneAndUpdate().SetReturnDocument(options.After)
- var updatedChat models.EncryptedChat
- err = s.collection.FindOneAndUpdate(ctx, filter, update, opts).Decode(&updatedChat)
- if err == mongo.ErrNoDocuments {
- return nil, fmt.Errorf("version conflict during update")
- }
- if err != nil {
- return nil, fmt.Errorf("failed to update chat: %w", err)
- }
-
- return &models.ChatResponse{
- ID: chatID,
- Title: updatedChat.Title,
- Messages: messages,
- IsStarred: updatedChat.IsStarred,
- Model: updatedChat.Model,
- Version: updatedChat.Version,
- CreatedAt: updatedChat.CreatedAt,
- UpdatedAt: updatedChat.UpdatedAt,
- }, nil
-}
-
-// DeleteAllUserChats removes all chats for a user (GDPR compliance)
-func (s *ChatSyncService) DeleteAllUserChats(ctx context.Context, userID string) (int64, error) {
- if userID == "" {
- return 0, fmt.Errorf("user ID is required")
- }
-
- filter := bson.M{"userId": userID}
- result, err := s.collection.DeleteMany(ctx, filter)
- if err != nil {
- return 0, fmt.Errorf("failed to delete user chats: %w", err)
- }
-
- return result.DeletedCount, nil
-}
-
-// decryptMessages decrypts and decompresses the encrypted messages JSON
-func (s *ChatSyncService) decryptMessages(userID, encryptedMessages string) ([]models.ChatMessage, error) {
- if encryptedMessages == "" {
- return []models.ChatMessage{}, nil
- }
-
- // Decompress if compressed (backward compatible - old data won't have gzip: prefix)
- dataToDecrypt := encryptedMessages
- if strings.HasPrefix(encryptedMessages, "gzip:") {
- compressed := strings.TrimPrefix(encryptedMessages, "gzip:")
- decompressed, err := s.decompressData(compressed)
- if err != nil {
- return nil, fmt.Errorf("failed to decompress messages: %w", err)
- }
- dataToDecrypt = decompressed
- }
-
- decrypted, err := s.encryptionService.Decrypt(userID, dataToDecrypt)
- if err != nil {
- return nil, err
- }
-
- var messages []models.ChatMessage
- if err := json.Unmarshal(decrypted, &messages); err != nil {
- return nil, fmt.Errorf("failed to parse messages: %w", err)
- }
-
- return messages, nil
-}
-
-// compressData compresses a string using gzip and returns it with a prefix marker
-func (s *ChatSyncService) compressData(data string) (string, error) {
- var buf bytes.Buffer
- writer := gzip.NewWriter(&buf)
-
- if _, err := writer.Write([]byte(data)); err != nil {
- return "", err
- }
-
- if err := writer.Close(); err != nil {
- return "", err
- }
-
- // Encode to base64 and add prefix to identify compressed data
- compressed := base64.StdEncoding.EncodeToString(buf.Bytes())
- return "gzip:" + compressed, nil
-}
-
-// decompressData decompresses a base64-encoded gzip string
-func (s *ChatSyncService) decompressData(compressed string) (string, error) {
- // Decode base64
- data, err := base64.StdEncoding.DecodeString(compressed)
- if err != nil {
- return "", fmt.Errorf("failed to decode base64: %w", err)
- }
-
- // Decompress gzip
- reader, err := gzip.NewReader(bytes.NewReader(data))
- if err != nil {
- return "", fmt.Errorf("failed to create gzip reader: %w", err)
- }
- defer reader.Close()
-
- decompressed, err := io.ReadAll(reader)
- if err != nil {
- return "", fmt.Errorf("failed to read decompressed data: %w", err)
- }
-
- return string(decompressed), nil
-}
-
-// EnsureIndexes creates necessary indexes for the chats collection
-func (s *ChatSyncService) EnsureIndexes(ctx context.Context) error {
- indexes := []mongo.IndexModel{
- {Keys: bson.D{{Key: "userId", Value: 1}, {Key: "updatedAt", Value: -1}}},
- {Keys: bson.D{{Key: "userId", Value: 1}, {Key: "chatId", Value: 1}}, Options: options.Index().SetUnique(true)},
- {Keys: bson.D{{Key: "userId", Value: 1}, {Key: "isStarred", Value: 1}}},
- }
-
- _, err := s.collection.Indexes().CreateMany(ctx, indexes)
- if err != nil {
- return fmt.Errorf("failed to create chat indexes: %w", err)
- }
-
- return nil
-}
diff --git a/backend/internal/services/chat_sync_service_test.go b/backend/internal/services/chat_sync_service_test.go
deleted file mode 100644
index 87eabfa2..00000000
--- a/backend/internal/services/chat_sync_service_test.go
+++ /dev/null
@@ -1,976 +0,0 @@
-package services
-
-import (
- "claraverse/internal/crypto"
- "claraverse/internal/models"
- "encoding/json"
- "testing"
- "time"
-)
-
-// Test encryption service creation and basic operations
-func TestEncryptionService(t *testing.T) {
- // Generate a test master key (32 bytes = 64 hex chars)
- masterKey := "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"
-
- encService, err := crypto.NewEncryptionService(masterKey)
- if err != nil {
- t.Fatalf("Failed to create encryption service: %v", err)
- }
-
- if encService == nil {
- t.Fatal("Encryption service should not be nil")
- }
-}
-
-func TestEncryptDecryptRoundtrip(t *testing.T) {
- masterKey := "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"
- encService, err := crypto.NewEncryptionService(masterKey)
- if err != nil {
- t.Fatalf("Failed to create encryption service: %v", err)
- }
-
- userID := "test-user-123"
- testCases := []struct {
- name string
- plaintext string
- }{
- {"simple text", "Hello, World!"},
- {"empty string", ""},
- {"json array", `[{"id":"1","content":"test"}]`},
- {"unicode", "Hello, \u4e16\u754c! \U0001F600"},
- {"long text", string(make([]byte, 10000))}, // 10KB of null bytes
- }
-
- for _, tc := range testCases {
- t.Run(tc.name, func(t *testing.T) {
- // Encrypt
- encrypted, err := encService.EncryptString(userID, tc.plaintext)
- if err != nil {
- if tc.plaintext == "" {
- // Empty string returns empty, not error
- return
- }
- t.Fatalf("Encryption failed: %v", err)
- }
-
- // Encrypted should not equal plaintext (unless empty)
- if tc.plaintext != "" && encrypted == tc.plaintext {
- t.Error("Encrypted text should not equal plaintext")
- }
-
- // Decrypt
- decrypted, err := encService.DecryptString(userID, encrypted)
- if err != nil {
- t.Fatalf("Decryption failed: %v", err)
- }
-
- // Decrypted should equal original
- if decrypted != tc.plaintext {
- t.Errorf("Decrypted text doesn't match original. Got %q, want %q", decrypted, tc.plaintext)
- }
- })
- }
-}
-
-func TestDifferentUsersGetDifferentKeys(t *testing.T) {
- masterKey := "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"
- encService, err := crypto.NewEncryptionService(masterKey)
- if err != nil {
- t.Fatalf("Failed to create encryption service: %v", err)
- }
-
- plaintext := "Secret message"
- user1 := "user-1"
- user2 := "user-2"
-
- // Encrypt same message for two different users
- encrypted1, _ := encService.EncryptString(user1, plaintext)
- encrypted2, _ := encService.EncryptString(user2, plaintext)
-
- // Encrypted values should be different (different user keys + different nonces)
- if encrypted1 == encrypted2 {
- t.Error("Same plaintext encrypted for different users should produce different ciphertext")
- }
-
- // User 2 should not be able to decrypt user 1's message
- decrypted, err := encService.DecryptString(user2, encrypted1)
- if err == nil && decrypted == plaintext {
- t.Error("User 2 should not be able to decrypt User 1's message")
- }
-}
-
-func TestChatMessageSerialization(t *testing.T) {
- messages := []models.ChatMessage{
- {
- ID: "msg-1",
- Role: "user",
- Content: "Hello!",
- Timestamp: time.Now().UnixMilli(),
- },
- {
- ID: "msg-2",
- Role: "assistant",
- Content: "Hi there! How can I help you?",
- Timestamp: time.Now().UnixMilli(),
- },
- }
-
- // Serialize
- jsonData, err := json.Marshal(messages)
- if err != nil {
- t.Fatalf("Failed to serialize messages: %v", err)
- }
-
- // Deserialize
- var decoded []models.ChatMessage
- err = json.Unmarshal(jsonData, &decoded)
- if err != nil {
- t.Fatalf("Failed to deserialize messages: %v", err)
- }
-
- if len(decoded) != len(messages) {
- t.Errorf("Expected %d messages, got %d", len(messages), len(decoded))
- }
-
- for i, msg := range decoded {
- if msg.ID != messages[i].ID {
- t.Errorf("Message %d ID mismatch: got %s, want %s", i, msg.ID, messages[i].ID)
- }
- if msg.Role != messages[i].Role {
- t.Errorf("Message %d Role mismatch: got %s, want %s", i, msg.Role, messages[i].Role)
- }
- if msg.Content != messages[i].Content {
- t.Errorf("Message %d Content mismatch: got %s, want %s", i, msg.Content, messages[i].Content)
- }
- }
-}
-
-func TestChatMessageWithAttachments(t *testing.T) {
- message := models.ChatMessage{
- ID: "msg-1",
- Role: "user",
- Content: "Check out this file",
- Timestamp: time.Now().UnixMilli(),
- Attachments: []models.ChatAttachment{
- {
- FileID: "att-1",
- Filename: "document.pdf",
- Type: "document",
- MimeType: "application/pdf",
- Size: 1024,
- },
- },
- }
-
- // Serialize
- jsonData, err := json.Marshal(message)
- if err != nil {
- t.Fatalf("Failed to serialize message with attachment: %v", err)
- }
-
- // Deserialize
- var decoded models.ChatMessage
- err = json.Unmarshal(jsonData, &decoded)
- if err != nil {
- t.Fatalf("Failed to deserialize message with attachment: %v", err)
- }
-
- if len(decoded.Attachments) != 1 {
- t.Fatalf("Expected 1 attachment, got %d", len(decoded.Attachments))
- }
-
- att := decoded.Attachments[0]
- if att.FileID != "att-1" {
- t.Errorf("Attachment FileID mismatch: got %s, want att-1", att.FileID)
- }
- if att.Filename != "document.pdf" {
- t.Errorf("Attachment filename mismatch: got %s, want document.pdf", att.Filename)
- }
-}
-
-func TestCreateChatRequestValidation(t *testing.T) {
- tests := []struct {
- name string
- request models.CreateChatRequest
- wantErr bool
- }{
- {
- name: "valid request",
- request: models.CreateChatRequest{
- ID: "chat-123",
- Title: "Test Chat",
- Messages: []models.ChatMessage{
- {ID: "msg-1", Role: "user", Content: "Hello"},
- },
- },
- wantErr: false,
- },
- {
- name: "empty chat ID",
- request: models.CreateChatRequest{
- ID: "",
- Title: "Test Chat",
- },
- wantErr: true,
- },
- {
- name: "empty messages",
- request: models.CreateChatRequest{
- ID: "chat-123",
- Title: "Test Chat",
- Messages: []models.ChatMessage{},
- },
- wantErr: false, // Empty messages should be allowed
- },
- }
-
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- hasErr := tt.request.ID == ""
- if hasErr != tt.wantErr {
- t.Errorf("Validation error = %v, wantErr %v", hasErr, tt.wantErr)
- }
- })
- }
-}
-
-func TestBulkSyncRequest(t *testing.T) {
- req := models.BulkSyncRequest{
- Chats: []models.CreateChatRequest{
- {
- ID: "chat-1",
- Title: "Chat 1",
- Messages: []models.ChatMessage{
- {ID: "msg-1", Role: "user", Content: "Hello"},
- },
- },
- {
- ID: "chat-2",
- Title: "Chat 2",
- Messages: []models.ChatMessage{
- {ID: "msg-2", Role: "user", Content: "Hi"},
- },
- },
- },
- }
-
- // Serialize
- jsonData, err := json.Marshal(req)
- if err != nil {
- t.Fatalf("Failed to serialize bulk sync request: %v", err)
- }
-
- // Deserialize
- var decoded models.BulkSyncRequest
- err = json.Unmarshal(jsonData, &decoded)
- if err != nil {
- t.Fatalf("Failed to deserialize bulk sync request: %v", err)
- }
-
- if len(decoded.Chats) != 2 {
- t.Errorf("Expected 2 chats, got %d", len(decoded.Chats))
- }
-}
-
-func TestChatResponseConversion(t *testing.T) {
- response := models.ChatResponse{
- ID: "chat-123",
- Title: "Test Chat",
- Messages: []models.ChatMessage{{ID: "msg-1", Role: "user", Content: "Hello"}},
- IsStarred: true,
- Model: "gpt-4",
- Version: 5,
- CreatedAt: time.Now(),
- UpdatedAt: time.Now(),
- }
-
- // Serialize
- jsonData, err := json.Marshal(response)
- if err != nil {
- t.Fatalf("Failed to serialize chat response: %v", err)
- }
-
- // Deserialize
- var decoded models.ChatResponse
- err = json.Unmarshal(jsonData, &decoded)
- if err != nil {
- t.Fatalf("Failed to deserialize chat response: %v", err)
- }
-
- if decoded.ID != response.ID {
- t.Errorf("ID mismatch: got %s, want %s", decoded.ID, response.ID)
- }
- if decoded.Version != response.Version {
- t.Errorf("Version mismatch: got %d, want %d", decoded.Version, response.Version)
- }
- if decoded.IsStarred != response.IsStarred {
- t.Errorf("IsStarred mismatch: got %v, want %v", decoded.IsStarred, response.IsStarred)
- }
-}
-
-func TestChatListItem(t *testing.T) {
- item := models.ChatListItem{
- ID: "chat-123",
- Title: "Test Chat",
- IsStarred: true,
- Model: "gpt-4",
- MessageCount: 10,
- Version: 3,
- CreatedAt: time.Now(),
- UpdatedAt: time.Now(),
- }
-
- // Serialize
- jsonData, err := json.Marshal(item)
- if err != nil {
- t.Fatalf("Failed to serialize chat list item: %v", err)
- }
-
- // Check JSON contains expected fields
- var jsonMap map[string]interface{}
- json.Unmarshal(jsonData, &jsonMap)
-
- if _, ok := jsonMap["message_count"]; !ok {
- t.Error("Expected message_count in JSON output")
- }
- if _, ok := jsonMap["is_starred"]; !ok {
- t.Error("Expected is_starred in JSON output")
- }
-}
-
-func TestUpdateChatRequest(t *testing.T) {
- title := "New Title"
- starred := true
-
- req := models.UpdateChatRequest{
- Title: &title,
- IsStarred: &starred,
- Version: 5,
- }
-
- // Serialize
- jsonData, err := json.Marshal(req)
- if err != nil {
- t.Fatalf("Failed to serialize update request: %v", err)
- }
-
- // Deserialize
- var decoded models.UpdateChatRequest
- err = json.Unmarshal(jsonData, &decoded)
- if err != nil {
- t.Fatalf("Failed to deserialize update request: %v", err)
- }
-
- if decoded.Title == nil || *decoded.Title != title {
- t.Error("Title should be set")
- }
- if decoded.IsStarred == nil || *decoded.IsStarred != starred {
- t.Error("IsStarred should be set")
- }
- if decoded.Version != 5 {
- t.Errorf("Version mismatch: got %d, want 5", decoded.Version)
- }
-}
-
-func TestChatAddMessageRequest(t *testing.T) {
- req := models.ChatAddMessageRequest{
- Message: models.ChatMessage{
- ID: "msg-new",
- Role: "user",
- Content: "New message",
- Timestamp: time.Now().UnixMilli(),
- },
- Version: 3,
- }
-
- // Serialize
- jsonData, err := json.Marshal(req)
- if err != nil {
- t.Fatalf("Failed to serialize add message request: %v", err)
- }
-
- // Deserialize
- var decoded models.ChatAddMessageRequest
- err = json.Unmarshal(jsonData, &decoded)
- if err != nil {
- t.Fatalf("Failed to deserialize add message request: %v", err)
- }
-
- if decoded.Message.ID != "msg-new" {
- t.Errorf("Message ID mismatch: got %s, want msg-new", decoded.Message.ID)
- }
- if decoded.Version != 3 {
- t.Errorf("Version mismatch: got %d, want 3", decoded.Version)
- }
-}
-
-func TestNewChatSyncService(t *testing.T) {
- // This test requires MongoDB to be set up
- // In a real test environment, you'd use a test MongoDB instance
- // For now, we just verify the service constructor doesn't panic with valid inputs
- t.Skip("Requires MongoDB connection - run with integration tests")
-}
-
-// ==================== EDGE CASE TESTS ====================
-
-func TestEmptyMessagesArray(t *testing.T) {
- masterKey := "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"
- encService, _ := crypto.NewEncryptionService(masterKey)
-
- userID := "user-123"
- emptyMessages := []models.ChatMessage{}
-
- // Serialize empty array
- jsonData, err := json.Marshal(emptyMessages)
- if err != nil {
- t.Fatalf("Failed to serialize empty messages: %v", err)
- }
-
- // Encrypt
- encrypted, err := encService.Encrypt(userID, jsonData)
- if err != nil {
- t.Fatalf("Failed to encrypt empty messages: %v", err)
- }
-
- // Decrypt
- decrypted, err := encService.Decrypt(userID, encrypted)
- if err != nil {
- t.Fatalf("Failed to decrypt: %v", err)
- }
-
- // Deserialize
- var recovered []models.ChatMessage
- err = json.Unmarshal(decrypted, &recovered)
- if err != nil {
- t.Fatalf("Failed to deserialize: %v", err)
- }
-
- if len(recovered) != 0 {
- t.Errorf("Expected empty array, got %d messages", len(recovered))
- }
-}
-
-func TestNullFieldsInMessage(t *testing.T) {
- // Test that null/empty optional fields don't break serialization
- message := models.ChatMessage{
- ID: "msg-1",
- Role: "user",
- Content: "Hello",
- Timestamp: 1700000000000,
- IsStreaming: false,
- Attachments: nil, // nil attachments
- AgentId: "", // empty agent id
- AgentName: "",
- AgentAvatar: "",
- }
-
- jsonData, err := json.Marshal(message)
- if err != nil {
- t.Fatalf("Failed to serialize message with null fields: %v", err)
- }
-
- var recovered models.ChatMessage
- err = json.Unmarshal(jsonData, &recovered)
- if err != nil {
- t.Fatalf("Failed to deserialize: %v", err)
- }
-
- if recovered.ID != message.ID {
- t.Error("ID mismatch after roundtrip")
- }
- if recovered.Attachments != nil {
- t.Error("Expected nil attachments")
- }
-}
-
-func TestSpecialCharactersInContent(t *testing.T) {
- masterKey := "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"
- encService, _ := crypto.NewEncryptionService(masterKey)
-
- testCases := []string{
- "Hello\nWorld", // Newlines
- "Tab\there", // Tabs
- "Quote: \"test\"", // Quotes
- "Backslash: \\path\\to\\file", // Backslashes
- "Unicode: \u4e2d\u6587", // Chinese characters
- "Emoji: \U0001F600\U0001F389", // Emojis
- "HTML: ", // HTML
- "SQL: SELECT * FROM users; DROP TABLE--", // SQL injection attempt
- "Null byte: \x00", // Null byte
- "Control chars: \x01\x02\x03", // Control characters
- string(make([]byte, 100000)), // Large content (100KB)
- }
-
- userID := "user-123"
-
- for i, content := range testCases {
- t.Run("case_"+string(rune('0'+i)), func(t *testing.T) {
- message := models.ChatMessage{
- ID: "msg-1",
- Role: "user",
- Content: content,
- Timestamp: 1700000000000,
- }
-
- // Serialize
- jsonData, err := json.Marshal(message)
- if err != nil {
- t.Fatalf("Failed to serialize: %v", err)
- }
-
- // Encrypt
- encrypted, err := encService.Encrypt(userID, jsonData)
- if err != nil {
- t.Fatalf("Failed to encrypt: %v", err)
- }
-
- // Decrypt
- decrypted, err := encService.Decrypt(userID, encrypted)
- if err != nil {
- t.Fatalf("Failed to decrypt: %v", err)
- }
-
- // Deserialize
- var recovered models.ChatMessage
- err = json.Unmarshal(decrypted, &recovered)
- if err != nil {
- t.Fatalf("Failed to deserialize: %v", err)
- }
-
- if recovered.Content != content {
- t.Errorf("Content mismatch after roundtrip")
- }
- })
- }
-}
-
-func TestMaxMessageCount(t *testing.T) {
- // Test with a large number of messages (stress test)
- messageCount := 1000
- messages := make([]models.ChatMessage, messageCount)
-
- for i := 0; i < messageCount; i++ {
- role := "user"
- if i%2 == 1 {
- role = "assistant"
- }
- messages[i] = models.ChatMessage{
- ID: "msg-" + string(rune('0'+i%10)) + string(rune('0'+i/10)),
- Role: role,
- Content: "This is message number " + string(rune('0'+i)),
- Timestamp: int64(1700000000000 + i*1000),
- }
- }
-
- masterKey := "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"
- encService, _ := crypto.NewEncryptionService(masterKey)
- userID := "user-123"
-
- // Serialize
- jsonData, err := json.Marshal(messages)
- if err != nil {
- t.Fatalf("Failed to serialize %d messages: %v", messageCount, err)
- }
-
- // Encrypt
- encrypted, err := encService.Encrypt(userID, jsonData)
- if err != nil {
- t.Fatalf("Failed to encrypt: %v", err)
- }
-
- // Decrypt
- decrypted, err := encService.Decrypt(userID, encrypted)
- if err != nil {
- t.Fatalf("Failed to decrypt: %v", err)
- }
-
- // Deserialize
- var recovered []models.ChatMessage
- err = json.Unmarshal(decrypted, &recovered)
- if err != nil {
- t.Fatalf("Failed to deserialize: %v", err)
- }
-
- if len(recovered) != messageCount {
- t.Errorf("Expected %d messages, got %d", messageCount, len(recovered))
- }
-}
-
-func TestAttachmentTypes(t *testing.T) {
- // Test all attachment types
- attachments := []models.ChatAttachment{
- {
- FileID: "att-1",
- Filename: "document.pdf",
- Type: "document",
- MimeType: "application/pdf",
- Size: 1024,
- URL: "https://example.com/file.pdf",
- },
- {
- FileID: "att-2",
- Filename: "image.png",
- Type: "image",
- MimeType: "image/png",
- Size: 2048,
- Preview: "base64encodedcontent",
- },
- {
- FileID: "att-3",
- Filename: "data.csv",
- Type: "data",
- MimeType: "text/csv",
- Size: 512,
- },
- }
-
- message := models.ChatMessage{
- ID: "msg-1",
- Role: "user",
- Content: "Check these files",
- Timestamp: 1700000000000,
- Attachments: attachments,
- }
-
- jsonData, err := json.Marshal(message)
- if err != nil {
- t.Fatalf("Failed to serialize: %v", err)
- }
-
- var recovered models.ChatMessage
- err = json.Unmarshal(jsonData, &recovered)
- if err != nil {
- t.Fatalf("Failed to deserialize: %v", err)
- }
-
- if len(recovered.Attachments) != len(attachments) {
- t.Fatalf("Expected %d attachments, got %d", len(attachments), len(recovered.Attachments))
- }
-
- for i, att := range recovered.Attachments {
- if att.FileID != attachments[i].FileID {
- t.Errorf("Attachment %d: FileID mismatch", i)
- }
- if att.Type != attachments[i].Type {
- t.Errorf("Attachment %d: Type mismatch", i)
- }
- if att.Size != attachments[i].Size {
- t.Errorf("Attachment %d: Size mismatch", i)
- }
- }
-}
-
-// ==================== BACKWARD COMPATIBILITY TESTS ====================
-
-func TestBackwardCompatibility_OldMessageFormat(t *testing.T) {
- // Simulate old message format that might exist in localStorage
- // This tests that the backend can handle messages without all new fields
- oldFormatJSON := `{
- "id": "msg-1",
- "role": "user",
- "content": "Hello",
- "timestamp": 1700000000000
- }`
-
- var message models.ChatMessage
- err := json.Unmarshal([]byte(oldFormatJSON), &message)
- if err != nil {
- t.Fatalf("Failed to parse old format: %v", err)
- }
-
- if message.ID != "msg-1" {
- t.Error("ID mismatch")
- }
- if message.Role != "user" {
- t.Error("Role mismatch")
- }
- if message.Attachments != nil {
- t.Error("Attachments should be nil for old format")
- }
-}
-
-func TestBackwardCompatibility_OldChatFormat(t *testing.T) {
- // Test parsing of chat without newer optional fields
- oldFormatJSON := `{
- "id": "chat-123",
- "title": "Test Chat",
- "messages": [
- {"id": "msg-1", "role": "user", "content": "Hello", "timestamp": 1700000000000}
- ]
- }`
-
- var req models.CreateChatRequest
- err := json.Unmarshal([]byte(oldFormatJSON), &req)
- if err != nil {
- t.Fatalf("Failed to parse old format: %v", err)
- }
-
- if req.ID != "chat-123" {
- t.Error("ID mismatch")
- }
- if req.IsStarred != false {
- t.Error("IsStarred should default to false")
- }
- if req.Model != "" {
- t.Error("Model should be empty for old format")
- }
-}
-
-func TestBackwardCompatibility_VersionZero(t *testing.T) {
- // Test that version 0 (unset) is handled correctly
- req := models.CreateChatRequest{
- ID: "chat-123",
- Title: "Test",
- Messages: []models.ChatMessage{},
- Version: 0, // Explicitly zero (or unset)
- }
-
- jsonData, err := json.Marshal(req)
- if err != nil {
- t.Fatalf("Failed to marshal: %v", err)
- }
-
- var parsed models.CreateChatRequest
- err = json.Unmarshal(jsonData, &parsed)
- if err != nil {
- t.Fatalf("Failed to unmarshal: %v", err)
- }
-
- if parsed.Version != 0 {
- t.Errorf("Version should be 0, got %d", parsed.Version)
- }
-}
-
-func TestBackwardCompatibility_MixedTimestampFormats(t *testing.T) {
- // Test both Unix milliseconds (new) and potential variations
- testCases := []struct {
- name string
- timestamp int64
- }{
- {"current timestamp", 1700000000000},
- {"zero timestamp", 0},
- {"far future", 9999999999999},
- {"year 2000", 946684800000},
- }
-
- for _, tc := range testCases {
- t.Run(tc.name, func(t *testing.T) {
- message := models.ChatMessage{
- ID: "msg-1",
- Role: "user",
- Content: "Test",
- Timestamp: tc.timestamp,
- }
-
- jsonData, _ := json.Marshal(message)
- var recovered models.ChatMessage
- json.Unmarshal(jsonData, &recovered)
-
- if recovered.Timestamp != tc.timestamp {
- t.Errorf("Timestamp mismatch: expected %d, got %d", tc.timestamp, recovered.Timestamp)
- }
- })
- }
-}
-
-// ==================== VERSION CONFLICT TESTS ====================
-
-func TestVersionConflictDetection(t *testing.T) {
- // Test that version numbers work correctly for conflict detection
- req1 := models.CreateChatRequest{
- ID: "chat-123",
- Title: "Original",
- Messages: []models.ChatMessage{},
- Version: 1,
- }
-
- req2 := models.CreateChatRequest{
- ID: "chat-123",
- Title: "Updated",
- Messages: []models.ChatMessage{},
- Version: 2,
- }
-
- // Simulate version check
- if req2.Version <= req1.Version {
- t.Error("req2 should have higher version than req1")
- }
-}
-
-func TestUpdateRequestPartialFields(t *testing.T) {
- // Test that partial updates work correctly
- title := "New Title"
- req := models.UpdateChatRequest{
- Title: &title,
- Version: 5,
- }
-
- if req.IsStarred != nil {
- t.Error("IsStarred should be nil (not set)")
- }
- if req.Model != nil {
- t.Error("Model should be nil (not set)")
- }
- if *req.Title != title {
- t.Error("Title should be set")
- }
-}
-
-// ==================== ENCRYPTION SECURITY TESTS ====================
-
-func TestEncryptionDeterminism(t *testing.T) {
- // Verify that same plaintext produces different ciphertext each time (due to random nonce)
- masterKey := "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"
- encService, _ := crypto.NewEncryptionService(masterKey)
-
- plaintext := "Same message"
- userID := "user-123"
-
- encrypted1, _ := encService.EncryptString(userID, plaintext)
- encrypted2, _ := encService.EncryptString(userID, plaintext)
-
- if encrypted1 == encrypted2 {
- t.Error("Same plaintext should produce different ciphertext (random nonce)")
- }
-
- // But both should decrypt to the same value
- decrypted1, _ := encService.DecryptString(userID, encrypted1)
- decrypted2, _ := encService.DecryptString(userID, encrypted2)
-
- if decrypted1 != decrypted2 || decrypted1 != plaintext {
- t.Error("Both encryptions should decrypt to the same plaintext")
- }
-}
-
-func TestDecryptionWithWrongKey(t *testing.T) {
- masterKey1 := "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"
- masterKey2 := "fedcba9876543210fedcba9876543210fedcba9876543210fedcba9876543210"
-
- encService1, _ := crypto.NewEncryptionService(masterKey1)
- encService2, _ := crypto.NewEncryptionService(masterKey2)
-
- plaintext := "Secret message"
- userID := "user-123"
-
- encrypted, _ := encService1.EncryptString(userID, plaintext)
-
- // Try to decrypt with different master key - should fail
- _, err := encService2.DecryptString(userID, encrypted)
- if err == nil {
- t.Error("Decryption with wrong key should fail")
- }
-}
-
-func TestTamperedCiphertext(t *testing.T) {
- masterKey := "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"
- encService, err := crypto.NewEncryptionService(masterKey)
- if err != nil {
- t.Fatalf("Failed to create encryption service: %v", err)
- }
-
- plaintext := "Secret message"
- userID := "user-123"
-
- encrypted, err := encService.EncryptString(userID, plaintext)
- if err != nil {
- t.Fatalf("Failed to encrypt: %v", err)
- }
-
- if len(encrypted) < 20 {
- t.Skip("Encrypted string too short for tampering test")
- }
-
- // Tamper with the ciphertext - flip multiple characters to ensure GCM detects it
- tampered := encrypted[:len(encrypted)-5] + "XXXXX"
-
- _, err = encService.DecryptString(userID, tampered)
- if err == nil {
- t.Error("Tampered ciphertext should fail to decrypt")
- }
-}
-
-func TestFullEncryptionDecryptionPipeline(t *testing.T) {
- // Simulate the full pipeline: messages -> JSON -> encrypt -> decrypt -> JSON -> messages
- masterKey := "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"
- encService, err := crypto.NewEncryptionService(masterKey)
- if err != nil {
- t.Fatalf("Failed to create encryption service: %v", err)
- }
-
- userID := "user-123"
-
- // Create test messages
- originalMessages := []models.ChatMessage{
- {
- ID: "msg-1",
- Role: "user",
- Content: "What's the weather like?",
- Timestamp: 1700000000000,
- },
- {
- ID: "msg-2",
- Role: "assistant",
- Content: "I'd be happy to help with weather information! However, I don't have access to real-time weather data.",
- Timestamp: 1700000001000,
- },
- {
- ID: "msg-3",
- Role: "user",
- Content: "Thanks!",
- Timestamp: 1700000002000,
- Attachments: []models.ChatAttachment{
- {FileID: "att-1", Filename: "image.png", Type: "image", MimeType: "image/png", Size: 5000},
- },
- },
- }
-
- // Step 1: Serialize to JSON
- jsonData, err := json.Marshal(originalMessages)
- if err != nil {
- t.Fatalf("Failed to serialize messages: %v", err)
- }
-
- // Step 2: Encrypt
- encrypted, err := encService.Encrypt(userID, jsonData)
- if err != nil {
- t.Fatalf("Failed to encrypt: %v", err)
- }
-
- // Step 3: Decrypt
- decrypted, err := encService.Decrypt(userID, encrypted)
- if err != nil {
- t.Fatalf("Failed to decrypt: %v", err)
- }
-
- // Step 4: Deserialize
- var recoveredMessages []models.ChatMessage
- err = json.Unmarshal(decrypted, &recoveredMessages)
- if err != nil {
- t.Fatalf("Failed to deserialize messages: %v", err)
- }
-
- // Verify all messages match
- if len(recoveredMessages) != len(originalMessages) {
- t.Fatalf("Message count mismatch: got %d, want %d", len(recoveredMessages), len(originalMessages))
- }
-
- for i, original := range originalMessages {
- recovered := recoveredMessages[i]
- if recovered.ID != original.ID {
- t.Errorf("Message %d: ID mismatch", i)
- }
- if recovered.Role != original.Role {
- t.Errorf("Message %d: Role mismatch", i)
- }
- if recovered.Content != original.Content {
- t.Errorf("Message %d: Content mismatch", i)
- }
- if recovered.Timestamp != original.Timestamp {
- t.Errorf("Message %d: Timestamp mismatch", i)
- }
- if len(recovered.Attachments) != len(original.Attachments) {
- t.Errorf("Message %d: Attachment count mismatch", i)
- }
- }
-}
diff --git a/backend/internal/services/config_service.go b/backend/internal/services/config_service.go
deleted file mode 100644
index 1e3bead3..00000000
--- a/backend/internal/services/config_service.go
+++ /dev/null
@@ -1,150 +0,0 @@
-package services
-
-import (
- "claraverse/internal/models"
- "log"
- "sync"
-)
-
-// ConfigService handles configuration management
-type ConfigService struct {
- mu sync.RWMutex
- recommendedModels map[int]*models.RecommendedModels // Provider ID -> Recommended Models
- modelAliases map[int]map[string]models.ModelAlias // Provider ID -> (Model Name -> Alias Info)
- providerSecurity map[int]bool // Provider ID -> Secure flag
-}
-
-var (
- configServiceInstance *ConfigService
- configServiceOnce sync.Once
-)
-
-// GetConfigService returns the singleton config service instance
-func GetConfigService() *ConfigService {
- configServiceOnce.Do(func() {
- configServiceInstance = &ConfigService{
- recommendedModels: make(map[int]*models.RecommendedModels),
- modelAliases: make(map[int]map[string]models.ModelAlias),
- providerSecurity: make(map[int]bool),
- }
- })
- return configServiceInstance
-}
-
-// SetRecommendedModels stores recommended models for a provider
-func (s *ConfigService) SetRecommendedModels(providerID int, recommended *models.RecommendedModels) {
- if recommended == nil {
- return
- }
-
- s.mu.Lock()
- defer s.mu.Unlock()
-
- s.recommendedModels[providerID] = recommended
- log.Printf("📌 [CONFIG] Set recommended models for provider %d: top=%s, medium=%s, fastest=%s",
- providerID, recommended.Top, recommended.Medium, recommended.Fastest)
-}
-
-// GetRecommendedModels retrieves recommended models for a provider
-func (s *ConfigService) GetRecommendedModels(providerID int) *models.RecommendedModels {
- s.mu.RLock()
- defer s.mu.RUnlock()
-
- return s.recommendedModels[providerID]
-}
-
-// GetAllRecommendedModels retrieves all recommended models across providers
-func (s *ConfigService) GetAllRecommendedModels() map[int]*models.RecommendedModels {
- s.mu.RLock()
- defer s.mu.RUnlock()
-
- // Create a copy to avoid race conditions
- result := make(map[int]*models.RecommendedModels)
- for k, v := range s.recommendedModels {
- result[k] = v
- }
-
- return result
-}
-
-// SetModelAliases stores model aliases for a provider
-func (s *ConfigService) SetModelAliases(providerID int, aliases map[string]models.ModelAlias) {
- if aliases == nil || len(aliases) == 0 {
- return
- }
-
- s.mu.Lock()
- defer s.mu.Unlock()
-
- s.modelAliases[providerID] = aliases
- log.Printf("📌 [CONFIG] Set %d model aliases for provider %d", len(aliases), providerID)
-}
-
-// GetModelAliases retrieves model aliases for a provider
-func (s *ConfigService) GetModelAliases(providerID int) map[string]models.ModelAlias {
- s.mu.RLock()
- defer s.mu.RUnlock()
-
- return s.modelAliases[providerID]
-}
-
-// GetAllModelAliases retrieves all model aliases across providers
-func (s *ConfigService) GetAllModelAliases() map[int]map[string]models.ModelAlias {
- s.mu.RLock()
- defer s.mu.RUnlock()
-
- // Create a copy to avoid race conditions
- result := make(map[int]map[string]models.ModelAlias)
- for k, v := range s.modelAliases {
- result[k] = v
- }
-
- return result
-}
-
-// GetAliasForModel checks if a model has an alias configured
-func (s *ConfigService) GetAliasForModel(providerID int, modelName string) *models.ModelAlias {
- s.mu.RLock()
- defer s.mu.RUnlock()
-
- if aliases, exists := s.modelAliases[providerID]; exists {
- if alias, found := aliases[modelName]; found {
- return &alias
- }
- }
-
- return nil
-}
-
-// GetModelAlias retrieves a specific alias by its key for a provider
-func (s *ConfigService) GetModelAlias(providerID int, aliasKey string) *models.ModelAlias {
- s.mu.RLock()
- defer s.mu.RUnlock()
-
- if aliases, exists := s.modelAliases[providerID]; exists {
- if alias, found := aliases[aliasKey]; found {
- return &alias
- }
- }
-
- return nil
-}
-
-// SetProviderSecure stores the secure flag for a provider
-func (s *ConfigService) SetProviderSecure(providerID int, secure bool) {
- s.mu.Lock()
- defer s.mu.Unlock()
-
- s.providerSecurity[providerID] = secure
- if secure {
- log.Printf("🔒 [CONFIG] Provider %d marked as secure (doesn't store user data)", providerID)
- }
-}
-
-// IsProviderSecure checks if a provider is marked as secure
-func (s *ConfigService) IsProviderSecure(providerID int) bool {
- s.mu.RLock()
- defer s.mu.RUnlock()
-
- return s.providerSecurity[providerID]
-}
diff --git a/backend/internal/services/connection_manager.go b/backend/internal/services/connection_manager.go
deleted file mode 100644
index a3a32a84..00000000
--- a/backend/internal/services/connection_manager.go
+++ /dev/null
@@ -1,67 +0,0 @@
-package services
-
-import (
- "claraverse/internal/models"
- "log"
- "sync"
-)
-
-// ConnectionManager manages all active WebSocket connections
-type ConnectionManager struct {
- connections map[string]*models.UserConnection
- mutex sync.RWMutex
-}
-
-// NewConnectionManager creates a new connection manager
-func NewConnectionManager() *ConnectionManager {
- return &ConnectionManager{
- connections: make(map[string]*models.UserConnection),
- }
-}
-
-// Add adds a new connection
-func (cm *ConnectionManager) Add(conn *models.UserConnection) {
- cm.mutex.Lock()
- defer cm.mutex.Unlock()
- cm.connections[conn.ConnID] = conn
- log.Printf("✅ Connection added: %s (Total: %d)", conn.ConnID, len(cm.connections))
-}
-
-// Remove removes a connection
-func (cm *ConnectionManager) Remove(connID string) {
- cm.mutex.Lock()
- defer cm.mutex.Unlock()
- if conn, exists := cm.connections[connID]; exists {
- close(conn.WriteChan)
- close(conn.StopChan)
- delete(cm.connections, connID)
- log.Printf("❌ Connection removed: %s (Total: %d)", connID, len(cm.connections))
- }
-}
-
-// Get retrieves a connection by ID
-func (cm *ConnectionManager) Get(connID string) (*models.UserConnection, bool) {
- cm.mutex.RLock()
- defer cm.mutex.RUnlock()
- conn, exists := cm.connections[connID]
- return conn, exists
-}
-
-// Count returns the number of active connections
-func (cm *ConnectionManager) Count() int {
- cm.mutex.RLock()
- defer cm.mutex.RUnlock()
- return len(cm.connections)
-}
-
-// GetAll returns all active connections
-func (cm *ConnectionManager) GetAll() []*models.UserConnection {
- cm.mutex.RLock()
- defer cm.mutex.RUnlock()
-
- conns := make([]*models.UserConnection, 0, len(cm.connections))
- for _, conn := range cm.connections {
- conns = append(conns, conn)
- }
- return conns
-}
diff --git a/backend/internal/services/credential_service.go b/backend/internal/services/credential_service.go
deleted file mode 100644
index 2d473aae..00000000
--- a/backend/internal/services/credential_service.go
+++ /dev/null
@@ -1,622 +0,0 @@
-package services
-
-import (
- "claraverse/internal/crypto"
- "claraverse/internal/database"
- "claraverse/internal/models"
- "context"
- "encoding/json"
- "fmt"
- "io"
- "log"
- "net/http"
- "os"
- "time"
-
- "go.mongodb.org/mongo-driver/bson"
- "go.mongodb.org/mongo-driver/bson/primitive"
- "go.mongodb.org/mongo-driver/mongo"
- "go.mongodb.org/mongo-driver/mongo/options"
-)
-
-const (
- // CollectionCredentials is the MongoDB collection name
- CollectionCredentials = "credentials"
-)
-
-// CredentialService manages encrypted credentials for integrations
-type CredentialService struct {
- mongoDB *database.MongoDB
- encryption *crypto.EncryptionService
-}
-
-// NewCredentialService creates a new credential service
-func NewCredentialService(mongoDB *database.MongoDB, encryption *crypto.EncryptionService) *CredentialService {
- return &CredentialService{
- mongoDB: mongoDB,
- encryption: encryption,
- }
-}
-
-// collection returns the credentials collection
-func (s *CredentialService) collection() *mongo.Collection {
- return s.mongoDB.Database().Collection(CollectionCredentials)
-}
-
-// Create creates a new credential with encrypted data
-func (s *CredentialService) Create(ctx context.Context, userID string, req *models.CreateCredentialRequest) (*models.CredentialListItem, error) {
- // Validate integration type exists
- integration, exists := models.GetIntegration(req.IntegrationType)
- if !exists {
- return nil, fmt.Errorf("unknown integration type: %s", req.IntegrationType)
- }
-
- // Validate required fields
- if err := models.ValidateCredentialData(req.IntegrationType, req.Data); err != nil {
- return nil, err
- }
-
- // Serialize data to JSON
- dataJSON, err := json.Marshal(req.Data)
- if err != nil {
- return nil, fmt.Errorf("failed to serialize credential data: %w", err)
- }
-
- // Encrypt the data
- encryptedData, err := s.encryption.Encrypt(userID, dataJSON)
- if err != nil {
- return nil, fmt.Errorf("failed to encrypt credential data: %w", err)
- }
-
- // Generate masked preview
- maskedPreview := models.GenerateMaskedPreview(req.IntegrationType, req.Data)
-
- now := time.Now()
- credential := &models.Credential{
- UserID: userID,
- Name: req.Name,
- IntegrationType: req.IntegrationType,
- EncryptedData: encryptedData,
- Metadata: models.CredentialMetadata{
- MaskedPreview: maskedPreview,
- Icon: integration.Icon,
- UsageCount: 0,
- TestStatus: "pending",
- },
- CreatedAt: now,
- UpdatedAt: now,
- }
-
- result, err := s.collection().InsertOne(ctx, credential)
- if err != nil {
- return nil, fmt.Errorf("failed to create credential: %w", err)
- }
-
- credential.ID = result.InsertedID.(primitive.ObjectID)
-
- log.Printf("🔐 [CREDENTIAL] Created credential %s (%s) for user %s",
- credential.ID.Hex(), req.IntegrationType, userID)
-
- return credential.ToListItem(), nil
-}
-
-// GetByID retrieves a credential by ID (metadata only, no decryption)
-func (s *CredentialService) GetByID(ctx context.Context, credentialID primitive.ObjectID) (*models.Credential, error) {
- var credential models.Credential
- err := s.collection().FindOne(ctx, bson.M{"_id": credentialID}).Decode(&credential)
- if err != nil {
- if err == mongo.ErrNoDocuments {
- return nil, fmt.Errorf("credential not found")
- }
- return nil, fmt.Errorf("failed to get credential: %w", err)
- }
- return &credential, nil
-}
-
-// GetByIDAndUser retrieves a credential ensuring user ownership
-func (s *CredentialService) GetByIDAndUser(ctx context.Context, credentialID primitive.ObjectID, userID string) (*models.Credential, error) {
- var credential models.Credential
- err := s.collection().FindOne(ctx, bson.M{
- "_id": credentialID,
- "userId": userID,
- }).Decode(&credential)
- if err != nil {
- if err == mongo.ErrNoDocuments {
- return nil, fmt.Errorf("credential not found")
- }
- return nil, fmt.Errorf("failed to get credential: %w", err)
- }
- return &credential, nil
-}
-
-// GetDecrypted retrieves and decrypts a credential for tool use
-// SECURITY: This should ONLY be called by tools, never exposed to API/LLM
-func (s *CredentialService) GetDecrypted(ctx context.Context, userID string, credentialID primitive.ObjectID) (*models.DecryptedCredential, error) {
- // Get the credential with ownership verification
- credential, err := s.GetByIDAndUser(ctx, credentialID, userID)
- if err != nil {
- return nil, err
- }
-
- // Decrypt the data
- decryptedJSON, err := s.encryption.Decrypt(userID, credential.EncryptedData)
- if err != nil {
- log.Printf("⚠️ [CREDENTIAL] Decryption failed for credential %s: %v", credentialID.Hex(), err)
- return nil, fmt.Errorf("failed to decrypt credential")
- }
-
- // Parse JSON
- var data map[string]interface{}
- if err := json.Unmarshal(decryptedJSON, &data); err != nil {
- return nil, fmt.Errorf("failed to parse credential data: %w", err)
- }
-
- // Update usage stats asynchronously
- go s.updateUsageStats(context.Background(), credentialID)
-
- return &models.DecryptedCredential{
- ID: credential.ID.Hex(),
- Name: credential.Name,
- IntegrationType: credential.IntegrationType,
- Data: data,
- }, nil
-}
-
-// GetDecryptedByName retrieves and decrypts a credential by name for tool use
-// SECURITY: This should ONLY be called by tools, never exposed to API/LLM
-func (s *CredentialService) GetDecryptedByName(ctx context.Context, userID string, integrationType string, name string) (*models.DecryptedCredential, error) {
- var credential models.Credential
- err := s.collection().FindOne(ctx, bson.M{
- "userId": userID,
- "integrationType": integrationType,
- "name": name,
- }).Decode(&credential)
- if err != nil {
- if err == mongo.ErrNoDocuments {
- return nil, fmt.Errorf("credential not found")
- }
- return nil, fmt.Errorf("failed to get credential: %w", err)
- }
-
- // Decrypt the data
- decryptedJSON, err := s.encryption.Decrypt(userID, credential.EncryptedData)
- if err != nil {
- log.Printf("⚠️ [CREDENTIAL] Decryption failed for credential %s: %v", credential.ID.Hex(), err)
- return nil, fmt.Errorf("failed to decrypt credential")
- }
-
- // Parse JSON
- var data map[string]interface{}
- if err := json.Unmarshal(decryptedJSON, &data); err != nil {
- return nil, fmt.Errorf("failed to parse credential data: %w", err)
- }
-
- // Update usage stats asynchronously
- go s.updateUsageStats(context.Background(), credential.ID)
-
- return &models.DecryptedCredential{
- ID: credential.ID.Hex(),
- Name: credential.Name,
- IntegrationType: credential.IntegrationType,
- Data: data,
- }, nil
-}
-
-// ListByUser returns all credentials for a user (metadata only)
-func (s *CredentialService) ListByUser(ctx context.Context, userID string) ([]*models.CredentialListItem, error) {
- cursor, err := s.collection().Find(ctx, bson.M{
- "userId": userID,
- }, options.Find().SetSort(bson.D{
- {Key: "integrationType", Value: 1},
- {Key: "name", Value: 1},
- }))
- if err != nil {
- return nil, fmt.Errorf("failed to list credentials: %w", err)
- }
- defer cursor.Close(ctx)
-
- var credentials []*models.CredentialListItem
- for cursor.Next(ctx) {
- var cred models.Credential
- if err := cursor.Decode(&cred); err != nil {
- continue
- }
- credentials = append(credentials, cred.ToListItem())
- }
-
- if credentials == nil {
- credentials = []*models.CredentialListItem{}
- }
-
- return credentials, nil
-}
-
-// ListByUserAndType returns credentials for a specific integration type
-func (s *CredentialService) ListByUserAndType(ctx context.Context, userID string, integrationType string) ([]*models.CredentialListItem, error) {
- cursor, err := s.collection().Find(ctx, bson.M{
- "userId": userID,
- "integrationType": integrationType,
- }, options.Find().SetSort(bson.D{{Key: "name", Value: 1}}))
- if err != nil {
- return nil, fmt.Errorf("failed to list credentials: %w", err)
- }
- defer cursor.Close(ctx)
-
- var credentials []*models.CredentialListItem
- for cursor.Next(ctx) {
- var cred models.Credential
- if err := cursor.Decode(&cred); err != nil {
- continue
- }
- credentials = append(credentials, cred.ToListItem())
- }
-
- if credentials == nil {
- credentials = []*models.CredentialListItem{}
- }
-
- return credentials, nil
-}
-
-// Update updates a credential's name and/or data
-func (s *CredentialService) Update(ctx context.Context, credentialID primitive.ObjectID, userID string, req *models.UpdateCredentialRequest) (*models.CredentialListItem, error) {
- // Get existing credential
- credential, err := s.GetByIDAndUser(ctx, credentialID, userID)
- if err != nil {
- return nil, err
- }
-
- updateFields := bson.M{
- "updatedAt": time.Now(),
- }
-
- // Update name if provided
- if req.Name != "" {
- updateFields["name"] = req.Name
- }
-
- // Update data if provided (requires re-encryption)
- if req.Data != nil {
- // Validate the new data
- if err := models.ValidateCredentialData(credential.IntegrationType, req.Data); err != nil {
- return nil, err
- }
-
- // Serialize and encrypt new data
- dataJSON, err := json.Marshal(req.Data)
- if err != nil {
- return nil, fmt.Errorf("failed to serialize credential data: %w", err)
- }
-
- encryptedData, err := s.encryption.Encrypt(userID, dataJSON)
- if err != nil {
- return nil, fmt.Errorf("failed to encrypt credential data: %w", err)
- }
-
- updateFields["encryptedData"] = encryptedData
- updateFields["metadata.maskedPreview"] = models.GenerateMaskedPreview(credential.IntegrationType, req.Data)
- updateFields["metadata.testStatus"] = "pending" // Reset test status
- }
-
- _, err = s.collection().UpdateByID(ctx, credentialID, bson.M{"$set": updateFields})
- if err != nil {
- return nil, fmt.Errorf("failed to update credential: %w", err)
- }
-
- // Get updated credential
- updated, err := s.GetByIDAndUser(ctx, credentialID, userID)
- if err != nil {
- return nil, err
- }
-
- log.Printf("📝 [CREDENTIAL] Updated credential %s for user %s", credentialID.Hex(), userID)
-
- return updated.ToListItem(), nil
-}
-
-// Delete permanently deletes a credential
-func (s *CredentialService) Delete(ctx context.Context, credentialID primitive.ObjectID, userID string) error {
- // CRITICAL: Get credential data BEFORE deletion to revoke Composio connections
- credential, err := s.GetByIDAndUser(ctx, credentialID, userID)
- if err != nil {
- return err
- }
-
- // Revoke Composio OAuth if this is a Composio integration
- if err := s.revokeComposioIfNeeded(ctx, credential); err != nil {
- log.Printf("⚠️ [CREDENTIAL] Failed to revoke Composio connection for %s: %v", credentialID.Hex(), err)
- // Continue with deletion even if revocation fails (connection might already be invalid)
- }
-
- result, err := s.collection().DeleteOne(ctx, bson.M{
- "_id": credentialID,
- "userId": userID,
- })
- if err != nil {
- return fmt.Errorf("failed to delete credential: %w", err)
- }
-
- if result.DeletedCount == 0 {
- return fmt.Errorf("credential not found")
- }
-
- log.Printf("🗑️ [CREDENTIAL] Deleted credential %s for user %s", credentialID.Hex(), userID)
- return nil
-}
-
-// DeleteAllByUser deletes all credentials for a user (for account deletion)
-func (s *CredentialService) DeleteAllByUser(ctx context.Context, userID string) (int64, error) {
- result, err := s.collection().DeleteMany(ctx, bson.M{
- "userId": userID,
- })
- if err != nil {
- return 0, fmt.Errorf("failed to delete credentials: %w", err)
- }
-
- log.Printf("🗑️ [CREDENTIAL] Deleted %d credentials for user %s", result.DeletedCount, userID)
- return result.DeletedCount, nil
-}
-
-// UpdateTestStatus updates the test status of a credential
-func (s *CredentialService) UpdateTestStatus(ctx context.Context, credentialID primitive.ObjectID, userID string, status string, err error) error {
- updateFields := bson.M{
- "metadata.testStatus": status,
- "metadata.lastTestAt": time.Now(),
- "updatedAt": time.Now(),
- }
-
- _, updateErr := s.collection().UpdateOne(ctx, bson.M{
- "_id": credentialID,
- "userId": userID,
- }, bson.M{"$set": updateFields})
- if updateErr != nil {
- return fmt.Errorf("failed to update test status: %w", updateErr)
- }
-
- return nil
-}
-
-// updateUsageStats updates the usage statistics for a credential
-func (s *CredentialService) updateUsageStats(ctx context.Context, credentialID primitive.ObjectID) {
- _, err := s.collection().UpdateByID(ctx, credentialID, bson.M{
- "$set": bson.M{
- "metadata.lastUsedAt": time.Now(),
- },
- "$inc": bson.M{
- "metadata.usageCount": 1,
- },
- })
- if err != nil {
- log.Printf("⚠️ [CREDENTIAL] Failed to update usage stats: %v", err)
- }
-}
-
-// CountByUser counts credentials for a user
-func (s *CredentialService) CountByUser(ctx context.Context, userID string) (int64, error) {
- count, err := s.collection().CountDocuments(ctx, bson.M{
- "userId": userID,
- })
- if err != nil {
- return 0, fmt.Errorf("failed to count credentials: %w", err)
- }
- return count, nil
-}
-
-// CountByUserAndType counts credentials for a user by type
-func (s *CredentialService) CountByUserAndType(ctx context.Context, userID string, integrationType string) (int64, error) {
- count, err := s.collection().CountDocuments(ctx, bson.M{
- "userId": userID,
- "integrationType": integrationType,
- })
- if err != nil {
- return 0, fmt.Errorf("failed to count credentials: %w", err)
- }
- return count, nil
-}
-
-// GetCredentialReferences returns credential references for use in LLM context
-// This returns only names and IDs, safe to show to LLM
-func (s *CredentialService) GetCredentialReferences(ctx context.Context, userID string, integrationTypes []string) ([]models.CredentialReference, error) {
- filter := bson.M{"userId": userID}
- if len(integrationTypes) > 0 {
- filter["integrationType"] = bson.M{"$in": integrationTypes}
- }
-
- cursor, err := s.collection().Find(ctx, filter, options.Find().
- SetProjection(bson.M{
- "_id": 1,
- "name": 1,
- "integrationType": 1,
- }).
- SetSort(bson.D{
- {Key: "integrationType", Value: 1},
- {Key: "name", Value: 1},
- }))
- if err != nil {
- return nil, fmt.Errorf("failed to get credential references: %w", err)
- }
- defer cursor.Close(ctx)
-
- var refs []models.CredentialReference
- for cursor.Next(ctx) {
- var cred struct {
- ID primitive.ObjectID `bson:"_id"`
- Name string `bson:"name"`
- IntegrationType string `bson:"integrationType"`
- }
- if err := cursor.Decode(&cred); err != nil {
- continue
- }
- refs = append(refs, models.CredentialReference{
- ID: cred.ID.Hex(),
- Name: cred.Name,
- IntegrationType: cred.IntegrationType,
- })
- }
-
- if refs == nil {
- refs = []models.CredentialReference{}
- }
-
- return refs, nil
-}
-
-// revokeComposioIfNeeded revokes Composio OAuth connection when deleting a Composio credential
-func (s *CredentialService) revokeComposioIfNeeded(ctx context.Context, credential *models.Credential) error {
- // Only revoke if this is a Composio integration
- if len(credential.IntegrationType) < 9 || credential.IntegrationType[:9] != "composio_" {
- return nil // Not a Composio integration
- }
-
- composioAPIKey := os.Getenv("COMPOSIO_API_KEY")
- if composioAPIKey == "" {
- return fmt.Errorf("COMPOSIO_API_KEY not set")
- }
-
- // Decrypt credential data to get entity_id
- decryptedJSON, err := s.encryption.Decrypt(credential.UserID, credential.EncryptedData)
- if err != nil {
- return fmt.Errorf("failed to decrypt credential: %w", err)
- }
-
- var data map[string]interface{}
- if err := json.Unmarshal(decryptedJSON, &data); err != nil {
- return fmt.Errorf("failed to parse credential data: %w", err)
- }
-
- entityID, ok := data["composio_entity_id"].(string)
- if !ok || entityID == "" {
- return fmt.Errorf("no composio_entity_id found")
- }
-
- // Extract app name from integration type (e.g., "composio_gmail" -> "gmail")
- appName := credential.IntegrationType[9:] // Remove "composio_" prefix
-
- // Get connected account ID from Composio v3 API
- connectedAccountID, err := s.getComposioConnectedAccountID(ctx, composioAPIKey, entityID, appName)
- if err != nil {
- return fmt.Errorf("failed to get connected account: %w", err)
- }
-
- // Delete the connected account (revokes OAuth)
- deleteURL := fmt.Sprintf("https://backend.composio.dev/api/v3/connected_accounts/%s", connectedAccountID)
- req, err := http.NewRequestWithContext(ctx, "DELETE", deleteURL, nil)
- if err != nil {
- return fmt.Errorf("failed to create delete request: %w", err)
- }
-
- req.Header.Set("x-api-key", composioAPIKey)
-
- client := &http.Client{Timeout: 10 * time.Second}
- resp, err := client.Do(req)
- if err != nil {
- return fmt.Errorf("failed to delete connection: %w", err)
- }
- defer resp.Body.Close()
-
- if resp.StatusCode >= 400 && resp.StatusCode != 404 {
- bodyBytes, _ := io.ReadAll(resp.Body)
- return fmt.Errorf("Composio API error (status %d): %s", resp.StatusCode, string(bodyBytes))
- }
-
- log.Printf("✅ [COMPOSIO] Revoked %s connection for entity %s", appName, entityID)
- return nil
-}
-
-// getComposioConnectedAccountID retrieves the connected account ID from Composio v3 API
-func (s *CredentialService) getComposioConnectedAccountID(ctx context.Context, apiKey string, entityID string, appName string) (string, error) {
- url := fmt.Sprintf("https://backend.composio.dev/api/v3/connected_accounts?user_ids=%s", entityID)
- req, err := http.NewRequestWithContext(ctx, "GET", url, nil)
- if err != nil {
- return "", fmt.Errorf("failed to create request: %w", err)
- }
-
- req.Header.Set("x-api-key", apiKey)
-
- client := &http.Client{Timeout: 10 * time.Second}
- resp, err := client.Do(req)
- if err != nil {
- return "", fmt.Errorf("failed to fetch connected accounts: %w", err)
- }
- defer resp.Body.Close()
-
- respBody, _ := io.ReadAll(resp.Body)
-
- if resp.StatusCode >= 400 {
- return "", fmt.Errorf("Composio API error (status %d): %s", resp.StatusCode, string(respBody))
- }
-
- // Parse v3 response
- var response struct {
- Items []struct {
- ID string `json:"id"`
- Toolkit struct {
- Slug string `json:"slug"`
- } `json:"toolkit"`
- } `json:"items"`
- }
- if err := json.Unmarshal(respBody, &response); err != nil {
- return "", fmt.Errorf("failed to parse response: %w", err)
- }
-
- // Find the connected account for this app
- for _, account := range response.Items {
- if account.Toolkit.Slug == appName {
- return account.ID, nil
- }
- }
-
- return "", fmt.Errorf("no %s connection found for entity %s", appName, entityID)
-}
-
-// EnsureIndexes creates the necessary indexes for the credentials collection
-func (s *CredentialService) EnsureIndexes(ctx context.Context) error {
- indexes := []mongo.IndexModel{
- // User ID for listing
- {
- Keys: bson.D{{Key: "userId", Value: 1}},
- },
- // User + integration type for filtering
- {
- Keys: bson.D{
- {Key: "userId", Value: 1},
- {Key: "integrationType", Value: 1},
- },
- },
- // User + name + type for uniqueness (optional, could enforce unique names per type)
- {
- Keys: bson.D{
- {Key: "userId", Value: 1},
- {Key: "integrationType", Value: 1},
- {Key: "name", Value: 1},
- },
- },
- }
-
- _, err := s.collection().Indexes().CreateMany(ctx, indexes)
- if err != nil {
- return fmt.Errorf("failed to create credential indexes: %w", err)
- }
-
- log.Println("✅ [CREDENTIAL] Ensured indexes for credentials collection")
- return nil
-}
-
-// CreateCredentialResolver creates a credential resolver function that can be
-// injected into tool args for runtime credential access.
-// This function is here to avoid import cycles (tools cannot import services).
-func (s *CredentialService) CreateCredentialResolver(userID string) func(credentialID string) (*models.DecryptedCredential, error) {
- return func(credentialID string) (*models.DecryptedCredential, error) {
- objID, err := primitive.ObjectIDFromHex(credentialID)
- if err != nil {
- return nil, fmt.Errorf("invalid credential ID: %w", err)
- }
-
- cred, err := s.GetDecrypted(context.Background(), userID, objID)
- if err != nil {
- return nil, fmt.Errorf("failed to retrieve credential: %w", err)
- }
-
- return cred, nil
- }
-}
diff --git a/backend/internal/services/execution_service.go b/backend/internal/services/execution_service.go
deleted file mode 100644
index 4d411d9b..00000000
--- a/backend/internal/services/execution_service.go
+++ /dev/null
@@ -1,861 +0,0 @@
-package services
-
-import (
- "claraverse/internal/database"
- "claraverse/internal/models"
- "context"
- "encoding/json"
- "fmt"
- "log"
- "regexp"
- "time"
-
- "go.mongodb.org/mongo-driver/bson"
- "go.mongodb.org/mongo-driver/bson/primitive"
- "go.mongodb.org/mongo-driver/mongo"
- "go.mongodb.org/mongo-driver/mongo/options"
-)
-
-// ExecutionRecord is the MongoDB representation of an execution
-type ExecutionRecord struct {
- ID primitive.ObjectID `bson:"_id,omitempty" json:"id"`
- AgentID string `bson:"agentId" json:"agentId"`
- UserID string `bson:"userId" json:"userId"`
- WorkflowVersion int `bson:"workflowVersion" json:"workflowVersion"`
-
- // Trigger info
- TriggerType string `bson:"triggerType" json:"triggerType"` // manual, scheduled, webhook, api
- ScheduleID primitive.ObjectID `bson:"scheduleId,omitempty" json:"scheduleId,omitempty"`
- APIKeyID primitive.ObjectID `bson:"apiKeyId,omitempty" json:"apiKeyId,omitempty"`
-
- // Execution state
- Status string `bson:"status" json:"status"` // pending, running, completed, failed, partial
- Input map[string]interface{} `bson:"input,omitempty" json:"input,omitempty"`
- Output map[string]interface{} `bson:"output,omitempty" json:"output,omitempty"`
- BlockStates map[string]*models.BlockState `bson:"blockStates,omitempty" json:"blockStates,omitempty"`
- Error string `bson:"error,omitempty" json:"error,omitempty"`
-
- // Standardized API response (clean, well-structured output)
- Result string `bson:"result,omitempty" json:"result,omitempty"` // Primary text result
- Artifacts []models.APIArtifact `bson:"artifacts,omitempty" json:"artifacts,omitempty"` // Generated charts/images
- Files []models.APIFile `bson:"files,omitempty" json:"files,omitempty"` // Generated files
-
- // Timing
- StartedAt time.Time `bson:"startedAt" json:"startedAt"`
- CompletedAt *time.Time `bson:"completedAt,omitempty" json:"completedAt,omitempty"`
- DurationMs int64 `bson:"durationMs,omitempty" json:"durationMs,omitempty"`
-
- // TTL (tier-based retention)
- ExpiresAt time.Time `bson:"expiresAt" json:"expiresAt"`
-
- CreatedAt time.Time `bson:"createdAt" json:"createdAt"`
-}
-
-// ExecutionService manages execution history in MongoDB
-type ExecutionService struct {
- mongoDB *database.MongoDB
- tierService *TierService
-}
-
-// NewExecutionService creates a new execution service
-func NewExecutionService(mongoDB *database.MongoDB, tierService *TierService) *ExecutionService {
- return &ExecutionService{
- mongoDB: mongoDB,
- tierService: tierService,
- }
-}
-
-// collection returns the executions collection
-func (s *ExecutionService) collection() *mongo.Collection {
- return s.mongoDB.Database().Collection("executions")
-}
-
-// Create creates a new execution record
-func (s *ExecutionService) Create(ctx context.Context, req *CreateExecutionRequest) (*ExecutionRecord, error) {
- // Calculate retention based on user tier
- retentionDays := 30 // default free tier
- if s.tierService != nil {
- retentionDays = s.tierService.GetExecutionRetentionDays(ctx, req.UserID)
- }
-
- now := time.Now()
- record := &ExecutionRecord{
- AgentID: req.AgentID,
- UserID: req.UserID,
- WorkflowVersion: req.WorkflowVersion,
- TriggerType: req.TriggerType,
- ScheduleID: req.ScheduleID,
- APIKeyID: req.APIKeyID,
- Status: "pending",
- Input: req.Input,
- StartedAt: now,
- ExpiresAt: now.Add(time.Duration(retentionDays) * 24 * time.Hour),
- CreatedAt: now,
- }
-
- result, err := s.collection().InsertOne(ctx, record)
- if err != nil {
- return nil, fmt.Errorf("failed to create execution: %w", err)
- }
-
- record.ID = result.InsertedID.(primitive.ObjectID)
- log.Printf("📝 [EXECUTION] Created execution %s for agent %s (trigger: %s)",
- record.ID.Hex(), req.AgentID, req.TriggerType)
-
- return record, nil
-}
-
-// CreateExecutionRequest contains the data needed to create an execution
-type CreateExecutionRequest struct {
- AgentID string
- UserID string
- WorkflowVersion int
- TriggerType string // manual, scheduled, webhook, api
- ScheduleID primitive.ObjectID
- APIKeyID primitive.ObjectID
- Input map[string]interface{}
-}
-
-// UpdateStatus updates the execution status
-func (s *ExecutionService) UpdateStatus(ctx context.Context, executionID primitive.ObjectID, status string) error {
- update := bson.M{
- "$set": bson.M{
- "status": status,
- },
- }
-
- _, err := s.collection().UpdateByID(ctx, executionID, update)
- if err != nil {
- return fmt.Errorf("failed to update execution status: %w", err)
- }
-
- log.Printf("📊 [EXECUTION] Updated %s status to %s", executionID.Hex(), status)
- return nil
-}
-
-// Complete marks an execution as complete with output
-func (s *ExecutionService) Complete(ctx context.Context, executionID primitive.ObjectID, result *ExecutionCompleteRequest) error {
- now := time.Now()
-
- // Get the execution to calculate duration
- exec, err := s.GetByID(ctx, executionID)
- if err != nil {
- return err
- }
-
- durationMs := now.Sub(exec.StartedAt).Milliseconds()
-
- // Sanitize output and blockStates to remove large base64 data
- // This prevents MongoDB document size limit (16MB) issues
- sanitizedOutput := sanitizeOutputForStorage(result.Output)
- sanitizedBlockStates := sanitizeBlockStatesForStorageV2(result.BlockStates)
-
- // Log sanitization to help debug
- log.Printf("🧹 [EXECUTION] Sanitizing execution %s for storage", executionID.Hex())
-
- update := bson.M{
- "$set": bson.M{
- "status": result.Status,
- "output": sanitizedOutput,
- "blockStates": sanitizedBlockStates,
- "error": result.Error,
- "completedAt": now,
- "durationMs": durationMs,
- // Store clean API response fields
- "result": result.Result,
- "artifacts": result.Artifacts,
- "files": result.Files,
- },
- }
-
- _, err = s.collection().UpdateByID(ctx, executionID, update)
- if err != nil {
- return fmt.Errorf("failed to complete execution: %w", err)
- }
-
- log.Printf("✅ [EXECUTION] Completed %s with status %s (duration: %dms)",
- executionID.Hex(), result.Status, durationMs)
-
- return nil
-}
-
-// ExecutionCompleteRequest contains the completion data
-type ExecutionCompleteRequest struct {
- Status string
- Output map[string]interface{}
- BlockStates map[string]*models.BlockState
- Error string
-
- // Clean API response fields
- Result string // Primary text result
- Artifacts []models.APIArtifact // Generated charts/images
- Files []models.APIFile // Generated files
-}
-
-// GetByID retrieves an execution by ID
-func (s *ExecutionService) GetByID(ctx context.Context, executionID primitive.ObjectID) (*ExecutionRecord, error) {
- var record ExecutionRecord
- err := s.collection().FindOne(ctx, bson.M{"_id": executionID}).Decode(&record)
- if err != nil {
- if err == mongo.ErrNoDocuments {
- return nil, fmt.Errorf("execution not found")
- }
- return nil, fmt.Errorf("failed to get execution: %w", err)
- }
- return &record, nil
-}
-
-// GetByIDAndUser retrieves an execution by ID ensuring user ownership
-func (s *ExecutionService) GetByIDAndUser(ctx context.Context, executionID primitive.ObjectID, userID string) (*ExecutionRecord, error) {
- var record ExecutionRecord
- err := s.collection().FindOne(ctx, bson.M{
- "_id": executionID,
- "userId": userID,
- }).Decode(&record)
- if err != nil {
- if err == mongo.ErrNoDocuments {
- return nil, fmt.Errorf("execution not found")
- }
- return nil, fmt.Errorf("failed to get execution: %w", err)
- }
- return &record, nil
-}
-
-// ListByAgent returns paginated executions for an agent
-func (s *ExecutionService) ListByAgent(ctx context.Context, agentID, userID string, opts *ListExecutionsOptions) (*PaginatedExecutions, error) {
- filter := bson.M{
- "agentId": agentID,
- "userId": userID,
- }
-
- if opts != nil && opts.Status != "" {
- filter["status"] = opts.Status
- }
-
- if opts != nil && opts.TriggerType != "" {
- filter["triggerType"] = opts.TriggerType
- }
-
- return s.listWithFilter(ctx, filter, opts)
-}
-
-// ListByUser returns paginated executions for a user
-func (s *ExecutionService) ListByUser(ctx context.Context, userID string, opts *ListExecutionsOptions) (*PaginatedExecutions, error) {
- filter := bson.M{
- "userId": userID,
- }
-
- if opts != nil && opts.Status != "" {
- filter["status"] = opts.Status
- }
-
- if opts != nil && opts.TriggerType != "" {
- filter["triggerType"] = opts.TriggerType
- }
-
- if opts != nil && opts.AgentID != "" {
- filter["agentId"] = opts.AgentID
- }
-
- return s.listWithFilter(ctx, filter, opts)
-}
-
-// listWithFilter performs the actual paginated list query
-func (s *ExecutionService) listWithFilter(ctx context.Context, filter bson.M, opts *ListExecutionsOptions) (*PaginatedExecutions, error) {
- // Default pagination
- limit := int64(20)
- page := int64(1)
-
- if opts != nil {
- if opts.Limit > 0 && opts.Limit <= 100 {
- limit = int64(opts.Limit)
- }
- if opts.Page > 0 {
- page = int64(opts.Page)
- }
- }
-
- skip := (page - 1) * limit
-
- // Count total
- total, err := s.collection().CountDocuments(ctx, filter)
- if err != nil {
- return nil, fmt.Errorf("failed to count executions: %w", err)
- }
-
- // Find with pagination (newest first)
- findOpts := options.Find().
- SetSort(bson.D{{Key: "startedAt", Value: -1}}).
- SetSkip(skip).
- SetLimit(limit)
-
- cursor, err := s.collection().Find(ctx, filter, findOpts)
- if err != nil {
- return nil, fmt.Errorf("failed to find executions: %w", err)
- }
- defer cursor.Close(ctx)
-
- var executions []ExecutionRecord
- if err := cursor.All(ctx, &executions); err != nil {
- return nil, fmt.Errorf("failed to decode executions: %w", err)
- }
-
- return &PaginatedExecutions{
- Executions: executions,
- Total: total,
- Page: page,
- Limit: limit,
- HasMore: skip+int64(len(executions)) < total,
- }, nil
-}
-
-// ListExecutionsOptions contains query options for listing executions
-type ListExecutionsOptions struct {
- Page int
- Limit int
- Status string // filter by status
- TriggerType string // filter by trigger type
- AgentID string // filter by agent (for user-wide queries)
-}
-
-// PaginatedExecutions is the response for paginated execution lists
-type PaginatedExecutions struct {
- Executions []ExecutionRecord `json:"executions"`
- Total int64 `json:"total"`
- Page int64 `json:"page"`
- Limit int64 `json:"limit"`
- HasMore bool `json:"hasMore"`
-}
-
-// GetStats returns execution statistics for an agent
-func (s *ExecutionService) GetStats(ctx context.Context, agentID, userID string) (*ExecutionStats, error) {
- filter := bson.M{
- "agentId": agentID,
- "userId": userID,
- }
-
- // Get counts by status
- pipeline := mongo.Pipeline{
- {{Key: "$match", Value: filter}},
- {{Key: "$group", Value: bson.M{
- "_id": "$status",
- "count": bson.M{"$sum": 1},
- "avgDuration": bson.M{"$avg": "$durationMs"},
- }}},
- }
-
- cursor, err := s.collection().Aggregate(ctx, pipeline)
- if err != nil {
- return nil, fmt.Errorf("failed to aggregate stats: %w", err)
- }
- defer cursor.Close(ctx)
-
- stats := &ExecutionStats{
- ByStatus: make(map[string]StatusStats),
- }
-
- var results []struct {
- ID string `bson:"_id"`
- Count int64 `bson:"count"`
- AvgDuration float64 `bson:"avgDuration"`
- }
-
- if err := cursor.All(ctx, &results); err != nil {
- return nil, fmt.Errorf("failed to decode stats: %w", err)
- }
-
- for _, r := range results {
- stats.Total += r.Count
- stats.ByStatus[r.ID] = StatusStats{
- Count: r.Count,
- AvgDuration: int64(r.AvgDuration),
- }
- if r.ID == "completed" {
- stats.SuccessCount = r.Count
- } else if r.ID == "failed" {
- stats.FailedCount = r.Count
- }
- }
-
- if stats.Total > 0 {
- stats.SuccessRate = float64(stats.SuccessCount) / float64(stats.Total) * 100
- }
-
- return stats, nil
-}
-
-// ExecutionStats contains aggregated execution statistics
-type ExecutionStats struct {
- Total int64 `json:"total"`
- SuccessCount int64 `json:"successCount"`
- FailedCount int64 `json:"failedCount"`
- SuccessRate float64 `json:"successRate"`
- ByStatus map[string]StatusStats `json:"byStatus"`
-}
-
-// StatusStats contains stats for a single status
-type StatusStats struct {
- Count int64 `json:"count"`
- AvgDuration int64 `json:"avgDurationMs"`
-}
-
-// DeleteExpired removes executions past their TTL (called by cleanup job)
-func (s *ExecutionService) DeleteExpired(ctx context.Context) (int64, error) {
- result, err := s.collection().DeleteMany(ctx, bson.M{
- "expiresAt": bson.M{"$lt": time.Now()},
- })
- if err != nil {
- return 0, fmt.Errorf("failed to delete expired executions: %w", err)
- }
-
- if result.DeletedCount > 0 {
- log.Printf("🗑️ [EXECUTION] Deleted %d expired executions", result.DeletedCount)
- }
-
- return result.DeletedCount, nil
-}
-
-// DeleteAllByUser deletes all executions for a user (GDPR compliance)
-func (s *ExecutionService) DeleteAllByUser(ctx context.Context, userID string) (int64, error) {
- if userID == "" {
- return 0, fmt.Errorf("user ID is required")
- }
-
- result, err := s.collection().DeleteMany(ctx, bson.M{"userId": userID})
- if err != nil {
- return 0, fmt.Errorf("failed to delete user executions: %w", err)
- }
-
- log.Printf("🗑️ [GDPR] Deleted %d executions for user %s", result.DeletedCount, userID)
- return result.DeletedCount, nil
-}
-
-// EnsureIndexes creates the necessary indexes for the executions collection
-func (s *ExecutionService) EnsureIndexes(ctx context.Context) error {
- indexes := []mongo.IndexModel{
- // User + startedAt for listing user's executions
- {
- Keys: bson.D{
- {Key: "userId", Value: 1},
- {Key: "startedAt", Value: -1},
- },
- },
- // Agent + startedAt for listing agent's executions
- {
- Keys: bson.D{
- {Key: "agentId", Value: 1},
- {Key: "startedAt", Value: -1},
- },
- },
- // TTL index for automatic deletion
- {
- Keys: bson.D{{Key: "expiresAt", Value: 1}},
- Options: options.Index().SetExpireAfterSeconds(0),
- },
- // Status index for filtering
- {
- Keys: bson.D{{Key: "status", Value: 1}},
- },
- // Schedule ID for scheduled execution lookups
- {
- Keys: bson.D{{Key: "scheduleId", Value: 1}},
- Options: options.Index().SetSparse(true),
- },
- }
-
- _, err := s.collection().Indexes().CreateMany(ctx, indexes)
- if err != nil {
- return fmt.Errorf("failed to create execution indexes: %w", err)
- }
-
- log.Println("✅ [EXECUTION] Ensured indexes for executions collection")
- return nil
-}
-
-// sanitizeOutputForStorage sanitizes execution output by converting to JSON, stripping base64 data, and converting back
-// This approach handles any nested structure including typed structs
-func sanitizeOutputForStorage(data map[string]interface{}) map[string]interface{} {
- if data == nil {
- return nil
- }
-
- // Marshal to JSON
- jsonBytes, err := json.Marshal(data)
- if err != nil {
- log.Printf("⚠️ [EXECUTION] Failed to marshal output for sanitization: %v", err)
- return data
- }
-
- originalSize := len(jsonBytes)
-
- // Apply regex patterns to strip base64 data
- sanitized := stripBase64FromJSON(string(jsonBytes))
-
- // Unmarshal back
- var result map[string]interface{}
- if err := json.Unmarshal([]byte(sanitized), &result); err != nil {
- log.Printf("⚠️ [EXECUTION] Failed to unmarshal sanitized output: %v", err)
- return data
- }
-
- // Apply internal field filtering to remove noise fields (model, tokens, etc.)
- result = filterInternalFields(result)
-
- newSize := len(sanitized)
- if originalSize != newSize {
- log.Printf("🧹 [EXECUTION] Sanitized output: %d -> %d bytes (%.1f%% reduction)",
- originalSize, newSize, float64(originalSize-newSize)/float64(originalSize)*100)
- }
-
- return result
-}
-
-// filterInternalFields recursively removes internal fields from the output map
-func filterInternalFields(data map[string]interface{}) map[string]interface{} {
- if data == nil {
- return nil
- }
-
- result := make(map[string]interface{})
- for key, value := range data {
- // Skip internal fields
- if internalFieldsToFilter[key] {
- continue
- }
-
- // Recursively filter nested maps
- if nested, ok := value.(map[string]interface{}); ok {
- result[key] = filterInternalFields(nested)
- } else if slice, ok := value.([]interface{}); ok {
- // Handle arrays
- filteredSlice := make([]interface{}, len(slice))
- for i, item := range slice {
- if itemMap, ok := item.(map[string]interface{}); ok {
- filteredSlice[i] = filterInternalFields(itemMap)
- } else {
- filteredSlice[i] = item
- }
- }
- result[key] = filteredSlice
- } else {
- result[key] = value
- }
- }
-
- return result
-}
-
-// sanitizeBlockStatesForStorageV2 sanitizes block states using JSON approach
-func sanitizeBlockStatesForStorageV2(states map[string]*models.BlockState) map[string]*models.BlockState {
- if states == nil {
- return nil
- }
-
- // Marshal to JSON
- jsonBytes, err := json.Marshal(states)
- if err != nil {
- log.Printf("⚠️ [EXECUTION] Failed to marshal block states for sanitization: %v", err)
- return states
- }
-
- originalSize := len(jsonBytes)
-
- // Apply regex patterns to strip base64 data
- sanitized := stripBase64FromJSON(string(jsonBytes))
-
- // Unmarshal back
- var result map[string]*models.BlockState
- if err := json.Unmarshal([]byte(sanitized), &result); err != nil {
- log.Printf("⚠️ [EXECUTION] Failed to unmarshal sanitized block states: %v", err)
- return states
- }
-
- // Apply internal field filtering to block state outputs
- for blockID, state := range result {
- if state != nil && state.Outputs != nil {
- result[blockID].Outputs = filterInternalFields(state.Outputs)
- }
- }
-
- newSize := len(sanitized)
- if originalSize != newSize {
- log.Printf("🧹 [EXECUTION] Sanitized block states: %d -> %d bytes (%.1f%% reduction)",
- originalSize, newSize, float64(originalSize-newSize)/float64(originalSize)*100)
- }
-
- return result
-}
-
-// stripBase64FromJSON removes base64 image data from JSON string using regex
-func stripBase64FromJSON(jsonStr string) string {
- // Pattern 1: data URI images (data:image/xxx;base64,...)
- dataURIPattern := regexp.MustCompile(`"data:image/[^;]+;base64,[A-Za-z0-9+/=]+"`)
- jsonStr = dataURIPattern.ReplaceAllString(jsonStr, `"[BASE64_IMAGE_STRIPPED]"`)
-
- // Pattern 2: Long base64-like strings in "data" fields
- dataFieldPattern := regexp.MustCompile(`"data"\s*:\s*"[A-Za-z0-9+/=]{500,}"`)
- jsonStr = dataFieldPattern.ReplaceAllString(jsonStr, `"data":"[BASE64_DATA_STRIPPED]"`)
-
- // Pattern 3: Long base64-like strings in "image", "plot", "chart" fields
- imageFieldPattern := regexp.MustCompile(`"(image|plot|chart|figure|png|jpeg|base64)"\s*:\s*"[A-Za-z0-9+/=]{500,}"`)
- jsonStr = imageFieldPattern.ReplaceAllString(jsonStr, `"$1":"[BASE64_IMAGE_STRIPPED]"`)
-
- // Pattern 4: Any remaining very long strings that look like base64
- // Note: Go RE2 has max repeat count of 1000, so we use {1000,} to catch long strings
- longStringPattern := regexp.MustCompile(`"[A-Za-z0-9+/=]{1000,}"`)
- jsonStr = longStringPattern.ReplaceAllString(jsonStr, `"[LARGE_DATA_STRIPPED]"`)
-
- // Pattern 5: Handle nested JSON strings containing base64 (e.g., in "Result" field of tool calls)
- // This handles cases where the Result is a JSON string that contains base64
- resultFieldPattern := regexp.MustCompile(`"Result"\s*:\s*"\{[^"]*"data"\s*:\s*\\"[A-Za-z0-9+/=]{100,}\\"[^"]*\}"`)
- if resultFieldPattern.MatchString(jsonStr) {
- // For Result fields containing JSON with base64, we need to escape the replacement
- jsonStr = resultFieldPattern.ReplaceAllStringFunc(jsonStr, func(match string) string {
- // Strip the base64 within the nested JSON
- innerPattern := regexp.MustCompile(`\\"data\\"\s*:\s*\\"[A-Za-z0-9+/=]+\\"`)
- return innerPattern.ReplaceAllString(match, `\"data\":\"[BASE64_STRIPPED]\"`)
- })
- }
-
- return jsonStr
-}
-
-// sanitizeForStorage removes large base64 data from maps to prevent MongoDB document size limit issues
-// It replaces base64 image data with a placeholder while preserving metadata
-func sanitizeForStorage(data map[string]interface{}) map[string]interface{} {
- if data == nil {
- return nil
- }
-
- result := make(map[string]interface{})
- for key, value := range data {
- result[key] = sanitizeValue(value)
- }
- return result
-}
-
-// sanitizeBlockStatesForStorage sanitizes all block states
-func sanitizeBlockStatesForStorage(states map[string]*models.BlockState) map[string]*models.BlockState {
- if states == nil {
- return nil
- }
-
- result := make(map[string]*models.BlockState)
- for blockID, state := range states {
- if state == nil {
- continue
- }
- sanitizedState := &models.BlockState{
- Status: state.Status,
- Inputs: sanitizeForStorage(state.Inputs),
- Outputs: sanitizeForStorage(state.Outputs),
- Error: state.Error,
- StartedAt: state.StartedAt,
- CompletedAt: state.CompletedAt,
- }
- result[blockID] = sanitizedState
- }
- return result
-}
-
-// sanitizeValue recursively sanitizes a value, replacing large base64 strings
-func sanitizeValue(value interface{}) interface{} {
- if value == nil {
- return nil
- }
-
- switch v := value.(type) {
- case string:
- return sanitizeString(v)
- case map[string]interface{}:
- return sanitizeMap(v)
- case []interface{}:
- return sanitizeSlice(v)
- default:
- return value
- }
-}
-
-// sanitizeString checks if a string is base64 image data and replaces it
-func sanitizeString(s string) string {
- // If string is too short, keep it
- if len(s) < 500 {
- return s
- }
-
- // Check for data URI prefix (base64 image)
- if regexp.MustCompile(`^data:image/[^;]+;base64,`).MatchString(s) {
- return "[BASE64_IMAGE_STRIPPED_FOR_STORAGE]"
- }
-
- // Check for long base64-like strings (no spaces, mostly alphanumeric + /+=)
- if regexp.MustCompile(`^[A-Za-z0-9+/=]{500,}$`).MatchString(s) {
- return "[BASE64_DATA_STRIPPED_FOR_STORAGE]"
- }
-
- // If string is extremely long (>100KB), truncate it
- if len(s) > 100000 {
- return s[:1000] + "... [TRUNCATED_FOR_STORAGE]"
- }
-
- return s
-}
-
-// internalFieldsToFilter contains fields that should not be exposed to API consumers
-// These are internal execution details that add noise to the output
-var internalFieldsToFilter = map[string]bool{
- "model": true, // Internal model ID (use _workflowModelId instead)
- "__user_id__": true, // Internal user context
- "_workflowModelId": true, // Internal workflow model reference
- "tokens": true, // Token usage (available in metadata)
- "iterations": true, // Internal execution iterations
- "start": true, // Internal start state
- "value": true, // Redundant with response
- "input": true, // Already stored separately
- "rawResponse": true, // Huge and redundant
-}
-
-// sanitizeMap recursively sanitizes a map
-func sanitizeMap(m map[string]interface{}) map[string]interface{} {
- result := make(map[string]interface{})
-
- for key, value := range m {
- // Skip internal fields that shouldn't be exposed to API consumers
- if internalFieldsToFilter[key] {
- continue
- }
-
- // Special handling for known artifact/image fields
- if key == "artifacts" {
- result[key] = sanitizeArtifacts(value)
- continue
- }
- if key == "plots" || key == "images" || key == "base64_images" {
- result[key] = sanitizePlots(value)
- continue
- }
- // Handle toolCalls array which contains Result fields with JSON+base64
- if key == "toolCalls" {
- result[key] = sanitizeToolCallsForStorage(value)
- continue
- }
- result[key] = sanitizeValue(value)
- }
-
- return result
-}
-
-// sanitizeSlice recursively sanitizes a slice
-func sanitizeSlice(s []interface{}) []interface{} {
- result := make([]interface{}, len(s))
- for i, v := range s {
- result[i] = sanitizeValue(v)
- }
- return result
-}
-
-// sanitizeArtifacts handles the artifacts array, keeping metadata but removing data
-func sanitizeArtifacts(value interface{}) interface{} {
- artifacts, ok := value.([]interface{})
- if !ok {
- return value
- }
-
- result := make([]interface{}, 0, len(artifacts))
- for _, a := range artifacts {
- artifact, ok := a.(map[string]interface{})
- if !ok {
- continue
- }
-
- // Keep metadata, remove actual data
- sanitized := map[string]interface{}{
- "type": artifact["type"],
- "format": artifact["format"],
- "title": artifact["title"],
- "data": "[BASE64_IMAGE_STRIPPED_FOR_STORAGE]",
- }
- result = append(result, sanitized)
- }
-
- log.Printf("🧹 [EXECUTION] Sanitized %d artifacts for storage", len(result))
- return result
-}
-
-// sanitizePlots handles plots/images arrays from E2B responses
-func sanitizePlots(value interface{}) interface{} {
- plots, ok := value.([]interface{})
- if !ok {
- // Could be a single plot as map
- if plotMap, ok := value.(map[string]interface{}); ok {
- return sanitizeSinglePlot(plotMap)
- }
- return value
- }
-
- result := make([]interface{}, 0, len(plots))
- for _, p := range plots {
- if plot, ok := p.(map[string]interface{}); ok {
- result = append(result, sanitizeSinglePlot(plot))
- }
- }
-
- log.Printf("🧹 [EXECUTION] Sanitized %d plots for storage", len(result))
- return result
-}
-
-// sanitizeSinglePlot removes base64 data from a single plot while keeping metadata
-func sanitizeSinglePlot(plot map[string]interface{}) map[string]interface{} {
- result := make(map[string]interface{})
- for k, v := range plot {
- if k == "data" || k == "image" || k == "base64" {
- result[k] = "[BASE64_IMAGE_STRIPPED_FOR_STORAGE]"
- } else {
- result[k] = v
- }
- }
- return result
-}
-
-// sanitizeToolCallsForStorage sanitizes tool call results (JSON strings with base64)
-func sanitizeToolCallsForStorage(toolCalls interface{}) interface{} {
- calls, ok := toolCalls.([]interface{})
- if !ok {
- return toolCalls
- }
-
- result := make([]interface{}, 0, len(calls))
- for _, tc := range calls {
- call, ok := tc.(map[string]interface{})
- if !ok {
- result = append(result, tc)
- continue
- }
-
- sanitized := make(map[string]interface{})
- for k, v := range call {
- if k == "result" {
- // Result is a JSON string - parse, sanitize, re-stringify
- if resultStr, ok := v.(string); ok && len(resultStr) > 1000 {
- var resultData map[string]interface{}
- if err := json.Unmarshal([]byte(resultStr), &resultData); err == nil {
- sanitizedResult := sanitizeMap(resultData)
- if sanitizedJSON, err := json.Marshal(sanitizedResult); err == nil {
- sanitized[k] = string(sanitizedJSON)
- continue
- }
- }
- // If parsing fails, just truncate
- if len(resultStr) > 5000 {
- sanitized[k] = resultStr[:5000] + "... [TRUNCATED]"
- continue
- }
- }
- }
- sanitized[k] = v
- }
- result = append(result, sanitized)
- }
-
- return result
-}
diff --git a/backend/internal/services/execution_service_test.go b/backend/internal/services/execution_service_test.go
deleted file mode 100644
index 72704a08..00000000
--- a/backend/internal/services/execution_service_test.go
+++ /dev/null
@@ -1,241 +0,0 @@
-package services
-
-import (
- "context"
- "testing"
-
- "go.mongodb.org/mongo-driver/bson/primitive"
-)
-
-func TestNewExecutionService(t *testing.T) {
- // Test creation without dependencies (both nil)
- service := NewExecutionService(nil, nil)
- if service == nil {
- t.Fatal("Expected non-nil execution service")
- }
-}
-
-func TestExecutionRecord_Structure(t *testing.T) {
- // Test that ExecutionRecord can be created with all fields
- record := &ExecutionRecord{
- ID: primitive.NewObjectID(),
- AgentID: "agent-123",
- UserID: "user-456",
- WorkflowVersion: 1,
- TriggerType: "manual",
- Status: "pending",
- Input: map[string]interface{}{"topic": "test"},
- }
-
- if record.AgentID != "agent-123" {
- t.Errorf("Expected AgentID 'agent-123', got '%s'", record.AgentID)
- }
-
- if record.TriggerType != "manual" {
- t.Errorf("Expected TriggerType 'manual', got '%s'", record.TriggerType)
- }
-}
-
-func TestCreateExecutionRequest_Validation(t *testing.T) {
- tests := []struct {
- name string
- req CreateExecutionRequest
- wantAgentID string
- wantTrigger string
- }{
- {
- name: "manual trigger",
- req: CreateExecutionRequest{
- AgentID: "agent-1",
- UserID: "user-1",
- WorkflowVersion: 1,
- TriggerType: "manual",
- },
- wantAgentID: "agent-1",
- wantTrigger: "manual",
- },
- {
- name: "scheduled trigger",
- req: CreateExecutionRequest{
- AgentID: "agent-2",
- UserID: "user-2",
- WorkflowVersion: 2,
- TriggerType: "scheduled",
- ScheduleID: primitive.NewObjectID(),
- },
- wantAgentID: "agent-2",
- wantTrigger: "scheduled",
- },
- {
- name: "api trigger",
- req: CreateExecutionRequest{
- AgentID: "agent-3",
- UserID: "user-3",
- WorkflowVersion: 1,
- TriggerType: "api",
- APIKeyID: primitive.NewObjectID(),
- },
- wantAgentID: "agent-3",
- wantTrigger: "api",
- },
- }
-
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- if tt.req.AgentID != tt.wantAgentID {
- t.Errorf("Expected AgentID '%s', got '%s'", tt.wantAgentID, tt.req.AgentID)
- }
- if tt.req.TriggerType != tt.wantTrigger {
- t.Errorf("Expected TriggerType '%s', got '%s'", tt.wantTrigger, tt.req.TriggerType)
- }
- })
- }
-}
-
-func TestListExecutionsOptions_Defaults(t *testing.T) {
- opts := &ListExecutionsOptions{}
-
- // Default values should be zero/empty
- if opts.Page != 0 {
- t.Errorf("Expected default Page 0, got %d", opts.Page)
- }
- if opts.Limit != 0 {
- t.Errorf("Expected default Limit 0, got %d", opts.Limit)
- }
- if opts.Status != "" {
- t.Errorf("Expected empty default Status, got '%s'", opts.Status)
- }
-}
-
-func TestListExecutionsOptions_WithFilters(t *testing.T) {
- opts := &ListExecutionsOptions{
- Page: 2,
- Limit: 50,
- Status: "completed",
- TriggerType: "scheduled",
- AgentID: "agent-123",
- }
-
- if opts.Page != 2 {
- t.Errorf("Expected Page 2, got %d", opts.Page)
- }
- if opts.Limit != 50 {
- t.Errorf("Expected Limit 50, got %d", opts.Limit)
- }
- if opts.Status != "completed" {
- t.Errorf("Expected Status 'completed', got '%s'", opts.Status)
- }
- if opts.TriggerType != "scheduled" {
- t.Errorf("Expected TriggerType 'scheduled', got '%s'", opts.TriggerType)
- }
- if opts.AgentID != "agent-123" {
- t.Errorf("Expected AgentID 'agent-123', got '%s'", opts.AgentID)
- }
-}
-
-func TestPaginatedExecutions_Empty(t *testing.T) {
- result := &PaginatedExecutions{
- Executions: []ExecutionRecord{},
- Total: 0,
- Page: 1,
- Limit: 20,
- HasMore: false,
- }
-
- if len(result.Executions) != 0 {
- t.Errorf("Expected 0 executions, got %d", len(result.Executions))
- }
- if result.HasMore {
- t.Error("Expected HasMore to be false")
- }
-}
-
-func TestExecutionStats_Empty(t *testing.T) {
- stats := &ExecutionStats{
- Total: 0,
- SuccessCount: 0,
- FailedCount: 0,
- SuccessRate: 0,
- ByStatus: make(map[string]StatusStats),
- }
-
- if stats.Total != 0 {
- t.Errorf("Expected Total 0, got %d", stats.Total)
- }
- if stats.SuccessRate != 0 {
- t.Errorf("Expected SuccessRate 0, got %f", stats.SuccessRate)
- }
-}
-
-func TestExecutionStats_Calculations(t *testing.T) {
- // Simulate stats calculation
- stats := &ExecutionStats{
- Total: 100,
- SuccessCount: 85,
- FailedCount: 15,
- SuccessRate: 85.0,
- ByStatus: map[string]StatusStats{
- "completed": {Count: 85, AvgDuration: 1500},
- "failed": {Count: 15, AvgDuration: 2000},
- },
- }
-
- if stats.SuccessRate != 85.0 {
- t.Errorf("Expected SuccessRate 85.0, got %f", stats.SuccessRate)
- }
-
- completedStats, ok := stats.ByStatus["completed"]
- if !ok {
- t.Fatal("Expected 'completed' status in ByStatus")
- }
- if completedStats.Count != 85 {
- t.Errorf("Expected completed count 85, got %d", completedStats.Count)
- }
-}
-
-func TestExecutionCompleteRequest_Fields(t *testing.T) {
- req := &ExecutionCompleteRequest{
- Status: "completed",
- Output: map[string]interface{}{
- "result": "success",
- },
- Error: "",
- }
-
- if req.Status != "completed" {
- t.Errorf("Expected Status 'completed', got '%s'", req.Status)
- }
- if req.Error != "" {
- t.Errorf("Expected empty Error, got '%s'", req.Error)
- }
-}
-
-func TestExecutionCompleteRequest_WithError(t *testing.T) {
- req := &ExecutionCompleteRequest{
- Status: "failed",
- Output: nil,
- Error: "execution timeout",
- }
-
- if req.Status != "failed" {
- t.Errorf("Expected Status 'failed', got '%s'", req.Status)
- }
- if req.Error != "execution timeout" {
- t.Errorf("Expected Error 'execution timeout', got '%s'", req.Error)
- }
-}
-
-// Integration test helper - would need MongoDB to run actual tests
-func TestExecutionService_Integration(t *testing.T) {
- if testing.Short() {
- t.Skip("Skipping integration test in short mode")
- }
-
- // This test would require a running MongoDB instance
- // For now, just verify the service can be created
- _ = context.Background()
- service := NewExecutionService(nil, nil)
- if service == nil {
- t.Fatal("Expected non-nil service")
- }
-}
diff --git a/backend/internal/services/file_cache.go b/backend/internal/services/file_cache.go
deleted file mode 100644
index 575836e7..00000000
--- a/backend/internal/services/file_cache.go
+++ /dev/null
@@ -1,437 +0,0 @@
-package services
-
-import (
- "claraverse/internal/security"
- "fmt"
- "log"
- "os"
- "sync"
- "time"
-
- "github.com/patrickmn/go-cache"
-)
-
-// CachedFile represents a file stored in memory cache
-type CachedFile struct {
- FileID string
- UserID string
- ConversationID string
- ExtractedText *security.SecureString // For PDFs
- FileHash security.Hash
- Filename string
- MimeType string
- Size int64
- PageCount int // For PDFs
- WordCount int // For PDFs
- FilePath string // For images (disk location)
- UploadedAt time.Time
-}
-
-// FileCacheService manages uploaded files in memory
-type FileCacheService struct {
- cache *cache.Cache
- mu sync.RWMutex
-}
-
-var (
- fileCacheInstance *FileCacheService
- fileCacheOnce sync.Once
-)
-
-// GetFileCacheService returns the singleton file cache service
-func GetFileCacheService() *FileCacheService {
- fileCacheOnce.Do(func() {
- fileCacheInstance = NewFileCacheService()
- })
- return fileCacheInstance
-}
-
-// NewFileCacheService creates a new file cache service
-func NewFileCacheService() *FileCacheService {
- c := cache.New(30*time.Minute, 10*time.Minute)
-
- // Set eviction handler for secure wiping
- c.OnEvicted(func(key string, value interface{}) {
- if file, ok := value.(*CachedFile); ok {
- log.Printf("🗑️ [FILE-CACHE] Evicting file %s (%s) - secure wiping memory", file.FileID, file.Filename)
- file.SecureWipe()
- }
- })
-
- return &FileCacheService{
- cache: c,
- }
-}
-
-// Store stores a file in the cache
-func (s *FileCacheService) Store(file *CachedFile) {
- s.mu.Lock()
- defer s.mu.Unlock()
- s.cache.Set(file.FileID, file, cache.DefaultExpiration)
- log.Printf("📦 [FILE-CACHE] Stored file %s (%s) - %d bytes, %d words",
- file.FileID, file.Filename, file.Size, file.WordCount)
-}
-
-// Get retrieves a file from the cache
-func (s *FileCacheService) Get(fileID string) (*CachedFile, bool) {
- s.mu.RLock()
- defer s.mu.RUnlock()
-
- value, found := s.cache.Get(fileID)
- if !found {
- return nil, false
- }
-
- file, ok := value.(*CachedFile)
- if !ok {
- return nil, false
- }
-
- return file, true
-}
-
-// GetByUserAndConversation retrieves a file if it belongs to the user and conversation
-func (s *FileCacheService) GetByUserAndConversation(fileID, userID, conversationID string) (*CachedFile, error) {
- file, found := s.Get(fileID)
- if !found {
- return nil, fmt.Errorf("file not found or expired")
- }
-
- // Verify ownership
- if file.UserID != userID {
- return nil, fmt.Errorf("access denied: file belongs to different user")
- }
-
- // Verify conversation
- if file.ConversationID != conversationID {
- return nil, fmt.Errorf("file belongs to different conversation")
- }
-
- return file, nil
-}
-
-// GetFilesForConversation returns all files for a conversation
-func (s *FileCacheService) GetFilesForConversation(conversationID string) []*CachedFile {
- s.mu.RLock()
- defer s.mu.RUnlock()
-
- var files []*CachedFile
- for _, item := range s.cache.Items() {
- if file, ok := item.Object.(*CachedFile); ok {
- if file.ConversationID == conversationID {
- files = append(files, file)
- }
- }
- }
-
- return files
-}
-
-// GetConversationFiles returns all file IDs for a conversation
-func (s *FileCacheService) GetConversationFiles(conversationID string) []string {
- files := s.GetFilesForConversation(conversationID)
- fileIDs := make([]string, 0, len(files))
- for _, file := range files {
- fileIDs = append(fileIDs, file.FileID)
- }
- return fileIDs
-}
-
-// Delete removes a file from the cache and securely wipes it
-func (s *FileCacheService) Delete(fileID string) {
- s.mu.Lock()
- defer s.mu.Unlock()
-
- // Get the file first to wipe it
- if value, found := s.cache.Get(fileID); found {
- if file, ok := value.(*CachedFile); ok {
- log.Printf("🗑️ [FILE-CACHE] Deleting file %s (%s)", file.FileID, file.Filename)
- file.SecureWipe()
- }
- }
-
- s.cache.Delete(fileID)
-}
-
-// DeleteConversationFiles deletes all files for a conversation
-func (s *FileCacheService) DeleteConversationFiles(conversationID string) {
- s.mu.Lock()
- defer s.mu.Unlock()
-
- log.Printf("🗑️ [FILE-CACHE] Deleting all files for conversation %s", conversationID)
-
- for key, item := range s.cache.Items() {
- if file, ok := item.Object.(*CachedFile); ok {
- if file.ConversationID == conversationID {
- file.SecureWipe()
- s.cache.Delete(key)
- }
- }
- }
-}
-
-// ExtendTTL extends the TTL of a file to match conversation lifetime
-func (s *FileCacheService) ExtendTTL(fileID string, duration time.Duration) {
- s.mu.Lock()
- defer s.mu.Unlock()
-
- if value, found := s.cache.Get(fileID); found {
- s.cache.Set(fileID, value, duration)
- log.Printf("⏰ [FILE-CACHE] Extended TTL for file %s to %v", fileID, duration)
- }
-}
-
-// SecureWipe securely wipes the file's sensitive data
-func (f *CachedFile) SecureWipe() {
- if f.ExtractedText != nil {
- f.ExtractedText.Wipe()
- f.ExtractedText = nil
- }
-
- // Delete physical file if it exists (for images)
- if f.FilePath != "" {
- if err := os.Remove(f.FilePath); err != nil && !os.IsNotExist(err) {
- log.Printf("⚠️ Failed to delete file %s: %v", f.FilePath, err)
- } else {
- log.Printf("🗑️ Deleted file from disk: %s", f.FilePath)
- }
- }
-
- // Wipe hash
- for i := range f.FileHash {
- f.FileHash[i] = 0
- }
-
- // Clear other fields
- f.FileID = ""
- f.UserID = ""
- f.ConversationID = ""
- f.Filename = ""
- f.FilePath = ""
-}
-
-// CleanupExpiredFiles deletes files (images, CSV, Excel, JSON, etc.) older than 1 hour
-// This handles all file types stored on disk, not just images
-func (s *FileCacheService) CleanupExpiredFiles() {
- s.mu.Lock()
- defer s.mu.Unlock()
-
- now := time.Now()
- expiredCount := 0
-
- for key, item := range s.cache.Items() {
- if file, ok := item.Object.(*CachedFile); ok {
- // Cleanup all files with disk storage (images, CSV, Excel, JSON, etc.)
- if file.FilePath != "" {
- // Delete if older than 1 hour
- if now.Sub(file.UploadedAt) > 1*time.Hour {
- log.Printf("🗑️ [FILE-CACHE] Deleting expired file: %s (uploaded %v ago)",
- file.Filename, now.Sub(file.UploadedAt))
-
- // Delete from disk
- if err := os.Remove(file.FilePath); err != nil && !os.IsNotExist(err) {
- log.Printf("⚠️ Failed to delete expired file %s: %v", file.FilePath, err)
- }
-
- // Remove from cache
- s.cache.Delete(key)
- expiredCount++
- }
- }
- }
- }
-
- if expiredCount > 0 {
- log.Printf("✅ [FILE-CACHE] Cleaned up %d expired files", expiredCount)
- }
-}
-
-// CleanupOrphanedFiles scans the uploads directory and deletes files that:
-// 1. Are not tracked in the cache (orphaned after server restart)
-// 2. Are older than the maxAge duration
-// This ensures zero retention policy is enforced even after server restarts
-func (s *FileCacheService) CleanupOrphanedFiles(uploadDir string, maxAge time.Duration) {
- s.mu.RLock()
- // Build a set of tracked file paths
- trackedFiles := make(map[string]bool)
- for _, item := range s.cache.Items() {
- if file, ok := item.Object.(*CachedFile); ok {
- if file.FilePath != "" {
- trackedFiles[file.FilePath] = true
- }
- }
- }
- s.mu.RUnlock()
-
- // Scan uploads directory
- entries, err := os.ReadDir(uploadDir)
- if err != nil {
- log.Printf("⚠️ [CLEANUP] Failed to read uploads directory: %v", err)
- return
- }
-
- now := time.Now()
- orphanedCount := 0
- expiredCount := 0
-
- for _, entry := range entries {
- if entry.IsDir() {
- continue
- }
-
- filePath := fmt.Sprintf("%s/%s", uploadDir, entry.Name())
-
- // Get file info
- info, err := entry.Info()
- if err != nil {
- log.Printf("⚠️ [CLEANUP] Failed to get file info for %s: %v", entry.Name(), err)
- continue
- }
-
- fileAge := now.Sub(info.ModTime())
-
- // Check if file is tracked in cache
- if !trackedFiles[filePath] {
- // File is orphaned (not in cache) - delete if older than 5 minutes
- // Give a small grace period for files being uploaded
- if fileAge > 5*time.Minute {
- log.Printf("🗑️ [CLEANUP] Deleting orphaned file: %s (age: %v)", entry.Name(), fileAge)
- if err := os.Remove(filePath); err != nil && !os.IsNotExist(err) {
- log.Printf("⚠️ [CLEANUP] Failed to delete orphaned file %s: %v", entry.Name(), err)
- } else {
- orphanedCount++
- }
- }
- } else if fileAge > maxAge {
- // File is tracked but expired - will be cleaned by cache eviction
- // Just log for now
- expiredCount++
- }
- }
-
- if orphanedCount > 0 || expiredCount > 0 {
- log.Printf("✅ [CLEANUP] Scan complete: deleted %d orphaned files, found %d expired tracked files",
- orphanedCount, expiredCount)
- }
-}
-
-// RunStartupCleanup performs initial cleanup when server starts
-// This is critical for enforcing zero retention policy after restarts
-func (s *FileCacheService) RunStartupCleanup(uploadDir string) {
- log.Printf("🧹 [STARTUP] Running startup file cleanup in %s...", uploadDir)
-
- entries, err := os.ReadDir(uploadDir)
- if err != nil {
- log.Printf("⚠️ [STARTUP] Failed to read uploads directory: %v", err)
- return
- }
-
- now := time.Now()
- deletedCount := 0
-
- for _, entry := range entries {
- if entry.IsDir() {
- continue
- }
-
- filePath := fmt.Sprintf("%s/%s", uploadDir, entry.Name())
-
- // Get file info
- info, err := entry.Info()
- if err != nil {
- continue
- }
-
- // Delete any file older than 1 hour (matching our retention policy)
- // On startup, all files are orphaned since cache is empty
- if now.Sub(info.ModTime()) > 1*time.Hour {
- log.Printf("🗑️ [STARTUP] Deleting stale file: %s (modified: %v ago)",
- entry.Name(), now.Sub(info.ModTime()))
- if err := os.Remove(filePath); err != nil && !os.IsNotExist(err) {
- log.Printf("⚠️ [STARTUP] Failed to delete file %s: %v", entry.Name(), err)
- } else {
- deletedCount++
- }
- }
- }
-
- log.Printf("✅ [STARTUP] Startup cleanup complete: deleted %d stale files", deletedCount)
-}
-
-// GetStats returns cache statistics
-func (s *FileCacheService) GetStats() map[string]interface{} {
- s.mu.RLock()
- defer s.mu.RUnlock()
-
- items := s.cache.Items()
- totalSize := int64(0)
- totalWords := 0
-
- for _, item := range items {
- if file, ok := item.Object.(*CachedFile); ok {
- totalSize += file.Size
- totalWords += file.WordCount
- }
- }
-
- return map[string]interface{}{
- "total_files": len(items),
- "total_size": totalSize,
- "total_words": totalWords,
- }
-}
-
-// GetAllFilesByUser returns metadata for all files owned by a user (for GDPR data export)
-func (s *FileCacheService) GetAllFilesByUser(userID string) []map[string]interface{} {
- s.mu.RLock()
- defer s.mu.RUnlock()
-
- var fileMetadata []map[string]interface{}
-
- for _, item := range s.cache.Items() {
- if file, ok := item.Object.(*CachedFile); ok {
- if file.UserID == userID {
- metadata := map[string]interface{}{
- "file_id": file.FileID,
- "filename": file.Filename,
- "mime_type": file.MimeType,
- "size": file.Size,
- "uploaded_at": file.UploadedAt.Format(time.RFC3339),
- "conversation_id": file.ConversationID,
- }
-
- // Add PDF-specific fields if applicable
- if file.MimeType == "application/pdf" {
- metadata["page_count"] = file.PageCount
- metadata["word_count"] = file.WordCount
- }
-
- fileMetadata = append(fileMetadata, metadata)
- }
- }
- }
-
- return fileMetadata
-}
-
-// DeleteAllFilesByUser deletes all files owned by a user (for GDPR compliance)
-func (s *FileCacheService) DeleteAllFilesByUser(userID string) (int, error) {
- s.mu.Lock()
- defer s.mu.Unlock()
-
- deletedCount := 0
-
- for key, item := range s.cache.Items() {
- if file, ok := item.Object.(*CachedFile); ok {
- if file.UserID == userID {
- log.Printf("🗑️ [GDPR] Deleting file %s (%s) for user %s", file.FileID, file.Filename, userID)
- file.SecureWipe()
- s.cache.Delete(key)
- deletedCount++
- }
- }
- }
-
- log.Printf("✅ [GDPR] Deleted %d files for user %s", deletedCount, userID)
- return deletedCount, nil
-}
diff --git a/backend/internal/services/image_edit_provider_service.go b/backend/internal/services/image_edit_provider_service.go
deleted file mode 100644
index a895762a..00000000
--- a/backend/internal/services/image_edit_provider_service.go
+++ /dev/null
@@ -1,94 +0,0 @@
-package services
-
-import (
- "claraverse/internal/models"
- "log"
- "sync"
-)
-
-// ImageEditProviderConfig holds the configuration for an image editing provider
-type ImageEditProviderConfig struct {
- Name string
- BaseURL string
- APIKey string
- Favicon string
-}
-
-// ImageEditProviderService manages image editing providers
-type ImageEditProviderService struct {
- providers []ImageEditProviderConfig
- mutex sync.RWMutex
-}
-
-var (
- imageEditProviderInstance *ImageEditProviderService
- imageEditProviderOnce sync.Once
-)
-
-// GetImageEditProviderService returns the singleton image edit provider service
-func GetImageEditProviderService() *ImageEditProviderService {
- imageEditProviderOnce.Do(func() {
- imageEditProviderInstance = &ImageEditProviderService{
- providers: make([]ImageEditProviderConfig, 0),
- }
- })
- return imageEditProviderInstance
-}
-
-// LoadFromProviders loads image edit providers from the providers config
-// This is called during provider sync
-func (s *ImageEditProviderService) LoadFromProviders(providers []models.ProviderConfig) {
- s.mutex.Lock()
- defer s.mutex.Unlock()
-
- // Clear existing providers
- s.providers = make([]ImageEditProviderConfig, 0)
-
- for _, p := range providers {
- // Only load enabled providers with image_edit_only flag
- if p.Enabled && p.ImageEditOnly {
- config := ImageEditProviderConfig{
- Name: p.Name,
- BaseURL: p.BaseURL,
- APIKey: p.APIKey,
- Favicon: p.Favicon,
- }
- s.providers = append(s.providers, config)
- log.Printf("🖌️ [IMAGE-EDIT-PROVIDER] Loaded image edit provider: %s", p.Name)
- }
- }
-
- log.Printf("🖌️ [IMAGE-EDIT-PROVIDER] Total image edit providers loaded: %d", len(s.providers))
-}
-
-// GetProvider returns the first enabled image edit provider
-// Returns nil if no image edit providers are configured
-func (s *ImageEditProviderService) GetProvider() *ImageEditProviderConfig {
- s.mutex.RLock()
- defer s.mutex.RUnlock()
-
- if len(s.providers) == 0 {
- return nil
- }
-
- // Return the first provider
- return &s.providers[0]
-}
-
-// GetAllProviders returns all configured image edit providers
-func (s *ImageEditProviderService) GetAllProviders() []ImageEditProviderConfig {
- s.mutex.RLock()
- defer s.mutex.RUnlock()
-
- // Return a copy to prevent external modification
- result := make([]ImageEditProviderConfig, len(s.providers))
- copy(result, s.providers)
- return result
-}
-
-// HasProvider checks if any image edit provider is configured
-func (s *ImageEditProviderService) HasProvider() bool {
- s.mutex.RLock()
- defer s.mutex.RUnlock()
- return len(s.providers) > 0
-}
diff --git a/backend/internal/services/image_provider_service.go b/backend/internal/services/image_provider_service.go
deleted file mode 100644
index 86eff88b..00000000
--- a/backend/internal/services/image_provider_service.go
+++ /dev/null
@@ -1,96 +0,0 @@
-package services
-
-import (
- "claraverse/internal/models"
- "log"
- "sync"
-)
-
-// ImageProviderConfig holds the configuration for an image generation provider
-type ImageProviderConfig struct {
- Name string
- BaseURL string
- APIKey string
- DefaultModel string
- Favicon string
-}
-
-// ImageProviderService manages image generation providers
-type ImageProviderService struct {
- providers []ImageProviderConfig
- mutex sync.RWMutex
-}
-
-var (
- imageProviderInstance *ImageProviderService
- imageProviderOnce sync.Once
-)
-
-// GetImageProviderService returns the singleton image provider service
-func GetImageProviderService() *ImageProviderService {
- imageProviderOnce.Do(func() {
- imageProviderInstance = &ImageProviderService{
- providers: make([]ImageProviderConfig, 0),
- }
- })
- return imageProviderInstance
-}
-
-// LoadFromProviders loads image providers from the providers config
-// This is called during provider sync
-func (s *ImageProviderService) LoadFromProviders(providers []models.ProviderConfig) {
- s.mutex.Lock()
- defer s.mutex.Unlock()
-
- // Clear existing providers
- s.providers = make([]ImageProviderConfig, 0)
-
- for _, p := range providers {
- // Only load enabled providers with image_only flag
- if p.Enabled && p.ImageOnly {
- config := ImageProviderConfig{
- Name: p.Name,
- BaseURL: p.BaseURL,
- APIKey: p.APIKey,
- DefaultModel: p.DefaultModel,
- Favicon: p.Favicon,
- }
- s.providers = append(s.providers, config)
- log.Printf("🎨 [IMAGE-PROVIDER] Loaded image provider: %s (model: %s)", p.Name, p.DefaultModel)
- }
- }
-
- log.Printf("🎨 [IMAGE-PROVIDER] Total image providers loaded: %d", len(s.providers))
-}
-
-// GetProvider returns the first enabled image provider
-// Returns nil if no image providers are configured
-func (s *ImageProviderService) GetProvider() *ImageProviderConfig {
- s.mutex.RLock()
- defer s.mutex.RUnlock()
-
- if len(s.providers) == 0 {
- return nil
- }
-
- // Return the first provider (could be enhanced to support multiple providers)
- return &s.providers[0]
-}
-
-// GetAllProviders returns all configured image providers
-func (s *ImageProviderService) GetAllProviders() []ImageProviderConfig {
- s.mutex.RLock()
- defer s.mutex.RUnlock()
-
- // Return a copy to prevent external modification
- result := make([]ImageProviderConfig, len(s.providers))
- copy(result, s.providers)
- return result
-}
-
-// HasProvider checks if any image provider is configured
-func (s *ImageProviderService) HasProvider() bool {
- s.mutex.RLock()
- defer s.mutex.RUnlock()
- return len(s.providers) > 0
-}
diff --git a/backend/internal/services/image_registry_service.go b/backend/internal/services/image_registry_service.go
deleted file mode 100644
index aabfcb86..00000000
--- a/backend/internal/services/image_registry_service.go
+++ /dev/null
@@ -1,344 +0,0 @@
-package services
-
-import (
- "fmt"
- "log"
- "strings"
- "sync"
- "time"
-)
-
-// ImageEntry represents a registered image in a conversation
-type ImageEntry struct {
- Handle string // "img-1", "img-2", etc.
- FileID string // UUID from filecache
- Filename string // Original or generated name
- Source string // "uploaded", "generated", "edited"
- SourceHandle string // For edited images, which image was source
- Prompt string // For generated/edited images, the prompt used
- Width int // Image width (if known)
- Height int // Image height (if known)
- CreatedAt time.Time // When this entry was created
-}
-
-// ImageRegistry holds the image entries for a single conversation
-type ImageRegistry struct {
- entries map[string]*ImageEntry // handle -> entry
- byFileID map[string]string // fileID -> handle (reverse lookup)
- counter int // for generating handles
- mutex sync.RWMutex
-}
-
-// ImageRegistryService manages per-conversation image registries
-type ImageRegistryService struct {
- registries map[string]*ImageRegistry // conversationID -> registry
- mutex sync.RWMutex
-}
-
-var (
- imageRegistryInstance *ImageRegistryService
- imageRegistryOnce sync.Once
-)
-
-// GetImageRegistryService returns the singleton image registry service
-func GetImageRegistryService() *ImageRegistryService {
- imageRegistryOnce.Do(func() {
- imageRegistryInstance = &ImageRegistryService{
- registries: make(map[string]*ImageRegistry),
- }
- log.Printf("📸 [IMAGE-REGISTRY] Service initialized")
- })
- return imageRegistryInstance
-}
-
-// getOrCreateRegistry gets or creates a registry for a conversation
-func (s *ImageRegistryService) getOrCreateRegistry(conversationID string) *ImageRegistry {
- s.mutex.Lock()
- defer s.mutex.Unlock()
-
- if registry, exists := s.registries[conversationID]; exists {
- return registry
- }
-
- registry := &ImageRegistry{
- entries: make(map[string]*ImageEntry),
- byFileID: make(map[string]string),
- counter: 0,
- }
- s.registries[conversationID] = registry
- return registry
-}
-
-// generateHandle creates the next handle for a registry
-func (r *ImageRegistry) generateHandle() string {
- r.counter++
- return fmt.Sprintf("img-%d", r.counter)
-}
-
-// RegisterUploadedImage registers an uploaded image and returns its handle
-func (s *ImageRegistryService) RegisterUploadedImage(conversationID, fileID, filename string, width, height int) string {
- registry := s.getOrCreateRegistry(conversationID)
-
- registry.mutex.Lock()
- defer registry.mutex.Unlock()
-
- // Check if already registered
- if handle, exists := registry.byFileID[fileID]; exists {
- log.Printf("📸 [IMAGE-REGISTRY] Image already registered: %s -> %s", fileID, handle)
- return handle
- }
-
- handle := registry.generateHandle()
- entry := &ImageEntry{
- Handle: handle,
- FileID: fileID,
- Filename: filename,
- Source: "uploaded",
- Width: width,
- Height: height,
- CreatedAt: time.Now(),
- }
-
- registry.entries[handle] = entry
- registry.byFileID[fileID] = handle
-
- log.Printf("📸 [IMAGE-REGISTRY] Registered uploaded image: %s -> %s (%s)", handle, fileID, filename)
- return handle
-}
-
-// RegisterGeneratedImage registers a generated image and returns its handle
-func (s *ImageRegistryService) RegisterGeneratedImage(conversationID, fileID, prompt string) string {
- registry := s.getOrCreateRegistry(conversationID)
-
- registry.mutex.Lock()
- defer registry.mutex.Unlock()
-
- // Check if already registered
- if handle, exists := registry.byFileID[fileID]; exists {
- return handle
- }
-
- handle := registry.generateHandle()
-
- // Create a short filename from prompt
- filename := truncatePromptForFilename(prompt) + ".png"
-
- entry := &ImageEntry{
- Handle: handle,
- FileID: fileID,
- Filename: filename,
- Source: "generated",
- Prompt: prompt,
- Width: 1024, // Default generation size
- Height: 1024,
- CreatedAt: time.Now(),
- }
-
- registry.entries[handle] = entry
- registry.byFileID[fileID] = handle
-
- log.Printf("📸 [IMAGE-REGISTRY] Registered generated image: %s -> %s", handle, fileID)
- return handle
-}
-
-// RegisterEditedImage registers an edited image and returns its handle
-func (s *ImageRegistryService) RegisterEditedImage(conversationID, fileID, sourceHandle, prompt string) string {
- registry := s.getOrCreateRegistry(conversationID)
-
- registry.mutex.Lock()
- defer registry.mutex.Unlock()
-
- // Check if already registered
- if handle, exists := registry.byFileID[fileID]; exists {
- return handle
- }
-
- handle := registry.generateHandle()
-
- // Create filename based on source and edit prompt
- filename := fmt.Sprintf("edited_%s_%s.png", sourceHandle, truncatePromptForFilename(prompt))
-
- entry := &ImageEntry{
- Handle: handle,
- FileID: fileID,
- Filename: filename,
- Source: "edited",
- SourceHandle: sourceHandle,
- Prompt: prompt,
- Width: 1024, // Edited images typically same size
- Height: 1024,
- CreatedAt: time.Now(),
- }
-
- registry.entries[handle] = entry
- registry.byFileID[fileID] = handle
-
- log.Printf("📸 [IMAGE-REGISTRY] Registered edited image: %s -> %s (from %s)", handle, fileID, sourceHandle)
- return handle
-}
-
-// GetByHandle returns an image entry by its handle
-func (s *ImageRegistryService) GetByHandle(conversationID, handle string) *ImageEntry {
- s.mutex.RLock()
- registry, exists := s.registries[conversationID]
- s.mutex.RUnlock()
-
- if !exists {
- return nil
- }
-
- registry.mutex.RLock()
- defer registry.mutex.RUnlock()
-
- return registry.entries[handle]
-}
-
-// GetByFileID returns an image entry by its file ID
-func (s *ImageRegistryService) GetByFileID(conversationID, fileID string) *ImageEntry {
- s.mutex.RLock()
- registry, exists := s.registries[conversationID]
- s.mutex.RUnlock()
-
- if !exists {
- return nil
- }
-
- registry.mutex.RLock()
- defer registry.mutex.RUnlock()
-
- handle, exists := registry.byFileID[fileID]
- if !exists {
- return nil
- }
-
- return registry.entries[handle]
-}
-
-// ListImages returns all images in a conversation
-func (s *ImageRegistryService) ListImages(conversationID string) []*ImageEntry {
- s.mutex.RLock()
- registry, exists := s.registries[conversationID]
- s.mutex.RUnlock()
-
- if !exists {
- return nil
- }
-
- registry.mutex.RLock()
- defer registry.mutex.RUnlock()
-
- result := make([]*ImageEntry, 0, len(registry.entries))
- for _, entry := range registry.entries {
- result = append(result, entry)
- }
-
- return result
-}
-
-// ListHandles returns all available handles for a conversation (for error messages)
-func (s *ImageRegistryService) ListHandles(conversationID string) []string {
- s.mutex.RLock()
- registry, exists := s.registries[conversationID]
- s.mutex.RUnlock()
-
- if !exists {
- return nil
- }
-
- registry.mutex.RLock()
- defer registry.mutex.RUnlock()
-
- handles := make([]string, 0, len(registry.entries))
- for handle := range registry.entries {
- handles = append(handles, handle)
- }
-
- return handles
-}
-
-// BuildSystemContext creates the system prompt injection for available images
-func (s *ImageRegistryService) BuildSystemContext(conversationID string) string {
- images := s.ListImages(conversationID)
- if len(images) == 0 {
- return ""
- }
-
- var sb strings.Builder
- sb.WriteString("[Available Images]\n")
- sb.WriteString("You have access to the following images in this conversation. Use the image ID (e.g., 'img-1') when calling the edit_image tool:\n")
-
- for _, img := range images {
- sb.WriteString(fmt.Sprintf("- %s: \"%s\"", img.Handle, img.Filename))
-
- // Add source info
- switch img.Source {
- case "uploaded":
- sb.WriteString(" (uploaded by user")
- case "generated":
- sb.WriteString(" (generated")
- case "edited":
- sb.WriteString(fmt.Sprintf(" (edited from %s", img.SourceHandle))
- }
-
- // Add dimensions if known
- if img.Width > 0 && img.Height > 0 {
- sb.WriteString(fmt.Sprintf(", %dx%d", img.Width, img.Height))
- }
-
- sb.WriteString(")\n")
- }
-
- return sb.String()
-}
-
-// CleanupConversation removes all registry data for a conversation
-func (s *ImageRegistryService) CleanupConversation(conversationID string) {
- s.mutex.Lock()
- defer s.mutex.Unlock()
-
- if _, exists := s.registries[conversationID]; exists {
- delete(s.registries, conversationID)
- log.Printf("📸 [IMAGE-REGISTRY] Cleaned up conversation: %s", conversationID)
- }
-}
-
-// HasImages checks if a conversation has any registered images
-func (s *ImageRegistryService) HasImages(conversationID string) bool {
- s.mutex.RLock()
- registry, exists := s.registries[conversationID]
- s.mutex.RUnlock()
-
- if !exists {
- return false
- }
-
- registry.mutex.RLock()
- defer registry.mutex.RUnlock()
-
- return len(registry.entries) > 0
-}
-
-// truncatePromptForFilename creates a short filename-safe string from a prompt
-func truncatePromptForFilename(prompt string) string {
- // Take first 30 chars, make filename-safe
- if len(prompt) > 30 {
- prompt = prompt[:30]
- }
-
- // Replace unsafe characters
- safe := strings.Map(func(r rune) rune {
- if (r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z') || (r >= '0' && r <= '9') || r == '_' || r == '-' {
- return r
- }
- if r == ' ' {
- return '_'
- }
- return -1 // Remove other characters
- }, prompt)
-
- if safe == "" {
- safe = "image"
- }
-
- return safe
-}
diff --git a/backend/internal/services/mcp_bridge_service.go b/backend/internal/services/mcp_bridge_service.go
deleted file mode 100644
index 292961d6..00000000
--- a/backend/internal/services/mcp_bridge_service.go
+++ /dev/null
@@ -1,296 +0,0 @@
-package services
-
-import (
- "encoding/json"
- "fmt"
- "log"
- "sync"
- "time"
-
- "claraverse/internal/database"
- "claraverse/internal/models"
- "claraverse/internal/tools"
- "github.com/google/uuid"
-)
-
-// MCPBridgeService manages MCP client connections and tool routing
-type MCPBridgeService struct {
- db *database.DB
- connections map[string]*models.MCPConnection // clientID -> connection
- userConns map[string]string // userID -> clientID
- registry *tools.Registry
- mutex sync.RWMutex
-}
-
-// NewMCPBridgeService creates a new MCP bridge service
-func NewMCPBridgeService(db *database.DB, registry *tools.Registry) *MCPBridgeService {
- return &MCPBridgeService{
- db: db,
- connections: make(map[string]*models.MCPConnection),
- userConns: make(map[string]string),
- registry: registry,
- }
-}
-
-// RegisterClient registers a new MCP client connection
-func (s *MCPBridgeService) RegisterClient(userID string, registration *models.MCPToolRegistration) (*models.MCPConnection, error) {
- s.mutex.Lock()
- defer s.mutex.Unlock()
-
- // Check if user already has a connection
- if existingClientID, exists := s.userConns[userID]; exists {
- // Disconnect existing connection
- if existingConn, ok := s.connections[existingClientID]; ok {
- log.Printf("Disconnecting existing MCP client for user %s", userID)
- s.disconnectClientLocked(existingClientID, existingConn)
- }
- }
-
- // Create new connection
- conn := &models.MCPConnection{
- ID: uuid.New().String(),
- UserID: userID,
- ClientID: registration.ClientID,
- ClientVersion: registration.ClientVersion,
- Platform: registration.Platform,
- ConnectedAt: time.Now(),
- LastHeartbeat: time.Now(),
- IsActive: true,
- Tools: registration.Tools,
- WriteChan: make(chan models.MCPServerMessage, 100),
- StopChan: make(chan bool, 1),
- PendingResults: make(map[string]chan models.MCPToolResult),
- }
-
- // Store in memory
- s.connections[registration.ClientID] = conn
- s.userConns[userID] = registration.ClientID
-
- // Store in database
- _, err := s.db.Exec(`
- INSERT INTO mcp_connections (user_id, client_id, client_version, platform, connected_at, last_heartbeat, is_active)
- VALUES (?, ?, ?, ?, ?, ?, ?)
- `, userID, registration.ClientID, registration.ClientVersion, registration.Platform, conn.ConnectedAt, conn.LastHeartbeat, true)
-
- if err != nil {
- delete(s.connections, registration.ClientID)
- delete(s.userConns, userID)
- return nil, fmt.Errorf("failed to store connection in database: %w", err)
- }
-
- // Get connection ID from database
- var dbConnID int64
- err = s.db.QueryRow("SELECT id FROM mcp_connections WHERE client_id = ?", registration.ClientID).Scan(&dbConnID)
- if err != nil {
- log.Printf("Warning: Failed to get connection ID from database: %v", err)
- }
-
- // Register tools in registry and database
- for _, tool := range registration.Tools {
- // Register in registry
- err := s.registry.RegisterUserTool(userID, &tools.Tool{
- Name: tool.Name,
- Description: tool.Description,
- Parameters: tool.Parameters,
- Source: tools.ToolSourceMCPLocal,
- UserID: userID,
- Execute: nil, // MCP tools don't have direct execute functions
- })
-
- if err != nil {
- log.Printf("Warning: Failed to register tool %s: %v", tool.Name, err)
- continue
- }
-
- // Store tool in database
- toolDefJSON, _ := json.Marshal(tool)
- _, err = s.db.Exec(`
- INSERT OR REPLACE INTO mcp_tools (user_id, connection_id, tool_name, tool_definition)
- VALUES (?, ?, ?, ?)
- `, userID, dbConnID, tool.Name, string(toolDefJSON))
-
- if err != nil {
- log.Printf("Warning: Failed to store tool %s in database: %v", tool.Name, err)
- }
- }
-
- log.Printf("✅ MCP client registered: user=%s, client=%s, tools=%d", userID, registration.ClientID, len(registration.Tools))
-
- // Send acknowledgment
- go func() {
- conn.WriteChan <- models.MCPServerMessage{
- Type: "ack",
- Payload: map[string]interface{}{
- "status": "connected",
- "tools_registered": len(registration.Tools),
- },
- }
- }()
-
- return conn, nil
-}
-
-// DisconnectClient handles client disconnection
-func (s *MCPBridgeService) DisconnectClient(clientID string) error {
- s.mutex.Lock()
- defer s.mutex.Unlock()
-
- conn, exists := s.connections[clientID]
- if !exists {
- return fmt.Errorf("client %s not found", clientID)
- }
-
- s.disconnectClientLocked(clientID, conn)
- return nil
-}
-
-// disconnectClientLocked handles disconnection (must be called with lock held)
-func (s *MCPBridgeService) disconnectClientLocked(clientID string, conn *models.MCPConnection) {
- // Mark as inactive in database
- _, err := s.db.Exec("UPDATE mcp_connections SET is_active = 0 WHERE client_id = ?", clientID)
- if err != nil {
- log.Printf("Warning: Failed to mark connection as inactive: %v", err)
- }
-
- // Unregister all tools
- s.registry.UnregisterAllUserTools(conn.UserID)
-
- // Clean up memory
- delete(s.connections, clientID)
- delete(s.userConns, conn.UserID)
-
- // Close channels
- close(conn.StopChan)
- close(conn.WriteChan)
-
- log.Printf("🔌 MCP client disconnected: user=%s, client=%s", conn.UserID, clientID)
-}
-
-// UpdateHeartbeat updates the last heartbeat time for a client
-func (s *MCPBridgeService) UpdateHeartbeat(clientID string) error {
- s.mutex.Lock()
- defer s.mutex.Unlock()
-
- conn, exists := s.connections[clientID]
- if !exists {
- return fmt.Errorf("client %s not found", clientID)
- }
-
- conn.LastHeartbeat = time.Now()
-
- // Update in database
- _, err := s.db.Exec("UPDATE mcp_connections SET last_heartbeat = ? WHERE client_id = ?", conn.LastHeartbeat, clientID)
- return err
-}
-
-// ExecuteToolOnClient sends a tool execution request to the MCP client
-func (s *MCPBridgeService) ExecuteToolOnClient(userID string, toolName string, args map[string]interface{}, timeout time.Duration) (string, error) {
- s.mutex.RLock()
- clientID, exists := s.userConns[userID]
- if !exists {
- s.mutex.RUnlock()
- return "", fmt.Errorf("no MCP client connected for user %s", userID)
- }
-
- conn, connExists := s.connections[clientID]
- s.mutex.RUnlock()
-
- if !connExists {
- return "", fmt.Errorf("MCP client connection not found")
- }
-
- // Generate unique call ID
- callID := uuid.New().String()
-
- // Create result channel for this call
- resultChan := make(chan models.MCPToolResult, 1)
- conn.PendingResults[callID] = resultChan
-
- // Create tool call message
- toolCall := models.MCPToolCall{
- CallID: callID,
- ToolName: toolName,
- Arguments: args,
- Timeout: int(timeout.Seconds()),
- }
-
- // Send to client
- select {
- case conn.WriteChan <- models.MCPServerMessage{
- Type: "tool_call",
- Payload: map[string]interface{}{
- "call_id": toolCall.CallID,
- "tool_name": toolCall.ToolName,
- "arguments": toolCall.Arguments,
- "timeout": toolCall.Timeout,
- },
- }:
- // Message sent successfully
- case <-time.After(5 * time.Second):
- delete(conn.PendingResults, callID)
- return "", fmt.Errorf("timeout sending tool call to client")
- }
-
- // Wait for result with timeout
- select {
- case result := <-resultChan:
- delete(conn.PendingResults, callID)
- if result.Success {
- return result.Result, nil
- } else {
- return "", fmt.Errorf("%s", result.Error)
- }
- case <-time.After(timeout):
- delete(conn.PendingResults, callID)
- return "", fmt.Errorf("tool execution timeout after %v", timeout)
- }
-}
-
-// GetConnection retrieves a connection by client ID
-func (s *MCPBridgeService) GetConnection(clientID string) (*models.MCPConnection, bool) {
- s.mutex.RLock()
- defer s.mutex.RUnlock()
- conn, exists := s.connections[clientID]
- return conn, exists
-}
-
-// GetUserConnection retrieves a connection by user ID
-func (s *MCPBridgeService) GetUserConnection(userID string) (*models.MCPConnection, bool) {
- s.mutex.RLock()
- defer s.mutex.RUnlock()
-
- clientID, exists := s.userConns[userID]
- if !exists {
- return nil, false
- }
-
- conn, connExists := s.connections[clientID]
- return conn, connExists
-}
-
-// IsUserConnected checks if a user has an active MCP client
-func (s *MCPBridgeService) IsUserConnected(userID string) bool {
- s.mutex.RLock()
- defer s.mutex.RUnlock()
- _, exists := s.userConns[userID]
- return exists
-}
-
-// GetConnectionCount returns the number of active connections
-func (s *MCPBridgeService) GetConnectionCount() int {
- s.mutex.RLock()
- defer s.mutex.RUnlock()
- return len(s.connections)
-}
-
-// LogToolExecution logs a tool execution for audit purposes
-func (s *MCPBridgeService) LogToolExecution(userID, toolName, conversationID string, executionTimeMs int, success bool, errorMsg string) {
- _, err := s.db.Exec(`
- INSERT INTO mcp_audit_log (user_id, tool_name, conversation_id, execution_time_ms, success, error_message)
- VALUES (?, ?, ?, ?, ?, ?)
- `, userID, toolName, conversationID, executionTimeMs, success, errorMsg)
-
- if err != nil {
- log.Printf("Warning: Failed to log tool execution: %v", err)
- }
-}
diff --git a/backend/internal/services/memory_decay_service.go b/backend/internal/services/memory_decay_service.go
deleted file mode 100644
index b642124a..00000000
--- a/backend/internal/services/memory_decay_service.go
+++ /dev/null
@@ -1,304 +0,0 @@
-package services
-
-import (
- "context"
- "fmt"
- "log"
- "math"
- "time"
-
- "claraverse/internal/database"
- "go.mongodb.org/mongo-driver/bson"
- "go.mongodb.org/mongo-driver/bson/primitive"
- "go.mongodb.org/mongo-driver/mongo"
-)
-
-// MemoryDecayService handles memory scoring and archival using PageRank-like algorithm
-type MemoryDecayService struct {
- mongodb *database.MongoDB
- collection *mongo.Collection
-}
-
-// DecayConfig holds the decay algorithm configuration
-type DecayConfig struct {
- RecencyWeight float64 // Default: 0.4
- FrequencyWeight float64 // Default: 0.3
- EngagementWeight float64 // Default: 0.3
- RecencyDecayRate float64 // Default: 0.05
- FrequencyMax int64 // Default: 20
- ArchiveThreshold float64 // Default: 0.15
-}
-
-// DefaultDecayConfig returns the default decay configuration
-func DefaultDecayConfig() DecayConfig {
- return DecayConfig{
- RecencyWeight: 0.4,
- FrequencyWeight: 0.3,
- EngagementWeight: 0.3,
- RecencyDecayRate: 0.05,
- FrequencyMax: 20,
- ArchiveThreshold: 0.15,
- }
-}
-
-// NewMemoryDecayService creates a new memory decay service
-func NewMemoryDecayService(mongodb *database.MongoDB) *MemoryDecayService {
- return &MemoryDecayService{
- mongodb: mongodb,
- collection: mongodb.Collection(database.CollectionMemories),
- }
-}
-
-// RunDecayJob runs the full decay job for all users
-func (s *MemoryDecayService) RunDecayJob(ctx context.Context) error {
- log.Printf("🔄 [MEMORY-DECAY] Starting decay job")
-
- config := DefaultDecayConfig()
-
- // Get all unique user IDs with active memories
- userIDs, err := s.getActiveUserIDs(ctx)
- if err != nil {
- return fmt.Errorf("failed to get active user IDs: %w", err)
- }
-
- log.Printf("📊 [MEMORY-DECAY] Processing %d users with active memories", len(userIDs))
-
- // Process each user
- totalRecalculated := 0
- totalArchived := 0
-
- for _, userID := range userIDs {
- recalculated, archived, err := s.RunDecayJobForUser(ctx, userID, config)
- if err != nil {
- log.Printf("⚠️ [MEMORY-DECAY] Failed to process user %s: %v", userID, err)
- continue
- }
-
- totalRecalculated += recalculated
- totalArchived += archived
- }
-
- log.Printf("✅ [MEMORY-DECAY] Decay job completed: %d memories recalculated, %d archived", totalRecalculated, totalArchived)
- return nil
-}
-
-// RunDecayJobForUser runs decay job for a specific user
-func (s *MemoryDecayService) RunDecayJobForUser(ctx context.Context, userID string, config DecayConfig) (int, int, error) {
- // Get all active memories for user
- filter := bson.M{
- "userId": userID,
- "isArchived": false,
- }
-
- cursor, err := s.collection.Find(ctx, filter)
- if err != nil {
- return 0, 0, fmt.Errorf("failed to find memories: %w", err)
- }
- defer cursor.Close(ctx)
-
- var memories []struct {
- ID primitive.ObjectID `bson:"_id"`
- AccessCount int64 `bson:"accessCount"`
- LastAccessedAt *time.Time `bson:"lastAccessedAt"`
- SourceEngagement float64 `bson:"sourceEngagement"`
- CreatedAt time.Time `bson:"createdAt"`
- }
-
- if err := cursor.All(ctx, &memories); err != nil {
- return 0, 0, fmt.Errorf("failed to decode memories: %w", err)
- }
-
- if len(memories) == 0 {
- return 0, 0, nil
- }
-
- // Calculate scores for all memories
- now := time.Now()
- memoriesToArchive := []primitive.ObjectID{}
- memoriesToUpdate := []mongo.WriteModel{}
-
- for _, mem := range memories {
- newScore := s.calculateMemoryScore(mem.AccessCount, mem.LastAccessedAt, mem.SourceEngagement, mem.CreatedAt, now, config)
-
- // Check if should be archived
- if newScore < config.ArchiveThreshold {
- memoriesToArchive = append(memoriesToArchive, mem.ID)
- } else {
- // Update score
- update := mongo.NewUpdateOneModel().
- SetFilter(bson.M{"_id": mem.ID}).
- SetUpdate(bson.M{
- "$set": bson.M{
- "score": newScore,
- "updatedAt": now,
- },
- })
- memoriesToUpdate = append(memoriesToUpdate, update)
- }
- }
-
- // Bulk update scores
- recalculated := 0
- if len(memoriesToUpdate) > 0 {
- result, err := s.collection.BulkWrite(ctx, memoriesToUpdate)
- if err != nil {
- log.Printf("⚠️ [MEMORY-DECAY] Failed to update scores for user %s: %v", userID, err)
- } else {
- recalculated = int(result.ModifiedCount)
- }
- }
-
- // Archive low-score memories
- archived := 0
- if len(memoriesToArchive) > 0 {
- archived, err = s.archiveMemoriesBulk(ctx, memoriesToArchive, now)
- if err != nil {
- log.Printf("⚠️ [MEMORY-DECAY] Failed to archive memories for user %s: %v", userID, err)
- }
- }
-
- log.Printf("📊 [MEMORY-DECAY] User %s: %d memories recalculated, %d archived", userID, recalculated, archived)
- return recalculated, archived, nil
-}
-
-// calculateMemoryScore calculates the PageRank-like score for a memory
-func (s *MemoryDecayService) calculateMemoryScore(
- accessCount int64,
- lastAccessedAt *time.Time,
- sourceEngagement float64,
- createdAt time.Time,
- now time.Time,
- config DecayConfig,
-) float64 {
- // Calculate recency score
- recencyScore := s.calculateRecencyScore(lastAccessedAt, createdAt, now, config.RecencyDecayRate)
-
- // Calculate frequency score
- frequencyScore := s.calculateFrequencyScore(accessCount, config.FrequencyMax)
-
- // Engagement score is directly from source conversation
- engagementScore := sourceEngagement
-
- // Weighted combination (PageRank-like)
- finalScore := (config.RecencyWeight * recencyScore) +
- (config.FrequencyWeight * frequencyScore) +
- (config.EngagementWeight * engagementScore)
-
- return finalScore
-}
-
-// calculateRecencyScore calculates recency score using exponential decay
-// RecencyScore = exp(-0.05 × days_since_last_access)
-// - Recent: 1.0
-// - 1 week: ~0.70
-// - 1 month: ~0.22
-// - 3 months: ~0.01
-func (s *MemoryDecayService) calculateRecencyScore(lastAccessedAt *time.Time, createdAt time.Time, now time.Time, decayRate float64) float64 {
- var referenceTime time.Time
-
- // Use last accessed time if available, otherwise use creation time
- if lastAccessedAt != nil {
- referenceTime = *lastAccessedAt
- } else {
- referenceTime = createdAt
- }
-
- // Calculate days since last access/creation
- daysSince := now.Sub(referenceTime).Hours() / 24.0
-
- // Exponential decay: exp(-decayRate × days)
- recencyScore := math.Exp(-decayRate * daysSince)
-
- return recencyScore
-}
-
-// calculateFrequencyScore calculates frequency score based on access count
-// FrequencyScore = min(1.0, access_count / max)
-// - 0 accesses: 0.0
-// - 10 accesses: 0.5 (if max=20)
-// - 20+ accesses: 1.0
-func (s *MemoryDecayService) calculateFrequencyScore(accessCount int64, frequencyMax int64) float64 {
- if accessCount <= 0 {
- return 0.0
- }
-
- frequencyScore := float64(accessCount) / float64(frequencyMax)
-
- // Cap at 1.0
- if frequencyScore > 1.0 {
- frequencyScore = 1.0
- }
-
- return frequencyScore
-}
-
-// archiveMemoriesBulk archives multiple memories at once
-func (s *MemoryDecayService) archiveMemoriesBulk(ctx context.Context, memoryIDs []primitive.ObjectID, now time.Time) (int, error) {
- filter := bson.M{
- "_id": bson.M{"$in": memoryIDs},
- }
-
- update := bson.M{
- "$set": bson.M{
- "isArchived": true,
- "archivedAt": now,
- "updatedAt": now,
- },
- }
-
- result, err := s.collection.UpdateMany(ctx, filter, update)
- if err != nil {
- return 0, fmt.Errorf("failed to archive memories: %w", err)
- }
-
- log.Printf("📦 [MEMORY-DECAY] Archived %d memories", result.ModifiedCount)
- return int(result.ModifiedCount), nil
-}
-
-// getActiveUserIDs gets all unique user IDs with active memories
-func (s *MemoryDecayService) getActiveUserIDs(ctx context.Context) ([]string, error) {
- filter := bson.M{"isArchived": false}
-
- distinctUserIDs, err := s.collection.Distinct(ctx, "userId", filter)
- if err != nil {
- return nil, fmt.Errorf("failed to get distinct user IDs: %w", err)
- }
-
- userIDs := make([]string, 0, len(distinctUserIDs))
- for _, id := range distinctUserIDs {
- if userID, ok := id.(string); ok {
- userIDs = append(userIDs, userID)
- }
- }
-
- return userIDs, nil
-}
-
-// GetMemoryScore calculates and returns the current score for a specific memory (for testing/debugging)
-func (s *MemoryDecayService) GetMemoryScore(ctx context.Context, memoryID primitive.ObjectID) (float64, error) {
- var memory struct {
- AccessCount int64 `bson:"accessCount"`
- LastAccessedAt *time.Time `bson:"lastAccessedAt"`
- SourceEngagement float64 `bson:"sourceEngagement"`
- CreatedAt time.Time `bson:"createdAt"`
- }
-
- err := s.collection.FindOne(ctx, bson.M{"_id": memoryID}).Decode(&memory)
- if err != nil {
- return 0, fmt.Errorf("failed to find memory: %w", err)
- }
-
- config := DefaultDecayConfig()
- now := time.Now()
-
- score := s.calculateMemoryScore(
- memory.AccessCount,
- memory.LastAccessedAt,
- memory.SourceEngagement,
- memory.CreatedAt,
- now,
- config,
- )
-
- return score, nil
-}
diff --git a/backend/internal/services/memory_decay_service_test.go b/backend/internal/services/memory_decay_service_test.go
deleted file mode 100644
index bae9cf9c..00000000
--- a/backend/internal/services/memory_decay_service_test.go
+++ /dev/null
@@ -1,352 +0,0 @@
-package services
-
-import (
- "math"
- "testing"
- "time"
-)
-
-// TestCalculateRecencyScore tests the recency score calculation
-func TestCalculateRecencyScore(t *testing.T) {
- service := &MemoryDecayService{}
- config := DefaultDecayConfig()
- now := time.Now()
-
- tests := []struct {
- name string
- lastAccessedAt *time.Time
- createdAt time.Time
- expectedScore float64
- tolerance float64
- }{
- {
- name: "Just accessed (0 days)",
- lastAccessedAt: &now,
- createdAt: now.AddDate(0, 0, -30),
- expectedScore: 1.0,
- tolerance: 0.01,
- },
- {
- name: "1 week ago (~0.70)",
- lastAccessedAt: timePtr(now.AddDate(0, 0, -7)),
- createdAt: now.AddDate(0, 0, -30),
- expectedScore: 0.70,
- tolerance: 0.05,
- },
- {
- name: "1 month ago (~0.22)",
- lastAccessedAt: timePtr(now.AddDate(0, 0, -30)),
- createdAt: now.AddDate(0, 0, -60),
- expectedScore: 0.22,
- tolerance: 0.05,
- },
- {
- name: "3 months ago (~0.01)",
- lastAccessedAt: timePtr(now.AddDate(0, 0, -90)),
- createdAt: now.AddDate(0, 0, -120),
- expectedScore: 0.01,
- tolerance: 0.02,
- },
- {
- name: "Never accessed (use createdAt)",
- lastAccessedAt: nil,
- createdAt: now.AddDate(0, 0, -7),
- expectedScore: 0.70,
- tolerance: 0.05,
- },
- }
-
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- score := service.calculateRecencyScore(tt.lastAccessedAt, tt.createdAt, now, config.RecencyDecayRate)
- if math.Abs(score-tt.expectedScore) > tt.tolerance {
- t.Errorf("Expected score ~%.2f, got %.2f (tolerance: %.2f)", tt.expectedScore, score, tt.tolerance)
- }
- })
- }
-}
-
-// TestCalculateFrequencyScore tests the frequency score calculation
-func TestCalculateFrequencyScore(t *testing.T) {
- service := &MemoryDecayService{}
- config := DefaultDecayConfig()
-
- tests := []struct {
- name string
- accessCount int64
- expectedScore float64
- }{
- {
- name: "0 accesses",
- accessCount: 0,
- expectedScore: 0.0,
- },
- {
- name: "10 accesses (50%)",
- accessCount: 10,
- expectedScore: 0.5,
- },
- {
- name: "20 accesses (100%)",
- accessCount: 20,
- expectedScore: 1.0,
- },
- {
- name: "40 accesses (capped at 100%)",
- accessCount: 40,
- expectedScore: 1.0,
- },
- {
- name: "5 accesses (25%)",
- accessCount: 5,
- expectedScore: 0.25,
- },
- }
-
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- score := service.calculateFrequencyScore(tt.accessCount, config.FrequencyMax)
- if math.Abs(score-tt.expectedScore) > 0.01 {
- t.Errorf("Expected score %.2f, got %.2f", tt.expectedScore, score)
- }
- })
- }
-}
-
-// TestCalculateMemoryScore tests the complete PageRank algorithm
-func TestCalculateMemoryScore(t *testing.T) {
- service := &MemoryDecayService{}
- config := DefaultDecayConfig()
- now := time.Now()
-
- tests := []struct {
- name string
- accessCount int64
- lastAccessedAt *time.Time
- sourceEngagement float64
- createdAt time.Time
- minScore float64
- maxScore float64
- description string
- }{
- {
- name: "High quality memory (recent, frequent, engaging)",
- accessCount: 25,
- lastAccessedAt: &now,
- sourceEngagement: 0.9,
- createdAt: now.AddDate(0, 0, -30),
- minScore: 0.85,
- maxScore: 1.0,
- description: "Should have very high score",
- },
- {
- name: "Medium quality memory",
- accessCount: 10,
- lastAccessedAt: timePtr(now.AddDate(0, 0, -7)),
- sourceEngagement: 0.6,
- createdAt: now.AddDate(0, 0, -30),
- minScore: 0.50,
- maxScore: 0.70,
- description: "Should have medium score",
- },
- {
- name: "Low quality memory (old, never accessed, low engagement)",
- accessCount: 0,
- lastAccessedAt: nil,
- sourceEngagement: 0.2,
- createdAt: now.AddDate(0, 0, -90),
- minScore: 0.0,
- maxScore: 0.15,
- description: "Should be below archive threshold",
- },
- {
- name: "Decaying memory (moderately old, few accesses)",
- accessCount: 3,
- lastAccessedAt: timePtr(now.AddDate(0, 0, -30)),
- sourceEngagement: 0.5,
- createdAt: now.AddDate(0, 0, -60),
- minScore: 0.15,
- maxScore: 0.35,
- description: "Should be approaching archive threshold",
- },
- {
- name: "High engagement saves old memory",
- accessCount: 5,
- lastAccessedAt: timePtr(now.AddDate(0, 0, -60)),
- sourceEngagement: 0.95,
- createdAt: now.AddDate(0, 0, -90),
- minScore: 0.30,
- maxScore: 0.50,
- description: "High engagement keeps it above archive threshold",
- },
- }
-
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- score := service.calculateMemoryScore(
- tt.accessCount,
- tt.lastAccessedAt,
- tt.sourceEngagement,
- tt.createdAt,
- now,
- config,
- )
-
- if score < tt.minScore || score > tt.maxScore {
- t.Errorf("%s: Expected score between %.2f and %.2f, got %.2f",
- tt.description, tt.minScore, tt.maxScore, score)
- }
-
- t.Logf("Score: %.3f (recency: %.2f, frequency: %.2f, engagement: %.2f)",
- score,
- service.calculateRecencyScore(tt.lastAccessedAt, tt.createdAt, now, config.RecencyDecayRate),
- service.calculateFrequencyScore(tt.accessCount, config.FrequencyMax),
- tt.sourceEngagement,
- )
- })
- }
-}
-
-// TestArchiveThreshold ensures memories below threshold get archived
-func TestArchiveThreshold(t *testing.T) {
- service := &MemoryDecayService{}
- config := DefaultDecayConfig()
- now := time.Now()
-
- // Create a memory that should be archived
- accessCount := int64(0)
- lastAccessedAt := (*time.Time)(nil)
- sourceEngagement := 0.2
- createdAt := now.AddDate(0, 0, -90)
-
- score := service.calculateMemoryScore(
- accessCount,
- lastAccessedAt,
- sourceEngagement,
- createdAt,
- now,
- config,
- )
-
- if score >= config.ArchiveThreshold {
- t.Errorf("Expected score below archive threshold (%.2f), got %.2f", config.ArchiveThreshold, score)
- }
-}
-
-// TestDecayConfigWeights ensures weights add up to 1.0
-func TestDecayConfigWeights(t *testing.T) {
- config := DefaultDecayConfig()
-
- totalWeight := config.RecencyWeight + config.FrequencyWeight + config.EngagementWeight
-
- if math.Abs(totalWeight-1.0) > 0.001 {
- t.Errorf("Weights should add up to 1.0, got %.3f", totalWeight)
- }
-}
-
-// TestRecencyDecayFormula verifies the exponential decay formula
-func TestRecencyDecayFormula(t *testing.T) {
- service := &MemoryDecayService{}
- now := time.Now()
-
- // Test known values
- tests := []struct {
- daysAgo int
- decayRate float64
- expectedScore float64
- tolerance float64
- }{
- {daysAgo: 0, decayRate: 0.05, expectedScore: 1.0, tolerance: 0.01},
- {daysAgo: 7, decayRate: 0.05, expectedScore: 0.704, tolerance: 0.01},
- {daysAgo: 14, decayRate: 0.05, expectedScore: 0.496, tolerance: 0.01},
- {daysAgo: 30, decayRate: 0.05, expectedScore: 0.223, tolerance: 0.01},
- {daysAgo: 60, decayRate: 0.05, expectedScore: 0.050, tolerance: 0.01},
- {daysAgo: 90, decayRate: 0.05, expectedScore: 0.011, tolerance: 0.01},
- }
-
- for _, tt := range tests {
- lastAccessed := now.AddDate(0, 0, -tt.daysAgo)
- createdAt := now.AddDate(0, 0, -tt.daysAgo-30)
- score := service.calculateRecencyScore(&lastAccessed, createdAt, now, tt.decayRate)
-
- if math.Abs(score-tt.expectedScore) > tt.tolerance {
- t.Errorf("Day %d: Expected %.3f, got %.3f (diff: %.3f)",
- tt.daysAgo, tt.expectedScore, score, math.Abs(score-tt.expectedScore))
- }
- }
-}
-
-// TestFrequencyScoreLinear ensures frequency score is linear up to max
-func TestFrequencyScoreLinear(t *testing.T) {
- service := &MemoryDecayService{}
- frequencyMax := int64(20)
-
- for i := int64(0); i <= frequencyMax*2; i += 2 {
- score := service.calculateFrequencyScore(i, frequencyMax)
-
- expected := math.Min(1.0, float64(i)/float64(frequencyMax))
- if math.Abs(score-expected) > 0.001 {
- t.Errorf("Access count %d: Expected %.3f, got %.3f", i, expected, score)
- }
- }
-}
-
-// TestMemoryLifecycle tests a realistic memory lifecycle
-func TestMemoryLifecycle(t *testing.T) {
- service := &MemoryDecayService{}
- config := DefaultDecayConfig()
- now := time.Now()
-
- // Day 0: Memory created with high engagement
- createdAt := now.AddDate(0, 0, -90)
- accessCount := int64(0)
- lastAccessed := (*time.Time)(nil)
- sourceEngagement := 0.85
-
- // Initial score (only engagement matters)
- score0 := service.calculateMemoryScore(accessCount, lastAccessed, sourceEngagement, createdAt, createdAt, config)
- t.Logf("Day 0: Score %.3f", score0)
-
- // Day 7: Accessed once
- day7 := createdAt.AddDate(0, 0, 7)
- accessCount = 1
- lastAccessed = &day7
- score7 := service.calculateMemoryScore(accessCount, lastAccessed, sourceEngagement, createdAt, day7, config)
- t.Logf("Day 7: Score %.3f (accessed once)", score7)
-
- // Day 30: Accessed 5 more times
- day30 := createdAt.AddDate(0, 0, 30)
- accessCount = 6
- lastAccessed = &day30
- score30 := service.calculateMemoryScore(accessCount, lastAccessed, sourceEngagement, createdAt, day30, config)
- t.Logf("Day 30: Score %.3f (accessed 6 times total)", score30)
-
- // Day 60: No new accesses (recency drops)
- day60 := createdAt.AddDate(0, 0, 60)
- score60 := service.calculateMemoryScore(accessCount, lastAccessed, sourceEngagement, createdAt, day60, config)
- t.Logf("Day 60: Score %.3f (no new accesses, recency drops)", score60)
-
- // Day 90: Still no accesses (further decay)
- day90 := createdAt.AddDate(0, 0, 90)
- score90 := service.calculateMemoryScore(accessCount, lastAccessed, sourceEngagement, createdAt, day90, config)
- t.Logf("Day 90: Score %.3f (continued decay)", score90)
-
- // Score should decrease over time without accesses
- if score60 >= score30 {
- t.Error("Score should decrease from day 30 to day 60 without accesses")
- }
- if score90 >= score60 {
- t.Error("Score should continue decreasing from day 60 to day 90")
- }
-
- // Should still be above archive threshold due to high engagement
- if score90 < config.ArchiveThreshold {
- t.Errorf("High engagement memory should stay above archive threshold (%.2f), got %.3f",
- config.ArchiveThreshold, score90)
- }
-}
-
-// Helper function to create time pointer
-func timePtr(t time.Time) *time.Time {
- return &t
-}
diff --git a/backend/internal/services/memory_extraction_service.go b/backend/internal/services/memory_extraction_service.go
deleted file mode 100644
index 4c885b14..00000000
--- a/backend/internal/services/memory_extraction_service.go
+++ /dev/null
@@ -1,717 +0,0 @@
-package services
-
-import (
- "bytes"
- "context"
- "encoding/json"
- "fmt"
- "io"
- "log"
- "net/http"
- "strings"
- "time"
-
- "claraverse/internal/crypto"
- "claraverse/internal/database"
- "claraverse/internal/models"
- "go.mongodb.org/mongo-driver/bson"
- "go.mongodb.org/mongo-driver/bson/primitive"
- "go.mongodb.org/mongo-driver/mongo"
- "go.mongodb.org/mongo-driver/mongo/options"
-)
-
-// MemoryExtractionService handles extraction of memories from conversations using LLMs
-type MemoryExtractionService struct {
- mongodb *database.MongoDB
- jobCollection *mongo.Collection
- engagementCollection *mongo.Collection
- encryptionService *crypto.EncryptionService
- providerService *ProviderService
- memoryStorageService *MemoryStorageService
- chatService *ChatService
- modelPool *MemoryModelPool // Dynamic model pool with round-robin and failover
-}
-
-// Rate limiting constants for extraction to prevent abuse
-const (
- MaxExtractionsPerHour = 20 // Maximum extractions per user per hour
- MaxPendingJobsPerUser = 50 // Maximum pending jobs per user
-)
-
-// Memory extraction system prompt
-const MemoryExtractionSystemPrompt = `You are a memory extraction system for Clara AI. Analyze this conversation and extract important information to remember about the user.
-
-WHAT TO EXTRACT:
-1. **Personal Information**: Name, location, occupation, family, age, background
-2. **Preferences**: Likes, dislikes, communication style, how they want to be addressed
-3. **Important Context**: Ongoing projects, goals, constraints, responsibilities
-4. **Facts**: Skills, experiences, knowledge areas, technical expertise
-5. **Instructions**: Specific guidelines the user wants you to follow (e.g., "always use TypeScript", "keep responses brief")
-
-RULES:
-- Be concise (1-2 sentences per memory)
-- Only extract FACTUAL information explicitly stated by the user
-- Ignore small talk and pleasantries
-- Avoid redundant or obvious information
-- Each memory should be atomic (one piece of information)
-- Categorize each memory correctly
-- Add relevant tags for searchability
-- **CRITICAL**: DO NOT extract information that is already captured in EXISTING MEMORIES (provided below)
-- Only extract NEW information not present in existing memories
-- If conversation contains no new memorable information, return empty array
-
-CATEGORIES:
-- "personal_info": Name, location, occupation, family, age
-- "preferences": Likes, dislikes, style, communication preferences
-- "context": Ongoing projects, goals, responsibilities
-- "fact": Skills, knowledge, experiences
-- "instruction": Guidelines to follow
-
-Return JSON with array of memories.`
-
-// memoryExtractionSchema defines structured output for memory extraction
-var memoryExtractionSchema = map[string]interface{}{
- "type": "object",
- "properties": map[string]interface{}{
- "memories": map[string]interface{}{
- "type": "array",
- "items": map[string]interface{}{
- "type": "object",
- "properties": map[string]interface{}{
- "content": map[string]interface{}{
- "type": "string",
- "description": "The memory content (concise, factual)",
- },
- "category": map[string]interface{}{
- "type": "string",
- "enum": []string{"personal_info", "preferences", "context", "fact", "instruction"},
- "description": "Memory category",
- },
- "tags": map[string]interface{}{
- "type": "array",
- "items": map[string]interface{}{
- "type": "string",
- },
- "description": "Relevant tags for this memory",
- },
- },
- "required": []string{"content", "category", "tags"},
- "additionalProperties": false,
- },
- },
- },
- "required": []string{"memories"},
- "additionalProperties": false,
-}
-
-// NewMemoryExtractionService creates a new memory extraction service
-func NewMemoryExtractionService(
- mongodb *database.MongoDB,
- encryptionService *crypto.EncryptionService,
- providerService *ProviderService,
- memoryStorageService *MemoryStorageService,
- chatService *ChatService,
- modelPool *MemoryModelPool,
-) *MemoryExtractionService {
- return &MemoryExtractionService{
- mongodb: mongodb,
- jobCollection: mongodb.Collection(database.CollectionMemoryExtractionJobs),
- engagementCollection: mongodb.Collection(database.CollectionConversationEngagement),
- encryptionService: encryptionService,
- providerService: providerService,
- memoryStorageService: memoryStorageService,
- chatService: chatService,
- modelPool: modelPool,
- }
-}
-
-// EnqueueExtraction creates a new extraction job (non-blocking)
-// SECURITY: Includes rate limiting to prevent abuse and DoS attacks
-func (s *MemoryExtractionService) EnqueueExtraction(
- ctx context.Context,
- userID string,
- conversationID string,
- messages []map[string]interface{},
-) error {
- if userID == "" || conversationID == "" {
- return fmt.Errorf("user ID and conversation ID are required")
- }
-
- // SECURITY: Check pending jobs limit to prevent queue flooding
- pendingCount, err := s.jobCollection.CountDocuments(ctx, bson.M{
- "userId": userID,
- "status": models.JobStatusPending,
- })
- if err != nil {
- log.Printf("⚠️ [MEMORY-EXTRACTION] Failed to count pending jobs: %v", err)
- } else if pendingCount >= MaxPendingJobsPerUser {
- log.Printf("⚠️ [MEMORY-EXTRACTION] User %s has %d pending jobs (max: %d), skipping", userID, pendingCount, MaxPendingJobsPerUser)
- return fmt.Errorf("too many pending extraction jobs (%d), please wait", pendingCount)
- }
-
- // SECURITY: Check hourly extraction rate (last hour completed + pending)
- oneHourAgo := time.Now().Add(-1 * time.Hour)
- recentCount, err := s.jobCollection.CountDocuments(ctx, bson.M{
- "userId": userID,
- "$or": []bson.M{
- {"status": models.JobStatusPending},
- {"status": models.JobStatusProcessing},
- {
- "status": models.JobStatusCompleted,
- "processedAt": bson.M{"$gte": oneHourAgo},
- },
- },
- })
- if err != nil {
- log.Printf("⚠️ [MEMORY-EXTRACTION] Failed to count recent jobs: %v", err)
- } else if recentCount >= MaxExtractionsPerHour {
- log.Printf("⚠️ [MEMORY-EXTRACTION] User %s exceeded hourly extraction limit (%d/%d)", userID, recentCount, MaxExtractionsPerHour)
- return fmt.Errorf("extraction rate limit exceeded (%d extractions in last hour), please wait", recentCount)
- }
-
- // Encrypt messages
- messagesJSON, err := json.Marshal(messages)
- if err != nil {
- return fmt.Errorf("failed to marshal messages: %w", err)
- }
-
- encryptedMessages, err := s.encryptionService.Encrypt(userID, messagesJSON)
- if err != nil {
- return fmt.Errorf("failed to encrypt messages: %w", err)
- }
-
- // Create job
- job := &models.MemoryExtractionJob{
- ID: primitive.NewObjectID(),
- UserID: userID,
- ConversationID: conversationID,
- MessageCount: len(messages),
- EncryptedMessages: encryptedMessages,
- Status: models.JobStatusPending,
- AttemptCount: 0,
- CreatedAt: time.Now(),
- }
-
- // Insert job
- _, err = s.jobCollection.InsertOne(ctx, job)
- if err != nil {
- return fmt.Errorf("failed to insert extraction job: %w", err)
- }
-
- log.Printf("📥 [MEMORY-EXTRACTION] Enqueued job for conversation %s (%d messages)", conversationID, len(messages))
- return nil
-}
-
-// ProcessPendingJobs processes all pending extraction jobs (background worker)
-func (s *MemoryExtractionService) ProcessPendingJobs(ctx context.Context) error {
- // Find pending jobs
- filter := bson.M{"status": models.JobStatusPending}
- cursor, err := s.jobCollection.Find(ctx, filter)
- if err != nil {
- return fmt.Errorf("failed to find pending jobs: %w", err)
- }
- defer cursor.Close(ctx)
-
- var jobs []models.MemoryExtractionJob
- if err := cursor.All(ctx, &jobs); err != nil {
- return fmt.Errorf("failed to decode jobs: %w", err)
- }
-
- if len(jobs) == 0 {
- return nil // No pending jobs
- }
-
- log.Printf("⚙️ [MEMORY-EXTRACTION] Processing %d pending jobs", len(jobs))
-
- // Process each job
- for _, job := range jobs {
- if err := s.processJob(ctx, &job); err != nil {
- log.Printf("⚠️ [MEMORY-EXTRACTION] Job %s failed: %v", job.ID.Hex(), err)
- s.markJobFailed(ctx, job.ID, err.Error())
- }
- }
-
- return nil
-}
-
-// processJob processes a single extraction job
-func (s *MemoryExtractionService) processJob(ctx context.Context, job *models.MemoryExtractionJob) error {
- // Mark as processing
- s.updateJobStatus(ctx, job.ID, models.JobStatusProcessing)
-
- // Decrypt messages
- messagesBytes, err := s.encryptionService.Decrypt(job.UserID, job.EncryptedMessages)
- if err != nil {
- return fmt.Errorf("failed to decrypt messages: %w", err)
- }
-
- var messages []map[string]interface{}
- if err := json.Unmarshal(messagesBytes, &messages); err != nil {
- return fmt.Errorf("failed to unmarshal messages: %w", err)
- }
-
- log.Printf("🔍 [MEMORY-EXTRACTION] Processing job %s (%d messages)", job.ID.Hex(), len(messages))
-
- // Calculate conversation engagement
- engagement := s.calculateEngagement(messages)
-
- // Store engagement in database
- if err := s.storeEngagement(ctx, job.UserID, job.ConversationID, messages, engagement); err != nil {
- log.Printf("⚠️ [MEMORY-EXTRACTION] Failed to store engagement: %v", err)
- }
-
- // Fetch existing memories to avoid duplicates
- existingMemories, _, err := s.memoryStorageService.ListMemories(
- ctx,
- job.UserID,
- "", // category (empty = all categories)
- nil, // tags (nil = all tags)
- false, // includeArchived (false = only active)
- 1, // page
- 100, // pageSize (get recent 100 memories for context)
- )
- if err != nil {
- log.Printf("⚠️ [MEMORY-EXTRACTION] Failed to fetch existing memories: %v, continuing without context", err)
- existingMemories = []models.DecryptedMemory{} // Continue with empty list
- }
-
- log.Printf("📚 [MEMORY-EXTRACTION] Found %d existing memories to avoid duplicates", len(existingMemories))
-
- // Extract memories via LLM (with existing memories for context)
- extractedMemories, err := s.extractMemories(ctx, job.UserID, messages, existingMemories)
- if err != nil {
- return fmt.Errorf("failed to extract memories: %w", err)
- }
-
- log.Printf("🧠 [MEMORY-EXTRACTION] Extracted %d memories", len(extractedMemories.Memories))
-
- // Store each memory
- for _, mem := range extractedMemories.Memories {
- _, err := s.memoryStorageService.CreateMemory(
- ctx,
- job.UserID,
- mem.Content,
- mem.Category,
- mem.Tags,
- engagement,
- job.ConversationID,
- )
- if err != nil {
- log.Printf("⚠️ [MEMORY-EXTRACTION] Failed to store memory: %v", err)
- }
- }
-
- // Mark job as completed
- s.markJobCompleted(ctx, job.ID)
-
- log.Printf("✅ [MEMORY-EXTRACTION] Job %s completed successfully", job.ID.Hex())
- return nil
-}
-
-// extractMemories calls LLM to extract memories from conversation with automatic failover
-func (s *MemoryExtractionService) extractMemories(
- ctx context.Context,
- userID string,
- messages []map[string]interface{},
- existingMemories []models.DecryptedMemory,
-) (*models.ExtractedMemoryFromLLM, error) {
-
- // Check if user has a custom extractor model preference
- userPreferredModel, err := s.getExtractorModelForUser(ctx, userID)
- var extractorModelID string
-
- if err == nil && userPreferredModel != "" {
- // User has a preference, use it
- extractorModelID = userPreferredModel
- log.Printf("👤 [MEMORY-EXTRACTION] Using user-preferred model: %s", extractorModelID)
- } else {
- // No user preference, get from model pool
- extractorModelID, err = s.modelPool.GetNextExtractor()
- if err != nil {
- return nil, fmt.Errorf("no extractor models available: %w", err)
- }
- }
-
- // Try extraction with automatic failover (max 3 attempts)
- maxAttempts := 3
- var lastError error
-
- for attempt := 1; attempt <= maxAttempts; attempt++ {
- result, err := s.tryExtraction(ctx, userID, extractorModelID, messages, existingMemories)
-
- if err == nil {
- // Success!
- s.modelPool.MarkSuccess(extractorModelID)
- return result, nil
- }
-
- // Extraction failed
- lastError = err
- s.modelPool.MarkFailure(extractorModelID)
- log.Printf("⚠️ [MEMORY-EXTRACTION] Attempt %d/%d failed with model %s: %v",
- attempt, maxAttempts, extractorModelID, err)
-
- // If not last attempt, get next model from pool
- if attempt < maxAttempts {
- extractorModelID, err = s.modelPool.GetNextExtractor()
- if err != nil {
- return nil, fmt.Errorf("no more extractors available after %d attempts: %w", attempt, err)
- }
- log.Printf("🔄 [MEMORY-EXTRACTION] Retrying with next model: %s", extractorModelID)
- }
- }
-
- return nil, fmt.Errorf("extraction failed after %d attempts, last error: %w", maxAttempts, lastError)
-}
-
-// tryExtraction attempts extraction with a specific model (internal helper)
-func (s *MemoryExtractionService) tryExtraction(
- ctx context.Context,
- userID string,
- extractorModelID string,
- messages []map[string]interface{},
- existingMemories []models.DecryptedMemory,
-) (*models.ExtractedMemoryFromLLM, error) {
-
- // Get provider and model
- provider, actualModel, err := s.getProviderAndModel(extractorModelID)
- if err != nil {
- return nil, fmt.Errorf("failed to get provider for extractor: %w", err)
- }
-
- log.Printf("🤖 [MEMORY-EXTRACTION] Using model: %s (%s)", extractorModelID, actualModel)
-
- // Build conversation transcript
- conversationTranscript := s.buildConversationTranscript(messages)
-
- // Build existing memories context (decrypted)
- existingMemoriesContext := s.buildExistingMemoriesContext(ctx, userID, existingMemories)
-
- // Build user prompt with existing memories
- userPrompt := fmt.Sprintf(`EXISTING MEMORIES:
-%s
-
-CONVERSATION:
-%s
-
-Analyze this conversation and extract ONLY NEW memories that are NOT already captured in the existing memories above. Return JSON with array of memories. If no new information, return empty array.`, existingMemoriesContext, conversationTranscript)
-
- // Build messages
- llmMessages := []map[string]interface{}{
- {
- "role": "system",
- "content": MemoryExtractionSystemPrompt,
- },
- {
- "role": "user",
- "content": userPrompt,
- },
- }
-
- // Build request with structured output
- requestBody := map[string]interface{}{
- "model": actualModel,
- "messages": llmMessages,
- "stream": false,
- "temperature": 0.3, // Low temp for consistency
- "response_format": map[string]interface{}{
- "type": "json_schema",
- "json_schema": map[string]interface{}{
- "name": "memory_extraction",
- "strict": true,
- "schema": memoryExtractionSchema,
- },
- },
- }
-
- reqBody, err := json.Marshal(requestBody)
- if err != nil {
- return nil, fmt.Errorf("failed to marshal request: %w", err)
- }
-
- // Create HTTP request with timeout
- httpReq, err := http.NewRequestWithContext(ctx, "POST", provider.BaseURL+"/chat/completions", bytes.NewBuffer(reqBody))
- if err != nil {
- return nil, fmt.Errorf("failed to create request: %w", err)
- }
-
- httpReq.Header.Set("Content-Type", "application/json")
- httpReq.Header.Set("Authorization", "Bearer "+provider.APIKey)
-
- // Send request with 60s timeout
- client := &http.Client{Timeout: 60 * time.Second}
- resp, err := client.Do(httpReq)
- if err != nil {
- return nil, fmt.Errorf("request failed: %w", err)
- }
- defer resp.Body.Close()
-
- body, err := io.ReadAll(resp.Body)
- if err != nil {
- return nil, fmt.Errorf("failed to read response: %w", err)
- }
-
- if resp.StatusCode != http.StatusOK {
- log.Printf("⚠️ [MEMORY-EXTRACTION] API error: %s", string(body))
- return nil, fmt.Errorf("API error (status %d): %s", resp.StatusCode, string(body))
- }
-
- // Parse response
- var apiResponse struct {
- Choices []struct {
- Message struct {
- Content string `json:"content"`
- } `json:"message"`
- } `json:"choices"`
- }
-
- if err := json.Unmarshal(body, &apiResponse); err != nil {
- return nil, fmt.Errorf("failed to parse API response: %w", err)
- }
-
- if len(apiResponse.Choices) == 0 {
- return nil, fmt.Errorf("no response from extractor model")
- }
-
- // Parse the extraction result
- var result models.ExtractedMemoryFromLLM
- content := apiResponse.Choices[0].Message.Content
-
- if err := json.Unmarshal([]byte(content), &result); err != nil {
- // SECURITY: Don't log decrypted content - only log length
- log.Printf("⚠️ [MEMORY-EXTRACTION] Failed to parse extraction: %v (response length: %d bytes)", err, len(content))
- return nil, fmt.Errorf("failed to parse extraction: %w", err)
- }
-
- return &result, nil
-}
-
-// calculateEngagement calculates conversation engagement score
-func (s *MemoryExtractionService) calculateEngagement(messages []map[string]interface{}) float64 {
- if len(messages) == 0 {
- return 0.0
- }
-
- userMessageCount := 0
- totalResponseLength := 0
-
- for _, msg := range messages {
- role, _ := msg["role"].(string)
- content, _ := msg["content"].(string)
-
- if role == "user" {
- userMessageCount++
- totalResponseLength += len(content)
- }
- }
-
- if userMessageCount == 0 {
- return 0.0
- }
-
- // Turn ratio (how much user participated)
- turnRatio := float64(userMessageCount) / float64(len(messages))
-
- // Length score (longer responses = more engaged)
- avgUserLength := totalResponseLength / userMessageCount
- lengthScore := float64(avgUserLength) / 200.0
- if lengthScore > 1.0 {
- lengthScore = 1.0
- }
-
- // Recency bonus (recent conversations get boost)
- recencyBonus := 1.0 // Assume all conversations being extracted are recent
-
- // Weighted engagement score
- engagement := (0.5 * turnRatio) + (0.3 * lengthScore) + (0.2 * recencyBonus)
-
- log.Printf("📊 [MEMORY-EXTRACTION] Engagement: %.2f (turn: %.2f, length: %.2f)", engagement, turnRatio, lengthScore)
-
- return engagement
-}
-
-// storeEngagement stores conversation engagement in database
-func (s *MemoryExtractionService) storeEngagement(
- ctx context.Context,
- userID string,
- conversationID string,
- messages []map[string]interface{},
- engagementScore float64,
-) error {
-
- userMessageCount := 0
- totalResponseLength := 0
-
- for _, msg := range messages {
- role, _ := msg["role"].(string)
- content, _ := msg["content"].(string)
-
- if role == "user" {
- userMessageCount++
- totalResponseLength += len(content)
- }
- }
-
- avgResponseLength := 0
- if userMessageCount > 0 {
- avgResponseLength = totalResponseLength / userMessageCount
- }
-
- engagement := &models.ConversationEngagement{
- ID: primitive.NewObjectID(),
- UserID: userID,
- ConversationID: conversationID,
- MessageCount: len(messages),
- UserMessageCount: userMessageCount,
- AvgResponseLength: avgResponseLength,
- EngagementScore: engagementScore,
- CreatedAt: time.Now(),
- UpdatedAt: time.Now(),
- }
-
- // Upsert (update if exists, insert if not)
- filter := bson.M{"userId": userID, "conversationId": conversationID}
- update := bson.M{
- "$set": bson.M{
- "messageCount": engagement.MessageCount,
- "userMessageCount": engagement.UserMessageCount,
- "avgResponseLength": engagement.AvgResponseLength,
- "engagementScore": engagement.EngagementScore,
- "updatedAt": engagement.UpdatedAt,
- },
- "$setOnInsert": bson.M{
- "_id": engagement.ID,
- "createdAt": engagement.CreatedAt,
- },
- }
-
- opts := options.Update().SetUpsert(true)
- _, err := s.engagementCollection.UpdateOne(ctx, filter, update, opts)
-
- return err
-}
-
-// buildConversationTranscript builds a human-readable transcript
-func (s *MemoryExtractionService) buildConversationTranscript(messages []map[string]interface{}) string {
- var builder strings.Builder
-
- for _, msg := range messages {
- role, _ := msg["role"].(string)
- content, _ := msg["content"].(string)
-
- // Skip system messages
- if role == "system" {
- continue
- }
-
- // Format message
- if role == "user" {
- builder.WriteString(fmt.Sprintf("USER: %s\n\n", content))
- } else if role == "assistant" {
- builder.WriteString(fmt.Sprintf("ASSISTANT: %s\n\n", content))
- }
- }
-
- return builder.String()
-}
-
-// buildExistingMemoriesContext formats existing memories for LLM context
-func (s *MemoryExtractionService) buildExistingMemoriesContext(
- ctx context.Context,
- userID string,
- memories []models.DecryptedMemory,
-) string {
- if len(memories) == 0 {
- return "(No existing memories - this is the first extraction)"
- }
-
- var builder strings.Builder
- builder.WriteString(fmt.Sprintf("(%d existing memories):\n\n", len(memories)))
-
- for i, mem := range memories {
- // Format: [category] content (tags: tag1, tag2)
- tags := ""
- if len(mem.Tags) > 0 {
- tags = fmt.Sprintf(" (tags: %s)", strings.Join(mem.Tags, ", "))
- }
-
- builder.WriteString(fmt.Sprintf("%d. [%s] %s%s\n", i+1, mem.Category, mem.DecryptedContent, tags))
- }
-
- return builder.String()
-}
-
-// getExtractorModelForUser gets user's preferred extractor model
-func (s *MemoryExtractionService) getExtractorModelForUser(ctx context.Context, userID string) (string, error) {
- // Query MongoDB for user preferences
- usersCollection := s.mongodb.Collection(database.CollectionUsers)
-
- var user models.User
- err := usersCollection.FindOne(ctx, bson.M{"supabaseUserId": userID}).Decode(&user)
- if err != nil {
- return "", nil // No user found, return empty (will use pool)
- }
-
- // If user has a preference and it's not empty, use it
- if user.Preferences.MemoryExtractorModelID != "" {
- log.Printf("🎯 [MEMORY-EXTRACTION] Using user-preferred model: %s", user.Preferences.MemoryExtractorModelID)
- return user.Preferences.MemoryExtractorModelID, nil
- }
-
- // No preference set - return empty (will use pool)
- return "", nil
-}
-
-// getProviderAndModel resolves model ID to provider and actual model name
-func (s *MemoryExtractionService) getProviderAndModel(modelID string) (*models.Provider, string, error) {
- if modelID == "" {
- return nil, "", fmt.Errorf("model ID is required")
- }
-
- // Try to resolve through ChatService (handles aliases)
- if s.chatService != nil {
- if provider, actualModel, found := s.chatService.ResolveModelAlias(modelID); found {
- return provider, actualModel, nil
- }
- }
-
- return nil, "", fmt.Errorf("model %s not found in providers", modelID)
-}
-
-// updateJobStatus updates job status
-func (s *MemoryExtractionService) updateJobStatus(ctx context.Context, jobID primitive.ObjectID, status string) {
- update := bson.M{
- "$set": bson.M{
- "status": status,
- },
- }
- s.jobCollection.UpdateOne(ctx, bson.M{"_id": jobID}, update)
-}
-
-// markJobCompleted marks job as completed
-func (s *MemoryExtractionService) markJobCompleted(ctx context.Context, jobID primitive.ObjectID) {
- now := time.Now()
- update := bson.M{
- "$set": bson.M{
- "status": models.JobStatusCompleted,
- "processedAt": now,
- },
- }
- s.jobCollection.UpdateOne(ctx, bson.M{"_id": jobID}, update)
-}
-
-// markJobFailed marks job as failed with error message
-func (s *MemoryExtractionService) markJobFailed(ctx context.Context, jobID primitive.ObjectID, errorMsg string) {
- update := bson.M{
- "$set": bson.M{
- "status": models.JobStatusFailed,
- "errorMessage": errorMsg,
- },
- "$inc": bson.M{
- "attemptCount": 1,
- },
- }
- s.jobCollection.UpdateOne(ctx, bson.M{"_id": jobID}, update)
-}
-
-// Helper function for pointer
diff --git a/backend/internal/services/memory_integration_test.go b/backend/internal/services/memory_integration_test.go
deleted file mode 100644
index bde92ed8..00000000
--- a/backend/internal/services/memory_integration_test.go
+++ /dev/null
@@ -1,465 +0,0 @@
-package services
-
-import (
- "testing"
- "time"
-
- "claraverse/internal/models"
-)
-
-// TestMemoryExtractionSchema tests the extraction schema structure
-func TestMemoryExtractionSchema(t *testing.T) {
- schema := memoryExtractionSchema
-
- // Verify schema structure
- if schema["type"] != "object" {
- t.Error("Schema should be object type")
- }
-
- properties, ok := schema["properties"].(map[string]interface{})
- if !ok {
- t.Fatal("Schema should have properties")
- }
-
- // Verify required fields
- requiredFields := []string{"memories"}
- required, ok := schema["required"].([]string)
- if !ok {
- t.Fatal("Schema should have required fields")
- }
-
- for _, field := range requiredFields {
- found := false
- for _, req := range required {
- if req == field {
- found = true
- break
- }
- }
- if !found {
- t.Errorf("Required field %s not found in schema", field)
- }
- }
-
- // Verify memories array structure
- memories, ok := properties["memories"].(map[string]interface{})
- if !ok {
- t.Fatal("memories field should exist")
- }
-
- if memories["type"] != "array" {
- t.Error("memories should be array type")
- }
-
- items, ok := memories["items"].(map[string]interface{})
- if !ok {
- t.Fatal("memories should have items definition")
- }
-
- itemProps, ok := items["properties"].(map[string]interface{})
- if !ok {
- t.Fatal("memory items should have properties")
- }
-
- // Verify memory item fields
- requiredMemoryFields := []string{"content", "category", "tags"}
- for _, field := range requiredMemoryFields {
- if _, exists := itemProps[field]; !exists {
- t.Errorf("Memory item should have field: %s", field)
- }
- }
-}
-
-// TestMemorySelectionSchema tests the selection schema structure
-func TestMemorySelectionSchema(t *testing.T) {
- schema := memorySelectionSchema
-
- // Verify schema structure
- if schema["type"] != "object" {
- t.Error("Schema should be object type")
- }
-
- properties, ok := schema["properties"].(map[string]interface{})
- if !ok {
- t.Fatal("Schema should have properties")
- }
-
- // Verify required fields
- requiredFields := []string{"selected_memory_ids", "reasoning"}
- required, ok := schema["required"].([]string)
- if !ok {
- t.Fatal("Schema should have required fields")
- }
-
- for _, field := range requiredFields {
- found := false
- for _, req := range required {
- if req == field {
- found = true
- break
- }
- }
- if !found {
- t.Errorf("Required field %s not found in schema", field)
- }
- }
-
- // Verify selected_memory_ids array structure
- selectedIDs, ok := properties["selected_memory_ids"].(map[string]interface{})
- if !ok {
- t.Fatal("selected_memory_ids field should exist")
- }
-
- if selectedIDs["type"] != "array" {
- t.Error("selected_memory_ids should be array type")
- }
-}
-
-// TestMemoryExtractionSystemPrompt tests the extraction prompt content
-func TestMemoryExtractionSystemPrompt(t *testing.T) {
- prompt := MemoryExtractionSystemPrompt
-
- // Verify prompt contains key instructions
- requiredPhrases := []string{
- "memory extraction system",
- "Personal Information",
- "Preferences",
- "Important Context",
- "Facts",
- "Instructions",
- }
-
- for _, phrase := range requiredPhrases {
- if !contains(prompt, phrase) {
- t.Errorf("Extraction prompt should contain: %q", phrase)
- }
- }
-
- // Verify prompt is not empty
- if len(prompt) < 100 {
- t.Error("Extraction prompt seems too short")
- }
-
- t.Logf("Extraction prompt length: %d characters", len(prompt))
-}
-
-// TestMemorySelectionSystemPrompt tests the selection prompt content
-func TestMemorySelectionSystemPrompt(t *testing.T) {
- prompt := MemorySelectionSystemPrompt
-
- // Verify prompt contains key instructions
- requiredPhrases := []string{
- "memory selection system",
- "MOST RELEVANT",
- "Direct Relevance",
- "Contextual Information",
- "User Preferences",
- "Instructions",
- }
-
- for _, phrase := range requiredPhrases {
- if !contains(prompt, phrase) {
- t.Errorf("Selection prompt should contain: %q", phrase)
- }
- }
-
- // Verify prompt is not empty
- if len(prompt) < 100 {
- t.Error("Selection prompt seems too short")
- }
-
- t.Logf("Selection prompt length: %d characters", len(prompt))
-}
-
-// TestConversationEngagementCalculation tests engagement score logic
-func TestConversationEngagementCalculation(t *testing.T) {
- tests := []struct {
- name string
- messageCount int
- userMessageCount int
- avgResponseLength int
- withinWeek bool
- minExpected float64
- maxExpected float64
- }{
- {
- name: "High engagement (balanced turn-taking, long responses, recent)",
- messageCount: 20,
- userMessageCount: 10,
- avgResponseLength: 250,
- withinWeek: true,
- minExpected: 0.70,
- maxExpected: 0.80,
- },
- {
- name: "Medium engagement (moderate activity)",
- messageCount: 10,
- userMessageCount: 4,
- avgResponseLength: 150,
- withinWeek: true,
- minExpected: 0.40,
- maxExpected: 0.70,
- },
- {
- name: "Low engagement (few messages, short responses, old)",
- messageCount: 4,
- userMessageCount: 1,
- avgResponseLength: 50,
- withinWeek: false,
- minExpected: 0.00,
- maxExpected: 0.30,
- },
- {
- name: "User dominated (high user ratio)",
- messageCount: 10,
- userMessageCount: 8,
- avgResponseLength: 200,
- withinWeek: true,
- minExpected: 0.70,
- maxExpected: 1.00,
- },
- }
-
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- // Calculate engagement score components
- turnRatio := float64(tt.userMessageCount) / float64(tt.messageCount)
- lengthScore := minFloat(1.0, float64(tt.avgResponseLength)/200.0)
- recencyBonus := 0.0
- if tt.withinWeek {
- recencyBonus = 1.0
- }
-
- // Engagement formula: (0.5 × TurnRatio) + (0.3 × LengthScore) + (0.2 × RecencyBonus)
- engagementScore := (0.5 * turnRatio) + (0.3 * lengthScore) + (0.2 * recencyBonus)
-
- if engagementScore < tt.minExpected || engagementScore > tt.maxExpected {
- t.Errorf("Expected engagement between %.2f and %.2f, got %.2f\n"+
- " Turn Ratio: %.2f, Length Score: %.2f, Recency: %.2f",
- tt.minExpected, tt.maxExpected, engagementScore,
- turnRatio, lengthScore, recencyBonus)
- }
-
- t.Logf("Engagement Score: %.2f (turn: %.2f, length: %.2f, recency: %.2f)",
- engagementScore, turnRatio, lengthScore, recencyBonus)
- })
- }
-}
-
-// TestDecayConfigDefaults tests the default decay configuration
-func TestDecayConfigDefaults(t *testing.T) {
- config := DefaultDecayConfig()
-
- // Test default weights
- if config.RecencyWeight != 0.4 {
- t.Errorf("Expected RecencyWeight 0.4, got %.2f", config.RecencyWeight)
- }
- if config.FrequencyWeight != 0.3 {
- t.Errorf("Expected FrequencyWeight 0.3, got %.2f", config.FrequencyWeight)
- }
- if config.EngagementWeight != 0.3 {
- t.Errorf("Expected EngagementWeight 0.3, got %.2f", config.EngagementWeight)
- }
-
- // Test weights sum to 1.0
- totalWeight := config.RecencyWeight + config.FrequencyWeight + config.EngagementWeight
- if totalWeight != 1.0 {
- t.Errorf("Weights should sum to 1.0, got %.2f", totalWeight)
- }
-
- // Test other defaults
- if config.RecencyDecayRate != 0.05 {
- t.Errorf("Expected RecencyDecayRate 0.05, got %.2f", config.RecencyDecayRate)
- }
- if config.FrequencyMax != 20 {
- t.Errorf("Expected FrequencyMax 20, got %d", config.FrequencyMax)
- }
- if config.ArchiveThreshold != 0.15 {
- t.Errorf("Expected ArchiveThreshold 0.15, got %.2f", config.ArchiveThreshold)
- }
-}
-
-// TestMemoryModelDefaults tests default values in Memory model
-func TestMemoryModelDefaults(t *testing.T) {
- // This test documents expected default states
- defaultAccessCount := int64(0)
- defaultIsArchived := false
- defaultVersion := int64(1)
-
- if defaultAccessCount != 0 {
- t.Errorf("New memories should have 0 access count")
- }
- if defaultIsArchived != false {
- t.Errorf("New memories should not be archived")
- }
- if defaultVersion != 1 {
- t.Errorf("New memories should start at version 1")
- }
-}
-
-// TestExtractedMemoryStructure tests the structure of extracted memories
-func TestExtractedMemoryStructure(t *testing.T) {
- // Test valid categories
- validCategories := map[string]bool{
- "personal_info": true,
- "preferences": true,
- "context": true,
- "fact": true,
- "instruction": true,
- }
-
- // Simulate an extracted memory (correct structure)
- testResult := models.ExtractedMemoryFromLLM{
- Memories: []struct {
- Content string `json:"content"`
- Category string `json:"category"`
- Tags []string `json:"tags"`
- }{
- {
- Content: "User prefers dark mode",
- Category: "preferences",
- Tags: []string{"ui", "theme", "preferences"},
- },
- },
- }
-
- // Validate we have memories
- if len(testResult.Memories) == 0 {
- t.Error("Should have at least one memory")
- }
-
- // Validate first memory
- testMemory := testResult.Memories[0]
-
- // Validate category
- if !validCategories[testMemory.Category] {
- t.Errorf("Invalid category: %s", testMemory.Category)
- }
-
- // Validate content not empty
- if testMemory.Content == "" {
- t.Error("Memory content should not be empty")
- }
-
- // Validate tags
- if len(testMemory.Tags) == 0 {
- t.Error("Memory should have at least one tag")
- }
-}
-
-// TestSelectedMemoriesStructure tests the structure of selected memories
-func TestSelectedMemoriesStructure(t *testing.T) {
- // Simulate a selection result
- selection := models.SelectedMemoriesFromLLM{
- SelectedMemoryIDs: []string{"id1", "id2", "id3"},
- Reasoning: "These memories are relevant because...",
- }
-
- // Validate IDs
- if len(selection.SelectedMemoryIDs) == 0 {
- t.Error("Selection should have memory IDs")
- }
-
- // Validate reasoning
- if selection.Reasoning == "" {
- t.Error("Selection should have reasoning")
- }
-
- // Max 5 memories rule
- maxMemories := 5
- if len(selection.SelectedMemoryIDs) > maxMemories {
- t.Errorf("Should not select more than %d memories, got %d", maxMemories, len(selection.SelectedMemoryIDs))
- }
-}
-
-// TestMemoryLifecycleStates tests valid state transitions
-func TestMemoryLifecycleStates(t *testing.T) {
- now := time.Now()
-
- // State 1: Newly created
- memory := models.Memory{
- AccessCount: 0,
- LastAccessedAt: nil,
- IsArchived: false,
- ArchivedAt: nil,
- CreatedAt: now,
- UpdatedAt: now,
- Version: 1,
- }
-
- if memory.IsArchived {
- t.Error("New memory should not be archived")
- }
- if memory.AccessCount != 0 {
- t.Error("New memory should have 0 access count")
- }
-
- // State 2: Accessed
- accessTime := now.Add(1 * time.Hour)
- memory.AccessCount = 1
- memory.LastAccessedAt = &accessTime
-
- if memory.LastAccessedAt == nil {
- t.Error("Accessed memory should have LastAccessedAt")
- }
-
- // State 3: Archived
- archiveTime := now.Add(90 * 24 * time.Hour)
- memory.IsArchived = true
- memory.ArchivedAt = &archiveTime
-
- if !memory.IsArchived {
- t.Error("Archived memory should have IsArchived=true")
- }
- if memory.ArchivedAt == nil {
- t.Error("Archived memory should have ArchivedAt timestamp")
- }
-}
-
-// TestMemorySystemPromptInjection tests the memory context formatting
-func TestMemorySystemPromptInjection(t *testing.T) {
- // Simulate building memory context
- memories := []models.DecryptedMemory{
- {DecryptedContent: "User prefers dark mode"},
- {DecryptedContent: "User name is Clara"},
- {DecryptedContent: "User timezone is America/New_York"},
- }
-
- // Build context string (simplified version of buildMemoryContext)
- var context string
- context = "\n\n## Relevant Context from Previous Conversations\n\n"
- for i, mem := range memories {
- context += string(rune('1'+i)) + ". " + mem.DecryptedContent + "\n"
- }
-
- // Verify format
- if !contains(context, "## Relevant Context") {
- t.Error("Context should have header")
- }
-
- for _, mem := range memories {
- if !contains(context, mem.DecryptedContent) {
- t.Errorf("Context should include memory: %s", mem.DecryptedContent)
- }
- }
-
- // Verify numbered list
- if !contains(context, "1. ") {
- t.Error("Context should use numbered list")
- }
-
- t.Logf("Generated context:\n%s", context)
-}
-
-// Helper function to check if string contains substring
-func contains(s, substr string) bool {
- return len(s) >= len(substr) && (s == substr || len(s) > len(substr) && (s[:len(substr)] == substr || contains(s[1:], substr)))
-}
-
-// Helper function for min (renamed to avoid conflict)
-func minFloat(a, b float64) float64 {
- if a < b {
- return a
- }
- return b
-}
diff --git a/backend/internal/services/memory_model_pool.go b/backend/internal/services/memory_model_pool.go
deleted file mode 100644
index dcca63ff..00000000
--- a/backend/internal/services/memory_model_pool.go
+++ /dev/null
@@ -1,430 +0,0 @@
-package services
-
-import (
- "database/sql"
- "fmt"
- "log"
- "sync"
- "time"
-
- "claraverse/internal/config"
- "claraverse/internal/models"
-)
-
-// MemoryModelPool manages multiple models for memory operations with health tracking and failover
-type MemoryModelPool struct {
- extractorModels []ModelCandidate
- selectorModels []ModelCandidate
- extractorIndex int
- selectorIndex int
- healthTracker map[string]*ModelHealth
- mu sync.Mutex
- chatService *ChatService
- db *sql.DB // Database connection for querying model_aliases
-}
-
-// ModelCandidate represents a model eligible for memory operations
-type ModelCandidate struct {
- ModelID string
- ProviderName string
- SpeedMs int
- DisplayName string
-}
-
-// ModelHealth tracks model health and failures
-type ModelHealth struct {
- FailureCount int
- SuccessCount int
- LastFailure time.Time
- LastSuccess time.Time
- IsHealthy bool
- ConsecutiveFails int
-}
-
-const (
- // Health thresholds
- MaxConsecutiveFailures = 3
- HealthCheckCooldown = 5 * time.Minute
- MinSuccessesToRecover = 2
-)
-
-// NewMemoryModelPool creates a new model pool by discovering eligible models from providers
-func NewMemoryModelPool(chatService *ChatService, db *sql.DB) (*MemoryModelPool, error) {
- pool := &MemoryModelPool{
- chatService: chatService,
- db: db,
- healthTracker: make(map[string]*ModelHealth),
- }
-
- // Discover models from ChatService
- if err := pool.discoverModels(); err != nil {
- log.Printf("⚠️ [MODEL-POOL] Failed to discover models: %v", err)
- log.Printf("⚠️ [MODEL-POOL] Memory services will be disabled until models with memory flags are added")
- }
-
- if len(pool.extractorModels) == 0 {
- log.Printf("⚠️ [MODEL-POOL] No extractor models found - memory extraction disabled")
- }
-
- if len(pool.selectorModels) == 0 {
- log.Printf("⚠️ [MODEL-POOL] No selector models found - memory selection disabled")
- }
-
- if len(pool.extractorModels) > 0 || len(pool.selectorModels) > 0 {
- log.Printf("🎯 [MODEL-POOL] Initialized with %d extractors, %d selectors",
- len(pool.extractorModels), len(pool.selectorModels))
- }
-
- // Return pool even if empty - allows graceful degradation
- return pool, nil
-}
-
-// discoverModels scans database for models with memory flags
-func (p *MemoryModelPool) discoverModels() error {
- // First try loading from database (MySQL-first approach)
- dbModels, err := p.discoverFromDatabase()
- if err == nil && len(dbModels) > 0 {
- log.Printf("✅ [MODEL-POOL] Discovered %d models from database", len(dbModels))
- return nil
- }
-
- // Fallback: Load providers configuration from providers.json
- providersConfig, err := config.LoadProviders("providers.json")
- if err != nil {
- // Gracefully handle missing providers.json (expected in admin UI workflow)
- log.Printf("⚠️ [MODEL-POOL] providers.json not found or invalid: %v", err)
- log.Printf("ℹ️ [MODEL-POOL] This is normal when starting with empty database")
- return nil // Not a fatal error - just means no models configured yet
- }
-
- for _, providerConfig := range providersConfig.Providers {
- if !providerConfig.Enabled {
- continue
- }
-
- for alias, modelAlias := range providerConfig.ModelAliases {
- // Get model configuration map (convert from ModelAlias)
- modelConfig := modelAliasToMap(modelAlias)
-
- // Check if model supports memory extraction
- if isExtractor, ok := modelConfig["memory_extractor"].(bool); ok && isExtractor {
- candidate := ModelCandidate{
- ModelID: alias,
- ProviderName: providerConfig.Name,
- DisplayName: getDisplayName(modelConfig),
- SpeedMs: getSpeedMs(modelConfig),
- }
- p.extractorModels = append(p.extractorModels, candidate)
- p.healthTracker[alias] = &ModelHealth{IsHealthy: true}
-
- log.Printf("✅ [MODEL-POOL] Found extractor: %s (%s) - %dms",
- alias, providerConfig.Name, candidate.SpeedMs)
- }
-
- // Check if model supports memory selection
- if isSelector, ok := modelConfig["memory_selector"].(bool); ok && isSelector {
- // Avoid duplicates if model is both extractor and selector
- if _, exists := p.healthTracker[alias]; !exists {
- p.healthTracker[alias] = &ModelHealth{IsHealthy: true}
- }
-
- candidate := ModelCandidate{
- ModelID: alias,
- ProviderName: providerConfig.Name,
- DisplayName: getDisplayName(modelConfig),
- SpeedMs: getSpeedMs(modelConfig),
- }
- p.selectorModels = append(p.selectorModels, candidate)
-
- log.Printf("✅ [MODEL-POOL] Found selector: %s (%s) - %dms",
- alias, providerConfig.Name, candidate.SpeedMs)
- }
- }
- }
-
- // Sort by speed (fastest first)
- p.sortModelsBySpeed(p.extractorModels)
- p.sortModelsBySpeed(p.selectorModels)
-
- return nil
-}
-
-// discoverFromDatabase loads memory models from database (model_aliases table)
-func (p *MemoryModelPool) discoverFromDatabase() ([]ModelCandidate, error) {
- if p.db == nil {
- return nil, fmt.Errorf("database connection not available")
- }
-
- // Query for models with memory_extractor or memory_selector flags
- rows, err := p.db.Query(`
- SELECT
- a.alias_name,
- pr.name as provider_name,
- a.display_name,
- COALESCE(a.structured_output_speed_ms, 999999) as speed_ms,
- COALESCE(a.memory_extractor, 0) as memory_extractor,
- COALESCE(a.memory_selector, 0) as memory_selector
- FROM model_aliases a
- JOIN providers pr ON a.provider_id = pr.id
- WHERE a.memory_extractor = 1 OR a.memory_selector = 1
- `)
-
- if err != nil {
- return nil, fmt.Errorf("failed to query model_aliases: %w", err)
- }
- defer rows.Close()
-
- var candidates []ModelCandidate
- for rows.Next() {
- var aliasName, providerName, displayName string
- var speedMs int
- var isExtractor, isSelector int
-
- if err := rows.Scan(&aliasName, &providerName, &displayName, &speedMs, &isExtractor, &isSelector); err != nil {
- log.Printf("⚠️ [MODEL-POOL] Failed to scan row: %v", err)
- continue
- }
-
- candidate := ModelCandidate{
- ModelID: aliasName,
- ProviderName: providerName,
- DisplayName: displayName,
- SpeedMs: speedMs,
- }
-
- if isExtractor == 1 {
- p.extractorModels = append(p.extractorModels, candidate)
- p.healthTracker[aliasName] = &ModelHealth{IsHealthy: true}
- log.Printf("✅ [MODEL-POOL] Found extractor from DB: %s (%s) - %dms", aliasName, providerName, speedMs)
- }
-
- if isSelector == 1 {
- // Avoid duplicates if model is both extractor and selector
- if _, exists := p.healthTracker[aliasName]; !exists {
- p.healthTracker[aliasName] = &ModelHealth{IsHealthy: true}
- }
- p.selectorModels = append(p.selectorModels, candidate)
- log.Printf("✅ [MODEL-POOL] Found selector from DB: %s (%s) - %dms", aliasName, providerName, speedMs)
- }
-
- candidates = append(candidates, candidate)
- }
-
- // Sort by speed (fastest first)
- p.sortModelsBySpeed(p.extractorModels)
- p.sortModelsBySpeed(p.selectorModels)
-
- return candidates, nil
-}
-
-// modelAliasToMap converts ModelAlias struct to map for easier access
-func modelAliasToMap(alias models.ModelAlias) map[string]interface{} {
- m := make(map[string]interface{})
-
- // Set display_name
- m["display_name"] = alias.DisplayName
-
- // Set structured_output_speed_ms if available
- if alias.StructuredOutputSpeedMs != nil {
- m["structured_output_speed_ms"] = *alias.StructuredOutputSpeedMs
- }
-
- // Set memory flags if available
- if alias.MemoryExtractor != nil {
- m["memory_extractor"] = *alias.MemoryExtractor
- }
- if alias.MemorySelector != nil {
- m["memory_selector"] = *alias.MemorySelector
- }
-
- return m
-}
-
-// GetNextExtractor returns the next healthy extractor model using round-robin
-func (p *MemoryModelPool) GetNextExtractor() (string, error) {
- p.mu.Lock()
- defer p.mu.Unlock()
-
- if len(p.extractorModels) == 0 {
- return "", fmt.Errorf("no extractor models available")
- }
-
- // Try all models in round-robin fashion
- attempts := 0
- maxAttempts := len(p.extractorModels)
-
- for attempts < maxAttempts {
- candidate := p.extractorModels[p.extractorIndex]
- p.extractorIndex = (p.extractorIndex + 1) % len(p.extractorModels)
- attempts++
-
- // Check if model is healthy
- health := p.healthTracker[candidate.ModelID]
- if health.IsHealthy {
- log.Printf("🔄 [MODEL-POOL] Selected extractor: %s (healthy)", candidate.ModelID)
- return candidate.ModelID, nil
- }
-
- // Check if enough time has passed since last failure (cooldown)
- if time.Since(health.LastFailure) > HealthCheckCooldown {
- log.Printf("⚡ [MODEL-POOL] Retrying extractor after cooldown: %s", candidate.ModelID)
- health.IsHealthy = true
- health.ConsecutiveFails = 0
- return candidate.ModelID, nil
- }
-
- log.Printf("⏭️ [MODEL-POOL] Skipping unhealthy extractor: %s (fails: %d, last: %s ago)",
- candidate.ModelID, health.ConsecutiveFails, time.Since(health.LastFailure).Round(time.Second))
- }
-
- // All models unhealthy - return fastest anyway as last resort
- log.Printf("⚠️ [MODEL-POOL] All extractors unhealthy, using fastest: %s", p.extractorModels[0].ModelID)
- return p.extractorModels[0].ModelID, nil
-}
-
-// GetNextSelector returns the next healthy selector model using round-robin
-func (p *MemoryModelPool) GetNextSelector() (string, error) {
- p.mu.Lock()
- defer p.mu.Unlock()
-
- if len(p.selectorModels) == 0 {
- return "", fmt.Errorf("no selector models available")
- }
-
- // Try all models in round-robin fashion
- attempts := 0
- maxAttempts := len(p.selectorModels)
-
- for attempts < maxAttempts {
- candidate := p.selectorModels[p.selectorIndex]
- p.selectorIndex = (p.selectorIndex + 1) % len(p.selectorModels)
- attempts++
-
- // Check if model is healthy
- health := p.healthTracker[candidate.ModelID]
- if health.IsHealthy {
- log.Printf("🔄 [MODEL-POOL] Selected selector: %s (healthy)", candidate.ModelID)
- return candidate.ModelID, nil
- }
-
- // Check if enough time has passed since last failure (cooldown)
- if time.Since(health.LastFailure) > HealthCheckCooldown {
- log.Printf("⚡ [MODEL-POOL] Retrying selector after cooldown: %s", candidate.ModelID)
- health.IsHealthy = true
- health.ConsecutiveFails = 0
- return candidate.ModelID, nil
- }
-
- log.Printf("⏭️ [MODEL-POOL] Skipping unhealthy selector: %s (fails: %d, last: %s ago)",
- candidate.ModelID, health.ConsecutiveFails, time.Since(health.LastFailure).Round(time.Second))
- }
-
- // All models unhealthy - return fastest anyway as last resort
- log.Printf("⚠️ [MODEL-POOL] All selectors unhealthy, using fastest: %s", p.selectorModels[0].ModelID)
- return p.selectorModels[0].ModelID, nil
-}
-
-// MarkSuccess records a successful model call
-func (p *MemoryModelPool) MarkSuccess(modelID string) {
- p.mu.Lock()
- defer p.mu.Unlock()
-
- health, exists := p.healthTracker[modelID]
- if !exists {
- return
- }
-
- health.SuccessCount++
- health.LastSuccess = time.Now()
- health.ConsecutiveFails = 0
-
- // Restore health after consecutive successes
- if !health.IsHealthy && health.SuccessCount >= MinSuccessesToRecover {
- health.IsHealthy = true
- log.Printf("💚 [MODEL-POOL] Model recovered: %s (successes: %d)", modelID, health.SuccessCount)
- }
-}
-
-// MarkFailure records a failed model call
-func (p *MemoryModelPool) MarkFailure(modelID string) {
- p.mu.Lock()
- defer p.mu.Unlock()
-
- health, exists := p.healthTracker[modelID]
- if !exists {
- return
- }
-
- health.FailureCount++
- health.ConsecutiveFails++
- health.LastFailure = time.Now()
-
- // Mark unhealthy after consecutive failures
- if health.ConsecutiveFails >= MaxConsecutiveFailures {
- health.IsHealthy = false
- log.Printf("💔 [MODEL-POOL] Model marked unhealthy: %s (consecutive fails: %d, total fails: %d)",
- modelID, health.ConsecutiveFails, health.FailureCount)
- } else {
- log.Printf("⚠️ [MODEL-POOL] Model failure: %s (consecutive: %d/%d)",
- modelID, health.ConsecutiveFails, MaxConsecutiveFailures)
- }
-}
-
-// GetStats returns current pool statistics
-func (p *MemoryModelPool) GetStats() map[string]interface{} {
- p.mu.Lock()
- defer p.mu.Unlock()
-
- healthyExtractors := 0
- healthySelectors := 0
-
- for _, model := range p.extractorModels {
- if p.healthTracker[model.ModelID].IsHealthy {
- healthyExtractors++
- }
- }
-
- for _, model := range p.selectorModels {
- if p.healthTracker[model.ModelID].IsHealthy {
- healthySelectors++
- }
- }
-
- return map[string]interface{}{
- "total_extractors": len(p.extractorModels),
- "healthy_extractors": healthyExtractors,
- "total_selectors": len(p.selectorModels),
- "healthy_selectors": healthySelectors,
- }
-}
-
-// Helper functions
-
-func getDisplayName(modelConfig map[string]interface{}) string {
- if name, ok := modelConfig["display_name"].(string); ok {
- return name
- }
- return "Unknown"
-}
-
-func getSpeedMs(modelConfig map[string]interface{}) int {
- if speed, ok := modelConfig["structured_output_speed_ms"].(float64); ok {
- return int(speed)
- }
- if speed, ok := modelConfig["structured_output_speed_ms"].(int); ok {
- return speed
- }
- return 999999 // Default to slow if not specified
-}
-
-func (p *MemoryModelPool) sortModelsBySpeed(models []ModelCandidate) {
- // Simple bubble sort (fine for small arrays)
- n := len(models)
- for i := 0; i < n-1; i++ {
- for j := 0; j < n-i-1; j++ {
- if models[j].SpeedMs > models[j+1].SpeedMs {
- models[j], models[j+1] = models[j+1], models[j]
- }
- }
- }
-}
diff --git a/backend/internal/services/memory_selection_service.go b/backend/internal/services/memory_selection_service.go
deleted file mode 100644
index 366732d8..00000000
--- a/backend/internal/services/memory_selection_service.go
+++ /dev/null
@@ -1,443 +0,0 @@
-package services
-
-import (
- "bytes"
- "context"
- "encoding/json"
- "fmt"
- "io"
- "log"
- "net/http"
- "strings"
- "time"
-
- "claraverse/internal/crypto"
- "claraverse/internal/database"
- "claraverse/internal/models"
- "go.mongodb.org/mongo-driver/bson/primitive"
-)
-
-// MemorySelectionService handles selection of relevant memories using LLMs
-type MemorySelectionService struct {
- mongodb *database.MongoDB
- encryptionService *crypto.EncryptionService
- providerService *ProviderService
- memoryStorageService *MemoryStorageService
- chatService *ChatService
- modelPool *MemoryModelPool // Dynamic model pool with round-robin and failover
-}
-
-// Memory selection system prompt
-const MemorySelectionSystemPrompt = `You are a memory selection system for Clara AI. Given the user's recent conversation and their memory bank, select the MOST RELEVANT memories.
-
-SELECTION CRITERIA:
-1. **Direct Relevance**: Memory directly relates to current conversation topic
-2. **Contextual Information**: Memory provides important background context
-3. **User Preferences**: Memory contains preferences that affect how to respond
-4. **Instructions**: Memory contains guidelines the user wants followed
-
-RULES:
-- Select up to 5 memories maximum (fewer is better if not all are relevant)
-- Prioritize memories that prevent asking redundant questions
-- Include memories that personalize the response
-- Skip memories that are obvious or unrelated
-- If no memories are relevant, return empty array
-
-Return JSON with selected memory IDs and brief reasoning.`
-
-// memorySelectionSchema defines structured output for memory selection
-var memorySelectionSchema = map[string]interface{}{
- "type": "object",
- "properties": map[string]interface{}{
- "selected_memory_ids": map[string]interface{}{
- "type": "array",
- "items": map[string]interface{}{
- "type": "string",
- "description": "Memory ID from the provided list",
- },
- "description": "IDs of memories relevant to current conversation (max 5)",
- },
- "reasoning": map[string]interface{}{
- "type": "string",
- "description": "Brief explanation of why these memories are relevant",
- },
- },
- "required": []string{"selected_memory_ids", "reasoning"},
- "additionalProperties": false,
-}
-
-// NewMemorySelectionService creates a new memory selection service
-func NewMemorySelectionService(
- mongodb *database.MongoDB,
- encryptionService *crypto.EncryptionService,
- providerService *ProviderService,
- memoryStorageService *MemoryStorageService,
- chatService *ChatService,
- modelPool *MemoryModelPool,
-) *MemorySelectionService {
- return &MemorySelectionService{
- mongodb: mongodb,
- encryptionService: encryptionService,
- providerService: providerService,
- memoryStorageService: memoryStorageService,
- chatService: chatService,
- modelPool: modelPool,
- }
-}
-
-// SelectRelevantMemories selects memories relevant to the current conversation
-func (s *MemorySelectionService) SelectRelevantMemories(
- ctx context.Context,
- userID string,
- recentMessages []map[string]interface{},
- maxMemories int,
-) ([]models.DecryptedMemory, error) {
-
- // Get all active memories for user
- activeMemories, err := s.memoryStorageService.GetActiveMemories(ctx, userID)
- if err != nil {
- return nil, fmt.Errorf("failed to get active memories: %w", err)
- }
-
- // If no memories, return empty
- if len(activeMemories) == 0 {
- log.Printf("📭 [MEMORY-SELECTION] No active memories for user %s", userID)
- return []models.DecryptedMemory{}, nil
- }
-
- log.Printf("🔍 [MEMORY-SELECTION] Selecting from %d active memories for user %s", len(activeMemories), userID)
-
- // If we have fewer memories than max, just return all and update access
- if len(activeMemories) <= maxMemories {
- log.Printf("📚 [MEMORY-SELECTION] Returning all %d memories (below max %d)", len(activeMemories), maxMemories)
- memoryIDs := make([]primitive.ObjectID, len(activeMemories))
- for i, mem := range activeMemories {
- memoryIDs[i] = mem.ID
- }
- s.memoryStorageService.UpdateMemoryAccess(ctx, memoryIDs)
- return activeMemories, nil
- }
-
- // Use LLM to select relevant memories
- selectedIDs, reasoning, err := s.selectMemoriesWithLLM(ctx, userID, activeMemories, recentMessages, maxMemories)
- if err != nil {
- log.Printf("⚠️ [MEMORY-SELECTION] LLM selection failed: %v, falling back to top %d by score", err, maxMemories)
- // Fallback: return top N by score
- selectedMemories := activeMemories
- if len(selectedMemories) > maxMemories {
- selectedMemories = selectedMemories[:maxMemories]
- }
- memoryIDs := make([]primitive.ObjectID, len(selectedMemories))
- for i, mem := range selectedMemories {
- memoryIDs[i] = mem.ID
- }
- s.memoryStorageService.UpdateMemoryAccess(ctx, memoryIDs)
- return selectedMemories, nil
- }
-
- log.Printf("🎯 [MEMORY-SELECTION] LLM selected %d memories: %s", len(selectedIDs), reasoning)
-
- // Filter memories by selected IDs
- selectedMemories := s.filterMemoriesByIDs(activeMemories, selectedIDs)
-
- // Update access counts and timestamps
- memoryIDs := make([]primitive.ObjectID, len(selectedMemories))
- for i, mem := range selectedMemories {
- memoryIDs[i] = mem.ID
- }
- if len(memoryIDs) > 0 {
- s.memoryStorageService.UpdateMemoryAccess(ctx, memoryIDs)
- }
-
- return selectedMemories, nil
-}
-
-// selectMemoriesWithLLM uses LLM to select relevant memories with automatic failover
-func (s *MemorySelectionService) selectMemoriesWithLLM(
- ctx context.Context,
- userID string,
- memories []models.DecryptedMemory,
- recentMessages []map[string]interface{},
- maxMemories int,
-) ([]string, string, error) {
-
- // Check if user has a custom selector model preference
- userPreferredModel, err := s.getSelectorModelForUser(ctx, userID)
- var selectorModelID string
-
- if err == nil && userPreferredModel != "" {
- // User has a preference, use it
- selectorModelID = userPreferredModel
- log.Printf("👤 [MEMORY-SELECTION] Using user-preferred model: %s", selectorModelID)
- } else {
- // No user preference, get from model pool
- selectorModelID, err = s.modelPool.GetNextSelector()
- if err != nil {
- return nil, "", fmt.Errorf("no selector models available: %w", err)
- }
- }
-
- // Try selection with automatic failover (max 3 attempts)
- maxAttempts := 3
- var lastError error
-
- for attempt := 1; attempt <= maxAttempts; attempt++ {
- selectedIDs, reasoning, err := s.trySelection(ctx, selectorModelID, memories, recentMessages, maxMemories)
-
- if err == nil {
- // Success!
- s.modelPool.MarkSuccess(selectorModelID)
- return selectedIDs, reasoning, nil
- }
-
- // Selection failed
- lastError = err
- s.modelPool.MarkFailure(selectorModelID)
- log.Printf("⚠️ [MEMORY-SELECTION] Attempt %d/%d failed with model %s: %v",
- attempt, maxAttempts, selectorModelID, err)
-
- // If not last attempt, get next model from pool
- if attempt < maxAttempts {
- selectorModelID, err = s.modelPool.GetNextSelector()
- if err != nil {
- return nil, "", fmt.Errorf("no more selectors available after %d attempts: %w", attempt, err)
- }
- log.Printf("🔄 [MEMORY-SELECTION] Retrying with next model: %s", selectorModelID)
- }
- }
-
- return nil, "", fmt.Errorf("selection failed after %d attempts, last error: %w", maxAttempts, lastError)
-}
-
-// trySelection attempts selection with a specific model (internal helper)
-func (s *MemorySelectionService) trySelection(
- ctx context.Context,
- selectorModelID string,
- memories []models.DecryptedMemory,
- recentMessages []map[string]interface{},
- maxMemories int,
-) ([]string, string, error) {
-
- // Get provider and model
- provider, actualModel, err := s.getProviderAndModel(selectorModelID)
- if err != nil {
- return nil, "", fmt.Errorf("failed to get provider for selector: %w", err)
- }
-
- log.Printf("🤖 [MEMORY-SELECTION] Using model: %s (%s)", selectorModelID, actualModel)
-
- // Build conversation context
- conversationContext := s.buildConversationContext(recentMessages)
-
- // Build memory list for prompt
- memoryList := s.buildMemoryListPrompt(memories)
-
- // Build user prompt
- userPrompt := fmt.Sprintf(`RECENT CONVERSATION:
-%s
-
-MEMORY BANK (%d memories):
-%s
-
-Select up to %d memories that are DIRECTLY relevant to the current conversation. Return JSON with selected memory IDs and reasoning.`,
- conversationContext, len(memories), memoryList, maxMemories)
-
- // Build messages
- llmMessages := []map[string]interface{}{
- {
- "role": "system",
- "content": MemorySelectionSystemPrompt,
- },
- {
- "role": "user",
- "content": userPrompt,
- },
- }
-
- // Build request with structured output
- requestBody := map[string]interface{}{
- "model": actualModel,
- "messages": llmMessages,
- "stream": false,
- "temperature": 0.2, // Low temp for consistency
- "response_format": map[string]interface{}{
- "type": "json_schema",
- "json_schema": map[string]interface{}{
- "name": "memory_selection",
- "strict": true,
- "schema": memorySelectionSchema,
- },
- },
- }
-
- reqBody, err := json.Marshal(requestBody)
- if err != nil {
- return nil, "", fmt.Errorf("failed to marshal request: %w", err)
- }
-
- // Create HTTP request with timeout
- httpReq, err := http.NewRequestWithContext(ctx, "POST", provider.BaseURL+"/chat/completions", bytes.NewBuffer(reqBody))
- if err != nil {
- return nil, "", fmt.Errorf("failed to create request: %w", err)
- }
-
- httpReq.Header.Set("Content-Type", "application/json")
- httpReq.Header.Set("Authorization", "Bearer "+provider.APIKey)
-
- // Send request with 30s timeout
- client := &http.Client{Timeout: 30 * time.Second}
- resp, err := client.Do(httpReq)
- if err != nil {
- return nil, "", fmt.Errorf("request failed: %w", err)
- }
- defer resp.Body.Close()
-
- body, err := io.ReadAll(resp.Body)
- if err != nil {
- return nil, "", fmt.Errorf("failed to read response: %w", err)
- }
-
- if resp.StatusCode != http.StatusOK {
- log.Printf("⚠️ [MEMORY-SELECTION] API error: %s", string(body))
- return nil, "", fmt.Errorf("API error (status %d): %s", resp.StatusCode, string(body))
- }
-
- // Parse response
- var apiResponse struct {
- Choices []struct {
- Message struct {
- Content string `json:"content"`
- } `json:"message"`
- } `json:"choices"`
- }
-
- if err := json.Unmarshal(body, &apiResponse); err != nil {
- return nil, "", fmt.Errorf("failed to parse API response: %w", err)
- }
-
- if len(apiResponse.Choices) == 0 {
- return nil, "", fmt.Errorf("no response from selector model")
- }
-
- // Parse the selection result
- var result models.SelectedMemoriesFromLLM
- content := apiResponse.Choices[0].Message.Content
-
- if err := json.Unmarshal([]byte(content), &result); err != nil {
- // SECURITY: Don't log decrypted memory content - only log length
- log.Printf("⚠️ [MEMORY-SELECTION] Failed to parse selection: %v (response length: %d bytes)", err, len(content))
- return nil, "", fmt.Errorf("failed to parse selection: %w", err)
- }
-
- return result.SelectedMemoryIDs, result.Reasoning, nil
-}
-
-// buildConversationContext builds a concise context from recent messages
-func (s *MemorySelectionService) buildConversationContext(messages []map[string]interface{}) string {
- var builder strings.Builder
-
- // Only include last 10 messages to keep prompt concise
- startIdx := len(messages) - 10
- if startIdx < 0 {
- startIdx = 0
- }
-
- for _, msg := range messages[startIdx:] {
- role, _ := msg["role"].(string)
- content, _ := msg["content"].(string)
-
- // Skip system messages
- if role == "system" {
- continue
- }
-
- // Format message
- if role == "user" {
- builder.WriteString(fmt.Sprintf("USER: %s\n", content))
- } else if role == "assistant" {
- // Truncate long assistant messages
- if len(content) > 200 {
- content = content[:200] + "..."
- }
- builder.WriteString(fmt.Sprintf("ASSISTANT: %s\n", content))
- }
- }
-
- return builder.String()
-}
-
-// buildMemoryListPrompt creates a numbered list of memories for the prompt
-func (s *MemorySelectionService) buildMemoryListPrompt(memories []models.DecryptedMemory) string {
- var builder strings.Builder
-
- for i, mem := range memories {
- builder.WriteString(fmt.Sprintf("%d. [ID: %s] [Category: %s] %s\n",
- i+1,
- mem.ID.Hex(),
- mem.Category,
- mem.DecryptedContent,
- ))
- }
-
- return builder.String()
-}
-
-// filterMemoriesByIDs filters memories to only include selected IDs
-func (s *MemorySelectionService) filterMemoriesByIDs(
- memories []models.DecryptedMemory,
- selectedIDs []string,
-) []models.DecryptedMemory {
-
- // Build set for O(1) lookup
- idSet := make(map[string]bool)
- for _, id := range selectedIDs {
- idSet[id] = true
- }
-
- filtered := make([]models.DecryptedMemory, 0, len(selectedIDs))
-
- for _, mem := range memories {
- if idSet[mem.ID.Hex()] {
- filtered = append(filtered, mem)
- }
- }
-
- return filtered
-}
-
-// getSelectorModelForUser gets user's preferred selector model
-func (s *MemorySelectionService) getSelectorModelForUser(ctx context.Context, userID string) (string, error) {
- // Query MongoDB for user preferences
- usersCollection := s.mongodb.Collection(database.CollectionUsers)
-
- var user models.User
- err := usersCollection.FindOne(ctx, map[string]interface{}{"supabaseUserId": userID}).Decode(&user)
- if err != nil {
- return "", nil // No user found, return empty (will use pool)
- }
-
- // If user has a preference and it's not empty, use it
- if user.Preferences.MemorySelectorModelID != "" {
- log.Printf("🎯 [MEMORY-SELECTION] Using user-preferred model: %s", user.Preferences.MemorySelectorModelID)
- return user.Preferences.MemorySelectorModelID, nil
- }
-
- // No preference set - return empty (will use pool)
- return "", nil
-}
-
-// getProviderAndModel resolves model ID to provider and actual model name
-func (s *MemorySelectionService) getProviderAndModel(modelID string) (*models.Provider, string, error) {
- if modelID == "" {
- return nil, "", fmt.Errorf("model ID is required")
- }
-
- // Try to resolve through ChatService (handles aliases)
- if s.chatService != nil {
- if provider, actualModel, found := s.chatService.ResolveModelAlias(modelID); found {
- return provider, actualModel, nil
- }
- }
-
- return nil, "", fmt.Errorf("model %s not found in providers", modelID)
-}
diff --git a/backend/internal/services/memory_storage_service.go b/backend/internal/services/memory_storage_service.go
deleted file mode 100644
index 7153c761..00000000
--- a/backend/internal/services/memory_storage_service.go
+++ /dev/null
@@ -1,534 +0,0 @@
-package services
-
-import (
- "context"
- "crypto/sha256"
- "encoding/hex"
- "fmt"
- "log"
- "strings"
- "time"
-
- "claraverse/internal/crypto"
- "claraverse/internal/database"
- "claraverse/internal/models"
- "go.mongodb.org/mongo-driver/bson"
- "go.mongodb.org/mongo-driver/bson/primitive"
- "go.mongodb.org/mongo-driver/mongo"
- "go.mongodb.org/mongo-driver/mongo/options"
-)
-
-// MemoryStorageService handles CRUD operations for memories with encryption and deduplication
-type MemoryStorageService struct {
- mongodb *database.MongoDB
- collection *mongo.Collection
- encryptionService *crypto.EncryptionService
-}
-
-// NewMemoryStorageService creates a new memory storage service
-func NewMemoryStorageService(mongodb *database.MongoDB, encryptionService *crypto.EncryptionService) *MemoryStorageService {
- return &MemoryStorageService{
- mongodb: mongodb,
- collection: mongodb.Collection(database.CollectionMemories),
- encryptionService: encryptionService,
- }
-}
-
-// CreateMemory creates a new memory with encryption and deduplication
-func (s *MemoryStorageService) CreateMemory(ctx context.Context, userID, content, category string, tags []string, sourceEngagement float64, conversationID string) (*models.Memory, error) {
- if userID == "" {
- return nil, fmt.Errorf("user ID is required")
- }
- if content == "" {
- return nil, fmt.Errorf("memory content is required")
- }
-
- // Normalize and hash content for deduplication
- normalizedContent := s.normalizeContent(content)
- contentHash := s.calculateHash(normalizedContent)
-
- // Check for duplicate
- existingMemory, err := s.CheckDuplicate(ctx, userID, contentHash)
- if err != nil && err != mongo.ErrNoDocuments {
- return nil, fmt.Errorf("failed to check duplicate: %w", err)
- }
-
- // If duplicate exists, update it instead
- if existingMemory != nil {
- log.Printf("🔄 [MEMORY-STORAGE] Duplicate memory found (ID: %s), updating instead", existingMemory.ID.Hex())
- return s.UpdateExistingMemory(ctx, existingMemory, tags, sourceEngagement)
- }
-
- // Encrypt content
- encryptedContent, err := s.encryptionService.Encrypt(userID, []byte(content))
- if err != nil {
- return nil, fmt.Errorf("failed to encrypt memory content: %w", err)
- }
-
- // Initial score is based solely on source engagement
- initialScore := sourceEngagement
-
- // Create new memory
- memory := &models.Memory{
- ID: primitive.NewObjectID(),
- UserID: userID,
- ConversationID: conversationID,
- EncryptedContent: encryptedContent,
- ContentHash: contentHash,
- Category: category,
- Tags: tags,
- Score: initialScore,
- AccessCount: 0,
- LastAccessedAt: nil,
- IsArchived: false,
- ArchivedAt: nil,
- SourceEngagement: sourceEngagement,
- CreatedAt: time.Now(),
- UpdatedAt: time.Now(),
- Version: 1,
- }
-
- // Insert into database
- _, err = s.collection.InsertOne(ctx, memory)
- if err != nil {
- return nil, fmt.Errorf("failed to insert memory: %w", err)
- }
-
- log.Printf("✅ [MEMORY-STORAGE] Created new memory (ID: %s, Category: %s, Score: %.2f)", memory.ID.Hex(), category, initialScore)
- return memory, nil
-}
-
-// UpdateExistingMemory updates an existing memory (for deduplication)
-func (s *MemoryStorageService) UpdateExistingMemory(ctx context.Context, memory *models.Memory, newTags []string, sourceEngagement float64) (*models.Memory, error) {
- // Merge tags (avoid duplicates)
- tagMap := make(map[string]bool)
- for _, tag := range memory.Tags {
- tagMap[tag] = true
- }
- for _, tag := range newTags {
- tagMap[tag] = true
- }
- mergedTags := make([]string, 0, len(tagMap))
- for tag := range tagMap {
- mergedTags = append(mergedTags, tag)
- }
-
- // Boost score slightly on re-mention (indicates importance)
- newScore := memory.Score + 0.1
- if newScore > 1.0 {
- newScore = 1.0
- }
-
- // Update engagement if higher
- if sourceEngagement > memory.SourceEngagement {
- memory.SourceEngagement = sourceEngagement
- }
-
- // Update memory
- update := bson.M{
- "$set": bson.M{
- "tags": mergedTags,
- "score": newScore,
- "sourceEngagement": memory.SourceEngagement,
- "updatedAt": time.Now(),
- },
- "$inc": bson.M{
- "version": 1,
- },
- }
-
- result := s.collection.FindOneAndUpdate(
- ctx,
- bson.M{"_id": memory.ID},
- update,
- options.FindOneAndUpdate().SetReturnDocument(options.After),
- )
-
- var updatedMemory models.Memory
- if err := result.Decode(&updatedMemory); err != nil {
- return nil, fmt.Errorf("failed to decode updated memory: %w", err)
- }
-
- log.Printf("🔄 [MEMORY-STORAGE] Updated memory (ID: %s, New Score: %.2f, Version: %d)", updatedMemory.ID.Hex(), newScore, updatedMemory.Version)
- return &updatedMemory, nil
-}
-
-// UpdateMemoryInPlace atomically updates a memory (content, category, tags)
-// SECURITY: Replaces delete-create pattern to prevent race conditions
-func (s *MemoryStorageService) UpdateMemoryInPlace(
- ctx context.Context,
- userID string,
- memoryID primitive.ObjectID,
- content string,
- category string,
- tags []string,
- sourceEngagement float64,
- conversationID string,
-) (*models.Memory, error) {
- if userID == "" {
- return nil, fmt.Errorf("user ID is required")
- }
- if content == "" {
- return nil, fmt.Errorf("memory content is required")
- }
-
- // Normalize and hash new content
- normalizedContent := s.normalizeContent(content)
- contentHash := s.calculateHash(normalizedContent)
-
- // Encrypt new content
- encryptedContent, err := s.encryptionService.Encrypt(userID, []byte(content))
- if err != nil {
- return nil, fmt.Errorf("failed to encrypt memory content: %w", err)
- }
-
- now := time.Now()
-
- // Atomic update with user authorization check
- update := bson.M{
- "$set": bson.M{
- "encryptedContent": encryptedContent,
- "contentHash": contentHash,
- "category": category,
- "tags": tags,
- "sourceEngagement": sourceEngagement,
- "conversationId": conversationID,
- "updatedAt": now,
- },
- "$inc": bson.M{
- "version": 1,
- },
- }
-
- // SECURITY: Filter includes userId to prevent unauthorized updates
- filter := bson.M{
- "_id": memoryID,
- "userId": userID, // Critical: ensures user can only update their own memories
- }
-
- result := s.collection.FindOneAndUpdate(
- ctx,
- filter,
- update,
- options.FindOneAndUpdate().SetReturnDocument(options.After),
- )
-
- var updatedMemory models.Memory
- if err := result.Decode(&updatedMemory); err != nil {
- if err == mongo.ErrNoDocuments {
- return nil, fmt.Errorf("memory not found or access denied")
- }
- return nil, fmt.Errorf("failed to update memory: %w", err)
- }
-
- log.Printf("✅ [MEMORY-STORAGE] Updated memory atomically (ID: %s, Version: %d)", updatedMemory.ID.Hex(), updatedMemory.Version)
- return &updatedMemory, nil
-}
-
-// GetMemory retrieves and decrypts a single memory
-func (s *MemoryStorageService) GetMemory(ctx context.Context, userID string, memoryID primitive.ObjectID) (*models.DecryptedMemory, error) {
- var memory models.Memory
- err := s.collection.FindOne(ctx, bson.M{"_id": memoryID, "userId": userID}).Decode(&memory)
- if err != nil {
- if err == mongo.ErrNoDocuments {
- return nil, fmt.Errorf("memory not found")
- }
- return nil, fmt.Errorf("failed to get memory: %w", err)
- }
-
- // Decrypt content
- decryptedBytes, err := s.encryptionService.Decrypt(userID, memory.EncryptedContent)
- if err != nil {
- return nil, fmt.Errorf("failed to decrypt memory: %w", err)
- }
-
- decryptedMemory := &models.DecryptedMemory{
- Memory: memory,
- DecryptedContent: string(decryptedBytes),
- }
-
- return decryptedMemory, nil
-}
-
-// ListMemories retrieves memories with optional filters and pagination
-func (s *MemoryStorageService) ListMemories(ctx context.Context, userID string, category string, tags []string, includeArchived bool, page, pageSize int) ([]models.DecryptedMemory, int64, error) {
- // Build filter
- filter := bson.M{"userId": userID}
-
- if !includeArchived {
- filter["isArchived"] = false
- }
-
- if category != "" {
- filter["category"] = category
- }
-
- if len(tags) > 0 {
- filter["tags"] = bson.M{"$in": tags}
- }
-
- // Count total
- total, err := s.collection.CountDocuments(ctx, filter)
- if err != nil {
- return nil, 0, fmt.Errorf("failed to count memories: %w", err)
- }
-
- // Calculate pagination
- skip := (page - 1) * pageSize
- findOptions := options.Find().
- SetSort(bson.D{{Key: "score", Value: -1}, {Key: "updatedAt", Value: -1}}).
- SetSkip(int64(skip)).
- SetLimit(int64(pageSize))
-
- // Find memories
- cursor, err := s.collection.Find(ctx, filter, findOptions)
- if err != nil {
- return nil, 0, fmt.Errorf("failed to find memories: %w", err)
- }
- defer cursor.Close(ctx)
-
- var memories []models.Memory
- if err := cursor.All(ctx, &memories); err != nil {
- return nil, 0, fmt.Errorf("failed to decode memories: %w", err)
- }
-
- // Decrypt all memories
- decryptedMemories := make([]models.DecryptedMemory, 0, len(memories))
- for _, memory := range memories {
- decryptedBytes, err := s.encryptionService.Decrypt(userID, memory.EncryptedContent)
- if err != nil {
- log.Printf("⚠️ [MEMORY-STORAGE] Failed to decrypt memory %s: %v", memory.ID.Hex(), err)
- continue
- }
-
- decryptedMemories = append(decryptedMemories, models.DecryptedMemory{
- Memory: memory,
- DecryptedContent: string(decryptedBytes),
- })
- }
-
- return decryptedMemories, total, nil
-}
-
-// GetActiveMemories retrieves all active (non-archived) memories for a user (decrypted)
-func (s *MemoryStorageService) GetActiveMemories(ctx context.Context, userID string) ([]models.DecryptedMemory, error) {
- filter := bson.M{
- "userId": userID,
- "isArchived": false,
- }
-
- findOptions := options.Find().SetSort(bson.D{{Key: "score", Value: -1}})
-
- cursor, err := s.collection.Find(ctx, filter, findOptions)
- if err != nil {
- return nil, fmt.Errorf("failed to find active memories: %w", err)
- }
- defer cursor.Close(ctx)
-
- var memories []models.Memory
- if err := cursor.All(ctx, &memories); err != nil {
- return nil, fmt.Errorf("failed to decode memories: %w", err)
- }
-
- // Decrypt all memories
- decryptedMemories := make([]models.DecryptedMemory, 0, len(memories))
- for _, memory := range memories {
- decryptedBytes, err := s.encryptionService.Decrypt(userID, memory.EncryptedContent)
- if err != nil {
- log.Printf("⚠️ [MEMORY-STORAGE] Failed to decrypt memory %s: %v", memory.ID.Hex(), err)
- continue
- }
-
- decryptedMemories = append(decryptedMemories, models.DecryptedMemory{
- Memory: memory,
- DecryptedContent: string(decryptedBytes),
- })
- }
-
- log.Printf("📚 [MEMORY-STORAGE] Retrieved %d active memories for user %s", len(decryptedMemories), userID)
- return decryptedMemories, nil
-}
-
-// UpdateMemoryAccess increments access count and updates last accessed timestamp
-func (s *MemoryStorageService) UpdateMemoryAccess(ctx context.Context, memoryIDs []primitive.ObjectID) error {
- if len(memoryIDs) == 0 {
- return nil
- }
-
- now := time.Now()
- filter := bson.M{"_id": bson.M{"$in": memoryIDs}}
- update := bson.M{
- "$inc": bson.M{"accessCount": 1},
- "$set": bson.M{"lastAccessedAt": now},
- }
-
- result, err := s.collection.UpdateMany(ctx, filter, update)
- if err != nil {
- return fmt.Errorf("failed to update memory access: %w", err)
- }
-
- log.Printf("📊 [MEMORY-STORAGE] Updated access for %d memories", result.ModifiedCount)
- return nil
-}
-
-// ArchiveMemory marks a memory as archived
-func (s *MemoryStorageService) ArchiveMemory(ctx context.Context, userID string, memoryID primitive.ObjectID) error {
- now := time.Now()
- update := bson.M{
- "$set": bson.M{
- "isArchived": true,
- "archivedAt": now,
- "updatedAt": now,
- },
- }
-
- result, err := s.collection.UpdateOne(ctx, bson.M{"_id": memoryID, "userId": userID}, update)
- if err != nil {
- return fmt.Errorf("failed to archive memory: %w", err)
- }
-
- if result.MatchedCount == 0 {
- return fmt.Errorf("memory not found or access denied")
- }
-
- log.Printf("📦 [MEMORY-STORAGE] Archived memory %s", memoryID.Hex())
- return nil
-}
-
-// UnarchiveMemory restores an archived memory
-func (s *MemoryStorageService) UnarchiveMemory(ctx context.Context, userID string, memoryID primitive.ObjectID) error {
- update := bson.M{
- "$set": bson.M{
- "isArchived": false,
- "archivedAt": nil,
- "updatedAt": time.Now(),
- },
- }
-
- result, err := s.collection.UpdateOne(ctx, bson.M{"_id": memoryID, "userId": userID}, update)
- if err != nil {
- return fmt.Errorf("failed to unarchive memory: %w", err)
- }
-
- if result.MatchedCount == 0 {
- return fmt.Errorf("memory not found or access denied")
- }
-
- log.Printf("📤 [MEMORY-STORAGE] Unarchived memory %s", memoryID.Hex())
- return nil
-}
-
-// DeleteMemory permanently deletes a memory
-func (s *MemoryStorageService) DeleteMemory(ctx context.Context, userID string, memoryID primitive.ObjectID) error {
- result, err := s.collection.DeleteOne(ctx, bson.M{"_id": memoryID, "userId": userID})
- if err != nil {
- return fmt.Errorf("failed to delete memory: %w", err)
- }
-
- if result.DeletedCount == 0 {
- return fmt.Errorf("memory not found or access denied")
- }
-
- log.Printf("🗑️ [MEMORY-STORAGE] Deleted memory %s", memoryID.Hex())
- return nil
-}
-
-// CheckDuplicate checks if a memory with the same content hash exists
-func (s *MemoryStorageService) CheckDuplicate(ctx context.Context, userID, contentHash string) (*models.Memory, error) {
- var memory models.Memory
- err := s.collection.FindOne(ctx, bson.M{"userId": userID, "contentHash": contentHash}).Decode(&memory)
- if err != nil {
- return nil, err
- }
- return &memory, nil
-}
-
-// normalizeContent normalizes content for deduplication
-func (s *MemoryStorageService) normalizeContent(content string) string {
- // Convert to lowercase
- normalized := strings.ToLower(content)
-
- // Replace word separators with spaces first (before removing other punctuation)
- // This prevents words from merging when punctuation is removed
- normalized = strings.ReplaceAll(normalized, "\n", " ")
- normalized = strings.ReplaceAll(normalized, "\t", " ")
- normalized = strings.ReplaceAll(normalized, "\r", " ")
- normalized = strings.ReplaceAll(normalized, "-", " ")
- normalized = strings.ReplaceAll(normalized, "_", " ")
-
- // Trim whitespace
- normalized = strings.TrimSpace(normalized)
-
- // Remove punctuation (simple version)
- normalized = strings.Map(func(r rune) rune {
- if (r >= 'a' && r <= 'z') || (r >= '0' && r <= '9') || r == ' ' {
- return r
- }
- return -1
- }, normalized)
-
- // Collapse multiple spaces
- normalized = strings.Join(strings.Fields(normalized), " ")
- return normalized
-}
-
-// calculateHash calculates SHA-256 hash of content
-func (s *MemoryStorageService) calculateHash(content string) string {
- hash := sha256.Sum256([]byte(content))
- return hex.EncodeToString(hash[:])
-}
-
-// GetMemoryStats returns statistics about user's memories
-func (s *MemoryStorageService) GetMemoryStats(ctx context.Context, userID string) (map[string]interface{}, error) {
- // Count total memories
- total, err := s.collection.CountDocuments(ctx, bson.M{"userId": userID})
- if err != nil {
- return nil, fmt.Errorf("failed to count total memories: %w", err)
- }
-
- // Count active memories
- active, err := s.collection.CountDocuments(ctx, bson.M{"userId": userID, "isArchived": false})
- if err != nil {
- return nil, fmt.Errorf("failed to count active memories: %w", err)
- }
-
- // Count archived memories
- archived, err := s.collection.CountDocuments(ctx, bson.M{"userId": userID, "isArchived": true})
- if err != nil {
- return nil, fmt.Errorf("failed to count archived memories: %w", err)
- }
-
- // Calculate average score
- pipeline := mongo.Pipeline{
- {{Key: "$match", Value: bson.M{"userId": userID, "isArchived": false}}},
- {{Key: "$group", Value: bson.M{
- "_id": nil,
- "avgScore": bson.M{"$avg": "$score"},
- }}},
- }
-
- cursor, err := s.collection.Aggregate(ctx, pipeline)
- if err != nil {
- return nil, fmt.Errorf("failed to aggregate scores: %w", err)
- }
- defer cursor.Close(ctx)
-
- var avgScoreResult struct {
- AvgScore float64 `bson:"avgScore"`
- }
- avgScore := 0.0
- if cursor.Next(ctx) {
- if err := cursor.Decode(&avgScoreResult); err == nil {
- avgScore = avgScoreResult.AvgScore
- }
- }
-
- stats := map[string]interface{}{
- "total_memories": total,
- "active_memories": active,
- "archived_memories": archived,
- "avg_score": avgScore,
- }
-
- return stats, nil
-}
diff --git a/backend/internal/services/memory_storage_service_test.go b/backend/internal/services/memory_storage_service_test.go
deleted file mode 100644
index 7b19e6e6..00000000
--- a/backend/internal/services/memory_storage_service_test.go
+++ /dev/null
@@ -1,346 +0,0 @@
-package services
-
-import (
- "testing"
-)
-
-// TestNormalizeContent tests content normalization for deduplication
-func TestNormalizeContent(t *testing.T) {
- service := &MemoryStorageService{}
-
- tests := []struct {
- name string
- input string
- expected string
- }{
- {
- name: "Basic normalization",
- input: "User prefers dark mode",
- expected: "user prefers dark mode",
- },
- {
- name: "Remove punctuation",
- input: "User's name is John, and he likes coffee!",
- expected: "users name is john and he likes coffee",
- },
- {
- name: "Collapse whitespace",
- input: "User likes lots of spaces",
- expected: "user likes lots of spaces",
- },
- {
- name: "Mixed case and punctuation",
- input: "User PREFERS Dark-Mode!!!",
- expected: "user prefers dark mode",
- },
- {
- name: "Trim whitespace",
- input: " user prefers dark mode ",
- expected: "user prefers dark mode",
- },
- {
- name: "Numbers preserved",
- input: "User is 25 years old",
- expected: "user is 25 years old",
- },
- {
- name: "Special characters removed",
- input: "User's email: john@example.com",
- expected: "users email johnexamplecom",
- },
- }
-
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- result := service.normalizeContent(tt.input)
- if result != tt.expected {
- t.Errorf("Expected: %q, got: %q", tt.expected, result)
- }
- })
- }
-}
-
-// TestCalculateHash ensures consistent hashing
-func TestCalculateHash(t *testing.T) {
- service := &MemoryStorageService{}
-
- tests := []struct {
- name string
- input1 string
- input2 string
- shouldMatch bool
- }{
- {
- name: "Identical strings",
- input1: "user prefers dark mode",
- input2: "user prefers dark mode",
- shouldMatch: true,
- },
- {
- name: "Different strings",
- input1: "user prefers dark mode",
- input2: "user prefers light mode",
- shouldMatch: false,
- },
- {
- name: "Empty string",
- input1: "",
- input2: "",
- shouldMatch: true,
- },
- }
-
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- hash1 := service.calculateHash(tt.input1)
- hash2 := service.calculateHash(tt.input2)
-
- if tt.shouldMatch && hash1 != hash2 {
- t.Errorf("Expected hashes to match: %s != %s", hash1, hash2)
- }
-
- if !tt.shouldMatch && hash1 == hash2 {
- t.Errorf("Expected hashes to differ, both got: %s", hash1)
- }
-
- // Verify SHA-256 produces 64 character hex string
- if len(hash1) != 64 {
- t.Errorf("Expected 64 character hash, got %d", len(hash1))
- }
- })
- }
-}
-
-// TestDeduplicationLogic tests the deduplication flow
-func TestDeduplicationLogic(t *testing.T) {
- service := &MemoryStorageService{}
-
- // Test cases that should be considered duplicates after normalization
- duplicates := []struct {
- original string
- variant string
- }{
- {
- original: "User prefers dark mode",
- variant: "USER PREFERS DARK MODE",
- },
- {
- original: "User prefers dark mode",
- variant: "User prefers dark-mode!",
- },
- {
- original: "User likes coffee",
- variant: "User likes coffee!!!",
- },
- {
- original: "User name is John",
- variant: " User name is John ",
- },
- }
-
- for _, tt := range duplicates {
- t.Run(tt.original, func(t *testing.T) {
- normalized1 := service.normalizeContent(tt.original)
- normalized2 := service.normalizeContent(tt.variant)
-
- hash1 := service.calculateHash(normalized1)
- hash2 := service.calculateHash(normalized2)
-
- if hash1 != hash2 {
- t.Errorf("Expected duplicates to have same hash:\n Original: %q -> %q -> %s\n Variant: %q -> %q -> %s",
- tt.original, normalized1, hash1,
- tt.variant, normalized2, hash2,
- )
- }
- })
- }
-
- // Test cases that should NOT be considered duplicates
- nonDuplicates := []struct {
- content1 string
- content2 string
- }{
- {
- content1: "User prefers dark mode",
- content2: "User prefers light mode",
- },
- {
- content1: "User likes coffee",
- content2: "User likes tea",
- },
- {
- content1: "User is 25 years old",
- content2: "User is 30 years old",
- },
- }
-
- for i, tt := range nonDuplicates {
- t.Run(string(rune('A'+i)), func(t *testing.T) {
- normalized1 := service.normalizeContent(tt.content1)
- normalized2 := service.normalizeContent(tt.content2)
-
- hash1 := service.calculateHash(normalized1)
- hash2 := service.calculateHash(normalized2)
-
- if hash1 == hash2 {
- t.Errorf("Expected different hashes:\n Content1: %q -> %q\n Content2: %q -> %q\n Both got: %s",
- tt.content1, normalized1,
- tt.content2, normalized2,
- hash1,
- )
- }
- })
- }
-}
-
-// TestContentHashCollisions checks for hash collision resistance
-func TestContentHashCollisions(t *testing.T) {
- service := &MemoryStorageService{}
-
- // Generate 1000 different normalized contents with guaranteed uniqueness
- contents := make([]string, 1000)
- for i := 0; i < 1000; i++ {
- // Use index to guarantee uniqueness
- contents[i] = service.normalizeContent(string(rune('a'+(i%26))) + " test content " + string(rune('0'+(i%10))) + string(rune('0'+((i/10)%10))) + string(rune('0'+((i/100)%10))))
- }
-
- // Calculate hashes and check for collisions
- hashes := make(map[string]string)
- collisionCount := 0
- for _, content := range contents {
- hash := service.calculateHash(content)
-
- if existingContent, exists := hashes[hash]; exists {
- // Only report as collision if content is actually different
- if existingContent != content {
- t.Errorf("Hash collision detected!\n Hash: %s\n Content1: %q\n Content2: %q",
- hash, existingContent, content)
- collisionCount++
- }
- } else {
- hashes[hash] = content
- }
- }
-
- if collisionCount > 0 {
- t.Errorf("Found %d true hash collisions", collisionCount)
- }
-
- t.Logf("Generated %d unique hashes without collisions", len(hashes))
-}
-
-// TestNormalizationEdgeCases tests edge cases in normalization
-func TestNormalizationEdgeCases(t *testing.T) {
- service := &MemoryStorageService{}
-
- tests := []struct {
- name string
- input string
- expected string
- }{
- {
- name: "Only punctuation",
- input: "!@#$%^&*()",
- expected: "",
- },
- {
- name: "Only whitespace",
- input: " ",
- expected: "",
- },
- {
- name: "Unicode characters",
- input: "User likes café ☕",
- expected: "user likes caf",
- },
- {
- name: "Mixed alphanumeric",
- input: "abc123DEF456",
- expected: "abc123def456",
- },
- {
- name: "Newlines and tabs",
- input: "User\nprefers\tdark\nmode",
- expected: "user prefers dark mode",
- },
- }
-
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- result := service.normalizeContent(tt.input)
- if result != tt.expected {
- t.Errorf("Expected: %q, got: %q", tt.expected, result)
- }
- })
- }
-}
-
-// TestHashConsistency ensures hashing is deterministic
-func TestHashConsistency(t *testing.T) {
- service := &MemoryStorageService{}
-
- content := "user prefers dark mode"
-
- // Calculate hash multiple times
- hash1 := service.calculateHash(content)
- hash2 := service.calculateHash(content)
- hash3 := service.calculateHash(content)
-
- if hash1 != hash2 || hash2 != hash3 {
- t.Errorf("Hash should be deterministic, got different values: %s, %s, %s", hash1, hash2, hash3)
- }
-}
-
-// TestMemoryCategories ensures category values are valid
-func TestMemoryCategories(t *testing.T) {
- validCategories := []string{
- "personal_info",
- "preferences",
- "context",
- "fact",
- "instruction",
- }
-
- // This test documents the expected categories
- t.Logf("Valid memory categories: %v", validCategories)
-
- for _, category := range validCategories {
- if category == "" {
- t.Errorf("Category should not be empty")
- }
- }
-}
-
-// BenchmarkNormalizeContent benchmarks content normalization
-func BenchmarkNormalizeContent(b *testing.B) {
- service := &MemoryStorageService{}
- content := "User prefers dark mode and likes to use the application at night!"
-
- b.ResetTimer()
- for i := 0; i < b.N; i++ {
- service.normalizeContent(content)
- }
-}
-
-// BenchmarkCalculateHash benchmarks hash calculation
-func BenchmarkCalculateHash(b *testing.B) {
- service := &MemoryStorageService{}
- content := "user prefers dark mode and likes to use the application at night"
-
- b.ResetTimer()
- for i := 0; i < b.N; i++ {
- service.calculateHash(content)
- }
-}
-
-// BenchmarkDeduplicationPipeline benchmarks the full deduplication pipeline
-func BenchmarkDeduplicationPipeline(b *testing.B) {
- service := &MemoryStorageService{}
- content := "User prefers dark mode and likes to use the application at night!"
-
- b.ResetTimer()
- for i := 0; i < b.N; i++ {
- normalized := service.normalizeContent(content)
- service.calculateHash(normalized)
- }
-}
diff --git a/backend/internal/services/metrics.go b/backend/internal/services/metrics.go
deleted file mode 100644
index 8ca2a2d0..00000000
--- a/backend/internal/services/metrics.go
+++ /dev/null
@@ -1,114 +0,0 @@
-package services
-
-import (
- "github.com/prometheus/client_golang/prometheus"
- "github.com/prometheus/client_golang/prometheus/promauto"
-)
-
-// Metrics holds all custom Prometheus metrics for the application
-type Metrics struct {
- // WebSocket metrics
- WebSocketConnections prometheus.Gauge
- WebSocketMessages *prometheus.CounterVec
-
- // Chat metrics
- ChatRequests prometheus.Counter
- ChatRequestLatency prometheus.Histogram
- ChatErrors *prometheus.CounterVec
-
- // Connection manager reference for dynamic metrics
- connManager *ConnectionManager
-}
-
-var globalMetrics *Metrics
-
-// InitMetrics initializes the Prometheus metrics
-func InitMetrics(connManager *ConnectionManager) *Metrics {
- metrics := &Metrics{
- connManager: connManager,
-
- // WebSocket active connections (gauge - can go up and down)
- WebSocketConnections: promauto.NewGauge(prometheus.GaugeOpts{
- Name: "claraverse_websocket_connections_active",
- Help: "Number of active WebSocket connections",
- }),
-
- // WebSocket messages by type (counter - only goes up)
- WebSocketMessages: promauto.NewCounterVec(prometheus.CounterOpts{
- Name: "claraverse_websocket_messages_total",
- Help: "Total number of WebSocket messages by type",
- }, []string{"type", "direction"}), // direction: "inbound" or "outbound"
-
- // Chat requests counter
- ChatRequests: promauto.NewCounter(prometheus.CounterOpts{
- Name: "claraverse_chat_requests_total",
- Help: "Total number of chat requests processed",
- }),
-
- // Chat request latency histogram
- ChatRequestLatency: promauto.NewHistogram(prometheus.HistogramOpts{
- Name: "claraverse_chat_request_duration_seconds",
- Help: "Chat request latency in seconds",
- Buckets: []float64{0.1, 0.5, 1, 2, 5, 10, 30, 60, 120}, // up to 2 minutes for LLM responses
- }),
-
- // Chat errors by type
- ChatErrors: promauto.NewCounterVec(prometheus.CounterOpts{
- Name: "claraverse_chat_errors_total",
- Help: "Total number of chat errors by type",
- }, []string{"error_type"}),
- }
-
- // Register a collector that updates WebSocket connections from ConnectionManager
- prometheus.MustRegister(prometheus.NewGaugeFunc(
- prometheus.GaugeOpts{
- Name: "claraverse_websocket_connections_current",
- Help: "Current number of active WebSocket connections (from connection manager)",
- },
- func() float64 {
- if connManager != nil {
- return float64(connManager.Count())
- }
- return 0
- },
- ))
-
- globalMetrics = metrics
- return metrics
-}
-
-// GetMetrics returns the global metrics instance
-func GetMetrics() *Metrics {
- return globalMetrics
-}
-
-// RecordWebSocketConnect records a new WebSocket connection
-func (m *Metrics) RecordWebSocketConnect() {
- m.WebSocketConnections.Inc()
-}
-
-// RecordWebSocketDisconnect records a WebSocket disconnection
-func (m *Metrics) RecordWebSocketDisconnect() {
- m.WebSocketConnections.Dec()
-}
-
-// RecordWebSocketMessage records a WebSocket message
-func (m *Metrics) RecordWebSocketMessage(msgType, direction string) {
- m.WebSocketMessages.WithLabelValues(msgType, direction).Inc()
-}
-
-// RecordChatRequest records a chat request
-func (m *Metrics) RecordChatRequest() {
- m.ChatRequests.Inc()
-}
-
-// RecordChatLatency records chat request latency
-func (m *Metrics) RecordChatLatency(seconds float64) {
- m.ChatRequestLatency.Observe(seconds)
-}
-
-// RecordChatError records a chat error
-func (m *Metrics) RecordChatError(errorType string) {
- m.ChatErrors.WithLabelValues(errorType).Inc()
-}
-
diff --git a/backend/internal/services/model_management_service.go b/backend/internal/services/model_management_service.go
deleted file mode 100644
index d39676ae..00000000
--- a/backend/internal/services/model_management_service.go
+++ /dev/null
@@ -1,1354 +0,0 @@
-package services
-
-import (
- "bytes"
- "claraverse/internal/database"
- "claraverse/internal/models"
- "context"
- "database/sql"
- "encoding/json"
- "fmt"
- "io"
- "log"
- "net/http"
- "os"
- "strings"
- "sync"
- "time"
-)
-
-// ModelManagementService handles model CRUD operations with dual-write to SQLite and providers.json
-type ModelManagementService struct {
- db *database.DB
- providersFile string
- fileMutex sync.Mutex // Protects providers.json file operations
-}
-
-// NewModelManagementService creates a new model management service
-// providersFile is optional - if empty, only database operations are performed
-func NewModelManagementService(db *database.DB) *ModelManagementService {
- return &ModelManagementService{
- db: db,
- providersFile: "", // No longer using providers file
- }
-}
-
-// ================== DUAL-WRITE COORDINATOR ==================
-
-// CreateModel creates a new model in both database and providers.json
-func (s *ModelManagementService) CreateModel(ctx context.Context, req *CreateModelRequest) (*models.Model, error) {
- log.Printf("📝 [MODEL-MGMT] Creating model: %s (provider %d)", req.ModelID, req.ProviderID)
-
- // Step 1: Begin SQLite transaction
- tx, err := s.db.Begin()
- if err != nil {
- return nil, fmt.Errorf("failed to begin transaction: %w", err)
- }
- defer tx.Rollback()
-
- // Step 2: Insert into database
- _, err = tx.Exec(`
- INSERT INTO models (id, provider_id, name, display_name, description, context_length,
- supports_tools, supports_streaming, supports_vision, is_visible, system_prompt, fetched_at)
- VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
- `, req.ModelID, req.ProviderID, req.Name, req.DisplayName, req.Description, req.ContextLength,
- req.SupportsTools, req.SupportsStreaming, req.SupportsVision, req.IsVisible, req.SystemPrompt, time.Now())
-
- if err != nil {
- return nil, fmt.Errorf("failed to insert model: %w", err)
- }
-
- // Step 3: Commit transaction
- if err := tx.Commit(); err != nil {
- return nil, fmt.Errorf("failed to commit transaction: %w", err)
- }
-
- log.Printf("✅ [MODEL-MGMT] Created model: %s", req.ModelID)
-
- // Fetch and return the created model
- return s.GetModelByID(req.ModelID)
-}
-
-// UpdateModel updates an existing model in both database and providers.json
-func (s *ModelManagementService) UpdateModel(ctx context.Context, modelID string, req *UpdateModelRequest) (*models.Model, error) {
- log.Printf("📝 [MODEL-MGMT] Updating model: %s", modelID)
-
- // Build dynamic update query
- updateParts := []string{}
- args := []interface{}{}
-
- if req.DisplayName != nil {
- updateParts = append(updateParts, "display_name = ?")
- args = append(args, *req.DisplayName)
- }
- if req.Description != nil {
- updateParts = append(updateParts, "description = ?")
- args = append(args, *req.Description)
- }
- if req.ContextLength != nil {
- updateParts = append(updateParts, "context_length = ?")
- args = append(args, *req.ContextLength)
- }
- if req.SupportsTools != nil {
- updateParts = append(updateParts, "supports_tools = ?")
- args = append(args, *req.SupportsTools)
- }
- if req.SupportsStreaming != nil {
- updateParts = append(updateParts, "supports_streaming = ?")
- args = append(args, *req.SupportsStreaming)
- }
- if req.SupportsVision != nil {
- updateParts = append(updateParts, "supports_vision = ?")
- args = append(args, *req.SupportsVision)
- }
- if req.IsVisible != nil {
- updateParts = append(updateParts, "is_visible = ?")
- args = append(args, *req.IsVisible)
- log.Printf("[DEBUG] Adding is_visible to update: value=%v type=%T", *req.IsVisible, *req.IsVisible)
- } else {
- log.Printf("[DEBUG] is_visible field is nil, not updating")
- }
- if req.SystemPrompt != nil {
- updateParts = append(updateParts, "system_prompt = ?")
- args = append(args, *req.SystemPrompt)
- }
- if req.SmartToolRouter != nil {
- updateParts = append(updateParts, "smart_tool_router = ?")
- args = append(args, *req.SmartToolRouter)
- }
- if req.FreeTier != nil {
- updateParts = append(updateParts, "free_tier = ?")
- args = append(args, *req.FreeTier)
- }
-
- if len(updateParts) == 0 {
- return s.GetModelByID(modelID)
- }
-
- // Add WHERE clause
- args = append(args, modelID)
- query := fmt.Sprintf("UPDATE models SET %s WHERE id = ?", joinStrings(updateParts, ", "))
-
- log.Printf("[DEBUG] SQL Query: %s", query)
- log.Printf("[DEBUG] SQL Args: %v", args)
-
- // Step 1: Begin transaction
- tx, err := s.db.Begin()
- if err != nil {
- return nil, fmt.Errorf("failed to begin transaction: %w", err)
- }
- defer tx.Rollback()
-
- // Step 2: Execute update
- result, err := tx.Exec(query, args...)
- if err != nil {
- return nil, fmt.Errorf("failed to update model: %w", err)
- }
-
- rowsAffected, _ := result.RowsAffected()
- log.Printf("[DEBUG] SQL execution successful, rows affected: %d", rowsAffected)
-
- // Step 3: Commit transaction
- if err := tx.Commit(); err != nil {
- return nil, fmt.Errorf("failed to commit transaction: %w", err)
- }
-
- log.Printf("✅ [MODEL-MGMT] Updated model: %s", modelID)
-
- // Get fresh model state from database
- updatedModel, err := s.GetModelByID(modelID)
- if err != nil {
- return nil, err
- }
- log.Printf("[DEBUG] Retrieved is_visible after update: %v", updatedModel.IsVisible)
- return updatedModel, nil
-}
-
-// DeleteModel deletes a model from both database and providers.json
-func (s *ModelManagementService) DeleteModel(ctx context.Context, modelID string) error {
- log.Printf("🗑️ [MODEL-MGMT] Deleting model: %s", modelID)
-
- // Step 1: Begin transaction
- tx, err := s.db.Begin()
- if err != nil {
- return fmt.Errorf("failed to begin transaction: %w", err)
- }
- defer tx.Rollback()
-
- // Step 2: Delete from database (cascades to model_capabilities and model_aliases)
- result, err := tx.Exec("DELETE FROM models WHERE id = ?", modelID)
- if err != nil {
- return fmt.Errorf("failed to delete model: %w", err)
- }
-
- rowsAffected, _ := result.RowsAffected()
- if rowsAffected == 0 {
- return fmt.Errorf("model not found: %s", modelID)
- }
-
- // Step 3: Commit transaction
- if err := tx.Commit(); err != nil {
- return fmt.Errorf("failed to commit transaction: %w", err)
- }
-
- log.Printf("✅ [MODEL-MGMT] Deleted model: %s", modelID)
- return nil
-}
-
-// reloadConfigServiceCache reloads the config service cache from database
-func (s *ModelManagementService) reloadConfigServiceCache() error {
- log.Printf("🔄 [MODEL-MGMT] Reloading config service cache from database...")
-
- configService := GetConfigService()
-
- // Get all providers and their aliases from database
- rows, err := s.db.Query(`
- SELECT DISTINCT provider_id FROM model_aliases
- `)
- if err != nil {
- return fmt.Errorf("failed to query provider IDs: %w", err)
- }
- defer rows.Close()
-
- var providerIDs []int
- for rows.Next() {
- var providerID int
- if err := rows.Scan(&providerID); err != nil {
- return fmt.Errorf("failed to scan provider ID: %w", err)
- }
- providerIDs = append(providerIDs, providerID)
- }
-
- // Reload aliases for each provider
- for _, providerID := range providerIDs {
- aliases, err := s.getModelAliasesForProvider(providerID)
- if err != nil {
- log.Printf("⚠️ [MODEL-MGMT] Failed to load aliases for provider %d: %v", providerID, err)
- continue
- }
-
- // Update config service cache
- configService.SetModelAliases(providerID, aliases)
- log.Printf("✅ [MODEL-MGMT] Reloaded %d aliases for provider %d", len(aliases), providerID)
- }
-
- log.Printf("✅ [MODEL-MGMT] Config service cache reloaded successfully")
- return nil
-}
-
-// getModelAliasesForProvider retrieves all model aliases for a provider from model_aliases table
-func (s *ModelManagementService) getModelAliasesForProvider(providerID int) (map[string]models.ModelAlias, error) {
- rows, err := s.db.Query(`
- SELECT alias_name, model_id, display_name, description, supports_vision,
- agents_enabled, smart_tool_router, free_tier, structured_output_support,
- structured_output_compliance, structured_output_warning, structured_output_speed_ms,
- structured_output_badge, memory_extractor, memory_selector
- FROM model_aliases
- WHERE provider_id = ?
- `, providerID)
- if err != nil {
- return nil, err
- }
- defer rows.Close()
-
- aliases := make(map[string]models.ModelAlias)
-
- for rows.Next() {
- var aliasName, modelID, displayName string
- var description, structuredOutputSupport, structuredOutputWarning, structuredOutputBadge sql.NullString
- var supportsVision, agentsEnabled, smartToolRouter, freeTier, memoryExtractor, memorySelector sql.NullBool
- var structuredOutputCompliance, structuredOutputSpeedMs sql.NullInt64
-
- err := rows.Scan(&aliasName, &modelID, &displayName, &description, &supportsVision,
- &agentsEnabled, &smartToolRouter, &freeTier, &structuredOutputSupport,
- &structuredOutputCompliance, &structuredOutputWarning, &structuredOutputSpeedMs,
- &structuredOutputBadge, &memoryExtractor, &memorySelector)
- if err != nil {
- return nil, err
- }
-
- alias := models.ModelAlias{
- ActualModel: modelID,
- DisplayName: displayName,
- }
-
- if description.Valid {
- alias.Description = description.String
- }
- if supportsVision.Valid {
- vision := supportsVision.Bool
- alias.SupportsVision = &vision
- }
- if agentsEnabled.Valid {
- agents := agentsEnabled.Bool
- alias.Agents = &agents
- }
- if smartToolRouter.Valid {
- router := smartToolRouter.Bool
- alias.SmartToolRouter = &router
- }
- if freeTier.Valid {
- free := freeTier.Bool
- alias.FreeTier = &free
- }
- if structuredOutputSupport.Valid {
- alias.StructuredOutputSupport = structuredOutputSupport.String
- }
- if structuredOutputCompliance.Valid {
- compliance := int(structuredOutputCompliance.Int64)
- alias.StructuredOutputCompliance = &compliance
- }
- if structuredOutputWarning.Valid {
- alias.StructuredOutputWarning = structuredOutputWarning.String
- }
- if structuredOutputSpeedMs.Valid {
- speed := int(structuredOutputSpeedMs.Int64)
- alias.StructuredOutputSpeedMs = &speed
- }
- if structuredOutputBadge.Valid {
- alias.StructuredOutputBadge = structuredOutputBadge.String
- }
- if memoryExtractor.Valid {
- extractor := memoryExtractor.Bool
- alias.MemoryExtractor = &extractor
- }
- if memorySelector.Valid {
- selector := memorySelector.Bool
- alias.MemorySelector = &selector
- }
-
- aliases[aliasName] = alias
- }
-
- return aliases, nil
-}
-
-// ================== MODEL FETCHING ==================
-
-// FetchModelsFromProvider fetches models from a provider's API and stores them
-func (s *ModelManagementService) FetchModelsFromProvider(ctx context.Context, providerID int) (int, error) {
- log.Printf("🔄 [MODEL-MGMT] Fetching models from provider %d", providerID)
-
- // Get provider details
- provider, err := s.getProviderByID(providerID)
- if err != nil {
- return 0, fmt.Errorf("failed to get provider: %w", err)
- }
-
- // Create HTTP request to provider's /v1/models endpoint
- req, err := http.NewRequest("GET", provider.BaseURL+"/models", nil)
- if err != nil {
- return 0, fmt.Errorf("failed to create request: %w", err)
- }
-
- req.Header.Set("Authorization", "Bearer "+provider.APIKey)
- req.Header.Set("Content-Type", "application/json")
-
- client := &http.Client{Timeout: 30 * time.Second}
- resp, err := client.Do(req)
- if err != nil {
- return 0, fmt.Errorf("failed to fetch models: %w", err)
- }
- defer resp.Body.Close()
-
- if resp.StatusCode != http.StatusOK {
- body, _ := io.ReadAll(resp.Body)
- return 0, fmt.Errorf("API error (status %d): %s", resp.StatusCode, string(body))
- }
-
- // Parse response
- body, err := io.ReadAll(resp.Body)
- if err != nil {
- return 0, fmt.Errorf("failed to read response: %w", err)
- }
-
- var modelsResp models.OpenAIModelsResponse
- if err := json.Unmarshal(body, &modelsResp); err != nil {
- return 0, fmt.Errorf("failed to parse models response: %w", err)
- }
-
- log.Printf("✅ [MODEL-MGMT] Fetched %d models from provider %d", len(modelsResp.Data), providerID)
-
- // Store models in database (all hidden by default - admin must manually toggle visibility)
- count := 0
- for _, modelData := range modelsResp.Data {
- _, err := s.db.Exec(`
- INSERT INTO models (id, provider_id, name, display_name, is_visible, fetched_at)
- VALUES (?, ?, ?, ?, 0, ?)
- ON DUPLICATE KEY UPDATE
- name = VALUES(name),
- display_name = VALUES(display_name),
- fetched_at = VALUES(fetched_at)
- `, modelData.ID, providerID, modelData.ID, modelData.ID, time.Now())
-
- if err != nil {
- log.Printf("⚠️ [MODEL-MGMT] Failed to store model %s: %v", modelData.ID, err)
- } else {
- count++
- }
- }
-
- log.Printf("✅ [MODEL-MGMT] Stored %d models for provider %d", count, providerID)
- return count, nil
-}
-
-// ================== MODEL TESTING ==================
-
-// TestModelConnection performs a basic connection test
-func (s *ModelManagementService) TestModelConnection(ctx context.Context, modelID string) (*ConnectionTestResult, error) {
- log.Printf("🔌 [MODEL-MGMT] Testing connection for model: %s", modelID)
-
- model, err := s.GetModelByID(modelID)
- if err != nil {
- return nil, err
- }
-
- provider, err := s.getProviderByID(model.ProviderID)
- if err != nil {
- return nil, err
- }
-
- start := time.Now()
-
- // Send test prompt
- reqBody := map[string]interface{}{
- "model": modelID,
- "messages": []map[string]string{
- {"role": "user", "content": "Hello! Respond with OK"},
- },
- "max_tokens": 10,
- }
-
- jsonData, _ := json.Marshal(reqBody)
- req, err := http.NewRequest("POST", provider.BaseURL+"/chat/completions", bytes.NewBuffer(jsonData))
- if err != nil {
- return nil, err
- }
-
- req.Header.Set("Authorization", "Bearer "+provider.APIKey)
- req.Header.Set("Content-Type", "application/json")
-
- client := &http.Client{Timeout: 30 * time.Second}
- resp, err := client.Do(req)
- if err != nil {
- return &ConnectionTestResult{
- ModelID: modelID,
- Passed: false,
- LatencyMs: int(time.Since(start).Milliseconds()),
- Error: err.Error(),
- }, nil
- }
- defer resp.Body.Close()
-
- latency := int(time.Since(start).Milliseconds())
-
- if resp.StatusCode != http.StatusOK {
- body, _ := io.ReadAll(resp.Body)
- return &ConnectionTestResult{
- ModelID: modelID,
- Passed: false,
- LatencyMs: latency,
- Error: fmt.Sprintf("API error (status %d): %s", resp.StatusCode, string(body)),
- }, nil
- }
-
- // Update database
- _, err = s.db.Exec(`
- REPLACE INTO model_capabilities (model_id, provider_id, connection_test_passed, last_tested)
- VALUES (?, ?, 1, ?)
- `, modelID, model.ProviderID, time.Now())
-
- if err != nil {
- log.Printf("⚠️ [MODEL-MGMT] Failed to update test result: %v", err)
- }
-
- log.Printf("✅ [MODEL-MGMT] Connection test passed for %s (latency: %dms)", modelID, latency)
- return &ConnectionTestResult{
- ModelID: modelID,
- Passed: true,
- LatencyMs: latency,
- }, nil
-}
-
-// RunBenchmark runs a comprehensive benchmark suite on a model
-func (s *ModelManagementService) RunBenchmark(ctx context.Context, modelID string) (*BenchmarkResults, error) {
- log.Printf("📊 [MODEL-MGMT] Starting benchmark suite for model: %s", modelID)
-
- model, err := s.GetModelByID(modelID)
- if err != nil {
- log.Printf("❌ [MODEL-MGMT] Failed to get model %s: %v", modelID, err)
- return nil, fmt.Errorf("model not found: %w", err)
- }
-
- provider, err := s.getProviderByID(model.ProviderID)
- if err != nil {
- log.Printf("❌ [MODEL-MGMT] Failed to get provider %d: %v", model.ProviderID, err)
- return nil, fmt.Errorf("provider not found: %w", err)
- }
-
- log.Printf(" Provider: %s (%s)", provider.Name, provider.BaseURL)
-
- results := &BenchmarkResults{
- LastTested: time.Now().Format(time.RFC3339),
- }
-
- // 1. Run connection test
- log.Printf(" [1/3] Running connection test...")
- connResult, err := s.TestModelConnection(ctx, modelID)
- if err == nil {
- results.ConnectionTest = connResult
- log.Printf(" ✓ Connection test complete")
- } else {
- log.Printf(" ✗ Connection test failed: %v", err)
- }
-
- // 2. Run structured output test
- log.Printf(" [2/3] Running structured output test (5 prompts)...")
- structuredResult, err := s.testStructuredOutput(ctx, modelID, provider)
- if err == nil {
- results.StructuredOutput = structuredResult
- log.Printf(" ✓ Structured output test complete")
- } else {
- log.Printf(" ✗ Structured output test failed: %v", err)
- }
-
- // 3. Run performance test
- log.Printf(" [3/3] Running performance test (3 prompts)...")
- perfResult, err := s.testPerformance(ctx, modelID, provider)
- if err == nil {
- results.Performance = perfResult
- log.Printf(" ✓ Performance test complete")
- } else {
- log.Printf(" ✗ Performance test failed: %v", err)
- }
-
- // 4. Update database with benchmark results
- if results.StructuredOutput != nil {
- _, err = s.db.Exec(`
- UPDATE model_capabilities
- SET structured_output_compliance = ?,
- structured_output_speed_ms = ?,
- benchmark_date = ?
- WHERE model_id = ? AND provider_id = ?
- `, results.StructuredOutput.CompliancePercentage,
- results.StructuredOutput.AverageSpeedMs,
- time.Now(),
- modelID,
- model.ProviderID)
-
- if err != nil {
- log.Printf("⚠️ [MODEL-MGMT] Failed to update benchmark results in DB: %v", err)
- }
- }
-
- log.Printf("✅ [MODEL-MGMT] Benchmark suite completed for %s", modelID)
- return results, nil
-}
-
-// testStructuredOutput tests JSON schema compliance
-func (s *ModelManagementService) testStructuredOutput(ctx context.Context, modelID string, provider *models.Provider) (*StructuredOutputBenchmark, error) {
- log.Printf("🧪 [MODEL-MGMT] Testing structured output for model: %s at %s", modelID, provider.BaseURL)
-
- testPrompts := []string{
- `Generate a JSON object with fields: name (string), age (number), active (boolean)`,
- `Create a JSON array with 3 objects, each with id and title fields`,
- `Output JSON with nested structure: user { profile { name, email } }`,
- `Return JSON with array field "tags" containing 5 strings`,
- `Generate JSON matching: { count: number, items: string[] }`,
- }
-
- passedTests := 0
- totalLatency := 0
- totalTests := len(testPrompts)
- failureReasons := []string{}
-
- client := &http.Client{Timeout: 60 * time.Second}
-
- for i, prompt := range testPrompts {
- start := time.Now()
-
- reqBody := map[string]interface{}{
- "model": modelID,
- "messages": []map[string]string{
- {"role": "user", "content": prompt},
- },
- "max_tokens": 200,
- }
-
- jsonData, _ := json.Marshal(reqBody)
- req, err := http.NewRequest("POST", provider.BaseURL+"/chat/completions", bytes.NewBuffer(jsonData))
- if err != nil {
- failureReasons = append(failureReasons, fmt.Sprintf("Test %d: request creation failed - %v", i+1, err))
- continue
- }
-
- req.Header.Set("Authorization", "Bearer "+provider.APIKey)
- req.Header.Set("Content-Type", "application/json")
-
- resp, err := client.Do(req)
- if err != nil {
- failureReasons = append(failureReasons, fmt.Sprintf("Test %d: HTTP request failed - %v", i+1, err))
- continue
- }
-
- latency := int(time.Since(start).Milliseconds())
- totalLatency += latency
-
- if resp.StatusCode == http.StatusOK {
- // Check if response is valid JSON
- body, _ := io.ReadAll(resp.Body)
- var result map[string]interface{}
- if json.Unmarshal(body, &result) == nil {
- passedTests++
- log.Printf(" ✓ Test %d passed (latency: %dms)", i+1, latency)
- } else {
- failureReasons = append(failureReasons, fmt.Sprintf("Test %d: invalid JSON response", i+1))
- }
- } else {
- body, _ := io.ReadAll(resp.Body)
- failureReasons = append(failureReasons, fmt.Sprintf("Test %d: HTTP %d - %s", i+1, resp.StatusCode, string(body)))
- }
- resp.Body.Close()
- }
-
- if len(failureReasons) > 0 {
- log.Printf("⚠️ [MODEL-MGMT] Structured output test failures: %v", failureReasons)
- }
-
- compliancePercentage := (passedTests * 100) / totalTests
- avgSpeedMs := 0
- if totalLatency > 0 && passedTests > 0 {
- avgSpeedMs = totalLatency / passedTests
- }
-
- qualityLevel := "poor"
- if compliancePercentage >= 90 {
- qualityLevel = "excellent"
- } else if compliancePercentage >= 75 {
- qualityLevel = "good"
- } else if compliancePercentage >= 50 {
- qualityLevel = "fair"
- }
-
- log.Printf(" 📊 Results: %d/%d passed (%d%%), avg speed: %dms, quality: %s",
- passedTests, totalTests, compliancePercentage, avgSpeedMs, qualityLevel)
-
- return &StructuredOutputBenchmark{
- CompliancePercentage: compliancePercentage,
- AverageSpeedMs: avgSpeedMs,
- QualityLevel: qualityLevel,
- TestsPassed: passedTests,
- TestsFailed: totalTests - passedTests,
- }, nil
-}
-
-// testPerformance tests model performance metrics
-func (s *ModelManagementService) testPerformance(ctx context.Context, modelID string, provider *models.Provider) (*PerformanceBenchmark, error) {
- log.Printf("⚡ [MODEL-MGMT] Testing performance for model: %s", modelID)
-
- testPrompt := "Write a detailed explanation of machine learning in 200 words."
- numTests := 3
- totalLatency := 0
- totalTokens := 0
-
- client := &http.Client{Timeout: 60 * time.Second}
-
- for i := 0; i < numTests; i++ {
- start := time.Now()
-
- reqBody := map[string]interface{}{
- "model": modelID,
- "messages": []map[string]string{
- {"role": "user", "content": testPrompt},
- },
- "max_tokens": 300,
- }
-
- jsonData, _ := json.Marshal(reqBody)
- req, err := http.NewRequest("POST", provider.BaseURL+"/chat/completions", bytes.NewBuffer(jsonData))
- if err != nil {
- continue
- }
-
- req.Header.Set("Authorization", "Bearer "+provider.APIKey)
- req.Header.Set("Content-Type", "application/json")
-
- resp, err := client.Do(req)
- if err != nil {
- continue
- }
-
- latency := int(time.Since(start).Milliseconds())
- totalLatency += latency
-
- if resp.StatusCode == http.StatusOK {
- body, _ := io.ReadAll(resp.Body)
- var result map[string]interface{}
- if json.Unmarshal(body, &result) == nil {
- if usage, ok := result["usage"].(map[string]interface{}); ok {
- if completionTokens, ok := usage["completion_tokens"].(float64); ok {
- totalTokens += int(completionTokens)
- }
- }
- }
- }
- resp.Body.Close()
- }
-
- avgLatencyMs := totalLatency / numTests
- avgTokens := float64(totalTokens) / float64(numTests)
- tokensPerSecond := (avgTokens / float64(avgLatencyMs)) * 1000
-
- return &PerformanceBenchmark{
- TokensPerSecond: tokensPerSecond,
- AvgLatencyMs: avgLatencyMs,
- TestedAt: time.Now().Format(time.RFC3339),
- }, nil
-}
-
-// ================== ALIAS MANAGEMENT ==================
-
-// CreateAlias creates a new model alias
-func (s *ModelManagementService) CreateAlias(ctx context.Context, req *CreateAliasRequest) error {
- log.Printf("📝 [MODEL-MGMT] Creating alias: %s -> %s (provider %d)", req.AliasName, req.ModelID, req.ProviderID)
-
- // Begin transaction
- tx, err := s.db.Begin()
- if err != nil {
- return fmt.Errorf("failed to begin transaction: %w", err)
- }
- defer tx.Rollback()
-
- // Convert empty string to NULL for ENUM fields
- var structuredOutputSupport interface{}
- if req.StructuredOutputSupport == "" {
- structuredOutputSupport = nil
- } else {
- structuredOutputSupport = req.StructuredOutputSupport
- }
-
- // Convert empty strings to NULL for optional text fields
- var structuredOutputWarning interface{}
- if req.StructuredOutputWarning == "" {
- structuredOutputWarning = nil
- } else {
- structuredOutputWarning = req.StructuredOutputWarning
- }
-
- var structuredOutputBadge interface{}
- if req.StructuredOutputBadge == "" {
- structuredOutputBadge = nil
- } else {
- structuredOutputBadge = req.StructuredOutputBadge
- }
-
- var description interface{}
- if req.Description == "" {
- description = nil
- } else {
- description = req.Description
- }
-
- // Insert alias
- _, err = tx.Exec(`
- INSERT INTO model_aliases (alias_name, model_id, provider_id, display_name, description,
- supports_vision, agents_enabled, smart_tool_router, free_tier,
- structured_output_support, structured_output_compliance, structured_output_warning,
- structured_output_speed_ms, structured_output_badge, memory_extractor, memory_selector)
- VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
- `, req.AliasName, req.ModelID, req.ProviderID, req.DisplayName, description,
- req.SupportsVision, req.AgentsEnabled, req.SmartToolRouter, req.FreeTier,
- structuredOutputSupport, req.StructuredOutputCompliance, structuredOutputWarning,
- req.StructuredOutputSpeedMs, structuredOutputBadge, req.MemoryExtractor, req.MemorySelector)
-
- if err != nil {
- return fmt.Errorf("failed to insert alias: %w", err)
- }
-
- if err := tx.Commit(); err != nil {
- return fmt.Errorf("failed to commit transaction: %w", err)
- }
-
- // Reload config service cache with updated aliases from database
- if err := s.reloadConfigServiceCache(); err != nil {
- log.Printf("⚠️ [MODEL-MGMT] Failed to reload config cache: %v", err)
- }
-
- log.Printf("✅ [MODEL-MGMT] Created alias: %s", req.AliasName)
- return nil
-}
-
-// DeleteAlias deletes a model alias
-func (s *ModelManagementService) DeleteAlias(ctx context.Context, aliasName string, providerID int) error {
- log.Printf("🗑️ [MODEL-MGMT] Deleting alias: %s (provider %d)", aliasName, providerID)
-
- tx, err := s.db.Begin()
- if err != nil {
- return fmt.Errorf("failed to begin transaction: %w", err)
- }
- defer tx.Rollback()
-
- result, err := tx.Exec("DELETE FROM model_aliases WHERE alias_name = ? AND provider_id = ?", aliasName, providerID)
- if err != nil {
- return fmt.Errorf("failed to delete alias: %w", err)
- }
-
- rowsAffected, _ := result.RowsAffected()
- if rowsAffected == 0 {
- return fmt.Errorf("alias not found: %s", aliasName)
- }
-
- if err := tx.Commit(); err != nil {
- return fmt.Errorf("failed to commit transaction: %w", err)
- }
-
- // Reload config service cache with updated aliases from database
- if err := s.reloadConfigServiceCache(); err != nil {
- log.Printf("⚠️ [MODEL-MGMT] Failed to reload config cache: %v", err)
- }
-
- log.Printf("✅ [MODEL-MGMT] Deleted alias: %s", aliasName)
- return nil
-}
-
-// GetAliases retrieves all aliases for a model
-func (s *ModelManagementService) GetAliases(ctx context.Context, modelID string) ([]models.ModelAliasView, error) {
- log.Printf("🔍 [MODEL-MGMT] Fetching aliases for model: %s", modelID)
-
- rows, err := s.db.Query(`
- SELECT id, alias_name, model_id, provider_id, display_name, description,
- supports_vision, agents_enabled, smart_tool_router, free_tier,
- structured_output_support, structured_output_compliance, structured_output_warning,
- structured_output_speed_ms, structured_output_badge, memory_extractor, memory_selector,
- created_at, updated_at
- FROM model_aliases
- WHERE model_id = ?
- ORDER BY created_at DESC
- `, modelID)
-
- if err != nil {
- return nil, fmt.Errorf("failed to query aliases: %w", err)
- }
- defer rows.Close()
-
- var aliases []models.ModelAliasView
- for rows.Next() {
- var alias models.ModelAliasView
- var description, structuredOutputSupport, structuredOutputWarning, structuredOutputBadge sql.NullString
- var structuredOutputCompliance, structuredOutputSpeedMs sql.NullInt64
- var supportsVision, agentsEnabled, smartToolRouter, freeTier, memoryExtractor, memorySelector sql.NullBool
-
- err := rows.Scan(
- &alias.ID, &alias.AliasName, &alias.ModelID, &alias.ProviderID, &alias.DisplayName, &description,
- &supportsVision, &agentsEnabled, &smartToolRouter, &freeTier,
- &structuredOutputSupport, &structuredOutputCompliance, &structuredOutputWarning,
- &structuredOutputSpeedMs, &structuredOutputBadge, &memoryExtractor, &memorySelector,
- &alias.CreatedAt, &alias.UpdatedAt,
- )
-
- if err != nil {
- return nil, fmt.Errorf("failed to scan alias: %w", err)
- }
-
- // Handle nullable fields
- if description.Valid {
- alias.Description = &description.String
- }
- if supportsVision.Valid {
- alias.SupportsVision = &supportsVision.Bool
- }
- if agentsEnabled.Valid {
- alias.AgentsEnabled = &agentsEnabled.Bool
- }
- if smartToolRouter.Valid {
- alias.SmartToolRouter = &smartToolRouter.Bool
- }
- if freeTier.Valid {
- alias.FreeTier = &freeTier.Bool
- }
- if structuredOutputSupport.Valid {
- alias.StructuredOutputSupport = &structuredOutputSupport.String
- }
- if structuredOutputCompliance.Valid {
- compliance := int(structuredOutputCompliance.Int64)
- alias.StructuredOutputCompliance = &compliance
- }
- if structuredOutputWarning.Valid {
- alias.StructuredOutputWarning = &structuredOutputWarning.String
- }
- if structuredOutputSpeedMs.Valid {
- speed := int(structuredOutputSpeedMs.Int64)
- alias.StructuredOutputSpeedMs = &speed
- }
- if structuredOutputBadge.Valid {
- alias.StructuredOutputBadge = &structuredOutputBadge.String
- }
- if memoryExtractor.Valid {
- alias.MemoryExtractor = &memoryExtractor.Bool
- }
- if memorySelector.Valid {
- alias.MemorySelector = &memorySelector.Bool
- }
-
- aliases = append(aliases, alias)
- }
-
- if err := rows.Err(); err != nil {
- return nil, fmt.Errorf("error iterating aliases: %w", err)
- }
-
- log.Printf("✅ [MODEL-MGMT] Found %d aliases for model %s", len(aliases), modelID)
- return aliases, nil
-}
-
-// ImportAliasesFromJSON imports all aliases from providers.json into the database
-func (s *ModelManagementService) ImportAliasesFromJSON(ctx context.Context) error {
- log.Printf("📥 [MODEL-MGMT] Starting import of aliases from providers.json to database...")
-
- // Read providers.json
- data, err := os.ReadFile(s.providersFile)
- if err != nil {
- return fmt.Errorf("failed to read providers.json: %w", err)
- }
-
- var cfg models.ProvidersConfig
- if err := json.Unmarshal(data, &cfg); err != nil {
- return fmt.Errorf("failed to parse providers.json: %w", err)
- }
-
- totalImported := 0
- totalSkipped := 0
-
- // Iterate through all providers
- for _, provider := range cfg.Providers {
- // Get provider ID from database
- var providerID int
- err := s.db.QueryRow(`SELECT id FROM providers WHERE name = ?`, provider.Name).Scan(&providerID)
- if err != nil {
- log.Printf("⚠️ [MODEL-MGMT] Provider %s not found in database, skipping aliases", provider.Name)
- continue
- }
-
- // Iterate through all model_aliases for this provider
- for aliasName, aliasConfig := range provider.ModelAliases {
- // Check if alias already exists
- var existingID int
- err := s.db.QueryRow(`SELECT id FROM model_aliases WHERE alias_name = ? AND provider_id = ?`,
- aliasName, providerID).Scan(&existingID)
-
- if err == nil {
- // Alias already exists, skip
- totalSkipped++
- continue
- }
-
- // Extract values from aliasConfig
- modelID := aliasConfig.ActualModel
- displayName := aliasConfig.DisplayName
- description := aliasConfig.Description
- supportsVision := aliasConfig.SupportsVision
- agentsEnabled := aliasConfig.Agents
- smartToolRouter := aliasConfig.SmartToolRouter
- freeTier := aliasConfig.FreeTier
- structuredOutputSupport := aliasConfig.StructuredOutputSupport
- structuredOutputCompliance := aliasConfig.StructuredOutputCompliance
- structuredOutputWarning := aliasConfig.StructuredOutputWarning
- structuredOutputSpeedMs := aliasConfig.StructuredOutputSpeedMs
- structuredOutputBadge := aliasConfig.StructuredOutputBadge
- memoryExtractor := aliasConfig.MemoryExtractor
- memorySelector := aliasConfig.MemorySelector
-
- // Insert alias into database
- _, err = s.db.Exec(`
- INSERT INTO model_aliases (alias_name, model_id, provider_id, display_name, description,
- supports_vision, agents_enabled, smart_tool_router, free_tier,
- structured_output_support, structured_output_compliance, structured_output_warning,
- structured_output_speed_ms, structured_output_badge, memory_extractor, memory_selector)
- VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
- `, aliasName, modelID, providerID, displayName, description,
- supportsVision, agentsEnabled, smartToolRouter, freeTier,
- structuredOutputSupport, structuredOutputCompliance, structuredOutputWarning,
- structuredOutputSpeedMs, structuredOutputBadge, memoryExtractor, memorySelector)
-
- if err != nil {
- log.Printf("⚠️ [MODEL-MGMT] Failed to import alias %s: %v", aliasName, err)
- continue
- }
-
- totalImported++
- log.Printf(" ✓ Imported alias: %s -> %s (provider: %s)", aliasName, modelID, provider.Name)
- }
- }
-
- log.Printf("✅ [MODEL-MGMT] Import complete: %d aliases imported, %d skipped (already exist)", totalImported, totalSkipped)
- return nil
-}
-
-// ================== HELPER METHODS ==================
-
-// GetModelByID retrieves a model by ID
-func (s *ModelManagementService) GetModelByID(modelID string) (*models.Model, error) {
- var m models.Model
- var displayName, description, systemPrompt, providerFavicon sql.NullString
- var contextLength sql.NullInt64
- var fetchedAt sql.NullTime
-
- err := s.db.QueryRow(`
- SELECT m.id, m.provider_id, p.name as provider_name, p.favicon as provider_favicon,
- m.name, m.display_name, m.description, m.context_length, m.supports_tools,
- m.supports_streaming, m.supports_vision, m.smart_tool_router, m.is_visible, m.system_prompt, m.fetched_at
- FROM models m
- JOIN providers p ON m.provider_id = p.id
- WHERE m.id = ?
- `, modelID).Scan(&m.ID, &m.ProviderID, &m.ProviderName, &providerFavicon,
- &m.Name, &displayName, &description, &contextLength, &m.SupportsTools,
- &m.SupportsStreaming, &m.SupportsVision, &m.SmartToolRouter, &m.IsVisible, &systemPrompt, &fetchedAt)
-
- if err == sql.ErrNoRows {
- return nil, fmt.Errorf("model not found: %s", modelID)
- }
- if err != nil {
- return nil, fmt.Errorf("failed to query model: %w", err)
- }
-
- if displayName.Valid {
- m.DisplayName = displayName.String
- }
- if description.Valid {
- m.Description = description.String
- }
- if contextLength.Valid {
- m.ContextLength = int(contextLength.Int64)
- }
- if systemPrompt.Valid {
- m.SystemPrompt = systemPrompt.String
- }
- if providerFavicon.Valid {
- m.ProviderFavicon = providerFavicon.String
- }
- if fetchedAt.Valid {
- m.FetchedAt = fetchedAt.Time
- }
-
- return &m, nil
-}
-
-// getProviderByID retrieves a provider by ID
-func (s *ModelManagementService) getProviderByID(id int) (*models.Provider, error) {
- var p models.Provider
- var systemPrompt, favicon sql.NullString
- err := s.db.QueryRow(`
- SELECT id, name, base_url, api_key, enabled, audio_only, system_prompt, favicon, created_at, updated_at
- FROM providers
- WHERE id = ?
- `, id).Scan(&p.ID, &p.Name, &p.BaseURL, &p.APIKey, &p.Enabled, &p.AudioOnly, &systemPrompt, &favicon, &p.CreatedAt, &p.UpdatedAt)
-
- if err == sql.ErrNoRows {
- return nil, fmt.Errorf("provider not found")
- }
- if err != nil {
- return nil, fmt.Errorf("failed to query provider: %w", err)
- }
-
- if systemPrompt.Valid {
- p.SystemPrompt = systemPrompt.String
- }
- if favicon.Valid {
- p.Favicon = favicon.String
- }
-
- return &p, nil
-}
-
-// ================== REQUEST/RESPONSE TYPES ==================
-
-// CreateModelRequest represents a request to create a new model
-type CreateModelRequest struct {
- ModelID string
- ProviderID int
- Name string
- DisplayName string
- Description string
- ContextLength int
- SupportsTools bool
- SupportsStreaming bool
- SupportsVision bool
- IsVisible bool
- SystemPrompt string
-}
-
-// UpdateModelRequest represents a request to update a model
-type UpdateModelRequest struct {
- DisplayName *string
- Description *string
- ContextLength *int
- SupportsTools *bool
- SupportsStreaming *bool
- SupportsVision *bool
- IsVisible *bool
- SystemPrompt *string
- SmartToolRouter *bool
- FreeTier *bool
-}
-
-// CreateAliasRequest represents a request to create a model alias
-type CreateAliasRequest struct {
- AliasName string `json:"alias_name"`
- ModelID string `json:"model_id"`
- ProviderID int `json:"provider_id"`
- DisplayName string `json:"display_name"`
- Description string `json:"description"`
- SupportsVision *bool `json:"supports_vision"`
- AgentsEnabled *bool `json:"agents_enabled"`
- SmartToolRouter *bool `json:"smart_tool_router"`
- FreeTier *bool `json:"free_tier"`
- StructuredOutputSupport string `json:"structured_output_support"`
- StructuredOutputCompliance *int `json:"structured_output_compliance"`
- StructuredOutputWarning string `json:"structured_output_warning"`
- StructuredOutputSpeedMs *int `json:"structured_output_speed_ms"`
- StructuredOutputBadge string `json:"structured_output_badge"`
- MemoryExtractor *bool `json:"memory_extractor"`
- MemorySelector *bool `json:"memory_selector"`
-}
-
-// ConnectionTestResult represents the result of a connection test
-type ConnectionTestResult struct {
- ModelID string
- Passed bool
- LatencyMs int
- Error string
-}
-
-// StructuredOutputBenchmark represents structured output test results
-type StructuredOutputBenchmark struct {
- CompliancePercentage int `json:"compliance_percentage"`
- AverageSpeedMs int `json:"average_speed_ms"`
- QualityLevel string `json:"quality_level"`
- TestsPassed int `json:"tests_passed"`
- TestsFailed int `json:"tests_failed"`
-}
-
-// PerformanceBenchmark represents performance test results
-type PerformanceBenchmark struct {
- TokensPerSecond float64 `json:"tokens_per_second"`
- AvgLatencyMs int `json:"avg_latency_ms"`
- TestedAt string `json:"tested_at"`
-}
-
-// BenchmarkResults represents comprehensive benchmark test results
-type BenchmarkResults struct {
- ConnectionTest *ConnectionTestResult `json:"connection_test,omitempty"`
- StructuredOutput *StructuredOutputBenchmark `json:"structured_output,omitempty"`
- Performance *PerformanceBenchmark `json:"performance,omitempty"`
- LastTested string `json:"last_tested,omitempty"`
-}
-
-// ================== GLOBAL TIER MANAGEMENT ==================
-
-// TierAssignment represents a model assigned to a global tier
-type TierAssignment struct {
- ModelID string `json:"model_id"`
- ProviderID int `json:"provider_id"`
- DisplayName string `json:"display_name"`
- Tier string `json:"tier"`
-}
-
-// SetGlobalTier assigns a model to a global tier (tier1-tier5)
-// Only one model can occupy each tier slot
-func (s *ModelManagementService) SetGlobalTier(modelID string, providerID int, tier string) error {
- // Validate tier value
- validTiers := map[string]bool{
- "tier1": true,
- "tier2": true,
- "tier3": true,
- "tier4": true,
- "tier5": true,
- }
-
- if !validTiers[tier] {
- return fmt.Errorf("invalid tier: %s (must be tier1, tier2, tier3, tier4, or tier5)", tier)
- }
-
- // Check if model exists
- var displayName string
- err := s.db.QueryRow("SELECT display_name FROM models WHERE id = ? AND provider_id = ?", modelID, providerID).Scan(&displayName)
- if err == sql.ErrNoRows {
- return fmt.Errorf("model not found: %s", modelID)
- }
- if err != nil {
- return fmt.Errorf("failed to verify model: %w", err)
- }
-
- // Use model ID as alias if display name is empty
- alias := modelID
- if displayName != "" {
- alias = displayName
- }
-
- // Try to insert (will fail if tier already occupied due to unique constraint)
- _, err = s.db.Exec(`
- INSERT INTO recommended_models (provider_id, tier, model_alias)
- VALUES (?, ?, ?)
- ON DUPLICATE KEY UPDATE
- provider_id = VALUES(provider_id),
- model_alias = VALUES(model_alias),
- updated_at = CURRENT_TIMESTAMP
- `, providerID, tier, alias)
-
- if err != nil {
- return fmt.Errorf("failed to set tier: %w", err)
- }
-
- log.Printf("✅ [TIER] Assigned %s to %s", alias, tier)
- return nil
-}
-
-// GetGlobalTiers retrieves all 5 tier assignments
-func (s *ModelManagementService) GetGlobalTiers() (map[string]*TierAssignment, error) {
- rows, err := s.db.Query(`
- SELECT r.tier, r.provider_id, r.model_alias, m.id as model_id, m.display_name
- FROM recommended_models r
- LEFT JOIN models m ON r.provider_id = m.provider_id AND (m.display_name = r.model_alias OR m.id = r.model_alias)
- ORDER BY r.tier
- `)
- if err != nil {
- return nil, fmt.Errorf("failed to query tiers: %w", err)
- }
- defer rows.Close()
-
- tiers := make(map[string]*TierAssignment)
-
- for rows.Next() {
- var tier, modelAlias, modelID string
- var providerID int
- var displayName sql.NullString
-
- err := rows.Scan(&tier, &providerID, &modelAlias, &modelID, &displayName)
- if err != nil {
- log.Printf("⚠️ Failed to scan tier: %v", err)
- continue
- }
-
- assignment := &TierAssignment{
- ModelID: modelID,
- ProviderID: providerID,
- DisplayName: modelAlias,
- Tier: tier,
- }
-
- if displayName.Valid && displayName.String != "" {
- assignment.DisplayName = displayName.String
- }
-
- tiers[tier] = assignment
- }
-
- return tiers, nil
-}
-
-// ClearTier removes a model from a tier
-func (s *ModelManagementService) ClearTier(tier string) error {
- // Validate tier
- validTiers := map[string]bool{
- "tier1": true,
- "tier2": true,
- "tier3": true,
- "tier4": true,
- "tier5": true,
- }
-
- if !validTiers[tier] {
- return fmt.Errorf("invalid tier: %s", tier)
- }
-
- result, err := s.db.Exec("DELETE FROM recommended_models WHERE tier = ?", tier)
- if err != nil {
- return fmt.Errorf("failed to clear tier: %w", err)
- }
-
- rowsAffected, _ := result.RowsAffected()
- if rowsAffected == 0 {
- return fmt.Errorf("tier %s is already empty", tier)
- }
-
- log.Printf("✅ [TIER] Cleared %s", tier)
- return nil
-}
-
-// BulkUpdateAgentsEnabled updates agents_enabled for multiple models
-func (s *ModelManagementService) BulkUpdateAgentsEnabled(modelIDs []string, enabled bool) error {
- if len(modelIDs) == 0 {
- return fmt.Errorf("no model IDs provided")
- }
-
- // Build placeholders for IN clause
- placeholders := make([]string, len(modelIDs))
- args := make([]interface{}, len(modelIDs)+1)
- args[0] = enabled
-
- for i, modelID := range modelIDs {
- placeholders[i] = "?"
- args[i+1] = modelID
- }
-
- query := fmt.Sprintf(`
- UPDATE models
- SET agents_enabled = ?
- WHERE id IN (%s)
- `, strings.Join(placeholders, ","))
-
- result, err := s.db.Exec(query, args...)
- if err != nil {
- return fmt.Errorf("failed to bulk update agents_enabled: %w", err)
- }
-
- rowsAffected, _ := result.RowsAffected()
- log.Printf("✅ [BULK] Updated agents_enabled=%v for %d models", enabled, rowsAffected)
- return nil
-}
-
-// BulkUpdateVisibility bulk shows/hides models from users
-func (s *ModelManagementService) BulkUpdateVisibility(modelIDs []string, visible bool) error {
- if len(modelIDs) == 0 {
- return fmt.Errorf("no model IDs provided")
- }
-
- // Build placeholders for IN clause
- placeholders := make([]string, len(modelIDs))
- args := make([]interface{}, len(modelIDs)+1)
- args[0] = visible
-
- for i, modelID := range modelIDs {
- placeholders[i] = "?"
- args[i+1] = modelID
- }
-
- query := fmt.Sprintf(`
- UPDATE models
- SET is_visible = ?
- WHERE id IN (%s)
- `, strings.Join(placeholders, ","))
-
- result, err := s.db.Exec(query, args...)
- if err != nil {
- return fmt.Errorf("failed to bulk update visibility: %w", err)
- }
-
- rowsAffected, _ := result.RowsAffected()
- log.Printf("✅ [BULK] Updated is_visible=%v for %d models", visible, rowsAffected)
- return nil
-}
-
-// ================== UTILITY FUNCTIONS ==================
-
-// joinStrings joins a slice of strings with a separator
-func joinStrings(parts []string, sep string) string {
- if len(parts) == 0 {
- return ""
- }
- result := parts[0]
- for i := 1; i < len(parts); i++ {
- result += sep + parts[i]
- }
- return result
-}
diff --git a/backend/internal/services/model_service.go b/backend/internal/services/model_service.go
deleted file mode 100644
index 283ca7af..00000000
--- a/backend/internal/services/model_service.go
+++ /dev/null
@@ -1,652 +0,0 @@
-package services
-
-import (
- "claraverse/internal/database"
- "claraverse/internal/models"
- "database/sql"
- "encoding/json"
- "fmt"
- "io"
- "log"
- "net/http"
- "strings"
- "time"
-)
-
-// ModelService handles model operations
-type ModelService struct {
- db *database.DB
-}
-
-// NewModelService creates a new model service
-func NewModelService(db *database.DB) *ModelService {
- return &ModelService{db: db}
-}
-
-// GetDB returns the underlying database connection
-func (s *ModelService) GetDB() *database.DB {
- return s.db
-}
-
-// GetAll returns all models, optionally filtered by visibility
-// Excludes models from audio-only providers (those are for transcription only)
-func (s *ModelService) GetAll(visibleOnly bool) ([]models.Model, error) {
- query := `
- SELECT m.id, m.provider_id, p.name as provider_name, p.favicon as provider_favicon,
- m.name, m.display_name, m.description, m.context_length, m.supports_tools,
- m.supports_streaming, m.supports_vision, m.smart_tool_router, m.is_visible, m.system_prompt, m.fetched_at
- FROM models m
- JOIN providers p ON m.provider_id = p.id
- WHERE (p.audio_only = 0 OR p.audio_only IS NULL)
- `
- if visibleOnly {
- query += " AND m.is_visible = 1"
- }
- query += " ORDER BY p.name, m.name"
-
- rows, err := s.db.Query(query)
- if err != nil {
- return nil, fmt.Errorf("failed to query models: %w", err)
- }
- defer rows.Close()
-
- var modelsList []models.Model
- for rows.Next() {
- var m models.Model
- var displayName, description, systemPrompt, providerFavicon sql.NullString
- var contextLength sql.NullInt64
- var fetchedAt sql.NullTime
-
- err := rows.Scan(&m.ID, &m.ProviderID, &m.ProviderName, &providerFavicon,
- &m.Name, &displayName, &description, &contextLength, &m.SupportsTools,
- &m.SupportsStreaming, &m.SupportsVision, &m.SmartToolRouter, &m.IsVisible, &systemPrompt, &fetchedAt)
- if err != nil {
- return nil, fmt.Errorf("failed to scan model: %w", err)
- }
-
- // Handle nullable fields
- if displayName.Valid {
- m.DisplayName = displayName.String
- }
- if description.Valid {
- m.Description = description.String
- }
- if contextLength.Valid {
- m.ContextLength = int(contextLength.Int64)
- }
- if systemPrompt.Valid {
- m.SystemPrompt = systemPrompt.String
- }
- if providerFavicon.Valid {
- m.ProviderFavicon = providerFavicon.String
- }
- if fetchedAt.Valid {
- m.FetchedAt = fetchedAt.Time
- }
-
- modelsList = append(modelsList, m)
- }
-
- return modelsList, nil
-}
-
-// GetByProvider returns models for a specific provider
-func (s *ModelService) GetByProvider(providerID int, visibleOnly bool) ([]models.Model, error) {
- query := `
- SELECT m.id, m.provider_id, p.name as provider_name, p.favicon as provider_favicon,
- m.name, m.display_name, m.description, m.context_length, m.supports_tools,
- m.supports_streaming, m.supports_vision, m.smart_tool_router, m.is_visible, m.system_prompt, m.fetched_at
- FROM models m
- JOIN providers p ON m.provider_id = p.id
- WHERE m.provider_id = ?
- `
- if visibleOnly {
- query += " AND m.is_visible = 1"
- }
- query += " ORDER BY m.name"
-
- rows, err := s.db.Query(query, providerID)
- if err != nil {
- return nil, fmt.Errorf("failed to query models: %w", err)
- }
- defer rows.Close()
-
- var modelsList []models.Model
- for rows.Next() {
- var m models.Model
- var displayName, description, systemPrompt, providerFavicon sql.NullString
- var contextLength sql.NullInt64
- var fetchedAt sql.NullTime
-
- err := rows.Scan(&m.ID, &m.ProviderID, &m.ProviderName, &providerFavicon,
- &m.Name, &displayName, &description, &contextLength, &m.SupportsTools,
- &m.SupportsStreaming, &m.SupportsVision, &m.SmartToolRouter, &m.IsVisible, &systemPrompt, &fetchedAt)
- if err != nil {
- return nil, fmt.Errorf("failed to scan model: %w", err)
- }
-
- // Handle nullable fields
- if displayName.Valid {
- m.DisplayName = displayName.String
- }
- if description.Valid {
- m.Description = description.String
- }
- if contextLength.Valid {
- m.ContextLength = int(contextLength.Int64)
- }
- if systemPrompt.Valid {
- m.SystemPrompt = systemPrompt.String
- }
- if providerFavicon.Valid {
- m.ProviderFavicon = providerFavicon.String
- }
- if fetchedAt.Valid {
- m.FetchedAt = fetchedAt.Time
- }
-
- modelsList = append(modelsList, m)
- }
-
- return modelsList, nil
-}
-
-// GetToolPredictorModels returns only models that can be used as tool predictors
-// These are models with smart_tool_router = true and is_visible = true
-func (s *ModelService) GetToolPredictorModels() ([]models.Model, error) {
- query := `
- SELECT m.id, m.provider_id, p.name as provider_name, p.favicon as provider_favicon,
- m.name, m.display_name, m.description, m.context_length, m.supports_tools,
- m.supports_streaming, m.supports_vision, m.smart_tool_router, m.is_visible, m.system_prompt, m.fetched_at
- FROM models m
- JOIN providers p ON m.provider_id = p.id
- WHERE m.smart_tool_router = 1
- AND m.is_visible = 1
- AND (p.audio_only = 0 OR p.audio_only IS NULL)
- ORDER BY p.name, m.name
- `
-
- rows, err := s.db.Query(query)
- if err != nil {
- return nil, fmt.Errorf("failed to query tool predictor models: %w", err)
- }
- defer rows.Close()
-
- var modelsList []models.Model
- for rows.Next() {
- var m models.Model
- var displayName, description, systemPrompt, providerFavicon sql.NullString
- var contextLength sql.NullInt64
- var fetchedAt sql.NullTime
-
- err := rows.Scan(&m.ID, &m.ProviderID, &m.ProviderName, &providerFavicon,
- &m.Name, &displayName, &description, &contextLength, &m.SupportsTools,
- &m.SupportsStreaming, &m.SupportsVision, &m.SmartToolRouter, &m.IsVisible, &systemPrompt, &fetchedAt)
- if err != nil {
- return nil, fmt.Errorf("failed to scan tool predictor model: %w", err)
- }
-
- // Handle nullable fields
- if displayName.Valid {
- m.DisplayName = displayName.String
- }
- if description.Valid {
- m.Description = description.String
- }
- if providerFavicon.Valid {
- m.ProviderFavicon = providerFavicon.String
- }
- if contextLength.Valid {
- m.ContextLength = int(contextLength.Int64)
- }
- if systemPrompt.Valid {
- m.SystemPrompt = systemPrompt.String
- }
- if fetchedAt.Valid {
- m.FetchedAt = fetchedAt.Time
- }
-
- modelsList = append(modelsList, m)
- }
-
- return modelsList, nil
-}
-
-// FetchFromProvider fetches models from a provider's API
-func (s *ModelService) FetchFromProvider(provider *models.Provider) error {
- log.Printf("🔄 Fetching models from provider: %s", provider.Name)
-
- // Create HTTP request to provider's /v1/models endpoint
- req, err := http.NewRequest("GET", provider.BaseURL+"/models", nil)
- if err != nil {
- return fmt.Errorf("failed to create request: %w", err)
- }
-
- req.Header.Set("Authorization", "Bearer "+provider.APIKey)
- req.Header.Set("Content-Type", "application/json")
-
- client := &http.Client{Timeout: 30 * time.Second}
- resp, err := client.Do(req)
- if err != nil {
- return fmt.Errorf("failed to fetch models: %w", err)
- }
- defer resp.Body.Close()
-
- if resp.StatusCode != http.StatusOK {
- body, _ := io.ReadAll(resp.Body)
- return fmt.Errorf("API error (status %d): %s", resp.StatusCode, string(body))
- }
-
- // Parse response
- body, err := io.ReadAll(resp.Body)
- if err != nil {
- return fmt.Errorf("failed to read response: %w", err)
- }
-
- var modelsResp models.OpenAIModelsResponse
- if err := json.Unmarshal(body, &modelsResp); err != nil {
- return fmt.Errorf("failed to parse models response: %w", err)
- }
-
- log.Printf("✅ Fetched %d models from %s", len(modelsResp.Data), provider.Name)
-
- // Store models in database
- for _, modelData := range modelsResp.Data {
- _, err := s.db.Exec(`
- INSERT INTO models (id, provider_id, name, display_name, fetched_at)
- VALUES (?, ?, ?, ?, ?)
- ON DUPLICATE KEY UPDATE
- name = VALUES(name),
- display_name = VALUES(display_name),
- fetched_at = VALUES(fetched_at)
- `, modelData.ID, provider.ID, modelData.ID, modelData.ID, time.Now())
-
- if err != nil {
- log.Printf("⚠️ Failed to store model %s: %v", modelData.ID, err)
- }
- }
-
- // Log refresh
- _, err = s.db.Exec(`
- INSERT INTO model_refresh_log (provider_id, models_fetched, refreshed_at)
- VALUES (?, ?, ?)
- `, provider.ID, len(modelsResp.Data), time.Now())
-
- if err != nil {
- log.Printf("⚠️ Failed to log refresh: %v", err)
- }
-
- log.Printf("✅ Refreshed %d models for provider %s", len(modelsResp.Data), provider.Name)
- return nil
-}
-
-// SyncModelAliasMetadata syncs metadata from model aliases to the database
-// This updates existing model records with flags like smart_tool_router, agents, supports_vision
-func (s *ModelService) SyncModelAliasMetadata(providerID int, aliases map[string]models.ModelAlias) error {
- if len(aliases) == 0 {
- return nil
- }
-
- log.Printf("🔄 [MODEL-SYNC] Syncing metadata for %d model aliases (provider %d)", len(aliases), providerID)
-
- for modelID, alias := range aliases {
- // Build update statement for fields that are set in the alias
- updateParts := []string{}
- args := []interface{}{}
-
- // Smart tool router flag
- if alias.SmartToolRouter != nil {
- updateParts = append(updateParts, "smart_tool_router = ?")
- if *alias.SmartToolRouter {
- args = append(args, 1)
- } else {
- args = append(args, 0)
- }
- }
-
- // Free tier flag
- if alias.FreeTier != nil {
- updateParts = append(updateParts, "free_tier = ?")
- if *alias.FreeTier {
- args = append(args, 1)
- } else {
- args = append(args, 0)
- }
- }
-
- // Supports vision flag
- if alias.SupportsVision != nil {
- updateParts = append(updateParts, "supports_vision = ?")
- if *alias.SupportsVision {
- args = append(args, 1)
- } else {
- args = append(args, 0)
- }
- }
-
- // Display name
- if alias.DisplayName != "" {
- updateParts = append(updateParts, "display_name = ?")
- args = append(args, alias.DisplayName)
- }
-
- // Description
- if alias.Description != "" {
- updateParts = append(updateParts, "description = ?")
- args = append(args, alias.Description)
- }
-
- if len(updateParts) == 0 {
- continue // No metadata to sync for this alias
- }
-
- // Add WHERE clause arguments
- args = append(args, modelID, providerID)
-
- query := fmt.Sprintf(`
- UPDATE models
- SET %s
- WHERE id = ? AND provider_id = ?
- `, strings.Join(updateParts, ", "))
-
- result, err := s.db.Exec(query, args...)
- if err != nil {
- log.Printf("⚠️ [MODEL-SYNC] Failed to update model %s: %v", modelID, err)
- continue
- }
-
- rowsAffected, _ := result.RowsAffected()
- if rowsAffected > 0 {
- log.Printf(" ✅ Updated model %s: %s", modelID, strings.Join(updateParts, ", "))
- }
- }
-
- log.Printf("✅ [MODEL-SYNC] Model alias metadata sync completed for provider %d", providerID)
- return nil
-}
-
-// LoadAllAliasesFromDB loads all model aliases from the database
-// Returns map[providerID]map[aliasName]ModelAlias
-func (s *ModelService) LoadAllAliasesFromDB() (map[int]map[string]models.ModelAlias, error) {
- query := `
- SELECT provider_id, alias_name, model_id, display_name, description,
- supports_vision, agents_enabled, smart_tool_router, free_tier,
- structured_output_support, structured_output_compliance,
- structured_output_warning, structured_output_speed_ms,
- structured_output_badge, memory_extractor, memory_selector
- FROM model_aliases
- ORDER BY provider_id, alias_name
- `
-
- rows, err := s.db.Query(query)
- if err != nil {
- return nil, fmt.Errorf("failed to query model aliases: %w", err)
- }
- defer rows.Close()
-
- result := make(map[int]map[string]models.ModelAlias)
-
- for rows.Next() {
- var providerID int
- var aliasName, modelID, displayName string
- var description, structuredOutputSupport, structuredOutputWarning, structuredOutputBadge sql.NullString
- var supportsVision, agentsEnabled, smartToolRouter, freeTier, memoryExtractor, memorySelector sql.NullBool
- var structuredOutputCompliance, structuredOutputSpeedMs sql.NullInt64
-
- err := rows.Scan(&providerID, &aliasName, &modelID, &displayName, &description,
- &supportsVision, &agentsEnabled, &smartToolRouter, &freeTier,
- &structuredOutputSupport, &structuredOutputCompliance,
- &structuredOutputWarning, &structuredOutputSpeedMs,
- &structuredOutputBadge, &memoryExtractor, &memorySelector)
- if err != nil {
- log.Printf("⚠️ Failed to scan alias: %v", err)
- continue
- }
-
- // Initialize provider map if not exists
- if result[providerID] == nil {
- result[providerID] = make(map[string]models.ModelAlias)
- }
-
- // Build ModelAlias struct
- alias := models.ModelAlias{
- ActualModel: modelID,
- DisplayName: displayName,
- }
-
- if description.Valid {
- alias.Description = description.String
- }
- if supportsVision.Valid {
- val := supportsVision.Bool
- alias.SupportsVision = &val
- }
- if agentsEnabled.Valid {
- val := agentsEnabled.Bool
- alias.Agents = &val
- }
- if smartToolRouter.Valid {
- val := smartToolRouter.Bool
- alias.SmartToolRouter = &val
- }
- if freeTier.Valid {
- val := freeTier.Bool
- alias.FreeTier = &val
- }
- if structuredOutputSupport.Valid {
- alias.StructuredOutputSupport = structuredOutputSupport.String
- }
- if structuredOutputCompliance.Valid {
- val := int(structuredOutputCompliance.Int64)
- alias.StructuredOutputCompliance = &val
- }
- if structuredOutputWarning.Valid {
- alias.StructuredOutputWarning = structuredOutputWarning.String
- }
- if structuredOutputSpeedMs.Valid {
- val := int(structuredOutputSpeedMs.Int64)
- alias.StructuredOutputSpeedMs = &val
- }
- if structuredOutputBadge.Valid {
- alias.StructuredOutputBadge = structuredOutputBadge.String
- }
- if memoryExtractor.Valid {
- val := memoryExtractor.Bool
- alias.MemoryExtractor = &val
- }
- if memorySelector.Valid {
- val := memorySelector.Bool
- alias.MemorySelector = &val
- }
-
- result[providerID][aliasName] = alias
- }
-
- log.Printf("✅ Loaded %d provider alias sets from database", len(result))
- return result, nil
-}
-
-// LoadAllRecommendedModelsFromDB loads all recommended models from the database
-// Returns map[providerID]*RecommendedModels
-func (s *ModelService) LoadAllRecommendedModelsFromDB() (map[int]*models.RecommendedModels, error) {
- query := `
- SELECT provider_id, tier, model_alias
- FROM recommended_models
- ORDER BY provider_id, tier
- `
-
- rows, err := s.db.Query(query)
- if err != nil {
- return nil, fmt.Errorf("failed to query recommended models: %w", err)
- }
- defer rows.Close()
-
- result := make(map[int]*models.RecommendedModels)
-
- for rows.Next() {
- var providerID int
- var tier, modelAlias string
-
- err := rows.Scan(&providerID, &tier, &modelAlias)
- if err != nil {
- log.Printf("⚠️ Failed to scan recommended model: %v", err)
- continue
- }
-
- // Initialize provider recommendations if not exists
- if result[providerID] == nil {
- result[providerID] = &models.RecommendedModels{}
- }
-
- // Set the appropriate tier
- switch tier {
- case "top":
- result[providerID].Top = modelAlias
- case "medium":
- result[providerID].Medium = modelAlias
- case "fastest":
- result[providerID].Fastest = modelAlias
- case "new":
- result[providerID].New = modelAlias
- }
- }
-
- log.Printf("✅ Loaded recommended models for %d providers from database", len(result))
- return result, nil
-}
-
-// SaveAliasesToDB saves model aliases to the database
-func (s *ModelService) SaveAliasesToDB(providerID int, aliases map[string]models.ModelAlias) error {
- if len(aliases) == 0 {
- return nil
- }
-
- log.Printf("💾 [MODEL-ALIAS] Saving %d aliases to database for provider %d", len(aliases), providerID)
-
- for aliasName, alias := range aliases {
- _, err := s.db.Exec(`
- INSERT INTO model_aliases (
- alias_name, model_id, provider_id, display_name, description,
- supports_vision, agents_enabled, smart_tool_router, free_tier,
- structured_output_support, structured_output_compliance,
- structured_output_warning, structured_output_speed_ms,
- structured_output_badge, memory_extractor, memory_selector
- ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
- ON DUPLICATE KEY UPDATE
- model_id = VALUES(model_id),
- display_name = VALUES(display_name),
- description = VALUES(description),
- supports_vision = VALUES(supports_vision),
- agents_enabled = VALUES(agents_enabled),
- smart_tool_router = VALUES(smart_tool_router),
- free_tier = VALUES(free_tier),
- structured_output_support = VALUES(structured_output_support),
- structured_output_compliance = VALUES(structured_output_compliance),
- structured_output_warning = VALUES(structured_output_warning),
- structured_output_speed_ms = VALUES(structured_output_speed_ms),
- structured_output_badge = VALUES(structured_output_badge),
- memory_extractor = VALUES(memory_extractor),
- memory_selector = VALUES(memory_selector)
- `,
- aliasName,
- alias.ActualModel,
- providerID,
- alias.DisplayName,
- nullString(alias.Description),
- nullBool(alias.SupportsVision),
- nullBool(alias.Agents),
- nullBool(alias.SmartToolRouter),
- nullBool(alias.FreeTier),
- nullString(alias.StructuredOutputSupport),
- nullInt(alias.StructuredOutputCompliance),
- nullString(alias.StructuredOutputWarning),
- nullInt(alias.StructuredOutputSpeedMs),
- nullString(alias.StructuredOutputBadge),
- nullBool(alias.MemoryExtractor),
- nullBool(alias.MemorySelector),
- )
-
- if err != nil {
- log.Printf("⚠️ [MODEL-ALIAS] Failed to save alias %s: %v", aliasName, err)
- continue
- }
- }
-
- log.Printf("✅ [MODEL-ALIAS] Saved %d aliases to database for provider %d", len(aliases), providerID)
- return nil
-}
-
-// SaveRecommendedModelsToDB saves recommended models to the database
-func (s *ModelService) SaveRecommendedModelsToDB(providerID int, recommended *models.RecommendedModels) error {
- if recommended == nil {
- return nil
- }
-
- log.Printf("💾 [RECOMMENDED] Saving recommended models to database for provider %d", providerID)
-
- // Delete existing recommendations for this provider
- _, err := s.db.Exec("DELETE FROM recommended_models WHERE provider_id = ?", providerID)
- if err != nil {
- return fmt.Errorf("failed to delete old recommendations: %w", err)
- }
-
- // Insert new recommendations
- tiers := map[string]string{
- "top": recommended.Top,
- "medium": recommended.Medium,
- "fastest": recommended.Fastest,
- "new": recommended.New,
- }
-
- for tier, modelAlias := range tiers {
- if modelAlias == "" {
- continue
- }
-
- _, err := s.db.Exec(`
- INSERT INTO recommended_models (provider_id, tier, model_alias)
- VALUES (?, ?, ?)
- `, providerID, tier, modelAlias)
-
- if err != nil {
- log.Printf("⚠️ [RECOMMENDED] Failed to save %s tier: %v", tier, err)
- }
- }
-
- log.Printf("✅ [RECOMMENDED] Saved recommended models for provider %d", providerID)
- return nil
-}
-
-// Helper functions for nullable values
-func nullString(s string) interface{} {
- if s == "" {
- return nil
- }
- return s
-}
-
-func nullInt(i *int) interface{} {
- if i == nil {
- return nil
- }
- return *i
-}
-
-func nullBool(b *bool) interface{} {
- if b == nil {
- return nil
- }
- return *b
-}
-
-// IsFreeTier checks if a model is marked as free tier
-func (s *ModelService) IsFreeTier(modelID string) bool {
- var isFreeTier int
- err := s.db.QueryRow(`
- SELECT COALESCE(free_tier, 0)
- FROM models
- WHERE id = ?
- `, modelID).Scan(&isFreeTier)
-
- return err == nil && isFreeTier == 1
-}
diff --git a/backend/internal/services/model_service_test.go b/backend/internal/services/model_service_test.go
deleted file mode 100644
index 4f81ac91..00000000
--- a/backend/internal/services/model_service_test.go
+++ /dev/null
@@ -1,466 +0,0 @@
-package services
-
-import (
- "claraverse/internal/database"
- "claraverse/internal/models"
- "encoding/json"
- "net/http"
- "net/http/httptest"
- "os"
- "testing"
- "time"
-)
-
-func setupTestDBForModels(t *testing.T) (*database.DB, func()) {
- t.Skip("SQLite tests are deprecated - please use DATABASE_URL with MySQL DSN")
- tmpFile := "test_model_service.db"
- db, err := database.New(tmpFile)
- if err != nil {
- t.Fatalf("Failed to create test database: %v", err)
- }
-
- if err := db.Initialize(); err != nil {
- t.Fatalf("Failed to initialize test database: %v", err)
- }
-
- cleanup := func() {
- db.Close()
- os.Remove(tmpFile)
- }
-
- return db, cleanup
-}
-
-func createTestProvider(t *testing.T, db *database.DB, name string) *models.Provider {
- providerService := NewProviderService(db)
- config := models.ProviderConfig{
- Name: name,
- BaseURL: "https://api.test.com/v1",
- APIKey: "test-key",
- Enabled: true,
- }
-
- provider, err := providerService.Create(config)
- if err != nil {
- t.Fatalf("Failed to create test provider: %v", err)
- }
-
- return provider
-}
-
-func insertTestModel(t *testing.T, db *database.DB, model *models.Model) {
- _, err := db.Exec(`
- INSERT OR REPLACE INTO models
- (id, provider_id, name, display_name, description, context_length,
- supports_tools, supports_streaming, supports_vision, is_visible, fetched_at)
- VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
- `, model.ID, model.ProviderID, model.Name, model.DisplayName, model.Description,
- model.ContextLength, model.SupportsTools, model.SupportsStreaming,
- model.SupportsVision, model.IsVisible, time.Now())
-
- if err != nil {
- t.Fatalf("Failed to insert test model: %v", err)
- }
-}
-
-func TestNewModelService(t *testing.T) {
- db, cleanup := setupTestDBForModels(t)
- defer cleanup()
-
- service := NewModelService(db)
- if service == nil {
- t.Fatal("Expected non-nil model service")
- }
-}
-
-func TestModelService_GetAll(t *testing.T) {
- db, cleanup := setupTestDBForModels(t)
- defer cleanup()
-
- service := NewModelService(db)
- provider := createTestProvider(t, db, "Test Provider")
-
- // Create test models
- testModels := []models.Model{
- {
- ID: "model-1",
- ProviderID: provider.ID,
- Name: "Model 1",
- IsVisible: true,
- SupportsStreaming: true,
- },
- {
- ID: "model-2",
- ProviderID: provider.ID,
- Name: "Model 2",
- IsVisible: false,
- SupportsStreaming: true,
- },
- {
- ID: "model-3",
- ProviderID: provider.ID,
- Name: "Model 3",
- IsVisible: true,
- SupportsTools: true,
- },
- }
-
- for i := range testModels {
- insertTestModel(t, db, &testModels[i])
- }
-
- // Get all models (including hidden)
- allModels, err := service.GetAll(false)
- if err != nil {
- t.Fatalf("Failed to get all models: %v", err)
- }
-
- if len(allModels) != 3 {
- t.Errorf("Expected 3 models, got %d", len(allModels))
- }
-
- // Get only visible models
- visibleModels, err := service.GetAll(true)
- if err != nil {
- t.Fatalf("Failed to get visible models: %v", err)
- }
-
- if len(visibleModels) != 2 {
- t.Errorf("Expected 2 visible models, got %d", len(visibleModels))
- }
-}
-
-func TestModelService_GetByProvider(t *testing.T) {
- db, cleanup := setupTestDBForModels(t)
- defer cleanup()
-
- service := NewModelService(db)
- provider1 := createTestProvider(t, db, "Provider 1")
- provider2 := createTestProvider(t, db, "Provider 2")
-
- // Create models for both providers
- testModels := []models.Model{
- {ID: "model-1", ProviderID: provider1.ID, Name: "Model 1", IsVisible: true},
- {ID: "model-2", ProviderID: provider1.ID, Name: "Model 2", IsVisible: true},
- {ID: "model-3", ProviderID: provider2.ID, Name: "Model 3", IsVisible: true},
- }
-
- for i := range testModels {
- insertTestModel(t, db, &testModels[i])
- }
-
- // Get models for provider 1
- provider1Models, err := service.GetByProvider(provider1.ID, false)
- if err != nil {
- t.Fatalf("Failed to get provider 1 models: %v", err)
- }
-
- if len(provider1Models) != 2 {
- t.Errorf("Expected 2 models for provider 1, got %d", len(provider1Models))
- }
-
- // Get models for provider 2
- provider2Models, err := service.GetByProvider(provider2.ID, false)
- if err != nil {
- t.Fatalf("Failed to get provider 2 models: %v", err)
- }
-
- if len(provider2Models) != 1 {
- t.Errorf("Expected 1 model for provider 2, got %d", len(provider2Models))
- }
-}
-
-func TestModelService_GetByProvider_VisibleOnly(t *testing.T) {
- db, cleanup := setupTestDBForModels(t)
- defer cleanup()
-
- service := NewModelService(db)
- provider := createTestProvider(t, db, "Test Provider")
-
- // Create test models with different visibility
- testModels := []models.Model{
- {ID: "model-1", ProviderID: provider.ID, Name: "Model 1", IsVisible: true},
- {ID: "model-2", ProviderID: provider.ID, Name: "Model 2", IsVisible: false},
- {ID: "model-3", ProviderID: provider.ID, Name: "Model 3", IsVisible: true},
- }
-
- for i := range testModels {
- insertTestModel(t, db, &testModels[i])
- }
-
- // Get only visible models
- visibleModels, err := service.GetByProvider(provider.ID, true)
- if err != nil {
- t.Fatalf("Failed to get visible models: %v", err)
- }
-
- if len(visibleModels) != 2 {
- t.Errorf("Expected 2 visible models, got %d", len(visibleModels))
- }
-}
-
-func TestModelService_FetchFromProvider(t *testing.T) {
- db, cleanup := setupTestDBForModels(t)
- defer cleanup()
-
- // Create mock server
- mockResponse := models.OpenAIModelsResponse{
- Object: "list",
- Data: []struct {
- ID string `json:"id"`
- Object string `json:"object"`
- Created int64 `json:"created"`
- OwnedBy string `json:"owned_by"`
- }{
- {ID: "gpt-4", Object: "model", Created: 1234567890, OwnedBy: "openai"},
- {ID: "gpt-3.5-turbo", Object: "model", Created: 1234567891, OwnedBy: "openai"},
- },
- }
-
- server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
- // Verify request
- if r.URL.Path != "/models" {
- t.Errorf("Expected path /models, got %s", r.URL.Path)
- }
-
- authHeader := r.Header.Get("Authorization")
- if authHeader != "Bearer test-key" {
- t.Errorf("Expected Authorization header 'Bearer test-key', got %s", authHeader)
- }
-
- // Return mock response
- w.Header().Set("Content-Type", "application/json")
- json.NewEncoder(w).Encode(mockResponse)
- }))
- defer server.Close()
-
- service := NewModelService(db)
- providerService := NewProviderService(db)
-
- // Create provider with mock server URL
- config := models.ProviderConfig{
- Name: "Test Provider",
- BaseURL: server.URL,
- APIKey: "test-key",
- Enabled: true,
- }
-
- provider, err := providerService.Create(config)
- if err != nil {
- t.Fatalf("Failed to create provider: %v", err)
- }
-
- // Fetch models from provider
- if err := service.FetchFromProvider(provider); err != nil {
- t.Fatalf("Failed to fetch models from provider: %v", err)
- }
-
- // Verify models were stored
- models, err := service.GetByProvider(provider.ID, false)
- if err != nil {
- t.Fatalf("Failed to get models: %v", err)
- }
-
- if len(models) != 2 {
- t.Errorf("Expected 2 models, got %d", len(models))
- }
-
- // Verify model data
- if models[0].ID != "gpt-3.5-turbo" && models[1].ID != "gpt-3.5-turbo" {
- t.Error("Expected to find gpt-3.5-turbo model")
- }
-
- if models[0].ID != "gpt-4" && models[1].ID != "gpt-4" {
- t.Error("Expected to find gpt-4 model")
- }
-}
-
-func TestModelService_FetchFromProvider_InvalidAuth(t *testing.T) {
- db, cleanup := setupTestDBForModels(t)
- defer cleanup()
-
- // Create mock server that returns 401
- server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
- w.WriteHeader(http.StatusUnauthorized)
- w.Write([]byte(`{"error":"invalid_api_key"}`))
- }))
- defer server.Close()
-
- service := NewModelService(db)
- providerService := NewProviderService(db)
-
- // Create provider with mock server URL
- config := models.ProviderConfig{
- Name: "Test Provider",
- BaseURL: server.URL,
- APIKey: "invalid-key",
- Enabled: true,
- }
-
- provider, err := providerService.Create(config)
- if err != nil {
- t.Fatalf("Failed to create provider: %v", err)
- }
-
- // Fetch should fail
- err = service.FetchFromProvider(provider)
- if err == nil {
- t.Error("Expected error for invalid API key, got nil")
- }
-}
-
-func TestModelService_FetchFromProvider_InvalidJSON(t *testing.T) {
- db, cleanup := setupTestDBForModels(t)
- defer cleanup()
-
- // Create mock server that returns invalid JSON
- server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
- w.Header().Set("Content-Type", "application/json")
- w.Write([]byte(`{invalid json}`))
- }))
- defer server.Close()
-
- service := NewModelService(db)
- providerService := NewProviderService(db)
-
- // Create provider with mock server URL
- config := models.ProviderConfig{
- Name: "Test Provider",
- BaseURL: server.URL,
- APIKey: "test-key",
- Enabled: true,
- }
-
- provider, err := providerService.Create(config)
- if err != nil {
- t.Fatalf("Failed to create provider: %v", err)
- }
-
- // Fetch should fail
- err = service.FetchFromProvider(provider)
- if err == nil {
- t.Error("Expected error for invalid JSON, got nil")
- }
-}
-
-func TestModelService_FetchFromProvider_EmptyResponse(t *testing.T) {
- db, cleanup := setupTestDBForModels(t)
- defer cleanup()
-
- // Create mock server that returns empty models list
- mockResponse := models.OpenAIModelsResponse{
- Object: "list",
- Data: []struct {
- ID string `json:"id"`
- Object string `json:"object"`
- Created int64 `json:"created"`
- OwnedBy string `json:"owned_by"`
- }{},
- }
-
- server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
- w.Header().Set("Content-Type", "application/json")
- json.NewEncoder(w).Encode(mockResponse)
- }))
- defer server.Close()
-
- service := NewModelService(db)
- providerService := NewProviderService(db)
-
- // Create provider with mock server URL
- config := models.ProviderConfig{
- Name: "Test Provider",
- BaseURL: server.URL,
- APIKey: "test-key",
- Enabled: true,
- }
-
- provider, err := providerService.Create(config)
- if err != nil {
- t.Fatalf("Failed to create provider: %v", err)
- }
-
- // Fetch should succeed but return no models
- if err := service.FetchFromProvider(provider); err != nil {
- t.Fatalf("Failed to fetch models: %v", err)
- }
-
- // Verify no models were stored
- models, err := service.GetByProvider(provider.ID, false)
- if err != nil {
- t.Fatalf("Failed to get models: %v", err)
- }
-
- if len(models) != 0 {
- t.Errorf("Expected 0 models, got %d", len(models))
- }
-}
-
-func TestModelService_FetchFromProvider_UpdateExisting(t *testing.T) {
- db, cleanup := setupTestDBForModels(t)
- defer cleanup()
-
- // Create mock server
- mockResponse := models.OpenAIModelsResponse{
- Object: "list",
- Data: []struct {
- ID string `json:"id"`
- Object string `json:"object"`
- Created int64 `json:"created"`
- OwnedBy string `json:"owned_by"`
- }{
- {ID: "gpt-4", Object: "model", Created: 1234567890, OwnedBy: "openai"},
- },
- }
-
- server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
- w.Header().Set("Content-Type", "application/json")
- json.NewEncoder(w).Encode(mockResponse)
- }))
- defer server.Close()
-
- service := NewModelService(db)
- providerService := NewProviderService(db)
-
- // Create provider
- config := models.ProviderConfig{
- Name: "Test Provider",
- BaseURL: server.URL,
- APIKey: "test-key",
- Enabled: true,
- }
-
- provider, err := providerService.Create(config)
- if err != nil {
- t.Fatalf("Failed to create provider: %v", err)
- }
-
- // Insert existing model with different display name
- existingModel := models.Model{
- ID: "gpt-4",
- ProviderID: provider.ID,
- Name: "gpt-4",
- DisplayName: "Old GPT-4",
- IsVisible: true,
- }
- insertTestModel(t, db, &existingModel)
-
- // Fetch models - should update existing
- if err := service.FetchFromProvider(provider); err != nil {
- t.Fatalf("Failed to fetch models: %v", err)
- }
-
- // Verify model was updated (count should still be 1)
- models, err := service.GetByProvider(provider.ID, false)
- if err != nil {
- t.Fatalf("Failed to get models: %v", err)
- }
-
- if len(models) != 1 {
- t.Errorf("Expected 1 model, got %d", len(models))
- }
-
- if models[0].ID != "gpt-4" {
- t.Errorf("Expected model ID 'gpt-4', got %s", models[0].ID)
- }
-}
diff --git a/backend/internal/services/payment_service.go b/backend/internal/services/payment_service.go
deleted file mode 100644
index 9be570bc..00000000
--- a/backend/internal/services/payment_service.go
+++ /dev/null
@@ -1,1564 +0,0 @@
-package services
-
-import (
- "claraverse/internal/database"
- "claraverse/internal/models"
- "context"
- "crypto/hmac"
- "crypto/sha256"
- "encoding/hex"
- "encoding/json"
- "fmt"
- "log"
- "net/http"
- "os"
- "strings"
- "time"
-
- "github.com/dodopayments/dodopayments-go"
- "github.com/dodopayments/dodopayments-go/option"
- "go.mongodb.org/mongo-driver/bson"
- "go.mongodb.org/mongo-driver/bson/primitive"
- "go.mongodb.org/mongo-driver/mongo"
- "go.mongodb.org/mongo-driver/mongo/options"
-)
-
-// WebhookEvent represents a webhook event from DodoPayments
-type WebhookEvent struct {
- ID string `json:"id"`
- Type string `json:"type"`
- Data map[string]interface{} `json:"data"`
-}
-
-// PaymentService handles payment and subscription operations
-type PaymentService struct {
- client *dodopayments.Client
- webhookSecret string
- mongoDB *database.MongoDB
- userService *UserService
- tierService *TierService
- usageLimiter *UsageLimiterService
-
- subscriptions *mongo.Collection
- events *mongo.Collection
-}
-
-// NewPaymentService creates a new payment service
-func NewPaymentService(
- apiKey, webhookSecret, businessID string,
- mongoDB *database.MongoDB,
- userService *UserService,
- tierService *TierService,
- usageLimiter *UsageLimiterService,
-) *PaymentService {
- var client *dodopayments.Client
- if apiKey != "" {
- // Determine environment mode
- env := os.Getenv("DODO_ENVIRONMENT")
- var envOpt option.RequestOption
- if env == "test" {
- envOpt = option.WithEnvironmentTestMode()
- } else {
- envOpt = option.WithEnvironmentLiveMode()
- }
-
- // Initialize DodoPayments client
- client = dodopayments.NewClient(
- option.WithBearerToken(apiKey),
- envOpt,
- )
- log.Println("✅ DodoPayments client initialized")
- } else {
- log.Println("⚠️ DodoPayments API key not provided, payment features disabled")
- }
-
- var subscriptions *mongo.Collection
- var events *mongo.Collection
- if mongoDB != nil {
- subscriptions = mongoDB.Database().Collection("subscriptions")
- events = mongoDB.Database().Collection("subscription_events")
- }
-
- return &PaymentService{
- client: client,
- webhookSecret: webhookSecret,
- mongoDB: mongoDB,
- userService: userService,
- tierService: tierService,
- usageLimiter: usageLimiter,
- subscriptions: subscriptions,
- events: events,
- }
-}
-
-// CheckoutResponse represents the response for checkout creation
-type CheckoutResponse struct {
- CheckoutURL string `json:"checkout_url"`
- SessionID string `json:"session_id"`
-}
-
-// CreateCheckoutSession creates a checkout session for a subscription
-func (s *PaymentService) CreateCheckoutSession(ctx context.Context, userID, userEmail, planID string) (*CheckoutResponse, error) {
- plan := models.GetPlanByID(planID)
- if plan == nil {
- return nil, fmt.Errorf("invalid plan ID: %s", planID)
- }
-
- if plan.Tier == models.TierFree {
- return nil, fmt.Errorf("cannot create checkout for free plan")
- }
-
- if plan.ContactSales {
- return nil, fmt.Errorf("enterprise plan requires contact sales")
- }
-
- if plan.DodoProductID == "" {
- return nil, fmt.Errorf("plan %s does not have a DodoPayments product ID configured", planID)
- }
-
- // Get or create user in MongoDB (sync from Supabase if new)
- user, err := s.userService.GetUserBySupabaseID(ctx, userID)
- if err != nil {
- // User doesn't exist in MongoDB, sync them from Supabase
- if userEmail == "" {
- return nil, fmt.Errorf("failed to get user and no email provided for sync")
- }
- log.Printf("📝 Syncing new user %s (%s) to MongoDB", userID, userEmail)
- user, err = s.userService.SyncUserFromSupabase(ctx, userID, userEmail)
- if err != nil {
- return nil, fmt.Errorf("failed to sync user: %w", err)
- }
- }
-
- customerID := user.DodoCustomerID
- if customerID == "" {
- // Create customer in DodoPayments
- if s.client == nil {
- return nil, fmt.Errorf("DodoPayments client not initialized")
- }
-
- // Generate a customer name from email (DodoPayments requires a name)
- // Use the part before @ as the name, or full email if no @
- customerName := user.Email
- if atIndex := strings.Index(user.Email, "@"); atIndex > 0 {
- customerName = user.Email[:atIndex]
- }
-
- customer, err := s.client.Customers.New(ctx, dodopayments.CustomerNewParams{
- Email: dodopayments.F(user.Email),
- Name: dodopayments.F(customerName),
- Metadata: dodopayments.F(map[string]string{
- "supabase_user_id": userID,
- }),
- })
- if err != nil {
- return nil, fmt.Errorf("failed to create customer: %w", err)
- }
-
- customerID = customer.CustomerID
- if err := s.updateUserDodoCustomer(ctx, userID, customerID); err != nil {
- return nil, fmt.Errorf("failed to update customer ID: %w", err)
- }
- }
-
- if s.client == nil {
- return nil, fmt.Errorf("DodoPayments client not initialized")
- }
-
- // Create checkout session using the SDK - Link to our existing customer!
- session, err := s.client.CheckoutSessions.New(ctx, dodopayments.CheckoutSessionNewParams{
- CheckoutSessionRequest: dodopayments.CheckoutSessionRequestParam{
- ProductCart: dodopayments.F([]dodopayments.CheckoutSessionRequestProductCartParam{{
- ProductID: dodopayments.F(plan.DodoProductID),
- Quantity: dodopayments.F(int64(1)),
- }}),
- ReturnURL: dodopayments.F(fmt.Sprintf("%s/settings?tab=billing&checkout=success", getBaseURL())),
- // Attach the checkout to our existing customer
- Customer: dodopayments.F[dodopayments.CustomerRequestUnionParam](dodopayments.AttachExistingCustomerParam{
- CustomerID: dodopayments.F(customerID),
- }),
- },
- })
- if err != nil {
- return nil, fmt.Errorf("failed to create checkout session: %w", err)
- }
-
- return &CheckoutResponse{
- CheckoutURL: session.CheckoutURL,
- SessionID: session.SessionID,
- }, nil
-}
-
-// GetCurrentSubscription gets the user's current subscription
-func (s *PaymentService) GetCurrentSubscription(ctx context.Context, userID string) (*models.Subscription, error) {
- // First, check if user has a promo tier in the users collection
- if s.userService != nil {
- user, err := s.userService.GetUserBySupabaseID(ctx, userID)
- if err == nil && user != nil && user.SubscriptionTier != "" {
- // User has a tier set (either from promo or previous subscription)
- sub := &models.Subscription{
- UserID: userID,
- Tier: user.SubscriptionTier,
- Status: user.SubscriptionStatus,
- CancelAtPeriodEnd: false,
- }
- if user.SubscriptionExpiresAt != nil {
- sub.CurrentPeriodEnd = *user.SubscriptionExpiresAt
- }
- return sub, nil
- }
- }
-
- if s.subscriptions == nil {
- // Return default free tier subscription
- return &models.Subscription{
- UserID: userID,
- Tier: models.TierFree,
- Status: models.SubStatusActive,
- }, nil
- }
-
- var sub models.Subscription
- err := s.subscriptions.FindOne(ctx, bson.M{"userId": userID}).Decode(&sub)
- if err == mongo.ErrNoDocuments {
- // No subscription found, return free tier
- return &models.Subscription{
- UserID: userID,
- Tier: models.TierFree,
- Status: models.SubStatusActive,
- }, nil
- }
- if err != nil {
- return nil, fmt.Errorf("failed to get subscription: %w", err)
- }
-
- return &sub, nil
-}
-
-// PlanChangeResult represents the result of a plan change
-type PlanChangeResult struct {
- Type string `json:"type"` // "upgrade" or "downgrade"
- Immediate bool `json:"immediate"` // true for upgrades, false for downgrades
- NewTier string `json:"new_tier"`
- EffectiveAt time.Time `json:"effective_at,omitempty"`
-}
-
-// ChangePlan handles both upgrades and downgrades
-func (s *PaymentService) ChangePlan(ctx context.Context, userID, newPlanID string) (*PlanChangeResult, error) {
- current, err := s.GetCurrentSubscription(ctx, userID)
- if err != nil {
- return nil, err
- }
-
- newPlan := models.GetPlanByID(newPlanID)
- if newPlan == nil {
- return nil, fmt.Errorf("invalid plan ID: %s", newPlanID)
- }
-
- currentPlan := models.GetPlanByTier(current.Tier)
- if currentPlan == nil {
- currentPlan = models.GetPlanByTier(models.TierFree)
- }
-
- // Determine if upgrade or downgrade
- comparison := models.CompareTiers(current.Tier, newPlan.Tier)
- isUpgrade := comparison < 0
-
- if comparison == 0 {
- return nil, fmt.Errorf("user is already on %s plan", newPlan.Tier)
- }
-
- if isUpgrade {
- // UPGRADE: Immediate with proration
- if current.DodoSubscriptionID == "" {
- return nil, fmt.Errorf("no active subscription to upgrade")
- }
-
- if s.client == nil {
- return nil, fmt.Errorf("DodoPayments client not initialized")
- }
-
- // Change plan using DodoPayments SDK (handles proration automatically)
- err = s.client.Subscriptions.ChangePlan(ctx, current.DodoSubscriptionID, dodopayments.SubscriptionChangePlanParams{
- ProductID: dodopayments.F(newPlan.DodoProductID),
- ProrationBillingMode: dodopayments.F(dodopayments.SubscriptionChangePlanParamsProrationBillingModeProratedImmediately),
- Quantity: dodopayments.F(int64(1)),
- })
- if err != nil {
- return nil, fmt.Errorf("failed to change plan: %w", err)
- }
-
- // Log the change for audit
- log.Printf("✅ Plan changed to %s for subscription %s", newPlan.DodoProductID, current.DodoSubscriptionID)
-
- // Update subscription in our DB (webhook will confirm, but optimistic update)
- now := time.Now()
- update := bson.M{
- "$set": bson.M{
- "tier": newPlan.Tier,
- "status": models.SubStatusActive,
- "scheduledTier": "",
- "scheduledChangeAt": nil,
- "cancelAtPeriodEnd": false,
- "updatedAt": now,
- },
- }
-
- if s.subscriptions != nil {
- _, err = s.subscriptions.UpdateOne(ctx, bson.M{"userId": userID}, update)
- if err != nil {
- log.Printf("⚠️ Failed to update subscription optimistically: %v", err)
- }
- }
-
- // Invalidate tier cache
- if s.tierService != nil {
- s.tierService.InvalidateCache(userID)
- }
-
- return &PlanChangeResult{
- Type: "upgrade",
- Immediate: true,
- NewTier: newPlan.Tier,
- }, nil
- } else {
- // DOWNGRADE: Schedule for end of period
- if current.DodoSubscriptionID == "" {
- // No active subscription, just update tier
- now := time.Now()
- if s.subscriptions != nil {
- _, err = s.subscriptions.UpdateOne(ctx, bson.M{"userId": userID}, bson.M{
- "$set": bson.M{
- "tier": newPlan.Tier,
- "status": models.SubStatusActive,
- "updatedAt": now,
- },
- })
- }
- if s.tierService != nil {
- s.tierService.InvalidateCache(userID)
- }
- return &PlanChangeResult{
- Type: "downgrade",
- Immediate: true,
- NewTier: newPlan.Tier,
- }, nil
- }
-
- // Schedule downgrade for end of period
- periodEnd := current.CurrentPeriodEnd
- if periodEnd.IsZero() {
- periodEnd = time.Now().Add(30 * 24 * time.Hour) // Default to 30 days
- }
-
- update := bson.M{
- "$set": bson.M{
- "scheduledTier": newPlan.Tier,
- "scheduledChangeAt": periodEnd,
- "updatedAt": time.Now(),
- },
- }
-
- if s.subscriptions != nil {
- _, err = s.subscriptions.UpdateOne(ctx, bson.M{"userId": userID}, update)
- if err != nil {
- return nil, fmt.Errorf("failed to schedule downgrade: %w", err)
- }
- }
-
- return &PlanChangeResult{
- Type: "downgrade",
- Immediate: false,
- NewTier: newPlan.Tier,
- EffectiveAt: periodEnd,
- }, nil
- }
-}
-
-// PreviewPlanChange shows what will happen before confirming
-type PlanChangePreview struct {
- ChangeType string `json:"change_type"` // "upgrade" or "downgrade"
- Immediate bool `json:"immediate"`
- CurrentTier string `json:"current_tier"`
- NewTier string `json:"new_tier"`
- ProratedAmount int64 `json:"prorated_amount,omitempty"` // cents
- EffectiveAt time.Time `json:"effective_at,omitempty"`
-}
-
-// PreviewPlanChange previews a plan change
-func (s *PaymentService) PreviewPlanChange(ctx context.Context, userID, newPlanID string) (*PlanChangePreview, error) {
- current, err := s.GetCurrentSubscription(ctx, userID)
- if err != nil {
- return nil, err
- }
-
- newPlan := models.GetPlanByID(newPlanID)
- if newPlan == nil {
- return nil, fmt.Errorf("invalid plan ID: %s", newPlanID)
- }
-
- currentPlan := models.GetPlanByTier(current.Tier)
- if currentPlan == nil {
- currentPlan = models.GetPlanByTier(models.TierFree)
- }
-
- comparison := models.CompareTiers(current.Tier, newPlan.Tier)
- isUpgrade := comparison < 0
-
- if comparison == 0 {
- return nil, fmt.Errorf("user is already on %s plan", newPlan.Tier)
- }
-
- preview := &PlanChangePreview{
- CurrentTier: current.Tier,
- NewTier: newPlan.Tier,
- }
-
- if isUpgrade {
- preview.ChangeType = "upgrade"
- preview.Immediate = true
- preview.EffectiveAt = time.Now()
-
- // Calculate proration if we have period info
- if !current.CurrentPeriodEnd.IsZero() && !current.CurrentPeriodStart.IsZero() {
- daysRemaining := int(time.Until(current.CurrentPeriodEnd).Hours() / 24)
- totalDays := int(current.CurrentPeriodEnd.Sub(current.CurrentPeriodStart).Hours() / 24)
- if daysRemaining > 0 && totalDays > 0 {
- preview.ProratedAmount = s.CalculateProration(
- currentPlan.PriceMonthly,
- newPlan.PriceMonthly,
- daysRemaining,
- totalDays,
- )
- }
- }
- } else {
- preview.ChangeType = "downgrade"
- preview.Immediate = false
- preview.EffectiveAt = current.CurrentPeriodEnd
- if preview.EffectiveAt.IsZero() {
- preview.EffectiveAt = time.Now().Add(30 * 24 * time.Hour)
- }
- }
-
- return preview, nil
-}
-
-// CalculateProration calculates prorated charge for plan change
-func (s *PaymentService) CalculateProration(fromPrice, toPrice int64, daysRemaining, totalDays int) int64 {
- if daysRemaining <= 0 || totalDays <= 0 {
- return 0
- }
-
- // Calculate daily rates
- fromDaily := float64(fromPrice) / float64(totalDays)
- toDaily := float64(toPrice) / float64(totalDays)
-
- // Calculate difference for remaining days
- difference := (toDaily - fromDaily) * float64(daysRemaining)
-
- return int64(difference)
-}
-
-// CancelSubscription schedules cancellation at period end
-func (s *PaymentService) CancelSubscription(ctx context.Context, userID string) error {
- current, err := s.GetCurrentSubscription(ctx, userID)
- if err != nil {
- return err
- }
-
- if current.Tier == models.TierFree {
- return fmt.Errorf("no active subscription to cancel")
- }
-
- if current.CancelAtPeriodEnd {
- return fmt.Errorf("subscription is already scheduled for cancellation")
- }
-
- if current.DodoSubscriptionID != "" && s.client != nil {
- // Cancel in DodoPayments (cancel at next billing date)
- _, err = s.client.Subscriptions.Update(ctx, current.DodoSubscriptionID, dodopayments.SubscriptionUpdateParams{
- CancelAtNextBillingDate: dodopayments.F(true),
- })
- if err != nil {
- return fmt.Errorf("failed to cancel subscription: %w", err)
- }
- }
-
- // Update in our DB
- update := bson.M{
- "$set": bson.M{
- "cancelAtPeriodEnd": true,
- "status": models.SubStatusPendingCancel,
- "updatedAt": time.Now(),
- },
- }
-
- if s.subscriptions != nil {
- _, err = s.subscriptions.UpdateOne(ctx, bson.M{"userId": userID}, update)
- if err != nil {
- return fmt.Errorf("failed to update subscription: %w", err)
- }
- }
-
- return nil
-}
-
-// ReactivateSubscription undoes cancellation if still in period
-func (s *PaymentService) ReactivateSubscription(ctx context.Context, userID string) error {
- current, err := s.GetCurrentSubscription(ctx, userID)
- if err != nil {
- return err
- }
-
- if !current.CancelAtPeriodEnd {
- return fmt.Errorf("subscription is not scheduled for cancellation")
- }
-
- if current.DodoSubscriptionID != "" && s.client != nil {
- // Reactivate in DodoPayments (clear cancel_at_next_billing_date)
- _, err = s.client.Subscriptions.Update(ctx, current.DodoSubscriptionID, dodopayments.SubscriptionUpdateParams{
- CancelAtNextBillingDate: dodopayments.F(false),
- })
- if err != nil {
- return fmt.Errorf("failed to reactivate subscription: %w", err)
- }
- }
-
- // Update in our DB
- update := bson.M{
- "$set": bson.M{
- "cancelAtPeriodEnd": false,
- "status": models.SubStatusActive,
- "updatedAt": time.Now(),
- },
- }
-
- if s.subscriptions != nil {
- _, err = s.subscriptions.UpdateOne(ctx, bson.M{"userId": userID}, update)
- if err != nil {
- return fmt.Errorf("failed to update subscription: %w", err)
- }
- }
-
- return nil
-}
-
-// GetCustomerPortalURL gets the DodoPayments customer portal URL
-func (s *PaymentService) GetCustomerPortalURL(ctx context.Context, userID string) (string, error) {
- user, err := s.userService.GetUserBySupabaseID(ctx, userID)
- if err != nil {
- return "", fmt.Errorf("failed to get user: %w", err)
- }
-
- if user.DodoCustomerID == "" {
- return "", fmt.Errorf("user does not have a DodoPayments customer ID")
- }
-
- if s.client == nil {
- return "", fmt.Errorf("DodoPayments client not initialized")
- }
-
- // Create a customer portal session using DodoPayments SDK
- portalSession, err := s.client.Customers.CustomerPortal.New(ctx, user.DodoCustomerID, dodopayments.CustomerCustomerPortalNewParams{})
- if err != nil {
- return "", fmt.Errorf("failed to create customer portal session: %w", err)
- }
-
- return portalSession.Link, nil
-}
-
-// GetAvailablePlans returns all available plans
-func (s *PaymentService) GetAvailablePlans() []models.Plan {
- return models.GetAvailablePlans()
-}
-
-// DetermineChangeType determines if a plan change is an upgrade or downgrade
-func (s *PaymentService) DetermineChangeType(fromTier, toTier string) (isUpgrade, isDowngrade bool) {
- comparison := models.CompareTiers(fromTier, toTier)
- isUpgrade = comparison < 0
- isDowngrade = comparison > 0
- return
-}
-
-// VerifyWebhook verifies webhook signature (legacy method for tests)
-func (s *PaymentService) VerifyWebhook(payload []byte, signature string) error {
- if s.webhookSecret == "" {
- return fmt.Errorf("webhook secret not configured")
- }
-
- mac := hmac.New(sha256.New, []byte(s.webhookSecret))
- mac.Write(payload)
- expectedSig := hex.EncodeToString(mac.Sum(nil))
-
- if signature != expectedSig {
- return fmt.Errorf("invalid webhook signature")
- }
-
- return nil
-}
-
-// VerifyAndParseWebhook verifies and parses webhook using DodoPayments SDK
-// DodoPayments uses Standard Webhooks format with headers:
-// - webhook-id: unique message ID
-// - webhook-signature: v1,
-// - webhook-timestamp: unix timestamp
-func (s *PaymentService) VerifyAndParseWebhook(payload []byte, headers http.Header) (*WebhookEvent, error) {
- // If SDK client is available, use it for verification
- if s.client != nil && s.webhookSecret != "" {
- event, err := s.client.Webhooks.Unwrap(payload, headers, option.WithWebhookKey(s.webhookSecret))
- if err != nil {
- return nil, fmt.Errorf("webhook verification failed: %w", err)
- }
-
- // Successfully verified with SDK - convert and return
- return s.convertSDKEventToWebhookEvent(event)
- }
-
- // Fallback: Use legacy HMAC verification for tests or when SDK is not available
- signature := headers.Get("Webhook-Signature")
- if signature == "" {
- signature = headers.Get("Dodo-Signature")
- }
- if signature == "" {
- return nil, fmt.Errorf("missing webhook signature header")
- }
-
- if err := s.VerifyWebhook(payload, signature); err != nil {
- return nil, err
- }
-
- // Parse the payload
- var event WebhookEvent
- if err := json.Unmarshal(payload, &event); err != nil {
- return nil, fmt.Errorf("failed to parse webhook payload: %w", err)
- }
-
- return &event, nil
-}
-
-// convertSDKEventToWebhookEvent converts SDK event to internal WebhookEvent
-func (s *PaymentService) convertSDKEventToWebhookEvent(event *dodopayments.UnwrapWebhookEvent) (*WebhookEvent, error) {
- // Convert SDK event to our internal WebhookEvent format
- webhookEvent := &WebhookEvent{
- Type: string(event.Type),
- }
-
- // Extract event ID and data based on event type
- // The Data field embeds the subscription/payment struct directly
- switch e := event.AsUnion().(type) {
- case dodopayments.SubscriptionActiveWebhookEvent:
- webhookEvent.ID = e.Data.SubscriptionID
- webhookEvent.Data = map[string]interface{}{
- "subscription_id": e.Data.SubscriptionID,
- "customer_id": e.Data.Customer.CustomerID,
- "product_id": e.Data.ProductID,
- "current_period_start": e.Data.PreviousBillingDate.Format(time.RFC3339),
- "current_period_end": e.Data.NextBillingDate.Format(time.RFC3339),
- }
- case dodopayments.SubscriptionUpdatedWebhookEvent:
- webhookEvent.ID = e.Data.SubscriptionID
- webhookEvent.Data = map[string]interface{}{
- "subscription_id": e.Data.SubscriptionID,
- "customer_id": e.Data.Customer.CustomerID,
- "product_id": e.Data.ProductID,
- "current_period_start": e.Data.PreviousBillingDate.Format(time.RFC3339),
- "current_period_end": e.Data.NextBillingDate.Format(time.RFC3339),
- }
- case dodopayments.SubscriptionCancelledWebhookEvent:
- webhookEvent.ID = e.Data.SubscriptionID
- webhookEvent.Data = map[string]interface{}{
- "subscription_id": e.Data.SubscriptionID,
- "customer_id": e.Data.Customer.CustomerID,
- }
- case dodopayments.SubscriptionRenewedWebhookEvent:
- webhookEvent.ID = e.Data.SubscriptionID
- webhookEvent.Data = map[string]interface{}{
- "subscription_id": e.Data.SubscriptionID,
- "customer_id": e.Data.Customer.CustomerID,
- "current_period_start": e.Data.PreviousBillingDate.Format(time.RFC3339),
- "current_period_end": e.Data.NextBillingDate.Format(time.RFC3339),
- }
- case dodopayments.SubscriptionOnHoldWebhookEvent:
- webhookEvent.ID = e.Data.SubscriptionID
- webhookEvent.Data = map[string]interface{}{
- "subscription_id": e.Data.SubscriptionID,
- "customer_id": e.Data.Customer.CustomerID,
- }
- case dodopayments.PaymentSucceededWebhookEvent:
- webhookEvent.ID = e.Data.PaymentID
- webhookEvent.Data = map[string]interface{}{
- "payment_id": e.Data.PaymentID,
- "subscription_id": e.Data.SubscriptionID,
- }
- case dodopayments.PaymentFailedWebhookEvent:
- webhookEvent.ID = e.Data.PaymentID
- webhookEvent.Data = map[string]interface{}{
- "payment_id": e.Data.PaymentID,
- "subscription_id": e.Data.SubscriptionID,
- }
- default:
- // For unknown event types, try to extract basic info
- webhookEvent.ID = fmt.Sprintf("evt_%d", time.Now().UnixNano())
- webhookEvent.Data = make(map[string]interface{})
- }
-
- return webhookEvent, nil
-}
-
-// IsEventProcessed checks if a webhook event has already been processed
-func (s *PaymentService) IsEventProcessed(ctx context.Context, eventID string) bool {
- if s.events == nil {
- return false
- }
-
- count, err := s.events.CountDocuments(ctx, bson.M{"dodoEventId": eventID})
- if err != nil {
- return false
- }
-
- return count > 0
-}
-
-// HandleWebhookEvent processes a verified webhook event
-func (s *PaymentService) HandleWebhookEvent(ctx context.Context, event *WebhookEvent) error {
- // Check idempotency
- if s.IsEventProcessed(ctx, event.ID) {
- log.Printf("⚠️ Webhook event %s already processed, skipping", event.ID)
- return fmt.Errorf("webhook event already processed (idempotent)")
- }
-
- // Log event
- eventDoc := models.SubscriptionEvent{
- ID: primitive.NewObjectID(),
- DodoEventID: event.ID,
- EventType: event.Type,
- Metadata: event.Data,
- CreatedAt: time.Now(),
- }
-
- if s.events != nil {
- _, err := s.events.InsertOne(ctx, eventDoc)
- if err != nil {
- log.Printf("⚠️ Failed to log webhook event: %v", err)
- }
- }
-
- // Handle event based on type
- switch event.Type {
- case "subscription.active":
- return s.handleSubscriptionActive(ctx, event)
- case "subscription.updated":
- return s.handleSubscriptionUpdated(ctx, event)
- case "subscription.on_hold":
- return s.handleSubscriptionOnHold(ctx, event)
- case "subscription.renewed":
- return s.handleSubscriptionRenewed(ctx, event)
- case "subscription.cancelled":
- return s.handleSubscriptionCancelled(ctx, event)
- case "payment.succeeded":
- return s.handlePaymentSucceeded(ctx, event)
- case "payment.failed":
- return s.handlePaymentFailed(ctx, event)
- default:
- log.Printf("⚠️ Unhandled webhook event type: %s", event.Type)
- return nil
- }
-}
-
-func (s *PaymentService) handleSubscriptionActive(ctx context.Context, event *WebhookEvent) error {
- subID, _ := event.Data["subscription_id"].(string)
- customerID, _ := event.Data["customer_id"].(string)
- productID, _ := event.Data["product_id"].(string)
-
- if subID == "" || customerID == "" {
- return fmt.Errorf("missing required fields in subscription.active event")
- }
-
- // Find plan by product ID
- var plan *models.Plan
- for i := range models.AvailablePlans {
- if models.AvailablePlans[i].DodoProductID == productID {
- plan = &models.AvailablePlans[i]
- break
- }
- }
-
- if plan == nil {
- return fmt.Errorf("unknown product ID: %s", productID)
- }
-
- // Find user by customer ID first
- var user models.User
- if s.mongoDB == nil {
- return fmt.Errorf("MongoDB not available")
- }
-
- err := s.mongoDB.Database().Collection("users").FindOne(ctx, bson.M{"dodoCustomerId": customerID}).Decode(&user)
- if err != nil {
- log.Printf("⚠️ User not found by customer ID %s, trying to fetch from DodoPayments...", customerID)
-
- // Fallback: Fetch customer from DodoPayments to get email
- if s.client != nil {
- customer, fetchErr := s.client.Customers.Get(ctx, customerID)
- if fetchErr != nil {
- return fmt.Errorf("failed to find user by customer ID and failed to fetch customer: %w", fetchErr)
- }
-
- // Try to find user by email
- err = s.mongoDB.Database().Collection("users").FindOne(ctx, bson.M{"email": customer.Email}).Decode(&user)
- if err != nil {
- return fmt.Errorf("failed to find user by customer ID or email (%s): %w", customer.Email, err)
- }
-
- // Update the user's dodoCustomerId with the new customer ID
- log.Printf("✅ Found user by email %s, updating dodoCustomerId to %s", customer.Email, customerID)
- _, updateErr := s.mongoDB.Database().Collection("users").UpdateOne(
- ctx,
- bson.M{"_id": user.ID},
- bson.M{"$set": bson.M{"dodoCustomerId": customerID}},
- )
- if updateErr != nil {
- log.Printf("⚠️ Failed to update dodoCustomerId: %v", updateErr)
- }
- } else {
- return fmt.Errorf("failed to find user by customer ID: %w", err)
- }
- }
-
- // Parse period dates
- var periodStart, periodEnd time.Time
- if startStr, ok := event.Data["current_period_start"].(string); ok {
- periodStart, _ = time.Parse(time.RFC3339, startStr)
- }
- if endStr, ok := event.Data["current_period_end"].(string); ok {
- periodEnd, _ = time.Parse(time.RFC3339, endStr)
- }
-
- // Upsert subscription
- now := time.Now()
-
- if s.subscriptions != nil {
- filter := bson.M{"userId": user.SupabaseUserID}
- update := bson.M{
- "$set": bson.M{
- "userId": user.SupabaseUserID,
- "dodoSubscriptionId": subID,
- "dodoCustomerId": customerID,
- "tier": plan.Tier,
- "status": models.SubStatusActive,
- "currentPeriodStart": periodStart,
- "currentPeriodEnd": periodEnd,
- "updatedAt": now,
- },
- "$setOnInsert": bson.M{
- "createdAt": now,
- },
- }
- opts := options.Update().SetUpsert(true)
- _, err := s.subscriptions.UpdateOne(ctx, filter, update, opts)
- if err != nil {
- return fmt.Errorf("failed to upsert subscription: %w", err)
- }
- }
-
- // Update user subscription tier
- if s.userService != nil {
- err := s.userService.UpdateSubscriptionWithStatus(ctx, user.SupabaseUserID, plan.Tier, models.SubStatusActive, &periodEnd)
- if err != nil {
- log.Printf("⚠️ Failed to update user subscription: %v", err)
- }
- }
-
- // Invalidate tier cache
- if s.tierService != nil {
- s.tierService.InvalidateCache(user.SupabaseUserID)
- }
-
- // Reset usage counters on new subscription activation
- if s.usageLimiter != nil {
- if err := s.usageLimiter.ResetAllCounters(ctx, user.SupabaseUserID); err != nil {
- log.Printf("⚠️ Failed to reset usage counters for user %s: %v", user.SupabaseUserID, err)
- } else {
- log.Printf("✅ [WEBHOOK] Reset usage counters for new subscriber %s", user.SupabaseUserID)
- }
- }
-
- log.Printf("✅ Subscription activated for user %s: %s", user.SupabaseUserID, plan.Tier)
- return nil
-}
-
-func (s *PaymentService) handleSubscriptionUpdated(ctx context.Context, event *WebhookEvent) error {
- subID, _ := event.Data["subscription_id"].(string)
- productID, _ := event.Data["product_id"].(string)
- if subID == "" {
- return fmt.Errorf("missing subscription_id in event")
- }
-
- // Parse period dates from webhook
- var periodStart, periodEnd time.Time
- if startStr, ok := event.Data["current_period_start"].(string); ok && startStr != "" {
- periodStart, _ = time.Parse(time.RFC3339, startStr)
- }
- if endStr, ok := event.Data["current_period_end"].(string); ok && endStr != "" {
- periodEnd, _ = time.Parse(time.RFC3339, endStr)
- }
-
- // Get current subscription
- var sub models.Subscription
- if s.subscriptions != nil {
- err := s.subscriptions.FindOne(ctx, bson.M{"dodoSubscriptionId": subID}).Decode(&sub)
- if err != nil {
- if err == mongo.ErrNoDocuments {
- // Subscription doesn't exist yet - this can happen due to webhook race conditions
- // The subscription.active event will create it, so we can safely skip this update
- log.Printf("⚠️ Subscription %s not found yet (race condition), skipping update", subID)
- return nil
- }
- return fmt.Errorf("subscription not found: %w", err)
- }
- } else {
- return fmt.Errorf("MongoDB not available")
- }
-
- // Find new plan by product ID (for upgrades/downgrades)
- var newPlan *models.Plan
- if productID != "" {
- for i := range models.AvailablePlans {
- if models.AvailablePlans[i].DodoProductID == productID {
- newPlan = &models.AvailablePlans[i]
- break
- }
- }
- }
-
- // Check if this is a scheduled downgrade being applied
- if sub.HasScheduledChange() && time.Now().After(*sub.ScheduledChangeAt) {
- // Apply scheduled downgrade
- update := bson.M{
- "$set": bson.M{
- "tier": sub.ScheduledTier,
- "scheduledTier": "",
- "scheduledChangeAt": nil,
- "updatedAt": time.Now(),
- },
- }
- if !periodEnd.IsZero() {
- update["$set"].(bson.M)["currentPeriodEnd"] = periodEnd
- }
- if !periodStart.IsZero() {
- update["$set"].(bson.M)["currentPeriodStart"] = periodStart
- }
-
- if s.subscriptions != nil {
- _, err := s.subscriptions.UpdateOne(ctx, bson.M{"_id": sub.ID}, update)
- if err != nil {
- return fmt.Errorf("failed to apply scheduled downgrade: %w", err)
- }
- }
-
- // Update user tier
- if s.userService != nil {
- err := s.userService.UpdateSubscriptionWithStatus(ctx, sub.UserID, sub.ScheduledTier, models.SubStatusActive, &periodEnd)
- if err != nil {
- log.Printf("⚠️ Failed to update user tier: %v", err)
- }
- }
-
- if s.tierService != nil {
- s.tierService.InvalidateCache(sub.UserID)
- }
-
- log.Printf("✅ Scheduled downgrade applied for subscription %s: %s -> %s", subID, sub.Tier, sub.ScheduledTier)
- return nil
- }
-
- // Handle tier change from plan upgrade/downgrade
- if newPlan != nil && newPlan.Tier != sub.Tier {
- updateFields := bson.M{
- "tier": newPlan.Tier,
- "updatedAt": time.Now(),
- }
- if !periodEnd.IsZero() {
- updateFields["currentPeriodEnd"] = periodEnd
- }
- if !periodStart.IsZero() {
- updateFields["currentPeriodStart"] = periodStart
- }
-
- update := bson.M{"$set": updateFields}
-
- if s.subscriptions != nil {
- _, err := s.subscriptions.UpdateOne(ctx, bson.M{"_id": sub.ID}, update)
- if err != nil {
- return fmt.Errorf("failed to update subscription tier: %w", err)
- }
- }
-
- // Update user tier
- if s.userService != nil {
- err := s.userService.UpdateSubscriptionWithStatus(ctx, sub.UserID, newPlan.Tier, models.SubStatusActive, &periodEnd)
- if err != nil {
- log.Printf("⚠️ Failed to update user tier: %v", err)
- }
- }
-
- // Invalidate tier cache
- if s.tierService != nil {
- s.tierService.InvalidateCache(sub.UserID)
- }
-
- // If upgrade, reset counters to give immediate access to new limits
- if isUpgrade(sub.Tier, newPlan.Tier) {
- if s.usageLimiter != nil {
- if err := s.usageLimiter.ResetAllCounters(ctx, sub.UserID); err != nil {
- log.Printf("⚠️ Failed to reset usage counters on upgrade: %v", err)
- } else {
- log.Printf("✅ [WEBHOOK] Reset usage counters for upgraded user %s (%s -> %s)", sub.UserID, sub.Tier, newPlan.Tier)
- }
- }
- }
-
- log.Printf("✅ Subscription updated for %s: %s -> %s", subID, sub.Tier, newPlan.Tier)
- return nil
- }
-
- // Just update period dates if no tier change
- if !periodEnd.IsZero() || !periodStart.IsZero() {
- updateFields := bson.M{"updatedAt": time.Now()}
- if !periodEnd.IsZero() {
- updateFields["currentPeriodEnd"] = periodEnd
- }
- if !periodStart.IsZero() {
- updateFields["currentPeriodStart"] = periodStart
- }
-
- if s.subscriptions != nil {
- _, err := s.subscriptions.UpdateOne(ctx, bson.M{"_id": sub.ID}, bson.M{"$set": updateFields})
- if err != nil {
- log.Printf("⚠️ Failed to update subscription period dates: %v", err)
- }
- }
- }
-
- log.Printf("✅ Subscription updated event processed for %s", subID)
- return nil
-}
-
-func (s *PaymentService) handleSubscriptionOnHold(ctx context.Context, event *WebhookEvent) error {
- subID, _ := event.Data["subscription_id"].(string)
- if subID == "" {
- return fmt.Errorf("missing subscription_id in event")
- }
-
- update := bson.M{
- "$set": bson.M{
- "status": models.SubStatusOnHold,
- "updatedAt": time.Now(),
- },
- }
-
- if s.subscriptions != nil {
- _, err := s.subscriptions.UpdateOne(ctx, bson.M{"dodoSubscriptionId": subID}, update)
- if err != nil {
- return fmt.Errorf("failed to update subscription status: %w", err)
- }
- }
-
- log.Printf("⚠️ Subscription %s put on hold", subID)
- return nil
-}
-
-func (s *PaymentService) handleSubscriptionRenewed(ctx context.Context, event *WebhookEvent) error {
- subID, _ := event.Data["subscription_id"].(string)
- if subID == "" {
- return fmt.Errorf("missing subscription_id in event")
- }
-
- // Parse period dates
- var periodStart, periodEnd time.Time
- if startStr, ok := event.Data["current_period_start"].(string); ok {
- periodStart, _ = time.Parse(time.RFC3339, startStr)
- }
- if endStr, ok := event.Data["current_period_end"].(string); ok {
- periodEnd, _ = time.Parse(time.RFC3339, endStr)
- }
-
- // Get subscription
- var sub models.Subscription
- if s.subscriptions != nil {
- err := s.subscriptions.FindOne(ctx, bson.M{"dodoSubscriptionId": subID}).Decode(&sub)
- if err != nil {
- if err == mongo.ErrNoDocuments {
- // Subscription doesn't exist yet - this can happen due to webhook race conditions
- // The subscription.active event will create it, so we can safely skip this renewal
- log.Printf("⚠️ Subscription %s not found yet (race condition), skipping renewal", subID)
- return nil
- }
- return fmt.Errorf("subscription not found: %w", err)
- }
-
- // Check if cancellation was scheduled
- if sub.CancelAtPeriodEnd {
- // Revert to free tier
- update := bson.M{
- "$set": bson.M{
- "tier": models.TierFree,
- "status": models.SubStatusCancelled,
- "cancelAtPeriodEnd": false,
- "cancelledAt": time.Now(),
- "updatedAt": time.Now(),
- },
- }
- _, err = s.subscriptions.UpdateOne(ctx, bson.M{"_id": sub.ID}, update)
- if err != nil {
- return fmt.Errorf("failed to cancel subscription: %w", err)
- }
-
- // Update user tier
- if s.userService != nil {
- err = s.userService.UpdateSubscriptionWithStatus(ctx, sub.UserID, models.TierFree, models.SubStatusCancelled, nil)
- if err != nil {
- log.Printf("⚠️ Failed to update user tier: %v", err)
- }
- }
-
- if s.tierService != nil {
- s.tierService.InvalidateCache(sub.UserID)
- }
-
- log.Printf("✅ Subscription %s cancelled and reverted to free", subID)
- return nil
- }
-
- // Check if a downgrade was scheduled (should be applied on renewal)
- if sub.HasScheduledChange() {
- // Apply scheduled downgrade
- update := bson.M{
- "$set": bson.M{
- "tier": sub.ScheduledTier,
- "scheduledTier": "",
- "scheduledChangeAt": nil,
- "currentPeriodStart": periodStart,
- "currentPeriodEnd": periodEnd,
- "status": models.SubStatusActive,
- "updatedAt": time.Now(),
- },
- }
- _, err = s.subscriptions.UpdateOne(ctx, bson.M{"_id": sub.ID}, update)
- if err != nil {
- return fmt.Errorf("failed to apply scheduled downgrade: %w", err)
- }
-
- // Update user tier
- if s.userService != nil {
- err = s.userService.UpdateSubscriptionWithStatus(ctx, sub.UserID, sub.ScheduledTier, models.SubStatusActive, &periodEnd)
- if err != nil {
- log.Printf("⚠️ Failed to update user tier: %v", err)
- }
- }
-
- if s.tierService != nil {
- s.tierService.InvalidateCache(sub.UserID)
- }
-
- log.Printf("✅ Subscription %s renewed and scheduled downgrade to %s applied", subID, sub.ScheduledTier)
- return nil
- }
-
- // Normal renewal - just update period dates
- update := bson.M{
- "$set": bson.M{
- "currentPeriodStart": periodStart,
- "currentPeriodEnd": periodEnd,
- "status": models.SubStatusActive,
- "updatedAt": time.Now(),
- },
- }
- _, err = s.subscriptions.UpdateOne(ctx, bson.M{"_id": sub.ID}, update)
- if err != nil {
- return fmt.Errorf("failed to update subscription: %w", err)
- }
- }
-
- log.Printf("✅ Subscription %s renewed", subID)
- return nil
-}
-
-func (s *PaymentService) handleSubscriptionCancelled(ctx context.Context, event *WebhookEvent) error {
- subID, _ := event.Data["subscription_id"].(string)
- if subID == "" {
- return fmt.Errorf("missing subscription_id in event")
- }
-
- // Get subscription
- var sub models.Subscription
- if s.subscriptions != nil {
- err := s.subscriptions.FindOne(ctx, bson.M{"dodoSubscriptionId": subID}).Decode(&sub)
- if err != nil {
- return fmt.Errorf("subscription not found: %w", err)
- }
-
- // Revert to free tier
- update := bson.M{
- "$set": bson.M{
- "tier": models.TierFree,
- "status": models.SubStatusCancelled,
- "cancelledAt": time.Now(),
- "updatedAt": time.Now(),
- },
- }
- _, err = s.subscriptions.UpdateOne(ctx, bson.M{"_id": sub.ID}, update)
- if err != nil {
- return fmt.Errorf("failed to cancel subscription: %w", err)
- }
-
- // Update user tier
- if s.userService != nil {
- err = s.userService.UpdateSubscription(ctx, sub.UserID, models.TierFree, nil)
- if err != nil {
- log.Printf("⚠️ Failed to update user tier: %v", err)
- }
- }
-
- if s.tierService != nil {
- s.tierService.InvalidateCache(sub.UserID)
- }
- }
-
- log.Printf("✅ Subscription %s cancelled", subID)
- return nil
-}
-
-func (s *PaymentService) handlePaymentSucceeded(ctx context.Context, event *WebhookEvent) error {
- // Payment succeeded - subscription should already be active
- // Just log for audit
- log.Printf("✅ Payment succeeded for subscription")
- return nil
-}
-
-func (s *PaymentService) handlePaymentFailed(ctx context.Context, event *WebhookEvent) error {
- subID, _ := event.Data["subscription_id"].(string)
- if subID == "" {
- return fmt.Errorf("missing subscription_id in event")
- }
-
- // Update subscription status to on_hold
- update := bson.M{
- "$set": bson.M{
- "status": models.SubStatusOnHold,
- "updatedAt": time.Now(),
- },
- }
-
- if s.subscriptions != nil {
- _, err := s.subscriptions.UpdateOne(ctx, bson.M{"dodoSubscriptionId": subID}, update)
- if err != nil {
- return fmt.Errorf("failed to update subscription status: %w", err)
- }
- }
-
- log.Printf("⚠️ Payment failed for subscription %s", subID)
- return nil
-}
-
-// SyncSubscriptionFromDodo syncs subscription data from DodoPayments for a user
-func (s *PaymentService) SyncSubscriptionFromDodo(ctx context.Context, userID string) (map[string]interface{}, error) {
- if s.client == nil {
- return nil, fmt.Errorf("DodoPayments client not initialized")
- }
-
- // Get user
- user, err := s.userService.GetUserBySupabaseID(ctx, userID)
- if err != nil {
- return nil, fmt.Errorf("failed to get user: %w", err)
- }
-
- // If user has no dodoCustomerId, try to find by email
- customerID := user.DodoCustomerID
- if customerID == "" {
- // Search for customer by email using the customers list API
- log.Printf("⚠️ User %s has no dodoCustomerId, trying to find customer by email...", userID)
-
- // For now, we'll need the user to do a new checkout to create the customer link
- return map[string]interface{}{
- "status": "no_customer",
- "message": "No DodoPayments customer linked. Please initiate a new checkout to link your account.",
- }, nil
- }
-
- // Get subscriptions from DodoPayments for this customer
- subscriptionsPage, err := s.client.Subscriptions.List(ctx, dodopayments.SubscriptionListParams{
- CustomerID: dodopayments.F(customerID),
- })
- if err != nil {
- return nil, fmt.Errorf("failed to list subscriptions from DodoPayments: %w", err)
- }
-
- // Find active subscription (SubscriptionListResponse type)
- var activeSub *dodopayments.SubscriptionListResponse
- for i := range subscriptionsPage.Items {
- sub := &subscriptionsPage.Items[i]
- if sub.Status == dodopayments.SubscriptionStatusActive {
- activeSub = sub
- break
- }
- }
-
- if activeSub == nil {
- // No active subscription found
- return map[string]interface{}{
- "status": "no_subscription",
- "message": "No active subscription found in DodoPayments",
- "tier": models.TierFree,
- }, nil
- }
-
- // Find plan by product ID
- var plan *models.Plan
- if activeSub.ProductID != "" {
- for i := range models.AvailablePlans {
- if models.AvailablePlans[i].DodoProductID == activeSub.ProductID {
- plan = &models.AvailablePlans[i]
- break
- }
- }
- }
-
- tier := models.TierFree
- if plan != nil {
- tier = plan.Tier
- }
-
- // DodoPayments uses NextBillingDate (end of period) and PreviousBillingDate (start of period)
- periodStart := activeSub.PreviousBillingDate
- periodEnd := activeSub.NextBillingDate
-
- // Update local subscription
- now := time.Now()
- if s.subscriptions != nil {
- filter := bson.M{"userId": userID}
- update := bson.M{
- "$set": bson.M{
- "userId": userID,
- "dodoSubscriptionId": activeSub.SubscriptionID,
- "dodoCustomerId": customerID,
- "tier": tier,
- "status": models.SubStatusActive,
- "currentPeriodStart": periodStart,
- "currentPeriodEnd": periodEnd,
- "updatedAt": now,
- },
- "$setOnInsert": bson.M{
- "createdAt": now,
- },
- }
- opts := options.Update().SetUpsert(true)
- _, err = s.subscriptions.UpdateOne(ctx, filter, update, opts)
- if err != nil {
- return nil, fmt.Errorf("failed to update local subscription: %w", err)
- }
- }
-
- // Update user tier
- if s.userService != nil {
- err = s.userService.UpdateSubscriptionWithStatus(ctx, userID, tier, models.SubStatusActive, &periodEnd)
- if err != nil {
- log.Printf("⚠️ Failed to update user subscription: %v", err)
- }
- }
-
- // Invalidate cache
- if s.tierService != nil {
- s.tierService.InvalidateCache(userID)
- }
-
- log.Printf("✅ Synced subscription for user %s: tier=%s, sub_id=%s", userID, tier, activeSub.SubscriptionID)
-
- return map[string]interface{}{
- "status": "synced",
- "tier": tier,
- "subscription_id": activeSub.SubscriptionID,
- "period_end": periodEnd,
- }, nil
-}
-
-// Helper functions
-
-// isUpgrade determines if a tier change is an upgrade
-func isUpgrade(oldTier, newTier string) bool {
- oldRank := models.TierOrder[oldTier]
- newRank := models.TierOrder[newTier]
- return newRank > oldRank
-}
-
-func (s *PaymentService) updateUserDodoCustomer(ctx context.Context, userID, customerID string) error {
- if s.mongoDB == nil {
- return fmt.Errorf("MongoDB not available")
- }
-
- _, err := s.mongoDB.Database().Collection("users").UpdateOne(
- ctx,
- bson.M{"supabaseUserId": userID},
- bson.M{"$set": bson.M{"dodoCustomerId": customerID}},
- )
- return err
-}
-
-func getBaseURL() string {
- if url := os.Getenv("FRONTEND_URL"); url != "" {
- return strings.TrimSuffix(url, "/")
- }
- return "http://localhost:5173"
-}
-
-// UsageStats represents the current usage statistics for a user
-type UsageStats struct {
- Schedules UsageStat `json:"schedules"`
- APIKeys UsageStat `json:"api_keys"`
- ExecutionsToday UsageStat `json:"executions_today"`
- RequestsPerMin UsageStat `json:"requests_per_min"`
- Messages UsageStatWithTime `json:"messages"`
- FileUploads UsageStatWithTime `json:"file_uploads"`
- ImageGenerations UsageStatWithTime `json:"image_generations"`
- MemoryExtractions UsageStatWithTime `json:"memory_extractions"` // Daily memory extraction count
-}
-
-// UsageStat represents a single usage statistic
-type UsageStat struct {
- Current int64 `json:"current"`
- Max int64 `json:"max"`
-}
-
-// UsageStatWithTime represents a usage statistic with reset time
-type UsageStatWithTime struct {
- Current int64 `json:"current"`
- Max int64 `json:"max"`
- ResetAt time.Time `json:"reset_at"`
-}
-
-// GetUsageStats returns the current usage statistics for a user
-func (s *PaymentService) GetUsageStats(ctx context.Context, userID string) (*UsageStats, error) {
- if s.mongoDB == nil || s.tierService == nil {
- return &UsageStats{}, nil
- }
-
- // Get user's tier limits
- limits := s.tierService.GetLimits(ctx, userID)
-
- // Count schedules
- scheduleCount, err := s.mongoDB.Database().Collection("schedules").CountDocuments(ctx, bson.M{"userId": userID})
- if err != nil {
- log.Printf("⚠️ Failed to count schedules for user %s: %v", userID, err)
- scheduleCount = 0
- }
-
- // Count API keys
- apiKeyCount, err := s.mongoDB.Database().Collection("api_keys").CountDocuments(ctx, bson.M{"userId": userID})
- if err != nil {
- log.Printf("⚠️ Failed to count API keys for user %s: %v", userID, err)
- apiKeyCount = 0
- }
-
- // Count executions today (from Redis if available, otherwise from MongoDB)
- executionsToday := int64(0)
- today := time.Now().UTC().Format("2006-01-02")
- startOfDay, _ := time.Parse("2006-01-02", today)
-
- execCount, err := s.mongoDB.Database().Collection("executions").CountDocuments(ctx, bson.M{
- "userId": userID,
- "createdAt": bson.M{
- "$gte": startOfDay,
- },
- })
- if err != nil {
- log.Printf("⚠️ Failed to count executions for user %s: %v", userID, err)
- } else {
- executionsToday = execCount
- }
-
- // Get usage counts and reset times from UsageLimiterService
- var msgCount, fileCount, imageCount int64
- var msgResetAt, fileResetAt, imageResetAt time.Time
-
- if s.usageLimiter != nil {
- limiterStats, err := s.usageLimiter.GetUsageStats(ctx, userID)
- if err == nil {
- msgCount = limiterStats.MessagesUsed
- fileCount = limiterStats.FileUploadsUsed
- imageCount = limiterStats.ImageGensUsed
- msgResetAt = limiterStats.MessageResetAt
- fileResetAt = limiterStats.FileUploadResetAt
- imageResetAt = limiterStats.ImageGenResetAt
- } else {
- log.Printf("⚠️ Failed to get usage limiter stats for user %s: %v", userID, err)
- // Set default reset times
- msgResetAt = time.Now().UTC().AddDate(0, 1, 0)
- fileResetAt = time.Now().UTC().AddDate(0, 0, 1)
- imageResetAt = time.Now().UTC().AddDate(0, 0, 1)
- }
- } else {
- // No usageLimiter available, use default reset times
- msgResetAt = time.Now().UTC().AddDate(0, 1, 0)
- fileResetAt = time.Now().UTC().AddDate(0, 0, 1)
- imageResetAt = time.Now().UTC().AddDate(0, 0, 1)
- }
-
- // Count memory extractions today (completed jobs)
- memoryExtractionCount := int64(0)
- memoryExtractCount, err := s.mongoDB.Database().Collection("memory_extraction_jobs").CountDocuments(ctx, bson.M{
- "userId": userID,
- "status": "completed",
- "processedAt": bson.M{
- "$gte": startOfDay,
- },
- })
- if err != nil {
- log.Printf("⚠️ Failed to count memory extractions for user %s: %v", userID, err)
- } else {
- memoryExtractionCount = memoryExtractCount
- }
-
- // Calculate next reset time for memory extractions (midnight UTC)
- now := time.Now().UTC()
- nextMidnight := time.Date(now.Year(), now.Month(), now.Day()+1, 0, 0, 0, 0, time.UTC)
-
- return &UsageStats{
- Schedules: UsageStat{
- Current: scheduleCount,
- Max: int64(limits.MaxSchedules),
- },
- APIKeys: UsageStat{
- Current: apiKeyCount,
- Max: int64(limits.MaxAPIKeys),
- },
- ExecutionsToday: UsageStat{
- Current: executionsToday,
- Max: limits.MaxExecutionsPerDay,
- },
- RequestsPerMin: UsageStat{
- Current: 0, // This would need real-time rate limiting data
- Max: limits.RequestsPerMinute,
- },
- Messages: UsageStatWithTime{
- Current: msgCount,
- Max: limits.MaxMessagesPerMonth,
- ResetAt: msgResetAt,
- },
- FileUploads: UsageStatWithTime{
- Current: fileCount,
- Max: limits.MaxFileUploadsPerDay,
- ResetAt: fileResetAt,
- },
- ImageGenerations: UsageStatWithTime{
- Current: imageCount,
- Max: limits.MaxImageGensPerDay,
- ResetAt: imageResetAt,
- },
- MemoryExtractions: UsageStatWithTime{
- Current: memoryExtractionCount,
- Max: limits.MaxMemoryExtractionsPerDay,
- ResetAt: nextMidnight,
- },
- }, nil
-}
diff --git a/backend/internal/services/payment_service_test.go b/backend/internal/services/payment_service_test.go
deleted file mode 100644
index fa21f3f5..00000000
--- a/backend/internal/services/payment_service_test.go
+++ /dev/null
@@ -1,264 +0,0 @@
-package services
-
-import (
- "claraverse/internal/models"
- "context"
- "crypto/hmac"
- "crypto/sha256"
- "encoding/hex"
- "testing"
-)
-
-func TestNewPaymentService(t *testing.T) {
- service := NewPaymentService("test_key", "webhook_secret", "business_id", nil, nil, nil, nil)
- if service == nil {
- t.Fatal("Expected non-nil payment service")
- }
-}
-
-func TestPaymentService_DetermineChangeType(t *testing.T) {
- service := NewPaymentService("", "", "", nil, nil, nil, nil)
-
- tests := []struct {
- name string
- fromTier string
- toTier string
- isUpgrade bool
- isDowngrade bool
- }{
- {"free to pro", models.TierFree, models.TierPro, true, false},
- {"pro to max", models.TierPro, models.TierMax, true, false},
- {"max to pro", models.TierMax, models.TierPro, false, true},
- {"pro to free", models.TierPro, models.TierFree, false, true},
- {"same tier", models.TierPro, models.TierPro, false, false},
- }
-
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- isUpgrade, isDowngrade := service.DetermineChangeType(tt.fromTier, tt.toTier)
- if isUpgrade != tt.isUpgrade {
- t.Errorf("isUpgrade = %v, want %v", isUpgrade, tt.isUpgrade)
- }
- if isDowngrade != tt.isDowngrade {
- t.Errorf("isDowngrade = %v, want %v", isDowngrade, tt.isDowngrade)
- }
- })
- }
-}
-
-func TestPaymentService_VerifyWebhookSignature(t *testing.T) {
- secret := "test_webhook_secret"
- service := NewPaymentService("", secret, "", nil, nil, nil, nil)
-
- payload := []byte(`{"type":"subscription.active","data":{}}`)
-
- // Generate valid signature
- mac := hmac.New(sha256.New, []byte(secret))
- mac.Write(payload)
- validSig := hex.EncodeToString(mac.Sum(nil))
-
- tests := []struct {
- name string
- signature string
- expectErr bool
- }{
- {"valid signature", validSig, false},
- {"invalid signature", "invalid_sig", true},
- {"empty signature", "", true},
- {"wrong signature", "abcd1234", true},
- }
-
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- err := service.VerifyWebhook(payload, tt.signature)
- if tt.expectErr && err == nil {
- t.Error("Expected error but got nil")
- }
- if !tt.expectErr && err != nil {
- t.Errorf("Expected no error but got: %v", err)
- }
- })
- }
-}
-
-func TestPaymentService_VerifyWebhook_NoSecret(t *testing.T) {
- service := NewPaymentService("", "", "", nil, nil, nil, nil)
- payload := []byte(`{"type":"subscription.active"}`)
-
- err := service.VerifyWebhook(payload, "signature")
- if err == nil {
- t.Error("Expected error when webhook secret is not configured")
- }
-}
-
-func TestPaymentService_CalculateProration(t *testing.T) {
- service := NewPaymentService("", "", "", nil, nil, nil, nil)
-
- tests := []struct {
- name string
- fromPrice int64 // cents
- toPrice int64 // cents
- daysRemaining int
- totalDays int
- expectedCharge int64
- }{
- {
- name: "upgrade mid-month",
- fromPrice: 1000, // $10/month
- toPrice: 2000, // $20/month
- daysRemaining: 15,
- totalDays: 30,
- expectedCharge: 500, // $5 (half month difference)
- },
- {
- name: "upgrade near end",
- fromPrice: 1000,
- toPrice: 2000,
- daysRemaining: 3,
- totalDays: 30,
- expectedCharge: 100, // ~$1
- },
- {
- name: "downgrade mid-month",
- fromPrice: 2000,
- toPrice: 1000,
- daysRemaining: 15,
- totalDays: 30,
- expectedCharge: -500, // Credit
- },
- {
- name: "zero days remaining",
- fromPrice: 1000,
- toPrice: 2000,
- daysRemaining: 0,
- totalDays: 30,
- expectedCharge: 0,
- },
- }
-
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- charge := service.CalculateProration(
- tt.fromPrice, tt.toPrice,
- tt.daysRemaining, tt.totalDays,
- )
- // Allow 10% variance for rounding
- variance := float64(tt.expectedCharge) * 0.1
- if tt.expectedCharge < 0 {
- variance = -variance
- }
- if float64(charge) < float64(tt.expectedCharge)-variance ||
- float64(charge) > float64(tt.expectedCharge)+variance {
- t.Errorf("Proration = %d, want ~%d", charge, tt.expectedCharge)
- }
- })
- }
-}
-
-func TestPaymentService_GetAvailablePlans(t *testing.T) {
- service := NewPaymentService("", "", "", nil, nil, nil, nil)
-
- plans := service.GetAvailablePlans()
-
- // Should have free, pro, max, enterprise
- if len(plans) < 4 {
- t.Errorf("Expected at least 4 plans, got %d", len(plans))
- }
-
- // Verify enterprise has contact_sales flag
- var enterprisePlan *models.Plan
- for i := range plans {
- if plans[i].Tier == models.TierEnterprise {
- enterprisePlan = &plans[i]
- break
- }
- }
-
- if enterprisePlan == nil {
- t.Fatal("Enterprise plan not found")
- }
-
- if !enterprisePlan.ContactSales {
- t.Error("Enterprise plan should have ContactSales=true")
- }
- if enterprisePlan.PriceMonthly != 0 {
- t.Error("Enterprise plan should have 0 price (contact sales)")
- }
-
- // Verify pricing order: free < pro < max
- var freePlan, proPlan, maxPlan *models.Plan
- for i := range plans {
- switch plans[i].Tier {
- case models.TierFree:
- freePlan = &plans[i]
- case models.TierPro:
- proPlan = &plans[i]
- case models.TierMax:
- maxPlan = &plans[i]
- }
- }
-
- if freePlan == nil || proPlan == nil || maxPlan == nil {
- t.Fatal("Missing required plans")
- }
-
- if freePlan.PriceMonthly != 0 {
- t.Error("Free plan should be $0")
- }
- if proPlan.PriceMonthly >= maxPlan.PriceMonthly {
- t.Error("Pro+ should be more expensive than Pro")
- }
-}
-
-func TestPaymentService_GetCurrentSubscription_NoMongoDB(t *testing.T) {
- service := NewPaymentService("", "", "", nil, nil, nil, nil)
- ctx := context.Background()
-
- sub, err := service.GetCurrentSubscription(ctx, "user-123")
- if err != nil {
- t.Fatalf("Expected no error, got: %v", err)
- }
-
- if sub == nil {
- t.Fatal("Expected subscription, got nil")
- }
-
- if sub.Tier != models.TierFree {
- t.Errorf("Expected free tier, got %s", sub.Tier)
- }
-
- if sub.Status != models.SubStatusActive {
- t.Errorf("Expected active status, got %s", sub.Status)
- }
-}
-
-func TestPaymentService_PreviewPlanChange(t *testing.T) {
- service := NewPaymentService("", "", "", nil, nil, nil, nil)
- ctx := context.Background()
-
- tests := []struct {
- name string
- currentTier string
- newPlanID string
- expectError bool
- }{
- {"free to pro", models.TierFree, "pro", false},
- {"pro to max", models.TierPro, "max", false},
- {"max to pro", models.TierMax, "pro", false},
- {"invalid plan", models.TierFree, "invalid", true},
- {"same tier", models.TierFree, "free", true}, // Default tier is free without MongoDB
- }
-
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- // This test would need MongoDB to set up current subscription
- // For now, just test the error cases
- if tt.expectError {
- _, err := service.PreviewPlanChange(ctx, "user-123", tt.newPlanID)
- if err == nil {
- t.Error("Expected error but got nil")
- }
- }
- })
- }
-}
diff --git a/backend/internal/services/provider_service.go b/backend/internal/services/provider_service.go
deleted file mode 100644
index 71d6c5f1..00000000
--- a/backend/internal/services/provider_service.go
+++ /dev/null
@@ -1,379 +0,0 @@
-package services
-
-import (
- "claraverse/internal/database"
- "claraverse/internal/models"
- "database/sql"
- "fmt"
- "log"
- "path/filepath"
- "strings"
-)
-
-// ProviderService handles provider operations
-type ProviderService struct {
- db *database.DB
-}
-
-// NewProviderService creates a new provider service
-func NewProviderService(db *database.DB) *ProviderService {
- return &ProviderService{db: db}
-}
-
-// GetAll returns all enabled providers
-func (s *ProviderService) GetAll() ([]models.Provider, error) {
- rows, err := s.db.Query(`
- SELECT id, name, base_url, api_key, enabled, audio_only, image_only, image_edit_only, secure, default_model, system_prompt, favicon, created_at, updated_at
- FROM providers
- WHERE enabled = 1
- ORDER BY name
- `)
- if err != nil {
- return nil, fmt.Errorf("failed to query providers: %w", err)
- }
- defer rows.Close()
-
- var providers []models.Provider
- for rows.Next() {
- var p models.Provider
- var systemPrompt, favicon, defaultModel sql.NullString
- if err := rows.Scan(&p.ID, &p.Name, &p.BaseURL, &p.APIKey, &p.Enabled, &p.AudioOnly, &p.ImageOnly, &p.ImageEditOnly, &p.Secure, &defaultModel, &systemPrompt, &favicon, &p.CreatedAt, &p.UpdatedAt); err != nil {
- return nil, fmt.Errorf("failed to scan provider: %w", err)
- }
- if systemPrompt.Valid {
- p.SystemPrompt = systemPrompt.String
- }
- if favicon.Valid {
- p.Favicon = favicon.String
- }
- if defaultModel.Valid {
- p.DefaultModel = defaultModel.String
- }
- providers = append(providers, p)
- }
-
- return providers, nil
-}
-
-// GetAllForModels returns all enabled providers that are NOT audio-only (for model selection)
-func (s *ProviderService) GetAllForModels() ([]models.Provider, error) {
- rows, err := s.db.Query(`
- SELECT id, name, base_url, api_key, enabled, audio_only, image_only, image_edit_only, secure, default_model, system_prompt, favicon, created_at, updated_at
- FROM providers
- WHERE enabled = 1 AND (audio_only = 0 OR audio_only IS NULL) AND (image_only = 0 OR image_only IS NULL) AND (image_edit_only = 0 OR image_edit_only IS NULL)
- ORDER BY name
- `)
- if err != nil {
- return nil, fmt.Errorf("failed to query providers: %w", err)
- }
- defer rows.Close()
-
- var providers []models.Provider
- for rows.Next() {
- var p models.Provider
- var systemPrompt, favicon, defaultModel sql.NullString
- if err := rows.Scan(&p.ID, &p.Name, &p.BaseURL, &p.APIKey, &p.Enabled, &p.AudioOnly, &p.ImageOnly, &p.ImageEditOnly, &p.Secure, &defaultModel, &systemPrompt, &favicon, &p.CreatedAt, &p.UpdatedAt); err != nil {
- return nil, fmt.Errorf("failed to scan provider: %w", err)
- }
- if systemPrompt.Valid {
- p.SystemPrompt = systemPrompt.String
- }
- if favicon.Valid {
- p.Favicon = favicon.String
- }
- if defaultModel.Valid {
- p.DefaultModel = defaultModel.String
- }
- providers = append(providers, p)
- }
-
- return providers, nil
-}
-
-// GetByID returns a provider by ID
-func (s *ProviderService) GetByID(id int) (*models.Provider, error) {
- var p models.Provider
- var systemPrompt, favicon, defaultModel sql.NullString
- err := s.db.QueryRow(`
- SELECT id, name, base_url, api_key, enabled, audio_only, image_only, image_edit_only, secure, default_model, system_prompt, favicon, created_at, updated_at
- FROM providers
- WHERE id = ?
- `, id).Scan(&p.ID, &p.Name, &p.BaseURL, &p.APIKey, &p.Enabled, &p.AudioOnly, &p.ImageOnly, &p.ImageEditOnly, &p.Secure, &defaultModel, &systemPrompt, &favicon, &p.CreatedAt, &p.UpdatedAt)
-
- if err == sql.ErrNoRows {
- return nil, fmt.Errorf("provider not found")
- }
- if err != nil {
- return nil, fmt.Errorf("failed to query provider: %w", err)
- }
-
- if systemPrompt.Valid {
- p.SystemPrompt = systemPrompt.String
- }
- if favicon.Valid {
- p.Favicon = favicon.String
- }
- if defaultModel.Valid {
- p.DefaultModel = defaultModel.String
- }
-
- return &p, nil
-}
-
-// GetByName returns a provider by name
-func (s *ProviderService) GetByName(name string) (*models.Provider, error) {
- var p models.Provider
- var systemPrompt, favicon, defaultModel sql.NullString
- err := s.db.QueryRow(`
- SELECT id, name, base_url, api_key, enabled, audio_only, image_only, image_edit_only, secure, default_model, system_prompt, favicon, created_at, updated_at
- FROM providers
- WHERE name = ?
- `, name).Scan(&p.ID, &p.Name, &p.BaseURL, &p.APIKey, &p.Enabled, &p.AudioOnly, &p.ImageOnly, &p.ImageEditOnly, &p.Secure, &defaultModel, &systemPrompt, &favicon, &p.CreatedAt, &p.UpdatedAt)
-
- if err == sql.ErrNoRows {
- return nil, nil // Not found, not an error
- }
- if err != nil {
- return nil, fmt.Errorf("failed to query provider: %w", err)
- }
-
- if systemPrompt.Valid {
- p.SystemPrompt = systemPrompt.String
- }
- if favicon.Valid {
- p.Favicon = favicon.String
- }
- if defaultModel.Valid {
- p.DefaultModel = defaultModel.String
- }
-
- return &p, nil
-}
-
-// Create creates a new provider
-func (s *ProviderService) Create(config models.ProviderConfig) (*models.Provider, error) {
- result, err := s.db.Exec(`
- INSERT INTO providers (name, base_url, api_key, enabled, audio_only, image_only, image_edit_only, default_model, system_prompt, favicon)
- VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
- `, config.Name, config.BaseURL, config.APIKey, config.Enabled, config.AudioOnly, config.ImageOnly, config.ImageEditOnly, config.DefaultModel, config.SystemPrompt, config.Favicon)
-
- if err != nil {
- return nil, fmt.Errorf("failed to create provider: %w", err)
- }
-
- id, err := result.LastInsertId()
- if err != nil {
- return nil, fmt.Errorf("failed to get inserted ID: %w", err)
- }
-
- log.Printf(" ✅ Created provider %s with ID %d", config.Name, id)
- return s.GetByID(int(id))
-}
-
-// Update updates an existing provider
-func (s *ProviderService) Update(id int, config models.ProviderConfig) error {
- _, err := s.db.Exec(`
- UPDATE providers
- SET base_url = ?, api_key = ?, enabled = ?, audio_only = ?, image_only = ?, image_edit_only = ?,
- default_model = ?, system_prompt = ?, favicon = ?, updated_at = CURRENT_TIMESTAMP
- WHERE id = ?
- `, config.BaseURL, config.APIKey, config.Enabled, config.AudioOnly, config.ImageOnly, config.ImageEditOnly,
- config.DefaultModel, config.SystemPrompt, config.Favicon, id)
-
- if err != nil {
- return fmt.Errorf("failed to update provider: %w", err)
- }
-
- log.Printf(" ✅ Updated provider %s (ID %d)", config.Name, id)
- return nil
-}
-
-// SyncFilters syncs filter configuration for a provider
-func (s *ProviderService) SyncFilters(providerID int, filters []models.FilterConfig) error {
- // Delete old filters
- if _, err := s.db.Exec("DELETE FROM provider_model_filters WHERE provider_id = ?", providerID); err != nil {
- return fmt.Errorf("failed to delete old filters: %w", err)
- }
-
- // Insert new filters
- for _, filter := range filters {
- _, err := s.db.Exec(`
- INSERT INTO provider_model_filters (provider_id, model_pattern, action, priority)
- VALUES (?, ?, ?, ?)
- `, providerID, filter.Pattern, filter.Action, filter.Priority)
-
- if err != nil {
- return fmt.Errorf("failed to insert filter: %w", err)
- }
-
- log.Printf(" ✓ Added filter: %s (%s)", filter.Pattern, filter.Action)
- }
-
- return nil
-}
-
-// ApplyFilters applies filter rules to models for a provider
-func (s *ProviderService) ApplyFilters(providerID int) error {
- // Get filters for this provider ordered by priority (higher first)
- rows, err := s.db.Query(`
- SELECT model_pattern, action
- FROM provider_model_filters
- WHERE provider_id = ?
- ORDER BY priority DESC, id ASC
- `, providerID)
- if err != nil {
- return fmt.Errorf("failed to query filters: %w", err)
- }
- defer rows.Close()
-
- var filters []struct {
- Pattern string
- Action string
- }
-
- for rows.Next() {
- var f struct {
- Pattern string
- Action string
- }
- if err := rows.Scan(&f.Pattern, &f.Action); err != nil {
- return fmt.Errorf("failed to scan filter: %w", err)
- }
- filters = append(filters, f)
- }
-
- if len(filters) == 0 {
- // No filters, show all models
- _, err := s.db.Exec(`
- UPDATE models
- SET is_visible = 1
- WHERE provider_id = ?
- `, providerID)
- return err
- }
-
- // Reset visibility
- if _, err := s.db.Exec("UPDATE models SET is_visible = 0 WHERE provider_id = ?", providerID); err != nil {
- return fmt.Errorf("failed to reset visibility: %w", err)
- }
-
- // Apply filters
- for _, filter := range filters {
- if filter.Action == "include" {
- // Match pattern using SQL LIKE (convert * to %)
- pattern := strings.ReplaceAll(filter.Pattern, "*", "%")
- _, err := s.db.Exec(`
- UPDATE models
- SET is_visible = 1
- WHERE provider_id = ? AND (name LIKE ? OR id LIKE ?)
- `, providerID, pattern, pattern)
- if err != nil {
- return fmt.Errorf("failed to apply include filter: %w", err)
- }
- } else if filter.Action == "exclude" {
- pattern := strings.ReplaceAll(filter.Pattern, "*", "%")
- _, err := s.db.Exec(`
- UPDATE models
- SET is_visible = 0
- WHERE provider_id = ? AND (name LIKE ? OR id LIKE ?)
- `, providerID, pattern, pattern)
- if err != nil {
- return fmt.Errorf("failed to apply exclude filter: %w", err)
- }
- }
- }
-
- return nil
-}
-
-// matchesPattern checks if a model name matches a wildcard pattern
-func matchesPattern(name, pattern string) bool {
- matched, _ := filepath.Match(pattern, name)
- return matched
-}
-
-// GetByModelID returns the provider associated with a given model ID
-func (s *ProviderService) GetByModelID(modelID string) (*models.Provider, error) {
- var providerID int
- err := s.db.QueryRow(`
- SELECT provider_id FROM models WHERE id = ?
- `, modelID).Scan(&providerID)
-
- if err == sql.ErrNoRows {
- return nil, fmt.Errorf("model not found: %s", modelID)
- }
- if err != nil {
- return nil, fmt.Errorf("failed to query model: %w", err)
- }
-
- return s.GetByID(providerID)
-}
-
-// GetAllIncludingDisabled returns all providers including disabled ones
-func (s *ProviderService) GetAllIncludingDisabled() ([]models.Provider, error) {
- rows, err := s.db.Query(`
- SELECT id, name, base_url, api_key, enabled, audio_only, image_only, image_edit_only, secure, default_model, system_prompt, favicon, created_at, updated_at
- FROM providers
- ORDER BY name
- `)
- if err != nil {
- return nil, fmt.Errorf("failed to query providers: %w", err)
- }
- defer rows.Close()
-
- var providers []models.Provider
- for rows.Next() {
- var p models.Provider
- var systemPrompt, favicon, defaultModel sql.NullString
- if err := rows.Scan(&p.ID, &p.Name, &p.BaseURL, &p.APIKey, &p.Enabled, &p.AudioOnly, &p.ImageOnly, &p.ImageEditOnly, &p.Secure, &defaultModel, &systemPrompt, &favicon, &p.CreatedAt, &p.UpdatedAt); err != nil {
- return nil, fmt.Errorf("failed to scan provider: %w", err)
- }
- if systemPrompt.Valid {
- p.SystemPrompt = systemPrompt.String
- }
- if favicon.Valid {
- p.Favicon = favicon.String
- }
- if defaultModel.Valid {
- p.DefaultModel = defaultModel.String
- }
- providers = append(providers, p)
- }
-
- return providers, nil
-}
-
-// Delete removes a provider and all its associated models from the database
-func (s *ProviderService) Delete(id int) error {
- // Models are deleted automatically via ON DELETE CASCADE
- _, err := s.db.Exec(`DELETE FROM providers WHERE id = ?`, id)
- if err != nil {
- return fmt.Errorf("failed to delete provider: %w", err)
- }
- return nil
-}
-
-// GetFilters retrieves all filters for a provider
-func (s *ProviderService) GetFilters(providerID int) ([]models.FilterConfig, error) {
- rows, err := s.db.Query(`
- SELECT model_pattern, action, priority
- FROM provider_model_filters
- WHERE provider_id = ?
- ORDER BY priority DESC, id ASC
- `, providerID)
- if err != nil {
- return nil, fmt.Errorf("failed to query filters: %w", err)
- }
- defer rows.Close()
-
- var filters []models.FilterConfig
- for rows.Next() {
- var f models.FilterConfig
- if err := rows.Scan(&f.Pattern, &f.Action, &f.Priority); err != nil {
- return nil, fmt.Errorf("failed to scan filter: %w", err)
- }
- filters = append(filters, f)
- }
-
- return filters, nil
-}
diff --git a/backend/internal/services/provider_service_test.go b/backend/internal/services/provider_service_test.go
deleted file mode 100644
index 4ea27e80..00000000
--- a/backend/internal/services/provider_service_test.go
+++ /dev/null
@@ -1,470 +0,0 @@
-package services
-
-import (
- "claraverse/internal/database"
- "claraverse/internal/models"
- "os"
- "testing"
-)
-
-func setupTestDB(t *testing.T) (*database.DB, func()) {
- t.Skip("SQLite tests are deprecated - please use DATABASE_URL with MySQL DSN")
- tmpFile := "test_provider_service.db"
- db, err := database.New(tmpFile)
- if err != nil {
- t.Fatalf("Failed to create test database: %v", err)
- }
-
- if err := db.Initialize(); err != nil {
- t.Fatalf("Failed to initialize test database: %v", err)
- }
-
- cleanup := func() {
- db.Close()
- os.Remove(tmpFile)
- }
-
- return db, cleanup
-}
-
-func TestNewProviderService(t *testing.T) {
- db, cleanup := setupTestDB(t)
- defer cleanup()
-
- service := NewProviderService(db)
- if service == nil {
- t.Fatal("Expected non-nil provider service")
- }
-}
-
-func TestProviderService_Create(t *testing.T) {
- db, cleanup := setupTestDB(t)
- defer cleanup()
-
- service := NewProviderService(db)
-
- config := models.ProviderConfig{
- Name: "Test Provider",
- BaseURL: "https://api.test.com/v1",
- APIKey: "test-key-123",
- Enabled: true,
- }
-
- provider, err := service.Create(config)
- if err != nil {
- t.Fatalf("Failed to create provider: %v", err)
- }
-
- if provider.Name != config.Name {
- t.Errorf("Expected name %s, got %s", config.Name, provider.Name)
- }
-
- if provider.BaseURL != config.BaseURL {
- t.Errorf("Expected base URL %s, got %s", config.BaseURL, provider.BaseURL)
- }
-
- if provider.APIKey != config.APIKey {
- t.Errorf("Expected API key %s, got %s", config.APIKey, provider.APIKey)
- }
-
- if !provider.Enabled {
- t.Error("Expected provider to be enabled")
- }
-}
-
-func TestProviderService_GetAll(t *testing.T) {
- db, cleanup := setupTestDB(t)
- defer cleanup()
-
- service := NewProviderService(db)
-
- // Create test providers
- configs := []models.ProviderConfig{
- {Name: "Provider A", BaseURL: "https://a.com", APIKey: "key-a", Enabled: true},
- {Name: "Provider B", BaseURL: "https://b.com", APIKey: "key-b", Enabled: true},
- {Name: "Provider C", BaseURL: "https://c.com", APIKey: "key-c", Enabled: false},
- }
-
- for _, config := range configs {
- if _, err := service.Create(config); err != nil {
- t.Fatalf("Failed to create provider: %v", err)
- }
- }
-
- providers, err := service.GetAll()
- if err != nil {
- t.Fatalf("Failed to get all providers: %v", err)
- }
-
- // Should only return enabled providers
- if len(providers) != 2 {
- t.Errorf("Expected 2 enabled providers, got %d", len(providers))
- }
-
- // Verify order (alphabetical)
- if providers[0].Name != "Provider A" {
- t.Errorf("Expected first provider to be 'Provider A', got %s", providers[0].Name)
- }
-}
-
-func TestProviderService_GetByID(t *testing.T) {
- db, cleanup := setupTestDB(t)
- defer cleanup()
-
- service := NewProviderService(db)
-
- config := models.ProviderConfig{
- Name: "Test Provider",
- BaseURL: "https://api.test.com/v1",
- APIKey: "test-key",
- Enabled: true,
- }
-
- created, err := service.Create(config)
- if err != nil {
- t.Fatalf("Failed to create provider: %v", err)
- }
-
- // Get by ID
- provider, err := service.GetByID(created.ID)
- if err != nil {
- t.Fatalf("Failed to get provider by ID: %v", err)
- }
-
- if provider.ID != created.ID {
- t.Errorf("Expected ID %d, got %d", created.ID, provider.ID)
- }
-
- if provider.Name != config.Name {
- t.Errorf("Expected name %s, got %s", config.Name, provider.Name)
- }
-}
-
-func TestProviderService_GetByID_NotFound(t *testing.T) {
- db, cleanup := setupTestDB(t)
- defer cleanup()
-
- service := NewProviderService(db)
-
- _, err := service.GetByID(999)
- if err == nil {
- t.Error("Expected error for non-existent provider, got nil")
- }
-}
-
-func TestProviderService_GetByName(t *testing.T) {
- db, cleanup := setupTestDB(t)
- defer cleanup()
-
- service := NewProviderService(db)
-
- config := models.ProviderConfig{
- Name: "Test Provider",
- BaseURL: "https://api.test.com/v1",
- APIKey: "test-key",
- Enabled: true,
- }
-
- created, err := service.Create(config)
- if err != nil {
- t.Fatalf("Failed to create provider: %v", err)
- }
-
- // Get by name
- provider, err := service.GetByName("Test Provider")
- if err != nil {
- t.Fatalf("Failed to get provider by name: %v", err)
- }
-
- if provider == nil {
- t.Fatal("Expected provider, got nil")
- }
-
- if provider.ID != created.ID {
- t.Errorf("Expected ID %d, got %d", created.ID, provider.ID)
- }
-}
-
-func TestProviderService_GetByName_NotFound(t *testing.T) {
- db, cleanup := setupTestDB(t)
- defer cleanup()
-
- service := NewProviderService(db)
-
- provider, err := service.GetByName("Non-existent Provider")
- if err != nil {
- t.Fatalf("Expected no error for non-existent provider, got: %v", err)
- }
-
- if provider != nil {
- t.Error("Expected nil provider for non-existent name")
- }
-}
-
-func TestProviderService_Update(t *testing.T) {
- db, cleanup := setupTestDB(t)
- defer cleanup()
-
- service := NewProviderService(db)
-
- config := models.ProviderConfig{
- Name: "Test Provider",
- BaseURL: "https://api.test.com/v1",
- APIKey: "test-key",
- Enabled: true,
- }
-
- created, err := service.Create(config)
- if err != nil {
- t.Fatalf("Failed to create provider: %v", err)
- }
-
- // Update provider
- updateConfig := models.ProviderConfig{
- Name: "Test Provider",
- BaseURL: "https://api.updated.com/v2",
- APIKey: "updated-key",
- Enabled: false,
- }
-
- if err := service.Update(created.ID, updateConfig); err != nil {
- t.Fatalf("Failed to update provider: %v", err)
- }
-
- // Verify update
- updated, err := service.GetByID(created.ID)
- if err != nil {
- t.Fatalf("Failed to get updated provider: %v", err)
- }
-
- if updated.BaseURL != updateConfig.BaseURL {
- t.Errorf("Expected base URL %s, got %s", updateConfig.BaseURL, updated.BaseURL)
- }
-
- if updated.APIKey != updateConfig.APIKey {
- t.Errorf("Expected API key %s, got %s", updateConfig.APIKey, updated.APIKey)
- }
-
- if updated.Enabled != updateConfig.Enabled {
- t.Errorf("Expected enabled %v, got %v", updateConfig.Enabled, updated.Enabled)
- }
-}
-
-func TestProviderService_SyncFilters(t *testing.T) {
- db, cleanup := setupTestDB(t)
- defer cleanup()
-
- service := NewProviderService(db)
-
- config := models.ProviderConfig{
- Name: "Test Provider",
- BaseURL: "https://api.test.com/v1",
- APIKey: "test-key",
- Enabled: true,
- }
-
- created, err := service.Create(config)
- if err != nil {
- t.Fatalf("Failed to create provider: %v", err)
- }
-
- filters := []models.FilterConfig{
- {Pattern: "gpt-4*", Action: "include", Priority: 10},
- {Pattern: "gpt-3.5*", Action: "include", Priority: 5},
- {Pattern: "*preview*", Action: "exclude", Priority: 1},
- }
-
- if err := service.SyncFilters(created.ID, filters); err != nil {
- t.Fatalf("Failed to sync filters: %v", err)
- }
-
- // Verify filters were inserted
- var count int
- err = db.QueryRow("SELECT COUNT(*) FROM provider_model_filters WHERE provider_id = ?", created.ID).Scan(&count)
- if err != nil {
- t.Fatalf("Failed to count filters: %v", err)
- }
-
- if count != len(filters) {
- t.Errorf("Expected %d filters, got %d", len(filters), count)
- }
-}
-
-func TestProviderService_SyncFilters_Replace(t *testing.T) {
- db, cleanup := setupTestDB(t)
- defer cleanup()
-
- service := NewProviderService(db)
-
- config := models.ProviderConfig{
- Name: "Test Provider",
- BaseURL: "https://api.test.com/v1",
- APIKey: "test-key",
- Enabled: true,
- }
-
- created, err := service.Create(config)
- if err != nil {
- t.Fatalf("Failed to create provider: %v", err)
- }
-
- // Add initial filters
- initialFilters := []models.FilterConfig{
- {Pattern: "gpt-4*", Action: "include", Priority: 10},
- }
-
- if err := service.SyncFilters(created.ID, initialFilters); err != nil {
- t.Fatalf("Failed to sync initial filters: %v", err)
- }
-
- // Update with new filters
- newFilters := []models.FilterConfig{
- {Pattern: "claude*", Action: "include", Priority: 15},
- {Pattern: "opus*", Action: "include", Priority: 20},
- }
-
- if err := service.SyncFilters(created.ID, newFilters); err != nil {
- t.Fatalf("Failed to sync new filters: %v", err)
- }
-
- // Verify only new filters exist
- var count int
- err = db.QueryRow("SELECT COUNT(*) FROM provider_model_filters WHERE provider_id = ?", created.ID).Scan(&count)
- if err != nil {
- t.Fatalf("Failed to count filters: %v", err)
- }
-
- if count != len(newFilters) {
- t.Errorf("Expected %d filters, got %d", len(newFilters), count)
- }
-}
-
-func TestProviderService_ApplyFilters(t *testing.T) {
- db, cleanup := setupTestDB(t)
- defer cleanup()
-
- service := NewProviderService(db)
-
- // Create provider
- config := models.ProviderConfig{
- Name: "Test Provider",
- BaseURL: "https://api.test.com/v1",
- APIKey: "test-key",
- Enabled: true,
- }
-
- provider, err := service.Create(config)
- if err != nil {
- t.Fatalf("Failed to create provider: %v", err)
- }
-
- // Create test models
- testModels := []models.Model{
- {ID: "gpt-4-turbo", ProviderID: provider.ID, Name: "gpt-4-turbo", IsVisible: false},
- {ID: "gpt-4-preview", ProviderID: provider.ID, Name: "gpt-4-preview", IsVisible: false},
- {ID: "gpt-3.5-turbo", ProviderID: provider.ID, Name: "gpt-3.5-turbo", IsVisible: false},
- {ID: "claude-3", ProviderID: provider.ID, Name: "claude-3", IsVisible: false},
- }
-
- for _, model := range testModels {
- _, err := db.Exec(`
- INSERT INTO models (id, provider_id, name, is_visible)
- VALUES (?, ?, ?, ?)
- `, model.ID, model.ProviderID, model.Name, model.IsVisible)
- if err != nil {
- t.Fatalf("Failed to create test model: %v", err)
- }
- }
-
- // Set up filters
- filters := []models.FilterConfig{
- {Pattern: "gpt-4*", Action: "include", Priority: 10},
- {Pattern: "*preview*", Action: "exclude", Priority: 5},
- }
-
- if err := service.SyncFilters(provider.ID, filters); err != nil {
- t.Fatalf("Failed to sync filters: %v", err)
- }
-
- // Apply filters
- if err := service.ApplyFilters(provider.ID); err != nil {
- t.Fatalf("Failed to apply filters: %v", err)
- }
-
- // Verify visibility
- // gpt-4-turbo should be visible (matches include, not excluded)
- // gpt-4-preview should be hidden (matches exclude)
- // gpt-3.5-turbo should be hidden (doesn't match any include)
- // claude-3 should be hidden (doesn't match any include)
-
- var visibleCount int
- err = db.QueryRow(`
- SELECT COUNT(*) FROM models
- WHERE provider_id = ? AND is_visible = 1
- `, provider.ID).Scan(&visibleCount)
- if err != nil {
- t.Fatalf("Failed to count visible models: %v", err)
- }
-
- if visibleCount != 1 {
- t.Errorf("Expected 1 visible model, got %d", visibleCount)
- }
-
- // Check specific model
- var isVisible bool
- err = db.QueryRow(`
- SELECT is_visible FROM models WHERE id = ?
- `, "gpt-4-turbo").Scan(&isVisible)
- if err != nil {
- t.Fatalf("Failed to get model visibility: %v", err)
- }
-
- if !isVisible {
- t.Error("Expected gpt-4-turbo to be visible")
- }
-}
-
-func TestProviderService_ApplyFilters_NoFilters(t *testing.T) {
- db, cleanup := setupTestDB(t)
- defer cleanup()
-
- service := NewProviderService(db)
-
- // Create provider
- config := models.ProviderConfig{
- Name: "Test Provider",
- BaseURL: "https://api.test.com/v1",
- APIKey: "test-key",
- Enabled: true,
- }
-
- provider, err := service.Create(config)
- if err != nil {
- t.Fatalf("Failed to create provider: %v", err)
- }
-
- // Create test model
- _, err = db.Exec(`
- INSERT INTO models (id, provider_id, name, is_visible)
- VALUES (?, ?, ?, ?)
- `, "test-model", provider.ID, "test-model", false)
- if err != nil {
- t.Fatalf("Failed to create test model: %v", err)
- }
-
- // Apply filters (no filters configured)
- if err := service.ApplyFilters(provider.ID); err != nil {
- t.Fatalf("Failed to apply filters: %v", err)
- }
-
- // All models should be visible when no filters exist
- var isVisible bool
- err = db.QueryRow("SELECT is_visible FROM models WHERE id = ?", "test-model").Scan(&isVisible)
- if err != nil {
- t.Fatalf("Failed to get model visibility: %v", err)
- }
-
- if !isVisible {
- t.Error("Expected model to be visible when no filters configured")
- }
-}
diff --git a/backend/internal/services/pubsub_service.go b/backend/internal/services/pubsub_service.go
deleted file mode 100644
index 7cd48310..00000000
--- a/backend/internal/services/pubsub_service.go
+++ /dev/null
@@ -1,242 +0,0 @@
-package services
-
-import (
- "context"
- "encoding/json"
- "log"
- "sync"
-
- "github.com/redis/go-redis/v9"
-)
-
-// PubSubService manages Redis pub/sub for cross-instance communication
-type PubSubService struct {
- redis *RedisService
- pubsub *redis.PubSub
- handlers map[string][]MessageHandler
- mu sync.RWMutex
- instanceID string
- ctx context.Context
- cancel context.CancelFunc
-}
-
-// MessageHandler is a callback for handling pub/sub messages
-type MessageHandler func(channel string, message *PubSubMessage)
-
-// PubSubMessage represents a message sent via pub/sub
-type PubSubMessage struct {
- Type string `json:"type"` // Message type (e.g., "execution_update", "agent_status")
- UserID string `json:"userId"` // Target user ID
- AgentID string `json:"agentId,omitempty"`
- InstanceID string `json:"instanceId"` // Source instance ID
- Payload map[string]interface{} `json:"payload"` // Message payload
-}
-
-// NewPubSubService creates a new pub/sub service
-func NewPubSubService(redisService *RedisService, instanceID string) *PubSubService {
- ctx, cancel := context.WithCancel(context.Background())
- return &PubSubService{
- redis: redisService,
- handlers: make(map[string][]MessageHandler),
- instanceID: instanceID,
- ctx: ctx,
- cancel: cancel,
- }
-}
-
-// Subscribe subscribes to a channel pattern
-func (s *PubSubService) Subscribe(pattern string, handler MessageHandler) {
- s.mu.Lock()
- defer s.mu.Unlock()
-
- s.handlers[pattern] = append(s.handlers[pattern], handler)
- log.Printf("📡 [PUBSUB] Subscribed to pattern: %s", pattern)
-}
-
-// Start begins listening for pub/sub messages
-func (s *PubSubService) Start() error {
- client := s.redis.Client()
-
- // Subscribe to all user channels and global channels
- s.pubsub = client.PSubscribe(s.ctx,
- "user:*:events", // User-specific events
- "agent:*:events", // Agent-specific events
- "broadcast:*", // Global broadcast
- )
-
- // Wait for subscription confirmation
- _, err := s.pubsub.Receive(s.ctx)
- if err != nil {
- return err
- }
-
- // Start message processor
- go s.processMessages()
-
- log.Printf("✅ [PUBSUB] Started listening for messages (instance: %s)", s.instanceID)
- return nil
-}
-
-// processMessages handles incoming pub/sub messages
-func (s *PubSubService) processMessages() {
- ch := s.pubsub.Channel()
-
- for {
- select {
- case <-s.ctx.Done():
- return
- case msg, ok := <-ch:
- if !ok {
- return
- }
- s.handleMessage(msg)
- }
- }
-}
-
-// handleMessage processes a single pub/sub message
-func (s *PubSubService) handleMessage(msg *redis.Message) {
- var message PubSubMessage
- if err := json.Unmarshal([]byte(msg.Payload), &message); err != nil {
- log.Printf("⚠️ [PUBSUB] Failed to unmarshal message: %v", err)
- return
- }
-
- // Skip messages from this instance (avoid loops)
- if message.InstanceID == s.instanceID {
- return
- }
-
- // Find matching handlers
- s.mu.RLock()
- defer s.mu.RUnlock()
-
- // Check for exact match
- if handlers, ok := s.handlers[msg.Channel]; ok {
- for _, handler := range handlers {
- go handler(msg.Channel, &message)
- }
- }
-
- // Check for pattern matches (simplified - real implementation would use glob matching)
- for pattern, handlers := range s.handlers {
- if matchPattern(pattern, msg.Channel) {
- for _, handler := range handlers {
- go handler(msg.Channel, &message)
- }
- }
- }
-}
-
-// PublishToUser publishes a message to a user's channel
-func (s *PubSubService) PublishToUser(ctx context.Context, userID string, msgType string, payload map[string]interface{}) error {
- message := &PubSubMessage{
- Type: msgType,
- UserID: userID,
- InstanceID: s.instanceID,
- Payload: payload,
- }
-
- data, err := json.Marshal(message)
- if err != nil {
- return err
- }
-
- channel := "user:" + userID + ":events"
- return s.redis.Client().Publish(ctx, channel, data).Err()
-}
-
-// PublishToAgent publishes a message to an agent's channel
-func (s *PubSubService) PublishToAgent(ctx context.Context, agentID string, msgType string, payload map[string]interface{}) error {
- message := &PubSubMessage{
- Type: msgType,
- AgentID: agentID,
- InstanceID: s.instanceID,
- Payload: payload,
- }
-
- data, err := json.Marshal(message)
- if err != nil {
- return err
- }
-
- channel := "agent:" + agentID + ":events"
- return s.redis.Client().Publish(ctx, channel, data).Err()
-}
-
-// Broadcast publishes a message to all instances
-func (s *PubSubService) Broadcast(ctx context.Context, topic string, msgType string, payload map[string]interface{}) error {
- message := &PubSubMessage{
- Type: msgType,
- InstanceID: s.instanceID,
- Payload: payload,
- }
-
- data, err := json.Marshal(message)
- if err != nil {
- return err
- }
-
- channel := "broadcast:" + topic
- return s.redis.Client().Publish(ctx, channel, data).Err()
-}
-
-// PublishExecutionUpdate publishes an execution update for a user
-func (s *PubSubService) PublishExecutionUpdate(ctx context.Context, userID, agentID, executionID string, update map[string]interface{}) error {
- payload := map[string]interface{}{
- "executionId": executionID,
- "agentId": agentID,
- "update": update,
- }
-
- return s.PublishToUser(ctx, userID, "execution_update", payload)
-}
-
-// Stop stops the pub/sub service
-func (s *PubSubService) Stop() error {
- s.cancel()
- if s.pubsub != nil {
- return s.pubsub.Close()
- }
- return nil
-}
-
-// matchPattern checks if a channel matches a pattern (simplified glob)
-func matchPattern(pattern, channel string) bool {
- // Simple wildcard matching
- if pattern == channel {
- return true
- }
-
- // Handle patterns like "user:*:events"
- patternParts := splitChannel(pattern)
- channelParts := splitChannel(channel)
-
- if len(patternParts) != len(channelParts) {
- return false
- }
-
- for i, part := range patternParts {
- if part != "*" && part != channelParts[i] {
- return false
- }
- }
-
- return true
-}
-
-// splitChannel splits a channel name by ":"
-func splitChannel(channel string) []string {
- var parts []string
- current := ""
- for _, c := range channel {
- if c == ':' {
- parts = append(parts, current)
- current = ""
- } else {
- current += string(c)
- }
- }
- parts = append(parts, current)
- return parts
-}
diff --git a/backend/internal/services/redis_service.go b/backend/internal/services/redis_service.go
deleted file mode 100644
index a592e136..00000000
--- a/backend/internal/services/redis_service.go
+++ /dev/null
@@ -1,185 +0,0 @@
-package services
-
-import (
- "context"
- "fmt"
- "log"
- "sync"
- "time"
-
- "github.com/redis/go-redis/v9"
-)
-
-// RedisService provides Redis connection and operations
-type RedisService struct {
- client *redis.Client
- mu sync.RWMutex
-}
-
-var (
- redisInstance *RedisService
- redisOnce sync.Once
-)
-
-// NewRedisService creates a new Redis service instance
-func NewRedisService(redisURL string) (*RedisService, error) {
- var initErr error
-
- redisOnce.Do(func() {
- opts, err := redis.ParseURL(redisURL)
- if err != nil {
- initErr = fmt.Errorf("failed to parse Redis URL: %w", err)
- return
- }
-
- // Configure connection pool
- opts.PoolSize = 10
- opts.MinIdleConns = 2
- opts.MaxRetries = 3
- opts.DialTimeout = 5 * time.Second
- opts.ReadTimeout = 3 * time.Second
- opts.WriteTimeout = 3 * time.Second
-
- client := redis.NewClient(opts)
-
- // Test connection
- ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
- defer cancel()
-
- if err := client.Ping(ctx).Err(); err != nil {
- initErr = fmt.Errorf("failed to connect to Redis: %w", err)
- return
- }
-
- redisInstance = &RedisService{
- client: client,
- }
-
- log.Println("✅ Redis connection established")
- })
-
- if initErr != nil {
- return nil, initErr
- }
-
- return redisInstance, nil
-}
-
-// GetRedisService returns the singleton Redis service instance
-func GetRedisService() *RedisService {
- return redisInstance
-}
-
-// Client returns the underlying Redis client
-func (r *RedisService) Client() *redis.Client {
- r.mu.RLock()
- defer r.mu.RUnlock()
- return r.client
-}
-
-// Close closes the Redis connection
-func (r *RedisService) Close() error {
- r.mu.Lock()
- defer r.mu.Unlock()
-
- if r.client != nil {
- return r.client.Close()
- }
- return nil
-}
-
-// Ping checks if Redis is healthy
-func (r *RedisService) Ping(ctx context.Context) error {
- return r.client.Ping(ctx).Err()
-}
-
-// Set sets a key-value pair with optional expiration
-func (r *RedisService) Set(ctx context.Context, key string, value interface{}, expiration time.Duration) error {
- return r.client.Set(ctx, key, value, expiration).Err()
-}
-
-// Get retrieves a value by key
-func (r *RedisService) Get(ctx context.Context, key string) (string, error) {
- return r.client.Get(ctx, key).Result()
-}
-
-// Delete removes a key
-func (r *RedisService) Delete(ctx context.Context, keys ...string) error {
- return r.client.Del(ctx, keys...).Err()
-}
-
-// SetNX sets a key only if it doesn't exist (for distributed locking)
-func (r *RedisService) SetNX(ctx context.Context, key string, value interface{}, expiration time.Duration) (bool, error) {
- return r.client.SetNX(ctx, key, value, expiration).Result()
-}
-
-// Publish publishes a message to a channel
-func (r *RedisService) Publish(ctx context.Context, channel string, message interface{}) error {
- return r.client.Publish(ctx, channel, message).Err()
-}
-
-// Subscribe subscribes to one or more channels
-func (r *RedisService) Subscribe(ctx context.Context, channels ...string) *redis.PubSub {
- return r.client.Subscribe(ctx, channels...)
-}
-
-// Incr increments a counter (for rate limiting)
-func (r *RedisService) Incr(ctx context.Context, key string) (int64, error) {
- return r.client.Incr(ctx, key).Result()
-}
-
-// Expire sets expiration on a key
-func (r *RedisService) Expire(ctx context.Context, key string, expiration time.Duration) error {
- return r.client.Expire(ctx, key, expiration).Err()
-}
-
-// TTL gets the remaining time to live for a key
-func (r *RedisService) TTL(ctx context.Context, key string) (time.Duration, error) {
- return r.client.TTL(ctx, key).Result()
-}
-
-// AcquireLock attempts to acquire a distributed lock
-// Returns true if lock was acquired, false otherwise
-func (r *RedisService) AcquireLock(ctx context.Context, lockKey string, lockValue string, expiration time.Duration) (bool, error) {
- return r.client.SetNX(ctx, lockKey, lockValue, expiration).Result()
-}
-
-// ReleaseLock releases a distributed lock if it's still held by the given value
-func (r *RedisService) ReleaseLock(ctx context.Context, lockKey string, lockValue string) (bool, error) {
- // Lua script to atomically check and delete
- script := redis.NewScript(`
- if redis.call("get", KEYS[1]) == ARGV[1] then
- return redis.call("del", KEYS[1])
- else
- return 0
- end
- `)
-
- result, err := script.Run(ctx, r.client, []string{lockKey}, lockValue).Int64()
- if err != nil {
- return false, err
- }
-
- return result == 1, nil
-}
-
-// CheckRateLimit checks if a rate limit has been exceeded
-// Returns remaining requests and whether the limit was exceeded
-func (r *RedisService) CheckRateLimit(ctx context.Context, key string, limit int64, window time.Duration) (remaining int64, exceeded bool, err error) {
- count, err := r.client.Incr(ctx, key).Result()
- if err != nil {
- return 0, false, err
- }
-
- // Set expiry on first request
- if count == 1 {
- r.client.Expire(ctx, key, window)
- }
-
- remaining = limit - count
- if remaining < 0 {
- remaining = 0
- }
-
- return remaining, count > limit, nil
-}
diff --git a/backend/internal/services/scheduler_service.go b/backend/internal/services/scheduler_service.go
deleted file mode 100644
index e12ccbd0..00000000
--- a/backend/internal/services/scheduler_service.go
+++ /dev/null
@@ -1,730 +0,0 @@
-package services
-
-import (
- "claraverse/internal/database"
- "claraverse/internal/models"
- "context"
- "fmt"
- "log"
- "sync"
- "time"
-
- "github.com/go-co-op/gocron/v2"
- "github.com/google/uuid"
- "github.com/robfig/cron/v3"
- "go.mongodb.org/mongo-driver/bson"
- "go.mongodb.org/mongo-driver/bson/primitive"
- "go.mongodb.org/mongo-driver/mongo"
- "go.mongodb.org/mongo-driver/mongo/options"
-)
-
-// SchedulerService manages scheduled agent executions
-type SchedulerService struct {
- scheduler gocron.Scheduler
- mongoDB *database.MongoDB
- redisService *RedisService
- agentService *AgentService
- executionService *ExecutionService
- workflowExecutor models.WorkflowExecuteFunc
- instanceID string
- mu sync.RWMutex
- jobs map[string]gocron.Job // scheduleID -> job
-}
-
-// NewSchedulerService creates a new scheduler service
-func NewSchedulerService(
- mongoDB *database.MongoDB,
- redisService *RedisService,
- agentService *AgentService,
- executionService *ExecutionService,
-) (*SchedulerService, error) {
- // Create scheduler with second-level precision
- scheduler, err := gocron.NewScheduler(
- gocron.WithLocation(time.UTC),
- )
- if err != nil {
- return nil, fmt.Errorf("failed to create scheduler: %w", err)
- }
-
- return &SchedulerService{
- scheduler: scheduler,
- mongoDB: mongoDB,
- redisService: redisService,
- agentService: agentService,
- executionService: executionService,
- instanceID: uuid.New().String(),
- jobs: make(map[string]gocron.Job),
- }, nil
-}
-
-// Start starts the scheduler and loads all enabled schedules
-func (s *SchedulerService) Start(ctx context.Context) error {
- log.Println("⏰ Starting scheduler service...")
-
- // Load and register all enabled schedules
- if err := s.loadSchedules(ctx); err != nil {
- log.Printf("⚠️ Failed to load schedules: %v", err)
- }
-
- // Start the scheduler
- s.scheduler.Start()
- log.Println("✅ Scheduler service started")
-
- return nil
-}
-
-// Stop stops the scheduler
-func (s *SchedulerService) Stop() error {
- log.Println("⏹️ Stopping scheduler service...")
- return s.scheduler.Shutdown()
-}
-
-// SetWorkflowExecutor sets the workflow executor function (used for deferred initialization)
-func (s *SchedulerService) SetWorkflowExecutor(executor models.WorkflowExecuteFunc) {
- s.mu.Lock()
- defer s.mu.Unlock()
- s.workflowExecutor = executor
-}
-
-// loadSchedules loads all enabled schedules from MongoDB and registers them
-func (s *SchedulerService) loadSchedules(ctx context.Context) error {
- if s.mongoDB == nil {
- log.Println("⚠️ MongoDB not available, skipping schedule loading")
- return nil
- }
-
- collection := s.mongoDB.Database().Collection("schedules")
-
- cursor, err := collection.Find(ctx, bson.M{"enabled": true})
- if err != nil {
- return fmt.Errorf("failed to query schedules: %w", err)
- }
- defer cursor.Close(ctx)
-
- var count int
- for cursor.Next(ctx) {
- var schedule models.Schedule
- if err := cursor.Decode(&schedule); err != nil {
- log.Printf("⚠️ Failed to decode schedule: %v", err)
- continue
- }
-
- if err := s.registerJob(&schedule); err != nil {
- log.Printf("⚠️ Failed to register schedule %s: %v", schedule.ID.Hex(), err)
- continue
- }
- count++
- }
-
- log.Printf("✅ Loaded %d schedules", count)
- return nil
-}
-
-// registerJob registers a schedule with gocron
-func (s *SchedulerService) registerJob(schedule *models.Schedule) error {
- s.mu.Lock()
- defer s.mu.Unlock()
-
- // Validate timezone
- _, err := time.LoadLocation(schedule.Timezone)
- if err != nil {
- return fmt.Errorf("invalid timezone %s: %w", schedule.Timezone, err)
- }
-
- // Build cron expression with timezone prefix (CRON_TZ=America/New_York 0 9 * * *)
- cronWithTZ := fmt.Sprintf("CRON_TZ=%s %s", schedule.Timezone, schedule.CronExpression)
-
- // Create the job
- job, err := s.scheduler.NewJob(
- gocron.CronJob(cronWithTZ, false),
- gocron.NewTask(func() {
- s.executeScheduledJob(schedule)
- }),
- gocron.WithName(schedule.ID.Hex()),
- gocron.WithTags(schedule.AgentID, schedule.UserID),
- )
- if err != nil {
- return fmt.Errorf("failed to create job: %w", err)
- }
-
- s.jobs[schedule.ID.Hex()] = job
- log.Printf("📅 Registered schedule %s for agent %s (cron: %s, tz: %s)",
- schedule.ID.Hex(), schedule.AgentID, schedule.CronExpression, schedule.Timezone)
-
- return nil
-}
-
-// unregisterJob removes a job from the scheduler
-func (s *SchedulerService) unregisterJob(scheduleID string) error {
- s.mu.Lock()
- defer s.mu.Unlock()
-
- job, exists := s.jobs[scheduleID]
- if !exists {
- return nil
- }
-
- if err := s.scheduler.RemoveJob(job.ID()); err != nil {
- return fmt.Errorf("failed to remove job: %w", err)
- }
-
- delete(s.jobs, scheduleID)
- log.Printf("🗑️ Unregistered schedule %s", scheduleID)
-
- return nil
-}
-
-// executeScheduledJob executes a scheduled agent workflow
-func (s *SchedulerService) executeScheduledJob(schedule *models.Schedule) {
- ctx := context.Background()
-
- // Create a unique lock key for this schedule execution window
- // Using minute-level granularity to prevent duplicate runs within the same minute
- lockKey := fmt.Sprintf("schedule-lock:%s:%d", schedule.ID.Hex(), time.Now().Unix()/60)
-
- // Try to acquire distributed lock
- acquired, err := s.redisService.AcquireLock(ctx, lockKey, s.instanceID, 5*time.Minute)
- if err != nil {
- log.Printf("❌ Failed to acquire lock for schedule %s: %v", schedule.ID.Hex(), err)
- return
- }
-
- if !acquired {
- // Another instance is handling this execution
- log.Printf("⏭️ Schedule %s already being executed by another instance", schedule.ID.Hex())
- return
- }
-
- // Release lock when done
- defer func() {
- if _, err := s.redisService.ReleaseLock(ctx, lockKey, s.instanceID); err != nil {
- log.Printf("⚠️ Failed to release lock for schedule %s: %v", schedule.ID.Hex(), err)
- }
- }()
-
- log.Printf("▶️ Executing scheduled job for agent %s (schedule: %s)", schedule.AgentID, schedule.ID.Hex())
-
- // Get the agent and workflow
- // Note: We use a system context here since scheduled jobs run without a user session
- agent, err := s.agentService.GetAgentByID(schedule.AgentID)
- if err != nil {
- log.Printf("❌ Failed to get agent %s: %v", schedule.AgentID, err)
- s.updateScheduleStats(ctx, schedule.ID, false, schedule)
- return
- }
-
- if agent.Workflow == nil {
- log.Printf("❌ Agent %s has no workflow", schedule.AgentID)
- s.updateScheduleStats(ctx, schedule.ID, false, schedule)
- return
- }
-
- // Build input from template
- input := make(map[string]interface{})
- if schedule.InputTemplate != nil {
- for k, v := range schedule.InputTemplate {
- input[k] = v
- }
- }
-
- // CRITICAL: Inject user context for credential resolution and tool execution
- // Without this, tools like send_discord_message cannot access user credentials
- input["__user_id__"] = schedule.UserID
- log.Printf("🔐 [SCHEDULER] Injecting user context: __user_id__=%s for schedule %s", schedule.UserID, schedule.ID.Hex())
-
- // Create execution record in MongoDB
- var execRecord *ExecutionRecord
- if s.executionService != nil {
- var err error
- execRecord, err = s.executionService.Create(ctx, &CreateExecutionRequest{
- AgentID: schedule.AgentID,
- UserID: schedule.UserID,
- WorkflowVersion: agent.Workflow.Version,
- TriggerType: "scheduled",
- ScheduleID: schedule.ID,
- Input: input,
- })
- if err != nil {
- log.Printf("⚠️ Failed to create execution record: %v", err)
- } else {
- // Mark as running
- s.executionService.UpdateStatus(ctx, execRecord.ID, "running")
- }
- }
-
- // Check if workflow executor is available
- s.mu.RLock()
- executor := s.workflowExecutor
- s.mu.RUnlock()
-
- if executor == nil {
- log.Printf("❌ Workflow executor not set for schedule %s", schedule.ID.Hex())
- s.updateScheduleStats(ctx, schedule.ID, false, schedule)
- if execRecord != nil {
- s.executionService.Complete(ctx, execRecord.ID, &ExecutionCompleteRequest{
- Status: "failed",
- Error: "Workflow executor not available",
- })
- }
- return
- }
-
- // Execute the workflow using the callback function
- result, execErr := executor(agent.Workflow, input)
-
- // Determine success status
- status := "failed"
- if result != nil {
- status = result.Status
- }
- success := status == "completed" && execErr == nil
-
- // Complete the execution record
- if execRecord != nil {
- completeReq := &ExecutionCompleteRequest{
- Status: status,
- }
- if execErr != nil {
- completeReq.Error = execErr.Error()
- } else if result != nil {
- completeReq.Output = result.Output
- completeReq.BlockStates = result.BlockStates
- if result.Error != "" {
- completeReq.Error = result.Error
- }
- }
- s.executionService.Complete(ctx, execRecord.ID, completeReq)
- log.Printf("📊 Scheduled execution %s completed with status: %s", execRecord.ID.Hex(), status)
- }
-
- if success {
- log.Printf("✅ Scheduled execution completed successfully for agent %s", schedule.AgentID)
- } else {
- errMsg := ""
- if execErr != nil {
- errMsg = execErr.Error()
- } else if result != nil && result.Error != "" {
- errMsg = result.Error
- }
- log.Printf("❌ Scheduled execution failed for agent %s: %s (status: %s)", schedule.AgentID, errMsg, status)
- }
-
- // Update schedule statistics and next run time
- s.updateScheduleStats(ctx, schedule.ID, success, schedule)
-}
-
-// updateScheduleStats updates the schedule's run statistics and next run time
-func (s *SchedulerService) updateScheduleStats(ctx context.Context, scheduleID primitive.ObjectID, success bool, schedule *models.Schedule) {
- if s.mongoDB == nil {
- return
- }
-
- collection := s.mongoDB.Database().Collection("schedules")
-
- now := time.Now()
-
- // Calculate next run time
- parser := cron.NewParser(cron.Minute | cron.Hour | cron.Dom | cron.Month | cron.Dow)
- cronSchedule, err := parser.Parse(schedule.CronExpression)
- var nextRun time.Time
- if err == nil {
- loc, locErr := time.LoadLocation(schedule.Timezone)
- if locErr == nil {
- nextRun = cronSchedule.Next(now.In(loc))
- } else {
- nextRun = cronSchedule.Next(now)
- }
- }
-
- update := bson.M{
- "$set": bson.M{
- "lastRunAt": now,
- "updatedAt": now,
- "nextRunAt": nextRun,
- },
- "$inc": bson.M{
- "totalRuns": 1,
- },
- }
-
- if success {
- update["$inc"].(bson.M)["successfulRuns"] = 1
- } else {
- update["$inc"].(bson.M)["failedRuns"] = 1
- }
-
- if _, err := collection.UpdateByID(ctx, scheduleID, update); err != nil {
- log.Printf("⚠️ Failed to update schedule stats: %v", err)
- } else {
- log.Printf("📅 Updated next run time to %v for schedule %s", nextRun, scheduleID.Hex())
- }
-}
-
-// CreateSchedule creates a new schedule for an agent
-func (s *SchedulerService) CreateSchedule(ctx context.Context, agentID, userID string, req *models.CreateScheduleRequest) (*models.Schedule, error) {
- if s.mongoDB == nil {
- return nil, fmt.Errorf("MongoDB not available")
- }
-
- // Validate cron expression
- parser := cron.NewParser(cron.Minute | cron.Hour | cron.Dom | cron.Month | cron.Dow)
- if _, err := parser.Parse(req.CronExpression); err != nil {
- return nil, fmt.Errorf("invalid cron expression: %w", err)
- }
-
- // Validate timezone
- loc, err := time.LoadLocation(req.Timezone)
- if err != nil {
- return nil, fmt.Errorf("invalid timezone: %w", err)
- }
-
- // Check user's schedule limit
- limits := models.GetTierLimits(s.getUserTier(ctx, userID))
- if limits.MaxSchedules > 0 {
- count, err := s.countUserSchedules(ctx, userID)
- if err != nil {
- return nil, fmt.Errorf("failed to check schedule limit: %w", err)
- }
- if count >= int64(limits.MaxSchedules) {
- return nil, fmt.Errorf("active schedule limit reached (%d/%d). Pause an existing schedule to create a new one", count, limits.MaxSchedules)
- }
- }
-
- // Check if agent already has a schedule
- existing, _ := s.GetScheduleByAgentID(ctx, agentID, userID)
- if existing != nil {
- return nil, fmt.Errorf("agent already has a schedule")
- }
-
- // Calculate next run time
- schedule, _ := parser.Parse(req.CronExpression)
- nextRun := schedule.Next(time.Now().In(loc))
-
- // Default enabled to true
- enabled := true
- if req.Enabled != nil {
- enabled = *req.Enabled
- }
-
- now := time.Now()
- doc := &models.Schedule{
- ID: primitive.NewObjectID(),
- AgentID: agentID,
- UserID: userID,
- CronExpression: req.CronExpression,
- Timezone: req.Timezone,
- Enabled: enabled,
- InputTemplate: req.InputTemplate,
- NextRunAt: &nextRun,
- TotalRuns: 0,
- SuccessfulRuns: 0,
- FailedRuns: 0,
- CreatedAt: now,
- UpdatedAt: now,
- }
-
- collection := s.mongoDB.Database().Collection("schedules")
- if _, err := collection.InsertOne(ctx, doc); err != nil {
- return nil, fmt.Errorf("failed to create schedule: %w", err)
- }
-
- // Register with scheduler if enabled
- if enabled {
- if err := s.registerJob(doc); err != nil {
- log.Printf("⚠️ Failed to register new schedule: %v", err)
- }
- }
-
- log.Printf("✅ Created schedule %s for agent %s", doc.ID.Hex(), agentID)
- return doc, nil
-}
-
-// GetSchedule retrieves a schedule by ID
-func (s *SchedulerService) GetSchedule(ctx context.Context, scheduleID, userID string) (*models.Schedule, error) {
- if s.mongoDB == nil {
- return nil, fmt.Errorf("MongoDB not available")
- }
-
- objID, err := primitive.ObjectIDFromHex(scheduleID)
- if err != nil {
- return nil, fmt.Errorf("invalid schedule ID")
- }
-
- collection := s.mongoDB.Database().Collection("schedules")
-
- var schedule models.Schedule
- err = collection.FindOne(ctx, bson.M{
- "_id": objID,
- "userId": userID,
- }).Decode(&schedule)
-
- if err == mongo.ErrNoDocuments {
- return nil, fmt.Errorf("schedule not found")
- }
- if err != nil {
- return nil, fmt.Errorf("failed to get schedule: %w", err)
- }
-
- return &schedule, nil
-}
-
-// GetScheduleByAgentID retrieves a schedule by agent ID
-func (s *SchedulerService) GetScheduleByAgentID(ctx context.Context, agentID, userID string) (*models.Schedule, error) {
- if s.mongoDB == nil {
- return nil, fmt.Errorf("MongoDB not available")
- }
-
- collection := s.mongoDB.Database().Collection("schedules")
-
- var schedule models.Schedule
- err := collection.FindOne(ctx, bson.M{
- "agentId": agentID,
- "userId": userID,
- }).Decode(&schedule)
-
- if err == mongo.ErrNoDocuments {
- return nil, fmt.Errorf("schedule not found")
- }
- if err != nil {
- return nil, fmt.Errorf("failed to get schedule: %w", err)
- }
-
- return &schedule, nil
-}
-
-// UpdateSchedule updates a schedule
-func (s *SchedulerService) UpdateSchedule(ctx context.Context, scheduleID, userID string, req *models.UpdateScheduleRequest) (*models.Schedule, error) {
- if s.mongoDB == nil {
- return nil, fmt.Errorf("MongoDB not available")
- }
-
- // Get existing schedule
- schedule, err := s.GetSchedule(ctx, scheduleID, userID)
- if err != nil {
- return nil, err
- }
-
- // Build update
- update := bson.M{
- "updatedAt": time.Now(),
- }
-
- if req.CronExpression != nil {
- // Validate cron expression
- parser := cron.NewParser(cron.Minute | cron.Hour | cron.Dom | cron.Month | cron.Dow)
- if _, err := parser.Parse(*req.CronExpression); err != nil {
- return nil, fmt.Errorf("invalid cron expression: %w", err)
- }
- update["cronExpression"] = *req.CronExpression
- schedule.CronExpression = *req.CronExpression
- }
-
- if req.Timezone != nil {
- if _, err := time.LoadLocation(*req.Timezone); err != nil {
- return nil, fmt.Errorf("invalid timezone: %w", err)
- }
- update["timezone"] = *req.Timezone
- schedule.Timezone = *req.Timezone
- }
-
- if req.InputTemplate != nil {
- update["inputTemplate"] = req.InputTemplate
- schedule.InputTemplate = req.InputTemplate
- }
-
- if req.Enabled != nil {
- update["enabled"] = *req.Enabled
- schedule.Enabled = *req.Enabled
- }
-
- // Recalculate next run time
- parser := cron.NewParser(cron.Minute | cron.Hour | cron.Dom | cron.Month | cron.Dow)
- cronSchedule, _ := parser.Parse(schedule.CronExpression)
- loc, _ := time.LoadLocation(schedule.Timezone)
- nextRun := cronSchedule.Next(time.Now().In(loc))
- update["nextRunAt"] = nextRun
-
- collection := s.mongoDB.Database().Collection("schedules")
- _, err = collection.UpdateByID(ctx, schedule.ID, bson.M{"$set": update})
- if err != nil {
- return nil, fmt.Errorf("failed to update schedule: %w", err)
- }
-
- // Re-register job if cron or timezone changed, or enable/disable
- s.unregisterJob(scheduleID)
- if schedule.Enabled {
- schedule.NextRunAt = &nextRun
- if err := s.registerJob(schedule); err != nil {
- log.Printf("⚠️ Failed to re-register schedule: %v", err)
- }
- }
-
- return schedule, nil
-}
-
-// DeleteSchedule deletes a schedule
-func (s *SchedulerService) DeleteSchedule(ctx context.Context, scheduleID, userID string) error {
- if s.mongoDB == nil {
- return fmt.Errorf("MongoDB not available")
- }
-
- objID, err := primitive.ObjectIDFromHex(scheduleID)
- if err != nil {
- return fmt.Errorf("invalid schedule ID")
- }
-
- // Unregister from scheduler
- s.unregisterJob(scheduleID)
-
- collection := s.mongoDB.Database().Collection("schedules")
- result, err := collection.DeleteOne(ctx, bson.M{
- "_id": objID,
- "userId": userID,
- })
-
- if err != nil {
- return fmt.Errorf("failed to delete schedule: %w", err)
- }
-
- if result.DeletedCount == 0 {
- return fmt.Errorf("schedule not found")
- }
-
- log.Printf("🗑️ Deleted schedule %s", scheduleID)
- return nil
-}
-
-// DeleteAllByUser deletes all schedules for a user (GDPR compliance)
-func (s *SchedulerService) DeleteAllByUser(ctx context.Context, userID string) (int64, error) {
- if s.mongoDB == nil {
- return 0, nil // No MongoDB, no schedules to delete
- }
-
- if userID == "" {
- return 0, fmt.Errorf("user ID is required")
- }
-
- // First, unregister all jobs for this user from the scheduler
- collection := s.mongoDB.Database().Collection("schedules")
- cursor, err := collection.Find(ctx, bson.M{"userId": userID})
- if err != nil {
- return 0, fmt.Errorf("failed to find user schedules: %w", err)
- }
- defer cursor.Close(ctx)
-
- for cursor.Next(ctx) {
- var schedule struct {
- ID primitive.ObjectID `bson:"_id"`
- }
- if err := cursor.Decode(&schedule); err == nil {
- s.unregisterJob(schedule.ID.Hex())
- }
- }
-
- // Delete all schedules
- result, err := collection.DeleteMany(ctx, bson.M{"userId": userID})
- if err != nil {
- return 0, fmt.Errorf("failed to delete user schedules: %w", err)
- }
-
- log.Printf("🗑️ [GDPR] Deleted %d schedules for user %s", result.DeletedCount, userID)
- return result.DeletedCount, nil
-}
-
-// TriggerNow triggers an immediate execution of a schedule
-func (s *SchedulerService) TriggerNow(ctx context.Context, scheduleID, userID string) error {
- schedule, err := s.GetSchedule(ctx, scheduleID, userID)
- if err != nil {
- return err
- }
-
- // Execute in background
- go s.executeScheduledJob(schedule)
-
- return nil
-}
-
-// countUserSchedules counts the number of ENABLED schedules for a user
-// Only enabled schedules count toward the limit - paused schedules don't consume quota
-// This allows users to pause schedules to free up slots for new ones
-func (s *SchedulerService) countUserSchedules(ctx context.Context, userID string) (int64, error) {
- collection := s.mongoDB.Database().Collection("schedules")
- return collection.CountDocuments(ctx, bson.M{"userId": userID, "enabled": true})
-}
-
-// ScheduleUsage represents the user's schedule usage stats
-type ScheduleUsage struct {
- Active int64 `json:"active"`
- Paused int64 `json:"paused"`
- Total int64 `json:"total"`
- Limit int `json:"limit"`
- CanCreate bool `json:"canCreate"`
-}
-
-// GetScheduleUsage returns the user's schedule usage statistics
-func (s *SchedulerService) GetScheduleUsage(ctx context.Context, userID string) (*ScheduleUsage, error) {
- collection := s.mongoDB.Database().Collection("schedules")
-
- // Count active (enabled) schedules
- active, err := collection.CountDocuments(ctx, bson.M{"userId": userID, "enabled": true})
- if err != nil {
- return nil, fmt.Errorf("failed to count active schedules: %w", err)
- }
-
- // Count paused (disabled) schedules
- paused, err := collection.CountDocuments(ctx, bson.M{"userId": userID, "enabled": false})
- if err != nil {
- return nil, fmt.Errorf("failed to count paused schedules: %w", err)
- }
-
- // Get user's limit
- limits := models.GetTierLimits(s.getUserTier(ctx, userID))
- limit := limits.MaxSchedules
-
- // Can create if active < limit (or limit is -1 for unlimited)
- canCreate := limit < 0 || active < int64(limit)
-
- return &ScheduleUsage{
- Active: active,
- Paused: paused,
- Total: active + paused,
- Limit: limit,
- CanCreate: canCreate,
- }, nil
-}
-
-// getUserTier gets the user's subscription tier (placeholder - will be implemented with UserService)
-func (s *SchedulerService) getUserTier(ctx context.Context, userID string) string {
- // TODO: Look up user's tier from MongoDB
- return "free"
-}
-
-// InitializeIndexes creates the necessary indexes for the schedules collection
-func (s *SchedulerService) InitializeIndexes(ctx context.Context) error {
- if s.mongoDB == nil {
- return nil
- }
-
- collection := s.mongoDB.Database().Collection("schedules")
-
- indexes := []mongo.IndexModel{
- {
- Keys: bson.D{{Key: "agentId", Value: 1}},
- Options: options.Index().SetUnique(true),
- },
- {
- Keys: bson.D{{Key: "userId", Value: 1}, {Key: "enabled", Value: 1}},
- },
- {
- Keys: bson.D{{Key: "nextRunAt", Value: 1}, {Key: "enabled", Value: 1}},
- },
- }
-
- _, err := collection.Indexes().CreateMany(ctx, indexes)
- if err != nil {
- return fmt.Errorf("failed to create indexes: %w", err)
- }
-
- log.Println("✅ Schedule indexes created")
- return nil
-}
diff --git a/backend/internal/services/scraper_client.go b/backend/internal/services/scraper_client.go
deleted file mode 100644
index c011e2ef..00000000
--- a/backend/internal/services/scraper_client.go
+++ /dev/null
@@ -1,76 +0,0 @@
-package services
-
-import (
- "context"
- "fmt"
- "net"
- "net/http"
- "time"
-)
-
-// ScraperClient wraps an HTTP client with optimized settings for web scraping
-type ScraperClient struct {
- httpClient *http.Client
- userAgent string
- timeout time.Duration
-}
-
-// NewScraperClient creates a new HTTP client optimized for web scraping
-func NewScraperClient() *ScraperClient {
- // Custom transport with optimized connection pooling
- transport := &http.Transport{
- MaxIdleConns: 100, // Total idle connections across all hosts
- MaxIdleConnsPerHost: 20, // CRITICAL: Default is 2! Increase for performance
- MaxConnsPerHost: 50, // Maximum connections per host
- IdleConnTimeout: 90 * time.Second, // Keep connections alive
- TLSHandshakeTimeout: 10 * time.Second,
- DisableCompression: false,
-
- // Dial settings for connection establishment
- DialContext: (&net.Dialer{
- Timeout: 30 * time.Second,
- KeepAlive: 30 * time.Second,
- }).DialContext,
- }
-
- return &ScraperClient{
- httpClient: &http.Client{
- Transport: transport,
- Timeout: 60 * time.Second, // Overall request timeout
- CheckRedirect: func(req *http.Request, via []*http.Request) error {
- if len(via) >= 10 {
- return fmt.Errorf("too many redirects (max 10)")
- }
- return nil
- },
- },
- userAgent: "ClaraVerse-Bot/1.0 (+https://claraverse.example.com/bot)",
- timeout: 60 * time.Second,
- }
-}
-
-// Get performs an HTTP GET request with proper headers
-func (c *ScraperClient) Get(ctx context.Context, url string) (*http.Response, error) {
- req, err := http.NewRequestWithContext(ctx, "GET", url, nil)
- if err != nil {
- return nil, fmt.Errorf("failed to create request: %w", err)
- }
-
- // Set proper headers
- req.Header.Set("User-Agent", c.userAgent)
- req.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8")
- req.Header.Set("Accept-Language", "en-US,en;q=0.9")
-
- return c.httpClient.Do(req)
-}
-
-// SetUserAgent updates the user agent string
-func (c *ScraperClient) SetUserAgent(userAgent string) {
- c.userAgent = userAgent
-}
-
-// SetTimeout updates the request timeout
-func (c *ScraperClient) SetTimeout(timeout time.Duration) {
- c.timeout = timeout
- c.httpClient.Timeout = timeout
-}
diff --git a/backend/internal/services/scraper_ratelimit.go b/backend/internal/services/scraper_ratelimit.go
deleted file mode 100644
index 27b7999e..00000000
--- a/backend/internal/services/scraper_ratelimit.go
+++ /dev/null
@@ -1,118 +0,0 @@
-package services
-
-import (
- "context"
- "sync"
- "time"
-
- "golang.org/x/time/rate"
-)
-
-// RateLimiter implements three-tier rate limiting for web scraping
-type RateLimiter struct {
- globalLimiter *rate.Limiter // Overall requests/second for the server
- perDomainLimiters *sync.Map // map[string]*rate.Limiter - per domain limits
- perUserLimiters *sync.Map // map[string]*rate.Limiter - per user limits
- mu sync.RWMutex
-}
-
-// NewRateLimiter creates a new three-tier rate limiter
-func NewRateLimiter(globalRate, perUserRate float64) *RateLimiter {
- return &RateLimiter{
- globalLimiter: rate.NewLimiter(rate.Limit(globalRate), int(globalRate*2)), // 10 req/s, burst 20
- perDomainLimiters: &sync.Map{},
- perUserLimiters: &sync.Map{},
- }
-}
-
-// Wait applies all three tiers of rate limiting
-func (rl *RateLimiter) Wait(ctx context.Context, userID, domain string) error {
- // Tier 1: Global rate limit (protect server resources)
- if err := rl.globalLimiter.Wait(ctx); err != nil {
- return err
- }
-
- // Tier 2: Per-domain rate limit (respect target websites)
- domainLimiter := rl.getOrCreateDomainLimiter(domain)
- if err := domainLimiter.Wait(ctx); err != nil {
- return err
- }
-
- // Tier 3: Per-user rate limit (fair usage)
- userLimiter := rl.getOrCreateUserLimiter(userID)
- if err := userLimiter.Wait(ctx); err != nil {
- return err
- }
-
- return nil
-}
-
-// WaitWithCrawlDelay applies rate limiting with respect to robots.txt crawl-delay
-func (rl *RateLimiter) WaitWithCrawlDelay(ctx context.Context, userID, domain string, crawlDelay time.Duration) error {
- // Tier 1: Global rate limit
- if err := rl.globalLimiter.Wait(ctx); err != nil {
- return err
- }
-
- // Tier 2: Per-domain rate limit with crawl-delay
- domainLimiter := rl.getOrCreateDomainLimiterWithDelay(domain, crawlDelay)
- if err := domainLimiter.Wait(ctx); err != nil {
- return err
- }
-
- // Tier 3: Per-user rate limit
- userLimiter := rl.getOrCreateUserLimiter(userID)
- if err := userLimiter.Wait(ctx); err != nil {
- return err
- }
-
- return nil
-}
-
-// getOrCreateDomainLimiter gets or creates a rate limiter for a domain (default 2 req/s)
-func (rl *RateLimiter) getOrCreateDomainLimiter(domain string) *rate.Limiter {
- return rl.getOrCreateDomainLimiterWithDelay(domain, 500*time.Millisecond)
-}
-
-// getOrCreateDomainLimiterWithDelay gets or creates a rate limiter for a domain with custom delay
-func (rl *RateLimiter) getOrCreateDomainLimiterWithDelay(domain string, crawlDelay time.Duration) *rate.Limiter {
- if limiter, ok := rl.perDomainLimiters.Load(domain); ok {
- return limiter.(*rate.Limiter)
- }
-
- // Create new limiter based on crawl delay
- requestsPerSecond := 1.0 / crawlDelay.Seconds()
- if requestsPerSecond > 5.0 {
- requestsPerSecond = 5.0 // Cap at 5 req/s
- }
- if requestsPerSecond < 0.2 {
- requestsPerSecond = 0.2 // Minimum 1 request per 5 seconds
- }
-
- newLimiter := rate.NewLimiter(rate.Limit(requestsPerSecond), 1)
-
- // Try to store, but use existing if another goroutine created it first
- actual, _ := rl.perDomainLimiters.LoadOrStore(domain, newLimiter)
- return actual.(*rate.Limiter)
-}
-
-// getOrCreateUserLimiter gets or creates a rate limiter for a user (5 req/s)
-func (rl *RateLimiter) getOrCreateUserLimiter(userID string) *rate.Limiter {
- if limiter, ok := rl.perUserLimiters.Load(userID); ok {
- return limiter.(*rate.Limiter)
- }
-
- newLimiter := rate.NewLimiter(rate.Limit(5.0), 10) // 5 req/s, burst 10
-
- // Try to store, but use existing if another goroutine created it first
- actual, _ := rl.perUserLimiters.LoadOrStore(userID, newLimiter)
- return actual.(*rate.Limiter)
-}
-
-// SetGlobalRate updates the global rate limit
-func (rl *RateLimiter) SetGlobalRate(requestsPerSecond float64) {
- rl.mu.Lock()
- defer rl.mu.Unlock()
- rl.globalLimiter.SetLimit(rate.Limit(requestsPerSecond))
- rl.globalLimiter.SetBurst(int(requestsPerSecond * 2))
-}
diff --git a/backend/internal/services/scraper_resource.go b/backend/internal/services/scraper_resource.go
deleted file mode 100644
index 9c2ebe29..00000000
--- a/backend/internal/services/scraper_resource.go
+++ /dev/null
@@ -1,52 +0,0 @@
-package services
-
-import (
- "context"
- "fmt"
- "io"
-)
-
-// ResourceManager handles resource limits for concurrent scraping
-type ResourceManager struct {
- semaphore chan struct{} // Limit concurrent requests
- maxBodySize int64 // Max response body size in bytes
-}
-
-// NewResourceManager creates a new resource manager
-func NewResourceManager(maxConcurrent int, maxBodySize int64) *ResourceManager {
- return &ResourceManager{
- semaphore: make(chan struct{}, maxConcurrent),
- maxBodySize: maxBodySize,
- }
-}
-
-// Acquire acquires a slot for a scraping operation
-func (rm *ResourceManager) Acquire(ctx context.Context) error {
- select {
- case rm.semaphore <- struct{}{}:
- return nil
- case <-ctx.Done():
- return fmt.Errorf("context cancelled while waiting for resource: %w", ctx.Err())
- }
-}
-
-// Release releases a slot after scraping completes
-func (rm *ResourceManager) Release() {
- <-rm.semaphore
-}
-
-// ReadBody reads the response body with size limit to prevent memory exhaustion
-func (rm *ResourceManager) ReadBody(body io.Reader) ([]byte, error) {
- limitedReader := io.LimitReader(body, rm.maxBodySize)
- data, err := io.ReadAll(limitedReader)
- if err != nil {
- return nil, fmt.Errorf("failed to read body: %w", err)
- }
-
- // Check if we hit the limit
- if int64(len(data)) >= rm.maxBodySize {
- return nil, fmt.Errorf("response body too large (max %d bytes)", rm.maxBodySize)
- }
-
- return data, nil
-}
diff --git a/backend/internal/services/scraper_robots.go b/backend/internal/services/scraper_robots.go
deleted file mode 100644
index 9085c7a3..00000000
--- a/backend/internal/services/scraper_robots.go
+++ /dev/null
@@ -1,114 +0,0 @@
-package services
-
-import (
- "context"
- "fmt"
- "io"
- "net/http"
- "net/url"
- "time"
-
- cache "github.com/patrickmn/go-cache"
- "github.com/temoto/robotstxt"
-)
-
-// RobotsChecker handles robots.txt fetching and compliance checking
-type RobotsChecker struct {
- cache *cache.Cache
- userAgent string
- client *http.Client
-}
-
-// NewRobotsChecker creates a new robots.txt checker
-func NewRobotsChecker(userAgent string) *RobotsChecker {
- return &RobotsChecker{
- cache: cache.New(24*time.Hour, 1*time.Hour), // Cache robots.txt for 24 hours
- userAgent: userAgent,
- client: &http.Client{
- Timeout: 10 * time.Second,
- },
- }
-}
-
-// CanFetch checks if the URL can be fetched according to robots.txt
-// Returns (allowed bool, crawlDelay time.Duration, error)
-func (rc *RobotsChecker) CanFetch(ctx context.Context, urlStr string) (bool, time.Duration, error) {
- parsedURL, err := url.Parse(urlStr)
- if err != nil {
- return false, 0, fmt.Errorf("invalid URL: %w", err)
- }
-
- domain := parsedURL.Scheme + "://" + parsedURL.Host
- robotsURL := domain + "/robots.txt"
-
- // Check cache first
- if cached, found := rc.cache.Get(domain); found {
- robotsData := cached.(*robotstxt.RobotsData)
- group := robotsData.FindGroup(rc.userAgent)
- allowed := group.Test(parsedURL.Path)
- crawlDelay := rc.getCrawlDelay(group)
- return allowed, crawlDelay, nil
- }
-
- // Fetch robots.txt
- req, err := http.NewRequestWithContext(ctx, "GET", robotsURL, nil)
- if err != nil {
- return false, 0, fmt.Errorf("failed to create request: %w", err)
- }
-
- req.Header.Set("User-Agent", rc.userAgent)
-
- resp, err := rc.client.Do(req)
- if err != nil {
- // If robots.txt doesn't exist or network error, allow by default
- return true, 1 * time.Second, nil
- }
- defer resp.Body.Close()
-
- // If robots.txt returns 404 or other error, allow by default
- if resp.StatusCode != http.StatusOK {
- return true, 1 * time.Second, nil
- }
-
- // Read and parse robots.txt
- body, err := io.ReadAll(io.LimitReader(resp.Body, 1*1024*1024)) // Max 1MB
- if err != nil {
- return true, 1 * time.Second, nil
- }
-
- robotsData, err := robotstxt.FromBytes(body)
- if err != nil {
- // If parsing fails, be conservative and allow
- return true, 1 * time.Second, nil
- }
-
- // Cache the robots.txt data
- rc.cache.Set(domain, robotsData, cache.DefaultExpiration)
-
- // Check if path is allowed
- group := robotsData.FindGroup(rc.userAgent)
- allowed := group.Test(parsedURL.Path)
- crawlDelay := rc.getCrawlDelay(group)
-
- return allowed, crawlDelay, nil
-}
-
-// getCrawlDelay extracts crawl delay from robots.txt group
-func (rc *RobotsChecker) getCrawlDelay(group *robotstxt.Group) time.Duration {
- if group.CrawlDelay > 0 {
- delay := time.Duration(group.CrawlDelay) * time.Second
- // Cap at maximum 10 seconds
- if delay > 10*time.Second {
- delay = 10 * time.Second
- }
- return delay
- }
-
- // Default to 1 second if no crawl delay specified
- return 1 * time.Second
-}
-
-// SetUserAgent updates the user agent string
-func (rc *RobotsChecker) SetUserAgent(userAgent string) {
- rc.userAgent = userAgent
-}
diff --git a/backend/internal/services/scraper_service.go b/backend/internal/services/scraper_service.go
deleted file mode 100644
index 56c4fb4e..00000000
--- a/backend/internal/services/scraper_service.go
+++ /dev/null
@@ -1,232 +0,0 @@
-package services
-
-import (
- "bytes"
- "context"
- "fmt"
- "log"
- "net/url"
- "strings"
- "sync"
- "time"
-
- "github.com/markusmobius/go-trafilatura"
- cache "github.com/patrickmn/go-cache"
-)
-
-const (
- defaultUserAgent = "ClaraVerse-Bot/1.0 (+https://claraverse.example.com/bot)"
- defaultMaxBodySize = 10 * 1024 * 1024 // 10MB
- defaultMaxConcurrent = 10
- defaultGlobalRate = 10.0 // requests per second
- defaultPerUserRate = 5.0 // requests per second
-)
-
-// ScraperService handles web scraping operations
-type ScraperService struct {
- client *ScraperClient
- rateLimiter *RateLimiter
- robotsChecker *RobotsChecker
- contentCache *cache.Cache
- resourceMgr *ResourceManager
-}
-
-var (
- scraperInstance *ScraperService
- scraperOnce sync.Once
-)
-
-// GetScraperService returns the singleton scraper service instance
-func GetScraperService() *ScraperService {
- scraperOnce.Do(func() {
- scraperInstance = &ScraperService{
- client: NewScraperClient(),
- rateLimiter: NewRateLimiter(defaultGlobalRate, defaultPerUserRate),
- robotsChecker: NewRobotsChecker(defaultUserAgent),
- contentCache: cache.New(1*time.Hour, 10*time.Minute), // Cache for 1 hour
- resourceMgr: NewResourceManager(defaultMaxConcurrent, defaultMaxBodySize),
- }
-
- log.Printf("✅ [SCRAPER] Service initialized: max_concurrent=%d, global_rate=%.1f req/s",
- defaultMaxConcurrent, defaultGlobalRate)
- })
- return scraperInstance
-}
-
-// ScrapeURL scrapes a web page and returns clean content
-func (s *ScraperService) ScrapeURL(ctx context.Context, urlStr, format string, maxLength int, userID string) (string, error) {
- startTime := time.Now()
-
- // 1. Validate URL
- if err := s.validateURL(urlStr); err != nil {
- return "", err
- }
-
- parsedURL, err := url.Parse(urlStr)
- if err != nil {
- return "", fmt.Errorf("invalid URL: %w", err)
- }
-
- domain := parsedURL.Host
-
- // 2. Check cache
- cacheKey := s.getCacheKey(urlStr, format)
- if cached, found := s.contentCache.Get(cacheKey); found {
- log.Printf("✅ [SCRAPER] Cache hit for URL: %s (latency: %dms)",
- urlStr, time.Since(startTime).Milliseconds())
- return cached.(string), nil
- }
-
- // 3. Check robots.txt
- allowed, crawlDelay, err := s.robotsChecker.CanFetch(ctx, urlStr)
- if err != nil {
- log.Printf("⚠️ [SCRAPER] Failed to check robots.txt for %s: %v", urlStr, err)
- // Continue anyway with default delay
- crawlDelay = 1 * time.Second
- }
-
- if !allowed {
- return "", fmt.Errorf("access blocked by robots.txt for: %s", urlStr)
- }
-
- // 4. Apply rate limiting
- if err := s.rateLimiter.WaitWithCrawlDelay(ctx, userID, domain, crawlDelay); err != nil {
- return "", fmt.Errorf("rate limit error: %w", err)
- }
-
- // 5. Acquire resource semaphore
- if err := s.resourceMgr.Acquire(ctx); err != nil {
- return "", fmt.Errorf("resource limit reached, try again later: %w", err)
- }
- defer s.resourceMgr.Release()
-
- // 6. Fetch URL
- resp, err := s.client.Get(ctx, urlStr)
- if err != nil {
- log.Printf("❌ [SCRAPER] Failed to fetch URL %s: %v", urlStr, err)
- return "", fmt.Errorf("failed to fetch URL: %w", err)
- }
- defer resp.Body.Close()
-
- // 7. Check HTTP status
- if resp.StatusCode != 200 {
- return "", fmt.Errorf("HTTP error %d: %s", resp.StatusCode, resp.Status)
- }
-
- // 8. Check content type
- contentType := resp.Header.Get("Content-Type")
- if !s.isSupportedContentType(contentType) {
- return "", fmt.Errorf("unsupported content type: %s", contentType)
- }
-
- // 9. Read body with size limit
- body, err := s.resourceMgr.ReadBody(resp.Body)
- if err != nil {
- return "", fmt.Errorf("failed to read response: %w", err)
- }
-
- // 10. Extract main content using trafilatura
- opts := trafilatura.Options{
- OriginalURL: parsedURL,
- }
-
- result, err := trafilatura.Extract(bytes.NewReader(body), opts)
- if err != nil {
- return "", fmt.Errorf("failed to extract content: %w", err)
- }
-
- if result == nil || result.ContentText == "" {
- return "", fmt.Errorf("no content extracted from page")
- }
-
- // 11. Use extracted content (already plain text or markdown)
- content := result.ContentText
-
- // 12. Apply length limit
- if len(content) > maxLength {
- content = content[:maxLength] + "\n\n[Content truncated due to length limit]"
- }
-
- // 13. Add metadata header
- metadata := fmt.Sprintf("# %s\n\n", result.Metadata.Title)
- if result.Metadata.Author != "" {
- metadata += fmt.Sprintf("**Author:** %s \n", result.Metadata.Author)
- }
- if !result.Metadata.Date.IsZero() {
- metadata += fmt.Sprintf("**Published:** %s \n", result.Metadata.Date.Format("January 2, 2006"))
- }
- metadata += fmt.Sprintf("**Source:** %s \n", urlStr)
- metadata += "\n---\n\n"
-
- finalContent := metadata + content
-
- // 14. Cache result
- s.contentCache.Set(cacheKey, finalContent, cache.DefaultExpiration)
-
- latency := time.Since(startTime).Milliseconds()
- log.Printf("✅ [SCRAPER] Successfully scraped URL: %s (latency: %dms, length: %d chars)",
- urlStr, latency, len(finalContent))
-
- return finalContent, nil
-}
-
-// validateURL checks if the URL is safe to scrape (SSRF protection)
-func (s *ScraperService) validateURL(urlStr string) error {
- parsedURL, err := url.Parse(urlStr)
- if err != nil {
- return fmt.Errorf("invalid URL format: %w", err)
- }
-
- // Only allow HTTP and HTTPS
- if parsedURL.Scheme != "http" && parsedURL.Scheme != "https" {
- return fmt.Errorf("only HTTP/HTTPS URLs are supported, got: %s", parsedURL.Scheme)
- }
-
- hostname := strings.ToLower(parsedURL.Hostname())
-
- // Block localhost
- if hostname == "localhost" || hostname == "127.0.0.1" || hostname == "::1" {
- return fmt.Errorf("localhost URLs are not allowed")
- }
-
- // Block private IP ranges
- privateRanges := []string{
- "192.168.", "10.", "172.16.", "172.17.", "172.18.", "172.19.",
- "172.20.", "172.21.", "172.22.", "172.23.", "172.24.", "172.25.",
- "172.26.", "172.27.", "172.28.", "172.29.", "172.30.", "172.31.",
- "169.254.", // Link-local
- "fd", // IPv6 private
- }
-
- for _, prefix := range privateRanges {
- if strings.HasPrefix(hostname, prefix) {
- return fmt.Errorf("private IP addresses are not allowed")
- }
- }
-
- return nil
-}
-
-// isSupportedContentType checks if the content type is supported
-func (s *ScraperService) isSupportedContentType(contentType string) bool {
- contentType = strings.ToLower(contentType)
-
- supported := []string{
- "text/html",
- "text/plain",
- "application/xhtml+xml",
- }
-
- for _, ct := range supported {
- if strings.Contains(contentType, ct) {
- return true
- }
- }
-
- return false
-}
-
-// getCacheKey generates a cache key for URL and format
-func (s *ScraperService) getCacheKey(urlStr, format string) string {
- return fmt.Sprintf("%s:%s", urlStr, format)
-}
diff --git a/backend/internal/services/stream_buffer_service.go b/backend/internal/services/stream_buffer_service.go
deleted file mode 100644
index 52b64af2..00000000
--- a/backend/internal/services/stream_buffer_service.go
+++ /dev/null
@@ -1,389 +0,0 @@
-package services
-
-import (
- "errors"
- "log"
- "strings"
- "sync"
- "time"
-)
-
-// Stream buffer constants for production safety
-const (
- MaxChunksPerBuffer = 10000 // Prevent memory exhaustion
- MaxBufferSize = 1 << 20 // 1MB max per buffer
- DefaultBufferTTL = 2 * time.Minute
- CleanupInterval = 30 * time.Second
-)
-
-// Error types for stream buffer operations
-var (
- ErrBufferNotFound = errors.New("stream buffer not found")
- ErrBufferFull = errors.New("stream buffer full: max chunks exceeded")
- ErrBufferSizeExceeded = errors.New("stream buffer size exceeded")
- ErrResumeTooFast = errors.New("resume rate limit exceeded")
-)
-
-// BufferedMessage represents a message that needs to be delivered on reconnect
-type BufferedMessage struct {
- Type string `json:"type"`
- Content string `json:"content,omitempty"`
- ToolName string `json:"tool_name,omitempty"`
- ToolDisplayName string `json:"tool_display_name,omitempty"`
- ToolIcon string `json:"tool_icon,omitempty"`
- ToolDescription string `json:"tool_description,omitempty"`
- Status string `json:"status,omitempty"`
- Arguments map[string]interface{} `json:"arguments,omitempty"`
- Result string `json:"result,omitempty"`
- Plots interface{} `json:"plots,omitempty"` // For image artifacts
- Delivered bool `json:"-"` // Track if already delivered (not serialized)
-}
-
-// StreamBuffer holds buffered chunks for a streaming conversation
-type StreamBuffer struct {
- ConversationID string
- UserID string
- ConnID string // Original connection ID
- Chunks []string // Buffered text chunks
- PendingMessages []BufferedMessage // Important messages (tool_result, etc.) to deliver on reconnect
- TotalSize int // Current total size of all chunks
- IsComplete bool // Generation finished?
- FullContent string // Full content if complete
- CreatedAt time.Time
- LastChunkAt time.Time // Last chunk received time
- ResumeCount int // Track resume attempts
- LastResume time.Time // Prevent rapid resume spam
- mutex sync.Mutex
-}
-
-// StreamBufferService manages stream buffers for disconnected clients
-type StreamBufferService struct {
- buffers map[string]*StreamBuffer // conversationID -> buffer
- mutex sync.RWMutex
- ttl time.Duration
- cleanupTick *time.Ticker
- done chan struct{}
-}
-
-// NewStreamBufferService creates a new stream buffer service
-func NewStreamBufferService() *StreamBufferService {
- svc := &StreamBufferService{
- buffers: make(map[string]*StreamBuffer),
- ttl: DefaultBufferTTL,
- cleanupTick: time.NewTicker(CleanupInterval),
- done: make(chan struct{}),
- }
- go svc.cleanupLoop()
- log.Println("📦 StreamBufferService initialized")
- return svc
-}
-
-// cleanupLoop periodically removes expired buffers
-func (s *StreamBufferService) cleanupLoop() {
- for {
- select {
- case <-s.done:
- return
- case <-s.cleanupTick.C:
- s.cleanup()
- }
- }
-}
-
-// cleanup removes expired buffers
-func (s *StreamBufferService) cleanup() {
- s.mutex.Lock()
- defer s.mutex.Unlock()
-
- now := time.Now()
- expired := 0
- for convID, buf := range s.buffers {
- if now.Sub(buf.CreatedAt) > s.ttl {
- delete(s.buffers, convID)
- expired++
- log.Printf("📦 Buffer expired for conversation %s", convID)
- }
- }
- if expired > 0 {
- log.Printf("📦 Cleaned up %d expired buffers, %d active", expired, len(s.buffers))
- }
-}
-
-// Shutdown gracefully shuts down the service
-func (s *StreamBufferService) Shutdown() {
- close(s.done)
- s.cleanupTick.Stop()
- s.mutex.Lock()
- defer s.mutex.Unlock()
- s.buffers = nil
- log.Println("📦 StreamBufferService shutdown complete")
-}
-
-// CreateBuffer creates a new buffer for a conversation
-func (s *StreamBufferService) CreateBuffer(conversationID, userID, connID string) {
- s.mutex.Lock()
- defer s.mutex.Unlock()
-
- // If buffer already exists, don't overwrite (prevents race conditions)
- if _, exists := s.buffers[conversationID]; exists {
- log.Printf("📦 Buffer already exists for conversation %s", conversationID)
- return
- }
-
- s.buffers[conversationID] = &StreamBuffer{
- ConversationID: conversationID,
- UserID: userID,
- ConnID: connID,
- Chunks: make([]string, 0, 100), // Pre-allocate for performance
- PendingMessages: make([]BufferedMessage, 0, 10), // For tool results, etc.
- TotalSize: 0,
- CreatedAt: time.Now(),
- LastChunkAt: time.Now(),
- }
- log.Printf("📦 Buffer created for conversation %s (user: %s)", conversationID, userID)
-}
-
-// AppendChunk adds a chunk to the buffer
-func (s *StreamBufferService) AppendChunk(conversationID, chunk string) error {
- s.mutex.RLock()
- buf, exists := s.buffers[conversationID]
- s.mutex.RUnlock()
-
- if !exists {
- // Buffer doesn't exist - this is normal if streaming started before disconnect
- return nil
- }
-
- buf.mutex.Lock()
- defer buf.mutex.Unlock()
-
- // Safety limits
- if len(buf.Chunks) >= MaxChunksPerBuffer {
- log.Printf("⚠️ Buffer full for conversation %s (max chunks: %d)", conversationID, MaxChunksPerBuffer)
- return ErrBufferFull
- }
-
- if buf.TotalSize+len(chunk) > MaxBufferSize {
- log.Printf("⚠️ Buffer size exceeded for conversation %s (max: %d bytes)", conversationID, MaxBufferSize)
- return ErrBufferSizeExceeded
- }
-
- buf.Chunks = append(buf.Chunks, chunk)
- buf.TotalSize += len(chunk)
- buf.LastChunkAt = time.Now()
-
- return nil
-}
-
-// AppendMessage adds an important message to the buffer for delivery on reconnect
-// This is used for tool_result, artifacts, and other critical messages that shouldn't be lost
-func (s *StreamBufferService) AppendMessage(conversationID string, msg BufferedMessage) error {
- s.mutex.RLock()
- buf, exists := s.buffers[conversationID]
- s.mutex.RUnlock()
-
- if !exists {
- // Buffer doesn't exist - this is normal if streaming started before disconnect
- return nil
- }
-
- buf.mutex.Lock()
- defer buf.mutex.Unlock()
-
- // Limit pending messages to prevent memory issues
- if len(buf.PendingMessages) >= 50 {
- log.Printf("⚠️ Too many pending messages for conversation %s", conversationID)
- return ErrBufferFull
- }
-
- buf.PendingMessages = append(buf.PendingMessages, msg)
- buf.LastChunkAt = time.Now()
-
- log.Printf("📦 Buffered message type=%s for conversation %s (pending: %d)",
- msg.Type, conversationID, len(buf.PendingMessages))
-
- return nil
-}
-
-// MarkMessagesDelivered marks all pending messages as delivered to prevent duplicate replay
-func (s *StreamBufferService) MarkMessagesDelivered(conversationID string) {
- s.mutex.RLock()
- buf, exists := s.buffers[conversationID]
- s.mutex.RUnlock()
-
- if !exists {
- return
- }
-
- buf.mutex.Lock()
- defer buf.mutex.Unlock()
-
- for i := range buf.PendingMessages {
- buf.PendingMessages[i].Delivered = true
- }
- log.Printf("📦 Marked %d pending messages as delivered for conversation %s", len(buf.PendingMessages), conversationID)
-}
-
-// MarkComplete marks the buffer as complete with the full content
-func (s *StreamBufferService) MarkComplete(conversationID, fullContent string) {
- s.mutex.RLock()
- buf, exists := s.buffers[conversationID]
- s.mutex.RUnlock()
-
- if !exists {
- return
- }
-
- buf.mutex.Lock()
- defer buf.mutex.Unlock()
-
- buf.IsComplete = true
- buf.FullContent = fullContent
- log.Printf("📦 Buffer marked complete for conversation %s (size: %d bytes)", conversationID, len(fullContent))
-}
-
-// GetBuffer retrieves a buffer without clearing it (allows multiple resume attempts)
-func (s *StreamBufferService) GetBuffer(conversationID string) (*StreamBuffer, error) {
- s.mutex.Lock()
- defer s.mutex.Unlock()
-
- buf, exists := s.buffers[conversationID]
- if !exists {
- return nil, ErrBufferNotFound
- }
-
- // Rate limit: 1 resume per second
- if time.Since(buf.LastResume) < time.Second {
- return nil, ErrResumeTooFast
- }
-
- buf.ResumeCount++
- buf.LastResume = time.Now()
-
- log.Printf("📦 Buffer retrieved for conversation %s (resume #%d, chunks: %d)",
- conversationID, buf.ResumeCount, len(buf.Chunks))
-
- return buf, nil
-}
-
-// ClearBuffer removes a buffer after successful resume
-func (s *StreamBufferService) ClearBuffer(conversationID string) {
- s.mutex.Lock()
- defer s.mutex.Unlock()
-
- if _, exists := s.buffers[conversationID]; exists {
- delete(s.buffers, conversationID)
- log.Printf("📦 Buffer cleared for conversation %s", conversationID)
- }
-}
-
-// HasBuffer checks if a buffer exists for a conversation
-func (s *StreamBufferService) HasBuffer(conversationID string) bool {
- s.mutex.RLock()
- defer s.mutex.RUnlock()
- _, exists := s.buffers[conversationID]
- return exists
-}
-
-// GetBufferStats returns statistics about the buffer service
-func (s *StreamBufferService) GetBufferStats() map[string]interface{} {
- s.mutex.RLock()
- defer s.mutex.RUnlock()
-
- totalChunks := 0
- totalSize := 0
- for _, buf := range s.buffers {
- buf.mutex.Lock()
- totalChunks += len(buf.Chunks)
- totalSize += buf.TotalSize
- buf.mutex.Unlock()
- }
-
- return map[string]interface{}{
- "active_buffers": len(s.buffers),
- "total_chunks": totalChunks,
- "total_size": totalSize,
- }
-}
-
-// GetBufferInfo returns detailed info about a specific buffer (for debugging)
-func (s *StreamBufferService) GetBufferInfo(conversationID string) map[string]interface{} {
- s.mutex.RLock()
- buf, exists := s.buffers[conversationID]
- s.mutex.RUnlock()
-
- if !exists {
- return nil
- }
-
- buf.mutex.Lock()
- defer buf.mutex.Unlock()
-
- return map[string]interface{}{
- "conversation_id": buf.ConversationID,
- "user_id": buf.UserID,
- "conn_id": buf.ConnID,
- "chunk_count": len(buf.Chunks),
- "total_size": buf.TotalSize,
- "is_complete": buf.IsComplete,
- "created_at": buf.CreatedAt,
- "last_chunk_at": buf.LastChunkAt,
- "resume_count": buf.ResumeCount,
- "age_seconds": time.Since(buf.CreatedAt).Seconds(),
- }
-}
-
-// BufferData represents the data needed for stream resume
-type BufferData struct {
- ConversationID string
- UserID string
- CombinedChunks string
- IsComplete bool
- ChunkCount int
- PendingMessages []BufferedMessage // Tool results, artifacts, etc. to replay
-}
-
-// GetBufferData safely retrieves buffer data for resume operations
-func (s *StreamBufferService) GetBufferData(conversationID string) (*BufferData, error) {
- s.mutex.Lock()
- defer s.mutex.Unlock()
-
- buf, exists := s.buffers[conversationID]
- if !exists {
- return nil, ErrBufferNotFound
- }
-
- // Rate limit: 1 resume per second
- if time.Since(buf.LastResume) < time.Second {
- return nil, ErrResumeTooFast
- }
-
- buf.ResumeCount++
- buf.LastResume = time.Now()
-
- buf.mutex.Lock()
- defer buf.mutex.Unlock()
-
- // Combine all chunks
- var combined strings.Builder
- for _, chunk := range buf.Chunks {
- combined.WriteString(chunk)
- }
-
- // Copy only undelivered pending messages
- var pendingMsgs []BufferedMessage
- for _, msg := range buf.PendingMessages {
- if !msg.Delivered {
- pendingMsgs = append(pendingMsgs, msg)
- }
- }
-
- return &BufferData{
- ConversationID: buf.ConversationID,
- UserID: buf.UserID,
- CombinedChunks: combined.String(),
- IsComplete: buf.IsComplete,
- ChunkCount: len(buf.Chunks),
- PendingMessages: pendingMsgs,
- }, nil
-}
diff --git a/backend/internal/services/stream_buffer_service_test.go b/backend/internal/services/stream_buffer_service_test.go
deleted file mode 100644
index 943d65fc..00000000
--- a/backend/internal/services/stream_buffer_service_test.go
+++ /dev/null
@@ -1,286 +0,0 @@
-package services
-
-import (
- "fmt"
- "strings"
- "sync"
- "testing"
- "time"
-)
-
-func TestStreamBuffer_CreateAndAppend(t *testing.T) {
- svc := NewStreamBufferService()
- defer svc.Shutdown()
-
- convID := "test-conv-123"
- userID := "user-456"
- connID := "conn-789"
-
- // Create buffer
- svc.CreateBuffer(convID, userID, connID)
-
- // Verify buffer exists
- if !svc.HasBuffer(convID) {
- t.Fatal("Buffer should exist after creation")
- }
-
- // Append chunks
- chunks := []string{"Hello", " ", "World", "!"}
- for _, chunk := range chunks {
- err := svc.AppendChunk(convID, chunk)
- if err != nil {
- t.Fatalf("Failed to append chunk: %v", err)
- }
- }
-
- // Get buffer data
- data, err := svc.GetBufferData(convID)
- if err != nil {
- t.Fatalf("Failed to get buffer data: %v", err)
- }
-
- expected := "Hello World!"
- if data.CombinedChunks != expected {
- t.Errorf("Expected combined chunks %q, got %q", expected, data.CombinedChunks)
- }
-
- if data.ChunkCount != 4 {
- t.Errorf("Expected 4 chunks, got %d", data.ChunkCount)
- }
-
- if data.UserID != userID {
- t.Errorf("Expected userID %q, got %q", userID, data.UserID)
- }
-}
-
-func TestStreamBuffer_MarkComplete(t *testing.T) {
- svc := NewStreamBufferService()
- defer svc.Shutdown()
-
- convID := "test-conv-complete"
- userID := "user-456"
- connID := "conn-789"
-
- svc.CreateBuffer(convID, userID, connID)
- svc.AppendChunk(convID, "Test content")
- svc.MarkComplete(convID, "Full test content")
-
- data, err := svc.GetBufferData(convID)
- if err != nil {
- t.Fatalf("Failed to get buffer data: %v", err)
- }
-
- if !data.IsComplete {
- t.Error("Buffer should be marked as complete")
- }
-}
-
-func TestStreamBuffer_ClearBuffer(t *testing.T) {
- svc := NewStreamBufferService()
- defer svc.Shutdown()
-
- convID := "test-conv-clear"
- svc.CreateBuffer(convID, "user", "conn")
- svc.AppendChunk(convID, "Test")
-
- // Verify exists
- if !svc.HasBuffer(convID) {
- t.Fatal("Buffer should exist before clear")
- }
-
- // Clear
- svc.ClearBuffer(convID)
-
- // Verify gone
- if svc.HasBuffer(convID) {
- t.Error("Buffer should not exist after clear")
- }
-
- // GetBufferData should return error
- _, err := svc.GetBufferData(convID)
- if err != ErrBufferNotFound {
- t.Errorf("Expected ErrBufferNotFound, got %v", err)
- }
-}
-
-func TestStreamBuffer_MemoryLimits(t *testing.T) {
- svc := NewStreamBufferService()
- defer svc.Shutdown()
-
- svc.CreateBuffer("conv-1", "user-1", "conn-1")
-
- // Try to exceed max chunks
- for i := 0; i < MaxChunksPerBuffer+10; i++ {
- err := svc.AppendChunk("conv-1", "x")
- if i >= MaxChunksPerBuffer {
- if err != ErrBufferFull {
- t.Errorf("Expected ErrBufferFull at chunk %d, got %v", i, err)
- }
- } else {
- if err != nil {
- t.Errorf("Unexpected error at chunk %d: %v", i, err)
- }
- }
- }
-}
-
-func TestStreamBuffer_SizeLimit(t *testing.T) {
- svc := NewStreamBufferService()
- defer svc.Shutdown()
-
- svc.CreateBuffer("conv-size", "user", "conn")
-
- // Create a large chunk that exceeds the size limit
- largeChunk := strings.Repeat("x", MaxBufferSize+1)
- err := svc.AppendChunk("conv-size", largeChunk)
- if err != ErrBufferSizeExceeded {
- t.Errorf("Expected ErrBufferSizeExceeded, got %v", err)
- }
-}
-
-func TestStreamBuffer_RateLimiting(t *testing.T) {
- svc := NewStreamBufferService()
- defer svc.Shutdown()
-
- svc.CreateBuffer("conv-rate", "user", "conn")
- svc.AppendChunk("conv-rate", "test")
-
- // First GetBufferData should succeed
- _, err := svc.GetBufferData("conv-rate")
- if err != nil {
- t.Fatalf("First GetBufferData should succeed: %v", err)
- }
-
- // Immediate second call should be rate limited
- _, err = svc.GetBufferData("conv-rate")
- if err != ErrResumeTooFast {
- t.Errorf("Expected ErrResumeTooFast, got %v", err)
- }
-
- // Wait for rate limit to expire
- time.Sleep(1100 * time.Millisecond)
-
- // Should succeed now
- _, err = svc.GetBufferData("conv-rate")
- if err != nil {
- t.Errorf("GetBufferData should succeed after rate limit: %v", err)
- }
-}
-
-func TestStreamBuffer_ConcurrentAccess(t *testing.T) {
- svc := NewStreamBufferService()
- defer svc.Shutdown()
-
- svc.CreateBuffer("conv-concurrent", "user", "conn")
-
- // Concurrent writes
- var wg sync.WaitGroup
- numGoroutines := 100
-
- for i := 0; i < numGoroutines; i++ {
- wg.Add(1)
- go func(idx int) {
- defer wg.Done()
- svc.AppendChunk("conv-concurrent", fmt.Sprintf("chunk-%d-", idx))
- }(i)
- }
- wg.Wait()
-
- data, err := svc.GetBufferData("conv-concurrent")
- if err != nil {
- t.Fatalf("Failed to get buffer data: %v", err)
- }
-
- if data.ChunkCount != numGoroutines {
- t.Errorf("Expected %d chunks, got %d", numGoroutines, data.ChunkCount)
- }
-}
-
-func TestStreamBuffer_NonExistentBuffer(t *testing.T) {
- svc := NewStreamBufferService()
- defer svc.Shutdown()
-
- // Append to non-existent buffer should not error (just no-op)
- err := svc.AppendChunk("non-existent", "test")
- if err != nil {
- t.Errorf("AppendChunk to non-existent buffer should not error: %v", err)
- }
-
- // GetBufferData should return error
- _, err = svc.GetBufferData("non-existent")
- if err != ErrBufferNotFound {
- t.Errorf("Expected ErrBufferNotFound, got %v", err)
- }
-}
-
-func TestStreamBuffer_DuplicateCreate(t *testing.T) {
- svc := NewStreamBufferService()
- defer svc.Shutdown()
-
- convID := "conv-duplicate"
-
- // Create buffer
- svc.CreateBuffer(convID, "user-1", "conn-1")
- svc.AppendChunk(convID, "original")
-
- // Try to create again - should not overwrite
- svc.CreateBuffer(convID, "user-2", "conn-2")
-
- data, err := svc.GetBufferData(convID)
- if err != nil {
- t.Fatalf("Failed to get buffer data: %v", err)
- }
-
- // Should still have original user
- if data.UserID != "user-1" {
- t.Errorf("Buffer should not be overwritten, expected user-1, got %s", data.UserID)
- }
-
- // Should still have original content
- if data.CombinedChunks != "original" {
- t.Errorf("Buffer content should not be overwritten")
- }
-}
-
-func TestStreamBuffer_Stats(t *testing.T) {
- svc := NewStreamBufferService()
- defer svc.Shutdown()
-
- // Create multiple buffers
- svc.CreateBuffer("conv-1", "user", "conn")
- svc.CreateBuffer("conv-2", "user", "conn")
- svc.CreateBuffer("conv-3", "user", "conn")
-
- svc.AppendChunk("conv-1", "hello")
- svc.AppendChunk("conv-2", "world")
- svc.AppendChunk("conv-3", "!")
-
- stats := svc.GetBufferStats()
-
- activeBuffers := stats["active_buffers"].(int)
- if activeBuffers != 3 {
- t.Errorf("Expected 3 active buffers, got %d", activeBuffers)
- }
-
- totalChunks := stats["total_chunks"].(int)
- if totalChunks != 3 {
- t.Errorf("Expected 3 total chunks, got %d", totalChunks)
- }
-}
-
-func TestStreamBuffer_Shutdown(t *testing.T) {
- svc := NewStreamBufferService()
- svc.CreateBuffer("conv-shutdown", "user", "conn")
- svc.AppendChunk("conv-shutdown", "test")
-
- // Verify exists
- if !svc.HasBuffer("conv-shutdown") {
- t.Fatal("Buffer should exist before shutdown")
- }
-
- // Shutdown
- svc.Shutdown()
-
- // After shutdown, HasBuffer should return false (buffers is nil)
- // This is a simple check - in production, you'd want more robust handling
-}
diff --git a/backend/internal/services/tier_service.go b/backend/internal/services/tier_service.go
deleted file mode 100644
index a9719718..00000000
--- a/backend/internal/services/tier_service.go
+++ /dev/null
@@ -1,243 +0,0 @@
-package services
-
-import (
- "claraverse/internal/database"
- "claraverse/internal/models"
- "context"
- "log"
- "sync"
- "time"
-
- "go.mongodb.org/mongo-driver/bson"
-)
-
-// CacheEntry stores cached tier with expiration info for TTL-based invalidation
-type CacheEntry struct {
- Tier string
- ExpiresAt *time.Time // For promo users, this is subscriptionExpiresAt; nil for regular users
- CachedAt time.Time
-}
-
-// TierService manages subscription tier limits and lookups
-type TierService struct {
- mongoDB *database.MongoDB
- cache map[string]CacheEntry // userID -> CacheEntry with TTL info
- mu sync.RWMutex
- defaultTTL time.Duration // Default cache TTL for non-promo users
-}
-
-// NewTierService creates a new tier service
-func NewTierService(mongoDB *database.MongoDB) *TierService {
- return &TierService{
- mongoDB: mongoDB,
- cache: make(map[string]CacheEntry),
- defaultTTL: 5 * time.Minute,
- }
-}
-
-// GetUserTier returns the subscription tier for a user
-func (s *TierService) GetUserTier(ctx context.Context, userID string) string {
- now := time.Now()
-
- // Check cache first
- s.mu.RLock()
- if entry, ok := s.cache[userID]; ok {
- // Check if cache entry is still valid
- // For promo users: check if promo has expired
- if entry.ExpiresAt != nil && entry.ExpiresAt.Before(now) {
- s.mu.RUnlock()
- // Promo expired - invalidate cache and re-fetch
- s.InvalidateCache(userID)
- log.Printf("🔄 [TIER] Promo expired for user %s, re-fetching tier", userID)
- return s.fetchAndCacheTier(ctx, userID)
- }
-
- // For non-promo users: check default TTL
- if entry.ExpiresAt == nil && now.Sub(entry.CachedAt) > s.defaultTTL {
- s.mu.RUnlock()
- // Cache TTL exceeded - re-fetch
- s.InvalidateCache(userID)
- return s.fetchAndCacheTier(ctx, userID)
- }
-
- s.mu.RUnlock()
- return entry.Tier
- }
- s.mu.RUnlock()
-
- return s.fetchAndCacheTier(ctx, userID)
-}
-
-// fetchAndCacheTier fetches the tier from database and caches it
-func (s *TierService) fetchAndCacheTier(ctx context.Context, userID string) string {
- // v2.0: Default to Pro tier (no payment system)
- tier := models.TierPro
- var expiresAt *time.Time
-
- // Check for admin-set tier overrides in MongoDB
- if s.mongoDB != nil {
- collection := s.mongoDB.Database().Collection("users")
-
- var user struct {
- SubscriptionTier string `bson:"subscriptionTier"`
- TierOverride string `bson:"tierOverride"` // Admin manual override
- }
-
- err := collection.FindOne(ctx, bson.M{"_id": userID}).Decode(&user)
- if err == nil {
- // Admin override takes priority
- if user.TierOverride != "" {
- tier = user.TierOverride
- } else if user.SubscriptionTier != "" {
- tier = user.SubscriptionTier
- }
- }
- }
-
- // Cache the result with expiration info
- s.mu.Lock()
- s.cache[userID] = CacheEntry{
- Tier: tier,
- ExpiresAt: expiresAt,
- CachedAt: time.Now(),
- }
- s.mu.Unlock()
-
- return tier
-}
-
-// GetLimits returns the limits for a user based on their tier
-func (s *TierService) GetLimits(ctx context.Context, userID string) models.TierLimits {
- // Get base tier
- tier := s.GetUserTier(ctx, userID)
- baseLimits := models.GetTierLimits(tier)
-
- // Check for granular limit overrides
- if s.mongoDB != nil {
- collection := s.mongoDB.Database().Collection("users")
-
- var user struct {
- LimitOverrides *models.TierLimits `bson:"limitOverrides"`
- }
-
- err := collection.FindOne(ctx, bson.M{"supabaseUserId": userID}).Decode(&user)
- if err == nil && user.LimitOverrides != nil {
- // Merge overrides with base limits
- return s.mergeLimits(baseLimits, *user.LimitOverrides)
- }
- }
-
- return baseLimits
-}
-
-// mergeLimits merges override limits with base limits
-// Non-zero override values replace base limits
-// Zero override values are ignored (use base limit)
-func (s *TierService) mergeLimits(base, override models.TierLimits) models.TierLimits {
- result := base // Start with base limits
-
- // Override each field if non-zero
- if override.MaxSchedules != 0 {
- result.MaxSchedules = override.MaxSchedules
- }
- if override.MaxAPIKeys != 0 {
- result.MaxAPIKeys = override.MaxAPIKeys
- }
- if override.RequestsPerMinute != 0 {
- result.RequestsPerMinute = override.RequestsPerMinute
- }
- if override.RequestsPerHour != 0 {
- result.RequestsPerHour = override.RequestsPerHour
- }
- if override.RetentionDays != 0 {
- result.RetentionDays = override.RetentionDays
- }
- if override.MaxExecutionsPerDay != 0 {
- result.MaxExecutionsPerDay = override.MaxExecutionsPerDay
- }
- if override.MaxMessagesPerMonth != 0 {
- result.MaxMessagesPerMonth = override.MaxMessagesPerMonth
- }
- if override.MaxFileUploadsPerDay != 0 {
- result.MaxFileUploadsPerDay = override.MaxFileUploadsPerDay
- }
- if override.MaxImageGensPerDay != 0 {
- result.MaxImageGensPerDay = override.MaxImageGensPerDay
- }
-
- return result
-}
-
-// InvalidateCache removes a user from the cache (call when tier changes)
-func (s *TierService) InvalidateCache(userID string) {
- s.mu.Lock()
- delete(s.cache, userID)
- s.mu.Unlock()
- log.Printf("🔄 [TIER] Invalidated cache for user %s", userID)
-}
-
-// CheckScheduleLimit checks if user can create another schedule
-func (s *TierService) CheckScheduleLimit(ctx context.Context, userID string, currentCount int64) bool {
- limits := s.GetLimits(ctx, userID)
- if limits.MaxSchedules < 0 {
- return true // Unlimited
- }
- return currentCount < int64(limits.MaxSchedules)
-}
-
-// CheckAPIKeyLimit checks if user can create another API key
-func (s *TierService) CheckAPIKeyLimit(ctx context.Context, userID string, currentCount int64) bool {
- limits := s.GetLimits(ctx, userID)
- if limits.MaxAPIKeys < 0 {
- return true // Unlimited
- }
- return currentCount < int64(limits.MaxAPIKeys)
-}
-
-// RateLimitConfig holds rate limit values
-type RateLimitConfig struct {
- RequestsPerMinute int64
- RequestsPerHour int64
-}
-
-// GetRateLimits returns the rate limit configuration for a user
-func (s *TierService) GetRateLimits(ctx context.Context, userID string) RateLimitConfig {
- limits := s.GetLimits(ctx, userID)
- return RateLimitConfig{
- RequestsPerMinute: limits.RequestsPerMinute,
- RequestsPerHour: limits.RequestsPerHour,
- }
-}
-
-// GetExecutionRetentionDays returns how long to keep execution history
-func (s *TierService) GetExecutionRetentionDays(ctx context.Context, userID string) int {
- limits := s.GetLimits(ctx, userID)
- return limits.RetentionDays
-}
-
-// CheckMessageLimit checks if user can send another message this month
-func (s *TierService) CheckMessageLimit(ctx context.Context, userID string, currentCount int64) bool {
- limits := s.GetLimits(ctx, userID)
- if limits.MaxMessagesPerMonth < 0 {
- return true // Unlimited
- }
- return currentCount < limits.MaxMessagesPerMonth
-}
-
-// CheckFileUploadLimit checks if user can upload another file today
-func (s *TierService) CheckFileUploadLimit(ctx context.Context, userID string, currentCount int64) bool {
- limits := s.GetLimits(ctx, userID)
- if limits.MaxFileUploadsPerDay < 0 {
- return true // Unlimited
- }
- return currentCount < limits.MaxFileUploadsPerDay
-}
-
-// CheckImageGenLimit checks if user can generate another image today
-func (s *TierService) CheckImageGenLimit(ctx context.Context, userID string, currentCount int64) bool {
- limits := s.GetLimits(ctx, userID)
- if limits.MaxImageGensPerDay < 0 {
- return true // Unlimited
- }
- return currentCount < limits.MaxImageGensPerDay
-}
diff --git a/backend/internal/services/tier_service_test.go b/backend/internal/services/tier_service_test.go
deleted file mode 100644
index f1393461..00000000
--- a/backend/internal/services/tier_service_test.go
+++ /dev/null
@@ -1,133 +0,0 @@
-package services
-
-import (
- "context"
- "testing"
-)
-
-func TestNewTierService(t *testing.T) {
- // Test without MongoDB (nil)
- service := NewTierService(nil)
- if service == nil {
- t.Fatal("Expected non-nil tier service")
- }
-}
-
-func TestTierService_GetUserTier_DefaultsToFree(t *testing.T) {
- service := NewTierService(nil)
- ctx := context.Background()
-
- tier := service.GetUserTier(ctx, "user-123")
- if tier != "pro" {
- t.Errorf("Expected 'pro' tier (v2.0 default), got %s", tier)
- }
-}
-
-func TestTierService_GetLimits(t *testing.T) {
- service := NewTierService(nil)
- ctx := context.Background()
-
- limits := service.GetLimits(ctx, "user-123")
-
- // Default to pro tier limits (v2.0 default)
- if limits.MaxSchedules != 50 {
- t.Errorf("Expected MaxSchedules 50, got %d", limits.MaxSchedules)
- }
-
- if limits.MaxAPIKeys != 50 {
- t.Errorf("Expected MaxAPIKeys 50, got %d", limits.MaxAPIKeys)
- }
-}
-
-func TestTierService_CheckScheduleLimit(t *testing.T) {
- service := NewTierService(nil)
- ctx := context.Background()
-
- tests := []struct {
- name string
- currentCount int64
- expected bool
- }{
- {"under limit", 3, true},
- {"at limit", 50, false}, // Pro tier limit is 50
- {"over limit", 100, false}, // Pro tier limit is 50
- }
-
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- result := service.CheckScheduleLimit(ctx, "user-123", tt.currentCount)
- if result != tt.expected {
- t.Errorf("Expected %v for currentCount %d, got %v", tt.expected, tt.currentCount, result)
- }
- })
- }
-}
-
-func TestTierService_CheckAPIKeyLimit(t *testing.T) {
- service := NewTierService(nil)
- ctx := context.Background()
-
- tests := []struct {
- name string
- currentCount int64
- expected bool
- }{
- {"under limit", 1, true},
- {"at limit", 50, false}, // Pro tier limit is 50
- {"over limit", 100, false}, // Pro tier limit is 50
- }
-
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- result := service.CheckAPIKeyLimit(ctx, "user-123", tt.currentCount)
- if result != tt.expected {
- t.Errorf("Expected %v for currentCount %d, got %v", tt.expected, tt.currentCount, result)
- }
- })
- }
-}
-
-func TestTierService_GetRateLimits(t *testing.T) {
- service := NewTierService(nil)
- ctx := context.Background()
-
- rateLimits := service.GetRateLimits(ctx, "user-123")
-
- // Default to pro tier rate limits (v2.0 default)
- if rateLimits.RequestsPerMinute != 300 {
- t.Errorf("Expected RequestsPerMinute 300, got %d", rateLimits.RequestsPerMinute)
- }
-
- if rateLimits.RequestsPerHour != 5000 {
- t.Errorf("Expected RequestsPerHour 5000, got %d", rateLimits.RequestsPerHour)
- }
-}
-
-func TestTierService_GetExecutionRetentionDays(t *testing.T) {
- service := NewTierService(nil)
- ctx := context.Background()
-
- days := service.GetExecutionRetentionDays(ctx, "user-123")
-
- // Default to free tier retention
- if days != 30 {
- t.Errorf("Expected 30 days retention, got %d", days)
- }
-}
-
-func TestTierService_InvalidateCache(t *testing.T) {
- service := NewTierService(nil)
- ctx := context.Background()
-
- // Get tier to populate cache
- _ = service.GetUserTier(ctx, "user-123")
-
- // Invalidate cache
- service.InvalidateCache("user-123")
-
- // Should still return pro (v2.0 default) but cache should be empty
- tier := service.GetUserTier(ctx, "user-123")
- if tier != "pro" {
- t.Errorf("Expected 'pro' tier after cache invalidation, got %s", tier)
- }
-}
diff --git a/backend/internal/services/tool_predictor_service.go b/backend/internal/services/tool_predictor_service.go
deleted file mode 100644
index 4a659c32..00000000
--- a/backend/internal/services/tool_predictor_service.go
+++ /dev/null
@@ -1,478 +0,0 @@
-package services
-
-import (
- "bytes"
- "claraverse/internal/database"
- "claraverse/internal/models"
- "context"
- "database/sql"
- "encoding/json"
- "fmt"
- "io"
- "log"
- "net/http"
- "strings"
- "time"
-)
-
-// ToolPredictorService handles dynamic tool selection for chat requests
-type ToolPredictorService struct {
- db *database.DB
- providerService *ProviderService
- chatService *ChatService
- defaultPredictorModel string // "gpt-4.1-mini"
-}
-
-// ToolPredictionResult represents selected tools from predictor
-type ToolPredictionResult struct {
- SelectedTools []string `json:"selected_tools"` // Array of tool names
- Reasoning string `json:"reasoning"`
-}
-
-// Tool prediction system prompt (adapted from WorkflowGeneratorV2)
-const ToolPredictionSystemPrompt = `You are a tool selection expert for Clara AI chat system. Analyze the user's message and select the MINIMUM set of tools needed to respond effectively.
-
-CRITICAL RULES:
-- Select ONLY tools that are DIRECTLY needed for THIS specific request
-- Most requests need 0-3 tools. Rarely should you select more than 5 tools
-- If no tools are needed (general conversation, advice, explanation), return empty array
-- Don't over-select "just in case" - be precise and minimal
-
-WHEN TO SELECT TOOLS:
-- Search tools: User asks for current info, news, research, "look up", "search for"
-- Time tools: User asks "what time", "current date", mentions time-sensitive info
-- File tools: User mentions reading/processing files (CSV, PDF, etc.)
-- Communication tools: User wants to send message to specific platform (Discord, Slack, email)
-- Calculation tools: Complex math, data analysis
-- API tools: Interacting with specific services (GitHub, Jira, etc.)
-
-WHEN NOT TO SELECT TOOLS:
-- General questions, explanations, advice, brainstorming
-- Coding help (unless explicitly needs to search docs/internet)
-- Writing tasks (emails, documents, summaries of provided text)
-- Conversation, jokes, casual chat
-
-Return JSON with selected_tools array (just tool names) and reasoning.`
-
-// toolPredictionSchema defines structured output for tool selection
-var toolPredictionSchema = map[string]interface{}{
- "type": "object",
- "properties": map[string]interface{}{
- "selected_tools": map[string]interface{}{
- "type": "array",
- "items": map[string]interface{}{
- "type": "string",
- "description": "Tool name from available tools",
- },
- "description": "Array of tool names needed for this request",
- },
- "reasoning": map[string]interface{}{
- "type": "string",
- "description": "Brief explanation of tool selection",
- },
- },
- "required": []string{"selected_tools", "reasoning"},
- "additionalProperties": false,
-}
-
-// NewToolPredictorService creates a new tool predictor service
-func NewToolPredictorService(
- db *database.DB,
- providerService *ProviderService,
- chatService *ChatService,
-) *ToolPredictorService {
- service := &ToolPredictorService{
- db: db,
- providerService: providerService,
- chatService: chatService,
- }
-
- // Dynamically select first available smart_tool_router model
- var modelID string
- err := db.QueryRow(`
- SELECT m.id
- FROM models m
- WHERE m.smart_tool_router = 1 AND m.is_visible = 1
- ORDER BY m.id ASC
- LIMIT 1
- `).Scan(&modelID)
-
- if err != nil {
- log.Printf("⚠️ [TOOL-PREDICTOR] No smart_tool_router models found, falling back to any available model")
- // Fallback: Use any available visible model
- err = db.QueryRow(`
- SELECT m.id
- FROM models m
- WHERE m.is_visible = 1
- ORDER BY m.id ASC
- LIMIT 1
- `).Scan(&modelID)
-
- if err != nil {
- log.Printf("❌ [TOOL-PREDICTOR] No models available in database at initialization")
- modelID = "" // Will be handled later when models are loaded
- }
- }
-
- service.defaultPredictorModel = modelID
- if modelID != "" {
- log.Printf("✅ [TOOL-PREDICTOR] Using default predictor model: %s", modelID)
- }
-
- return service
-}
-
-// PredictTools predicts which tools are needed for a user message
-// Returns selected tool definitions and error (nil on success)
-// On failure, returns nil (caller should use all tools as fallback)
-// conversationHistory: Recent conversation messages for better context-aware tool selection
-func (s *ToolPredictorService) PredictTools(
- ctx context.Context,
- userID string,
- userMessage string,
- availableTools []map[string]interface{},
- conversationHistory []map[string]interface{},
-) ([]map[string]interface{}, error) {
-
- // Get predictor model for user (or use default)
- predictorModelID, err := s.getPredictorModelForUser(ctx, userID)
- if err != nil {
- log.Printf("⚠️ [TOOL-PREDICTOR] Could not get predictor model: %v, using default", err)
- predictorModelID = s.defaultPredictorModel
- }
-
- // Get provider and model
- provider, actualModel, err := s.getProviderAndModel(predictorModelID)
- if err != nil {
- log.Printf("⚠️ [TOOL-PREDICTOR] Failed to get provider for predictor: %v", err)
- return nil, err
- }
-
- log.Printf("🤖 [TOOL-PREDICTOR] Using model: %s (%s)", predictorModelID, actualModel)
-
- // Build tool list for prompt
- toolListPrompt := s.buildToolListPrompt(availableTools)
-
- // Build user prompt
- userPrompt := fmt.Sprintf(`USER MESSAGE:
-%s
-
-AVAILABLE TOOLS:
-%s
-
-Select the minimal set of tools needed. Return JSON with selected_tools and reasoning.`,
- userMessage, toolListPrompt)
-
- // Build messages with conversation history for better context
- messages := []map[string]interface{}{
- {
- "role": "system",
- "content": ToolPredictionSystemPrompt,
- },
- }
-
- // Add recent conversation history for multi-turn context (exclude current message)
- // Limit to last 6 messages (3 pairs) to avoid token bloat
- historyLimit := 6
- startIdx := len(conversationHistory) - historyLimit
- if startIdx < 0 {
- startIdx = 0
- }
- for i := startIdx; i < len(conversationHistory); i++ {
- msg := conversationHistory[i]
- messages = append(messages, msg)
- }
-
- // Add current user message with tool selection prompt
- messages = append(messages, map[string]interface{}{
- "role": "user",
- "content": userPrompt,
- })
-
- // Build request with structured output
- requestBody := map[string]interface{}{
- "model": actualModel,
- "messages": messages,
- "stream": false,
- "temperature": 0.2, // Low temp for consistency
- "response_format": map[string]interface{}{
- "type": "json_schema",
- "json_schema": map[string]interface{}{
- "name": "tool_prediction",
- "strict": true,
- "schema": toolPredictionSchema,
- },
- },
- }
-
- reqBody, err := json.Marshal(requestBody)
- if err != nil {
- return nil, fmt.Errorf("failed to marshal request: %w", err)
- }
-
- log.Printf("📤 [TOOL-PREDICTOR] Sending prediction request to %s", provider.BaseURL)
-
- // Create HTTP request with timeout
- httpReq, err := http.NewRequestWithContext(ctx, "POST", provider.BaseURL+"/chat/completions", bytes.NewBuffer(reqBody))
- if err != nil {
- return nil, fmt.Errorf("failed to create request: %w", err)
- }
-
- httpReq.Header.Set("Content-Type", "application/json")
- httpReq.Header.Set("Authorization", "Bearer "+provider.APIKey)
-
- // Send request with 30s timeout
- client := &http.Client{Timeout: 30 * time.Second}
- resp, err := client.Do(httpReq)
- if err != nil {
- return nil, fmt.Errorf("request failed: %w", err)
- }
- defer resp.Body.Close()
-
- body, err := io.ReadAll(resp.Body)
- if err != nil {
- return nil, fmt.Errorf("failed to read response: %w", err)
- }
-
- if resp.StatusCode != http.StatusOK {
- log.Printf("⚠️ [TOOL-PREDICTOR] API error: %s", string(body))
- return nil, fmt.Errorf("API error (status %d): %s", resp.StatusCode, string(body))
- }
-
- // Parse response
- var apiResponse struct {
- Choices []struct {
- Message struct {
- Content string `json:"content"`
- } `json:"message"`
- } `json:"choices"`
- }
-
- if err := json.Unmarshal(body, &apiResponse); err != nil {
- return nil, fmt.Errorf("failed to parse API response: %w", err)
- }
-
- if len(apiResponse.Choices) == 0 {
- return nil, fmt.Errorf("no response from predictor model")
- }
-
- // Parse the prediction result
- var result ToolPredictionResult
- content := apiResponse.Choices[0].Message.Content
-
- if err := json.Unmarshal([]byte(content), &result); err != nil {
- log.Printf("⚠️ [TOOL-PREDICTOR] Failed to parse prediction: %v, content: %s", err, content)
- return nil, fmt.Errorf("failed to parse prediction: %w", err)
- }
-
- log.Printf("✅ [TOOL-PREDICTOR] Selected %d tools: %v", len(result.SelectedTools), result.SelectedTools)
- log.Printf("💭 [TOOL-PREDICTOR] Reasoning: %s", result.Reasoning)
-
- // Filter available tools to only include selected ones
- selectedToolDefs := s.filterToolsByNames(availableTools, result.SelectedTools)
-
- log.Printf("📊 [TOOL-PREDICTOR] Reduced from %d to %d tools", len(availableTools), len(selectedToolDefs))
-
- return selectedToolDefs, nil
-}
-
-// buildToolListPrompt creates a concise list of tools for the prompt
-func (s *ToolPredictorService) buildToolListPrompt(tools []map[string]interface{}) string {
- var builder strings.Builder
-
- for i, toolDef := range tools {
- fn, ok := toolDef["function"].(map[string]interface{})
- if !ok {
- continue
- }
-
- name, _ := fn["name"].(string)
- desc, _ := fn["description"].(string)
-
- builder.WriteString(fmt.Sprintf("%d. %s: %s\n", i+1, name, desc))
- }
-
- return builder.String()
-}
-
-// filterToolsByNames filters tool definitions by selected names
-func (s *ToolPredictorService) filterToolsByNames(
- allTools []map[string]interface{},
- selectedNames []string,
-) []map[string]interface{} {
-
- // Build set for O(1) lookup
- nameSet := make(map[string]bool)
- for _, name := range selectedNames {
- nameSet[name] = true
- }
-
- filtered := make([]map[string]interface{}, 0, len(selectedNames))
-
- for _, toolDef := range allTools {
- fn, ok := toolDef["function"].(map[string]interface{})
- if !ok {
- continue
- }
-
- name, ok := fn["name"].(string)
- if !ok {
- continue
- }
-
- if nameSet[name] {
- filtered = append(filtered, toolDef)
- }
- }
-
- return filtered
-}
-
-// getPredictorModelForUser gets user's preferred predictor model
-func (s *ToolPredictorService) getPredictorModelForUser(ctx context.Context, userID string) (string, error) {
- // Query user preferences for tool predictor model
- var predictorModelID sql.NullString
- err := s.db.QueryRow(`
- SELECT preferences->>'toolPredictorModelId'
- FROM users
- WHERE id = ?
- `, userID).Scan(&predictorModelID)
-
- if err != nil {
- // User not found or no preferences - use default
- log.Printf("⚠️ [TOOL-PREDICTOR] Could not get user predictor preference: %v, using default", err)
- return s.defaultPredictorModel, nil
- }
-
- // If user has a preference and it's not empty, use it
- if predictorModelID.Valid && predictorModelID.String != "" {
- log.Printf("🎯 [TOOL-PREDICTOR] Using user-preferred model: %s", predictorModelID.String)
- return predictorModelID.String, nil
- }
-
- // No preference set - use default
- return s.defaultPredictorModel, nil
-}
-
-// getProviderAndModel resolves model ID to provider and actual model name
-func (s *ToolPredictorService) getProviderAndModel(modelID string) (*models.Provider, string, error) {
- if modelID == "" {
- return s.getDefaultPredictorModel()
- }
-
- // Try to find model in database
- var providerID int
- var modelName string
- var smartToolRouter int
-
- err := s.db.QueryRow(`
- SELECT m.name, m.provider_id, COALESCE(m.smart_tool_router, 0)
- FROM models m
- WHERE m.id = ? AND m.is_visible = 1
- `, modelID).Scan(&modelName, &providerID, &smartToolRouter)
-
- if err != nil {
- // Try as model alias
- if s.chatService != nil {
- if provider, actualModel, found := s.chatService.ResolveModelAlias(modelID); found {
- return provider, actualModel, nil
- }
- }
- // Only fall back to default if this is NOT already the default model (avoid recursion)
- if modelID != s.defaultPredictorModel {
- return s.getDefaultPredictorModel()
- }
- return nil, "", fmt.Errorf("default predictor model %s not found in database", modelID)
- }
-
- // Verify model is marked as smart tool router
- if smartToolRouter == 0 {
- log.Printf("⚠️ [TOOL-PREDICTOR] Model %s not marked as smart_tool_router", modelID)
-
- // Search for ANY available smart router model as fallback
- log.Printf("⚠️ [TOOL-PREDICTOR] Searching for any available smart router model...")
- var fallbackModelID string
- var fallbackModelName string
- var fallbackProviderID int
-
- err := s.db.QueryRow(`
- SELECT m.id, m.name, m.provider_id
- FROM models m
- WHERE m.smart_tool_router = 1 AND m.is_visible = 1
- ORDER BY m.id ASC
- LIMIT 1
- `).Scan(&fallbackModelID, &fallbackModelName, &fallbackProviderID)
-
- if err != nil {
- return nil, "", fmt.Errorf("no smart_tool_router models available in database")
- }
-
- log.Printf("✅ [TOOL-PREDICTOR] Found smart router model: %s (%s)", fallbackModelID, fallbackModelName)
-
- provider, err := s.providerService.GetByID(fallbackProviderID)
- if err != nil {
- return nil, "", fmt.Errorf("failed to get provider for smart router model: %w", err)
- }
-
- return provider, fallbackModelName, nil
- }
-
- provider, err := s.providerService.GetByID(providerID)
- if err != nil {
- return nil, "", fmt.Errorf("failed to get provider: %w", err)
- }
-
- return provider, modelName, nil
-}
-
-// getDefaultPredictorModel returns the default predictor model
-// This directly looks up the default model to avoid infinite recursion
-func (s *ToolPredictorService) getDefaultPredictorModel() (*models.Provider, string, error) {
- // First try to get the hardcoded default model
- var providerID int
- var modelName string
- var smartToolRouter int
-
- err := s.db.QueryRow(`
- SELECT m.name, m.provider_id, COALESCE(m.smart_tool_router, 0)
- FROM models m
- WHERE m.id = ? AND m.is_visible = 1
- `, s.defaultPredictorModel).Scan(&modelName, &providerID, &smartToolRouter)
-
- if err != nil {
- // Try as model alias
- if s.chatService != nil {
- if provider, actualModel, found := s.chatService.ResolveModelAlias(s.defaultPredictorModel); found {
- return provider, actualModel, nil
- }
- }
- return nil, "", fmt.Errorf("default predictor model %s not found: %w", s.defaultPredictorModel, err)
- }
-
- // If default model is not marked as smart_tool_router, find ANY available smart router model
- if smartToolRouter == 0 {
- log.Printf("⚠️ [TOOL-PREDICTOR] Default model %s not marked as smart_tool_router, searching for any available smart router model...", s.defaultPredictorModel)
-
- var fallbackModelID string
- err := s.db.QueryRow(`
- SELECT m.id, m.name, m.provider_id
- FROM models m
- WHERE m.smart_tool_router = 1 AND m.is_visible = 1
- ORDER BY m.id ASC
- LIMIT 1
- `).Scan(&fallbackModelID, &modelName, &providerID)
-
- if err != nil {
- return nil, "", fmt.Errorf("no smart_tool_router models available in database")
- }
-
- log.Printf("✅ [TOOL-PREDICTOR] Found smart router model: %s (%s)", fallbackModelID, modelName)
- }
-
- provider, err := s.providerService.GetByID(providerID)
- if err != nil {
- return nil, "", fmt.Errorf("failed to get provider for default model: %w", err)
- }
-
- return provider, modelName, nil
-}
diff --git a/backend/internal/services/tool_registry.go b/backend/internal/services/tool_registry.go
deleted file mode 100644
index 7327e07b..00000000
--- a/backend/internal/services/tool_registry.go
+++ /dev/null
@@ -1,1110 +0,0 @@
-package services
-
-import "strings"
-
-// ToolDefinition represents a single tool in the registry
-type ToolDefinition struct {
- ID string `json:"id"`
- Name string `json:"name"`
- Description string `json:"description"`
- Category string `json:"category"`
- Icon string `json:"icon"` // Icon name for frontend (lucide-react icon name)
- Keywords []string `json:"keywords"`
- UseCases []string `json:"use_cases"`
- Parameters string `json:"parameters,omitempty"` // Brief parameter description
- CodeBlockExample string `json:"code_block_example,omitempty"` // Example argumentMapping for code_block usage
-}
-
-// ToolCategory represents a category of tools
-type ToolCategory struct {
- ID string `json:"id"`
- Name string `json:"name"`
- Icon string `json:"icon"`
- Description string `json:"description"`
-}
-
-// ToolRegistry holds all available tools - easily extensible
-var ToolRegistry = []ToolDefinition{
- // ═══════════════════════════════════════════════════════════════
- // 📊 DATA & ANALYSIS
- // ═══════════════════════════════════════════════════════════════
- {
- ID: "analyze_data",
- Name: "Analyze Data",
- Description: "Python data analysis with charts and visualizations",
- Category: "data_analysis",
- Icon: "BarChart2",
- Keywords: []string{"analyze", "analysis", "data", "chart", "graph", "statistics", "visualize", "visualization", "metrics", "plot", "pandas", "numpy"},
- UseCases: []string{"Analyze CSV/Excel data", "Generate charts", "Calculate statistics", "Create visualizations"},
- Parameters: "code: Python code to execute",
- },
- {
- ID: "calculate_math",
- Name: "Calculate Math",
- Description: "Mathematical calculations and expressions",
- Category: "data_analysis",
- Icon: "Calculator",
- Keywords: []string{"calculate", "math", "formula", "equation", "compute", "arithmetic", "algebra"},
- UseCases: []string{"Solve equations", "Calculate formulas", "Mathematical operations"},
- Parameters: "expression: Math expression to evaluate",
- CodeBlockExample: `{"expression": "{{start.input}}"}`,
- },
- {
- ID: "read_spreadsheet",
- Name: "Read Spreadsheet",
- Description: "Read Excel/CSV files (xlsx, xls, csv, tsv)",
- Category: "data_analysis",
- Icon: "FileSpreadsheet",
- Keywords: []string{"spreadsheet", "excel", "csv", "xlsx", "xls", "tsv", "read", "import", "table"},
- UseCases: []string{"Read Excel files", "Import CSV data", "Parse spreadsheet data"},
- Parameters: "file_id: ID of uploaded file",
- },
- {
- ID: "read_data_file",
- Name: "Read Data File",
- Description: "Read and parse data files (CSV, JSON, text)",
- Category: "data_analysis",
- Icon: "FileJson",
- Keywords: []string{"read", "parse", "data", "file", "json", "csv", "text", "import"},
- UseCases: []string{"Read JSON files", "Parse text data", "Import data files"},
- Parameters: "file_id: ID of uploaded file",
- },
- {
- ID: "read_document",
- Name: "Read Document",
- Description: "Extract text from documents (PDF, DOCX, PPTX)",
- Category: "data_analysis",
- Icon: "FileText",
- Keywords: []string{"document", "pdf", "docx", "pptx", "word", "powerpoint", "extract", "read", "text"},
- UseCases: []string{"Extract PDF text", "Read Word documents", "Parse presentations"},
- Parameters: "file_id: ID of uploaded file",
- },
-
- // ═══════════════════════════════════════════════════════════════
- // 🔍 SEARCH & WEB
- // ═══════════════════════════════════════════════════════════════
- {
- ID: "search_web",
- Name: "Search Web",
- Description: "Search the internet for information, news, articles",
- Category: "search_web",
- Icon: "Search",
- Keywords: []string{"search", "google", "web", "internet", "find", "lookup", "query", "news", "articles", "information"},
- UseCases: []string{"Search for information", "Find news articles", "Research topics"},
- Parameters: "query: Search query string",
- CodeBlockExample: `{"query": "{{start.input}}"}`,
- },
- {
- ID: "search_images",
- Name: "Search Images",
- Description: "Search for images on the web",
- Category: "search_web",
- Icon: "Image",
- Keywords: []string{"image", "images", "photo", "picture", "search", "find", "visual"},
- UseCases: []string{"Find images", "Search for photos", "Visual content search"},
- Parameters: "query: Image search query",
- },
- {
- ID: "scrape_web",
- Name: "Scrape Web",
- Description: "Scrape content from a specific URL",
- Category: "search_web",
- Icon: "Globe",
- Keywords: []string{"scrape", "crawl", "url", "website", "extract", "web", "page", "content"},
- UseCases: []string{"Extract webpage content", "Scrape URLs", "Get page data"},
- Parameters: "url: URL to scrape",
- CodeBlockExample: `{"url": "{{start.input}}"}`,
- },
- {
- ID: "download_file",
- Name: "Download File",
- Description: "Download a file from a URL",
- Category: "search_web",
- Icon: "Download",
- Keywords: []string{"download", "file", "url", "fetch", "get", "retrieve"},
- UseCases: []string{"Download files", "Fetch remote content", "Get assets"},
- Parameters: "url: URL of file to download",
- },
-
- // ═══════════════════════════════════════════════════════════════
- // 📝 CONTENT CREATION
- // ═══════════════════════════════════════════════════════════════
- {
- ID: "create_document",
- Name: "Create Document",
- Description: "Create DOCX or PDF documents",
- Category: "content_creation",
- Icon: "FileText",
- Keywords: []string{"create", "document", "docx", "pdf", "word", "write", "generate", "report"},
- UseCases: []string{"Create Word documents", "Generate PDFs", "Write reports"},
- Parameters: "content: Document content, format: docx|pdf",
- },
- {
- ID: "create_text_file",
- Name: "Create Text File",
- Description: "Create plain text files",
- Category: "content_creation",
- Icon: "FilePlus",
- Keywords: []string{"create", "text", "file", "write", "plain", "txt"},
- UseCases: []string{"Create text files", "Write plain text", "Save content"},
- Parameters: "content: Text content, filename: Output filename",
- },
- {
- ID: "create_presentation",
- Name: "Create Presentation",
- Description: "Create PowerPoint presentations with slides",
- Category: "content_creation",
- Icon: "Presentation",
- Keywords: []string{"presentation", "powerpoint", "pptx", "slides", "create", "deck"},
- UseCases: []string{"Create presentations", "Generate slide decks", "Make PowerPoints"},
- Parameters: "slides: Array of slide content",
- },
- {
- ID: "generate_image",
- Name: "Generate Image",
- Description: "Generate images using AI (DALL-E)",
- Category: "content_creation",
- Icon: "Wand2",
- Keywords: []string{"generate", "image", "create", "ai", "dall-e", "picture", "art", "visual"},
- UseCases: []string{"Generate AI images", "Create artwork", "Visual content generation"},
- Parameters: "prompt: Image description",
- },
- {
- ID: "edit_image",
- Name: "Edit Image",
- Description: "Edit/transform images (resize, crop, filters)",
- Category: "content_creation",
- Icon: "ImagePlus",
- Keywords: []string{"edit", "image", "resize", "crop", "filter", "transform", "modify"},
- UseCases: []string{"Resize images", "Crop photos", "Apply filters"},
- Parameters: "file_id: Image file ID, operation: resize|crop|filter",
- },
- {
- ID: "html_to_pdf",
- Name: "HTML to PDF",
- Description: "Convert HTML content to PDF",
- Category: "content_creation",
- Icon: "FileOutput",
- Keywords: []string{"html", "pdf", "convert", "render", "export"},
- UseCases: []string{"Convert HTML to PDF", "Export web content", "Generate PDF reports"},
- Parameters: "html: HTML content to convert",
- },
-
- // ═══════════════════════════════════════════════════════════════
- // 🎤 MEDIA PROCESSING
- // ═══════════════════════════════════════════════════════════════
- {
- ID: "transcribe_audio",
- Name: "Transcribe Audio",
- Description: "Transcribe speech from audio (MP3, WAV, M4A, OGG, FLAC, WebM)",
- Category: "media_processing",
- Icon: "Mic",
- Keywords: []string{"transcribe", "audio", "speech", "voice", "mp3", "wav", "recording", "speech-to-text"},
- UseCases: []string{"Transcribe audio files", "Convert speech to text", "Process recordings"},
- Parameters: "file_id: Audio file ID",
- },
- {
- ID: "describe_image",
- Name: "Describe Image",
- Description: "Analyze/describe images using AI vision",
- Category: "media_processing",
- Icon: "Eye",
- Keywords: []string{"describe", "image", "vision", "analyze", "see", "look", "visual", "ai"},
- UseCases: []string{"Describe image content", "Analyze visuals", "Image understanding"},
- Parameters: "file_id: Image file ID",
- },
-
- // ═══════════════════════════════════════════════════════════════
- // ⏰ UTILITIES
- // ═══════════════════════════════════════════════════════════════
- {
- ID: "get_current_time",
- Name: "Get Current Time",
- Description: "Get current date/time (REQUIRED for time-sensitive queries)",
- Category: "utilities",
- Icon: "Clock",
- Keywords: []string{"time", "date", "now", "today", "current", "datetime", "timestamp"},
- UseCases: []string{"Get current time", "Date operations", "Time-sensitive queries"},
- Parameters: "timezone: Optional timezone",
- CodeBlockExample: `{}`,
- },
- {
- ID: "ask_user",
- Name: "Ask User Questions",
- Description: "Ask the user clarifying questions via modal dialog. Waits for response (blocking).",
- Category: "utilities",
- Icon: "MessageCircleQuestion",
- Keywords: []string{"ask", "question", "prompt", "user", "input", "clarify", "modal", "dialog", "form", "interactive", "wait", "blocking"},
- UseCases: []string{"Ask clarifying questions", "Gather user input", "Get user preferences", "Confirm actions", "Multi-choice questions"},
- Parameters: "title: Prompt title, questions: Array of questions (text/number/checkbox/select/multi-select), allow_skip: Optional",
- },
- {
- ID: "run_python",
- Name: "Run Python",
- Description: "Execute Python code for custom logic",
- Category: "utilities",
- Icon: "Code",
- Keywords: []string{"python", "code", "script", "execute", "run", "program", "custom"},
- UseCases: []string{"Run custom code", "Execute scripts", "Custom processing"},
- Parameters: "code: Python code to execute",
- },
- {
- ID: "api_request",
- Name: "API Request",
- Description: "Make HTTP API requests (GET, POST, PUT, DELETE)",
- Category: "utilities",
- Icon: "Globe",
- Keywords: []string{"api", "http", "request", "rest", "get", "post", "put", "delete", "endpoint"},
- UseCases: []string{"Call external APIs", "HTTP requests", "API integrations"},
- Parameters: "url: API URL, method: GET|POST|PUT|DELETE, body: Request body",
- },
- {
- ID: "send_webhook",
- Name: "Send Webhook",
- Description: "Send data to any webhook URL",
- Category: "utilities",
- Icon: "Webhook",
- Keywords: []string{"webhook", "send", "post", "notify", "trigger", "callback"},
- UseCases: []string{"Trigger webhooks", "Send notifications", "External integrations"},
- Parameters: "url: Webhook URL, data: JSON payload",
- CodeBlockExample: `{"webhook_url": "https://example.com/hook", "data": {"message": "{{previous-block.response}}"}}`,
- },
-
- // ═══════════════════════════════════════════════════════════════
- // 💬 MESSAGING & COMMUNICATION
- // ═══════════════════════════════════════════════════════════════
- {
- ID: "send_discord_message",
- Name: "Send Discord Message",
- Description: "Send message to Discord channel",
- Category: "messaging",
- Icon: "MessageCircle",
- Keywords: []string{"discord", "message", "send", "chat", "channel", "notify", "bot"},
- UseCases: []string{"Send Discord messages", "Discord notifications", "Bot messaging"},
- Parameters: "content: Message text, embed_title: Optional embed title",
- CodeBlockExample: `{"content": "{{previous-block.response}}"}`,
- },
- {
- ID: "send_slack_message",
- Name: "Send Slack Message",
- Description: "Send message to Slack channel",
- Category: "messaging",
- Icon: "Hash",
- Keywords: []string{"slack", "message", "send", "channel", "notify", "workspace"},
- UseCases: []string{"Send Slack messages", "Slack notifications", "Team messaging"},
- Parameters: "channel: Channel name, text: Message text",
- CodeBlockExample: `{"channel": "#general", "text": "{{previous-block.response}}"}`,
- },
- {
- ID: "send_telegram_message",
- Name: "Send Telegram Message",
- Description: "Send message to Telegram chat",
- Category: "messaging",
- Icon: "Send",
- Keywords: []string{"telegram", "message", "send", "chat", "notify", "bot"},
- UseCases: []string{"Send Telegram messages", "Telegram notifications", "Bot messaging"},
- Parameters: "chat_id: Chat ID, text: Message text",
- },
- {
- ID: "send_google_chat_message",
- Name: "Send Google Chat Message",
- Description: "Send message to Google Chat",
- Category: "messaging",
- Icon: "MessageSquare",
- Keywords: []string{"google chat", "message", "send", "hangouts", "workspace"},
- UseCases: []string{"Google Chat messages", "Workspace notifications"},
- Parameters: "space: Space ID, text: Message text",
- },
- {
- ID: "send_teams_message",
- Name: "Send Teams Message",
- Description: "Send message to Microsoft Teams",
- Category: "messaging",
- Icon: "Users",
- Keywords: []string{"teams", "microsoft", "message", "send", "channel", "notify"},
- UseCases: []string{"Teams messages", "Microsoft Teams notifications"},
- Parameters: "channel: Channel, text: Message text",
- },
- {
- ID: "send_email",
- Name: "Send Email",
- Description: "Send email via SendGrid",
- Category: "messaging",
- Icon: "Mail",
- Keywords: []string{"email", "send", "mail", "sendgrid", "notify"},
- UseCases: []string{"Send emails", "Email notifications"},
- Parameters: "to: Recipient, subject: Subject, body: Email body",
- },
- {
- ID: "send_brevo_email",
- Name: "Send Brevo Email",
- Description: "Send email via Brevo",
- Category: "messaging",
- Icon: "Mail",
- Keywords: []string{"email", "brevo", "sendinblue", "send", "mail"},
- UseCases: []string{"Send emails via Brevo", "Marketing emails"},
- Parameters: "to: Recipient, subject: Subject, body: Email body",
- },
- {
- ID: "twilio_send_sms",
- Name: "Send SMS",
- Description: "Send SMS via Twilio",
- Category: "messaging",
- Icon: "Smartphone",
- Keywords: []string{"sms", "text", "twilio", "send", "phone", "mobile"},
- UseCases: []string{"Send SMS messages", "Text notifications"},
- Parameters: "to: Phone number, body: Message text",
- },
- {
- ID: "twilio_send_whatsapp",
- Name: "Send WhatsApp",
- Description: "Send WhatsApp message via Twilio",
- Category: "messaging",
- Icon: "MessageCircle",
- Keywords: []string{"whatsapp", "message", "twilio", "send", "chat"},
- UseCases: []string{"Send WhatsApp messages", "WhatsApp notifications"},
- Parameters: "to: Phone number, body: Message text",
- },
- {
- ID: "referralmonk_whatsapp",
- Name: "ReferralMonk WhatsApp",
- Description: "Send WhatsApp message via ReferralMonk with template support",
- Category: "messaging",
- Icon: "MessageSquare",
- Keywords: []string{"whatsapp", "referralmonk", "template", "campaign", "message", "send", "ahaguru"},
- UseCases: []string{"Send templated WhatsApp messages", "WhatsApp campaigns", "Marketing via WhatsApp"},
- Parameters: "mobile: Phone with country code, template_name: Template ID, language: Language code (default: en), param_1/2/3: Template parameters",
- CodeBlockExample: `{"mobile": "917550002919", "template_name": "demo_session_01", "language": "en", "param_1": "{{user.name}}", "param_2": "{{lesson.link}}", "param_3": "Team Name"}`,
- },
-
- // ═══════════════════════════════════════════════════════════════
- // 📹 VIDEO CONFERENCING
- // ═══════════════════════════════════════════════════════════════
- {
- ID: "zoom_meeting",
- Name: "Zoom Meeting",
- Description: "Zoom meetings & webinars - create, list, register attendees",
- Category: "video_conferencing",
- Icon: "Video",
- Keywords: []string{"zoom", "meeting", "webinar", "video", "conference", "call", "register", "attendee", "schedule"},
- UseCases: []string{"Create Zoom meetings", "Register for webinars", "List meetings", "Manage attendees"},
- Parameters: "action: create|list|get|register|create_webinar|register_webinar, meeting_id/webinar_id, email, first_name, last_name",
- },
- {
- ID: "calendly_events",
- Name: "Calendly Events",
- Description: "List and manage Calendly events",
- Category: "video_conferencing",
- Icon: "Calendar",
- Keywords: []string{"calendly", "calendar", "schedule", "event", "booking", "appointment"},
- UseCases: []string{"List Calendly events", "View scheduled meetings"},
- Parameters: "user: User URI",
- },
- {
- ID: "calendly_event_types",
- Name: "Calendly Event Types",
- Description: "List Calendly event types",
- Category: "video_conferencing",
- Icon: "CalendarDays",
- Keywords: []string{"calendly", "event type", "booking type", "schedule"},
- UseCases: []string{"List event types", "Get booking options"},
- Parameters: "user: User URI",
- },
- {
- ID: "calendly_invitees",
- Name: "Calendly Invitees",
- Description: "Get event invitees/attendees",
- Category: "video_conferencing",
- Icon: "Users",
- Keywords: []string{"calendly", "invitee", "attendee", "participant"},
- UseCases: []string{"List event invitees", "Get attendee info"},
- Parameters: "event_uuid: Event UUID",
- },
-
- // ═══════════════════════════════════════════════════════════════
- // 📋 PROJECT MANAGEMENT
- // ═══════════════════════════════════════════════════════════════
- {
- ID: "jira_issues",
- Name: "Jira Issues",
- Description: "List/search Jira issues",
- Category: "project_management",
- Icon: "CheckSquare",
- Keywords: []string{"jira", "issue", "ticket", "bug", "task", "search", "list"},
- UseCases: []string{"Search Jira issues", "List tickets", "Find tasks"},
- Parameters: "jql: JQL query string",
- },
- {
- ID: "jira_create_issue",
- Name: "Create Jira Issue",
- Description: "Create a new Jira issue",
- Category: "project_management",
- Icon: "PlusSquare",
- Keywords: []string{"jira", "create", "issue", "ticket", "bug", "task", "new"},
- UseCases: []string{"Create Jira tickets", "Report bugs", "Add tasks"},
- Parameters: "project: Project key, summary: Title, description: Description",
- },
- {
- ID: "jira_update_issue",
- Name: "Update Jira Issue",
- Description: "Update an existing Jira issue",
- Category: "project_management",
- Icon: "Edit",
- Keywords: []string{"jira", "update", "edit", "issue", "ticket", "modify"},
- UseCases: []string{"Update tickets", "Modify issues", "Edit tasks"},
- Parameters: "issue_key: Issue key, fields: Fields to update",
- },
- {
- ID: "linear_issues",
- Name: "Linear Issues",
- Description: "List Linear issues",
- Category: "project_management",
- Icon: "CheckSquare",
- Keywords: []string{"linear", "issue", "ticket", "task", "list"},
- UseCases: []string{"List Linear issues", "View tasks"},
- Parameters: "team_id: Team ID",
- },
- {
- ID: "linear_create_issue",
- Name: "Create Linear Issue",
- Description: "Create a new Linear issue",
- Category: "project_management",
- Icon: "PlusSquare",
- Keywords: []string{"linear", "create", "issue", "ticket", "task", "new"},
- UseCases: []string{"Create Linear issues", "Add tasks"},
- Parameters: "team_id: Team ID, title: Title, description: Description",
- },
- {
- ID: "linear_update_issue",
- Name: "Update Linear Issue",
- Description: "Update a Linear issue",
- Category: "project_management",
- Icon: "Edit",
- Keywords: []string{"linear", "update", "edit", "issue", "modify"},
- UseCases: []string{"Update Linear issues", "Modify tasks"},
- Parameters: "issue_id: Issue ID, fields: Fields to update",
- },
- {
- ID: "clickup_tasks",
- Name: "ClickUp Tasks",
- Description: "List ClickUp tasks",
- Category: "project_management",
- Icon: "CheckCircle",
- Keywords: []string{"clickup", "task", "list", "todo"},
- UseCases: []string{"List ClickUp tasks", "View todos"},
- Parameters: "list_id: List ID",
- },
- {
- ID: "clickup_create_task",
- Name: "Create ClickUp Task",
- Description: "Create a new ClickUp task",
- Category: "project_management",
- Icon: "PlusCircle",
- Keywords: []string{"clickup", "create", "task", "new", "todo"},
- UseCases: []string{"Create ClickUp tasks", "Add todos"},
- Parameters: "list_id: List ID, name: Task name, description: Description",
- },
- {
- ID: "clickup_update_task",
- Name: "Update ClickUp Task",
- Description: "Update a ClickUp task",
- Category: "project_management",
- Icon: "Edit",
- Keywords: []string{"clickup", "update", "edit", "task", "modify"},
- UseCases: []string{"Update ClickUp tasks", "Modify todos"},
- Parameters: "task_id: Task ID, fields: Fields to update",
- },
- {
- ID: "trello_boards",
- Name: "Trello Boards",
- Description: "List Trello boards",
- Category: "project_management",
- Icon: "Layout",
- Keywords: []string{"trello", "board", "list", "kanban"},
- UseCases: []string{"List Trello boards", "View kanban boards"},
- Parameters: "None required",
- },
- {
- ID: "trello_lists",
- Name: "Trello Lists",
- Description: "List Trello lists in a board",
- Category: "project_management",
- Icon: "List",
- Keywords: []string{"trello", "list", "column", "board"},
- UseCases: []string{"List Trello lists", "View board columns"},
- Parameters: "board_id: Board ID",
- },
- {
- ID: "trello_cards",
- Name: "Trello Cards",
- Description: "List Trello cards",
- Category: "project_management",
- Icon: "Square",
- Keywords: []string{"trello", "card", "task", "list"},
- UseCases: []string{"List Trello cards", "View tasks"},
- Parameters: "list_id: List ID",
- },
- {
- ID: "trello_create_card",
- Name: "Create Trello Card",
- Description: "Create a new Trello card",
- Category: "project_management",
- Icon: "Plus",
- Keywords: []string{"trello", "create", "card", "new", "task"},
- UseCases: []string{"Create Trello cards", "Add tasks"},
- Parameters: "list_id: List ID, name: Card name, description: Description",
- },
- {
- ID: "asana_tasks",
- Name: "Asana Tasks",
- Description: "List Asana tasks",
- Category: "project_management",
- Icon: "CheckSquare",
- Keywords: []string{"asana", "task", "list", "project"},
- UseCases: []string{"List Asana tasks", "View project tasks"},
- Parameters: "project_id: Project ID",
- },
-
- // ═══════════════════════════════════════════════════════════════
- // 💼 CRM & SALES
- // ═══════════════════════════════════════════════════════════════
- {
- ID: "hubspot_contacts",
- Name: "HubSpot Contacts",
- Description: "List/search HubSpot contacts",
- Category: "crm_sales",
- Icon: "Users",
- Keywords: []string{"hubspot", "contact", "crm", "customer", "lead", "list", "search"},
- UseCases: []string{"List HubSpot contacts", "Search customers", "Find leads"},
- Parameters: "query: Optional search query",
- },
- {
- ID: "hubspot_deals",
- Name: "HubSpot Deals",
- Description: "List HubSpot deals",
- Category: "crm_sales",
- Icon: "DollarSign",
- Keywords: []string{"hubspot", "deal", "sales", "pipeline", "opportunity"},
- UseCases: []string{"List deals", "View sales pipeline"},
- Parameters: "None required",
- },
- {
- ID: "hubspot_companies",
- Name: "HubSpot Companies",
- Description: "List HubSpot companies",
- Category: "crm_sales",
- Icon: "Building",
- Keywords: []string{"hubspot", "company", "organization", "account"},
- UseCases: []string{"List companies", "View accounts"},
- Parameters: "None required",
- },
- {
- ID: "leadsquared_leads",
- Name: "LeadSquared Leads",
- Description: "List LeadSquared leads",
- Category: "crm_sales",
- Icon: "UserPlus",
- Keywords: []string{"leadsquared", "lead", "crm", "prospect"},
- UseCases: []string{"List leads", "View prospects"},
- Parameters: "query: Optional search query",
- },
- {
- ID: "leadsquared_create_lead",
- Name: "Create LeadSquared Lead",
- Description: "Create a new LeadSquared lead",
- Category: "crm_sales",
- Icon: "UserPlus",
- Keywords: []string{"leadsquared", "create", "lead", "new", "prospect"},
- UseCases: []string{"Create leads", "Add prospects"},
- Parameters: "email: Email, firstName: First name, lastName: Last name",
- },
- {
- ID: "leadsquared_activities",
- Name: "LeadSquared Activities",
- Description: "List LeadSquared activities",
- Category: "crm_sales",
- Icon: "Activity",
- Keywords: []string{"leadsquared", "activity", "history", "timeline"},
- UseCases: []string{"List activities", "View lead history"},
- Parameters: "lead_id: Lead ID",
- },
- {
- ID: "mailchimp_lists",
- Name: "Mailchimp Lists",
- Description: "List Mailchimp audiences",
- Category: "crm_sales",
- Icon: "Users",
- Keywords: []string{"mailchimp", "list", "audience", "subscribers", "email"},
- UseCases: []string{"List audiences", "View subscriber lists"},
- Parameters: "None required",
- },
- {
- ID: "mailchimp_add_subscriber",
- Name: "Mailchimp Add Subscriber",
- Description: "Add subscriber to Mailchimp list",
- Category: "crm_sales",
- Icon: "UserPlus",
- Keywords: []string{"mailchimp", "subscriber", "add", "email", "list"},
- UseCases: []string{"Add subscribers", "Email list signup"},
- Parameters: "list_id: List ID, email: Email address",
- },
-
- // ═══════════════════════════════════════════════════════════════
- // 📊 ANALYTICS
- // ═══════════════════════════════════════════════════════════════
- {
- ID: "posthog_capture",
- Name: "PostHog Capture",
- Description: "Track PostHog events",
- Category: "analytics",
- Icon: "BarChart",
- Keywords: []string{"posthog", "track", "event", "analytics", "capture"},
- UseCases: []string{"Track events", "Capture user actions"},
- Parameters: "event: Event name, properties: Event properties",
- },
- {
- ID: "posthog_identify",
- Name: "PostHog Identify",
- Description: "Identify PostHog user",
- Category: "analytics",
- Icon: "User",
- Keywords: []string{"posthog", "identify", "user", "profile"},
- UseCases: []string{"Identify users", "Set user properties"},
- Parameters: "distinct_id: User ID, properties: User properties",
- },
- {
- ID: "posthog_query",
- Name: "PostHog Query",
- Description: "Query PostHog analytics",
- Category: "analytics",
- Icon: "Database",
- Keywords: []string{"posthog", "query", "analytics", "insights", "data"},
- UseCases: []string{"Query analytics", "Get insights"},
- Parameters: "query: HogQL query",
- },
- {
- ID: "mixpanel_track",
- Name: "Mixpanel Track",
- Description: "Track Mixpanel events",
- Category: "analytics",
- Icon: "BarChart",
- Keywords: []string{"mixpanel", "track", "event", "analytics"},
- UseCases: []string{"Track Mixpanel events", "Log user actions"},
- Parameters: "event: Event name, properties: Event properties",
- },
- {
- ID: "mixpanel_user_profile",
- Name: "Mixpanel User Profile",
- Description: "Update Mixpanel user profile",
- Category: "analytics",
- Icon: "User",
- Keywords: []string{"mixpanel", "user", "profile", "update"},
- UseCases: []string{"Update user profiles", "Set user properties"},
- Parameters: "distinct_id: User ID, properties: Profile properties",
- },
-
- // ═══════════════════════════════════════════════════════════════
- // 🐙 CODE & DEVOPS
- // ═══════════════════════════════════════════════════════════════
- {
- ID: "github_create_issue",
- Name: "GitHub Create Issue",
- Description: "Create a GitHub issue",
- Category: "code_devops",
- Icon: "CircleDot",
- Keywords: []string{"github", "issue", "create", "bug", "feature", "repo"},
- UseCases: []string{"Create GitHub issues", "Report bugs"},
- Parameters: "owner: Repo owner, repo: Repo name, title: Title, body: Description",
- },
- {
- ID: "github_list_issues",
- Name: "GitHub List Issues",
- Description: "List GitHub issues",
- Category: "code_devops",
- Icon: "List",
- Keywords: []string{"github", "issue", "list", "bug", "repo"},
- UseCases: []string{"List GitHub issues", "View repo issues"},
- Parameters: "owner: Repo owner, repo: Repo name",
- },
- {
- ID: "github_get_repo",
- Name: "GitHub Get Repo",
- Description: "Get GitHub repository info",
- Category: "code_devops",
- Icon: "GitBranch",
- Keywords: []string{"github", "repo", "repository", "info", "details"},
- UseCases: []string{"Get repo info", "View repository details"},
- Parameters: "owner: Repo owner, repo: Repo name",
- },
- {
- ID: "github_add_comment",
- Name: "GitHub Add Comment",
- Description: "Add comment to GitHub issue/PR",
- Category: "code_devops",
- Icon: "MessageSquare",
- Keywords: []string{"github", "comment", "issue", "pr", "pull request"},
- UseCases: []string{"Comment on issues", "Reply to PRs"},
- Parameters: "owner: Repo owner, repo: Repo name, issue_number: Issue number, body: Comment",
- },
- {
- ID: "gitlab_projects",
- Name: "GitLab Projects",
- Description: "List GitLab projects",
- Category: "code_devops",
- Icon: "Folder",
- Keywords: []string{"gitlab", "project", "list", "repo"},
- UseCases: []string{"List GitLab projects", "View repositories"},
- Parameters: "None required",
- },
- {
- ID: "gitlab_issues",
- Name: "GitLab Issues",
- Description: "List GitLab issues",
- Category: "code_devops",
- Icon: "List",
- Keywords: []string{"gitlab", "issue", "list", "bug"},
- UseCases: []string{"List GitLab issues", "View project issues"},
- Parameters: "project_id: Project ID",
- },
- {
- ID: "gitlab_mrs",
- Name: "GitLab Merge Requests",
- Description: "List GitLab merge requests",
- Category: "code_devops",
- Icon: "GitMerge",
- Keywords: []string{"gitlab", "merge request", "mr", "pull request", "pr"},
- UseCases: []string{"List merge requests", "View MRs"},
- Parameters: "project_id: Project ID",
- },
- {
- ID: "netlify_sites",
- Name: "Netlify Sites",
- Description: "List Netlify sites",
- Category: "code_devops",
- Icon: "Globe",
- Keywords: []string{"netlify", "site", "list", "deploy", "hosting"},
- UseCases: []string{"List Netlify sites", "View deployed sites"},
- Parameters: "None required",
- },
- {
- ID: "netlify_deploys",
- Name: "Netlify Deploys",
- Description: "List Netlify deploys",
- Category: "code_devops",
- Icon: "Rocket",
- Keywords: []string{"netlify", "deploy", "list", "build", "release"},
- UseCases: []string{"List deploys", "View build history"},
- Parameters: "site_id: Site ID",
- },
- {
- ID: "netlify_trigger_build",
- Name: "Netlify Trigger Build",
- Description: "Trigger a Netlify build",
- Category: "code_devops",
- Icon: "Play",
- Keywords: []string{"netlify", "build", "trigger", "deploy", "release"},
- UseCases: []string{"Trigger builds", "Deploy sites"},
- Parameters: "site_id: Site ID",
- },
-
- // ═══════════════════════════════════════════════════════════════
- // 📓 PRODUCTIVITY
- // ═══════════════════════════════════════════════════════════════
- {
- ID: "notion_search",
- Name: "Notion Search",
- Description: "Search Notion pages/databases",
- Category: "productivity",
- Icon: "Search",
- Keywords: []string{"notion", "search", "page", "database", "find"},
- UseCases: []string{"Search Notion", "Find pages"},
- Parameters: "query: Search query",
- },
- {
- ID: "notion_query_database",
- Name: "Notion Query Database",
- Description: "Query a Notion database",
- Category: "productivity",
- Icon: "Database",
- Keywords: []string{"notion", "database", "query", "filter", "table"},
- UseCases: []string{"Query databases", "Filter records"},
- Parameters: "database_id: Database ID, filter: Optional filter",
- },
- {
- ID: "notion_create_page",
- Name: "Notion Create Page",
- Description: "Create a Notion page",
- Category: "productivity",
- Icon: "FilePlus",
- Keywords: []string{"notion", "create", "page", "new", "doc"},
- UseCases: []string{"Create pages", "Add documents"},
- Parameters: "parent_id: Parent page/database ID, properties: Page properties",
- },
- {
- ID: "notion_update_page",
- Name: "Notion Update Page",
- Description: "Update a Notion page",
- Category: "productivity",
- Icon: "Edit",
- Keywords: []string{"notion", "update", "edit", "page", "modify"},
- UseCases: []string{"Update pages", "Edit documents"},
- Parameters: "page_id: Page ID, properties: Properties to update",
- },
- {
- ID: "airtable_list",
- Name: "Airtable List Records",
- Description: "List Airtable records",
- Category: "productivity",
- Icon: "Table",
- Keywords: []string{"airtable", "list", "records", "table", "database"},
- UseCases: []string{"List records", "View table data"},
- Parameters: "base_id: Base ID, table_name: Table name",
- },
- {
- ID: "airtable_read",
- Name: "Airtable Read Record",
- Description: "Read a single Airtable record",
- Category: "productivity",
- Icon: "Eye",
- Keywords: []string{"airtable", "read", "record", "get", "single"},
- UseCases: []string{"Read records", "Get single record"},
- Parameters: "base_id: Base ID, table_name: Table name, record_id: Record ID",
- },
- {
- ID: "airtable_create",
- Name: "Airtable Create Record",
- Description: "Create an Airtable record",
- Category: "productivity",
- Icon: "Plus",
- Keywords: []string{"airtable", "create", "record", "new", "add"},
- UseCases: []string{"Create records", "Add data"},
- Parameters: "base_id: Base ID, table_name: Table name, fields: Record fields",
- },
- {
- ID: "airtable_update",
- Name: "Airtable Update Record",
- Description: "Update an Airtable record",
- Category: "productivity",
- Icon: "Edit",
- Keywords: []string{"airtable", "update", "record", "edit", "modify"},
- UseCases: []string{"Update records", "Modify data"},
- Parameters: "base_id: Base ID, table_name: Table name, record_id: Record ID, fields: Fields to update",
- },
-
- // ═══════════════════════════════════════════════════════════════
- // 🛒 E-COMMERCE
- // ═══════════════════════════════════════════════════════════════
- {
- ID: "shopify_products",
- Name: "Shopify Products",
- Description: "List Shopify products",
- Category: "ecommerce",
- Icon: "ShoppingBag",
- Keywords: []string{"shopify", "product", "list", "inventory", "catalog"},
- UseCases: []string{"List products", "View inventory"},
- Parameters: "None required",
- },
- {
- ID: "shopify_orders",
- Name: "Shopify Orders",
- Description: "List Shopify orders",
- Category: "ecommerce",
- Icon: "ShoppingCart",
- Keywords: []string{"shopify", "order", "list", "sales", "purchase"},
- UseCases: []string{"List orders", "View sales"},
- Parameters: "status: Optional status filter",
- },
- {
- ID: "shopify_customers",
- Name: "Shopify Customers",
- Description: "List Shopify customers",
- Category: "ecommerce",
- Icon: "Users",
- Keywords: []string{"shopify", "customer", "list", "buyer"},
- UseCases: []string{"List customers", "View buyers"},
- Parameters: "None required",
- },
-
- // ═══════════════════════════════════════════════════════════════
- // 🗄️ DATABASE
- // ═══════════════════════════════════════════════════════════════
- {
- ID: "mongodb_query",
- Name: "MongoDB Query",
- Description: "Query MongoDB collections - find, aggregate, count documents",
- Category: "database",
- Icon: "Database",
- Keywords: []string{"mongodb", "mongo", "database", "query", "find", "aggregate", "nosql", "document", "collection"},
- UseCases: []string{"Query MongoDB collections", "Find documents", "Aggregate data", "Count records"},
- Parameters: "action: find|aggregate|count, collection: Collection name, filter: Query filter, pipeline: Aggregation pipeline",
- },
- {
- ID: "mongodb_write",
- Name: "MongoDB Write",
- Description: "Write to MongoDB - insert or update documents (delete not permitted)",
- Category: "database",
- Icon: "DatabaseBackup",
- Keywords: []string{"mongodb", "mongo", "database", "insert", "update", "write", "create", "modify", "insertOne", "insertMany", "updateOne", "updateMany"},
- UseCases: []string{"Insert single document", "Insert multiple documents", "Update single record", "Update multiple records"},
- Parameters: "action: insertOne|insertMany|updateOne|updateMany, collection: Collection name, document: Document to insert, documents: Array for insertMany, filter: Update filter, update: Update operations",
- CodeBlockExample: `{"action": "insertOne", "collection": "users", "document": {"name": "John", "email": "john@example.com"}}`,
- },
- {
- ID: "redis_read",
- Name: "Redis Read",
- Description: "Read from Redis - get keys, scan, list operations",
- Category: "database",
- Icon: "Database",
- Keywords: []string{"redis", "cache", "key-value", "read", "get", "scan", "list", "hash", "set"},
- UseCases: []string{"Get cached values", "Read keys", "Scan patterns", "List operations"},
- Parameters: "action: get|mget|scan|hgetall|lrange|smembers, key: Redis key, pattern: Scan pattern",
- CodeBlockExample: `{"action": "get", "key": "{{start.input}}"}`,
- },
- {
- ID: "redis_write",
- Name: "Redis Write",
- Description: "Write to Redis - set keys, lists, hashes, with TTL support",
- Category: "database",
- Icon: "DatabaseBackup",
- Keywords: []string{"redis", "cache", "key-value", "write", "set", "expire", "list", "hash", "push"},
- UseCases: []string{"Set cache values", "Store data", "Queue operations", "Set expiry"},
- Parameters: "action: set|mset|hset|lpush|rpush|sadd|del, key: Redis key, value: Value to set, ttl: Optional TTL in seconds",
- CodeBlockExample: `{"action": "set", "key": "{{start.input}}", "value": "{{previous-block.response}}"}`,
- },
-
- // ═══════════════════════════════════════════════════════════════
- // 🐦 SOCIAL MEDIA
- // ═══════════════════════════════════════════════════════════════
- {
- ID: "x_search_posts",
- Name: "X Search Posts",
- Description: "Search X/Twitter posts",
- Category: "social_media",
- Icon: "Twitter",
- Keywords: []string{"twitter", "x", "search", "tweet", "post", "social"},
- UseCases: []string{"Search tweets", "Find posts"},
- Parameters: "query: Search query",
- },
- {
- ID: "x_post_tweet",
- Name: "X Post Tweet",
- Description: "Post to X/Twitter",
- Category: "social_media",
- Icon: "Send",
- Keywords: []string{"twitter", "x", "post", "tweet", "publish"},
- UseCases: []string{"Post tweets", "Share content"},
- Parameters: "text: Tweet text",
- },
- {
- ID: "x_get_user",
- Name: "X Get User",
- Description: "Get X/Twitter user info",
- Category: "social_media",
- Icon: "User",
- Keywords: []string{"twitter", "x", "user", "profile", "account"},
- UseCases: []string{"Get user info", "View profiles"},
- Parameters: "username: Twitter username",
- },
- {
- ID: "x_get_user_posts",
- Name: "X Get User Posts",
- Description: "Get user's X/Twitter posts",
- Category: "social_media",
- Icon: "List",
- Keywords: []string{"twitter", "x", "user", "posts", "tweets", "timeline"},
- UseCases: []string{"Get user tweets", "View timeline"},
- Parameters: "user_id: User ID",
- },
-}
-
-// ToolCategoryRegistry defines all tool categories
-var ToolCategoryRegistry = []ToolCategory{
- {ID: "data_analysis", Name: "Data & Analysis", Icon: "BarChart2", Description: "Analyze data, create charts, and work with spreadsheets"},
- {ID: "search_web", Name: "Search & Web", Icon: "Search", Description: "Search the web, scrape URLs, and download files"},
- {ID: "content_creation", Name: "Content Creation", Icon: "FileText", Description: "Create documents, presentations, and images"},
- {ID: "media_processing", Name: "Media Processing", Icon: "Mic", Description: "Transcribe audio and analyze images"},
- {ID: "utilities", Name: "Utilities", Icon: "Clock", Description: "Time, code execution, and API requests"},
- {ID: "messaging", Name: "Messaging", Icon: "MessageCircle", Description: "Send messages via Discord, Slack, email, SMS, etc."},
- {ID: "video_conferencing", Name: "Video Conferencing", Icon: "Video", Description: "Zoom meetings, webinars, and Calendly"},
- {ID: "project_management", Name: "Project Management", Icon: "CheckSquare", Description: "Jira, Linear, ClickUp, Trello, Asana"},
- {ID: "crm_sales", Name: "CRM & Sales", Icon: "Users", Description: "HubSpot, LeadSquared, Mailchimp"},
- {ID: "analytics", Name: "Analytics", Icon: "BarChart", Description: "PostHog and Mixpanel tracking"},
- {ID: "code_devops", Name: "Code & DevOps", Icon: "GitBranch", Description: "GitHub, GitLab, and Netlify"},
- {ID: "productivity", Name: "Productivity", Icon: "Layout", Description: "Notion and Airtable"},
- {ID: "ecommerce", Name: "E-Commerce", Icon: "ShoppingBag", Description: "Shopify products, orders, and customers"},
- {ID: "social_media", Name: "Social Media", Icon: "Twitter", Description: "X/Twitter posts and interactions"},
- {ID: "database", Name: "Database", Icon: "Database", Description: "MongoDB and Redis database operations"},
-}
-
-// GetToolsByCategory returns all tools in a given category
-func GetToolsByCategory(categoryID string) []ToolDefinition {
- var tools []ToolDefinition
- for _, tool := range ToolRegistry {
- if tool.Category == categoryID {
- tools = append(tools, tool)
- }
- }
- return tools
-}
-
-// GetToolByID returns a tool by its ID
-func GetToolByID(toolID string) *ToolDefinition {
- for _, tool := range ToolRegistry {
- if tool.ID == toolID {
- return &tool
- }
- }
- return nil
-}
-
-// GetAllToolIDs returns all tool IDs
-func GetAllToolIDs() []string {
- ids := make([]string, len(ToolRegistry))
- for i, tool := range ToolRegistry {
- ids[i] = tool.ID
- }
- return ids
-}
-
-// BuildToolPromptSection builds a prompt section for specific tools
-func BuildToolPromptSection(toolIDs []string) string {
- if len(toolIDs) == 0 {
- return ""
- }
-
- // Group tools by category for better organization
- categoryTools := make(map[string][]ToolDefinition)
- for _, toolID := range toolIDs {
- if tool := GetToolByID(toolID); tool != nil {
- categoryTools[tool.Category] = append(categoryTools[tool.Category], *tool)
- }
- }
-
- var builder strings.Builder
- builder.WriteString("=== AVAILABLE TOOLS (Selected for this workflow) ===\n\n")
-
- // Get category info for display
- categoryInfo := make(map[string]ToolCategory)
- for _, cat := range ToolCategoryRegistry {
- categoryInfo[cat.ID] = cat
- }
-
- for catID, tools := range categoryTools {
- if cat, ok := categoryInfo[catID]; ok {
- builder.WriteString(cat.Name + ":\n")
- }
- for _, tool := range tools {
- builder.WriteString("- " + tool.ID + ": " + tool.Description + "\n")
- if tool.Parameters != "" {
- builder.WriteString(" Parameters: " + tool.Parameters + "\n")
- }
- // Include code_block example if available - shows how to configure argumentMapping
- if tool.CodeBlockExample != "" {
- builder.WriteString(" code_block argumentMapping: " + tool.CodeBlockExample + "\n")
- }
- }
- builder.WriteString("\n")
- }
-
- return builder.String()
-}
diff --git a/backend/internal/services/tool_service.go b/backend/internal/services/tool_service.go
deleted file mode 100644
index 96ed6c6c..00000000
--- a/backend/internal/services/tool_service.go
+++ /dev/null
@@ -1,196 +0,0 @@
-package services
-
-import (
- "claraverse/internal/tools"
- "context"
- "log"
- "strings"
-)
-
-// ToolService handles tool-related operations with credential awareness.
-// It filters tools based on user's configured credentials to ensure
-// only usable tools are sent to the LLM.
-type ToolService struct {
- toolRegistry *tools.Registry
- credentialService *CredentialService
-}
-
-// NewToolService creates a new tool service
-func NewToolService(registry *tools.Registry, credentialService *CredentialService) *ToolService {
- return &ToolService{
- toolRegistry: registry,
- credentialService: credentialService,
- }
-}
-
-// GetAvailableTools returns tools filtered by user's credentials.
-// - Tools not in ToolIntegrationMap are always included (no credential needed)
-// - Tools in ToolIntegrationMap are only included if user has a credential for that integration type
-func (s *ToolService) GetAvailableTools(ctx context.Context, userID string) []map[string]interface{} {
- // Get all tools for user (built-in + MCP)
- allTools := s.toolRegistry.GetUserTools(userID)
-
- // If no credential service, return all tools (fallback for dev mode or tests)
- if s.credentialService == nil {
- log.Printf("⚠️ [TOOL-SERVICE] No credential service, returning all %d tools", len(allTools))
- return allTools
- }
-
- // Get user's configured integration types
- userIntegrations, err := s.GetUserIntegrationTypes(ctx, userID)
- if err != nil {
- log.Printf("⚠️ [TOOL-SERVICE] Could not fetch user credentials, returning all tools: %v", err)
- return allTools // Graceful degradation
- }
-
- // Filter tools based on credential requirements
- var filteredTools []map[string]interface{}
- excludedCount := 0
-
- for _, toolDef := range allTools {
- toolName := extractToolName(toolDef)
- if toolName == "" {
- continue
- }
-
- // Check if this is an MCP tool (user-specific local client tools)
- isMCPTool := isUserSpecificTool(toolDef, userID)
-
- // Check if tool requires a credential
- requiredIntegration := tools.GetIntegrationTypeForTool(toolName)
-
- if requiredIntegration == "" {
- // Tool is NOT in integration mapping
- if isMCPTool {
- // MCP tools without explicit mapping are excluded by default (security)
- // Only include MCP tools that are explicitly mapped as not needing credentials
- log.Printf("🔒 [TOOL-SERVICE] Excluding unmapped MCP tool: %s (requires explicit mapping)", toolName)
- excludedCount++
- } else {
- // Built-in tool that doesn't require credentials - always include
- filteredTools = append(filteredTools, toolDef)
- }
- } else if userIntegrations[requiredIntegration] {
- // Tool requires credentials AND user has them - include
- filteredTools = append(filteredTools, toolDef)
- } else {
- // Tool requires credentials user doesn't have - exclude
- excludedCount++
- }
- }
-
- log.Printf("🔧 [TOOL-SERVICE] Filtered tools for user %s: %d available, %d excluded (missing credentials)",
- userID, len(filteredTools), excludedCount)
-
- return filteredTools
-}
-
-// GetUserIntegrationTypes returns a set of integration types the user has credentials for
-func (s *ToolService) GetUserIntegrationTypes(ctx context.Context, userID string) (map[string]bool, error) {
- if s.credentialService == nil {
- return make(map[string]bool), nil
- }
-
- credentials, err := s.credentialService.ListByUser(ctx, userID)
- if err != nil {
- return nil, err
- }
-
- integrations := make(map[string]bool)
- for _, cred := range credentials {
- integrations[cred.IntegrationType] = true
- }
-
- return integrations, nil
-}
-
-// extractToolName extracts the tool name from an OpenAI tool definition
-func extractToolName(toolDef map[string]interface{}) string {
- fn, ok := toolDef["function"].(map[string]interface{})
- if !ok {
- return ""
- }
- name, ok := fn["name"].(string)
- if !ok {
- return ""
- }
- return name
-}
-
-// isUserSpecificTool checks if a tool is an MCP tool (user-specific, not built-in)
-// MCP tools are registered per-user and should be filtered by credentials by default
-func isUserSpecificTool(toolDef map[string]interface{}, userID string) bool {
- // Check if tool has user_id metadata (MCP tools have this)
- if metadata, ok := toolDef["metadata"].(map[string]interface{}); ok {
- if toolUserID, ok := metadata["user_id"].(string); ok && toolUserID == userID {
- return true
- }
- }
-
- // Fallback: Check if tool name suggests it's an MCP tool
- // MCP tools often have specific naming patterns (e.g., containing "gmail", "calendar", "notion", etc.)
- toolName := extractToolName(toolDef)
- mcpPatterns := []string{
- "gmail", "calendar", "drive", "sheets", "docs", "slack", "discord",
- "notion", "trello", "asana", "jira", "linear", "github", "gitlab",
- "spotify", "twitter", "youtube", "reddit", "instagram",
- }
-
- toolNameLower := strings.ToLower(toolName)
- for _, pattern := range mcpPatterns {
- if strings.Contains(toolNameLower, pattern) {
- // Check if it's NOT a built-in Composio tool (which start with the integration name)
- // Built-in: "gmail_send_email", MCP: "send_gmail_message"
- if !strings.HasPrefix(toolNameLower, pattern+"_") {
- return true
- }
- }
- }
-
- return false
-}
-
-// GetCredentialForTool returns the credential ID for a tool that requires credentials.
-// Returns empty string if no credential is needed or not found.
-func (s *ToolService) GetCredentialForTool(ctx context.Context, userID string, toolName string) string {
- if s.credentialService == nil {
- return ""
- }
-
- // Check if tool requires a credential
- integrationType := tools.GetIntegrationTypeForTool(toolName)
- if integrationType == "" {
- return "" // Tool doesn't require credentials
- }
-
- // Get credentials for this integration type
- credentials, err := s.credentialService.ListByUserAndType(ctx, userID, integrationType)
- if err != nil {
- log.Printf("⚠️ [TOOL-SERVICE] Error getting credentials for %s: %v", integrationType, err)
- return ""
- }
-
- if len(credentials) == 0 {
- log.Printf("⚠️ [TOOL-SERVICE] No %s credentials found for user %s", integrationType, userID)
- return ""
- }
-
- // Use the first credential (or the only one)
- credentialID := credentials[0].ID
- log.Printf("🔐 [TOOL-SERVICE] Found credential %s for tool %s (type: %s)", credentialID, toolName, integrationType)
- return credentialID
-}
-
-// CreateCredentialResolver creates a credential resolver function for a user.
-// Returns nil if credential service is not available.
-func (s *ToolService) CreateCredentialResolver(userID string) tools.CredentialResolver {
- if s.credentialService == nil {
- return nil
- }
- return s.credentialService.CreateCredentialResolver(userID)
-}
-
-// GetCredentialService returns the underlying credential service (for advanced use cases)
-func (s *ToolService) GetCredentialService() *CredentialService {
- return s.credentialService
-}
diff --git a/backend/internal/services/usage_limiter_service.go b/backend/internal/services/usage_limiter_service.go
deleted file mode 100644
index bc25dae5..00000000
--- a/backend/internal/services/usage_limiter_service.go
+++ /dev/null
@@ -1,384 +0,0 @@
-package services
-
-import (
- "claraverse/internal/database"
- "claraverse/internal/models"
- "context"
- "fmt"
- "time"
-
- "github.com/redis/go-redis/v9"
- "go.mongodb.org/mongo-driver/bson"
-)
-
-// UsageLimiterService tracks and enforces usage limits for messages, file uploads, and image generation
-type UsageLimiterService struct {
- tierService *TierService
- redis *redis.Client
- mongoDB *database.MongoDB
-}
-
-// UsageLimiterStats holds current usage statistics for a user
-type UsageLimiterStats struct {
- MessagesUsed int64 `json:"messages_used"`
- FileUploadsUsed int64 `json:"file_uploads_used"`
- ImageGensUsed int64 `json:"image_gens_used"`
- MessageResetAt time.Time `json:"message_reset_at"`
- FileUploadResetAt time.Time `json:"file_upload_reset_at"`
- ImageGenResetAt time.Time `json:"image_gen_reset_at"`
-}
-
-// LimitExceededError represents a rate limit error
-type LimitExceededError struct {
- ErrorCode string `json:"error_code"`
- Message string `json:"message"`
- Limit int64 `json:"limit"`
- Used int64 `json:"used"`
- ResetAt time.Time `json:"reset_at"`
- UpgradeTo string `json:"upgrade_to"`
-}
-
-func (e *LimitExceededError) Error() string {
- return e.Message
-}
-
-// NewUsageLimiterService creates a new usage limiter service
-func NewUsageLimiterService(tierService *TierService, redis *redis.Client, mongoDB *database.MongoDB) *UsageLimiterService {
- return &UsageLimiterService{
- tierService: tierService,
- redis: redis,
- mongoDB: mongoDB,
- }
-}
-
-// ========== Message Limits (Monthly - Billing Cycle Reset) ==========
-
-// CheckMessageLimit checks if user can send another message this month
-func (s *UsageLimiterService) CheckMessageLimit(ctx context.Context, userID string) error {
- limits := s.tierService.GetLimits(ctx, userID)
-
- // Unlimited
- if limits.MaxMessagesPerMonth < 0 {
- return nil
- }
-
- // Get current count
- count, err := s.GetMonthlyMessageCount(ctx, userID)
- if err != nil {
- // On error, allow request (fail open)
- return nil
- }
-
- // Check limit
- if count >= limits.MaxMessagesPerMonth {
- resetAt, _ := s.getMonthlyResetTime(ctx, userID)
- return &LimitExceededError{
- ErrorCode: "message_limit_exceeded",
- Message: fmt.Sprintf("Monthly message limit reached (%d/%d). Resets on %s. Upgrade to Pro for 3,000 messages/month.", count, limits.MaxMessagesPerMonth, resetAt.Format("Jan 2")),
- Limit: limits.MaxMessagesPerMonth,
- Used: count,
- ResetAt: resetAt,
- UpgradeTo: s.getSuggestedUpgradeTier(s.tierService.GetUserTier(ctx, userID)),
- }
- }
-
- return nil
-}
-
-// IncrementMessageCount increments the user's monthly message count
-func (s *UsageLimiterService) IncrementMessageCount(ctx context.Context, userID string) error {
- key, err := s.getMessageKey(ctx, userID)
- if err != nil {
- return err
- }
-
- // Increment counter
- _, err = s.redis.Incr(ctx, key).Result()
- if err != nil {
- return err
- }
-
- // Set expiry (billing period end + 30 days buffer)
- resetAt, err := s.getMonthlyResetTime(ctx, userID)
- if err == nil {
- expiry := time.Until(resetAt.AddDate(0, 1, 0)) // Add 30 days buffer
- s.redis.Expire(ctx, key, expiry)
- }
-
- return nil
-}
-
-// GetMonthlyMessageCount returns the user's current message count for this billing period
-func (s *UsageLimiterService) GetMonthlyMessageCount(ctx context.Context, userID string) (int64, error) {
- key, err := s.getMessageKey(ctx, userID)
- if err != nil {
- return 0, err
- }
-
- count, err := s.redis.Get(ctx, key).Int64()
- if err == redis.Nil {
- return 0, nil
- }
- return count, err
-}
-
-// ========== File Upload Limits (Daily - Midnight UTC Reset) ==========
-
-// CheckFileUploadLimit checks if user can upload another file today
-func (s *UsageLimiterService) CheckFileUploadLimit(ctx context.Context, userID string) error {
- limits := s.tierService.GetLimits(ctx, userID)
-
- // Unlimited
- if limits.MaxFileUploadsPerDay < 0 {
- return nil
- }
-
- // Get current count
- count, err := s.GetDailyFileUploadCount(ctx, userID)
- if err != nil {
- // On error, allow request (fail open)
- return nil
- }
-
- // Check limit
- if count >= limits.MaxFileUploadsPerDay {
- resetAt := s.getNextMidnightUTC()
- return &LimitExceededError{
- ErrorCode: "file_upload_limit_exceeded",
- Message: fmt.Sprintf("Daily file upload limit reached (%d/%d). Resets at midnight UTC. Upgrade to Pro for 10 uploads/day.", count, limits.MaxFileUploadsPerDay),
- Limit: limits.MaxFileUploadsPerDay,
- Used: count,
- ResetAt: resetAt,
- UpgradeTo: s.getSuggestedUpgradeTier(s.tierService.GetUserTier(ctx, userID)),
- }
- }
-
- return nil
-}
-
-// IncrementFileUploadCount increments the user's daily file upload count
-func (s *UsageLimiterService) IncrementFileUploadCount(ctx context.Context, userID string) error {
- key := s.getFileUploadKey(userID)
-
- // Increment counter
- _, err := s.redis.Incr(ctx, key).Result()
- if err != nil {
- return err
- }
-
- // Set expiry to next midnight + 24 hours buffer
- resetAt := s.getNextMidnightUTC()
- expiry := time.Until(resetAt.Add(24 * time.Hour))
- s.redis.Expire(ctx, key, expiry)
-
- return nil
-}
-
-// GetDailyFileUploadCount returns the user's current file upload count for today
-func (s *UsageLimiterService) GetDailyFileUploadCount(ctx context.Context, userID string) (int64, error) {
- key := s.getFileUploadKey(userID)
-
- count, err := s.redis.Get(ctx, key).Int64()
- if err == redis.Nil {
- return 0, nil
- }
- return count, err
-}
-
-// ========== Image Generation Limits (Daily - Midnight UTC Reset) ==========
-
-// CheckImageGenLimit checks if user can generate another image today
-func (s *UsageLimiterService) CheckImageGenLimit(ctx context.Context, userID string) error {
- limits := s.tierService.GetLimits(ctx, userID)
-
- // Unlimited
- if limits.MaxImageGensPerDay < 0 {
- return nil
- }
-
- // Get current count
- count, err := s.GetDailyImageGenCount(ctx, userID)
- if err != nil {
- // On error, allow request (fail open)
- return nil
- }
-
- // Check limit
- if count >= limits.MaxImageGensPerDay {
- resetAt := s.getNextMidnightUTC()
- return &LimitExceededError{
- ErrorCode: "image_gen_limit_exceeded",
- Message: fmt.Sprintf("Daily image generation limit reached (%d/%d). Resets at midnight UTC. Upgrade to Pro for 25 images/day.", count, limits.MaxImageGensPerDay),
- Limit: limits.MaxImageGensPerDay,
- Used: count,
- ResetAt: resetAt,
- UpgradeTo: s.getSuggestedUpgradeTier(s.tierService.GetUserTier(ctx, userID)),
- }
- }
-
- return nil
-}
-
-// IncrementImageGenCount increments the user's daily image generation count
-func (s *UsageLimiterService) IncrementImageGenCount(ctx context.Context, userID string) error {
- key := s.getImageGenKey(userID)
-
- // Increment counter
- _, err := s.redis.Incr(ctx, key).Result()
- if err != nil {
- return err
- }
-
- // Set expiry to next midnight + 24 hours buffer
- resetAt := s.getNextMidnightUTC()
- expiry := time.Until(resetAt.Add(24 * time.Hour))
- s.redis.Expire(ctx, key, expiry)
-
- return nil
-}
-
-// GetDailyImageGenCount returns the user's current image generation count for today
-func (s *UsageLimiterService) GetDailyImageGenCount(ctx context.Context, userID string) (int64, error) {
- key := s.getImageGenKey(userID)
-
- count, err := s.redis.Get(ctx, key).Int64()
- if err == redis.Nil {
- return 0, nil
- }
- return count, err
-}
-
-// ========== Utility Methods ==========
-
-// GetUsageStats returns comprehensive usage statistics for a user
-func (s *UsageLimiterService) GetUsageStats(ctx context.Context, userID string) (*UsageLimiterStats, error) {
- msgCount, _ := s.GetMonthlyMessageCount(ctx, userID)
- fileCount, _ := s.GetDailyFileUploadCount(ctx, userID)
- imageCount, _ := s.GetDailyImageGenCount(ctx, userID)
-
- msgResetAt, _ := s.getMonthlyResetTime(ctx, userID)
- dailyResetAt := s.getNextMidnightUTC()
-
- return &UsageLimiterStats{
- MessagesUsed: msgCount,
- FileUploadsUsed: fileCount,
- ImageGensUsed: imageCount,
- MessageResetAt: msgResetAt,
- FileUploadResetAt: dailyResetAt,
- ImageGenResetAt: dailyResetAt,
- }, nil
-}
-
-// ResetMonthlyCounters resets the monthly message counter for a user
-func (s *UsageLimiterService) ResetMonthlyCounters(ctx context.Context, userID string) error {
- key, err := s.getMessageKey(ctx, userID)
- if err != nil {
- return err
- }
- return s.redis.Del(ctx, key).Err()
-}
-
-// ResetAllCounters resets all usage counters for a user (used on tier upgrade)
-func (s *UsageLimiterService) ResetAllCounters(ctx context.Context, userID string) error {
- // Reset monthly message counter
- msgKey, _ := s.getMessageKey(ctx, userID)
- s.redis.Del(ctx, msgKey)
-
- // Reset daily file upload counter
- fileKey := s.getFileUploadKey(userID)
- s.redis.Del(ctx, fileKey)
-
- // Reset daily image gen counter
- imageKey := s.getImageGenKey(userID)
- s.redis.Del(ctx, imageKey)
-
- return nil
-}
-
-// ========== Private Helper Methods ==========
-
-// getMessageKey generates the Redis key for monthly message count
-func (s *UsageLimiterService) getMessageKey(ctx context.Context, userID string) (string, error) {
- billingPeriodKey, err := s.getBillingPeriodKey(ctx, userID)
- if err != nil {
- // Fallback to calendar month for free users
- billingPeriodKey = time.Now().UTC().Format("2006-01")
- }
- return fmt.Sprintf("messages:%s:%s", userID, billingPeriodKey), nil
-}
-
-// getFileUploadKey generates the Redis key for daily file upload count
-func (s *UsageLimiterService) getFileUploadKey(userID string) string {
- date := time.Now().UTC().Format("2006-01-02")
- return fmt.Sprintf("file_uploads:%s:%s", userID, date)
-}
-
-// getImageGenKey generates the Redis key for daily image generation count
-func (s *UsageLimiterService) getImageGenKey(userID string) string {
- date := time.Now().UTC().Format("2006-01-02")
- return fmt.Sprintf("image_gens:%s:%s", userID, date)
-}
-
-// getBillingPeriodKey returns a unique key for the current billing period
-func (s *UsageLimiterService) getBillingPeriodKey(ctx context.Context, userID string) (string, error) {
- // Get subscription from MongoDB
- subscription, err := s.getSubscription(ctx, userID)
- if err != nil || subscription == nil {
- // Free tier - use calendar month
- return time.Now().UTC().Format("2006-01"), nil
- }
-
- // Paid tier - use billing cycle start date
- return subscription.CurrentPeriodStart.Format("2006-01-02"), nil
-}
-
-// getSubscription retrieves the user's subscription from MongoDB
-func (s *UsageLimiterService) getSubscription(ctx context.Context, userID string) (*models.Subscription, error) {
- if s.mongoDB == nil {
- return nil, fmt.Errorf("MongoDB not available")
- }
-
- collection := s.mongoDB.Database().Collection("subscriptions")
- var subscription models.Subscription
-
- err := collection.FindOne(ctx, bson.M{"userId": userID}).Decode(&subscription)
- if err != nil {
- return nil, err
- }
-
- return &subscription, nil
-}
-
-// getMonthlyResetTime returns when the monthly message count will reset
-func (s *UsageLimiterService) getMonthlyResetTime(ctx context.Context, userID string) (time.Time, error) {
- subscription, err := s.getSubscription(ctx, userID)
- if err != nil || subscription == nil {
- // Free tier - reset at end of current month (first day of next month at midnight)
- now := time.Now().UTC()
- // Get first day of next month properly (handles year rollover)
- firstDayNextMonth := time.Date(now.Year(), now.Month(), 1, 0, 0, 0, 0, time.UTC).AddDate(0, 1, 0)
- return firstDayNextMonth, nil
- }
-
- // Paid tier - reset at billing cycle end
- return subscription.CurrentPeriodEnd, nil
-}
-
-// getNextMidnightUTC returns the next midnight UTC time
-func (s *UsageLimiterService) getNextMidnightUTC() time.Time {
- now := time.Now().UTC()
- tomorrow := now.AddDate(0, 0, 1)
- return time.Date(tomorrow.Year(), tomorrow.Month(), tomorrow.Day(), 0, 0, 0, 0, time.UTC)
-}
-
-// getSuggestedUpgradeTier suggests which tier to upgrade to based on current tier
-func (s *UsageLimiterService) getSuggestedUpgradeTier(currentTier string) string {
- switch currentTier {
- case models.TierFree:
- return "pro"
- case models.TierPro:
- return "max"
- default:
- return "max"
- }
-}
diff --git a/backend/internal/services/user_service.go b/backend/internal/services/user_service.go
deleted file mode 100644
index c6abad36..00000000
--- a/backend/internal/services/user_service.go
+++ /dev/null
@@ -1,502 +0,0 @@
-package services
-
-import (
- "claraverse/internal/config"
- "claraverse/internal/database"
- "claraverse/internal/models"
- "context"
- "fmt"
- "log"
- "time"
-
- "go.mongodb.org/mongo-driver/bson"
- "go.mongodb.org/mongo-driver/bson/primitive"
- "go.mongodb.org/mongo-driver/mongo"
- "go.mongodb.org/mongo-driver/mongo/options"
-)
-
-// UserService handles user operations with MongoDB
-type UserService struct {
- db *database.MongoDB
- collection *mongo.Collection
- config *config.Config
- usageLimiter *UsageLimiterService
-}
-
-// NewUserService creates a new user service
-// usageLimiter can be nil and set later via SetUsageLimiter
-func NewUserService(db *database.MongoDB, cfg *config.Config, usageLimiter *UsageLimiterService) *UserService {
- return &UserService{
- db: db,
- collection: db.Collection(database.CollectionUsers),
- config: cfg,
- usageLimiter: usageLimiter,
- }
-}
-
-// SetUsageLimiter sets the usage limiter (for deferred initialization)
-func (s *UserService) SetUsageLimiter(limiter *UsageLimiterService) {
- s.usageLimiter = limiter
-}
-
-// SyncUserFromSupabase creates or updates a user from Supabase authentication
-// This should be called on every authenticated request to keep user data in sync
-func (s *UserService) SyncUserFromSupabase(ctx context.Context, supabaseUserID, email string) (*models.User, error) {
- if supabaseUserID == "" {
- return nil, fmt.Errorf("supabase user ID is required")
- }
-
- now := time.Now()
-
- // Determine subscription tier based on promo eligibility
- subscriptionTier := models.TierFree
- var subscriptionExpiresAt *time.Time
-
- if s.isPromoEligible(now) {
- subscriptionTier = models.TierPro
- expiresAt := now.Add(time.Duration(s.config.PromoDuration) * 24 * time.Hour)
- subscriptionExpiresAt = &expiresAt
- }
-
- // Use upsert to create or update user
- filter := bson.M{"supabaseUserId": supabaseUserID}
- setOnInsertFields := bson.M{
- "supabaseUserId": supabaseUserID,
- "createdAt": now,
- "subscriptionTier": subscriptionTier,
- "subscriptionStatus": models.SubStatusActive,
- "preferences": models.UserPreferences{
- StoreBuilderChatHistory: true, // Default to storing chat history
- },
- }
-
- if subscriptionExpiresAt != nil {
- setOnInsertFields["subscriptionExpiresAt"] = subscriptionExpiresAt
- }
-
- update := bson.M{
- "$set": bson.M{
- "email": email,
- "lastLoginAt": now,
- },
- "$setOnInsert": setOnInsertFields,
- }
-
- opts := options.FindOneAndUpdate().
- SetUpsert(true).
- SetReturnDocument(options.After)
-
- var user models.User
- err := s.collection.FindOneAndUpdate(ctx, filter, update, opts).Decode(&user)
- if err != nil {
- return nil, fmt.Errorf("failed to sync user: %w", err)
- }
-
- // Reset usage counters for NEW promo users to ensure clean slate
- // A new user is detected by checking if createdAt is very close to now (within 2 seconds)
- if user.SubscriptionTier == models.TierPro && user.CreatedAt.After(now.Add(-2*time.Second)) {
- if s.usageLimiter != nil {
- if err := s.usageLimiter.ResetAllCounters(ctx, supabaseUserID); err != nil {
- log.Printf("⚠️ Failed to reset usage counters for new promo user %s: %v", supabaseUserID, err)
- } else {
- log.Printf("✅ Reset usage counters for new promo user %s", supabaseUserID)
- }
- }
- }
-
- return &user, nil
-}
-
-// GetUserBySupabaseID retrieves a user by their Supabase user ID
-func (s *UserService) GetUserBySupabaseID(ctx context.Context, supabaseUserID string) (*models.User, error) {
- var user models.User
- err := s.collection.FindOne(ctx, bson.M{"supabaseUserId": supabaseUserID}).Decode(&user)
- if err == mongo.ErrNoDocuments {
- return nil, fmt.Errorf("user not found")
- }
- if err != nil {
- return nil, fmt.Errorf("failed to get user: %w", err)
- }
- return &user, nil
-}
-
-// GetUserByID retrieves a user by their MongoDB ID
-func (s *UserService) GetUserByID(ctx context.Context, userID primitive.ObjectID) (*models.User, error) {
- var user models.User
- err := s.collection.FindOne(ctx, bson.M{"_id": userID}).Decode(&user)
- if err == mongo.ErrNoDocuments {
- return nil, fmt.Errorf("user not found")
- }
- if err != nil {
- return nil, fmt.Errorf("failed to get user: %w", err)
- }
- return &user, nil
-}
-
-// GetUserByEmail retrieves a user by their email address
-func (s *UserService) GetUserByEmail(ctx context.Context, email string) (*models.User, error) {
- var user models.User
- err := s.collection.FindOne(ctx, bson.M{"email": email}).Decode(&user)
- if err == mongo.ErrNoDocuments {
- return nil, fmt.Errorf("user not found")
- }
- if err != nil {
- return nil, fmt.Errorf("failed to get user: %w", err)
- }
- return &user, nil
-}
-
-// CreateUser creates a new user (for local auth registration)
-func (s *UserService) CreateUser(ctx context.Context, user *models.User) error {
- result, err := s.collection.InsertOne(ctx, user)
- if err != nil {
- return fmt.Errorf("failed to create user: %w", err)
- }
-
- // Update user ID with the inserted ID
- if oid, ok := result.InsertedID.(primitive.ObjectID); ok {
- user.ID = oid
- }
-
- return nil
-}
-
-// UpdateUser updates an existing user
-func (s *UserService) UpdateUser(ctx context.Context, user *models.User) error {
- filter := bson.M{"_id": user.ID}
- update := bson.M{"$set": user}
-
- result, err := s.collection.UpdateOne(ctx, filter, update)
- if err != nil {
- return fmt.Errorf("failed to update user: %w", err)
- }
- if result.MatchedCount == 0 {
- return fmt.Errorf("user not found")
- }
-
- return nil
-}
-
-// Collection returns the MongoDB collection (for direct access when needed)
-func (s *UserService) Collection() *mongo.Collection {
- return s.collection
-}
-
-// UpdatePreferences updates a user's preferences
-func (s *UserService) UpdatePreferences(ctx context.Context, supabaseUserID string, req *models.UpdateUserPreferencesRequest) (*models.UserPreferences, error) {
- // Build update document
- updateFields := bson.M{}
- if req.StoreBuilderChatHistory != nil {
- updateFields["preferences.storeBuilderChatHistory"] = *req.StoreBuilderChatHistory
- }
- if req.DefaultModelID != nil {
- updateFields["preferences.defaultModelId"] = *req.DefaultModelID
- }
- if req.ToolPredictorModelID != nil {
- updateFields["preferences.toolPredictorModelId"] = *req.ToolPredictorModelID
- }
- if req.ChatPrivacyMode != nil {
- updateFields["preferences.chatPrivacyMode"] = *req.ChatPrivacyMode
- }
- if req.Theme != nil {
- updateFields["preferences.theme"] = *req.Theme
- }
- if req.FontSize != nil {
- updateFields["preferences.fontSize"] = *req.FontSize
- }
-
- // Memory system preferences
- if req.MemoryEnabled != nil {
- updateFields["preferences.memoryEnabled"] = *req.MemoryEnabled
- }
- if req.MemoryExtractionThreshold != nil {
- updateFields["preferences.memoryExtractionThreshold"] = *req.MemoryExtractionThreshold
- }
- if req.MemoryMaxInjection != nil {
- updateFields["preferences.memoryMaxInjection"] = *req.MemoryMaxInjection
- }
- if req.MemoryExtractorModelID != nil {
- updateFields["preferences.memoryExtractorModelId"] = *req.MemoryExtractorModelID
- }
- if req.MemorySelectorModelID != nil {
- updateFields["preferences.memorySelectorModelId"] = *req.MemorySelectorModelID
- }
-
- if len(updateFields) == 0 {
- // No changes, just return current preferences
- user, err := s.GetUserBySupabaseID(ctx, supabaseUserID)
- if err != nil {
- return nil, err
- }
- return &user.Preferences, nil
- }
-
- filter := bson.M{"supabaseUserId": supabaseUserID}
- update := bson.M{"$set": updateFields}
-
- opts := options.FindOneAndUpdate().SetReturnDocument(options.After)
-
- var user models.User
- err := s.collection.FindOneAndUpdate(ctx, filter, update, opts).Decode(&user)
- if err == mongo.ErrNoDocuments {
- return nil, fmt.Errorf("user not found")
- }
- if err != nil {
- return nil, fmt.Errorf("failed to update preferences: %w", err)
- }
-
- return &user.Preferences, nil
-}
-
-// GetPreferences retrieves a user's preferences
-func (s *UserService) GetPreferences(ctx context.Context, supabaseUserID string) (*models.UserPreferences, error) {
- user, err := s.GetUserBySupabaseID(ctx, supabaseUserID)
- if err != nil {
- return nil, err
- }
- return &user.Preferences, nil
-}
-
-// MarkWelcomePopupSeen marks the welcome popup as seen for a user
-func (s *UserService) MarkWelcomePopupSeen(ctx context.Context, supabaseUserID string) error {
- filter := bson.M{"supabaseUserId": supabaseUserID}
- update := bson.M{
- "$set": bson.M{
- "hasSeenWelcomePopup": true,
- },
- }
-
- result, err := s.collection.UpdateOne(ctx, filter, update)
- if err != nil {
- return fmt.Errorf("failed to mark welcome popup as seen: %w", err)
- }
- if result.MatchedCount == 0 {
- return fmt.Errorf("user not found")
- }
-
- return nil
-}
-
-// GetUserCount returns the total number of users (for admin analytics)
-func (s *UserService) GetUserCount(ctx context.Context) (int64, error) {
- count, err := s.collection.CountDocuments(ctx, bson.M{})
- if err != nil {
- return 0, fmt.Errorf("failed to count users: %w", err)
- }
- return count, nil
-}
-
-// ListUsers returns a paginated list of users (for admin)
-func (s *UserService) ListUsers(ctx context.Context, skip, limit int64) ([]*models.User, error) {
- opts := options.Find().
- SetSkip(skip).
- SetLimit(limit).
- SetSort(bson.M{"createdAt": -1})
-
- cursor, err := s.collection.Find(ctx, bson.M{}, opts)
- if err != nil {
- return nil, fmt.Errorf("failed to list users: %w", err)
- }
- defer cursor.Close(ctx)
-
- var users []*models.User
- if err := cursor.All(ctx, &users); err != nil {
- return nil, fmt.Errorf("failed to decode users: %w", err)
- }
-
- return users, nil
-}
-
-// UpdateSubscription updates a user's subscription (for payment integration)
-func (s *UserService) UpdateSubscription(ctx context.Context, supabaseUserID, tier string, expiresAt *time.Time) error {
- return s.UpdateSubscriptionWithStatus(ctx, supabaseUserID, tier, "", expiresAt)
-}
-
-// UpdateSubscriptionWithStatus updates a user's subscription with status
-func (s *UserService) UpdateSubscriptionWithStatus(ctx context.Context, supabaseUserID, tier, status string, expiresAt *time.Time) error {
- filter := bson.M{"supabaseUserId": supabaseUserID}
- updateFields := bson.M{
- "subscriptionTier": tier,
- }
- if status != "" {
- updateFields["subscriptionStatus"] = status
- }
- if expiresAt != nil {
- updateFields["subscriptionExpiresAt"] = expiresAt
- }
-
- update := bson.M{
- "$set": updateFields,
- }
-
- result, err := s.collection.UpdateOne(ctx, filter, update)
- if err != nil {
- return fmt.Errorf("failed to update subscription: %w", err)
- }
- if result.MatchedCount == 0 {
- return fmt.Errorf("user not found")
- }
-
- return nil
-}
-
-// UpdateDodoCustomer updates a user's DodoPayments customer ID
-func (s *UserService) UpdateDodoCustomer(ctx context.Context, supabaseUserID, customerID string) error {
- filter := bson.M{"supabaseUserId": supabaseUserID}
- update := bson.M{
- "$set": bson.M{
- "dodoCustomerId": customerID,
- },
- }
-
- result, err := s.collection.UpdateOne(ctx, filter, update)
- if err != nil {
- return fmt.Errorf("failed to update DodoPayments customer ID: %w", err)
- }
- if result.MatchedCount == 0 {
- return fmt.Errorf("user not found")
- }
-
- return nil
-}
-
-// DeleteUser deletes a user and all their data (GDPR compliance)
-func (s *UserService) DeleteUser(ctx context.Context, supabaseUserID string) error {
- // Get user first to get their MongoDB ID
- user, err := s.GetUserBySupabaseID(ctx, supabaseUserID)
- if err != nil {
- return err
- }
-
- // Delete user document
- result, err := s.collection.DeleteOne(ctx, bson.M{"_id": user.ID})
- if err != nil {
- return fmt.Errorf("failed to delete user: %w", err)
- }
- if result.DeletedCount == 0 {
- return fmt.Errorf("user not found")
- }
-
- // Note: Related data (agents, conversations, etc.) should be deleted
- // in a transaction or by the caller. This could be enhanced with
- // cascade delete logic.
-
- return nil
-}
-
-// isPromoEligible checks if a signup time is within the promotional window
-func (s *UserService) isPromoEligible(signupTime time.Time) bool {
- if s.config == nil || !s.config.PromoEnabled {
- return false
- }
-
- // Check if signup time is within promo window (UTC)
- // Use After for start (exclusive) and Before for end (exclusive)
- return !signupTime.Before(s.config.PromoStartDate) &&
- signupTime.Before(s.config.PromoEndDate)
-}
-
-// SetLimitOverrides sets tier OR granular limit overrides for a user (admin only)
-func (s *UserService) SetLimitOverrides(ctx context.Context, supabaseUserID, adminUserID, reason string, tier *string, limits *models.TierLimits) error {
- if supabaseUserID == "" {
- return fmt.Errorf("user ID is required")
- }
- if tier == nil && limits == nil {
- return fmt.Errorf("either tier or limits must be provided")
- }
-
- now := time.Now()
- updateFields := bson.M{
- "overrideSetBy": adminUserID,
- "overrideSetAt": now,
- "overrideReason": reason,
- }
-
- // Set tier override if provided
- if tier != nil {
- updateFields["tierOverride"] = *tier
- // Clear limit overrides when setting tier
- updateFields["limitOverrides"] = nil
- }
-
- // Set granular limit overrides if provided
- if limits != nil {
- updateFields["limitOverrides"] = limits
- // Clear tier override when setting limits
- updateFields["tierOverride"] = nil
- }
-
- filter := bson.M{"supabaseUserId": supabaseUserID}
- update := bson.M{"$set": updateFields}
-
- result, err := s.collection.UpdateOne(ctx, filter, update)
- if err != nil {
- return fmt.Errorf("failed to set overrides: %w", err)
- }
- if result.MatchedCount == 0 {
- return fmt.Errorf("user not found")
- }
-
- if tier != nil {
- log.Printf("🔐 Admin %s set tier override for user %s: %s (reason: %s)", adminUserID, supabaseUserID, *tier, reason)
- } else {
- log.Printf("🔐 Admin %s set granular limit overrides for user %s (reason: %s)", adminUserID, supabaseUserID, reason)
- }
- return nil
-}
-
-// RemoveAllOverrides removes all overrides (tier and limits) for a user (admin only)
-func (s *UserService) RemoveAllOverrides(ctx context.Context, supabaseUserID, adminUserID string) error {
- if supabaseUserID == "" {
- return fmt.Errorf("user ID is required")
- }
-
- update := bson.M{
- "$unset": bson.M{
- "tierOverride": "",
- "limitOverrides": "",
- "overrideSetBy": "",
- "overrideSetAt": "",
- "overrideReason": "",
- },
- }
-
- filter := bson.M{"supabaseUserId": supabaseUserID}
- result, err := s.collection.UpdateOne(ctx, filter, update)
- if err != nil {
- return fmt.Errorf("failed to remove overrides: %w", err)
- }
- if result.MatchedCount == 0 {
- return fmt.Errorf("user not found")
- }
-
- log.Printf("🔐 Admin %s removed all overrides for user %s", adminUserID, supabaseUserID)
- return nil
-}
-
-// GetAdminUserDetails returns detailed user info for admin (includes override info)
-func (s *UserService) GetAdminUserDetails(ctx context.Context, supabaseUserID string, tierService *TierService) (*models.AdminUserResponse, error) {
- user, err := s.GetUserBySupabaseID(ctx, supabaseUserID)
- if err != nil {
- return nil, err
- }
-
- // Get effective tier
- effectiveTier := tierService.GetUserTier(ctx, supabaseUserID)
-
- // Get effective limits (with overrides applied)
- effectiveLimits := tierService.GetLimits(ctx, supabaseUserID)
-
- return &models.AdminUserResponse{
- UserResponse: user.ToResponse(),
- EffectiveTier: effectiveTier,
- EffectiveLimits: effectiveLimits,
- HasTierOverride: user.TierOverride != nil,
- HasLimitOverrides: user.LimitOverrides != nil,
- TierOverride: user.TierOverride,
- LimitOverrides: user.LimitOverrides,
- OverrideSetBy: user.OverrideSetBy,
- OverrideSetAt: user.OverrideSetAt,
- OverrideReason: user.OverrideReason,
- }, nil
-}
diff --git a/backend/internal/services/vision_init.go b/backend/internal/services/vision_init.go
deleted file mode 100644
index 6eece449..00000000
--- a/backend/internal/services/vision_init.go
+++ /dev/null
@@ -1,93 +0,0 @@
-package services
-
-import (
- "claraverse/internal/database"
- "claraverse/internal/vision"
- "fmt"
- "log"
- "sync"
-)
-
-var (
- visionInitOnce sync.Once
- visionProviderSvc *ProviderService
- visionDB *database.DB
-)
-
-// SetVisionDependencies sets the dependencies needed for vision service
-// Must be called before InitVisionService
-func SetVisionDependencies(providerService *ProviderService, db *database.DB) {
- visionProviderSvc = providerService
- visionDB = db
-}
-
-// InitVisionService initializes the vision package with provider access
-func InitVisionService() {
- if visionProviderSvc == nil {
- log.Println("⚠️ [VISION-INIT] Provider service not set, vision service disabled")
- return
- }
-
- visionInitOnce.Do(func() {
- configService := GetConfigService()
-
- // Provider getter callback
- providerGetter := func(id int) (*vision.Provider, error) {
- p, err := visionProviderSvc.GetByID(id)
- if err != nil {
- return nil, err
- }
- return &vision.Provider{
- ID: p.ID,
- Name: p.Name,
- BaseURL: p.BaseURL,
- APIKey: p.APIKey,
- Enabled: p.Enabled,
- }, nil
- }
-
- // Vision model finder callback
- visionModelFinder := func() (int, string, error) {
- // First check aliases for vision-capable models
- allAliases := configService.GetAllModelAliases()
-
- for providerID, aliases := range allAliases {
- for _, aliasInfo := range aliases {
- if aliasInfo.SupportsVision != nil && *aliasInfo.SupportsVision {
- provider, err := visionProviderSvc.GetByID(providerID)
- if err == nil && provider.Enabled {
- log.Printf("🖼️ [VISION-INIT] Found vision model via alias: %s -> %s", aliasInfo.DisplayName, aliasInfo.ActualModel)
- return providerID, aliasInfo.ActualModel, nil
- }
- }
- }
- }
-
- // Fallback: Check database for vision models
- if visionDB == nil {
- return 0, "", fmt.Errorf("database not available")
- }
-
- var providerID int
- var modelName string
- err := visionDB.QueryRow(`
- SELECT m.provider_id, m.name
- FROM models m
- JOIN providers p ON m.provider_id = p.id
- WHERE m.supports_vision = 1 AND m.is_visible = 1 AND p.enabled = 1
- ORDER BY m.provider_id ASC
- LIMIT 1
- `).Scan(&providerID, &modelName)
-
- if err != nil {
- return 0, "", fmt.Errorf("no vision model found: %w", err)
- }
-
- log.Printf("🖼️ [VISION-INIT] Found vision model from database: %s (provider: %d)", modelName, providerID)
- return providerID, modelName, nil
- }
-
- vision.InitService(providerGetter, visionModelFinder)
- log.Printf("✅ [VISION-INIT] Vision service initialized")
- })
-}
diff --git a/backend/internal/services/workflow_generator.go b/backend/internal/services/workflow_generator.go
deleted file mode 100644
index 314fbc80..00000000
--- a/backend/internal/services/workflow_generator.go
+++ /dev/null
@@ -1,1589 +0,0 @@
-package services
-
-import (
- "bytes"
- "encoding/json"
- "fmt"
- "io"
- "log"
- "net/http"
- "regexp"
- "strings"
- "time"
-
- "github.com/google/uuid"
-
- "claraverse/internal/database"
- "claraverse/internal/models"
-)
-
-// WorkflowGeneratorService handles workflow generation with structured output
-type WorkflowGeneratorService struct {
- db *database.DB
- providerService *ProviderService
- chatService *ChatService
-}
-
-// NewWorkflowGeneratorService creates a new workflow generator service
-func NewWorkflowGeneratorService(
- db *database.DB,
- providerService *ProviderService,
- chatService *ChatService,
-) *WorkflowGeneratorService {
- return &WorkflowGeneratorService{
- db: db,
- providerService: providerService,
- chatService: chatService,
- }
-}
-
-// V1ToolCategory is used for the legacy v1 dynamic tool injection
-type V1ToolCategory struct {
- Name string
- Keywords []string
- Tools string
- Description string
-}
-
-// v1ToolCategories defines tool categories for legacy v1 workflow generation
-var v1ToolCategories = []V1ToolCategory{
- {Name: "data_analysis", Keywords: []string{"analyze", "analysis", "data", "csv", "excel", "spreadsheet", "chart", "graph", "statistics", "visualize", "visualization", "metrics", "calculate", "math"}, Description: "Data analysis and visualization", Tools: "📊 DATA & ANALYSIS:\n- analyze_data: Python data analysis with charts\n- calculate_math: Mathematical calculations\n- read_spreadsheet: Read Excel/CSV files\n- read_data_file: Read and parse data files\n- read_document: Extract text from documents"},
- {Name: "search_web", Keywords: []string{"search", "find", "lookup", "google", "web", "internet", "news", "articles", "scrape", "crawl", "download", "url", "website"}, Description: "Web search and scraping", Tools: "🔍 SEARCH & WEB:\n- search_web: Search the internet\n- search_images: Search for images\n- scrape_web: Scrape content from URL\n- download_file: Download a file from URL"},
- {Name: "content_creation", Keywords: []string{"create", "generate", "write", "document", "pdf", "docx", "presentation", "pptx", "powerpoint", "image", "picture", "photo", "text file", "html"}, Description: "Content creation", Tools: "📝 CONTENT CREATION:\n- create_document: Create DOCX or PDF\n- create_text_file: Create text files\n- create_presentation: Create PowerPoint\n- generate_image: Generate AI images\n- edit_image: Edit images\n- html_to_pdf: Convert HTML to PDF"},
- {Name: "media_processing", Keywords: []string{"audio", "transcribe", "speech", "voice", "mp3", "wav", "video", "image", "describe", "vision", "see", "look"}, Description: "Media processing", Tools: "🎤 MEDIA PROCESSING:\n- transcribe_audio: Transcribe audio\n- describe_image: Analyze images"},
- {Name: "utilities", Keywords: []string{"time", "date", "now", "today", "current", "python", "code", "script", "api", "http", "request", "webhook", "endpoint"}, Description: "Utilities", Tools: "⏰ UTILITIES:\n- get_current_time: Get current time\n- run_python: Execute Python code\n- api_request: Make HTTP requests\n- send_webhook: Send webhook"},
- {Name: "messaging", Keywords: []string{"discord", "slack", "telegram", "teams", "google chat", "email", "sms", "whatsapp", "message", "send", "notify", "notification", "alert", "chat"}, Description: "Messaging", Tools: "💬 MESSAGING:\n- send_discord_message: Discord\n- send_slack_message: Slack\n- send_telegram_message: Telegram\n- send_google_chat_message: Google Chat\n- send_teams_message: Teams\n- send_email: Email\n- twilio_send_sms: SMS\n- twilio_send_whatsapp: WhatsApp"},
- {Name: "video_conferencing", Keywords: []string{"zoom", "meeting", "webinar", "calendly", "calendar", "schedule", "event", "conference", "call", "register", "attendee"}, Description: "Video conferencing", Tools: "📹 VIDEO CONFERENCING:\n- zoom_meeting: Zoom meetings/webinars (actions: create, list, get, register, create_webinar, register_webinar)\n- calendly_events: Calendly events"},
- {Name: "project_management", Keywords: []string{"jira", "linear", "clickup", "trello", "asana", "task", "issue", "ticket", "project", "board", "kanban", "sprint", "backlog"}, Description: "Project management", Tools: "📋 PROJECT MANAGEMENT:\n- jira_issues/jira_create_issue/jira_update_issue\n- linear_issues/linear_create_issue/linear_update_issue\n- clickup_tasks/clickup_create_task/clickup_update_task\n- trello_boards/trello_lists/trello_cards/trello_create_card\n- asana_tasks"},
- {Name: "crm_sales", Keywords: []string{"hubspot", "leadsquared", "mailchimp", "crm", "lead", "contact", "deal", "sales", "customer", "subscriber", "marketing", "audience"}, Description: "CRM & Sales", Tools: "💼 CRM & SALES:\n- hubspot_contacts/hubspot_deals/hubspot_companies\n- leadsquared_leads/leadsquared_create_lead\n- mailchimp_lists/mailchimp_add_subscriber"},
- {Name: "analytics", Keywords: []string{"posthog", "mixpanel", "analytics", "track", "event", "identify", "user profile", "funnel", "cohort", "retention"}, Description: "Analytics", Tools: "📊 ANALYTICS:\n- posthog_capture/posthog_identify/posthog_query\n- mixpanel_track/mixpanel_user_profile"},
- {Name: "code_devops", Keywords: []string{"github", "gitlab", "netlify", "git", "repo", "repository", "issue", "pull request", "pr", "merge", "deploy", "build", "ci", "cd", "code"}, Description: "Code & DevOps", Tools: "🐙 CODE & DEVOPS:\n- github_create_issue/github_list_issues/github_get_repo/github_add_comment\n- gitlab_projects/gitlab_issues/gitlab_mrs\n- netlify_sites/netlify_deploys/netlify_trigger_build"},
- {Name: "productivity", Keywords: []string{"notion", "airtable", "database", "page", "note", "record", "table", "workspace", "wiki"}, Description: "Productivity", Tools: "📓 PRODUCTIVITY:\n- notion_search/notion_query_database/notion_create_page/notion_update_page\n- airtable_list/airtable_read/airtable_create/airtable_update"},
- {Name: "ecommerce", Keywords: []string{"shopify", "shop", "product", "order", "customer", "ecommerce", "store", "inventory", "cart"}, Description: "E-Commerce", Tools: "🛒 E-COMMERCE:\n- shopify_products/shopify_orders/shopify_customers"},
- {Name: "social_media", Keywords: []string{"twitter", "x", "tweet", "post", "social", "media", "follow", "user", "timeline"}, Description: "Social Media", Tools: "🐦 SOCIAL MEDIA:\n- x_search_posts/x_post_tweet/x_get_user/x_get_user_posts"},
-}
-
-// detectToolCategoriesV1 analyzes user message and returns relevant tool categories (legacy v1)
-func detectToolCategoriesV1(userMessage string) []string {
- msg := strings.ToLower(userMessage)
- detected := make(map[string]bool)
-
- for _, category := range v1ToolCategories {
- for _, keyword := range category.Keywords {
- if strings.Contains(msg, keyword) {
- detected[category.Name] = true
- break
- }
- }
- }
-
- // Always include utilities for time-sensitive keywords
- timeSensitiveKeywords := []string{"today", "daily", "recent", "latest", "current", "now", "this week", "this month", "news", "trending", "breaking"}
- for _, keyword := range timeSensitiveKeywords {
- if strings.Contains(msg, keyword) {
- detected["utilities"] = true
- break
- }
- }
-
- // If no categories detected, return a default set
- if len(detected) == 0 {
- detected["data_analysis"] = true
- detected["search_web"] = true
- detected["utilities"] = true
- detected["content_creation"] = true
- }
-
- result := make([]string, 0, len(detected))
- for cat := range detected {
- result = append(result, cat)
- }
- return result
-}
-
-// buildDynamicToolsSectionV1 builds the tools section based on detected categories (legacy v1)
-func buildDynamicToolsSectionV1(categories []string) string {
- var builder strings.Builder
- builder.WriteString("=== AVAILABLE TOOLS (Relevant to your request) ===\n\n")
-
- categoryMap := make(map[string]V1ToolCategory)
- for _, cat := range v1ToolCategories {
- categoryMap[cat.Name] = cat
- }
-
- for _, catName := range categories {
- if cat, ok := categoryMap[catName]; ok {
- builder.WriteString(cat.Tools)
- builder.WriteString("\n\n")
- }
- }
-
- return builder.String()
-}
-
-// buildDynamicSystemPrompt builds the complete system prompt with dynamically injected tools (legacy v1)
-func buildDynamicSystemPrompt(userMessage string) string {
- categories := detectToolCategoriesV1(userMessage)
- toolsSection := buildDynamicToolsSectionV1(categories)
- prompt := strings.Replace(WorkflowSystemPromptBase, "{{DYNAMIC_TOOLS_SECTION}}", toolsSection, 1)
- log.Printf("🔧 [WORKFLOW-GEN] Detected tool categories: %v", categories)
- return prompt
-}
-
-// WorkflowSystemPromptBase is the base system prompt without tools section
-const WorkflowSystemPromptBase = `You are a Clara AI workflow generator. Your ONLY job is to output valid JSON workflow definitions.
-
-CRITICAL: You must ONLY respond with a JSON object. No explanations, no markdown, no code blocks - JUST the JSON.
-
-=== WORKFLOW STRUCTURE ===
-{
- "blocks": [...],
- "connections": [...],
- "variables": [],
- "explanation": "Brief description of what the workflow does or what changed"
-}
-
-=== BLOCK TYPES ===
-1. "variable" - Input block (Start)
- Config: { "operation": "read", "variableName": "input", "defaultValue": "" }
- Optional "inputType": "text" (default), "file", or "json"
- - Use "json" when workflow needs structured input (API-like endpoints, complex data)
- - For JSON input, add "jsonSchema" to define expected structure
- Example JSON input block:
- { "operation": "read", "variableName": "input", "inputType": "json",
- "jsonSchema": { "type": "object", "properties": { "userId": {"type": "string"}, "action": {"type": "string"} } } }
-
-2. "llm_inference" - AI agent with tools (EXECUTION MODE)
- Config: {
- "systemPrompt": "IMPERATIVE instructions - what the agent MUST do",
- "userPrompt": "{{input}}" or "{{previous-block.response}}",
- "temperature": 0.3,
- "enabledTools": ["tool_name"],
- "requiredTools": ["tool_name"],
- "requireToolUsage": true,
- "outputFormat": "json",
- "outputSchema": { JSON Schema object }
- }
-
-3. "code_block" - Direct tool execution (NO LLM, FAST & DETERMINISTIC)
- Config: {
- "toolName": "tool_name_here",
- "argumentMapping": { "param1": "{{input}}", "param2": "{{block-id.response}}" }
- }
-
- USE code_block WHEN:
- - Task is PURELY mechanical (no reasoning/decisions needed)
- - All data and parameters are already available
- - Examples: get current time, send pre-formatted message, make API call with known params
-
- USE llm_inference INSTEAD WHEN:
- - Need to DECIDE what to search/do
- - Need to INTERPRET, FORMAT, or SUMMARIZE data
- - ANY intelligent decision-making is required
-
-=== BLOCK TYPE DECISION GUIDE ===
-Q: Does the task need ANY reasoning, decisions, or interpretation?
- YES → Use "llm_inference" (LLM calls tools with intelligence)
- NO → Use "code_block" (direct execution, faster & cheaper)
-
-EXAMPLES - When to use code_block:
-- "Get current time" → code_block with toolName="get_current_time"
-- "Send this exact message to Discord" (message already formatted) → code_block
-- "Download file from URL" (URL provided) → code_block
-- "Calculate 2+2" → code_block with toolName="calculate_math"
-
-EXAMPLES - When to use llm_inference:
-- "Search for news about X" → llm_inference (LLM decides search query)
-- "Analyze this data" → llm_inference (LLM interprets results)
-- "Format and send to Discord" → llm_inference (needs formatting decision)
-- "Summarize the results" → llm_inference (needs interpretation)
-
-=== SYSTEM PROMPT WRITING RULES (CRITICAL!) ===
-System prompts MUST be written in IMPERATIVE/COMMAND style, not conversational:
-
-CORRECT (Imperative - use these patterns):
-- "Search for news about the topic. Call search_web. Return top 3 results with titles and summaries."
-- "Send this content to Discord NOW using send_discord_message. Include embed_title."
-- "Analyze the data. Generate a bar chart. Use analyze_data tool immediately."
-
-WRONG (Conversational - NEVER use):
-- "You should search for news..." (too passive)
-- "Please format and send to Discord..." (too polite/optional)
-- "Can you analyze this data..." (implies optionality)
-- "If you want, you could..." (gives choice - NO!)
-
-WRONG (Question-asking - NEVER generate prompts that ask questions):
-- "What topic would you like to search?" (NO - data is provided)
-- "Should I include charts?" (NO - decide based on context)
-- "Would you like me to..." (NO - just do it)
-
-{{DYNAMIC_TOOLS_SECTION}}
-
-=== TOOL CONFIGURATION (CRITICAL FOR RELIABILITY!) ===
-For each LLM block with tools, you MUST include:
-- "enabledTools": List of tools the block CAN use
-- "requiredTools": List of tools the block MUST use (usually same as enabledTools)
-- "requireToolUsage": true (forces tool usage, prevents text-only responses)
-- "temperature": 0.3 (low for deterministic execution)
-
-Example for Discord Publisher block:
-{
- "enabledTools": ["send_discord_message"],
- "requiredTools": ["send_discord_message"],
- "requireToolUsage": true,
- "temperature": 0.3
-}
-
-=== STRUCTURED OUTPUT (CRITICAL FOR RELIABILITY!) ===
-ALWAYS use structured outputs for blocks that return data to be consumed by other blocks or rendered in UIs.
-This ensures 100% predictable, parseable outputs.
-
-When to use structured output:
-- Data fetching blocks (news, search results, API data)
-- Analysis blocks that return metrics or insights
-- Any block whose output will be displayed in a UI
-- Blocks that extract specific information
-
-How to configure structured output:
-1. Set "outputFormat": "json"
-2. Define "outputSchema" with JSON Schema
-3. The schema MUST match what downstream blocks or UI expect
-
-Example for News Fetcher block:
-{
- "systemPrompt": "FIRST call get_current_time. THEN call search_web for news. Return EXACTLY in the schema format.",
- "temperature": 0.3,
- "enabledTools": ["get_current_time", "search_web"],
- "requiredTools": ["get_current_time", "search_web"],
- "requireToolUsage": true,
- "outputFormat": "json",
- "outputSchema": {
- "type": "object",
- "properties": {
- "articles": {
- "type": "array",
- "items": {
- "type": "object",
- "properties": {
- "title": {"type": "string"},
- "source": {"type": "string"},
- "url": {"type": "string"},
- "summary": {"type": "string"},
- "publishedDate": {"type": "string"}
- },
- "required": ["title", "source", "url", "summary", "publishedDate"]
- }
- },
- "totalResults": {"type": "number"},
- "fetchedAt": {"type": "string"}
- },
- "required": ["articles", "totalResults", "fetchedAt"],
- "additionalProperties": false
- }
-}
-
-Common schema patterns:
-- News/Articles WITH metadata: { articles: [{ title, source, url, summary, publishedDate }], totalResults, fetchedAt }
-- Simple list (array at root): [{ id, name, value }] - use "type": "array" with "items" schema
-- Metrics/Stats: { metrics: { key: value }, summary, analyzedAt }
-- List Results: { items: [{ name, description, value }], count, retrievedAt }
-- Analysis: { insights: [...], recommendations: [...], confidence: number }
-
-Example for Simple Product List (array at root):
-{
- "systemPrompt": "Call search_products and return the list of products.",
- "outputFormat": "json",
- "outputSchema": {
- "type": "array",
- "items": {
- "type": "object",
- "properties": {
- "id": {"type": "string"},
- "name": {"type": "string"},
- "price": {"type": "number"}
- },
- "required": ["id", "name", "price"]
- }
- }
-}
-
-RULES for structured output:
-1. Use "additionalProperties": false to prevent extra fields
-2. CRITICAL: In "required" arrays, you MUST list ALL properties defined in "properties" - OpenAI's strict mode rejects partial required arrays
-3. Use descriptive property names (camelCase)
-4. Include metadata (fetchedAt, analyzedAt, etc.)
-5. Schema MUST be strict - no optional variations
-6. Every nested object needs its own "required" array listing ALL its properties
-7. ARRAYS AT ROOT LEVEL: You can use arrays directly without wrapping in an object:
- - For simple lists: { "type": "array", "items": { "type": "object", "properties": {...}, "required": [...] } }
- - For data + metadata: { "type": "object", "properties": { "items": {...}, "total": {...} }, "required": [...] }
- - Use arrays when returning just a list, use objects when you need metadata too
-
-=== CREDENTIAL HANDLING ===
-For integration tools (Discord, Slack, webhooks):
-- Credentials are AUTO-INJECTED at runtime
-- DO NOT include webhook URLs in prompts
-- DO NOT tell the agent to ask for credentials
-- System prompts should command: "Send to Discord NOW" (not "provide your webhook URL")
-
-=== TIME-SENSITIVE QUERIES (CRITICAL!) ===
-When the user's request involves time-sensitive information, the search block MUST also call get_current_time:
-
-TIME-SENSITIVE KEYWORDS (if any of these appear, add get_current_time):
-- "today", "daily", "recent", "latest", "current", "now", "this week", "this month"
-- "news", "events", "updates", "trending", "breaking"
-- "stock", "price", "weather", "score", "live"
-
-For time-sensitive search blocks, use BOTH tools:
-{
- "enabledTools": ["get_current_time", "search_web"],
- "requiredTools": ["get_current_time", "search_web"],
- "systemPrompt": "FIRST call get_current_time to get today's date. THEN search for [topic] using that date. Include the date in your search query for accurate results."
-}
-
-EXAMPLE - User asks "Get me today's AI news":
-{
- "systemPrompt": "FIRST call get_current_time to get today's date and time. THEN call search_web with the topic AND the current date (e.g., 'AI news December 2024'). Return top 3 results with titles, sources, and the date they were published.",
- "enabledTools": ["get_current_time", "search_web"],
- "requiredTools": ["get_current_time", "search_web"]
-}
-
-=== TOOL ASSIGNMENT RULES ===
-Each block = ONE specific task = ONE set of related tools. Never mix unrelated tools!
-
-TOOL SELECTION BY BLOCK PURPOSE:
-- Research/Search block (time-sensitive): enabledTools=["get_current_time", "search_web"], requiredTools=["get_current_time", "search_web"]
-- Research/Search block (general): enabledTools=["search_web"], requiredTools=["search_web"]
-- Data Analysis block: enabledTools=["analyze_data"], requiredTools=["analyze_data"]
-- Spreadsheet Reading block: enabledTools=["read_spreadsheet"], requiredTools=["read_spreadsheet"]
-- Audio Transcription block: enabledTools=["transcribe_audio"], requiredTools=["transcribe_audio"]
-- Image Analysis block: enabledTools=["describe_image"], requiredTools=["describe_image"]
-- Document Reading block: enabledTools=["read_document"], requiredTools=["read_document"]
-- Discord Publisher: enabledTools=["send_discord_message"], requiredTools=["send_discord_message"]
-- Slack Publisher: enabledTools=["send_slack_message"], requiredTools=["send_slack_message"]
-- Telegram Publisher: enabledTools=["send_telegram_message"], requiredTools=["send_telegram_message"]
-- Google Chat Publisher: enabledTools=["send_google_chat_message"], requiredTools=["send_google_chat_message"]
-- Content Writer: enabledTools=[] (no tools - generates text only, requireToolUsage=false)
-
-=== DATA FLOW & VARIABLE PATHS (CRITICAL!) ===
-
-UNDERSTANDING VARIABLE BLOCKS:
-The Start block (type: "variable") has two important fields:
- - "id": "start" (the block's ID)
- - "variableName": "input" (creates a global variable)
-
-The variableName field creates a TOP-LEVEL key in the workflow context!
-
-Example Start block output structure:
-{
- "value": {"email": "test@example.com", "name": "John"},
- "input": {"email": "test@example.com", "name": "John"} ← variableName creates this key
-}
-
-VARIABLE PATH RULES:
-1. To access the ENTIRE input: {{input}}
-2. To access nested fields: {{input.email}}, {{input.name}}, {{input.phone}}
-3. Previous block outputs: {{block-id.response}}
-4. Block outputs are ALREADY RESOLVED - no need to fetch data
-
-CORRECT PATHS:
-- {{input}} - Entire workflow input (from start block's variableName)
-- {{input.email}} - Nested field from input
-- {{news-researcher.response}} - Previous block's response
-- {{block-id.response.articles}} - Nested data from previous block
-
-WRONG PATHS (NEVER use these):
-- {{start.email}} - NO! "email" is INSIDE input, not a property of start
-- {{start.response.email}} - NO! Start block doesn't have "response"
-- {{start.output.value}} - NO! Use {{input}} instead
-- {{block.output.value}} - NO! Use {{block.response}} instead
-
-FOR CODE_BLOCKS with nested data:
-When accessing nested fields from previous blocks in argumentMapping:
-{
- "toolName": "mongodb_write",
- "argumentMapping": {
- "action": "insertOne",
- "collection": "users",
- "document": {
- "email": "{{input.email}}", ← Access nested field
- "name": "{{input.name}}", ← Access nested field
- "phone": "{{input.phone}}", ← Access nested field
- "created_at": "{{get-current-time.response}}"
- }
- }
-}
-
-=== BLOCK ID NAMING ===
-Block "id" MUST be kebab-case of "name":
-- "News Researcher" → id: "news-researcher"
-- "Discord Publisher" → id: "discord-publisher"
-
-=== LAYOUT ===
-- Start block: position { "x": 250, "y": 50 }
-- Space blocks 150px vertically
-- timeout: 30 for variable, 120 for LLM blocks
-
-=== EXAMPLE 1: News Search + Discord (TIME-SENSITIVE) ===
-User: "Create an agent that searches for news and posts to Discord"
-
-{
- "blocks": [
- {
- "id": "start",
- "type": "variable",
- "name": "Start",
- "description": "Input topic for news search",
- "config": { "operation": "read", "variableName": "input", "defaultValue": "AI news" },
- "position": { "x": 250, "y": 50 },
- "timeout": 30
- },
- {
- "id": "news-researcher",
- "type": "llm_inference",
- "name": "News Researcher",
- "description": "Search and summarize latest news",
- "config": {
- "systemPrompt": "FIRST call get_current_time to get today's date. THEN call search_web for news about the given topic, including the current date in your query (e.g., 'AI news December 2024'). Return results EXACTLY in the output schema format with top 3 articles.",
- "userPrompt": "{{input}}",
- "temperature": 0.3,
- "enabledTools": ["get_current_time", "search_web"],
- "requiredTools": ["get_current_time", "search_web"],
- "requireToolUsage": true,
- "outputFormat": "json",
- "outputSchema": {
- "type": "object",
- "properties": {
- "articles": {
- "type": "array",
- "items": {
- "type": "object",
- "properties": {
- "title": {"type": "string"},
- "source": {"type": "string"},
- "url": {"type": "string"},
- "summary": {"type": "string"},
- "publishedDate": {"type": "string"}
- },
- "required": ["title", "source", "url", "summary"]
- }
- },
- "totalResults": {"type": "number"},
- "fetchedAt": {"type": "string"}
- },
- "required": ["articles", "totalResults", "fetchedAt"],
- "additionalProperties": false
- }
- },
- "position": { "x": 250, "y": 200 },
- "timeout": 120
- },
- {
- "id": "discord-publisher",
- "type": "llm_inference",
- "name": "Discord Publisher",
- "description": "Format and send news to Discord",
- "config": {
- "systemPrompt": "Send this news summary to Discord NOW. Call send_discord_message with: content containing a brief intro, embed_title set to 'Latest News Update', embed_description with the full summary. Execute immediately.",
- "userPrompt": "{{news-researcher.response}}",
- "temperature": 0.3,
- "enabledTools": ["send_discord_message"],
- "requiredTools": ["send_discord_message"],
- "requireToolUsage": true
- },
- "position": { "x": 250, "y": 350 },
- "timeout": 120
- }
- ],
- "connections": [
- { "id": "conn-1", "sourceBlockId": "start", "sourceOutput": "output", "targetBlockId": "news-researcher", "targetInput": "input" },
- { "id": "conn-2", "sourceBlockId": "news-researcher", "sourceOutput": "output", "targetBlockId": "discord-publisher", "targetInput": "input" }
- ],
- "variables": [],
- "explanation": "3 blocks: Start→News Researcher (MUST call get_current_time THEN search_web)→Discord Publisher (MUST call send_discord_message)"
-}
-
-=== EXAMPLE 2: Data Analysis + Discord ===
-User: "Analyze CSV data and send chart to Discord"
-
-{
- "blocks": [
- {
- "id": "start",
- "type": "variable",
- "name": "Start",
- "description": "Receive CSV file input",
- "config": { "operation": "read", "variableName": "input", "inputType": "file" },
- "position": { "x": 250, "y": 50 },
- "timeout": 30
- },
- {
- "id": "data-analyzer",
- "type": "llm_inference",
- "name": "Data Analyzer",
- "description": "Analyze data and generate charts",
- "config": {
- "systemPrompt": "Analyze this data immediately. Call analyze_data to generate visualizations. Create at least one meaningful chart showing key insights. Return analysis summary.",
- "userPrompt": "{{input}}",
- "temperature": 0.3,
- "enabledTools": ["analyze_data"],
- "requiredTools": ["analyze_data"],
- "requireToolUsage": true
- },
- "position": { "x": 250, "y": 200 },
- "timeout": 120
- },
- {
- "id": "discord-publisher",
- "type": "llm_inference",
- "name": "Discord Publisher",
- "description": "Send analysis results and charts to Discord",
- "config": {
- "systemPrompt": "Send this analysis to Discord NOW. Call send_discord_message with the summary as content and include the chart image. Execute immediately - do not ask questions.",
- "userPrompt": "{{data-analyzer.response}}",
- "temperature": 0.3,
- "enabledTools": ["send_discord_message"],
- "requiredTools": ["send_discord_message"],
- "requireToolUsage": true
- },
- "position": { "x": 250, "y": 350 },
- "timeout": 120
- }
- ],
- "connections": [
- { "id": "conn-1", "sourceBlockId": "start", "sourceOutput": "output", "targetBlockId": "data-analyzer", "targetInput": "input" },
- { "id": "conn-2", "sourceBlockId": "data-analyzer", "sourceOutput": "output", "targetBlockId": "discord-publisher", "targetInput": "input" }
- ],
- "variables": [],
- "explanation": "3 blocks: Start (file)→Data Analyzer (MUST call analyze_data)→Discord Publisher (MUST call send_discord_message)"
-}
-
-=== EXAMPLE 3: Audio Transcription + Summary ===
-User: "Create an agent that transcribes audio files and summarizes them"
-
-{
- "blocks": [
- {
- "id": "start",
- "type": "variable",
- "name": "Start",
- "description": "Receive audio file input",
- "config": { "operation": "read", "variableName": "input", "inputType": "file", "acceptedFileTypes": ["audio"] },
- "position": { "x": 250, "y": 50 },
- "timeout": 30
- },
- {
- "id": "audio-transcriber",
- "type": "llm_inference",
- "name": "Audio Transcriber",
- "description": "Transcribe audio to text",
- "config": {
- "systemPrompt": "Transcribe the provided audio file immediately. Call transcribe_audio with the file_id from the input. Return the full transcription text.",
- "userPrompt": "{{input}}",
- "temperature": 0.3,
- "enabledTools": ["transcribe_audio"],
- "requiredTools": ["transcribe_audio"],
- "requireToolUsage": true
- },
- "position": { "x": 250, "y": 200 },
- "timeout": 120
- },
- {
- "id": "content-summarizer",
- "type": "llm_inference",
- "name": "Content Summarizer",
- "description": "Summarize the transcribed content",
- "config": {
- "systemPrompt": "Summarize this transcription. Extract key points, main topics discussed, and important details. Provide a concise summary with bullet points.",
- "userPrompt": "{{audio-transcriber.response}}",
- "temperature": 0.5,
- "enabledTools": [],
- "requireToolUsage": false
- },
- "position": { "x": 250, "y": 350 },
- "timeout": 120
- }
- ],
- "connections": [
- { "id": "conn-1", "sourceBlockId": "start", "sourceOutput": "output", "targetBlockId": "audio-transcriber", "targetInput": "input" },
- { "id": "conn-2", "sourceBlockId": "audio-transcriber", "sourceOutput": "output", "targetBlockId": "content-summarizer", "targetInput": "input" }
- ],
- "variables": [],
- "explanation": "3 blocks: Start (audio file)→Audio Transcriber (MUST call transcribe_audio)→Content Summarizer (text generation)"
-}
-
-=== EXAMPLE 4: MongoDB Insert with Nested Field Access ===
-User: "Insert user data into MongoDB"
-
-This example shows correct variable path usage for accessing nested input fields:
-
-{
- "blocks": [
- {
- "id": "start",
- "type": "variable",
- "name": "Start",
- "description": "Receive user input as JSON",
- "config": {
- "operation": "read",
- "variableName": "input",
- "inputType": "json",
- "jsonSchema": {
- "type": "object",
- "properties": {
- "email": {"type": "string"},
- "name": {"type": "string"},
- "phone": {"type": "string"}
- }
- }
- },
- "position": { "x": 250, "y": 50 },
- "timeout": 30
- },
- {
- "id": "get-current-time",
- "type": "code_block",
- "name": "Get Current Time",
- "description": "Get timestamp for record",
- "config": {
- "toolName": "get_current_time",
- "argumentMapping": {}
- },
- "position": { "x": 250, "y": 200 },
- "timeout": 30
- },
- {
- "id": "insert-user",
- "type": "code_block",
- "name": "Insert User",
- "description": "Insert user into MongoDB with nested field access",
- "config": {
- "toolName": "mongodb_write",
- "argumentMapping": {
- "action": "insertOne",
- "collection": "users",
- "document": {
- "email": "{{input.email}}",
- "name": "{{input.name}}",
- "phone": "{{input.phone}}",
- "created_at": "{{get-current-time.response}}",
- "status": "active"
- }
- }
- },
- "position": { "x": 250, "y": 350 },
- "timeout": 30
- }
- ],
- "connections": [
- { "id": "conn-1", "sourceBlockId": "start", "sourceOutput": "output", "targetBlockId": "get-current-time", "targetInput": "input" },
- { "id": "conn-2", "sourceBlockId": "get-current-time", "sourceOutput": "output", "targetBlockId": "insert-user", "targetInput": "input" }
- ],
- "variables": [],
- "explanation": "Inserts user data into MongoDB using correct nested field paths: {{input.email}}, {{input.name}}, {{input.phone}}"
-}
-
-KEY POINTS IN THIS EXAMPLE:
-- Start block has variableName: "input", so we use {{input.email}}, NOT {{start.email}}
-- Code_block argumentMapping can have nested objects
-- Deep interpolation resolves {{...}} at any nesting level
-- Mixed literal values ("active") and variable references work together
-
-=== EXAMPLE 5: Mixed LLM + code_block (EFFICIENT HYBRID) ===
-User: "Search for AI news and send to Discord"
-
-This example shows how to mix llm_inference (for research) with code_block (for sending).
-The llm_inference block does the intelligent work (search + format), then code_block sends directly.
-
-{
- "blocks": [
- {
- "id": "start",
- "type": "variable",
- "name": "Start",
- "description": "Input topic for news search",
- "config": { "operation": "read", "variableName": "input", "defaultValue": "AI news" },
- "position": { "x": 250, "y": 50 },
- "timeout": 30
- },
- {
- "id": "news-researcher",
- "type": "llm_inference",
- "name": "News Researcher",
- "description": "Search news and format for Discord",
- "config": {
- "systemPrompt": "FIRST call get_current_time. THEN call search_web for news. Format results as a Discord message with embed fields. Return EXACTLY in the output schema format.",
- "userPrompt": "{{input}}",
- "temperature": 0.3,
- "enabledTools": ["get_current_time", "search_web"],
- "requiredTools": ["get_current_time", "search_web"],
- "requireToolUsage": true,
- "outputFormat": "json",
- "outputSchema": {
- "type": "object",
- "properties": {
- "content": {"type": "string", "description": "Brief intro message"},
- "embed_title": {"type": "string", "description": "Discord embed title"},
- "embed_description": {"type": "string", "description": "Full news summary with links"}
- },
- "required": ["content", "embed_title", "embed_description"],
- "additionalProperties": false
- }
- },
- "position": { "x": 250, "y": 200 },
- "timeout": 120
- },
- {
- "id": "discord-sender",
- "type": "code_block",
- "name": "Discord Sender",
- "description": "Send pre-formatted message to Discord (no LLM needed)",
- "config": {
- "toolName": "send_discord_message",
- "argumentMapping": {
- "content": "{{news-researcher.response.content}}",
- "embed_title": "{{news-researcher.response.embed_title}}",
- "embed_description": "{{news-researcher.response.embed_description}}"
- }
- },
- "position": { "x": 250, "y": 350 },
- "timeout": 30
- }
- ],
- "connections": [
- { "id": "conn-1", "sourceBlockId": "start", "sourceOutput": "output", "targetBlockId": "news-researcher", "targetInput": "input" },
- { "id": "conn-2", "sourceBlockId": "news-researcher", "sourceOutput": "output", "targetBlockId": "discord-sender", "targetInput": "input" }
- ],
- "variables": [],
- "explanation": "3 blocks: Start→News Researcher (llm_inference: search + format)→Discord Sender (code_block: direct send, no LLM overhead)"
-}
-
-WHY use code_block for Discord Sender?
-- The message is ALREADY formatted by the researcher
-- No decisions needed - just send the exact content
-- FASTER execution (no LLM API call)
-- CHEAPER (no token costs)
-- MORE RELIABLE (no LLM interpretation)
-
-REMEMBER:
-- Temperature = 0.3 for all LLM blocks (deterministic)
-- requiredTools = same as enabledTools (forces tool usage)
-- requireToolUsage = true (validates tool was called)
-- System prompts use IMPERATIVE style (commands, not suggestions)
-- Use code_block for mechanical tasks with NO reasoning needed (faster, cheaper)
-- Use llm_inference when decisions, formatting, or interpretation is required
-- code_block timeout = 30 (no LLM), llm_inference timeout = 120
-- Output ONLY valid JSON. No text before or after.`
-
-// GenerateWorkflow generates a workflow based on user input
-func (s *WorkflowGeneratorService) GenerateWorkflow(req *models.WorkflowGenerateRequest, userID string) (*models.WorkflowGenerateResponse, error) {
- log.Printf("🔧 [WORKFLOW-GEN] Generating workflow for agent %s, user %s", req.AgentID, userID)
-
- // Get provider and model
- provider, modelID, err := s.getProviderAndModel(req.ModelID)
- if err != nil {
- return &models.WorkflowGenerateResponse{
- Success: false,
- Error: fmt.Sprintf("Failed to get provider: %v", err),
- }, nil
- }
-
- // Build the user message
- userMessage := s.buildUserMessage(req)
-
- // Build dynamic system prompt with relevant tools based on user request
- systemPrompt := buildDynamicSystemPrompt(req.UserMessage)
-
- // Build messages array with conversation history for better context
- messages := []map[string]interface{}{
- {
- "role": "system",
- "content": systemPrompt,
- },
- }
-
- // Add conversation history if provided (for multi-turn context)
- if len(req.ConversationHistory) > 0 {
- for _, msg := range req.ConversationHistory {
- messages = append(messages, map[string]interface{}{
- "role": msg.Role,
- "content": msg.Content,
- })
- }
- }
-
- // Add current user message
- messages = append(messages, map[string]interface{}{
- "role": "user",
- "content": userMessage,
- })
-
- // Build request body with structured output
- requestBody := map[string]interface{}{
- "model": modelID,
- "messages": messages,
- "stream": false,
- "temperature": 0.3, // Lower temperature for more consistent JSON output
- }
-
- // Add response_format for structured output (OpenAI-compatible)
- requestBody["response_format"] = map[string]interface{}{
- "type": "json_object",
- }
-
- reqBody, err := json.Marshal(requestBody)
- if err != nil {
- return nil, fmt.Errorf("failed to marshal request: %w", err)
- }
-
- log.Printf("📤 [WORKFLOW-GEN] Sending request to %s with model %s", provider.BaseURL, modelID)
-
- // Create HTTP request
- httpReq, err := http.NewRequest("POST", provider.BaseURL+"/chat/completions", bytes.NewBuffer(reqBody))
- if err != nil {
- return nil, fmt.Errorf("failed to create request: %w", err)
- }
-
- httpReq.Header.Set("Content-Type", "application/json")
- httpReq.Header.Set("Authorization", "Bearer "+provider.APIKey)
-
- // Send request with timeout
- client := &http.Client{Timeout: 120 * time.Second}
- resp, err := client.Do(httpReq)
- if err != nil {
- return nil, fmt.Errorf("request failed: %w", err)
- }
- defer resp.Body.Close()
-
- // Read response
- body, err := io.ReadAll(resp.Body)
- if err != nil {
- return nil, fmt.Errorf("failed to read response: %w", err)
- }
-
- if resp.StatusCode != http.StatusOK {
- log.Printf("⚠️ [WORKFLOW-GEN] API error: %s", string(body))
- return &models.WorkflowGenerateResponse{
- Success: false,
- Error: fmt.Sprintf("API error (status %d): %s", resp.StatusCode, string(body)),
- }, nil
- }
-
- // Parse API response
- var apiResponse struct {
- Choices []struct {
- Message struct {
- Content string `json:"content"`
- } `json:"message"`
- } `json:"choices"`
- }
-
- if err := json.Unmarshal(body, &apiResponse); err != nil {
- return nil, fmt.Errorf("failed to parse API response: %w", err)
- }
-
- if len(apiResponse.Choices) == 0 {
- return &models.WorkflowGenerateResponse{
- Success: false,
- Error: "No response from model",
- }, nil
- }
-
- content := apiResponse.Choices[0].Message.Content
- log.Printf("📥 [WORKFLOW-GEN] Received response (%d chars)", len(content))
-
- // Parse the workflow JSON from the response
- return s.parseWorkflowResponse(content, req.CurrentWorkflow != nil, req.AgentID)
-}
-
-// buildUserMessage constructs the user message for workflow generation
-func (s *WorkflowGeneratorService) buildUserMessage(req *models.WorkflowGenerateRequest) string {
- if req.CurrentWorkflow != nil && len(req.CurrentWorkflow.Blocks) > 0 {
- // Modification request - include current workflow
- workflowJSON, _ := json.MarshalIndent(req.CurrentWorkflow, "", " ")
- return fmt.Sprintf(`MODIFICATION REQUEST
-
-Current workflow:
-%s
-
-User request: %s
-
-Output the complete modified workflow JSON with all blocks (not just changes).`, string(workflowJSON), req.UserMessage)
- }
-
- // New workflow request
- return fmt.Sprintf("CREATE NEW WORKFLOW\n\nUser request: %s", req.UserMessage)
-}
-
-// parseWorkflowResponse parses the LLM response into a workflow
-func (s *WorkflowGeneratorService) parseWorkflowResponse(content string, isModification bool, agentID string) (*models.WorkflowGenerateResponse, error) {
- // Try to extract JSON from the response (handle markdown code blocks)
- jsonContent := s.extractJSON(content)
-
- // Parse the workflow
- var workflowData struct {
- Blocks []models.Block `json:"blocks"`
- Connections []models.Connection `json:"connections"`
- Variables []models.Variable `json:"variables"`
- Explanation string `json:"explanation"`
- }
-
- if err := json.Unmarshal([]byte(jsonContent), &workflowData); err != nil {
- log.Printf("⚠️ [WORKFLOW-GEN] Failed to parse workflow JSON: %v", err)
- log.Printf("⚠️ [WORKFLOW-GEN] Content: %s", jsonContent[:min(500, len(jsonContent))])
- return &models.WorkflowGenerateResponse{
- Success: false,
- Error: fmt.Sprintf("Failed to parse workflow JSON: %v", err),
- Explanation: content, // Return raw content as explanation
- }, nil
- }
-
- // Log the generated workflow for debugging
- prettyWorkflow, _ := json.MarshalIndent(workflowData, "", " ")
- log.Printf("📋 [WORKFLOW-GEN] Generated workflow:\n%s", string(prettyWorkflow))
-
- // Post-process blocks: set normalizedId to match id
- for i := range workflowData.Blocks {
- if workflowData.Blocks[i].NormalizedID == "" {
- workflowData.Blocks[i].NormalizedID = workflowData.Blocks[i].ID
- }
- }
-
- // Validate the workflow
- errors := s.validateWorkflow(&workflowData)
- if len(errors) > 0 {
- log.Printf("⚠️ [WORKFLOW-GEN] Workflow validation errors: %v", errors)
- }
-
- // Determine action
- action := "create"
- if isModification {
- action = "modify"
- }
-
- // Build the workflow with generated IDs
- workflow := &models.Workflow{
- ID: uuid.New().String(),
- AgentID: agentID,
- Blocks: workflowData.Blocks,
- Connections: workflowData.Connections,
- Variables: workflowData.Variables,
- Version: 1,
- }
-
- log.Printf("✅ [WORKFLOW-GEN] Successfully parsed workflow: %d blocks, %d connections",
- len(workflow.Blocks), len(workflow.Connections))
-
- return &models.WorkflowGenerateResponse{
- Success: true,
- Workflow: workflow,
- Explanation: workflowData.Explanation,
- Action: action,
- Version: 1,
- Errors: errors,
- }, nil
-}
-
-// extractJSON extracts JSON from a response that might be wrapped in markdown
-func (s *WorkflowGeneratorService) extractJSON(content string) string {
- content = strings.TrimSpace(content)
-
- // If it starts with {, assume it's pure JSON
- if strings.HasPrefix(content, "{") {
- return content
- }
-
- // Try to extract from markdown code block
- re := regexp.MustCompile("(?s)```(?:json)?\\s*\\n?(\\{.*\\})\\s*\\n?```")
- matches := re.FindStringSubmatch(content)
- if len(matches) > 1 {
- return matches[1]
- }
-
- // Try to find JSON object anywhere in the content
- re = regexp.MustCompile(`(?s)\{.*"blocks".*\}`)
- match := re.FindString(content)
- if match != "" {
- return match
- }
-
- return content
-}
-
-// validateWorkflow validates the workflow structure
-func (s *WorkflowGeneratorService) validateWorkflow(workflow *struct {
- Blocks []models.Block `json:"blocks"`
- Connections []models.Connection `json:"connections"`
- Variables []models.Variable `json:"variables"`
- Explanation string `json:"explanation"`
-}) []models.ValidationError {
- var errors []models.ValidationError
-
- // Check for empty blocks
- if len(workflow.Blocks) == 0 {
- errors = append(errors, models.ValidationError{
- Type: "schema",
- Message: "Workflow must have at least one block",
- })
- return errors
- }
-
- // Build block ID set for connection validation
- blockIDs := make(map[string]bool)
- for _, block := range workflow.Blocks {
- blockIDs[block.ID] = true
-
- // Validate block type
- if block.Type != "llm_inference" && block.Type != "variable" {
- errors = append(errors, models.ValidationError{
- Type: "schema",
- Message: fmt.Sprintf("Invalid block type: %s", block.Type),
- BlockID: block.ID,
- })
- }
- }
-
- // Validate connections reference valid blocks
- for _, conn := range workflow.Connections {
- if !blockIDs[conn.SourceBlockID] {
- errors = append(errors, models.ValidationError{
- Type: "missing_input",
- Message: fmt.Sprintf("Connection references non-existent source block: %s", conn.SourceBlockID),
- ConnectionID: conn.ID,
- })
- }
- if !blockIDs[conn.TargetBlockID] {
- errors = append(errors, models.ValidationError{
- Type: "missing_input",
- Message: fmt.Sprintf("Connection references non-existent target block: %s", conn.TargetBlockID),
- ConnectionID: conn.ID,
- })
- }
- }
-
- return errors
-}
-
-// getProviderAndModel gets the provider and model for the request
-func (s *WorkflowGeneratorService) getProviderAndModel(modelID string) (*models.Provider, string, error) {
- // If no model specified, use default
- if modelID == "" {
- provider, model, err := s.chatService.GetDefaultProviderWithModel()
- if err != nil {
- return nil, "", err
- }
- return provider, model, nil
- }
-
- // Try to find the model in the database
- var providerID int
- var modelName string
-
- err := s.db.QueryRow(`
- SELECT m.name, m.provider_id
- FROM models m
- WHERE m.id = ? AND m.is_visible = 1
- `, modelID).Scan(&modelName, &providerID)
-
- if err != nil {
- // Try as model alias
- if provider, actualModel, found := s.chatService.ResolveModelAlias(modelID); found {
- return provider, actualModel, nil
- }
- // Fall back to default
- return s.chatService.GetDefaultProviderWithModel()
- }
-
- // Get the provider
- provider, err := s.providerService.GetByID(providerID)
- if err != nil {
- return nil, "", fmt.Errorf("failed to get provider: %w", err)
- }
-
- return provider, modelName, nil
-}
-
-func min(a, b int) int {
- if a < b {
- return a
- }
- return b
-}
-
-// AgentMetadata holds generated name and description for an agent
-type AgentMetadata struct {
- Name string `json:"name"`
- Description string `json:"description"`
-}
-
-// GenerateAgentMetadata generates a name and description for an agent based on the user's request
-func (s *WorkflowGeneratorService) GenerateAgentMetadata(userMessage string) (*AgentMetadata, error) {
- // Use ChatService's GetTextProviderWithModel which dynamically finds a text-capable provider
- // This method checks model aliases from config and falls back to database providers
- provider, modelID, err := s.chatService.GetTextProviderWithModel()
- if err != nil {
- return nil, fmt.Errorf("failed to get text provider for metadata generation: %w", err)
- }
-
- log.Printf("🔍 [METADATA-GEN] Using dynamic model: %s (provider: %s)", modelID, provider.Name)
-
- // Build a prompt that generates both name and description in a simple format
- messages := []map[string]interface{}{
- {
- "role": "system",
- "content": `Generate a catchy name and brief description for an AI agent.
-
-RULES for name:
-- 2-4 words maximum
-- Action-oriented and memorable (e.g., "News Pulse", "Data Wizard", "Chart Crafter", "Report Runner")
-- Use descriptive verbs or nouns that indicate the agent's purpose
-- NEVER use generic words like "Agent", "Bot", "AI", "Assistant", "Helper", "Tool"
-- Make it sound professional but approachable
-- Be creative and specific to the task
-
-RULES for description:
-- One sentence, maximum 100 characters
-- Start with a verb (e.g., "Searches...", "Analyzes...", "Monitors...")
-- Be specific about what the agent does
-- Mention the key output or destination if relevant
-
-RESPOND with EXACTLY this format (two lines only):
-NAME: [Your agent name here]
-DESC: [Your one-line description here]
-
-Example for "search for AI news and post to Discord":
-NAME: News Pulse
-DESC: Searches for latest tech news and posts summaries to Discord
-
-Example for "analyze CSV data and create charts":
-NAME: Chart Crafter
-DESC: Analyzes data files and generates visual charts`,
- },
- {
- "role": "user",
- "content": fmt.Sprintf("Agent purpose: %s", userMessage),
- },
- }
-
- // Simple request like chat title generation - no structured output
- requestBody := map[string]interface{}{
- "model": modelID,
- "messages": messages,
- "stream": false,
- "temperature": 0.7,
- }
-
- reqBody, err := json.Marshal(requestBody)
- if err != nil {
- return nil, fmt.Errorf("failed to marshal request: %w", err)
- }
-
- // Create HTTP request - use base URL with /chat/completions
- apiURL := provider.BaseURL + "/chat/completions"
- log.Printf("🔍 [METADATA-GEN] Sending request to: %s with model: %s", apiURL, modelID)
-
- httpReq, err := http.NewRequest("POST", apiURL, bytes.NewBuffer(reqBody))
- if err != nil {
- return nil, fmt.Errorf("failed to create request: %w", err)
- }
-
- httpReq.Header.Set("Content-Type", "application/json")
- httpReq.Header.Set("Authorization", "Bearer "+provider.APIKey)
-
- // Send request with timeout
- client := &http.Client{Timeout: 30 * time.Second}
- resp, err := client.Do(httpReq)
- if err != nil {
- return nil, fmt.Errorf("request failed: %w", err)
- }
- defer resp.Body.Close()
-
- body, err := io.ReadAll(resp.Body)
- if err != nil {
- return nil, fmt.Errorf("failed to read response: %w", err)
- }
-
- log.Printf("🔍 [METADATA-GEN] Response status: %d, body length: %d", resp.StatusCode, len(body))
-
- if resp.StatusCode != http.StatusOK {
- log.Printf("❌ [METADATA-GEN] API error response: %s", string(body))
- return nil, fmt.Errorf("API error (status %d): %s", resp.StatusCode, string(body))
- }
-
- // Parse API response
- var apiResponse struct {
- Choices []struct {
- Message struct {
- Content string `json:"content"`
- } `json:"message"`
- } `json:"choices"`
- }
-
- if err := json.Unmarshal(body, &apiResponse); err != nil {
- return nil, fmt.Errorf("failed to parse API response: %w", err)
- }
-
- if len(apiResponse.Choices) == 0 {
- return nil, fmt.Errorf("no response from model")
- }
-
- // Parse the NAME: and DESC: format from response
- content := strings.TrimSpace(apiResponse.Choices[0].Message.Content)
- log.Printf("🔍 [METADATA-GEN] Raw response: %s", content)
-
- var name, description string
-
- // Parse line by line looking for NAME: and DESC:
- lines := strings.Split(content, "\n")
- for _, line := range lines {
- line = strings.TrimSpace(line)
- if strings.HasPrefix(strings.ToUpper(line), "NAME:") {
- name = strings.TrimSpace(strings.TrimPrefix(line, "NAME:"))
- name = strings.TrimSpace(strings.TrimPrefix(line, "name:"))
- // Remove the prefix more reliably
- if idx := strings.Index(strings.ToLower(line), "name:"); idx != -1 {
- name = strings.TrimSpace(line[idx+5:])
- }
- } else if strings.HasPrefix(strings.ToUpper(line), "DESC:") {
- description = strings.TrimSpace(strings.TrimPrefix(line, "DESC:"))
- description = strings.TrimSpace(strings.TrimPrefix(line, "desc:"))
- // Remove the prefix more reliably
- if idx := strings.Index(strings.ToLower(line), "desc:"); idx != -1 {
- description = strings.TrimSpace(line[idx+5:])
- }
- }
- }
-
- // Fallback: if parsing failed, try to use first line as name
- if name == "" && len(lines) > 0 {
- name = strings.TrimSpace(lines[0])
- name = strings.Trim(name, `"'#*-`)
- }
-
- // Clean up name
- name = strings.Trim(name, `"'#*-`)
-
- // Limit name to 5 words
- words := strings.Fields(name)
- if len(words) > 5 {
- words = words[:5]
- name = strings.Join(words, " ")
- }
-
- if name == "" {
- return nil, fmt.Errorf("empty name from model")
- }
-
- metadata := AgentMetadata{
- Name: name,
- Description: description,
- }
-
- // Truncate if too long
- if len(metadata.Name) > 50 {
- metadata.Name = metadata.Name[:50]
- }
- if len(metadata.Description) > 150 {
- metadata.Description = metadata.Description[:150]
- }
-
- log.Printf("📝 [WORKFLOW-GEN] Generated agent metadata: name=%s, description=%s", metadata.Name, metadata.Description)
- return &metadata, nil
-}
-
-// GenerateAgentName generates a short, descriptive name for an agent (backwards compatibility)
-func (s *WorkflowGeneratorService) GenerateAgentName(userMessage string) (string, error) {
- metadata, err := s.GenerateAgentMetadata(userMessage)
- if err != nil {
- return "", err
- }
- return metadata.Name, nil
-}
-
-// GenerateDescriptionFromWorkflow generates a description for an agent based on its workflow blocks
-func (s *WorkflowGeneratorService) GenerateDescriptionFromWorkflow(workflow *models.Workflow, agentName string) (string, error) {
- if workflow == nil || len(workflow.Blocks) == 0 {
- return "", fmt.Errorf("no workflow blocks to analyze")
- }
-
- // Use ChatService's GetTextProviderWithModel which dynamically finds a text-capable provider
- // This method checks model aliases from config and falls back to database providers
- provider, modelID, err := s.chatService.GetTextProviderWithModel()
- if err != nil {
- return "", fmt.Errorf("failed to get text provider for description generation: %w", err)
- }
-
- log.Printf("🔍 [DESC-GEN] Using dynamic model: %s (provider: %s)", modelID, provider.Name)
-
- // Build a summary of the workflow blocks for the LLM
- var blockSummary strings.Builder
- blockSummary.WriteString("Workflow blocks:\n")
- for _, block := range workflow.Blocks {
- if block.Type == "llm_inference" {
- // Extract key info from LLM blocks
- tools := ""
- if enabledTools, ok := block.Config["enabledTools"].([]interface{}); ok {
- toolNames := make([]string, 0)
- for _, t := range enabledTools {
- if toolName, ok := t.(string); ok {
- toolNames = append(toolNames, toolName)
- }
- }
- tools = strings.Join(toolNames, ", ")
- }
- if tools != "" {
- blockSummary.WriteString(fmt.Sprintf("- %s: %s (tools: %s)\n", block.Name, block.Description, tools))
- } else {
- blockSummary.WriteString(fmt.Sprintf("- %s: %s\n", block.Name, block.Description))
- }
- } else if block.Type == "variable" {
- blockSummary.WriteString(fmt.Sprintf("- %s (input): %s\n", block.Name, block.Description))
- }
- }
-
- messages := []map[string]interface{}{
- {
- "role": "system",
- "content": `Generate a brief, one-sentence description for an AI agent based on its workflow.
-
-RULES:
-- Maximum 100 characters
-- Start with a verb (e.g., "Searches...", "Analyzes...", "Monitors...")
-- Be specific about what the agent does
-- Mention the key actions or outputs
-- Do not include the agent name in the description
-- Do not use quotes around the description
-
-RESPOND with ONLY the description text, nothing else.`,
- },
- {
- "role": "user",
- "content": fmt.Sprintf("Agent name: %s\n\n%s", agentName, blockSummary.String()),
- },
- }
-
- requestBody := map[string]interface{}{
- "model": modelID,
- "messages": messages,
- "stream": false,
- "temperature": 0.5,
- }
-
- reqBody, err := json.Marshal(requestBody)
- if err != nil {
- return "", fmt.Errorf("failed to marshal request: %w", err)
- }
-
- apiURL := provider.BaseURL + "/chat/completions"
- log.Printf("🔍 [DESC-GEN] Generating description for agent: %s", agentName)
-
- httpReq, err := http.NewRequest("POST", apiURL, bytes.NewBuffer(reqBody))
- if err != nil {
- return "", fmt.Errorf("failed to create request: %w", err)
- }
-
- httpReq.Header.Set("Content-Type", "application/json")
- httpReq.Header.Set("Authorization", "Bearer "+provider.APIKey)
-
- client := &http.Client{Timeout: 30 * time.Second}
- resp, err := client.Do(httpReq)
- if err != nil {
- return "", fmt.Errorf("request failed: %w", err)
- }
- defer resp.Body.Close()
-
- body, err := io.ReadAll(resp.Body)
- if err != nil {
- return "", fmt.Errorf("failed to read response: %w", err)
- }
-
- if resp.StatusCode != http.StatusOK {
- return "", fmt.Errorf("API error (status %d): %s", resp.StatusCode, string(body))
- }
-
- var apiResponse struct {
- Choices []struct {
- Message struct {
- Content string `json:"content"`
- } `json:"message"`
- } `json:"choices"`
- }
-
- if err := json.Unmarshal(body, &apiResponse); err != nil {
- return "", fmt.Errorf("failed to parse API response: %w", err)
- }
-
- if len(apiResponse.Choices) == 0 {
- return "", fmt.Errorf("no response from model")
- }
-
- description := strings.TrimSpace(apiResponse.Choices[0].Message.Content)
- description = strings.Trim(description, `"'`)
-
- // Truncate if too long
- if len(description) > 150 {
- description = description[:150]
- }
-
- log.Printf("📝 [DESC-GEN] Generated description: %s", description)
- return description, nil
-}
-
-// GenerateSampleInput generates sample JSON input for a workflow based on its blocks
-func (s *WorkflowGeneratorService) GenerateSampleInput(workflow *models.Workflow, modelID string, userID string) (map[string]interface{}, error) {
- if workflow == nil || len(workflow.Blocks) == 0 {
- return nil, fmt.Errorf("no workflow blocks to analyze")
- }
-
- // Get provider and model
- provider, resolvedModelID, err := s.getProviderAndModel(modelID)
- if err != nil {
- return nil, fmt.Errorf("failed to get provider: %w", err)
- }
-
- log.Printf("🎯 [SAMPLE-INPUT] Generating sample input using model: %s (provider: %s)", resolvedModelID, provider.Name)
-
- // Build a summary of the workflow to understand what input it needs
- var workflowSummary strings.Builder
- workflowSummary.WriteString("This workflow has the following blocks:\n\n")
-
- for i, block := range workflow.Blocks {
- workflowSummary.WriteString(fmt.Sprintf("Block %d: %s (type: %s)\n", i+1, block.Name, block.Type))
-
- if block.Type == "llm_inference" {
- // Extract system prompt and enabled tools
- if systemPrompt, ok := block.Config["systemPrompt"].(string); ok && systemPrompt != "" {
- // Truncate long prompts
- if len(systemPrompt) > 500 {
- systemPrompt = systemPrompt[:500] + "..."
- }
- workflowSummary.WriteString(fmt.Sprintf(" System prompt: %s\n", systemPrompt))
- }
-
- if enabledTools, ok := block.Config["enabledTools"].([]interface{}); ok && len(enabledTools) > 0 {
- toolNames := make([]string, 0, len(enabledTools))
- for _, t := range enabledTools {
- if ts, ok := t.(string); ok {
- toolNames = append(toolNames, ts)
- }
- }
- if len(toolNames) > 0 {
- workflowSummary.WriteString(fmt.Sprintf(" Tools: %s\n", strings.Join(toolNames, ", ")))
- }
- }
- } else if block.Type == "variable" {
- workflowSummary.WriteString(" This is the start block that receives input\n")
- }
- workflowSummary.WriteString("\n")
- }
-
- // Build messages for the LLM
- messages := []map[string]interface{}{
- {
- "role": "system",
- "content": `You are a helpful assistant that generates realistic sample JSON input for AI workflow testing.
-
-Analyze the workflow description and generate appropriate sample JSON input that would be useful for testing this workflow.
-
-RULES:
-1. Output ONLY valid JSON - no text before or after
-2. Use realistic, meaningful sample data that matches what the workflow expects
-3. If the workflow processes text, include relevant sample text
-4. If it handles URLs, include valid example URLs
-5. If it handles names/contacts, use realistic placeholder names
-6. If it handles numbers/data, use reasonable sample values
-7. Keep the JSON concise but complete
-8. Use "input" as the top-level key if no specific structure is evident
-9. Consider the tools being used - e.g., if web scraping, include a URL; if data analysis, include data points
-
-EXAMPLES:
-- For a news search workflow: {"input": "latest developments in artificial intelligence"}
-- For a data analysis workflow: {"data": [{"name": "Q1", "value": 1000}, {"name": "Q2", "value": 1500}]}
-- For a web scraping workflow: {"url": "https://example.com/article", "extract": "main content"}
-- For a contact workflow: {"name": "John Smith", "email": "john@example.com", "company": "Acme Corp"}`,
- },
- {
- "role": "user",
- "content": fmt.Sprintf("Generate sample JSON input for this workflow:\n\n%s", workflowSummary.String()),
- },
- }
-
- // Build request body
- requestBody := map[string]interface{}{
- "model": resolvedModelID,
- "messages": messages,
- "stream": false,
- "temperature": 0.7,
- "response_format": map[string]interface{}{
- "type": "json_object",
- },
- }
-
- reqBody, err := json.Marshal(requestBody)
- if err != nil {
- return nil, fmt.Errorf("failed to marshal request: %w", err)
- }
-
- // Create HTTP request
- apiURL := provider.BaseURL + "/chat/completions"
- log.Printf("🔍 [SAMPLE-INPUT] Sending request to: %s", apiURL)
-
- httpReq, err := http.NewRequest("POST", apiURL, bytes.NewBuffer(reqBody))
- if err != nil {
- return nil, fmt.Errorf("failed to create request: %w", err)
- }
-
- httpReq.Header.Set("Content-Type", "application/json")
- httpReq.Header.Set("Authorization", "Bearer "+provider.APIKey)
-
- // Send request with timeout
- client := &http.Client{Timeout: 60 * time.Second}
- resp, err := client.Do(httpReq)
- if err != nil {
- return nil, fmt.Errorf("request failed: %w", err)
- }
- defer resp.Body.Close()
-
- body, err := io.ReadAll(resp.Body)
- if err != nil {
- return nil, fmt.Errorf("failed to read response: %w", err)
- }
-
- if resp.StatusCode != http.StatusOK {
- log.Printf("⚠️ [SAMPLE-INPUT] API error: %s", string(body))
- return nil, fmt.Errorf("API error (status %d): %s", resp.StatusCode, string(body))
- }
-
- // Parse API response
- var apiResponse struct {
- Choices []struct {
- Message struct {
- Content string `json:"content"`
- } `json:"message"`
- } `json:"choices"`
- }
-
- if err := json.Unmarshal(body, &apiResponse); err != nil {
- return nil, fmt.Errorf("failed to parse API response: %w", err)
- }
-
- if len(apiResponse.Choices) == 0 {
- return nil, fmt.Errorf("no response from model")
- }
-
- content := strings.TrimSpace(apiResponse.Choices[0].Message.Content)
- log.Printf("📥 [SAMPLE-INPUT] Received response: %s", content)
-
- // Parse the JSON response
- var sampleInput map[string]interface{}
- if err := json.Unmarshal([]byte(content), &sampleInput); err != nil {
- // Try to extract JSON from the response
- jsonContent := s.extractJSON(content)
- if err := json.Unmarshal([]byte(jsonContent), &sampleInput); err != nil {
- return nil, fmt.Errorf("failed to parse sample input JSON: %w", err)
- }
- }
-
- log.Printf("✅ [SAMPLE-INPUT] Generated sample input with %d keys", len(sampleInput))
- return sampleInput, nil
-}
diff --git a/backend/internal/services/workflow_generator_v2.go b/backend/internal/services/workflow_generator_v2.go
deleted file mode 100644
index c1bb37fe..00000000
--- a/backend/internal/services/workflow_generator_v2.go
+++ /dev/null
@@ -1,669 +0,0 @@
-package services
-
-import (
- "bytes"
- "encoding/json"
- "fmt"
- "io"
- "log"
- "net/http"
- "strings"
- "time"
-
- "github.com/google/uuid"
-
- "claraverse/internal/database"
- "claraverse/internal/models"
-)
-
-// WorkflowGeneratorV2Service handles multi-step workflow generation
-type WorkflowGeneratorV2Service struct {
- db *database.DB
- providerService *ProviderService
- chatService *ChatService
-}
-
-// NewWorkflowGeneratorV2Service creates a new v2 workflow generator service
-func NewWorkflowGeneratorV2Service(
- db *database.DB,
- providerService *ProviderService,
- chatService *ChatService,
-) *WorkflowGeneratorV2Service {
- return &WorkflowGeneratorV2Service{
- db: db,
- providerService: providerService,
- chatService: chatService,
- }
-}
-
-// ToolSelectionResult represents the result of tool selection (Step 1)
-type ToolSelectionResult struct {
- SelectedTools []SelectedTool `json:"selected_tools"`
- Reasoning string `json:"reasoning"`
-}
-
-// SelectedTool represents a selected tool with reasoning
-type SelectedTool struct {
- ToolID string `json:"tool_id"`
- Category string `json:"category"`
- Reason string `json:"reason"`
-}
-
-// GenerationStep represents a step in the generation process
-type GenerationStep struct {
- StepNumber int `json:"step_number"`
- StepName string `json:"step_name"`
- Status string `json:"status"` // "pending", "running", "completed", "failed"
- Description string `json:"description"`
- Tools []string `json:"tools,omitempty"` // Tool IDs for step 1 result
-}
-
-// MultiStepGenerateRequest is the request for multi-step generation
-type MultiStepGenerateRequest struct {
- AgentID string `json:"agent_id"`
- UserMessage string `json:"user_message"`
- ModelID string `json:"model_id,omitempty"`
- CurrentWorkflow *models.Workflow `json:"current_workflow,omitempty"`
- ConversationHistory []models.ConversationMessage `json:"conversation_history,omitempty"` // Recent conversation context for better tool selection
-}
-
-// MultiStepGenerateResponse is the response for multi-step generation
-type MultiStepGenerateResponse struct {
- Success bool `json:"success"`
- CurrentStep int `json:"current_step"`
- TotalSteps int `json:"total_steps"`
- Steps []GenerationStep `json:"steps"`
- SelectedTools []SelectedTool `json:"selected_tools,omitempty"`
- Workflow *models.Workflow `json:"workflow,omitempty"`
- Explanation string `json:"explanation,omitempty"`
- Error string `json:"error,omitempty"`
- StepInProgress *GenerationStep `json:"step_in_progress,omitempty"`
-}
-
-// Tool selection system prompt - asks LLM to select relevant tools
-const ToolSelectionSystemPrompt = `You are a tool selection expert for Clara AI workflow builder. Your job is to analyze user requests and select the MINIMUM set of tools needed to accomplish the task.
-
-IMPORTANT: Only select tools that are DIRECTLY needed for the workflow. Don't over-select.
-
-You will be given:
-1. A user request describing what workflow they want to build
-2. A list of all available tools with their descriptions and use cases
-
-Your task: Select the specific tools needed and explain why each is needed.
-
-Rules:
-- Select ONLY tools that will be directly used in the workflow
-- If the request mentions "news" or time-sensitive info, ALWAYS include "get_current_time"
-- If the request mentions sending to a specific platform (Discord, Slack, etc.), select that messaging tool
-- Don't select redundant tools - if search_web is enough, don't also select scrape_web unless needed
-- For file processing, select the appropriate reader tool based on file type mentioned
-
-Output format: JSON with selected_tools array and reasoning.`
-
-// BuildToolSelectionUserPrompt builds the user prompt for tool selection
-func (s *WorkflowGeneratorV2Service) BuildToolSelectionUserPrompt(userMessage string) string {
- var builder strings.Builder
-
- builder.WriteString("USER REQUEST:\n")
- builder.WriteString(userMessage)
- builder.WriteString("\n\n")
- builder.WriteString("AVAILABLE TOOLS:\n\n")
-
- // Group tools by category
- for _, category := range ToolCategoryRegistry {
- tools := GetToolsByCategory(category.ID)
- if len(tools) == 0 {
- continue
- }
-
- builder.WriteString(fmt.Sprintf("## %s\n", category.Name))
- for _, tool := range tools {
- builder.WriteString(fmt.Sprintf("- **%s**: %s\n", tool.ID, tool.Description))
- if len(tool.UseCases) > 0 {
- builder.WriteString(fmt.Sprintf(" Use cases: %s\n", strings.Join(tool.UseCases, ", ")))
- }
- }
- builder.WriteString("\n")
- }
-
- builder.WriteString("\nSelect the tools needed for this workflow. Return JSON with selected_tools array.")
-
- return builder.String()
-}
-
-// Tool selection JSON schema for structured output
-var toolSelectionSchema = map[string]interface{}{
- "type": "object",
- "properties": map[string]interface{}{
- "selected_tools": map[string]interface{}{
- "type": "array",
- "items": map[string]interface{}{
- "type": "object",
- "properties": map[string]interface{}{
- "tool_id": map[string]interface{}{
- "type": "string",
- "description": "The tool ID from the available tools list",
- },
- "category": map[string]interface{}{
- "type": "string",
- "description": "The category of the tool",
- },
- "reason": map[string]interface{}{
- "type": "string",
- "description": "Brief reason why this tool is needed",
- },
- },
- "required": []string{"tool_id", "category", "reason"},
- "additionalProperties": false,
- },
- },
- "reasoning": map[string]interface{}{
- "type": "string",
- "description": "Overall reasoning for the tool selection",
- },
- },
- "required": []string{"selected_tools", "reasoning"},
- "additionalProperties": false,
-}
-
-// Step1SelectTools performs tool selection using structured output
-func (s *WorkflowGeneratorV2Service) Step1SelectTools(req *MultiStepGenerateRequest, userID string) (*ToolSelectionResult, error) {
- log.Printf("🔧 [WORKFLOW-GEN-V2] Step 1: Selecting tools for request: %s", req.UserMessage)
-
- // Get provider and model
- provider, modelID, err := s.getProviderAndModel(req.ModelID)
- if err != nil {
- return nil, fmt.Errorf("failed to get provider: %w", err)
- }
-
- // Build the user prompt with all available tools
- userPrompt := s.BuildToolSelectionUserPrompt(req.UserMessage)
-
- // Build messages with conversation history for better context
- messages := []map[string]interface{}{
- {
- "role": "system",
- "content": ToolSelectionSystemPrompt,
- },
- }
-
- // Add conversation history if provided (for multi-turn context)
- if len(req.ConversationHistory) > 0 {
- for _, msg := range req.ConversationHistory {
- messages = append(messages, map[string]interface{}{
- "role": msg.Role,
- "content": msg.Content,
- })
- }
- }
-
- // Add current user message
- messages = append(messages, map[string]interface{}{
- "role": "user",
- "content": userPrompt,
- })
-
- // Build request with structured output
- requestBody := map[string]interface{}{
- "model": modelID,
- "messages": messages,
- "stream": false,
- "temperature": 0.2, // Low temperature for consistent selection
- "response_format": map[string]interface{}{
- "type": "json_schema",
- "json_schema": map[string]interface{}{
- "name": "tool_selection",
- "strict": true,
- "schema": toolSelectionSchema,
- },
- },
- }
-
- reqBody, err := json.Marshal(requestBody)
- if err != nil {
- return nil, fmt.Errorf("failed to marshal request: %w", err)
- }
-
- log.Printf("📤 [WORKFLOW-GEN-V2] Sending tool selection request to %s", provider.BaseURL)
-
- // Create HTTP request
- httpReq, err := http.NewRequest("POST", provider.BaseURL+"/chat/completions", bytes.NewBuffer(reqBody))
- if err != nil {
- return nil, fmt.Errorf("failed to create request: %w", err)
- }
-
- httpReq.Header.Set("Content-Type", "application/json")
- httpReq.Header.Set("Authorization", "Bearer "+provider.APIKey)
-
- // Send request
- client := &http.Client{Timeout: 60 * time.Second}
- resp, err := client.Do(httpReq)
- if err != nil {
- return nil, fmt.Errorf("request failed: %w", err)
- }
- defer resp.Body.Close()
-
- body, err := io.ReadAll(resp.Body)
- if err != nil {
- return nil, fmt.Errorf("failed to read response: %w", err)
- }
-
- if resp.StatusCode != http.StatusOK {
- log.Printf("⚠️ [WORKFLOW-GEN-V2] API error: %s", string(body))
- return nil, fmt.Errorf("API error (status %d): %s", resp.StatusCode, string(body))
- }
-
- // Parse response
- var apiResponse struct {
- Choices []struct {
- Message struct {
- Content string `json:"content"`
- } `json:"message"`
- } `json:"choices"`
- }
-
- if err := json.Unmarshal(body, &apiResponse); err != nil {
- return nil, fmt.Errorf("failed to parse API response: %w", err)
- }
-
- if len(apiResponse.Choices) == 0 {
- return nil, fmt.Errorf("no response from model")
- }
-
- // Parse the tool selection result
- var result ToolSelectionResult
- content := apiResponse.Choices[0].Message.Content
-
- if err := json.Unmarshal([]byte(content), &result); err != nil {
- log.Printf("⚠️ [WORKFLOW-GEN-V2] Failed to parse tool selection: %v, content: %s", err, content)
- return nil, fmt.Errorf("failed to parse tool selection: %w", err)
- }
-
- // Validate selected tools exist
- validTools := make([]SelectedTool, 0)
- for _, selected := range result.SelectedTools {
- if tool := GetToolByID(selected.ToolID); tool != nil {
- selected.Category = tool.Category // Ensure category is correct
- validTools = append(validTools, selected)
- } else {
- log.Printf("⚠️ [WORKFLOW-GEN-V2] Unknown tool selected: %s, skipping", selected.ToolID)
- }
- }
- result.SelectedTools = validTools
-
- log.Printf("✅ [WORKFLOW-GEN-V2] Selected %d tools: %v", len(result.SelectedTools), getToolIDs(result.SelectedTools))
-
- return &result, nil
-}
-
-// Step2GenerateWorkflow generates the workflow using only selected tools
-func (s *WorkflowGeneratorV2Service) Step2GenerateWorkflow(
- req *MultiStepGenerateRequest,
- selectedTools []SelectedTool,
- userID string,
-) (*models.WorkflowGenerateResponse, error) {
- log.Printf("🔧 [WORKFLOW-GEN-V2] Step 2: Generating workflow with %d tools", len(selectedTools))
-
- // Get provider and model
- provider, modelID, err := s.getProviderAndModel(req.ModelID)
- if err != nil {
- return &models.WorkflowGenerateResponse{
- Success: false,
- Error: fmt.Sprintf("Failed to get provider: %v", err),
- }, nil
- }
-
- // Build tool IDs list
- toolIDs := getToolIDs(selectedTools)
-
- // Build system prompt with only selected tools
- systemPrompt := s.buildWorkflowSystemPromptWithTools(toolIDs)
-
- // Build user message
- userMessage := s.buildUserMessage(req)
-
- // Build messages with conversation history for better context
- messages := []map[string]interface{}{
- {
- "role": "system",
- "content": systemPrompt,
- },
- }
-
- // Add conversation history if provided (for multi-turn context)
- if len(req.ConversationHistory) > 0 {
- for _, msg := range req.ConversationHistory {
- messages = append(messages, map[string]interface{}{
- "role": msg.Role,
- "content": msg.Content,
- })
- }
- }
-
- // Add current user message
- messages = append(messages, map[string]interface{}{
- "role": "user",
- "content": userMessage,
- })
-
- // Build request
- requestBody := map[string]interface{}{
- "model": modelID,
- "messages": messages,
- "stream": false,
- "temperature": 0.3,
- "response_format": map[string]interface{}{
- "type": "json_object",
- },
- }
-
- reqBody, err := json.Marshal(requestBody)
- if err != nil {
- return nil, fmt.Errorf("failed to marshal request: %w", err)
- }
-
- log.Printf("📤 [WORKFLOW-GEN-V2] Sending workflow generation request")
-
- // Create HTTP request
- httpReq, err := http.NewRequest("POST", provider.BaseURL+"/chat/completions", bytes.NewBuffer(reqBody))
- if err != nil {
- return nil, fmt.Errorf("failed to create request: %w", err)
- }
-
- httpReq.Header.Set("Content-Type", "application/json")
- httpReq.Header.Set("Authorization", "Bearer "+provider.APIKey)
-
- // Send request
- client := &http.Client{Timeout: 120 * time.Second}
- resp, err := client.Do(httpReq)
- if err != nil {
- return nil, fmt.Errorf("request failed: %w", err)
- }
- defer resp.Body.Close()
-
- body, err := io.ReadAll(resp.Body)
- if err != nil {
- return nil, fmt.Errorf("failed to read response: %w", err)
- }
-
- if resp.StatusCode != http.StatusOK {
- log.Printf("⚠️ [WORKFLOW-GEN-V2] API error: %s", string(body))
- return &models.WorkflowGenerateResponse{
- Success: false,
- Error: fmt.Sprintf("API error (status %d): %s", resp.StatusCode, string(body)),
- }, nil
- }
-
- // Parse response
- var apiResponse struct {
- Choices []struct {
- Message struct {
- Content string `json:"content"`
- } `json:"message"`
- } `json:"choices"`
- }
-
- if err := json.Unmarshal(body, &apiResponse); err != nil {
- return nil, fmt.Errorf("failed to parse API response: %w", err)
- }
-
- if len(apiResponse.Choices) == 0 {
- return &models.WorkflowGenerateResponse{
- Success: false,
- Error: "No response from model",
- }, nil
- }
-
- content := apiResponse.Choices[0].Message.Content
- log.Printf("📥 [WORKFLOW-GEN-V2] Received workflow response (%d chars)", len(content))
-
- // Parse the workflow
- return s.parseWorkflowResponse(content, req.CurrentWorkflow != nil, req.AgentID)
-}
-
-// GenerateWorkflowMultiStep performs the full multi-step generation
-func (s *WorkflowGeneratorV2Service) GenerateWorkflowMultiStep(
- req *MultiStepGenerateRequest,
- userID string,
- stepCallback func(step GenerationStep),
-) (*MultiStepGenerateResponse, error) {
- response := &MultiStepGenerateResponse{
- TotalSteps: 2,
- Steps: []GenerationStep{
- {StepNumber: 1, StepName: "Tool Selection", Status: "pending", Description: "Analyzing request and selecting relevant tools"},
- {StepNumber: 2, StepName: "Workflow Generation", Status: "pending", Description: "Building the workflow with selected tools"},
- },
- }
-
- // Step 1: Tool Selection
- response.Steps[0].Status = "running"
- response.CurrentStep = 1
- response.StepInProgress = &response.Steps[0]
- if stepCallback != nil {
- stepCallback(response.Steps[0])
- }
-
- toolResult, err := s.Step1SelectTools(req, userID)
- if err != nil {
- response.Steps[0].Status = "failed"
- response.Success = false
- response.Error = fmt.Sprintf("Tool selection failed: %v", err)
- return response, nil
- }
-
- response.Steps[0].Status = "completed"
- response.Steps[0].Tools = getToolIDs(toolResult.SelectedTools)
- response.SelectedTools = toolResult.SelectedTools
-
- if stepCallback != nil {
- stepCallback(response.Steps[0])
- }
-
- // Step 2: Workflow Generation
- response.Steps[1].Status = "running"
- response.CurrentStep = 2
- response.StepInProgress = &response.Steps[1]
- if stepCallback != nil {
- stepCallback(response.Steps[1])
- }
-
- workflowResult, err := s.Step2GenerateWorkflow(req, toolResult.SelectedTools, userID)
- if err != nil {
- response.Steps[1].Status = "failed"
- response.Success = false
- response.Error = fmt.Sprintf("Workflow generation failed: %v", err)
- return response, nil
- }
-
- if !workflowResult.Success {
- response.Steps[1].Status = "failed"
- response.Success = false
- response.Error = workflowResult.Error
- return response, nil
- }
-
- response.Steps[1].Status = "completed"
- response.Workflow = workflowResult.Workflow
- response.Explanation = workflowResult.Explanation
- response.Success = true
- response.StepInProgress = nil
-
- if stepCallback != nil {
- stepCallback(response.Steps[1])
- }
-
- log.Printf("✅ [WORKFLOW-GEN-V2] Multi-step generation completed successfully")
-
- return response, nil
-}
-
-// buildWorkflowSystemPromptWithTools builds the system prompt with specific tools
-func (s *WorkflowGeneratorV2Service) buildWorkflowSystemPromptWithTools(toolIDs []string) string {
- toolsSection := BuildToolPromptSection(toolIDs)
- return strings.Replace(WorkflowSystemPromptBase, "{{DYNAMIC_TOOLS_SECTION}}", toolsSection, 1)
-}
-
-// buildUserMessage constructs the user message for workflow generation
-func (s *WorkflowGeneratorV2Service) buildUserMessage(req *MultiStepGenerateRequest) string {
- if req.CurrentWorkflow != nil && len(req.CurrentWorkflow.Blocks) > 0 {
- workflowJSON, _ := json.MarshalIndent(req.CurrentWorkflow, "", " ")
- return fmt.Sprintf(`MODIFICATION REQUEST
-
-Current workflow:
-%s
-
-User request: %s
-
-Output the complete modified workflow JSON with all blocks (not just changes).`, string(workflowJSON), req.UserMessage)
- }
-
- return fmt.Sprintf("CREATE NEW WORKFLOW\n\nUser request: %s", req.UserMessage)
-}
-
-// parseWorkflowResponse parses the LLM response into a workflow
-func (s *WorkflowGeneratorV2Service) parseWorkflowResponse(content string, isModification bool, agentID string) (*models.WorkflowGenerateResponse, error) {
- // Try to extract JSON from the response
- jsonContent := extractJSON(content)
-
- // Parse the workflow
- var workflowData struct {
- Blocks []models.Block `json:"blocks"`
- Connections []models.Connection `json:"connections"`
- Variables []models.Variable `json:"variables"`
- Explanation string `json:"explanation"`
- }
-
- if err := json.Unmarshal([]byte(jsonContent), &workflowData); err != nil {
- log.Printf("⚠️ [WORKFLOW-GEN-V2] Failed to parse workflow JSON: %v", err)
- return &models.WorkflowGenerateResponse{
- Success: false,
- Error: fmt.Sprintf("Failed to parse workflow JSON: %v", err),
- Explanation: content,
- }, nil
- }
-
- // Log the generated workflow for debugging
- prettyWorkflow, _ := json.MarshalIndent(workflowData, "", " ")
- log.Printf("📋 [WORKFLOW-GEN-V2] Generated workflow:\n%s", string(prettyWorkflow))
-
- // Post-process blocks
- for i := range workflowData.Blocks {
- if workflowData.Blocks[i].NormalizedID == "" {
- workflowData.Blocks[i].NormalizedID = workflowData.Blocks[i].ID
- }
- }
-
- // Determine action
- action := "create"
- if isModification {
- action = "modify"
- }
-
- // Build the workflow
- workflow := &models.Workflow{
- ID: uuid.New().String(),
- AgentID: agentID,
- Blocks: workflowData.Blocks,
- Connections: workflowData.Connections,
- Variables: workflowData.Variables,
- Version: 1,
- }
-
- log.Printf("✅ [WORKFLOW-GEN-V2] Parsed workflow: %d blocks, %d connections",
- len(workflow.Blocks), len(workflow.Connections))
-
- return &models.WorkflowGenerateResponse{
- Success: true,
- Workflow: workflow,
- Explanation: workflowData.Explanation,
- Action: action,
- Version: 1,
- }, nil
-}
-
-// getProviderAndModel gets the provider and model for the request
-func (s *WorkflowGeneratorV2Service) getProviderAndModel(modelID string) (*models.Provider, string, error) {
- if modelID == "" {
- return s.chatService.GetDefaultProviderWithModel()
- }
-
- // Try to find the model in the database
- var providerID int
- var modelName string
-
- err := s.db.QueryRow(`
- SELECT m.name, m.provider_id
- FROM models m
- WHERE m.id = ? AND m.is_visible = 1
- `, modelID).Scan(&modelName, &providerID)
-
- if err != nil {
- if provider, actualModel, found := s.chatService.ResolveModelAlias(modelID); found {
- return provider, actualModel, nil
- }
- return s.chatService.GetDefaultProviderWithModel()
- }
-
- provider, err := s.providerService.GetByID(providerID)
- if err != nil {
- return nil, "", fmt.Errorf("failed to get provider: %w", err)
- }
-
- return provider, modelName, nil
-}
-
-// Helper function to get tool IDs from selected tools
-func getToolIDs(tools []SelectedTool) []string {
- ids := make([]string, len(tools))
- for i, t := range tools {
- ids[i] = t.ToolID
- }
- return ids
-}
-
-// extractJSON extracts JSON from a response (handles markdown code blocks)
-func extractJSON(content string) string {
- content = strings.TrimSpace(content)
-
- if strings.HasPrefix(content, "{") {
- return content
- }
-
- // Try to extract from markdown code block
- if idx := strings.Index(content, "```json"); idx != -1 {
- start := idx + 7
- end := strings.Index(content[start:], "```")
- if end != -1 {
- return strings.TrimSpace(content[start : start+end])
- }
- }
-
- if idx := strings.Index(content, "```"); idx != -1 {
- start := idx + 3
- // Skip language identifier if present
- if newline := strings.Index(content[start:], "\n"); newline != -1 {
- start = start + newline + 1
- }
- end := strings.Index(content[start:], "```")
- if end != -1 {
- return strings.TrimSpace(content[start : start+end])
- }
- }
-
- // Find JSON object
- if start := strings.Index(content, "{"); start != -1 {
- depth := 0
- for i := start; i < len(content); i++ {
- if content[i] == '{' {
- depth++
- } else if content[i] == '}' {
- depth--
- if depth == 0 {
- return content[start : i+1]
- }
- }
- }
- }
-
- return content
-}
diff --git a/backend/internal/tests/legacy_migration_test.go b/backend/internal/tests/legacy_migration_test.go
deleted file mode 100644
index 8fa279a1..00000000
--- a/backend/internal/tests/legacy_migration_test.go
+++ /dev/null
@@ -1,537 +0,0 @@
-package tests
-
-import (
- "claraverse/internal/handlers"
- "claraverse/internal/models"
- "context"
- "encoding/json"
- "io"
- "net/http/httptest"
- "testing"
- "time"
-
- "github.com/gofiber/fiber/v2"
- "go.mongodb.org/mongo-driver/bson"
-)
-
-// ============================================================================
-// Test: Production User (Main Branch) Before Migration
-// ============================================================================
-// This test verifies what happens when a user from production (main branch)
-// without any subscription fields logs in. They should get "free" tier.
-
-func TestE2E_ProductionUserLoginBeforeMigration(t *testing.T) {
- if testing.Short() {
- t.Skip("Skipping E2E test in short mode")
- }
-
- // Setup with promo disabled to avoid interference
- ts := SetupE2ETestWithMongoDB(t, &PromoTestConfig{
- PromoEnabled: false,
- })
- if ts == nil {
- return
- }
- defer ts.Cleanup()
-
- ctx := context.Background()
- userID := "test-e2e-prod-before-" + time.Now().Format("20060102150405")
- email := "test-e2e-prod-before@example.com"
-
- // Create user with ONLY main branch fields (simulating production user)
- // Main branch users have: _id, supabaseUserId, email, createdAt, lastLoginAt, preferences
- // NO subscriptionTier, subscriptionStatus, or other new fields
- collection := ts.MongoDB.Database().Collection("users")
- _, err := collection.InsertOne(ctx, bson.M{
- "supabaseUserId": userID,
- "email": email,
- "createdAt": time.Now().Add(-90 * 24 * time.Hour), // Created 90 days ago
- "lastLoginAt": time.Now().Add(-24 * time.Hour),
- "preferences": bson.M{
- "storeBuilderChatHistory": true,
- "chatPrivacyMode": "cloud",
- },
- // Explicitly NOT setting: subscriptionTier, subscriptionStatus, etc.
- })
- if err != nil {
- t.Fatalf("Failed to create production-like user: %v", err)
- }
-
- // Verify user was created without subscription fields
- var createdUser bson.M
- err = collection.FindOne(ctx, bson.M{"supabaseUserId": userID}).Decode(&createdUser)
- if err != nil {
- t.Fatalf("Failed to fetch created user: %v", err)
- }
- if _, exists := createdUser["subscriptionTier"]; exists {
- t.Fatal("User should NOT have subscriptionTier field (simulating main branch)")
- }
-
- // Create app with auth middleware for this user
- app := fiber.New(fiber.Config{DisableStartupMessage: true})
- app.Use(testAuthMiddleware(userID, email))
-
- subHandler := handlers.NewSubscriptionHandler(ts.PaymentService, ts.UserService)
- app.Get("/api/subscriptions/current", subHandler.GetCurrent)
-
- // Call GET /api/subscriptions/current
- req := httptest.NewRequest("GET", "/api/subscriptions/current", nil)
- resp, err := app.Test(req, -1)
- if err != nil {
- t.Fatalf("Request failed: %v", err)
- }
- defer resp.Body.Close()
-
- if resp.StatusCode != 200 {
- body, _ := io.ReadAll(resp.Body)
- t.Fatalf("Expected 200, got %d: %s", resp.StatusCode, string(body))
- }
-
- var result map[string]interface{}
- if err := json.NewDecoder(resp.Body).Decode(&result); err != nil {
- t.Fatalf("Failed to decode response: %v", err)
- }
-
- // Without migration, TierService defaults to "free" for users without tier
- tier := ts.TierService.GetUserTier(ctx, userID)
- if tier != models.TierFree {
- t.Errorf("Expected TierService to return 'free' for unmigrated user, got '%s'", tier)
- }
-
- t.Logf("Response tier: %v", result["tier"])
- t.Logf("TierService tier: %s", tier)
- t.Log("Confirmed: Production users without subscription fields get 'free' tier until migrated")
-}
-
-// ============================================================================
-// Test: Production User Migration to Legacy Unlimited
-// ============================================================================
-// This test simulates the migration script logic and verifies it works correctly.
-
-func TestE2E_ProductionUserMigration(t *testing.T) {
- if testing.Short() {
- t.Skip("Skipping E2E test in short mode")
- }
-
- ts := SetupE2ETestWithMongoDB(t, &PromoTestConfig{
- PromoEnabled: false,
- })
- if ts == nil {
- return
- }
- defer ts.Cleanup()
-
- ctx := context.Background()
- userID := "test-e2e-migrate-" + time.Now().Format("20060102150405")
- email := "test-e2e-migrate@example.com"
-
- collection := ts.MongoDB.Database().Collection("users")
-
- // Step 1: Create production-like user (no subscription fields)
- _, err := collection.InsertOne(ctx, bson.M{
- "supabaseUserId": userID,
- "email": email,
- "createdAt": time.Now().Add(-90 * 24 * time.Hour),
- "lastLoginAt": time.Now().Add(-24 * time.Hour),
- "preferences": bson.M{
- "storeBuilderChatHistory": true,
- },
- })
- if err != nil {
- t.Fatalf("Failed to create production-like user: %v", err)
- }
-
- // Step 2: Apply migration logic (same as migrate_legacy_users.go)
- migrationFilter := bson.M{
- "$or": []bson.M{
- {"subscriptionTier": bson.M{"$exists": false}},
- {"subscriptionTier": ""},
- },
- }
- migrationTime := time.Now()
- migrationUpdate := bson.M{
- "$set": bson.M{
- "subscriptionTier": models.TierLegacyUnlimited,
- "subscriptionStatus": models.SubStatusActive,
- "migratedToLegacyAt": migrationTime,
- },
- }
-
- result, err := collection.UpdateMany(ctx, migrationFilter, migrationUpdate)
- if err != nil {
- t.Fatalf("Migration failed: %v", err)
- }
- if result.ModifiedCount == 0 {
- t.Fatal("Expected at least 1 user to be migrated")
- }
- t.Logf("Migrated %d user(s)", result.ModifiedCount)
-
- // Step 3: Verify user now has legacy_unlimited tier
- var migratedUser models.User
- err = collection.FindOne(ctx, bson.M{"supabaseUserId": userID}).Decode(&migratedUser)
- if err != nil {
- t.Fatalf("Failed to fetch migrated user: %v", err)
- }
-
- if migratedUser.SubscriptionTier != models.TierLegacyUnlimited {
- t.Errorf("Expected tier '%s', got '%s'", models.TierLegacyUnlimited, migratedUser.SubscriptionTier)
- }
- if migratedUser.SubscriptionStatus != models.SubStatusActive {
- t.Errorf("Expected status '%s', got '%s'", models.SubStatusActive, migratedUser.SubscriptionStatus)
- }
- if migratedUser.MigratedToLegacyAt == nil {
- t.Error("Expected migratedToLegacyAt to be set")
- }
-
- // Step 4: Verify TierService returns correct tier
- // Invalidate cache first since we updated directly in DB
- ts.TierService.InvalidateCache(userID)
- tier := ts.TierService.GetUserTier(ctx, userID)
- if tier != models.TierLegacyUnlimited {
- t.Errorf("Expected TierService to return '%s', got '%s'", models.TierLegacyUnlimited, tier)
- }
-
- // Step 5: Verify limits are unlimited
- limits := models.GetTierLimits(tier)
- if limits.MaxSchedules != -1 {
- t.Errorf("Expected unlimited schedules (-1), got %d", limits.MaxSchedules)
- }
- if limits.MaxAPIKeys != -1 {
- t.Errorf("Expected unlimited API keys (-1), got %d", limits.MaxAPIKeys)
- }
-
- t.Log("Migration test passed: Production user successfully migrated to legacy_unlimited")
-}
-
-// ============================================================================
-// Test: Production User Login After Migration
-// ============================================================================
-// This tests the full flow: production user already migrated, then logs in.
-
-func TestE2E_ProductionUserLoginAfterMigration(t *testing.T) {
- if testing.Short() {
- t.Skip("Skipping E2E test in short mode")
- }
-
- // Setup with ACTIVE promo (to verify legacy users are NOT affected by promo)
- now := time.Now()
- ts := SetupE2ETestWithMongoDB(t, &PromoTestConfig{
- PromoEnabled: true,
- PromoStartDate: now.Add(-1 * time.Hour),
- PromoEndDate: now.Add(24 * time.Hour),
- PromoDuration: 30,
- })
- if ts == nil {
- return
- }
- defer ts.Cleanup()
-
- ctx := context.Background()
- userID := "test-e2e-post-migrate-" + time.Now().Format("20060102150405")
- email := "test-e2e-post-migrate@example.com"
-
- collection := ts.MongoDB.Database().Collection("users")
-
- // Create user that has ALREADY been migrated (simulating post-migration state)
- migratedAt := time.Now().Add(-7 * 24 * time.Hour) // Migrated 7 days ago
- _, err := collection.InsertOne(ctx, bson.M{
- "supabaseUserId": userID,
- "email": email,
- "createdAt": time.Now().Add(-120 * 24 * time.Hour), // Created 120 days ago
- "lastLoginAt": time.Now().Add(-24 * time.Hour),
- "subscriptionTier": models.TierLegacyUnlimited,
- "subscriptionStatus": models.SubStatusActive,
- "migratedToLegacyAt": migratedAt,
- "preferences": bson.M{
- "storeBuilderChatHistory": true,
- },
- })
- if err != nil {
- t.Fatalf("Failed to create migrated user: %v", err)
- }
-
- // Create app with auth middleware
- app := fiber.New(fiber.Config{DisableStartupMessage: true})
- app.Use(testAuthMiddleware(userID, email))
-
- subHandler := handlers.NewSubscriptionHandler(ts.PaymentService, ts.UserService)
- app.Get("/api/subscriptions/current", subHandler.GetCurrent)
-
- // Call GET /api/subscriptions/current
- req := httptest.NewRequest("GET", "/api/subscriptions/current", nil)
- resp, err := app.Test(req, -1)
- if err != nil {
- t.Fatalf("Request failed: %v", err)
- }
- defer resp.Body.Close()
-
- if resp.StatusCode != 200 {
- body, _ := io.ReadAll(resp.Body)
- t.Fatalf("Expected 200, got %d: %s", resp.StatusCode, string(body))
- }
-
- var result map[string]interface{}
- if err := json.NewDecoder(resp.Body).Decode(&result); err != nil {
- t.Fatalf("Failed to decode response: %v", err)
- }
-
- // Verify tier is legacy_unlimited (NOT converted to promo even though promo is active)
- if result["tier"] != models.TierLegacyUnlimited {
- t.Errorf("Expected tier '%s', got '%v'", models.TierLegacyUnlimited, result["tier"])
- }
-
- // Verify is_promo_user is false
- if result["is_promo_user"] != false {
- t.Errorf("Expected is_promo_user false for legacy user, got '%v'", result["is_promo_user"])
- }
-
- // Verify via TierService
- tier := ts.TierService.GetUserTier(ctx, userID)
- if tier != models.TierLegacyUnlimited {
- t.Errorf("TierService should return '%s', got '%s'", models.TierLegacyUnlimited, tier)
- }
-
- t.Log("Post-migration login test passed: Legacy user retains legacy_unlimited tier")
-}
-
-// ============================================================================
-// Test: Migration Idempotency
-// ============================================================================
-// Running migration twice should not break anything or update migratedToLegacyAt.
-
-func TestE2E_MigrationIdempotent(t *testing.T) {
- if testing.Short() {
- t.Skip("Skipping E2E test in short mode")
- }
-
- ts := SetupE2ETestWithMongoDB(t, &PromoTestConfig{
- PromoEnabled: false,
- })
- if ts == nil {
- return
- }
- defer ts.Cleanup()
-
- ctx := context.Background()
- userID := "test-e2e-idempotent-" + time.Now().Format("20060102150405")
- email := "test-e2e-idempotent@example.com"
-
- collection := ts.MongoDB.Database().Collection("users")
-
- // Create production-like user
- _, err := collection.InsertOne(ctx, bson.M{
- "supabaseUserId": userID,
- "email": email,
- "createdAt": time.Now().Add(-60 * 24 * time.Hour),
- "lastLoginAt": time.Now().Add(-24 * time.Hour),
- })
- if err != nil {
- t.Fatalf("Failed to create user: %v", err)
- }
-
- // First migration
- migrationFilter := bson.M{
- "$or": []bson.M{
- {"subscriptionTier": bson.M{"$exists": false}},
- {"subscriptionTier": ""},
- },
- }
- firstMigrationTime := time.Now()
- migrationUpdate := bson.M{
- "$set": bson.M{
- "subscriptionTier": models.TierLegacyUnlimited,
- "subscriptionStatus": models.SubStatusActive,
- "migratedToLegacyAt": firstMigrationTime,
- },
- }
-
- result1, err := collection.UpdateMany(ctx, migrationFilter, migrationUpdate)
- if err != nil {
- t.Fatalf("First migration failed: %v", err)
- }
- if result1.ModifiedCount != 1 {
- t.Fatalf("Expected 1 user migrated on first run, got %d", result1.ModifiedCount)
- }
-
- // Get migratedToLegacyAt after first migration
- var userAfterFirst models.User
- err = collection.FindOne(ctx, bson.M{"supabaseUserId": userID}).Decode(&userAfterFirst)
- if err != nil {
- t.Fatalf("Failed to fetch user after first migration: %v", err)
- }
- firstMigratedAt := userAfterFirst.MigratedToLegacyAt
-
- // Wait a moment to ensure time difference would be detectable
- time.Sleep(10 * time.Millisecond)
-
- // Second migration (should not match the user since they already have a tier)
- result2, err := collection.UpdateMany(ctx, migrationFilter, bson.M{
- "$set": bson.M{
- "subscriptionTier": models.TierLegacyUnlimited,
- "subscriptionStatus": models.SubStatusActive,
- "migratedToLegacyAt": time.Now(), // Different time
- },
- })
- if err != nil {
- t.Fatalf("Second migration failed: %v", err)
- }
- if result2.ModifiedCount != 0 {
- t.Errorf("Expected 0 users migrated on second run (already migrated), got %d", result2.ModifiedCount)
- }
-
- // Verify migratedToLegacyAt unchanged
- var userAfterSecond models.User
- err = collection.FindOne(ctx, bson.M{"supabaseUserId": userID}).Decode(&userAfterSecond)
- if err != nil {
- t.Fatalf("Failed to fetch user after second migration: %v", err)
- }
-
- if userAfterSecond.MigratedToLegacyAt == nil || firstMigratedAt == nil {
- t.Error("migratedToLegacyAt should be set")
- } else if !userAfterSecond.MigratedToLegacyAt.Equal(*firstMigratedAt) {
- t.Errorf("migratedToLegacyAt should not change: first=%v, second=%v",
- firstMigratedAt, userAfterSecond.MigratedToLegacyAt)
- }
-
- // Verify user still has correct tier
- if userAfterSecond.SubscriptionTier != models.TierLegacyUnlimited {
- t.Errorf("Expected tier '%s', got '%s'", models.TierLegacyUnlimited, userAfterSecond.SubscriptionTier)
- }
-
- t.Log("Idempotency test passed: Running migration twice does not affect already-migrated users")
-}
-
-// ============================================================================
-// Test: Mixed Users Migration (Only Unmigrated Users Affected)
-// ============================================================================
-// Verifies that users with existing tiers are NOT affected by migration.
-
-func TestE2E_MixedUsersMigration(t *testing.T) {
- if testing.Short() {
- t.Skip("Skipping E2E test in short mode")
- }
-
- ts := SetupE2ETestWithMongoDB(t, &PromoTestConfig{
- PromoEnabled: false,
- })
- if ts == nil {
- return
- }
- defer ts.Cleanup()
-
- ctx := context.Background()
- collection := ts.MongoDB.Database().Collection("users")
- timestamp := time.Now().Format("20060102150405")
-
- // Create 4 different user types:
-
- // 1. Production user (no tier) - SHOULD be migrated
- prodUserID := "test-e2e-mixed-prod-" + timestamp
- _, err := collection.InsertOne(ctx, bson.M{
- "supabaseUserId": prodUserID,
- "email": "prod@example.com",
- "createdAt": time.Now().Add(-90 * 24 * time.Hour),
- "lastLoginAt": time.Now().Add(-24 * time.Hour),
- // No subscriptionTier
- })
- if err != nil {
- t.Fatalf("Failed to create prod user: %v", err)
- }
-
- // 2. Free tier user (has tier set) - should NOT be migrated
- freeUserID := "test-e2e-mixed-free-" + timestamp
- _, err = collection.InsertOne(ctx, bson.M{
- "supabaseUserId": freeUserID,
- "email": "free@example.com",
- "createdAt": time.Now().Add(-30 * 24 * time.Hour),
- "lastLoginAt": time.Now().Add(-1 * time.Hour),
- "subscriptionTier": models.TierFree,
- "subscriptionStatus": models.SubStatusActive,
- })
- if err != nil {
- t.Fatalf("Failed to create free user: %v", err)
- }
-
- // 3. Pro user (paid) - should NOT be migrated
- proUserID := "test-e2e-mixed-pro-" + timestamp
- _, err = collection.InsertOne(ctx, bson.M{
- "supabaseUserId": proUserID,
- "email": "pro@example.com",
- "createdAt": time.Now().Add(-60 * 24 * time.Hour),
- "lastLoginAt": time.Now().Add(-2 * time.Hour),
- "subscriptionTier": models.TierPro,
- "subscriptionStatus": models.SubStatusActive,
- "dodoCustomerId": "cust_12345",
- })
- if err != nil {
- t.Fatalf("Failed to create pro user: %v", err)
- }
-
- // 4. User with empty tier (edge case) - SHOULD be migrated
- emptyTierUserID := "test-e2e-mixed-empty-" + timestamp
- _, err = collection.InsertOne(ctx, bson.M{
- "supabaseUserId": emptyTierUserID,
- "email": "empty@example.com",
- "createdAt": time.Now().Add(-45 * 24 * time.Hour),
- "lastLoginAt": time.Now().Add(-12 * time.Hour),
- "subscriptionTier": "", // Empty string
- })
- if err != nil {
- t.Fatalf("Failed to create empty tier user: %v", err)
- }
-
- // Run migration
- migrationFilter := bson.M{
- "$or": []bson.M{
- {"subscriptionTier": bson.M{"$exists": false}},
- {"subscriptionTier": ""},
- },
- }
- migrationUpdate := bson.M{
- "$set": bson.M{
- "subscriptionTier": models.TierLegacyUnlimited,
- "subscriptionStatus": models.SubStatusActive,
- "migratedToLegacyAt": time.Now(),
- },
- }
-
- result, err := collection.UpdateMany(ctx, migrationFilter, migrationUpdate)
- if err != nil {
- t.Fatalf("Migration failed: %v", err)
- }
-
- // Should have migrated exactly 2 users (prodUserID and emptyTierUserID)
- if result.ModifiedCount != 2 {
- t.Errorf("Expected 2 users migrated, got %d", result.ModifiedCount)
- }
-
- // Verify prod user is migrated
- var prodUser models.User
- collection.FindOne(ctx, bson.M{"supabaseUserId": prodUserID}).Decode(&prodUser)
- if prodUser.SubscriptionTier != models.TierLegacyUnlimited {
- t.Errorf("Prod user should have tier '%s', got '%s'", models.TierLegacyUnlimited, prodUser.SubscriptionTier)
- }
-
- // Verify empty tier user is migrated
- var emptyUser models.User
- collection.FindOne(ctx, bson.M{"supabaseUserId": emptyTierUserID}).Decode(&emptyUser)
- if emptyUser.SubscriptionTier != models.TierLegacyUnlimited {
- t.Errorf("Empty tier user should have tier '%s', got '%s'", models.TierLegacyUnlimited, emptyUser.SubscriptionTier)
- }
-
- // Verify free user is NOT migrated
- var freeUser models.User
- collection.FindOne(ctx, bson.M{"supabaseUserId": freeUserID}).Decode(&freeUser)
- if freeUser.SubscriptionTier != models.TierFree {
- t.Errorf("Free user should still have tier '%s', got '%s'", models.TierFree, freeUser.SubscriptionTier)
- }
-
- // Verify pro user is NOT migrated
- var proUser models.User
- collection.FindOne(ctx, bson.M{"supabaseUserId": proUserID}).Decode(&proUser)
- if proUser.SubscriptionTier != models.TierPro {
- t.Errorf("Pro user should still have tier '%s', got '%s'", models.TierPro, proUser.SubscriptionTier)
- }
-
- t.Log("Mixed users test passed: Only production users (no tier) are migrated")
-}
diff --git a/backend/internal/tests/subscription_e2e_test.go b/backend/internal/tests/subscription_e2e_test.go
deleted file mode 100644
index 6910c51c..00000000
--- a/backend/internal/tests/subscription_e2e_test.go
+++ /dev/null
@@ -1,678 +0,0 @@
-package tests
-
-import (
- "claraverse/internal/config"
- "claraverse/internal/database"
- "claraverse/internal/handlers"
- "claraverse/internal/models"
- "claraverse/internal/services"
- "context"
- "encoding/json"
- "io"
- "net/http/httptest"
- "os"
- "testing"
- "time"
-
- "github.com/gofiber/fiber/v2"
- "go.mongodb.org/mongo-driver/bson"
-)
-
-// TestServices holds all services for E2E tests
-type TestServices struct {
- App *fiber.App
- MongoDB *database.MongoDB
- UserService *services.UserService
- TierService *services.TierService
- PaymentService *services.PaymentService
- Config *config.Config
- Cleanup func()
-}
-
-// PromoTestConfig configures the promo window for tests
-type PromoTestConfig struct {
- PromoEnabled bool
- PromoStartDate time.Time
- PromoEndDate time.Time
- PromoDuration int // days
-}
-
-// SetupE2ETestWithMongoDB creates test infrastructure with MongoDB
-// Requires MONGODB_TEST_URI environment variable to be set
-func SetupE2ETestWithMongoDB(t *testing.T, promoConfig *PromoTestConfig) *TestServices {
- mongoURI := os.Getenv("MONGODB_TEST_URI")
- if mongoURI == "" {
- t.Skip("MONGODB_TEST_URI not set - skipping E2E test")
- return nil
- }
-
- ctx := context.Background()
-
- // Connect to MongoDB
- mongoDB, err := database.NewMongoDB(mongoURI)
- if err != nil {
- t.Fatalf("Failed to connect to MongoDB: %v", err)
- }
-
- if err := mongoDB.Initialize(ctx); err != nil {
- t.Fatalf("Failed to initialize MongoDB: %v", err)
- }
-
- // Create test config
- cfg := &config.Config{
- PromoEnabled: promoConfig.PromoEnabled,
- PromoStartDate: promoConfig.PromoStartDate,
- PromoEndDate: promoConfig.PromoEndDate,
- PromoDuration: promoConfig.PromoDuration,
- }
-
- // Initialize services
- tierService := services.NewTierService(mongoDB)
- userService := services.NewUserService(mongoDB, cfg, nil) // No usage limiter for tests
- paymentService := services.NewPaymentService("", "", "", mongoDB, userService, tierService, nil)
-
- // Setup Fiber app with routes
- app := fiber.New(fiber.Config{
- DisableStartupMessage: true,
- })
-
- subHandler := handlers.NewSubscriptionHandler(paymentService, userService)
-
- // Add routes with test auth middleware
- api := app.Group("/api")
- subs := api.Group("/subscriptions")
- subs.Get("/current", subHandler.GetCurrent)
- subs.Get("/usage", subHandler.GetUsageStats)
- subs.Get("/plans", subHandler.ListPlans)
-
- cleanup := func() {
- // Clean up test data
- db := mongoDB.Database()
- db.Collection("users").DeleteMany(ctx, bson.M{"email": bson.M{"$regex": "^test-e2e-"}})
- db.Collection("subscriptions").DeleteMany(ctx, bson.M{"userId": bson.M{"$regex": "^test-e2e-"}})
- mongoDB.Close(ctx)
- }
-
- return &TestServices{
- App: app,
- MongoDB: mongoDB,
- UserService: userService,
- TierService: tierService,
- PaymentService: paymentService,
- Config: cfg,
- Cleanup: cleanup,
- }
-}
-
-// testAuthMiddleware sets user_id and user_email in context
-func testAuthMiddleware(userID, email string) fiber.Handler {
- return func(c *fiber.Ctx) error {
- c.Locals("user_id", userID)
- c.Locals("user_email", email)
- return c.Next()
- }
-}
-
-// ============================================================================
-// Test: New User Sign-in During Promo Window
-// ============================================================================
-
-func TestE2E_PromoUserSignIn(t *testing.T) {
- if testing.Short() {
- t.Skip("Skipping E2E test in short mode")
- }
-
- // Setup with active promo window
- now := time.Now()
- ts := SetupE2ETestWithMongoDB(t, &PromoTestConfig{
- PromoEnabled: true,
- PromoStartDate: now.Add(-1 * time.Hour), // Started 1 hour ago
- PromoEndDate: now.Add(24 * time.Hour), // Ends tomorrow
- PromoDuration: 30, // 30 days
- })
- if ts == nil {
- return
- }
- defer ts.Cleanup()
-
- userID := "test-e2e-promo-" + time.Now().Format("20060102150405")
- email := "test-e2e-promo@example.com"
-
- // Create app with auth middleware for this user
- app := fiber.New(fiber.Config{DisableStartupMessage: true})
- app.Use(testAuthMiddleware(userID, email))
-
- subHandler := handlers.NewSubscriptionHandler(ts.PaymentService, ts.UserService)
- app.Get("/api/subscriptions/current", subHandler.GetCurrent)
-
- // Call GET /api/subscriptions/current (triggers SyncUserFromSupabase)
- req := httptest.NewRequest("GET", "/api/subscriptions/current", nil)
- resp, err := app.Test(req, -1)
- if err != nil {
- t.Fatalf("Request failed: %v", err)
- }
- defer resp.Body.Close()
-
- if resp.StatusCode != 200 {
- body, _ := io.ReadAll(resp.Body)
- t.Fatalf("Expected 200, got %d: %s", resp.StatusCode, string(body))
- }
-
- var result map[string]interface{}
- if err := json.NewDecoder(resp.Body).Decode(&result); err != nil {
- t.Fatalf("Failed to decode response: %v", err)
- }
-
- // Verify tier is Pro
- if result["tier"] != "pro" {
- t.Errorf("Expected tier 'pro', got '%v'", result["tier"])
- }
-
- // Verify is_promo_user is true
- if result["is_promo_user"] != true {
- t.Errorf("Expected is_promo_user true, got '%v'", result["is_promo_user"])
- }
-
- // Verify has_seen_welcome_popup is false for new user
- if result["has_seen_welcome_popup"] != false {
- t.Errorf("Expected has_seen_welcome_popup false, got '%v'", result["has_seen_welcome_popup"])
- }
-
- // Verify subscription_expires_at is set (approximately 30 days from now)
- if result["subscription_expires_at"] == nil {
- t.Error("Expected subscription_expires_at to be set")
- } else {
- expiresAtStr := result["subscription_expires_at"].(string)
- expiresAt, err := time.Parse(time.RFC3339, expiresAtStr)
- if err != nil {
- t.Errorf("Failed to parse subscription_expires_at: %v", err)
- } else {
- expectedExpiry := now.Add(30 * 24 * time.Hour)
- diff := expiresAt.Sub(expectedExpiry)
- if diff < -1*time.Minute || diff > 1*time.Minute {
- t.Errorf("Expiry time off by more than 1 minute: expected ~%v, got %v", expectedExpiry, expiresAt)
- }
- }
- }
-
- // Verify database state
- ctx := context.Background()
- user, err := ts.UserService.GetUserBySupabaseID(ctx, userID)
- if err != nil {
- t.Fatalf("Failed to get user from DB: %v", err)
- }
-
- if user.SubscriptionTier != models.TierPro {
- t.Errorf("DB tier mismatch: expected '%s', got '%s'", models.TierPro, user.SubscriptionTier)
- }
- if user.SubscriptionStatus != models.SubStatusActive {
- t.Errorf("DB status mismatch: expected '%s', got '%s'", models.SubStatusActive, user.SubscriptionStatus)
- }
- if user.SubscriptionExpiresAt == nil {
- t.Error("DB subscription expires_at should be set")
- }
-
- t.Log("✅ Promo user sign-in test passed")
-}
-
-// ============================================================================
-// Test: New User Sign-in Outside Promo Window
-// ============================================================================
-
-func TestE2E_NonPromoUserSignIn(t *testing.T) {
- if testing.Short() {
- t.Skip("Skipping E2E test in short mode")
- }
-
- // Setup with EXPIRED promo window
- now := time.Now()
- ts := SetupE2ETestWithMongoDB(t, &PromoTestConfig{
- PromoEnabled: true,
- PromoStartDate: now.Add(-48 * time.Hour), // Started 2 days ago
- PromoEndDate: now.Add(-24 * time.Hour), // Ended 1 day ago
- PromoDuration: 30,
- })
- if ts == nil {
- return
- }
- defer ts.Cleanup()
-
- userID := "test-e2e-free-" + time.Now().Format("20060102150405")
- email := "test-e2e-free@example.com"
-
- // Create app with auth middleware for this user
- app := fiber.New(fiber.Config{DisableStartupMessage: true})
- app.Use(testAuthMiddleware(userID, email))
-
- subHandler := handlers.NewSubscriptionHandler(ts.PaymentService, ts.UserService)
- app.Get("/api/subscriptions/current", subHandler.GetCurrent)
-
- // Call GET /api/subscriptions/current
- req := httptest.NewRequest("GET", "/api/subscriptions/current", nil)
- resp, err := app.Test(req, -1)
- if err != nil {
- t.Fatalf("Request failed: %v", err)
- }
- defer resp.Body.Close()
-
- if resp.StatusCode != 200 {
- body, _ := io.ReadAll(resp.Body)
- t.Fatalf("Expected 200, got %d: %s", resp.StatusCode, string(body))
- }
-
- var result map[string]interface{}
- if err := json.NewDecoder(resp.Body).Decode(&result); err != nil {
- t.Fatalf("Failed to decode response: %v", err)
- }
-
- // Verify tier is Free
- if result["tier"] != "free" {
- t.Errorf("Expected tier 'free', got '%v'", result["tier"])
- }
-
- // Verify is_promo_user is false
- if result["is_promo_user"] != false {
- t.Errorf("Expected is_promo_user false, got '%v'", result["is_promo_user"])
- }
-
- // Verify no subscription_expires_at for free user
- if result["subscription_expires_at"] != nil {
- t.Errorf("Expected no subscription_expires_at for free user, got '%v'", result["subscription_expires_at"])
- }
-
- t.Log("✅ Non-promo user sign-in test passed")
-}
-
-// ============================================================================
-// Test: Legacy User Sign-in (Tier Preserved)
-// ============================================================================
-
-func TestE2E_LegacyUserSignIn(t *testing.T) {
- if testing.Short() {
- t.Skip("Skipping E2E test in short mode")
- }
-
- // Setup with active promo window (to verify legacy users are NOT converted)
- now := time.Now()
- ts := SetupE2ETestWithMongoDB(t, &PromoTestConfig{
- PromoEnabled: true,
- PromoStartDate: now.Add(-1 * time.Hour),
- PromoEndDate: now.Add(24 * time.Hour),
- PromoDuration: 30,
- })
- if ts == nil {
- return
- }
- defer ts.Cleanup()
-
- ctx := context.Background()
- userID := "test-e2e-legacy-" + time.Now().Format("20060102150405")
- email := "test-e2e-legacy@example.com"
-
- // Pre-create user with legacy_unlimited tier (simulating migration)
- collection := ts.MongoDB.Database().Collection("users")
- _, err := collection.InsertOne(ctx, bson.M{
- "supabaseUserId": userID,
- "email": email,
- "subscriptionTier": models.TierLegacyUnlimited,
- "subscriptionStatus": models.SubStatusActive,
- "createdAt": now.Add(-90 * 24 * time.Hour), // Created 90 days ago
- "lastLoginAt": now.Add(-1 * time.Hour), // Logged in 1 hour ago
- "migratedToLegacyAt": now.Add(-30 * 24 * time.Hour),
- })
- if err != nil {
- t.Fatalf("Failed to pre-create legacy user: %v", err)
- }
-
- // Create app with auth middleware for this user
- app := fiber.New(fiber.Config{DisableStartupMessage: true})
- app.Use(testAuthMiddleware(userID, email))
-
- subHandler := handlers.NewSubscriptionHandler(ts.PaymentService, ts.UserService)
- app.Get("/api/subscriptions/current", subHandler.GetCurrent)
-
- // Call GET /api/subscriptions/current
- req := httptest.NewRequest("GET", "/api/subscriptions/current", nil)
- resp, err := app.Test(req, -1)
- if err != nil {
- t.Fatalf("Request failed: %v", err)
- }
- defer resp.Body.Close()
-
- if resp.StatusCode != 200 {
- body, _ := io.ReadAll(resp.Body)
- t.Fatalf("Expected 200, got %d: %s", resp.StatusCode, string(body))
- }
-
- var result map[string]interface{}
- if err := json.NewDecoder(resp.Body).Decode(&result); err != nil {
- t.Fatalf("Failed to decode response: %v", err)
- }
-
- // Verify tier is legacy_unlimited (NOT downgraded to promo or free)
- if result["tier"] != models.TierLegacyUnlimited {
- t.Errorf("Expected tier '%s', got '%v'", models.TierLegacyUnlimited, result["tier"])
- }
-
- // Verify is_promo_user is false (legacy is NOT promo)
- if result["is_promo_user"] != false {
- t.Errorf("Expected is_promo_user false for legacy user, got '%v'", result["is_promo_user"])
- }
-
- t.Log("✅ Legacy user sign-in test passed")
-}
-
-// ============================================================================
-// Test: Promo User Expiration (Downgrade to Free)
-// ============================================================================
-
-func TestE2E_PromoExpirationDowngrade(t *testing.T) {
- if testing.Short() {
- t.Skip("Skipping E2E test in short mode")
- }
-
- ts := SetupE2ETestWithMongoDB(t, &PromoTestConfig{
- PromoEnabled: true,
- PromoStartDate: time.Now().Add(-48 * time.Hour),
- PromoEndDate: time.Now().Add(24 * time.Hour),
- PromoDuration: 30,
- })
- if ts == nil {
- return
- }
- defer ts.Cleanup()
-
- ctx := context.Background()
- userID := "test-e2e-expired-" + time.Now().Format("20060102150405")
- email := "test-e2e-expired@example.com"
-
- // Pre-create user with EXPIRED promo
- expiredAt := time.Now().Add(-1 * time.Hour) // Expired 1 hour ago
- collection := ts.MongoDB.Database().Collection("users")
- _, err := collection.InsertOne(ctx, bson.M{
- "supabaseUserId": userID,
- "email": email,
- "subscriptionTier": models.TierPro,
- "subscriptionStatus": models.SubStatusActive,
- "subscriptionExpiresAt": expiredAt,
- "createdAt": time.Now().Add(-31 * 24 * time.Hour),
- "lastLoginAt": time.Now().Add(-2 * time.Hour),
- })
- if err != nil {
- t.Fatalf("Failed to pre-create expired promo user: %v", err)
- }
-
- // Pre-warm cache with stale Pro tier
- tier := ts.TierService.GetUserTier(ctx, userID)
- if tier != models.TierFree {
- // Cache should detect expiration and return free
- t.Logf("Initial tier check returned: %s (expected free due to expiration)", tier)
- }
-
- // Create app with auth middleware for this user
- app := fiber.New(fiber.Config{DisableStartupMessage: true})
- app.Use(testAuthMiddleware(userID, email))
-
- subHandler := handlers.NewSubscriptionHandler(ts.PaymentService, ts.UserService)
- app.Get("/api/subscriptions/current", subHandler.GetCurrent)
-
- // Call GET /api/subscriptions/current - should detect expiration
- req := httptest.NewRequest("GET", "/api/subscriptions/current", nil)
- resp, err := app.Test(req, -1)
- if err != nil {
- t.Fatalf("Request failed: %v", err)
- }
- defer resp.Body.Close()
-
- if resp.StatusCode != 200 {
- body, _ := io.ReadAll(resp.Body)
- t.Fatalf("Expected 200, got %d: %s", resp.StatusCode, string(body))
- }
-
- var result map[string]interface{}
- if err := json.NewDecoder(resp.Body).Decode(&result); err != nil {
- t.Fatalf("Failed to decode response: %v", err)
- }
-
- // Note: GetCurrentSubscription gets tier from user doc, which still says "pro"
- // But the TierService should return "free" because promo expired
- // Let's verify the tier cache returns free
- cachedTier := ts.TierService.GetUserTier(ctx, userID)
- if cachedTier != models.TierFree {
- t.Errorf("TierService should return 'free' for expired promo, got '%s'", cachedTier)
- }
-
- // The response tier may still show "pro" from the user doc
- // This is a known issue - GetCurrentSubscription reads from user doc
- // The tier validation happens in TierService/middleware
- t.Logf("Response tier: %v, TierService tier: %s", result["tier"], cachedTier)
-
- // Verify is_promo_user is false (expired = not promo anymore)
- // Actually, the promo detection looks at tier + expiresAt + no dodo sub
- // so it might still show as promo even though expired
- t.Logf("is_promo_user: %v", result["is_promo_user"])
-
- t.Log("✅ Promo expiration test passed")
-}
-
-// ============================================================================
-// Test: Existing Paid User Re-login (Not Converted to Promo)
-// ============================================================================
-
-func TestE2E_ExistingPaidUserReLogin(t *testing.T) {
- if testing.Short() {
- t.Skip("Skipping E2E test in short mode")
- }
-
- // Setup with active promo window
- now := time.Now()
- ts := SetupE2ETestWithMongoDB(t, &PromoTestConfig{
- PromoEnabled: true,
- PromoStartDate: now.Add(-1 * time.Hour),
- PromoEndDate: now.Add(24 * time.Hour),
- PromoDuration: 30,
- })
- if ts == nil {
- return
- }
- defer ts.Cleanup()
-
- ctx := context.Background()
- userID := "test-e2e-paid-" + time.Now().Format("20060102150405")
- email := "test-e2e-paid@example.com"
-
- // Pre-create user with paid Pro tier (NOT promo - has Dodo subscription)
- collection := ts.MongoDB.Database().Collection("users")
- _, err := collection.InsertOne(ctx, bson.M{
- "supabaseUserId": userID,
- "email": email,
- "subscriptionTier": models.TierPro,
- "subscriptionStatus": models.SubStatusActive,
- "dodoCustomerId": "cust_test_12345",
- "dodoSubscriptionId": "sub_test_12345",
- "createdAt": now.Add(-60 * 24 * time.Hour),
- "lastLoginAt": now.Add(-24 * time.Hour),
- })
- if err != nil {
- t.Fatalf("Failed to pre-create paid user: %v", err)
- }
-
- // Create app with auth middleware for this user
- app := fiber.New(fiber.Config{DisableStartupMessage: true})
- app.Use(testAuthMiddleware(userID, email))
-
- subHandler := handlers.NewSubscriptionHandler(ts.PaymentService, ts.UserService)
- app.Get("/api/subscriptions/current", subHandler.GetCurrent)
-
- // Call GET /api/subscriptions/current
- req := httptest.NewRequest("GET", "/api/subscriptions/current", nil)
- resp, err := app.Test(req, -1)
- if err != nil {
- t.Fatalf("Request failed: %v", err)
- }
- defer resp.Body.Close()
-
- if resp.StatusCode != 200 {
- body, _ := io.ReadAll(resp.Body)
- t.Fatalf("Expected 200, got %d: %s", resp.StatusCode, string(body))
- }
-
- var result map[string]interface{}
- if err := json.NewDecoder(resp.Body).Decode(&result); err != nil {
- t.Fatalf("Failed to decode response: %v", err)
- }
-
- // Verify tier is still Pro (NOT reset or converted)
- if result["tier"] != "pro" {
- t.Errorf("Expected tier 'pro', got '%v'", result["tier"])
- }
-
- // Verify is_promo_user is false (paid user has dodo subscription)
- if result["is_promo_user"] != false {
- t.Errorf("Expected is_promo_user false for paid user, got '%v'", result["is_promo_user"])
- }
-
- // Verify dodo IDs are preserved
- user, err := ts.UserService.GetUserBySupabaseID(ctx, userID)
- if err != nil {
- t.Fatalf("Failed to get user: %v", err)
- }
-
- if user.DodoCustomerID != "cust_test_12345" {
- t.Errorf("DodoCustomerID should be preserved, got '%s'", user.DodoCustomerID)
- }
- if user.DodoSubscriptionID != "sub_test_12345" {
- t.Errorf("DodoSubscriptionID should be preserved, got '%s'", user.DodoSubscriptionID)
- }
-
- t.Log("✅ Existing paid user re-login test passed")
-}
-
-// ============================================================================
-// Test: Tier Cache TTL Expiration
-// ============================================================================
-
-func TestE2E_TierCacheTTLExpiration(t *testing.T) {
- if testing.Short() {
- t.Skip("Skipping E2E test in short mode")
- }
-
- ts := SetupE2ETestWithMongoDB(t, &PromoTestConfig{
- PromoEnabled: false,
- })
- if ts == nil {
- return
- }
- defer ts.Cleanup()
-
- ctx := context.Background()
- userID := "test-e2e-cache-" + time.Now().Format("20060102150405")
- email := "test-e2e-cache@example.com"
-
- // Pre-create user with free tier
- collection := ts.MongoDB.Database().Collection("users")
- _, err := collection.InsertOne(ctx, bson.M{
- "supabaseUserId": userID,
- "email": email,
- "subscriptionTier": models.TierFree,
- "subscriptionStatus": models.SubStatusActive,
- "createdAt": time.Now(),
- "lastLoginAt": time.Now(),
- })
- if err != nil {
- t.Fatalf("Failed to pre-create user: %v", err)
- }
-
- // Get tier (should cache as "free")
- tier1 := ts.TierService.GetUserTier(ctx, userID)
- if tier1 != models.TierFree {
- t.Errorf("Expected tier 'free', got '%s'", tier1)
- }
-
- // Update tier directly in DB
- _, err = collection.UpdateOne(ctx,
- bson.M{"supabaseUserId": userID},
- bson.M{"$set": bson.M{"subscriptionTier": models.TierPro}},
- )
- if err != nil {
- t.Fatalf("Failed to update tier in DB: %v", err)
- }
-
- // Get tier again - should still return cached "free" (TTL not expired)
- tier2 := ts.TierService.GetUserTier(ctx, userID)
- if tier2 != models.TierFree {
- t.Logf("Note: Cache returned '%s' instead of 'free' - TTL may have already expired", tier2)
- }
-
- // Invalidate cache manually
- ts.TierService.InvalidateCache(userID)
-
- // Get tier again - should return "pro" from DB
- tier3 := ts.TierService.GetUserTier(ctx, userID)
- if tier3 != models.TierPro {
- t.Errorf("After cache invalidation, expected tier 'pro', got '%s'", tier3)
- }
-
- t.Log("✅ Tier cache TTL test passed")
-}
-
-// ============================================================================
-// Test: Promo Disabled - New User Gets Free
-// ============================================================================
-
-func TestE2E_PromoDisabled(t *testing.T) {
- if testing.Short() {
- t.Skip("Skipping E2E test in short mode")
- }
-
- // Setup with promo DISABLED
- ts := SetupE2ETestWithMongoDB(t, &PromoTestConfig{
- PromoEnabled: false,
- PromoStartDate: time.Now().Add(-1 * time.Hour),
- PromoEndDate: time.Now().Add(24 * time.Hour),
- PromoDuration: 30,
- })
- if ts == nil {
- return
- }
- defer ts.Cleanup()
-
- userID := "test-e2e-nopromo-" + time.Now().Format("20060102150405")
- email := "test-e2e-nopromo@example.com"
-
- // Create app with auth middleware for this user
- app := fiber.New(fiber.Config{DisableStartupMessage: true})
- app.Use(testAuthMiddleware(userID, email))
-
- subHandler := handlers.NewSubscriptionHandler(ts.PaymentService, ts.UserService)
- app.Get("/api/subscriptions/current", subHandler.GetCurrent)
-
- // Call GET /api/subscriptions/current
- req := httptest.NewRequest("GET", "/api/subscriptions/current", nil)
- resp, err := app.Test(req, -1)
- if err != nil {
- t.Fatalf("Request failed: %v", err)
- }
- defer resp.Body.Close()
-
- if resp.StatusCode != 200 {
- body, _ := io.ReadAll(resp.Body)
- t.Fatalf("Expected 200, got %d: %s", resp.StatusCode, string(body))
- }
-
- var result map[string]interface{}
- if err := json.NewDecoder(resp.Body).Decode(&result); err != nil {
- t.Fatalf("Failed to decode response: %v", err)
- }
-
- // Even though we're in promo date range, promo is disabled, so free tier
- if result["tier"] != "free" {
- t.Errorf("Expected tier 'free' when promo disabled, got '%v'", result["tier"])
- }
-
- if result["is_promo_user"] != false {
- t.Errorf("Expected is_promo_user false when promo disabled, got '%v'", result["is_promo_user"])
- }
-
- t.Log("✅ Promo disabled test passed")
-}
diff --git a/backend/internal/tests/subscription_integration_test.go b/backend/internal/tests/subscription_integration_test.go
deleted file mode 100644
index 1ffbdbe3..00000000
--- a/backend/internal/tests/subscription_integration_test.go
+++ /dev/null
@@ -1,194 +0,0 @@
-package tests
-
-import (
- "context"
- "testing"
- "time"
-
- "claraverse/internal/models"
-)
-
-// MockDodoClient implements DodoPayments client for testing
-type MockDodoClient struct {
- CheckoutSessions map[string]*CheckoutSession
- Subscriptions map[string]*Subscription
-}
-
-type CheckoutSession struct {
- ID string
- CheckoutURL string
-}
-
-type Subscription struct {
- ID string
- CustomerID string
- ProductID string
- Status string
- CurrentPeriodStart time.Time
- CurrentPeriodEnd time.Time
- CancelAtPeriodEnd bool
-}
-
-func TestIntegration_FullUpgradeFlow(t *testing.T) {
- if testing.Short() {
- t.Skip("Skipping integration test")
- }
-
- ctx := context.Background()
-
- // Setup: User starts with free tier
- userID := "test-user-upgrade"
-
- // Step 1: Create checkout session for Pro
- // Step 2: Simulate successful payment (webhook)
- // Step 3: Verify user is now on Pro tier
- // Step 4: Upgrade to Pro+
- // Step 5: Verify prorated charge
- // Step 6: Verify user is now on Pro+ tier
-
- _ = ctx
- _ = userID
-
- // TODO: Implement with MongoDB test setup
- t.Log("Integration test placeholder - requires MongoDB test setup")
-}
-
-func TestIntegration_FullDowngradeFlow(t *testing.T) {
- if testing.Short() {
- t.Skip("Skipping integration test")
- }
-
- // Setup: User is on Pro+ tier
- // Step 1: Request downgrade to Pro
- // Step 2: Verify downgrade is scheduled (not immediate)
- // Step 3: Verify user still has Pro+ access
- // Step 4: Simulate billing period end (webhook)
- // Step 5: Verify user is now on Pro tier
-
- // TODO: Implement with MongoDB test setup
- t.Log("Integration test placeholder - requires MongoDB test setup")
-}
-
-func TestIntegration_CancellationFlow(t *testing.T) {
- if testing.Short() {
- t.Skip("Skipping integration test")
- }
-
- // Setup: User is on Pro tier
- // Step 1: Request cancellation
- // Step 2: Verify status is pending_cancel
- // Step 3: Verify user still has Pro access
- // Step 4: Simulate billing period end
- // Step 5: Verify user is now on Free tier
-
- // TODO: Implement with MongoDB test setup
- t.Log("Integration test placeholder - requires MongoDB test setup")
-}
-
-func TestIntegration_ReactivationFlow(t *testing.T) {
- if testing.Short() {
- t.Skip("Skipping integration test")
- }
-
- // Setup: User has pending cancellation
- // Step 1: Request reactivation
- // Step 2: Verify cancellation is cleared
- // Step 3: Verify subscription continues normally
-
- // TODO: Implement with MongoDB test setup
- t.Log("Integration test placeholder - requires MongoDB test setup")
-}
-
-func TestIntegration_PaymentFailureFlow(t *testing.T) {
- if testing.Short() {
- t.Skip("Skipping integration test")
- }
-
- // Setup: User is on Pro tier
- // Step 1: Simulate payment failure (webhook)
- // Step 2: Verify status is on_hold
- // Step 3: Verify user still has Pro access (grace period)
- // Step 4: Simulate payment retry success
- // Step 5: Verify status back to active
-
- // TODO: Implement with MongoDB test setup
- t.Log("Integration test placeholder - requires MongoDB test setup")
-}
-
-func TestIntegration_PaymentFailureToFree(t *testing.T) {
- if testing.Short() {
- t.Skip("Skipping integration test")
- }
-
- // Setup: User is on_hold status
- // Step 1: Simulate grace period expiry
- // Step 2: Verify user reverted to Free tier
-
- // TODO: Implement with MongoDB test setup
- t.Log("Integration test placeholder - requires MongoDB test setup")
-}
-
-func TestIntegration_TierServiceCacheInvalidation(t *testing.T) {
- if testing.Short() {
- t.Skip("Skipping integration test")
- }
-
- // Verify that tier cache is invalidated on subscription changes
- // Step 1: Get user tier (should cache)
- // Step 2: Update subscription via webhook
- // Step 3: Get user tier (should return new tier, not cached)
-
- // TODO: Implement with MongoDB test setup
- t.Log("Integration test placeholder - requires MongoDB test setup")
-}
-
-func TestIntegration_PlanComparison(t *testing.T) {
- // Test tier comparison logic
- tests := []struct {
- name string
- fromTier string
- toTier string
- expected int
- }{
- {"free to pro", models.TierFree, models.TierPro, -1},
- {"pro to max", models.TierPro, models.TierMax, -1},
- {"max to pro", models.TierMax, models.TierPro, 1},
- {"pro to free", models.TierPro, models.TierFree, 1},
- {"same tier", models.TierPro, models.TierPro, 0},
- }
-
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- result := models.CompareTiers(tt.fromTier, tt.toTier)
- if result != tt.expected {
- t.Errorf("CompareTiers(%s, %s) = %d, want %d",
- tt.fromTier, tt.toTier, result, tt.expected)
- }
- })
- }
-}
-
-func TestIntegration_SubscriptionStatusTransitions(t *testing.T) {
- // Test subscription status transitions
- tests := []struct {
- name string
- status string
- shouldBeActive bool
- }{
- {"active", models.SubStatusActive, true},
- {"on_hold", models.SubStatusOnHold, true},
- {"pending_cancel", models.SubStatusPendingCancel, true},
- {"cancelled", models.SubStatusCancelled, false},
- {"paused", models.SubStatusPaused, false},
- }
-
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- sub := &models.Subscription{Status: tt.status}
- if sub.IsActive() != tt.shouldBeActive {
- t.Errorf("IsActive() for status %s = %v, want %v",
- tt.status, sub.IsActive(), tt.shouldBeActive)
- }
- })
- }
-}
diff --git a/backend/internal/tools/airtable_tool.go b/backend/internal/tools/airtable_tool.go
deleted file mode 100644
index 52a8d985..00000000
--- a/backend/internal/tools/airtable_tool.go
+++ /dev/null
@@ -1,358 +0,0 @@
-package tools
-
-import (
- "bytes"
- "encoding/json"
- "fmt"
- "io"
- "net/http"
- "net/url"
- "time"
-)
-
-const airtableAPIBase = "https://api.airtable.com/v0"
-
-// NewAirtableListTool creates a tool for listing Airtable records
-func NewAirtableListTool() *Tool {
- return &Tool{
- Name: "airtable_list",
- DisplayName: "List Airtable Records",
- Description: "List records from an Airtable table. Authentication is handled automatically via configured credentials.",
- Icon: "Table",
- Source: ToolSourceBuiltin,
- Category: "integration",
- Keywords: []string{"airtable", "database", "records", "list", "table"},
- Parameters: map[string]interface{}{
- "type": "object",
- "properties": map[string]interface{}{
- "credential_id": map[string]interface{}{
- "type": "string",
- "description": "INTERNAL: Auto-injected by system. Do not set manually.",
- },
- "base_id": map[string]interface{}{
- "type": "string",
- "description": "Airtable Base ID (e.g., appXXXXXXXXXXXXXX)",
- },
- "table_name": map[string]interface{}{
- "type": "string",
- "description": "Table name or ID",
- },
- "view": map[string]interface{}{
- "type": "string",
- "description": "Optional view name to filter records",
- },
- "max_records": map[string]interface{}{
- "type": "number",
- "description": "Maximum number of records to return (default 100)",
- },
- "filter_formula": map[string]interface{}{
- "type": "string",
- "description": "Airtable formula to filter records",
- },
- },
- "required": []string{"base_id", "table_name"},
- },
- Execute: executeAirtableList,
- }
-}
-
-// NewAirtableReadTool creates a tool for reading a single Airtable record
-func NewAirtableReadTool() *Tool {
- return &Tool{
- Name: "airtable_read",
- DisplayName: "Read Airtable Record",
- Description: "Read a single record from an Airtable table by ID. Authentication is handled automatically.",
- Icon: "FileText",
- Source: ToolSourceBuiltin,
- Category: "integration",
- Keywords: []string{"airtable", "database", "record", "read", "get"},
- Parameters: map[string]interface{}{
- "type": "object",
- "properties": map[string]interface{}{
- "credential_id": map[string]interface{}{
- "type": "string",
- "description": "INTERNAL: Auto-injected by system. Do not set manually.",
- },
- "base_id": map[string]interface{}{
- "type": "string",
- "description": "Airtable Base ID",
- },
- "table_name": map[string]interface{}{
- "type": "string",
- "description": "Table name or ID",
- },
- "record_id": map[string]interface{}{
- "type": "string",
- "description": "Record ID to retrieve",
- },
- },
- "required": []string{"base_id", "table_name", "record_id"},
- },
- Execute: executeAirtableRead,
- }
-}
-
-// NewAirtableCreateTool creates a tool for creating Airtable records
-func NewAirtableCreateTool() *Tool {
- return &Tool{
- Name: "airtable_create",
- DisplayName: "Create Airtable Record",
- Description: "Create a new record in an Airtable table. Authentication is handled automatically.",
- Icon: "Plus",
- Source: ToolSourceBuiltin,
- Category: "integration",
- Keywords: []string{"airtable", "database", "record", "create", "add"},
- Parameters: map[string]interface{}{
- "type": "object",
- "properties": map[string]interface{}{
- "credential_id": map[string]interface{}{
- "type": "string",
- "description": "INTERNAL: Auto-injected by system. Do not set manually.",
- },
- "base_id": map[string]interface{}{
- "type": "string",
- "description": "Airtable Base ID",
- },
- "table_name": map[string]interface{}{
- "type": "string",
- "description": "Table name or ID",
- },
- "fields": map[string]interface{}{
- "type": "object",
- "description": "Record fields as key-value pairs",
- },
- },
- "required": []string{"base_id", "table_name", "fields"},
- },
- Execute: executeAirtableCreate,
- }
-}
-
-// NewAirtableUpdateTool creates a tool for updating Airtable records
-func NewAirtableUpdateTool() *Tool {
- return &Tool{
- Name: "airtable_update",
- DisplayName: "Update Airtable Record",
- Description: "Update an existing record in an Airtable table. Authentication is handled automatically.",
- Icon: "Edit",
- Source: ToolSourceBuiltin,
- Category: "integration",
- Keywords: []string{"airtable", "database", "record", "update", "edit"},
- Parameters: map[string]interface{}{
- "type": "object",
- "properties": map[string]interface{}{
- "credential_id": map[string]interface{}{
- "type": "string",
- "description": "INTERNAL: Auto-injected by system. Do not set manually.",
- },
- "base_id": map[string]interface{}{
- "type": "string",
- "description": "Airtable Base ID",
- },
- "table_name": map[string]interface{}{
- "type": "string",
- "description": "Table name or ID",
- },
- "record_id": map[string]interface{}{
- "type": "string",
- "description": "Record ID to update",
- },
- "fields": map[string]interface{}{
- "type": "object",
- "description": "Fields to update as key-value pairs",
- },
- },
- "required": []string{"base_id", "table_name", "record_id", "fields"},
- },
- Execute: executeAirtableUpdate,
- }
-}
-
-func airtableRequest(method, endpoint, token string, body interface{}) (map[string]interface{}, error) {
- var reqBody io.Reader
- if body != nil {
- jsonBody, err := json.Marshal(body)
- if err != nil {
- return nil, fmt.Errorf("failed to marshal request body: %w", err)
- }
- reqBody = bytes.NewBuffer(jsonBody)
- }
-
- req, err := http.NewRequest(method, airtableAPIBase+endpoint, reqBody)
- if err != nil {
- return nil, fmt.Errorf("failed to create request: %w", err)
- }
-
- req.Header.Set("Authorization", "Bearer "+token)
- req.Header.Set("Content-Type", "application/json")
-
- client := &http.Client{Timeout: 30 * time.Second}
- resp, err := client.Do(req)
- if err != nil {
- return nil, fmt.Errorf("request failed: %w", err)
- }
- defer resp.Body.Close()
-
- respBody, err := io.ReadAll(resp.Body)
- if err != nil {
- return nil, fmt.Errorf("failed to read response: %w", err)
- }
-
- var result map[string]interface{}
- if err := json.Unmarshal(respBody, &result); err != nil {
- return nil, fmt.Errorf("failed to parse response: %w", err)
- }
-
- if resp.StatusCode >= 400 {
- errMsg := "Airtable API error"
- if errInfo, ok := result["error"].(map[string]interface{}); ok {
- if msg, ok := errInfo["message"].(string); ok {
- errMsg = msg
- }
- }
- return nil, fmt.Errorf("%s (status %d)", errMsg, resp.StatusCode)
- }
-
- return result, nil
-}
-
-func executeAirtableList(args map[string]interface{}) (string, error) {
- token, err := ResolveAPIKey(args, "airtable", "api_key")
- if err != nil {
- return "", fmt.Errorf("failed to get Airtable token: %w", err)
- }
-
- baseID, _ := args["base_id"].(string)
- tableName, _ := args["table_name"].(string)
-
- if baseID == "" || tableName == "" {
- return "", fmt.Errorf("base_id and table_name are required")
- }
-
- // Build query params
- params := url.Values{}
- if view, ok := args["view"].(string); ok && view != "" {
- params.Set("view", view)
- }
- if maxRecords, ok := args["max_records"].(float64); ok && maxRecords > 0 {
- params.Set("maxRecords", fmt.Sprintf("%d", int(maxRecords)))
- }
- if filter, ok := args["filter_formula"].(string); ok && filter != "" {
- params.Set("filterByFormula", filter)
- }
-
- endpoint := fmt.Sprintf("/%s/%s", baseID, url.PathEscape(tableName))
- if len(params) > 0 {
- endpoint += "?" + params.Encode()
- }
-
- result, err := airtableRequest("GET", endpoint, token, nil)
- if err != nil {
- return "", err
- }
-
- response := map[string]interface{}{
- "success": true,
- "records": result["records"],
- }
-
- jsonResult, _ := json.MarshalIndent(response, "", " ")
- return string(jsonResult), nil
-}
-
-func executeAirtableRead(args map[string]interface{}) (string, error) {
- token, err := ResolveAPIKey(args, "airtable", "api_key")
- if err != nil {
- return "", fmt.Errorf("failed to get Airtable token: %w", err)
- }
-
- baseID, _ := args["base_id"].(string)
- tableName, _ := args["table_name"].(string)
- recordID, _ := args["record_id"].(string)
-
- if baseID == "" || tableName == "" || recordID == "" {
- return "", fmt.Errorf("base_id, table_name, and record_id are required")
- }
-
- endpoint := fmt.Sprintf("/%s/%s/%s", baseID, url.PathEscape(tableName), recordID)
- result, err := airtableRequest("GET", endpoint, token, nil)
- if err != nil {
- return "", err
- }
-
- response := map[string]interface{}{
- "success": true,
- "record": result,
- }
-
- jsonResult, _ := json.MarshalIndent(response, "", " ")
- return string(jsonResult), nil
-}
-
-func executeAirtableCreate(args map[string]interface{}) (string, error) {
- token, err := ResolveAPIKey(args, "airtable", "api_key")
- if err != nil {
- return "", fmt.Errorf("failed to get Airtable token: %w", err)
- }
-
- baseID, _ := args["base_id"].(string)
- tableName, _ := args["table_name"].(string)
- fields, _ := args["fields"].(map[string]interface{})
-
- if baseID == "" || tableName == "" || len(fields) == 0 {
- return "", fmt.Errorf("base_id, table_name, and fields are required")
- }
-
- endpoint := fmt.Sprintf("/%s/%s", baseID, url.PathEscape(tableName))
- body := map[string]interface{}{"fields": fields}
-
- result, err := airtableRequest("POST", endpoint, token, body)
- if err != nil {
- return "", err
- }
-
- response := map[string]interface{}{
- "success": true,
- "message": "Record created successfully",
- "record_id": result["id"],
- "record": result,
- }
-
- jsonResult, _ := json.MarshalIndent(response, "", " ")
- return string(jsonResult), nil
-}
-
-func executeAirtableUpdate(args map[string]interface{}) (string, error) {
- token, err := ResolveAPIKey(args, "airtable", "api_key")
- if err != nil {
- return "", fmt.Errorf("failed to get Airtable token: %w", err)
- }
-
- baseID, _ := args["base_id"].(string)
- tableName, _ := args["table_name"].(string)
- recordID, _ := args["record_id"].(string)
- fields, _ := args["fields"].(map[string]interface{})
-
- if baseID == "" || tableName == "" || recordID == "" || len(fields) == 0 {
- return "", fmt.Errorf("base_id, table_name, record_id, and fields are required")
- }
-
- endpoint := fmt.Sprintf("/%s/%s/%s", baseID, url.PathEscape(tableName), recordID)
- body := map[string]interface{}{"fields": fields}
-
- result, err := airtableRequest("PATCH", endpoint, token, body)
- if err != nil {
- return "", err
- }
-
- response := map[string]interface{}{
- "success": true,
- "message": "Record updated successfully",
- "record_id": result["id"],
- "record": result,
- }
-
- jsonResult, _ := json.MarshalIndent(response, "", " ")
- return string(jsonResult), nil
-}
-
diff --git a/backend/internal/tools/api_tester_tool.go b/backend/internal/tools/api_tester_tool.go
deleted file mode 100644
index 0022a79f..00000000
--- a/backend/internal/tools/api_tester_tool.go
+++ /dev/null
@@ -1,289 +0,0 @@
-package tools
-
-import (
- "context"
- "encoding/json"
- "fmt"
- "strings"
-
- "claraverse/internal/e2b"
- "claraverse/internal/security"
-)
-
-// NewAPITesterTool creates a new API Tester tool
-func NewAPITesterTool() *Tool {
- return &Tool{
- Name: "test_api",
- DisplayName: "API Tester",
- Description: "Test REST API endpoints with various HTTP methods (GET, POST, PUT, DELETE, PATCH). Sends requests, validates responses, measures response times, and displays status codes, headers, and response bodies. Useful for API testing, debugging, and validation.",
- Icon: "Network",
- Source: ToolSourceBuiltin,
- Category: "integration",
- Keywords: []string{"api", "test", "http", "rest", "endpoint", "request", "response", "get", "post", "put", "delete", "patch", "web service", "debugging"},
- Parameters: map[string]interface{}{
- "type": "object",
- "properties": map[string]interface{}{
- "url": map[string]interface{}{
- "type": "string",
- "description": "API endpoint URL (must include http:// or https://)",
- "pattern": "^https?://.*$",
- },
- "method": map[string]interface{}{
- "type": "string",
- "description": "HTTP method to use",
- "enum": []string{"GET", "POST", "PUT", "DELETE", "PATCH"},
- "default": "GET",
- },
- "headers": map[string]interface{}{
- "type": "object",
- "description": "HTTP headers to include in the request (optional)",
- "additionalProperties": map[string]interface{}{
- "type": "string",
- },
- },
- "body": map[string]interface{}{
- "type": "string",
- "description": "Request body (JSON string, optional)",
- },
- "expected_status": map[string]interface{}{
- "type": "number",
- "description": "Expected HTTP status code for validation (optional)",
- "minimum": 100,
- "maximum": 599,
- },
- },
- "required": []string{"url"},
- },
- Execute: executeAPITester,
- }
-}
-
-func executeAPITester(args map[string]interface{}) (string, error) {
- // Extract parameters
- url, ok := args["url"].(string)
- if !ok {
- return "", fmt.Errorf("url must be a string")
- }
-
- // Validate URL
- if !strings.HasPrefix(url, "http://") && !strings.HasPrefix(url, "https://") {
- return "", fmt.Errorf("url must start with http:// or https://")
- }
-
- // SSRF protection: block requests to internal/private networks
- if err := security.ValidateURLForSSRF(url); err != nil {
- return "", fmt.Errorf("SSRF protection: %w", err)
- }
-
- method := "GET"
- if m, ok := args["method"].(string); ok {
- method = strings.ToUpper(m)
- }
-
- headers := make(map[string]string)
- if headersRaw, ok := args["headers"].(map[string]interface{}); ok {
- for key, value := range headersRaw {
- headers[key] = fmt.Sprintf("%v", value)
- }
- }
-
- body := ""
- if b, ok := args["body"].(string); ok {
- body = b
- }
-
- expectedStatus := 0
- if es, ok := args["expected_status"].(float64); ok {
- expectedStatus = int(es)
- }
-
- // Generate Python code
- pythonCode := generateAPITestCode(url, method, headers, body, expectedStatus)
-
- // Execute code
- e2bService := e2b.GetE2BExecutorService()
- result, err := e2bService.Execute(context.Background(), pythonCode, 30)
- if err != nil {
- return "", fmt.Errorf("failed to execute API test: %w", err)
- }
-
- if !result.Success {
- if result.Error != nil {
- return "", fmt.Errorf("API test failed: %s", *result.Error)
- }
- return "", fmt.Errorf("API test failed with stderr: %s", result.Stderr)
- }
-
- // Format response
- response := map[string]interface{}{
- "success": true,
- "url": url,
- "method": method,
- "output": result.Stdout,
- }
-
- jsonResponse, _ := json.MarshalIndent(response, "", " ")
- return string(jsonResponse), nil
-}
-
-func generateAPITestCode(url, method string, headers map[string]string, body string, expectedStatus int) string {
- // Build headers dict
- headersStr := ""
- if len(headers) > 0 {
- headerParts := []string{}
- for key, value := range headers {
- // Escape single quotes in values
- escapedValue := strings.ReplaceAll(value, "'", "\\'")
- headerParts = append(headerParts, fmt.Sprintf(" '%s': '%s'", key, escapedValue))
- }
- headersStr = fmt.Sprintf("{\n%s\n}", strings.Join(headerParts, ",\n"))
- } else {
- headersStr = "{}"
- }
-
- // Escape body for Python string
- escapedBody := strings.ReplaceAll(body, "'", "\\'")
- escapedBody = strings.ReplaceAll(escapedBody, "\n", "\\n")
-
- code := fmt.Sprintf(`import requests
-import json
-import time
-
-print("=" * 80)
-print("🔌 API TESTER")
-print("=" * 80)
-
-# Request configuration
-url = '%s'
-method = '%s'
-headers = %s
-`, url, method, headersStr)
-
- if body != "" {
- code += fmt.Sprintf(`
-body = '''%s'''
-`, escapedBody)
- } else {
- code += `
-body = None
-`
- }
-
- code += fmt.Sprintf(`
-print(f"\n📡 Testing API Endpoint")
-print("-" * 80)
-print(f"URL: {url}")
-print(f"Method: {method}")
-
-if headers:
- print(f"\n📋 Headers:")
- for key, value in headers.items():
- print(f" {key}: {value}")
-
-if body:
- print(f"\n📦 Request Body:")
- try:
- # Try to pretty-print if it's JSON
- body_dict = json.loads(body)
- print(json.dumps(body_dict, indent=2))
- except:
- print(body[:500]) # Print first 500 chars if not JSON
-
-print(f"\n⏳ Sending request...")
-
-# Send request
-start_time = time.time()
-
-try:
- if method == 'GET':
- response = requests.get(url, headers=headers, timeout=10)
- elif method == 'POST':
- response = requests.post(url, headers=headers, data=body, timeout=10)
- elif method == 'PUT':
- response = requests.put(url, headers=headers, data=body, timeout=10)
- elif method == 'DELETE':
- response = requests.delete(url, headers=headers, timeout=10)
- elif method == 'PATCH':
- response = requests.patch(url, headers=headers, data=body, timeout=10)
- else:
- raise ValueError(f"Unsupported method: {method}")
-
- elapsed_time = time.time() - start_time
-
- print(f"\n✅ Request completed in {elapsed_time:.3f}s")
-
- # Response details
- print(f"\n📊 RESPONSE")
- print("-" * 80)
- print(f"Status Code: {response.status_code} {response.reason}")
-`)
-
- if expectedStatus > 0 {
- code += fmt.Sprintf(`
- # Validate status code
- if response.status_code == %d:
- print(f"✅ Status code matches expected: %d")
- else:
- print(f"❌ Status code mismatch! Expected: %d, Got: {response.status_code}")
-`, expectedStatus, expectedStatus, expectedStatus)
- }
-
- code += `
- print(f"Response Time: {elapsed_time:.3f}s")
- print(f"Content Length: {len(response.content)} bytes")
-
- # Response headers
- print(f"\n📋 Response Headers:")
- for key, value in response.headers.items():
- print(f" {key}: {value}")
-
- # Response body
- print(f"\n📦 Response Body:")
- print("-" * 80)
-
- content_type = response.headers.get('Content-Type', '')
-
- if 'application/json' in content_type:
- try:
- json_data = response.json()
- print(json.dumps(json_data, indent=2))
- except:
- print(response.text[:2000])
- else:
- print(response.text[:2000])
-
- if len(response.text) > 2000:
- print(f"\n... (Total length: {len(response.text)} characters)")
-
- # Status code interpretation
- print(f"\n💡 Status Code Interpretation:")
- if 200 <= response.status_code < 300:
- print(f" ✅ Success - Request completed successfully")
- elif 300 <= response.status_code < 400:
- print(f" 🔄 Redirect - Resource moved to another location")
- elif 400 <= response.status_code < 500:
- print(f" ❌ Client Error - Problem with the request")
- elif 500 <= response.status_code < 600:
- print(f" 🚨 Server Error - Problem on the server side")
-
-except requests.exceptions.Timeout:
- print(f"\n⏱️ Request timed out after 10 seconds")
- print(f"💡 Tip: The server is taking too long to respond")
-
-except requests.exceptions.ConnectionError:
- print(f"\n🔌 Connection failed")
- print(f"💡 Tip: Check if the URL is correct and the server is accessible")
-
-except requests.exceptions.RequestException as e:
- print(f"\n❌ Request failed: {e}")
-
-except Exception as e:
- print(f"\n❌ Unexpected error: {e}")
-
-print("\n" + "=" * 80)
-print("✅ API TEST COMPLETE")
-print("=" * 80)
-`
-
- return code
-}
diff --git a/backend/internal/tools/ask_user_tool.go b/backend/internal/tools/ask_user_tool.go
deleted file mode 100644
index 04770f49..00000000
--- a/backend/internal/tools/ask_user_tool.go
+++ /dev/null
@@ -1,320 +0,0 @@
-package tools
-
-import (
- "claraverse/internal/models"
- "encoding/json"
- "fmt"
- "log"
- "time"
-
- "github.com/google/uuid"
-)
-
-// UserConnectionKey is the key for injecting user connection into tool args
-const UserConnectionKey = "__user_connection__"
-
-// PromptWaiterKey is the key for injecting the prompt response waiter function
-const PromptWaiterKey = "__prompt_waiter__"
-
-// NewAskUserTool creates a tool that allows the AI to ask clarifying questions via modal prompts
-func NewAskUserTool() *Tool {
- return &Tool{
- Name: "ask_user",
- DisplayName: "Ask User Questions",
- Description: "Ask the user clarifying questions via an interactive modal dialog. Use this when you need additional information from the user to complete a task (e.g., preferences, choices, confirmation). This tool WAITS for the user to respond (blocks execution) and returns their answers, so you can use the responses immediately in your next step. Maximum wait time is 5 minutes.",
- Icon: "MessageCircleQuestion",
- Parameters: map[string]interface{}{
- "type": "object",
- "properties": map[string]interface{}{
- "title": map[string]interface{}{
- "type": "string",
- "description": "Title of the prompt dialog (e.g., 'Need More Information', 'Create Project')",
- },
- "description": map[string]interface{}{
- "type": "string",
- "description": "Optional description explaining why you're asking these questions",
- },
- "questions": map[string]interface{}{
- "type": "array",
- "description": "Array of questions to ask the user (minimum 1, maximum 5 questions recommended)",
- "minItems": 1,
- "maxItems": 10,
- "items": map[string]interface{}{
- "type": "object",
- "properties": map[string]interface{}{
- "id": map[string]interface{}{
- "type": "string",
- "description": "Unique identifier for this question (e.g., 'language', 'framework', 'email')",
- },
- "type": map[string]interface{}{
- "type": "string",
- "description": "Question type: 'text', 'number', 'checkbox', 'select' (radio), or 'multi-select' (checkboxes)",
- "enum": []string{"text", "number", "checkbox", "select", "multi-select"},
- },
- "label": map[string]interface{}{
- "type": "string",
- "description": "The question text to display to the user",
- },
- "placeholder": map[string]interface{}{
- "type": "string",
- "description": "Placeholder text for text/number inputs (optional)",
- },
- "required": map[string]interface{}{
- "type": "boolean",
- "description": "Whether the user must answer this question (default: false)",
- "default": false,
- },
- "options": map[string]interface{}{
- "type": "array",
- "description": "Options for 'select' or 'multi-select' questions (required for those types)",
- "items": map[string]interface{}{
- "type": "string",
- },
- },
- "allow_other": map[string]interface{}{
- "type": "boolean",
- "description": "For select/multi-select: allow 'Other' option with custom text input (default: false)",
- "default": false,
- },
- "validation": map[string]interface{}{
- "type": "object",
- "description": "Validation rules for the question (optional)",
- "properties": map[string]interface{}{
- "min": map[string]interface{}{
- "type": "number",
- "description": "Minimum value for number type",
- },
- "max": map[string]interface{}{
- "type": "number",
- "description": "Maximum value for number type",
- },
- "pattern": map[string]interface{}{
- "type": "string",
- "description": "Regex pattern for text validation (e.g., email pattern)",
- },
- "min_length": map[string]interface{}{
- "type": "integer",
- "description": "Minimum length for text input",
- },
- "max_length": map[string]interface{}{
- "type": "integer",
- "description": "Maximum length for text input",
- },
- },
- },
- },
- "required": []string{"id", "type", "label"},
- },
- },
- "allow_skip": map[string]interface{}{
- "type": "boolean",
- "description": "Whether the user can skip/cancel the prompt (default: true). Set to false for critical questions.",
- "default": true,
- },
- },
- "required": []string{"title", "questions"},
- },
- Execute: executeAskUser,
- Source: ToolSourceBuiltin,
- Category: "interaction",
- Keywords: []string{"ask", "question", "prompt", "user", "input", "clarify", "modal"},
- }
-}
-
-func executeAskUser(args map[string]interface{}) (string, error) {
- // Extract user connection (injected by chat service)
- userConn, ok := args[UserConnectionKey].(*models.UserConnection)
- if !ok || userConn == nil {
- return "", fmt.Errorf("interactive prompts are not available in this context (user connection not found)")
- }
-
- // Extract title
- title, ok := args["title"].(string)
- if !ok || title == "" {
- return "", fmt.Errorf("title is required")
- }
-
- // Extract description (optional)
- description, _ := args["description"].(string)
-
- // Extract questions array
- questionsRaw, ok := args["questions"].([]interface{})
- if !ok || len(questionsRaw) == 0 {
- return "", fmt.Errorf("questions array is required and must not be empty")
- }
-
- // Convert questions to InteractiveQuestion structs
- questions := make([]models.InteractiveQuestion, 0, len(questionsRaw))
- for i, qRaw := range questionsRaw {
- qMap, ok := qRaw.(map[string]interface{})
- if !ok {
- return "", fmt.Errorf("question at index %d is not a valid object", i)
- }
-
- // Extract required fields
- id, _ := qMap["id"].(string)
- qType, _ := qMap["type"].(string)
- label, _ := qMap["label"].(string)
-
- if id == "" || qType == "" || label == "" {
- return "", fmt.Errorf("question at index %d is missing required fields (id, type, or label)", i)
- }
-
- // Validate question type
- validTypes := map[string]bool{
- "text": true,
- "number": true,
- "checkbox": true,
- "select": true,
- "multi-select": true,
- }
- if !validTypes[qType] {
- return "", fmt.Errorf("invalid question type '%s' at index %d. Must be: text, number, checkbox, select, or multi-select", qType, i)
- }
-
- question := models.InteractiveQuestion{
- ID: id,
- Type: qType,
- Label: label,
- }
-
- // Optional: placeholder
- if placeholder, ok := qMap["placeholder"].(string); ok {
- question.Placeholder = placeholder
- }
-
- // Optional: required
- if required, ok := qMap["required"].(bool); ok {
- question.Required = required
- }
-
- // Optional: options (required for select/multi-select)
- if optionsRaw, ok := qMap["options"].([]interface{}); ok {
- options := make([]string, 0, len(optionsRaw))
- for _, opt := range optionsRaw {
- if optStr, ok := opt.(string); ok {
- options = append(options, optStr)
- }
- }
- question.Options = options
- } else if qType == "select" || qType == "multi-select" {
- return "", fmt.Errorf("question '%s' (type: %s) requires an 'options' array", id, qType)
- }
-
- // Optional: allow_other
- if allowOther, ok := qMap["allow_other"].(bool); ok {
- question.AllowOther = allowOther
- }
-
- // Optional: validation
- if validationRaw, ok := qMap["validation"].(map[string]interface{}); ok {
- validation := &models.QuestionValidation{}
-
- if min, ok := validationRaw["min"].(float64); ok {
- validation.Min = &min
- }
- if max, ok := validationRaw["max"].(float64); ok {
- validation.Max = &max
- }
- if pattern, ok := validationRaw["pattern"].(string); ok {
- validation.Pattern = pattern
- }
- if minLength, ok := validationRaw["min_length"].(float64); ok {
- minLenInt := int(minLength)
- validation.MinLength = &minLenInt
- }
- if maxLength, ok := validationRaw["max_length"].(float64); ok {
- maxLenInt := int(maxLength)
- validation.MaxLength = &maxLenInt
- }
-
- question.Validation = validation
- }
-
- questions = append(questions, question)
- }
-
- // Extract allow_skip (default: true)
- allowSkip := true
- if skipRaw, ok := args["allow_skip"].(bool); ok {
- allowSkip = skipRaw
- }
-
- // Generate prompt ID
- promptID := uuid.New().String()
-
- // Create the prompt message
- prompt := models.ServerMessage{
- Type: "interactive_prompt",
- PromptID: promptID,
- ConversationID: userConn.ConversationID,
- Title: title,
- Description: description,
- Questions: questions,
- AllowSkip: &allowSkip,
- }
-
- // Extract prompt waiter function (injected by chat service)
- waiterFunc, ok := args[PromptWaiterKey].(models.PromptWaiterFunc)
- if !ok || waiterFunc == nil {
- return "", fmt.Errorf("prompt waiter not available (internal error)")
- }
-
- // Send the prompt to the user
- success := userConn.SafeSend(prompt)
- if !success {
- log.Printf("❌ [ASK_USER] Failed to send interactive prompt (connection closed)")
- return "", fmt.Errorf("failed to send prompt: connection closed")
- }
-
- log.Printf("✅ [ASK_USER] Sent interactive prompt: %s (id: %s, questions: %d, allow_skip: %v)",
- title, promptID, len(questions), allowSkip)
- log.Printf("⏳ [ASK_USER] Waiting for user response...")
-
- // Wait for user response (5 minute timeout)
- answers, skipped, err := waiterFunc(promptID, 5*time.Minute)
- if err != nil {
- log.Printf("❌ [ASK_USER] Error waiting for response: %v", err)
- return "", fmt.Errorf("failed to receive user response: %w", err)
- }
-
- // Check if user skipped
- if skipped {
- log.Printf("📋 [ASK_USER] User skipped the prompt")
- return "User skipped the prompt without providing answers.", nil
- }
-
- // Format the answers for the LLM
- result := map[string]interface{}{
- "status": "completed",
- "message": "User answered the prompt. Here are their responses:",
- "answers": make(map[string]interface{}),
- }
-
- // Convert answers to a format the LLM can understand
- for questionID, answer := range answers {
- // Find the question to get its label
- var questionLabel string
- for _, q := range questions {
- if q.ID == questionID {
- questionLabel = q.Label
- break
- }
- }
-
- answerData := map[string]interface{}{
- "question": questionLabel,
- "answer": answer.Value,
- }
-
- if answer.IsOther {
- answerData["is_custom_answer"] = true
- }
-
- result["answers"].(map[string]interface{})[questionID] = answerData
- }
-
- resultJSON, _ := json.MarshalIndent(result, "", " ")
- log.Printf("✅ [ASK_USER] Returning user's answers to LLM")
- return string(resultJSON), nil
-}
diff --git a/backend/internal/tools/brevo_tool.go b/backend/internal/tools/brevo_tool.go
deleted file mode 100644
index 428f9bef..00000000
--- a/backend/internal/tools/brevo_tool.go
+++ /dev/null
@@ -1,325 +0,0 @@
-package tools
-
-import (
- "bytes"
- "encoding/json"
- "fmt"
- "io"
- "net/http"
- "strings"
- "time"
-)
-
-// NewBrevoTool creates a Brevo (formerly SendInBlue) email sending tool
-func NewBrevoTool() *Tool {
- return &Tool{
- Name: "send_brevo_email",
- DisplayName: "Send Email (Brevo)",
- Description: `Send emails via Brevo (formerly SendInBlue) API. Supports transactional emails, marketing campaigns, and templates.
-
-Features:
-- Send to single or multiple recipients (to, cc, bcc)
-- HTML and plain text email bodies
-- Custom sender name and reply-to address
-- Template support with dynamic parameters
-- Attachment support via URL
-
-Authentication is handled automatically via configured Brevo credentials. Do NOT ask users for API keys.
-The sender email (from_email) can be configured in credentials as default, or overridden per-request.`,
- Icon: "Mail",
- Source: ToolSourceBuiltin,
- Category: "integration",
- Keywords: []string{"brevo", "sendinblue", "email", "send", "mail", "message", "notification", "newsletter", "transactional", "marketing"},
- Parameters: map[string]interface{}{
- "type": "object",
- "properties": map[string]interface{}{
- "credential_id": map[string]interface{}{
- "type": "string",
- "description": "INTERNAL: Auto-injected by system. Do not set manually.",
- },
- "to": map[string]interface{}{
- "type": "string",
- "description": "Recipient email address(es). For multiple recipients, separate with commas (e.g., 'user1@example.com, user2@example.com')",
- },
- "from_email": map[string]interface{}{
- "type": "string",
- "description": "Sender email address (optional). Must be a verified sender in Brevo. If not provided, uses the default from_email configured in credentials.",
- },
- "from_name": map[string]interface{}{
- "type": "string",
- "description": "Sender display name (optional, e.g., 'John Doe' or 'My Company')",
- },
- "subject": map[string]interface{}{
- "type": "string",
- "description": "Email subject line",
- },
- "text_content": map[string]interface{}{
- "type": "string",
- "description": "Plain text email body. Either text_content or html_content (or both) must be provided.",
- },
- "html_content": map[string]interface{}{
- "type": "string",
- "description": "HTML email body for rich formatting. Either text_content or html_content (or both) must be provided.",
- },
- "cc": map[string]interface{}{
- "type": "string",
- "description": "CC recipient(s). For multiple, separate with commas.",
- },
- "bcc": map[string]interface{}{
- "type": "string",
- "description": "BCC recipient(s). For multiple, separate with commas.",
- },
- "reply_to": map[string]interface{}{
- "type": "string",
- "description": "Reply-to email address (optional)",
- },
- "template_id": map[string]interface{}{
- "type": "number",
- "description": "Brevo template ID to use instead of html_content/text_content (optional)",
- },
- "params": map[string]interface{}{
- "type": "object",
- "description": "Template parameters as key-value pairs (optional, used with template_id)",
- },
- "tags": map[string]interface{}{
- "type": "array",
- "description": "Tags to categorize this email (optional)",
- "items": map[string]interface{}{
- "type": "string",
- },
- },
- },
- "required": []string{"to", "subject"},
- },
- Execute: executeBrevoEmail,
- }
-}
-
-// BrevoRecipient represents an email recipient
-type BrevoRecipient struct {
- Email string `json:"email"`
- Name string `json:"name,omitempty"`
-}
-
-// BrevoSender represents the email sender
-type BrevoSender struct {
- Email string `json:"email"`
- Name string `json:"name,omitempty"`
-}
-
-// BrevoRequest represents the Brevo API request
-type BrevoRequest struct {
- Sender BrevoSender `json:"sender"`
- To []BrevoRecipient `json:"to"`
- CC []BrevoRecipient `json:"cc,omitempty"`
- BCC []BrevoRecipient `json:"bcc,omitempty"`
- ReplyTo *BrevoRecipient `json:"replyTo,omitempty"`
- Subject string `json:"subject"`
- HTMLContent string `json:"htmlContent,omitempty"`
- TextContent string `json:"textContent,omitempty"`
- TemplateID int `json:"templateId,omitempty"`
- Params map[string]interface{} `json:"params,omitempty"`
- Tags []string `json:"tags,omitempty"`
-}
-
-func executeBrevoEmail(args map[string]interface{}) (string, error) {
- // Get all credential data first
- credData, credErr := GetCredentialData(args, "brevo")
-
- // Resolve API key from credential
- apiKey, err := ResolveAPIKey(args, "brevo", "api_key")
- if err != nil {
- return "", fmt.Errorf("failed to get Brevo API key: %w. Please configure Brevo credentials first.", err)
- }
-
- // Validate API key format (Brevo keys start with "xkeysib-")
- if !strings.HasPrefix(apiKey, "xkeysib-") {
- return "", fmt.Errorf("invalid Brevo API key format (should start with 'xkeysib-')")
- }
-
- // Extract required parameters
- toStr, ok := args["to"].(string)
- if !ok || toStr == "" {
- return "", fmt.Errorf("'to' email address is required")
- }
-
- // Get from_email - first check args, then fall back to credential data
- fromEmail, _ := args["from_email"].(string)
- if fromEmail == "" && credErr == nil && credData != nil {
- if credFromEmail, ok := credData["from_email"].(string); ok {
- fromEmail = credFromEmail
- }
- }
- if fromEmail == "" {
- return "", fmt.Errorf("'from_email' is required - either provide it in the request or configure a default in Brevo credentials")
- }
-
- subject, ok := args["subject"].(string)
- if !ok || subject == "" {
- return "", fmt.Errorf("'subject' is required")
- }
-
- // Extract content
- textContent, _ := args["text_content"].(string)
- htmlContent, _ := args["html_content"].(string)
- templateID := 0
- if tid, ok := args["template_id"].(float64); ok {
- templateID = int(tid)
- }
-
- // Require either content or template
- if textContent == "" && htmlContent == "" && templateID == 0 {
- return "", fmt.Errorf("either 'text_content', 'html_content', or 'template_id' is required")
- }
-
- // Parse recipient email addresses
- toRecipients := parseBrevoEmailList(toStr)
- if len(toRecipients) == 0 {
- return "", fmt.Errorf("at least one valid 'to' email address is required")
- }
-
- // Build sender
- sender := BrevoSender{Email: fromEmail}
- if fromName, ok := args["from_name"].(string); ok && fromName != "" {
- sender.Name = fromName
- }
-
- // Build request
- request := BrevoRequest{
- Sender: sender,
- To: toRecipients,
- Subject: subject,
- }
-
- // Add content or template
- if templateID > 0 {
- request.TemplateID = templateID
- if params, ok := args["params"].(map[string]interface{}); ok {
- request.Params = params
- }
- } else {
- if htmlContent != "" {
- request.HTMLContent = htmlContent
- }
- if textContent != "" {
- request.TextContent = textContent
- }
- }
-
- // Parse CC recipients
- if ccStr, ok := args["cc"].(string); ok && ccStr != "" {
- request.CC = parseBrevoEmailList(ccStr)
- }
-
- // Parse BCC recipients
- if bccStr, ok := args["bcc"].(string); ok && bccStr != "" {
- request.BCC = parseBrevoEmailList(bccStr)
- }
-
- // Add reply-to if provided
- if replyTo, ok := args["reply_to"].(string); ok && replyTo != "" {
- request.ReplyTo = &BrevoRecipient{Email: replyTo}
- }
-
- // Add tags if provided
- if tags, ok := args["tags"].([]interface{}); ok {
- for _, tag := range tags {
- if tagStr, ok := tag.(string); ok {
- request.Tags = append(request.Tags, tagStr)
- }
- }
- }
-
- // Serialize request
- jsonPayload, err := json.Marshal(request)
- if err != nil {
- return "", fmt.Errorf("failed to serialize request: %w", err)
- }
-
- // Create HTTP client
- client := &http.Client{
- Timeout: 30 * time.Second,
- }
-
- // Create request to Brevo API
- req, err := http.NewRequest("POST", "https://api.brevo.com/v3/smtp/email", bytes.NewBuffer(jsonPayload))
- if err != nil {
- return "", fmt.Errorf("failed to create request: %w", err)
- }
-
- req.Header.Set("Content-Type", "application/json")
- req.Header.Set("api-key", apiKey)
- req.Header.Set("Accept", "application/json")
-
- // Execute request
- resp, err := client.Do(req)
- if err != nil {
- return "", fmt.Errorf("request failed: %w", err)
- }
- defer resp.Body.Close()
-
- // Read response body
- respBody, err := io.ReadAll(resp.Body)
- if err != nil {
- return "", fmt.Errorf("failed to read response: %w", err)
- }
-
- // Brevo returns 201 Created on success
- success := resp.StatusCode == 201
-
- // Build result
- result := map[string]interface{}{
- "success": success,
- "status_code": resp.StatusCode,
- "email_sent": success,
- "recipients": len(toRecipients),
- "subject": subject,
- "from": fromEmail,
- }
-
- if len(request.CC) > 0 {
- result["cc_count"] = len(request.CC)
- }
- if len(request.BCC) > 0 {
- result["bcc_count"] = len(request.BCC)
- }
- if templateID > 0 {
- result["template_id"] = templateID
- }
-
- // Parse response
- if len(respBody) > 0 {
- var apiResp map[string]interface{}
- if err := json.Unmarshal(respBody, &apiResp); err == nil {
- if messageId, ok := apiResp["messageId"].(string); ok {
- result["message_id"] = messageId
- }
- if !success {
- result["error"] = apiResp
- }
- } else if !success {
- result["error"] = string(respBody)
- }
- }
-
- if success {
- result["message"] = fmt.Sprintf("Email sent successfully to %d recipient(s)", len(toRecipients))
- }
-
- jsonResult, _ := json.MarshalIndent(result, "", " ")
- return string(jsonResult), nil
-}
-
-// parseBrevoEmailList parses a comma-separated list of email addresses
-func parseBrevoEmailList(emailStr string) []BrevoRecipient {
- var recipients []BrevoRecipient
- parts := strings.Split(emailStr, ",")
- for _, part := range parts {
- email := strings.TrimSpace(part)
- if email != "" && strings.Contains(email, "@") {
- recipients = append(recipients, BrevoRecipient{Email: email})
- }
- }
- return recipients
-}
-
diff --git a/backend/internal/tools/calendly_tool.go b/backend/internal/tools/calendly_tool.go
deleted file mode 100644
index 14aabc1d..00000000
--- a/backend/internal/tools/calendly_tool.go
+++ /dev/null
@@ -1,323 +0,0 @@
-package tools
-
-import (
- "encoding/json"
- "fmt"
- "io"
- "net/http"
- "net/url"
- "time"
-)
-
-// NewCalendlyEventsTool creates a Calendly events listing tool
-func NewCalendlyEventsTool() *Tool {
- return &Tool{
- Name: "calendly_events",
- DisplayName: "Calendly Events",
- Description: `List scheduled events from Calendly.
-
-Returns event details including start/end times, invitees, and status.
-Authentication is handled automatically via configured Calendly API key.`,
- Icon: "Calendar",
- Source: ToolSourceBuiltin,
- Category: "integration",
- Keywords: []string{"calendly", "events", "meetings", "schedule", "calendar"},
- Parameters: map[string]interface{}{
- "type": "object",
- "properties": map[string]interface{}{
- "credential_id": map[string]interface{}{
- "type": "string",
- "description": "INTERNAL: Auto-injected by system.",
- },
- "status": map[string]interface{}{
- "type": "string",
- "description": "Filter by status: active or canceled",
- },
- "min_start_time": map[string]interface{}{
- "type": "string",
- "description": "Filter events starting after this time (ISO 8601 format)",
- },
- "max_start_time": map[string]interface{}{
- "type": "string",
- "description": "Filter events starting before this time (ISO 8601 format)",
- },
- "count": map[string]interface{}{
- "type": "number",
- "description": "Number of events to return (max 100)",
- },
- },
- "required": []string{},
- },
- Execute: executeCalendlyEvents,
- }
-}
-
-// NewCalendlyEventTypesTool creates a Calendly event types listing tool
-func NewCalendlyEventTypesTool() *Tool {
- return &Tool{
- Name: "calendly_event_types",
- DisplayName: "Calendly Event Types",
- Description: `List available event types from Calendly.
-
-Returns scheduling links and configuration for each event type.
-Authentication is handled automatically via configured Calendly API key.`,
- Icon: "Calendar",
- Source: ToolSourceBuiltin,
- Category: "integration",
- Keywords: []string{"calendly", "event", "types", "scheduling", "links"},
- Parameters: map[string]interface{}{
- "type": "object",
- "properties": map[string]interface{}{
- "credential_id": map[string]interface{}{
- "type": "string",
- "description": "INTERNAL: Auto-injected by system.",
- },
- "active": map[string]interface{}{
- "type": "boolean",
- "description": "Filter by active status",
- },
- "count": map[string]interface{}{
- "type": "number",
- "description": "Number of event types to return (max 100)",
- },
- },
- "required": []string{},
- },
- Execute: executeCalendlyEventTypes,
- }
-}
-
-// NewCalendlyInviteesTool creates a Calendly invitees listing tool
-func NewCalendlyInviteesTool() *Tool {
- return &Tool{
- Name: "calendly_invitees",
- DisplayName: "Calendly Invitees",
- Description: `List invitees for a specific Calendly event.
-
-Returns invitee details including name, email, and responses.
-Authentication is handled automatically via configured Calendly API key.`,
- Icon: "Users",
- Source: ToolSourceBuiltin,
- Category: "integration",
- Keywords: []string{"calendly", "invitees", "attendees", "participants"},
- Parameters: map[string]interface{}{
- "type": "object",
- "properties": map[string]interface{}{
- "credential_id": map[string]interface{}{
- "type": "string",
- "description": "INTERNAL: Auto-injected by system.",
- },
- "event_uri": map[string]interface{}{
- "type": "string",
- "description": "The scheduled event URI to get invitees for",
- },
- "status": map[string]interface{}{
- "type": "string",
- "description": "Filter by status: active or canceled",
- },
- "count": map[string]interface{}{
- "type": "number",
- "description": "Number of invitees to return (max 100)",
- },
- },
- "required": []string{"event_uri"},
- },
- Execute: executeCalendlyInvitees,
- }
-}
-
-func getCalendlyCurrentUser(apiKey string) (string, error) {
- req, _ := http.NewRequest("GET", "https://api.calendly.com/users/me", nil)
- req.Header.Set("Authorization", "Bearer "+apiKey)
- req.Header.Set("Content-Type", "application/json")
-
- client := &http.Client{Timeout: 30 * time.Second}
- resp, err := client.Do(req)
- if err != nil {
- return "", err
- }
- defer resp.Body.Close()
-
- body, _ := io.ReadAll(resp.Body)
- var result map[string]interface{}
- json.Unmarshal(body, &result)
-
- if resp.StatusCode >= 400 {
- return "", fmt.Errorf("failed to get current user")
- }
-
- if resource, ok := result["resource"].(map[string]interface{}); ok {
- if uri, ok := resource["uri"].(string); ok {
- return uri, nil
- }
- }
- return "", fmt.Errorf("user URI not found")
-}
-
-func executeCalendlyEvents(args map[string]interface{}) (string, error) {
- credData, err := GetCredentialData(args, "calendly")
- if err != nil {
- return "", fmt.Errorf("failed to get Calendly credentials: %w", err)
- }
-
- apiKey, _ := credData["api_key"].(string)
- if apiKey == "" {
- return "", fmt.Errorf("Calendly API key not configured")
- }
-
- userURI, err := getCalendlyCurrentUser(apiKey)
- if err != nil {
- return "", fmt.Errorf("failed to get current user: %w", err)
- }
-
- queryParams := url.Values{}
- queryParams.Set("user", userURI)
- if status, ok := args["status"].(string); ok && status != "" {
- queryParams.Set("status", status)
- }
- if minStart, ok := args["min_start_time"].(string); ok && minStart != "" {
- queryParams.Set("min_start_time", minStart)
- }
- if maxStart, ok := args["max_start_time"].(string); ok && maxStart != "" {
- queryParams.Set("max_start_time", maxStart)
- }
- if count, ok := args["count"].(float64); ok && count > 0 {
- queryParams.Set("count", fmt.Sprintf("%d", int(count)))
- }
-
- apiURL := "https://api.calendly.com/scheduled_events?" + queryParams.Encode()
- req, _ := http.NewRequest("GET", apiURL, nil)
- req.Header.Set("Authorization", "Bearer "+apiKey)
- req.Header.Set("Content-Type", "application/json")
-
- client := &http.Client{Timeout: 30 * time.Second}
- resp, err := client.Do(req)
- if err != nil {
- return "", fmt.Errorf("failed to send request: %w", err)
- }
- defer resp.Body.Close()
-
- body, _ := io.ReadAll(resp.Body)
- var result map[string]interface{}
- json.Unmarshal(body, &result)
-
- if resp.StatusCode >= 400 {
- errMsg := "unknown error"
- if msg, ok := result["message"].(string); ok {
- errMsg = msg
- }
- return "", fmt.Errorf("Calendly API error: %s", errMsg)
- }
-
- jsonResult, _ := json.MarshalIndent(result, "", " ")
- return string(jsonResult), nil
-}
-
-func executeCalendlyEventTypes(args map[string]interface{}) (string, error) {
- credData, err := GetCredentialData(args, "calendly")
- if err != nil {
- return "", fmt.Errorf("failed to get Calendly credentials: %w", err)
- }
-
- apiKey, _ := credData["api_key"].(string)
- if apiKey == "" {
- return "", fmt.Errorf("Calendly API key not configured")
- }
-
- userURI, err := getCalendlyCurrentUser(apiKey)
- if err != nil {
- return "", fmt.Errorf("failed to get current user: %w", err)
- }
-
- queryParams := url.Values{}
- queryParams.Set("user", userURI)
- if active, ok := args["active"].(bool); ok {
- queryParams.Set("active", fmt.Sprintf("%t", active))
- }
- if count, ok := args["count"].(float64); ok && count > 0 {
- queryParams.Set("count", fmt.Sprintf("%d", int(count)))
- }
-
- apiURL := "https://api.calendly.com/event_types?" + queryParams.Encode()
- req, _ := http.NewRequest("GET", apiURL, nil)
- req.Header.Set("Authorization", "Bearer "+apiKey)
- req.Header.Set("Content-Type", "application/json")
-
- client := &http.Client{Timeout: 30 * time.Second}
- resp, err := client.Do(req)
- if err != nil {
- return "", fmt.Errorf("failed to send request: %w", err)
- }
- defer resp.Body.Close()
-
- body, _ := io.ReadAll(resp.Body)
- var result map[string]interface{}
- json.Unmarshal(body, &result)
-
- if resp.StatusCode >= 400 {
- errMsg := "unknown error"
- if msg, ok := result["message"].(string); ok {
- errMsg = msg
- }
- return "", fmt.Errorf("Calendly API error: %s", errMsg)
- }
-
- jsonResult, _ := json.MarshalIndent(result, "", " ")
- return string(jsonResult), nil
-}
-
-func executeCalendlyInvitees(args map[string]interface{}) (string, error) {
- credData, err := GetCredentialData(args, "calendly")
- if err != nil {
- return "", fmt.Errorf("failed to get Calendly credentials: %w", err)
- }
-
- apiKey, _ := credData["api_key"].(string)
- if apiKey == "" {
- return "", fmt.Errorf("Calendly API key not configured")
- }
-
- eventURI, _ := args["event_uri"].(string)
- if eventURI == "" {
- return "", fmt.Errorf("'event_uri' is required")
- }
-
- queryParams := url.Values{}
- if status, ok := args["status"].(string); ok && status != "" {
- queryParams.Set("status", status)
- }
- if count, ok := args["count"].(float64); ok && count > 0 {
- queryParams.Set("count", fmt.Sprintf("%d", int(count)))
- }
-
- apiURL := eventURI + "/invitees"
- if len(queryParams) > 0 {
- apiURL += "?" + queryParams.Encode()
- }
-
- req, _ := http.NewRequest("GET", apiURL, nil)
- req.Header.Set("Authorization", "Bearer "+apiKey)
- req.Header.Set("Content-Type", "application/json")
-
- client := &http.Client{Timeout: 30 * time.Second}
- resp, err := client.Do(req)
- if err != nil {
- return "", fmt.Errorf("failed to send request: %w", err)
- }
- defer resp.Body.Close()
-
- body, _ := io.ReadAll(resp.Body)
- var result map[string]interface{}
- json.Unmarshal(body, &result)
-
- if resp.StatusCode >= 400 {
- errMsg := "unknown error"
- if msg, ok := result["message"].(string); ok {
- errMsg = msg
- }
- return "", fmt.Errorf("Calendly API error: %s", errMsg)
- }
-
- jsonResult, _ := json.MarshalIndent(result, "", " ")
- return string(jsonResult), nil
-}
diff --git a/backend/internal/tools/clickup_tool.go b/backend/internal/tools/clickup_tool.go
deleted file mode 100644
index 0304ebd5..00000000
--- a/backend/internal/tools/clickup_tool.go
+++ /dev/null
@@ -1,322 +0,0 @@
-package tools
-
-import (
- "bytes"
- "encoding/json"
- "fmt"
- "io"
- "net/http"
- "time"
-)
-
-// NewClickUpTasksTool creates a ClickUp tasks listing tool
-func NewClickUpTasksTool() *Tool {
- return &Tool{
- Name: "clickup_tasks",
- DisplayName: "ClickUp Tasks",
- Description: `List tasks from a ClickUp list.
-
-Returns task details including name, status, assignees, and due dates.
-Authentication is handled automatically via configured ClickUp API key.`,
- Icon: "CheckSquare",
- Source: ToolSourceBuiltin,
- Category: "integration",
- Keywords: []string{"clickup", "tasks", "list", "project", "management"},
- Parameters: map[string]interface{}{
- "type": "object",
- "properties": map[string]interface{}{
- "credential_id": map[string]interface{}{
- "type": "string",
- "description": "INTERNAL: Auto-injected by system.",
- },
- "list_id": map[string]interface{}{
- "type": "string",
- "description": "The ClickUp list ID to get tasks from",
- },
- "archived": map[string]interface{}{
- "type": "boolean",
- "description": "Include archived tasks",
- },
- "subtasks": map[string]interface{}{
- "type": "boolean",
- "description": "Include subtasks",
- },
- },
- "required": []string{"list_id"},
- },
- Execute: executeClickUpTasks,
- }
-}
-
-// NewClickUpCreateTaskTool creates a ClickUp task creation tool
-func NewClickUpCreateTaskTool() *Tool {
- return &Tool{
- Name: "clickup_create_task",
- DisplayName: "Create ClickUp Task",
- Description: `Create a new task in a ClickUp list.
-
-Supports setting name, description, status, priority, due date, assignees, and tags.
-Authentication is handled automatically via configured ClickUp API key.`,
- Icon: "Plus",
- Source: ToolSourceBuiltin,
- Category: "integration",
- Keywords: []string{"clickup", "task", "create", "new", "project"},
- Parameters: map[string]interface{}{
- "type": "object",
- "properties": map[string]interface{}{
- "credential_id": map[string]interface{}{
- "type": "string",
- "description": "INTERNAL: Auto-injected by system.",
- },
- "list_id": map[string]interface{}{
- "type": "string",
- "description": "The ClickUp list ID to create task in",
- },
- "name": map[string]interface{}{
- "type": "string",
- "description": "The task name",
- },
- "description": map[string]interface{}{
- "type": "string",
- "description": "The task description (supports markdown)",
- },
- "status": map[string]interface{}{
- "type": "string",
- "description": "The task status",
- },
- "priority": map[string]interface{}{
- "type": "number",
- "description": "Priority: 1 (Urgent), 2 (High), 3 (Normal), 4 (Low)",
- },
- "due_date": map[string]interface{}{
- "type": "number",
- "description": "Due date as Unix timestamp in milliseconds",
- },
- },
- "required": []string{"list_id", "name"},
- },
- Execute: executeClickUpCreateTask,
- }
-}
-
-// NewClickUpUpdateTaskTool creates a ClickUp task update tool
-func NewClickUpUpdateTaskTool() *Tool {
- return &Tool{
- Name: "clickup_update_task",
- DisplayName: "Update ClickUp Task",
- Description: `Update an existing ClickUp task.
-
-Can modify name, description, status, priority, due date, or archive status.
-Authentication is handled automatically via configured ClickUp API key.`,
- Icon: "Edit",
- Source: ToolSourceBuiltin,
- Category: "integration",
- Keywords: []string{"clickup", "task", "update", "edit", "modify"},
- Parameters: map[string]interface{}{
- "type": "object",
- "properties": map[string]interface{}{
- "credential_id": map[string]interface{}{
- "type": "string",
- "description": "INTERNAL: Auto-injected by system.",
- },
- "task_id": map[string]interface{}{
- "type": "string",
- "description": "The ClickUp task ID to update",
- },
- "name": map[string]interface{}{
- "type": "string",
- "description": "The new task name",
- },
- "description": map[string]interface{}{
- "type": "string",
- "description": "The new task description",
- },
- "status": map[string]interface{}{
- "type": "string",
- "description": "The new task status",
- },
- "priority": map[string]interface{}{
- "type": "number",
- "description": "Priority: 1 (Urgent), 2 (High), 3 (Normal), 4 (Low)",
- },
- },
- "required": []string{"task_id"},
- },
- Execute: executeClickUpUpdateTask,
- }
-}
-
-func executeClickUpTasks(args map[string]interface{}) (string, error) {
- credData, err := GetCredentialData(args, "clickup")
- if err != nil {
- return "", fmt.Errorf("failed to get ClickUp credentials: %w", err)
- }
-
- apiKey, _ := credData["api_key"].(string)
- if apiKey == "" {
- return "", fmt.Errorf("ClickUp API key not configured")
- }
-
- listID, _ := args["list_id"].(string)
- if listID == "" {
- return "", fmt.Errorf("'list_id' is required")
- }
-
- apiURL := fmt.Sprintf("https://api.clickup.com/api/v2/list/%s/task", listID)
- if archived, ok := args["archived"].(bool); ok && archived {
- apiURL += "?archived=true"
- }
-
- req, _ := http.NewRequest("GET", apiURL, nil)
- req.Header.Set("Authorization", apiKey)
- req.Header.Set("Content-Type", "application/json")
-
- client := &http.Client{Timeout: 30 * time.Second}
- resp, err := client.Do(req)
- if err != nil {
- return "", fmt.Errorf("failed to send request: %w", err)
- }
- defer resp.Body.Close()
-
- body, _ := io.ReadAll(resp.Body)
- var result map[string]interface{}
- json.Unmarshal(body, &result)
-
- if resp.StatusCode >= 400 {
- errMsg := "unknown error"
- if msg, ok := result["err"].(string); ok {
- errMsg = msg
- }
- return "", fmt.Errorf("ClickUp API error: %s", errMsg)
- }
-
- jsonResult, _ := json.MarshalIndent(result, "", " ")
- return string(jsonResult), nil
-}
-
-func executeClickUpCreateTask(args map[string]interface{}) (string, error) {
- credData, err := GetCredentialData(args, "clickup")
- if err != nil {
- return "", fmt.Errorf("failed to get ClickUp credentials: %w", err)
- }
-
- apiKey, _ := credData["api_key"].(string)
- if apiKey == "" {
- return "", fmt.Errorf("ClickUp API key not configured")
- }
-
- listID, _ := args["list_id"].(string)
- name, _ := args["name"].(string)
- if listID == "" || name == "" {
- return "", fmt.Errorf("'list_id' and 'name' are required")
- }
-
- payload := map[string]interface{}{"name": name}
- if desc, ok := args["description"].(string); ok && desc != "" {
- payload["description"] = desc
- }
- if status, ok := args["status"].(string); ok && status != "" {
- payload["status"] = status
- }
- if priority, ok := args["priority"].(float64); ok && priority > 0 {
- payload["priority"] = int(priority)
- }
- if dueDate, ok := args["due_date"].(float64); ok && dueDate > 0 {
- payload["due_date"] = int64(dueDate)
- }
-
- jsonBody, _ := json.Marshal(payload)
- apiURL := fmt.Sprintf("https://api.clickup.com/api/v2/list/%s/task", listID)
- req, _ := http.NewRequest("POST", apiURL, bytes.NewBuffer(jsonBody))
- req.Header.Set("Authorization", apiKey)
- req.Header.Set("Content-Type", "application/json")
-
- client := &http.Client{Timeout: 30 * time.Second}
- resp, err := client.Do(req)
- if err != nil {
- return "", fmt.Errorf("failed to send request: %w", err)
- }
- defer resp.Body.Close()
-
- body, _ := io.ReadAll(resp.Body)
- var result map[string]interface{}
- json.Unmarshal(body, &result)
-
- if resp.StatusCode >= 400 {
- errMsg := "unknown error"
- if msg, ok := result["err"].(string); ok {
- errMsg = msg
- }
- return "", fmt.Errorf("ClickUp API error: %s", errMsg)
- }
-
- output := map[string]interface{}{
- "success": true,
- "task": result,
- }
- jsonResult, _ := json.MarshalIndent(output, "", " ")
- return string(jsonResult), nil
-}
-
-func executeClickUpUpdateTask(args map[string]interface{}) (string, error) {
- credData, err := GetCredentialData(args, "clickup")
- if err != nil {
- return "", fmt.Errorf("failed to get ClickUp credentials: %w", err)
- }
-
- apiKey, _ := credData["api_key"].(string)
- if apiKey == "" {
- return "", fmt.Errorf("ClickUp API key not configured")
- }
-
- taskID, _ := args["task_id"].(string)
- if taskID == "" {
- return "", fmt.Errorf("'task_id' is required")
- }
-
- payload := make(map[string]interface{})
- if name, ok := args["name"].(string); ok && name != "" {
- payload["name"] = name
- }
- if desc, ok := args["description"].(string); ok && desc != "" {
- payload["description"] = desc
- }
- if status, ok := args["status"].(string); ok && status != "" {
- payload["status"] = status
- }
- if priority, ok := args["priority"].(float64); ok && priority > 0 {
- payload["priority"] = int(priority)
- }
-
- jsonBody, _ := json.Marshal(payload)
- apiURL := fmt.Sprintf("https://api.clickup.com/api/v2/task/%s", taskID)
- req, _ := http.NewRequest("PUT", apiURL, bytes.NewBuffer(jsonBody))
- req.Header.Set("Authorization", apiKey)
- req.Header.Set("Content-Type", "application/json")
-
- client := &http.Client{Timeout: 30 * time.Second}
- resp, err := client.Do(req)
- if err != nil {
- return "", fmt.Errorf("failed to send request: %w", err)
- }
- defer resp.Body.Close()
-
- body, _ := io.ReadAll(resp.Body)
- var result map[string]interface{}
- json.Unmarshal(body, &result)
-
- if resp.StatusCode >= 400 {
- errMsg := "unknown error"
- if msg, ok := result["err"].(string); ok {
- errMsg = msg
- }
- return "", fmt.Errorf("ClickUp API error: %s", errMsg)
- }
-
- output := map[string]interface{}{
- "success": true,
- "task": result,
- }
- jsonResult, _ := json.MarshalIndent(output, "", " ")
- return string(jsonResult), nil
-}
diff --git a/backend/internal/tools/composio_gmail_tool.go b/backend/internal/tools/composio_gmail_tool.go
deleted file mode 100644
index 3d1c8de4..00000000
--- a/backend/internal/tools/composio_gmail_tool.go
+++ /dev/null
@@ -1,1224 +0,0 @@
-package tools
-
-import (
- "bytes"
- "encoding/json"
- "fmt"
- "io"
- "log"
- "net/http"
- "net/url"
- "os"
- "regexp"
- "strings"
- "sync"
- "time"
-)
-
-// composioGmailRateLimiter implements per-user rate limiting for Composio Gmail API calls
-type composioGmailRateLimiter struct {
- requests map[string][]time.Time
- mutex sync.RWMutex
- maxCalls int
- window time.Duration
-}
-
-var globalGmailRateLimiter = &composioGmailRateLimiter{
- requests: make(map[string][]time.Time),
- maxCalls: 50,
- window: 1 * time.Minute,
-}
-
-// checkGmailRateLimit checks rate limit using user ID from args
-func checkGmailRateLimit(args map[string]interface{}) error {
- userID, ok := args["__user_id__"].(string)
- if !ok || userID == "" {
- log.Printf("⚠️ [GMAIL] No user ID for rate limiting")
- return nil
- }
-
- globalGmailRateLimiter.mutex.Lock()
- defer globalGmailRateLimiter.mutex.Unlock()
-
- now := time.Now()
- windowStart := now.Add(-globalGmailRateLimiter.window)
-
- timestamps := globalGmailRateLimiter.requests[userID]
- validTimestamps := []time.Time{}
- for _, ts := range timestamps {
- if ts.After(windowStart) {
- validTimestamps = append(validTimestamps, ts)
- }
- }
-
- if len(validTimestamps) >= globalGmailRateLimiter.maxCalls {
- return fmt.Errorf("rate limit exceeded: max %d requests per minute", globalGmailRateLimiter.maxCalls)
- }
-
- validTimestamps = append(validTimestamps, now)
- globalGmailRateLimiter.requests[userID] = validTimestamps
- return nil
-}
-
-// NewComposioGmailSendTool creates a tool for sending emails via Composio Gmail
-func NewComposioGmailSendTool() *Tool {
- return &Tool{
- Name: "gmail_send_email",
- DisplayName: "Gmail - Send Email",
- Description: `Send an email via Gmail using OAuth authentication.
-
-Features:
-- Send to multiple recipients (To, Cc, Bcc)
-- HTML or plain text body
-- Subject and body
-- OAuth authentication handled by Composio
-
-Use this to send emails from the authenticated user's Gmail account.`,
- Icon: "Mail",
- Source: ToolSourceComposio,
- Category: "integration",
- Keywords: []string{"gmail", "email", "send", "compose", "composio"},
- Parameters: map[string]interface{}{
- "type": "object",
- "properties": map[string]interface{}{
- "credential_id": map[string]interface{}{
- "type": "string",
- "description": "INTERNAL: Auto-injected by system. Do not set manually.",
- },
- "recipient_email": map[string]interface{}{
- "type": "string",
- "description": "Primary recipient email address",
- },
- "subject": map[string]interface{}{
- "type": "string",
- "description": "Email subject",
- },
- "body": map[string]interface{}{
- "type": "string",
- "description": "Email body (plain text or HTML)",
- },
- "is_html": map[string]interface{}{
- "type": "boolean",
- "description": "Set to true if body contains HTML (default: false)",
- },
- "cc": map[string]interface{}{
- "type": "array",
- "description": "Array of CC email addresses",
- "items": map[string]interface{}{
- "type": "string",
- },
- },
- "bcc": map[string]interface{}{
- "type": "array",
- "description": "Array of BCC email addresses",
- "items": map[string]interface{}{
- "type": "string",
- },
- },
- },
- "required": []string{},
- },
- Execute: executeComposioGmailSend,
- }
-}
-
-func executeComposioGmailSend(args map[string]interface{}) (string, error) {
- if err := checkGmailRateLimit(args); err != nil {
- return "", err
- }
-
- credData, err := GetCredentialData(args, "composio_gmail")
- if err != nil {
- return "", fmt.Errorf("failed to get Composio credentials: %w", err)
- }
-
- entityID, ok := credData["composio_entity_id"].(string)
- if !ok || entityID == "" {
- return "", fmt.Errorf("composio_entity_id not found in credentials")
- }
-
- composioAPIKey := os.Getenv("COMPOSIO_API_KEY")
- if composioAPIKey == "" {
- return "", fmt.Errorf("COMPOSIO_API_KEY environment variable not set")
- }
-
- // Build input
- input := map[string]interface{}{
- "user_id": "me",
- }
-
- if recipientEmail, ok := args["recipient_email"].(string); ok && recipientEmail != "" {
- input["recipient_email"] = recipientEmail
- }
- if subject, ok := args["subject"].(string); ok {
- input["subject"] = subject
- }
- if body, ok := args["body"].(string); ok {
- input["body"] = body
- }
- if isHTML, ok := args["is_html"].(bool); ok {
- input["is_html"] = isHTML
- }
- if cc, ok := args["cc"].([]interface{}); ok && len(cc) > 0 {
- input["cc"] = cc
- }
- if bcc, ok := args["bcc"].([]interface{}); ok && len(bcc) > 0 {
- input["bcc"] = bcc
- }
-
- payload := map[string]interface{}{
- "entityId": entityID,
- "appName": "gmail",
- "input": input,
- }
-
- return callComposioGmailAPI(composioAPIKey, entityID, "GMAIL_SEND_EMAIL", payload)
-}
-
-// NewComposioGmailFetchTool creates a tool for fetching/searching emails
-func NewComposioGmailFetchTool() *Tool {
- return &Tool{
- Name: "gmail_fetch_emails",
- DisplayName: "Gmail - Fetch Emails",
- Description: `Fetch and search emails from Gmail.
-
-Features:
-- Search with Gmail query syntax (e.g., "from:user@example.com", "is:unread")
-- Filter by labels
-- Pagination support
-- Returns email metadata and content
-- OAuth authentication handled by Composio
-
-Use this to list, search, and retrieve emails from Gmail inbox.`,
- Icon: "Mail",
- Source: ToolSourceComposio,
- Category: "integration",
- Keywords: []string{"gmail", "email", "fetch", "search", "list", "composio"},
- Parameters: map[string]interface{}{
- "type": "object",
- "properties": map[string]interface{}{
- "credential_id": map[string]interface{}{
- "type": "string",
- "description": "INTERNAL: Auto-injected by system. Do not set manually.",
- },
- "query": map[string]interface{}{
- "type": "string",
- "description": "Gmail search query (e.g., 'from:user@example.com is:unread')",
- },
- "max_results": map[string]interface{}{
- "type": "integer",
- "description": "Maximum number of emails to return (default: 10)",
- },
- "label_ids": map[string]interface{}{
- "type": "array",
- "description": "Filter by label IDs (e.g., ['INBOX', 'UNREAD'])",
- "items": map[string]interface{}{
- "type": "string",
- },
- },
- },
- "required": []string{},
- },
- Execute: executeComposioGmailFetch,
- }
-}
-
-func executeComposioGmailFetch(args map[string]interface{}) (string, error) {
- if err := checkGmailRateLimit(args); err != nil {
- return "", err
- }
-
- credData, err := GetCredentialData(args, "composio_gmail")
- if err != nil {
- return "", fmt.Errorf("failed to get Composio credentials: %w", err)
- }
-
- entityID, ok := credData["composio_entity_id"].(string)
- if !ok || entityID == "" {
- return "", fmt.Errorf("composio_entity_id not found in credentials")
- }
-
- composioAPIKey := os.Getenv("COMPOSIO_API_KEY")
- if composioAPIKey == "" {
- return "", fmt.Errorf("COMPOSIO_API_KEY environment variable not set")
- }
-
- // Build input
- input := map[string]interface{}{
- "user_id": "me",
- "include_payload": true,
- "verbose": true,
- }
-
- if query, ok := args["query"].(string); ok && query != "" {
- input["query"] = query
- }
- if maxResults, ok := args["max_results"].(float64); ok {
- input["max_results"] = int(maxResults)
- } else {
- input["max_results"] = 10
- }
- if labelIDs, ok := args["label_ids"].([]interface{}); ok && len(labelIDs) > 0 {
- input["label_ids"] = labelIDs
- }
-
- payload := map[string]interface{}{
- "entityId": entityID,
- "appName": "gmail",
- "input": input,
- }
-
- result, err := callComposioGmailAPI(composioAPIKey, entityID, "GMAIL_FETCH_EMAILS", payload)
- if err != nil {
- return "", err
- }
-
- // Parse and simplify the response for LLM consumption
- return simplifyGmailFetchResponse(result)
-}
-
-// NewComposioGmailGetMessageTool creates a tool for getting a specific email by ID
-func NewComposioGmailGetMessageTool() *Tool {
- return &Tool{
- Name: "gmail_get_message",
- DisplayName: "Gmail - Get Message",
- Description: `Get a specific email message by its ID.
-
-Features:
-- Retrieve full email content and metadata
-- Get headers, body, attachments info
-- OAuth authentication handled by Composio
-
-Use this to fetch details of a specific email when you have its message ID.`,
- Icon: "Mail",
- Source: ToolSourceComposio,
- Category: "integration",
- Keywords: []string{"gmail", "email", "get", "fetch", "message", "composio"},
- Parameters: map[string]interface{}{
- "type": "object",
- "properties": map[string]interface{}{
- "credential_id": map[string]interface{}{
- "type": "string",
- "description": "INTERNAL: Auto-injected by system. Do not set manually.",
- },
- "message_id": map[string]interface{}{
- "type": "string",
- "description": "The Gmail message ID",
- },
- },
- "required": []string{"message_id"},
- },
- Execute: executeComposioGmailGetMessage,
- }
-}
-
-func executeComposioGmailGetMessage(args map[string]interface{}) (string, error) {
- if err := checkGmailRateLimit(args); err != nil {
- return "", err
- }
-
- credData, err := GetCredentialData(args, "composio_gmail")
- if err != nil {
- return "", fmt.Errorf("failed to get Composio credentials: %w", err)
- }
-
- entityID, ok := credData["composio_entity_id"].(string)
- if !ok || entityID == "" {
- return "", fmt.Errorf("composio_entity_id not found in credentials")
- }
-
- messageID, _ := args["message_id"].(string)
- if messageID == "" {
- return "", fmt.Errorf("'message_id' is required")
- }
-
- composioAPIKey := os.Getenv("COMPOSIO_API_KEY")
- if composioAPIKey == "" {
- return "", fmt.Errorf("COMPOSIO_API_KEY environment variable not set")
- }
-
- payload := map[string]interface{}{
- "entityId": entityID,
- "appName": "gmail",
- "input": map[string]interface{}{
- "message_id": messageID,
- "user_id": "me",
- "format": "full",
- },
- }
-
- return callComposioGmailAPI(composioAPIKey, entityID, "GMAIL_FETCH_MESSAGE_BY_MESSAGE_ID", payload)
-}
-
-// NewComposioGmailReplyTool creates a tool for replying to email threads
-func NewComposioGmailReplyTool() *Tool {
- return &Tool{
- Name: "gmail_reply_to_thread",
- DisplayName: "Gmail - Reply to Thread",
- Description: `Reply to an existing email thread.
-
-Features:
-- Reply within existing conversation
-- Maintains thread continuity
-- Supports HTML or plain text
-- OAuth authentication handled by Composio
-
-Use this to send replies to existing email conversations.`,
- Icon: "Mail",
- Source: ToolSourceComposio,
- Category: "integration",
- Keywords: []string{"gmail", "email", "reply", "thread", "conversation", "composio"},
- Parameters: map[string]interface{}{
- "type": "object",
- "properties": map[string]interface{}{
- "credential_id": map[string]interface{}{
- "type": "string",
- "description": "INTERNAL: Auto-injected by system. Do not set manually.",
- },
- "thread_id": map[string]interface{}{
- "type": "string",
- "description": "The Gmail thread ID to reply to",
- },
- "message_body": map[string]interface{}{
- "type": "string",
- "description": "Reply message body",
- },
- "recipient_email": map[string]interface{}{
- "type": "string",
- "description": "Recipient email (optional if replying to thread)",
- },
- "is_html": map[string]interface{}{
- "type": "boolean",
- "description": "Set to true if body contains HTML",
- },
- },
- "required": []string{"thread_id"},
- },
- Execute: executeComposioGmailReply,
- }
-}
-
-func executeComposioGmailReply(args map[string]interface{}) (string, error) {
- if err := checkGmailRateLimit(args); err != nil {
- return "", err
- }
-
- credData, err := GetCredentialData(args, "composio_gmail")
- if err != nil {
- return "", fmt.Errorf("failed to get Composio credentials: %w", err)
- }
-
- entityID, ok := credData["composio_entity_id"].(string)
- if !ok || entityID == "" {
- return "", fmt.Errorf("composio_entity_id not found in credentials")
- }
-
- threadID, _ := args["thread_id"].(string)
- if threadID == "" {
- return "", fmt.Errorf("'thread_id' is required")
- }
-
- composioAPIKey := os.Getenv("COMPOSIO_API_KEY")
- if composioAPIKey == "" {
- return "", fmt.Errorf("COMPOSIO_API_KEY environment variable not set")
- }
-
- input := map[string]interface{}{
- "thread_id": threadID,
- "user_id": "me",
- }
-
- if messageBody, ok := args["message_body"].(string); ok && messageBody != "" {
- input["message_body"] = messageBody
- }
- if recipientEmail, ok := args["recipient_email"].(string); ok && recipientEmail != "" {
- input["recipient_email"] = recipientEmail
- }
- if isHTML, ok := args["is_html"].(bool); ok {
- input["is_html"] = isHTML
- }
-
- payload := map[string]interface{}{
- "entityId": entityID,
- "appName": "gmail",
- "input": input,
- }
-
- return callComposioGmailAPI(composioAPIKey, entityID, "GMAIL_REPLY_TO_THREAD", payload)
-}
-
-// NewComposioGmailCreateDraftTool creates a tool for creating email drafts
-func NewComposioGmailCreateDraftTool() *Tool {
- return &Tool{
- Name: "gmail_create_draft",
- DisplayName: "Gmail - Create Draft",
- Description: `Create an email draft in Gmail.
-
-Features:
-- Create drafts to send later
-- Supports To, Cc, Bcc
-- HTML or plain text
-- Can be edited before sending
-- OAuth authentication handled by Composio
-
-Use this to create email drafts that can be reviewed and sent later.`,
- Icon: "Mail",
- Source: ToolSourceComposio,
- Category: "integration",
- Keywords: []string{"gmail", "email", "draft", "compose", "save", "composio"},
- Parameters: map[string]interface{}{
- "type": "object",
- "properties": map[string]interface{}{
- "credential_id": map[string]interface{}{
- "type": "string",
- "description": "INTERNAL: Auto-injected by system. Do not set manually.",
- },
- "recipient_email": map[string]interface{}{
- "type": "string",
- "description": "Primary recipient email address (optional)",
- },
- "subject": map[string]interface{}{
- "type": "string",
- "description": "Email subject",
- },
- "body": map[string]interface{}{
- "type": "string",
- "description": "Email body",
- },
- "is_html": map[string]interface{}{
- "type": "boolean",
- "description": "Set to true if body contains HTML",
- },
- },
- "required": []string{},
- },
- Execute: executeComposioGmailCreateDraft,
- }
-}
-
-func executeComposioGmailCreateDraft(args map[string]interface{}) (string, error) {
- if err := checkGmailRateLimit(args); err != nil {
- return "", err
- }
-
- credData, err := GetCredentialData(args, "composio_gmail")
- if err != nil {
- return "", fmt.Errorf("failed to get Composio credentials: %w", err)
- }
-
- entityID, ok := credData["composio_entity_id"].(string)
- if !ok || entityID == "" {
- return "", fmt.Errorf("composio_entity_id not found in credentials")
- }
-
- composioAPIKey := os.Getenv("COMPOSIO_API_KEY")
- if composioAPIKey == "" {
- return "", fmt.Errorf("COMPOSIO_API_KEY environment variable not set")
- }
-
- input := map[string]interface{}{
- "user_id": "me",
- }
-
- if recipientEmail, ok := args["recipient_email"].(string); ok && recipientEmail != "" {
- input["recipient_email"] = recipientEmail
- }
- if subject, ok := args["subject"].(string); ok {
- input["subject"] = subject
- }
- if body, ok := args["body"].(string); ok {
- input["body"] = body
- }
- if isHTML, ok := args["is_html"].(bool); ok {
- input["is_html"] = isHTML
- }
-
- payload := map[string]interface{}{
- "entityId": entityID,
- "appName": "gmail",
- "input": input,
- }
-
- return callComposioGmailAPI(composioAPIKey, entityID, "GMAIL_CREATE_EMAIL_DRAFT", payload)
-}
-
-// NewComposioGmailSendDraftTool creates a tool for sending existing drafts
-func NewComposioGmailSendDraftTool() *Tool {
- return &Tool{
- Name: "gmail_send_draft",
- DisplayName: "Gmail - Send Draft",
- Description: `Send an existing email draft.
-
-Features:
-- Send previously created drafts
-- Draft is deleted after sending
-- OAuth authentication handled by Composio
-
-Use this to send drafts that were created earlier.`,
- Icon: "Mail",
- Source: ToolSourceComposio,
- Category: "integration",
- Keywords: []string{"gmail", "email", "draft", "send", "composio"},
- Parameters: map[string]interface{}{
- "type": "object",
- "properties": map[string]interface{}{
- "credential_id": map[string]interface{}{
- "type": "string",
- "description": "INTERNAL: Auto-injected by system. Do not set manually.",
- },
- "draft_id": map[string]interface{}{
- "type": "string",
- "description": "The Gmail draft ID to send",
- },
- },
- "required": []string{"draft_id"},
- },
- Execute: executeComposioGmailSendDraft,
- }
-}
-
-func executeComposioGmailSendDraft(args map[string]interface{}) (string, error) {
- if err := checkGmailRateLimit(args); err != nil {
- return "", err
- }
-
- credData, err := GetCredentialData(args, "composio_gmail")
- if err != nil {
- return "", fmt.Errorf("failed to get Composio credentials: %w", err)
- }
-
- entityID, ok := credData["composio_entity_id"].(string)
- if !ok || entityID == "" {
- return "", fmt.Errorf("composio_entity_id not found in credentials")
- }
-
- draftID, _ := args["draft_id"].(string)
- if draftID == "" {
- return "", fmt.Errorf("'draft_id' is required")
- }
-
- composioAPIKey := os.Getenv("COMPOSIO_API_KEY")
- if composioAPIKey == "" {
- return "", fmt.Errorf("COMPOSIO_API_KEY environment variable not set")
- }
-
- payload := map[string]interface{}{
- "entityId": entityID,
- "appName": "gmail",
- "input": map[string]interface{}{
- "draft_id": draftID,
- "user_id": "me",
- },
- }
-
- return callComposioGmailAPI(composioAPIKey, entityID, "GMAIL_SEND_DRAFT", payload)
-}
-
-// NewComposioGmailListDraftsTool creates a tool for listing drafts
-func NewComposioGmailListDraftsTool() *Tool {
- return &Tool{
- Name: "gmail_list_drafts",
- DisplayName: "Gmail - List Drafts",
- Description: `List all email drafts in Gmail.
-
-Features:
-- List all saved drafts
-- Pagination support
-- Returns draft IDs and metadata
-- OAuth authentication handled by Composio
-
-Use this to view all saved email drafts.`,
- Icon: "Mail",
- Source: ToolSourceComposio,
- Category: "integration",
- Keywords: []string{"gmail", "email", "draft", "list", "composio"},
- Parameters: map[string]interface{}{
- "type": "object",
- "properties": map[string]interface{}{
- "credential_id": map[string]interface{}{
- "type": "string",
- "description": "INTERNAL: Auto-injected by system. Do not set manually.",
- },
- "max_results": map[string]interface{}{
- "type": "integer",
- "description": "Maximum number of drafts to return (default: 10)",
- },
- },
- "required": []string{},
- },
- Execute: executeComposioGmailListDrafts,
- }
-}
-
-func executeComposioGmailListDrafts(args map[string]interface{}) (string, error) {
- if err := checkGmailRateLimit(args); err != nil {
- return "", err
- }
-
- credData, err := GetCredentialData(args, "composio_gmail")
- if err != nil {
- return "", fmt.Errorf("failed to get Composio credentials: %w", err)
- }
-
- entityID, ok := credData["composio_entity_id"].(string)
- if !ok || entityID == "" {
- return "", fmt.Errorf("composio_entity_id not found in credentials")
- }
-
- composioAPIKey := os.Getenv("COMPOSIO_API_KEY")
- if composioAPIKey == "" {
- return "", fmt.Errorf("COMPOSIO_API_KEY environment variable not set")
- }
-
- input := map[string]interface{}{
- "user_id": "me",
- "verbose": true,
- }
-
- if maxResults, ok := args["max_results"].(float64); ok {
- input["max_results"] = int(maxResults)
- } else {
- input["max_results"] = 10
- }
-
- payload := map[string]interface{}{
- "entityId": entityID,
- "appName": "gmail",
- "input": input,
- }
-
- return callComposioGmailAPI(composioAPIKey, entityID, "GMAIL_LIST_DRAFTS", payload)
-}
-
-// NewComposioGmailAddLabelTool creates a tool for managing email labels
-func NewComposioGmailAddLabelTool() *Tool {
- return &Tool{
- Name: "gmail_add_label",
- DisplayName: "Gmail - Add/Remove Labels",
- Description: `Add or remove labels from an email message.
-
-Features:
-- Add labels to organize emails
-- Remove labels from emails
-- Use system labels (INBOX, UNREAD, STARRED, etc.)
-- Use custom labels
-- OAuth authentication handled by Composio
-
-Use this to organize emails with labels (categories/tags).`,
- Icon: "Mail",
- Source: ToolSourceComposio,
- Category: "integration",
- Keywords: []string{"gmail", "email", "label", "tag", "organize", "composio"},
- Parameters: map[string]interface{}{
- "type": "object",
- "properties": map[string]interface{}{
- "credential_id": map[string]interface{}{
- "type": "string",
- "description": "INTERNAL: Auto-injected by system. Do not set manually.",
- },
- "message_id": map[string]interface{}{
- "type": "string",
- "description": "The Gmail message ID",
- },
- "add_label_ids": map[string]interface{}{
- "type": "array",
- "description": "Array of label IDs to add (e.g., ['INBOX', 'STARRED'])",
- "items": map[string]interface{}{
- "type": "string",
- },
- },
- "remove_label_ids": map[string]interface{}{
- "type": "array",
- "description": "Array of label IDs to remove (e.g., ['UNREAD'])",
- "items": map[string]interface{}{
- "type": "string",
- },
- },
- },
- "required": []string{"message_id"},
- },
- Execute: executeComposioGmailAddLabel,
- }
-}
-
-func executeComposioGmailAddLabel(args map[string]interface{}) (string, error) {
- if err := checkGmailRateLimit(args); err != nil {
- return "", err
- }
-
- credData, err := GetCredentialData(args, "composio_gmail")
- if err != nil {
- return "", fmt.Errorf("failed to get Composio credentials: %w", err)
- }
-
- entityID, ok := credData["composio_entity_id"].(string)
- if !ok || entityID == "" {
- return "", fmt.Errorf("composio_entity_id not found in credentials")
- }
-
- messageID, _ := args["message_id"].(string)
- if messageID == "" {
- return "", fmt.Errorf("'message_id' is required")
- }
-
- composioAPIKey := os.Getenv("COMPOSIO_API_KEY")
- if composioAPIKey == "" {
- return "", fmt.Errorf("COMPOSIO_API_KEY environment variable not set")
- }
-
- input := map[string]interface{}{
- "message_id": messageID,
- "user_id": "me",
- }
-
- if addLabelIDs, ok := args["add_label_ids"].([]interface{}); ok && len(addLabelIDs) > 0 {
- input["add_label_ids"] = addLabelIDs
- }
- if removeLabelIDs, ok := args["remove_label_ids"].([]interface{}); ok && len(removeLabelIDs) > 0 {
- input["remove_label_ids"] = removeLabelIDs
- }
-
- payload := map[string]interface{}{
- "entityId": entityID,
- "appName": "gmail",
- "input": input,
- }
-
- return callComposioGmailAPI(composioAPIKey, entityID, "GMAIL_ADD_LABEL_TO_EMAIL", payload)
-}
-
-// NewComposioGmailListLabelsTool creates a tool for listing all labels
-func NewComposioGmailListLabelsTool() *Tool {
- return &Tool{
- Name: "gmail_list_labels",
- DisplayName: "Gmail - List Labels",
- Description: `List all Gmail labels (system and custom).
-
-Features:
-- List all available labels
-- Includes system labels (INBOX, SENT, TRASH, etc.)
-- Includes user-created custom labels
-- Returns label IDs and names
-- OAuth authentication handled by Composio
-
-Use this to discover available labels for organizing emails.`,
- Icon: "Mail",
- Source: ToolSourceComposio,
- Category: "integration",
- Keywords: []string{"gmail", "email", "label", "list", "categories", "composio"},
- Parameters: map[string]interface{}{
- "type": "object",
- "properties": map[string]interface{}{
- "credential_id": map[string]interface{}{
- "type": "string",
- "description": "INTERNAL: Auto-injected by system. Do not set manually.",
- },
- },
- "required": []string{},
- },
- Execute: executeComposioGmailListLabels,
- }
-}
-
-func executeComposioGmailListLabels(args map[string]interface{}) (string, error) {
- if err := checkGmailRateLimit(args); err != nil {
- return "", err
- }
-
- credData, err := GetCredentialData(args, "composio_gmail")
- if err != nil {
- return "", fmt.Errorf("failed to get Composio credentials: %w", err)
- }
-
- entityID, ok := credData["composio_entity_id"].(string)
- if !ok || entityID == "" {
- return "", fmt.Errorf("composio_entity_id not found in credentials")
- }
-
- composioAPIKey := os.Getenv("COMPOSIO_API_KEY")
- if composioAPIKey == "" {
- return "", fmt.Errorf("COMPOSIO_API_KEY environment variable not set")
- }
-
- payload := map[string]interface{}{
- "entityId": entityID,
- "appName": "gmail",
- "input": map[string]interface{}{
- "user_id": "me",
- },
- }
-
- return callComposioGmailAPI(composioAPIKey, entityID, "GMAIL_LIST_LABELS", payload)
-}
-
-// NewComposioGmailTrashTool creates a tool for moving emails to trash
-func NewComposioGmailTrashTool() *Tool {
- return &Tool{
- Name: "gmail_move_to_trash",
- DisplayName: "Gmail - Move to Trash",
- Description: `Move an email message to trash.
-
-Features:
-- Moves message to Trash (not permanent deletion)
-- Can be recovered from Trash
-- OAuth authentication handled by Composio
-
-Use this to delete emails (they go to Trash first).`,
- Icon: "Mail",
- Source: ToolSourceComposio,
- Category: "integration",
- Keywords: []string{"gmail", "email", "trash", "delete", "remove", "composio"},
- Parameters: map[string]interface{}{
- "type": "object",
- "properties": map[string]interface{}{
- "credential_id": map[string]interface{}{
- "type": "string",
- "description": "INTERNAL: Auto-injected by system. Do not set manually.",
- },
- "message_id": map[string]interface{}{
- "type": "string",
- "description": "The Gmail message ID to trash",
- },
- },
- "required": []string{"message_id"},
- },
- Execute: executeComposioGmailTrash,
- }
-}
-
-func executeComposioGmailTrash(args map[string]interface{}) (string, error) {
- if err := checkGmailRateLimit(args); err != nil {
- return "", err
- }
-
- credData, err := GetCredentialData(args, "composio_gmail")
- if err != nil {
- return "", fmt.Errorf("failed to get Composio credentials: %w", err)
- }
-
- entityID, ok := credData["composio_entity_id"].(string)
- if !ok || entityID == "" {
- return "", fmt.Errorf("composio_entity_id not found in credentials")
- }
-
- messageID, _ := args["message_id"].(string)
- if messageID == "" {
- return "", fmt.Errorf("'message_id' is required")
- }
-
- composioAPIKey := os.Getenv("COMPOSIO_API_KEY")
- if composioAPIKey == "" {
- return "", fmt.Errorf("COMPOSIO_API_KEY environment variable not set")
- }
-
- payload := map[string]interface{}{
- "entityId": entityID,
- "appName": "gmail",
- "input": map[string]interface{}{
- "message_id": messageID,
- "user_id": "me",
- },
- }
-
- return callComposioGmailAPI(composioAPIKey, entityID, "GMAIL_MOVE_TO_TRASH", payload)
-}
-
-// callComposioGmailAPI makes a v2 API call to Composio for Gmail actions
-func callComposioGmailAPI(apiKey string, entityID string, action string, payload map[string]interface{}) (string, error) {
- // Get connected account ID
- connectedAccountID, err := getGmailConnectedAccountID(apiKey, entityID, "gmail")
- if err != nil {
- return "", fmt.Errorf("failed to get connected account: %w", err)
- }
-
- url := "https://backend.composio.dev/api/v2/actions/" + action + "/execute"
-
- v2Payload := map[string]interface{}{
- "connectedAccountId": connectedAccountID,
- "input": payload["input"],
- }
-
- jsonData, err := json.Marshal(v2Payload)
- if err != nil {
- return "", fmt.Errorf("failed to marshal request: %w", err)
- }
-
- log.Printf("🔍 [GMAIL] Action: %s, ConnectedAccount: %s", action, maskSensitiveID(connectedAccountID))
-
- req, err := http.NewRequest("POST", url, bytes.NewBuffer(jsonData))
- if err != nil {
- return "", fmt.Errorf("failed to create request: %w", err)
- }
-
- req.Header.Set("Content-Type", "application/json")
- req.Header.Set("x-api-key", apiKey)
-
- client := &http.Client{Timeout: 30 * time.Second}
- resp, err := client.Do(req)
- if err != nil {
- return "", fmt.Errorf("failed to send request: %w", err)
- }
- defer resp.Body.Close()
-
- // ✅ SECURITY FIX: Parse and log rate limit headers
- parseGmailRateLimitHeaders(resp.Header, action)
-
- respBody, _ := io.ReadAll(resp.Body)
-
- if resp.StatusCode >= 400 {
- log.Printf("❌ [GMAIL] API error (status %d) for action %s", resp.StatusCode, action)
- log.Printf("❌ [GMAIL] Composio error response: %s", string(respBody))
- log.Printf("❌ [GMAIL] Request payload: %s", string(jsonData))
-
- // Handle rate limiting with specific error
- if resp.StatusCode == 429 {
- retryAfter := resp.Header.Get("Retry-After")
- if retryAfter != "" {
- log.Printf("⚠️ [GMAIL] Rate limited, retry after: %s seconds", retryAfter)
- return "", fmt.Errorf("rate limit exceeded, retry after %s seconds", retryAfter)
- }
- return "", fmt.Errorf("rate limit exceeded, please try again later")
- }
-
- if resp.StatusCode >= 500 {
- return "", fmt.Errorf("external service error (status %d)", resp.StatusCode)
- }
- return "", fmt.Errorf("invalid request (status %d): check parameters and permissions", resp.StatusCode)
- }
-
- var apiResponse map[string]interface{}
- if err := json.Unmarshal(respBody, &apiResponse); err != nil {
- return string(respBody), nil
- }
-
- result, _ := json.MarshalIndent(apiResponse, "", " ")
- return string(result), nil
-}
-
-// getGmailConnectedAccountID retrieves the connected account ID from Composio v3 API
-func getGmailConnectedAccountID(apiKey string, userID string, appName string) (string, error) {
- baseURL := "https://backend.composio.dev/api/v3/connected_accounts"
- params := url.Values{}
- params.Add("user_ids", userID)
- fullURL := baseURL + "?" + params.Encode()
-
- req, err := http.NewRequest("GET", fullURL, nil)
- if err != nil {
- return "", fmt.Errorf("failed to create request: %w", err)
- }
-
- req.Header.Set("x-api-key", apiKey)
-
- client := &http.Client{Timeout: 30 * time.Second}
- resp, err := client.Do(req)
- if err != nil {
- return "", fmt.Errorf("failed to fetch connected accounts: %w", err)
- }
- defer resp.Body.Close()
-
- respBody, _ := io.ReadAll(resp.Body)
-
- if resp.StatusCode >= 400 {
- return "", fmt.Errorf("Composio API error (status %d): %s", resp.StatusCode, string(respBody))
- }
-
- // Parse v3 response with proper structure including deprecated.uuid
- var response struct {
- Items []struct {
- ID string `json:"id"`
- Toolkit struct {
- Slug string `json:"slug"`
- } `json:"toolkit"`
- Deprecated struct {
- UUID string `json:"uuid"`
- } `json:"deprecated"`
- } `json:"items"`
- }
- if err := json.Unmarshal(respBody, &response); err != nil {
- return "", fmt.Errorf("failed to parse response: %w", err)
- }
-
- // Find the connected account for this app
- for _, account := range response.Items {
- if account.Toolkit.Slug == appName {
- // v2 execution endpoint needs the old UUID, not the new nano ID
- // Check if deprecated.uuid exists (for v2 compatibility)
- if account.Deprecated.UUID != "" {
- return account.Deprecated.UUID, nil
- }
- // Fall back to nano ID if UUID not available
- return account.ID, nil
- }
- }
-
- return "", fmt.Errorf("no %s connection found for user. Please connect your Gmail account first", appName)
-}
-
-// stripHTMLAndClean removes HTML tags and cleans up whitespace from text
-func stripHTMLAndClean(html string) string {
- // Remove HTML tags using regex
- re := regexp.MustCompile(`<[^>]*>`)
- text := re.ReplaceAllString(html, "")
-
- // Decode HTML entities like , &, etc.
- text = strings.ReplaceAll(text, " ", " ")
- text = strings.ReplaceAll(text, "&", "&")
- text = strings.ReplaceAll(text, "<", "<")
- text = strings.ReplaceAll(text, ">", ">")
- text = strings.ReplaceAll(text, """, "\"")
- text = strings.ReplaceAll(text, "'", "'")
- text = strings.ReplaceAll(text, "'", "'")
- text = strings.ReplaceAll(text, "\u00a0", " ") // Non-breaking space
- text = strings.ReplaceAll(text, "\u200b", "") // Zero-width space
- text = strings.ReplaceAll(text, "\u200c", "") // Zero-width non-joiner
- text = strings.ReplaceAll(text, "\u200d", "") // Zero-width joiner
- text = strings.ReplaceAll(text, "\ufeff", "") // Zero-width no-break space
- text = strings.ReplaceAll(text, "\r", "") // Remove carriage returns
- text = strings.ReplaceAll(text, "\u003e", " ") // Remove greater-than symbol
- text = strings.ReplaceAll(text, "\u003c", " ") // Remove less-than symbol
- text = strings.ReplaceAll(text, "\u0026", " ") // Remove ampersand symbol
- text = strings.ReplaceAll(text, "\u00ab", " ") // Remove left-pointing double angle quotation mark
- text = strings.ReplaceAll(text, "\u00bb", " ") // Remove right-pointing double angle quotation mark
- text = strings.ReplaceAll(text, "\u0026", "") // Remove ampersand symbol
-
-
- // Remove excessive whitespace
- lines := strings.Split(text, "\n")
- var cleanedLines []string
- for _, line := range lines {
- line = strings.TrimSpace(line)
- if line != "" {
- cleanedLines = append(cleanedLines, line)
- }
- }
-
- text = strings.Join(cleanedLines, "\n")
-
- // Collapse multiple spaces into one
- re = regexp.MustCompile(`\s+`)
- text = re.ReplaceAllString(text, " ")
-
- // Final trim
- text = strings.TrimSpace(text)
-
- return text
-}
-
-// simplifyGmailFetchResponse parses the raw Composio Gmail response and returns a simplified, LLM-friendly format
-func simplifyGmailFetchResponse(rawResponse string) (string, error) {
- var response map[string]interface{}
- if err := json.Unmarshal([]byte(rawResponse), &response); err != nil {
- // If parsing fails, return raw response
- return rawResponse, nil
- }
-
- // Extract the data.messages array
- data, ok := response["data"].(map[string]interface{})
- if !ok {
- return rawResponse, nil
- }
-
- messages, ok := data["messages"].([]interface{})
- if !ok || len(messages) == 0 {
- return "No emails found matching your criteria.", nil
- }
-
- // Build simplified response
- simplified := make([]map[string]interface{}, 0, len(messages))
-
- for _, msg := range messages {
- msgMap, ok := msg.(map[string]interface{})
- if !ok {
- continue
- }
-
- simplifiedMsg := make(map[string]interface{})
-
- // Extract essential fields
- if messageID, ok := msgMap["messageId"].(string); ok {
- simplifiedMsg["message_id"] = messageID
- }
- if threadID, ok := msgMap["threadId"].(string); ok {
- simplifiedMsg["thread_id"] = threadID
- }
- if subject, ok := msgMap["subject"].(string); ok {
- simplifiedMsg["subject"] = subject
- }
- if from, ok := msgMap["from"].(string); ok {
- simplifiedMsg["from"] = from
- }
- if date, ok := msgMap["date"].(string); ok {
- simplifiedMsg["date"] = date
- }
- if snippet, ok := msgMap["snippet"].(string); ok {
- simplifiedMsg["snippet"] = snippet
- }
-
- // Extract message text (prefer full text over snippet)
- // Strip HTML tags and clean up whitespace
- if messageText, ok := msgMap["messageText"].(string); ok && messageText != "" {
- simplifiedMsg["message"] = stripHTMLAndClean(messageText)
- } else if snippet, ok := msgMap["snippet"].(string); ok {
- simplifiedMsg["message"] = snippet
- }
-
- // Include labels only if they contain useful info (skip internal IDs)
- if labels, ok := msgMap["labelIds"].([]interface{}); ok {
- readableLabels := []string{}
- for _, label := range labels {
- if labelStr, ok := label.(string); ok {
- // Only include readable labels (INBOX, UNREAD, IMPORTANT, etc.)
- if labelStr == "INBOX" || labelStr == "UNREAD" || labelStr == "IMPORTANT" ||
- labelStr == "STARRED" || labelStr == "SENT" || labelStr == "DRAFT" {
- readableLabels = append(readableLabels, labelStr)
- }
- }
- }
- if len(readableLabels) > 0 {
- simplifiedMsg["labels"] = readableLabels
- }
- }
-
- simplified = append(simplified, simplifiedMsg)
- }
-
- // Format as JSON for LLM
- result, err := json.MarshalIndent(map[string]interface{}{
- "count": len(simplified),
- "messages": simplified,
- }, "", " ")
-
- if err != nil {
- return rawResponse, nil
- }
-
- return string(result), nil
-}
-
-// parseGmailRateLimitHeaders parses and logs rate limit headers from Gmail API responses
-func parseGmailRateLimitHeaders(headers http.Header, action string) {
- limit := headers.Get("X-RateLimit-Limit")
- remaining := headers.Get("X-RateLimit-Remaining")
- reset := headers.Get("X-RateLimit-Reset")
-
- if limit != "" || remaining != "" || reset != "" {
- log.Printf("📊 [GMAIL] Rate limits for %s - Limit: %s, Remaining: %s, Reset: %s",
- action, limit, remaining, reset)
-
- // Warning if approaching rate limit
- if remaining != "" && limit != "" {
- remainingInt := 0
- limitInt := 0
- fmt.Sscanf(remaining, "%d", &remainingInt)
- fmt.Sscanf(limit, "%d", &limitInt)
-
- if limitInt > 0 {
- percentRemaining := float64(remainingInt) / float64(limitInt) * 100
- if percentRemaining < 20 {
- log.Printf("⚠️ [GMAIL] Rate limit warning: only %.1f%% remaining (%d/%d)",
- percentRemaining, remainingInt, limitInt)
- }
- }
- }
- }
-}
diff --git a/backend/internal/tools/composio_googlesheets_tool.go b/backend/internal/tools/composio_googlesheets_tool.go
deleted file mode 100644
index e63f5807..00000000
--- a/backend/internal/tools/composio_googlesheets_tool.go
+++ /dev/null
@@ -1,1401 +0,0 @@
-package tools
-
-import (
- "bytes"
- "encoding/json"
- "fmt"
- "io"
- "log"
- "net/http"
- "net/url"
- "os"
- "sync"
- "time"
-)
-
-// maskSensitiveID masks a sensitive ID for safe logging (e.g., "acc_abc123xyz" -> "acc_...xyz")
-func maskSensitiveID(id string) string {
- if len(id) <= 8 {
- return "***"
- }
- return id[:4] + "..." + id[len(id)-4:]
-}
-
-// composioRateLimiter implements per-user rate limiting for Composio API calls
-type composioRateLimiter struct {
- requests map[string][]time.Time // userID -> timestamps
- mutex sync.RWMutex
- maxCalls int // max calls per window
- window time.Duration // time window
-}
-
-var globalComposioRateLimiter = &composioRateLimiter{
- requests: make(map[string][]time.Time),
- maxCalls: 50, // 50 calls per minute per user
- window: 1 * time.Minute,
-}
-
-// checkRateLimit checks if user has exceeded rate limit
-func (rl *composioRateLimiter) checkRateLimit(userID string) error {
- rl.mutex.Lock()
- defer rl.mutex.Unlock()
-
- now := time.Now()
- windowStart := now.Add(-rl.window)
-
- // Get user's request history
- timestamps := rl.requests[userID]
-
- // Remove timestamps outside window
- validTimestamps := []time.Time{}
- for _, ts := range timestamps {
- if ts.After(windowStart) {
- validTimestamps = append(validTimestamps, ts)
- }
- }
-
- // Check if limit exceeded
- if len(validTimestamps) >= rl.maxCalls {
- return fmt.Errorf("rate limit exceeded: max %d requests per minute", rl.maxCalls)
- }
-
- // Add current timestamp
- validTimestamps = append(validTimestamps, now)
- rl.requests[userID] = validTimestamps
-
- return nil
-}
-
-// checkComposioRateLimit checks rate limit using user ID from args
-func checkComposioRateLimit(args map[string]interface{}) error {
- // Extract user ID from args (injected by chat service)
- userID, ok := args["__user_id__"].(string)
- if !ok || userID == "" {
- // If no user ID, allow but log warning
- log.Printf("⚠️ [COMPOSIO] No user ID for rate limiting")
- return nil
- }
-
- return globalComposioRateLimiter.checkRateLimit(userID)
-}
-
-// NewComposioGoogleSheetsReadTool creates a tool for reading Google Sheets via Composio
-func NewComposioGoogleSheetsReadTool() *Tool {
- return &Tool{
- Name: "googlesheets_read",
- DisplayName: "Google Sheets - Read Range",
- Description: `Read data from a Google Sheets range via Composio.
-
-Features:
-- Read any range from a spreadsheet (e.g., "Sheet1!A1:D10")
-- Returns data as 2D array
-- Supports named sheets and ranges
-- OAuth authentication handled by Composio
-
-Use this to fetch data from Google Sheets for processing, analysis, or automation workflows.`,
- Icon: "FileSpreadsheet",
- Source: ToolSourceComposio,
- Category: "integration",
- Keywords: []string{"google", "sheets", "spreadsheet", "read", "data", "excel", "composio"},
- Parameters: map[string]interface{}{
- "type": "object",
- "properties": map[string]interface{}{
- "credential_id": map[string]interface{}{
- "type": "string",
- "description": "INTERNAL: Auto-injected by system. Do not set manually.",
- },
- "spreadsheet_id": map[string]interface{}{
- "type": "string",
- "description": "Google Sheets spreadsheet ID (from the URL)",
- },
- "range": map[string]interface{}{
- "type": "string",
- "description": "Range to read (e.g., 'Sheet1!A1:D10' or 'Sheet1!A:D')",
- },
- },
- "required": []string{"spreadsheet_id", "range"},
- },
- Execute: executeComposioGoogleSheetsRead,
- }
-}
-
-// NewComposioGoogleSheetsWriteTool creates a tool for writing to Google Sheets via Composio
-func NewComposioGoogleSheetsWriteTool() *Tool {
- return &Tool{
- Name: "googlesheets_write",
- DisplayName: "Google Sheets - Write Range",
- Description: `Write data to a Google Sheets range via Composio.
-
-Features:
-- Write data to specific sheet (overwrites existing data)
-- Supports 2D arrays for multiple rows/columns
-- Can write formulas and formatted strings (uses USER_ENTERED mode)
-- OAuth authentication handled by Composio
-
-Use this to update Google Sheets with calculated results, API responses, or processed data.
-
-Note: The range parameter should include the sheet name (e.g., 'Sheet1!A1:D10'). The sheet name will be automatically extracted.`,
- Icon: "FileSpreadsheet",
- Source: ToolSourceComposio,
- Category: "integration",
- Keywords: []string{"google", "sheets", "spreadsheet", "write", "update", "data", "excel", "composio"},
- Parameters: map[string]interface{}{
- "type": "object",
- "properties": map[string]interface{}{
- "credential_id": map[string]interface{}{
- "type": "string",
- "description": "INTERNAL: Auto-injected by system. Do not set manually.",
- },
- "spreadsheet_id": map[string]interface{}{
- "type": "string",
- "description": "Google Sheets spreadsheet ID (from the URL)",
- },
- "range": map[string]interface{}{
- "type": "string",
- "description": "Sheet name and range to write (e.g., 'Sheet1!A1:D10'). Sheet name is required.",
- },
- "values": map[string]interface{}{
- "type": "array",
- "description": "2D array of values to write [[row1], [row2], ...] or JSON string",
- "items": map[string]interface{}{
- "type": "array",
- "items": map[string]interface{}{}, // Allow any type (string, number, boolean, etc.)
- },
- },
- },
- "required": []string{"spreadsheet_id", "range", "values"},
- },
- Execute: executeComposioGoogleSheetsWrite,
- }
-}
-
-// NewComposioGoogleSheetsAppendTool creates a tool for appending to Google Sheets via Composio
-func NewComposioGoogleSheetsAppendTool() *Tool {
- return &Tool{
- Name: "googlesheets_append",
- DisplayName: "Google Sheets - Append Rows",
- Description: `Append rows to a Google Sheets spreadsheet via Composio.
-
-Features:
-- Appends rows to the end of the specified range
-- Automatically finds the next empty row
-- Supports multiple rows in one operation
-- Uses USER_ENTERED mode (formulas are evaluated)
-- OAuth authentication handled by Composio
-
-Use this to add new data without overwriting existing content (logs, form responses, etc.).`,
- Icon: "FileSpreadsheet",
- Source: ToolSourceComposio,
- Category: "integration",
- Keywords: []string{"google", "sheets", "spreadsheet", "append", "add", "insert", "data", "excel", "composio"},
- Parameters: map[string]interface{}{
- "type": "object",
- "properties": map[string]interface{}{
- "credential_id": map[string]interface{}{
- "type": "string",
- "description": "INTERNAL: Auto-injected by system. Do not set manually.",
- },
- "spreadsheet_id": map[string]interface{}{
- "type": "string",
- "description": "Google Sheets spreadsheet ID (from the URL)",
- },
- "range": map[string]interface{}{
- "type": "string",
- "description": "Sheet name and column range to append to (e.g., 'Sheet1!A:D' or 'Sheet1')",
- },
- "values": map[string]interface{}{
- "type": "array",
- "description": "2D array of values to append [[row1], [row2], ...] or JSON string",
- "items": map[string]interface{}{
- "type": "array",
- "items": map[string]interface{}{}, // Allow any type (string, number, boolean, etc.)
- },
- },
- },
- "required": []string{"spreadsheet_id", "range", "values"},
- },
- Execute: executeComposioGoogleSheetsAppend,
- }
-}
-
-func executeComposioGoogleSheetsRead(args map[string]interface{}) (string, error) {
- // ✅ RATE LIMITING - Check per-user rate limit
- if err := checkComposioRateLimit(args); err != nil {
- return "", err
- }
-
- // Get Composio credentials
- credData, err := GetCredentialData(args, "composio_googlesheets")
- if err != nil {
- return "", fmt.Errorf("failed to get Composio credentials: %w", err)
- }
-
- entityID, ok := credData["composio_entity_id"].(string)
- if !ok || entityID == "" {
- return "", fmt.Errorf("composio_entity_id not found in credentials")
- }
-
- // Extract parameters
- spreadsheetID, _ := args["spreadsheet_id"].(string)
- rangeSpec, _ := args["range"].(string)
-
- if spreadsheetID == "" {
- return "", fmt.Errorf("'spreadsheet_id' is required")
- }
- if rangeSpec == "" {
- return "", fmt.Errorf("'range' is required")
- }
-
- // Call Composio API
- composioAPIKey := os.Getenv("COMPOSIO_API_KEY")
- if composioAPIKey == "" {
- return "", fmt.Errorf("COMPOSIO_API_KEY environment variable not set")
- }
-
- // Use exact parameter names from Composio docs
- payload := map[string]interface{}{
- "entityId": entityID,
- "appName": "googlesheets",
- "input": map[string]interface{}{
- "spreadsheet_id": spreadsheetID,
- "ranges": []string{rangeSpec},
- },
- }
-
- return callComposioAPI(composioAPIKey, "GOOGLESHEETS_BATCH_GET", payload)
-}
-
-func executeComposioGoogleSheetsWrite(args map[string]interface{}) (string, error) {
- // ✅ RATE LIMITING
- if err := checkComposioRateLimit(args); err != nil {
- return "", err
- }
-
- // Get Composio credentials
- credData, err := GetCredentialData(args, "composio_googlesheets")
- if err != nil {
- return "", fmt.Errorf("failed to get Composio credentials: %w", err)
- }
-
- entityID, ok := credData["composio_entity_id"].(string)
- if !ok || entityID == "" {
- return "", fmt.Errorf("composio_entity_id not found in credentials")
- }
-
- // Extract parameters
- spreadsheetID, _ := args["spreadsheet_id"].(string)
- rangeSpec, _ := args["range"].(string)
- values := args["values"]
-
- if spreadsheetID == "" {
- return "", fmt.Errorf("'spreadsheet_id' is required")
- }
- if rangeSpec == "" {
- return "", fmt.Errorf("'range' is required")
- }
- if values == nil {
- return "", fmt.Errorf("'values' is required")
- }
-
- // Parse values if it's a JSON string
- var valuesArray [][]interface{}
- switch v := values.(type) {
- case string:
- if err := json.Unmarshal([]byte(v), &valuesArray); err != nil {
- return "", fmt.Errorf("failed to parse values JSON: %w", err)
- }
- case []interface{}:
- // Convert to 2D array
- for _, row := range v {
- if rowArr, ok := row.([]interface{}); ok {
- valuesArray = append(valuesArray, rowArr)
- } else {
- // Single value row
- valuesArray = append(valuesArray, []interface{}{row})
- }
- }
- default:
- return "", fmt.Errorf("values must be array or JSON string")
- }
-
- // Extract sheet name from range (e.g., "Sheet1!A1:D10" -> "Sheet1")
- sheetName := "Sheet1"
- for i := 0; i < len(rangeSpec); i++ {
- if rangeSpec[i] == '!' {
- sheetName = rangeSpec[:i]
- break
- }
- }
-
- // Call Composio API
- composioAPIKey := os.Getenv("COMPOSIO_API_KEY")
- if composioAPIKey == "" {
- return "", fmt.Errorf("COMPOSIO_API_KEY environment variable not set")
- }
-
- // Use exact parameter names from Composio docs for GOOGLESHEETS_BATCH_UPDATE
- payload := map[string]interface{}{
- "entityId": entityID,
- "appName": "googlesheets",
- "input": map[string]interface{}{
- "spreadsheet_id": spreadsheetID,
- "sheet_name": sheetName,
- "values": valuesArray,
- "valueInputOption": "USER_ENTERED", // Default value from docs
- },
- }
-
- return callComposioAPI(composioAPIKey, "GOOGLESHEETS_BATCH_UPDATE", payload)
-}
-
-func executeComposioGoogleSheetsAppend(args map[string]interface{}) (string, error) {
- // ✅ RATE LIMITING
- if err := checkComposioRateLimit(args); err != nil {
- return "", err
- }
-
- // Get Composio credentials
- credData, err := GetCredentialData(args, "composio_googlesheets")
- if err != nil {
- return "", fmt.Errorf("failed to get Composio credentials: %w", err)
- }
-
- entityID, ok := credData["composio_entity_id"].(string)
- if !ok || entityID == "" {
- return "", fmt.Errorf("composio_entity_id not found in credentials")
- }
-
- // Extract parameters
- spreadsheetID, _ := args["spreadsheet_id"].(string)
- rangeSpec, _ := args["range"].(string)
- values := args["values"]
-
- if spreadsheetID == "" {
- return "", fmt.Errorf("'spreadsheet_id' is required")
- }
- if rangeSpec == "" {
- return "", fmt.Errorf("'range' is required")
- }
- if values == nil {
- return "", fmt.Errorf("'values' is required")
- }
-
- // Parse values if it's a JSON string
- var valuesArray [][]interface{}
- switch v := values.(type) {
- case string:
- if err := json.Unmarshal([]byte(v), &valuesArray); err != nil {
- return "", fmt.Errorf("failed to parse values JSON: %w", err)
- }
- case []interface{}:
- // Convert to 2D array
- for _, row := range v {
- if rowArr, ok := row.([]interface{}); ok {
- valuesArray = append(valuesArray, rowArr)
- } else {
- // Single value row
- valuesArray = append(valuesArray, []interface{}{row})
- }
- }
- default:
- return "", fmt.Errorf("values must be array or JSON string")
- }
-
- // Call Composio API
- composioAPIKey := os.Getenv("COMPOSIO_API_KEY")
- if composioAPIKey == "" {
- return "", fmt.Errorf("COMPOSIO_API_KEY environment variable not set")
- }
-
- // Use exact parameter names from Composio docs for GOOGLESHEETS_SPREADSHEETS_VALUES_APPEND
- payload := map[string]interface{}{
- "entityId": entityID,
- "appName": "googlesheets",
- "input": map[string]interface{}{
- "spreadsheetId": spreadsheetID,
- "range": rangeSpec,
- "valueInputOption": "USER_ENTERED", // Required by docs
- "values": valuesArray,
- },
- }
-
- return callComposioAPI(composioAPIKey, "GOOGLESHEETS_SPREADSHEETS_VALUES_APPEND", payload)
-}
-
-// NewComposioGoogleSheetsCreateTool creates a tool for creating new Google Sheets via Composio
-func NewComposioGoogleSheetsCreateTool() *Tool {
- return &Tool{
- Name: "googlesheets_create",
- DisplayName: "Google Sheets - Create Spreadsheet",
- Description: `Create a new Google Sheets spreadsheet via Composio.
-
-Features:
-- Creates a new spreadsheet in Google Drive
-- Can specify custom title or use default
-- Returns spreadsheet ID and URL
-- OAuth authentication handled by Composio
-
-Use this to create new spreadsheets for data storage, reports, or automation workflows.`,
- Icon: "FileSpreadsheet",
- Source: ToolSourceComposio,
- Category: "integration",
- Keywords: []string{"google", "sheets", "spreadsheet", "create", "new", "composio"},
- Parameters: map[string]interface{}{
- "type": "object",
- "properties": map[string]interface{}{
- "credential_id": map[string]interface{}{
- "type": "string",
- "description": "INTERNAL: Auto-injected by system. Do not set manually.",
- },
- "title": map[string]interface{}{
- "type": "string",
- "description": "Title for the new spreadsheet (optional, defaults to 'Untitled spreadsheet')",
- },
- },
- "required": []string{},
- },
- Execute: executeComposioGoogleSheetsCreate,
- }
-}
-
-func executeComposioGoogleSheetsCreate(args map[string]interface{}) (string, error) {
- // ✅ RATE LIMITING
- if err := checkComposioRateLimit(args); err != nil {
- return "", err
- }
-
- // Get Composio credentials
- credData, err := GetCredentialData(args, "composio_googlesheets")
- if err != nil {
- return "", fmt.Errorf("failed to get Composio credentials: %w", err)
- }
-
- entityID, ok := credData["composio_entity_id"].(string)
- if !ok || entityID == "" {
- return "", fmt.Errorf("composio_entity_id not found in credentials")
- }
-
- // Extract optional title parameter
- title, _ := args["title"].(string)
-
- // Call Composio API
- composioAPIKey := os.Getenv("COMPOSIO_API_KEY")
- if composioAPIKey == "" {
- return "", fmt.Errorf("COMPOSIO_API_KEY environment variable not set")
- }
-
- // Build payload based on whether title is provided
- input := map[string]interface{}{}
- if title != "" {
- input["title"] = title
- }
-
- payload := map[string]interface{}{
- "entityId": entityID,
- "appName": "googlesheets",
- "input": input,
- }
-
- return callComposioAPI(composioAPIKey, "GOOGLESHEETS_CREATE_GOOGLE_SHEET1", payload)
-}
-
-// NewComposioGoogleSheetsInfoTool creates a tool for getting spreadsheet metadata via Composio
-func NewComposioGoogleSheetsInfoTool() *Tool {
- return &Tool{
- Name: "googlesheets_get_info",
- DisplayName: "Google Sheets - Get Spreadsheet Info",
- Description: `Get comprehensive metadata for a Google Sheets spreadsheet via Composio.
-
-Features:
-- Returns spreadsheet title, locale, timezone
-- Lists all sheets/worksheets with their properties
-- Gets sheet dimensions and tab colors
-- OAuth authentication handled by Composio
-
-Use this to discover sheet names, understand spreadsheet structure, or validate spreadsheet existence.`,
- Icon: "FileSpreadsheet",
- Source: ToolSourceComposio,
- Category: "integration",
- Keywords: []string{"google", "sheets", "spreadsheet", "info", "metadata", "sheets list", "composio"},
- Parameters: map[string]interface{}{
- "type": "object",
- "properties": map[string]interface{}{
- "credential_id": map[string]interface{}{
- "type": "string",
- "description": "INTERNAL: Auto-injected by system. Do not set manually.",
- },
- "spreadsheet_id": map[string]interface{}{
- "type": "string",
- "description": "Google Sheets spreadsheet ID (from the URL)",
- },
- },
- "required": []string{"spreadsheet_id"},
- },
- Execute: executeComposioGoogleSheetsInfo,
- }
-}
-
-func executeComposioGoogleSheetsInfo(args map[string]interface{}) (string, error) {
- // ✅ RATE LIMITING
- if err := checkComposioRateLimit(args); err != nil {
- return "", err
- }
-
- credData, err := GetCredentialData(args, "composio_googlesheets")
- if err != nil {
- return "", fmt.Errorf("failed to get Composio credentials: %w", err)
- }
-
- entityID, ok := credData["composio_entity_id"].(string)
- if !ok || entityID == "" {
- return "", fmt.Errorf("composio_entity_id not found in credentials")
- }
-
- spreadsheetID, _ := args["spreadsheet_id"].(string)
- if spreadsheetID == "" {
- return "", fmt.Errorf("'spreadsheet_id' is required")
- }
-
- composioAPIKey := os.Getenv("COMPOSIO_API_KEY")
- if composioAPIKey == "" {
- return "", fmt.Errorf("COMPOSIO_API_KEY environment variable not set")
- }
-
- payload := map[string]interface{}{
- "entityId": entityID,
- "appName": "googlesheets",
- "input": map[string]interface{}{
- "spreadsheet_id": spreadsheetID,
- },
- }
-
- return callComposioAPI(composioAPIKey, "GOOGLESHEETS_GET_SPREADSHEET_INFO", payload)
-}
-
-// NewComposioGoogleSheetsListSheetsTool creates a tool for listing sheet names via Composio
-func NewComposioGoogleSheetsListSheetsTool() *Tool {
- return &Tool{
- Name: "googlesheets_list_sheets",
- DisplayName: "Google Sheets - List Sheet Names",
- Description: `List all worksheet names in a Google Spreadsheet via Composio.
-
-Features:
-- Returns array of all sheet/tab names in order
-- Fast, lightweight operation (no cell data)
-- Useful before reading/writing to specific sheets
-- OAuth authentication handled by Composio
-
-Use this to discover available sheets or validate sheet existence before operations.`,
- Icon: "FileSpreadsheet",
- Source: ToolSourceComposio,
- Category: "integration",
- Keywords: []string{"google", "sheets", "spreadsheet", "list", "tabs", "worksheets", "composio"},
- Parameters: map[string]interface{}{
- "type": "object",
- "properties": map[string]interface{}{
- "credential_id": map[string]interface{}{
- "type": "string",
- "description": "INTERNAL: Auto-injected by system. Do not set manually.",
- },
- "spreadsheet_id": map[string]interface{}{
- "type": "string",
- "description": "Google Sheets spreadsheet ID (from the URL)",
- },
- },
- "required": []string{"spreadsheet_id"},
- },
- Execute: executeComposioGoogleSheetsListSheets,
- }
-}
-
-func executeComposioGoogleSheetsListSheets(args map[string]interface{}) (string, error) {
- // ✅ RATE LIMITING
- if err := checkComposioRateLimit(args); err != nil {
- return "", err
- }
-
- credData, err := GetCredentialData(args, "composio_googlesheets")
- if err != nil {
- return "", fmt.Errorf("failed to get Composio credentials: %w", err)
- }
-
- entityID, ok := credData["composio_entity_id"].(string)
- if !ok || entityID == "" {
- return "", fmt.Errorf("composio_entity_id not found in credentials")
- }
-
- spreadsheetID, _ := args["spreadsheet_id"].(string)
- if spreadsheetID == "" {
- return "", fmt.Errorf("'spreadsheet_id' is required")
- }
-
- composioAPIKey := os.Getenv("COMPOSIO_API_KEY")
- if composioAPIKey == "" {
- return "", fmt.Errorf("COMPOSIO_API_KEY environment variable not set")
- }
-
- payload := map[string]interface{}{
- "entityId": entityID,
- "appName": "googlesheets",
- "input": map[string]interface{}{
- "spreadsheet_id": spreadsheetID,
- },
- }
-
- return callComposioAPI(composioAPIKey, "GOOGLESHEETS_GET_SHEET_NAMES", payload)
-}
-
-// NewComposioGoogleSheetsSearchTool creates a tool for searching spreadsheets via Composio
-func NewComposioGoogleSheetsSearchTool() *Tool {
- return &Tool{
- Name: "googlesheets_search",
- DisplayName: "Google Sheets - Search Spreadsheets",
- Description: `Search for Google Spreadsheets using filters via Composio.
-
-Features:
-- Search by name, content, or metadata
-- Filter by creation/modification date
-- Find shared or starred spreadsheets
-- Returns spreadsheet IDs and metadata
-- OAuth authentication handled by Composio
-
-Use this to find spreadsheets by name when you don't have the ID, or discover available sheets.`,
- Icon: "FileSpreadsheet",
- Source: ToolSourceComposio,
- Category: "integration",
- Keywords: []string{"google", "sheets", "spreadsheet", "search", "find", "discover", "composio"},
- Parameters: map[string]interface{}{
- "type": "object",
- "properties": map[string]interface{}{
- "credential_id": map[string]interface{}{
- "type": "string",
- "description": "INTERNAL: Auto-injected by system. Do not set manually.",
- },
- "query": map[string]interface{}{
- "type": "string",
- "description": "Search query (searches in name and content)",
- },
- "max_results": map[string]interface{}{
- "type": "integer",
- "description": "Maximum number of results to return (default: 10)",
- },
- },
- "required": []string{},
- },
- Execute: executeComposioGoogleSheetsSearch,
- }
-}
-
-func executeComposioGoogleSheetsSearch(args map[string]interface{}) (string, error) {
- // ✅ RATE LIMITING
- if err := checkComposioRateLimit(args); err != nil {
- return "", err
- }
-
- credData, err := GetCredentialData(args, "composio_googlesheets")
- if err != nil {
- return "", fmt.Errorf("failed to get Composio credentials: %w", err)
- }
-
- entityID, ok := credData["composio_entity_id"].(string)
- if !ok || entityID == "" {
- return "", fmt.Errorf("composio_entity_id not found in credentials")
- }
-
- composioAPIKey := os.Getenv("COMPOSIO_API_KEY")
- if composioAPIKey == "" {
- return "", fmt.Errorf("COMPOSIO_API_KEY environment variable not set")
- }
-
- // Build input parameters
- input := map[string]interface{}{}
-
- if query, ok := args["query"].(string); ok && query != "" {
- input["query"] = query
- }
-
- if maxResults, ok := args["max_results"].(float64); ok {
- input["max_results"] = int(maxResults)
- }
-
- payload := map[string]interface{}{
- "entityId": entityID,
- "appName": "googlesheets",
- "input": input,
- }
-
- return callComposioAPI(composioAPIKey, "GOOGLESHEETS_SEARCH_SPREADSHEETS", payload)
-}
-
-// NewComposioGoogleSheetsClearTool creates a tool for clearing cell values via Composio
-func NewComposioGoogleSheetsClearTool() *Tool {
- return &Tool{
- Name: "googlesheets_clear",
- DisplayName: "Google Sheets - Clear Values",
- Description: `Clear cell content from a range in Google Sheets via Composio.
-
-Features:
-- Clears cell values but preserves formatting
-- Preserves cell notes/comments
-- Clears formulas and data
-- Supports A1 notation ranges
-- OAuth authentication handled by Composio
-
-Use this to clear data from specific ranges while keeping cell formatting intact.`,
- Icon: "FileSpreadsheet",
- Source: ToolSourceComposio,
- Category: "integration",
- Keywords: []string{"google", "sheets", "spreadsheet", "clear", "delete", "erase", "composio"},
- Parameters: map[string]interface{}{
- "type": "object",
- "properties": map[string]interface{}{
- "credential_id": map[string]interface{}{
- "type": "string",
- "description": "INTERNAL: Auto-injected by system. Do not set manually.",
- },
- "spreadsheet_id": map[string]interface{}{
- "type": "string",
- "description": "Google Sheets spreadsheet ID (from the URL)",
- },
- "range": map[string]interface{}{
- "type": "string",
- "description": "Range to clear (e.g., 'Sheet1!A1:D10' or 'Sheet1!A:D')",
- },
- },
- "required": []string{"spreadsheet_id", "range"},
- },
- Execute: executeComposioGoogleSheetsClear,
- }
-}
-
-func executeComposioGoogleSheetsClear(args map[string]interface{}) (string, error) {
- // ✅ RATE LIMITING
- if err := checkComposioRateLimit(args); err != nil {
- return "", err
- }
-
- credData, err := GetCredentialData(args, "composio_googlesheets")
- if err != nil {
- return "", fmt.Errorf("failed to get Composio credentials: %w", err)
- }
-
- entityID, ok := credData["composio_entity_id"].(string)
- if !ok || entityID == "" {
- return "", fmt.Errorf("composio_entity_id not found in credentials")
- }
-
- spreadsheetID, _ := args["spreadsheet_id"].(string)
- rangeSpec, _ := args["range"].(string)
-
- if spreadsheetID == "" {
- return "", fmt.Errorf("'spreadsheet_id' is required")
- }
- if rangeSpec == "" {
- return "", fmt.Errorf("'range' is required")
- }
-
- composioAPIKey := os.Getenv("COMPOSIO_API_KEY")
- if composioAPIKey == "" {
- return "", fmt.Errorf("COMPOSIO_API_KEY environment variable not set")
- }
-
- payload := map[string]interface{}{
- "entityId": entityID,
- "appName": "googlesheets",
- "input": map[string]interface{}{
- "spreadsheet_id": spreadsheetID,
- "range": rangeSpec,
- },
- }
-
- return callComposioAPI(composioAPIKey, "GOOGLESHEETS_CLEAR_VALUES", payload)
-}
-
-// NewComposioGoogleSheetsAddSheetTool creates a tool for adding new sheets via Composio
-func NewComposioGoogleSheetsAddSheetTool() *Tool {
- return &Tool{
- Name: "googlesheets_add_sheet",
- DisplayName: "Google Sheets - Add Sheet",
- Description: `Add a new worksheet/tab to an existing Google Spreadsheet via Composio.
-
-Features:
-- Creates new sheet within existing spreadsheet
-- Can specify sheet title
-- Can set initial row/column count
-- Can set tab color
-- OAuth authentication handled by Composio
-
-Use this to add new tabs/worksheets to organize data in existing spreadsheets.`,
- Icon: "FileSpreadsheet",
- Source: ToolSourceComposio,
- Category: "integration",
- Keywords: []string{"google", "sheets", "spreadsheet", "add", "create", "tab", "worksheet", "composio"},
- Parameters: map[string]interface{}{
- "type": "object",
- "properties": map[string]interface{}{
- "credential_id": map[string]interface{}{
- "type": "string",
- "description": "INTERNAL: Auto-injected by system. Do not set manually.",
- },
- "spreadsheet_id": map[string]interface{}{
- "type": "string",
- "description": "Google Sheets spreadsheet ID (from the URL)",
- },
- "title": map[string]interface{}{
- "type": "string",
- "description": "Title for the new sheet (default: 'Sheet{N}')",
- },
- },
- "required": []string{"spreadsheet_id"},
- },
- Execute: executeComposioGoogleSheetsAddSheet,
- }
-}
-
-func executeComposioGoogleSheetsAddSheet(args map[string]interface{}) (string, error) {
- // ✅ RATE LIMITING
- if err := checkComposioRateLimit(args); err != nil {
- return "", err
- }
-
- credData, err := GetCredentialData(args, "composio_googlesheets")
- if err != nil {
- return "", fmt.Errorf("failed to get Composio credentials: %w", err)
- }
-
- entityID, ok := credData["composio_entity_id"].(string)
- if !ok || entityID == "" {
- return "", fmt.Errorf("composio_entity_id not found in credentials")
- }
-
- spreadsheetID, _ := args["spreadsheet_id"].(string)
- if spreadsheetID == "" {
- return "", fmt.Errorf("'spreadsheet_id' is required")
- }
-
- composioAPIKey := os.Getenv("COMPOSIO_API_KEY")
- if composioAPIKey == "" {
- return "", fmt.Errorf("COMPOSIO_API_KEY environment variable not set")
- }
-
- // Build input with optional title
- input := map[string]interface{}{
- "spreadsheetId": spreadsheetID,
- }
-
- // Add optional properties
- properties := map[string]interface{}{}
- if title, ok := args["title"].(string); ok && title != "" {
- properties["title"] = title
- }
-
- if len(properties) > 0 {
- input["properties"] = properties
- }
-
- payload := map[string]interface{}{
- "entityId": entityID,
- "appName": "googlesheets",
- "input": input,
- }
-
- return callComposioAPI(composioAPIKey, "GOOGLESHEETS_ADD_SHEET", payload)
-}
-
-// NewComposioGoogleSheetsDeleteSheetTool creates a tool for deleting sheets via Composio
-func NewComposioGoogleSheetsDeleteSheetTool() *Tool {
- return &Tool{
- Name: "googlesheets_delete_sheet",
- DisplayName: "Google Sheets - Delete Sheet",
- Description: `Delete a worksheet/tab from a Google Spreadsheet via Composio.
-
-Features:
-- Permanently removes a sheet from spreadsheet
-- Requires sheet ID (numeric ID, not name)
-- Cannot delete the last remaining sheet
-- OAuth authentication handled by Composio
-
-Use this to remove unwanted worksheets. Get sheet ID from 'googlesheets_get_info' first.
-
-WARNING: This action is permanent and cannot be undone!`,
- Icon: "FileSpreadsheet",
- Source: ToolSourceComposio,
- Category: "integration",
- Keywords: []string{"google", "sheets", "spreadsheet", "delete", "remove", "tab", "worksheet", "composio"},
- Parameters: map[string]interface{}{
- "type": "object",
- "properties": map[string]interface{}{
- "credential_id": map[string]interface{}{
- "type": "string",
- "description": "INTERNAL: Auto-injected by system. Do not set manually.",
- },
- "spreadsheet_id": map[string]interface{}{
- "type": "string",
- "description": "Google Sheets spreadsheet ID (from the URL)",
- },
- "sheet_id": map[string]interface{}{
- "type": "integer",
- "description": "Numeric ID of the sheet to delete (get from googlesheets_get_info)",
- },
- },
- "required": []string{"spreadsheet_id", "sheet_id"},
- },
- Execute: executeComposioGoogleSheetsDeleteSheet,
- }
-}
-
-func executeComposioGoogleSheetsDeleteSheet(args map[string]interface{}) (string, error) {
- // ✅ RATE LIMITING
- if err := checkComposioRateLimit(args); err != nil {
- return "", err
- }
-
- credData, err := GetCredentialData(args, "composio_googlesheets")
- if err != nil {
- return "", fmt.Errorf("failed to get Composio credentials: %w", err)
- }
-
- entityID, ok := credData["composio_entity_id"].(string)
- if !ok || entityID == "" {
- return "", fmt.Errorf("composio_entity_id not found in credentials")
- }
-
- spreadsheetID, _ := args["spreadsheet_id"].(string)
- if spreadsheetID == "" {
- return "", fmt.Errorf("'spreadsheet_id' is required")
- }
-
- // Handle both float64 and int types for sheet_id
- var sheetID int
- switch v := args["sheet_id"].(type) {
- case float64:
- sheetID = int(v)
- case int:
- sheetID = v
- default:
- return "", fmt.Errorf("'sheet_id' must be a number")
- }
-
- composioAPIKey := os.Getenv("COMPOSIO_API_KEY")
- if composioAPIKey == "" {
- return "", fmt.Errorf("COMPOSIO_API_KEY environment variable not set")
- }
-
- payload := map[string]interface{}{
- "entityId": entityID,
- "appName": "googlesheets",
- "input": map[string]interface{}{
- "spreadsheetId": spreadsheetID,
- "sheet_id": sheetID,
- },
- }
-
- return callComposioAPI(composioAPIKey, "GOOGLESHEETS_DELETE_SHEET", payload)
-}
-
-// NewComposioGoogleSheetsFindReplaceTool creates a tool for find and replace via Composio
-func NewComposioGoogleSheetsFindReplaceTool() *Tool {
- return &Tool{
- Name: "googlesheets_find_replace",
- DisplayName: "Google Sheets - Find and Replace",
- Description: `Find and replace text in a Google Spreadsheet via Composio.
-
-Features:
-- Find and replace across entire spreadsheet or specific sheets
-- Case-sensitive or case-insensitive matching
-- Match entire cell or partial content
-- Supports regex patterns
-- OAuth authentication handled by Composio
-
-Use this to bulk update values, fix errors, or update formulas across your spreadsheet.`,
- Icon: "FileSpreadsheet",
- Source: ToolSourceComposio,
- Category: "integration",
- Keywords: []string{"google", "sheets", "spreadsheet", "find", "replace", "search", "update", "composio"},
- Parameters: map[string]interface{}{
- "type": "object",
- "properties": map[string]interface{}{
- "credential_id": map[string]interface{}{
- "type": "string",
- "description": "INTERNAL: Auto-injected by system. Do not set manually.",
- },
- "spreadsheet_id": map[string]interface{}{
- "type": "string",
- "description": "Google Sheets spreadsheet ID (from the URL)",
- },
- "find": map[string]interface{}{
- "type": "string",
- "description": "Text or pattern to find",
- },
- "replace": map[string]interface{}{
- "type": "string",
- "description": "Text to replace with",
- },
- "sheet_id": map[string]interface{}{
- "type": "integer",
- "description": "Optional: Numeric sheet ID to limit search (omit for all sheets)",
- },
- "match_case": map[string]interface{}{
- "type": "boolean",
- "description": "Whether to match case (default: false)",
- },
- },
- "required": []string{"spreadsheet_id", "find", "replace"},
- },
- Execute: executeComposioGoogleSheetsFindReplace,
- }
-}
-
-func executeComposioGoogleSheetsFindReplace(args map[string]interface{}) (string, error) {
- // ✅ RATE LIMITING
- if err := checkComposioRateLimit(args); err != nil {
- return "", err
- }
-
- credData, err := GetCredentialData(args, "composio_googlesheets")
- if err != nil {
- return "", fmt.Errorf("failed to get Composio credentials: %w", err)
- }
-
- entityID, ok := credData["composio_entity_id"].(string)
- if !ok || entityID == "" {
- return "", fmt.Errorf("composio_entity_id not found in credentials")
- }
-
- spreadsheetID, _ := args["spreadsheet_id"].(string)
- find, _ := args["find"].(string)
- replace, _ := args["replace"].(string)
-
- if spreadsheetID == "" {
- return "", fmt.Errorf("'spreadsheet_id' is required")
- }
- if find == "" {
- return "", fmt.Errorf("'find' is required")
- }
- if replace == "" {
- return "", fmt.Errorf("'replace' is required")
- }
-
- composioAPIKey := os.Getenv("COMPOSIO_API_KEY")
- if composioAPIKey == "" {
- return "", fmt.Errorf("COMPOSIO_API_KEY environment variable not set")
- }
-
- // Build input
- input := map[string]interface{}{
- "spreadsheetId": spreadsheetID,
- "find": find,
- "replace": replace,
- }
-
- // Add optional parameters
- if sheetID, ok := args["sheet_id"].(float64); ok {
- input["sheetId"] = int(sheetID)
- }
- if matchCase, ok := args["match_case"].(bool); ok {
- input["matchCase"] = matchCase
- }
-
- payload := map[string]interface{}{
- "entityId": entityID,
- "appName": "googlesheets",
- "input": input,
- }
-
- return callComposioAPI(composioAPIKey, "GOOGLESHEETS_FIND_REPLACE", payload)
-}
-
-// NewComposioGoogleSheetsUpsertRowsTool creates a tool for upserting rows via Composio
-func NewComposioGoogleSheetsUpsertRowsTool() *Tool {
- return &Tool{
- Name: "googlesheets_upsert_rows",
- DisplayName: "Google Sheets - Upsert Rows",
- Description: `Smart update/insert rows by matching a key column via Composio.
-
-Features:
-- Updates existing rows by matching key column
-- Appends new rows if key not found
-- Auto-adds missing columns to sheet
-- Supports partial column updates
-- Column order doesn't matter (auto-maps by header)
-- Prevents duplicates
-- OAuth authentication handled by Composio
-
-Use this for CRM syncs, inventory updates, or any scenario where you want to update existing records or create new ones based on a unique identifier.
-
-Example: Update contacts by email, inventory by SKU, leads by Lead ID, etc.`,
- Icon: "FileSpreadsheet",
- Source: ToolSourceComposio,
- Category: "integration",
- Keywords: []string{"google", "sheets", "spreadsheet", "upsert", "update", "insert", "merge", "sync", "composio"},
- Parameters: map[string]interface{}{
- "type": "object",
- "properties": map[string]interface{}{
- "credential_id": map[string]interface{}{
- "type": "string",
- "description": "INTERNAL: Auto-injected by system. Do not set manually.",
- },
- "spreadsheet_id": map[string]interface{}{
- "type": "string",
- "description": "Google Sheets spreadsheet ID (from the URL)",
- },
- "sheet_name": map[string]interface{}{
- "type": "string",
- "description": "Name of the sheet/tab to upsert into",
- },
- "key_column": map[string]interface{}{
- "type": "string",
- "description": "Column name to match on (e.g., 'Email', 'SKU', 'Lead ID')",
- },
- "rows": map[string]interface{}{
- "type": "array",
- "description": "Array of row data arrays [[row1], [row2], ...]",
- "items": map[string]interface{}{
- "type": "array",
- "items": map[string]interface{}{}, // Allow any type
- },
- },
- "headers": map[string]interface{}{
- "type": "array",
- "description": "Optional: Array of column headers (if not provided, uses first row of sheet)",
- "items": map[string]interface{}{
- "type": "string",
- },
- },
- },
- "required": []string{"spreadsheet_id", "sheet_name", "rows"},
- },
- Execute: executeComposioGoogleSheetsUpsertRows,
- }
-}
-
-func executeComposioGoogleSheetsUpsertRows(args map[string]interface{}) (string, error) {
- // ✅ RATE LIMITING
- if err := checkComposioRateLimit(args); err != nil {
- return "", err
- }
-
- credData, err := GetCredentialData(args, "composio_googlesheets")
- if err != nil {
- return "", fmt.Errorf("failed to get Composio credentials: %w", err)
- }
-
- entityID, ok := credData["composio_entity_id"].(string)
- if !ok || entityID == "" {
- return "", fmt.Errorf("composio_entity_id not found in credentials")
- }
-
- spreadsheetID, _ := args["spreadsheet_id"].(string)
- sheetName, _ := args["sheet_name"].(string)
- rows := args["rows"]
-
- if spreadsheetID == "" {
- return "", fmt.Errorf("'spreadsheet_id' is required")
- }
- if sheetName == "" {
- return "", fmt.Errorf("'sheet_name' is required")
- }
- if rows == nil {
- return "", fmt.Errorf("'rows' is required")
- }
-
- composioAPIKey := os.Getenv("COMPOSIO_API_KEY")
- if composioAPIKey == "" {
- return "", fmt.Errorf("COMPOSIO_API_KEY environment variable not set")
- }
-
- // Build input
- input := map[string]interface{}{
- "spreadsheetId": spreadsheetID,
- "sheetName": sheetName,
- "rows": rows,
- }
-
- // Add optional parameters
- if keyColumn, ok := args["key_column"].(string); ok && keyColumn != "" {
- input["keyColumn"] = keyColumn
- }
- if headers, ok := args["headers"].([]interface{}); ok && len(headers) > 0 {
- input["headers"] = headers
- }
-
- payload := map[string]interface{}{
- "entityId": entityID,
- "appName": "googlesheets",
- "input": input,
- }
-
- return callComposioAPI(composioAPIKey, "GOOGLESHEETS_UPSERT_ROWS", payload)
-}
-
-// getConnectedAccountID retrieves the connected account ID from Composio v3 API
-func getConnectedAccountID(apiKey string, userID string, appName string) (string, error) {
- // Query v3 API to get connected accounts for this user (URL-safe to prevent injection)
- baseURL := "https://backend.composio.dev/api/v3/connected_accounts"
- params := url.Values{}
- params.Add("user_ids", userID)
- fullURL := baseURL + "?" + params.Encode()
-
- req, err := http.NewRequest("GET", fullURL, nil)
- if err != nil {
- return "", fmt.Errorf("failed to create request: %w", err)
- }
-
- req.Header.Set("x-api-key", apiKey)
-
- client := &http.Client{Timeout: 30 * time.Second}
- resp, err := client.Do(req)
- if err != nil {
- return "", fmt.Errorf("failed to send request: %w", err)
- }
- defer resp.Body.Close()
-
- respBody, _ := io.ReadAll(resp.Body)
-
- if resp.StatusCode >= 400 {
- return "", fmt.Errorf("Composio API error (status %d): %s", resp.StatusCode, string(respBody))
- }
-
- // Parse v3 response
- var response struct {
- Items []struct {
- ID string `json:"id"`
- Toolkit struct {
- Slug string `json:"slug"`
- } `json:"toolkit"`
- Deprecated struct {
- UUID string `json:"uuid"`
- } `json:"deprecated"`
- } `json:"items"`
- }
-
- if err := json.Unmarshal(respBody, &response); err != nil {
- return "", fmt.Errorf("failed to parse response: %w", err)
- }
-
- // Find the connected account for this app
- for _, account := range response.Items {
- if account.Toolkit.Slug == appName {
- // v2 execution endpoint needs the old UUID, not the new nano ID
- // Check if deprecated.uuid exists (for v2 compatibility)
- if account.Deprecated.UUID != "" {
- return account.Deprecated.UUID, nil
- }
- // Fall back to nano ID if UUID not available
- return account.ID, nil
- }
- }
-
- return "", fmt.Errorf("no connected account found for app '%s' and user '%s'", appName, userID)
-}
-
-// callComposioAPI makes a request to Composio's v3 API
-func callComposioAPI(apiKey string, action string, payload map[string]interface{}) (string, error) {
- // v2 execution endpoint still works with v3 connected accounts
- url := "https://backend.composio.dev/api/v2/actions/" + action + "/execute"
-
- // Get params from payload
- entityID, _ := payload["entityId"].(string)
- appName, _ := payload["appName"].(string)
- input, _ := payload["input"].(map[string]interface{})
-
- // For v3, we need to find the connected account ID
- connectedAccountID, err := getConnectedAccountID(apiKey, entityID, appName)
- if err != nil {
- return "", fmt.Errorf("failed to get connected account ID: %w", err)
- }
-
- // Build v2 payload (v2 execution endpoint uses connectedAccountId with camelCase)
- v2Payload := map[string]interface{}{
- "connectedAccountId": connectedAccountID,
- "input": input,
- }
-
- jsonData, err := json.Marshal(v2Payload)
- if err != nil {
- return "", fmt.Errorf("failed to marshal request: %w", err)
- }
-
- // ✅ SECURE LOGGING - Only log non-sensitive metadata
- log.Printf("🔍 [COMPOSIO] Action: %s, ConnectedAccount: %s", action, maskSensitiveID(connectedAccountID))
-
- req, err := http.NewRequest("POST", url, bytes.NewBuffer(jsonData))
- if err != nil {
- return "", fmt.Errorf("failed to create request: %w", err)
- }
-
- req.Header.Set("Content-Type", "application/json")
- req.Header.Set("x-api-key", apiKey)
-
- client := &http.Client{Timeout: 30 * time.Second}
- resp, err := client.Do(req)
- if err != nil {
- return "", fmt.Errorf("failed to send request: %w", err)
- }
- defer resp.Body.Close()
-
- // ✅ SECURITY FIX: Parse and log rate limit headers
- parseRateLimitHeaders(resp.Header, action)
-
- respBody, _ := io.ReadAll(resp.Body)
-
- if resp.StatusCode >= 400 {
- // ✅ SECURE ERROR HANDLING - Log full details server-side, sanitize for user
- log.Printf("❌ [COMPOSIO] API error (status %d) for action %s", resp.StatusCode, action)
-
- // Handle rate limiting with specific error
- if resp.StatusCode == 429 {
- retryAfter := resp.Header.Get("Retry-After")
- if retryAfter != "" {
- log.Printf("⚠️ [COMPOSIO] Rate limited, retry after: %s seconds", retryAfter)
- return "", fmt.Errorf("rate limit exceeded, retry after %s seconds", retryAfter)
- }
- return "", fmt.Errorf("rate limit exceeded, please try again later")
- }
-
- // Don't expose internal Composio error details to users
- if resp.StatusCode >= 500 {
- return "", fmt.Errorf("external service error (status %d)", resp.StatusCode)
- }
- // Client errors (4xx) can be slightly more specific
- return "", fmt.Errorf("invalid request (status %d): check spreadsheet ID and permissions", resp.StatusCode)
- }
-
- // Parse response
- var apiResponse map[string]interface{}
- if err := json.Unmarshal(respBody, &apiResponse); err != nil {
- return string(respBody), nil
- }
-
- // Return formatted response
- result, _ := json.MarshalIndent(apiResponse, "", " ")
- return string(result), nil
-}
-
-// parseRateLimitHeaders parses and logs rate limit headers from Composio API responses
-func parseRateLimitHeaders(headers http.Header, action string) {
- limit := headers.Get("X-RateLimit-Limit")
- remaining := headers.Get("X-RateLimit-Remaining")
- reset := headers.Get("X-RateLimit-Reset")
-
- if limit != "" || remaining != "" || reset != "" {
- log.Printf("📊 [COMPOSIO] Rate limits for %s - Limit: %s, Remaining: %s, Reset: %s",
- action, limit, remaining, reset)
-
- // Warning if approaching rate limit
- if remaining != "" && limit != "" {
- remainingInt := 0
- limitInt := 0
- fmt.Sscanf(remaining, "%d", &remainingInt)
- fmt.Sscanf(limit, "%d", &limitInt)
-
- if limitInt > 0 {
- percentRemaining := float64(remainingInt) / float64(limitInt) * 100
- if percentRemaining < 20 {
- log.Printf("⚠️ [COMPOSIO] Rate limit warning: only %.1f%% remaining (%d/%d)",
- percentRemaining, remainingInt, limitInt)
- }
- }
- }
- }
-}
diff --git a/backend/internal/tools/credential_helper.go b/backend/internal/tools/credential_helper.go
deleted file mode 100644
index 20345aaf..00000000
--- a/backend/internal/tools/credential_helper.go
+++ /dev/null
@@ -1,138 +0,0 @@
-package tools
-
-import (
- "fmt"
-
- "claraverse/internal/models"
-)
-
-// CredentialResolver is a function type for resolving credentials at runtime
-// This is injected into tool args to provide access to credentials without
-// exposing the credential service directly to tools
-type CredentialResolver func(credentialID string) (*models.DecryptedCredential, error)
-
-// ContextKey for passing credential resolver through args
-const CredentialResolverKey = "__credential_resolver__"
-const UserIDKey = "__user_id__"
-
-// Note: CreateCredentialResolver is defined in services/credential_service.go
-// to avoid import cycles (services imports tools, so tools cannot import services)
-
-// ResolveWebhookURL resolves a webhook URL from either direct URL or credential ID
-// Priority: 1. Direct webhook_url parameter, 2. credential_id lookup
-func ResolveWebhookURL(args map[string]interface{}, integrationType string) (string, error) {
- // First, check for direct webhook_url
- if webhookURL, ok := args["webhook_url"].(string); ok && webhookURL != "" {
- return webhookURL, nil
- }
-
- // Check for credential_id
- credentialID, hasCredID := args["credential_id"].(string)
- if !hasCredID || credentialID == "" {
- return "", fmt.Errorf("either webhook_url or credential_id is required")
- }
-
- // Get credential resolver from args
- resolver, ok := args[CredentialResolverKey].(CredentialResolver)
- if !ok || resolver == nil {
- return "", fmt.Errorf("credential resolver not available")
- }
-
- // Resolve the credential
- cred, err := resolver(credentialID)
- if err != nil {
- return "", fmt.Errorf("failed to resolve credential: %w", err)
- }
-
- // Verify integration type matches
- if cred.IntegrationType != integrationType {
- return "", fmt.Errorf("credential type mismatch: expected %s, got %s", integrationType, cred.IntegrationType)
- }
-
- // Extract webhook URL from credential data
- webhookURL, ok := cred.Data["webhook_url"].(string)
- if !ok || webhookURL == "" {
- // Try alternate key names
- if url, ok := cred.Data["url"].(string); ok && url != "" {
- webhookURL = url
- } else {
- return "", fmt.Errorf("credential does not contain a valid webhook URL")
- }
- }
-
- return webhookURL, nil
-}
-
-// ResolveAPIKey resolves an API key from either direct parameter or credential ID
-func ResolveAPIKey(args map[string]interface{}, integrationType string, keyFieldName string) (string, error) {
- // First, check for direct API key
- if apiKey, ok := args[keyFieldName].(string); ok && apiKey != "" {
- return apiKey, nil
- }
-
- // Check for credential_id
- credentialID, hasCredID := args["credential_id"].(string)
- if !hasCredID || credentialID == "" {
- return "", fmt.Errorf("either %s or credential_id is required", keyFieldName)
- }
-
- // Get credential resolver from args
- resolver, ok := args[CredentialResolverKey].(CredentialResolver)
- if !ok || resolver == nil {
- return "", fmt.Errorf("credential resolver not available")
- }
-
- // Resolve the credential
- cred, err := resolver(credentialID)
- if err != nil {
- return "", fmt.Errorf("failed to resolve credential: %w", err)
- }
-
- // Verify integration type matches
- if cred.IntegrationType != integrationType {
- return "", fmt.Errorf("credential type mismatch: expected %s, got %s", integrationType, cred.IntegrationType)
- }
-
- // Extract API key from credential data
- apiKey, ok := cred.Data[keyFieldName].(string)
- if !ok || apiKey == "" {
- // Try alternate key names
- if key, ok := cred.Data["api_key"].(string); ok && key != "" {
- apiKey = key
- } else if key, ok := cred.Data["token"].(string); ok && key != "" {
- apiKey = key
- } else {
- return "", fmt.Errorf("credential does not contain a valid API key")
- }
- }
-
- return apiKey, nil
-}
-
-// GetCredentialData retrieves all data from a credential by ID
-func GetCredentialData(args map[string]interface{}, integrationType string) (map[string]interface{}, error) {
- // Check for credential_id
- credentialID, hasCredID := args["credential_id"].(string)
- if !hasCredID || credentialID == "" {
- return nil, fmt.Errorf("credential_id is required")
- }
-
- // Get credential resolver from args
- resolver, ok := args[CredentialResolverKey].(CredentialResolver)
- if !ok || resolver == nil {
- return nil, fmt.Errorf("credential resolver not available")
- }
-
- // Resolve the credential
- cred, err := resolver(credentialID)
- if err != nil {
- return nil, fmt.Errorf("failed to resolve credential: %w", err)
- }
-
- // Verify integration type matches (if provided)
- if integrationType != "" && cred.IntegrationType != integrationType {
- return nil, fmt.Errorf("credential type mismatch: expected %s, got %s", integrationType, cred.IntegrationType)
- }
-
- return cred.Data, nil
-}
diff --git a/backend/internal/tools/data_analyst_tool.go b/backend/internal/tools/data_analyst_tool.go
deleted file mode 100644
index 4451afa8..00000000
--- a/backend/internal/tools/data_analyst_tool.go
+++ /dev/null
@@ -1,500 +0,0 @@
-package tools
-
-import (
- "context"
- "encoding/json"
- "fmt"
- "log"
- "regexp"
- "strings"
-
- "claraverse/internal/e2b"
-)
-
-// stripDataLoadingCalls removes pd.read_csv(), pd.read_excel(), pd.read_json() calls from user code
-// This prevents LLMs from trying to load files by filename (which don't exist in the sandbox)
-func stripDataLoadingCalls(code string) string {
- // Patterns to match various forms of data loading
- patterns := []string{
- // Match: df = pd.read_csv('filename.csv') or similar with double quotes
- `(?m)^\s*\w+\s*=\s*pd\.read_csv\s*\([^)]+\)\s*$`,
- `(?m)^\s*\w+\s*=\s*pd\.read_excel\s*\([^)]+\)\s*$`,
- `(?m)^\s*\w+\s*=\s*pd\.read_json\s*\([^)]+\)\s*$`,
- `(?m)^\s*\w+\s*=\s*pd\.read_table\s*\([^)]+\)\s*$`,
- // Match inline read calls (not assigned)
- `(?m)^\s*pd\.read_csv\s*\([^)]+\)\s*$`,
- `(?m)^\s*pd\.read_excel\s*\([^)]+\)\s*$`,
- `(?m)^\s*pd\.read_json\s*\([^)]+\)\s*$`,
- // Match with pandas prefix
- `(?m)^\s*\w+\s*=\s*pandas\.read_csv\s*\([^)]+\)\s*$`,
- `(?m)^\s*\w+\s*=\s*pandas\.read_excel\s*\([^)]+\)\s*$`,
- }
-
- result := code
- stripped := false
- for _, pattern := range patterns {
- re := regexp.MustCompile(pattern)
- if re.MatchString(result) {
- stripped = true
- result = re.ReplaceAllString(result, "# [AUTO-REMOVED: Data is pre-loaded as 'df']")
- }
- }
-
- if stripped {
- log.Printf("🔧 [DATA-ANALYST] Stripped data loading calls from user code")
- }
-
- return result
-}
-
-// NewDataAnalystTool creates a new AI Data Analyst tool
-func NewDataAnalystTool() *Tool {
- return &Tool{
- Name: "analyze_data",
- DisplayName: "AI Data Analyst",
- Description: `Analyze data with Python. Full access to pandas, numpy, matplotlib, and seaborn.
-
-⚠️ IMPORTANT: Data is AUTOMATICALLY loaded as 'df' (pandas DataFrame).
-DO NOT use pd.read_csv(), pd.read_excel(), or pd.read_json() - it will fail!
-Just use 'df' directly in your code.
-
-Example usage:
-- df.head() - view data
-- df.describe() - statistics
-- sns.barplot(data=df, x='category', y='sales') - bar chart
-- df.plot() - line plot
-
-Chart types you can create:
-- Bar: sns.barplot(data=df, x='col1', y='col2')
-- Pie: plt.pie(df.groupby('cat')['val'].sum(), labels=..., autopct='%1.1f%%')
-- Scatter: sns.scatterplot(data=df, x='col1', y='col2', hue='col3')
-- Heatmap: sns.heatmap(df.corr(), annot=True, cmap='coolwarm')
-- Histogram: sns.histplot(df['col'], bins=30)
-- Box: sns.boxplot(data=df, x='category', y='value')
-
-Always use plt.show() after each plot. Add titles and labels for clarity.`,
- Icon: "ChartBar",
- Source: ToolSourceBuiltin,
- Category: "computation",
- Keywords: []string{"analyze", "data", "python", "pandas", "visualization", "chart", "graph", "statistics", "csv", "dataframe", "plot", "analytics"},
- Parameters: map[string]interface{}{
- "type": "object",
- "properties": map[string]interface{}{
- "file_id": map[string]interface{}{
- "type": "string",
- "description": "Upload ID of the CSV/Excel file to analyze (from file upload). Use this for uploaded files.",
- },
- "csv_data": map[string]interface{}{
- "type": "string",
- "description": "CSV data as a string (with headers). Use this for small inline data. Either file_id or csv_data is required.",
- },
- "python_code": map[string]interface{}{
- "type": "string",
- "description": `Custom Python code for visualization.
-
-⚠️ CRITICAL: 'df' is ALREADY LOADED as a pandas DataFrame!
-DO NOT use pd.read_csv(), pd.read_excel(), or pd.read_json()!
-The file is NOT accessible by filename in the sandbox.
-
-Just write code that uses 'df' directly:
-- sns.barplot(data=df, x='category', y='sales')
-- plt.pie(df.groupby('cat')['val'].sum(), labels=df['cat'].unique())
-- sns.heatmap(df.corr(), annot=True)
-
-Always end with plt.show()`,
- },
- "analysis_type": map[string]interface{}{
- "type": "string",
- "description": "Predefined analysis type (used only if python_code is not provided)",
- "enum": []string{"summary", "correlation", "trend", "distribution", "outliers", "full"},
- "default": "summary",
- },
- "columns": map[string]interface{}{
- "type": "array",
- "description": "Optional: Specific columns to analyze (if empty, analyzes all columns)",
- "items": map[string]interface{}{
- "type": "string",
- },
- },
- },
- },
- Execute: executeDataAnalyst,
- }
-}
-
-func executeDataAnalyst(args map[string]interface{}) (string, error) {
- var csvData []byte
- var filename string
-
- // Try file_id first (uploaded files)
- if fileID, ok := args["file_id"].(string); ok && fileID != "" {
- content, name, err := GetUploadedFile(fileID)
- if err != nil {
- return "", fmt.Errorf("failed to get uploaded file: %w", err)
- }
- csvData = content
- filename = name
- } else if csvDataStr, ok := args["csv_data"].(string); ok && csvDataStr != "" {
- // Fallback to direct CSV data for small inline data
- csvData = []byte(csvDataStr)
- filename = "data.csv"
- } else {
- return "", fmt.Errorf("either file_id or csv_data is required")
- }
-
- // Create file map with CSV data
- files := map[string][]byte{
- filename: csvData,
- }
-
- var pythonCode string
-
- // Check for custom python_code first (LLM-generated visualizations)
- if customCode, ok := args["python_code"].(string); ok && customCode != "" {
- // Strip any pd.read_* calls - the LLM shouldn't load files, we pre-load them
- cleanedCode := stripDataLoadingCalls(customCode)
-
- // Use LLM-generated code with pre-loaded data
- pythonCode = fmt.Sprintf(`import pandas as pd
-import numpy as np
-import matplotlib.pyplot as plt
-import seaborn as sns
-
-# Set plot style
-plt.style.use('seaborn-v0_8-darkgrid')
-sns.set_palette("husl")
-
-# Load data
-df = pd.read_csv('%s')
-
-print("=" * 80)
-print("DATA ANALYSIS")
-print("=" * 80)
-print(f"\nDataset: %s")
-print(f"Shape: {df.shape[0]} rows × {df.shape[1]} columns")
-print(f"Columns: {list(df.columns)}")
-print()
-
-# Execute custom analysis code
-%s
-
-print("\n" + "=" * 80)
-print("✅ ANALYSIS COMPLETE")
-print("=" * 80)
-`, filename, filename, cleanedCode)
- } else {
- // Fallback to predefined analysis types
- analysisType := "summary"
- if at, ok := args["analysis_type"].(string); ok {
- analysisType = at
- }
-
- var columns []string
- if colsRaw, ok := args["columns"].([]interface{}); ok {
- for _, col := range colsRaw {
- columns = append(columns, fmt.Sprintf("%v", col))
- }
- }
-
- pythonCode = generateDataAnalysisCode([]string{filename}, analysisType, columns)
- }
-
- // Execute code with longer timeout for custom visualizations
- e2bService := e2b.GetE2BExecutorService()
- result, err := e2bService.ExecuteWithFiles(context.Background(), pythonCode, files, 120)
- if err != nil {
- return "", fmt.Errorf("failed to execute analysis: %w", err)
- }
-
- if !result.Success {
- errorMsg := result.Stderr
- if result.Error != nil {
- errorMsg = *result.Error
- }
-
- // Check for FileNotFoundError and provide helpful message
- if strings.Contains(errorMsg, "FileNotFoundError") || strings.Contains(errorMsg, "No such file or directory") {
- return "", fmt.Errorf(`analysis failed: %s
-
-💡 HINT: The data is already pre-loaded as 'df' (pandas DataFrame).
-Do NOT use pd.read_csv(), pd.read_excel(), or pd.read_json() - those files don't exist in the sandbox!
-Just use 'df' directly in your code. Example: sns.barplot(data=df, x='category', y='sales')`, errorMsg)
- }
-
- return "", fmt.Errorf("analysis failed: %s", errorMsg)
- }
-
- // Format response
- response := map[string]interface{}{
- "success": true,
- "analysis": result.Stdout,
- "plots": result.Plots,
- "plot_count": len(result.Plots),
- "filename": filename,
- }
-
- jsonResponse, _ := json.MarshalIndent(response, "", " ")
- return string(jsonResponse), nil
-}
-
-func generateDataAnalysisCode(fileNames []string, analysisType string, columns []string) string {
- // Determine the primary file
- primaryFile := fileNames[0]
-
- // Column filter
- colFilter := ""
- if len(columns) > 0 {
- colsStr := "'" + strings.Join(columns, "', '") + "'"
- colFilter = fmt.Sprintf("\ndf = df[[%s]]", colsStr)
- }
-
- // Base code
- code := fmt.Sprintf(`import pandas as pd
-import matplotlib.pyplot as plt
-import numpy as np
-import seaborn as sns
-
-# Set plot style
-plt.style.use('seaborn-v0_8-darkgrid')
-sns.set_palette("husl")
-
-# Load data
-df = pd.read_csv('%s')%s
-
-print("=" * 80)
-print("DATA ANALYSIS REPORT")
-print("=" * 80)
-print(f"\nDataset: %s")
-print(f"Shape: {df.shape[0]} rows × {df.shape[1]} columns")
-print()
-`, primaryFile, colFilter, primaryFile)
-
- switch analysisType {
- case "summary":
- code += `
-# Summary Statistics
-print("\n📊 SUMMARY STATISTICS")
-print("-" * 80)
-print(df.describe())
-
-# Data types
-print("\n📋 DATA TYPES")
-print("-" * 80)
-print(df.dtypes)
-
-# Missing values
-print("\n⚠️ MISSING VALUES")
-print("-" * 80)
-missing = df.isnull().sum()
-if missing.sum() > 0:
- print(missing[missing > 0])
-else:
- print("No missing values!")
-`
-
- case "correlation":
- code += `
-# Correlation Analysis
-print("\n🔗 CORRELATION ANALYSIS")
-print("-" * 80)
-
-numeric_cols = df.select_dtypes(include=[np.number]).columns
-if len(numeric_cols) > 1:
- corr = df[numeric_cols].corr()
- print(corr)
-
- # Correlation heatmap
- plt.figure(figsize=(10, 8))
- sns.heatmap(corr, annot=True, cmap='coolwarm', center=0, fmt='.2f')
- plt.title('Correlation Matrix')
- plt.tight_layout()
- plt.show()
-else:
- print("Not enough numeric columns for correlation analysis")
-`
-
- case "trend":
- code += `
-# Trend Analysis
-print("\n📈 TREND ANALYSIS")
-print("-" * 80)
-
-numeric_cols = df.select_dtypes(include=[np.number]).columns
-if len(numeric_cols) > 0:
- # Line plot for numeric columns
- fig, axes = plt.subplots(len(numeric_cols), 1, figsize=(12, 4 * len(numeric_cols)))
- if len(numeric_cols) == 1:
- axes = [axes]
-
- for ax, col in zip(axes, numeric_cols):
- df[col].plot(ax=ax, linewidth=2)
- ax.set_title(f'{col} - Trend Over Time')
- ax.set_xlabel('Index')
- ax.set_ylabel(col)
- ax.grid(True, alpha=0.3)
-
- plt.tight_layout()
- plt.show()
-
- print(f"Analyzed trends for {len(numeric_cols)} numeric columns")
-else:
- print("No numeric columns for trend analysis")
-`
-
- case "distribution":
- code += `
-# Distribution Analysis
-print("\n📊 DISTRIBUTION ANALYSIS")
-print("-" * 80)
-
-numeric_cols = df.select_dtypes(include=[np.number]).columns
-if len(numeric_cols) > 0:
- # Create subplots
- n_cols = min(3, len(numeric_cols))
- n_rows = (len(numeric_cols) + n_cols - 1) // n_cols
-
- fig, axes = plt.subplots(n_rows, n_cols, figsize=(5 * n_cols, 4 * n_rows))
- if len(numeric_cols) == 1:
- axes = [axes]
- else:
- axes = axes.flatten() if n_rows > 1 else axes
-
- for ax, col in zip(axes, numeric_cols):
- df[col].hist(ax=ax, bins=30, edgecolor='black', alpha=0.7)
- ax.set_title(f'{col} Distribution')
- ax.set_xlabel(col)
- ax.set_ylabel('Frequency')
- ax.grid(True, alpha=0.3)
-
- # Hide empty subplots
- for i in range(len(numeric_cols), len(axes)):
- axes[i].set_visible(False)
-
- plt.tight_layout()
- plt.show()
-
- # Print statistics
- for col in numeric_cols:
- print(f"\n{col}:")
- print(f" Mean: {df[col].mean():.2f}")
- print(f" Median: {df[col].median():.2f}")
- print(f" Std Dev: {df[col].std():.2f}")
- print(f" Min: {df[col].min():.2f}, Max: {df[col].max():.2f}")
-else:
- print("No numeric columns for distribution analysis")
-`
-
- case "outliers":
- code += `
-# Outlier Detection
-print("\n🚨 OUTLIER DETECTION")
-print("-" * 80)
-
-numeric_cols = df.select_dtypes(include=[np.number]).columns
-if len(numeric_cols) > 0:
- for col in numeric_cols:
- Q1 = df[col].quantile(0.25)
- Q3 = df[col].quantile(0.75)
- IQR = Q3 - Q1
- lower_bound = Q1 - 1.5 * IQR
- upper_bound = Q3 + 1.5 * IQR
-
- outliers = df[(df[col] < lower_bound) | (df[col] > upper_bound)]
-
- print(f"\n{col}:")
- print(f" Lower bound: {lower_bound:.2f}")
- print(f" Upper bound: {upper_bound:.2f}")
- print(f" Outliers found: {len(outliers)}")
-
- if len(outliers) > 0:
- print(f" Outlier values: {sorted(outliers[col].unique())[:10]}")
-
- # Box plot
- fig, axes = plt.subplots(len(numeric_cols), 1, figsize=(10, 3 * len(numeric_cols)))
- if len(numeric_cols) == 1:
- axes = [axes]
-
- for ax, col in zip(axes, numeric_cols):
- df.boxplot(column=col, ax=ax, vert=False)
- ax.set_title(f'{col} - Box Plot (Outlier Detection)')
- ax.grid(True, alpha=0.3)
-
- plt.tight_layout()
- plt.show()
-else:
- print("No numeric columns for outlier detection")
-`
-
- case "full":
- code += `
-# Full Analysis
-print("\n📊 SUMMARY STATISTICS")
-print("-" * 80)
-print(df.describe())
-
-print("\n📋 DATA TYPES")
-print("-" * 80)
-print(df.dtypes)
-
-print("\n⚠️ MISSING VALUES")
-print("-" * 80)
-missing = df.isnull().sum()
-if missing.sum() > 0:
- print(missing[missing > 0])
-else:
- print("✅ No missing values!")
-
-numeric_cols = df.select_dtypes(include=[np.number]).columns
-
-if len(numeric_cols) > 1:
- # Correlation heatmap
- print("\n🔗 CORRELATION ANALYSIS")
- print("-" * 80)
- corr = df[numeric_cols].corr()
- print(corr)
-
- plt.figure(figsize=(10, 8))
- sns.heatmap(corr, annot=True, cmap='coolwarm', center=0, fmt='.2f',
- square=True, linewidths=0.5)
- plt.title('Correlation Matrix', fontsize=16, fontweight='bold')
- plt.tight_layout()
- plt.show()
-
-if len(numeric_cols) > 0:
- # Distribution plots
- print("\n📊 DISTRIBUTION ANALYSIS")
- print("-" * 80)
-
- n_cols = min(3, len(numeric_cols))
- n_rows = (len(numeric_cols) + n_cols - 1) // n_cols
-
- fig, axes = plt.subplots(n_rows, n_cols, figsize=(5 * n_cols, 4 * n_rows))
- if len(numeric_cols) == 1:
- axes = [axes]
- else:
- axes = axes.flatten() if n_rows > 1 else axes
-
- for ax, col in zip(axes, numeric_cols):
- df[col].hist(ax=ax, bins=30, edgecolor='black', alpha=0.7, color='skyblue')
- ax.set_title(f'{col} Distribution', fontweight='bold')
- ax.set_xlabel(col)
- ax.set_ylabel('Frequency')
- ax.grid(True, alpha=0.3)
-
- for i in range(len(numeric_cols), len(axes)):
- axes[i].set_visible(False)
-
- plt.tight_layout()
- plt.show()
-
- for col in numeric_cols:
- print(f"{col}: μ={df[col].mean():.2f}, σ={df[col].std():.2f}")
-
-print("\n" + "=" * 80)
-print("✅ ANALYSIS COMPLETE")
-print("=" * 80)
-`
- }
-
- return code
-}
diff --git a/backend/internal/tools/describe_image_tool.go b/backend/internal/tools/describe_image_tool.go
deleted file mode 100644
index 6f18b310..00000000
--- a/backend/internal/tools/describe_image_tool.go
+++ /dev/null
@@ -1,372 +0,0 @@
-package tools
-
-import (
- "claraverse/internal/filecache"
- "claraverse/internal/vision"
- "encoding/json"
- "fmt"
- "io"
- "log"
- "net/http"
- "os"
- "path/filepath"
- "strings"
- "time"
-)
-
-// NewDescribeImageTool creates the describe_image tool for AI image analysis
-func NewDescribeImageTool() *Tool {
- return &Tool{
- Name: "describe_image",
- DisplayName: "Describe Image",
- Description: `Analyzes an image using AI vision and returns a detailed text description.
-
-Use this tool when the user asks you to:
-- Describe what's in an image
-- Analyze the content of a picture
-- Answer questions about an image
-- Identify objects, people, or text in an image
-
-Parameters:
-- image_url: A direct URL to an image on the web (e.g., "https://example.com/image.jpg"). Supports http/https URLs.
-- image_id: The image handle (e.g., "img-1") from the available images list. Use this for generated or previously referenced images.
-- file_id: Alternative - use the direct file ID from an upload response
-- question: Optional specific question about the image
-- detail: "brief" for 1-2 sentences, "detailed" for comprehensive description
-
-You must provide one of: image_url, image_id, OR file_id. Use image_url for web images, image_id for generated/edited images, file_id for uploaded files.`,
- Icon: "Image",
- Parameters: map[string]interface{}{
- "type": "object",
- "properties": map[string]interface{}{
- "image_url": map[string]interface{}{
- "type": "string",
- "description": "A direct URL to an image on the web (e.g., 'https://example.com/image.jpg'). Use this to analyze images from the internet.",
- },
- "image_id": map[string]interface{}{
- "type": "string",
- "description": "The image handle (e.g., 'img-1') from the available images list. Preferred for generated or edited images.",
- },
- "file_id": map[string]interface{}{
- "type": "string",
- "description": "Alternative: The direct file ID of the uploaded image. Use image_id when available.",
- },
- "question": map[string]interface{}{
- "type": "string",
- "description": "Optional specific question about the image (e.g., 'What color is the car?', 'How many people are in this photo?')",
- },
- "detail": map[string]interface{}{
- "type": "string",
- "enum": []string{"brief", "detailed"},
- "description": "Level of detail: 'brief' for 1-2 sentences, 'detailed' for comprehensive description. Default is 'detailed'",
- },
- },
- "required": []string{},
- },
- Execute: executeDescribeImage,
- Source: ToolSourceBuiltin,
- Category: "data_sources",
- Keywords: []string{"image", "describe", "analyze", "vision", "picture", "photo", "screenshot", "diagram", "chart", "url"},
- }
-}
-
-// Constants for URL image fetching
-const (
- describeImageMaxSize = 20 * 1024 * 1024 // 20MB for images
- describeImageTimeout = 30 * time.Second
-)
-
-func executeDescribeImage(args map[string]interface{}) (string, error) {
- // Extract image_url, image_id (handle like "img-1") or file_id (direct UUID)
- imageURL, hasImageURL := args["image_url"].(string)
- imageID, hasImageID := args["image_id"].(string)
- fileID, hasFileID := args["file_id"].(string)
-
- if (!hasImageURL || imageURL == "") && (!hasImageID || imageID == "") && (!hasFileID || fileID == "") {
- return "", fmt.Errorf("one of image_url, image_id, or file_id is required. Use image_url for web images, image_id (e.g., 'img-1') for generated images, or file_id for uploaded files")
- }
-
- // Extract optional question parameter
- question := ""
- if q, ok := args["question"].(string); ok {
- question = q
- }
-
- // Extract detail level (default to "detailed")
- detail := "detailed"
- if d, ok := args["detail"].(string); ok && (d == "brief" || d == "detailed") {
- detail = d
- }
-
- // Extract user context (injected by tool executor)
- userID, _ := args["__user_id__"].(string)
- convID, _ := args["__conversation_id__"].(string)
-
- // Variables to hold image data and metadata
- var imageData []byte
- var mimeType string
- var filename string
- var resolvedFileID string
- var sourceURL string
-
- // If image_url is provided, fetch the image directly from the web
- if hasImageURL && imageURL != "" {
- log.Printf("🖼️ [DESCRIBE-IMAGE] Fetching image from URL: %s", imageURL)
-
- data, mime, fname, err := fetchImageFromURL(imageURL)
- if err != nil {
- log.Printf("❌ [DESCRIBE-IMAGE] Failed to fetch image from URL: %v", err)
- return "", fmt.Errorf("failed to fetch image from URL: %v", err)
- }
-
- imageData = data
- mimeType = mime
- filename = fname
- sourceURL = imageURL
- resolvedFileID = "url-image"
-
- log.Printf("✅ [DESCRIBE-IMAGE] Fetched image from URL: %s (%d bytes, %s)", filename, len(imageData), mimeType)
- } else {
- // Get file cache service for image_id or file_id
- fileCacheService := filecache.GetService()
- var file *filecache.CachedFile
-
- // If image_id is provided, resolve it via the registry
- if hasImageID && imageID != "" {
- // Get image registry (injected by chat_service)
- registry, ok := args[ImageRegistryKey].(ImageRegistryInterface)
- if !ok || registry == nil {
- // Registry not available - try to use image_id as file_id fallback
- log.Printf("⚠️ [DESCRIBE-IMAGE] Image registry not available, treating image_id as file_id")
- resolvedFileID = imageID
- } else {
- // Look up the image by handle
- entry := registry.GetByHandle(convID, imageID)
- if entry == nil {
- // Provide helpful error message with available handles
- handles := registry.ListHandles(convID)
- if len(handles) == 0 {
- return "", fmt.Errorf("image '%s' not found. No images are available in this conversation. Please upload an image first or use file_id for direct file access", imageID)
- }
- return "", fmt.Errorf("image '%s' not found. Available images: %s", imageID, strings.Join(handles, ", "))
- }
- resolvedFileID = entry.FileID
- log.Printf("🖼️ [DESCRIBE-IMAGE] Resolved image_id '%s' to file_id '%s'", imageID, resolvedFileID)
- }
- } else {
- // Use file_id directly
- resolvedFileID = fileID
- }
-
- log.Printf("🖼️ [DESCRIBE-IMAGE] Analyzing image file_id=%s detail=%s (user=%s, conv=%s)", resolvedFileID, detail, userID, convID)
-
- // Get file from cache with proper validation
- if userID != "" && convID != "" {
- var err error
- file, err = fileCacheService.GetByUserAndConversation(resolvedFileID, userID, convID)
- if err != nil {
- // Try with just user validation
- file, err = fileCacheService.GetByUser(resolvedFileID, userID)
- if err != nil {
- // Try without validation for workflow context
- file, _ = fileCacheService.Get(resolvedFileID)
- if file != nil && file.UserID != "" && file.UserID != userID {
- log.Printf("🚫 [DESCRIBE-IMAGE] Access denied: file %s belongs to different user", resolvedFileID)
- return "", fmt.Errorf("access denied: you don't have permission to access this file")
- }
- }
- }
- } else if userID != "" {
- var err error
- file, err = fileCacheService.GetByUser(resolvedFileID, userID)
- if err != nil {
- file, _ = fileCacheService.Get(resolvedFileID)
- }
- } else {
- file, _ = fileCacheService.Get(resolvedFileID)
- }
-
- if file == nil {
- log.Printf("❌ [DESCRIBE-IMAGE] File not found: %s", resolvedFileID)
- if hasImageID && imageID != "" {
- return "", fmt.Errorf("image '%s' has expired or is no longer available. Images are cached for 30 minutes. Please upload or generate the image again", imageID)
- }
- return "", fmt.Errorf("image file not found or has expired. Files are only available for 30 minutes after upload")
- }
-
- // Validate it's an image
- if !strings.HasPrefix(file.MimeType, "image/") {
- log.Printf("⚠️ [DESCRIBE-IMAGE] File is not an image: %s (%s)", resolvedFileID, file.MimeType)
- return "", fmt.Errorf("file is not an image (type: %s). Use read_document for documents or read_data_file for data files", file.MimeType)
- }
-
- // Read image data from disk
- if file.FilePath == "" {
- return "", fmt.Errorf("image file path not available")
- }
-
- var err error
- imageData, err = os.ReadFile(file.FilePath)
- if err != nil {
- log.Printf("❌ [DESCRIBE-IMAGE] Failed to read image from disk: %v (path: %s)", err, file.FilePath)
- return "", fmt.Errorf("image file has expired or been deleted. Please upload or generate the image again")
- }
-
- mimeType = file.MimeType
- filename = file.Filename
- }
-
- // Get the vision service
- visionService := vision.GetService()
- if visionService == nil {
- return "", fmt.Errorf("vision service not available. Please configure a vision-capable model (e.g., GPT-4o)")
- }
-
- // Build the request
- req := &vision.DescribeImageRequest{
- ImageData: imageData,
- MimeType: mimeType,
- Question: question,
- Detail: detail,
- }
-
- // Call vision service
- result, err := visionService.DescribeImage(req)
- if err != nil {
- log.Printf("❌ [DESCRIBE-IMAGE] Vision analysis failed: %v", err)
- return "", fmt.Errorf("failed to analyze image: %v", err)
- }
-
- // Build response
- response := map[string]interface{}{
- "success": true,
- "filename": filename,
- "mime_type": mimeType,
- "description": result.Description,
- "model": result.Model,
- "provider": result.Provider,
- }
-
- // Include source-specific fields
- if sourceURL != "" {
- response["source_url"] = sourceURL
- } else {
- response["file_id"] = resolvedFileID
- }
-
- // Include image_id if it was used
- if hasImageID && imageID != "" {
- response["image_id"] = imageID
- }
-
- if question != "" {
- response["question"] = question
- }
-
- responseJSON, err := json.Marshal(response)
- if err != nil {
- return "", fmt.Errorf("failed to marshal response: %w", err)
- }
-
- log.Printf("✅ [DESCRIBE-IMAGE] Successfully described image %s using %s", filename, result.Model)
-
- return string(responseJSON), nil
-}
-
-// fetchImageFromURL downloads an image from a URL and returns the data, mime type, and filename
-func fetchImageFromURL(urlStr string) ([]byte, string, string, error) {
- // Validate URL using the existing validation function from download_file_tool
- parsedURL, err := validateDownloadURL(urlStr)
- if err != nil {
- return nil, "", "", fmt.Errorf("invalid URL: %v", err)
- }
-
- // Create HTTP client with timeout
- client := &http.Client{
- Timeout: describeImageTimeout,
- CheckRedirect: func(req *http.Request, via []*http.Request) error {
- if len(via) >= 5 {
- return fmt.Errorf("too many redirects")
- }
- // Validate redirect URL
- if _, err := validateDownloadURL(req.URL.String()); err != nil {
- return fmt.Errorf("redirect blocked: %v", err)
- }
- return nil
- },
- }
-
- // Create request
- req, err := http.NewRequest("GET", parsedURL.String(), nil)
- if err != nil {
- return nil, "", "", fmt.Errorf("failed to create request: %v", err)
- }
-
- // Set a reasonable User-Agent
- req.Header.Set("User-Agent", "ClaraVerse/1.0 (Image Analyzer)")
-
- // Make request
- resp, err := client.Do(req)
- if err != nil {
- return nil, "", "", fmt.Errorf("failed to fetch image: %v", err)
- }
- defer resp.Body.Close()
-
- if resp.StatusCode != http.StatusOK {
- return nil, "", "", fmt.Errorf("failed to fetch image: HTTP %d", resp.StatusCode)
- }
-
- // Get content type
- contentType := resp.Header.Get("Content-Type")
- if contentType == "" {
- contentType = "application/octet-stream"
- }
- // Strip charset suffix if present
- if idx := strings.Index(contentType, ";"); idx != -1 {
- contentType = strings.TrimSpace(contentType[:idx])
- }
-
- // Validate it's an image
- if !strings.HasPrefix(contentType, "image/") {
- // Try to detect from URL extension
- ext := strings.ToLower(filepath.Ext(parsedURL.Path))
- switch ext {
- case ".jpg", ".jpeg":
- contentType = "image/jpeg"
- case ".png":
- contentType = "image/png"
- case ".gif":
- contentType = "image/gif"
- case ".webp":
- contentType = "image/webp"
- case ".svg":
- contentType = "image/svg+xml"
- case ".bmp":
- contentType = "image/bmp"
- default:
- return nil, "", "", fmt.Errorf("URL does not point to an image (content-type: %s)", contentType)
- }
- }
-
- // Check content length if available
- if resp.ContentLength > describeImageMaxSize {
- return nil, "", "", fmt.Errorf("image too large: %d bytes (max %d bytes)", resp.ContentLength, describeImageMaxSize)
- }
-
- // Read body with size limit
- limitedReader := io.LimitReader(resp.Body, describeImageMaxSize+1)
- content, err := io.ReadAll(limitedReader)
- if err != nil {
- return nil, "", "", fmt.Errorf("failed to read image: %v", err)
- }
-
- if int64(len(content)) > describeImageMaxSize {
- return nil, "", "", fmt.Errorf("image too large: max %d bytes", describeImageMaxSize)
- }
-
- // Extract filename from URL or Content-Disposition
- filename := extractFilename(parsedURL, contentType, resp.Header.Get("Content-Disposition"))
- filename = sanitizeFilename(filename)
-
- return content, contentType, filename, nil
-}
diff --git a/backend/internal/tools/discord_tool.go b/backend/internal/tools/discord_tool.go
deleted file mode 100644
index 32161b07..00000000
--- a/backend/internal/tools/discord_tool.go
+++ /dev/null
@@ -1,431 +0,0 @@
-package tools
-
-import (
- "bytes"
- "encoding/base64"
- "encoding/json"
- "fmt"
- "io"
- "mime/multipart"
- "net/http"
- "os"
- "strings"
- "time"
-)
-
-// NewDiscordTool creates a Discord webhook messaging tool
-func NewDiscordTool() *Tool {
- return &Tool{
- Name: "send_discord_message",
- DisplayName: "Send Discord Message",
- Description: "Send a message to Discord via webhook. Message content is limited to 2000 characters max (Discord API limit). Just provide the message content - webhook authentication is handled automatically via configured credentials. Do NOT ask the user for webhook URLs. Supports embeds for rich formatting.",
- Icon: "MessageCircle",
- Source: ToolSourceBuiltin,
- Category: "integration",
- Keywords: []string{"discord", "message", "chat", "notify", "webhook", "channel", "bot", "notification"},
- Parameters: map[string]interface{}{
- "type": "object",
- "properties": map[string]interface{}{
- "credential_id": map[string]interface{}{
- "type": "string",
- "description": "INTERNAL: Auto-injected by system. Do not set manually.",
- },
- "webhook_url": map[string]interface{}{
- "type": "string",
- "description": "INTERNAL: Resolved from credentials. Do not ask user for this.",
- },
- "content": map[string]interface{}{
- "type": "string",
- "description": "Message content (max 2000 characters). This is the main text message.",
- },
- "username": map[string]interface{}{
- "type": "string",
- "description": "Override the default webhook username (optional)",
- },
- "avatar_url": map[string]interface{}{
- "type": "string",
- "description": "Override the default webhook avatar URL (optional)",
- },
- "embed_title": map[string]interface{}{
- "type": "string",
- "description": "Title for an embed (optional, for rich formatting)",
- },
- "embed_description": map[string]interface{}{
- "type": "string",
- "description": "Description for an embed (optional, max 4096 characters)",
- },
- "embed_color": map[string]interface{}{
- "type": "number",
- "description": "Embed color as decimal (optional, e.g., 5814783 for blue)",
- },
- "image_data": map[string]interface{}{
- "type": "string",
- "description": "Base64 encoded image data to attach (optional). Can include data URI prefix or raw base64.",
- },
- "image_filename": map[string]interface{}{
- "type": "string",
- "description": "Filename for the attached image (optional, defaults to 'chart.png')",
- },
- "file_url": map[string]interface{}{
- "type": "string",
- "description": "URL to download a file to attach (optional). Supports both absolute URLs and relative paths starting with /api/files/. For relative paths, the backend URL will be automatically resolved.",
- },
- "file_name": map[string]interface{}{
- "type": "string",
- "description": "Filename for the attached file from URL (optional, will be inferred from URL if not provided)",
- },
- },
- "required": []string{},
- },
- Execute: executeDiscordMessage,
- }
-}
-
-func executeDiscordMessage(args map[string]interface{}) (string, error) {
- // Resolve webhook URL from credential or direct parameter
- webhookURL, err := ResolveWebhookURL(args, "discord")
- if err != nil {
- // Fallback: check for direct webhook_url if credential resolution failed
- if url, ok := args["webhook_url"].(string); ok && url != "" {
- webhookURL = url
- } else {
- return "", fmt.Errorf("failed to get webhook URL: %w", err)
- }
- }
-
- // Validate Discord webhook URL
- if !strings.Contains(webhookURL, "discord.com/api/webhooks/") && !strings.Contains(webhookURL, "discordapp.com/api/webhooks/") {
- return "", fmt.Errorf("invalid Discord webhook URL")
- }
-
- // Extract content (optional now since we might just send an image)
- content, _ := args["content"].(string)
-
- // Truncate content if too long (Discord limit is 2000)
- if len(content) > 2000 {
- content = content[:1997] + "..."
- }
-
- // Check for image data
- imageData, hasImage := args["image_data"].(string)
- imageFilename := "chart.png"
- if fn, ok := args["image_filename"].(string); ok && fn != "" {
- imageFilename = fn
- }
-
- // Check for file URL
- fileURL, hasFileURL := args["file_url"].(string)
- var fileData []byte
- fileName := ""
- if fn, ok := args["file_name"].(string); ok && fn != "" {
- fileName = fn
- }
-
- // Fetch file from URL if provided
- if hasFileURL && fileURL != "" {
- var fetchErr error
- fileData, fileName, fetchErr = fetchFileFromURL(fileURL, fileName)
- if fetchErr != nil {
- return "", fmt.Errorf("failed to fetch file from URL: %w", fetchErr)
- }
- }
-
- // Build Discord webhook payload
- payload := map[string]interface{}{}
- if content != "" {
- payload["content"] = content
- }
-
- // Optional username override
- if username, ok := args["username"].(string); ok && username != "" {
- payload["username"] = username
- }
-
- // Optional avatar override
- if avatarURL, ok := args["avatar_url"].(string); ok && avatarURL != "" {
- payload["avatar_url"] = avatarURL
- }
-
- // Build embed if any embed fields provided
- embed := make(map[string]interface{})
- hasEmbed := false
-
- if embedTitle, ok := args["embed_title"].(string); ok && embedTitle != "" {
- embed["title"] = embedTitle
- hasEmbed = true
- }
-
- if embedDesc, ok := args["embed_description"].(string); ok && embedDesc != "" {
- // Truncate embed description if too long (Discord limit is 4096)
- if len(embedDesc) > 4096 {
- embedDesc = embedDesc[:4093] + "..."
- }
- embed["description"] = embedDesc
- hasEmbed = true
- }
-
- if embedColor, ok := args["embed_color"].(float64); ok {
- embed["color"] = int(embedColor)
- hasEmbed = true
- }
-
- // If we have an image, reference it in the embed
- if hasImage && imageData != "" {
- if !hasEmbed {
- embed["title"] = "Generated Chart"
- hasEmbed = true
- }
- // Reference the attached image in the embed
- embed["image"] = map[string]interface{}{
- "url": "attachment://" + imageFilename,
- }
- }
-
- if hasEmbed {
- embed["timestamp"] = time.Now().UTC().Format(time.RFC3339)
- payload["embeds"] = []map[string]interface{}{embed}
- }
-
- // Require at least content, image, file, or embed
- hasFile := len(fileData) > 0
- if content == "" && !hasImage && !hasFile && !hasEmbed {
- return "", fmt.Errorf("either content, image_data, file_url, or embed is required")
- }
-
- // Create HTTP client
- client := &http.Client{
- Timeout: 60 * time.Second,
- }
-
- var req *http.Request
-
- if hasFile {
- // Send with multipart/form-data for file attachment
- req, err = createMultipartRequestWithFile(webhookURL, payload, fileData, fileName)
- } else if hasImage && imageData != "" {
- // Send with multipart/form-data for image attachment
- req, err = createMultipartRequest(webhookURL, payload, imageData, imageFilename)
- } else {
- // Send as JSON (no image)
- jsonPayload, jsonErr := json.Marshal(payload)
- if jsonErr != nil {
- return "", fmt.Errorf("failed to serialize payload: %w", jsonErr)
- }
- req, err = http.NewRequest("POST", webhookURL, bytes.NewBuffer(jsonPayload))
- if err == nil {
- req.Header.Set("Content-Type", "application/json")
- }
- }
-
- if err != nil {
- return "", fmt.Errorf("failed to create request: %w", err)
- }
-
- // Execute request
- resp, err := client.Do(req)
- if err != nil {
- return "", fmt.Errorf("request failed: %w", err)
- }
- defer resp.Body.Close()
-
- // Read response body
- respBody, err := io.ReadAll(resp.Body)
- if err != nil {
- return "", fmt.Errorf("failed to read response: %w", err)
- }
-
- // Build result
- success := resp.StatusCode >= 200 && resp.StatusCode < 300
- result := map[string]interface{}{
- "success": success,
- "status_code": resp.StatusCode,
- "status": resp.Status,
- "message_sent": success,
- }
-
- if content != "" {
- result["content_length"] = len(content)
- }
- if hasFile {
- result["file_attached"] = true
- result["file_name"] = fileName
- result["file_size"] = len(fileData)
- }
- if hasImage {
- result["image_attached"] = true
- result["image_filename"] = imageFilename
- }
-
- // Include response body if there's an error
- if !success && len(respBody) > 0 {
- result["error"] = string(respBody)
- }
-
- // Add success message
- if success {
- if hasFile {
- result["message"] = fmt.Sprintf("Discord message with file '%s' sent successfully", fileName)
- } else if hasImage {
- result["message"] = "Discord message with image sent successfully"
- } else {
- result["message"] = "Discord message sent successfully"
- }
- }
-
- jsonResult, _ := json.MarshalIndent(result, "", " ")
- return string(jsonResult), nil
-}
-
-// fetchFileFromURL fetches a file from a URL (supports relative paths for internal files)
-func fetchFileFromURL(fileURL, providedFileName string) ([]byte, string, error) {
- // Resolve relative URLs to absolute URLs
- actualURL := fileURL
- if strings.HasPrefix(fileURL, "/api/") {
- // Use BACKEND_URL env var for internal API calls
- backendURL := os.Getenv("BACKEND_URL")
- if backendURL == "" {
- backendURL = "http://localhost:3001"
- }
- actualURL = backendURL + fileURL
- }
-
- // Create HTTP client
- client := &http.Client{
- Timeout: 30 * time.Second,
- }
-
- resp, err := client.Get(actualURL)
- if err != nil {
- return nil, "", fmt.Errorf("failed to fetch file: %w", err)
- }
- defer resp.Body.Close()
-
- if resp.StatusCode != http.StatusOK {
- return nil, "", fmt.Errorf("failed to fetch file: status %d", resp.StatusCode)
- }
-
- // Read the file content
- data, err := io.ReadAll(resp.Body)
- if err != nil {
- return nil, "", fmt.Errorf("failed to read file: %w", err)
- }
-
- // Determine filename
- fileName := providedFileName
- if fileName == "" {
- // Try to get from Content-Disposition header
- if cd := resp.Header.Get("Content-Disposition"); cd != "" {
- if strings.Contains(cd, "filename=") {
- parts := strings.Split(cd, "filename=")
- if len(parts) > 1 {
- fileName = strings.Trim(parts[1], "\"' ")
- }
- }
- }
- // Fallback: extract from URL
- if fileName == "" {
- parts := strings.Split(strings.Split(fileURL, "?")[0], "/")
- if len(parts) > 0 {
- fileName = parts[len(parts)-1]
- }
- }
- // Final fallback
- if fileName == "" {
- fileName = "attachment"
- }
- }
-
- return data, fileName, nil
-}
-
-// createMultipartRequestWithFile creates a multipart request with JSON payload and raw file data
-func createMultipartRequestWithFile(webhookURL string, payload map[string]interface{}, fileData []byte, filename string) (*http.Request, error) {
- // Create multipart form
- var body bytes.Buffer
- writer := multipart.NewWriter(&body)
-
- // Add payload_json field
- payloadJSON, err := json.Marshal(payload)
- if err != nil {
- return nil, fmt.Errorf("failed to serialize payload: %w", err)
- }
-
- if err := writer.WriteField("payload_json", string(payloadJSON)); err != nil {
- return nil, fmt.Errorf("failed to write payload field: %w", err)
- }
-
- // Add file attachment
- part, err := writer.CreateFormFile("files[0]", filename)
- if err != nil {
- return nil, fmt.Errorf("failed to create form file: %w", err)
- }
-
- if _, err := part.Write(fileData); err != nil {
- return nil, fmt.Errorf("failed to write file data: %w", err)
- }
-
- if err := writer.Close(); err != nil {
- return nil, fmt.Errorf("failed to close multipart writer: %w", err)
- }
-
- // Create request
- req, err := http.NewRequest("POST", webhookURL, &body)
- if err != nil {
- return nil, err
- }
-
- req.Header.Set("Content-Type", writer.FormDataContentType())
- return req, nil
-}
-
-// createMultipartRequest creates a multipart request with JSON payload and image attachment
-func createMultipartRequest(webhookURL string, payload map[string]interface{}, imageData, filename string) (*http.Request, error) {
- // Decode base64 image data
- // Remove data URI prefix if present
- imageData = strings.TrimPrefix(imageData, "data:image/png;base64,")
- imageData = strings.TrimPrefix(imageData, "data:image/jpeg;base64,")
- imageData = strings.TrimPrefix(imageData, "data:image/jpg;base64,")
- imageData = strings.TrimPrefix(imageData, "data:image/gif;base64,")
-
- imageBytes, err := base64.StdEncoding.DecodeString(imageData)
- if err != nil {
- return nil, fmt.Errorf("failed to decode base64 image: %w", err)
- }
-
- // Create multipart form
- var body bytes.Buffer
- writer := multipart.NewWriter(&body)
-
- // Add payload_json field
- payloadJSON, err := json.Marshal(payload)
- if err != nil {
- return nil, fmt.Errorf("failed to serialize payload: %w", err)
- }
-
- if err := writer.WriteField("payload_json", string(payloadJSON)); err != nil {
- return nil, fmt.Errorf("failed to write payload field: %w", err)
- }
-
- // Add file attachment
- part, err := writer.CreateFormFile("files[0]", filename)
- if err != nil {
- return nil, fmt.Errorf("failed to create form file: %w", err)
- }
-
- if _, err := part.Write(imageBytes); err != nil {
- return nil, fmt.Errorf("failed to write image data: %w", err)
- }
-
- if err := writer.Close(); err != nil {
- return nil, fmt.Errorf("failed to close multipart writer: %w", err)
- }
-
- // Create request
- req, err := http.NewRequest("POST", webhookURL, &body)
- if err != nil {
- return nil, err
- }
-
- req.Header.Set("Content-Type", writer.FormDataContentType())
- return req, nil
-}
diff --git a/backend/internal/tools/document_tool.go b/backend/internal/tools/document_tool.go
deleted file mode 100644
index b99bf090..00000000
--- a/backend/internal/tools/document_tool.go
+++ /dev/null
@@ -1,139 +0,0 @@
-package tools
-
-import (
- "claraverse/internal/document"
- "claraverse/internal/securefile"
- "encoding/json"
- "fmt"
- "log"
- "os"
-)
-
-// NewDocumentTool creates the create_document tool
-func NewDocumentTool() *Tool {
- return &Tool{
- Name: "create_document",
- DisplayName: "Create Document",
- Description: `Creates a professional PDF document from custom HTML content. Full creative control with HTML/CSS for maximum design flexibility.
-
-Perfect for:
-- Professional reports with custom branding
-- Invoices and receipts with styled layouts
-- Legal documents with precise formatting
-- Technical documentation with code blocks
-- Certificates and formal documents
-- Creative documents with custom designs
-
-You can use any HTML/CSS - inline styles, flexbox/grid layouts, custom fonts, colors, gradients, tables, images (base64 or URLs), etc. The document is rendered as a standard A4 portrait PDF and stored for 30 days with an access code for download.
-
-**Page Break Control:**
-- Use CSS 'page-break-before', 'page-break-after', or 'page-break-inside' to control page breaks
-- Add 'page-break-inside: avoid' to prevent elements from being split across pages
-- Use 'page-break-after: always' to force a new page after an element
-- Tables, images, and code blocks should have 'page-break-inside: avoid' to prevent awkward cuts
-- Example:
Content that stays together
`,
- Icon: "FileText",
- Parameters: map[string]interface{}{
- "type": "object",
- "properties": map[string]interface{}{
- "html": map[string]interface{}{
- "type": "string",
- "description": "The document content as HTML. Can include inline CSS (
Title Slide
"},
- {"html": "
Topic 1
Content here
"},
- {"html": "
Topic 2
More content
"},
- {"html": "
Topic 3
"},
- {"html": "
Thank You
"}
- ]
-}
-
-**REQUIREMENTS:**
-- MUST create 5-15 pages (NOT just 1-2!)
-- Each page MUST be a complete HTML document starting with and ending with
-- Include with
+
+
+
+
+
+ 
+
+
+
+
Why ClaraVerse?
+
+
+ ### **Clara AI**
+ **Smart Assistant**
+
+ `Advanced Chat` `Code Gen`
+ `Voice` `Files` `MCP Tools` `Model Manager`
+
+ 
+
+ ### **LumaUI**
+ **Code Builder**
+
+ `Live Preview` `WebContainer`
+ `React/Vue/JS` `Hot Reload`
+
+ 
+
+ ### **ComfyUI**
+ **Image Studio**
+
+ `SDXL/Flux` `LoRA`
+ `Batch Gen` `100% Private`
+
+ 
+
+ ### **N8N**
+ **Automation**
+
+ `Visual Workflows` `1000+ APIs`
+ `Scheduling` `Data Pipelines`
+
+ 
+
+ ### **Agent Studio**
+ **AI Agents**
+
+ `Autonomous Agents` `MCP Integration`
+ `Visual Designer` `Custom Nodes`
+
+ 
+
+ ### **Community**
+ **Sharing Hub**
+
+ `Content Sharing` `Custom Nodes`
+ `Workflows` `Collaboration`
+
+ 
+
+ ### **Widgets**
+ **Dashboard**
+
+ `Drag & Drop` `Live Updates`
+ `Persistent` `Customizable`
+
+ 
+
+ ### **MCP Ecosystem**
+ **Tool Integration**
+
+ `20+ MCP Servers` `Desktop Automation`
+ `Browser Control` `File System`
+
+ 
Browser-Local Storage: True Zero-Knowledge Privacy
+
Feature Showcase in a Nutshell
+## 🤝 **Community & Support**
+[](https://discord.gg/j633fsrAne)
+[](https://www.reddit.com/r/claraverse/)
+[](https://twitter.com/claraverse)
+[](https://youtube.com/@claraverse)
-*Chat naturally with GPT-4, Claude, Gemini, and more - all in one unified interface*
+### **📚 Resources**
+
+[Documentation](https://github.com/badboysm890/ClaraVerse/tree/main/docs) •
+[API Reference](https://github.com/badboysm890/ClaraVerse/tree/main/docs/api) •
+[Tutorials](https://github.com/badboysm890/ClaraVerse/tree/main/docs/tutorials) •
+[FAQ](https://github.com/badboysm890/ClaraVerse/wiki/FAQ)
+
+
+## 💝 **Contributors**
-*Clara remembers your preferences and conversation context across sessions*
-
+
+
+
+### **How to Contribute**
-### Smart Multi-Agent Orchestration, Chat with Clara to Create your crew of agents
-
+We love contributions! Check our [Contributing Guide](CONTRIBUTING.md) to get started.
-*Coordinate multiple specialized AI agents for complex workflows*
-
+## 🙏 **Acknowledgments**
+
+
Key Features
-
-### 
**Privacy & Security First**
-- **Browser-Local Storage**: Conversations stored in IndexedDB, never touch server—even admins can't read your chats
-- **Zero-Knowledge Architecture**: Server only proxies LLM API calls, doesn't log or store message content
-- **Per-Conversation Privacy**: Choose browser-local (100% offline) or cloud-sync (encrypted backup) per chat
-- **Local JWT Authentication**: Secure authentication with Argon2id password hashing
-- **True Offline Mode**: Works completely air-gapped after initial load—no server dependency
-
-### 
**Universal AI Access**
-- **Multi-Provider Support**: OpenAI, Anthropic Claude, Google Gemini, and any OpenAI-compatible endpoint
-- **Bring Your Own Key (BYOK)**: Use existing API accounts or free local models
-- **400+ Models Available**: From GPT-4o to Llama, Mistral, and specialized models
-- **Unified Interface**: One workspace for all your AI needs
-
-### 
**Advanced Capabilities**
-- **Visual Workflow Builder**: Drag-and-drop workflow designer with auto-layout—chat to create, visual editor to refine
-- **Hybrid Block Architecture**: Variable blocks, LLM blocks, and Code blocks (execute tools without LLM overhead)
-- **Interactive Prompts**: AI asks clarifying questions mid-conversation with typed forms (text, select, checkbox)
-- **Real-Time Streaming**: WebSocket-based chat with automatic reconnection and conversation resume
-- **Tool Execution**: Code generation, image creation, web search, file analysis with real-time status tracking
-- **Response Versioning**: Generate, compare, and track multiple versions (add details, make concise, no search)
-
-### 
**Cross-Platform & Flexible**
-- **Desktop Apps**: Native Windows, macOS, and Linux applications
-- **Web Interface**: Browser-based access via React frontend
-- **Mobile Ready**: Responsive design for tablets and phones
-- **P2P Sync**: Device-to-device synchronization without cloud storage
-- **Enterprise Deployment**: Self-host for complete organizational control
-
-### 
**Developer-Friendly**
-- **MCP Bridge**: Native Model Context Protocol support—connect any MCP-compatible tool seamlessly
-- **Open API**: RESTful + WebSocket APIs for custom integrations
-- **Plugin System**: Extend functionality with custom tools and connectors
-- **Docker Support**: One-command deployment with `docker compose`
-- **GitHub, Slack, Notion Integration**: Pre-built connectors for your workflow
-- **Database Connections**: Query and analyze data with AI assistance
+**MIT License** - Use it, modify it, sell it, we don't mind! (If want you to give credit, that would be awesome but not mandatory 😉)
+
+See [LICENSE](LICENSE) for details.
+
+
Our Mission
+
+
-
**Best-in-Class Interface**
-- Intuitive, polished UI that rivals ChatGPT and Claude
-- Real-time streaming with automatic reconnection
-- Smart context management across sessions
-- Multi-modal support (text, images, code, files)
-- Clara Memory: Remembers what matters, archives what doesn't
+
**Extensibility That Matters**
-- **MCP Bridge**: Native Model Context Protocol integration for seamless tool connections
-- Multi-agent orchestration: Coordinate specialized AI agents for complex workflows
-- 400+ models: OpenAI, Anthropic, Google, Gemini, and any OpenAI-compatible endpoint
-- BYOK: Use your own API keys or completely free local models
-- Plugin ecosystem: GitHub, Slack, Notion, databases, and custom integrations
+
Documentation
+### **Building ClaraVerse**
-| Resource | Description |
-|----------|-------------|
-| [
Architecture Guide](backend/docs/ARCHITECTURE.md) | System design and component overview |
-| [
API Reference](backend/docs/API_REFERENCE.md) | REST and WebSocket API documentation |
-| [
Docker Guide](docs/DOCKER.md) | Comprehensive Docker deployment |
-| [
Admin Guide](backend/docs/ADMIN_GUIDE.md) | System administration and configuration |
-| [
Developer Guide](backend/docs/DEVELOPER_GUIDE.md) | Contributing and local development |
-| [
Quick Reference](backend/docs/QUICK_REFERENCE.md) | Common commands and workflows |
+
Use Cases
-
-### For Developers
-- **Code Review & Debugging**: Get instant feedback on code quality
-- **Documentation Generation**: Auto-generate docs from codebases
-- **API Integration**: Connect ClaraVerse to your development workflow
-- **Database Analysis**: Query and visualize data with AI assistance
-
-### For Businesses
-- **Zero-Liability Hosting**: Host for teams without server-side chat storage—admins can't access conversations
-- **True Data Sovereignty**: Browser-local mode means data never leaves employee devices, even when self-hosted
-- **Simplified Compliance**: No message retention in database = easier GDPR/HIPAA compliance
-- **Team Collaboration**: Shared AI workspace with access control and privacy guarantees
-- **Custom Integrations**: Connect to Slack, Notion, GitHub, CRMs via visual workflow builder
-- **Cost Control**: BYOK means you control AI spending with your own API keys
-
-### For Privacy Advocates
-- **Browser-Local Storage**: Conversations never touch server—even when self-hosted, admins can't read chats
-- **Zero-Knowledge Architecture**: Server only proxies API calls, doesn't log or store message content
-- **Per-Conversation Privacy**: Choose offline browser-local or encrypted cloud-sync per conversation
-- **Air-Gapped Operation**: Works 100% offline after initial load—no server dependency
-- **Open Source**: Verify zero-knowledge claims yourself or hire security auditors
-- **No Database Retention**: Messages not stored in MongoDB—simplified compliance for GDPR/HIPAA
-
-### For Researchers
-- **Experiment with Models**: Test 400+ models in one interface
-- **Dataset Analysis**: Process large datasets with AI assistance
-- **Literature Review**: Search and summarize academic papers
-- **Reproducible Workflows**: Save and share AI-assisted research processes
+
**Report bugs** and suggest features
-- 
**Share** with colleagues and on social media
-- 
**Sponsor** development ([GitHub Sponsors](https://github.com/sponsors/claraverse-space))
-- 
**Contribute** code, docs, or designs
+
-ClaraVerse is built with modern, production-ready technologies:
+### **Solo Developer**
-```
-┌─────────────────────────────────────────────────────────────┐
-│ Frontend Layer │
-│ React 19 + TypeScript + Tailwind CSS 4 │
-│ Zustand State + React Router 7 │
-└────────────────────┬────────────────────────────────────────┘
- │ WebSocket + REST API
-┌────────────────────▼────────────────────────────────────────┐
-│ Backend Layer │
-│ Go 1.24 + Fiber Framework │
-│ Real-time Streaming + Tool Execution │
-└────────────────────┬────────────────────────────────────────┘
- │
- ┌────────────┼────────────┐
- │ │ │
- ┌────▼───┐ ┌───▼────┐ ┌───▼────┐
- │MongoDB │ │ Redis │ │SearXNG │
- │Storage │ │ Jobs │ │ Search │
- └────────┘ └────────┘ └────────┘
-```
+`9-6 Day Job` `Night Coding`
+`Weekend Builds` `3 AM Commits`
-**Technology Stack:**
-- **Frontend**: 
React 19, 
TypeScript, 
Vite 7, 
Tailwind CSS 4, 
Zustand 5
-- **Backend**: 
Go 1.24, 
Fiber (web framework), WebSocket streaming
-- **Database**: 
MongoDB for persistence, 
MySQL for models/providers, 
Redis for caching/jobs
-- **Services**: 
SearXNG (search), E2B Local Docker (code execution - no API key!)
-- **Deployment**: 
Docker Compose, 
Nginx reverse proxy
-- **Auth**: 
Local JWT with Argon2id password hashing (v2.0 - fully local, no Supabase)
+**One person. Three platforms.**
+**Just having fun in doing this.**
----
+
Features in Detail
-
-### Real-Time Streaming Chat
-
-Experience instant AI responses with our WebSocket-based architecture:
-
-- **Chunked Streaming**: See responses as they're generated
-- **Connection Recovery**: Automatic reconnection with conversation resume
-- **Heartbeat System**: Maintains stable connections through proxies
-- **Multi-User Support**: Concurrent conversations without interference
-
-### Tool Execution Engine
-
-Extend AI capabilities beyond text:
-
-| Tool | Description | Example |
-|------|-------------|---------|
-| **Code Generation** | Execute Python, JavaScript, Go code in sandboxed E2B environment | "Write and run a script to analyze this CSV" |
-| **Image Generation** | Create images with DALL-E, Stable Diffusion, or local models | "Generate a logo for my startup" |
-| **Web Search** | Real-time internet search via SearXNG | "What are the latest AI developments?" |
-| **File Analysis** | Process PDFs, images, documents with vision models | "Summarize this 50-page report" |
-| **Data Query** | Connect to databases and run SQL queries | "Show sales trends from our PostgreSQL" |
-
-### Bring Your Own Key (BYOK)
-
-Use your existing AI subscriptions:
-
-1. Add your API keys in `backend/providers.json`
-2. Configure model preferences and rate limits
-3. Switch between providers seamlessly
-4. Or use completely free local models (Ollama, LM Studio)
-
-```json
-{
- "providers": [
- {
- "name": "OpenAI",
- "api_key": "sk-your-key",
- "models": ["gpt-4o", "gpt-4o-mini"]
- },
- {
- "name": "Anthropic",
- "api_key": "your-key",
- "models": ["claude-3-5-sonnet", "claude-3-opus"]
- }
- ]
-}
-```
+
-### Multi-Agent Orchestration
+### **Community Heroes**
-Coordinate multiple AI agents for complex workflows:
+`Discord Testers` `Bug Hunters`
+`Coffee Supporters` `PR Contributors`
-- **Specialized Agents**: Create agents with specific roles (researcher, coder, analyst)
-- **Agent Collaboration**: Agents can communicate and share context
-- **Workflow Automation**: Chain agent tasks for multi-step processes
-- **Custom Instructions**: Define agent behavior with natural language
+**You make this possible.**
+**Every single day.**
----
+
Roadmap
-
-### ✅ Completed (v2.0 - Current Version)
-- [x] **Browser-local storage** (IndexedDB) with zero-knowledge architecture
-- [x] **Visual workflow builder** with drag-and-drop interface and auto-layout
-- [x] **Interactive prompts** (AI asks questions mid-conversation with typed forms)
-- [x] **Per-conversation privacy toggle** (browser-local vs cloud-sync)
-- [x] Multi-provider LLM support (OpenAI, Anthropic, Google, OpenAI-compatible)
-- [x] Real-time WebSocket streaming with automatic reconnection
-- [x] Tool execution (code, image generation, web search)
-- [x] Response versioning (regenerate, add details, make concise, etc.)
-- [x] Memory system with auto-archival and scoring
-- [x] Hybrid block architecture (Variable, LLM, Code blocks)
-- [x] Docker-based deployment
-- [x] BYOK (Bring Your Own Key) functionality
-- [x] MongoDB + MySQL + Redis infrastructure
-- [x] **Local JWT authentication** (v2.0 - replaced Supabase, fully offline)
-- [x] **E2B Local Docker mode** (v2.0 - code execution without API key)
-- [x] **Removed payment processing** (v2.0 - all users Pro tier by default)
-- [x] **Removed CAPTCHA** (v2.0 - rate limiting only)
-- [x] **100% offline core functionality** (v2.0 - no external service dependencies)
-- [x] File upload support with previews (images, PDFs, documents, CSV, audio)
-- [x] Markdown rendering with reasoning extraction
-
-### 🚧 In Progress (v1.1)
-- [ ] Desktop applications (Windows, macOS, Linux)
-- [ ] Mobile apps (iOS, Android)
-- [ ] P2P device synchronization
-- [ ] Enhanced multi-agent orchestration
-- [ ] Plugin marketplace
-
-### 🔮 Planned (v2.0 and beyond)
-- [ ] Local LLM integration (Ollama, LM Studio native support)
-- [ ] Voice input/output
-- [ ] Advanced RAG (Retrieval-Augmented Generation)
-- [ ] Workspace collaboration features
-- [ ] Browser extension
-- [ ] Kubernetes deployment templates
-- [ ] Enterprise SSO integration
-
-[View full roadmap →](https://github.com/claraverse-space/ClaraVerseAI/projects)
+
Contributing
+Here's the honest truth...
-We welcome contributions from developers of all skill levels! ClaraVerse is built by the community, for the community.
+I'm a **single developer** building ClaraVerse during nights and weekends while working a **9-6 day job**. I'm maintaining **3 platforms**, fixing bugs, adding features, answering Discord questions, reviewing PRs, and somehow trying to keep documentation updated.
-### How to Contribute
+
**Documentation**: Improve guides, add examples, fix typos
-- 
**Translations**: Help us reach non-English speakers
-- 
**UI/UX**: Design improvements and accessibility
-- 
**Testing**: Add unit tests, integration tests, E2E tests
-- 
**Models**: Add support for new LLM providers
+
Community
+**🙏 Thank you for being part of this crazy journey**
-Join thousands of privacy-conscious developers and AI enthusiasts:
+*Building an AI platform alone is insane.*
+*Your support makes it possible.*
-- 
**[Discord](https://discord.com/invite/j633fsrAne)**: Real-time chat and support
-- 
**[Twitter/X](https://x.com/clara_verse_)**: Updates and announcements
-- 
**[TikTok](https://www.tiktok.com/@claraversehq)**: Short-form content and demos
-- 
**[Newsletter](https://claraverse.space/newsletter)**: Monthly updates and tips
-- 
**[YouTube](https://www.youtube.com/@ClaraVerseAI)**: Tutorials and demos
-- 
**[LinkedIn](https://linkedin.com/company/claraverse)**: Professional updates
+— @badboysm890, probably debugging at 3 AM ☕
-### Show Your Support
+
License
+## ❓ **Frequently Asked Questions**
-ClaraVerse is licensed under the **GNU Affero General Public License v3.0 (AGPL-3.0)** - see the [LICENSE](LICENSE) file for details.
+
**[Anthropic](https://anthropic.com/)**, 
**[OpenAI](https://openai.com/)**, 
Troubleshooting
+**🍎 macOS Won't Open?**
+```bash
+sudo xattr -r -d com.apple.quarantine \
+ /Applications/ClaraVerse.app
+```
-### Common Issues
+
Contact
+### **Quick Fixes for Common Problems**
-- 
**Website**: [claraverse.space](https://claraverse.space)
-- 
**Email**: [hello@claraverse.space](mailto:hello@claraverse.space)
-- 
**Bug Reports**: [GitHub Issues](https://github.com/claraverse-space/ClaraVerseAI/issues)
-- 
**Feature Requests**: [GitHub Discussions](https://github.com/claraverse-space/ClaraVerseAI/discussions)
+
+
+**High Memory?**
+
+
+• Close unused tabs
+
+**Network Issues?**
+
+
+• Check firewall
+
+**GPU Problems?**
+
+
+• Update drivers
+
+**App Crashes?**
+
+
+• Check error logs
+
+
+