diff --git a/.github/workflows/fork-pre-pr.yml b/.github/workflows/fork-pre-pr.yml new file mode 100644 index 00000000..8458637e --- /dev/null +++ b/.github/workflows/fork-pre-pr.yml @@ -0,0 +1,35 @@ +name: Fork Pre-PR Check + +# 在 fork 仓库推送 feat/* 分支时自动运行， +# 提 PR 到上游前提前发现 clippy / 编译问题。 +on: + push: + branches: + - "feat/**" + - "fix/**" + - "chore/**" + +env: + CARGO_TERM_COLOR: always + SQLX_OFFLINE: "true" + # 无需真实 DB / Embedding key，仅做静态检查 + DATABASE_URL: mysql://root:111@localhost:6001/memoria_test + +jobs: + check-and-clippy: + name: Check & Clippy + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: dtolnay/rust-toolchain@1.85.0 + with: + components: clippy + + - uses: Swatinem/rust-cache@v2 + with: + workspaces: memoria + shared-key: fork-pre-pr + + - name: Check & Clippy + run: cd memoria && cargo check && cargo clippy -- -D warnings diff --git a/README.md b/README.md index df2a1e38..8dd3ed2d 100644 --- a/README.md +++ b/README.md @@ -166,7 +166,7 @@ This creates: The generated `mcp.json` includes all environment variables (empty = not configured). Edit the file to fill in your values. -⚠️ **Configure embedding BEFORE the MCP server starts for the first time.** Tables are created on first startup with the configured dimension. +💡 **Embedding dimension** defaults to `EMBEDDING_DIM=0` (auto-infer): Memoria probes your embedding service on first startup and uses the returned vector length. Set `EMBEDDING_DIM` explicitly if the service may be unavailable at boot time. Either way, the dimension is locked into the database schema on first run — changing it later requires dropping the schema. ### 4. Restart & verify @@ -529,8 +529,7 @@ Leave all empty to use local embedding (all-MiniLM-L6-v2, dim=384). **💡 Local Embedding Tips:** Local embedding requires building from source with `--features local-embedding` (pre-built binaries don't include it). See [Local Embedding Guide](skills/local-embedding/SKILL.md) for build instructions, supported models, and troubleshooting. -**⚠️ CRITICAL: Configure embedding BEFORE the MCP server starts for the first time.** - Tables are created on first startup with the configured dimension. Changing it later requires re-creating the embedding column (destructive). +**💡 Embedding dimension is auto-inferred by default** (`EMBEDDING_DIM=0`): Memoria probes your embedding service on startup to detect the correct dimension. The dimension is then locked into the database schema — changing models later requires dropping the schema (destructive). Set `EMBEDDING_DIM` explicitly if you need deterministic startup without a probe call. --- diff --git a/memoria/crates/memoria-cli/src/main.rs b/memoria/crates/memoria-cli/src/main.rs index 8402b3c4..9b76643b 100644 --- a/memoria/crates/memoria-cli/src/main.rs +++ b/memoria/crates/memoria-cli/src/main.rs @@ -327,9 +327,17 @@ async fn cmd_serve(db_url: Option, port: u16, master_key: String) -> Res cfg.db_url = v; } + // Auto-infer embedding dimension when EMBEDDING_DIM=0 (or unset). + // Probes the embedding service with a test call and uses the returned + // vector length as the actual dimension. + if cfg.embedding_dim == 0 && cfg.has_embedding() { + cfg.embedding_dim = probe_embedding_dim(&cfg).await?; + } + tracing::info!( db_url = %cfg.db_url, port = port, instance_id = %cfg.instance_id, + embedding_dim = cfg.embedding_dim, has_llm = cfg.has_llm(), has_embedding = cfg.has_embedding(), governance_plugin_binding = %cfg.governance_plugin_binding, "Starting Memoria API server" @@ -337,6 +345,7 @@ async fn cmd_serve(db_url: Option, port: u16, master_key: String) -> Res let store = SqlMemoryStore::connect(&cfg.db_url, cfg.embedding_dim).await?; store.migrate().await?; + store.check_embedding_dim_compat().await?; let pool = MySqlPool::connect(&cfg.db_url).await?; let git = Arc::new(GitForDataService::new(pool, &cfg.db_name)); @@ -431,9 +440,15 @@ async fn cmd_mcp( cfg.db_name = v; } + // Auto-infer embedding dimension when EMBEDDING_DIM=0 (or unset). + if cfg.embedding_dim == 0 && cfg.has_embedding() { + cfg.embedding_dim = probe_embedding_dim(&cfg).await?; + } + tracing::info!( db_url = %cfg.db_url, embedding_provider = %cfg.embedding_provider, + embedding_dim = cfg.embedding_dim, has_llm = cfg.has_llm(), governance_plugin_binding = %cfg.governance_plugin_binding, user = %cfg.user, @@ -442,6 +457,7 @@ async fn cmd_mcp( let store = SqlMemoryStore::connect(&cfg.db_url, cfg.embedding_dim).await?; store.migrate().await?; + store.check_embedding_dim_compat().await?; let pool = MySqlPool::connect(&cfg.db_url).await?; let git = Arc::new(GitForDataService::new(pool, &cfg.db_name)); @@ -788,6 +804,56 @@ fn cmd_plugin_dev_keygen(dir: &Path) -> Result<()> { // ── Shared helpers ──────────────────────────────────────────────────────────── +/// Probe the configured embedding service to determine the vector dimension. +/// +/// Called when `EMBEDDING_DIM=0` (the default). Makes a single embedding +/// request with a short probe string and returns `vec.len()` as the +/// actual dimension, which is then used to create or validate the database +/// schema. +/// +/// # Errors +/// Returns an error if the embedding service is unreachable or returns an +/// empty vector, with a suggestion to set `EMBEDDING_DIM` explicitly. +async fn probe_embedding_dim(cfg: &memoria_service::Config) -> Result { + use memoria_core::interfaces::EmbeddingProvider; + use memoria_embedding::HttpEmbedder; + + // Build a temporary embedder with dim=0 (dim is not used by embed()). + let embedder = HttpEmbedder::new( + &cfg.embedding_base_url, + &cfg.embedding_api_key, + &cfg.embedding_model, + 0, + ); + + tracing::info!( + model = %cfg.embedding_model, + base_url = %cfg.embedding_base_url, + "EMBEDDING_DIM=0: probing embedding service to auto-infer dimension" + ); + + let vec = embedder + .embed("dimension probe") + .await + .map_err(|e| anyhow::anyhow!( + "EMBEDDING_DIM=0 but the embedding probe failed: {e}. \ + Set EMBEDDING_DIM explicitly (e.g. EMBEDDING_DIM=768 for \ + nomic-embed-text, EMBEDDING_DIM=1024 for BAAI/bge-m3) or \ + check that your embedding service is reachable." + ))?; + + if vec.is_empty() { + return Err(anyhow::anyhow!( + "EMBEDDING_DIM=0: embedding service returned an empty vector. \ + Set EMBEDDING_DIM explicitly." + )); + } + + let dim = vec.len(); + tracing::info!(embedding_dim = dim, "Auto-inferred embedding dimension"); + Ok(dim) +} + fn build_embedder( cfg: &memoria_service::Config, ) -> Option> { diff --git a/memoria/crates/memoria-service/src/config.rs b/memoria/crates/memoria-service/src/config.rs index 2f3afd0d..444cc926 100644 --- a/memoria/crates/memoria-service/src/config.rs +++ b/memoria/crates/memoria-service/src/config.rs @@ -64,7 +64,7 @@ impl Config { let embedding_dim = std::env::var("EMBEDDING_DIM") .ok() .and_then(|s| s.parse().ok()) - .unwrap_or(1024usize); + .unwrap_or(0usize); // 0 = auto-infer from embedding service at startup let llm_api_key = std::env::var("LLM_API_KEY").ok().filter(|s| !s.is_empty()); diff --git a/memoria/crates/memoria-storage/src/store.rs b/memoria/crates/memoria-storage/src/store.rs index 2e7831a7..fa0d9867 100644 --- a/memoria/crates/memoria-storage/src/store.rs +++ b/memoria/crates/memoria-storage/src/store.rs @@ -462,6 +462,52 @@ impl SqlMemoryStore { Ok(()) } + /// Check that the configured embedding dimension matches the dimension + /// already stored in the database schema. + /// + /// If `mem_memories` already exists with a different dimension, returning + /// an error here is far better than silently failing on the first INSERT. + /// Called after `migrate()` so the table is guaranteed to exist. + /// + /// # Errors + /// Returns [`MemoriaError::Internal`] when a mismatch is detected, + /// with a human-readable message explaining how to resolve it. + pub async fn check_embedding_dim_compat(&self) -> Result<(), MemoriaError> { + // Query the actual column type stored in the schema, e.g. "vecf32(768)" + let col_type: Option = sqlx::query_scalar( + "SELECT column_type \ + FROM information_schema.columns \ + WHERE table_schema = DATABASE() \ + AND table_name = 'mem_memories' \ + AND column_name = 'embedding'", + ) + .fetch_optional(&self.pool) + .await + .map_err(db_err)?; + + if let Some(ct) = col_type { + // Parse "vecf32(768)" → 768 + if let Ok(schema_dim) = ct + .trim_start_matches("vecf32(") + .trim_end_matches(')') + .parse::() + { + if schema_dim != self.embedding_dim { + return Err(MemoriaError::Internal(format!( + "Embedding dimension mismatch: the database schema has \ + {}d vectors but Memoria is configured for {}d. \ + To fix: either set EMBEDDING_DIM={} to match the \ + existing schema, or drop the database (data loss) and \ + restart to rebuild with the new dimension.", + schema_dim, self.embedding_dim, schema_dim + ))); + } + } + } + + Ok(()) + } + // ── Audit log ───────────────────────────────────────────────────────────── /// Create a safety snapshot before destructive operations. Best-effort. diff --git a/skills/deployment/SKILL.md b/skills/deployment/SKILL.md index 0e557144..1d40cc58 100644 --- a/skills/deployment/SKILL.md +++ b/skills/deployment/SKILL.md @@ -39,7 +39,7 @@ Services: API on `:8100`, MatrixOne on `:6001`. Verify: `curl http://localhost:8 | `MEMORIA_EMBEDDING_MODEL` | `all-MiniLM-L6-v2` | Model name | | `MEMORIA_EMBEDDING_API_KEY` | — | Required if provider is `openai` | | `MEMORIA_EMBEDDING_BASE_URL` | — | Custom endpoint (OpenAI-compatible) | -| `MEMORIA_EMBEDDING_DIM` | `0` (auto) | Embedding dimension | +| `MEMORIA_EMBEDDING_DIM` | `0` (auto) | Embedding dimension. `0` = auto-infer: Memoria probes the embedding service on startup and uses the returned vector length. Set explicitly (e.g. `768`, `1024`) to skip the probe or when the embedding service may be unavailable at boot time. | ### Distributed