Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions src/cli/commands.rs
Original file line number Diff line number Diff line change
Expand Up @@ -516,7 +516,7 @@ fn run_index(
}

let db = Database::new(&config.db_path()?)?;
let indexer = ServerIndexer::new(db, client, max_size);
let mut indexer = ServerIndexer::new(db, client, max_size);
indexer.index_directory(&path, force)?;
}
Mode::Local => {
Expand All @@ -531,7 +531,7 @@ fn run_index(

let db = Database::new(&config.db_path()?)?;
let engine = EmbeddingEngine::new(config)?;
let indexer = Indexer::new(db, engine, max_size);
let mut indexer = Indexer::new(db, engine, max_size);
indexer.index_directory(&path, force)?;
}
}
Expand Down
46 changes: 45 additions & 1 deletion src/core/db.rs
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
use anyhow::Result;
use anyhow::{Context, Result};
use chrono::{DateTime, Utc};
use rusqlite::{params, Connection};
use std::path::{Path, PathBuf};
Expand Down Expand Up @@ -104,6 +104,50 @@ impl Database {
}
}

pub fn insert_file_with_chunks(
&mut self,
path: &Path,
hash: &str,
chunks: &[ChunkEntry],
) -> Result<()> {
let path_str = path.to_string_lossy();
let now = Utc::now().to_rfc3339();

let tx = self.conn.transaction()?;

// Insert file
tx.execute(
"INSERT OR REPLACE INTO files (path, hash, indexed_at) VALUES (?, ?, ?)",
params![path_str.as_ref(), hash, now],
)?;

let file_id = tx.last_insert_rowid();

// Insert chunks
{
let mut stmt = tx.prepare(
r"INSERT OR REPLACE INTO chunks
(file_id, chunk_index, content, start_line, end_line, embedding)
VALUES (?, ?, ?, ?, ?, ?)",
)?;

for chunk in chunks {
let embedding_bytes = embedding_to_bytes(&chunk.embedding);
stmt.execute(params![
file_id,
chunk.chunk_index,
chunk.content,
chunk.start_line,
chunk.end_line,
embedding_bytes
])?;
}
}

tx.commit()?;
Ok(())
}

pub fn insert_file(&self, path: &Path, hash: &str) -> Result<i64> {
let path_str = path.to_string_lossy();
let now = Utc::now().to_rfc3339();
Expand Down
53 changes: 29 additions & 24 deletions src/core/indexer.rs
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ use sha2::{Digest, Sha256};
use std::fs;
use std::path::{Path, PathBuf};

use super::db::Database;
use super::db::{ChunkEntry, Database};
use super::embeddings::EmbeddingEngine;
use crate::ui;

Expand Down Expand Up @@ -42,7 +42,7 @@ impl Indexer {
}
}

pub fn index_directory(&self, path: &Path, force: bool) -> Result<()> {
pub fn index_directory(&mut self, path: &Path, force: bool) -> Result<()> {
let abs_path = fs::canonicalize(path).context("Failed to resolve path")?;

println!(
Expand Down Expand Up @@ -159,23 +159,25 @@ impl Indexer {
let mut indexed = 0;

for pending in &pending_files {
// Insert file
let file_id = self.db.insert_file(&pending.path, &pending.hash)?;

// Insert chunks with their embeddings
// Prepare chunks with embeddings
let mut chunks_to_insert = Vec::new();
for (chunk_idx, chunk) in pending.chunks.iter().enumerate() {
let embedding = &all_embeddings[embedding_idx];
self.db.insert_chunk(
file_id,
chunk_idx as i32,
&chunk.content,
chunk.start_line,
chunk.end_line,
embedding,
)?;
chunks_to_insert.push(ChunkEntry {
id: 0, // Will be auto-incremented
file_id: 0, // Will be set after file insertion
chunk_index: chunk_idx as i32,
content: chunk.content.clone(),
start_line: chunk.start_line,
end_line: chunk.end_line,
embedding: embedding.clone(),
});
embedding_idx += 1;
}

// Insert file and chunks atomically
self.db.insert_file_with_chunks(&pending.path, &pending.hash, &chunks_to_insert)?;

indexed += 1;
pb.inc(1);
}
Expand Down Expand Up @@ -410,7 +412,7 @@ impl ServerIndexer {
}
}

pub fn index_directory(&self, path: &Path, force: bool) -> Result<()> {
pub fn index_directory(&mut self, path: &Path, force: bool) -> Result<()> {
let abs_path = fs::canonicalize(path).context("Failed to resolve path")?;

println!(
Expand Down Expand Up @@ -538,21 +540,24 @@ impl ServerIndexer {
let mut indexed = 0;

for pending in &pending_files {
let file_id = self.db.insert_file(&pending.path, &pending.hash)?;
let mut chunks_to_insert = Vec::new();

for (chunk_idx, chunk) in pending.chunks.iter().enumerate() {
let embedding = &all_embeddings[embedding_idx];
self.db.insert_chunk(
file_id,
chunk_idx as i32,
&chunk.content,
chunk.start_line,
chunk.end_line,
embedding,
)?;
chunks_to_insert.push(ChunkEntry {
id: 0,
file_id: 0,
chunk_index: chunk_idx as i32,
content: chunk.content.clone(),
start_line: chunk.start_line,
end_line: chunk.end_line,
embedding: embedding.clone(),
});
embedding_idx += 1;
}

self.db.insert_file_with_chunks(&pending.path, &pending.hash, &chunks_to_insert)?;

indexed += 1;
pb.inc(1);
}
Expand Down
4 changes: 2 additions & 2 deletions src/watcher.rs
Original file line number Diff line number Diff line change
Expand Up @@ -270,7 +270,7 @@ impl FileWatcher {
Mode::Server => {
let db = Database::new(&self.config.db_path()?)?;
let client = Client::new(&self.config.server_host, self.config.server_port);
let indexer = ServerIndexer::new(db, client, self.config.max_file_size);
let mut indexer = ServerIndexer::new(db, client, self.config.max_file_size);
indexer.index_directory(&self.root_path, false)?;
}
Mode::Local => {
Expand All @@ -279,7 +279,7 @@ impl FileWatcher {
}
let db = Database::new(&self.config.db_path()?)?;
let engine = crate::core::EmbeddingEngine::new(&self.config)?;
let indexer = Indexer::new(db, engine, self.config.max_file_size);
let mut indexer = Indexer::new(db, engine, self.config.max_file_size);
indexer.index_directory(&self.root_path, false)?;
}
}
Expand Down
Loading