feat(cli): comprehensive indexer logging

packages/cli/src/commands/gh.ts

@@ -5,10 +5,11 @@
 
 import { getStorageFilePaths, getStoragePath } from '@lytics/dev-agent-core';
 import { GitHubIndexer } from '@lytics/dev-agent-subagents';
+import { createLogger } from '@lytics/kero';
 import chalk from 'chalk';
 import { Command } from 'commander';
 import ora from 'ora';
-import { logger } from '../utils/logger.js';
+import { keroLogger, logger } from '../utils/logger.js';
 
 /**
  * Create GitHub indexer with centralized storage
@@ -52,9 +53,15 @@ export const ghCommand = new Command('gh')
       .option('--prs-only', 'Index only pull requests')
       .option('--state <state>', 'Filter by state (open, closed, merged, all)', 'all')
       .option('--limit <number>', 'Limit number of items to fetch', Number.parseInt)
+      .option('-v, --verbose', 'Verbose output', false)
       .action(async (options) => {
         const spinner = ora('Loading configuration...').start();
 
+        // Create logger for indexing
+        const indexLogger = options.verbose
+          ? createLogger({ level: 'debug', format: 'pretty' })
+          : keroLogger.child({ command: 'gh-index' });
+
         try {
           spinner.text = 'Initializing indexers...';
 
@@ -82,6 +89,14 @@ export const ghCommand = new Command('gh')
             types: types as ('issue' | 'pull_request')[],
             state: state as ('open' | 'closed' | 'merged')[] | undefined,
             limit: options.limit,
+            logger: indexLogger,
+            onProgress: (progress) => {
+              if (progress.phase === 'fetching') {
+                spinner.text = 'Fetching GitHub issues/PRs...';
+              } else if (progress.phase === 'embedding') {
+                spinner.text = `Embedding ${progress.documentsProcessed}/${progress.totalDocuments} GitHub docs`;
+              }
+            },
           });
 
           spinner.succeed(chalk.green('GitHub data indexed!'));

packages/cli/src/commands/git.ts

@@ -10,10 +10,11 @@ import {
   LocalGitExtractor,
   VectorStorage,
 } from '@lytics/dev-agent-core';
+import { createLogger } from '@lytics/kero';
 import chalk from 'chalk';
 import { Command } from 'commander';
 import ora from 'ora';
-import { logger } from '../utils/logger.js';
+import { keroLogger, logger } from '../utils/logger.js';
 
 /**
  * Create Git indexer with centralized storage
@@ -51,9 +52,15 @@ export const gitCommand = new Command('git')
         '--since <date>',
         'Only index commits after this date (e.g., "2024-01-01", "6 months ago")'
       )
+      .option('-v, --verbose', 'Verbose output', false)
       .action(async (options) => {
         const spinner = ora('Loading configuration...').start();
 
+        // Create logger for indexing
+        const indexLogger = options.verbose
+          ? createLogger({ level: 'debug', format: 'pretty' })
+          : keroLogger.child({ command: 'git-index' });
+
         try {
           spinner.text = 'Initializing git indexer...';
 
@@ -64,6 +71,13 @@ export const gitCommand = new Command('git')
           const stats = await indexer.index({
             limit: options.limit,
             since: options.since,
+            logger: indexLogger,
+            onProgress: (progress) => {
+              if (progress.phase === 'storing' && progress.totalCommits > 0) {
+                const pct = Math.round((progress.commitsProcessed / progress.totalCommits) * 100);
+                spinner.text = `Embedding ${progress.commitsProcessed}/${progress.totalCommits} commits (${pct}%)`;
+              }
+            },
           });
 
           spinner.succeed(chalk.green('Git history indexed!'));

packages/cli/src/commands/index.ts

@@ -17,7 +17,7 @@ import { Command } from 'commander';
 import ora from 'ora';
 import { getDefaultConfig, loadConfig } from '../utils/config.js';
 import { formatBytes, getDirectorySize } from '../utils/file.js';
-import { logger } from '../utils/logger.js';
+import { createIndexLogger, logger } from '../utils/logger.js';
 
 /**
  * Check if a command is available
@@ -129,18 +129,27 @@ export const indexCommand = new Command('index')
 
       spinner.text = 'Scanning repository...';
 
+      // Create logger for indexing (verbose mode shows debug logs)
+      const indexLogger = createIndexLogger(options.verbose);
+
       const startTime = Date.now();
       let lastUpdate = startTime;
 
       const stats = await indexer.index({
         force: options.force,
+        logger: indexLogger,
         onProgress: (progress) => {
           const now = Date.now();
           // Update spinner every 100ms to avoid flickering
           if (now - lastUpdate > 100) {
-            const percent = progress.percentComplete || 0;
-            const currentFile = progress.currentFile ? ` ${progress.currentFile}` : '';
-            spinner.text = `${progress.phase}:${currentFile} (${percent.toFixed(0)}%)`;
+            if (progress.phase === 'storing' && progress.totalDocuments) {
+              // Show document count with percentage
+              const pct = Math.round((progress.documentsIndexed / progress.totalDocuments) * 100);
+              spinner.text = `Embedding ${progress.documentsIndexed}/${progress.totalDocuments} documents (${pct}%)`;
+            } else {
+              const percent = progress.percentComplete || 0;
+              spinner.text = `${progress.phase} (${percent.toFixed(0)}%)`;
+            }
             lastUpdate = now;
           }
         },
@@ -184,7 +193,16 @@ export const indexCommand = new Command('index')
           vectorStorage: gitVectorStore,
         });
 
-        gitStats = await gitIndexer.index({ limit: options.gitLimit });
+        gitStats = await gitIndexer.index({
+          limit: options.gitLimit,
+          logger: indexLogger,
+          onProgress: (progress) => {
+            if (progress.phase === 'storing' && progress.totalCommits > 0) {
+              const pct = Math.round((progress.commitsProcessed / progress.totalCommits) * 100);
+              spinner.text = `Embedding ${progress.commitsProcessed}/${progress.totalCommits} commits (${pct}%)`;
+            }
+          },
+        });
         await gitVectorStore.close();
 
         spinner.succeed(chalk.green('Git history indexed!'));
@@ -210,6 +228,14 @@ export const indexCommand = new Command('index')
 
         ghStats = await ghIndexer.index({
           limit: options.ghLimit,
+          logger: indexLogger,
+          onProgress: (progress) => {
+            if (progress.phase === 'fetching') {
+              spinner.text = 'Fetching GitHub issues/PRs...';
+            } else if (progress.phase === 'embedding') {
+              spinner.text = `Embedding ${progress.documentsProcessed}/${progress.totalDocuments} GitHub docs`;
+            }
+          },
         });
         spinner.succeed(chalk.green('GitHub indexed!'));
         logger.log('');

packages/cli/src/utils/logger.ts

@@ -2,10 +2,10 @@
  * CLI Logger using @lytics/kero
  */
 
-import { createLogger } from '@lytics/kero';
+import { createLogger, type Logger, type LogLevel } from '@lytics/kero';
 
 // Create a logger with pretty output and icons
-const keroLogger = createLogger({
+export const keroLogger = createLogger({
   preset: 'development',
   format: 'pretty',
 });
@@ -36,3 +36,14 @@ export const logger = {
     keroLogger.debug(message);
   },
 };
+
+/**
+ * Create a logger for indexing operations with configurable verbosity
+ */
+export function createIndexLogger(verbose: boolean): Logger {
+  const level: LogLevel = verbose ? 'debug' : 'info';
+  return createLogger({
+    level,
+    format: 'pretty',
+  });
+}

packages/core/src/git/indexer.ts

@@ -4,6 +4,7 @@
  * Indexes git commits into the vector store for semantic search.
  */
 
+import type { Logger } from '@lytics/kero';
 import type { VectorStorage } from '../vector';
 import type { EmbeddingDocument } from '../vector/types';
 import type { GitExtractor } from './extractor';
@@ -39,6 +40,8 @@ export interface GitIndexOptions {
   noMerges?: boolean;
   /** Progress callback */
   onProgress?: (progress: GitIndexProgress) => void;
+  /** Logger instance */
+  logger?: Logger;
 }
 
 /**
@@ -81,6 +84,9 @@ export class GitIndexer {
 
     const limit = options.limit ?? this.commitLimit;
     const onProgress = options.onProgress;
+    const logger = options.logger?.child({ component: 'git-indexer' });
+
+    logger?.info({ limit }, 'Starting git commit extraction');
 
     // Phase 1: Extract commits
     onProgress?.({
@@ -101,9 +107,11 @@ export class GitIndexer {
     let commits: GitCommit[];
     try {
       commits = await this.extractor.getCommits(extractOptions);
+      logger?.info({ commits: commits.length }, 'Extracted commits');
     } catch (error) {
       const message = `Failed to extract commits: ${error instanceof Error ? error.message : String(error)}`;
       errors.push(message);
+      logger?.error({ error: message }, 'Failed to extract commits');
       return {
         commitsIndexed: 0,
         durationMs: Date.now() - startTime,
@@ -112,6 +120,7 @@ export class GitIndexer {
     }
 
     if (commits.length === 0) {
+      logger?.info('No commits to index');
       onProgress?.({
         phase: 'complete',
         commitsProcessed: 0,
@@ -126,6 +135,7 @@ export class GitIndexer {
     }
 
     // Phase 2: Prepare documents for embedding
+    logger?.debug({ commits: commits.length }, 'Preparing commit documents for embedding');
     onProgress?.({
       phase: 'embedding',
       commitsProcessed: 0,
@@ -136,6 +146,10 @@ export class GitIndexer {
     const documents = this.prepareCommitDocuments(commits);
 
     // Phase 3: Store in batches
+    logger?.info(
+      { documents: documents.length, batchSize: this.batchSize },
+      'Starting commit embedding'
+    );
     onProgress?.({
       phase: 'storing',
       commitsProcessed: 0,
@@ -144,26 +158,43 @@ export class GitIndexer {
     });
 
     let commitsIndexed = 0;
+    const totalBatches = Math.ceil(documents.length / this.batchSize);
     for (let i = 0; i < documents.length; i += this.batchSize) {
       const batch = documents.slice(i, i + this.batchSize);
+      const batchNum = Math.floor(i / this.batchSize) + 1;
 
       try {
         await this.vectorStorage.addDocuments(batch);
         commitsIndexed += batch.length;
 
+        // Log every 10 batches
+        if (batchNum % 10 === 0 || batchNum === totalBatches) {
+          logger?.info(
+            { batch: batchNum, totalBatches, commitsIndexed, total: commits.length },
+            `Embedded ${commitsIndexed}/${commits.length} commits`
+          );
+        }
+
         onProgress?.({
           phase: 'storing',
           commitsProcessed: commitsIndexed,
           totalCommits: commits.length,
           percentComplete: 50 + (commitsIndexed / commits.length) * 50,
         });
       } catch (error) {
-        const message = `Failed to store batch ${i / this.batchSize}: ${error instanceof Error ? error.message : String(error)}`;
+        const message = `Failed to store batch ${batchNum}: ${error instanceof Error ? error.message : String(error)}`;
         errors.push(message);
+        logger?.error({ batch: batchNum, error: message }, 'Failed to store commit batch');
       }
     }
 
     // Phase 4: Complete
+    const durationMs = Date.now() - startTime;
+    logger?.info(
+      { commitsIndexed, duration: `${durationMs}ms`, errors: errors.length },
+      'Git indexing complete'
+    );
+
     onProgress?.({
       phase: 'complete',
       commitsProcessed: commitsIndexed,
@@ -173,7 +204,7 @@ export class GitIndexer {
 
     return {
       commitsIndexed,
-      durationMs: Date.now() - startTime,
+      durationMs,
       errors,
     };
   }

packages/core/src/indexer/index.ts

@@ -87,12 +87,16 @@ export class RepositoryIndexer {
         include: options.languages?.map((lang) => `**/*.${getExtensionForLanguage(lang)}`),
         exclude: [...this.config.excludePatterns, ...(options.excludePatterns || [])],
         languages: options.languages,
+        logger: options.logger,
       });
 
       filesScanned = scanResult.documents.length;
       documentsExtracted = scanResult.documents.length;
 
       // Phase 2: Prepare documents for embedding
+      const logger = options.logger?.child({ component: 'indexer' });
+      logger?.info({ documents: documentsExtracted }, 'Preparing documents for embedding');
+
       onProgress?.({
         phase: 'embedding',
         filesProcessed: filesScanned,
@@ -104,27 +108,49 @@ export class RepositoryIndexer {
       const embeddingDocuments = prepareDocumentsForEmbedding(scanResult.documents);
 
       // Phase 3: Batch embed and store
+      logger?.info(
+        {
+          documents: embeddingDocuments.length,
+          batchSize: options.batchSize || this.config.batchSize,
+        },
+        'Starting embedding and storage'
+      );
+
       onProgress?.({
         phase: 'storing',
         filesProcessed: filesScanned,
         totalFiles: filesScanned,
         documentsIndexed: 0,
+        totalDocuments: embeddingDocuments.length,
         percentComplete: 66,
       });
 
       const batchSize = options.batchSize || this.config.batchSize;
+      const totalBatches = Math.ceil(embeddingDocuments.length / batchSize);
+      let batchNum = 0;
+
       for (let i = 0; i < embeddingDocuments.length; i += batchSize) {
         const batch = embeddingDocuments.slice(i, i + batchSize);
+        batchNum++;
 
         try {
           await this.vectorStorage.addDocuments(batch);
           documentsIndexed += batch.length;
 
+          // Log progress every 10 batches or on last batch
+          if (batchNum % 10 === 0 || batchNum === totalBatches) {
+            logger?.info(
+              { batch: batchNum, totalBatches, documentsIndexed, total: embeddingDocuments.length },
+              `Embedded ${documentsIndexed}/${embeddingDocuments.length} documents`
+            );
+          }
+
           onProgress?.({
             phase: 'storing',
             filesProcessed: filesScanned,
             totalFiles: filesScanned,
             documentsIndexed,
+            totalDocuments: embeddingDocuments.length,
             percentComplete: 66 + (documentsIndexed / embeddingDocuments.length) * 33,
           });
         } catch (error) {
@@ -134,9 +160,15 @@ export class RepositoryIndexer {
             error: error instanceof Error ? error : undefined,
             timestamp: new Date(),
           });
+          logger?.error(
+            { batch: batchNum, error: error instanceof Error ? error.message : String(error) },
+            'Batch embedding failed'
+          );
         }
       }
 
+      logger?.info({ documentsIndexed, errors: errors.length }, 'Embedding complete');
+
       // Update state
       await this.updateState(scanResult.documents);
 
@@ -253,6 +285,7 @@ export class RepositoryIndexer {
         repoRoot: this.config.repositoryPath,
         include: filesToReindex,
         exclude: this.config.excludePatterns,
+        logger: options.logger,
       });
 
       documentsExtracted = scanResult.documents.length;