From d8c742f1314503e91b5ca14320c8d5df379c70f7 Mon Sep 17 00:00:00 2001 From: Mladen Mihajlovic Date: Sun, 22 Feb 2026 11:42:33 +0100 Subject: [PATCH 01/14] feat(framework): add Vue SFC processing for search/trace with source remapping --- cli/watch.go | 102 ++++++++-- cli/watch_framework_test.go | 81 ++++++++ config/config.go | 120 +++++++++-- config/config_test.go | 73 +++++++ docs/src/content/docs/contributing.md | 45 +++++ framework/astro_processor.go | 19 ++ framework/errors.go | 8 + framework/map.go | 74 +++++++ framework/registry.go | 121 ++++++++++++ framework/registry_test.go | 153 ++++++++++++++ framework/scripts/vue_processor.mjs | 172 ++++++++++++++++ framework/solid_processor.go | 20 ++ framework/svelte_processor.go | 19 ++ framework/types.go | 47 +++++ framework/util.go | 10 + framework/vue_matrix_test.go | 139 +++++++++++++ framework/vue_processor.go | 241 +++++++++++++++++++++++ framework/vue_processor_test.go | 56 ++++++ indexer/chunker.go | 51 ++--- indexer/indexer.go | 70 ++++++- indexer/indexer_test.go | 71 ++++++- scripts/test-vue-framework-matrix.sh | 158 +++++++++++++++ scripts/test-vue-framework-processing.sh | 151 ++++++++++++++ trace/extractor.go | 66 +++++++ trace/extractor_test.go | 42 ++++ 25 files changed, 2052 insertions(+), 57 deletions(-) create mode 100644 cli/watch_framework_test.go create mode 100644 framework/astro_processor.go create mode 100644 framework/errors.go create mode 100644 framework/map.go create mode 100644 framework/registry.go create mode 100644 framework/registry_test.go create mode 100644 framework/scripts/vue_processor.mjs create mode 100644 framework/solid_processor.go create mode 100644 framework/svelte_processor.go create mode 100644 framework/types.go create mode 100644 framework/util.go create mode 100644 framework/vue_matrix_test.go create mode 100644 framework/vue_processor.go create mode 100644 framework/vue_processor_test.go create mode 100755 scripts/test-vue-framework-matrix.sh create mode 100755 scripts/test-vue-framework-processing.sh diff --git a/cli/watch.go b/cli/watch.go index d56c1bc..7cb2038 100644 --- a/cli/watch.go +++ b/cli/watch.go @@ -19,6 +19,7 @@ import ( "github.com/yoanbernabeu/grepai/config" "github.com/yoanbernabeu/grepai/daemon" "github.com/yoanbernabeu/grepai/embedder" + "github.com/yoanbernabeu/grepai/framework" "github.com/yoanbernabeu/grepai/git" "github.com/yoanbernabeu/grepai/indexer" "github.com/yoanbernabeu/grepai/rpg" @@ -679,7 +680,7 @@ func startRPGRealtimeWorkers(ctx context.Context, projectLabel string, symbolSto } //nolint:unused // Retained for upcoming watch-loop refactor across fg/bg modes. -func runWatchLoop(ctx context.Context, st store.VectorStore, symbolStore *trace.GOBSymbolStore, w *watcher.Watcher, idx *indexer.Indexer, scanner *indexer.Scanner, extractor *trace.RegexExtractor, tracedLanguages []string, projectRoot string, cfg *config.Config, isBackgroundChild bool) error { +func runWatchLoop(ctx context.Context, st store.VectorStore, symbolStore *trace.GOBSymbolStore, w *watcher.Watcher, idx *indexer.Indexer, scanner *indexer.Scanner, extractor *trace.RegexExtractor, tracedLanguages []string, projectRoot string, cfg *config.Config, isBackgroundChild bool, processors ...*framework.ProcessorRegistry) error { // Handle signals sigChan := make(chan os.Signal, 1) signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM) @@ -733,12 +734,12 @@ func runWatchLoop(ctx context.Context, st store.VectorStore, symbolStore *trace. } case event := <-w.Events(): - handleFileEvent(ctx, idx, scanner, extractor, symbolStore, nil, nil, tracedLanguages, projectRoot, cfg, &lastConfigWrite, nil, event, nil, nil) + handleFileEvent(ctx, idx, scanner, extractor, symbolStore, nil, nil, tracedLanguages, projectRoot, cfg, &lastConfigWrite, nil, event, nil, nil, processors...) } } } -func runInitialScan(ctx context.Context, idx *indexer.Indexer, scanner *indexer.Scanner, extractor *trace.RegexExtractor, symbolStore *trace.GOBSymbolStore, tracedLanguages []string, lastIndexTime time.Time, isBackgroundChild bool, onScan func(current, total int, file string), onEmbed func(info indexer.BatchProgressInfo)) (*indexer.IndexStats, error) { +func runInitialScan(ctx context.Context, idx *indexer.Indexer, scanner *indexer.Scanner, extractor *trace.RegexExtractor, symbolStore *trace.GOBSymbolStore, tracedLanguages []string, lastIndexTime time.Time, isBackgroundChild bool, onScan func(current, total int, file string), onEmbed func(info indexer.BatchProgressInfo), processors ...*framework.ProcessorRegistry) (*indexer.IndexStats, error) { // Initial scan with progress if !isBackgroundChild { fmt.Println("\nPerforming initial scan...") @@ -828,7 +829,7 @@ func runInitialScan(ctx context.Context, idx *indexer.Indexer, scanner *indexer. continue } - symbols, refs, err := extractor.ExtractAll(ctx, fileInfo.Path, fileInfo.Content) + symbols, refs, err := extractSymbolsWithFramework(ctx, extractor, fileInfo.Path, fileInfo.Content, processors...) if err != nil { log.Printf("Warning: failed to extract symbols from %s: %v", fileInfo.Path, err) continue @@ -917,6 +918,26 @@ func canonicalPath(path string) string { return filepath.Clean(path) } +func buildFrameworkRegistry(cfg *config.Config) *framework.ProcessorRegistry { + regCfg := framework.RegistryConfig{ + Enabled: cfg.Framework.Enabled, + Mode: cfg.Framework.Mode, + NodePath: cfg.Framework.NodePath, + EnableVue: cfg.Framework.Frameworks.Vue.Enabled, + EnableSvelte: cfg.Framework.Frameworks.Svelte.Enabled, + EnableAstro: cfg.Framework.Frameworks.Astro.Enabled, + EnableSolid: cfg.Framework.Frameworks.Solid.Enabled, + } + + return framework.NewProcessorRegistry( + regCfg, + framework.NewVueProcessor(cfg.Framework.NodePath), + &framework.SvelteProcessor{}, + &framework.AstroProcessor{}, + &framework.SolidProcessor{}, + ) +} + // watchProject runs the full watch lifecycle for a single project. // The embedder is shared across all projects to avoid duplicate connections. // If onReady is non-nil, it is called once after initial indexing and watcher start. @@ -962,9 +983,10 @@ func watchProjectWithEventObserver(ctx context.Context, projectRoot string, emb // Initialize chunker chunker := indexer.NewChunker(cfg.Chunking.Size, cfg.Chunking.Overlap) + processorRegistry := buildFrameworkRegistry(cfg) // Initialize indexer - idx := indexer.NewIndexer(projectRoot, st, emb, chunker, scanner, cfg.Watch.LastIndexTime) + idx := indexer.NewIndexer(projectRoot, st, emb, chunker, scanner, cfg.Watch.LastIndexTime, processorRegistry) // Initialize symbol store and extractor symbolStore := trace.NewGOBSymbolStore(config.GetSymbolIndexPath(projectRoot)) @@ -1016,12 +1038,12 @@ func watchProjectWithEventObserver(ctx context.Context, projectRoot string, emb tracedLanguages := cfg.Trace.EnabledLanguages if len(tracedLanguages) == 0 { - tracedLanguages = []string{".go", ".js", ".ts", ".jsx", ".tsx", ".py", ".php", ".lua", ".java", ".cs", ".fs", ".fsx", ".fsi"} + tracedLanguages = []string{".go", ".js", ".ts", ".jsx", ".tsx", ".vue", ".py", ".php", ".lua", ".java", ".cs", ".fs", ".fsx", ".fsi"} } // In multi-worktree mode callers pass isBackgroundChild=true for non-interactive output. // Run initial scan and build symbol index. // In multi-worktree mode callers pass isBackgroundChild=true for non-interactive output. - stats, err := runInitialScan(ctx, idx, scanner, extractor, symbolStore, tracedLanguages, cfg.Watch.LastIndexTime, isBackgroundChild, onScan, onEmbed) + stats, err := runInitialScan(ctx, idx, scanner, extractor, symbolStore, tracedLanguages, cfg.Watch.LastIndexTime, isBackgroundChild, onScan, onEmbed, processorRegistry) if err != nil { return err } @@ -1069,7 +1091,7 @@ func watchProjectWithEventObserver(ctx context.Context, projectRoot string, emb } // Run watch loop (responds to ctx.Done() for graceful shutdown) - return runProjectWatchLoop(ctx, st, symbolStore, w, idx, scanner, extractor, rpgEncoder, rpgStore, tracedLanguages, projectRoot, cfg, onEvent, onActivity, onStats) + return runProjectWatchLoop(ctx, st, symbolStore, w, idx, scanner, extractor, rpgEncoder, rpgStore, tracedLanguages, projectRoot, cfg, onEvent, onActivity, onStats, processorRegistry) } func emitInitialStatsSnapshot(ctx context.Context, vectorStore store.VectorStore, symbolStore trace.SymbolStore, projectRoot string, onStats watchStatsObserver) { @@ -1107,7 +1129,7 @@ func emitInitialStatsSnapshot(ctx context.Context, vectorStore store.VectorStore } } -func runProjectWatchLoop(ctx context.Context, st store.VectorStore, symbolStore *trace.GOBSymbolStore, w *watcher.Watcher, idx *indexer.Indexer, scanner *indexer.Scanner, extractor *trace.RegexExtractor, rpgEncoder *rpg.RPGEncoder, rpgStore rpg.RPGStore, tracedLanguages []string, projectRoot string, cfg *config.Config, onEvent watchEventObserver, onActivity watchActivityObserver, onStats watchStatsObserver) error { +func runProjectWatchLoop(ctx context.Context, st store.VectorStore, symbolStore *trace.GOBSymbolStore, w *watcher.Watcher, idx *indexer.Indexer, scanner *indexer.Scanner, extractor *trace.RegexExtractor, rpgEncoder *rpg.RPGEncoder, rpgStore rpg.RPGStore, tracedLanguages []string, projectRoot string, cfg *config.Config, onEvent watchEventObserver, onActivity watchActivityObserver, onStats watchStatsObserver, processors ...*framework.ProcessorRegistry) error { persistTicker := time.NewTicker(30 * time.Second) defer persistTicker.Stop() @@ -1151,7 +1173,7 @@ func runProjectWatchLoop(ctx context.Context, st store.VectorStore, symbolStore if onEvent != nil { onEvent(projectRoot, event) } - handleFileEvent(ctx, idx, scanner, extractor, symbolStore, rpgEncoder, st, tracedLanguages, projectRoot, cfg, &lastConfigWrite, rpgManager, event, onActivity, onStats) + handleFileEvent(ctx, idx, scanner, extractor, symbolStore, rpgEncoder, st, tracedLanguages, projectRoot, cfg, &lastConfigWrite, rpgManager, event, onActivity, onStats, processors...) } } } @@ -1966,7 +1988,53 @@ func runWatchForeground() error { ) } -func handleFileEvent(ctx context.Context, idx *indexer.Indexer, scanner *indexer.Scanner, extractor *trace.RegexExtractor, symbolStore *trace.GOBSymbolStore, rpgEncoder *rpg.RPGEncoder, vectorStore store.VectorStore, enabledLanguages []string, projectRoot string, cfg *config.Config, lastConfigWrite *time.Time, rpgManager *rpgRealtimeManager, event watcher.FileEvent, onActivity watchActivityObserver, onStats watchStatsObserver) { +func extractSymbolsWithFramework(ctx context.Context, extractor trace.SymbolExtractor, filePath, source string, processors ...*framework.ProcessorRegistry) ([]trace.Symbol, []trace.Reference, error) { + if len(processors) == 0 || processors[0] == nil { + return extractor.ExtractAll(ctx, filePath, source) + } + + result, err := processors[0].TransformForTrace(ctx, filePath, source) + if err != nil { + return nil, nil, err + } + for _, w := range result.Warnings { + log.Printf("Warning: %s", w) + } + + inputPath := result.VirtualPath + if inputPath == "" { + inputPath = filePath + } + inputText := result.Text + if inputText == "" { + inputText = source + } + + symbols, refs, err := extractor.ExtractAll(ctx, inputPath, inputText) + if err != nil { + return nil, nil, err + } + + for i := range symbols { + symbols[i].File = filePath + symbols[i].Line = framework.RemapLine(result.GeneratedToSourceLine, symbols[i].Line) + if symbols[i].EndLine > 0 { + symbols[i].EndLine = framework.RemapLine(result.GeneratedToSourceLine, symbols[i].EndLine) + } + } + for i := range refs { + refs[i].File = filePath + refs[i].CallerFile = filePath + refs[i].Line = framework.RemapLine(result.GeneratedToSourceLine, refs[i].Line) + if refs[i].CallerLine > 0 { + refs[i].CallerLine = framework.RemapLine(result.GeneratedToSourceLine, refs[i].CallerLine) + } + } + + return symbols, refs, nil +} + +func handleFileEvent(ctx context.Context, idx *indexer.Indexer, scanner *indexer.Scanner, extractor *trace.RegexExtractor, symbolStore *trace.GOBSymbolStore, rpgEncoder *rpg.RPGEncoder, vectorStore store.VectorStore, enabledLanguages []string, projectRoot string, cfg *config.Config, lastConfigWrite *time.Time, rpgManager *rpgRealtimeManager, event watcher.FileEvent, onActivity watchActivityObserver, onStats watchStatsObserver, processors ...*framework.ProcessorRegistry) { if onActivity != nil { op := "processing" if event.Type == watcher.EventDelete { @@ -2047,7 +2115,7 @@ func handleFileEvent(ctx context.Context, idx *indexer.Indexer, scanner *indexer // Extract symbols if language is supported ext := strings.ToLower(filepath.Ext(event.Path)) if isTracedLanguage(ext, enabledLanguages) { - symbols, refs, err := extractor.ExtractAll(ctx, fileInfo.Path, fileInfo.Content) + symbols, refs, err := extractSymbolsWithFramework(ctx, extractor, fileInfo.Path, fileInfo.Content, processors...) if err != nil { log.Printf("Failed to extract symbols from %s: %v", event.Path, err) } else if err := symbolStore.SaveFileWithContentHash(ctx, fileInfo.Path, fileInfo.Hash, symbols, refs); err != nil { @@ -2564,6 +2632,7 @@ func runWorkspaceWatchForeground(logDir string, ws *config.Workspace) error { event.event, nil, nil, + runtime.processor, ) } } @@ -2580,6 +2649,7 @@ type workspaceProjectRuntime struct { idx *indexer.Indexer scanner *indexer.Scanner extractor *trace.RegexExtractor + processor *framework.ProcessorRegistry symbolStore *trace.GOBSymbolStore rpgEncoder *rpg.RPGEncoder rpgStore rpg.RPGStore @@ -2608,13 +2678,14 @@ func initializeWorkspaceRuntime(ctx context.Context, ws *config.Workspace, proje scanner := indexer.NewScanner(project.Path, ignoreMatcher) chunker := indexer.NewChunker(projectCfg.Chunking.Size, projectCfg.Chunking.Overlap) + processorRegistry := buildFrameworkRegistry(projectCfg) vectorStore := &projectPrefixStore{ store: sharedStore, workspaceName: ws.Name, projectName: project.Name, projectPath: project.Path, } - idx := indexer.NewIndexer(project.Path, vectorStore, emb, chunker, scanner, projectCfg.Watch.LastIndexTime) + idx := indexer.NewIndexer(project.Path, vectorStore, emb, chunker, scanner, projectCfg.Watch.LastIndexTime, processorRegistry) extractor := trace.NewRegexExtractor() symbolStore := trace.NewGOBSymbolStore(config.GetSymbolIndexPath(project.Path)) if err := symbolStore.Load(ctx); err != nil { @@ -2623,10 +2694,10 @@ func initializeWorkspaceRuntime(ctx context.Context, ws *config.Workspace, proje tracedLanguages := projectCfg.Trace.EnabledLanguages if len(tracedLanguages) == 0 { - tracedLanguages = []string{".go", ".js", ".ts", ".jsx", ".tsx", ".py", ".php", ".lua", ".java", ".cs", ".fs", ".fsx", ".fsi"} + tracedLanguages = []string{".go", ".js", ".ts", ".jsx", ".tsx", ".vue", ".py", ".php", ".lua", ".java", ".cs", ".fs", ".fsx", ".fsi"} } - stats, err := runInitialScan(ctx, idx, scanner, extractor, symbolStore, tracedLanguages, projectCfg.Watch.LastIndexTime, isBackgroundChild, nil, nil) + stats, err := runInitialScan(ctx, idx, scanner, extractor, symbolStore, tracedLanguages, projectCfg.Watch.LastIndexTime, isBackgroundChild, nil, nil, processorRegistry) if err != nil { _ = symbolStore.Close() return nil, nil, err @@ -2705,6 +2776,7 @@ func initializeWorkspaceRuntime(ctx context.Context, ws *config.Workspace, proje idx: idx, scanner: scanner, extractor: extractor, + processor: processorRegistry, symbolStore: symbolStore, rpgEncoder: rpgEncoder, rpgStore: rpgStore, diff --git a/cli/watch_framework_test.go b/cli/watch_framework_test.go new file mode 100644 index 0000000..da3075e --- /dev/null +++ b/cli/watch_framework_test.go @@ -0,0 +1,81 @@ +package cli + +import ( + "context" + "testing" + + "github.com/yoanbernabeu/grepai/framework" + "github.com/yoanbernabeu/grepai/trace" +) + +type mockSymbolExtractor struct { + symbols []trace.Symbol + refs []trace.Reference +} + +func (m *mockSymbolExtractor) ExtractSymbols(ctx context.Context, filePath string, content string) ([]trace.Symbol, error) { + return m.symbols, nil +} +func (m *mockSymbolExtractor) ExtractReferences(ctx context.Context, filePath string, content string) ([]trace.Reference, error) { + return m.refs, nil +} +func (m *mockSymbolExtractor) ExtractAll(ctx context.Context, filePath string, content string) ([]trace.Symbol, []trace.Reference, error) { + return m.symbols, m.refs, nil +} +func (m *mockSymbolExtractor) SupportedLanguages() []string { return []string{".ts"} } +func (m *mockSymbolExtractor) Mode() string { return "fast" } + +type mockFrameworkProcessor struct{} + +func (m *mockFrameworkProcessor) Name() string { return "vue" } +func (m *mockFrameworkProcessor) Supports(filePath string) bool { + return true +} +func (m *mockFrameworkProcessor) Capabilities() framework.ProcessorCapabilities { + return framework.ProcessorCapabilities{Embedding: true, Trace: true} +} +func (m *mockFrameworkProcessor) TransformForEmbedding(ctx context.Context, filePath, source string) (framework.TransformResult, error) { + return framework.TransformResult{FilePath: filePath, VirtualPath: filePath, Text: source}, nil +} +func (m *mockFrameworkProcessor) TransformForTrace(ctx context.Context, filePath, source string) (framework.TransformResult, error) { + return framework.TransformResult{ + FilePath: filePath, + VirtualPath: filePath + ".__trace__.ts", + Text: "transformed", + GeneratedToSourceLine: []int{10, 20, 30}, + }, nil +} + +func TestExtractSymbolsWithFramework_RemapsFileAndLines(t *testing.T) { + ex := &mockSymbolExtractor{ + symbols: []trace.Symbol{{Name: "fn", File: "virtual.ts", Line: 2, EndLine: 3}}, + refs: []trace.Reference{{SymbolName: "callee", File: "virtual.ts", Line: 3, CallerFile: "virtual.ts", CallerLine: 1}}, + } + + reg := framework.NewProcessorRegistry( + framework.RegistryConfig{Enabled: true, Mode: framework.ModeAuto, EnableVue: true}, + &mockFrameworkProcessor{}, + ) + + symbols, refs, err := extractSymbolsWithFramework(context.Background(), ex, "src/Test.vue", "original", reg) + if err != nil { + t.Fatalf("extractSymbolsWithFramework failed: %v", err) + } + if len(symbols) != 1 || len(refs) != 1 { + t.Fatalf("unexpected symbols/refs counts: %d/%d", len(symbols), len(refs)) + } + + if symbols[0].File != "src/Test.vue" { + t.Fatalf("symbol file = %q, want src/Test.vue", symbols[0].File) + } + if symbols[0].Line != 20 || symbols[0].EndLine != 30 { + t.Fatalf("symbol lines = %d-%d, want 20-30", symbols[0].Line, symbols[0].EndLine) + } + + if refs[0].File != "src/Test.vue" || refs[0].CallerFile != "src/Test.vue" { + t.Fatalf("reference file remap failed: file=%q caller=%q", refs[0].File, refs[0].CallerFile) + } + if refs[0].Line != 30 || refs[0].CallerLine != 10 { + t.Fatalf("reference lines = %d/%d, want 30/10", refs[0].Line, refs[0].CallerLine) + } +} diff --git a/config/config.go b/config/config.go index 26b8099..2c446e2 100644 --- a/config/config.go +++ b/config/config.go @@ -53,17 +53,18 @@ const ( ) type Config struct { - Version int `yaml:"version"` - Embedder EmbedderConfig `yaml:"embedder"` - Store StoreConfig `yaml:"store"` - Chunking ChunkingConfig `yaml:"chunking"` - Watch WatchConfig `yaml:"watch"` - Search SearchConfig `yaml:"search"` - Trace TraceConfig `yaml:"trace"` - RPG RPGConfig `yaml:"rpg"` - Update UpdateConfig `yaml:"update"` - Ignore []string `yaml:"ignore"` - ExternalGitignore string `yaml:"external_gitignore,omitempty"` + Version int `yaml:"version"` + Embedder EmbedderConfig `yaml:"embedder"` + Store StoreConfig `yaml:"store"` + Chunking ChunkingConfig `yaml:"chunking"` + Framework FrameworkConfig `yaml:"framework_processing"` + Watch WatchConfig `yaml:"watch"` + Search SearchConfig `yaml:"search"` + Trace TraceConfig `yaml:"trace"` + RPG RPGConfig `yaml:"rpg"` + Update UpdateConfig `yaml:"update"` + Ignore []string `yaml:"ignore"` + ExternalGitignore string `yaml:"external_gitignore,omitempty"` } // UpdateConfig holds auto-update settings @@ -200,6 +201,62 @@ func DefaultStoreForBackend(backend string) StoreConfig { return cfg } +type FrameworkConfig struct { + Enabled bool `yaml:"enabled"` + Mode string `yaml:"mode"` // auto | require | off + NodePath string `yaml:"node_path,omitempty"` + Frameworks FrameworkFeatureFlags `yaml:"frameworks"` + isSet bool `yaml:"-"` + enabledSet bool `yaml:"-"` +} + +type FrameworkFeatureFlags struct { + Vue FrameworkFeatureConfig `yaml:"vue"` + Svelte FrameworkFeatureConfig `yaml:"svelte"` + Astro FrameworkFeatureConfig `yaml:"astro"` + Solid FrameworkFeatureConfig `yaml:"solid"` +} + +type FrameworkFeatureConfig struct { + Enabled bool `yaml:"enabled"` + isSet bool `yaml:"-"` + enabledSet bool `yaml:"-"` +} + +func (c *FrameworkConfig) UnmarshalYAML(value *yaml.Node) error { + type raw FrameworkConfig + var aux raw + if err := value.Decode(&aux); err != nil { + return err + } + *c = FrameworkConfig(aux) + c.isSet = true + for i := 0; i+1 < len(value.Content); i += 2 { + if value.Content[i].Value == "enabled" { + c.enabledSet = true + break + } + } + return nil +} + +func (c *FrameworkFeatureConfig) UnmarshalYAML(value *yaml.Node) error { + type raw FrameworkFeatureConfig + var aux raw + if err := value.Decode(&aux); err != nil { + return err + } + *c = FrameworkFeatureConfig(aux) + c.isSet = true + for i := 0; i+1 < len(value.Content); i += 2 { + if value.Content[i].Value == "enabled" { + c.enabledSet = true + break + } + } + return nil +} + type WatchConfig struct { DebounceMs int `yaml:"debounce_ms"` LastIndexTime time.Time `yaml:"last_index_time,omitempty"` @@ -278,6 +335,17 @@ func DefaultConfig() *Config { Size: 512, Overlap: 50, }, + Framework: FrameworkConfig{ + Enabled: true, + Mode: "auto", + NodePath: "node", + Frameworks: FrameworkFeatureFlags{ + Vue: FrameworkFeatureConfig{Enabled: true}, + Svelte: FrameworkFeatureConfig{Enabled: false}, + Astro: FrameworkFeatureConfig{Enabled: false}, + Solid: FrameworkFeatureConfig{Enabled: false}, + }, + }, Watch: WatchConfig{ DebounceMs: 500, RPGPersistIntervalMs: DefaultWatchRPGPersistIntervalMs, @@ -327,7 +395,7 @@ func DefaultConfig() *Config { Trace: TraceConfig{ Mode: "fast", EnabledLanguages: []string{ - ".go", ".js", ".ts", ".jsx", ".tsx", ".py", ".php", + ".go", ".js", ".ts", ".jsx", ".tsx", ".vue", ".py", ".php", ".lua", ".c", ".h", ".cpp", ".hpp", ".cc", ".cxx", ".rs", ".zig", ".cs", ".java", @@ -462,6 +530,34 @@ func (c *Config) applyDefaults() { c.Chunking.Overlap = defaults.Chunking.Overlap } + // Framework processing defaults + hasFrameworkConfig := c.Framework.isSet + if !hasFrameworkConfig { + c.Framework = defaults.Framework + } else { + if !c.Framework.enabledSet { + c.Framework.Enabled = defaults.Framework.Enabled + } + if c.Framework.Mode == "" { + c.Framework.Mode = defaults.Framework.Mode + } + if c.Framework.NodePath == "" { + c.Framework.NodePath = defaults.Framework.NodePath + } + if !c.Framework.Frameworks.Vue.enabledSet { + c.Framework.Frameworks.Vue.Enabled = defaults.Framework.Frameworks.Vue.Enabled + } + if !c.Framework.Frameworks.Svelte.enabledSet { + c.Framework.Frameworks.Svelte.Enabled = defaults.Framework.Frameworks.Svelte.Enabled + } + if !c.Framework.Frameworks.Astro.enabledSet { + c.Framework.Frameworks.Astro.Enabled = defaults.Framework.Frameworks.Astro.Enabled + } + if !c.Framework.Frameworks.Solid.enabledSet { + c.Framework.Frameworks.Solid.Enabled = defaults.Framework.Frameworks.Solid.Enabled + } + } + // Watch defaults if c.Watch.DebounceMs == 0 { c.Watch.DebounceMs = defaults.Watch.DebounceMs diff --git a/config/config_test.go b/config/config_test.go index c06ab29..5ec9b19 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -3,6 +3,7 @@ package config import ( "os" "path/filepath" + "strings" "testing" ) @@ -42,6 +43,19 @@ func TestDefaultConfig(t *testing.T) { t.Errorf("expected chunk overlap 50, got %d", cfg.Chunking.Overlap) } + if !cfg.Framework.Enabled { + t.Error("expected framework_processing.enabled=true by default") + } + if cfg.Framework.Mode != "auto" { + t.Errorf("expected framework_processing.mode=auto, got %s", cfg.Framework.Mode) + } + if !cfg.Framework.Frameworks.Vue.Enabled { + t.Error("expected framework_processing.frameworks.vue.enabled=true by default") + } + if cfg.Framework.Frameworks.Svelte.Enabled || cfg.Framework.Frameworks.Astro.Enabled || cfg.Framework.Frameworks.Solid.Enabled { + t.Error("expected non-vue framework scaffolds disabled by default") + } + if cfg.Watch.DebounceMs != 500 { t.Errorf("expected debounce 500ms, got %d", cfg.Watch.DebounceMs) } @@ -100,6 +114,65 @@ func TestDefaultStoreForBackend(t *testing.T) { } } +func TestConfigLoad_FrameworkProcessingDefaultsRespectExplicitFalseAndNestedDefaults(t *testing.T) { + tmpDir := t.TempDir() + if err := os.MkdirAll(filepath.Join(tmpDir, ConfigDir), 0o755); err != nil { + t.Fatalf("mkdir config dir: %v", err) + } + cfgPath := GetConfigPath(tmpDir) + + tests := []struct { + name string + yaml string + wantEnabled bool + wantVueEnabled bool + wantSvelteEnabled bool + }{ + { + name: "explicit framework disabled", + yaml: "framework_processing:\n enabled: false\n", + wantEnabled: false, + wantVueEnabled: true, + wantSvelteEnabled: false, + }, + { + name: "framework section present without nested flags keeps vue default", + yaml: "framework_processing:\n mode: auto\n", + wantEnabled: true, + wantVueEnabled: true, + wantSvelteEnabled: false, + }, + { + name: "explicit vue disabled preserved", + yaml: "framework_processing:\n frameworks:\n vue:\n enabled: false\n", + wantEnabled: true, + wantVueEnabled: false, + wantSvelteEnabled: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if err := os.WriteFile(cfgPath, []byte(strings.TrimSpace(tt.yaml)+"\n"), 0o644); err != nil { + t.Fatalf("write config: %v", err) + } + cfg, err := Load(tmpDir) + if err != nil { + t.Fatalf("Load failed: %v", err) + } + if cfg.Framework.Enabled != tt.wantEnabled { + t.Fatalf("framework enabled = %v, want %v", cfg.Framework.Enabled, tt.wantEnabled) + } + if cfg.Framework.Frameworks.Vue.Enabled != tt.wantVueEnabled { + t.Fatalf("framework vue enabled = %v, want %v", cfg.Framework.Frameworks.Vue.Enabled, tt.wantVueEnabled) + } + if cfg.Framework.Frameworks.Svelte.Enabled != tt.wantSvelteEnabled { + t.Fatalf("framework svelte enabled = %v, want %v", cfg.Framework.Frameworks.Svelte.Enabled, tt.wantSvelteEnabled) + } + }) + } +} + func TestConfigSaveAndLoad(t *testing.T) { tmpDir := t.TempDir() diff --git a/docs/src/content/docs/contributing.md b/docs/src/content/docs/contributing.md index 8525c9f..ea46c5d 100644 --- a/docs/src/content/docs/contributing.md +++ b/docs/src/content/docs/contributing.md @@ -171,6 +171,51 @@ func (s *MyStore) Close() error { 3. Wire it up in CLI commands +## Adding a Framework Processor + +Framework processors normalize non-standard source files (Vue/Svelte/Astro/etc.) into index/trace friendly text while preserving source mapping. + +### 1. Implement the processor contract + +Create a new file under `framework/` and implement: + +- `Name() string` +- `Supports(filePath string) bool` +- `Capabilities() ProcessorCapabilities` +- `TransformForEmbedding(ctx, filePath, source)` +- `TransformForTrace(ctx, filePath, source)` + +Return `ErrNotImplemented` for scaffold placeholders and `ErrUnavailable` when runtime/compiler dependencies are missing. + +### 2. Respect mapping invariants + +- `GeneratedToSourceLine` is 1-based. +- Use `0` for generated lines without a reliable source line. +- Persisted symbols/chunks must always reference the original source path. +- Keep embedding text and displayed snippet separate when transformed output differs from source. + +### 3. Register processor and config flags + +- Add processor construction in `cli/watch.go` (`buildFrameworkRegistry`). +- Add per-framework enable toggle in `config/config.go` under `framework_processing.frameworks`. +- Honor global mode: `auto | require | off`. + +### 4. Required tests + +- Registry routing by extension. +- `auto` fallback behavior. +- `require` failure behavior. +- Source line remapping correctness. +- Regression tests for non-framework files. + +### 5. Scaffold examples + +- `framework/svelte_processor.go` +- `framework/astro_processor.go` +- `framework/solid_processor.go` + +These are intentionally non-functional placeholders showing expected structure. + ## Commit Convention Follow conventional commits: diff --git a/framework/astro_processor.go b/framework/astro_processor.go new file mode 100644 index 0000000..0ac9b69 --- /dev/null +++ b/framework/astro_processor.go @@ -0,0 +1,19 @@ +package framework + +import "context" + +type AstroProcessor struct{} + +func (p *AstroProcessor) Name() string { return "astro" } +func (p *AstroProcessor) Supports(filePath string) bool { + return hasExt(filePath, ".astro") +} +func (p *AstroProcessor) Capabilities() ProcessorCapabilities { + return ProcessorCapabilities{} +} +func (p *AstroProcessor) TransformForEmbedding(ctx context.Context, filePath, source string) (TransformResult, error) { + return TransformResult{}, ErrNotImplemented +} +func (p *AstroProcessor) TransformForTrace(ctx context.Context, filePath, source string) (TransformResult, error) { + return TransformResult{}, ErrNotImplemented +} diff --git a/framework/errors.go b/framework/errors.go new file mode 100644 index 0000000..841bff6 --- /dev/null +++ b/framework/errors.go @@ -0,0 +1,8 @@ +package framework + +import "errors" + +var ( + ErrNotImplemented = errors.New("framework processor not implemented") + ErrUnavailable = errors.New("framework processor unavailable") +) diff --git a/framework/map.go b/framework/map.go new file mode 100644 index 0000000..b7bd5e0 --- /dev/null +++ b/framework/map.go @@ -0,0 +1,74 @@ +package framework + +import "strings" + +// RemapLineRange maps a generated line range to source lines. +func RemapLineRange(generatedToSource []int, generatedStart, generatedEnd int) (int, int) { + if len(generatedToSource) == 0 { + return generatedStart, generatedEnd + } + if generatedStart < 1 { + generatedStart = 1 + } + if generatedEnd < generatedStart { + generatedEnd = generatedStart + } + if generatedStart > len(generatedToSource) { + return generatedStart, generatedEnd + } + if generatedEnd > len(generatedToSource) { + generatedEnd = len(generatedToSource) + } + + minLine := 0 + maxLine := 0 + for i := generatedStart; i <= generatedEnd; i++ { + src := generatedToSource[i-1] + if src <= 0 { + continue + } + if minLine == 0 || src < minLine { + minLine = src + } + if src > maxLine { + maxLine = src + } + } + if minLine == 0 { + return generatedStart, generatedEnd + } + return minLine, maxLine +} + +// RemapLine maps a generated line to source line. +func RemapLine(generatedToSource []int, generatedLine int) int { + if generatedLine < 1 || generatedLine > len(generatedToSource) { + return generatedLine + } + src := generatedToSource[generatedLine-1] + if src <= 0 { + return generatedLine + } + return src +} + +// SourceSnippet returns an inclusive line-range snippet from source. +func SourceSnippet(source string, startLine, endLine int) string { + if startLine < 1 { + startLine = 1 + } + if endLine < startLine { + endLine = startLine + } + lines := strings.Split(source, "\n") + if len(lines) == 0 { + return "" + } + if startLine > len(lines) { + return "" + } + if endLine > len(lines) { + endLine = len(lines) + } + return strings.Join(lines[startLine-1:endLine], "\n") +} diff --git a/framework/registry.go b/framework/registry.go new file mode 100644 index 0000000..f32ddd3 --- /dev/null +++ b/framework/registry.go @@ -0,0 +1,121 @@ +package framework + +import ( + "context" + "errors" + "fmt" +) + +// ProcessorRegistry resolves and executes framework processors. +type ProcessorRegistry struct { + cfg RegistryConfig + processors []FrameworkProcessor +} + +func NewProcessorRegistry(cfg RegistryConfig, processors ...FrameworkProcessor) *ProcessorRegistry { + if cfg.Mode == "" { + cfg.Mode = ModeAuto + } + if cfg.NodePath == "" { + cfg.NodePath = "node" + } + if !cfg.Enabled { + cfg.Mode = ModeOff + } + return &ProcessorRegistry{cfg: cfg, processors: processors} +} + +func (r *ProcessorRegistry) TransformForEmbedding(ctx context.Context, filePath, source string) (TransformResult, error) { + return r.transform(ctx, filePath, source, true) +} + +func (r *ProcessorRegistry) TransformForTrace(ctx context.Context, filePath, source string) (TransformResult, error) { + return r.transform(ctx, filePath, source, false) +} + +func (r *ProcessorRegistry) transform(ctx context.Context, filePath, source string, embedding bool) (TransformResult, error) { + if r.cfg.Mode == ModeOff { + return passthrough(filePath, source), nil + } + + p := r.findProcessor(filePath) + if p == nil { + return passthrough(filePath, source), nil + } + + var ( + res TransformResult + err error + ) + if embedding { + res, err = p.TransformForEmbedding(ctx, filePath, source) + } else { + res, err = p.TransformForTrace(ctx, filePath, source) + } + if err == nil { + if res.FilePath == "" { + res.FilePath = filePath + } + if res.VirtualPath == "" { + res.VirtualPath = filePath + } + if res.Processor == "" { + res.Processor = p.Name() + } + if res.Text == "" { + res.Text = source + } + return res, nil + } + + if errors.Is(err, ErrUnavailable) || errors.Is(err, ErrNotImplemented) { + if r.cfg.Mode == ModeRequire { + return TransformResult{}, fmt.Errorf("%s processor failed for %s: %w", p.Name(), filePath, err) + } + if res.Text != "" { + res.Warnings = append(res.Warnings, fmt.Sprintf("%s processor fallback: %v", p.Name(), err)) + return res, nil + } + out := passthrough(filePath, source) + out.Warnings = append(out.Warnings, fmt.Sprintf("%s processor fallback: %v", p.Name(), err)) + return out, nil + } + + return TransformResult{}, err +} + +func (r *ProcessorRegistry) findProcessor(filePath string) FrameworkProcessor { + for _, p := range r.processors { + if !r.processorEnabled(p.Name()) { + continue + } + if p.Supports(filePath) { + return p + } + } + return nil +} + +func (r *ProcessorRegistry) processorEnabled(name string) bool { + switch name { + case "vue": + return r.cfg.EnableVue + case "svelte": + return r.cfg.EnableSvelte + case "astro": + return r.cfg.EnableAstro + case "solid": + return r.cfg.EnableSolid + default: + return true + } +} + +func passthrough(filePath, source string) TransformResult { + return TransformResult{ + Processor: "passthrough", + FilePath: filePath, + VirtualPath: filePath, + Text: source, + } +} diff --git a/framework/registry_test.go b/framework/registry_test.go new file mode 100644 index 0000000..0cf4585 --- /dev/null +++ b/framework/registry_test.go @@ -0,0 +1,153 @@ +package framework + +import ( + "context" + "strings" + "testing" +) + +type stubProcessor struct { + name string + ext string + result TransformResult + err error + supports bool +} + +func (s *stubProcessor) Name() string { return s.name } +func (s *stubProcessor) Supports(filePath string) bool { + if !s.supports { + return false + } + return hasExt(filePath, s.ext) +} +func (s *stubProcessor) Capabilities() ProcessorCapabilities { return ProcessorCapabilities{} } +func (s *stubProcessor) TransformForEmbedding(ctx context.Context, filePath, source string) (TransformResult, error) { + if s.result.Text == "" { + s.result = TransformResult{Text: source, FilePath: filePath, VirtualPath: filePath} + } + return s.result, s.err +} +func (s *stubProcessor) TransformForTrace(ctx context.Context, filePath, source string) (TransformResult, error) { + if s.result.Text == "" { + s.result = TransformResult{Text: source, FilePath: filePath, VirtualPath: filePath} + } + return s.result, s.err +} + +func TestRegistryPassthroughWhenOff(t *testing.T) { + r := NewProcessorRegistry(RegistryConfig{Enabled: false, Mode: ModeOff}) + res, err := r.TransformForEmbedding(context.Background(), "a.vue", "hello") + if err != nil { + t.Fatalf("TransformForEmbedding err: %v", err) + } + if res.Processor != "passthrough" { + t.Fatalf("processor=%q want passthrough", res.Processor) + } +} + +func TestRegistryAutoFallbackOnUnavailable(t *testing.T) { + p := &stubProcessor{ + name: "vue", + ext: ".vue", + supports: true, + result: TransformResult{ + Text: "compiled", + FilePath: "Comp.vue", + VirtualPath: "Comp.vue.__trace__.ts", + GeneratedToSourceLine: []int{2}, + }, + err: ErrUnavailable, + } + r := NewProcessorRegistry(RegistryConfig{Enabled: true, Mode: ModeAuto, EnableVue: true}, p) + res, err := r.TransformForTrace(context.Background(), "Comp.vue", "") + if err != nil { + t.Fatalf("TransformForTrace err: %v", err) + } + if res.Text != "compiled" { + t.Fatalf("text=%q want compiled", res.Text) + } + if len(res.Warnings) == 0 { + t.Fatal("expected fallback warning") + } +} + +func TestRegistryRequireFailsOnUnavailable(t *testing.T) { + p := &stubProcessor{name: "vue", ext: ".vue", supports: true, err: ErrUnavailable} + r := NewProcessorRegistry(RegistryConfig{Enabled: true, Mode: ModeRequire, EnableVue: true}, p) + _, err := r.TransformForEmbedding(context.Background(), "Comp.vue", "x") + if err == nil { + t.Fatal("expected error in require mode") + } +} + +func TestRegistryHonorsFrameworkEnableFlags(t *testing.T) { + p := &stubProcessor{ + name: "vue", + ext: ".vue", + supports: true, + result: TransformResult{Text: "compiled"}, + } + r := NewProcessorRegistry(RegistryConfig{Enabled: true, Mode: ModeAuto, EnableVue: false}, p) + res, err := r.TransformForEmbedding(context.Background(), "Comp.vue", "source") + if err != nil { + t.Fatalf("TransformForEmbedding err: %v", err) + } + if res.Processor != "passthrough" { + t.Fatalf("processor=%q want passthrough when vue disabled", res.Processor) + } +} + +func TestRegistryRequireFailsForScaffoldProcessor(t *testing.T) { + r := NewProcessorRegistry( + RegistryConfig{Enabled: true, Mode: ModeRequire, EnableSvelte: true}, + &SvelteProcessor{}, + ) + _, err := r.TransformForEmbedding(context.Background(), "Comp.svelte", "") + if err == nil { + t.Fatal("expected error for scaffold processor in require mode") + } +} + +func TestVueProcessorFallbackWithMissingNode(t *testing.T) { + p := NewVueProcessor("node-does-not-exist") + r := NewProcessorRegistry(RegistryConfig{Enabled: true, Mode: ModeAuto, EnableVue: true}, p) + + source := `\n` + res, err := r.TransformForTrace(context.Background(), "Comp.vue", source) + if err != nil { + t.Fatalf("TransformForTrace err: %v", err) + } + if !strings.Contains(res.Text, "const n = 1") { + t.Fatalf("expected fallback script content, got: %q", res.Text) + } + if len(res.GeneratedToSourceLine) == 0 { + t.Fatal("expected line mapping") + } +} + +func TestScaffoldProcessorsReturnFallback(t *testing.T) { + r := NewProcessorRegistry(RegistryConfig{Enabled: true, Mode: ModeAuto, EnableSvelte: true, EnableAstro: true}, &SvelteProcessor{}, &AstroProcessor{}) + res, err := r.TransformForEmbedding(context.Background(), "A.svelte", "") + if err != nil { + t.Fatalf("svelte fallback failed: %v", err) + } + if res.Processor != "passthrough" { + t.Fatalf("processor=%q want passthrough", res.Processor) + } + + res, err = r.TransformForEmbedding(context.Background(), "A.astro", "---\nconst a = 1\n---") + if err != nil { + t.Fatalf("astro fallback failed: %v", err) + } + if res.Processor != "passthrough" { + t.Fatalf("processor=%q want passthrough", res.Processor) + } +} + +func TestRemapLineRange(t *testing.T) { + start, end := RemapLineRange([]int{0, 4, 5, 0}, 1, 4) + if start != 4 || end != 5 { + t.Fatalf("range=(%d,%d) want (4,5)", start, end) + } +} diff --git a/framework/scripts/vue_processor.mjs b/framework/scripts/vue_processor.mjs new file mode 100644 index 0000000..d3cd9ec --- /dev/null +++ b/framework/scripts/vue_processor.mjs @@ -0,0 +1,172 @@ +import { parse, compileTemplate } from "@vue/compiler-sfc"; + +const chunks = []; +for await (const chunk of process.stdin) { + chunks.push(chunk); +} + +const input = JSON.parse(Buffer.concat(chunks).toString("utf8")); +const source = input.source ?? ""; +const filePath = input.filePath ?? "Component.vue"; + +function countLines(text) { + if (text.length === 0) return 1; + return text.split("\n").length; +} + +function pushMapped(out, map, text, sourceStartLine) { + if (!text) return; + out.push(text); + const lineCount = countLines(text); + for (let i = 0; i < lineCount; i++) { + map.push(sourceStartLine + i); + } +} + +function pushUnmapped(out, map, text) { + if (!text) return; + out.push(text); + const lineCount = countLines(text); + for (let i = 0; i < lineCount; i++) { + map.push(0); + } +} + +function normalizeExpr(expr) { + const trimmed = expr.trim(); + const singleQuoted = /^'([^']+)'$/.exec(trimmed); + if (singleQuoted) return singleQuoted[1]; + const doubleQuoted = /^"([^"]+)"$/.exec(trimmed); + if (doubleQuoted) return doubleQuoted[1]; + return trimmed; +} + +function extractVBindExprsFromLine(line) { + const out = []; + let start = 0; + while (start < line.length) { + const idx = line.indexOf("v-bind(", start); + if (idx < 0) break; + let i = idx + "v-bind(".length; + let depth = 1; + while (i < line.length && depth > 0) { + const ch = line[i]; + if (ch === "(") depth++; + else if (ch === ")") depth--; + i++; + } + if (depth === 0) { + const expr = line.slice(idx + "v-bind(".length, i - 1); + out.push(normalizeExpr(expr)); + start = i; + } else { + break; + } + } + return out; +} + +function extractStyleVBindExpressions(styleContent, styleStartLine) { + const refs = []; + const lines = styleContent.split("\n"); + for (let i = 0; i < lines.length; i++) { + const exprs = extractVBindExprsFromLine(lines[i]); + for (const expr of exprs) { + refs.push({ + expr, + line: styleStartLine + i, + }); + } + } + return refs; +} + +function appendStyleBindings(out, map, refs, index) { + if (!refs.length) return; + out.push(`function __vue_style_bindings__${index}() {`); + map.push(refs[0].line); + for (const ref of refs) { + out.push(` __css_v_bind__(${ref.expr});`); + map.push(ref.line); + } + out.push("}"); + map.push(refs[refs.length - 1].line); +} + +try { + const parsed = parse(source, { filename: filePath }); + if (parsed.errors?.length) { + const msg = String(parsed.errors[0]); + throw new Error(msg); + } + + const d = parsed.descriptor; + const out = []; + const map = []; + + if (d.script?.content) { + pushMapped(out, map, d.script.content, d.script.loc.start.line + 1); + } + if (d.scriptSetup?.content) { + if (out.length > 0) { + out.push("\n"); + map.push(0); + } + pushMapped(out, map, d.scriptSetup.content, d.scriptSetup.loc.start.line + 1); + } + + if (d.template?.content) { + const compiled = compileTemplate({ + id: filePath, + source: d.template.content, + filename: filePath, + scoped: Boolean(d.styles?.some((s) => s.scoped)), + }); + if (compiled.errors?.length) { + const first = compiled.errors[0]; + throw new Error(typeof first === "string" ? first : first.message || String(first)); + } + if (compiled.code) { + if (out.length > 0) { + out.push("\n"); + map.push(0); + } + pushUnmapped(out, map, compiled.code); + } + } + + if (Array.isArray(d.styles) && d.styles.length > 0) { + for (let i = 0; i < d.styles.length; i++) { + const styleBlock = d.styles[i]; + if (!styleBlock?.content) continue; + if (out.length > 0) { + out.push("\n"); + map.push(0); + } + const styleStartLine = styleBlock.loc?.start?.line ? styleBlock.loc.start.line + 1 : 1; + // Include raw style content for semantic search. + pushMapped(out, map, styleBlock.content, styleStartLine); + + const refs = extractStyleVBindExpressions(styleBlock.content, styleStartLine); + if (refs.length > 0) { + out.push("\n"); + map.push(0); + appendStyleBindings(out, map, refs, i); + } + } + } + + const text = out.join("\n"); + process.stdout.write( + JSON.stringify({ + embeddingText: text, + traceText: text, + virtualPath: `${filePath}.__trace__.ts`, + generatedToSourceLine: map, + warnings: [], + }), + ); +} catch (err) { + process.stderr.write(String(err?.stack || err?.message || err)); + process.exit(1); +} diff --git a/framework/solid_processor.go b/framework/solid_processor.go new file mode 100644 index 0000000..44fb60f --- /dev/null +++ b/framework/solid_processor.go @@ -0,0 +1,20 @@ +package framework + +import "context" + +// SolidProcessor is a placeholder for future Solid-specific JSX/TSX transforms. +type SolidProcessor struct{} + +func (p *SolidProcessor) Name() string { return "solid" } +func (p *SolidProcessor) Supports(filePath string) bool { + return false +} +func (p *SolidProcessor) Capabilities() ProcessorCapabilities { + return ProcessorCapabilities{} +} +func (p *SolidProcessor) TransformForEmbedding(ctx context.Context, filePath, source string) (TransformResult, error) { + return TransformResult{}, ErrNotImplemented +} +func (p *SolidProcessor) TransformForTrace(ctx context.Context, filePath, source string) (TransformResult, error) { + return TransformResult{}, ErrNotImplemented +} diff --git a/framework/svelte_processor.go b/framework/svelte_processor.go new file mode 100644 index 0000000..4edddad --- /dev/null +++ b/framework/svelte_processor.go @@ -0,0 +1,19 @@ +package framework + +import "context" + +type SvelteProcessor struct{} + +func (p *SvelteProcessor) Name() string { return "svelte" } +func (p *SvelteProcessor) Supports(filePath string) bool { + return hasExt(filePath, ".svelte") +} +func (p *SvelteProcessor) Capabilities() ProcessorCapabilities { + return ProcessorCapabilities{} +} +func (p *SvelteProcessor) TransformForEmbedding(ctx context.Context, filePath, source string) (TransformResult, error) { + return TransformResult{}, ErrNotImplemented +} +func (p *SvelteProcessor) TransformForTrace(ctx context.Context, filePath, source string) (TransformResult, error) { + return TransformResult{}, ErrNotImplemented +} diff --git a/framework/types.go b/framework/types.go new file mode 100644 index 0000000..ca58617 --- /dev/null +++ b/framework/types.go @@ -0,0 +1,47 @@ +package framework + +import "context" + +const ( + ModeAuto = "auto" + ModeRequire = "require" + ModeOff = "off" +) + +// ProcessorCapabilities describes current support status for a processor. +type ProcessorCapabilities struct { + Embedding bool + Trace bool + Compiled bool +} + +// TransformResult is normalized output from framework processors. +type TransformResult struct { + Processor string + FilePath string + VirtualPath string + Text string + GeneratedToSourceLine []int // 1-indexed generated line -> source line. 0 means unmapped. + Transformed bool + Warnings []string +} + +// FrameworkProcessor transforms framework files into trace/index friendly text. +type FrameworkProcessor interface { + Name() string + Supports(filePath string) bool + Capabilities() ProcessorCapabilities + TransformForEmbedding(ctx context.Context, filePath, source string) (TransformResult, error) + TransformForTrace(ctx context.Context, filePath, source string) (TransformResult, error) +} + +// RegistryConfig controls processor behavior. +type RegistryConfig struct { + Enabled bool + Mode string + NodePath string + EnableVue bool + EnableSvelte bool + EnableAstro bool + EnableSolid bool +} diff --git a/framework/util.go b/framework/util.go new file mode 100644 index 0000000..23af670 --- /dev/null +++ b/framework/util.go @@ -0,0 +1,10 @@ +package framework + +import ( + "path/filepath" + "strings" +) + +func hasExt(filePath, ext string) bool { + return strings.EqualFold(filepath.Ext(filePath), ext) +} diff --git a/framework/vue_matrix_test.go b/framework/vue_matrix_test.go new file mode 100644 index 0000000..d2b0022 --- /dev/null +++ b/framework/vue_matrix_test.go @@ -0,0 +1,139 @@ +package framework + +import ( + "context" + "strings" + "testing" +) + +func TestVueProcessorFallback_Matrix(t *testing.T) { + p := NewVueProcessor("node") + + tests := []struct { + name string + source string + wantContains []string + wantErr bool + }{ + { + name: "script-only", + source: ` +`, + wantContains: []string{"export function one"}, + }, + { + name: "script-lang-js", + source: ` +`, + wantContains: []string{"export function jsFn"}, + }, + { + name: "script-no-lang", + source: ` +`, + wantContains: []string{"export function plainFn"}, + }, + { + name: "script-setup-only", + source: ` +`, + wantContains: []string{"const count = 1"}, + }, + { + name: "script-and-script-setup", + source: ` + +`, + wantContains: []string{"export function two", "const count = 2"}, + }, + { + name: "template-only", + source: ``, + wantErr: true, + }, + { + name: "style-heavy", + source: ` + +`, + wantContains: []string{"const msg = 'ok'", "__css_v_bind__(color)", "__css_v_bind__(getBg())"}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + res, err := p.fallback("Comp.vue", tt.source) + if tt.wantErr { + if err == nil { + t.Fatalf("expected error, got result: %+v", res) + } + return + } + if err != nil { + t.Fatalf("fallback failed: %v", err) + } + for _, s := range tt.wantContains { + if !strings.Contains(res.Text, s) { + t.Fatalf("fallback text missing %q in %q", s, res.Text) + } + } + if len(res.GeneratedToSourceLine) == 0 { + t.Fatal("expected generated->source line map") + } + }) + } +} + +func TestVueProcessorTransform_CompilerUnavailableFallsBackViaRegistryAuto(t *testing.T) { + p := NewVueProcessor("node-does-not-exist") + r := NewProcessorRegistry(RegistryConfig{Enabled: true, Mode: ModeAuto, EnableVue: true}, p) + + source := ` +` + + res, err := r.TransformForEmbedding(context.Background(), "Comp.vue", source) + if err != nil { + t.Fatalf("auto mode should not fail on unavailable compiler: %v", err) + } + if !strings.Contains(res.Text, "const value = 123") { + t.Fatalf("expected fallback script content, got: %q", res.Text) + } + if len(res.Warnings) == 0 { + t.Fatal("expected warning about compiler fallback") + } +} + +func TestVueProcessorTransform_CompilerUnavailableRequireFails(t *testing.T) { + p := NewVueProcessor("node-does-not-exist") + r := NewProcessorRegistry(RegistryConfig{Enabled: true, Mode: ModeRequire, EnableVue: true}, p) + + source := ` +` + + _, err := r.TransformForTrace(context.Background(), "Comp.vue", source) + if err == nil { + t.Fatal("expected require mode failure when compiler unavailable") + } +} diff --git a/framework/vue_processor.go b/framework/vue_processor.go new file mode 100644 index 0000000..1f44b6f --- /dev/null +++ b/framework/vue_processor.go @@ -0,0 +1,241 @@ +package framework + +import ( + "bytes" + "context" + _ "embed" + "encoding/json" + "fmt" + "os/exec" + "regexp" + "strings" +) + +//go:embed scripts/vue_processor.mjs +var vueProcessorScript string + +type VueProcessor struct { + nodePath string +} + +func NewVueProcessor(nodePath string) *VueProcessor { + if nodePath == "" { + nodePath = "node" + } + return &VueProcessor{nodePath: nodePath} +} + +func (p *VueProcessor) Name() string { return "vue" } +func (p *VueProcessor) Supports(filePath string) bool { + return hasExt(filePath, ".vue") +} +func (p *VueProcessor) Capabilities() ProcessorCapabilities { + return ProcessorCapabilities{Embedding: true, Trace: true, Compiled: true} +} + +func (p *VueProcessor) TransformForEmbedding(ctx context.Context, filePath, source string) (TransformResult, error) { + return p.transform(ctx, filePath, source) +} + +func (p *VueProcessor) TransformForTrace(ctx context.Context, filePath, source string) (TransformResult, error) { + return p.transform(ctx, filePath, source) +} + +type vueScriptInput struct { + FilePath string `json:"filePath"` + Source string `json:"source"` +} + +type vueScriptOutput struct { + EmbeddingText string `json:"embeddingText"` + TraceText string `json:"traceText"` + VirtualPath string `json:"virtualPath"` + GeneratedToSourceMap []int `json:"generatedToSourceLine"` + Warnings []string `json:"warnings"` +} + +func (p *VueProcessor) transform(ctx context.Context, filePath, source string) (TransformResult, error) { + in, _ := json.Marshal(vueScriptInput{FilePath: filePath, Source: source}) + cmd := exec.CommandContext(ctx, p.nodePath, "--input-type=module", "-e", vueProcessorScript) + cmd.Stdin = bytes.NewReader(in) + var stdout bytes.Buffer + var stderr bytes.Buffer + cmd.Stdout = &stdout + cmd.Stderr = &stderr + + if err := cmd.Run(); err != nil { + fallback, fbErr := p.fallback(filePath, source) + if fbErr == nil { + fallback.Warnings = append(fallback.Warnings, + fmt.Sprintf("vue compiler unavailable: %s", strings.TrimSpace(stderr.String()))) + return fallback, fmt.Errorf("%w: %v", ErrUnavailable, err) + } + return TransformResult{}, fmt.Errorf("%w: vue processor failed: %v (%s)", ErrUnavailable, err, strings.TrimSpace(stderr.String())) + } + + var out vueScriptOutput + if err := json.Unmarshal(stdout.Bytes(), &out); err != nil { + return TransformResult{}, fmt.Errorf("%w: invalid vue processor output: %v", ErrUnavailable, err) + } + text := out.EmbeddingText + if text == "" { + text = source + } + virtual := out.VirtualPath + if virtual == "" { + virtual = filePath + ".__trace__.ts" + } + return TransformResult{ + Processor: p.Name(), + FilePath: filePath, + VirtualPath: virtual, + Text: text, + GeneratedToSourceLine: out.GeneratedToSourceMap, + Warnings: out.Warnings, + Transformed: true, + }, nil +} + +var scriptBlockRE = regexp.MustCompile(`(?is)]*>(.*?)`) +var styleBlockRE = regexp.MustCompile(`(?is)]*>(.*?)`) + +func (p *VueProcessor) fallback(filePath, source string) (TransformResult, error) { + matches := scriptBlockRE.FindAllStringSubmatchIndex(source, -1) + var out strings.Builder + mapping := make([]int, 0, len(source)/20) + for idx, m := range matches { + if len(m) < 4 { + continue + } + start, end := m[2], m[3] + block := source[start:end] + if strings.TrimSpace(block) == "" { + continue + } + if idx > 0 && out.Len() > 0 { + out.WriteString("\n") + mapping = append(mapping, 0) + } + + sourceStart := strings.Count(source[:start], "\n") + 1 + lines := strings.Split(block, "\n") + for i, line := range lines { + if i > 0 { + out.WriteString("\n") + } + out.WriteString(line) + mapping = append(mapping, sourceStart+i) + } + } + + styleMatches := styleBlockRE.FindAllStringSubmatchIndex(source, -1) + styleBindingIndex := 0 + for _, m := range styleMatches { + if len(m) < 4 { + continue + } + start, end := m[2], m[3] + block := source[start:end] + if strings.TrimSpace(block) == "" { + continue + } + lineOffset := strings.Count(source[:start], "\n") + 1 + refs := extractStyleVBindRefs(block, lineOffset) + if len(refs) == 0 { + continue + } + if out.Len() > 0 { + out.WriteString("\n") + mapping = append(mapping, 0) + } + header := fmt.Sprintf("function __vue_style_bindings__%d() {", styleBindingIndex) + out.WriteString(header) + mapping = append(mapping, refs[0].line) + for _, ref := range refs { + out.WriteString("\n __css_v_bind__(" + ref.expr + ");") + mapping = append(mapping, ref.line) + } + out.WriteString("\n}") + mapping = append(mapping, refs[len(refs)-1].line) + styleBindingIndex++ + } + + if out.Len() == 0 { + return TransformResult{}, fmt.Errorf("no script blocks or style v-bind expressions found") + } + + return TransformResult{ + Processor: p.Name(), + FilePath: filePath, + VirtualPath: filePath + ".__trace__.ts", + Text: out.String(), + GeneratedToSourceLine: mapping, + Transformed: true, + }, nil +} + +type styleVBindRef struct { + expr string + line int +} + +func extractStyleVBindRefs(styleContent string, startLine int) []styleVBindRef { + lines := strings.Split(styleContent, "\n") + refs := make([]styleVBindRef, 0, len(lines)) + for i, line := range lines { + exprs := extractVBindExprsFromLine(line) + for _, raw := range exprs { + expr := normalizeStyleVBindExpr(raw) + if expr == "" { + continue + } + refs = append(refs, styleVBindRef{ + expr: expr, + line: startLine + i, + }) + } + } + return refs +} + +func extractVBindExprsFromLine(line string) []string { + const needle = "v-bind(" + out := make([]string, 0, 2) + start := 0 + + for start < len(line) { + idx := strings.Index(line[start:], needle) + if idx < 0 { + break + } + openIdx := start + idx + len(needle) + depth := 1 + i := openIdx + for i < len(line) && depth > 0 { + switch line[i] { + case '(': + depth++ + case ')': + depth-- + } + i++ + } + if depth != 0 { + break + } + out = append(out, line[openIdx:i-1]) + start = i + } + + return out +} + +func normalizeStyleVBindExpr(expr string) string { + trimmed := strings.TrimSpace(expr) + if len(trimmed) >= 2 { + if (trimmed[0] == '\'' && trimmed[len(trimmed)-1] == '\'') || (trimmed[0] == '"' && trimmed[len(trimmed)-1] == '"') { + return strings.TrimSpace(trimmed[1 : len(trimmed)-1]) + } + } + return trimmed +} diff --git a/framework/vue_processor_test.go b/framework/vue_processor_test.go new file mode 100644 index 0000000..575e0de --- /dev/null +++ b/framework/vue_processor_test.go @@ -0,0 +1,56 @@ +package framework + +import ( + "strings" + "testing" +) + +func TestVueProcessorFallback_ExtractsScriptBlocksAndMapsLines(t *testing.T) { + p := NewVueProcessor("node") + source := ` + + + +` + + res, err := p.fallback("Comp.vue", source) + if err != nil { + t.Fatalf("fallback failed: %v", err) + } + if !strings.Contains(res.Text, "export function a") { + t.Fatalf("missing first script block in fallback text: %q", res.Text) + } + if !strings.Contains(res.Text, "const b = 2") { + t.Fatalf("missing script setup block in fallback text: %q", res.Text) + } + if len(res.GeneratedToSourceLine) == 0 { + t.Fatal("expected generated->source map") + } + if got := RemapLine(res.GeneratedToSourceLine, 1); got < 3 { + t.Fatalf("expected mapped line to point into script block, got %d", got) + } +} + +func TestExtractStyleVBindRefs(t *testing.T) { + style := `.a { color: v-bind(color); background: v-bind(getBg()); } +.b { border-color: v-bind("borderColor"); }` + refs := extractStyleVBindRefs(style, 12) + if len(refs) != 3 { + t.Fatalf("expected 3 refs, got %d", len(refs)) + } + if refs[0].expr != "color" || refs[0].line != 12 { + t.Fatalf("unexpected first ref: %+v", refs[0]) + } + if refs[1].expr != "getBg()" || refs[1].line != 12 { + t.Fatalf("unexpected second ref: %+v", refs[1]) + } + if refs[2].expr != "borderColor" || refs[2].line != 13 { + t.Fatalf("unexpected third ref: %+v", refs[2]) + } +} diff --git a/indexer/chunker.go b/indexer/chunker.go index 1550301..f8e8181 100644 --- a/indexer/chunker.go +++ b/indexer/chunker.go @@ -15,13 +15,14 @@ const ( ) type ChunkInfo struct { - ID string - FilePath string - StartLine int - EndLine int - Content string - Hash string - ContentHash string // SHA256 of raw content text (without file path prefix) + ID string + FilePath string + StartLine int + EndLine int + Content string // Display content persisted in vector store. + EmbedContent string // Content used for embeddings and content hash. + Hash string + ContentHash string // SHA256 of raw content text (without file path prefix) } type Chunker struct { @@ -105,13 +106,14 @@ func (c *Chunker) Chunk(filePath string, content string) []ChunkInfo { chunkID := fmt.Sprintf("%s_%d", filePath, chunkIndex) chunks = append(chunks, ChunkInfo{ - ID: chunkID, - FilePath: filePath, - StartLine: startLine, - EndLine: endLine, - Content: chunkContent, - Hash: hex.EncodeToString(hash[:8]), - ContentHash: hex.EncodeToString(contentHash[:]), + ID: chunkID, + FilePath: filePath, + StartLine: startLine, + EndLine: endLine, + Content: chunkContent, + EmbedContent: chunkContent, + Hash: hex.EncodeToString(hash[:8]), + ContentHash: hex.EncodeToString(contentHash[:]), }) chunkIndex++ @@ -161,6 +163,7 @@ func (c *Chunker) ChunkWithContext(filePath string, content string) []ChunkInfo // Add file path context to each chunk for i := range chunks { chunks[i].Content = fmt.Sprintf("File: %s\n\n%s", filePath, chunks[i].Content) + chunks[i].EmbedContent = chunks[i].Content } return chunks @@ -171,7 +174,10 @@ func (c *Chunker) ChunkWithContext(filePath string, content string) []ChunkInfo // The parentIndex is used to generate unique sub-chunk IDs (e.g., "file.go_0_0", "file.go_0_1"). func (c *Chunker) ReChunk(parent ChunkInfo, parentIndex int) []ChunkInfo { // Strip the file context prefix if present (we'll re-add it later) - content := parent.Content + content := parent.EmbedContent + if content == "" { + content = parent.Content + } filePrefix := fmt.Sprintf("File: %s\n\n", parent.FilePath) hasContext := strings.HasPrefix(content, filePrefix) if hasContext { @@ -244,13 +250,14 @@ func (c *Chunker) ReChunk(parent ChunkInfo, parentIndex int) []ChunkInfo { } subChunks = append(subChunks, ChunkInfo{ - ID: subChunkID, - FilePath: parent.FilePath, - StartLine: absoluteStartLine, - EndLine: absoluteEndLine, - Content: finalContent, - Hash: hex.EncodeToString(hash[:8]), - ContentHash: hex.EncodeToString(contentHash[:]), + ID: subChunkID, + FilePath: parent.FilePath, + StartLine: absoluteStartLine, + EndLine: absoluteEndLine, + Content: finalContent, + EmbedContent: finalContent, + Hash: hex.EncodeToString(hash[:8]), + ContentHash: hex.EncodeToString(contentHash[:]), }) subIndex++ diff --git a/indexer/indexer.go b/indexer/indexer.go index 475cef3..34c4391 100644 --- a/indexer/indexer.go +++ b/indexer/indexer.go @@ -7,6 +7,7 @@ import ( "time" "github.com/yoanbernabeu/grepai/embedder" + "github.com/yoanbernabeu/grepai/framework" "github.com/yoanbernabeu/grepai/store" ) @@ -16,6 +17,7 @@ type Indexer struct { embedder embedder.Embedder chunker *Chunker scanner *Scanner + processor *framework.ProcessorRegistry lastIndexTime time.Time } @@ -59,13 +61,20 @@ func NewIndexer( chunker *Chunker, scanner *Scanner, lastIndexTime time.Time, + processors ...*framework.ProcessorRegistry, ) *Indexer { + var processor *framework.ProcessorRegistry + if len(processors) > 0 { + processor = processors[0] + } + return &Indexer{ root: root, store: st, embedder: emb, chunker: chunker, scanner: scanner, + processor: processor, lastIndexTime: lastIndexTime, } } @@ -214,6 +223,8 @@ type fileChunkData struct { fileIndex int // Index in the files slice (for result mapping) file FileInfo chunkInfos []ChunkInfo + lineMap []int + source string } // prepareFileChunks processes files by deleting existing chunks and creating new chunks. @@ -230,20 +241,23 @@ func (idx *Indexer) prepareFileChunks( return nil, nil, fmt.Errorf("failed to delete existing chunks for %s: %w", file.Path, err) } - chunkInfos := idx.chunker.ChunkWithContext(file.Path, file.Content) + embedContent, lineMap := idx.embeddingContent(ctx, file) + chunkInfos := idx.chunker.ChunkWithContext(file.Path, embedContent) if len(chunkInfos) == 0 { continue } contents := make([]string, len(chunkInfos)) for j, c := range chunkInfos { - contents[j] = c.Content + contents[j] = c.EmbedContent } fileData = append(fileData, fileChunkData{ fileIndex: i, file: file, chunkInfos: chunkInfos, + lineMap: lineMap, + source: file.Content, }) fileChunks = append(fileChunks, embedder.FileChunks{ @@ -393,6 +407,7 @@ func (idx *Indexer) indexFilesBatched( now := time.Now() for _, pf := range preFilledFiles { fd := fileData[pf.fdIndex] + idx.remapChunksToSource(fd.chunkInfos, fd.file.Path, fd.source, fd.lineMap) chunks, chunkIDs := createStoreChunks(fd.chunkInfos, pf.vectors, now) if err := idx.saveFileData(ctx, fd, chunks, chunkIDs); err != nil { return filesIndexed, chunksCreated, err @@ -418,6 +433,7 @@ func (idx *Indexer) indexFilesBatched( fd.file.Path, len(embeddings), len(fd.chunkInfos)) continue } + idx.remapChunksToSource(fd.chunkInfos, fd.file.Path, fd.source, fd.lineMap) chunks, chunkIDs := createStoreChunks(fd.chunkInfos, embeddings, now) if err := idx.saveFileData(ctx, fd, chunks, chunkIDs); err != nil { return filesIndexed, chunksCreated, err @@ -441,8 +457,10 @@ func (idx *Indexer) IndexFile(ctx context.Context, file FileInfo) (int, error) { return 0, fmt.Errorf("failed to delete existing chunks: %w", err) } + embedContent, lineMap := idx.embeddingContent(ctx, file) + // Chunk the file - chunkInfos := idx.chunker.ChunkWithContext(file.Path, file.Content) + chunkInfos := idx.chunker.ChunkWithContext(file.Path, embedContent) if len(chunkInfos) == 0 { return 0, nil } @@ -517,6 +535,8 @@ func (idx *Indexer) IndexFile(ctx context.Context, file FileInfo) (int, error) { } } + idx.remapChunksToSource(finalChunks, file.Path, file.Content, lineMap) + // Create store chunks now := time.Now() chunks := make([]store.Chunk, len(finalChunks)) @@ -567,7 +587,11 @@ func (idx *Indexer) embedWithReChunking(ctx context.Context, chunks []ChunkInfo) for attempt := 0; attempt < maxReChunkAttempts; attempt++ { contents := make([]string, len(currentChunks)) for i, c := range currentChunks { - contents[i] = c.Content + if c.EmbedContent != "" { + contents[i] = c.EmbedContent + } else { + contents[i] = c.Content + } } vectors, err := idx.embedder.EmbedBatch(ctx, contents) @@ -599,7 +623,11 @@ func (idx *Indexer) embedWithReChunking(ctx context.Context, chunks []ChunkInfo) if failedIndex > 0 { beforeContents := make([]string, failedIndex) for i := 0; i < failedIndex; i++ { - beforeContents[i] = currentChunks[i].Content + if currentChunks[i].EmbedContent != "" { + beforeContents[i] = currentChunks[i].EmbedContent + } else { + beforeContents[i] = currentChunks[i].Content + } } beforeVectors, err := idx.embedder.EmbedBatch(ctx, beforeContents) if err != nil { @@ -624,6 +652,38 @@ func (idx *Indexer) embedWithReChunking(ctx context.Context, chunks []ChunkInfo) return nil, nil, fmt.Errorf("exceeded maximum re-chunk attempts (%d) for file", maxReChunkAttempts) } +func (idx *Indexer) embeddingContent(ctx context.Context, file FileInfo) (string, []int) { + if idx.processor == nil { + return file.Content, nil + } + + res, err := idx.processor.TransformForEmbedding(ctx, file.Path, file.Content) + if err != nil { + log.Printf("Warning: framework embedding transform failed for %s: %v", file.Path, err) + return file.Content, nil + } + for _, w := range res.Warnings { + log.Printf("Warning: %s", w) + } + if res.Text == "" { + return file.Content, nil + } + return res.Text, res.GeneratedToSourceLine +} + +func (idx *Indexer) remapChunksToSource(chunks []ChunkInfo, filePath, source string, lineMap []int) { + for i := range chunks { + startLine, endLine := framework.RemapLineRange(lineMap, chunks[i].StartLine, chunks[i].EndLine) + snippet := framework.SourceSnippet(source, startLine, endLine) + if snippet == "" { + continue + } + chunks[i].StartLine = startLine + chunks[i].EndLine = endLine + chunks[i].Content = fmt.Sprintf("File: %s\n\n%s", filePath, snippet) + } +} + // lookupCachedEmbeddings checks if the store implements EmbeddingCache and returns // cached vectors for chunks with matching content hashes. The returned map maps // chunk index to cached vector. Chunks not in the map need fresh embedding. diff --git a/indexer/indexer_test.go b/indexer/indexer_test.go index 5b7535f..7e59cb3 100644 --- a/indexer/indexer_test.go +++ b/indexer/indexer_test.go @@ -11,6 +11,7 @@ import ( "time" "github.com/yoanbernabeu/grepai/embedder" + "github.com/yoanbernabeu/grepai/framework" "github.com/yoanbernabeu/grepai/store" ) @@ -166,6 +167,7 @@ func (m *mockStore) GetAllChunks(ctx context.Context) ([]store.Chunk, error) { // mockEmbedder implements embedder.Embedder for testing type mockEmbedder struct { embedCalled bool + lastBatch []string } func newMockEmbedder() *mockEmbedder { @@ -179,6 +181,7 @@ func (m *mockEmbedder) Embed(ctx context.Context, text string) ([]float32, error func (m *mockEmbedder) EmbedBatch(ctx context.Context, texts []string) ([][]float32, error) { m.embedCalled = true + m.lastBatch = append([]string(nil), texts...) vectors := make([][]float32, len(texts)) for i := range texts { vectors[i] = []float32{0.1, 0.2, 0.3} @@ -1111,8 +1114,8 @@ func TestEmbedWithReChunking_Success(t *testing.T) { } chunks := []ChunkInfo{ - {ID: "chunk1", FilePath: "test.go", Content: "small content", StartLine: 1, EndLine: 5}, - {ID: "chunk2", FilePath: "test.go", Content: "more content", StartLine: 6, EndLine: 10}, + {ID: "chunk1", FilePath: "test.go", Content: "small content", EmbedContent: "small content", StartLine: 1, EndLine: 5}, + {ID: "chunk2", FilePath: "test.go", Content: "more content", EmbedContent: "more content", StartLine: 6, EndLine: 10}, } vectors, finalChunks, err := indexer.embedWithReChunking(context.Background(), chunks) @@ -1147,7 +1150,7 @@ func TestEmbedWithReChunking_ReChunksOnError(t *testing.T) { // Create one large chunk that will exceed the limit largeContent := strings.Repeat("x", 1000) chunks := []ChunkInfo{ - {ID: "test.go_0", FilePath: "test.go", Content: largeContent, StartLine: 1, EndLine: 50}, + {ID: "test.go_0", FilePath: "test.go", Content: largeContent, EmbedContent: largeContent, StartLine: 1, EndLine: 50}, } vectors, finalChunks, err := indexer.embedWithReChunking(context.Background(), chunks) @@ -1165,3 +1168,65 @@ func TestEmbedWithReChunking_ReChunksOnError(t *testing.T) { t.Errorf("vectors count %d != chunks count %d", len(vectors), len(finalChunks)) } } + +type testTransformProcessor struct{} + +func (p *testTransformProcessor) Name() string { return "vue" } +func (p *testTransformProcessor) Supports(filePath string) bool { + return strings.HasSuffix(filePath, ".vue") +} +func (p *testTransformProcessor) Capabilities() framework.ProcessorCapabilities { + return framework.ProcessorCapabilities{Embedding: true, Trace: true} +} +func (p *testTransformProcessor) TransformForEmbedding(ctx context.Context, filePath, source string) (framework.TransformResult, error) { + return framework.TransformResult{ + Processor: "vue", + FilePath: filePath, + VirtualPath: filePath, + Text: "const transformed = true\nconsole.log(transformed)", + GeneratedToSourceLine: []int{2, 3}, + Transformed: true, + }, nil +} +func (p *testTransformProcessor) TransformForTrace(ctx context.Context, filePath, source string) (framework.TransformResult, error) { + return p.TransformForEmbedding(ctx, filePath, source) +} + +func TestIndexFile_UsesTransformedContentAndStoresSourceSnippet(t *testing.T) { + mockEmb := newMockEmbedder() + mockStore := newMockStore() + chunker := NewChunker(512, 50) + reg := framework.NewProcessorRegistry( + framework.RegistryConfig{Enabled: true, Mode: framework.ModeAuto, EnableVue: true}, + &testTransformProcessor{}, + ) + + idx := NewIndexer("/tmp", mockStore, mockEmb, chunker, nil, time.Time{}, reg) + file := FileInfo{ + Path: "Component.vue", + Hash: "h1", + ModTime: time.Now().Unix(), + Content: "\nconst fromSource = 1\nexport default {}", + } + + _, err := idx.IndexFile(context.Background(), file) + if err != nil { + t.Fatalf("IndexFile failed: %v", err) + } + chunks, err := mockStore.GetChunksForFile(context.Background(), "Component.vue") + if err != nil { + t.Fatalf("GetChunksForFile failed: %v", err) + } + if len(chunks) == 0 { + t.Fatal("expected at least one chunk") + } + if len(mockEmb.lastBatch) == 0 || !strings.Contains(mockEmb.lastBatch[0], "transformed") { + t.Fatalf("expected transformed embedding content, got batch: %#v", mockEmb.lastBatch) + } + if !strings.Contains(chunks[0].Content, "fromSource") { + t.Fatalf("expected stored source snippet, got: %q", chunks[0].Content) + } + if chunks[0].StartLine != 2 { + t.Fatalf("expected remapped start line 2, got %d", chunks[0].StartLine) + } +} diff --git a/scripts/test-vue-framework-matrix.sh b/scripts/test-vue-framework-matrix.sh new file mode 100755 index 0000000..11ccd9a --- /dev/null +++ b/scripts/test-vue-framework-matrix.sh @@ -0,0 +1,158 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT="/Users/mladenmihajlovic/Documents/git/github/grepai" +BIN="$ROOT/bin/grepai" +TEST_ROOT="/tmp/grepai-vue-matrix" + +cd "$ROOT" + +echo "[0/8] Ensure binary exists" +[ -x "$BIN" ] || make build +"$BIN" version + +echo "[1/8] Install compiler" +HAVE_COMPILER=1 +echo " (will install into temp project after creation)" + +echo "[2/8] Create matrix project" +rm -rf "$TEST_ROOT" +mkdir -p "$TEST_ROOT/src" + +cat > "$TEST_ROOT/src/ScriptOnly.vue" <<'VUE' + + +VUE + +cat > "$TEST_ROOT/src/ScriptJS.vue" <<'VUE' + + +VUE + +cat > "$TEST_ROOT/src/ScriptNoLang.vue" <<'VUE' + + +VUE + +cat > "$TEST_ROOT/src/ScriptSetupOnly.vue" <<'VUE' + + +VUE + +cat > "$TEST_ROOT/src/Mixed.vue" <<'VUE' + + + +VUE + +cat > "$TEST_ROOT/src/TemplateOnly.vue" <<'VUE' + +VUE + +cat > "$TEST_ROOT/src/StyleHeavy.vue" <<'VUE' + + + +VUE + +if ! ( cd "$TEST_ROOT" && npm init -y >/dev/null 2>&1 && timeout 60s npm install -D @vue/compiler-sfc >/dev/null 2>&1 ); then + HAVE_COMPILER=0 + echo "WARN: compiler install failed or timed out; continuing in fallback-only mode" +fi + +( cd "$TEST_ROOT" && "$BIN" init --yes >/dev/null 2>&1 || true ) +CFG="$TEST_ROOT/.grepai/config.yaml" + +if ! grep -q "framework_processing:" "$CFG"; then + cat >> "$CFG" <<'YAML' + +framework_processing: + enabled: true + mode: auto + node_path: node + frameworks: + vue: + enabled: true + svelte: + enabled: false + astro: + enabled: false + solid: + enabled: false +YAML +fi + +if ! grep -q "\.vue" "$CFG"; then + sed -i '' 's/ - ".tsx"/ - ".tsx"\ + - ".vue"/' "$CFG" +fi + +echo "[3/8] Index with compiler/fallback path" +( cd "$TEST_ROOT" && timeout 20s "$BIN" watch > /tmp/grepai-vue-matrix-watch.log 2>&1 || true ) + +echo "[4/8] Search assertions" +( cd "$TEST_ROOT" && "$BIN" search "mixed helper and styled function" --json ) > /tmp/grepai-vue-matrix-search.json +for f in ScriptOnly.vue ScriptSetupOnly.vue Mixed.vue StyleHeavy.vue; do + grep -q "$f" /tmp/grepai-vue-matrix-search.json + echo " - found $f" +done +for f in ScriptJS.vue ScriptNoLang.vue; do + grep -q "$f" /tmp/grepai-vue-matrix-search.json + echo " - found $f" +done + +echo "[5/8] Trace assertions" +( cd "$TEST_ROOT" && "$BIN" trace callees runMixed --json ) > /tmp/grepai-vue-matrix-trace-callees.json || true +grep -q "helperMixed" /tmp/grepai-vue-matrix-trace-callees.json + +( cd "$TEST_ROOT" && "$BIN" trace callers styledFn --json ) > /tmp/grepai-vue-matrix-trace-callers-style.json || true +grep -q "__vue_style_bindings__" /tmp/grepai-vue-matrix-trace-callers-style.json +echo " - style v-bind synthetic caller detected for styledFn" + +( cd "$TEST_ROOT" && "$BIN" trace callers scriptJsFn --json ) > /tmp/grepai-vue-matrix-trace-callers-js.json || true +grep -q "scriptJsFn" /tmp/grepai-vue-matrix-trace-callers-js.json +( cd "$TEST_ROOT" && "$BIN" trace callers scriptNoLangFn --json ) > /tmp/grepai-vue-matrix-trace-callers-nolang.json || true +grep -q "scriptNoLangFn" /tmp/grepai-vue-matrix-trace-callers-nolang.json +echo " - JS and no-lang script symbols detected" + +echo "[6/8] Fallback mode run" +if [ "$HAVE_COMPILER" -eq 1 ]; then + ( cd "$TEST_ROOT" && npm uninstall @vue/compiler-sfc >/dev/null ) || true +fi +( cd "$TEST_ROOT" && timeout 12s "$BIN" watch > /tmp/grepai-vue-matrix-watch-fallback.log 2>&1 || true ) + +echo "[7/8] Fallback still searchable" +( cd "$TEST_ROOT" && "$BIN" search "setup only function" --json ) > /tmp/grepai-vue-matrix-search-fallback.json +grep -q "ScriptSetupOnly.vue" /tmp/grepai-vue-matrix-search-fallback.json + +echo "[8/8] Done" +echo "Artifacts:" +echo " /tmp/grepai-vue-matrix-watch.log" +echo " /tmp/grepai-vue-matrix-search.json" +echo " /tmp/grepai-vue-matrix-trace-callees.json" +echo " /tmp/grepai-vue-matrix-trace-callers-style.json" +echo " /tmp/grepai-vue-matrix-trace-callers-js.json" +echo " /tmp/grepai-vue-matrix-trace-callers-nolang.json" +echo " /tmp/grepai-vue-matrix-watch-fallback.log" +echo " /tmp/grepai-vue-matrix-search-fallback.json" diff --git a/scripts/test-vue-framework-processing.sh b/scripts/test-vue-framework-processing.sh new file mode 100755 index 0000000..7b7fb04 --- /dev/null +++ b/scripts/test-vue-framework-processing.sh @@ -0,0 +1,151 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT="/Users/mladenmihajlovic/Documents/git/github/grepai" +BIN="$ROOT/bin/grepai" + +cd "$ROOT" + +echo "[0/9] Ensure grepai binary exists..." +if [ ! -x "$BIN" ]; then + echo "Building $BIN ..." + make build +fi +"$BIN" version + +echo "[1/9] Install Vue compiler (dev dep)..." +echo " (will install into temp project after creation)" + +echo "[2/9] Create test Vue file..." +rm -rf /tmp/grepai-vue-test +mkdir -p /tmp/grepai-vue-test/src +cat > /tmp/grepai-vue-test/src/UserUtils.vue <<'VUE' + + +VUE + +cat > /tmp/grepai-vue-test/src/ActivityPanel.vue <<'VUE' + + +VUE + +cat > /tmp/grepai-vue-test/src/Dashboard.vue <<'VUE' + + +VUE + +cat > /tmp/grepai-vue-test/src/App.vue <<'VUE' + + +VUE + +( cd /tmp/grepai-vue-test && npm init -y >/dev/null 2>&1 && npm install -D @vue/compiler-sfc >/dev/null ) + +echo "[3/9] Ensure grepai config exists..." +( cd /tmp/grepai-vue-test && "$BIN" init --yes >/dev/null 2>&1 || true ) + +CFG="/tmp/grepai-vue-test/.grepai/config.yaml" +if ! grep -q "framework_processing:" "$CFG"; then + cat >> "$CFG" <<'YAML' + +framework_processing: + enabled: true + mode: auto + node_path: node + frameworks: + vue: + enabled: true + svelte: + enabled: false + astro: + enabled: false + solid: + enabled: false +YAML +fi + +# Ensure Vue is included in traced languages for symbol extraction. +if ! grep -q "\.vue" "$CFG"; then + if grep -q ' - ".tsx"' "$CFG"; then + sed -i '' 's/ - ".tsx"/ - ".tsx"\ + - ".vue"/' "$CFG" + else + cat >> "$CFG" <<'YAML' + +trace: + enabled_languages: + - ".vue" +YAML + fi +fi + +echo "[4/9] Run watch briefly for initial index..." +( cd /tmp/grepai-vue-test && timeout 18s "$BIN" watch >/tmp/grepai-watch.log 2>&1 || true ) + +echo "[5/9] Run search..." +( cd /tmp/grepai-vue-test && "$BIN" search "user greeting and activity label formatting" --json ) | tee /tmp/grepai-search.json + +echo "[6/9] Check search includes multiple Vue files..." +grep -q "UserUtils.vue" /tmp/grepai-search.json +grep -q "ActivityPanel.vue" /tmp/grepai-search.json +grep -q "Dashboard.vue" /tmp/grepai-search.json +echo "OK: search returned multiple Vue files." + +echo "[7/9] Run trace..." +( cd /tmp/grepai-vue-test && "$BIN" trace callers formatUserName --json ) | tee /tmp/grepai-trace-callers.json || true +( cd /tmp/grepai-vue-test && "$BIN" trace callees buildHeader --json ) | tee /tmp/grepai-trace-callees.json || true +grep -q "formatUserName" /tmp/grepai-trace-callers.json +grep -q "buildActivityLabel" /tmp/grepai-trace-callers.json +grep -q "buildHeader" /tmp/grepai-trace-callees.json +grep -q "getGreeting" /tmp/grepai-trace-callees.json +echo "OK: trace captured cross-file function relationships." + +echo "[8/9] Optional fallback test (remove compiler, keep auto mode)..." +( cd /tmp/grepai-vue-test && npm uninstall @vue/compiler-sfc >/dev/null ) || true +( cd /tmp/grepai-vue-test && timeout 8s "$BIN" watch >/tmp/grepai-watch-fallback.log 2>&1 || true ) + +echo "[9/9] Done." +echo "Artifacts:" +echo " /tmp/grepai-search.json" +echo " /tmp/grepai-trace-callers.json" +echo " /tmp/grepai-trace-callees.json" +echo " /tmp/grepai-watch.log" +echo " /tmp/grepai-watch-fallback.log" diff --git a/trace/extractor.go b/trace/extractor.go index 4c204ff..d15bcb5 100644 --- a/trace/extractor.go +++ b/trace/extractor.go @@ -177,6 +177,9 @@ func (e *RegexExtractor) ExtractReferences(ctx context.Context, filePath string, continue } + if isDeclarationCallArtifact(content, match[0], name, patterns.Language) { + continue + } appendRef(name, match[0]) } } @@ -461,6 +464,69 @@ func buildIgnoredMask(content string, lang string) []bool { return mask } +// isDeclarationCallArtifact filters regex call matches that are actually +// function/method declarations in JS/TS-like languages. +func isDeclarationCallArtifact(content string, pos int, name string, lang string) bool { + if lang != "javascript" && lang != "typescript" { + return false + } + + lineStart := strings.LastIndex(content[:pos], "\n") + 1 + lineEndRel := strings.Index(content[pos:], "\n") + lineEnd := len(content) + if lineEndRel >= 0 { + lineEnd = pos + lineEndRel + } + line := strings.TrimSpace(content[lineStart:lineEnd]) + if line == "" { + return false + } + + declPrefixes := []string{ + "function " + name + "(", + "async function " + name + "(", + "export function " + name + "(", + "export async function " + name + "(", + "export default function " + name + "(", + "export default async function " + name + "(", + } + for _, p := range declPrefixes { + if strings.HasPrefix(line, p) { + return true + } + } + + // Class/object method declaration styles: + // - methodName(args) { ... } + // - public async methodName(args) { ... } + // These must include "{" on the same line. + if strings.Contains(line, "{") { + withoutMods := stripTSMethodModifiers(line) + if strings.HasPrefix(withoutMods, name+"(") { + return true + } + } + + return false +} + +func stripTSMethodModifiers(line string) string { + out := strings.TrimSpace(line) + mods := []string{"public ", "private ", "protected ", "static ", "readonly ", "abstract ", "override ", "async "} + for { + changed := false + for _, m := range mods { + if strings.HasPrefix(out, m) { + out = strings.TrimSpace(strings.TrimPrefix(out, m)) + changed = true + } + } + if !changed { + return out + } + } +} + // ExtractAll extracts both symbols and references in one pass. func (e *RegexExtractor) ExtractAll(ctx context.Context, filePath string, content string) ([]Symbol, []Reference, error) { symbols, err := e.ExtractSymbols(ctx, filePath, content) diff --git a/trace/extractor_test.go b/trace/extractor_test.go index b264343..f0a9e90 100644 --- a/trace/extractor_test.go +++ b/trace/extractor_test.go @@ -1014,6 +1014,48 @@ end` } } +func TestRegexExtractor_ExtractReferences_TypeScriptSkipsDeclarationArtifacts(t *testing.T) { + extractor := NewRegexExtractor() + ctx := context.Background() + + content := "export function formatUserName(name: string): string {\n" + + " return name.trim().toUpperCase()\n" + + "}\n\n" + + "export function getGreeting(name: string): string {\n" + + " return formatUserName(name)\n" + + "}\n\n" + + "export function buildActivityLabel(name: string, count: number): string {\n" + + " return formatUserName(name) + ' has ' + String(count) + ' alerts'\n" + + "}\n" + + refs, err := extractor.ExtractReferences(ctx, "test.ts", content) + if err != nil { + t.Fatalf("ExtractReferences failed: %v", err) + } + + var selfCall bool + callers := make(map[string]bool) + for _, ref := range refs { + if ref.SymbolName != "formatUserName" { + continue + } + if ref.CallerName == "formatUserName" { + selfCall = true + } + callers[ref.CallerName] = true + } + + if selfCall { + t.Fatal("unexpected self-call artifact for function declaration") + } + if !callers["getGreeting"] { + t.Fatal("expected getGreeting -> formatUserName reference") + } + if !callers["buildActivityLabel"] { + t.Fatal("expected buildActivityLabel -> formatUserName reference") + } +} + func TestIsKeyword(t *testing.T) { tests := []struct { name string From 39419ffea37400b84f932a9eb3aa9ed983832a5a Mon Sep 17 00:00:00 2001 From: Mladen Mihajlovic Date: Thu, 12 Mar 2026 11:45:18 +0100 Subject: [PATCH 02/14] feat(trace): synthesize Vue template read callers --- framework/vue_processor.go | 76 +++++++++++++++++++++++++++++++-- framework/vue_processor_test.go | 22 ++++++++++ 2 files changed, 95 insertions(+), 3 deletions(-) diff --git a/framework/vue_processor.go b/framework/vue_processor.go index 1f44b6f..ba06154 100644 --- a/framework/vue_processor.go +++ b/framework/vue_processor.go @@ -8,6 +8,7 @@ import ( "fmt" "os/exec" "regexp" + "sort" "strings" ) @@ -85,12 +86,13 @@ func (p *VueProcessor) transform(ctx context.Context, filePath, source string) ( if virtual == "" { virtual = filePath + ".__trace__.ts" } + text, mapping := appendTemplateCtxReadCalls(text, out.GeneratedToSourceMap) return TransformResult{ Processor: p.Name(), FilePath: filePath, VirtualPath: virtual, Text: text, - GeneratedToSourceLine: out.GeneratedToSourceMap, + GeneratedToSourceLine: mapping, Warnings: out.Warnings, Transformed: true, }, nil @@ -98,6 +100,7 @@ func (p *VueProcessor) transform(ctx context.Context, filePath, source string) ( var scriptBlockRE = regexp.MustCompile(`(?is)]*>(.*?)`) var styleBlockRE = regexp.MustCompile(`(?is)]*>(.*?)`) +var vueTemplateCtxIdentRE = regexp.MustCompile(`\b_ctx\.([A-Za-z_$][A-Za-z0-9_$]*)\b`) func (p *VueProcessor) fallback(filePath, source string) (TransformResult, error) { matches := scriptBlockRE.FindAllStringSubmatchIndex(source, -1) @@ -163,13 +166,14 @@ func (p *VueProcessor) fallback(filePath, source string) (TransformResult, error if out.Len() == 0 { return TransformResult{}, fmt.Errorf("no script blocks or style v-bind expressions found") } + text, mapped := appendTemplateCtxReadCalls(out.String(), mapping) return TransformResult{ Processor: p.Name(), FilePath: filePath, VirtualPath: filePath + ".__trace__.ts", - Text: out.String(), - GeneratedToSourceLine: mapping, + Text: text, + GeneratedToSourceLine: mapped, Transformed: true, }, nil } @@ -239,3 +243,69 @@ func normalizeStyleVBindExpr(expr string) string { } return trimmed } + +func appendTemplateCtxReadCalls(text string, generatedToSource []int) (string, []int) { + if strings.TrimSpace(text) == "" { + return text, generatedToSource + } + + lines := strings.Split(text, "\n") + sourceLineByIdent := make(map[string]int) + for i, line := range lines { + matches := vueTemplateCtxIdentRE.FindAllStringSubmatch(line, -1) + if len(matches) == 0 { + continue + } + lineSource := 0 + if i < len(generatedToSource) { + lineSource = generatedToSource[i] + } + for _, m := range matches { + if len(m) < 2 { + continue + } + name := m[1] + if strings.HasPrefix(name, "$") || strings.HasPrefix(name, "_") { + continue + } + if _, ok := sourceLineByIdent[name]; !ok { + sourceLineByIdent[name] = lineSource + } + } + } + + if len(sourceLineByIdent) == 0 { + return text, generatedToSource + } + + idents := make([]string, 0, len(sourceLineByIdent)) + for name := range sourceLineByIdent { + idents = append(idents, name) + } + sort.Strings(idents) + + baseSource := 0 + for _, name := range idents { + if sourceLineByIdent[name] > 0 { + baseSource = sourceLineByIdent[name] + break + } + } + + var b strings.Builder + b.Grow(len(text) + 64 + len(idents)*24) + b.WriteString(text) + b.WriteString("\nfunction __vue_template_reads__() {") + updatedMap := append([]int(nil), generatedToSource...) + updatedMap = append(updatedMap, baseSource) + for _, name := range idents { + b.WriteString("\n ") + b.WriteString(name) + b.WriteString("();") + updatedMap = append(updatedMap, sourceLineByIdent[name]) + } + b.WriteString("\n}") + updatedMap = append(updatedMap, baseSource) + + return b.String(), updatedMap +} diff --git a/framework/vue_processor_test.go b/framework/vue_processor_test.go index 575e0de..0192e71 100644 --- a/framework/vue_processor_test.go +++ b/framework/vue_processor_test.go @@ -54,3 +54,25 @@ func TestExtractStyleVBindRefs(t *testing.T) { t.Fatalf("unexpected third ref: %+v", refs[2]) } } + +func TestAppendTemplateCtxReadCalls(t *testing.T) { + input := "const _s = 1\nreturn _ctx.isAdmin && _ctx.store && _ctx.$slots.default && _ctx._hidden" + mapping := []int{3, 8} + + out, outMap := appendTemplateCtxReadCalls(input, mapping) + if !strings.Contains(out, "function __vue_template_reads__() {") { + t.Fatalf("missing synthetic template caller function: %q", out) + } + if !strings.Contains(out, "isAdmin();") { + t.Fatalf("missing isAdmin synthetic read caller: %q", out) + } + if !strings.Contains(out, "store();") { + t.Fatalf("missing store synthetic read caller: %q", out) + } + if strings.Contains(out, "$slots();") || strings.Contains(out, "_hidden();") { + t.Fatalf("unexpected internal ctx symbols emitted: %q", out) + } + if len(outMap) <= len(mapping) { + t.Fatalf("expected extended line map, got %d <= %d", len(outMap), len(mapping)) + } +} From d2e578d4e1a389c4aa266bc12538d30b6c887b8b Mon Sep 17 00:00:00 2001 From: Mladen Mihajlovic Date: Thu, 12 Mar 2026 15:53:27 +0100 Subject: [PATCH 03/14] fix(trace): disambiguate same-name symbols by file context --- cli/trace.go | 83 ++++++++++++++++++++++++++++++++++++++++++----- cli/trace_test.go | 33 +++++++++++++++++++ 2 files changed, 108 insertions(+), 8 deletions(-) diff --git a/cli/trace.go b/cli/trace.go index 8747c04..d507cbb 100644 --- a/cli/trace.go +++ b/cli/trace.go @@ -6,6 +6,7 @@ import ( "fmt" "log" "os" + "sort" "strings" "github.com/alpkeskin/gotoon" @@ -15,6 +16,59 @@ import ( "github.com/yoanbernabeu/grepai/trace" ) +func pickBestTargetSymbol(candidates []trace.Symbol, refs []trace.Reference) *trace.Symbol { + if len(candidates) == 0 { + return nil + } + if len(candidates) == 1 { + return &candidates[0] + } + + bestIdx := 0 + bestScore := -1 + for i, sym := range candidates { + score := 0 + for _, ref := range refs { + // Prefer symbols referenced from their own file first (component-local/private usage). + if ref.File == sym.File || ref.CallerFile == sym.File { + score++ + } + } + if score > bestScore { + bestScore = score + bestIdx = i + } + } + + // Deterministic fallback when all scores are equal. + if bestScore <= 0 { + sorted := append([]trace.Symbol(nil), candidates...) + sort.SliceStable(sorted, func(i, j int) bool { + if sorted[i].File == sorted[j].File { + return sorted[i].Line < sorted[j].Line + } + return sorted[i].File < sorted[j].File + }) + return &sorted[0] + } + + return &candidates[bestIdx] +} + +func pickBestSymbolForFile(candidates []trace.Symbol, preferredFile string) *trace.Symbol { + if len(candidates) == 0 { + return nil + } + if preferredFile != "" { + for i := range candidates { + if candidates[i].File == preferredFile { + return &candidates[i] + } + } + } + return &candidates[0] +} + var ( traceMode string traceDepth int @@ -118,16 +172,16 @@ func runTraceCallers(cmd *cobra.Command, args []string) error { result := trace.TraceResult{Query: symbolName, Mode: traceMode} for _, ss := range stores { + refs, err := ss.LookupCallers(ctx, symbolName) + if err != nil { + log.Printf("Warning: failed to lookup callers of %q: %v", symbolName, err) + } symbols, err := ss.LookupSymbol(ctx, symbolName) if err != nil { log.Printf("Warning: failed to lookup symbol %q: %v", symbolName, err) } if len(symbols) > 0 && result.Symbol == nil { - result.Symbol = &symbols[0] - } - refs, err := ss.LookupCallers(ctx, symbolName) - if err != nil { - log.Printf("Warning: failed to lookup callers of %q: %v", symbolName, err) + result.Symbol = pickBestTargetSymbol(symbols, refs) } for _, ref := range refs { callerSyms, err := ss.LookupSymbol(ctx, ref.CallerName) @@ -136,7 +190,11 @@ func runTraceCallers(cmd *cobra.Command, args []string) error { } var callerSym trace.Symbol if len(callerSyms) > 0 { - callerSym = callerSyms[0] + if picked := pickBestSymbolForFile(callerSyms, ref.CallerFile); picked != nil { + callerSym = *picked + } else { + callerSym = callerSyms[0] + } } else { callerSym = trace.Symbol{Name: ref.CallerName, File: ref.CallerFile, Line: ref.CallerLine} } @@ -209,10 +267,15 @@ func runTraceCallers(cmd *cobra.Command, args []string) error { return fmt.Errorf("failed to lookup callers: %w", err) } + target := pickBestTargetSymbol(symbols, refs) + if target == nil { + target = &symbols[0] + } + result := trace.TraceResult{ Query: symbolName, Mode: traceMode, - Symbol: &symbols[0], + Symbol: target, } // Convert refs to CallerInfo @@ -223,7 +286,11 @@ func runTraceCallers(cmd *cobra.Command, args []string) error { } var callerSym trace.Symbol if len(callerSyms) > 0 { - callerSym = callerSyms[0] + if picked := pickBestSymbolForFile(callerSyms, ref.CallerFile); picked != nil { + callerSym = *picked + } else { + callerSym = callerSyms[0] + } } else { callerSym = trace.Symbol{Name: ref.CallerName, File: ref.CallerFile, Line: ref.CallerLine} } diff --git a/cli/trace_test.go b/cli/trace_test.go index ba13628..1e3193d 100644 --- a/cli/trace_test.go +++ b/cli/trace_test.go @@ -261,6 +261,39 @@ func TestDisplayCallersResult_should_handle_empty_callers(t *testing.T) { } } +func TestPickBestTargetSymbol_prefersSameFileReferences(t *testing.T) { + candidates := []trace.Symbol{ + {Name: "isAdmin", File: "src/components/Navigation.vue", Line: 10}, + {Name: "isAdmin", File: "src/views/Home.vue", Line: 20}, + } + refs := []trace.Reference{ + {SymbolName: "isAdmin", File: "src/views/Home.vue", CallerFile: "src/views/Home.vue", Line: 49}, + } + + picked := pickBestTargetSymbol(candidates, refs) + if picked == nil { + t.Fatal("expected symbol to be selected") + } + if picked.File != "src/views/Home.vue" { + t.Fatalf("expected Home.vue symbol, got %s", picked.File) + } +} + +func TestPickBestSymbolForFile_prefersCallerFile(t *testing.T) { + candidates := []trace.Symbol{ + {Name: "__vue_template_reads__", File: "src/App.vue", Line: 100}, + {Name: "__vue_template_reads__", File: "src/views/Home.vue", Line: 200}, + } + + picked := pickBestSymbolForFile(candidates, "src/views/Home.vue") + if picked == nil { + t.Fatal("expected caller symbol") + } + if picked.File != "src/views/Home.vue" { + t.Fatalf("expected Home.vue caller symbol, got %s", picked.File) + } +} + func TestDisplayCallersResult_should_list_callers(t *testing.T) { result := trace.TraceResult{ Symbol: &trace.Symbol{Name: "Foo", Kind: trace.KindFunction, File: "foo.go", Line: 1}, From 6fe919cac82f70421003f8680f2bd69c0cda5fa1 Mon Sep 17 00:00:00 2001 From: Mladen Mihajlovic Date: Thu, 12 Mar 2026 17:29:43 +0100 Subject: [PATCH 04/14] feat(refs): add property readers/writers tracing for JS/TS and MCP --- cli/agent_setup.go | 36 ++++++- cli/mcp_serve.go | 2 + cli/refs.go | 220 ++++++++++++++++++++++++++++++++++++++++ mcp/server.go | 196 +++++++++++++++++++++++++++++++++++ trace/extractor.go | 54 ++++++++++ trace/extractor_test.go | 37 +++++++ trace/extractor_ts.go | 66 +++++++++++- trace/store.go | 58 ++++++++++- trace/store_test.go | 43 ++++++++ trace/trace.go | 13 +++ 10 files changed, 716 insertions(+), 9 deletions(-) create mode 100644 cli/refs.go diff --git a/cli/agent_setup.go b/cli/agent_setup.go index db46d77..da227c2 100644 --- a/cli/agent_setup.go +++ b/cli/agent_setup.go @@ -73,12 +73,25 @@ grepai trace callees "ProcessOrder" --json grepai trace graph "ValidateToken" --depth 3 --json ` + "```" + ` +### Property/Data Usage Tracing + +Use ` + "`grepai refs`" + ` to find non-call property/state usage (reads/writes): + +` + "```bash" + ` +# Find where a property is read +grepai refs readers "uid" --json + +# Find where a property is written +grepai refs writers "uid" --json +` + "```" + ` + ### Workflow 1. Start with ` + "`grepai search`" + ` to find relevant code 2. Use ` + "`grepai trace`" + ` to understand function relationships -3. Use ` + "`Read`" + ` tool to examine files from results -4. Only use Grep for exact string searches if needed +3. Use ` + "`grepai refs`" + ` for property/state readers and writers +4. Use ` + "`Read`" + ` tool to examine files from results +5. Only use Grep for exact string searches if needed ` @@ -123,6 +136,18 @@ grepai trace callees "ProcessOrder" --json grepai trace graph "ValidateToken" --depth 3 --json ` + "```" + ` +#### 3. Property/Data Usage Tracing: ` + "`grepai refs`" + ` + +Use this when the target is a property/state key rather than a function call: + +` + "```bash" + ` +# Find readers of a property +grepai refs readers "uid" --json + +# Find writers of a property +grepai refs writers "uid" --json +` + "```" + ` + Use ` + "`grepai trace`" + ` when you need to: - Find all callers of a function - Understand the call hierarchy @@ -140,9 +165,10 @@ Only fall back to Grep/Glob when: 1. Start with ` + "`grepai search`" + ` to find relevant code semantically 2. Use ` + "`grepai trace`" + ` to understand function relationships and call graphs -3. Use ` + "`Read`" + ` to examine promising files in detail -4. Use Grep only for exact string searches if needed -5. Synthesize findings into a clear summary +3. Use ` + "`grepai refs`" + ` for property/state readers and writers +4. Use ` + "`Read`" + ` to examine promising files in detail +5. Use Grep only for exact string searches if needed +6. Synthesize findings into a clear summary ` const subagentMarker = "name: deep-explore" diff --git a/cli/mcp_serve.go b/cli/mcp_serve.go index 4cf5c82..42eb790 100644 --- a/cli/mcp_serve.go +++ b/cli/mcp_serve.go @@ -22,6 +22,8 @@ The server communicates via stdio and exposes the following tools: - grepai_trace_callers: Find all functions that call a symbol - grepai_trace_callees: Find all functions called by a symbol - grepai_trace_graph: Build a call graph around a symbol + - grepai_refs_readers: Find property/state readers for a symbol name + - grepai_refs_writers: Find property/state writers for a symbol name - grepai_index_status: Check index health and statistics (includes RPG stats when enabled) - grepai_rpg_search: Search RPG graph nodes by feature semantics - grepai_rpg_fetch: Fetch hierarchy and edge context for a specific RPG node diff --git a/cli/refs.go b/cli/refs.go new file mode 100644 index 0000000..d399286 --- /dev/null +++ b/cli/refs.go @@ -0,0 +1,220 @@ +package cli + +import ( + "context" + "encoding/json" + "fmt" + "log" + "os" + "strings" + + "github.com/alpkeskin/gotoon" + "github.com/spf13/cobra" + "github.com/yoanbernabeu/grepai/config" + "github.com/yoanbernabeu/grepai/trace" +) + +var ( + refsJSON bool + refsTOON bool + refsWorkspace string + refsProject string +) + +type refsUsage struct { + Symbol trace.Symbol `json:"symbol"` + Access string `json:"access"` + AccessAt trace.CallSite `json:"access_at"` +} + +type refsResult struct { + Query string `json:"query"` + Kind string `json:"kind"` + Mode string `json:"mode"` + Readers []refsUsage `json:"readers,omitempty"` + Writers []refsUsage `json:"writers,omitempty"` +} + +var refsCmd = &cobra.Command{ + Use: "refs ", + Short: "Trace property/data readers and writers", + Long: `Refs command finds non-call data usage edges for a symbol name. + +Use this for property/state usage (e.g. store.uid), while 'trace' remains call-graph focused. + +Examples: + grepai refs readers "uid" + grepai refs writers "uid" --json`, +} + +var refsReadersCmd = &cobra.Command{ + Use: "readers ", + Short: "Find functions/components that read a property or data symbol", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + result, err := runRefs(args[0], true) + if err != nil { + return err + } + return outputRefsResult(result, true) + }, +} + +var refsWritersCmd = &cobra.Command{ + Use: "writers ", + Short: "Find functions/components that write a property or data symbol", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + result, err := runRefs(args[0], false) + if err != nil { + return err + } + return outputRefsResult(result, false) + }, +} + +func init() { + for _, cmd := range []*cobra.Command{refsReadersCmd, refsWritersCmd} { + cmd.Flags().BoolVar(&refsJSON, "json", false, "Output results in JSON format") + cmd.Flags().BoolVarP(&refsTOON, "toon", "t", false, "Output results in TOON format (token-efficient for AI agents)") + cmd.MarkFlagsMutuallyExclusive("json", "toon") + cmd.Flags().StringVar(&refsWorkspace, "workspace", "", "Workspace name for cross-project refs") + cmd.Flags().StringVar(&refsProject, "project", "", "Project name within workspace (requires --workspace)") + } + + refsCmd.AddCommand(refsReadersCmd) + refsCmd.AddCommand(refsWritersCmd) + rootCmd.AddCommand(refsCmd) +} + +func runRefs(symbolName string, readers bool) (refsResult, error) { + ctx := context.Background() + + if refsProject != "" && refsWorkspace == "" { + return refsResult{}, fmt.Errorf("--project requires --workspace") + } + + stores := []trace.SymbolStore{} + if refsWorkspace != "" { + var err error + stores, err = trace.LoadWorkspaceSymbolStores(ctx, refsWorkspace, refsProject) + if err != nil { + return refsResult{}, err + } + defer trace.CloseSymbolStores(stores) + } else { + projectRoot, err := config.FindProjectRoot() + if err != nil { + return refsResult{}, fmt.Errorf("failed to find project root: %w", err) + } + + symbolStore := trace.NewGOBSymbolStore(config.GetSymbolIndexPath(projectRoot)) + if err := symbolStore.Load(ctx); err != nil { + return refsResult{}, fmt.Errorf("failed to load symbol index: %w", err) + } + defer symbolStore.Close() + + stats, err := symbolStore.GetStats(ctx) + if err != nil || stats.TotalSymbols == 0 { + return refsResult{}, fmt.Errorf("symbol index is empty. Run 'grepai watch' first to build the index") + } + + stores = []trace.SymbolStore{symbolStore} + } + + result := refsResult{Query: symbolName, Kind: "property", Mode: "fast"} + for _, ss := range stores { + var refs []trace.Reference + var err error + if readers { + refs, err = ss.LookupReaders(ctx, symbolName) + } else { + refs, err = ss.LookupWriters(ctx, symbolName) + } + if err != nil { + log.Printf("Warning: failed to lookup refs for %q: %v", symbolName, err) + continue + } + + for _, ref := range refs { + sym := resolveRefCallerSymbol(ctx, ss, ref) + usage := refsUsage{ + Symbol: sym, + Access: ref.Kind, + AccessAt: trace.CallSite{ + File: ref.File, + Line: ref.Line, + Context: ref.Context, + }, + } + if readers { + result.Readers = append(result.Readers, usage) + } else { + result.Writers = append(result.Writers, usage) + } + } + } + + return result, nil +} + +func resolveRefCallerSymbol(ctx context.Context, ss trace.SymbolStore, ref trace.Reference) trace.Symbol { + if ref.CallerName == "" || ref.CallerName == "" { + return trace.Symbol{Name: ref.CallerName, File: ref.CallerFile, Line: ref.CallerLine} + } + + candidates, err := ss.LookupSymbol(ctx, ref.CallerName) + if err != nil || len(candidates) == 0 { + return trace.Symbol{Name: ref.CallerName, File: ref.CallerFile, Line: ref.CallerLine} + } + best := pickBestSymbolForFile(candidates, ref.CallerFile) + if best == nil { + return trace.Symbol{Name: ref.CallerName, File: ref.CallerFile, Line: ref.CallerLine} + } + return *best +} + +func outputRefsResult(result refsResult, readers bool) error { + if refsJSON { + enc := json.NewEncoder(os.Stdout) + enc.SetIndent("", " ") + return enc.Encode(result) + } + if refsTOON { + output, err := gotoon.Encode(result) + if err != nil { + return fmt.Errorf("failed to encode TOON: %w", err) + } + fmt.Println(output) + return nil + } + + label := "Readers" + usages := result.Readers + if !readers { + label = "Writers" + usages = result.Writers + } + + fmt.Printf("Symbol: %s (%s)\n", result.Query, result.Kind) + fmt.Printf("\n%s (%d):\n", label, len(usages)) + fmt.Println(strings.Repeat("-", 60)) + + if len(usages) == 0 { + fmt.Println("No references found.") + return nil + } + + for i, usage := range usages { + fmt.Printf("\n%d. %s\n", i+1, usage.Symbol.Name) + if usage.Symbol.File != "" { + fmt.Printf(" Defined: %s:%d\n", usage.Symbol.File, usage.Symbol.Line) + } + fmt.Printf(" Access: %s at %s:%d\n", usage.Access, usage.AccessAt.File, usage.AccessAt.Line) + if usage.AccessAt.Context != "" { + fmt.Printf(" Context: %s\n", truncate(usage.AccessAt.Context, 100)) + } + } + + return nil +} diff --git a/mcp/server.go b/mcp/server.go index 0ec5ce1..66eb751 100644 --- a/mcp/server.go +++ b/mcp/server.go @@ -71,6 +71,18 @@ type CalleeInfoCompact struct { CallSite CallSiteCompact `json:"call_site"` } +type RefUsageCompact struct { + Symbol trace.Symbol `json:"symbol"` + Access string `json:"access"` + AccessAt CallSiteCompact `json:"access_at"` +} + +type RefUsage struct { + Symbol trace.Symbol `json:"symbol"` + Access string `json:"access"` + AccessAt trace.CallSite `json:"access_at"` +} + // CallEdgeCompact is a compact version of trace.CallEdge for compact output. type CallEdgeCompact struct { CallerName string `json:"caller_name"` @@ -241,6 +253,48 @@ func (s *Server) registerTools() { ) s.mcpServer.AddTool(traceGraphTool, s.handleTraceGraph) + refsReadersTool := mcp.NewTool("grepai_refs_readers", + mcp.WithDescription("Find readers of a property/state symbol (non-call data usage such as store.uid reads)."), + mcp.WithString("symbol", + mcp.Required(), + mcp.Description("Property/state symbol name to find readers for"), + ), + mcp.WithBoolean("compact", + mcp.Description("Return minimal output without context (default: false)"), + ), + mcp.WithString("format", + mcp.Description("Output format: 'json' (default) or 'toon' (token-efficient)"), + ), + mcp.WithString("workspace", + mcp.Description("Workspace name for cross-project refs (optional)"), + ), + mcp.WithString("project", + mcp.Description("Project name within workspace (requires workspace)"), + ), + ) + s.mcpServer.AddTool(refsReadersTool, s.handleRefsReaders) + + refsWritersTool := mcp.NewTool("grepai_refs_writers", + mcp.WithDescription("Find writers of a property/state symbol (non-call data usage such as this.uid = ...)."), + mcp.WithString("symbol", + mcp.Required(), + mcp.Description("Property/state symbol name to find writers for"), + ), + mcp.WithBoolean("compact", + mcp.Description("Return minimal output without context (default: false)"), + ), + mcp.WithString("format", + mcp.Description("Output format: 'json' (default) or 'toon' (token-efficient)"), + ), + mcp.WithString("workspace", + mcp.Description("Workspace name for cross-project refs (optional)"), + ), + mcp.WithString("project", + mcp.Description("Project name within workspace (requires workspace)"), + ), + ) + s.mcpServer.AddTool(refsWritersTool, s.handleRefsWriters) + // grepai_index_status tool indexStatusTool := mcp.NewTool("grepai_index_status", mcp.WithDescription("Check the health and status of the grepai index. Returns statistics about indexed files, chunks, and configuration."), @@ -1437,6 +1491,148 @@ func (s *Server) handleTraceGraph(ctx context.Context, request mcp.CallToolReque return mcp.NewToolResultText(output), nil } +func (s *Server) handleRefsReaders(ctx context.Context, request mcp.CallToolRequest) (*mcp.CallToolResult, error) { + return s.handleRefsByKind(ctx, request, trace.RefKindRead) +} + +func (s *Server) handleRefsWriters(ctx context.Context, request mcp.CallToolRequest) (*mcp.CallToolResult, error) { + return s.handleRefsByKind(ctx, request, trace.RefKindWrite) +} + +func (s *Server) handleRefsByKind(ctx context.Context, request mcp.CallToolRequest, kind string) (*mcp.CallToolResult, error) { + symbolName, err := request.RequireString("symbol") + if err != nil { + return mcp.NewToolResultError("symbol parameter is required"), nil + } + + compact := request.GetBool("compact", false) + format := request.GetString("format", "json") + workspace := s.resolveWorkspace(request.GetString("workspace", "")) + project := request.GetString("project", "") + + if format != "json" && format != "toon" { + return mcp.NewToolResultError("format must be 'json' or 'toon'"), nil + } + + // Workspace mode + if workspace != "" { + stores, loadErr := trace.LoadWorkspaceSymbolStores(ctx, workspace, project) + if loadErr != nil { + return mcp.NewToolResultError(fmt.Sprintf("failed to load workspace symbol stores: %v", loadErr)), nil + } + defer trace.CloseSymbolStores(stores) + return s.handleRefsFromStores(ctx, symbolName, kind, compact, format, stores) + } + + // Single-project mode + if s.projectRoot == "" { + return mcp.NewToolResultError("refs requires a project context; use --workspace parameter or start mcp-serve from a project directory"), nil + } + + symbolStore := trace.NewGOBSymbolStore(config.GetSymbolIndexPath(s.projectRoot)) + if err := symbolStore.Load(ctx); err != nil { + return mcp.NewToolResultError(fmt.Sprintf("failed to load symbol index: %v. Run 'grepai watch' first", err)), nil + } + defer symbolStore.Close() + + stats, err := symbolStore.GetStats(ctx) + if err != nil || stats.TotalSymbols == 0 { + return mcp.NewToolResultError("symbol index is empty. Run 'grepai watch' first to build the index"), nil + } + + return s.handleRefsFromStores(ctx, symbolName, kind, compact, format, []trace.SymbolStore{symbolStore}) +} + +func (s *Server) handleRefsFromStores(ctx context.Context, symbolName string, kind string, compact bool, format string, stores []trace.SymbolStore) (*mcp.CallToolResult, error) { + usages := make([]RefUsage, 0) + + for _, ss := range stores { + var refs []trace.Reference + var err error + if kind == trace.RefKindWrite { + refs, err = ss.LookupWriters(ctx, symbolName) + } else { + refs, err = ss.LookupReaders(ctx, symbolName) + } + if err != nil { + log.Printf("Warning: failed to lookup refs of %q: %v", symbolName, err) + continue + } + + for _, ref := range refs { + usages = append(usages, RefUsage{ + Symbol: resolveRefCallerSymbol(ss, ctx, ref), + Access: ref.Kind, + AccessAt: trace.CallSite{ + File: ref.File, + Line: ref.Line, + Context: ref.Context, + }, + }) + } + } + + label := "readers" + if kind == trace.RefKindWrite { + label = "writers" + } + + var data any + if compact { + result := map[string]any{ + "query": symbolName, + "kind": "property", + "mode": "fast", + } + comp := make([]RefUsageCompact, 0, len(usages)) + for _, usage := range usages { + comp = append(comp, RefUsageCompact{ + Symbol: usage.Symbol, + Access: usage.Access, + AccessAt: CallSiteCompact{ + File: usage.AccessAt.File, + Line: usage.AccessAt.Line, + }, + }) + } + result[label] = comp + data = result + } else { + result := map[string]any{ + "query": symbolName, + "kind": "property", + "mode": "fast", + } + result[label] = usages + data = result + } + + output, err := encodeOutput(data, format) + if err != nil { + return mcp.NewToolResultError(fmt.Sprintf("failed to encode results: %v", err)), nil + } + + return mcp.NewToolResultText(output), nil +} + +func resolveRefCallerSymbol(ss trace.SymbolStore, ctx context.Context, ref trace.Reference) trace.Symbol { + if ref.CallerName == "" || ref.CallerName == "" { + return trace.Symbol{Name: ref.CallerName, File: ref.CallerFile, Line: ref.CallerLine} + } + + candidates, err := ss.LookupSymbol(ctx, ref.CallerName) + if err != nil || len(candidates) == 0 { + return trace.Symbol{Name: ref.CallerName, File: ref.CallerFile, Line: ref.CallerLine} + } + for _, sym := range candidates { + if sym.File == ref.CallerFile { + return sym + } + } + + return candidates[0] +} + // WorkspaceIndexStatus represents the status of a workspace index. type WorkspaceIndexStatus struct { Workspace string `json:"workspace"` diff --git a/trace/extractor.go b/trace/extractor.go index d15bcb5..cbbd024 100644 --- a/trace/extractor.go +++ b/trace/extractor.go @@ -225,6 +225,8 @@ func (e *RegexExtractor) extractLanguageSpecificReferences(filePath string, cont } switch patterns.Language { + case "javascript", "typescript": + return e.extractJSPropertyReferences(filePath, content, lines, functionBoundaries) case "lua": return e.extractLuaBracketKeyReferences(filePath, content, lines, patterns, functionBoundaries) default: @@ -232,6 +234,42 @@ func (e *RegexExtractor) extractLanguageSpecificReferences(filePath string, cont } } +var ( + jsPropertyReadRe = regexp.MustCompile(`\b(?:this\.)?(?:[A-Za-z_$][A-Za-z0-9_$]*\.)+([A-Za-z_$][A-Za-z0-9_$]*)\b`) + jsPropertyWriteRe = regexp.MustCompile(`\b(?:this\.)?(?:[A-Za-z_$][A-Za-z0-9_$]*\.)+([A-Za-z_$][A-Za-z0-9_$]*)\s*=`) +) + +func (e *RegexExtractor) extractJSPropertyReferences(filePath string, content string, lines []string, functionBoundaries []functionBoundary) []Reference { + writeMatches := jsPropertyWriteRe.FindAllStringSubmatchIndex(content, -1) + writeStarts := make(map[int]bool, len(writeMatches)) + refs := make([]Reference, 0, len(writeMatches)) + + for _, m := range writeMatches { + if len(m) < 4 { + continue + } + start := m[0] + writeStarts[start] = true + name := content[m[2]:m[3]] + refs = append(refs, buildDataReference(filePath, content, lines, name, start, RefKindWrite, functionBoundaries)) + } + + readMatches := jsPropertyReadRe.FindAllStringSubmatchIndex(content, -1) + for _, m := range readMatches { + if len(m) < 4 { + continue + } + start := m[0] + if writeStarts[start] { + continue + } + name := content[m[2]:m[3]] + refs = append(refs, buildDataReference(filePath, content, lines, name, start, RefKindRead, functionBoundaries)) + } + + return refs +} + // isDeclarationReferenceMatch reports whether a regex call match occurs in a declaration context. func isDeclarationReferenceMatch(content string, patterns *LanguagePatterns, pos int) bool { if patterns == nil { @@ -751,6 +789,22 @@ func buildReference(filePath, content string, lines []string, name string, pos i return Reference{ SymbolName: name, + Kind: RefKindCall, + File: filePath, + Line: line, + Context: getLineContext(lines, line-1, 0), + CallerName: caller.Name, + CallerFile: filePath, + CallerLine: caller.Line, + } +} + +func buildDataReference(filePath, content string, lines []string, name string, pos int, kind string, functionBoundaries []functionBoundary) Reference { + line := countLines(content[:pos]) + 1 + caller := findContainingFunction(pos, functionBoundaries) + return Reference{ + SymbolName: name, + Kind: kind, File: filePath, Line: line, Context: getLineContext(lines, line-1, 0), diff --git a/trace/extractor_test.go b/trace/extractor_test.go index f0a9e90..b8b3565 100644 --- a/trace/extractor_test.go +++ b/trace/extractor_test.go @@ -1303,3 +1303,40 @@ func TestRegexExtractor_ExtractReferences_FSharp_IgnoresCommentsAndStrings(t *te t.Error("nested block comment not fully masked: stillHidden leaked") } } + +func TestRegexExtractor_ExtractReferences_JSTypeScriptPropertyReadWrite(t *testing.T) { + extractor := NewRegexExtractor() + ctx := context.Background() + + content := `function login(store) { + const ok = store.uid !== null + this.store.uid = "next" + return ok +}` + + refs, err := extractor.ExtractReferences(ctx, "store.ts", content) + if err != nil { + t.Fatalf("ExtractReferences failed: %v", err) + } + + foundRead := false + foundWrite := false + for _, ref := range refs { + if ref.SymbolName != "uid" { + continue + } + if ref.Kind == RefKindRead { + foundRead = true + } + if ref.Kind == RefKindWrite { + foundWrite = true + } + } + + if !foundRead { + t.Fatal("expected at least one read reference for uid") + } + if !foundWrite { + t.Fatal("expected at least one write reference for uid") + } +} diff --git a/trace/extractor_ts.go b/trace/extractor_ts.go index 2f7aebf..a872173 100644 --- a/trace/extractor_ts.go +++ b/trace/extractor_ts.go @@ -649,12 +649,12 @@ func (e *TreeSitterExtractor) ExtractReferences(ctx context.Context, filePath st var refs []Reference root := tree.RootNode() - e.walkNodeForCalls(root, []byte(content), filePath, ext, &refs) + e.walkNodeForReferences(root, []byte(content), filePath, ext, &refs) return refs, nil } -func (e *TreeSitterExtractor) walkNodeForCalls(node *sitter.Node, content []byte, filePath string, ext string, refs *[]Reference) { +func (e *TreeSitterExtractor) walkNodeForReferences(node *sitter.Node, content []byte, filePath string, ext string, refs *[]Reference) { nodeType := node.Type() switch ext { @@ -676,6 +676,7 @@ func (e *TreeSitterExtractor) walkNodeForCalls(node *sitter.Node, content []byte *refs = append(*refs, Reference{ SymbolName: name, + Kind: RefKindCall, File: filePath, Line: int(node.StartPoint().Row) + 1, Column: int(node.StartPoint().Column), @@ -685,11 +686,16 @@ func (e *TreeSitterExtractor) walkNodeForCalls(node *sitter.Node, content []byte }) } } + if isJSLikeExt(ext) && nodeType == "member_expression" { + if ref, ok := e.extractMemberAccessReference(node, content, filePath, ext); ok { + *refs = append(*refs, ref) + } + } } for i := 0; i < int(node.ChildCount()); i++ { child := node.Child(i) - e.walkNodeForCalls(child, content, filePath, ext, refs) + e.walkNodeForReferences(child, content, filePath, ext, refs) } } @@ -728,6 +734,7 @@ func (e *TreeSitterExtractor) walkFSharpCalls(node *sitter.Node, nodeType string *refs = append(*refs, Reference{ SymbolName: name, + Kind: RefKindCall, File: filePath, Line: int(node.StartPoint().Row) + 1, Column: int(node.StartPoint().Column), @@ -737,6 +744,59 @@ func (e *TreeSitterExtractor) walkFSharpCalls(node *sitter.Node, nodeType string }) } +func isJSLikeExt(ext string) bool { + return ext == ".js" || ext == ".jsx" || ext == ".ts" || ext == ".tsx" +} + +func (e *TreeSitterExtractor) extractMemberAccessReference(node *sitter.Node, content []byte, filePath string, ext string) (Reference, bool) { + propertyNode := node.ChildByFieldName("property") + if propertyNode == nil { + return Reference{}, false + } + + name := strings.TrimSpace(propertyNode.Content(content)) + if name == "" { + return Reference{}, false + } + // Skip private/internal slots used by transformed Vue internals. + if strings.HasPrefix(name, "_") { + return Reference{}, false + } + // Ignore computed member access (obj["uid"]) for now; handled later by a dedicated parser path. + if strings.ContainsAny(name, "\"'[]") { + return Reference{}, false + } + + kind := RefKindRead + parent := node.Parent() + if parent != nil { + switch parent.Type() { + case "assignment_expression": + left := parent.ChildByFieldName("left") + if left != nil && left.ID() == node.ID() { + kind = RefKindWrite + } + case "update_expression": + arg := parent.ChildByFieldName("argument") + if arg != nil && arg.ID() == node.ID() { + kind = RefKindWrite + } + } + } + + caller := e.findContainingFunction(node, content, ext) + return Reference{ + SymbolName: name, + Kind: kind, + File: filePath, + Line: int(node.StartPoint().Row) + 1, + Column: int(node.StartPoint().Column), + Context: truncateContext(string(content[node.StartByte():node.EndByte()])), + CallerName: caller, + CallerFile: filePath, + }, true +} + func extractFSharpCallName(node *sitter.Node, content []byte) string { text := node.Content(content) // For dotted access like "String.Format" or "Logger.log", take the last part diff --git a/trace/store.go b/trace/store.go index a3b6033..b26c0a7 100644 --- a/trace/store.go +++ b/trace/store.go @@ -285,7 +285,7 @@ func (s *GOBSymbolStore) LookupCallers(ctx context.Context, symbolName string) ( if refs == nil { return []Reference{}, nil } - return refs, nil + return filterByReferenceKinds(refs, RefKindCall, ""), nil } // LookupCallees finds all symbols called by a function. @@ -307,6 +307,9 @@ func (s *GOBSymbolStore) LookupCallees(ctx context.Context, symbolName string, f // Find reference details if refs, ok := s.index.References[edge.Callee]; ok { for _, ref := range refs { + if !isCallReference(ref) { + continue + } if ref.CallerName == symbolName && ref.File == edge.File && ref.Line == edge.Line { callees = append(callees, ref) break @@ -328,6 +331,59 @@ func (s *GOBSymbolStore) LookupCallees(ctx context.Context, symbolName string, f return callees, nil } +// LookupReaders finds property/data readers for a symbol name. +func (s *GOBSymbolStore) LookupReaders(ctx context.Context, symbolName string) ([]Reference, error) { + s.mu.RLock() + defer s.mu.RUnlock() + + refs := s.index.References[symbolName] + if refs == nil { + return []Reference{}, nil + } + return filterByReferenceKinds(refs, RefKindRead), nil +} + +// LookupWriters finds property/data writers for a symbol name. +func (s *GOBSymbolStore) LookupWriters(ctx context.Context, symbolName string) ([]Reference, error) { + s.mu.RLock() + defer s.mu.RUnlock() + + refs := s.index.References[symbolName] + if refs == nil { + return []Reference{}, nil + } + return filterByReferenceKinds(refs, RefKindWrite), nil +} + +func filterByReferenceKinds(refs []Reference, kinds ...string) []Reference { + if len(refs) == 0 { + return []Reference{} + } + + allowed := make(map[string]bool, len(kinds)) + for _, kind := range kinds { + allowed[kind] = true + } + + filtered := make([]Reference, 0, len(refs)) + for _, ref := range refs { + if allowed[ref.Kind] { + filtered = append(filtered, ref) + continue + } + // Backward compatibility with older indices where kind wasn't persisted. + if ref.Kind == "" && allowed[RefKindCall] { + filtered = append(filtered, ref) + } + } + + return filtered +} + +func isCallReference(ref Reference) bool { + return ref.Kind == "" || ref.Kind == RefKindCall +} + // GetCallGraph builds a call graph from a starting symbol. func (s *GOBSymbolStore) GetCallGraph(ctx context.Context, symbolName string, depth int) (*CallGraph, error) { s.mu.RLock() diff --git a/trace/store_test.go b/trace/store_test.go index 0ba1350..04d39d9 100644 --- a/trace/store_test.go +++ b/trace/store_test.go @@ -603,3 +603,46 @@ func TestGOBSymbolStore_PersistCreatesLockFileAndNoTempFiles(t *testing.T) { } } } + +func TestGOBSymbolStore_ReferenceKindFilters(t *testing.T) { + ctx := context.Background() + indexPath := filepath.Join(t.TempDir(), "symbols.gob") + store := NewGOBSymbolStore(indexPath) + + symbols := []Symbol{ + {Name: "uidConsumer", Kind: KindFunction, File: "store.ts", Line: 1, Language: "typescript"}, + } + refs := []Reference{ + {SymbolName: "uid", Kind: RefKindRead, File: "store.ts", Line: 2, CallerName: "uidConsumer", CallerFile: "store.ts"}, + {SymbolName: "uid", Kind: RefKindWrite, File: "store.ts", Line: 3, CallerName: "uidConsumer", CallerFile: "store.ts"}, + {SymbolName: "uid", Kind: RefKindCall, File: "store.ts", Line: 4, CallerName: "uidConsumer", CallerFile: "store.ts"}, + } + + if err := store.SaveFile(ctx, "store.ts", symbols, refs); err != nil { + t.Fatalf("SaveFile failed: %v", err) + } + + callers, err := store.LookupCallers(ctx, "uid") + if err != nil { + t.Fatalf("LookupCallers failed: %v", err) + } + if len(callers) != 1 || callers[0].Kind != RefKindCall { + t.Fatalf("expected only call refs, got %+v", callers) + } + + readers, err := store.LookupReaders(ctx, "uid") + if err != nil { + t.Fatalf("LookupReaders failed: %v", err) + } + if len(readers) != 1 || readers[0].Kind != RefKindRead { + t.Fatalf("expected only read refs, got %+v", readers) + } + + writers, err := store.LookupWriters(ctx, "uid") + if err != nil { + t.Fatalf("LookupWriters failed: %v", err) + } + if len(writers) != 1 || writers[0].Kind != RefKindWrite { + t.Fatalf("expected only write refs, got %+v", writers) + } +} diff --git a/trace/trace.go b/trace/trace.go index 77abe7d..1f1d9e2 100644 --- a/trace/trace.go +++ b/trace/trace.go @@ -38,6 +38,7 @@ type Symbol struct { // Reference represents a usage/call of a symbol. type Reference struct { SymbolName string `json:"symbol_name"` + Kind string `json:"kind,omitempty"` File string `json:"file"` Line int `json:"line"` Column int `json:"column,omitempty"` @@ -47,6 +48,12 @@ type Reference struct { CallerLine int `json:"caller_line"` } +const ( + RefKindCall = "call" + RefKindRead = "read" + RefKindWrite = "write" +) + // CallEdge represents a caller -> callee relationship. type CallEdge struct { Caller string `json:"caller"` @@ -149,6 +156,12 @@ type SymbolStore interface { // LookupCallees finds all symbols called by a function. LookupCallees(ctx context.Context, symbolName string, file string) ([]Reference, error) + // LookupReaders finds property/data readers for a symbol name. + LookupReaders(ctx context.Context, symbolName string) ([]Reference, error) + + // LookupWriters finds property/data writers for a symbol name. + LookupWriters(ctx context.Context, symbolName string) ([]Reference, error) + // GetCallGraph builds a call graph from a starting symbol. GetCallGraph(ctx context.Context, symbolName string, depth int) (*CallGraph, error) From cdf5ffb3ebb9d890205ee1be26f40c739c9c3fcf Mon Sep 17 00:00:00 2001 From: Mladen Mihajlovic Date: Thu, 12 Mar 2026 17:47:13 +0100 Subject: [PATCH 05/14] test(cli): add end-to-end refs readers/writers index coverage --- cli/refs_test.go | 96 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 96 insertions(+) create mode 100644 cli/refs_test.go diff --git a/cli/refs_test.go b/cli/refs_test.go new file mode 100644 index 0000000..510e1a9 --- /dev/null +++ b/cli/refs_test.go @@ -0,0 +1,96 @@ +package cli + +import ( + "context" + "os" + "path/filepath" + "testing" + + "github.com/yoanbernabeu/grepai/config" + "github.com/yoanbernabeu/grepai/trace" +) + +func TestRunRefs_should_return_readers_and_writers_from_index(t *testing.T) { + projectRoot := t.TempDir() + if err := config.DefaultConfig().Save(projectRoot); err != nil { + t.Fatalf("failed to save config: %v", err) + } + + origWD, err := os.Getwd() + if err != nil { + t.Fatalf("failed to get cwd: %v", err) + } + defer func() { _ = os.Chdir(origWD) }() + if err := os.Chdir(projectRoot); err != nil { + t.Fatalf("failed to chdir to project root: %v", err) + } + + ctx := context.Background() + symbolStore := trace.NewGOBSymbolStore(config.GetSymbolIndexPath(projectRoot)) + refs := []trace.Reference{ + { + SymbolName: "uid", + Kind: trace.RefKindRead, + File: "src/views/Home.vue", + Line: 21, + Context: "return Boolean(this.store.uid)", + CallerName: "loggedIn", + CallerFile: "src/views/Home.vue", + CallerLine: 20, + }, + { + SymbolName: "uid", + Kind: trace.RefKindWrite, + File: "src/store/index.ts", + Line: 100, + Context: "this.uid = authUser.uid", + CallerName: "user_details", + CallerFile: "src/store/index.ts", + CallerLine: 99, + }, + } + symbols := []trace.Symbol{ + {Name: "loggedIn", Kind: trace.KindMethod, File: "src/views/Home.vue", Line: 20, Language: "typescript"}, + {Name: "user_details", Kind: trace.KindMethod, File: "src/store/index.ts", Line: 99, Language: "typescript"}, + } + if err := symbolStore.SaveFile(ctx, filepath.ToSlash("src/mix.ts"), symbols, refs); err != nil { + t.Fatalf("failed to save symbols/refs: %v", err) + } + if err := symbolStore.Persist(ctx); err != nil { + t.Fatalf("failed to persist symbol store: %v", err) + } + + origWorkspace := refsWorkspace + origProject := refsProject + defer func() { + refsWorkspace = origWorkspace + refsProject = origProject + }() + refsWorkspace = "" + refsProject = "" + + readers, err := runRefs("uid", true) + if err != nil { + t.Fatalf("runRefs readers failed: %v", err) + } + if readers.Query != "uid" || readers.Kind != "property" { + t.Fatalf("unexpected readers header: %+v", readers) + } + if len(readers.Readers) != 1 { + t.Fatalf("expected 1 reader, got %d", len(readers.Readers)) + } + if readers.Readers[0].Symbol.Name != "loggedIn" { + t.Fatalf("expected reader symbol loggedIn, got %q", readers.Readers[0].Symbol.Name) + } + + writers, err := runRefs("uid", false) + if err != nil { + t.Fatalf("runRefs writers failed: %v", err) + } + if len(writers.Writers) != 1 { + t.Fatalf("expected 1 writer, got %d", len(writers.Writers)) + } + if writers.Writers[0].Symbol.Name != "user_details" { + t.Fatalf("expected writer symbol user_details, got %q", writers.Writers[0].Symbol.Name) + } +} From 388ac8e5c171b118dddbcfe231473ae69a904ca3 Mon Sep 17 00:00:00 2001 From: Mladen Mihajlovic Date: Thu, 12 Mar 2026 17:49:33 +0100 Subject: [PATCH 06/14] feat(refs): support Composition API alias and storeToRefs usage --- trace/extractor.go | 114 +++++++++++++++++++++++++++++++++++++++- trace/extractor_test.go | 43 +++++++++++++++ 2 files changed, 155 insertions(+), 2 deletions(-) diff --git a/trace/extractor.go b/trace/extractor.go index cbbd024..daeef9d 100644 --- a/trace/extractor.go +++ b/trace/extractor.go @@ -237,6 +237,8 @@ func (e *RegexExtractor) extractLanguageSpecificReferences(filePath string, cont var ( jsPropertyReadRe = regexp.MustCompile(`\b(?:this\.)?(?:[A-Za-z_$][A-Za-z0-9_$]*\.)+([A-Za-z_$][A-Za-z0-9_$]*)\b`) jsPropertyWriteRe = regexp.MustCompile(`\b(?:this\.)?(?:[A-Za-z_$][A-Za-z0-9_$]*\.)+([A-Za-z_$][A-Za-z0-9_$]*)\s*=`) + jsStoreToRefsRe = regexp.MustCompile(`\bconst\s*{\s*([^}]*)\s*}\s*=\s*storeToRefs\s*\([^)]*\)`) + jsSimpleAliasRe = regexp.MustCompile(`\b(?:const|let|var)\s+([A-Za-z_$][A-Za-z0-9_$]*)\s*=\s*[A-Za-z_$][A-Za-z0-9_$]*\.([A-Za-z_$][A-Za-z0-9_$]*)\b`) ) func (e *RegexExtractor) extractJSPropertyReferences(filePath string, content string, lines []string, functionBoundaries []functionBoundary) []Reference { @@ -267,7 +269,115 @@ func (e *RegexExtractor) extractJSPropertyReferences(filePath string, content st refs = append(refs, buildDataReference(filePath, content, lines, name, start, RefKindRead, functionBoundaries)) } - return refs + refs = append(refs, extractJSAliasPropertyReferences(filePath, content, lines, functionBoundaries)...) + return dedupeReferences(refs) +} + +type jsAliasProperty struct { + alias string + propName string + isRef bool + declPos int +} + +func extractJSAliasPropertyReferences(filePath, content string, lines []string, functionBoundaries []functionBoundary) []Reference { + aliases := collectJSAliases(content) + if len(aliases) == 0 { + return nil + } + + refs := make([]Reference, 0) + for _, a := range aliases { + if a.isRef { + readRe := regexp.MustCompile(`\b` + regexp.QuoteMeta(a.alias) + `\s*\.\s*value\b`) + for _, m := range readRe.FindAllStringIndex(content, -1) { + if m[0] == a.declPos { + continue + } + refs = append(refs, buildDataReference(filePath, content, lines, a.propName, m[0], RefKindRead, functionBoundaries)) + } + writeEqRe := regexp.MustCompile(`\b` + regexp.QuoteMeta(a.alias) + `\s*\.\s*value\s*=`) + for _, m := range writeEqRe.FindAllStringIndex(content, -1) { + if m[0] == a.declPos { + continue + } + refs = append(refs, buildDataReference(filePath, content, lines, a.propName, m[0], RefKindWrite, functionBoundaries)) + } + writeUpdRe := regexp.MustCompile(`\b` + regexp.QuoteMeta(a.alias) + `\s*\.\s*value\s*(?:\+\+|--)`) + for _, m := range writeUpdRe.FindAllStringIndex(content, -1) { + if m[0] == a.declPos { + continue + } + refs = append(refs, buildDataReference(filePath, content, lines, a.propName, m[0], RefKindWrite, functionBoundaries)) + } + continue + } + + writeRe := regexp.MustCompile(`\b` + regexp.QuoteMeta(a.alias) + `\s*=`) + writeStarts := map[int]bool{} + for _, m := range writeRe.FindAllStringIndex(content, -1) { + if m[0] == a.declPos { + continue + } + writeStarts[m[0]] = true + refs = append(refs, buildDataReference(filePath, content, lines, a.propName, m[0], RefKindWrite, functionBoundaries)) + } + readRe := regexp.MustCompile(`\b` + regexp.QuoteMeta(a.alias) + `\b`) + for _, m := range readRe.FindAllStringIndex(content, -1) { + if m[0] == a.declPos || writeStarts[m[0]] { + continue + } + refs = append(refs, buildDataReference(filePath, content, lines, a.propName, m[0], RefKindRead, functionBoundaries)) + } + } + + return dedupeReferences(refs) +} + +func collectJSAliases(content string) []jsAliasProperty { + aliases := make([]jsAliasProperty, 0) + + destructured := jsStoreToRefsRe.FindAllStringSubmatchIndex(content, -1) + for _, m := range destructured { + if len(m) < 4 { + continue + } + inner := content[m[2]:m[3]] + parts := strings.Split(inner, ",") + for _, p := range parts { + part := strings.TrimSpace(p) + if part == "" { + continue + } + prop := part + alias := part + if strings.Contains(part, ":") { + sides := strings.SplitN(part, ":", 2) + prop = strings.TrimSpace(sides[0]) + alias = strings.TrimSpace(sides[1]) + } + if prop == "" || alias == "" { + continue + } + declPos := m[2] + strings.Index(inner, alias) + aliases = append(aliases, jsAliasProperty{alias: alias, propName: prop, isRef: true, declPos: declPos}) + } + } + + simple := jsSimpleAliasRe.FindAllStringSubmatchIndex(content, -1) + for _, m := range simple { + if len(m) < 6 { + continue + } + alias := strings.TrimSpace(content[m[2]:m[3]]) + prop := strings.TrimSpace(content[m[4]:m[5]]) + if alias == "" || prop == "" { + continue + } + aliases = append(aliases, jsAliasProperty{alias: alias, propName: prop, isRef: false, declPos: m[2]}) + } + + return aliases } // isDeclarationReferenceMatch reports whether a regex call match occurs in a declaration context. @@ -824,7 +934,7 @@ func dedupeReferences(refs []Reference) []Reference { deduped := make([]Reference, 0, len(refs)) for _, ref := range refs { - key := ref.SymbolName + "\x00" + ref.File + "\x00" + ref.CallerName + "\x00" + strconv.Itoa(ref.Line) + "\x00" + strconv.Itoa(ref.CallerLine) + key := ref.SymbolName + "\x00" + ref.Kind + "\x00" + ref.File + "\x00" + ref.CallerName + "\x00" + strconv.Itoa(ref.Line) + "\x00" + strconv.Itoa(ref.CallerLine) if seen[key] { continue } diff --git a/trace/extractor_test.go b/trace/extractor_test.go index b8b3565..cf0b204 100644 --- a/trace/extractor_test.go +++ b/trace/extractor_test.go @@ -1340,3 +1340,46 @@ func TestRegexExtractor_ExtractReferences_JSTypeScriptPropertyReadWrite(t *testi t.Fatal("expected at least one write reference for uid") } } + +func TestRegexExtractor_ExtractReferences_CompositionAPIAliases(t *testing.T) { + extractor := NewRegexExtractor() + ctx := context.Background() + + content := `function setup(store) { + const { uid: uidRef } = storeToRefs(store) + const roleLocal = store.role + const r1 = uidRef.value + uidRef.value = "x" + return roleLocal +}` + + refs, err := extractor.ExtractReferences(ctx, "comp.ts", content) + if err != nil { + t.Fatalf("ExtractReferences failed: %v", err) + } + + readUID := false + writeUID := false + readRole := false + for _, ref := range refs { + if ref.SymbolName == "uid" && ref.Kind == RefKindRead { + readUID = true + } + if ref.SymbolName == "uid" && ref.Kind == RefKindWrite { + writeUID = true + } + if ref.SymbolName == "role" && ref.Kind == RefKindRead { + readRole = true + } + } + + if !readUID { + t.Fatal("expected uid read via uidRef.value") + } + if !writeUID { + t.Fatal("expected uid write via uidRef.value assignment") + } + if !readRole { + t.Fatal("expected role read via simple alias") + } +} From 10a042f956d3662fceee26da31836fd0edb3a6d3 Mon Sep 17 00:00:00 2001 From: Mladen Mihajlovic Date: Thu, 12 Mar 2026 18:05:08 +0100 Subject: [PATCH 07/14] feat(refs): add bracket access, refs graph, and MCP refs graph tool --- cli/mcp_serve.go | 1 + cli/refs.go | 78 +++++++++++++++++++++- cli/refs_test.go | 60 +++++++++++++++++ mcp/server.go | 141 ++++++++++++++++++++++++++++++++++++++++ mcp/server_test.go | 33 ++++++++++ trace/extractor.go | 27 ++++++++ trace/extractor_test.go | 34 ++++++++++ trace/extractor_ts.go | 31 +++++++-- 8 files changed, 399 insertions(+), 6 deletions(-) diff --git a/cli/mcp_serve.go b/cli/mcp_serve.go index 42eb790..165f834 100644 --- a/cli/mcp_serve.go +++ b/cli/mcp_serve.go @@ -24,6 +24,7 @@ The server communicates via stdio and exposes the following tools: - grepai_trace_graph: Build a call graph around a symbol - grepai_refs_readers: Find property/state readers for a symbol name - grepai_refs_writers: Find property/state writers for a symbol name + - grepai_refs_graph: Build a property usage graph (readers + writers) - grepai_index_status: Check index health and statistics (includes RPG stats when enabled) - grepai_rpg_search: Search RPG graph nodes by feature semantics - grepai_rpg_fetch: Fetch hierarchy and edge context for a specific RPG node diff --git a/cli/refs.go b/cli/refs.go index d399286..ab8fe3a 100644 --- a/cli/refs.go +++ b/cli/refs.go @@ -35,6 +35,14 @@ type refsResult struct { Writers []refsUsage `json:"writers,omitempty"` } +type refsGraphResult struct { + Query string `json:"query"` + Kind string `json:"kind"` + Mode string `json:"mode"` + Readers []refsUsage `json:"readers,omitempty"` + Writers []refsUsage `json:"writers,omitempty"` +} + var refsCmd = &cobra.Command{ Use: "refs ", Short: "Trace property/data readers and writers", @@ -73,8 +81,33 @@ var refsWritersCmd = &cobra.Command{ }, } +var refsGraphCmd = &cobra.Command{ + Use: "graph ", + Short: "Show readers and writers graph for a property/data symbol", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + readersResult, err := runRefs(args[0], true) + if err != nil { + return err + } + writersResult, err := runRefs(args[0], false) + if err != nil { + return err + } + + graph := refsGraphResult{ + Query: args[0], + Kind: "property", + Mode: "fast", + Readers: readersResult.Readers, + Writers: writersResult.Writers, + } + return outputRefsGraphResult(graph) + }, +} + func init() { - for _, cmd := range []*cobra.Command{refsReadersCmd, refsWritersCmd} { + for _, cmd := range []*cobra.Command{refsReadersCmd, refsWritersCmd, refsGraphCmd} { cmd.Flags().BoolVar(&refsJSON, "json", false, "Output results in JSON format") cmd.Flags().BoolVarP(&refsTOON, "toon", "t", false, "Output results in TOON format (token-efficient for AI agents)") cmd.MarkFlagsMutuallyExclusive("json", "toon") @@ -84,6 +117,7 @@ func init() { refsCmd.AddCommand(refsReadersCmd) refsCmd.AddCommand(refsWritersCmd) + refsCmd.AddCommand(refsGraphCmd) rootCmd.AddCommand(refsCmd) } @@ -218,3 +252,45 @@ func outputRefsResult(result refsResult, readers bool) error { return nil } + +func outputRefsGraphResult(result refsGraphResult) error { + if refsJSON { + enc := json.NewEncoder(os.Stdout) + enc.SetIndent("", " ") + return enc.Encode(result) + } + if refsTOON { + output, err := gotoon.Encode(result) + if err != nil { + return fmt.Errorf("failed to encode TOON: %w", err) + } + fmt.Println(output) + return nil + } + + fmt.Printf("Symbol: %s (%s)\n", result.Query, result.Kind) + fmt.Printf("Readers: %d\n", len(result.Readers)) + fmt.Printf("Writers: %d\n", len(result.Writers)) + fmt.Println(strings.Repeat("-", 60)) + + if len(result.Readers) > 0 { + fmt.Println("Reader Sites:") + for i, usage := range result.Readers { + fmt.Printf("%d. %s @ %s:%d\n", i+1, usage.Symbol.Name, usage.AccessAt.File, usage.AccessAt.Line) + } + } + if len(result.Writers) > 0 { + if len(result.Readers) > 0 { + fmt.Println() + } + fmt.Println("Writer Sites:") + for i, usage := range result.Writers { + fmt.Printf("%d. %s @ %s:%d\n", i+1, usage.Symbol.Name, usage.AccessAt.File, usage.AccessAt.Line) + } + } + if len(result.Readers) == 0 && len(result.Writers) == 0 { + fmt.Println("No references found.") + } + + return nil +} diff --git a/cli/refs_test.go b/cli/refs_test.go index 510e1a9..1f4dd1f 100644 --- a/cli/refs_test.go +++ b/cli/refs_test.go @@ -1,7 +1,9 @@ package cli import ( + "bytes" "context" + "encoding/json" "os" "path/filepath" "testing" @@ -94,3 +96,61 @@ func TestRunRefs_should_return_readers_and_writers_from_index(t *testing.T) { t.Fatalf("expected writer symbol user_details, got %q", writers.Writers[0].Symbol.Name) } } + +func TestOutputRefsGraphResult_should_output_valid_json(t *testing.T) { + oldJSON := refsJSON + oldTOON := refsTOON + refsJSON = true + refsTOON = false + defer func() { + refsJSON = oldJSON + refsTOON = oldTOON + }() + + graph := refsGraphResult{ + Query: "uid", + Kind: "property", + Mode: "fast", + Readers: []refsUsage{ + { + Symbol: trace.Symbol{Name: "loggedIn", File: "src/Home.vue", Line: 10}, + Access: trace.RefKindRead, + AccessAt: trace.CallSite{ + File: "src/Home.vue", Line: 12, Context: "store.uid", + }, + }, + }, + Writers: []refsUsage{ + { + Symbol: trace.Symbol{Name: "user_details", File: "src/store.ts", Line: 20}, + Access: trace.RefKindWrite, + AccessAt: trace.CallSite{ + File: "src/store.ts", Line: 22, Context: "this.uid = x", + }, + }, + }, + } + + oldStdout := os.Stdout + r, w, _ := os.Pipe() + os.Stdout = w + err := outputRefsGraphResult(graph) + _ = w.Close() + os.Stdout = oldStdout + if err != nil { + t.Fatalf("outputRefsGraphResult failed: %v", err) + } + + var buf bytes.Buffer + _, _ = buf.ReadFrom(r) + var decoded refsGraphResult + if err := json.Unmarshal(buf.Bytes(), &decoded); err != nil { + t.Fatalf("invalid JSON output: %v", err) + } + if decoded.Query != "uid" { + t.Fatalf("decoded query = %q, want uid", decoded.Query) + } + if len(decoded.Readers) != 1 || len(decoded.Writers) != 1 { + t.Fatalf("expected one reader and one writer, got %d/%d", len(decoded.Readers), len(decoded.Writers)) + } +} diff --git a/mcp/server.go b/mcp/server.go index 66eb751..27b1a52 100644 --- a/mcp/server.go +++ b/mcp/server.go @@ -295,6 +295,27 @@ func (s *Server) registerTools() { ) s.mcpServer.AddTool(refsWritersTool, s.handleRefsWriters) + refsGraphTool := mcp.NewTool("grepai_refs_graph", + mcp.WithDescription("Build a property/state usage graph for a symbol by combining readers and writers."), + mcp.WithString("symbol", + mcp.Required(), + mcp.Description("Property/state symbol name to build graph for"), + ), + mcp.WithBoolean("compact", + mcp.Description("Return minimal output without context (default: false)"), + ), + mcp.WithString("format", + mcp.Description("Output format: 'json' (default) or 'toon' (token-efficient)"), + ), + mcp.WithString("workspace", + mcp.Description("Workspace name for cross-project refs (optional)"), + ), + mcp.WithString("project", + mcp.Description("Project name within workspace (requires workspace)"), + ), + ) + s.mcpServer.AddTool(refsGraphTool, s.handleRefsGraph) + // grepai_index_status tool indexStatusTool := mcp.NewTool("grepai_index_status", mcp.WithDescription("Check the health and status of the grepai index. Returns statistics about indexed files, chunks, and configuration."), @@ -1499,6 +1520,126 @@ func (s *Server) handleRefsWriters(ctx context.Context, request mcp.CallToolRequ return s.handleRefsByKind(ctx, request, trace.RefKindWrite) } +func (s *Server) handleRefsGraph(ctx context.Context, request mcp.CallToolRequest) (*mcp.CallToolResult, error) { + symbolName, err := request.RequireString("symbol") + if err != nil { + return mcp.NewToolResultError("symbol parameter is required"), nil + } + + compact := request.GetBool("compact", false) + format := request.GetString("format", "json") + workspace := s.resolveWorkspace(request.GetString("workspace", "")) + project := request.GetString("project", "") + + if format != "json" && format != "toon" { + return mcp.NewToolResultError("format must be 'json' or 'toon'"), nil + } + + var stores []trace.SymbolStore + if workspace != "" { + stores, err = trace.LoadWorkspaceSymbolStores(ctx, workspace, project) + if err != nil { + return mcp.NewToolResultError(fmt.Sprintf("failed to load workspace symbol stores: %v", err)), nil + } + defer trace.CloseSymbolStores(stores) + } else { + if s.projectRoot == "" { + return mcp.NewToolResultError("refs requires a project context; use --workspace parameter or start mcp-serve from a project directory"), nil + } + symbolStore := trace.NewGOBSymbolStore(config.GetSymbolIndexPath(s.projectRoot)) + if err := symbolStore.Load(ctx); err != nil { + return mcp.NewToolResultError(fmt.Sprintf("failed to load symbol index: %v. Run 'grepai watch' first", err)), nil + } + defer symbolStore.Close() + stats, err := symbolStore.GetStats(ctx) + if err != nil || stats.TotalSymbols == 0 { + return mcp.NewToolResultError("symbol index is empty. Run 'grepai watch' first to build the index"), nil + } + stores = []trace.SymbolStore{symbolStore} + } + + readers := make([]RefUsage, 0) + writers := make([]RefUsage, 0) + for _, ss := range stores { + r, err := ss.LookupReaders(ctx, symbolName) + if err == nil { + for _, ref := range r { + readers = append(readers, RefUsage{ + Symbol: resolveRefCallerSymbol(ss, ctx, ref), + Access: ref.Kind, + AccessAt: trace.CallSite{ + File: ref.File, + Line: ref.Line, + Context: ref.Context, + }, + }) + } + } + w, err := ss.LookupWriters(ctx, symbolName) + if err == nil { + for _, ref := range w { + writers = append(writers, RefUsage{ + Symbol: resolveRefCallerSymbol(ss, ctx, ref), + Access: ref.Kind, + AccessAt: trace.CallSite{ + File: ref.File, + Line: ref.Line, + Context: ref.Context, + }, + }) + } + } + } + + var data any + if compact { + rc := make([]RefUsageCompact, 0, len(readers)) + for _, usage := range readers { + rc = append(rc, RefUsageCompact{ + Symbol: usage.Symbol, + Access: usage.Access, + AccessAt: CallSiteCompact{ + File: usage.AccessAt.File, + Line: usage.AccessAt.Line, + }, + }) + } + wc := make([]RefUsageCompact, 0, len(writers)) + for _, usage := range writers { + wc = append(wc, RefUsageCompact{ + Symbol: usage.Symbol, + Access: usage.Access, + AccessAt: CallSiteCompact{ + File: usage.AccessAt.File, + Line: usage.AccessAt.Line, + }, + }) + } + data = map[string]any{ + "query": symbolName, + "kind": "property", + "mode": "fast", + "readers": rc, + "writers": wc, + } + } else { + data = map[string]any{ + "query": symbolName, + "kind": "property", + "mode": "fast", + "readers": readers, + "writers": writers, + } + } + + output, err := encodeOutput(data, format) + if err != nil { + return mcp.NewToolResultError(fmt.Sprintf("failed to encode results: %v", err)), nil + } + + return mcp.NewToolResultText(output), nil +} + func (s *Server) handleRefsByKind(ctx context.Context, request mcp.CallToolRequest, kind string) (*mcp.CallToolResult, error) { symbolName, err := request.RequireString("symbol") if err != nil { diff --git a/mcp/server_test.go b/mcp/server_test.go index 72c5039..fc93d5b 100644 --- a/mcp/server_test.go +++ b/mcp/server_test.go @@ -389,6 +389,39 @@ func TestRegisterTools_should_include_workspace_param_on_trace_graph(t *testing. } } +func TestRegisterTools_should_include_workspace_param_on_refs_readers(t *testing.T) { + props := helperGetToolSchemaProperties(t, "grepai_refs_readers") + + if _, ok := props["workspace"]; !ok { + t.Error("expected 'workspace' property in grepai_refs_readers schema") + } + if _, ok := props["project"]; !ok { + t.Error("expected 'project' property in grepai_refs_readers schema") + } +} + +func TestRegisterTools_should_include_workspace_param_on_refs_writers(t *testing.T) { + props := helperGetToolSchemaProperties(t, "grepai_refs_writers") + + if _, ok := props["workspace"]; !ok { + t.Error("expected 'workspace' property in grepai_refs_writers schema") + } + if _, ok := props["project"]; !ok { + t.Error("expected 'project' property in grepai_refs_writers schema") + } +} + +func TestRegisterTools_should_include_workspace_param_on_refs_graph(t *testing.T) { + props := helperGetToolSchemaProperties(t, "grepai_refs_graph") + + if _, ok := props["workspace"]; !ok { + t.Error("expected 'workspace' property in grepai_refs_graph schema") + } + if _, ok := props["project"]; !ok { + t.Error("expected 'project' property in grepai_refs_graph schema") + } +} + // TestRegisterTools_should_include_workspace_param_on_index_status verifies that // grepai_index_status has a workspace property in its schema. func TestRegisterTools_should_include_workspace_param_on_index_status(t *testing.T) { diff --git a/trace/extractor.go b/trace/extractor.go index daeef9d..e71d241 100644 --- a/trace/extractor.go +++ b/trace/extractor.go @@ -237,6 +237,8 @@ func (e *RegexExtractor) extractLanguageSpecificReferences(filePath string, cont var ( jsPropertyReadRe = regexp.MustCompile(`\b(?:this\.)?(?:[A-Za-z_$][A-Za-z0-9_$]*\.)+([A-Za-z_$][A-Za-z0-9_$]*)\b`) jsPropertyWriteRe = regexp.MustCompile(`\b(?:this\.)?(?:[A-Za-z_$][A-Za-z0-9_$]*\.)+([A-Za-z_$][A-Za-z0-9_$]*)\s*=`) + jsBracketReadRe = regexp.MustCompile(`\b(?:this\.)?(?:[A-Za-z_$][A-Za-z0-9_$]*\.)*[A-Za-z_$][A-Za-z0-9_$]*\s*\[\s*["']([A-Za-z_$][A-Za-z0-9_$]*)["']\s*\]`) + jsBracketWriteRe = regexp.MustCompile(`\b(?:this\.)?(?:[A-Za-z_$][A-Za-z0-9_$]*\.)*[A-Za-z_$][A-Za-z0-9_$]*\s*\[\s*["']([A-Za-z_$][A-Za-z0-9_$]*)["']\s*\]\s*=`) jsStoreToRefsRe = regexp.MustCompile(`\bconst\s*{\s*([^}]*)\s*}\s*=\s*storeToRefs\s*\([^)]*\)`) jsSimpleAliasRe = regexp.MustCompile(`\b(?:const|let|var)\s+([A-Za-z_$][A-Za-z0-9_$]*)\s*=\s*[A-Za-z_$][A-Za-z0-9_$]*\.([A-Za-z_$][A-Za-z0-9_$]*)\b`) ) @@ -269,6 +271,31 @@ func (e *RegexExtractor) extractJSPropertyReferences(filePath string, content st refs = append(refs, buildDataReference(filePath, content, lines, name, start, RefKindRead, functionBoundaries)) } + // Bracket property access: store["uid"], this.store['role'] + bracketWriteMatches := jsBracketWriteRe.FindAllStringSubmatchIndex(content, -1) + for _, m := range bracketWriteMatches { + if len(m) < 4 { + continue + } + start := m[0] + writeStarts[start] = true + name := content[m[2]:m[3]] + refs = append(refs, buildDataReference(filePath, content, lines, name, start, RefKindWrite, functionBoundaries)) + } + + bracketReadMatches := jsBracketReadRe.FindAllStringSubmatchIndex(content, -1) + for _, m := range bracketReadMatches { + if len(m) < 4 { + continue + } + start := m[0] + if writeStarts[start] { + continue + } + name := content[m[2]:m[3]] + refs = append(refs, buildDataReference(filePath, content, lines, name, start, RefKindRead, functionBoundaries)) + } + refs = append(refs, extractJSAliasPropertyReferences(filePath, content, lines, functionBoundaries)...) return dedupeReferences(refs) } diff --git a/trace/extractor_test.go b/trace/extractor_test.go index cf0b204..6d3f1f7 100644 --- a/trace/extractor_test.go +++ b/trace/extractor_test.go @@ -1383,3 +1383,37 @@ func TestRegexExtractor_ExtractReferences_CompositionAPIAliases(t *testing.T) { t.Fatal("expected role read via simple alias") } } + +func TestRegexExtractor_ExtractReferences_BracketPropertyAccess(t *testing.T) { + extractor := NewRegexExtractor() + ctx := context.Background() + + content := `function setup(store) { + const a = store["uid"] + this.store["role"] = "admin" + return a +}` + + refs, err := extractor.ExtractReferences(ctx, "comp.ts", content) + if err != nil { + t.Fatalf("ExtractReferences failed: %v", err) + } + + readUID := false + writeRole := false + for _, ref := range refs { + if ref.SymbolName == "uid" && ref.Kind == RefKindRead { + readUID = true + } + if ref.SymbolName == "role" && ref.Kind == RefKindWrite { + writeRole = true + } + } + + if !readUID { + t.Fatal("expected uid read via bracket access") + } + if !writeRole { + t.Fatal("expected role write via bracket access") + } +} diff --git a/trace/extractor_ts.go b/trace/extractor_ts.go index a872173..5ff8d5a 100644 --- a/trace/extractor_ts.go +++ b/trace/extractor_ts.go @@ -754,7 +754,7 @@ func (e *TreeSitterExtractor) extractMemberAccessReference(node *sitter.Node, co return Reference{}, false } - name := strings.TrimSpace(propertyNode.Content(content)) + name := normalizeJSPropertyName(propertyNode.Content(content)) if name == "" { return Reference{}, false } @@ -762,10 +762,6 @@ func (e *TreeSitterExtractor) extractMemberAccessReference(node *sitter.Node, co if strings.HasPrefix(name, "_") { return Reference{}, false } - // Ignore computed member access (obj["uid"]) for now; handled later by a dedicated parser path. - if strings.ContainsAny(name, "\"'[]") { - return Reference{}, false - } kind := RefKindRead parent := node.Parent() @@ -797,6 +793,31 @@ func (e *TreeSitterExtractor) extractMemberAccessReference(node *sitter.Node, co }, true } +func normalizeJSPropertyName(raw string) string { + name := strings.TrimSpace(raw) + if name == "" { + return "" + } + + // Handle computed string members such as obj["uid"] or obj['uid']. + if len(name) >= 2 && ((name[0] == '"' && name[len(name)-1] == '"') || (name[0] == '\'' && name[len(name)-1] == '\'')) { + name = name[1 : len(name)-1] + } + + name = strings.TrimSpace(name) + if name == "" { + return "" + } + + for _, r := range name { + if !(r == '$' || r == '_' || (r >= '0' && r <= '9') || (r >= 'A' && r <= 'Z') || (r >= 'a' && r <= 'z')) { + return "" + } + } + + return name +} + func extractFSharpCallName(node *sitter.Node, content []byte) string { text := node.Content(content) // For dotted access like "String.Format" or "Logger.log", take the last part From 87eda0537e60466315ea53b97ee2ad176f051511 Mon Sep 17 00:00:00 2001 From: Mladen Mihajlovic Date: Thu, 12 Mar 2026 18:13:45 +0100 Subject: [PATCH 08/14] test/docs(refs): harden vue matrix refs checks and document trace vs refs --- docs/src/content/docs/trace.md | 19 ++++++++ scripts/test-vue-framework-matrix.sh | 65 +++++++++++++++++++++++----- trace/extractor.go | 2 +- 3 files changed, 73 insertions(+), 13 deletions(-) diff --git a/docs/src/content/docs/trace.md b/docs/src/content/docs/trace.md index 4b525a5..77c9861 100644 --- a/docs/src/content/docs/trace.md +++ b/docs/src/content/docs/trace.md @@ -7,6 +7,22 @@ description: Analyze function relationships with grepai trace `grepai trace` provides call graph analysis for your codebase, allowing you to understand how functions relate to each other by tracking callers and callees. +### Trace vs Refs + +Use `trace` for call relationships, and `refs` for property/state usage. + +```bash +# Call graph (functions/methods) +grepai trace callers "isAdmin" + +# Property/state usage (reads/writes) +grepai refs readers "uid" +grepai refs writers "uid" +grepai refs graph "uid" +``` + +`grepai refs` is especially useful in Vue/Pinia code where state keys (for example `store.uid`) are read and written without direct function calls. + ### Features - **Find callers**: Discover which functions call a specific symbol @@ -198,3 +214,6 @@ grepai trace graph "AuthMiddleware" --depth 2 --json - [`grepai trace callers`](/grepai/commands/grepai_trace_callers/) - Find functions that call a symbol - [`grepai trace callees`](/grepai/commands/grepai_trace_callees/) - Find functions called by a symbol - [`grepai trace graph`](/grepai/commands/grepai_trace_graph/) - Build complete call graph +- [`grepai refs readers`](/grepai/commands/grepai_refs_readers/) - Find property/state readers +- [`grepai refs writers`](/grepai/commands/grepai_refs_writers/) - Find property/state writers +- [`grepai refs graph`](/grepai/commands/grepai_refs_graph/) - Build property usage graph diff --git a/scripts/test-vue-framework-matrix.sh b/scripts/test-vue-framework-matrix.sh index 11ccd9a..f429e5e 100755 --- a/scripts/test-vue-framework-matrix.sh +++ b/scripts/test-vue-framework-matrix.sh @@ -7,15 +7,15 @@ TEST_ROOT="/tmp/grepai-vue-matrix" cd "$ROOT" -echo "[0/8] Ensure binary exists" +echo "[0/10] Ensure binary exists" [ -x "$BIN" ] || make build "$BIN" version -echo "[1/8] Install compiler" +echo "[1/10] Install compiler" HAVE_COMPILER=1 echo " (will install into temp project after creation)" -echo "[2/8] Create matrix project" +echo "[2/10] Create matrix project" rm -rf "$TEST_ROOT" mkdir -p "$TEST_ROOT/src" @@ -76,6 +76,22 @@ export function styledFn() { return "ok" } VUE +cat > "$TEST_ROOT/src/StateRefs.vue" <<'VUE' + + +VUE + if ! ( cd "$TEST_ROOT" && npm init -y >/dev/null 2>&1 && timeout 60s npm install -D @vue/compiler-sfc >/dev/null 2>&1 ); then HAVE_COMPILER=0 echo "WARN: compiler install failed or timed out; continuing in fallback-only mode" @@ -108,21 +124,24 @@ if ! grep -q "\.vue" "$CFG"; then - ".vue"/' "$CFG" fi -echo "[3/8] Index with compiler/fallback path" +echo "[3/10] Index with compiler/fallback path" ( cd "$TEST_ROOT" && timeout 20s "$BIN" watch > /tmp/grepai-vue-matrix-watch.log 2>&1 || true ) -echo "[4/8] Search assertions" +echo "[4/10] Search assertions" ( cd "$TEST_ROOT" && "$BIN" search "mixed helper and styled function" --json ) > /tmp/grepai-vue-matrix-search.json -for f in ScriptOnly.vue ScriptSetupOnly.vue Mixed.vue StyleHeavy.vue; do +for f in ScriptOnly.vue ScriptSetupOnly.vue Mixed.vue StyleHeavy.vue StateRefs.vue; do grep -q "$f" /tmp/grepai-vue-matrix-search.json echo " - found $f" done for f in ScriptJS.vue ScriptNoLang.vue; do - grep -q "$f" /tmp/grepai-vue-matrix-search.json - echo " - found $f" + if grep -q "$f" /tmp/grepai-vue-matrix-search.json; then + echo " - found $f" + else + echo " - optional in semantic search: $f (validated later via trace)" + fi done -echo "[5/8] Trace assertions" +echo "[5/10] Trace assertions" ( cd "$TEST_ROOT" && "$BIN" trace callees runMixed --json ) > /tmp/grepai-vue-matrix-trace-callees.json || true grep -q "helperMixed" /tmp/grepai-vue-matrix-trace-callees.json @@ -136,17 +155,35 @@ grep -q "scriptJsFn" /tmp/grepai-vue-matrix-trace-callers-js.json grep -q "scriptNoLangFn" /tmp/grepai-vue-matrix-trace-callers-nolang.json echo " - JS and no-lang script symbols detected" -echo "[6/8] Fallback mode run" +echo "[6/10] Refs assertions" +( cd "$TEST_ROOT" && "$BIN" refs readers uid --json ) > /tmp/grepai-vue-matrix-refs-readers-uid.json +grep -q "StateRefs.vue" /tmp/grepai-vue-matrix-refs-readers-uid.json +grep -q "\"access\": \"read\"" /tmp/grepai-vue-matrix-refs-readers-uid.json + +( cd "$TEST_ROOT" && "$BIN" refs writers uid --json ) > /tmp/grepai-vue-matrix-refs-writers-uid.json +grep -q "StateRefs.vue" /tmp/grepai-vue-matrix-refs-writers-uid.json +grep -q "\"access\": \"write\"" /tmp/grepai-vue-matrix-refs-writers-uid.json + +( cd "$TEST_ROOT" && "$BIN" refs graph role --json ) > /tmp/grepai-vue-matrix-refs-graph-role.json +grep -q "\"readers\"" /tmp/grepai-vue-matrix-refs-graph-role.json +grep -q "\"writers\"" /tmp/grepai-vue-matrix-refs-graph-role.json +echo " - refs readers/writers/graph captured Composition API + bracket usage" + +echo "[7/10] Fallback mode run" if [ "$HAVE_COMPILER" -eq 1 ]; then ( cd "$TEST_ROOT" && npm uninstall @vue/compiler-sfc >/dev/null ) || true fi ( cd "$TEST_ROOT" && timeout 12s "$BIN" watch > /tmp/grepai-vue-matrix-watch-fallback.log 2>&1 || true ) -echo "[7/8] Fallback still searchable" +echo "[8/10] Fallback still searchable" ( cd "$TEST_ROOT" && "$BIN" search "setup only function" --json ) > /tmp/grepai-vue-matrix-search-fallback.json grep -q "ScriptSetupOnly.vue" /tmp/grepai-vue-matrix-search-fallback.json -echo "[8/8] Done" +echo "[9/10] Fallback refs still available" +( cd "$TEST_ROOT" && "$BIN" refs readers uid --json ) > /tmp/grepai-vue-matrix-refs-readers-uid-fallback.json +grep -q "StateRefs.vue" /tmp/grepai-vue-matrix-refs-readers-uid-fallback.json + +echo "[10/10] Done" echo "Artifacts:" echo " /tmp/grepai-vue-matrix-watch.log" echo " /tmp/grepai-vue-matrix-search.json" @@ -154,5 +191,9 @@ echo " /tmp/grepai-vue-matrix-trace-callees.json" echo " /tmp/grepai-vue-matrix-trace-callers-style.json" echo " /tmp/grepai-vue-matrix-trace-callers-js.json" echo " /tmp/grepai-vue-matrix-trace-callers-nolang.json" +echo " /tmp/grepai-vue-matrix-refs-readers-uid.json" +echo " /tmp/grepai-vue-matrix-refs-writers-uid.json" +echo " /tmp/grepai-vue-matrix-refs-graph-role.json" echo " /tmp/grepai-vue-matrix-watch-fallback.log" echo " /tmp/grepai-vue-matrix-search-fallback.json" +echo " /tmp/grepai-vue-matrix-refs-readers-uid-fallback.json" diff --git a/trace/extractor.go b/trace/extractor.go index e71d241..9b39621 100644 --- a/trace/extractor.go +++ b/trace/extractor.go @@ -239,7 +239,7 @@ var ( jsPropertyWriteRe = regexp.MustCompile(`\b(?:this\.)?(?:[A-Za-z_$][A-Za-z0-9_$]*\.)+([A-Za-z_$][A-Za-z0-9_$]*)\s*=`) jsBracketReadRe = regexp.MustCompile(`\b(?:this\.)?(?:[A-Za-z_$][A-Za-z0-9_$]*\.)*[A-Za-z_$][A-Za-z0-9_$]*\s*\[\s*["']([A-Za-z_$][A-Za-z0-9_$]*)["']\s*\]`) jsBracketWriteRe = regexp.MustCompile(`\b(?:this\.)?(?:[A-Za-z_$][A-Za-z0-9_$]*\.)*[A-Za-z_$][A-Za-z0-9_$]*\s*\[\s*["']([A-Za-z_$][A-Za-z0-9_$]*)["']\s*\]\s*=`) - jsStoreToRefsRe = regexp.MustCompile(`\bconst\s*{\s*([^}]*)\s*}\s*=\s*storeToRefs\s*\([^)]*\)`) + jsStoreToRefsRe = regexp.MustCompile(`\bconst\s*{\s*([^}]*)\s*}\s*=\s*(?:storeToRefs|toRefs)\s*\([^)]*\)`) jsSimpleAliasRe = regexp.MustCompile(`\b(?:const|let|var)\s+([A-Za-z_$][A-Za-z0-9_$]*)\s*=\s*[A-Za-z_$][A-Za-z0-9_$]*\.([A-Za-z_$][A-Za-z0-9_$]*)\b`) ) From 60304225329ea41105112c0df62d2a815b3ff1f1 Mon Sep 17 00:00:00 2001 From: Mladen Mihajlovic Date: Thu, 12 Mar 2026 20:26:40 +0100 Subject: [PATCH 09/14] feat(refs): reduce JS/Vue property noise from builtins and runtime internals --- trace/extractor.go | 73 +++++++++++++++++++++++++++++++++++++++-- trace/extractor_test.go | 52 +++++++++++++++++++++++++++++ trace/extractor_ts.go | 29 ++++++++++++++-- 3 files changed, 149 insertions(+), 5 deletions(-) diff --git a/trace/extractor.go b/trace/extractor.go index 9b39621..ef793e0 100644 --- a/trace/extractor.go +++ b/trace/extractor.go @@ -240,9 +240,23 @@ var ( jsBracketReadRe = regexp.MustCompile(`\b(?:this\.)?(?:[A-Za-z_$][A-Za-z0-9_$]*\.)*[A-Za-z_$][A-Za-z0-9_$]*\s*\[\s*["']([A-Za-z_$][A-Za-z0-9_$]*)["']\s*\]`) jsBracketWriteRe = regexp.MustCompile(`\b(?:this\.)?(?:[A-Za-z_$][A-Za-z0-9_$]*\.)*[A-Za-z_$][A-Za-z0-9_$]*\s*\[\s*["']([A-Za-z_$][A-Za-z0-9_$]*)["']\s*\]\s*=`) jsStoreToRefsRe = regexp.MustCompile(`\bconst\s*{\s*([^}]*)\s*}\s*=\s*(?:storeToRefs|toRefs)\s*\([^)]*\)`) - jsSimpleAliasRe = regexp.MustCompile(`\b(?:const|let|var)\s+([A-Za-z_$][A-Za-z0-9_$]*)\s*=\s*[A-Za-z_$][A-Za-z0-9_$]*\.([A-Za-z_$][A-Za-z0-9_$]*)\b`) + jsSimpleAliasRe = regexp.MustCompile(`\b(?:const|let|var)\s+([A-Za-z_$][A-Za-z0-9_$]*)\s*=\s*([A-Za-z_$][A-Za-z0-9_$]*)\.([A-Za-z_$][A-Za-z0-9_$]*)\b`) ) +var jsBuiltinRoots = map[string]bool{ + "Math": true, "JSON": true, "Object": true, "Array": true, "String": true, + "Number": true, "Boolean": true, "Date": true, "RegExp": true, "Promise": true, + "Reflect": true, "Intl": true, "Set": true, "Map": true, "WeakSet": true, + "WeakMap": true, "Symbol": true, "BigInt": true, "console": true, + "window": true, "document": true, "globalThis": true, +} + +var jsVueRuntimeInternalProps = map[string]bool{ + "$el": true, "$refs": true, "$slots": true, "$attrs": true, "$listeners": true, + "$parent": true, "$root": true, "$children": true, "$scopedSlots": true, + "$isServer": true, "$ssrContext": true, "$vnode": true, "$props": true, +} + func (e *RegexExtractor) extractJSPropertyReferences(filePath string, content string, lines []string, functionBoundaries []functionBoundary) []Reference { writeMatches := jsPropertyWriteRe.FindAllStringSubmatchIndex(content, -1) writeStarts := make(map[int]bool, len(writeMatches)) @@ -254,7 +268,11 @@ func (e *RegexExtractor) extractJSPropertyReferences(filePath string, content st } start := m[0] writeStarts[start] = true + expr := content[m[0]:m[1]] name := content[m[2]:m[3]] + if !keepJSPropertyReference(name, expr) { + continue + } refs = append(refs, buildDataReference(filePath, content, lines, name, start, RefKindWrite, functionBoundaries)) } @@ -267,7 +285,11 @@ func (e *RegexExtractor) extractJSPropertyReferences(filePath string, content st if writeStarts[start] { continue } + expr := content[m[0]:m[1]] name := content[m[2]:m[3]] + if !keepJSPropertyReference(name, expr) { + continue + } refs = append(refs, buildDataReference(filePath, content, lines, name, start, RefKindRead, functionBoundaries)) } @@ -279,7 +301,11 @@ func (e *RegexExtractor) extractJSPropertyReferences(filePath string, content st } start := m[0] writeStarts[start] = true + expr := content[m[0]:m[1]] name := content[m[2]:m[3]] + if !keepJSPropertyReference(name, expr) { + continue + } refs = append(refs, buildDataReference(filePath, content, lines, name, start, RefKindWrite, functionBoundaries)) } @@ -292,7 +318,11 @@ func (e *RegexExtractor) extractJSPropertyReferences(filePath string, content st if writeStarts[start] { continue } + expr := content[m[0]:m[1]] name := content[m[2]:m[3]] + if !keepJSPropertyReference(name, expr) { + continue + } refs = append(refs, buildDataReference(filePath, content, lines, name, start, RefKindRead, functionBoundaries)) } @@ -300,6 +330,39 @@ func (e *RegexExtractor) extractJSPropertyReferences(filePath string, content st return dedupeReferences(refs) } +func keepJSPropertyReference(name string, expr string) bool { + if name == "" || strings.HasPrefix(name, "_") { + return false + } + if jsVueRuntimeInternalProps[name] { + return false + } + root := extractJSRootIdentifier(expr) + if jsBuiltinRoots[root] { + return false + } + return true +} + +func extractJSRootIdentifier(expr string) string { + expr = strings.TrimSpace(expr) + if expr == "" { + return "" + } + if strings.HasPrefix(expr, "this.") { + expr = expr[len("this."):] + } + for i, r := range expr { + if r == '.' || r == '[' || r == '(' || r == ' ' || r == '\t' || r == '\n' { + if i == 0 { + return "" + } + return expr[:i] + } + } + return expr +} + type jsAliasProperty struct { alias string propName string @@ -393,14 +456,18 @@ func collectJSAliases(content string) []jsAliasProperty { simple := jsSimpleAliasRe.FindAllStringSubmatchIndex(content, -1) for _, m := range simple { - if len(m) < 6 { + if len(m) < 8 { continue } alias := strings.TrimSpace(content[m[2]:m[3]]) - prop := strings.TrimSpace(content[m[4]:m[5]]) + root := strings.TrimSpace(content[m[4]:m[5]]) + prop := strings.TrimSpace(content[m[6]:m[7]]) if alias == "" || prop == "" { continue } + if jsBuiltinRoots[root] || jsVueRuntimeInternalProps[prop] || strings.HasPrefix(prop, "_") { + continue + } aliases = append(aliases, jsAliasProperty{alias: alias, propName: prop, isRef: false, declPos: m[2]}) } diff --git a/trace/extractor_test.go b/trace/extractor_test.go index 6d3f1f7..8e35f90 100644 --- a/trace/extractor_test.go +++ b/trace/extractor_test.go @@ -1417,3 +1417,55 @@ func TestRegexExtractor_ExtractReferences_BracketPropertyAccess(t *testing.T) { t.Fatal("expected role write via bracket access") } } + +func TestRegexExtractor_ExtractReferences_FiltersBuiltinAndVueInternalNoise(t *testing.T) { + extractor := NewRegexExtractor() + ctx := context.Background() + + content := `function setup(store, obj) { + const n = Math.max(1, 2) + console.log(n) + const keys = Object.keys(obj) + this.$refs.input.focus() + const uid = store.uid + return uid +}` + + refs, err := extractor.ExtractReferences(ctx, "noise.ts", content) + if err != nil { + t.Fatalf("ExtractReferences failed: %v", err) + } + + hasUID := false + hasMax := false + hasLog := false + hasKeys := false + hasRefs := false + + for _, ref := range refs { + if ref.Kind == RefKindCall { + continue + } + switch ref.SymbolName { + case "uid": + if ref.Kind == RefKindRead { + hasUID = true + } + case "max": + hasMax = true + case "log": + hasLog = true + case "keys": + hasKeys = true + case "$refs": + hasRefs = true + } + } + + if !hasUID { + t.Fatal("expected store uid read to remain after filtering") + } + if hasMax || hasLog || hasKeys || hasRefs { + t.Fatalf("expected builtins/vue internals filtered, got max=%v log=%v keys=%v $refs=%v", hasMax, hasLog, hasKeys, hasRefs) + } +} diff --git a/trace/extractor_ts.go b/trace/extractor_ts.go index 5ff8d5a..ffb1000 100644 --- a/trace/extractor_ts.go +++ b/trace/extractor_ts.go @@ -758,10 +758,16 @@ func (e *TreeSitterExtractor) extractMemberAccessReference(node *sitter.Node, co if name == "" { return Reference{}, false } - // Skip private/internal slots used by transformed Vue internals. - if strings.HasPrefix(name, "_") { + if strings.HasPrefix(name, "_") || jsVueRuntimeInternalProps[name] { return Reference{}, false } + objectNode := node.ChildByFieldName("object") + if objectNode != nil { + root := extractJSRootFromNodeContent(objectNode.Content(content)) + if jsBuiltinRoots[root] { + return Reference{}, false + } + } kind := RefKindRead parent := node.Parent() @@ -793,6 +799,25 @@ func (e *TreeSitterExtractor) extractMemberAccessReference(node *sitter.Node, co }, true } +func extractJSRootFromNodeContent(objExpr string) string { + objExpr = strings.TrimSpace(objExpr) + if objExpr == "" { + return "" + } + if strings.HasPrefix(objExpr, "this.") { + objExpr = objExpr[len("this."):] + } + for i, r := range objExpr { + if r == '.' || r == '[' || r == '(' || r == ' ' || r == '\t' || r == '\n' { + if i == 0 { + return "" + } + return objExpr[:i] + } + } + return objExpr +} + func normalizeJSPropertyName(raw string) string { name := strings.TrimSpace(raw) if name == "" { From 87c27bbf5e7961f67d1c23597861a1afd37952b0 Mon Sep 17 00:00:00 2001 From: Mladen Mihajlovic Date: Mon, 16 Mar 2026 08:14:29 +0100 Subject: [PATCH 10/14] Fix CI lint failures --- cli/refs.go | 2 +- framework/vue_processor.go | 2 +- trace/extractor.go | 9 ++------- 3 files changed, 4 insertions(+), 9 deletions(-) diff --git a/cli/refs.go b/cli/refs.go index ab8fe3a..3cb58b5 100644 --- a/cli/refs.go +++ b/cli/refs.go @@ -128,7 +128,7 @@ func runRefs(symbolName string, readers bool) (refsResult, error) { return refsResult{}, fmt.Errorf("--project requires --workspace") } - stores := []trace.SymbolStore{} + var stores []trace.SymbolStore if refsWorkspace != "" { var err error stores, err = trace.LoadWorkspaceSymbolStores(ctx, refsWorkspace, refsProject) diff --git a/framework/vue_processor.go b/framework/vue_processor.go index ba06154..9cecb82 100644 --- a/framework/vue_processor.go +++ b/framework/vue_processor.go @@ -57,7 +57,7 @@ type vueScriptOutput struct { func (p *VueProcessor) transform(ctx context.Context, filePath, source string) (TransformResult, error) { in, _ := json.Marshal(vueScriptInput{FilePath: filePath, Source: source}) - cmd := exec.CommandContext(ctx, p.nodePath, "--input-type=module", "-e", vueProcessorScript) + cmd := exec.CommandContext(ctx, p.nodePath, "--input-type=module", "-e", vueProcessorScript) //nolint:gosec // nodePath is an executable path from trusted config; no shell is invoked cmd.Stdin = bytes.NewReader(in) var stdout bytes.Buffer var stderr bytes.Buffer diff --git a/trace/extractor.go b/trace/extractor.go index ef793e0..a14a199 100644 --- a/trace/extractor.go +++ b/trace/extractor.go @@ -338,10 +338,7 @@ func keepJSPropertyReference(name string, expr string) bool { return false } root := extractJSRootIdentifier(expr) - if jsBuiltinRoots[root] { - return false - } - return true + return !jsBuiltinRoots[root] } func extractJSRootIdentifier(expr string) string { @@ -349,9 +346,7 @@ func extractJSRootIdentifier(expr string) string { if expr == "" { return "" } - if strings.HasPrefix(expr, "this.") { - expr = expr[len("this."):] - } + expr = strings.TrimPrefix(expr, "this.") for i, r := range expr { if r == '.' || r == '[' || r == '(' || r == ' ' || r == '\t' || r == '\n' { if i == 0 { From a143a8846daafee706b32d0fa8dbf2945c5d8224 Mon Sep 17 00:00:00 2001 From: Mladen Mihajlovic Date: Mon, 16 Mar 2026 08:32:32 +0100 Subject: [PATCH 11/14] Add MCP refs handler tests --- mcp/server_test.go | 137 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 137 insertions(+) diff --git a/mcp/server_test.go b/mcp/server_test.go index fc93d5b..b5de40a 100644 --- a/mcp/server_test.go +++ b/mcp/server_test.go @@ -8,6 +8,7 @@ import ( "strings" "testing" + "github.com/mark3labs/mcp-go/mcp" "github.com/mark3labs/mcp-go/server" "github.com/yoanbernabeu/grepai/config" "github.com/yoanbernabeu/grepai/store" @@ -468,6 +469,142 @@ func TestRegisterTools_should_document_search_path_scope_and_examples(t *testing } } +func refsTestRequest(args map[string]any) mcp.CallToolRequest { + return mcp.CallToolRequest{ + Params: mcp.CallToolParams{ + Arguments: args, + }, + } +} + +func textResultPayload(t *testing.T, result *mcp.CallToolResult) string { + t.Helper() + if result == nil { + t.Fatal("expected non-nil result") + } + if len(result.Content) == 0 { + t.Fatal("expected non-empty MCP content") + } + textContent, ok := result.Content[0].(mcp.TextContent) + if !ok { + t.Fatalf("expected text content, got %T", result.Content[0]) + } + return textContent.Text +} + +func seedRefsTestStore(t *testing.T) string { + t.Helper() + + projectRoot := t.TempDir() + if err := os.MkdirAll(filepath.Join(projectRoot, config.ConfigDir), 0o755); err != nil { + t.Fatalf("mkdir config dir: %v", err) + } + + ctx := context.Background() + symbolStore := trace.NewGOBSymbolStore(config.GetSymbolIndexPath(projectRoot)) + if err := symbolStore.SaveFile(ctx, "src/store.ts", + []trace.Symbol{ + {Name: "uidConsumer", Kind: trace.KindFunction, File: "src/store.ts", Line: 10}, + }, + []trace.Reference{ + {SymbolName: "uid", Kind: trace.RefKindRead, File: "src/store.ts", Line: 12, Context: "const current = state.uid", CallerName: "uidConsumer", CallerFile: "src/store.ts", CallerLine: 10}, + {SymbolName: "uid", Kind: trace.RefKindWrite, File: "src/store.ts", Line: 13, Context: "state.uid = next", CallerName: "uidConsumer", CallerFile: "src/store.ts", CallerLine: 10}, + }, + ); err != nil { + t.Fatalf("SaveFile failed: %v", err) + } + if err := symbolStore.Close(); err != nil { + t.Fatalf("Close failed: %v", err) + } + + return projectRoot +} + +func TestHandleRefsReaders_requires_symbol(t *testing.T) { + s := &Server{} + + result, err := s.handleRefsReaders(context.Background(), refsTestRequest(map[string]any{"format": "json"})) + if err != nil { + t.Fatalf("handleRefsReaders returned error: %v", err) + } + + if got := textResultPayload(t, result); !strings.Contains(got, "symbol parameter is required") { + t.Fatalf("expected missing symbol error, got %q", got) + } +} + +func TestHandleRefsGraph_rejects_invalid_format(t *testing.T) { + s := &Server{} + + result, err := s.handleRefsGraph(context.Background(), refsTestRequest(map[string]any{ + "symbol": "uid", + "format": "xml", + })) + if err != nil { + t.Fatalf("handleRefsGraph returned error: %v", err) + } + + if got := textResultPayload(t, result); !strings.Contains(got, "format must be 'json' or 'toon'") { + t.Fatalf("expected invalid format error, got %q", got) + } +} + +func TestHandleRefsTools_return_expected_readers_and_graph(t *testing.T) { + projectRoot := seedRefsTestStore(t) + s := &Server{projectRoot: projectRoot} + + readersResult, err := s.handleRefsReaders(context.Background(), refsTestRequest(map[string]any{ + "symbol": "uid", + "format": "json", + })) + if err != nil { + t.Fatalf("handleRefsReaders returned error: %v", err) + } + + var readersPayload struct { + Query string `json:"query"` + Readers []RefUsage `json:"readers"` + } + if err := json.Unmarshal([]byte(textResultPayload(t, readersResult)), &readersPayload); err != nil { + t.Fatalf("failed to decode readers payload: %v", err) + } + if readersPayload.Query != "uid" { + t.Fatalf("query = %q, want uid", readersPayload.Query) + } + if len(readersPayload.Readers) != 1 { + t.Fatalf("expected 1 reader, got %d", len(readersPayload.Readers)) + } + if readersPayload.Readers[0].Access != trace.RefKindRead { + t.Fatalf("reader access = %q, want %q", readersPayload.Readers[0].Access, trace.RefKindRead) + } + if readersPayload.Readers[0].Symbol.Name != "uidConsumer" { + t.Fatalf("reader symbol = %q, want uidConsumer", readersPayload.Readers[0].Symbol.Name) + } + + graphResult, err := s.handleRefsGraph(context.Background(), refsTestRequest(map[string]any{ + "symbol": "uid", + "format": "json", + })) + if err != nil { + t.Fatalf("handleRefsGraph returned error: %v", err) + } + + var graphPayload struct { + Query string `json:"query"` + Readers []RefUsage `json:"readers"` + Writers []RefUsage `json:"writers"` + } + if err := json.Unmarshal([]byte(textResultPayload(t, graphResult)), &graphPayload); err != nil { + t.Fatalf("failed to decode graph payload: %v", err) + } + if len(graphPayload.Readers) != 1 || len(graphPayload.Writers) != 1 { + t.Fatalf("expected 1 reader and 1 writer, got %d/%d", len(graphPayload.Readers), len(graphPayload.Writers)) + } + if graphPayload.Writers[0].Access != trace.RefKindWrite { + t.Fatalf("writer access = %q, want %q", graphPayload.Writers[0].Access, trace.RefKindWrite) + } +} + func TestValidateWorkspacePathForProjects_should_return_structured_hint_for_invalid_path(t *testing.T) { projectRoot := filepath.Join(t.TempDir(), "ubermap_agent") if err := os.MkdirAll(filepath.Join(projectRoot, "MM32", "src"), 0755); err != nil { From b6e7d8687e741d868469a830e0d6ccbd0de5048b Mon Sep 17 00:00:00 2001 From: Mladen Mihajlovic Date: Mon, 16 Mar 2026 14:07:00 +0100 Subject: [PATCH 12/14] Add refs shell completions --- cli/completion.go | 11 +++++--- cli/completion_test.go | 61 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 68 insertions(+), 4 deletions(-) diff --git a/cli/completion.go b/cli/completion.go index 65f7d6e..51a72bc 100644 --- a/cli/completion.go +++ b/cli/completion.go @@ -143,18 +143,21 @@ func registerCompletions() { _ = searchCmd.RegisterFlagCompletionFunc("workspace", workspaceCompleter) _ = watchCmd.RegisterFlagCompletionFunc("workspace", workspaceCompleter) _ = mcpServeCmd.RegisterFlagCompletionFunc("workspace", workspaceCompleter) - for _, cmd := range []*cobra.Command{traceCallersCmd, traceCalleesCmd, traceGraphCmd} { + for _, cmd := range []*cobra.Command{traceCallersCmd, traceCalleesCmd, traceGraphCmd, refsReadersCmd, refsWritersCmd, refsGraphCmd} { _ = cmd.RegisterFlagCompletionFunc("workspace", workspaceCompleter) } - // Dynamic project completion for searchCmd --project (depends on --workspace value) - _ = searchCmd.RegisterFlagCompletionFunc("project", func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { + projectCompleter := func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { wsName, _ := cmd.Flags().GetString("workspace") if wsName == "" { return nil, cobra.ShellCompDirectiveNoFileComp } return completeProjectNames(wsName), cobra.ShellCompDirectiveNoFileComp - }) + } + _ = searchCmd.RegisterFlagCompletionFunc("project", projectCompleter) + for _, cmd := range []*cobra.Command{refsReadersCmd, refsWritersCmd, refsGraphCmd} { + _ = cmd.RegisterFlagCompletionFunc("project", projectCompleter) + } // Dynamic ValidArgsFunction for workspace subcommands workspaceShowCmd.ValidArgsFunction = func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { diff --git a/cli/completion_test.go b/cli/completion_test.go index c9a21f1..d9dcc81 100644 --- a/cli/completion_test.go +++ b/cli/completion_test.go @@ -133,3 +133,64 @@ func TestCompleteProjectNames_should_return_project_names(t *testing.T) { t.Fatalf("expected frontend and backend, got: %v", names) } } + +func TestRefsCompletion_should_suggest_subcommands(t *testing.T) { + var buf bytes.Buffer + rootCmd.SetOut(&buf) + rootCmd.SetErr(&buf) + rootCmd.SetArgs([]string{"__complete", "refs", ""}) + defer rootCmd.SetOut(nil) + defer rootCmd.SetErr(nil) + + if err := rootCmd.Execute(); err != nil { + t.Fatalf("__complete refs failed: %v", err) + } + + output := buf.String() + for _, sub := range []string{"readers", "writers", "graph"} { + if !strings.Contains(output, sub) { + t.Fatalf("expected completion output to contain %q, got: %s", sub, output) + } + } +} + +func TestRefsProjectCompletion_should_return_workspace_projects(t *testing.T) { + tmpDir := t.TempDir() + oldHome := os.Getenv("HOME") + os.Setenv("HOME", tmpDir) + defer os.Setenv("HOME", oldHome) + + wsCfg := &config.WorkspaceConfig{ + Version: 1, + Workspaces: map[string]config.Workspace{ + "test-ws": { + Name: "test-ws", + Projects: []config.ProjectEntry{ + {Name: "frontend", Path: "/tmp/frontend"}, + {Name: "backend", Path: "/tmp/backend"}, + }, + }, + }, + } + if err := config.SaveWorkspaceConfig(wsCfg); err != nil { + t.Fatalf("failed to save test workspace config: %v", err) + } + + var buf bytes.Buffer + rootCmd.SetOut(&buf) + rootCmd.SetErr(&buf) + rootCmd.SetArgs([]string{"__complete", "refs", "readers", "--workspace", "test-ws", "--project", ""}) + defer rootCmd.SetOut(nil) + defer rootCmd.SetErr(nil) + + if err := rootCmd.Execute(); err != nil { + t.Fatalf("__complete refs readers --project failed: %v", err) + } + + output := buf.String() + for _, project := range []string{"frontend", "backend"} { + if !strings.Contains(output, project) { + t.Fatalf("expected project completion output to contain %q, got: %s", project, output) + } + } +} From 6df0491abcfbdfa6ad4c1ad862c0c197826aff0a Mon Sep 17 00:00:00 2001 From: Mladen Mihajlovic Date: Tue, 17 Mar 2026 09:29:13 +0100 Subject: [PATCH 13/14] Improve Vue fallback warnings and no-ui watch output --- cli/watch.go | 96 ++++++++++++++++++++++++++--- cli/watch_workspace_test.go | 40 ++++++++++++ framework/registry.go | 4 +- framework/scripts/vue_processor.mjs | 36 ++++++++++- framework/vue_processor.go | 60 +++++++++++++++++- framework/vue_processor_test.go | 53 ++++++++++++++++ framework/warnings.go | 21 +++++++ indexer/indexer.go | 4 +- 8 files changed, 298 insertions(+), 16 deletions(-) create mode 100644 framework/warnings.go diff --git a/cli/watch.go b/cli/watch.go index 7cb2038..ccd4717 100644 --- a/cli/watch.go +++ b/cli/watch.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "io" "log" "os" "os/exec" @@ -46,6 +47,13 @@ var ( watchStopDaemonRunner = stopWatchDaemon ) +type watchProgressRenderer struct { + mu sync.Mutex + currentLine string +} + +var watchProgressOutput watchProgressRenderer + var watchCmd = &cobra.Command{ Use: "watch", Short: "Start the real-time file watcher daemon", @@ -766,8 +774,8 @@ func runInitialScan(ctx context.Context, idx *indexer.Indexer, scanner *indexer. } }, ) - // Clear progress line - fmt.Print("\r" + strings.Repeat(" ", 80) + "\r") + watchProgressOutput.clear() + fmt.Println() } else { stats, err = idx.IndexAllWithBatchProgress(ctx, func(info indexer.ProgressInfo) { if onScan != nil { @@ -1864,6 +1872,12 @@ func runWatchForeground() error { } } + var restoreLogs func() + if !isBackgroundChild && watchNoUI { + restoreLogs = captureWatchPlainLogs() + defer restoreLogs() + } + // Initialize shared embedder (reused across all worktrees) emb, err := initializeEmbedder(ctx, cfg) if err != nil { @@ -1997,9 +2011,7 @@ func extractSymbolsWithFramework(ctx context.Context, extractor trace.SymbolExtr if err != nil { return nil, nil, err } - for _, w := range result.Warnings { - log.Printf("Warning: %s", w) - } + framework.LogWarningsOnce(result.Warnings) inputPath := result.VirtualPath if inputPath == "" { @@ -2228,21 +2240,85 @@ func printProgress(current, total int, filePath string) { displayPath = "..." + filePath[len(filePath)-maxPathLen+3:] } - // Print with carriage return to overwrite previous line - fmt.Printf("\rIndexing [%s] %3.0f%% (%d/%d) %s", bar, percent, current, total, displayPath) + watchProgressOutput.render(fmt.Sprintf("Indexing [%s] %3.0f%% (%d/%d) %s", bar, percent, current, total, displayPath)) } func printBatchProgress(info indexer.BatchProgressInfo) { if info.Retrying { - fmt.Printf("\r%s\r", strings.Repeat(" ", 80)) reason := describeRetryReason(info.StatusCode) - fmt.Printf("%s - Retrying batch %d (attempt %d/5)...\n", reason, info.BatchIndex+1, info.Attempt) + watchProgressOutput.println(fmt.Sprintf("%s - Retrying batch %d (attempt %d/5)...", reason, info.BatchIndex+1, info.Attempt)) } else if info.TotalChunks > 0 { percentage := float64(info.CompletedChunks) / float64(info.TotalChunks) * 100 barWidth := 20 filled := int(float64(barWidth) * float64(info.CompletedChunks) / float64(info.TotalChunks)) bar := strings.Repeat("\u2588", filled) + strings.Repeat("\u2591", barWidth-filled) - fmt.Printf("\rEmbedding [%s] %3.0f%% (%d/%d)", bar, percentage, info.CompletedChunks, info.TotalChunks) + watchProgressOutput.render(fmt.Sprintf("Embedding [%s] %3.0f%% (%d/%d)", bar, percentage, info.CompletedChunks, info.TotalChunks)) + } +} + +func (r *watchProgressRenderer) render(line string) { + r.mu.Lock() + defer r.mu.Unlock() + r.currentLine = line + fmt.Printf("\r%s", line) +} + +func (r *watchProgressRenderer) println(line string) { + r.mu.Lock() + defer r.mu.Unlock() + r.clearLocked() + fmt.Println(line) + r.redrawLocked() +} + +func (r *watchProgressRenderer) clear() { + r.mu.Lock() + defer r.mu.Unlock() + r.clearLocked() +} + +func (r *watchProgressRenderer) clearLocked() { + if r.currentLine == "" { + return + } + fmt.Printf("\r%s\r", strings.Repeat(" ", len(r.currentLine))) +} + +func (r *watchProgressRenderer) redrawLocked() { + if r.currentLine == "" { + return + } + fmt.Printf("\r%s", r.currentLine) +} + +type watchPlainLogForwarder struct { + writer io.Writer +} + +func (f *watchPlainLogForwarder) Write(p []byte) (int, error) { + // Foreground plain mode keeps progress on a single line, so log writes need + // to temporarily clear and then redraw that line. + watchProgressOutput.mu.Lock() + defer watchProgressOutput.mu.Unlock() + watchProgressOutput.clearLocked() + n, err := f.writer.Write(p) + watchProgressOutput.redrawLocked() + return n, err +} + +func captureWatchPlainLogs() func() { + oldWriter := log.Writer() + oldFlags := log.Flags() + oldPrefix := log.Prefix() + + log.SetOutput(&watchPlainLogForwarder{writer: oldWriter}) + log.SetFlags(oldFlags) + log.SetPrefix(oldPrefix) + + return func() { + log.SetOutput(oldWriter) + log.SetFlags(oldFlags) + log.SetPrefix(oldPrefix) } } diff --git a/cli/watch_workspace_test.go b/cli/watch_workspace_test.go index 215ba79..d8f0f6e 100644 --- a/cli/watch_workspace_test.go +++ b/cli/watch_workspace_test.go @@ -1,7 +1,9 @@ package cli import ( + "bytes" "context" + "io" "os" "path/filepath" "strconv" @@ -277,3 +279,41 @@ func TestPrintProgressAndBatchProgress(t *testing.T) { CompletedChunks: 5, }) } + +func TestPrintProgressAndBatchProgress_NoUIModeUsesNewlines(t *testing.T) { + oldNoUI := watchNoUI + watchNoUI = true + defer func() { + watchNoUI = oldNoUI + }() + + oldStdout := os.Stdout + r, w, err := os.Pipe() + if err != nil { + t.Fatalf("Pipe() failed: %v", err) + } + os.Stdout = w + + printProgress(1, 2, filepath.Join("very", "long", "path", "to", "file.go")) + printBatchProgress(indexer.BatchProgressInfo{ + Retrying: false, + TotalChunks: 10, + CompletedChunks: 5, + }) + + _ = w.Close() + os.Stdout = oldStdout + + var buf bytes.Buffer + if _, err := io.Copy(&buf, r); err != nil { + t.Fatalf("io.Copy() failed: %v", err) + } + + out := buf.String() + if !strings.Contains(out, "\r") { + t.Fatalf("expected carriage returns in no-ui mode, got %q", out) + } + if !strings.Contains(out, "Indexing [") || !strings.Contains(out, "Embedding [") { + t.Fatalf("expected inline progress output, got %q", out) + } +} diff --git a/framework/registry.go b/framework/registry.go index f32ddd3..be6f9bb 100644 --- a/framework/registry.go +++ b/framework/registry.go @@ -73,7 +73,9 @@ func (r *ProcessorRegistry) transform(ctx context.Context, filePath, source stri return TransformResult{}, fmt.Errorf("%s processor failed for %s: %w", p.Name(), filePath, err) } if res.Text != "" { - res.Warnings = append(res.Warnings, fmt.Sprintf("%s processor fallback: %v", p.Name(), err)) + if len(res.Warnings) == 0 { + res.Warnings = append(res.Warnings, fmt.Sprintf("%s processor fallback: %v", p.Name(), err)) + } return res, nil } out := passthrough(filePath, source) diff --git a/framework/scripts/vue_processor.mjs b/framework/scripts/vue_processor.mjs index d3cd9ec..842a783 100644 --- a/framework/scripts/vue_processor.mjs +++ b/framework/scripts/vue_processor.mjs @@ -1,4 +1,6 @@ -import { parse, compileTemplate } from "@vue/compiler-sfc"; +import path from "node:path"; +import { createRequire } from "node:module"; +import { pathToFileURL } from "node:url"; const chunks = []; for await (const chunk of process.stdin) { @@ -8,6 +10,37 @@ for await (const chunk of process.stdin) { const input = JSON.parse(Buffer.concat(chunks).toString("utf8")); const source = input.source ?? ""; const filePath = input.filePath ?? "Component.vue"; +const absoluteFilePath = path.isAbsolute(filePath) ? filePath : path.resolve(process.cwd(), filePath); + +function candidateResolveDirs(startFilePath) { + const dirs = []; + let dir = path.dirname(startFilePath); + while (true) { + dirs.push(dir); + const parent = path.dirname(dir); + if (parent === dir) break; + dir = parent; + } + if (!dirs.includes(process.cwd())) { + dirs.push(process.cwd()); + } + return dirs; +} + +async function loadVueCompiler(startFilePath) { + for (const dir of candidateResolveDirs(startFilePath)) { + const req = createRequire(path.join(dir, "__grepai_vue_processor__.cjs")); + try { + const resolved = req.resolve("@vue/compiler-sfc"); + return import(pathToFileURL(resolved).href); + } catch (err) { + if (err?.code !== "MODULE_NOT_FOUND") { + throw err; + } + } + } + throw new Error(`Cannot resolve @vue/compiler-sfc from ${path.dirname(startFilePath)}`); +} function countLines(text) { if (text.length === 0) return 1; @@ -94,6 +127,7 @@ function appendStyleBindings(out, map, refs, index) { } try { + const { parse, compileTemplate } = await loadVueCompiler(absoluteFilePath); const parsed = parse(source, { filename: filePath }); if (parsed.errors?.length) { const msg = String(parsed.errors[0]); diff --git a/framework/vue_processor.go b/framework/vue_processor.go index 9cecb82..3dfa7cb 100644 --- a/framework/vue_processor.go +++ b/framework/vue_processor.go @@ -10,6 +10,7 @@ import ( "regexp" "sort" "strings" + "sync" ) //go:embed scripts/vue_processor.mjs @@ -17,6 +18,8 @@ var vueProcessorScript string type VueProcessor struct { nodePath string + mu sync.RWMutex + missing string } func NewVueProcessor(nodePath string) *VueProcessor { @@ -56,6 +59,10 @@ type vueScriptOutput struct { } func (p *VueProcessor) transform(ctx context.Context, filePath, source string) (TransformResult, error) { + if msg, ok := p.missingCompilerMessage(); ok { + return p.fallbackWithCompilerWarning(filePath, source, msg) + } + in, _ := json.Marshal(vueScriptInput{FilePath: filePath, Source: source}) cmd := exec.CommandContext(ctx, p.nodePath, "--input-type=module", "-e", vueProcessorScript) //nolint:gosec // nodePath is an executable path from trusted config; no shell is invoked cmd.Stdin = bytes.NewReader(in) @@ -65,10 +72,15 @@ func (p *VueProcessor) transform(ctx context.Context, filePath, source string) ( cmd.Stderr = &stderr if err := cmd.Run(); err != nil { + msg := strings.TrimSpace(stderr.String()) + if isMissingVueCompiler(msg) { + msg = normalizeMissingVueCompilerMessage(filePath) + p.setMissingCompilerMessage(msg) + } fallback, fbErr := p.fallback(filePath, source) if fbErr == nil { fallback.Warnings = append(fallback.Warnings, - fmt.Sprintf("vue compiler unavailable: %s", strings.TrimSpace(stderr.String()))) + fmt.Sprintf("vue compiler unavailable: %s", msg)) return fallback, fmt.Errorf("%w: %v", ErrUnavailable, err) } return TransformResult{}, fmt.Errorf("%w: vue processor failed: %v (%s)", ErrUnavailable, err, strings.TrimSpace(stderr.String())) @@ -98,6 +110,52 @@ func (p *VueProcessor) transform(ctx context.Context, filePath, source string) ( }, nil } +func (p *VueProcessor) fallbackWithCompilerWarning(filePath, source, msg string) (TransformResult, error) { + fallback, err := p.fallback(filePath, source) + if err != nil { + out := passthrough(filePath, source) + out.Processor = p.Name() + out.Warnings = append(out.Warnings, fmt.Sprintf("vue compiler unavailable: %s", msg)) + return out, fmt.Errorf("%w: vue compiler unavailable", ErrUnavailable) + } + fallback.Warnings = append(fallback.Warnings, fmt.Sprintf("vue compiler unavailable: %s", msg)) + return fallback, fmt.Errorf("%w: vue compiler unavailable", ErrUnavailable) +} + +func isMissingVueCompiler(msg string) bool { + return strings.Contains(msg, "@vue/compiler-sfc") && + (strings.Contains(msg, "Cannot find package") || strings.Contains(msg, "Cannot resolve @vue/compiler-sfc")) +} + +func normalizeMissingVueCompilerMessage(filePath string) string { + return fmt.Sprintf("Cannot resolve @vue/compiler-sfc from %s; install it in the Vue workspace to enable full SFC compilation", pathDir(filePath)) +} + +func pathDir(filePath string) string { + lastSlash := strings.LastIndexAny(filePath, `/\`) + if lastSlash < 0 { + return "." + } + if lastSlash == 0 { + return filePath[:1] + } + return filePath[:lastSlash] +} + +func (p *VueProcessor) missingCompilerMessage() (string, bool) { + p.mu.RLock() + defer p.mu.RUnlock() + return p.missing, p.missing != "" +} + +func (p *VueProcessor) setMissingCompilerMessage(msg string) { + p.mu.Lock() + defer p.mu.Unlock() + if p.missing == "" { + p.missing = msg + } +} + var scriptBlockRE = regexp.MustCompile(`(?is)]*>(.*?)`) var styleBlockRE = regexp.MustCompile(`(?is)]*>(.*?)`) var vueTemplateCtxIdentRE = regexp.MustCompile(`\b_ctx\.([A-Za-z_$][A-Za-z0-9_$]*)\b`) diff --git a/framework/vue_processor_test.go b/framework/vue_processor_test.go index 0192e71..02c6b9e 100644 --- a/framework/vue_processor_test.go +++ b/framework/vue_processor_test.go @@ -1,6 +1,9 @@ package framework import ( + "context" + "os" + "path/filepath" "strings" "testing" ) @@ -76,3 +79,53 @@ func TestAppendTemplateCtxReadCalls(t *testing.T) { t.Fatalf("expected extended line map, got %d <= %d", len(outMap), len(mapping)) } } + +func TestVueProcessorTransform_ResolvesCompilerFromFileHierarchy(t *testing.T) { + tmpDir := t.TempDir() + projectDir := filepath.Join(tmpDir, "project") + nodeModuleDir := filepath.Join(projectDir, "node_modules", "@vue", "compiler-sfc") + componentPath := filepath.Join(projectDir, "src", "Comp.vue") + + if err := os.MkdirAll(nodeModuleDir, 0o755); err != nil { + t.Fatalf("mkdir node module dir: %v", err) + } + if err := os.MkdirAll(filepath.Dir(componentPath), 0o755); err != nil { + t.Fatalf("mkdir component dir: %v", err) + } + + packageJSON := `{"name":"@vue/compiler-sfc","type":"module","exports":"./index.mjs"}` + if err := os.WriteFile(filepath.Join(nodeModuleDir, "package.json"), []byte(packageJSON), 0o644); err != nil { + t.Fatalf("write package.json: %v", err) + } + + moduleSource := `export function parse() { + return { + descriptor: { + script: { + content: "export const resolved = 1", + loc: { start: { line: 1 } } + }, + scriptSetup: null, + template: null, + styles: [] + }, + errors: [] + }; +} + +export function compileTemplate() { + return { code: "", errors: [] }; +}` + if err := os.WriteFile(filepath.Join(nodeModuleDir, "index.mjs"), []byte(moduleSource), 0o644); err != nil { + t.Fatalf("write compiler module: %v", err) + } + + p := NewVueProcessor("node") + res, err := p.TransformForEmbedding(context.Background(), componentPath, "") + if err != nil { + t.Fatalf("transform failed: %v", err) + } + if !strings.Contains(res.Text, "export const resolved = 1") { + t.Fatalf("expected resolved compiler output, got %q", res.Text) + } +} diff --git a/framework/warnings.go b/framework/warnings.go new file mode 100644 index 0000000..ee6ae95 --- /dev/null +++ b/framework/warnings.go @@ -0,0 +1,21 @@ +package framework + +import ( + "log" + "sync" +) + +var loggedFrameworkWarnings sync.Map + +// LogWarningsOnce suppresses identical framework warnings after the first log. +func LogWarningsOnce(warnings []string) { + for _, warning := range warnings { + if warning == "" { + continue + } + if _, loaded := loggedFrameworkWarnings.LoadOrStore(warning, struct{}{}); loaded { + continue + } + log.Printf("Warning: %s", warning) + } +} diff --git a/indexer/indexer.go b/indexer/indexer.go index 34c4391..cd3bf22 100644 --- a/indexer/indexer.go +++ b/indexer/indexer.go @@ -662,9 +662,7 @@ func (idx *Indexer) embeddingContent(ctx context.Context, file FileInfo) (string log.Printf("Warning: framework embedding transform failed for %s: %v", file.Path, err) return file.Content, nil } - for _, w := range res.Warnings { - log.Printf("Warning: %s", w) - } + framework.LogWarningsOnce(res.Warnings) if res.Text == "" { return file.Content, nil } From 4321ed6dc20ef8d2254383b07191192ebd081305 Mon Sep 17 00:00:00 2001 From: Mladen Mihajlovic Date: Thu, 26 Mar 2026 12:26:16 +0100 Subject: [PATCH 14/14] fix(config): format config test imports --- config/config_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/config_test.go b/config/config_test.go index ac07c5f..899072d 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -3,8 +3,8 @@ package config import ( "os" "path/filepath" - "strings" "runtime" + "strings" "testing" )