From d65d19bae618bd7600bea3af554a0b61293b58d6 Mon Sep 17 00:00:00 2001 From: Etan Joseph Heyman Date: Sun, 1 Mar 2026 04:07:06 +0200 Subject: [PATCH] nightshift: add by_persian index to words table, fix full table scan MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The getWordDetailsWithSongs query was loading ALL words from the database and filtering in memory. As the words table grows, this becomes a major performance bottleneck — called every time a user opens the word details modal. Now uses a by_persian index for O(log n) lookup. Co-Authored-By: Claude Opus 4.6 --- convex/schema.ts | 4 ++- convex/wordProgress.ts | 73 +++++++++++++++++------------------------- 2 files changed, 32 insertions(+), 45 deletions(-) diff --git a/convex/schema.ts b/convex/schema.ts index 7536f57..8f71fd3 100644 --- a/convex/schema.ts +++ b/convex/schema.ts @@ -48,7 +48,9 @@ export default defineSchema({ english: v.string(), grammarType: v.optional(v.string()), // noun, verb, preposition, adjective, etc. forvoAudioUrl: v.optional(v.string()), // Future: audio from Forvo or ElevenLabs - }).index("by_song_line", ["songId", "lineNumber"]), + }) + .index("by_song_line", ["songId", "lineNumber"]) + .index("by_persian", ["persian"]), // Track user's learning progress for individual words // Learning state is keyed by persian text so repeated words (e.g., "برای") sync across all instances diff --git a/convex/wordProgress.ts b/convex/wordProgress.ts index 64ccdef..cb6b199 100644 --- a/convex/wordProgress.ts +++ b/convex/wordProgress.ts @@ -8,7 +8,7 @@ export const getByUser = query({ handler: async (ctx) => { const userId = await getAuthUserId(ctx); if (!userId) return []; - + return await ctx.db .query("wordProgress") .withIndex("by_user", (q) => q.eq("userId", userId)) @@ -25,9 +25,7 @@ export const getByUserWord = query({ return await ctx.db .query("wordProgress") - .withIndex("by_user", (q) => - q.eq("userId", userId) - ) + .withIndex("by_user", (q) => q.eq("userId", userId)) .filter((q) => q.eq(q.field("wordId"), args.wordId)) .first(); }, @@ -42,9 +40,7 @@ export const getByUserPersian = query({ return await ctx.db .query("wordProgress") - .withIndex("by_user", (q) => - q.eq("userId", userId) - ) + .withIndex("by_user", (q) => q.eq("userId", userId)) .filter((q) => q.eq(q.field("persian"), args.persian)) .first(); }, @@ -64,13 +60,11 @@ export const getByUserPersians = query({ uniquePersians.map(async (persian) => { const progress = await ctx.db .query("wordProgress") - .withIndex("by_user", (q) => - q.eq("userId", userId) - ) + .withIndex("by_user", (q) => q.eq("userId", userId)) .filter((q) => q.eq(q.field("persian"), persian)) .first(); return { persian, progress }; - }) + }), ); return results; }, @@ -87,13 +81,11 @@ export const getByUserWords = query({ args.wordIds.map(async (wordId) => { const progress = await ctx.db .query("wordProgress") - .withIndex("by_user", (q) => - q.eq("userId", userId) - ) + .withIndex("by_user", (q) => q.eq("userId", userId)) .filter((q) => q.eq(q.field("wordId"), wordId)) .first(); return { wordId, progress }; - }) + }), ); return results; }, @@ -109,9 +101,7 @@ export const incrementViewCount = mutation({ // Look up by persian text first - this is the canonical key const existingPersian = await ctx.db .query("wordProgress") - .withIndex("by_user", (q) => - q.eq("userId", userId) - ) + .withIndex("by_user", (q) => q.eq("userId", userId)) .filter((q) => q.eq(q.field("persian"), args.persian)) .first(); @@ -148,9 +138,7 @@ export const incrementPlayCount = mutation({ // Look up by persian text first - this is the canonical key const existingPersian = await ctx.db .query("wordProgress") - .withIndex("by_user", (q) => - q.eq("userId", userId) - ) + .withIndex("by_user", (q) => q.eq("userId", userId)) .filter((q) => q.eq(q.field("persian"), args.persian)) .first(); @@ -186,27 +174,26 @@ export const toggleLearned = mutation({ // Find all progress records for this persian word const allMatching = await ctx.db .query("wordProgress") - .withIndex("by_user", (q) => - q.eq("userId", userId) - ) + .withIndex("by_user", (q) => q.eq("userId", userId)) .filter((q) => q.eq(q.field("persian"), args.persian)) .collect(); // Determine the new learned state (toggle from current) - const currentLearned = allMatching.length > 0 ? allMatching[0].learned : false; + const currentLearned = + allMatching.length > 0 ? allMatching[0].learned : false; const newLearned = !currentLearned; // Update ALL matching records // Only update lastSeen when marking as learned, not when unmarking - const updateData: { learned: boolean; lastSeen?: number } = { learned: newLearned }; + const updateData: { learned: boolean; lastSeen?: number } = { + learned: newLearned, + }; if (newLearned) { updateData.lastSeen = Date.now(); } await Promise.all( - allMatching.map((record) => - ctx.db.patch(record._id, updateData) - ) + allMatching.map((record) => ctx.db.patch(record._id, updateData)), ); // If no records exist yet, create one for this specific word instance @@ -237,23 +224,21 @@ export const setLearned = mutation({ // Find all progress records for this persian word const allMatching = await ctx.db .query("wordProgress") - .withIndex("by_user", (q) => - q.eq("userId", userId) - ) + .withIndex("by_user", (q) => q.eq("userId", userId)) .filter((q) => q.eq(q.field("persian"), args.persian)) .collect(); // Update ALL matching records // Only update lastSeen when marking as learned, not when unmarking - const updateData: { learned: boolean; lastSeen?: number } = { learned: args.learned }; + const updateData: { learned: boolean; lastSeen?: number } = { + learned: args.learned, + }; if (args.learned) { updateData.lastSeen = Date.now(); } await Promise.all( - allMatching.map((record) => - ctx.db.patch(record._id, updateData) - ) + allMatching.map((record) => ctx.db.patch(record._id, updateData)), ); // If no records exist yet, create one for this specific word instance @@ -428,7 +413,7 @@ export const getVocabularyByLanguage = query({ masteryLevel, sourceLanguage: song.sourceLanguage, }; - }) + }), ); // Filter out nulls and dedupe by persian text (keep highest practice count) @@ -485,8 +470,10 @@ export const getWordDetailsWithSongs = query({ args: { persian: v.string() }, handler: async (ctx, args) => { // Find all word instances with this persian text across all songs - const allWords = await ctx.db.query("words").collect(); - const matchingWords = allWords.filter((w) => w.persian === args.persian); + const matchingWords = await ctx.db + .query("words") + .withIndex("by_persian", (q) => q.eq("persian", args.persian)) + .collect(); if (matchingWords.length === 0) { return null; @@ -512,7 +499,7 @@ export const getWordDetailsWithSongs = query({ const line = await ctx.db .query("lyrics") .withIndex("by_song", (q) => - q.eq("songId", songId).eq("lineNumber", lineNumber) + q.eq("songId", songId).eq("lineNumber", lineNumber), ) .first(); @@ -524,7 +511,7 @@ export const getWordDetailsWithSongs = query({ lineNumber, linePreview: line?.original?.slice(0, 50) || "", }; - }) + }), ); const validSongs = songsWithContext.filter(Boolean) as NonNullable< @@ -562,9 +549,7 @@ export const getWordPracticeHistory = query({ // Get the progress record for this word const progress = await ctx.db .query("wordProgress") - .withIndex("by_user", (q) => - q.eq("userId", userId) - ) + .withIndex("by_user", (q) => q.eq("userId", userId)) .filter((q) => q.eq(q.field("persian"), args.persian)) .first();