Fix RAG zero-recall with HNSW index and add empty-state guidance

ai-pdf-chatbot/app/api/chat/route.ts

@@ -119,10 +119,72 @@ export async function POST(req: Request) {
   const stream = new ReadableStream<Uint8Array>({
     async start(controller) {
       const send = (obj: unknown) => controller.enqueue(encodeNdjson(obj));
+      // Streams a canned assistant message (no LLM call), persists it,
+      // and closes the stream. Used for the guidance paths below where
+      // retrieval has nothing to offer and an LLM round-trip would only
+      // produce a less helpful refusal.
+      const finishWithGuidance = async (text: string) => {
+        send({ type: 'citations', data: [] });
+        send({ type: 'delta', text });
+        await client.database.from('chat_messages').insert({
+          chat_id: resolvedChatId,
+          role: 'assistant',
+          content: text,
+          sort_order: nextOrder + 1,
+          citations: [],
+        });
+        send({ type: 'done' });
+        controller.close();
+      };
+
       try {
+        send({ type: 'chat', chatId: resolvedChatId });
+
+        // Look up the ready documents in scope first. If there are none,
+        // the user is asking into a void: guide them to upload instead of
+        // letting the model say "I couldn't find that".
+        let docsQuery = client.database
+          .from('documents')
+          .select('file_name, suggested_questions')
+          .eq('status', 'ready');
+        if (workspaceId) docsQuery = docsQuery.eq('workspace_id', workspaceId);
+        const docsRes = await docsQuery;
+        const readyDocs = (docsRes.data ?? []) as Array<{
+          file_name: string;
+          suggested_questions: string[] | null;
+        }>;
+
+        if (readyDocs.length === 0) {
+          await finishWithGuidance(
+            workspaceId
+              ? "This workspace doesn't have any ready PDFs yet. Open the workspace's Documents tab, upload a PDF (up to 10 MB), and ask me again once it finishes indexing."
+              : "You haven't uploaded any PDFs yet. Head to the Documents page, upload a PDF (up to 10 MB), and ask me again once it finishes indexing.",
+          );
+          return;
+        }
+
         const chunks = await retrieveForQuestion(client, ownerId, inputText, docIds, workspaceId);
+
+        if (chunks.length === 0) {
+          // Vector search came back empty (rare). Tell the user what IS
+          // available and reuse the cached per-document suggested
+          // questions as concrete next steps.
+          const names = readyDocs
+            .slice(0, 5)
+            .map((d) => d.file_name)
+            .join(', ');
+          const suggestions = readyDocs
+            .flatMap((d) => (Array.isArray(d.suggested_questions) ? d.suggested_questions : []))
+            .slice(0, 3);
+          let guidance = `I couldn't find anything relevant to that in your documents (${names}).`;
+          if (suggestions.length > 0) {
+            guidance += ` Try asking something like: ${suggestions.map((q) => `"${q}"`).join(' or ')}`;
+          }
+          await finishWithGuidance(guidance);
+          return;
+        }
+
         const citations = toCitations(chunks);
-        send({ type: 'chat', chatId: resolvedChatId });
         send({ type: 'citations', data: citations });
 
         const contextString = chunks

ai-pdf-chatbot/app/chat/page.tsx

@@ -41,6 +41,9 @@ function ChatHomePageInner() {
   const [pendingInput, setPendingInput] = useState<string | null>(null);
   const [citations] = useState<Citation[]>([]);
   const [suggestions, setSuggestions] = useState<string[]>([]);
+  // null = still loading. NotebookLM-style gating: with no ready docs the
+  // chat input is disabled and the empty state points at upload instead.
+  const [hasReadyDocs, setHasReadyDocs] = useState<boolean | null>(null);
   const setRailCitations = useSetRailCitations();
 
   // Pull a few suggested questions from the most recently uploaded, ready
@@ -54,8 +57,10 @@ function ChatHomePageInner() {
       if (!res.ok) return;
       const data = (await res.json()) as { documents: DocSummaryRow[] };
       if (cancelled) return;
-      const newest = data.documents?.find(
-        (d) => d.status === 'ready' && Array.isArray(d.suggested_questions) && d.suggested_questions.length > 0,
+      const ready = (data.documents ?? []).filter((d) => d.status === 'ready');
+      setHasReadyDocs(ready.length > 0);
+      const newest = ready.find(
+        (d) => Array.isArray(d.suggested_questions) && d.suggested_questions.length > 0,
       );
       setSuggestions(newest?.suggested_questions ?? []);
     })();
@@ -96,9 +101,16 @@ function ChatHomePageInner() {
         ) : (
           <div className="mx-auto flex max-w-xl flex-col items-center justify-center gap-6 py-12 text-center">
             <h1 className="font-display text-4xl">Chat with your PDFs</h1>
-            <p className="text-sm text-muted-foreground">
-              Upload a PDF on the <a className="underline" href="/documents">documents page</a>, then ask a question here.
-            </p>
+            {hasReadyDocs === false ? (
+              <p className="text-sm text-muted-foreground">
+                No PDFs yet. <a className="font-medium underline" href="/documents">Upload one on the documents page</a> to
+                start asking questions; chat unlocks as soon as a document finishes indexing.
+              </p>
+            ) : (
+              <p className="text-sm text-muted-foreground">
+                Upload a PDF on the <a className="underline" href="/documents">documents page</a>, then ask a question here.
+              </p>
+            )}
 
             {suggestions.length > 0 ? (
               <div className="w-full space-y-3">
@@ -123,7 +135,10 @@ function ChatHomePageInner() {
           </div>
         )}
       </div>
-      <ChatInput disabled={state.phase === 'streaming'} onSubmit={handleSubmit} />
+      <ChatInput
+        disabled={state.phase === 'streaming' || hasReadyDocs === false}
+        onSubmit={handleSubmit}
+      />
     </>
   );
 }

ai-pdf-chatbot/lib/ai/prompts.ts

@@ -1,10 +1,11 @@
 export const RAG_SYSTEM_PROMPT = `You are a helpful assistant answering questions about the user's uploaded PDFs.
 
 Rules:
-- Use only the sources provided below. If the answer is not in them, say "I couldn't find that in the provided documents."
+- Use only the sources provided below.
 - When you cite a source, use the bracket marker [n] that matches the numbered source. Cite at the END of each relevant sentence or claim.
 - Keep answers concise. Prefer 2-5 short paragraphs over walls of text.
 - Do not fabricate page numbers, file names, or quotes. Only repeat content verbatim if you wrap it in quotes and cite it.
+- If the sources don't answer the question, or the question is a greeting / test input / too vague to answer ("hi", "test", "what"), do NOT just refuse. Say in one sentence that you couldn't find it, then tell the user in one sentence what the provided sources DO cover, and suggest one concrete question they could ask instead.
 `;
 
 export function buildUserPrompt(

ai-pdf-chatbot/migrations/db_init.sql

@@ -101,10 +101,15 @@ create table if not exists public.document_chunks (
   unique (document_id, chunk_index)
 );
 
-create index if not exists document_chunks_embedding_idx
+-- HNSW, not ivfflat: ivfflat with lists=100 partitions vectors into 100
+-- clusters and probes 1 by default, which collapses recall to near zero
+-- when a user only has a handful of chunks (typical here: one PDF, 6-300
+-- chunks per user). HNSW has no cluster count to mistune and holds 95%+
+-- recall at any collection size.
+drop index if exists document_chunks_embedding_idx;
+create index if not exists document_chunks_embedding_hnsw_idx
   on public.document_chunks
-  using ivfflat (embedding vector_cosine_ops)
-  with (lists = 100);
+  using hnsw (embedding vector_cosine_ops);
 
 create index if not exists document_chunks_user_idx
   on public.document_chunks (user_id);