Fix RAG zero-recall with HNSW index and add empty-state guidance

ai-pdf-chatbot/app/api/chat/route.ts

@@ -119,10 +119,80 @@ export async function POST(req: Request) {
   const stream = new ReadableStream<Uint8Array>({
     async start(controller) {
       const send = (obj: unknown) => controller.enqueue(encodeNdjson(obj));
+      // Streams a canned assistant message (no LLM call), persists it,
+      // and closes the stream. Used for the guidance paths below where
+      // retrieval has nothing to offer and an LLM round-trip would only
+      // produce a less helpful refusal.
+      const finishWithGuidance = async (text: string) => {
+        send({ type: 'citations', data: [] });
+        send({ type: 'delta', text });
+        await client.database.from('chat_messages').insert({
+          chat_id: resolvedChatId,
+          role: 'assistant',
+          content: text,
+          sort_order: nextOrder + 1,
+          citations: [],
+        });
+        send({ type: 'done' });
+        controller.close();
+      };
+
       try {
+        send({ type: 'chat', chatId: resolvedChatId });
+
+        // Look up the ready documents in scope first. If there are none,
+        // the user is asking into a void: guide them to upload instead of
+        // letting the model say "I couldn't find that".
+        let docsQuery = client.database
+          .from('documents')
+          .select('file_name, suggested_questions')
+          .eq('status', 'ready');
+        if (workspaceId) docsQuery = docsQuery.eq('workspace_id', workspaceId);
+        const docsRes = await docsQuery;
+        if (docsRes.error) {
+          // A failed lookup must not masquerade as "no documents" — that
+          // would tell a user with perfectly good PDFs to go upload again.
+          // Log the real error server-side; the stream error payload goes
+          // to the client, so keep it generic.
+          console.error('[chat] documents lookup failed:', docsRes.error.message);
+          throw new Error('Failed to load documents');
+        }
+        const readyDocs = (docsRes.data ?? []) as Array<{
+          file_name: string;
+          suggested_questions: string[] | null;
+        }>;
+
+        if (readyDocs.length === 0) {
+          await finishWithGuidance(
+            workspaceId
+              ? "This workspace doesn't have any ready PDFs yet. Open the workspace's Documents tab, upload a PDF (up to 10 MB), and ask me again once it finishes indexing."
+              : "You haven't uploaded any PDFs yet. Head to the Documents page, upload a PDF (up to 10 MB), and ask me again once it finishes indexing.",
+          );
+          return;
+        }
+
         const chunks = await retrieveForQuestion(client, ownerId, inputText, docIds, workspaceId);
+
+        if (chunks.length === 0) {
+          // Vector search came back empty (rare). Tell the user what IS
+          // available and reuse the cached per-document suggested
+          // questions as concrete next steps.
+          const names = readyDocs
+            .slice(0, 5)
+            .map((d) => d.file_name)
+            .join(', ');
+          const suggestions = readyDocs
+            .flatMap((d) => (Array.isArray(d.suggested_questions) ? d.suggested_questions : []))
+            .slice(0, 3);
+          let guidance = `I couldn't find anything relevant to that in your documents (${names}).`;
+          if (suggestions.length > 0) {
+            guidance += ` Try asking something like: ${suggestions.map((q) => `"${q}"`).join(' or ')}`;
+          }
+          await finishWithGuidance(guidance);
+          return;
+        }
+
         const citations = toCitations(chunks);
-        send({ type: 'chat', chatId: resolvedChatId });
         send({ type: 'citations', data: citations });
 
         const contextString = chunks

ai-pdf-chatbot/app/chat/page.tsx

@@ -41,6 +41,9 @@ function ChatHomePageInner() {
   const [pendingInput, setPendingInput] = useState<string | null>(null);
   const [citations] = useState<Citation[]>([]);
   const [suggestions, setSuggestions] = useState<string[]>([]);
+  // null = still loading. NotebookLM-style gating: with no ready docs the
+  // chat input is disabled and the empty state points at upload instead.
+  const [hasReadyDocs, setHasReadyDocs] = useState<boolean | null>(null);
   const setRailCitations = useSetRailCitations();
 
   // Pull a few suggested questions from the most recently uploaded, ready
@@ -54,8 +57,10 @@ function ChatHomePageInner() {
       if (!res.ok) return;
       const data = (await res.json()) as { documents: DocSummaryRow[] };
       if (cancelled) return;
-      const newest = data.documents?.find(
-        (d) => d.status === 'ready' && Array.isArray(d.suggested_questions) && d.suggested_questions.length > 0,
+      const ready = (data.documents ?? []).filter((d) => d.status === 'ready');
+      setHasReadyDocs(ready.length > 0);
+      const newest = ready.find(
+        (d) => Array.isArray(d.suggested_questions) && d.suggested_questions.length > 0,
       );
       setSuggestions(newest?.suggested_questions ?? []);
     })();
@@ -96,9 +101,16 @@ function ChatHomePageInner() {
         ) : (
           <div className="mx-auto flex max-w-xl flex-col items-center justify-center gap-6 py-12 text-center">
             <h1 className="font-display text-4xl">Chat with your PDFs</h1>
-            <p className="text-sm text-muted-foreground">
-              Upload a PDF on the <a className="underline" href="/documents">documents page</a>, then ask a question here.
-            </p>
+            {hasReadyDocs === false ? (
+              <p className="text-sm text-muted-foreground">
+                No PDFs yet. <a className="font-medium underline" href="/documents">Upload one on the documents page</a> to
+                start asking questions; chat unlocks as soon as a document finishes indexing.
+              </p>
+            ) : (
+              <p className="text-sm text-muted-foreground">
+                Upload a PDF on the <a className="underline" href="/documents">documents page</a>, then ask a question here.
+              </p>
+            )}
 
             {suggestions.length > 0 ? (
               <div className="w-full space-y-3">
@@ -123,7 +135,10 @@ function ChatHomePageInner() {
           </div>
         )}
       </div>
-      <ChatInput disabled={state.phase === 'streaming'} onSubmit={handleSubmit} />
+      <ChatInput
+        disabled={state.phase === 'streaming' || hasReadyDocs === false}
+        onSubmit={handleSubmit}
+      />
     </>
   );
 }

ai-pdf-chatbot/app/layout.tsx

@@ -2,9 +2,56 @@ import type { Metadata } from 'next';
 import { Toaster } from 'sonner';
 import './globals.css';
 
+// One env var drives every absolute URL in the metadata (canonical, OG
+// image, sitemap). Template users get correct tags on their own domain
+// automatically because NEXT_PUBLIC_BETTER_AUTH_URL is already required
+// for auth to work.
+const SITE_URL = process.env.NEXT_PUBLIC_BETTER_AUTH_URL ?? 'http://localhost:3000';
+
+const TITLE = 'AI Notebook — chat, mindmap & podcast your PDFs';
+const DESCRIPTION =
+  'An open-source thinking partner for your PDFs: RAG chat with inline citations, auto-generated mindmaps, spaced-repetition flashcards, and podcast-style audio overviews. Self-hosted.';
+
 export const metadata: Metadata = {
-  title: 'AI PDF Chatbot',
-  description: 'Upload PDFs and chat with them. Powered by InsForge.',
+  metadataBase: new URL(SITE_URL),
+  title: {
+    default: TITLE,
+    template: '%s · AI Notebook',
+  },
+  description: DESCRIPTION,
+  alternates: {
+    canonical: '/',
+  },
+  openGraph: {
+    type: 'website',
+    siteName: 'AI Notebook',
+    title: TITLE,
+    description: DESCRIPTION,
+    url: '/',
+  },
+  twitter: {
+    card: 'summary_large_image',
+    title: TITLE,
+    description: DESCRIPTION,
+  },
+};
+
+// schema.org SoftwareApplication: one of the strongest signals AI
+// engines use to understand and cite a product.
+const JSON_LD = {
+  '@context': 'https://schema.org',
+  '@type': 'SoftwareApplication',
+  name: 'AI Notebook',
+  applicationCategory: 'EducationalApplication',
+  operatingSystem: 'Web',
+  url: SITE_URL,
+  description: DESCRIPTION,
+  offers: {
+    '@type': 'Offer',
+    price: '0',
+    priceCurrency: 'USD',
+  },
+  sameAs: ['https://github.com/InsForge/insforge-templates'],
 };
 
 export default function RootLayout({
@@ -15,6 +62,11 @@ export default function RootLayout({
   return (
     <html lang="en" suppressHydrationWarning>
       <body>
+        <script
+          type="application/ld+json"
+          // eslint-disable-next-line react/no-danger
+          dangerouslySetInnerHTML={{ __html: JSON.stringify(JSON_LD) }}
+        />
         {children}
         <Toaster position="top-center" richColors />
       </body>

ai-pdf-chatbot/app/robots.ts

@@ -0,0 +1,18 @@
+import type { MetadataRoute } from 'next';
+
+const SITE_URL = process.env.NEXT_PUBLIC_BETTER_AUTH_URL ?? 'http://localhost:3000';
+
+export default function robots(): MetadataRoute.Robots {
+  return {
+    rules: [
+      {
+        userAgent: '*',
+        allow: '/',
+        // API routes carry no indexable content; share links are
+        // unlisted-by-design and shouldn't end up in search results.
+        disallow: ['/api/', '/share/'],
+      },
+    ],
+    sitemap: `${SITE_URL}/sitemap.xml`,
+  };
+}

ai-pdf-chatbot/app/sitemap.ts

@@ -0,0 +1,13 @@
+import type { MetadataRoute } from 'next';
+
+const SITE_URL = process.env.NEXT_PUBLIC_BETTER_AUTH_URL ?? 'http://localhost:3000';
+
+// Everything behind auth is invisible to crawlers anyway; the public
+// surface is the landing redirect and the two auth pages.
+export default function sitemap(): MetadataRoute.Sitemap {
+  return [
+    { url: `${SITE_URL}/`, changeFrequency: 'weekly', priority: 1 },
+    { url: `${SITE_URL}/auth/sign-up`, changeFrequency: 'monthly', priority: 0.8 },
+    { url: `${SITE_URL}/auth/sign-in`, changeFrequency: 'monthly', priority: 0.5 },
+  ];
+}

ai-pdf-chatbot/lib/ai/prompts.ts

@@ -1,10 +1,11 @@
 export const RAG_SYSTEM_PROMPT = `You are a helpful assistant answering questions about the user's uploaded PDFs.
 
 Rules:
-- Use only the sources provided below. If the answer is not in them, say "I couldn't find that in the provided documents."
+- Use only the sources provided below.
 - When you cite a source, use the bracket marker [n] that matches the numbered source. Cite at the END of each relevant sentence or claim.
 - Keep answers concise. Prefer 2-5 short paragraphs over walls of text.
 - Do not fabricate page numbers, file names, or quotes. Only repeat content verbatim if you wrap it in quotes and cite it.
+- If the sources don't answer the question, or the question is a greeting / test input / too vague to answer ("hi", "test", "what"), do NOT just refuse. Say in one sentence that you couldn't find it, then tell the user in one sentence what the provided sources DO cover, and suggest one concrete question they could ask instead.
 `;
 
 export function buildUserPrompt(

ai-pdf-chatbot/migrations/db_init.sql

@@ -101,10 +101,15 @@ create table if not exists public.document_chunks (
   unique (document_id, chunk_index)
 );
 
-create index if not exists document_chunks_embedding_idx
+-- HNSW, not ivfflat: ivfflat with lists=100 partitions vectors into 100
+-- clusters and probes 1 by default, which collapses recall to near zero
+-- when a user only has a handful of chunks (typical here: one PDF, 6-300
+-- chunks per user). HNSW has no cluster count to mistune and holds 95%+
+-- recall at any collection size.
+drop index if exists document_chunks_embedding_idx;
+create index if not exists document_chunks_embedding_hnsw_idx
   on public.document_chunks
-  using ivfflat (embedding vector_cosine_ops)
-  with (lists = 100);
+  using hnsw (embedding vector_cosine_ops);
 
 create index if not exists document_chunks_user_idx
   on public.document_chunks (user_id);

ai-pdf-chatbot/public/llms.txt

@@ -0,0 +1,23 @@
+# AI Notebook
+
+> AI Notebook is an open-source, self-hosted study workspace for PDFs: upload documents into workspaces and get RAG chat with citations that highlight the exact passage inside the source PDF, auto-generated mindmaps, spaced-repetition flashcards, and NotebookLM-style two-host audio overviews. Built with Next.js, pgvector, and InsForge.
+
+AI Notebook is an InsForge template: developers fork it as a starting point for their own PDF / study / knowledge-base products. It is the open-source alternative to NotebookLM-style tools, with your data on your own infrastructure and any model via OpenRouter.
+
+## Product
+
+- [Live demo](https://aipdfchat.insforge.site): sign up with any email, upload a PDF, ask a question.
+- [Source code](https://github.com/InsForge/insforge-templates/tree/main/ai-pdf-chatbot): MIT-licensed Next.js app.
+- [Template marketplace](https://insforge.dev/templates): one-command scaffold via `npx @insforge/cli create`.
+
+## Key features
+
+- Workspaces: NotebookLM-style containers grouping PDFs, chats, mindmaps, flashcards, and audio.
+- RAG chat with inline citations: every answer cites sources; clicking a citation opens the PDF at the cited page with the passage highlighted.
+- Mindmaps: workspace-wide concept tree generated from document summaries, rendered with Markmap.
+- Spaced-repetition flashcards: SM-2 lite scheduler with a per-workspace review queue.
+- Audio Overview: two-host podcast summary of a workspace, generated with OpenAI TTS (opt-in).
+
+## Stack
+
+- Next.js App Router, Better Auth, InsForge (Postgres + pgvector + storage + AI gateway), react-pdf, Markmap.