feat: add page number to read PDFs (#815)

run-llama · May 7, 2024 · ce94780 · ce94780
1 parent 645fcf6
commit ce94780
Show file tree

Hide file tree

Showing 2 changed files with 12 additions and 4 deletions.
diff --git a/.changeset/ninety-doors-impress.md b/.changeset/ninety-doors-impress.md
@@ -0,0 +1,5 @@
+---
+"llamaindex": patch
+---
+
+Add page number to read PDFs and use generated IDs for PDF and markdown content
diff --git a/packages/core/src/readers/PDFReader.ts b/packages/core/src/readers/PDFReader.ts
@@ -12,10 +12,13 @@ export class PDFReader implements BaseReader {
     fs: GenericFileSystem = defaultFS,
   ): Promise<Document[]> {
     const content = await fs.readRawFile(file);
-    const text = await readPDF(content);
-    return text.map((text, page) => {
-      const id_ = `${file}_${page}`;
-      return new Document({ text, id_ });
+    const pages = await readPDF(content);
+    return pages.map((text, page) => {
+      const id_ = `${file}_${page + 1}`;
+      const metadata = {
+        page_number: page + 1,
+      };
+      return new Document({ text, id_, metadata });
     });
   }
 }