refactor: remove defaultFS from parameters (#841)

.changeset/new-shirts-reply.md

@@ -0,0 +1,10 @@
+---
+"llamaindex": patch
+"@llamaindex/env": patch
+---
+
+refactor: remove `defaultFS` from parameters
+
+We don't accept passing fs in the parameter since it's unnecessary for a determined JS environment.
+
+This was a polyfill way for the non-Node.js environment, but now we use another way to polyfill APIs.

packages/core/src/embeddings/utils.ts

@@ -1,4 +1,4 @@
-import { defaultFS } from "@llamaindex/env";
+import { fs } from "@llamaindex/env";
 import _ from "lodash";
 import { filetypemime } from "magic-bytes.js";
 import type { ImageType } from "../Node.js";
@@ -243,7 +243,7 @@ export async function imageToDataUrl(input: ImageType): Promise<string> {
     _.isString(input)
   ) {
     // string or file URL
-    const dataBuffer = await defaultFS.readFile(
+    const dataBuffer = await fs.readFile(
       input instanceof URL ? input.pathname : input,
     );
     input = new Blob([dataBuffer]);

packages/core/src/readers/CSVReader.ts

@@ -1,5 +1,4 @@
-import type { GenericFileSystem } from "@llamaindex/env";
-import { defaultFS } from "@llamaindex/env";
+import { fs } from "@llamaindex/env";
 import type { ParseConfig } from "papaparse";
 import Papa from "papaparse";
 import { Document } from "../Node.js";
@@ -40,11 +39,8 @@ export class PapaCSVReader implements FileReader {
    * @param {GenericFileSystem} [fs=DEFAULT_FS] - The file system to use for reading the file.
    * @returns {Promise<Document[]>}
    */
-  async loadData(
-    file: string,
-    fs: GenericFileSystem = defaultFS,
-  ): Promise<Document[]> {
-    const fileContent = await fs.readFile(file);
+  async loadData(file: string): Promise<Document[]> {
+    const fileContent = await fs.readFile(file, "utf-8");
     const result = Papa.parse(fileContent, this.papaConfig);
     const textList = result.data.map((row: any) => {
       // Compatible with header row mode

packages/core/src/readers/DocxReader.ts

@@ -1,16 +1,12 @@
-import type { GenericFileSystem } from "@llamaindex/env";
-import { defaultFS } from "@llamaindex/env";
+import { fs } from "@llamaindex/env";
 import mammoth from "mammoth";
 import { Document } from "../Node.js";
 import type { FileReader } from "./type.js";
 
 export class DocxReader implements FileReader {
   /** DocxParser */
-  async loadData(
-    file: string,
-    fs: GenericFileSystem = defaultFS,
-  ): Promise<Document[]> {
-    const dataBuffer = await fs.readRawFile(file);
+  async loadData(file: string): Promise<Document[]> {
+    const dataBuffer = await fs.readFile(file);
     const { value } = await mammoth.extractRawText({ buffer: dataBuffer });
     return [new Document({ text: value, id_: file })];
   }

packages/core/src/readers/HTMLReader.ts

@@ -1,5 +1,4 @@
-import type { GenericFileSystem } from "@llamaindex/env";
-import { defaultFS } from "@llamaindex/env";
+import { fs } from "@llamaindex/env";
 import { Document } from "../Node.js";
 import type { FileReader } from "./type.js";
 
@@ -15,14 +14,10 @@ export class HTMLReader implements FileReader {
    * Public method for this reader.
    * Required by BaseReader interface.
    * @param file Path/name of the file to be loaded.
-   * @param fs fs wrapper interface for getting the file content.
    * @returns Promise<Document[]> A Promise object, eventually yielding zero or one Document parsed from the HTML content of the specified file.
    */
-  async loadData(
-    file: string,
-    fs: GenericFileSystem = defaultFS,
-  ): Promise<Document[]> {
-    const dataBuffer = await fs.readFile(file);
+  async loadData(file: string): Promise<Document[]> {
+    const dataBuffer = await fs.readFile(file, "utf-8");
     const htmlOptions = this.getOptions();
     const content = await this.parseContent(dataBuffer, htmlOptions);
     return [new Document({ text: content, id_: file })];

packages/core/src/readers/ImageReader.ts

@@ -1,5 +1,4 @@
-import type { GenericFileSystem } from "@llamaindex/env";
-import { defaultFS } from "@llamaindex/env";
+import { fs } from "@llamaindex/env";
 import type { Document } from "../Node.js";
 import { ImageDocument } from "../Node.js";
 import type { FileReader } from "./type.js";
@@ -15,11 +14,8 @@ export class ImageReader implements FileReader {
    * @param fs fs wrapper interface for getting the file content.
    * @returns Promise<Document[]> A Promise object, eventually yielding zero or one ImageDocument of the specified file.
    */
-  async loadData(
-    file: string,
-    fs: GenericFileSystem = defaultFS,
-  ): Promise<Document[]> {
-    const dataBuffer = await fs.readRawFile(file);
+  async loadData(file: string): Promise<Document[]> {
+    const dataBuffer = await fs.readFile(file);
     const blob = new Blob([dataBuffer]);
     return [new ImageDocument({ image: blob, id_: file })];
   }

packages/core/src/readers/LlamaParseReader.ts

@@ -1,4 +1,4 @@
-import { defaultFS, getEnv, type GenericFileSystem } from "@llamaindex/env";
+import { fs, getEnv } from "@llamaindex/env";
 import { filetypemime } from "magic-bytes.js";
 import { Document } from "../Node.js";
 import type { FileReader, Language, ResultType } from "./type.js";
@@ -79,14 +79,11 @@ export class LlamaParseReader implements FileReader {
     this.apiKey = params.apiKey;
   }
 
-  async loadData(
-    file: string,
-    fs: GenericFileSystem = defaultFS,
-  ): Promise<Document[]> {
+  async loadData(file: string): Promise<Document[]> {
     const metadata = { file_path: file };
 
     // Load data, set the mime type
-    const data = await fs.readRawFile(file);
+    const data = await fs.readFile(file);
     const mimeType = await this.getMimeType(data);
 
     const body = new FormData();

packages/core/src/readers/MarkdownReader.ts

@@ -1,5 +1,4 @@
-import type { GenericFileSystem } from "@llamaindex/env";
-import { defaultFS } from "@llamaindex/env";
+import { fs } from "@llamaindex/env";
 import { Document } from "../Node.js";
 import type { FileReader } from "./type.js";
 
@@ -88,11 +87,8 @@ export class MarkdownReader implements FileReader {
     return this.markdownToTups(modifiedContent);
   }
 
-  async loadData(
-    file: string,
-    fs: GenericFileSystem = defaultFS,
-  ): Promise<Document[]> {
-    const content = await fs.readFile(file);
+  async loadData(file: string): Promise<Document[]> {
+    const content = await fs.readFile(file, "utf-8");
     const tups = this.parseTups(content);
     const results: Document[] = [];
     let counter = 0;

packages/core/src/readers/PDFReader.ts

@@ -1,17 +1,13 @@
-import type { GenericFileSystem } from "@llamaindex/env";
-import { defaultFS } from "@llamaindex/env";
+import { fs } from "@llamaindex/env";
 import { Document } from "../Node.js";
 import type { BaseReader } from "./type.js";
 
 /**
  * Read the text of a PDF
  */
 export class PDFReader implements BaseReader {
-  async loadData(
-    file: string,
-    fs: GenericFileSystem = defaultFS,
-  ): Promise<Document[]> {
-    const content = await fs.readRawFile(file);
+  async loadData(file: string): Promise<Document[]> {
+    const content = await fs.readFile(file);
     const pages = await readPDF(content);
     return pages.map((text, page) => {
       const id_ = `${file}_${page + 1}`;

packages/core/src/readers/SimpleDirectoryReader.edge.ts

@@ -1,5 +1,4 @@
-import type { CompleteFileSystem } from "@llamaindex/env";
-import { defaultFS, path } from "@llamaindex/env";
+import { fs, path } from "@llamaindex/env";
 import { Document, type Metadata } from "../Node.js";
 import { walk } from "../storage/FileSystem.js";
 import { TextFileReader } from "./TextFileReader.js";
@@ -19,7 +18,6 @@ enum ReaderStatus {
 
 export type SimpleDirectoryReaderLoadDataParams = {
   directoryPath: string;
-  fs?: CompleteFileSystem;
   defaultReader?: BaseReader | null;
   fileExtToReader?: Record<string, BaseReader>;
 };
@@ -45,7 +43,6 @@ export class SimpleDirectoryReader implements BaseReader {
 
     const {
       directoryPath,
-      fs = defaultFS,
       defaultReader = new TextFileReader(),
       fileExtToReader,
     } = params;
@@ -58,7 +55,7 @@ export class SimpleDirectoryReader implements BaseReader {
     }
 
     const docs: Document[] = [];
-    for await (const filePath of walk(fs, directoryPath)) {
+    for await (const filePath of walk(directoryPath)) {
       try {
         const fileExt = path.extname(filePath).slice(1).toLowerCase();
 

packages/core/src/readers/TextFileReader.ts

@@ -1,5 +1,4 @@
-import type { CompleteFileSystem } from "@llamaindex/env";
-import { defaultFS } from "@llamaindex/env";
+import { fs } from "@llamaindex/env";
 import { Document } from "../Node.js";
 import type { BaseReader } from "./type.js";
 
@@ -8,11 +7,8 @@ import type { BaseReader } from "./type.js";
  */
 
 export class TextFileReader implements BaseReader {
-  async loadData(
-    file: string,
-    fs: CompleteFileSystem = defaultFS,
-  ): Promise<Document[]> {
-    const dataBuffer = await fs.readFile(file);
+  async loadData(file: string): Promise<Document[]> {
+    const dataBuffer = await fs.readFile(file, "utf-8");
     return [new Document({ text: dataBuffer, id_: file })];
   }
 }

packages/core/src/readers/type.ts

@@ -1,4 +1,3 @@
-import type { CompleteFileSystem } from "@llamaindex/env";
 import type { Document } from "../Node.js";
 
 /**
@@ -12,7 +11,7 @@ export interface BaseReader {
  * A reader takes file paths and imports data into Document objects.
  */
 export interface FileReader extends BaseReader {
-  loadData(filePath: string, fs?: CompleteFileSystem): Promise<Document[]>;
+  loadData(filePath: string): Promise<Document[]>;
 }
 
 // For LlamaParseReader.ts

packages/core/src/storage/FileSystem.ts

@@ -1,5 +1,6 @@
-import type { GenericFileSystem, WalkableFileSystem } from "@llamaindex/env";
-// FS utility functions
+// FS utility helpers
+
+import { fs } from "@llamaindex/env";
 
 /**
  * Checks if a file exists.
@@ -8,10 +9,7 @@ import type { GenericFileSystem, WalkableFileSystem } from "@llamaindex/env";
  * @param path The path to the file to check.
  * @returns A promise that resolves to true if the file exists, false otherwise.
  */
-export async function exists(
-  fs: GenericFileSystem,
-  path: string,
-): Promise<boolean> {
+export async function exists(path: string): Promise<boolean> {
   try {
     await fs.access(path);
     return true;
@@ -22,19 +20,15 @@ export async function exists(
 
 /**
  * Recursively traverses a directory and yields all the paths to the files in it.
- * @param fs The filesystem to use.
  * @param dirPath The path to the directory to traverse.
  */
-export async function* walk(
-  fs: WalkableFileSystem,
-  dirPath: string,
-): AsyncIterable<string> {
+export async function* walk(dirPath: string): AsyncIterable<string> {
   const entries = await fs.readdir(dirPath);
   for (const entry of entries) {
     const fullPath = `${dirPath}/${entry}`;
     const stats = await fs.stat(fullPath);
     if (stats.isDirectory()) {
-      yield* walk(fs, fullPath);
+      yield* walk(fullPath);
     } else {
       yield fullPath;
     }

packages/core/src/storage/StorageContext.ts

@@ -1,5 +1,4 @@
-import type { GenericFileSystem } from "@llamaindex/env";
-import { defaultFS, path } from "@llamaindex/env";
+import { path } from "@llamaindex/env";
 import {
   DEFAULT_IMAGE_VECTOR_NAMESPACE,
   DEFAULT_NAMESPACE,
@@ -25,7 +24,6 @@ export type BuilderParams = {
   imageVectorStore: VectorStore;
   storeImages: boolean;
   persistDir: string;
-  fs: GenericFileSystem;
 };
 
 export async function storageContextFromDefaults({
@@ -35,30 +33,23 @@ export async function storageContextFromDefaults({
   imageVectorStore,
   storeImages,
   persistDir,
-  fs,
 }: Partial<BuilderParams>): Promise<StorageContext> {
   if (!persistDir) {
     docStore = docStore || new SimpleDocumentStore();
     indexStore = indexStore || new SimpleIndexStore();
     vectorStore = vectorStore || new SimpleVectorStore();
     imageVectorStore = storeImages ? new SimpleVectorStore() : imageVectorStore;
   } else {
-    fs = fs || defaultFS;
     docStore =
       docStore ||
-      (await SimpleDocumentStore.fromPersistDir(
-        persistDir,
-        DEFAULT_NAMESPACE,
-        fs,
-      ));
+      (await SimpleDocumentStore.fromPersistDir(persistDir, DEFAULT_NAMESPACE));
     indexStore =
-      indexStore || (await SimpleIndexStore.fromPersistDir(persistDir, fs));
+      indexStore || (await SimpleIndexStore.fromPersistDir(persistDir));
     vectorStore =
-      vectorStore || (await SimpleVectorStore.fromPersistDir(persistDir, fs));
+      vectorStore || (await SimpleVectorStore.fromPersistDir(persistDir));
     imageVectorStore = storeImages
       ? await SimpleVectorStore.fromPersistDir(
           path.join(persistDir, DEFAULT_IMAGE_VECTOR_NAMESPACE),
-          fs,
         )
       : imageVectorStore;
   }

packages/core/src/storage/docStore/SimpleDocumentStore.ts

@@ -1,5 +1,4 @@
-import type { GenericFileSystem } from "@llamaindex/env";
-import { defaultFS, path } from "@llamaindex/env";
+import { path } from "@llamaindex/env";
 import _ from "lodash";
 import {
   DEFAULT_DOC_STORE_PERSIST_FILENAME,
@@ -25,26 +24,19 @@ export class SimpleDocumentStore extends KVDocumentStore {
   static async fromPersistDir(
     persistDir: string = DEFAULT_PERSIST_DIR,
     namespace?: string,
-    fsModule?: GenericFileSystem,
   ): Promise<SimpleDocumentStore> {
     const persistPath = path.join(
       persistDir,
       DEFAULT_DOC_STORE_PERSIST_FILENAME,
     );
-    return await SimpleDocumentStore.fromPersistPath(
-      persistPath,
-      namespace,
-      fsModule,
-    );
+    return await SimpleDocumentStore.fromPersistPath(persistPath, namespace);
   }
 
   static async fromPersistPath(
     persistPath: string,
     namespace?: string,
-    fs?: GenericFileSystem,
   ): Promise<SimpleDocumentStore> {
-    fs = fs || defaultFS;
-    const simpleKVStore = await SimpleKVStore.fromPersistPath(persistPath, fs);
+    const simpleKVStore = await SimpleKVStore.fromPersistPath(persistPath);
     return new SimpleDocumentStore(simpleKVStore, namespace);
   }
 
@@ -53,14 +45,12 @@ export class SimpleDocumentStore extends KVDocumentStore {
       DEFAULT_PERSIST_DIR,
       DEFAULT_DOC_STORE_PERSIST_FILENAME,
     ),
-    fs?: GenericFileSystem,
   ): Promise<void> {
-    fs = fs || defaultFS;
     if (
       _.isObject(this.kvStore) &&
       this.kvStore instanceof BaseInMemoryKVStore
     ) {
-      await this.kvStore.persist(persistPath, fs);
+      await this.kvStore.persist(persistPath);
     }
   }