fix: Second try, BaseReader fixed in IngestionPipeline

run-llama · May 4, 2024 · 3c0f03e · 3c0f03e
1 parent cb2b673
commit 3c0f03e
Showing 1 changed file with 10 additions and 1 deletion.
diff --git a/packages/core/src/ingestion/IngestionPipeline.ts b/packages/core/src/ingestion/IngestionPipeline.ts
@@ -105,7 +105,16 @@ export class IngestionPipeline {
       inputNodes.push(this.documents);
     }
     if (this.reader) {
-      inputNodes.push(await this.reader.loadData());
+      try {
+        const loadedData = await this.reader.loadData();
+        if (Array.isArray(loadedData)) {
+          inputNodes.push(loadedData.flat()); // Ensure flat structure
+        } else {
+          inputNodes.push([loadedData]);
+        }
+      } catch (error) {
+        console.error(`Error loading data: ${error}`);
+      }
     }
     return inputNodes.flat();
   }