fix: excluding unnecessary metadata for reranker

Author: Mini256

src/core/services/llamaindex/retrieving.ts

@@ -221,7 +221,13 @@ function transform (result: RetrievedChunk): TextNode {
       ...result.metadata,
       ...result.document_metadata
     },
-    excludedLlmMetadataKeys: ['namespace_id'],
+    excludedLlmMetadataKeys: [
+      // Notice: Exclude several fields generated by the LLM to avoid passing too much text during rerank,
+      // which may lead to exceeding the model's token limit.
+      'sectionSummary',
+      'questionsThisExcerptCanAnswer',
+      'excerptKeywords'
+    ],
     relationships: {
       [NodeRelationship.NEXT]: result.relationships[NodeRelationship.NEXT] ? transformRef(result.relationships[NodeRelationship.NEXT]) : undefined,
       [NodeRelationship.PREVIOUS]: result.relationships[NodeRelationship.PREVIOUS] ? transformRef(result.relationships[NodeRelationship.PREVIOUS]) : undefined,

src/lib/llamaindex/postprocessors/postfilters/MetadataPostFilter.ts

@@ -75,12 +75,8 @@ export class MetadataPostFilter implements BaseNodePostprocessor {
     console.log('Nodes after filter:', filteredNodes.length, 'nodes');
 
     if (filteredNodes.length === 0) {
-      const requiredFilters = filters.filter(f => !f.optional);
-      console.info('No nodes left after filtering, fallback to filtering with required filters.', {
-        requiredFilters
-      });
-      filteredNodes = await this.filterNodes(nodes, requiredFilters);
-      console.log('Nodes after filtering with required filters:', filteredNodes.length, 'nodes');
+      console.warn('No nodes left after filtering, fallback to using all nodes.');
+      return nodes;
     }
 
     return filteredNodes;