fix(filesystem): preserve UTF-8 across head and tail chunks

Author: Excelius-Wang

src/filesystem/__tests__/lib.test.ts

@@ -26,6 +26,23 @@ import {
 vi.mock('fs/promises');
 const mockFs = fs as any;
 
+function createMockFileHandle(content: Buffer) {
+  return {
+    read: vi.fn(
+      async (buffer: Buffer, offset: number, length: number, position: number) => {
+        const bytesRead = content.copy(
+          buffer,
+          offset,
+          position,
+          Math.min(position + length, content.length),
+        );
+        return { bytesRead, buffer };
+      },
+    ),
+    close: vi.fn().mockResolvedValue(undefined),
+  };
+}
+
 describe('Lib Functions', () => {
   beforeEach(() => {
     vi.clearAllMocks();
@@ -643,6 +660,23 @@ describe('Lib Functions', () => {
         expect(mockFileHandle.close).toHaveBeenCalled();
       });
 
+      it('preserves UTF-8 characters split across chunk boundaries', async () => {
+        const content = Buffer.concat([
+          Buffer.from('discard\n'),
+          Buffer.from('界'),
+          Buffer.alloc(1017, 'a'),
+          Buffer.from('\nlast'),
+        ]);
+        const mockFileHandle = createMockFileHandle(content);
+
+        mockFs.stat.mockResolvedValue({ size: content.length } as any);
+        mockFs.open.mockResolvedValue(mockFileHandle);
+
+        const result = await tailFile('/test/file.txt', 2);
+
+        expect(result).toBe(`界${'a'.repeat(1017)}\nlast`);
+      });
+
       it('handles read errors gracefully', async () => {
         mockFs.stat.mockResolvedValue({ size: 100 } as any);
 
@@ -699,6 +733,20 @@ describe('Lib Functions', () => {
         expect(mockFileHandle.close).toHaveBeenCalled();
       });
 
+      it('preserves UTF-8 characters split across chunk boundaries', async () => {
+        const content = Buffer.concat([
+          Buffer.alloc(1023, 'a'),
+          Buffer.from('界\nsecond'),
+        ]);
+        const mockFileHandle = createMockFileHandle(content);
+
+        mockFs.open.mockResolvedValue(mockFileHandle);
+
+        const result = await headFile('/test/file.txt', 1);
+
+        expect(result).toBe(`${'a'.repeat(1023)}界`);
+      });
+
       it('handles files with leftover content', async () => {
         const mockFileHandle = {
           read: vi.fn(),

src/filesystem/lib.ts

@@ -2,6 +2,7 @@ import fs from "fs/promises";
 import path from "path";
 import os from 'os';
 import { randomBytes } from 'crypto';
+import { StringDecoder } from 'string_decoder';
 import { diffLines, createTwoFilesPatch } from 'diff';
 import { minimatch } from 'minimatch';
 import { normalizePath, expandHome } from './path-utils.js';
@@ -292,42 +293,28 @@ export async function tailFile(filePath: string, numLines: number): Promise<stri
   // Open file for reading
   const fileHandle = await fs.open(filePath, 'r');
   try {
-    const lines: string[] = [];
+    const chunks: Buffer[] = [];
     let position = fileSize;
-    let chunk = Buffer.alloc(CHUNK_SIZE);
-    let linesFound = 0;
-    let remainingText = '';
+    const chunk = Buffer.alloc(CHUNK_SIZE);
+    let newlinesFound = 0;
 
     // Read chunks from the end of the file until we have enough lines
-    while (position > 0 && linesFound < numLines) {
+    while (position > 0 && newlinesFound < numLines) {
       const size = Math.min(CHUNK_SIZE, position);
       position -= size;
 
       const { bytesRead } = await fileHandle.read(chunk, 0, size, position);
       if (!bytesRead) break;
-      
-      // Get the chunk as a string and prepend any remaining text from previous iteration
-      const readData = chunk.slice(0, bytesRead).toString('utf-8');
-      const chunkText = readData + remainingText;
-      
-      // Split by newlines and count
-      const chunkLines = normalizeLineEndings(chunkText).split('\n');
-      
-      // If this isn't the end of the file, the first line is likely incomplete
-      // Save it to prepend to the next chunk
-      if (position > 0) {
-        remainingText = chunkLines[0];
-        chunkLines.shift(); // Remove the first (incomplete) line
-      }
-      
-      // Add lines to our result (up to the number we need)
-      for (let i = chunkLines.length - 1; i >= 0 && linesFound < numLines; i--) {
-        lines.unshift(chunkLines[i]);
-        linesFound++;
+
+      const readData = Buffer.from(chunk.subarray(0, bytesRead));
+      chunks.unshift(readData);
+      for (const byte of readData) {
+        if (byte === 0x0a) newlinesFound++;
       }
     }
-    
-    return lines.join('\n');
+
+    const text = normalizeLineEndings(Buffer.concat(chunks).toString('utf-8'));
+    return text.split('\n').slice(-numLines).join('\n');
   } finally {
     await fileHandle.close();
   }
@@ -341,13 +328,14 @@ export async function headFile(filePath: string, numLines: number): Promise<stri
     let buffer = '';
     let bytesRead = 0;
     const chunk = Buffer.alloc(1024); // 1KB buffer
+    const decoder = new StringDecoder('utf-8');
 
     // Read chunks and count lines until we have enough or reach EOF
     while (lines.length < numLines) {
       const result = await fileHandle.read(chunk, 0, chunk.length, bytesRead);
       if (result.bytesRead === 0) break; // End of file
       bytesRead += result.bytesRead;
-      buffer += chunk.slice(0, result.bytesRead).toString('utf-8');
+      buffer += decoder.write(chunk.subarray(0, result.bytesRead));
 
       const newLineIndex = buffer.lastIndexOf('\n');
       if (newLineIndex !== -1) {
@@ -359,6 +347,8 @@ export async function headFile(filePath: string, numLines: number): Promise<stri
         }
       }
     }
+
+    buffer += decoder.end();
 
     // If there is leftover content and we still need lines, add it
     if (buffer.length > 0 && lines.length < numLines) {