Merge pull request #162 from lambdalisue/fix-bom

Fix BOM handling in GinEdit

Author: lambdalisue

.github/workflows/test.yml

@@ -13,10 +13,11 @@ on:
   workflow_dispatch:
     inputs:
       denops_branch:
-        description: "Denops revision to test"
+        description: "Denops branch to test"
         required: false
         default: "main"
 
+# Use 'bash' as default shell even on Windows
 defaults:
   run:
     shell: bash --noprofile --norc -eo pipefail {0}
@@ -31,13 +32,13 @@ jobs:
         runner:
           - ubuntu-latest
         deno_version:
-          - "1.x"
+          - "2.x"
     runs-on: ${{ matrix.runner }}
     steps:
       - run: git config --global core.autocrlf false
         if: runner.os == 'Windows'
       - uses: actions/checkout@v4
-      - uses: denoland/setup-deno@v1.1.4
+      - uses: denoland/setup-deno@v2
         with:
           deno-version: "${{ matrix.deno_version }}"
       - uses: actions/cache@v4
@@ -61,27 +62,22 @@ jobs:
           - macos-latest
           - ubuntu-latest
         deno_version:
-          - "1.45.0"
-          - "1.x"
+          - "2.x"
         host_version:
           - vim: "v9.1.0448"
             nvim: "v0.10.0"
+
     runs-on: ${{ matrix.runner }}
-    timeout-minutes: 15
+
     steps:
       - run: git config --global core.autocrlf false
         if: runner.os == 'Windows'
 
-      - run: |
-          git config --global user.email "[email protected]"
-          git config --global user.name "GitHub Action"
-          git version
-
       - uses: actions/checkout@v4
 
-      - uses: denoland/setup-deno@v1.1.4
+      - uses: denoland/setup-deno@v1
         with:
-          deno-version: "${{ matrix.deno_version }}"
+          deno-version: ${{ matrix.deno_version }}
 
       - name: Get denops
         run: |
@@ -96,13 +92,13 @@ jobs:
       - uses: rhysd/action-setup-vim@v1
         id: vim
         with:
-          version: "${{ matrix.host_version.vim }}"
+          version: ${{ matrix.host_version.vim }}
 
       - uses: rhysd/action-setup-vim@v1
         id: nvim
         with:
           neovim: true
-          version: "${{ matrix.host_version.nvim }}"
+          version: ${{ matrix.host_version.nvim }}
 
       - name: Export executables
         run: |
@@ -117,7 +113,8 @@ jobs:
 
       - name: Perform pre-cache
         run: |
-          deno cache ${DENOPS_TEST_DENOPS_PATH}/denops/@denops-private/mod.ts ./denops/gin/main.ts
+          deno cache --config ${DENOPS_TEST_DENOPS_PATH}/denops/@denops-private/deno.jsonc ${DENOPS_TEST_DENOPS_PATH}/denops/@denops-private/mod.ts
+          deno cache ./denops/gin/main.ts
 
       - name: Test
         run: deno task test:coverage

deno.jsonc

@@ -12,5 +12,12 @@
     "update": "deno run --allow-env --allow-read --allow-write --allow-run=git,deno --allow-net=jsr.io,registry.npmjs.org jsr:@molt/cli **/*.ts",
     "update:write": "deno task -q update --write",
     "update:commit": "deno task -q update --commit --prefix :package: --pre-commit=fmt,lint"
+  },
+  "lint": {
+    "rules": {
+      "exclude": [
+        "no-import-prefix"
+      ]
+    }
   }
 }

denops/gin/command/edit/edit.ts

@@ -9,6 +9,7 @@ import * as vars from "jsr:@denops/std@^7.0.0/variable";
 import { parseOpts, validateOpts } from "jsr:@denops/std@^7.0.0/argument";
 import { parse as parseBufname } from "jsr:@denops/std@^7.0.0/bufname";
 import { execute } from "../../git/executor.ts";
+import { hasBom } from "../../util/bom.ts";
 import { formatTreeish } from "./util.ts";
 
 export async function edit(
@@ -73,6 +74,7 @@ export async function exec(
       await denops.cmd("filetype detect");
       await option.swapfile.setLocal(denops, false);
       await option.bufhidden.setLocal(denops, "unload");
+      await option.bomb.setLocal(denops, hasBom(stdout));
       if (options.commitish) {
         await option.buftype.setLocal(denops, "nowrite");
         await option.modifiable.setLocal(denops, false);

denops/gin/command/edit/write.ts

@@ -5,7 +5,9 @@ import * as batch from "jsr:@denops/std@^7.0.0/batch";
 import * as fn from "jsr:@denops/std@^7.0.0/function";
 import * as option from "jsr:@denops/std@^7.0.0/option";
 import { parse as parseBufname } from "jsr:@denops/std@^7.0.0/bufname";
+import Encoding from "npm:[email protected]";
 import { findWorktreeFromDenops } from "../../git/worktree.ts";
+import { addBom } from "../../util/bom.ts";
 import { exec as execBare } from "../../command/bare/command.ts";
 
 export async function write(
@@ -32,15 +34,17 @@ export async function exec(
   relpath: string,
   options: ExecOptions,
 ): Promise<void> {
-  const [verbose, fileencoding, fileformat, content] = await batch.collect(
-    denops,
-    (denops) => [
-      option.verbose.get(denops),
-      option.fileencoding.getBuffer(denops, bufnr),
-      option.fileformat.getBuffer(denops, bufnr),
-      fn.getbufline(denops, bufnr, 1, "$"),
-    ],
-  );
+  const [verbose, fileencoding, fileformat, bomb, content] = await batch
+    .collect(
+      denops,
+      (denops) => [
+        option.verbose.get(denops),
+        option.fileencoding.getBuffer(denops, bufnr),
+        option.fileformat.getBuffer(denops, bufnr),
+        option.bomb.getBuffer(denops, bufnr),
+        fn.getbufline(denops, bufnr, 1, "$"),
+      ],
+    );
 
   const worktree = await findWorktreeFromDenops(denops, {
     worktree: options.worktree,
@@ -64,7 +68,8 @@ export async function exec(
   }
   try {
     await fs.copy(f, original);
-    await Deno.writeTextFile(original, `${content.join("\n")}\n`);
+    const data = encode(`${content.join("\n")}\n`, fileencoding);
+    await Deno.writeFile(original, bomb ? addBom(data) : data);
     await fn.setbufvar(denops, bufnr, "&modified", 0);
     await execBare(denops, [
       "add",
@@ -80,3 +85,12 @@ export async function exec(
     await restore();
   }
 }
+
+function encode(str: string, encoding: string): Uint8Array {
+  const utf8Encoder = new TextEncoder();
+  const utf8Bytes = utf8Encoder.encode(str);
+  return Uint8Array.from(Encoding.convert(utf8Bytes, {
+    to: encoding,
+    from: "UTF8",
+  }));
+}

denops/gin/git/executor.ts

@@ -124,7 +124,10 @@ async function postProcessor(
     env,
   });
   const proc = command.spawn();
-  const reader = new Blob([stdout]);
+  // Deno's Type definition doesn't allow ArrayBufferLike so we need to copy
+  // stdout (Uint8Array<ArrayBufferLike>) to safeStdout (Uint8Array<ArrayBuffer>)
+  const safeStdout = new Uint8Array(stdout);
+  const reader = new Blob([safeStdout.buffer]);
   const [_, result] = await Promise.all([
     reader.stream().pipeTo(proc.stdin),
     proc.output(),

denops/gin/git/finder_test.ts

@@ -141,8 +141,15 @@ Deno.test({
 async function prepare(): ReturnType<typeof sandbox> {
   const sbox = await sandbox();
   await $`git init`;
+  // Configure git user for tests to avoid commit failures
+  await $`git config user.email "[email protected]"`;
+  await $`git config user.name "Test User"`;
   await $`git commit --allow-empty -m 'Initial commit' --no-gpg-sign`;
-  await $`git switch -c main`;
+  // Check if we're already on main branch, if not switch to it
+  const currentBranch = await $`git branch --show-current`.text();
+  if (currentBranch.trim() !== "main") {
+    await $`git switch -c main`;
+  }
   await Deno.mkdir(join("a", "b", "c"), { recursive: true });
   return sbox;
 }

denops/gin/util/bom.ts

@@ -0,0 +1,27 @@
+export const BOM = Uint8Array.from([0xef, 0xbb, 0xbf]);
+
+export function addBom(data: Uint8Array): Uint8Array {
+  if (hasBom(data)) {
+    return data;
+  }
+  const withBom = new Uint8Array(data.length + BOM.length);
+  withBom.set(BOM, 0);
+  withBom.set(data, BOM.length);
+  return withBom;
+}
+
+export function removeBom(data: Uint8Array): Uint8Array {
+  if (!hasBom(data)) {
+    return data;
+  }
+  return data.subarray(BOM.length);
+}
+
+export function hasBom(data: Uint8Array): boolean {
+  return (
+    data.length >= BOM.length &&
+    data[0] === BOM[0] &&
+    data[1] === BOM[1] &&
+    data[2] === BOM[2]
+  );
+}

denops/gin/util/text.ts

@@ -11,7 +11,7 @@ export function encodeUtf8(input: string): Uint8Array {
 /**
  * Decode an UTF8 Uint8Array into a string
  */
-export function decodeUtf8(input: BufferSource): string {
+export function decodeUtf8(input: AllowSharedBufferSource): string {
   return textDecoder.decode(input);
 }