Setup custom packlist (#120)

.changeset/mighty-spiders-fetch.md

@@ -0,0 +1,5 @@
+---
+'@publint/packlist': minor
+---
+
+Initial release

.changeset/witty-coins-help.md

@@ -0,0 +1,9 @@
+---
+'publint': minor
+---
+
+`publint` now runs your project's package manager's `pack` command to get the list of packed files for linting. The previous `npm-packlist` dependency is now removed.
+
+A new `pack` option is added to the node API to allow configuring this. It defaults to `'auto'` and will automatically detect your project's package manager using [`package-manager-detector`](https://github.com/antfu-collective/package-manager-detector). See its JSDoc for more information of the option.
+
+This change is made as package managers have different behaviors for packing files, so running their `pack` command directly allows for more accurate linting. However, as a result of executing these commands in a child process, it may take 200-500ms longer to lint depending on the package manager used and the project size. For more information, see [this comment](https://github.com/bluwy/publint/issues/11#issuecomment-2176160022).

.github/workflows/ci.yml

@@ -25,7 +25,7 @@ jobs:
   test:
     name: Test
     runs-on: ${{ matrix.os }}
-    timeout-minutes: 5
+    timeout-minutes: 10
     strategy:
       matrix:
         os: [ubuntu-latest, windows-latest]
@@ -40,6 +40,8 @@ jobs:
         with:
           node-version: 18
           cache: pnpm
+      - name: Setup bun
+        uses: oven-sh/setup-bun@v2
       - name: Install dependencies
         run: pnpm install
       - if: ${{ matrix.os == 'ubuntu-latest' }}

package.json

@@ -8,7 +8,7 @@
     "lint": "prettier \"**/*.{js,ts,css,md,svelte,html}\" --check",
     "format": "prettier \"**/*.{js,ts,css,md,svelte,html}\" --write",
     "typecheck": "tsc -p packages/publint && tsc -p site && tsc -p analysis",
-    "test": "pnpm --dir packages/publint test"
+    "test": "pnpm --filter \"./packages/*\" test"
   },
   "packageManager": "[email protected]",
   "engines": {

packages/packlist/README.md

@@ -0,0 +1,53 @@
+# @publint/packlist
+
+Get a list of files packed by a package manager. Supports:
+
+- npm (v8, v9, v10)
+- yarn (v3, v4)
+- pnpm (v8, v9)
+
+## Usage
+
+```js
+import { packlist } from '@publint/packlist'
+
+const packageDir = process.cwd()
+
+const files = await packlist(packageDir, {
+  // options...
+})
+console.log(files)
+// => ['src/index.js', 'package.json']
+```
+
+### Options
+
+#### `packageManager`
+
+- Type: `'npm' | 'yarn' | 'pnpm' | 'bun'`
+- Default: `'npm'`
+
+The package manager to use for packing. An external package can be used to detect the preferred package manager if needed, e.g. [`package-manager-detector`](https://github.com/antfu-collective/package-manager-detector).
+
+#### `strategy`
+
+- Type: `'json' | 'pack' | 'json-and-pack'`
+- Default: `'json-and-pack'`
+
+How to pack the given directory to get the list of files:
+
+- `'json'`: Uses `<pm> pack --json` (works with all package manager except pnpm <9.14.1 and bun).
+- `'pack'`: Uses `<pm> pack --pack-destination` (works with all package managers).
+- `'json-and-pack'`: Tries to use `'json'` first, and if it fails, falls back to `'pack'`.
+
+NOTE: Theoretically, `'json'` should be faster than `'pack'`, but all package managers seem to only support it as an alternate stdout format and there's no significant speed difference in practice. However, `'json'` performs less fs operations internally so should still be slightly faster.
+
+## Comparison
+
+Compared to [`npm-packlist`](https://github.com/npm/npm-packlist), this package works at a higher level by invoking the package manager `pack` command to retrieve the list of files packed. While `npm-packlist` is abstracted away from `npm` to expose a more direct API, unfortunately not all package managers pack files the same way, e.g. the patterns in `"files"` may be interpreted differently. Plus, since `npm-packlist` v7, it requires `@npmcli/arborist` to be used together, which is a much larger dependency to include altogether.
+
+This package provides an alternative API that works across package managers with a much smaller package size. However, as it executes commands in a child process, it's usually slightly slower (around 200-500ms minimum depending on package manager used and the project size).
+
+## License
+
+MIT

packages/packlist/index.d.ts

@@ -0,0 +1,27 @@
+export interface Options {
+  /**
+   * The package manager to use for packing.
+   *
+   * @default 'npm'
+   */
+  packageManager?: 'npm' | 'yarn' | 'pnpm' | 'bun'
+  /**
+   * How to pack the given directory to get the list of files:
+   * - `'json'`: Uses `<pm> pack --json` (works with all package manager except pnpm <9.14.1 and bun).
+   * - `'pack'`: Uses `<pm> pack --pack-destination` (works with all package managers).
+   * - `'json-and-pack'`: Tries to use `'json'` first, and if it fails, falls back to `'pack'`.
+   *
+   * NOTE: Theoretically, `'json'` should be faster than `'pack'`, but all
+   * package managers seem to only support it as an alternate stdout format
+   * and there's no significant speed difference in practice. However, `'json'`
+   * performs less fs operations internally so should still be slightly faster.
+   *
+   * @default 'json-and-pack'
+   */
+  strategy?: 'json' | 'pack' | 'json-and-pack'
+}
+
+/**
+ * Packs the given directory and returns a list of relative file paths that were packed.
+ */
+export function packlist(dir: string, opts?: Options): Promise<string[]>

packages/packlist/package.json

@@ -0,0 +1,41 @@
+{
+  "name": "@publint/packlist",
+  "version": "0.0.1",
+  "description": "List files included in a package",
+  "type": "module",
+  "author": "Bjorn Lu",
+  "license": "MIT",
+  "types": "./index.d.ts",
+  "exports": {
+    "types": "./index.d.ts",
+    "default": "./src/index.js"
+  },
+  "scripts": {
+    "test": "uvu tests",
+    "prepublishOnly": "node ../publint/lib/cli.js"
+  },
+  "engines": {
+    "node": ">=16"
+  },
+  "files": [
+    "src",
+    "*.d.ts"
+  ],
+  "funding": "https://bjornlu.com/sponsor",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/bluwy/publint.git",
+    "directory": "packages/packlist"
+  },
+  "bugs": {
+    "url": "https://github.com/bluwy/publint/issues"
+  },
+  "keywords": [
+    "pack",
+    "list"
+  ],
+  "devDependencies": {
+    "fs-fixture": "^2.6.0",
+    "uvu": "^0.5.6"
+  }
+}

packages/packlist/src/index.js

@@ -0,0 +1,20 @@
+import { packlistWithJson } from './packlist-with-json.js'
+import { packlistWithPack } from './packlist-with-pack.js'
+
+/** @type {import('../index').packlist} */
+export async function packlist(dir, opts) {
+  const packageManager = opts?.packageManager ?? 'npm'
+
+  switch (opts?.strategy) {
+    case 'json':
+      return await packlistWithJson(dir, packageManager)
+    case 'pack':
+      return await packlistWithPack(dir, packageManager)
+    default:
+      try {
+        return await packlistWithJson(dir, packageManager)
+      } catch {
+        return await packlistWithPack(dir, packageManager)
+      }
+  }
+}

packages/packlist/src/packlist-with-json.js

@@ -0,0 +1,75 @@
+import fs from 'node:fs/promises'
+import util from 'node:util'
+import cp from 'node:child_process'
+import { getTempPackDir } from './temp.js'
+
+/**
+ * @param {string} dir
+ * @param {NonNullable<import('../index.d.ts').Options['packageManager']>} packageManager
+ * @returns {Promise<string[]>}
+ */
+export async function packlistWithJson(dir, packageManager) {
+  if (packageManager === 'bun') {
+    throw new Error('`packlistWithJson` is not supported for `bun`')
+  }
+
+  let command = `${packageManager} pack --json`
+
+  const supportsDryRun = packageManager === 'npm' || packageManager === 'yarn'
+  /** @type {string | undefined} */
+  let packDestination
+  if (supportsDryRun) {
+    command += ' --dry-run'
+  } else {
+    packDestination = await getTempPackDir()
+    command += ` --pack-destination ${packDestination}`
+  }
+
+  const { stdout } = await util.promisify(cp.exec)(command, { cwd: dir })
+
+  try {
+    const stdoutJson =
+      packageManager === 'yarn'
+        ? jsonParseYarnStdout(stdout)
+        : JSON.parse(stdout)
+
+    switch (packageManager) {
+      case 'npm':
+        return parseNpmPackJson(stdoutJson)
+      case 'yarn':
+        return parseYarnPackJson(stdoutJson)
+      case 'pnpm':
+        return parsePnpmPackJson(stdoutJson)
+    }
+  } finally {
+    if (!supportsDryRun && packDestination) {
+      await fs.rm(packDestination, { recursive: true })
+    }
+  }
+}
+
+// yarn outputs invalid json for some reason
+function jsonParseYarnStdout(stdout) {
+  const lines = stdout.split('\n')
+  const result = []
+  for (const line of lines) {
+    if (line) result.push(JSON.parse(line))
+  }
+  return result
+}
+
+function parseNpmPackJson(stdoutJson) {
+  return stdoutJson[0].files.map((file) => file.path)
+}
+
+function parseYarnPackJson(stdoutJson) {
+  const files = []
+  for (const value of stdoutJson) {
+    if (value.location) files.push(value.location)
+  }
+  return files
+}
+
+function parsePnpmPackJson(stdoutJson) {
+  return stdoutJson.files.map((file) => file.path)
+}

packages/packlist/src/packlist-with-pack.js

@@ -0,0 +1,84 @@
+import fs from 'node:fs/promises'
+import path from 'node:path'
+import util from 'node:util'
+import cp from 'node:child_process'
+import zlib from 'node:zlib'
+import { getTempPackDir } from './temp.js'
+
+/**
+ * @param {string} dir
+ * @param {NonNullable<import('../index.d.ts').Options['packageManager']>} packageManager
+ * @returns {Promise<string[]>}
+ */
+export async function packlistWithPack(dir, packageManager) {
+  let command = `${packageManager} pack`
+
+  const packDestination = await getTempPackDir()
+
+  if (packageManager === 'yarn') {
+    command += ` --out \"${path.join(packDestination, 'package.tgz')}\"`
+  } else if (packageManager === 'bun') {
+    command = command.replace('bun', 'bun pm')
+    command += ` --destination \"${packDestination}\"`
+  } else {
+    command += ` --pack-destination \"${packDestination}\"`
+  }
+
+  const output = await util.promisify(cp.exec)(command, { cwd: dir })
+
+  // Get first file that ends with `.tgz` in the pack destination
+  const tarballFile = await fs.readdir(packDestination).then((files) => {
+    return files.find((file) => file.endsWith('.tgz'))
+  })
+  if (!tarballFile) {
+    throw new Error(
+      `[publint] Failed to find packed tarball file in ${packDestination}\n${JSON.stringify(output, null, 2)}`
+    )
+  }
+
+  try {
+    const files = await unpack(path.join(packDestination, tarballFile))
+    // The tar file names have appended "package", except for `@types` packages very strangely
+    const pkgDir = files.length ? files[0].split('/')[0] : 'package'
+    return files.map((file) => file.slice(pkgDir.length + 1))
+  } finally {
+    await fs.rm(packDestination, { recursive: true })
+  }
+}
+
+async function unpack(tarballFile) {
+  const tarball = await fs.readFile(tarballFile)
+  const content = await util.promisify(zlib.gunzip)(tarball)
+
+  /** @type {string[]} */
+  const fileNames = []
+
+  let offset = 0
+  while (offset < content.length) {
+    // Skip empty blocks at end
+    if (content.subarray(offset, offset + 512).every((byte) => byte === 0))
+      break
+
+    // Read filename from header (100 bytes max)
+    const name = content
+      .subarray(offset, offset + 100)
+      .toString('ascii')
+      .split('\0')[0]
+
+    if (name) fileNames.push(name)
+
+    // Get file size from header (12 bytes octal at offset 124)
+    const size = parseInt(
+      content
+        .subarray(offset + 124, offset + 136)
+        .toString()
+        .trim(),
+      8
+    )
+
+    // Skip header and file content (padded to 512 bytes)
+    offset += 512 + Math.ceil(size / 512) * 512
+  }
+
+  return fileNames
+}

packages/packlist/src/temp.js

@@ -0,0 +1,9 @@
+import fs from 'node:fs/promises'
+import path from 'node:path'
+import os from 'node:os'
+
+export async function getTempPackDir() {
+  const tempDir = os.tmpdir() + path.sep
+  const tempPackDir = await fs.mkdtemp(tempDir + 'publint-pack-')
+  return await fs.realpath(tempPackDir)
+}