From 8fc24dc546f6c6912520cfe9179d1e88a8d90086 Mon Sep 17 00:00:00 2001 From: JamesPatrickGill Date: Tue, 7 Oct 2025 20:04:09 +0100 Subject: [PATCH 1/2] feat: simple processor -> dev packages not working yet --- jest.config.js | 2 + lib/dep-graph-builders-using-tooling/exec.ts | 80 +++++ .../npm7/index.ts | 216 ++++++++++++ .../npm7/npm-list-types.ts | 52 +++ lib/dep-graph-builders-using-tooling/types.ts | 0 lib/index.ts | 3 + package.json | 5 +- .../npm-module.test.ts | 85 +++++ test/utils/depgraph-comparison.ts | 316 ++++++++++++++++++ 9 files changed, 757 insertions(+), 2 deletions(-) create mode 100644 lib/dep-graph-builders-using-tooling/exec.ts create mode 100644 lib/dep-graph-builders-using-tooling/npm7/index.ts create mode 100644 lib/dep-graph-builders-using-tooling/npm7/npm-list-types.ts create mode 100644 lib/dep-graph-builders-using-tooling/types.ts create mode 100644 test/integration/dep-graph-builders-using-tooling/npm-module.test.ts create mode 100644 test/utils/depgraph-comparison.ts diff --git a/jest.config.js b/jest.config.js index 02532e62..1b7862b6 100644 --- a/jest.config.js +++ b/jest.config.js @@ -4,5 +4,7 @@ module.exports = { testMatch: [ '/test/jest/**/*.test.ts', '/test/jest/**/*.spec.ts', + '/test/integration/**/*.test.ts', + '/test/integration/**/*.spec.ts', ], }; diff --git a/lib/dep-graph-builders-using-tooling/exec.ts b/lib/dep-graph-builders-using-tooling/exec.ts new file mode 100644 index 00000000..d9ebfd96 --- /dev/null +++ b/lib/dep-graph-builders-using-tooling/exec.ts @@ -0,0 +1,80 @@ +import * as childProcess from 'child_process'; +import debugModule = require('debug'); +import { Shescape } from 'shescape'; + +const debugLogging = debugModule('snyk-gradle-plugin'); +// const shescape = new Shescape({ shell: false }); + +// Executes a subprocess. Resolves successfully with stdout contents if the exit code is 0. +export function execute( + command: string, + args: string[], + options: { cwd?: string; env?: NodeJS.ProcessEnv }, + perLineCallback?: (s: string) => Promise, +): Promise { + const spawnOptions: childProcess.SpawnOptions = { + shell: false, + cwd: options?.cwd, // You can do the same for cwd + env: { ...process.env, ...options?.env }, + }; + + // args = shescape.quoteAll(args); + + // Before spawning an external process, we look if we need to restore the system proxy configuration, + // which overides the cli internal proxy configuration. + // + // This top check is to satisfy ts + if (spawnOptions.env) { + if (process.env.SNYK_SYSTEM_HTTP_PROXY !== undefined) { + spawnOptions.env.HTTP_PROXY = process.env.SNYK_SYSTEM_HTTP_PROXY; + } + if (process.env.SNYK_SYSTEM_HTTPS_PROXY !== undefined) { + spawnOptions.env.HTTPS_PROXY = process.env.SNYK_SYSTEM_HTTPS_PROXY; + } + if (process.env.SNYK_SYSTEM_NO_PROXY !== undefined) { + spawnOptions.env.NO_PROXY = process.env.SNYK_SYSTEM_NO_PROXY; + } + } + + return new Promise((resolve, reject) => { + let stdout = ''; + let stderr = ''; + + const proc = childProcess.spawn(command, args, spawnOptions); + + proc.stdout?.on('data', (data: Buffer) => { + const strData = data.toString(); + stdout = stdout + strData; + if (perLineCallback) { + strData.split('\n').forEach(perLineCallback); + } + }); + + proc.stderr?.on('data', (data: Buffer) => { + stderr = stderr + data; + }); + + proc.on('close', (code: number) => { + if (code !== 0) { + const fullCommand = command + ' ' + args.join(' '); + return reject( + new Error(` +>>> command: ${fullCommand} +>>> exit code: ${code} +>>> stdout: +${stdout} +>>> stderr: +${stderr} +`), + ); + } + if (stderr) { + debugLogging( + 'subprocess exit code = 0, but stderr was not empty: ' + stderr, + ); + } + + resolve(stdout); + }); + }); +} diff --git a/lib/dep-graph-builders-using-tooling/npm7/index.ts b/lib/dep-graph-builders-using-tooling/npm7/index.ts new file mode 100644 index 00000000..94f3312e --- /dev/null +++ b/lib/dep-graph-builders-using-tooling/npm7/index.ts @@ -0,0 +1,216 @@ +import { DepGraph, DepGraphBuilder } from '@snyk/dep-graph'; + +import { execute } from '../exec'; +import { writeFileSync } from 'fs'; +import { + NpmDependency, + NpmListOutput, + isNpmListOutput, +} from './npm-list-types'; + +type NpmDependencyWithId = NpmDependency & { id: string; name: string }; + +export type Npm7ParseOptions = { + /** Whether to include development dependencies */ + includeDevDeps: boolean; + /** Whether to include optional dependencies */ + includeOptionalDeps: boolean; + /** Whether to include peer dependencies */ + includePeerDeps?: boolean; + /** Whether to prune cycles in the dependency graph */ + pruneCycles: boolean; + /** Whether to prune within top-level dependencies */ + pruneWithinTopLevelDeps: boolean; + /** Whether to honor package aliases */ + honorAliases?: boolean; +}; + +export async function processNpmProjDir( + dir: string, + options: Npm7ParseOptions, +): Promise { + const npmListJson = await getNpmListOutput(dir); + const dg = buildDepGraph(npmListJson, options); + return dg; +} + +async function getNpmListOutput(dir: string): Promise { + const npmListRawOutput = await execute( + 'npm', + ['list', '--all', '--json', '--package-lock-only'], + { cwd: dir }, + ); + + // Save the raw output for debugging + writeFileSync('./help.json', npmListRawOutput); + + try { + const parsed = JSON.parse(npmListRawOutput); + writeFileSync('./npm-list.json', JSON.stringify(parsed, null, 2)); + if (isNpmListOutput(parsed)) { + return parsed; + } else { + throw new Error( + 'Parsed JSON does not match expected NpmListOutput structure', + ); + } + } catch (e) { + throw new Error('Failed to parse JSON from npm list output'); + } +} + +function buildDepGraph( + npmListJson: NpmListOutput, + options: Npm7ParseOptions, +): DepGraph { + const depGraphBuilder = new DepGraphBuilder( + { name: 'npm' }, + { name: npmListJson.name, ...(npmListJson.version && { version: npmListJson.version }) }, + ); + + // First pass: Build a map of all full dependency definitions + const fullDependencyMap = new Map(); + collectFullDependencies(npmListJson.dependencies, fullDependencyMap); + console.log( + `Collected ${fullDependencyMap.size} full dependency definitions`, + ); + + const rootNode: NpmDependencyWithId = { + id: 'root-node', + name: npmListJson.name, + version: npmListJson.version || 'undefined', + dependencies: npmListJson.dependencies, + resolved: '', + overridden: false, + }; + + processNpmDependency( + depGraphBuilder, + rootNode, + options, + new Set(), + fullDependencyMap, + ); + + return depGraphBuilder.build(); +} + +/** + * Recursively collects all full dependency definitions (those with resolved, overridden, etc.) + * and stores them in a map keyed by "package-name@version" + */ +function collectFullDependencies( + dependencies: Record, + fullDependencyMap: Map, +): void { + for (const [name, dependency] of Object.entries(dependencies)) { + const key = `${name}@${dependency.version}`; + + // Store if this is a full definition (has resolved, overridden, or dependencies) + if ( + dependency.resolved || + dependency.overridden !== undefined || + dependency.dependencies + ) { + fullDependencyMap.set(key, dependency); + } + + // Recursively collect from nested dependencies + if (dependency.dependencies) { + collectFullDependencies(dependency.dependencies, fullDependencyMap); + } + } +} + +/** + * Checks if a dependency is deduplicated (only has version field) + */ +function isDeduplicatedDependency(dependency: any): boolean { + return ( + typeof dependency === 'object' && + dependency !== null && + typeof dependency.version === 'string' && + !dependency.resolved && + dependency.overridden === undefined && + !dependency.dependencies + ); +} + +/** + * Resolves a deduplicated dependency by looking up the full definition + */ +function resolveDeduplicatedDependency( + name: string, + version: string, + fullDependencyMap: Map, +): NpmDependency | null { + const key = `${name}@${version}`; + return fullDependencyMap.get(key) || null; +} + +function processNpmDependency( + depGraphBuilder: DepGraphBuilder, + node: NpmDependencyWithId, + options: Npm7ParseOptions, + visited: Set, + fullDependencyMap: Map, +) { + for (const [name, dependency] of Object.entries(node.dependencies || {})) { + let processedDependency = dependency; + + // Handle deduplicated dependencies + if (isDeduplicatedDependency(dependency)) { + const fullDefinition = resolveDeduplicatedDependency( + name, + dependency.version, + fullDependencyMap, + ); + + if (fullDefinition) { + processedDependency = fullDefinition; + } else { + // If we can't find the full definition, log a warning and continue with the deduplicated version + console.warn( + `Warning: Could not find full definition for deduplicated dependency ${name}@${dependency.version}`, + ); + // Create a minimal full definition from the deduplicated one + processedDependency = { + version: dependency.version, + resolved: '', + overridden: false, + dependencies: {}, + }; + } + } + + const childNode: NpmDependencyWithId = { + id: `${name}@${processedDependency.version}`, + name: name, + ...processedDependency, + }; + + if (visited.has(childNode.id) || childNode.id === 'root-node') { + depGraphBuilder.connectDep(node.id, childNode.id); + continue; + } + + depGraphBuilder.addPkgNode( + { name: childNode.name, version: childNode.version }, + childNode.id, + { + labels: { + scope: 'prod', + }, + }, + ); + depGraphBuilder.connectDep(node.id, childNode.id); + visited.add(childNode.id); + processNpmDependency( + depGraphBuilder, + childNode, + options, + visited, + fullDependencyMap, + ); + } +} diff --git a/lib/dep-graph-builders-using-tooling/npm7/npm-list-types.ts b/lib/dep-graph-builders-using-tooling/npm7/npm-list-types.ts new file mode 100644 index 00000000..4befff50 --- /dev/null +++ b/lib/dep-graph-builders-using-tooling/npm7/npm-list-types.ts @@ -0,0 +1,52 @@ +/** + * TypeScript types for help.json file structure + * This represents the output format of `npm list --all --json --package-lock-only` + */ + +export interface NpmDependency { + /** The version of the package */ + version: string; + /** The resolved URL where the package was downloaded from */ + resolved: string; + /** Whether this dependency was overridden */ + overridden: boolean; + /** Nested dependencies (optional) */ + dependencies?: Record; +} + +export interface NpmListOutput { + /** The name of the root package */ + name: string; + /** The version of the root package */ + version?: string; + /** Top-level dependencies */ + dependencies: Record; +} + +/** + * Type guard to check if an object is a valid NpmDependency + */ +export function isNpmDependency(obj: any): obj is NpmDependency { + return ( + typeof obj === 'object' && + obj !== null && + typeof obj.version === 'string' && + typeof obj.resolved === 'string' && + typeof obj.overridden === 'boolean' && + (obj.dependencies === undefined || + (typeof obj.dependencies === 'object' && obj.dependencies !== null)) + ); +} + +/** + * Type guard to check if an object is a valid NpmListOutput + */ +export function isNpmListOutput(obj: any): obj is NpmListOutput { + return ( + typeof obj === 'object' && + obj !== null && + typeof obj.name === 'string' && + typeof obj.dependencies === 'object' && + obj.dependencies !== null + ); +} diff --git a/lib/dep-graph-builders-using-tooling/types.ts b/lib/dep-graph-builders-using-tooling/types.ts new file mode 100644 index 00000000..e69de29b diff --git a/lib/index.ts b/lib/index.ts index 2df8f303..2602949c 100644 --- a/lib/index.ts +++ b/lib/index.ts @@ -37,6 +37,9 @@ export { ManifestFile, }; +import { processNpmProjDir } from './dep-graph-builders-using-tooling/npm7'; +export { processNpmProjDir }; + // Straight to Depgraph Functionality ************* // ************************************************ import { diff --git a/package.json b/package.json index 111a0602..d98d4fc6 100644 --- a/package.json +++ b/package.json @@ -9,11 +9,11 @@ "test": "npm run unit-test && npm run test:jest", "unit-test": "tap --ts -Rspec ./test/lib/*.test.[tj]s --timeout=300 --no-check-coverage", "test:jest": "jest --coverage --runInBand", + "test:integration": "jest test/integration --runInBand", "lint": "eslint --color --cache '{lib,test}/**/*.{js,ts}' && prettier --check '{lib,test}/**/*.{js,ts}'", "format": "prettier --write '{lib,test}/**/*.{js,ts,json}'", "build": "tsc", - "build-watch": "tsc -w", - "prepare": "npm run build" + "build-watch": "tsc -w" }, "types": "./dist/index.d.ts", "repository": { @@ -46,6 +46,7 @@ "micromatch": "^4.0.8", "p-map": "^4.0.0", "semver": "^7.6.0", + "shescape": "^2.1.6", "snyk-config": "^5.2.0", "tslib": "^1.9.3", "uuid": "^8.3.0" diff --git a/test/integration/dep-graph-builders-using-tooling/npm-module.test.ts b/test/integration/dep-graph-builders-using-tooling/npm-module.test.ts new file mode 100644 index 00000000..858db70e --- /dev/null +++ b/test/integration/dep-graph-builders-using-tooling/npm-module.test.ts @@ -0,0 +1,85 @@ +import { join } from 'path'; +import { readFileSync, writeFileSync } from 'fs'; +import { processNpmProjDir } from '../../../lib/dep-graph-builders-using-tooling/npm7'; +import { + assertDepGraphDataMatch, + compareDepGraphData, + formatComparisonResult, +} from '../../utils/depgraph-comparison'; + +describe('NPM Module Integration Tests', () => { + describe('Happy path tests', () => { + describe('Expected Result tests', () => { + describe.each([ + 'goof', + 'one-dep', + 'cyclic-dep', + // 'deeply-nested-packages', + // 'deeply-scoped', + // 'different-versions', + // 'local-pkg-without-workspaces', + // 'dist-tag-sub-dependency', + // 'bundled-top-level-dep', + ])('[simple tests] project: %s ', (fixtureName) => { + it('matches expected', async () => { + const fixtureDir = join( + __dirname, + `../../jest/dep-graph-builders/fixtures/npm-lock-v2/${fixtureName}`, + ); + + const newDepGraph = await processNpmProjDir(fixtureDir, { + includeDevDeps: false, + includeOptionalDeps: true, + pruneCycles: true, + pruneWithinTopLevelDeps: true, + }); + const newDepGraphData = newDepGraph.toJSON(); + + // Debug: Write the actual result to a file for manual verification + const debugOutputPath = join( + __dirname, + `../../jest/dep-graph-builders/fixtures/npm-lock-v2/${fixtureName}/expected-npm-list.json`, + ); + writeFileSync( + debugOutputPath, + JSON.stringify(newDepGraphData, null, 2) + '\n', + ); + console.log( + `Debug: Written actual DepGraphData to ${debugOutputPath}`, + ); + + const expectedDepGraphJson = JSON.parse( + readFileSync( + join( + __dirname, + `../../jest/dep-graph-builders/fixtures/npm-lock-v2/${fixtureName}/expected.json`, + ), + 'utf8', + ), + ); + + // Use the new comparison utility for better error messages + try { + assertDepGraphDataMatch(newDepGraphData, expectedDepGraphJson, { + ignoreOrder: true, + verbose: true, + }); + } catch (error) { + // If assertion fails, provide detailed comparison info + const comparison = compareDepGraphData( + newDepGraphData, + expectedDepGraphJson, + { + ignoreOrder: true, + verbose: true, + }, + ); + + console.log('\n' + formatComparisonResult(comparison)); + throw error; + } + }); + }); + }); + }); +}); diff --git a/test/utils/depgraph-comparison.ts b/test/utils/depgraph-comparison.ts new file mode 100644 index 00000000..54cbd80d --- /dev/null +++ b/test/utils/depgraph-comparison.ts @@ -0,0 +1,316 @@ +import { DepGraphData } from '@snyk/dep-graph'; + +export interface ComparisonResult { + matches: boolean; + differences: string[]; + summary: { + totalDifferences: number; + categories: { + schemaVersion?: number; + pkgManager?: number; + packages?: number; + graph?: number; + }; + }; +} + +export interface PackageInfo { + id: string; + name: string; + version: string; +} + +export interface GraphNode { + nodeId: string; + pkgId: string; + deps?: Array<{ nodeId: string }>; +} + +/** + * Compares two DepGraphData objects and provides detailed insights into differences + */ +export function compareDepGraphData( + actual: DepGraphData, + expected: DepGraphData, + options: { + ignoreOrder?: boolean; + verbose?: boolean; + } = {} +): ComparisonResult { + const { ignoreOrder = true, verbose = false } = options; + const differences: string[] = []; + const categories = { + schemaVersion: 0, + pkgManager: 0, + packages: 0, + graph: 0, + }; + + // Compare schema version + if (actual.schemaVersion !== expected.schemaVersion) { + differences.push( + `Schema version mismatch: actual=${actual.schemaVersion}, expected=${expected.schemaVersion}` + ); + categories.schemaVersion++; + } + + // Compare package manager + if (JSON.stringify(actual.pkgManager) !== JSON.stringify(expected.pkgManager)) { + differences.push( + `Package manager mismatch: actual=${JSON.stringify(actual.pkgManager)}, expected=${JSON.stringify(expected.pkgManager)}` + ); + categories.pkgManager++; + } + + // Compare packages + const packageDifferences = comparePackages(actual.pkgs, expected.pkgs, ignoreOrder); + differences.push(...packageDifferences); + categories.packages = packageDifferences.length; + + // Compare graph structure + const graphDifferences = compareGraph(actual.graph, expected.graph, ignoreOrder, verbose); + differences.push(...graphDifferences); + categories.graph = graphDifferences.length; + + return { + matches: differences.length === 0, + differences, + summary: { + totalDifferences: differences.length, + categories, + }, + }; +} + +function comparePackages( + actualPkgs: any[], + expectedPkgs: any[], + ignoreOrder: boolean +): string[] { + const differences: string[] = []; + + if (actualPkgs.length !== expectedPkgs.length) { + differences.push( + `Package count mismatch: actual=${actualPkgs.length}, expected=${expectedPkgs.length}` + ); + } + + // Create maps for easier comparison + const actualPkgMap = new Map(actualPkgs.map(pkg => [pkg.id, pkg])); + const expectedPkgMap = new Map(expectedPkgs.map(pkg => [pkg.id, pkg])); + + // Find missing packages + for (const [id, expectedPkg] of expectedPkgMap) { + if (!actualPkgMap.has(id)) { + differences.push(`Missing package: ${id} (${expectedPkg.info.name}@${expectedPkg.info.version})`); + } + } + + // Find extra packages + for (const [id, actualPkg] of actualPkgMap) { + if (!expectedPkgMap.has(id)) { + differences.push(`Extra package: ${id} (${actualPkg.info.name}@${actualPkg.info.version})`); + } + } + + // Compare common packages + for (const [id, expectedPkg] of expectedPkgMap) { + const actualPkg = actualPkgMap.get(id); + if (actualPkg) { + const pkgDiffs = comparePackageDetails(actualPkg, expectedPkg, id); + differences.push(...pkgDiffs); + } + } + + return differences; +} + +function comparePackageDetails(actual: any, expected: any, packageId: string): string[] { + const differences: string[] = []; + + if (actual.info.name !== expected.info.name) { + differences.push(`Package ${packageId}: name mismatch (actual=${actual.info.name}, expected=${expected.info.name})`); + } + + if (actual.info.version !== expected.info.version) { + differences.push(`Package ${packageId}: version mismatch (actual=${actual.info.version}, expected=${expected.info.version})`); + } + + // Compare other properties if they exist + const actualKeys = Object.keys(actual.info).sort(); + const expectedKeys = Object.keys(expected.info).sort(); + + // Filter out undefined values for comparison + const actualFilteredKeys = actualKeys.filter(key => actual.info[key] !== undefined); + const expectedFilteredKeys = expectedKeys.filter(key => expected.info[key] !== undefined); + + // Only report key differences if there are actual differences in the non-undefined keys + if (JSON.stringify(actualFilteredKeys) !== JSON.stringify(expectedFilteredKeys)) { + differences.push(`Package ${packageId}: info properties mismatch (actual keys: ${actualFilteredKeys.join(', ')}, expected keys: ${expectedFilteredKeys.join(', ')})`); + } + + // Compare all properties in the info object, ignoring undefined values + const allKeys = [...new Set([...actualKeys, ...expectedKeys])]; + for (const key of allKeys) { + const actualValue = actual.info[key]; + const expectedValue = expected.info[key]; + + // Skip comparison if both values are undefined + if (actualValue === undefined && expectedValue === undefined) { + continue; + } + + // Skip comparison if one is undefined and the other doesn't exist + if ((actualValue === undefined && !(key in expected.info)) || + (expectedValue === undefined && !(key in actual.info))) { + continue; + } + + if (actualValue !== expectedValue) { + differences.push(`Package ${packageId}: ${key} mismatch (actual=${actualValue}, expected=${expectedValue})`); + } + } + + return differences; +} + +function compareGraph( + actualGraph: any, + expectedGraph: any, + ignoreOrder: boolean, + verbose: boolean +): string[] { + const differences: string[] = []; + + // Compare root node ID + if (actualGraph.rootNodeId !== expectedGraph.rootNodeId) { + differences.push( + `Root node ID mismatch: actual=${actualGraph.rootNodeId}, expected=${expectedGraph.rootNodeId}` + ); + } + + // Compare node count + if (actualGraph.nodes.length !== expectedGraph.nodes.length) { + differences.push( + `Node count mismatch: actual=${actualGraph.nodes.length}, expected=${expectedGraph.nodes.length}` + ); + } + + // Create maps for easier comparison + const actualNodeMap = new Map(actualGraph.nodes.map((node: any) => [node.nodeId, node])); + const expectedNodeMap = new Map(expectedGraph.nodes.map((node: any) => [node.nodeId, node])); + + // Find missing nodes + for (const [nodeId, expectedNode] of expectedNodeMap) { + if (!actualNodeMap.has(nodeId)) { + differences.push(`Missing node: ${nodeId} (pkgId: ${(expectedNode as any).pkgId})`); + } + } + + // Find extra nodes + for (const [nodeId, actualNode] of actualNodeMap) { + if (!expectedNodeMap.has(nodeId)) { + differences.push(`Extra node: ${nodeId} (pkgId: ${(actualNode as any).pkgId})`); + } + } + + // Compare common nodes + for (const [nodeId, expectedNode] of expectedNodeMap) { + const actualNode = actualNodeMap.get(nodeId); + if (actualNode) { + const nodeDiffs = compareNodeDetails(actualNode as any, expectedNode as any, nodeId as string, verbose); + differences.push(...nodeDiffs); + } + } + + return differences; +} + +function compareNodeDetails( + actual: any, + expected: any, + nodeId: string, + verbose: boolean +): string[] { + const differences: string[] = []; + + if (actual.pkgId !== expected.pkgId) { + differences.push(`Node ${nodeId}: pkgId mismatch (actual=${actual.pkgId}, expected=${expected.pkgId})`); + } + + // Compare dependencies + const actualDeps = actual.deps || []; + const expectedDeps = expected.deps || []; + + if (actualDeps.length !== expectedDeps.length) { + differences.push( + `Node ${nodeId}: dependency count mismatch (actual=${actualDeps.length}, expected=${expectedDeps.length})` + ); + } + + // Compare dependency lists + const actualDepIds = actualDeps.map((dep: any) => dep.nodeId).sort(); + const expectedDepIds = expectedDeps.map((dep: any) => dep.nodeId).sort(); + + if (JSON.stringify(actualDepIds) !== JSON.stringify(expectedDepIds)) { + differences.push( + `Node ${nodeId}: dependencies mismatch (actual: [${actualDepIds.join(', ')}], expected: [${expectedDepIds.join(', ')}])` + ); + + if (verbose) { + // Find missing dependencies + const missingDeps = expectedDepIds.filter(id => !actualDepIds.includes(id)); + const extraDeps = actualDepIds.filter(id => !expectedDepIds.includes(id)); + + if (missingDeps.length > 0) { + differences.push(`Node ${nodeId}: missing dependencies: [${missingDeps.join(', ')}]`); + } + if (extraDeps.length > 0) { + differences.push(`Node ${nodeId}: extra dependencies: [${extraDeps.join(', ')}]`); + } + } + } + + return differences; +} + +/** + * Utility function to format comparison results for console output + */ +export function formatComparisonResult(result: ComparisonResult): string { + if (result.matches) { + return '✅ DepGraphData objects match perfectly!'; + } + + const output = [ + '❌ DepGraphData objects do not match:', + '', + `Total differences: ${result.summary.totalDifferences}`, + '', + 'Categories:', + ` - Schema Version: ${result.summary.categories.schemaVersion} differences`, + ` - Package Manager: ${result.summary.categories.pkgManager} differences`, + ` - Packages: ${result.summary.categories.packages} differences`, + ` - Graph Structure: ${result.summary.categories.graph} differences`, + '', + 'Detailed differences:', + ...result.differences.map(diff => ` • ${diff}`), + ]; + + return output.join('\n'); +} + +/** + * Simple assertion function for use in tests + */ +export function assertDepGraphDataMatch( + actual: DepGraphData, + expected: DepGraphData, + options?: { ignoreOrder?: boolean; verbose?: boolean } +): void { + const result = compareDepGraphData(actual, expected, options); + + if (!result.matches) { + throw new Error(`DepGraphData mismatch:\n${formatComparisonResult(result)}`); + } +} From a06410c8d662c99987159cfce6460e027ef784b4 Mon Sep 17 00:00:00 2001 From: JamesPatrickGill Date: Wed, 8 Oct 2025 17:10:10 +0100 Subject: [PATCH 2/2] feat: dev, peer and optional enabled --- .../index.ts => npm/depgraph-builder.ts} | 75 +++---------------- .../npm/index.ts | 15 ++++ .../npm/npm-list-processor.ts | 43 +++++++++++ .../{npm7/npm-list-types.ts => npm/types.ts} | 9 +++ lib/index.ts | 2 +- .../npm-module.test.ts | 30 ++------ 6 files changed, 86 insertions(+), 88 deletions(-) rename lib/dep-graph-builders-using-tooling/{npm7/index.ts => npm/depgraph-builder.ts} (65%) create mode 100644 lib/dep-graph-builders-using-tooling/npm/index.ts create mode 100644 lib/dep-graph-builders-using-tooling/npm/npm-list-processor.ts rename lib/dep-graph-builders-using-tooling/{npm7/npm-list-types.ts => npm/types.ts} (83%) diff --git a/lib/dep-graph-builders-using-tooling/npm7/index.ts b/lib/dep-graph-builders-using-tooling/npm/depgraph-builder.ts similarity index 65% rename from lib/dep-graph-builders-using-tooling/npm7/index.ts rename to lib/dep-graph-builders-using-tooling/npm/depgraph-builder.ts index 94f3312e..0aa00be6 100644 --- a/lib/dep-graph-builders-using-tooling/npm7/index.ts +++ b/lib/dep-graph-builders-using-tooling/npm/depgraph-builder.ts @@ -1,79 +1,27 @@ import { DepGraph, DepGraphBuilder } from '@snyk/dep-graph'; - -import { execute } from '../exec'; -import { writeFileSync } from 'fs'; import { NpmDependency, NpmListOutput, - isNpmListOutput, -} from './npm-list-types'; + NpmProjectProcessorOptions, +} from './types'; type NpmDependencyWithId = NpmDependency & { id: string; name: string }; -export type Npm7ParseOptions = { - /** Whether to include development dependencies */ - includeDevDeps: boolean; - /** Whether to include optional dependencies */ - includeOptionalDeps: boolean; - /** Whether to include peer dependencies */ - includePeerDeps?: boolean; - /** Whether to prune cycles in the dependency graph */ - pruneCycles: boolean; - /** Whether to prune within top-level dependencies */ - pruneWithinTopLevelDeps: boolean; - /** Whether to honor package aliases */ - honorAliases?: boolean; -}; - -export async function processNpmProjDir( - dir: string, - options: Npm7ParseOptions, -): Promise { - const npmListJson = await getNpmListOutput(dir); - const dg = buildDepGraph(npmListJson, options); - return dg; -} - -async function getNpmListOutput(dir: string): Promise { - const npmListRawOutput = await execute( - 'npm', - ['list', '--all', '--json', '--package-lock-only'], - { cwd: dir }, - ); - - // Save the raw output for debugging - writeFileSync('./help.json', npmListRawOutput); - - try { - const parsed = JSON.parse(npmListRawOutput); - writeFileSync('./npm-list.json', JSON.stringify(parsed, null, 2)); - if (isNpmListOutput(parsed)) { - return parsed; - } else { - throw new Error( - 'Parsed JSON does not match expected NpmListOutput structure', - ); - } - } catch (e) { - throw new Error('Failed to parse JSON from npm list output'); - } -} - -function buildDepGraph( +export function buildDepGraph( npmListJson: NpmListOutput, - options: Npm7ParseOptions, + options: NpmProjectProcessorOptions, ): DepGraph { const depGraphBuilder = new DepGraphBuilder( { name: 'npm' }, - { name: npmListJson.name, ...(npmListJson.version && { version: npmListJson.version }) }, + { + name: npmListJson.name, + ...(npmListJson.version && { version: npmListJson.version }), + }, ); // First pass: Build a map of all full dependency definitions const fullDependencyMap = new Map(); collectFullDependencies(npmListJson.dependencies, fullDependencyMap); - console.log( - `Collected ${fullDependencyMap.size} full dependency definitions`, - ); const rootNode: NpmDependencyWithId = { id: 'root-node', @@ -151,7 +99,7 @@ function resolveDeduplicatedDependency( function processNpmDependency( depGraphBuilder: DepGraphBuilder, node: NpmDependencyWithId, - options: Npm7ParseOptions, + options: NpmProjectProcessorOptions, visited: Set, fullDependencyMap: Map, ) { @@ -169,10 +117,7 @@ function processNpmDependency( if (fullDefinition) { processedDependency = fullDefinition; } else { - // If we can't find the full definition, log a warning and continue with the deduplicated version - console.warn( - `Warning: Could not find full definition for deduplicated dependency ${name}@${dependency.version}`, - ); + // If we can't find the full definition, continue with the deduplicated version // Create a minimal full definition from the deduplicated one processedDependency = { version: dependency.version, diff --git a/lib/dep-graph-builders-using-tooling/npm/index.ts b/lib/dep-graph-builders-using-tooling/npm/index.ts new file mode 100644 index 00000000..166313a6 --- /dev/null +++ b/lib/dep-graph-builders-using-tooling/npm/index.ts @@ -0,0 +1,15 @@ +import { DepGraph } from '@snyk/dep-graph'; +import { getNpmListOutput } from './npm-list-processor'; +import { buildDepGraph } from './depgraph-builder'; +import { NpmProjectProcessorOptions } from './types'; + +export { NpmProjectProcessorOptions }; + +export async function processNpmProjDir( + dir: string, + options: NpmProjectProcessorOptions, +): Promise { + const npmListJson = await getNpmListOutput(dir, options); + const dg = buildDepGraph(npmListJson, options); + return dg; +} diff --git a/lib/dep-graph-builders-using-tooling/npm/npm-list-processor.ts b/lib/dep-graph-builders-using-tooling/npm/npm-list-processor.ts new file mode 100644 index 00000000..01da8526 --- /dev/null +++ b/lib/dep-graph-builders-using-tooling/npm/npm-list-processor.ts @@ -0,0 +1,43 @@ +import { execute } from '../exec'; +import { writeFileSync } from 'fs'; +import { + NpmListOutput, + NpmProjectProcessorOptions, + isNpmListOutput, +} from './types'; + +export async function getNpmListOutput( + dir: string, + options: NpmProjectProcessorOptions, +): Promise { + const npmListRawOutput = await execute( + 'npm', + [ + 'list', + '--all', + '--json', + '--package-lock-only', + '--omit=dev', + '--omit=optional', + '--omit=peer', + ...(options.includeDevDeps ? ['--include=dev'] : []), + ...(options.includeOptionalDeps ? ['--include=optional'] : []), + ...(options.includePeerDeps ? ['--include=peer'] : []), + ], + { cwd: dir }, + ); + + try { + const parsed = JSON.parse(npmListRawOutput); + writeFileSync('./npm-list.json', JSON.stringify(parsed, null, 2)); + if (isNpmListOutput(parsed)) { + return parsed; + } else { + throw new Error( + 'Parsed JSON does not match expected NpmListOutput structure', + ); + } + } catch (e) { + throw new Error('Failed to parse JSON from npm list output'); + } +} diff --git a/lib/dep-graph-builders-using-tooling/npm7/npm-list-types.ts b/lib/dep-graph-builders-using-tooling/npm/types.ts similarity index 83% rename from lib/dep-graph-builders-using-tooling/npm7/npm-list-types.ts rename to lib/dep-graph-builders-using-tooling/npm/types.ts index 4befff50..91e418ca 100644 --- a/lib/dep-graph-builders-using-tooling/npm7/npm-list-types.ts +++ b/lib/dep-graph-builders-using-tooling/npm/types.ts @@ -3,6 +3,15 @@ * This represents the output format of `npm list --all --json --package-lock-only` */ +export interface NpmProjectProcessorOptions { + /** Whether to include development dependencies */ + includeDevDeps: boolean; + /** Whether to include optional dependencies */ + includeOptionalDeps: boolean; + /** Whether to include peer dependencies */ + includePeerDeps: boolean; +} + export interface NpmDependency { /** The version of the package */ version: string; diff --git a/lib/index.ts b/lib/index.ts index 2602949c..69159491 100644 --- a/lib/index.ts +++ b/lib/index.ts @@ -37,7 +37,7 @@ export { ManifestFile, }; -import { processNpmProjDir } from './dep-graph-builders-using-tooling/npm7'; +import { processNpmProjDir } from './dep-graph-builders-using-tooling/npm'; export { processNpmProjDir }; // Straight to Depgraph Functionality ************* diff --git a/test/integration/dep-graph-builders-using-tooling/npm-module.test.ts b/test/integration/dep-graph-builders-using-tooling/npm-module.test.ts index 858db70e..78a577d2 100644 --- a/test/integration/dep-graph-builders-using-tooling/npm-module.test.ts +++ b/test/integration/dep-graph-builders-using-tooling/npm-module.test.ts @@ -1,6 +1,6 @@ import { join } from 'path'; import { readFileSync, writeFileSync } from 'fs'; -import { processNpmProjDir } from '../../../lib/dep-graph-builders-using-tooling/npm7'; +import { processNpmProjDir } from '../../../lib/dep-graph-builders-using-tooling/npm'; import { assertDepGraphDataMatch, compareDepGraphData, @@ -14,12 +14,12 @@ describe('NPM Module Integration Tests', () => { 'goof', 'one-dep', 'cyclic-dep', - // 'deeply-nested-packages', - // 'deeply-scoped', - // 'different-versions', - // 'local-pkg-without-workspaces', - // 'dist-tag-sub-dependency', - // 'bundled-top-level-dep', + 'deeply-nested-packages', + 'deeply-scoped', + 'different-versions', + 'local-pkg-without-workspaces', + 'dist-tag-sub-dependency', + 'bundled-top-level-dep', ])('[simple tests] project: %s ', (fixtureName) => { it('matches expected', async () => { const fixtureDir = join( @@ -30,24 +30,10 @@ describe('NPM Module Integration Tests', () => { const newDepGraph = await processNpmProjDir(fixtureDir, { includeDevDeps: false, includeOptionalDeps: true, - pruneCycles: true, - pruneWithinTopLevelDeps: true, + includePeerDeps: true, }); const newDepGraphData = newDepGraph.toJSON(); - // Debug: Write the actual result to a file for manual verification - const debugOutputPath = join( - __dirname, - `../../jest/dep-graph-builders/fixtures/npm-lock-v2/${fixtureName}/expected-npm-list.json`, - ); - writeFileSync( - debugOutputPath, - JSON.stringify(newDepGraphData, null, 2) + '\n', - ); - console.log( - `Debug: Written actual DepGraphData to ${debugOutputPath}`, - ); - const expectedDepGraphJson = JSON.parse( readFileSync( join(