From f1b4be6725d2a86f0d7552b2d0e8044ee31eb16d Mon Sep 17 00:00:00 2001 From: malice00 Date: Sun, 22 Sep 2024 22:59:49 +0200 Subject: [PATCH] [Gradle] Scanning of all modules fixed (#1383) * Missed a part for deep-scanning Signed-off-by: Roland Asmann * Make sure the rootProject isn't added in the list of components either Signed-off-by: Roland Asmann * Added elasticsearch to repotests Signed-off-by: Roland Asmann * Made multithreaded gradle the default and removed singlethreaded code Signed-off-by: Roland Asmann * Removed printing output Signed-off-by: Roland Asmann * Minimized the adding/removing of the ':'-prefix Signed-off-by: Roland Asmann * Added the module-name to the metadata for all modules, not just those resolved from NPM Signed-off-by: Roland Asmann --------- Signed-off-by: Roland Asmann --- .github/workflows/repotests.yml | 14 +- docs/ENV.md | 2 +- lib/cli/index.js | 267 +++++++++++--------------------- lib/helpers/utils.js | 97 +++++------- lib/helpers/utils.test.js | 24 +-- 5 files changed, 155 insertions(+), 249 deletions(-) diff --git a/.github/workflows/repotests.yml b/.github/workflows/repotests.yml index 303a20d80..db31800f7 100644 --- a/.github/workflows/repotests.yml +++ b/.github/workflows/repotests.yml @@ -251,6 +251,10 @@ jobs: repository: 'malice00/cdxgen-expo-test' ref: 'android' path: 'repotests/expo-test' + - uses: actions/checkout@v4 + with: + repository: 'elastic/elasticsearch' + path: 'repotests/elasticsearch' - uses: dtolnay/rust-toolchain@stable - name: setup sdkman run: | @@ -451,10 +455,12 @@ jobs: - name: repotests expo run: | cd repotests/expo-test && npm ci && cd ../.. - GRADLE_ARGS_DEPENDENCIES="--configuration releaseRuntimeClasspath" bin/cdxgen.js -p -t gradle repotests/expo-test -o bomresults/bom-expo.json - GRADLE_ARGS_DEPENDENCIES="--configuration releaseRuntimeClasspath" GRADLE_MULTI_THREADED=true GRADLE_SKIP_MODULES=root bin/cdxgen.js -p -t gradle repotests/expo-test -o bomresults/bom-expo-multi.json - custom-json-diff -i bomresults/bom-expo.json bomresults/bom-expo-multi.json -o bomresults/diff-expo - GRADLE_ARGS_DEPENDENCIES="--configuration releaseRuntimeClasspath" GRADLE_MULTI_THREADED=true GRADLE_SKIP_MODULES=root GRADLE_RESOLVE_FROM_NODE=true bin/cdxgen.js -p -t gradle repotests/expo-test -o bomresults/bom-expo-npm.json + GRADLE_ARGS_DEPENDENCIES="--configuration releaseRuntimeClasspath" GRADLE_SKIP_MODULES=root bin/cdxgen.js -p -t gradle repotests/expo-test -o bomresults/bom-expo.json + GRADLE_ARGS_DEPENDENCIES="--configuration releaseRuntimeClasspath" GRADLE_SKIP_MODULES=root GRADLE_RESOLVE_FROM_NODE=true bin/cdxgen.js -p -t gradle repotests/expo-test -o bomresults/bom-expo-npm.json + shell: bash + - name: repotests elasticsearch + run: | + bin/cdxgen.js -t gradle repotests/elasticsearch -o bomresults/bom-elasticsearch.json shell: bash - name: jenkins plugins run: | diff --git a/docs/ENV.md b/docs/ENV.md index d0c01ac69..d68e26a35 100644 --- a/docs/ENV.md +++ b/docs/ENV.md @@ -25,7 +25,7 @@ The following environment variables are available to configure the bom generatio | GRADLE_CMD | Set to override gradle command | | GRADLE_DEPENDENCY_TASK | By default cdxgen use the task "dependencies" to collect packages. Set to override the task name. | | GRADLE_RESOLVE_FROM_NODE | If some of your gradle modules are included from node (eg when using expo or react-native), set this to true to use the npm-packages as your dependencies. The big advantage of this, is that the generated purls will be of actually known components (eg in OSS Index) instead of generic names for the packages. | -| GRADLE_SKIP_MODULES | Comma-separated list of modules to skip during the "dependencies" task. This can be useful if you have modules that would fail the gradle build, eg when they do not have dependencies in the given configuration. Use "root" if the top most module should be skipped, use the name (without leading ":") for all others. | +| GRADLE_SKIP_MODULES | Comma-separated list of modules to skip during the "dependencies" task. This can be useful if you have modules that would fail the gradle build, eg when they do not have dependencies in the given configuration. Use "root" if the top most module should be skipped, use their gradle-name (so WITH leading ":") for all others. | | SBT_CACHE_DIR | Specify sbt cache directory. Useful for class name resolving | | FETCH_LICENSE | Set this variable to `true` or `1` to fetch license information from the registry. npm and golang | | SEARCH_MAVEN_ORG | If maven metadata is missing in jar file, a search is performed on search.maven.org. Set to `false` or `0` to disable search. (defaults to `true`) | diff --git a/lib/cli/index.js b/lib/cli/index.js index a86ebc231..44d75cd00 100644 --- a/lib/cli/index.js +++ b/lib/cli/index.js @@ -1635,7 +1635,7 @@ export async function createJavaBom(path, options) { gradleFiles?.length && isPackageManagerAllowed("gradle", ["maven", "bazel", "sbt"], options) ) { - let retMap = executeGradleProperties(gradleRootPath, null); + const retMap = executeGradleProperties(gradleRootPath, null); const allProjectsStr = retMap.projects || []; const rootProject = retMap.rootProject; if (rootProject) { @@ -1647,50 +1647,31 @@ export async function createJavaBom(path, options) { } // Get the sub-project properties and set the root dependencies if (allProjectsStr?.length) { - if (process.env.GRADLE_MULTI_THREADED) { - const parallelPropTaskOut = executeParallelGradleProperties( - gradleRootPath, - allProjectsStr, - ); - const splitPropTaskOut = splitOutputByGradleProjects( - parallelPropTaskOut, - ["properties"], - ); + const parallelPropTaskOut = executeParallelGradleProperties( + gradleRootPath, + allProjectsStr, + ); + const splitPropTaskOut = splitOutputByGradleProjects( + parallelPropTaskOut, + ["properties"], + ); - for (const [key, propTaskOut] of splitPropTaskOut.entries()) { - const retMap = parseGradleProperties(propTaskOut); - const rootSubProject = retMap.rootProject; - if (rootSubProject) { - const rootSubProjectObj = await buildObjectForGradleModule( - rootSubProject.replace(/^:/, ""), - retMap.metadata, - ); - if (!allProjectsAddedPurls.includes(rootSubProjectObj["purl"])) { - allProjects.push(rootSubProjectObj); - rootDependsOn.push(rootSubProjectObj["bom-ref"]); - allProjectsAddedPurls.push(rootSubProjectObj["purl"]); - } - gradleModules.set(key.replace(/^:/, ""), rootSubProjectObj); - } - } - } else { - for (const spstr of allProjectsStr) { - retMap = executeGradleProperties(gradleRootPath, spstr); - const rootSubProject = retMap.rootProject; - if (rootSubProject) { - const rootSubProjectObj = await buildObjectForGradleModule( - rootSubProject.replace(/^:/, ""), - retMap.metadata, - ); - if (!allProjectsAddedPurls.includes(rootSubProjectObj["purl"])) { - allProjects.push(rootSubProjectObj); - rootDependsOn.push(rootSubProjectObj["bom-ref"]); - allProjectsAddedPurls.push(rootSubProjectObj["purl"]); - } - gradleModules.set(spstr.replace(/^:/, ""), rootSubProjectObj); + for (const [key, propTaskOut] of splitPropTaskOut.entries()) { + const retMap = parseGradleProperties(propTaskOut, key); + const rootSubProject = retMap.rootProject; + if (rootSubProject) { + const rootSubProjectObj = await buildObjectForGradleModule( + rootSubProject, + retMap.metadata, + ); + if (!allProjectsAddedPurls.includes(rootSubProjectObj["purl"])) { + allProjects.push(rootSubProjectObj); + rootDependsOn.push(rootSubProjectObj["bom-ref"]); + allProjectsAddedPurls.push(rootSubProjectObj["purl"]); } + gradleModules.set(key, rootSubProjectObj); } - } //end else + } // Bug #317 fix parentComponent.components = allProjects.flatMap((s) => { delete s.qualifiers; @@ -1714,155 +1695,87 @@ export async function createJavaBom(path, options) { ? process.env.GRADLE_DEPENDENCY_TASK : "dependencies"; - if (process.env.GRADLE_MULTI_THREADED) { - const gradleSubCommands = []; - const modulesToSkip = process.env.GRADLE_SKIP_MODULES - ? process.env.GRADLE_SKIP_MODULES.split(",") - : []; - if (!modulesToSkip.includes("root")) { - gradleSubCommands.push(gradleDepTask); - } - for (const [key, sp] of gradleModules) { - //create single command for dependencies tasks on all subprojects - if (sp.purl !== parentComponent.purl && !modulesToSkip.includes(key)) { - gradleSubCommands.push(`:${key}:${gradleDepTask}`); - } - } - const gradleArguments = buildGradleCommandArguments( - process.env.GRADLE_ARGS ? process.env.GRADLE_ARGS.split(" ") : [], - gradleSubCommands, - process.env.GRADLE_ARGS_DEPENDENCIES - ? process.env.GRADLE_ARGS_DEPENDENCIES.split(" ") - : [], - ); - console.log( - "Executing", - gradleCmd, - gradleArguments.join(" "), - "in", - gradleRootPath, - ); - const sresult = spawnSync(gradleCmd, gradleArguments, { - cwd: gradleRootPath, - encoding: "utf-8", - timeout: TIMEOUT_MS, - maxBuffer: MAX_BUFFER, - }); + const gradleSubCommands = []; + const modulesToSkip = process.env.GRADLE_SKIP_MODULES + ? process.env.GRADLE_SKIP_MODULES.split(",") + : []; + if (!modulesToSkip.includes("root")) { + gradleSubCommands.push(gradleDepTask); + } + for (const [key, sp] of gradleModules) { + //create single command for dependencies tasks on all subprojects + if (sp.purl !== parentComponent.purl && !modulesToSkip.includes(key)) { + gradleSubCommands.push(`${key}:${gradleDepTask}`); + } + } + const gradleArguments = buildGradleCommandArguments( + process.env.GRADLE_ARGS ? process.env.GRADLE_ARGS.split(" ") : [], + gradleSubCommands, + process.env.GRADLE_ARGS_DEPENDENCIES + ? process.env.GRADLE_ARGS_DEPENDENCIES.split(" ") + : [], + ); + console.log( + "Executing", + gradleCmd, + gradleArguments.join(" "), + "in", + gradleRootPath, + ); + const sresult = spawnSync(gradleCmd, gradleArguments, { + cwd: gradleRootPath, + encoding: "utf-8", + timeout: TIMEOUT_MS, + maxBuffer: MAX_BUFFER, + }); - if (sresult.status !== 0 || sresult.error) { - if (options.failOnError || DEBUG_MODE) { - console.error(sresult.stdout, sresult.stderr); - } - options.failOnError && process.exit(1); - } - const sstdout = sresult.stdout; - if (sstdout) { - const cmdOutput = Buffer.from(sstdout).toString(); - const perProjectOutput = splitOutputByGradleProjects(cmdOutput, [ - gradleDepTask, - ]); - for (const [key, sp] of gradleModules) { - const parsedList = await parseGradleDep( - perProjectOutput.has(key) ? perProjectOutput.get(key) : "", - key, - gradleModules, - gradleRootPath, - ); - const dlist = parsedList.pkgList; - if (parsedList.dependenciesList && parsedList.dependenciesList) { - dependencies = mergeDependencies( - dependencies, - parsedList.dependenciesList, - parentComponent, - ); - } - if (dlist?.length) { - if (DEBUG_MODE) { - console.log( - "Found", - dlist.length, - "packages in gradle project", - key, - ); - } - pkgList = pkgList.concat(dlist); - } - } - } - } else { - if (DEBUG_MODE) { - console.log( - "Try the new multi-threaded mode for gradle. Set the environment variable GRADLE_MULTI_THREADED to true to enable this.", - ); + if (sresult.status !== 0 || sresult.error) { + if (options.failOnError || DEBUG_MODE) { + console.error(sresult.stdout, sresult.stderr); } + options.failOnError && process.exit(1); + } + const sstdout = sresult.stdout; + if (sstdout) { + const cmdOutput = Buffer.from(sstdout).toString(); + const perProjectOutput = splitOutputByGradleProjects(cmdOutput, [ + gradleDepTask, + ]); for (const [key, sp] of gradleModules) { - const gradleArguments = buildGradleCommandArguments( - process.env.GRADLE_ARGS ? process.env.GRADLE_ARGS.split(" ") : [], - [ - sp.purl === parentComponent.purl - ? gradleDepTask - : `:${key}:${gradleDepTask}`, - ], - process.env.GRADLE_ARGS_DEPENDENCIES - ? process.env.GRADLE_ARGS_DEPENDENCIES.split(" ") - : [], - ); - - console.log( - "Executing", - gradleCmd, - gradleArguments.join(" "), - "in", + const parsedList = await parseGradleDep( + perProjectOutput.has(key) ? perProjectOutput.get(key) : "", + key, + gradleModules, gradleRootPath, ); - const sresult = spawnSync(gradleCmd, gradleArguments, { - cwd: gradleRootPath, - encoding: "utf-8", - timeout: TIMEOUT_MS, - maxBuffer: MAX_BUFFER, - }); - if (sresult.status !== 0 || sresult.error) { - if (options.failOnError || DEBUG_MODE) { - console.error(sresult.stdout, sresult.stderr); - } - options.failOnError && process.exit(1); - } - const sstdout = sresult.stdout; - if (sstdout) { - const cmdOutput = Buffer.from(sstdout).toString(); - const parsedList = await parseGradleDep( - cmdOutput, - key, - gradleModules, - gradleRootPath, + const dlist = parsedList.pkgList; + if (parsedList.dependenciesList && parsedList.dependenciesList) { + dependencies = mergeDependencies( + dependencies, + parsedList.dependenciesList, + parentComponent, ); - const dlist = parsedList.pkgList; - if (parsedList.dependenciesList && parsedList.dependenciesList) { - dependencies = mergeDependencies( - dependencies, - parsedList.dependenciesList, - parentComponent, + } + if (dlist?.length) { + if (DEBUG_MODE) { + console.log( + "Found", + dlist.length, + "packages in gradle project", + key, ); } - if (dlist?.length) { - if (DEBUG_MODE) { - console.log( - "Found", - dlist.length, - "packages in gradle project", - key, - ); - } - pkgList = pkgList.concat(dlist); - } + pkgList = pkgList.concat(dlist); } - } // for + } } if (pkgList.length) { if (parentComponent.components?.length) { for (const subProj of parentComponent.components) { pkgList = pkgList.filter( - (pkg) => pkg["bom-ref"] !== subProj["bom-ref"], + (pkg) => + pkg["bom-ref"] !== subProj["bom-ref"] && + pkg["bom-ref"] !== parentComponent["bom-ref"], ); } } diff --git a/lib/helpers/utils.js b/lib/helpers/utils.js index 58ce1b8f0..abec60104 100644 --- a/lib/helpers/utils.js +++ b/lib/helpers/utils.js @@ -2606,62 +2606,35 @@ export async function parseGradleDep( let scope = undefined; let profileName = undefined; if (retMap?.projects) { - if (process.env.GRADLE_MULTI_THREADED) { - const modulesToScan = retMap.projects.filter( - (module) => !gradleModules.has(module.replace(/^:/, "")), + const modulesToScan = retMap.projects.filter( + (module) => !gradleModules.has(module), + ); + if (modulesToScan.length > 0) { + const parallelPropTaskOut = executeParallelGradleProperties( + gradleRootPath, + modulesToScan, + ); + const splitPropTaskOut = splitOutputByGradleProjects( + parallelPropTaskOut, + ["properties"], ); - if (modulesToScan.length > 0) { - const parallelPropTaskOut = executeParallelGradleProperties( - gradleRootPath, - modulesToScan, - ); - const splitPropTaskOut = splitOutputByGradleProjects( - parallelPropTaskOut, - ["properties"], - ); - for (const [key, propTaskOut] of splitPropTaskOut.entries()) { - let propMap = {}; - // To optimize performance and reduce errors do not query for properties - // beyond the first level. Replicating behaviour from single-threaded Gradle generation. - if (key.includes(":")) { - propMap = { - rootProject: key, - projects: [], - metadata: { - version: "latest", - }, - }; - } else { - propMap = parseGradleProperties(propTaskOut); - } - const rootSubProject = propMap.rootProject; - if (rootSubProject) { - const rspName = rootSubProject.replace(/^:/, ""); - gradleModules.set( - rspName, - await buildObjectForGradleModule(rspName, propMap.metadata), - ); - } - } - } - } - const subDependsOn = []; - for (const sd of retMap.projects) { - const moduleName = sd.replace(":", ""); - if (!gradleModules.has(moduleName)) { - const propMap = executeGradleProperties(gradleRootPath, sd); + for (const [key, propTaskOut] of splitPropTaskOut.entries()) { + const propMap = parseGradleProperties(propTaskOut, key); const rootSubProject = propMap.rootProject; if (rootSubProject) { - const rspName = rootSubProject.replace(/^:/, ""); - gradleModules.set( - rspName, - await buildObjectForGradleModule(rspName, propMap.metadata), + const rootSubProjectObj = await buildObjectForGradleModule( + rootSubProject, + propMap.metadata, ); + gradleModules.set(key, rootSubProjectObj); } } - if (gradleModules.has(moduleName)) { - subDependsOn.push(gradleModules.get(moduleName)["bom-ref"]); + } + const subDependsOn = []; + for (const sd of retMap.projects) { + if (gradleModules.has(sd)) { + subDependsOn.push(gradleModules.get(sd)["bom-ref"]); } } level_trees[last_bomref] = subDependsOn; @@ -2722,7 +2695,7 @@ export async function parseGradleDep( const tmpA = rline.split("project "); if (tmpA && tmpA.length > 1) { group = rootProject.group; - name = tmpA[1].split(" ")[0].replace(/^:/, ""); + name = tmpA[1].split(" ")[0]; version = undefined; } } @@ -2735,7 +2708,7 @@ export async function parseGradleDep( purl = new PackageURL( "maven", group !== "project" ? group : rootProject.group, - name, + name.replace(/^:/, ""), version !== undefined ? version : rootProject.version, { type: "jar" }, null, @@ -2945,6 +2918,17 @@ export function parseGradleProjects(rawOutput) { } } } + } else if (l.includes("-> project ")) { + const tmpB = l.split("-> project "); + if (tmpB && tmpB.length > 1) { + const projName = tmpB[1]; + if (projName.startsWith(":")) { + const tmpName = projName.split(" ")[0]; + if (tmpName.length > 1) { + projects.add(tmpName); + } + } + } } }); } @@ -2958,11 +2942,15 @@ export function parseGradleProjects(rawOutput) { * Parse gradle properties output * * @param {string} rawOutput Raw string output + * @param {string} gradleModuleName The name (or 'path') of the module as seen from the root of the project */ -export function parseGradleProperties(rawOutput) { +export function parseGradleProperties(rawOutput, gradleModuleName = null) { let rootProject = "root"; const projects = new Set(); const metadata = { group: "", version: "latest", properties: [] }; + if (gradleModuleName) { + metadata.properties.push({ name: "GradleModule", value: gradleModuleName }); + } if (typeof rawOutput === "string") { const tmpA = rawOutput.split("\n"); tmpA.forEach((l) => { @@ -3020,6 +3008,7 @@ export function executeParallelGradleProperties(dir, allProjectsStr) { cwd: dir, encoding: "utf-8", shell: isWin, + maxBuffer: MAX_BUFFER, }); if (result.status !== 0 || result.error) { if (result.stderr) { @@ -3102,7 +3091,7 @@ export function executeGradleProperties(dir, subProject) { const stdout = result.stdout; if (stdout) { const cmdOutput = Buffer.from(stdout).toString(); - return parseGradleProperties(cmdOutput); + return parseGradleProperties(cmdOutput, subProject); } return {}; } @@ -10235,7 +10224,6 @@ export function splitOutputByGradleProjects(rawOutput, relevantTasks) { if (line.startsWith("Root project '") || line.startsWith("Project ':")) { currentProjectName = line.split("'")[1]; - currentProjectName = currentProjectName.replace(":", ""); outputSplitBySubprojects.set(currentProjectName, ""); } // if previous subProject has ended, push to array and reset subProject string @@ -10275,7 +10263,6 @@ export async function buildObjectForGradleModule(name, metadata) { component.properties = component.properties.concat( metadata.properties, ); - component.properties.push({ name: "GradleProjectName", value: name }); tmpDir = undefined; } else { tmpDir = tmpDir.substring(0, tmpDir.lastIndexOf("/")); diff --git a/lib/helpers/utils.test.js b/lib/helpers/utils.test.js index 1743976f6..04a64577d 100644 --- a/lib/helpers/utils.test.js +++ b/lib/helpers/utils.test.js @@ -178,14 +178,14 @@ test("splits parallel gradle properties output correctly", () => { expect(propOutputSplitBySubProject.size).toEqual(4); expect(propOutputSplitBySubProject.has("dependency-diff-check")).toBe(true); - expect(propOutputSplitBySubProject.has("dependency-diff-check-service")).toBe( - true, - ); expect( - propOutputSplitBySubProject.has("dependency-diff-check-common-core"), + propOutputSplitBySubProject.has(":dependency-diff-check-service"), + ).toBe(true); + expect( + propOutputSplitBySubProject.has(":dependency-diff-check-common-core"), ).toBe(true); expect( - propOutputSplitBySubProject.has("dependency-diff-check-client-starter"), + propOutputSplitBySubProject.has(":dependency-diff-check-client-starter"), ).toBe(true); const retMap = parseGradleProperties( @@ -210,14 +210,14 @@ test("splits parallel gradle dependencies output correctly", async () => { expect(depOutputSplitBySubProject.size).toEqual(4); expect(depOutputSplitBySubProject.has("dependency-diff-check")).toBe(true); - expect(depOutputSplitBySubProject.has("dependency-diff-check-service")).toBe( + expect(depOutputSplitBySubProject.has(":dependency-diff-check-service")).toBe( true, ); expect( - depOutputSplitBySubProject.has("dependency-diff-check-common-core"), + depOutputSplitBySubProject.has(":dependency-diff-check-common-core"), ).toBe(true); expect( - depOutputSplitBySubProject.has("dependency-diff-check-client-starter"), + depOutputSplitBySubProject.has(":dependency-diff-check-client-starter"), ).toBe(true); const retMap = await parseGradleDep( @@ -249,20 +249,20 @@ test("splits parallel custom gradle task outputs correctly", async () => { true, ); expect( - customDepTaskOuputSplitByProject.has("dependency-diff-check-service"), + customDepTaskOuputSplitByProject.has(":dependency-diff-check-service"), ).toBe(true); expect( - customDepTaskOuputSplitByProject.has("dependency-diff-check-common-core"), + customDepTaskOuputSplitByProject.has(":dependency-diff-check-common-core"), ).toBe(true); expect( customDepTaskOuputSplitByProject.has( - "dependency-diff-check-client-starter", + ":dependency-diff-check-client-starter", ), ).toBe(true); const retMap = await parseGradleDep( customDepTaskOuputSplitByProject.get( - "dependency-diff-check-client-starter", + ":dependency-diff-check-client-starter", ), "dependency-diff-check", new Map().set(