[Gradle] Added an option to fully scan 'includedBuilds' (#1388)

.github/workflows/repotests.yml

@@ -461,6 +461,8 @@ jobs:
       - name: repotests elasticsearch
         run: |
           bin/cdxgen.js -t gradle repotests/elasticsearch -o bomresults/bom-elasticsearch.json
+          GRADLE_INCLUDED_BUILDS=:build-conventions,:build-tools,:build-tools-internal bin/cdxgen.js -t gradle repotests/elasticsearch -o bomresults/bom-elasticsearch-with-included-builds.json
+          custom-json-diff -i bomresults/bom-elasticsearch.json bomresults/bom-elasticsearch-with-included-builds.json -o bomresults/diff-elasticsearch
         shell: bash
       - name: jenkins plugins
         run: |

docs/ENV.md

@@ -24,6 +24,7 @@ The following environment variables are available to configure the bom generatio
 | GRADLE_HOME                  | Specify gradle home                                                                                                                                                                                                                                                                                              |
 | GRADLE_CMD                   | Set to override gradle command                                                                                                                                                                                                                                                                                   |
 | GRADLE_DEPENDENCY_TASK       | By default cdxgen use the task "dependencies" to collect packages. Set to override the task name.                                                                                                                                                                                                                |
+| GRADLE_INCLUDED_BUILDS       | Comma-separated list of 'includedBuilds' modules that should be scanned on top of all the modules of your projects. Since includedBuilds can't be found automatically, they have to be listed here. Use gradle-conventions (include the ':'-prefix) for the names.                                               |
 | GRADLE_RESOLVE_FROM_NODE     | If some of your gradle modules are included from node (eg when using expo or react-native), set this to true to use the npm-packages as your dependencies. The big advantage of this, is that the generated purls will be of actually known components (eg in OSS Index) instead of generic names for the packages. |
 | GRADLE_SKIP_MODULES          | Comma-separated list of modules to skip during the "dependencies" task. This can be useful if you have modules that would fail the gradle build, eg when they do not have dependencies in the given configuration. Use "root" if the top most module should be skipped, use their gradle-name (so WITH leading ":") for all others. |
 | SBT_CACHE_DIR                | Specify sbt cache directory. Useful for class name resolving                                                                                                                                                                                                                                                     |

lib/cli/index.js

@@ -54,7 +54,6 @@ import {
   determineSbtVersion,
   dirNameStr,
   encodeForPurl,
-  executeGradleProperties,
   executeParallelGradleProperties,
   extractJarArchive,
   frameworksList,
@@ -1635,16 +1634,50 @@ export async function createJavaBom(path, options) {
     gradleFiles?.length &&
     isPackageManagerAllowed("gradle", ["maven", "bazel", "sbt"], options)
   ) {
-    const retMap = executeGradleProperties(gradleRootPath, null);
-    const allProjectsStr = retMap.projects || [];
-    const rootProject = retMap.rootProject;
-    if (rootProject) {
-      parentComponent = await buildObjectForGradleModule(
-        rootProject,
-        retMap.metadata,
+    let rootProjects = [null];
+    let allProjectsStr = [];
+    let rootGradleModule = {};
+    if (process.env.GRADLE_INCLUDED_BUILDS) {
+      rootProjects = rootProjects.concat(
+        process.env.GRADLE_INCLUDED_BUILDS.split(","),
       );
-      gradleModules.set(rootProject, parentComponent);
     }
+    const parallelPropTaskOut = executeParallelGradleProperties(
+      gradleRootPath,
+      rootProjects,
+    );
+    const splitPropTaskOut = splitOutputByGradleProjects(parallelPropTaskOut, [
+      "properties",
+    ]);
+
+    for (const [key, propTaskOut] of splitPropTaskOut.entries()) {
+      const retMap = parseGradleProperties(propTaskOut);
+      const rootProject = retMap.rootProject;
+      if (rootProject) {
+        const rootComponent = await buildObjectForGradleModule(
+          rootProject,
+          retMap.metadata,
+        );
+        gradleModules.set(key, rootComponent);
+        if (!rootProjects.includes(key)) {
+          if (rootGradleModule.name) {
+            console.error(
+              "Found more than 1 root-components! Maybe you made a mistake in the name of an included-build module?",
+            );
+            throw new Error(
+              "Found more than 1 root-components! Maybe you made a mistake in the name of an included-build module?",
+            );
+          }
+          rootGradleModule = rootComponent;
+        } else if (!allProjectsAddedPurls.includes(rootComponent["purl"])) {
+          allProjects.push(rootComponent);
+          rootDependsOn.push(rootComponent["bom-ref"]);
+          allProjectsAddedPurls.push(rootComponent["purl"]);
+        }
+        allProjectsStr = allProjectsStr.concat(retMap.projects);
+      }
+    }
+    parentComponent = rootGradleModule;
     // Get the sub-project properties and set the root dependencies
     if (allProjectsStr?.length) {
       const parallelPropTaskOut = executeParallelGradleProperties(
@@ -1741,7 +1774,7 @@ export async function createJavaBom(path, options) {
       const perProjectOutput = splitOutputByGradleProjects(cmdOutput, [
         gradleDepTask,
       ]);
-      for (const [key, sp] of gradleModules) {
+      for (const key of gradleModules.keys()) {
         const parsedList = await parseGradleDep(
           perProjectOutput.has(key) ? perProjectOutput.get(key) : "",
           key,

lib/helpers/utils.js

@@ -2955,6 +2955,16 @@ export function parseGradleProperties(rawOutput, gradleModuleName = null) {
     const tmpA = rawOutput.split("\n");
     tmpA.forEach((l) => {
       l = l.replace("\r", "");
+      if (
+        !gradleModuleName &&
+        (l.startsWith("Root project '") || l.startsWith("Project '"))
+      ) {
+        metadata.properties.push({
+          name: "GradleModule",
+          value: l.split("'")[1],
+        });
+        return;
+      }
       if (l.startsWith("----") || l.startsWith(">") || !l.includes(": ")) {
         return;
       }
@@ -2997,7 +3007,9 @@ export function executeParallelGradleProperties(dir, allProjectsStr) {
   const gradleCmd = getGradleCommand(dir, null);
   const gradleArgs = buildGradleCommandArguments(
     process.env.GRADLE_ARGS ? process.env.GRADLE_ARGS.split(" ") : [],
-    allProjectsStr.map((project) => `${project}:properties`),
+    allProjectsStr.map((project) =>
+      project ? `${project}:properties` : "properties",
+    ),
     process.env.GRADLE_ARGS_PROPERTIES
       ? process.env.GRADLE_ARGS_PROPERTIES.split(" ")
       : [],
@@ -3036,66 +3048,6 @@ export function executeParallelGradleProperties(dir, allProjectsStr) {
   return "";
 }
 
-/**
- * Execute gradle properties command and return parsed output
- *
- * @param {string} dir Directory to execute the command
- * @param {string} subProject Sub project name
- */
-export function executeGradleProperties(dir, subProject) {
-  const defaultProps = {
-    rootProject: subProject,
-    projects: [],
-    metadata: {
-      version: "latest",
-    },
-  };
-  const gradleCmd = getGradleCommand(dir, null);
-  const gradleArguments = buildGradleCommandArguments(
-    process.env.GRADLE_ARGS ? process.env.GRADLE_ARGS.split(" ") : [],
-    [subProject ? `${subProject}:properties` : "properties"],
-    process.env.GRADLE_ARGS_PROPERTIES
-      ? process.env.GRADLE_ARGS_PROPERTIES.split(" ")
-      : [],
-  );
-  console.log("Executing", gradleCmd, gradleArguments.join(" "), "in", dir);
-  const result = spawnSync(gradleCmd, gradleArguments, {
-    cwd: dir,
-    encoding: "utf-8",
-    shell: isWin,
-    maxBuffer: 10 * 1024 * 1024,
-  });
-  if (result.status !== 0 || result.error) {
-    if (result.stderr) {
-      if (result.stderr?.includes("does not exist")) {
-        return defaultProps;
-      }
-      console.error(result.stdout, result.stderr);
-      console.log(
-        "1. Check if the correct version of java and gradle are installed and available in PATH. For example, some project might require Java 11 with gradle 7.\n cdxgen container image bundles Java 23 with gradle 8 which might be incompatible.",
-      );
-      console.log(
-        "2. Try running cdxgen with the unofficial JDK11-based image `ghcr.io/appthreat/cdxgen-java:v10`.",
-      );
-      if (result.stderr?.includes("not get unknown property")) {
-        console.log(
-          "3. Check if the SBOM is generated for the correct root project for your application.",
-        );
-      } else if (result.stderr?.includes("Unable to find Git repository")) {
-        console.log(
-          "3. A plugin might be requiring a valid git repository. Retry by cloning this repo or performing 'git init' as a workaround.",
-        );
-      }
-    }
-  }
-  const stdout = result.stdout;
-  if (stdout) {
-    const cmdOutput = Buffer.from(stdout).toString();
-    return parseGradleProperties(cmdOutput, subProject);
-  }
-  return {};
-}
-
 /**
  * Parse bazel action graph output
  * @param {string} rawOutput Raw string output
@@ -10208,7 +10160,7 @@ export function splitOutputByGradleProjects(rawOutput, relevantTasks) {
   const outputSplitBySubprojects = new Map();
   let subProjectOut = "";
   const outSplitByLine = rawOutput.split("\n");
-  let currentProjectName = "";
+  let currentProjectName = "root";
   const regexPatternForRelevantTasks = `.*:(${relevantTasks.join("|")})(?=\s|\r|$)`;
   const regexForRelevantTasks = new RegExp(regexPatternForRelevantTasks);
   for (const [i, line] of outSplitByLine.entries()) {

lib/helpers/utils.test.js

@@ -604,6 +604,10 @@ test("parse gradle properties", () => {
       group: "com.ajmalab",
       version: "0.0.1-SNAPSHOT",
       properties: [
+        {
+          name: "GradleModule",
+          value: "dependency-diff-check",
+        },
         {
           name: "buildFile",
           value:
@@ -632,6 +636,10 @@ test("parse gradle properties", () => {
       group: "com.ajmalab.demo",
       version: "latest",
       properties: [
+        {
+          name: "GradleModule",
+          value: "java-test",
+        },
         {
           name: "buildFile",
           value: "/home/almalinux/work/sandbox/java-test/build.gradle",
@@ -656,6 +664,10 @@ test("parse gradle properties", () => {
       group: "com.ajmalab.demo",
       version: "latest",
       properties: [
+        {
+          name: "GradleModule",
+          value: "java-test",
+        },
         {
           name: "buildFile",
           value: "/home/almalinux/work/sandbox/java-test/build.gradle",