feat: Enable configuration of a lower resultsPerPage on NVD API (#6843)

jeremylong · Jul 16, 2024 · ffbc6d8 · ffbc6d8
1 parent ed0e8cb
commit ffbc6d8
Show file tree

Hide file tree

Showing 13 changed files with 66 additions and 1 deletion.
diff --git a/ant/src/main/java/org/owasp/dependencycheck/taskdefs/Update.java b/ant/src/main/java/org/owasp/dependencycheck/taskdefs/Update.java
@@ -70,6 +70,11 @@ public class Update extends Purge {
      */
     private int nvdApiDelay = 0;
 
+    /**
+     * The number of records per page of NVD API data.
+     */
+    private Integer nvdApiResultsPerPage;
+
     /**
      * The Proxy Server.
      */
@@ -287,6 +292,24 @@ public void setNvdApiDelay(int nvdApiDelay) {
         this.nvdApiDelay = nvdApiDelay;
     }
 
+    /**
+     * Get the value of nvdApiResultsPerPage.
+     *
+     * @return the value of nvdApiResultsPerPage
+     */
+    public int getNvdApiResultsPerPage() {
+        return nvdApiResultsPerPage;
+    }
+
+    /**
+     * Set the value of nvdApiResultsPerPage.
+     *
+     * @param nvdApiResultsPerPage new value of nvdApiResultsPerPage
+     */
+    public void setApiResultsPerPage(int nvdApiResultsPerPage) {
+        this.nvdApiResultsPerPage = nvdApiResultsPerPage;
+    }
+
     /**
      * Get the value of proxyServer.
      *
@@ -620,6 +643,7 @@ protected void populateSettings() throws BuildException {
         getSettings().setStringIfNotEmpty(Settings.KEYS.NVD_API_KEY, nvdApiKey);
         getSettings().setStringIfNotEmpty(Settings.KEYS.NVD_API_ENDPOINT, nvdApiEndpoint);
         getSettings().setIntIfNotNull(Settings.KEYS.NVD_API_DELAY, nvdApiDelay);
+        getSettings().setIntIfNotNull(Settings.KEYS.NVD_API_RESULTS_PER_PAGE, nvdApiResultsPerPage);
         getSettings().setStringIfNotEmpty(Settings.KEYS.NVD_API_DATAFEED_URL, nvdDatafeedUrl);
         getSettings().setStringIfNotEmpty(Settings.KEYS.NVD_API_DATAFEED_USER, nvdUser);
         getSettings().setStringIfNotEmpty(Settings.KEYS.NVD_API_DATAFEED_PASSWORD, nvdPassword);

diff --git a/ant/src/site/markdown/config-update.md b/ant/src/site/markdown/config-update.md
@@ -38,6 +38,7 @@ nvdApiKey            | The API Key to access the NVD API; obtained from https://
 nvdApiEndpoint       | The NVD API endpoint URL; setting this is uncommon.                                                                  | https://services.nvd.nist.gov/rest/json/cves/2.0
 nvdMaxRetryCount     | The maximum number of retry requests for a single call to the NVD API.                                               | 10
 nvdApiDelay          | The number of milliseconds to wait between calls to the NVD API.                                                     | 3500 with an NVD API Key or 8000 without an API Key
+nvdApiResultsPerPage | The number records for a single page from NVD API (must be <=2000).                                                  | 2000
 nvdDatafeedUrl       | The URL for the NVD API Data feed that can be generated using https://github.com/jeremylong/Open-Vulnerability-Project/tree/main/vulnz#caching-the-nvd-cve-data - example value `https://internal.server/cache/nvdcve-{0}.json.gz` | &nbsp;
 nvdUser              | Credentials used for basic authentication for the NVD API Data feed.                                                 | &nbsp;
 nvdPassword          | Credentials used for basic authentication for the NVD API Data feed.                                                 | &nbsp;

diff --git a/ant/src/site/markdown/configuration.md b/ant/src/site/markdown/configuration.md
@@ -148,6 +148,7 @@ nvdApiKey            | The API Key to access the NVD API; obtained from https://
 nvdApiEndpoint       | The NVD API endpoint URL; setting this is uncommon.                                                          | https://services.nvd.nist.gov/rest/json/cves/2.0
 nvdMaxRetryCount     | The maximum number of retry requests for a single call to the NVD API.                                       | 10
 nvdApiDelay          | The number of milliseconds to wait between calls to the NVD API.                                             | 3500 with an NVD API Key or 8000 without an API Key
+nvdApiResultsPerPage | The number records for a single page from NVD API (must be <=2000).                                          | 2000
 nvdDatafeedUrl       | The URL for the NVD API Data feed that can be generated using https://github.com/jeremylong/Open-Vulnerability-Project/tree/main/vulnz#caching-the-nvd-cve-data - example value `https://internal.server/cache/nvdcve-{0}.json.gz` | &nbsp;
 nvdUser              | Credentials used for basic authentication for the NVD API Data feed.                                         | &nbsp;
 nvdPassword          | Credentials used for basic authentication for the NVD API Data feed.                                         | &nbsp;

diff --git a/cli/src/main/java/org/owasp/dependencycheck/App.java b/cli/src/main/java/org/owasp/dependencycheck/App.java
@@ -661,6 +661,7 @@ protected void populateSettings(CliParser cli) throws InvalidSettingException {
         settings.setStringIfNotEmpty(Settings.KEYS.NVD_API_ENDPOINT,
                 cli.getStringArgument(CliParser.ARGUMENT.NVD_API_ENDPOINT));
         settings.setIntIfNotNull(Settings.KEYS.NVD_API_DELAY, cli.getIntegerValue(CliParser.ARGUMENT.NVD_API_DELAY));
+        settings.setIntIfNotNull(Settings.KEYS.NVD_API_RESULTS_PER_PAGE, cli.getIntegerValue(CliParser.ARGUMENT.NVD_API_RESULTS_PER_PAGE));
         settings.setStringIfNotEmpty(Settings.KEYS.NVD_API_DATAFEED_URL, cli.getStringArgument(CliParser.ARGUMENT.NVD_API_DATAFEED_URL));
         settings.setStringIfNotEmpty(Settings.KEYS.NVD_API_DATAFEED_USER, cli.getStringArgument(CliParser.ARGUMENT.NVD_API_DATAFEED_USER));
         settings.setStringIfNotEmpty(Settings.KEYS.NVD_API_DATAFEED_PASSWORD, cli.getStringArgument(CliParser.ARGUMENT.NVD_API_DATAFEED_PASSWORD));

diff --git a/cli/src/main/java/org/owasp/dependencycheck/CliParser.java b/cli/src/main/java/org/owasp/dependencycheck/CliParser.java
@@ -148,6 +148,17 @@ private void validateArgs() throws FileNotFoundException, ParseException {
                     throw new ParseException("Invalid Setting: nvdApiDelay must be a number greater than or equal to 0.");
                 }
             }
+            value = line.getOptionValue(ARGUMENT.NVD_API_RESULTS_PER_PAGE);
+            if (value != null) {
+                try {
+                    final int i = Integer.parseInt(value);
+                    if (i <= 0 || i > 2000) {
+                        throw new ParseException("Invalid Setting: nvdApiResultsPerPage must be a number in the range [1, 2000].");
+                    }
+                } catch (NumberFormatException ex) {
+                    throw new ParseException("Invalid Setting: nvdApiResultsPerPage must be a number in the range [1, 2000].");
+                }
+            }
         }
         if (isRunScan()) {
             validatePathExists(getScanFiles(), ARGUMENT.SCAN);
@@ -353,6 +364,8 @@ private void addAdvancedOptions(final Options options) {
                         "Only update the local NVD data cache; no scan will be executed."))
                 .addOption(newOptionWithArg(ARGUMENT.NVD_API_DELAY, "milliseconds",
                         "Time in milliseconds to wait between downloading from the NVD."))
+                .addOption(newOptionWithArg(ARGUMENT.NVD_API_RESULTS_PER_PAGE, "count",
+                        "The number records for a single page from NVD API (must be <=2000)."))
                 .addOption(newOptionWithArg(ARGUMENT.NVD_API_ENDPOINT, "endpoint",
                         "The NVD API Endpoint - setting this is rare."))
                 .addOption(newOptionWithArg(ARGUMENT.NVD_API_DATAFEED_URL, "url",
@@ -1163,6 +1176,10 @@ public static class ARGUMENT {
          * The time in milliseconds to wait between downloading NVD API data.
          */
         public static final String NVD_API_DELAY = "nvdApiDelay";
+        /**
+         * The number records for a single page from NVD API.
+         */
+        public static final String NVD_API_RESULTS_PER_PAGE = "nvdApiResultsPerPage";
         /**
          * The short CLI argument name for setting the location of the data
          * directory.

diff --git a/cli/src/site/markdown/arguments.md b/cli/src/site/markdown/arguments.md
@@ -31,6 +31,7 @@ Advanced Options
 |       | \-\-nvdApiEndpoint                    | \<endpoint\>    | The NVD API endpoint URL; setting this is uncommon.                                                                                                                                                    | https://services.nvd.nist.gov/rest/json/cves/2.0 |
 |       | \-\-nvdMaxRetryCount                  | \<count\>       | The maximum number of retry requests for a single call to the NVD API.                                                                                                                                 | 10            |
 |       | \-\-nvdApiDelay                       | \<milliseconds\>| The number of milliseconds to wait between calls to the NVD API.                                                                                                                                       | 3500 with an NVD API Key or 8000 without an API Key |
+|       | \-\-nvdApiResultsPerPage              | \<number\>      | The number records for a single page from NVD API (must be <=2000).                                                                                                                                    | 2000 |
 |       | \-\-nvdDatafeed                       | \<url\>         | The URL for the NVD API Data feed that can be generated using https://github.com/jeremylong/Open-Vulnerability-Project/tree/main/vulnz#caching-the-nvd-cve-data - example value `https://internal.server/cache/nvdcve-{0}.json.gz` | &nbsp; |
 |       | \-\-nvdUser                           | \<username\>    | Credentials used for basic authentication for the NVD API Data feed.                                                                                                                                   | &nbsp;        |
 |       | \-\-nvdPassword                       | \<password\>    | Credentials used for basic authentication for the NVD API Data feed.                                                                                                                                   | &nbsp;        |

diff --git a/core/src/main/java/org/owasp/dependencycheck/data/update/NvdApiDataSource.java b/core/src/main/java/org/owasp/dependencycheck/data/update/NvdApiDataSource.java
@@ -319,7 +319,10 @@ private boolean processApi() throws UpdateException {
                     + "an NVD API key as the update can take a VERY long time without an API Key");
             builder.withDelay(10000);
         }
-        builder.withResultsPerPage(RESULTS_PER_PAGE);
+
+        final int resultsPerPage = Math.min(settings.getInt(Settings.KEYS.NVD_API_RESULTS_PER_PAGE, RESULTS_PER_PAGE), RESULTS_PER_PAGE);
+
+        builder.withResultsPerPage(resultsPerPage);
         //removed due to the virtualMatch filter causing overhead with the NVD API
         //final String virtualMatch = settings.getString(Settings.KEYS.CVE_CPE_STARTS_WITH_FILTER);
         //if (virtualMatch != null) {

diff --git a/maven/src/main/java/org/owasp/dependencycheck/maven/BaseDependencyCheckMojo.java b/maven/src/main/java/org/owasp/dependencycheck/maven/BaseDependencyCheckMojo.java
@@ -992,6 +992,13 @@ public abstract class BaseDependencyCheckMojo extends AbstractMojo implements Ma
     @Parameter(property = "nvdApiDelay")
     private Integer nvdApiDelay;
 
+    /**
+     * The number records for a single page from NVD API (must be <=2000).
+     */
+    @SuppressWarnings("CanBeFinal")
+    @Parameter(property = "nvdApiResultsPerPage")
+    private Integer nvdApiResultsPerPage;
+
     /**
      * The path to dotnet core.
      */
@@ -2354,6 +2361,7 @@ protected void populateSettings() {
         settings.setStringIfNotEmpty(Settings.KEYS.DB_FILE_NAME, dbFilename);
         settings.setStringIfNotNull(Settings.KEYS.NVD_API_ENDPOINT, nvdApiEndpoint);
         settings.setIntIfNotNull(Settings.KEYS.NVD_API_DELAY, nvdApiDelay);
+        settings.setIntIfNotNull(Settings.KEYS.NVD_API_RESULTS_PER_PAGE, nvdApiResultsPerPage);
         settings.setStringIfNotEmpty(Settings.KEYS.NVD_API_DATAFEED_URL, nvdDatafeedUrl);
         settings.setIntIfNotNull(Settings.KEYS.NVD_API_VALID_FOR_HOURS, nvdValidForHours);
         settings.setIntIfNotNull(Settings.KEYS.NVD_API_MAX_RETRY_COUNT, nvdMaxRetryCount);

diff --git a/maven/src/site/markdown/configuration.md b/maven/src/site/markdown/configuration.md
@@ -150,6 +150,7 @@ nvdApiEndpoint           | The NVD API endpoint URL; setting this is uncommon.
 nvdApiServerId           | The id of a server defined in the settings.xml that configures the credentials (password is used as ApiKey) for accessing the NVD API.     | &nbsp;                                                              |
 nvdMaxRetryCount         | The maximum number of retry requests for a single call to the NVD API.                                                                     | 10                                                                  |
 nvdApiDelay              | The number of milliseconds to wait between calls to the NVD API.                                                                           | 3500 with an NVD API Key or 8000 without an API Key .               |
+nvdApiResultsPerPage     | The number records for a single page from NVD API (must be <=2000).                                                                        | 2000                                                                |
 nvdDatafeedUrl           | The URL for the NVD API Data feed that can be generated using https://github.com/jeremylong/Open-Vulnerability-Project/tree/main/vulnz#caching-the-nvd-cve-data - example value `https://internal.server/cache/nvdcve-{0}.json.gz` | &nbsp;                                         |
 nvdDatafeedServerId      | The id of a server defined in the settings.xml that configures the credentials (username and password) for accessing the NVD API Data feed.| &nbsp;                                                              |
 nvdUser                  | Credentials used for basic authentication for the NVD API Data feed.                                                                       | &nbsp;                                                              |

diff --git a/src/site/markdown/dependency-check-gradle/configuration-aggregate.md b/src/site/markdown/dependency-check-gradle/configuration-aggregate.md
@@ -88,6 +88,7 @@ nvd          | apiKey            | The API Key to access the NVD API; obtained f
 nvd          | endpoint          | The NVD API endpoint URL; setting this is uncommon.                                                          | https://services.nvd.nist.gov/rest/json/cves/2.0                    |
 nvd          | maxRetryCount     | The maximum number of retry requests for a single call to the NVD API.                                       | 10                                                                  |
 nvd          | delay             | The number of milliseconds to wait between calls to the NVD API.                                             | 3500 with an NVD API Key or 8000 without an API Key                 |
+nvd          | resultsPerPage    | The number records for a single page from NVD API (must be <=2000).                                          | 2000                                                                |
 nvd          | datafeedUrl       | The URL for the NVD API Data feed that can be generated using https://github.com/jeremylong/Open-Vulnerability-Project/tree/main/vulnz#caching-the-nvd-cve-data | &nbsp;           |
 nvd          | datafeedUser      | Credentials used for basic authentication for the NVD API Data feed.                                         | &nbsp;                                                              |
 nvd          | datafeedPassword  | Credentials used for basic authentication for the NVD API Data feed.                                         | &nbsp;                                                              |

diff --git a/src/site/markdown/dependency-check-gradle/configuration-update.md b/src/site/markdown/dependency-check-gradle/configuration-update.md
@@ -66,6 +66,7 @@ nvd          | apiKey            | The API Key to access the NVD API; obtained f
 nvd          | endpoint          | The NVD API endpoint URL; setting this is uncommon.                                                          | https://services.nvd.nist.gov/rest/json/cves/2.0                    |
 nvd          | maxRetryCount     | The maximum number of retry requests for a single call to the NVD API.                                       | 10                                                                  |
 nvd          | delay             | The number of milliseconds to wait between calls to the NVD API.                                             | 3500 with an NVD API Key or 8000 without an API Key .               |
+nvd          | resultsPerPage    | The number records for a single page from NVD API (must be <=2000).                                          | 2000                                                                |
 nvd          | datafeedUrl       | The URL for the NVD API Data feed that can be generated using https://github.com/jeremylong/Open-Vulnerability-Project/tree/main/vulnz#caching-the-nvd-cve-data | &nbsp;           |
 nvd          | datafeedUser      | Credentials used for basic authentication for the NVD API Data feed.                                         | &nbsp;                                                              |
 nvd          | datafeedPassword  | Credentials used for basic authentication for the NVD API Data feed.                                         | &nbsp;                                                              |

diff --git a/src/site/markdown/dependency-check-gradle/configuration.md b/src/site/markdown/dependency-check-gradle/configuration.md
@@ -70,6 +70,7 @@ nvd          | apiKey            | The API Key to access the NVD API; obtained f
 nvd          | endpoint          | The NVD API endpoint URL; setting this is uncommon.                                                          | https://services.nvd.nist.gov/rest/json/cves/2.0                            |
 nvd          | maxRetryCount     | The maximum number of retry requests for a single call to the NVD API.                                               | 10                                                                  |
 nvd          | delay             | The number of milliseconds to wait between calls to the NVD API.                                                     | 3500 with an NVD API Key or 8000 without an API Key                 |
+nvd          | resultsPerPage    | The number records for a single page from NVD API (must be <=2000).                                                  | 2000                                                                |
 nvd          | datafeedUrl       | The URL for the NVD API Data feed that can be generated using https://github.com/jeremylong/Open-Vulnerability-Project/tree/main/vulnz#caching-the-nvd-cve-data | &nbsp;                   |
 nvd          | datafeedUser      | Credentials used for basic authentication for the NVD API Data feed.                                                 | &nbsp;                                                              |
 nvd          | datafeedPassword  | Credentials used for basic authentication for the NVD API Data feed.                                                 | &nbsp;                                                              |

diff --git a/utils/src/main/java/org/owasp/dependencycheck/utils/Settings.java b/utils/src/main/java/org/owasp/dependencycheck/utils/Settings.java
@@ -183,6 +183,11 @@ public static final class KEYS {
          * updates.
          */
         public static final String NVD_API_VALID_FOR_HOURS = "nvd.api.check.validforhours";
+        /**
+         * The properties key to control the results per page lower than NVD's default of 2000
+         * See #6863 for the rationale on allowing lower configurations.
+         */
+        public static final String NVD_API_RESULTS_PER_PAGE = "nvd.api.results.per.page";
         /**
          * The properties key that indicates how often the NVD API data feed
          * needs to be updated before a full refresh is evaluated.