From 58ee055b7224984ccf265fff43d4c2eb1c36555a Mon Sep 17 00:00:00 2001 From: Jan Winz Date: Mon, 16 Oct 2023 09:38:29 +0200 Subject: [PATCH 1/7] Update documentation #2555 --- .../documents/shared/configuration/sechub_config.adoc | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/sechub-doc/src/docs/asciidoc/documents/shared/configuration/sechub_config.adoc b/sechub-doc/src/docs/asciidoc/documents/shared/configuration/sechub_config.adoc index 4bdafef0d6..2bc8b34e28 100644 --- a/sechub-doc/src/docs/asciidoc/documents/shared/configuration/sechub_config.adoc +++ b/sechub-doc/src/docs/asciidoc/documents/shared/configuration/sechub_config.adoc @@ -180,7 +180,9 @@ include::sechub_config_example2_webscan_anonyous.json[] <2> The `URL` to scan. This `URL` must be whitelisted in `{sechub}` project. Normally without a slash `/` at the end. <3> *Optional*: Define includes, if you have a special path that is linked nowhere, so the scanner can not detect it automatically while crawling the application. You can use wildcards by using the symbol `<*>` like in the example above. - To make the scan work the target URL will always be implicitly included with `"https://www.gamechanger.example.org<*>"` if no includes are specified. If includes are specified the scan is limited to this includes. + To make the scan work the target URL will always be implicitly included with `"https://www.gamechanger.example.org<*>"` if no includes are specified. If includes are specified the scan is limited to these includes. + In case you need to include certain parts of your application the scanner cannot detect, + but you want everything else to be scanned as well, please specify a wildcard as include explicitly: `"includes": [ "/hidden/from/crawler/", "/<*>" ]`. - Includes starting with a slash (`/`) like `"includes": [ "/special/include","/special/include/<*>"]` they are interpreted relative to the scan target `URL` provided before. - Includes not starting with a slash (`/`) like `"includes": [ "<*>/en/contacts/<*>","en/contacts/<*>","en/contacts","en/contacts/"`] are interpreted as enclosed by wildcards like the first include in the list example: `"<*>/en/contacts/<*>"`. <4> *Optional*: Define excludes, if you have a special path you want to exclude, from the scan. From 9669527f54e8bd39f1c7628a28eab335f5a1de3d Mon Sep 17 00:00:00 2001 From: Jan Winz Date: Mon, 16 Oct 2023 13:27:17 +0200 Subject: [PATCH 2/7] Zap wrapper handle wildcards for includes/excludes #2555 - implement wildcard handling for includes/excludes - implement tests - change ScanContext and ScanContextFactory to use String instead of URL for includes/excludes to handle wildcards - implement and change test cases for wildcards --- .../zapwrapper/config/ZapScanContext.java | 16 +++--- .../config/ZapScanContextFactory.java | 40 ++++++-------- .../helper/IncludeExcludeToZapURLHelper.java | 52 ++++++++++--------- .../sechub/zapwrapper/scan/ZapScanner.java | 15 +++--- .../util/TargetConnectionChecker.java | 14 +++-- .../sechub/zapwrapper/util/UrlUtil.java | 11 ++-- .../config/ZapScanContextFactoryTest.java | 25 ++++++++- .../IncludeExcludeToZapURLHelperTest.java | 49 ++++++++++------- .../zapwrapper/scan/ZapScannerTest.java | 5 +- .../sechub/zapwrapper/util/UrlUtilTest.java | 6 +-- .../no-auth-include-exclude.json | 8 +-- .../no-auth-without-includes-or-excludes.json | 6 +++ 12 files changed, 145 insertions(+), 102 deletions(-) create mode 100644 sechub-wrapper-owasp-zap/src/test/resources/sechub-config-examples/no-auth-without-includes-or-excludes.json diff --git a/sechub-wrapper-owasp-zap/src/main/java/com/mercedesbenz/sechub/zapwrapper/config/ZapScanContext.java b/sechub-wrapper-owasp-zap/src/main/java/com/mercedesbenz/sechub/zapwrapper/config/ZapScanContext.java index 1717e5c4aa..bf4d2d47e9 100644 --- a/sechub-wrapper-owasp-zap/src/main/java/com/mercedesbenz/sechub/zapwrapper/config/ZapScanContext.java +++ b/sechub-wrapper-owasp-zap/src/main/java/com/mercedesbenz/sechub/zapwrapper/config/ZapScanContext.java @@ -46,8 +46,8 @@ public class ZapScanContext { private List apiDefinitionFiles = new ArrayList<>(); // Using Set here to avoid duplicates - private Set zapURLsIncludeSet = new HashSet<>(); - private Set zapURLsExcludeSet = new HashSet<>(); + private Set zapURLsIncludeSet = new HashSet<>(); + private Set zapURLsExcludeSet = new HashSet<>(); private boolean connectionCheckEnabled; @@ -129,14 +129,14 @@ public List getApiDefinitionFiles() { return apiDefinitionFiles; } - public Set getZapURLsIncludeSet() { + public Set getZapURLsIncludeSet() { if (zapURLsIncludeSet == null) { return Collections.emptySet(); } return zapURLsIncludeSet; } - public Set getZapURLsExcludeSet() { + public Set getZapURLsExcludeSet() { if (zapURLsExcludeSet == null) { return Collections.emptySet(); } @@ -196,8 +196,8 @@ public static class ZapBasicScanContextBuilder { private List apiDefinitionFiles = new LinkedList<>(); // Using Set here to avoid duplicates - private Set zapURLsIncludeSet = new HashSet<>(); - private Set zapURLsExcludeSet = new HashSet<>(); + private Set zapURLsIncludeSet = new HashSet<>(); + private Set zapURLsExcludeSet = new HashSet<>(); private boolean connectionCheckEnabled; @@ -278,12 +278,12 @@ public ZapBasicScanContextBuilder addApiDefinitionFiles(List apiDefinition return this; } - public ZapBasicScanContextBuilder addZapURLsIncludeSet(Set zapURLsIncludeList) { + public ZapBasicScanContextBuilder addZapURLsIncludeSet(Set zapURLsIncludeList) { this.zapURLsIncludeSet.addAll(zapURLsIncludeList); return this; } - public ZapBasicScanContextBuilder addZapURLsExcludeSet(Set zapURLsExcludeList) { + public ZapBasicScanContextBuilder addZapURLsExcludeSet(Set zapURLsExcludeList) { this.zapURLsExcludeSet.addAll(zapURLsExcludeList); return this; } diff --git a/sechub-wrapper-owasp-zap/src/main/java/com/mercedesbenz/sechub/zapwrapper/config/ZapScanContextFactory.java b/sechub-wrapper-owasp-zap/src/main/java/com/mercedesbenz/sechub/zapwrapper/config/ZapScanContextFactory.java index cf2a8c7234..fc7f3c59c4 100644 --- a/sechub-wrapper-owasp-zap/src/main/java/com/mercedesbenz/sechub/zapwrapper/config/ZapScanContextFactory.java +++ b/sechub-wrapper-owasp-zap/src/main/java/com/mercedesbenz/sechub/zapwrapper/config/ZapScanContextFactory.java @@ -4,7 +4,6 @@ import java.io.File; import java.net.URL; import java.util.HashSet; -import java.util.LinkedList; import java.util.List; import java.util.Set; import java.util.UUID; @@ -12,7 +11,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.mercedesbenz.sechub.commons.model.SecHubMessage; import com.mercedesbenz.sechub.commons.model.SecHubScanConfiguration; import com.mercedesbenz.sechub.commons.model.SecHubWebScanConfiguration; import com.mercedesbenz.sechub.zapwrapper.cli.CommandLineSettings; @@ -27,9 +25,9 @@ import com.mercedesbenz.sechub.zapwrapper.helper.SecHubWebScanConfigurationHelper; import com.mercedesbenz.sechub.zapwrapper.helper.ZapPDSEventHandler; import com.mercedesbenz.sechub.zapwrapper.helper.ZapProductMessageHelper; -import com.mercedesbenz.sechub.zapwrapper.helper.ZapURLType; import com.mercedesbenz.sechub.zapwrapper.util.EnvironmentVariableConstants; import com.mercedesbenz.sechub.zapwrapper.util.EnvironmentVariableReader; +import com.mercedesbenz.sechub.zapwrapper.util.UrlUtil; public class ZapScanContextFactory { private static final Logger LOG = LoggerFactory.getLogger(ZapScanContextFactory.class); @@ -87,15 +85,12 @@ public ZapScanContext create(CommandLineSettings settings) { LOG.warn("The job UUID was not set. Using randomly generated UUID: {} as fallback.", contextName); } - List userMessages = new LinkedList<>(); - Set includeSet = createUrlsIncludedInContext(targetUrl, sechubWebConfig, userMessages); - Set excludeSet = createUrlsExcludedFromContext(targetUrl, sechubWebConfig, userMessages); + Set includeSet = createUrlsIncludedInContext(targetUrl, sechubWebConfig); + Set excludeSet = createUrlsExcludedFromContext(targetUrl, sechubWebConfig); ZapProductMessageHelper productMessagehelper = createZapProductMessageHelper(settings); ZapPDSEventHandler zapEventHandler = createZapEventhandler(settings); - checkForIncludeExcludeErrors(userMessages, productMessagehelper); - /* @formatter:off */ ZapScanContext scanContext = ZapScanContext.builder() .setTargetUrl(targetUrl) @@ -217,19 +212,24 @@ private List fetchApiDefinitionFiles(SecHubScanConfiguration sechubScanCon return apiDefinitionFileProvider.fetchApiDefinitionFiles(extractedSourcesFolderPath, sechubScanConfig); } - private Set createUrlsIncludedInContext(URL targetUrl, SecHubWebScanConfiguration sechubWebConfig, List userMessages) { - Set includeSet = new HashSet<>(); - includeSet.add(targetUrl); + private Set createUrlsIncludedInContext(URL targetUrl, SecHubWebScanConfiguration sechubWebConfig) { + Set includeSet = new HashSet<>(); if (sechubWebConfig.getIncludes().isPresent()) { - includeSet.addAll(includeExcludeToZapURLHelper.createListOfUrls(ZapURLType.INCLUDE, targetUrl, sechubWebConfig.getIncludes().get(), userMessages)); + includeSet.addAll(includeExcludeToZapURLHelper.createListOfUrls(targetUrl, sechubWebConfig.getIncludes().get())); + } + // if no includes are specified everything is included + if (includeSet.isEmpty()) { + includeSet.add(targetUrl + UrlUtil.PATTERN_STRING_MATCH_ALL); } + // needed as entry point to start the scan + includeSet.add(targetUrl.toString()); return includeSet; } - private Set createUrlsExcludedFromContext(URL targetUrl, SecHubWebScanConfiguration sechubWebConfig, List userMessages) { - Set excludeSet = new HashSet<>(); + private Set createUrlsExcludedFromContext(URL targetUrl, SecHubWebScanConfiguration sechubWebConfig) { + Set excludeSet = new HashSet<>(); if (sechubWebConfig.getExcludes().isPresent()) { - excludeSet.addAll(includeExcludeToZapURLHelper.createListOfUrls(ZapURLType.EXCLUDE, targetUrl, sechubWebConfig.getExcludes().get(), userMessages)); + excludeSet.addAll(includeExcludeToZapURLHelper.createListOfUrls(targetUrl, sechubWebConfig.getExcludes().get())); } return excludeSet; } @@ -258,14 +258,4 @@ private ZapPDSEventHandler createZapEventhandler(CommandLineSettings settings) { } return new ZapPDSEventHandler(pdsJobEventsFolder); } - - private void checkForIncludeExcludeErrors(List userMessages, ZapProductMessageHelper productMessageHelper) { - if (userMessages == null) { - return; - } - if (userMessages.isEmpty()) { - return; - } - productMessageHelper.writeProductMessages(userMessages); - } } diff --git a/sechub-wrapper-owasp-zap/src/main/java/com/mercedesbenz/sechub/zapwrapper/helper/IncludeExcludeToZapURLHelper.java b/sechub-wrapper-owasp-zap/src/main/java/com/mercedesbenz/sechub/zapwrapper/helper/IncludeExcludeToZapURLHelper.java index 8f9e6dcaef..d7704d571e 100644 --- a/sechub-wrapper-owasp-zap/src/main/java/com/mercedesbenz/sechub/zapwrapper/helper/IncludeExcludeToZapURLHelper.java +++ b/sechub-wrapper-owasp-zap/src/main/java/com/mercedesbenz/sechub/zapwrapper/helper/IncludeExcludeToZapURLHelper.java @@ -1,59 +1,61 @@ // SPDX-License-Identifier: MIT package com.mercedesbenz.sechub.zapwrapper.helper; -import java.net.MalformedURLException; import java.net.URL; +import java.util.Collections; import java.util.LinkedList; import java.util.List; -import com.mercedesbenz.sechub.commons.model.SecHubMessage; -import com.mercedesbenz.sechub.commons.model.SecHubMessageType; +import com.mercedesbenz.sechub.zapwrapper.util.UrlUtil; public class IncludeExcludeToZapURLHelper { + private UrlUtil urlUtil = new UrlUtil(); /** * Combine the targetUrl with all list of subSites.
*
* E.g. for the targetUrl http://localhost:8000 and the sub sites ["/api/v1/", - * "admin/profile"], results in ["http://localhost:8000/api/v1", - * "http://localhost:8000/admin/profile"]. + * "<*>admin/<*>", "api/users/"], results in ["http://localhost:8000/api/v1", + * "http://localhost:8000/.*admin/.*", "http://localhost:8000/.*api/users/.*"]. * - * @param urlType * @param targetUrl * @param subSites - * @param userMessages - * @return a list of full URLs + * @return a list of full URLs, or an empty list if subSites was empty. */ - public List createListOfUrls(ZapURLType urlType, URL targetUrl, List subSites, List userMessages) { + public List createListOfUrls(URL targetUrl, List subSites) { if (subSites == null) { - return new LinkedList(); + return Collections.emptyList(); } String targetUrlAsString = targetUrl.toString(); - List listOfUrls = new LinkedList<>(); + List listOfUrls = new LinkedList<>(); for (String subSite : subSites) { StringBuilder urlBuilder = new StringBuilder(); - + // append the base target url first if (targetUrlAsString.endsWith("/")) { - urlBuilder.append(targetUrlAsString.substring(0, targetUrlAsString.length() - 1)); + urlBuilder.append(targetUrlAsString); } else { urlBuilder.append(targetUrlAsString); - } - - if (!subSite.startsWith("/")) { urlBuilder.append("/"); } - if (subSite.endsWith("/")) { - urlBuilder.append(subSite.substring(0, subSite.length() - 1)); + + // replace wildcards with patterns + String replacedSubsite = urlUtil.replaceWebScanWildCardsWithRegexInString(subSite); + + // create include/exclude URL pattern + if (replacedSubsite.startsWith("/")) { + urlBuilder.append(replacedSubsite.substring(1)); } else { - urlBuilder.append(subSite); - } - try { - listOfUrls.add(new URL(urlBuilder.toString())); - } catch (MalformedURLException e) { - userMessages.add(new SecHubMessage(SecHubMessageType.ERROR, "The specified " + urlType.getId() + " " + subSite - + " combined with the target URL: " + targetUrl + " formed the invalid URL: " + urlBuilder.toString())); + if (!replacedSubsite.startsWith(UrlUtil.PATTERN_STRING_MATCH_ALL)) { + urlBuilder.append(UrlUtil.PATTERN_STRING_MATCH_ALL); + } + urlBuilder.append(replacedSubsite); + + if (!replacedSubsite.endsWith(UrlUtil.PATTERN_STRING_MATCH_ALL)) { + urlBuilder.append(UrlUtil.PATTERN_STRING_MATCH_ALL); + } } + listOfUrls.add(urlBuilder.toString()); } return listOfUrls; } diff --git a/sechub-wrapper-owasp-zap/src/main/java/com/mercedesbenz/sechub/zapwrapper/scan/ZapScanner.java b/sechub-wrapper-owasp-zap/src/main/java/com/mercedesbenz/sechub/zapwrapper/scan/ZapScanner.java index d162e9bcb3..9d9ba75f2a 100644 --- a/sechub-wrapper-owasp-zap/src/main/java/com/mercedesbenz/sechub/zapwrapper/scan/ZapScanner.java +++ b/sechub-wrapper-owasp-zap/src/main/java/com/mercedesbenz/sechub/zapwrapper/scan/ZapScanner.java @@ -4,7 +4,6 @@ import java.io.File; import java.io.IOException; import java.io.UnsupportedEncodingException; -import java.net.URL; import java.net.URLEncoder; import java.nio.charset.StandardCharsets; import java.nio.file.Files; @@ -217,7 +216,7 @@ void addReplacerRulesForHeaders() throws ClientApiException { for (String onlyForUrl : httpHeader.getOnlyForUrls().get()) { // we need to create a rule for each onlyForUrl pattern on each header description = onlyForUrl; - url = urlUtil.replaceWildCardsWithRegexInUrl(onlyForUrl); + url = urlUtil.replaceWebScanWildCardsWithRegexInString(onlyForUrl); clientApiFacade.addReplacerRule(description, enabled, matchtype, matchregex, matchstring, replacement, initiators, url); } } @@ -231,15 +230,15 @@ void addReplacerRulesForHeaders() throws ClientApiException { */ void addIncludedAndExcludedUrlsToContext() throws ClientApiException { LOG.info("For scan {}: Adding include parts.", scanContext.getContextName()); - for (URL url : scanContext.getZapURLsIncludeSet()) { - clientApiFacade.addIncludeUrlPatternToContext(scanContext.getContextName(), url + ".*"); - String followRedirects = "false"; - clientApiFacade.accessUrlViaZap(url.toString(), followRedirects); + String followRedirects = "false"; + for (String url : scanContext.getZapURLsIncludeSet()) { + clientApiFacade.addIncludeUrlPatternToContext(scanContext.getContextName(), url); + clientApiFacade.accessUrlViaZap(url, followRedirects); } LOG.info("For scan {}: Adding exclude parts.", scanContext.getContextName()); - for (URL url : scanContext.getZapURLsExcludeSet()) { - clientApiFacade.addExcludeUrlPatternToContext(scanContext.getContextName(), url + ".*"); + for (String url : scanContext.getZapURLsExcludeSet()) { + clientApiFacade.addExcludeUrlPatternToContext(scanContext.getContextName(), url); } } diff --git a/sechub-wrapper-owasp-zap/src/main/java/com/mercedesbenz/sechub/zapwrapper/util/TargetConnectionChecker.java b/sechub-wrapper-owasp-zap/src/main/java/com/mercedesbenz/sechub/zapwrapper/util/TargetConnectionChecker.java index 6bc270a096..e990804dbe 100644 --- a/sechub-wrapper-owasp-zap/src/main/java/com/mercedesbenz/sechub/zapwrapper/util/TargetConnectionChecker.java +++ b/sechub-wrapper-owasp-zap/src/main/java/com/mercedesbenz/sechub/zapwrapper/util/TargetConnectionChecker.java @@ -4,6 +4,7 @@ import java.io.IOException; import java.net.HttpURLConnection; import java.net.InetSocketAddress; +import java.net.MalformedURLException; import java.net.Proxy; import java.net.URL; import java.security.KeyManagementException; @@ -40,12 +41,19 @@ public class TargetConnectionChecker { public void assertApplicationIsReachable(ZapScanContext scanContext) { boolean isReachable = false; - Iterator iterator = scanContext.getZapURLsIncludeSet().iterator(); + Iterator iterator = scanContext.getZapURLsIncludeSet().iterator(); while (iterator.hasNext() && isReachable == false) { // trying to reach the target URL and all includes until the first reachable // URL is found. - isReachable = isSiteCurrentlyReachable(scanContext, iterator.next(), scanContext.getMaxNumberOfConnectionRetries(), - scanContext.getRetryWaittimeInMilliseconds()); + String nextUrl = iterator.next(); + try { + URL url = new URL(nextUrl); + isReachable = isSiteCurrentlyReachable(scanContext, url, scanContext.getMaxNumberOfConnectionRetries(), + scanContext.getRetryWaittimeInMilliseconds()); + } catch (MalformedURLException e) { + throw new ZapWrapperRuntimeException("URL: " + nextUrl + " is incalid. Cannot check if URL is reachable.", + ZapWrapperExitCode.TARGET_URL_INVALID); + } } if (!isReachable) { // Build error message containing proxy if it was set. diff --git a/sechub-wrapper-owasp-zap/src/main/java/com/mercedesbenz/sechub/zapwrapper/util/UrlUtil.java b/sechub-wrapper-owasp-zap/src/main/java/com/mercedesbenz/sechub/zapwrapper/util/UrlUtil.java index 683ce8f43a..abcf9c3a15 100644 --- a/sechub-wrapper-owasp-zap/src/main/java/com/mercedesbenz/sechub/zapwrapper/util/UrlUtil.java +++ b/sechub-wrapper-owasp-zap/src/main/java/com/mercedesbenz/sechub/zapwrapper/util/UrlUtil.java @@ -6,20 +6,23 @@ import com.mercedesbenz.sechub.commons.model.SecHubWebScanConfiguration; public class UrlUtil { + + public static final String PATTERN_STRING_MATCH_ALL = ".*"; + private static final String QUOTED_WEBSCAN_URL_WILDCARD_SYMBOL = Pattern.quote(SecHubWebScanConfiguration.WEBSCAN_URL_WILDCARD_SYMBOL); private static final Pattern PATTERN_QUOTED_WEBSCAN_URL_WILDCARD_SYMBOL = Pattern.compile(QUOTED_WEBSCAN_URL_WILDCARD_SYMBOL); /** * - * @param onlyForUrl + * @param string * @return URL that contains a regular expression instead of wildcards or an * empty String if the parameter was null. */ - public String replaceWildCardsWithRegexInUrl(String onlyForUrl) { - if (onlyForUrl == null) { + public String replaceWebScanWildCardsWithRegexInString(String string) { + if (string == null) { return ""; } - return PATTERN_QUOTED_WEBSCAN_URL_WILDCARD_SYMBOL.matcher(onlyForUrl).replaceAll(".*"); + return PATTERN_QUOTED_WEBSCAN_URL_WILDCARD_SYMBOL.matcher(string).replaceAll(".*"); } } diff --git a/sechub-wrapper-owasp-zap/src/test/java/com/mercedesbenz/sechub/zapwrapper/config/ZapScanContextFactoryTest.java b/sechub-wrapper-owasp-zap/src/test/java/com/mercedesbenz/sechub/zapwrapper/config/ZapScanContextFactoryTest.java index ecbfec0dbf..d4b10714d6 100644 --- a/sechub-wrapper-owasp-zap/src/test/java/com/mercedesbenz/sechub/zapwrapper/config/ZapScanContextFactoryTest.java +++ b/sechub-wrapper-owasp-zap/src/test/java/com/mercedesbenz/sechub/zapwrapper/config/ZapScanContextFactoryTest.java @@ -14,6 +14,7 @@ import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.never; @@ -487,8 +488,28 @@ void includes_and_excludes_from_sechub_json_are_inside_result() { ZapScanContext result = factoryToTest.create(settings); /* test */ - assertEquals(3, result.getZapURLsIncludeSet().size()); - assertEquals(2, result.getZapURLsExcludeSet().size()); + assertEquals(12, result.getZapURLsIncludeSet().size()); + assertTrue(result.getZapURLsIncludeSet().contains("https://www.targeturl.com")); + assertEquals(11, result.getZapURLsExcludeSet().size()); + } + + @Test + void includes_and_excludes_empty_from_sechub_json_result_in_empty_exclude_and_target_url_as_single_include() { + /* prepare */ + when(ruleProvider.fetchDeactivatedRuleReferences(any())).thenReturn(new DeactivatedRuleReferences()); + CommandLineSettings settings = createSettingsMockWithNecessaryPartsWithoutRuleFiles(); + + File sechubScanConfigFile = new File("src/test/resources/sechub-config-examples/no-auth-without-includes-or-excludes.json"); + when(settings.getSecHubConfigFile()).thenReturn(sechubScanConfigFile); + + /* execute */ + ZapScanContext result = factoryToTest.create(settings); + + /* test */ + assertEquals(2, result.getZapURLsIncludeSet().size()); + assertTrue(result.getZapURLsIncludeSet().contains("https://www.targeturl.com.*")); + assertTrue(result.getZapURLsIncludeSet().contains("https://www.targeturl.com")); + assertEquals(0, result.getZapURLsExcludeSet().size()); } @ParameterizedTest diff --git a/sechub-wrapper-owasp-zap/src/test/java/com/mercedesbenz/sechub/zapwrapper/helper/IncludeExcludeToZapURLHelperTest.java b/sechub-wrapper-owasp-zap/src/test/java/com/mercedesbenz/sechub/zapwrapper/helper/IncludeExcludeToZapURLHelperTest.java index a86e84c45f..59a305ba5d 100644 --- a/sechub-wrapper-owasp-zap/src/test/java/com/mercedesbenz/sechub/zapwrapper/helper/IncludeExcludeToZapURLHelperTest.java +++ b/sechub-wrapper-owasp-zap/src/test/java/com/mercedesbenz/sechub/zapwrapper/helper/IncludeExcludeToZapURLHelperTest.java @@ -1,12 +1,12 @@ // SPDX-License-Identifier: MIT package com.mercedesbenz.sechub.zapwrapper.helper; +import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; import java.net.MalformedURLException; import java.net.URL; import java.util.ArrayList; -import java.util.LinkedList; import java.util.List; import org.junit.jupiter.api.BeforeEach; @@ -14,18 +14,13 @@ import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.ValueSource; -import com.mercedesbenz.sechub.commons.model.SecHubMessage; - class IncludeExcludeToZapURLHelperTest { private IncludeExcludeToZapURLHelper helperToTest; - private List userMessages; - @BeforeEach void beforeEach() { helperToTest = new IncludeExcludeToZapURLHelper(); - userMessages = new LinkedList<>(); } @Test @@ -35,11 +30,10 @@ void list_of_subSites_is_null_returns_empty_list() throws MalformedURLException List sites = null; /* execute */ - List urls = helperToTest.createListOfUrls(ZapURLType.INCLUDE, targetUrl, sites, userMessages); + List urls = helperToTest.createListOfUrls(targetUrl, sites); /* test */ assertTrue(urls.isEmpty()); - assertTrue(userMessages.isEmpty()); } @Test @@ -49,11 +43,10 @@ void list_of_subSites_is_empty_returns_empty_list() throws MalformedURLException List sites = new ArrayList<>(); /* execute */ - List urls = helperToTest.createListOfUrls(ZapURLType.INCLUDE, targetUrl, sites, userMessages); + List urls = helperToTest.createListOfUrls(targetUrl, sites); /* test */ assertTrue(urls.isEmpty()); - assertTrue(userMessages.isEmpty()); } @ParameterizedTest @@ -64,16 +57,26 @@ void list_of_subsites_and_target_url_returns_list_of_combined_urls(String url) t List sites = createExampleListOfSites(); /* execute */ - List urls = helperToTest.createListOfUrls(ZapURLType.EXCLUDE, targetUrl, sites, userMessages); + List urls = helperToTest.createListOfUrls(targetUrl, sites); /* test */ - assertTrue(urls.contains(new URL("https://127.0.0.1:8080/sub"))); - assertTrue(urls.contains(new URL("https://127.0.0.1:8080/sub1/directory"))); - assertTrue(urls.contains(new URL("https://127.0.0.1:8080/sub2/directory"))); - assertTrue(urls.contains(new URL("https://127.0.0.1:8080/sub3/directory"))); - assertTrue(urls.contains(new URL("https://127.0.0.1:8080/sub4/directory"))); - - assertTrue(userMessages.isEmpty()); + assertEquals(15, urls.size()); + assertTrue(urls.contains("https://127.0.0.1:8080/sub/")); + assertTrue(urls.contains("https://127.0.0.1:8080/sub1/directory/")); + assertTrue(urls.contains("https://127.0.0.1:8080/sub2/directory")); + assertTrue(urls.contains("https://127.0.0.1:8080/.*sub3/directory/.*")); + assertTrue(urls.contains("https://127.0.0.1:8080/.*sub4/directory.*")); + assertTrue(urls.contains("https://127.0.0.1:8080/")); + + assertTrue(urls.contains("https://127.0.0.1:8080/")); + assertTrue(urls.contains("https://127.0.0.1:8080/.*")); + assertTrue(urls.contains("https://127.0.0.1:8080/.*")); + assertTrue(urls.contains("https://127.0.0.1:8080/.*/.*")); + assertTrue(urls.contains("https://127.0.0.1:8080/en/contacts/.*")); + assertTrue(urls.contains("https://127.0.0.1:8080/.*/en/contacts/.*")); + assertTrue(urls.contains("https://127.0.0.1:8080/.*/en/.*/.*/contacts/.*")); + assertTrue(urls.contains("https://127.0.0.1:8080/.*/en.*.*contacts/.*")); + assertTrue(urls.contains("https://127.0.0.1:8080/en/contacts/.*")); } private List createExampleListOfSites() { @@ -83,6 +86,16 @@ private List createExampleListOfSites() { subSites.add("/sub2/directory"); subSites.add("sub3/directory/"); subSites.add("sub4/directory"); + subSites.add("/"); + subSites.add("<*>"); + subSites.add("/<*>"); + subSites.add("<*>/<*>"); + subSites.add("/en/contacts/<*>"); + subSites.add("<*>/en/contacts/<*>"); + subSites.add("<*>/en/<*>/contacts/<*>"); + subSites.add("<*>/en/<*>/<*>/contacts/<*>"); + subSites.add("<*>/en<*><*>contacts/<*>"); + subSites.add("en/contacts/<*>"); return subSites; } diff --git a/sechub-wrapper-owasp-zap/src/test/java/com/mercedesbenz/sechub/zapwrapper/scan/ZapScannerTest.java b/sechub-wrapper-owasp-zap/src/test/java/com/mercedesbenz/sechub/zapwrapper/scan/ZapScannerTest.java index a09382e223..3770402538 100644 --- a/sechub-wrapper-owasp-zap/src/test/java/com/mercedesbenz/sechub/zapwrapper/scan/ZapScannerTest.java +++ b/sechub-wrapper-owasp-zap/src/test/java/com/mercedesbenz/sechub/zapwrapper/scan/ZapScannerTest.java @@ -53,7 +53,6 @@ import com.mercedesbenz.sechub.zapwrapper.helper.IncludeExcludeToZapURLHelper; import com.mercedesbenz.sechub.zapwrapper.helper.ZapPDSEventHandler; import com.mercedesbenz.sechub.zapwrapper.helper.ZapProductMessageHelper; -import com.mercedesbenz.sechub.zapwrapper.helper.ZapURLType; import com.mercedesbenz.sechub.zapwrapper.internal.scan.ClientApiFacade; import com.mercedesbenz.sechub.zapwrapper.scan.ZapScanner.UserInformation; import com.mercedesbenz.sechub.zapwrapper.util.SystemUtil; @@ -280,11 +279,11 @@ void set_includes_and_excludes_api_facade_is_called_once_for_each_include_and_on URL targetUrl = sechubWebScanConfig.getUrl().toURL(); List includesList = sechubWebScanConfig.getIncludes().get(); - Set includes = new HashSet<>(helper.createListOfUrls(ZapURLType.INCLUDE, targetUrl, includesList, new ArrayList<>())); + Set includes = new HashSet<>(helper.createListOfUrls(targetUrl, includesList)); when(scanContext.getZapURLsIncludeSet()).thenReturn(includes); List excludesList = sechubWebScanConfig.getExcludes().get(); - Set excludes = new HashSet<>(helper.createListOfUrls(ZapURLType.EXCLUDE, targetUrl, excludesList, new ArrayList<>())); + Set excludes = new HashSet<>(helper.createListOfUrls(targetUrl, excludesList)); when(scanContext.getZapURLsExcludeSet()).thenReturn(excludes); ApiResponse response = mock(ApiResponse.class); diff --git a/sechub-wrapper-owasp-zap/src/test/java/com/mercedesbenz/sechub/zapwrapper/util/UrlUtilTest.java b/sechub-wrapper-owasp-zap/src/test/java/com/mercedesbenz/sechub/zapwrapper/util/UrlUtilTest.java index b258d37fc1..73f88b4f27 100644 --- a/sechub-wrapper-owasp-zap/src/test/java/com/mercedesbenz/sechub/zapwrapper/util/UrlUtilTest.java +++ b/sechub-wrapper-owasp-zap/src/test/java/com/mercedesbenz/sechub/zapwrapper/util/UrlUtilTest.java @@ -23,7 +23,7 @@ void beforeEach() { @EmptySource void null_or_empty_url_leads_to_empty_result(String url) { /* execute */ - String result = urlUtilToTest.replaceWildCardsWithRegexInUrl(url); + String result = urlUtilToTest.replaceWebScanWildCardsWithRegexInString(url); /* test */ assertEquals("", result); @@ -36,7 +36,7 @@ void replacing_sechub_url_wildcard_results_in_url_with_regex() { String expectedUrl = "https://localhost/admin/.*/profile/.*"; /* execute */ - String result = urlUtilToTest.replaceWildCardsWithRegexInUrl(url); + String result = urlUtilToTest.replaceWebScanWildCardsWithRegexInString(url); /* test */ assertEquals(expectedUrl, result); @@ -49,7 +49,7 @@ void url_without_wildcard_is_not_changed_by_util_method() { String expectedUrl = "https://localhost/admin/profile"; /* execute */ - String result = urlUtilToTest.replaceWildCardsWithRegexInUrl(url); + String result = urlUtilToTest.replaceWebScanWildCardsWithRegexInString(url); /* test */ assertEquals(expectedUrl, result); diff --git a/sechub-wrapper-owasp-zap/src/test/resources/sechub-config-examples/no-auth-include-exclude.json b/sechub-wrapper-owasp-zap/src/test/resources/sechub-config-examples/no-auth-include-exclude.json index f5014a34ec..7accc93515 100644 --- a/sechub-wrapper-owasp-zap/src/test/resources/sechub-config-examples/no-auth-include-exclude.json +++ b/sechub-wrapper-owasp-zap/src/test/resources/sechub-config-examples/no-auth-include-exclude.json @@ -1,8 +1,10 @@ { "apiVersion" : "1.0", "webScan" : { - "url" : "https://127.0.0.1:8080", - "includes" : [ "/include", "healthcheck" ], - "excludes" : [ "/logout/", "/hello/" ] + "url" : "https://www.targeturl.com", + "includes" : [ "/", "<*>", "/<*>", "<*>/<*>", "/en/contacts", "/en/contacts/<*>", "<*>/en/contacts/<*>", "<*>/en/<*>/contacts/<*>", + "<*>/en/<*>/<*>/contacts/<*>", "<*>/en<*><*>contacts/<*>", "en/contacts/<*>", "en/contacts", "en/contacts/" ], + "excludes" : [ "/", "<*>", "/<*>", "<*>/<*>", "/en/contacts", "/en/contacts/<*>", "<*>/en/contacts/<*>", "<*>/en/<*>/contacts/<*>", + "<*>/en/<*>/<*>/contacts/<*>", "<*>/en<*><*>contacts/<*>", "en/contacts/<*>", "en/contacts", "en/contacts/" ] } } \ No newline at end of file diff --git a/sechub-wrapper-owasp-zap/src/test/resources/sechub-config-examples/no-auth-without-includes-or-excludes.json b/sechub-wrapper-owasp-zap/src/test/resources/sechub-config-examples/no-auth-without-includes-or-excludes.json new file mode 100644 index 0000000000..0245b868b0 --- /dev/null +++ b/sechub-wrapper-owasp-zap/src/test/resources/sechub-config-examples/no-auth-without-includes-or-excludes.json @@ -0,0 +1,6 @@ +{ + "apiVersion" : "1.0", + "webScan" : { + "url" : "https://127.0.0.1:8080" + } +} \ No newline at end of file From dc7a7898b890b7019e941ea85a5b9d8725c71d02 Mon Sep 17 00:00:00 2001 From: Jan Winz Date: Fri, 20 Oct 2023 10:34:09 +0200 Subject: [PATCH 3/7] Add comments #2555 --- .../zapwrapper/helper/IncludeExcludeToZapURLHelper.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/sechub-wrapper-owasp-zap/src/main/java/com/mercedesbenz/sechub/zapwrapper/helper/IncludeExcludeToZapURLHelper.java b/sechub-wrapper-owasp-zap/src/main/java/com/mercedesbenz/sechub/zapwrapper/helper/IncludeExcludeToZapURLHelper.java index d7704d571e..b934a049f8 100644 --- a/sechub-wrapper-owasp-zap/src/main/java/com/mercedesbenz/sechub/zapwrapper/helper/IncludeExcludeToZapURLHelper.java +++ b/sechub-wrapper-owasp-zap/src/main/java/com/mercedesbenz/sechub/zapwrapper/helper/IncludeExcludeToZapURLHelper.java @@ -5,6 +5,7 @@ import java.util.Collections; import java.util.LinkedList; import java.util.List; +import java.util.Objects; import com.mercedesbenz.sechub.zapwrapper.util.UrlUtil; @@ -12,17 +13,18 @@ public class IncludeExcludeToZapURLHelper { private UrlUtil urlUtil = new UrlUtil(); /** - * Combine the targetUrl with all list of subSites.
+ * Combine the targetUrl with the given list of subSites.
*
* E.g. for the targetUrl http://localhost:8000 and the sub sites ["/api/v1/", * "<*>admin/<*>", "api/users/"], results in ["http://localhost:8000/api/v1", * "http://localhost:8000/.*admin/.*", "http://localhost:8000/.*api/users/.*"]. * - * @param targetUrl + * @param targetUrl, must never be null * @param subSites * @return a list of full URLs, or an empty list if subSites was empty. */ public List createListOfUrls(URL targetUrl, List subSites) { + Objects.requireNonNull(targetUrl); if (subSites == null) { return Collections.emptyList(); } From 50a468433d54dce5c7e0bda6e2062b29317195d4 Mon Sep 17 00:00:00 2001 From: Jan Winz Date: Fri, 20 Oct 2023 10:49:23 +0200 Subject: [PATCH 4/7] Documentation updates #2555 --- .../sechub/zapwrapper/helper/IncludeExcludeToZapURLHelper.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sechub-wrapper-owasp-zap/src/main/java/com/mercedesbenz/sechub/zapwrapper/helper/IncludeExcludeToZapURLHelper.java b/sechub-wrapper-owasp-zap/src/main/java/com/mercedesbenz/sechub/zapwrapper/helper/IncludeExcludeToZapURLHelper.java index b934a049f8..763aab8f98 100644 --- a/sechub-wrapper-owasp-zap/src/main/java/com/mercedesbenz/sechub/zapwrapper/helper/IncludeExcludeToZapURLHelper.java +++ b/sechub-wrapper-owasp-zap/src/main/java/com/mercedesbenz/sechub/zapwrapper/helper/IncludeExcludeToZapURLHelper.java @@ -19,7 +19,7 @@ public class IncludeExcludeToZapURLHelper { * "<*>admin/<*>", "api/users/"], results in ["http://localhost:8000/api/v1", * "http://localhost:8000/.*admin/.*", "http://localhost:8000/.*api/users/.*"]. * - * @param targetUrl, must never be null + * @param targetUrl, must never be null * @param subSites * @return a list of full URLs, or an empty list if subSites was empty. */ From 4b21ecf7842efd87202e11a9bcac95669c446de0 Mon Sep 17 00:00:00 2001 From: Jan Winz Date: Fri, 20 Oct 2023 13:40:38 +0200 Subject: [PATCH 5/7] make returned list unmodifiable and rename constant #2555 --- .../zapwrapper/config/ZapScanContextFactory.java | 2 +- .../helper/IncludeExcludeToZapURLHelper.java | 12 ++++++------ .../mercedesbenz/sechub/zapwrapper/util/UrlUtil.java | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/sechub-wrapper-owasp-zap/src/main/java/com/mercedesbenz/sechub/zapwrapper/config/ZapScanContextFactory.java b/sechub-wrapper-owasp-zap/src/main/java/com/mercedesbenz/sechub/zapwrapper/config/ZapScanContextFactory.java index fc7f3c59c4..720d48691c 100644 --- a/sechub-wrapper-owasp-zap/src/main/java/com/mercedesbenz/sechub/zapwrapper/config/ZapScanContextFactory.java +++ b/sechub-wrapper-owasp-zap/src/main/java/com/mercedesbenz/sechub/zapwrapper/config/ZapScanContextFactory.java @@ -219,7 +219,7 @@ private Set createUrlsIncludedInContext(URL targetUrl, SecHubWebScanConf } // if no includes are specified everything is included if (includeSet.isEmpty()) { - includeSet.add(targetUrl + UrlUtil.PATTERN_STRING_MATCH_ALL); + includeSet.add(targetUrl + UrlUtil.REGEX_PATTERN_WILDCARD_STRING); } // needed as entry point to start the scan includeSet.add(targetUrl.toString()); diff --git a/sechub-wrapper-owasp-zap/src/main/java/com/mercedesbenz/sechub/zapwrapper/helper/IncludeExcludeToZapURLHelper.java b/sechub-wrapper-owasp-zap/src/main/java/com/mercedesbenz/sechub/zapwrapper/helper/IncludeExcludeToZapURLHelper.java index 763aab8f98..510afdd3e9 100644 --- a/sechub-wrapper-owasp-zap/src/main/java/com/mercedesbenz/sechub/zapwrapper/helper/IncludeExcludeToZapURLHelper.java +++ b/sechub-wrapper-owasp-zap/src/main/java/com/mercedesbenz/sechub/zapwrapper/helper/IncludeExcludeToZapURLHelper.java @@ -21,7 +21,7 @@ public class IncludeExcludeToZapURLHelper { * * @param targetUrl, must never be null * @param subSites - * @return a list of full URLs, or an empty list if subSites was empty. + * @return an unmodifiable list of full URLs, or an empty list if subSites was empty. */ public List createListOfUrls(URL targetUrl, List subSites) { Objects.requireNonNull(targetUrl); @@ -48,18 +48,18 @@ public List createListOfUrls(URL targetUrl, List subSites) { if (replacedSubsite.startsWith("/")) { urlBuilder.append(replacedSubsite.substring(1)); } else { - if (!replacedSubsite.startsWith(UrlUtil.PATTERN_STRING_MATCH_ALL)) { - urlBuilder.append(UrlUtil.PATTERN_STRING_MATCH_ALL); + if (!replacedSubsite.startsWith(UrlUtil.REGEX_PATTERN_WILDCARD_STRING)) { + urlBuilder.append(UrlUtil.REGEX_PATTERN_WILDCARD_STRING); } urlBuilder.append(replacedSubsite); - if (!replacedSubsite.endsWith(UrlUtil.PATTERN_STRING_MATCH_ALL)) { - urlBuilder.append(UrlUtil.PATTERN_STRING_MATCH_ALL); + if (!replacedSubsite.endsWith(UrlUtil.REGEX_PATTERN_WILDCARD_STRING)) { + urlBuilder.append(UrlUtil.REGEX_PATTERN_WILDCARD_STRING); } } listOfUrls.add(urlBuilder.toString()); } - return listOfUrls; + return Collections.unmodifiableList(listOfUrls); } } diff --git a/sechub-wrapper-owasp-zap/src/main/java/com/mercedesbenz/sechub/zapwrapper/util/UrlUtil.java b/sechub-wrapper-owasp-zap/src/main/java/com/mercedesbenz/sechub/zapwrapper/util/UrlUtil.java index abcf9c3a15..4118f0c849 100644 --- a/sechub-wrapper-owasp-zap/src/main/java/com/mercedesbenz/sechub/zapwrapper/util/UrlUtil.java +++ b/sechub-wrapper-owasp-zap/src/main/java/com/mercedesbenz/sechub/zapwrapper/util/UrlUtil.java @@ -7,7 +7,7 @@ public class UrlUtil { - public static final String PATTERN_STRING_MATCH_ALL = ".*"; + public static final String REGEX_PATTERN_WILDCARD_STRING = ".*"; private static final String QUOTED_WEBSCAN_URL_WILDCARD_SYMBOL = Pattern.quote(SecHubWebScanConfiguration.WEBSCAN_URL_WILDCARD_SYMBOL); private static final Pattern PATTERN_QUOTED_WEBSCAN_URL_WILDCARD_SYMBOL = Pattern.compile(QUOTED_WEBSCAN_URL_WILDCARD_SYMBOL); From 273a587693986189b7348534ec92b518e52bceff Mon Sep 17 00:00:00 2001 From: Jan Winz Date: Fri, 20 Oct 2023 13:44:05 +0200 Subject: [PATCH 6/7] execute spotlessApply #2555 --- .../sechub/zapwrapper/helper/IncludeExcludeToZapURLHelper.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sechub-wrapper-owasp-zap/src/main/java/com/mercedesbenz/sechub/zapwrapper/helper/IncludeExcludeToZapURLHelper.java b/sechub-wrapper-owasp-zap/src/main/java/com/mercedesbenz/sechub/zapwrapper/helper/IncludeExcludeToZapURLHelper.java index 510afdd3e9..fb24719728 100644 --- a/sechub-wrapper-owasp-zap/src/main/java/com/mercedesbenz/sechub/zapwrapper/helper/IncludeExcludeToZapURLHelper.java +++ b/sechub-wrapper-owasp-zap/src/main/java/com/mercedesbenz/sechub/zapwrapper/helper/IncludeExcludeToZapURLHelper.java @@ -21,7 +21,8 @@ public class IncludeExcludeToZapURLHelper { * * @param targetUrl, must never be null * @param subSites - * @return an unmodifiable list of full URLs, or an empty list if subSites was empty. + * @return an unmodifiable list of full URLs, or an empty list if subSites was + * empty. */ public List createListOfUrls(URL targetUrl, List subSites) { Objects.requireNonNull(targetUrl); From 1f19183a3645a9273a2f5f471f56f4ed290143a6 Mon Sep 17 00:00:00 2001 From: Jan Winz Date: Tue, 24 Oct 2023 16:18:13 +0200 Subject: [PATCH 7/7] PR review changes #2555 --- .../sechub/zapwrapper/util/TargetConnectionChecker.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sechub-wrapper-owasp-zap/src/main/java/com/mercedesbenz/sechub/zapwrapper/util/TargetConnectionChecker.java b/sechub-wrapper-owasp-zap/src/main/java/com/mercedesbenz/sechub/zapwrapper/util/TargetConnectionChecker.java index e990804dbe..bd094fcaac 100644 --- a/sechub-wrapper-owasp-zap/src/main/java/com/mercedesbenz/sechub/zapwrapper/util/TargetConnectionChecker.java +++ b/sechub-wrapper-owasp-zap/src/main/java/com/mercedesbenz/sechub/zapwrapper/util/TargetConnectionChecker.java @@ -51,7 +51,7 @@ public void assertApplicationIsReachable(ZapScanContext scanContext) { isReachable = isSiteCurrentlyReachable(scanContext, url, scanContext.getMaxNumberOfConnectionRetries(), scanContext.getRetryWaittimeInMilliseconds()); } catch (MalformedURLException e) { - throw new ZapWrapperRuntimeException("URL: " + nextUrl + " is incalid. Cannot check if URL is reachable.", + throw new ZapWrapperRuntimeException("URL: " + nextUrl + " is invalid. Cannot check if URL is reachable.", ZapWrapperExitCode.TARGET_URL_INVALID); } }