Merge pull request #2603 from mercedes-benz/feature-2555-zap-wrapper-…

…handle-includes-excludes-wildcards

Feature 2555 zap wrapper handle includes excludes wildcards

sechub-doc/src/docs/asciidoc/documents/shared/configuration/sechub_config.adoc

@@ -180,7 +180,9 @@ include::sechub_config_example2_webscan_anonyous.json[]
 <2> The `URL` to scan. This `URL` must be whitelisted in `{sechub}` project. Normally without a slash `/` at the end.
 <3> *Optional*: Define includes, if you have a special path that is linked nowhere,
 	so the scanner can not detect it automatically while crawling the application. You can use wildcards by using the symbol `<*>` like in the example above.
-	To make the scan work the target URL will always be implicitly included with `"https://www.gamechanger.example.org<*>"` if no includes are specified. If includes are specified the scan is limited to this includes.
+	To make the scan work the target URL will always be implicitly included with `"https://www.gamechanger.example.org<*>"` if no includes are specified. If includes are specified the scan is limited to these includes.
+	In case you need to include certain parts of your application the scanner cannot detect,
+	but you want everything else to be scanned as well, please specify a wildcard as include explicitly: `"includes": [ "/hidden/from/crawler/", "/<*>" ]`.
 	- Includes starting with a slash (`/`) like `"includes": [ "/special/include","/special/include/<*>"]` they are interpreted relative to the scan target `URL` provided before. 
 	- Includes not starting with a slash (`/`) like `"includes": [ "<*>/en/contacts/<*>","en/contacts/<*>","en/contacts","en/contacts/"`] are interpreted as enclosed by wildcards like the first include in the list example: `"<*>/en/contacts/<*>"`. 
 <4> *Optional*: Define excludes, if you have a special path you want to exclude, from the scan.

sechub-wrapper-owasp-zap/src/main/java/com/mercedesbenz/sechub/zapwrapper/config/ZapScanContext.java

@@ -46,8 +46,8 @@ public class ZapScanContext {
     private List<File> apiDefinitionFiles = new ArrayList<>();
 
     // Using Set here to avoid duplicates
-    private Set<URL> zapURLsIncludeSet = new HashSet<>();
-    private Set<URL> zapURLsExcludeSet = new HashSet<>();
+    private Set<String> zapURLsIncludeSet = new HashSet<>();
+    private Set<String> zapURLsExcludeSet = new HashSet<>();
 
     private boolean connectionCheckEnabled;
 
@@ -129,14 +129,14 @@ public List<File> getApiDefinitionFiles() {
         return apiDefinitionFiles;
     }
 
-    public Set<URL> getZapURLsIncludeSet() {
+    public Set<String> getZapURLsIncludeSet() {
         if (zapURLsIncludeSet == null) {
             return Collections.emptySet();
         }
         return zapURLsIncludeSet;
     }
 
-    public Set<URL> getZapURLsExcludeSet() {
+    public Set<String> getZapURLsExcludeSet() {
         if (zapURLsExcludeSet == null) {
             return Collections.emptySet();
         }
@@ -196,8 +196,8 @@ public static class ZapBasicScanContextBuilder {
         private List<File> apiDefinitionFiles = new LinkedList<>();
 
         // Using Set here to avoid duplicates
-        private Set<URL> zapURLsIncludeSet = new HashSet<>();
-        private Set<URL> zapURLsExcludeSet = new HashSet<>();
+        private Set<String> zapURLsIncludeSet = new HashSet<>();
+        private Set<String> zapURLsExcludeSet = new HashSet<>();
 
         private boolean connectionCheckEnabled;
 
@@ -278,12 +278,12 @@ public ZapBasicScanContextBuilder addApiDefinitionFiles(List<File> apiDefinition
             return this;
         }
 
-        public ZapBasicScanContextBuilder addZapURLsIncludeSet(Set<URL> zapURLsIncludeList) {
+        public ZapBasicScanContextBuilder addZapURLsIncludeSet(Set<String> zapURLsIncludeList) {
             this.zapURLsIncludeSet.addAll(zapURLsIncludeList);
             return this;
         }
 
-        public ZapBasicScanContextBuilder addZapURLsExcludeSet(Set<URL> zapURLsExcludeList) {
+        public ZapBasicScanContextBuilder addZapURLsExcludeSet(Set<String> zapURLsExcludeList) {
             this.zapURLsExcludeSet.addAll(zapURLsExcludeList);
             return this;
         }

sechub-wrapper-owasp-zap/src/main/java/com/mercedesbenz/sechub/zapwrapper/config/ZapScanContextFactory.java

@@ -4,15 +4,13 @@
 import java.io.File;
 import java.net.URL;
 import java.util.HashSet;
-import java.util.LinkedList;
 import java.util.List;
 import java.util.Set;
 import java.util.UUID;
 
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import com.mercedesbenz.sechub.commons.model.SecHubMessage;
 import com.mercedesbenz.sechub.commons.model.SecHubScanConfiguration;
 import com.mercedesbenz.sechub.commons.model.SecHubWebScanConfiguration;
 import com.mercedesbenz.sechub.zapwrapper.cli.CommandLineSettings;
@@ -27,9 +25,9 @@
 import com.mercedesbenz.sechub.zapwrapper.helper.SecHubWebScanConfigurationHelper;
 import com.mercedesbenz.sechub.zapwrapper.helper.ZapPDSEventHandler;
 import com.mercedesbenz.sechub.zapwrapper.helper.ZapProductMessageHelper;
-import com.mercedesbenz.sechub.zapwrapper.helper.ZapURLType;
 import com.mercedesbenz.sechub.zapwrapper.util.EnvironmentVariableConstants;
 import com.mercedesbenz.sechub.zapwrapper.util.EnvironmentVariableReader;
+import com.mercedesbenz.sechub.zapwrapper.util.UrlUtil;
 
 public class ZapScanContextFactory {
     private static final Logger LOG = LoggerFactory.getLogger(ZapScanContextFactory.class);
@@ -87,15 +85,12 @@ public ZapScanContext create(CommandLineSettings settings) {
             LOG.warn("The job UUID was not set. Using randomly generated UUID: {} as fallback.", contextName);
         }
 
-        List<SecHubMessage> userMessages = new LinkedList<>();
-        Set<URL> includeSet = createUrlsIncludedInContext(targetUrl, sechubWebConfig, userMessages);
-        Set<URL> excludeSet = createUrlsExcludedFromContext(targetUrl, sechubWebConfig, userMessages);
+        Set<String> includeSet = createUrlsIncludedInContext(targetUrl, sechubWebConfig);
+        Set<String> excludeSet = createUrlsExcludedFromContext(targetUrl, sechubWebConfig);
 
         ZapProductMessageHelper productMessagehelper = createZapProductMessageHelper(settings);
         ZapPDSEventHandler zapEventHandler = createZapEventhandler(settings);
 
-        checkForIncludeExcludeErrors(userMessages, productMessagehelper);
-
         /* @formatter:off */
 		ZapScanContext scanContext = ZapScanContext.builder()
 												.setTargetUrl(targetUrl)
@@ -217,19 +212,24 @@ private List<File> fetchApiDefinitionFiles(SecHubScanConfiguration sechubScanCon
         return apiDefinitionFileProvider.fetchApiDefinitionFiles(extractedSourcesFolderPath, sechubScanConfig);
     }
 
-    private Set<URL> createUrlsIncludedInContext(URL targetUrl, SecHubWebScanConfiguration sechubWebConfig, List<SecHubMessage> userMessages) {
-        Set<URL> includeSet = new HashSet<>();
-        includeSet.add(targetUrl);
+    private Set<String> createUrlsIncludedInContext(URL targetUrl, SecHubWebScanConfiguration sechubWebConfig) {
+        Set<String> includeSet = new HashSet<>();
         if (sechubWebConfig.getIncludes().isPresent()) {
-            includeSet.addAll(includeExcludeToZapURLHelper.createListOfUrls(ZapURLType.INCLUDE, targetUrl, sechubWebConfig.getIncludes().get(), userMessages));
+            includeSet.addAll(includeExcludeToZapURLHelper.createListOfUrls(targetUrl, sechubWebConfig.getIncludes().get()));
+        }
+        // if no includes are specified everything is included
+        if (includeSet.isEmpty()) {
+            includeSet.add(targetUrl + UrlUtil.REGEX_PATTERN_WILDCARD_STRING);
         }
+        // needed as entry point to start the scan
+        includeSet.add(targetUrl.toString());
         return includeSet;
     }
 
-    private Set<URL> createUrlsExcludedFromContext(URL targetUrl, SecHubWebScanConfiguration sechubWebConfig, List<SecHubMessage> userMessages) {
-        Set<URL> excludeSet = new HashSet<>();
+    private Set<String> createUrlsExcludedFromContext(URL targetUrl, SecHubWebScanConfiguration sechubWebConfig) {
+        Set<String> excludeSet = new HashSet<>();
         if (sechubWebConfig.getExcludes().isPresent()) {
-            excludeSet.addAll(includeExcludeToZapURLHelper.createListOfUrls(ZapURLType.EXCLUDE, targetUrl, sechubWebConfig.getExcludes().get(), userMessages));
+            excludeSet.addAll(includeExcludeToZapURLHelper.createListOfUrls(targetUrl, sechubWebConfig.getExcludes().get()));
         }
         return excludeSet;
     }
@@ -258,14 +258,4 @@ private ZapPDSEventHandler createZapEventhandler(CommandLineSettings settings) {
         }
         return new ZapPDSEventHandler(pdsJobEventsFolder);
     }
-
-    private void checkForIncludeExcludeErrors(List<SecHubMessage> userMessages, ZapProductMessageHelper productMessageHelper) {
-        if (userMessages == null) {
-            return;
-        }
-        if (userMessages.isEmpty()) {
-            return;
-        }
-        productMessageHelper.writeProductMessages(userMessages);
-    }
 }

sechub-wrapper-owasp-zap/src/main/java/com/mercedesbenz/sechub/zapwrapper/helper/IncludeExcludeToZapURLHelper.java

@@ -1,61 +1,66 @@
 // SPDX-License-Identifier: MIT
 package com.mercedesbenz.sechub.zapwrapper.helper;
 
-import java.net.MalformedURLException;
 import java.net.URL;
+import java.util.Collections;
 import java.util.LinkedList;
 import java.util.List;
+import java.util.Objects;
 
-import com.mercedesbenz.sechub.commons.model.SecHubMessage;
-import com.mercedesbenz.sechub.commons.model.SecHubMessageType;
+import com.mercedesbenz.sechub.zapwrapper.util.UrlUtil;
 
 public class IncludeExcludeToZapURLHelper {
+    private UrlUtil urlUtil = new UrlUtil();
 
     /**
-     * Combine the targetUrl with all list of subSites.<br>
+     * Combine the targetUrl with the given list of subSites.<br>
      * <br>
      * E.g. for the targetUrl http://localhost:8000 and the sub sites ["/api/v1/",
-     * "admin/profile"], results in ["http://localhost:8000/api/v1",
-     * "http://localhost:8000/admin/profile"].
+     * "<*>admin/<*>", "api/users/"], results in ["http://localhost:8000/api/v1",
+     * "http://localhost:8000/.*admin/.*", "http://localhost:8000/.*api/users/.*"].
      *
-     * @param urlType
-     * @param targetUrl
+     * @param targetUrl, must never be <code>null</code>
      * @param subSites
-     * @param userMessages
-     * @return a list of full URLs
+     * @return an unmodifiable list of full URLs, or an empty list if subSites was
+     *         empty.
      */
-    public List<URL> createListOfUrls(ZapURLType urlType, URL targetUrl, List<String> subSites, List<SecHubMessage> userMessages) {
+    public List<String> createListOfUrls(URL targetUrl, List<String> subSites) {
+        Objects.requireNonNull(targetUrl);
         if (subSites == null) {
-            return new LinkedList<URL>();
+            return Collections.emptyList();
         }
 
         String targetUrlAsString = targetUrl.toString();
-        List<URL> listOfUrls = new LinkedList<>();
+        List<String> listOfUrls = new LinkedList<>();
         for (String subSite : subSites) {
             StringBuilder urlBuilder = new StringBuilder();
-
+            // append the base target url first
             if (targetUrlAsString.endsWith("/")) {
-                urlBuilder.append(targetUrlAsString.substring(0, targetUrlAsString.length() - 1));
+                urlBuilder.append(targetUrlAsString);
             } else {
                 urlBuilder.append(targetUrlAsString);
-            }
-
-            if (!subSite.startsWith("/")) {
                 urlBuilder.append("/");
             }
-            if (subSite.endsWith("/")) {
-                urlBuilder.append(subSite.substring(0, subSite.length() - 1));
+
+            // replace wildcards with patterns
+            String replacedSubsite = urlUtil.replaceWebScanWildCardsWithRegexInString(subSite);
+
+            // create include/exclude URL pattern
+            if (replacedSubsite.startsWith("/")) {
+                urlBuilder.append(replacedSubsite.substring(1));
             } else {
-                urlBuilder.append(subSite);
-            }
-            try {
-                listOfUrls.add(new URL(urlBuilder.toString()));
-            } catch (MalformedURLException e) {
-                userMessages.add(new SecHubMessage(SecHubMessageType.ERROR, "The specified " + urlType.getId() + " " + subSite
-                        + " combined with the target URL: " + targetUrl + " formed the invalid URL: " + urlBuilder.toString()));
+                if (!replacedSubsite.startsWith(UrlUtil.REGEX_PATTERN_WILDCARD_STRING)) {
+                    urlBuilder.append(UrlUtil.REGEX_PATTERN_WILDCARD_STRING);
+                }
+                urlBuilder.append(replacedSubsite);
+
+                if (!replacedSubsite.endsWith(UrlUtil.REGEX_PATTERN_WILDCARD_STRING)) {
+                    urlBuilder.append(UrlUtil.REGEX_PATTERN_WILDCARD_STRING);
+                }
             }
+            listOfUrls.add(urlBuilder.toString());
         }
-        return listOfUrls;
+        return Collections.unmodifiableList(listOfUrls);
     }
 
 }

sechub-wrapper-owasp-zap/src/main/java/com/mercedesbenz/sechub/zapwrapper/scan/ZapScanner.java

@@ -4,7 +4,6 @@
 import java.io.File;
 import java.io.IOException;
 import java.io.UnsupportedEncodingException;
-import java.net.URL;
 import java.net.URLEncoder;
 import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
@@ -217,7 +216,7 @@ void addReplacerRulesForHeaders() throws ClientApiException {
                 for (String onlyForUrl : httpHeader.getOnlyForUrls().get()) {
                     // we need to create a rule for each onlyForUrl pattern on each header
                     description = onlyForUrl;
-                    url = urlUtil.replaceWildCardsWithRegexInUrl(onlyForUrl);
+                    url = urlUtil.replaceWebScanWildCardsWithRegexInString(onlyForUrl);
                     clientApiFacade.addReplacerRule(description, enabled, matchtype, matchregex, matchstring, replacement, initiators, url);
                 }
             }
@@ -231,15 +230,15 @@ void addReplacerRulesForHeaders() throws ClientApiException {
      */
     void addIncludedAndExcludedUrlsToContext() throws ClientApiException {
         LOG.info("For scan {}: Adding include parts.", scanContext.getContextName());
-        for (URL url : scanContext.getZapURLsIncludeSet()) {
-            clientApiFacade.addIncludeUrlPatternToContext(scanContext.getContextName(), url + ".*");
-            String followRedirects = "false";
-            clientApiFacade.accessUrlViaZap(url.toString(), followRedirects);
+        String followRedirects = "false";
+        for (String url : scanContext.getZapURLsIncludeSet()) {
+            clientApiFacade.addIncludeUrlPatternToContext(scanContext.getContextName(), url);
+            clientApiFacade.accessUrlViaZap(url, followRedirects);
         }
 
         LOG.info("For scan {}: Adding exclude parts.", scanContext.getContextName());
-        for (URL url : scanContext.getZapURLsExcludeSet()) {
-            clientApiFacade.addExcludeUrlPatternToContext(scanContext.getContextName(), url + ".*");
+        for (String url : scanContext.getZapURLsExcludeSet()) {
+            clientApiFacade.addExcludeUrlPatternToContext(scanContext.getContextName(), url);
         }
     }
 

sechub-wrapper-owasp-zap/src/main/java/com/mercedesbenz/sechub/zapwrapper/util/TargetConnectionChecker.java

@@ -4,6 +4,7 @@
 import java.io.IOException;
 import java.net.HttpURLConnection;
 import java.net.InetSocketAddress;
+import java.net.MalformedURLException;
 import java.net.Proxy;
 import java.net.URL;
 import java.security.KeyManagementException;
@@ -40,12 +41,19 @@ public class TargetConnectionChecker {
 
     public void assertApplicationIsReachable(ZapScanContext scanContext) {
         boolean isReachable = false;
-        Iterator<URL> iterator = scanContext.getZapURLsIncludeSet().iterator();
+        Iterator<String> iterator = scanContext.getZapURLsIncludeSet().iterator();
         while (iterator.hasNext() && isReachable == false) {
             // trying to reach the target URL and all includes until the first reachable
             // URL is found.
-            isReachable = isSiteCurrentlyReachable(scanContext, iterator.next(), scanContext.getMaxNumberOfConnectionRetries(),
-                    scanContext.getRetryWaittimeInMilliseconds());
+            String nextUrl = iterator.next();
+            try {
+                URL url = new URL(nextUrl);
+                isReachable = isSiteCurrentlyReachable(scanContext, url, scanContext.getMaxNumberOfConnectionRetries(),
+                        scanContext.getRetryWaittimeInMilliseconds());
+            } catch (MalformedURLException e) {
+                throw new ZapWrapperRuntimeException("URL: " + nextUrl + " is invalid. Cannot check if URL is reachable.",
+                        ZapWrapperExitCode.TARGET_URL_INVALID);
+            }
         }
         if (!isReachable) {
             // Build error message containing proxy if it was set.

sechub-wrapper-owasp-zap/src/main/java/com/mercedesbenz/sechub/zapwrapper/util/UrlUtil.java

@@ -6,20 +6,23 @@
 import com.mercedesbenz.sechub.commons.model.SecHubWebScanConfiguration;
 
 public class UrlUtil {
+
+    public static final String REGEX_PATTERN_WILDCARD_STRING = ".*";
+
     private static final String QUOTED_WEBSCAN_URL_WILDCARD_SYMBOL = Pattern.quote(SecHubWebScanConfiguration.WEBSCAN_URL_WILDCARD_SYMBOL);
     private static final Pattern PATTERN_QUOTED_WEBSCAN_URL_WILDCARD_SYMBOL = Pattern.compile(QUOTED_WEBSCAN_URL_WILDCARD_SYMBOL);
 
     /**
      *
-     * @param onlyForUrl
+     * @param string
      * @return URL that contains a regular expression instead of wildcards or an
      *         empty String if the parameter was <code>null</code>.
      */
-    public String replaceWildCardsWithRegexInUrl(String onlyForUrl) {
-        if (onlyForUrl == null) {
+    public String replaceWebScanWildCardsWithRegexInString(String string) {
+        if (string == null) {
             return "";
         }
-        return PATTERN_QUOTED_WEBSCAN_URL_WILDCARD_SYMBOL.matcher(onlyForUrl).replaceAll(".*");
+        return PATTERN_QUOTED_WEBSCAN_URL_WILDCARD_SYMBOL.matcher(string).replaceAll(".*");
     }
 
 }

sechub-wrapper-owasp-zap/src/test/java/com/mercedesbenz/sechub/zapwrapper/config/ZapScanContextFactoryTest.java

@@ -14,6 +14,7 @@
 import static org.junit.jupiter.api.Assertions.assertNotNull;
 import static org.junit.jupiter.api.Assertions.assertNull;
 import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
 import static org.mockito.ArgumentMatchers.any;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.never;
@@ -487,8 +488,28 @@ void includes_and_excludes_from_sechub_json_are_inside_result() {
         ZapScanContext result = factoryToTest.create(settings);
 
         /* test */
-        assertEquals(3, result.getZapURLsIncludeSet().size());
-        assertEquals(2, result.getZapURLsExcludeSet().size());
+        assertEquals(12, result.getZapURLsIncludeSet().size());
+        assertTrue(result.getZapURLsIncludeSet().contains("https://www.targeturl.com"));
+        assertEquals(11, result.getZapURLsExcludeSet().size());
+    }
+
+    @Test
+    void includes_and_excludes_empty_from_sechub_json_result_in_empty_exclude_and_target_url_as_single_include() {
+        /* prepare */
+        when(ruleProvider.fetchDeactivatedRuleReferences(any())).thenReturn(new DeactivatedRuleReferences());
+        CommandLineSettings settings = createSettingsMockWithNecessaryPartsWithoutRuleFiles();
+
+        File sechubScanConfigFile = new File("src/test/resources/sechub-config-examples/no-auth-without-includes-or-excludes.json");
+        when(settings.getSecHubConfigFile()).thenReturn(sechubScanConfigFile);
+
+        /* execute */
+        ZapScanContext result = factoryToTest.create(settings);
+
+        /* test */
+        assertEquals(2, result.getZapURLsIncludeSet().size());
+        assertTrue(result.getZapURLsIncludeSet().contains("https://www.targeturl.com.*"));
+        assertTrue(result.getZapURLsIncludeSet().contains("https://www.targeturl.com"));
+        assertEquals(0, result.getZapURLsExcludeSet().size());
     }
 
     @ParameterizedTest