diff --git a/apps/api-runtime/src/main/java/com/akto/dependency/DependencyAnalyser.java b/apps/api-runtime/src/main/java/com/akto/dependency/DependencyAnalyser.java index 05ac9bdc5a..f8d19b12fb 100644 --- a/apps/api-runtime/src/main/java/com/akto/dependency/DependencyAnalyser.java +++ b/apps/api-runtime/src/main/java/com/akto/dependency/DependencyAnalyser.java @@ -29,6 +29,7 @@ import java.util.*; import static com.akto.util.HttpRequestResponseUtils.extractValuesFromPayload; +import static com.akto.runtime.utils.Utils.parseCookie; public class DependencyAnalyser { Store valueStore; // this is to store all the values seen in response payload @@ -131,7 +132,7 @@ public void analyse(String message, int finalApiCollectionId) { for (String param: responseHeaders.keySet()) { List values = responseHeaders.get(param); if (param.equalsIgnoreCase("set-cookie")) { - Map cookieMap = AuthPolicy.parseCookie(values); + Map cookieMap = parseCookie(values); for (String cookieKey: cookieMap.keySet()) { String cookieVal = cookieMap.get(cookieKey); if (!filterValues(cookieVal)) continue; @@ -192,7 +193,7 @@ public void analyse(String message, int finalApiCollectionId) { List values = requestHeaders.get(param); if (param.equals("cookie")) { - Map cookieMap = AuthPolicy.parseCookie(values); + Map cookieMap = parseCookie(values); for (String cookieKey: cookieMap.keySet()) { String cookieValue = cookieMap.get(cookieKey); processRequestParam(cookieKey, new HashSet<>(Collections.singletonList(cookieValue)), combinedUrl, false, true, doInterCollectionMatch); diff --git a/apps/api-runtime/src/main/java/com/akto/parsers/HttpCallParser.java b/apps/api-runtime/src/main/java/com/akto/parsers/HttpCallParser.java index 2295b08f33..46fd0d3f6b 100644 --- a/apps/api-runtime/src/main/java/com/akto/parsers/HttpCallParser.java +++ b/apps/api-runtime/src/main/java/com/akto/parsers/HttpCallParser.java @@ -8,19 +8,26 @@ import com.akto.dao.traffic_metrics.TrafficMetricsDao; import com.akto.dependency.DependencyAnalyser; import com.akto.dto.*; +import com.akto.dto.ApiInfo.ApiInfoKey; import com.akto.dto.billing.FeatureAccess; import com.akto.dto.billing.SyncLimit; +import com.akto.dto.monitoring.FilterConfig; import com.akto.dto.billing.Organization; import com.akto.dto.settings.DefaultPayload; import com.akto.dto.traffic_metrics.TrafficMetrics; +import com.akto.dto.type.URLMethods.Method; import com.akto.dto.usage.MetricTypes; import com.akto.graphql.GraphQLUtils; import com.akto.log.LoggerMaker; import com.akto.log.LoggerMaker.LogDb; import com.akto.runtime.APICatalogSync; import com.akto.runtime.Main; +import com.akto.runtime.RuntimeUtil; import com.akto.runtime.URLAggregator; import com.akto.runtime.parser.SampleParser; +import com.akto.runtime.utils.Utils; +import com.akto.test_editor.execution.VariableResolver; +import com.akto.test_editor.filter.data_operands_impl.ValidationResult; import com.akto.usage.UsageMetricCalculator; import com.akto.util.JSONUtils; import com.akto.util.http_util.CoreHTTPClient; @@ -32,11 +39,11 @@ import com.mongodb.client.model.*; import okhttp3.*; import org.apache.commons.lang3.StringUtils; -import org.apache.commons.lang3.math.NumberUtils; import org.bson.conversions.Bson; import java.io.IOException; import java.util.*; +import java.util.Map.Entry; import java.util.concurrent.ConcurrentLinkedQueue; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; @@ -45,6 +52,7 @@ import java.util.regex.Pattern; import static com.akto.runtime.RuntimeUtil.matchesDefaultPayload; +import static com.akto.testing.Utils.validateFilter; public class HttpCallParser { private final int sync_threshold_count; @@ -162,13 +170,64 @@ public int createCollectionBasedOnHostName(int id, String host) throws Exceptio int numberOfSyncs = 0; + private List applyAdvancedFilters(List responseParams){ + Map filterMap = apiCatalogSync.advancedFilterMap; + + if (filterMap != null && !filterMap.isEmpty()) { + List filteredParams = new ArrayList<>(); + for (HttpResponseParams responseParam : responseParams) { + for (Entry apiFilterEntry : filterMap.entrySet()) { + try { + FilterConfig apiFilter = apiFilterEntry.getValue(); + String message = responseParam.getOrig(); + RawApi rawApi = RawApi.buildFromMessage(message); + int apiCollectionId = createApiCollectionId(responseParam); + responseParam.requestParams.setApiCollectionId(apiCollectionId); + String url = responseParam.getRequestParams().getURL(); + Method method = Method.valueOf(responseParam.getRequestParams().getMethod()); + ApiInfoKey apiInfoKey = new ApiInfoKey(apiCollectionId, url, method); + Map varMap = apiFilter.resolveVarMap(); + VariableResolver.resolveWordList(varMap, new HashMap>() { + { + put(apiInfoKey, Arrays.asList(message)); + } + }, apiInfoKey); + String filterExecutionLogId = UUID.randomUUID().toString(); + ValidationResult res = validateFilter(apiFilter.getFilter().getNode(), rawApi, + apiInfoKey, varMap, filterExecutionLogId); + if (res.getIsValid()) { + // TODO apply execution + + filteredParams.add(responseParam); + break; + } + } catch (Exception e) { + loggerMaker.errorAndAddToDb(e, String.format("Error in httpCallFilter %s", e.toString())); + return responseParams; + } + } + } + return filteredParams; + } + return responseParams; + } + public void syncFunction(List responseParams, boolean syncImmediately, boolean fetchAllSTI, AccountSettings accountSettings) { // USE ONLY filteredResponseParams and not responseParams List filteredResponseParams = responseParams; if (accountSettings != null && accountSettings.getDefaultPayloads() != null) { filteredResponseParams = filterDefaultPayloads(filteredResponseParams, accountSettings.getDefaultPayloads()); } - filteredResponseParams = filterHttpResponseParams(filteredResponseParams, accountSettings); + List redundantList = new ArrayList<>(); + if(accountSettings !=null && !accountSettings.getAllowRedundantEndpointsList().isEmpty()){ + redundantList = accountSettings.getAllowRedundantEndpointsList(); + } + Pattern regexPattern = Utils.createRegexPatternFromList(redundantList); + filteredResponseParams = filterHttpResponseParams(filteredResponseParams, redundantList, regexPattern); + + // add advanced filters + filteredResponseParams = applyAdvancedFilters(filteredResponseParams); + boolean isHarOrPcap = aggregate(filteredResponseParams, aggregatorMap); for (int apiCollectionId: aggregatorMap.keySet()) { @@ -336,17 +395,7 @@ public void incTrafficMetrics(TrafficMetrics.Key key, int value) { public static final String CONTENT_TYPE = "CONTENT-TYPE"; - public boolean isRedundantEndpoint(String url, List discardedUrlList){ - StringJoiner joiner = new StringJoiner("|", ".*\\.(", ")(\\?.*)?"); - for (String extension : discardedUrlList) { - if(extension.startsWith(CONTENT_TYPE)){ - continue; - } - joiner.add(extension); - } - String regex = joiner.toString(); - - Pattern pattern = Pattern.compile(regex); + public boolean isRedundantEndpoint(String url, Pattern pattern){ Matcher matcher = pattern.matcher(url); return matcher.matches(); } @@ -359,6 +408,54 @@ private boolean isInvalidContentType(String contentType){ return res; } + public int createApiCollectionId(HttpResponseParams httpResponseParam){ + int apiCollectionId; + String hostName = getHeaderValue(httpResponseParam.getRequestParams().getHeaders(), "host"); + + if (hostName != null && !hostNameToIdMap.containsKey(hostName) && RuntimeUtil.hasSpecialCharacters(hostName)) { + hostName = "Special_Char_Host"; + } + + int vxlanId = httpResponseParam.requestParams.getApiCollectionId(); + + if (useHostCondition(hostName, httpResponseParam.getSource())) { + hostName = hostName.toLowerCase(); + hostName = hostName.trim(); + + String key = hostName; + + if (hostNameToIdMap.containsKey(key)) { + apiCollectionId = hostNameToIdMap.get(key); + + } else { + int id = hostName.hashCode(); + try { + + apiCollectionId = createCollectionBasedOnHostName(id, hostName); + + hostNameToIdMap.put(key, apiCollectionId); + } catch (Exception e) { + loggerMaker.errorAndAddToDb("Failed to create collection for host : " + hostName, LogDb.RUNTIME); + createCollectionSimple(vxlanId); + hostNameToIdMap.put("null " + vxlanId, vxlanId); + apiCollectionId = httpResponseParam.requestParams.getApiCollectionId(); + } + } + + } else { + String key = "null" + " " + vxlanId; + if (!hostNameToIdMap.containsKey(key)) { + createCollectionSimple(vxlanId); + hostNameToIdMap.put(key, vxlanId); + } + + apiCollectionId = vxlanId; + } + return apiCollectionId; + } + + + private boolean isBlankResponseBodyForGET(String method, String contentType, String matchContentType, String responseBody) { boolean res = true; @@ -381,7 +478,7 @@ private boolean isBlankResponseBodyForGET(String method, String contentType, Str return res; } - public List filterHttpResponseParams(List httpResponseParamsList, AccountSettings accountSettings) { + public List filterHttpResponseParams(List httpResponseParamsList, List redundantUrlsList, Pattern pattern) { List filteredResponseParams = new ArrayList<>(); int originalSize = httpResponseParamsList.size(); for (HttpResponseParams httpResponseParam: httpResponseParamsList) { @@ -402,8 +499,8 @@ public List filterHttpResponseParams(List contentTypeList = (List) httpResponseParam.getRequestParams().getHeaders().getOrDefault("content-type", new ArrayList<>()); @@ -421,7 +518,7 @@ public List filterHttpResponseParams(List filterHttpResponseParams(List> reqHeaders = httpResponseParam.getRequestParams().getHeaders(); - String hostName = getHeaderValue(reqHeaders, "host"); - - if (StringUtils.isEmpty(hostName)) { - hostName = getHeaderValue(reqHeaders, "authority"); - if (StringUtils.isEmpty(hostName)) { - hostName = getHeaderValue(reqHeaders, ":authority"); - } - - if (!StringUtils.isEmpty(hostName)) { - reqHeaders.put("host", Collections.singletonList(hostName)); - } - } - - int vxlanId = httpResponseParam.requestParams.getApiCollectionId(); - int apiCollectionId ; - - if (useHostCondition(hostName, httpResponseParam.getSource())) { - hostName = hostName.toLowerCase(); - hostName = hostName.trim(); - - String key = hostName; - - if (hostNameToIdMap.containsKey(key)) { - apiCollectionId = hostNameToIdMap.get(key); - - } else { - int id = hostName.hashCode(); - try { - - apiCollectionId = createCollectionBasedOnHostName(id, hostName); - - hostNameToIdMap.put(key, apiCollectionId); - } catch (Exception e) { - loggerMaker.errorAndAddToDb("Failed to create collection for host : " + hostName, LogDb.RUNTIME); - createCollectionSimple(vxlanId); - hostNameToIdMap.put("null " + vxlanId, vxlanId); - apiCollectionId = httpResponseParam.requestParams.getApiCollectionId(); - } - } - - } else { - String key = "null" + " " + vxlanId; - if (!hostNameToIdMap.containsKey(key)) { - createCollectionSimple(vxlanId); - hostNameToIdMap.put(key, vxlanId); - } - - apiCollectionId = vxlanId; - } - + int apiCollectionId = createApiCollectionId(httpResponseParam); + httpResponseParam.requestParams.setApiCollectionId(apiCollectionId); List responseParamsList = GraphQLUtils.getUtils().parseGraphqlResponseParam(httpResponseParam); diff --git a/apps/api-runtime/src/main/java/com/akto/runtime/APICatalogSync.java b/apps/api-runtime/src/main/java/com/akto/runtime/APICatalogSync.java index 544ba667e3..012546bc8e 100644 --- a/apps/api-runtime/src/main/java/com/akto/runtime/APICatalogSync.java +++ b/apps/api-runtime/src/main/java/com/akto/runtime/APICatalogSync.java @@ -7,9 +7,13 @@ import com.akto.DaoInit; import com.akto.dao.*; import com.akto.dao.context.Context; +import com.akto.dao.monitoring.FilterYamlTemplateDao; +import com.akto.dao.runtime_filters.AdvancedTrafficFiltersDao; import com.akto.dto.*; import com.akto.dto.billing.SyncLimit; import com.akto.dto.dependency_flow.DependencyFlow; +import com.akto.dto.monitoring.FilterConfig; +import com.akto.dto.test_editor.YamlTemplate; import com.akto.dto.traffic.Key; import com.akto.dto.traffic.SampleData; import com.akto.dto.traffic.TrafficInfo; @@ -66,6 +70,7 @@ public class APICatalogSync { public Map sensitiveParamInfoBooleanMap; public static boolean mergeAsyncOutside = true; public BloomFilter existingAPIsInDb = BloomFilter.create(Funnels.stringFunnel(Charsets.UTF_8), 1_000_000, 0.001 ); + public Map advancedFilterMap = new HashMap<>(); public APICatalogSync(String userIdentifier,int thresh, boolean fetchAllSTI) { this(userIdentifier, thresh, fetchAllSTI, true); @@ -1534,7 +1539,8 @@ public void buildFromDB(boolean calcDiff, boolean fetchAllSTI) { this.delta = new HashMap<>(); } - + List advancedFilterTemplates = AdvancedTrafficFiltersDao.instance.findAll(Filters.ne(YamlTemplate.INACTIVE, true)); + advancedFilterMap = FilterYamlTemplateDao.instance.fetchFilterConfig(false, advancedFilterTemplates, true); try { // fetchAllSTI check added to make sure only runs in dashboard if (!fetchAllSTI) { diff --git a/apps/api-runtime/src/main/java/com/akto/runtime/RelationshipSync.java b/apps/api-runtime/src/main/java/com/akto/runtime/RelationshipSync.java index f13c58d553..d098746587 100644 --- a/apps/api-runtime/src/main/java/com/akto/runtime/RelationshipSync.java +++ b/apps/api-runtime/src/main/java/com/akto/runtime/RelationshipSync.java @@ -12,8 +12,6 @@ import com.fasterxml.jackson.databind.node.ArrayNode; import com.mongodb.client.model.*; import org.bson.conversions.Bson; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import com.akto.dto.HttpResponseParams; import com.akto.dto.HttpRequestParams; @@ -231,37 +229,7 @@ public static Map> extractAllValuesFromPayload(String payloa public static void extractAllValuesFromPayload(JsonNode node, List params, Map> values) { // TODO: null values remove - if (node == null) return; - if (node.isValueNode()) { - String textValue = node.asText(); - if (textValue != null) { - String param = String.join("",params); - if (param.startsWith("#")) { - param = param.substring(1); - } - if (!values.containsKey(param)) { - values.put(param, new HashSet<>()); - } - values.get(param).add(textValue); - } - } else if (node.isArray()) { - ArrayNode arrayNode = (ArrayNode) node; - for(int i = 0; i < arrayNode.size(); i++) { - JsonNode arrayElement = arrayNode.get(i); - params.add("#$"); - extractAllValuesFromPayload(arrayElement, params, values); - params.remove(params.size()-1); - } - } else { - Iterator fieldNames = node.fieldNames(); - while(fieldNames.hasNext()) { - String fieldName = fieldNames.next(); - params.add("#"+fieldName); - JsonNode fieldValue = node.get(fieldName); - extractAllValuesFromPayload(fieldValue, params,values); - params.remove(params.size()-1); - } - } + com.akto.runtime.RuntimeUtil.extractAllValuesFromPayload(node,params,values); } diff --git a/apps/api-runtime/src/main/java/com/akto/runtime/policies/AuthPolicy.java b/apps/api-runtime/src/main/java/com/akto/runtime/policies/AuthPolicy.java index 1b1a130247..d78a991232 100644 --- a/apps/api-runtime/src/main/java/com/akto/runtime/policies/AuthPolicy.java +++ b/apps/api-runtime/src/main/java/com/akto/runtime/policies/AuthPolicy.java @@ -12,6 +12,7 @@ import org.slf4j.LoggerFactory; import java.util.*; +import static com.akto.runtime.utils.Utils.parseCookie; public class AuthPolicy { @@ -33,25 +34,6 @@ private static List findBearerBasicAuth(String header, String return new ArrayList<>(); } - public static Map parseCookie(List cookieList){ - Map cookieMap = new HashMap<>(); - if(cookieList==null)return cookieMap; - for (String cookieValues : cookieList) { - String[] cookies = cookieValues.split(";"); - for (String cookie : cookies) { - cookie=cookie.trim(); - String[] cookieFields = cookie.split("="); - boolean twoCookieFields = cookieFields.length == 2; - if (twoCookieFields) { - if(!cookieMap.containsKey(cookieFields[0])){ - cookieMap.put(cookieFields[0], cookieFields[1]); - } - } - } - } - return cookieMap; - } - public static boolean findAuthType(HttpResponseParams httpResponseParams, ApiInfo apiInfo, RuntimeFilter filter, List customAuthTypes) { Set> allAuthTypesFound = apiInfo.getAllAuthTypesFound(); if (allAuthTypesFound == null) allAuthTypesFound = new HashSet<>(); diff --git a/apps/api-runtime/src/main/java/com/akto/utils/CustomAuthUtil.java b/apps/api-runtime/src/main/java/com/akto/utils/CustomAuthUtil.java index 3a70e4b03b..771a123515 100644 --- a/apps/api-runtime/src/main/java/com/akto/utils/CustomAuthUtil.java +++ b/apps/api-runtime/src/main/java/com/akto/utils/CustomAuthUtil.java @@ -18,9 +18,9 @@ import com.akto.dto.type.SingleTypeInfo; import com.akto.log.LoggerMaker; import com.akto.log.LoggerMaker.LogDb; -import com.akto.runtime.policies.AuthPolicy; import static com.akto.dto.ApiInfo.ALL_AUTH_TYPES_FOUND; +import static com.akto.runtime.utils.Utils.parseCookie; public class CustomAuthUtil { @@ -97,7 +97,7 @@ public static void customAuthTypeUtil(List customAuthTypes){ } SingleTypeInfo cookieSTI = SingleTypeInfoDao.instance.findOne(getFilters(apiInfo, true, COOKIE_LIST)); if(cookieSTI!=null){ - Map cookieMap = AuthPolicy.parseCookie(new ArrayList<>(cookieSTI.getValues().getElements())); + Map cookieMap = parseCookie(new ArrayList<>(cookieSTI.getValues().getElements())); headerAndCookieKeys.addAll(cookieMap.keySet()); } diff --git a/apps/api-runtime/src/main/java/com/akto/utils/RedactSampleData.java b/apps/api-runtime/src/main/java/com/akto/utils/RedactSampleData.java index e276c87263..eed330db4e 100644 --- a/apps/api-runtime/src/main/java/com/akto/utils/RedactSampleData.java +++ b/apps/api-runtime/src/main/java/com/akto/utils/RedactSampleData.java @@ -16,11 +16,11 @@ import com.fasterxml.jackson.databind.node.ArrayNode; import com.fasterxml.jackson.databind.node.ObjectNode; import com.fasterxml.jackson.databind.node.TextNode; -import com.mongodb.BasicDBObject; import java.util.*; import static com.akto.dto.RawApi.convertHeaders; +import static com.akto.runtime.utils.Utils.parseCookie; public class RedactSampleData { static ObjectMapper mapper = new ObjectMapper(); @@ -43,7 +43,7 @@ public static String redactDataTypes(String sample) throws Exception{ public static String redactCookie(Map> headers, String header) { String cookie = ""; List cookieList = headers.getOrDefault(header, new ArrayList<>()); - Map cookieMap = AuthPolicy.parseCookie(cookieList); + Map cookieMap = parseCookie(cookieList); for (String cookieKey : cookieMap.keySet()) { cookie += cookieKey + "=" + redactValue + ";"; } @@ -186,32 +186,6 @@ public static void change(String parentName, JsonNode parent, String newValue, b } } - - public static String convertOriginalReqRespToString(OriginalHttpRequest request, OriginalHttpResponse response) { - BasicDBObject req = new BasicDBObject(); - if (request != null) { - req.put("url", request.getUrl()); - req.put("method", request.getMethod()); - req.put("type", request.getType()); - req.put("queryParams", request.getQueryParams()); - req.put("body", request.getBody()); - req.put("headers", convertHeaders(request.getHeaders())); - } - - BasicDBObject resp = new BasicDBObject(); - if (response != null) { - resp.put("statusCode", response.getStatusCode()); - resp.put("body", response.getBody()); - resp.put("headers", convertHeaders(response.getHeaders())); - } - - BasicDBObject ret = new BasicDBObject(); - ret.put("request", req); - ret.put("response", resp); - - return ret.toString(); - } - public static String convertHttpRespToOriginalString(HttpResponseParams httpResponseParams) throws JsonProcessingException { Map m = new HashMap<>(); HttpRequestParams httpRequestParams = httpResponseParams.getRequestParams(); diff --git a/apps/api-runtime/src/test/java/com/akto/parsers/TestDBSync.java b/apps/api-runtime/src/test/java/com/akto/parsers/TestDBSync.java index 416e766186..c1b99d5f9a 100644 --- a/apps/api-runtime/src/test/java/com/akto/parsers/TestDBSync.java +++ b/apps/api-runtime/src/test/java/com/akto/parsers/TestDBSync.java @@ -4,6 +4,7 @@ import static org.junit.jupiter.api.Assertions.assertFalse; import java.util.*; +import java.util.regex.Pattern; import com.akto.MongoBasedTest; import com.akto.dao.*; @@ -25,6 +26,7 @@ import com.akto.runtime.APICatalogSync; import com.akto.runtime.Main; import com.akto.runtime.URLAggregator; +import com.akto.runtime.utils.Utils; import com.mongodb.BasicDBObject; import com.mongodb.client.model.Filters; @@ -254,7 +256,7 @@ public void testInitialiseFilters() throws InterruptedException { @Test public void testFilterHttpResponseParamsEmpty() { HttpCallParser httpCallParser = new HttpCallParser("",0,0,0, true); - List ss = httpCallParser.filterHttpResponseParams(new ArrayList<>(), null); + List ss = httpCallParser.filterHttpResponseParams(new ArrayList<>(), null, null); assertEquals(ss.size(),0); } @@ -278,7 +280,7 @@ public void testFilterHttpResponseParamsIpHost() { h2.requestParams.setApiCollectionId(1000); h2.setSource(Source.MIRRORING); - List ss = httpCallParser.filterHttpResponseParams(Arrays.asList(h1, h2), null); + List ss = httpCallParser.filterHttpResponseParams(Arrays.asList(h1, h2), null, null); assertEquals(ss.size(),2); assertEquals(h1.requestParams.getApiCollectionId(), 1000); assertTrue(h2.requestParams.getApiCollectionId() != 1000); @@ -302,7 +304,7 @@ public void testFilterHttpResponseParamsWithoutHost() { h1.statusCode = 200; h1.requestParams.setApiCollectionId(vxlanId1); - List filterHttpResponseParamsList = httpCallParser.filterHttpResponseParams(Collections.singletonList(h1),null); + List filterHttpResponseParamsList = httpCallParser.filterHttpResponseParams(Collections.singletonList(h1),null, null); Assertions.assertEquals(filterHttpResponseParamsList.size(),1); Assertions.assertEquals(filterHttpResponseParamsList.get(0).requestParams.getApiCollectionId(),vxlanId1); @@ -317,7 +319,7 @@ public void testFilterHttpResponseParamsWithoutHost() { h2.statusCode = 200; h2.requestParams.setApiCollectionId(vxlanId2); - filterHttpResponseParamsList = httpCallParser.filterHttpResponseParams(Collections.singletonList(h2),null); + filterHttpResponseParamsList = httpCallParser.filterHttpResponseParams(Collections.singletonList(h2),null, null); Assertions.assertEquals(filterHttpResponseParamsList.size(),1); Assertions.assertEquals(filterHttpResponseParamsList.get(0).requestParams.getApiCollectionId(),vxlanId2); @@ -332,7 +334,7 @@ public void testFilterHttpResponseParamsWithoutHost() { h3.statusCode = 400; h3.requestParams.setApiCollectionId(vxlanId2); - filterHttpResponseParamsList = httpCallParser.filterHttpResponseParams(Collections.singletonList(h3),null); + filterHttpResponseParamsList = httpCallParser.filterHttpResponseParams(Collections.singletonList(h3),null, null); Assertions.assertEquals(filterHttpResponseParamsList.size(),0); ApiCollection apiCollection3 = ApiCollectionsDao.instance.findOne("_id", vxlanId3); @@ -364,7 +366,7 @@ public void testFilterResponseParamsWithHost() { h1.statusCode = 200; h1.setSource(Source.MIRRORING); - httpCallParser.filterHttpResponseParams(Collections.singletonList(h1),null); + httpCallParser.filterHttpResponseParams(Collections.singletonList(h1),null, null); List apiCollections = ApiCollectionsDao.instance.findAll(new BasicDBObject()); Assertions.assertEquals(apiCollections.size(),2); @@ -390,7 +392,7 @@ public void testFilterResponseParamsWithHost() { h2.statusCode = 200; h2.setSource(Source.MIRRORING); - httpCallParser.filterHttpResponseParams(Collections.singletonList(h2),null); + httpCallParser.filterHttpResponseParams(Collections.singletonList(h2),null, null); apiCollections = ApiCollectionsDao.instance.findAll(new BasicDBObject()); Assertions.assertEquals(apiCollections.size(),3); @@ -414,7 +416,7 @@ public void testFilterResponseParamsWithHost() { h3.statusCode = 200; h3.setSource(Source.MIRRORING); - httpCallParser.filterHttpResponseParams(Collections.singletonList(h3),null); + httpCallParser.filterHttpResponseParams(Collections.singletonList(h3),null, null); apiCollections = ApiCollectionsDao.instance.findAll(new BasicDBObject()); Assertions.assertEquals(apiCollections.size(),4); @@ -447,7 +449,7 @@ public void testFilterResponseParamsWithHost() { ); httpCallParser.getHostNameToIdMap().put("hostRandom 1234", dupId); - httpCallParser.filterHttpResponseParams(Collections.singletonList(h4),null); + httpCallParser.filterHttpResponseParams(Collections.singletonList(h4),null, null); apiCollections = ApiCollectionsDao.instance.findAll(new BasicDBObject()); Assertions.assertEquals(apiCollections.size(),6); @@ -471,7 +473,7 @@ public void testCollisionHostNameCollection() { h1.statusCode = 200; HttpCallParser httpCallParser = new HttpCallParser("",0,0,0, true); - httpCallParser.filterHttpResponseParams(Collections.singletonList(h1),null); + httpCallParser.filterHttpResponseParams(Collections.singletonList(h1),null, null); List apiCollections = ApiCollectionsDao.instance.findAll(new BasicDBObject()); Assertions.assertEquals(apiCollections.size(), 1); @@ -589,10 +591,11 @@ public void testRedundantUrlCheck() { String url5 = "test5.js.js"; List allowedUrlType = Arrays.asList("js"); - assertEquals(httpCallParser.isRedundantEndpoint(url1, allowedUrlType), true); - assertEquals(httpCallParser.isRedundantEndpoint(url2, allowedUrlType), false); - assertEquals(httpCallParser.isRedundantEndpoint(url3, allowedUrlType), false); - assertEquals(httpCallParser.isRedundantEndpoint(url4, allowedUrlType), true); - assertEquals(httpCallParser.isRedundantEndpoint(url5, allowedUrlType), true); + Pattern pattern = Utils.createRegexPatternFromList(allowedUrlType); + assertEquals(httpCallParser.isRedundantEndpoint(url1, pattern), true); + assertEquals(httpCallParser.isRedundantEndpoint(url2, pattern), false); + assertEquals(httpCallParser.isRedundantEndpoint(url3, pattern), false); + assertEquals(httpCallParser.isRedundantEndpoint(url4, pattern), true); + assertEquals(httpCallParser.isRedundantEndpoint(url5, pattern), true); } } diff --git a/apps/api-threat-detection/src/main/java/com/akto/filters/HttpCallFilter.java b/apps/api-threat-detection/src/main/java/com/akto/filters/HttpCallFilter.java new file mode 100644 index 0000000000..051e0c6519 --- /dev/null +++ b/apps/api-threat-detection/src/main/java/com/akto/filters/HttpCallFilter.java @@ -0,0 +1,121 @@ +package com.akto.filters; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.UUID; + +import com.akto.dao.context.Context; +import com.akto.dao.monitoring.FilterYamlTemplateDao; +import com.akto.data_actor.DataActor; +import com.akto.data_actor.DataActorFactory; +import com.akto.dto.ApiInfo.ApiInfoKey; +import com.akto.dto.HttpResponseParams; +import com.akto.dto.RawApi; +import com.akto.dto.bulk_updates.BulkUpdates; +import com.akto.dto.bulk_updates.UpdatePayload; +import com.akto.dto.monitoring.FilterConfig; +import com.akto.dto.test_editor.YamlTemplate; +import com.akto.dto.traffic.SuspectSampleData; +import com.akto.dto.type.URLMethods.Method; +import com.akto.hybrid_parsers.HttpCallParser; +import com.akto.log.LoggerMaker; +import com.akto.log.LoggerMaker.LogDb; +import com.akto.rules.TestPlugin; +import com.akto.runtime.policies.ApiAccessTypePolicy; +import com.akto.test_editor.execution.VariableResolver; +import com.akto.test_editor.filter.data_operands_impl.ValidationResult; + +public class HttpCallFilter { + private static final LoggerMaker loggerMaker = new LoggerMaker(HttpCallFilter.class, LogDb.THREAT_DETECTION); + + private Map apiFilters; + private List bulkUpdates = new ArrayList<>(); + private final int sync_threshold_count; + private final int sync_threshold_time; + private int last_synced; + private int sync_count; + private HttpCallParser httpCallParser; + + private static final int FILTER_REFRESH_INTERVAL = 10 * 60; + private int lastFilterFetch; + + private static final DataActor dataActor = DataActorFactory.fetchInstance(); + + public HttpCallFilter(int sync_threshold_count, int sync_threshold_time) { + apiFilters = new HashMap<>(); + bulkUpdates = new ArrayList<>(); + this.sync_threshold_count = sync_threshold_count; + this.sync_threshold_time = sync_threshold_time; + last_synced = 0; + sync_count = 0; + lastFilterFetch = 0; + httpCallParser = new HttpCallParser(sync_threshold_count, sync_threshold_time); + } + + public void filterFunction(List responseParams) { + + int now = Context.now(); + if ((lastFilterFetch + FILTER_REFRESH_INTERVAL) < now) { + // TODO: add support for only active templates. + List templates = dataActor.fetchFilterYamlTemplates(); + apiFilters = FilterYamlTemplateDao.instance.fetchFilterConfig(false, templates, false); + lastFilterFetch = now; + } + + if (apiFilters != null && !apiFilters.isEmpty()) { + for (HttpResponseParams responseParam : responseParams) { + for (Entry apiFilterEntry : apiFilters.entrySet()) { + try { + FilterConfig apiFilter = apiFilterEntry.getValue(); + String filterId = apiFilterEntry.getKey(); + String message = responseParam.getOrig(); + List sourceIps = ApiAccessTypePolicy.getSourceIps(responseParam); + RawApi rawApi = RawApi.buildFromMessage(message); + int apiCollectionId = httpCallParser.createApiCollectionId(responseParam); + responseParam.requestParams.setApiCollectionId(apiCollectionId); + String url = responseParam.getRequestParams().getURL(); + Method method = Method.valueOf(responseParam.getRequestParams().getMethod()); + ApiInfoKey apiInfoKey = new ApiInfoKey(apiCollectionId, url, method); + Map varMap = apiFilter.resolveVarMap(); + VariableResolver.resolveWordList(varMap, new HashMap>() { + { + put(apiInfoKey, Arrays.asList(message)); + } + }, apiInfoKey); + String filterExecutionLogId = UUID.randomUUID().toString(); + ValidationResult res = TestPlugin.validateFilter(apiFilter.getFilter().getNode(), rawApi, + apiInfoKey, varMap, filterExecutionLogId); + if (res.getIsValid()) { + now = Context.now(); + SuspectSampleData sampleData = new SuspectSampleData( + sourceIps, apiCollectionId, url, method, + message, now, filterId); + Map filterMap = new HashMap<>(); + UpdatePayload updatePayload = new UpdatePayload("obj", sampleData, "set"); + ArrayList updates = new ArrayList<>(); + updates.add(updatePayload.toString()); + bulkUpdates.add(new BulkUpdates(filterMap, updates)); + } + } catch (Exception e) { + loggerMaker.errorAndAddToDb(e, String.format("Error in httpCallFilter %s", e.toString())); + } + } + } + } + sync_count = bulkUpdates.size(); + if (sync_count > 0 && (sync_count >= sync_threshold_count || + (Context.now() - last_synced) > sync_threshold_time)) { + List updates = new ArrayList<>(); + updates.addAll(bulkUpdates); + dataActor.bulkWriteSuspectSampleData(updates); + loggerMaker.infoAndAddToDb(String.format("Inserting %d records in SuspectSampleData", sync_count)); + last_synced = Context.now(); + sync_count = 0; + bulkUpdates.clear(); + } + } +} \ No newline at end of file diff --git a/apps/dashboard/src/main/java/com/akto/action/settings/AdvancedTrafficFiltersAction.java b/apps/dashboard/src/main/java/com/akto/action/settings/AdvancedTrafficFiltersAction.java new file mode 100644 index 0000000000..a42b753477 --- /dev/null +++ b/apps/dashboard/src/main/java/com/akto/action/settings/AdvancedTrafficFiltersAction.java @@ -0,0 +1,96 @@ +package com.akto.action.settings; + +import java.util.List; + +import org.bson.conversions.Bson; + +import com.akto.action.UserAction; +import com.akto.dao.monitoring.FilterConfigYamlParser; +import com.akto.dao.runtime_filters.AdvancedTrafficFiltersDao; +import com.akto.dto.monitoring.FilterConfig; +import com.akto.dto.test_editor.YamlTemplate; +import com.akto.util.Constants; +import com.akto.utils.TrafficFilterUtil; +import com.mongodb.client.model.Filters; +import com.mongodb.client.model.FindOneAndUpdateOptions; +import com.mongodb.client.model.Projections; +import com.mongodb.client.model.Updates; + +public class AdvancedTrafficFiltersAction extends UserAction { + + private List templatesList; + private String yamlContent; + private String templateId; + private boolean inactive; + + public String fetchAllFilterTemplates(){ + List yamlTemplates = AdvancedTrafficFiltersDao.instance.findAll( + Filters.empty(), + Projections.include( + Constants.ID, YamlTemplate.AUTHOR, YamlTemplate.CONTENT, YamlTemplate.INACTIVE + ) + ); + this.templatesList = yamlTemplates; + return SUCCESS.toUpperCase(); + } + + public String saveYamlTemplateForTrafficFilters(){ + FilterConfig filterConfig = new FilterConfig(); + try { + filterConfig = FilterConfigYamlParser.parseTemplate(yamlContent, true); + if (filterConfig.getId() == null) { + throw new Exception("id field cannot be empty"); + } + if (filterConfig.getFilter() == null) { + throw new Exception("filter field cannot be empty"); + } + List updates = TrafficFilterUtil.getDbUpdateForTemplate(this.yamlContent, getSUser().getLogin()); + AdvancedTrafficFiltersDao.instance.updateOne( + Filters.eq(Constants.ID, filterConfig.getId()), + Updates.combine(updates)); + + } catch (Exception e) { + e.printStackTrace(); + addActionError(e.getMessage()); + return ERROR.toUpperCase(); + } + return SUCCESS.toUpperCase(); + } + + public String changeActivityOfFilter() { + FindOneAndUpdateOptions options = new FindOneAndUpdateOptions(); + options.upsert(false); + AdvancedTrafficFiltersDao.instance.getMCollection().findOneAndUpdate( + Filters.eq(Constants.ID, this.templateId), + Updates.set(YamlTemplate.INACTIVE, this.inactive), + options + ); + + return SUCCESS.toUpperCase(); + } + + public String deleteAdvancedFilter(){ + AdvancedTrafficFiltersDao.instance.getMCollection().deleteOne( + Filters.eq(Constants.ID, this.templateId) + ); + return SUCCESS.toUpperCase(); + } + + public List getTemplatesList() { + return templatesList; + } + + public void setYamlContent(String yamlContent) { + this.yamlContent = yamlContent; + } + + public void setTemplateId(String templateId) { + this.templateId = templateId; + } + + public void setInactive(boolean inactive) { + this.inactive = inactive; + } + + +} diff --git a/apps/dashboard/src/main/java/com/akto/action/threat_detection/FilterYamlTemplateAction.java b/apps/dashboard/src/main/java/com/akto/action/threat_detection/FilterYamlTemplateAction.java new file mode 100644 index 0000000000..cb144e1919 --- /dev/null +++ b/apps/dashboard/src/main/java/com/akto/action/threat_detection/FilterYamlTemplateAction.java @@ -0,0 +1,70 @@ +package com.akto.action.threat_detection; + +import java.util.List; +import java.util.Map; + +import org.bson.conversions.Bson; + +import com.akto.action.UserAction; +import com.akto.dao.monitoring.FilterConfigYamlParser; +import com.akto.dao.monitoring.FilterYamlTemplateDao; +import com.akto.dto.monitoring.FilterConfig; +import com.akto.util.Constants; +import com.akto.utils.TrafficFilterUtil; +import com.mongodb.BasicDBList; +import com.mongodb.client.model.Filters; +import com.mongodb.client.model.Updates; + +public class FilterYamlTemplateAction extends UserAction { + + BasicDBList templates; + String content; + + public String fetchFilterYamlTemplate() { + Map configs = FilterYamlTemplateDao.instance.fetchFilterConfig(true, false); + this.templates = TrafficFilterUtil.getFilterTemplates(configs); + return SUCCESS.toUpperCase(); + } + + public String saveFilterYamlTemplate() { + + FilterConfig filterConfig = new FilterConfig(); + try { + filterConfig = FilterConfigYamlParser.parseTemplate(content, false); + if (filterConfig.getId() == null) { + throw new Exception("id field cannot be empty"); + } + if (filterConfig.getFilter() == null) { + throw new Exception("filter field cannot be empty"); + } + List updates = TrafficFilterUtil.getDbUpdateForTemplate(this.content, getSUser().getLogin()); + FilterYamlTemplateDao.instance.updateOne( + Filters.eq(Constants.ID, filterConfig.getId()), + Updates.combine(updates)); + + } catch (Exception e) { + e.printStackTrace(); + addActionError(e.getMessage()); + return ERROR.toUpperCase(); + } + + return SUCCESS.toUpperCase(); + } + + public BasicDBList getTemplates() { + return templates; + } + + public void setTemplates(BasicDBList templates) { + this.templates = templates; + } + + public String getContent() { + return content; + } + + public void setContent(String content) { + this.content = content; + } + +} diff --git a/apps/dashboard/src/main/java/com/akto/listener/InitializerListener.java b/apps/dashboard/src/main/java/com/akto/listener/InitializerListener.java index cb1a361a2c..e9ad0cc832 100644 --- a/apps/dashboard/src/main/java/com/akto/listener/InitializerListener.java +++ b/apps/dashboard/src/main/java/com/akto/listener/InitializerListener.java @@ -141,6 +141,7 @@ import static com.akto.task.Cluster.callDibs; import static com.akto.utils.billing.OrganizationUtils.syncOrganizationWithAkto; import static com.mongodb.client.model.Filters.eq; +import static com.akto.runtime.utils.Utils.convertOriginalReqRespToString; public class InitializerListener implements ServletContextListener { private static final Logger logger = LoggerFactory.getLogger(InitializerListener.class); @@ -842,7 +843,7 @@ public static void webhookSenderUtil(CustomWebhook webhook) { String message = null; try { - message = RedactSampleData.convertOriginalReqRespToString(request, response); + message = convertOriginalReqRespToString(request, response); } catch (Exception e) { errors.add("Failed converting sample data"); } diff --git a/apps/dashboard/src/main/java/com/akto/utils/TrafficFilterUtil.java b/apps/dashboard/src/main/java/com/akto/utils/TrafficFilterUtil.java new file mode 100644 index 0000000000..980eef9084 --- /dev/null +++ b/apps/dashboard/src/main/java/com/akto/utils/TrafficFilterUtil.java @@ -0,0 +1,57 @@ +package com.akto.utils; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; + +import org.bson.conversions.Bson; + +import com.akto.dao.context.Context; +import com.akto.dto.monitoring.FilterConfig; +import com.akto.dto.test_editor.YamlTemplate; +import com.akto.util.enums.GlobalEnums.YamlTemplateSource; +import com.mongodb.BasicDBList; +import com.mongodb.BasicDBObject; +import com.mongodb.client.model.Updates; + +public class TrafficFilterUtil { + public static BasicDBList getFilterTemplates(Map configs){ + BasicDBList templates = new BasicDBList(); + if (configs != null && !configs.isEmpty()) { + for (Entry apiFilterEntry : configs.entrySet()) { + FilterConfig config = apiFilterEntry.getValue(); + BasicDBObject template = new BasicDBObject(); + template.append(FilterConfig.ID, config.getId()); + template.append(FilterConfig._CONTENT, config.getContent()); + template.append(FilterConfig._AUTHOR, config.getAuthor()); + template.append(FilterConfig.CREATED_AT, config.getCreatedAt()); + template.append(FilterConfig.UPDATED_AT, config.getUpdatedAt()); + templates.add(template); + } + } + + return templates; + } + + public static List getDbUpdateForTemplate(String content, String userEmail) throws Exception{ + try { + String author = userEmail; + int createdAt = Context.now(); + int updatedAt = Context.now(); + + List updates = new ArrayList<>( + Arrays.asList( + Updates.setOnInsert(YamlTemplate.CREATED_AT, createdAt), + Updates.setOnInsert(YamlTemplate.AUTHOR, author), + Updates.set(YamlTemplate.UPDATED_AT, updatedAt), + Updates.set(YamlTemplate.CONTENT, content), + Updates.setOnInsert(YamlTemplate.SOURCE, YamlTemplateSource.CUSTOM))); + return updates; + + } catch (Exception e) { + throw e; + } + } +} diff --git a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/components/shared/SampleData.js b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/components/shared/SampleData.js index 7465688c2b..546337db63 100644 --- a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/components/shared/SampleData.js +++ b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/components/shared/SampleData.js @@ -4,6 +4,7 @@ import "./style.css"; import func from "@/util/func" import editorSetup from './customEditor'; import yamlEditorSetup from "../../pages/test_editor/components/editor_config/editorSetup" +import keywords from "../../pages/test_editor/components/editor_config/keywords" function highlightPaths(highlightPathMap, ref){ highlightPathMap && Object.keys(highlightPathMap).forEach((key) => { @@ -224,6 +225,13 @@ function SampleData(props) { editorSetup.setTokenizer() yamlEditorSetup.setEditorTheme() } + if(editorLanguage.includes("custom_yaml")){ + options['theme']= "customTheme" + yamlEditorSetup.registerLanguage() + yamlEditorSetup.setTokenizer() + yamlEditorSetup.setEditorTheme() + yamlEditorSetup.setAutoComplete(keywords) + } if(showDiff){ instance = monaco.editor.createDiffEditor(ref.current, options) } else { diff --git a/apps/database-abstractor/src/main/java/com/akto/action/DbAction.java b/apps/database-abstractor/src/main/java/com/akto/action/DbAction.java index 50dbb12ca4..a3ee7bf8c5 100644 --- a/apps/database-abstractor/src/main/java/com/akto/action/DbAction.java +++ b/apps/database-abstractor/src/main/java/com/akto/action/DbAction.java @@ -144,6 +144,7 @@ public void setTestSourceConfig(TestSourceConfig testSourceConfig) { List newEps; String logicalGroupName; BasicDBList issuesIds; + List activeAdvancedFilters; public BasicDBList getIssuesIds() { return issuesIds; @@ -1644,6 +1645,15 @@ public String fetchLatestEndpointsForTesting() { return Action.SUCCESS.toUpperCase(); } + public String fetchActiveAdvancedFilters(){ + try { + this.activeAdvancedFilters = DbLayer.fetchActiveFilterTemplates(); + } catch (Exception e) { + return Action.ERROR.toUpperCase(); + } + return Action.SUCCESS.toUpperCase(); + } + public List getCustomDataTypes() { return customDataTypes; } @@ -2522,4 +2532,12 @@ public void setNewEps(List newEps) { this.newEps = newEps; } + public List getActiveAdvancedFilters() { + return activeAdvancedFilters; + } + + public void setActiveAdvancedFilters(List activeAdvancedFilters) { + this.activeAdvancedFilters = activeAdvancedFilters; + } + } diff --git a/apps/database-abstractor/src/main/resources/struts.xml b/apps/database-abstractor/src/main/resources/struts.xml index 0b2adcc875..e4f5043569 100644 --- a/apps/database-abstractor/src/main/resources/struts.xml +++ b/apps/database-abstractor/src/main/resources/struts.xml @@ -1135,6 +1135,17 @@ + + + + + + 422 + false + ^actionErrors.* + + + diff --git a/apps/mini-runtime/src/main/java/com/akto/hybrid_parsers/HttpCallParser.java b/apps/mini-runtime/src/main/java/com/akto/hybrid_parsers/HttpCallParser.java index de0181b428..f24493a98e 100644 --- a/apps/mini-runtime/src/main/java/com/akto/hybrid_parsers/HttpCallParser.java +++ b/apps/mini-runtime/src/main/java/com/akto/hybrid_parsers/HttpCallParser.java @@ -11,33 +11,43 @@ import com.akto.data_actor.DataActorFactory; import com.akto.dto.billing.FeatureAccess; import com.akto.dto.*; +import com.akto.dto.ApiInfo.ApiInfoKey; import com.akto.dto.billing.Organization; import com.akto.dto.billing.SyncLimit; import com.akto.dto.bulk_updates.BulkUpdates; import com.akto.dto.bulk_updates.UpdatePayload; +import com.akto.dto.monitoring.FilterConfig; +import com.akto.dto.monitoring.FilterConfig.FILTER_TYPE; import com.akto.dto.settings.DefaultPayload; +import com.akto.dto.test_editor.ExecutorNode; import com.akto.dto.traffic_metrics.TrafficMetrics; +import com.akto.dto.type.URLMethods.Method; import com.akto.dto.usage.MetricTypes; import com.akto.graphql.GraphQLUtils; import com.akto.log.LoggerMaker; import com.akto.log.LoggerMaker.LogDb; +import com.akto.runtime.RuntimeUtil; import com.akto.runtime.parser.SampleParser; +import com.akto.runtime.utils.Utils; +import com.akto.test_editor.execution.ParseAndExecute; +import com.akto.test_editor.execution.VariableResolver; +import com.akto.test_editor.filter.data_operands_impl.ValidationResult; import com.akto.hybrid_runtime.APICatalogSync; import com.akto.hybrid_runtime.Main; import com.akto.hybrid_runtime.MergeLogicLocal; import com.akto.hybrid_runtime.URLAggregator; import com.akto.util.JSONUtils; +import com.akto.util.Pair; import com.akto.util.Constants; import com.akto.util.HttpRequestResponseUtils; import com.akto.util.http_util.CoreHTTPClient; import com.mongodb.BasicDBObject; import com.mongodb.client.model.*; import okhttp3.*; -import org.apache.commons.lang3.math.NumberUtils; -import org.bson.conversions.Bson; import java.io.IOException; import java.util.*; +import java.util.Map.Entry; import java.util.concurrent.ConcurrentLinkedQueue; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; @@ -47,6 +57,7 @@ import java.util.regex.Pattern; import static com.akto.runtime.RuntimeUtil.matchesDefaultPayload; +import static com.akto.testing.Utils.validateFilter; public class HttpCallParser { private final int sync_threshold_count; @@ -192,6 +203,64 @@ private SyncLimit fetchSyncLimit() { return syncLimit; } + public static FILTER_TYPE isValidResponseParam(HttpResponseParams responseParam, Map filterMap, Map> executorNodesMap){ + FILTER_TYPE filterType = FILTER_TYPE.UNCHANGED; + String message = responseParam.getOrig(); + RawApi rawApi = RawApi.buildFromMessage(message); + int apiCollectionId = responseParam.requestParams.getApiCollectionId(); + String url = responseParam.getRequestParams().getURL(); + Method method = Method.fromString(responseParam.getRequestParams().getMethod()); + ApiInfoKey apiInfoKey = new ApiInfoKey(apiCollectionId, url, method); + for (Entry apiFilterEntry : filterMap.entrySet()) { + try { + FilterConfig apiFilter = apiFilterEntry.getValue(); + Map varMap = apiFilter.resolveVarMap(); + VariableResolver.resolveWordList(varMap, new HashMap>() { + { + put(apiInfoKey, Arrays.asList(message)); + } + }, apiInfoKey); + String filterExecutionLogId = UUID.randomUUID().toString(); + ValidationResult res = validateFilter(apiFilter.getFilter().getNode(), rawApi, + apiInfoKey, varMap, filterExecutionLogId); + if (res.getIsValid()) { + // handle custom filters here + if(apiFilter.getId().equals(FilterConfig.DEFAULT_BLOCK_FILTER)){ + return FILTER_TYPE.BLOCKED; + } + + // handle execute here + List nodes = executorNodesMap.getOrDefault(apiFilter.getId(), new ArrayList<>()); + if(!nodes.isEmpty()){ + RawApi modifiedApi = new ParseAndExecute().execute(nodes, rawApi, apiInfoKey, varMap, filterExecutionLogId); + responseParam = Utils.convertRawApiToHttpResponseParams(modifiedApi, responseParam); + filterType = FILTER_TYPE.MODIFIED; + }else{ + filterType = FILTER_TYPE.ALLOWED; + } + + } + } catch (Exception e) { + loggerMaker.errorAndAddToDb(e, String.format("Error in httpCallFilter %s", e.toString())); + filterType = FILTER_TYPE.UNCHANGED; + } + } + return filterType; + } + + + public static Pair applyAdvancedFilters(HttpResponseParams responseParams, Map> executorNodesMap, Map filterMap){ + if (filterMap != null && !filterMap.isEmpty()) { + FILTER_TYPE filterType = isValidResponseParam(responseParams, filterMap, executorNodesMap); + if(filterType.equals(FILTER_TYPE.BLOCKED)){ + return new Pair(null, FILTER_TYPE.BLOCKED); + }else{ + return new Pair(responseParams, filterType); + } + } + return new Pair(responseParams, FILTER_TYPE.UNCHANGED); + } + public void syncFunction(List responseParams, boolean syncImmediately, boolean fetchAllSTI, AccountSettings accountSettings) { // USE ONLY filteredResponseParams and not responseParams List filteredResponseParams = responseParams; @@ -389,17 +458,7 @@ public void incTrafficMetrics(TrafficMetrics.Key key, int value) { public static final String CONTENT_TYPE = "CONTENT-TYPE"; - public boolean isRedundantEndpoint(String url, List discardedUrlList){ - StringJoiner joiner = new StringJoiner("|", ".*\\.(", ")(\\?.*)?"); - for (String extension : discardedUrlList) { - if(extension.startsWith(CONTENT_TYPE)){ - continue; - } - joiner.add(extension); - } - String regex = joiner.toString(); - - Pattern pattern = Pattern.compile(regex); + public boolean isRedundantEndpoint(String url, Pattern pattern){ Matcher matcher = pattern.matcher(url); return matcher.matches(); } @@ -412,6 +471,54 @@ private boolean isInvalidContentType(String contentType){ return res; } + public int createApiCollectionId(HttpResponseParams httpResponseParam){ + int apiCollectionId; + String hostName = getHeaderValue(httpResponseParam.getRequestParams().getHeaders(), "host"); + + if (hostName != null && !hostNameToIdMap.containsKey(hostName) && RuntimeUtil.hasSpecialCharacters(hostName)) { + hostName = "Special_Char_Host"; + } + + int vxlanId = httpResponseParam.requestParams.getApiCollectionId(); + + if (useHostCondition(hostName, httpResponseParam.getSource())) { + hostName = hostName.toLowerCase(); + hostName = hostName.trim(); + + String key = hostName; + + if (hostNameToIdMap.containsKey(key)) { + apiCollectionId = hostNameToIdMap.get(key); + + } else { + int id = hostName.hashCode(); + try { + + apiCollectionId = createCollectionBasedOnHostName(id, hostName); + + hostNameToIdMap.put(key, apiCollectionId); + } catch (Exception e) { + loggerMaker.errorAndAddToDb("Failed to create collection for host : " + hostName, LogDb.RUNTIME); + createCollectionSimple(vxlanId); + hostNameToIdMap.put("null " + vxlanId, vxlanId); + apiCollectionId = httpResponseParam.requestParams.getApiCollectionId(); + } + } + + } else { + String key = "null" + " " + vxlanId; + if (!hostNameToIdMap.containsKey(key)) { + createCollectionSimple(vxlanId); + hostNameToIdMap.put(key, vxlanId); + } + + apiCollectionId = vxlanId; + } + return apiCollectionId; + } + + + private boolean isBlankResponseBodyForGET(String method, String contentType, String matchContentType, String responseBody) { boolean res = true; @@ -437,6 +544,13 @@ private boolean isBlankResponseBodyForGET(String method, String contentType, Str public List filterHttpResponseParams(List httpResponseParamsList, AccountSettings accountSettings) { List filteredResponseParams = new ArrayList<>(); int originalSize = httpResponseParamsList.size(); + + List redundantList = new ArrayList<>(); + if(accountSettings !=null && !accountSettings.getAllowRedundantEndpointsList().isEmpty()){ + redundantList = accountSettings.getAllowRedundantEndpointsList(); + } + Pattern regexPattern = Utils.createRegexPatternFromList(redundantList); + Map> executorNodesMap = ParseAndExecute.createExecutorNodeMap(apiCatalogSync.advancedFilterMap); for (HttpResponseParams httpResponseParam: httpResponseParamsList) { if (httpResponseParam.getSource().equals(HttpResponseParams.Source.MIRRORING)) { @@ -459,8 +573,8 @@ public List filterHttpResponseParams(List contentTypeList = (List) httpResponseParam.getRequestParams().getHeaders().getOrDefault("content-type", new ArrayList<>()); @@ -497,45 +611,15 @@ public List filterHttpResponseParams(List temp = applyAdvancedFilters(httpResponseParam, executorNodesMap, apiCatalogSync.advancedFilterMap); + HttpResponseParams param = temp.getFirst(); + if(param == null){ + continue; + }else{ + httpResponseParam = param; } + + int apiCollectionId = createApiCollectionId(httpResponseParam); httpResponseParam.requestParams.setApiCollectionId(apiCollectionId); diff --git a/apps/mini-runtime/src/main/java/com/akto/hybrid_runtime/APICatalogSync.java b/apps/mini-runtime/src/main/java/com/akto/hybrid_runtime/APICatalogSync.java index 2995dcf7e5..44ef069f5c 100644 --- a/apps/mini-runtime/src/main/java/com/akto/hybrid_runtime/APICatalogSync.java +++ b/apps/mini-runtime/src/main/java/com/akto/hybrid_runtime/APICatalogSync.java @@ -7,11 +7,15 @@ import com.akto.dao.*; import com.akto.dao.context.Context; +import com.akto.dao.monitoring.FilterYamlTemplateDao; +import com.akto.dao.runtime_filters.AdvancedTrafficFiltersDao; import com.akto.dto.*; import com.akto.dto.billing.SyncLimit; import com.akto.dto.bulk_updates.BulkUpdates; import com.akto.dto.bulk_updates.UpdatePayload; +import com.akto.dto.monitoring.FilterConfig; import com.akto.dto.sql.SampleDataAlt; +import com.akto.dto.test_editor.YamlTemplate; import com.akto.dto.traffic.Key; import com.akto.dto.traffic.SampleData; import com.akto.dto.traffic.TrafficInfo; @@ -62,6 +66,7 @@ public class APICatalogSync { public static boolean mergeAsyncOutside = true; public int lastStiFetchTs = 0; public BloomFilter existingAPIsInDb = BloomFilter.create(Funnels.stringFunnel(Charsets.UTF_8), 1_000_000, 0.001 ); + public Map advancedFilterMap = new HashMap<>(); private static DataActor dataActor = DataActorFactory.fetchInstance(); @@ -1105,6 +1110,9 @@ public void buildFromDB(boolean calcDiff, boolean fetchAllSTI) { this.sensitiveParamInfoBooleanMap.put(sensitiveParamInfo, false); } + List advancedFilterTemplates = dataActor.fetchActiveAdvancedFilters(); + advancedFilterMap = FilterYamlTemplateDao.instance.fetchFilterConfig(false, advancedFilterTemplates, true); + try { loggerMaker.infoAndAddToDb("Started clearing values in db ", LogDb.RUNTIME); clearValuesInDB(); diff --git a/apps/mini-testing/src/main/java/com/akto/rules/TestPlugin.java b/apps/mini-testing/src/main/java/com/akto/rules/TestPlugin.java index 8627fc8963..bb5756bb6b 100644 --- a/apps/mini-testing/src/main/java/com/akto/rules/TestPlugin.java +++ b/apps/mini-testing/src/main/java/com/akto/rules/TestPlugin.java @@ -15,7 +15,6 @@ import com.akto.store.TestingUtil; import com.akto.test_editor.filter.Filter; import com.akto.testing.StatusCodeAnalyser; -import com.akto.utils.RedactSampleData; import com.fasterxml.jackson.core.JsonFactory; import com.fasterxml.jackson.core.JsonParser; import com.fasterxml.jackson.databind.JsonNode; @@ -28,6 +27,8 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; +import static com.akto.runtime.utils.Utils.convertOriginalReqRespToString; + public abstract class TestPlugin { static ObjectMapper mapper = new ObjectMapper(); @@ -144,7 +145,7 @@ public Result addWithoutRequestError(String originalMessage, TestResult.TestErro public TestResult buildFailedTestResultWithOriginalMessage(String originalMessage, TestResult.TestError testError, OriginalHttpRequest request, TestInfo testInfo) { String message = null; try { - message = RedactSampleData.convertOriginalReqRespToString(request, null); + message = convertOriginalReqRespToString(request, null); } catch (Exception e) { loggerMaker.errorAndAddToDb("Error while converting testRequest object to string : " + e, LogDb.TESTING); } @@ -165,11 +166,11 @@ public TestResult buildTestResult(OriginalHttpRequest request, OriginalHttpRespo List errors = new ArrayList<>(); String message = null; try { - message = RedactSampleData.convertOriginalReqRespToString(request, response); + message = convertOriginalReqRespToString(request, response); } catch (Exception e) { // TODO: logger.error("Error while converting OriginalHttpRequest to string", e); - message = RedactSampleData.convertOriginalReqRespToString(new OriginalHttpRequest(), new OriginalHttpResponse()); + message = convertOriginalReqRespToString(new OriginalHttpRequest(), new OriginalHttpResponse()); errors.add(TestResult.TestError.FAILED_TO_CONVERT_TEST_REQUEST_TO_STRING.getMessage()); } diff --git a/apps/mini-testing/src/main/java/com/akto/test_editor/execution/Build.java b/apps/mini-testing/src/main/java/com/akto/test_editor/execution/Build.java index 3ecab13e43..8d4b50bb4d 100644 --- a/apps/mini-testing/src/main/java/com/akto/test_editor/execution/Build.java +++ b/apps/mini-testing/src/main/java/com/akto/test_editor/execution/Build.java @@ -13,7 +13,6 @@ import com.akto.dto.traffic.SampleData; import com.akto.dto.type.URLMethods; import com.akto.log.LoggerMaker; -import com.akto.runtime.policies.AuthPolicy; import com.akto.testing.ApiExecutor; import com.akto.util.Constants; import com.akto.util.HttpRequestResponseUtils; @@ -30,6 +29,7 @@ import static com.akto.util.HttpRequestResponseUtils.FORM_URL_ENCODED_CONTENT_TYPE; import static com.akto.util.HttpRequestResponseUtils.extractValuesFromPayload; +import static com.akto.runtime.utils.Utils.parseCookie; public class Build { @@ -408,7 +408,7 @@ public static Map> getValuesMap(OriginalHttpResponse respons if (values == null) continue; if (headerKey.equalsIgnoreCase("set-cookie")) { - Map cookieMap = AuthPolicy.parseCookie(values); + Map cookieMap = parseCookie(values); for (String cookieKey : cookieMap.keySet()) { String cookieVal = cookieMap.get(cookieKey); valuesMap.put(cookieKey, new HashSet<>(Collections.singletonList(cookieVal))); diff --git a/apps/mini-testing/src/main/java/com/akto/test_editor/execution/Executor.java b/apps/mini-testing/src/main/java/com/akto/test_editor/execution/Executor.java index 7b9ca5f9ad..724e3228ac 100644 --- a/apps/mini-testing/src/main/java/com/akto/test_editor/execution/Executor.java +++ b/apps/mini-testing/src/main/java/com/akto/test_editor/execution/Executor.java @@ -41,6 +41,7 @@ import com.mongodb.BasicDBObject; import static com.akto.test_editor.Utils.bodyValuesUnchanged; import static com.akto.test_editor.Utils.headerValuesUnchanged; +import static com.akto.runtime.utils.Utils.convertOriginalReqRespToString; import org.apache.commons.lang3.StringUtils; @@ -365,7 +366,7 @@ public TestResult validate(ExecutionResult attempt, RawApi rawApi, Map cookieList = headers.getOrDefault(key, new ArrayList<>()); - Map cookieMap = AuthPolicy.parseCookie(cookieList); + Map cookieMap = parseCookie(cookieList); for (String cookieKey : cookieMap.keySet()) { dataOperandFilterRequest = new DataOperandFilterRequest(cookieKey, filterActionRequest.getQuerySet(), filterActionRequest.getOperand()); res = invokeFilter(dataOperandFilterRequest); @@ -596,7 +594,7 @@ public DataOperandsFilterResponse applyFiltersOnHeaders(FilterActionRequest filt if (!res && (key.equals("cookie") || key.equals("set-cookie"))) { List cookieList = headers.getOrDefault("cookie", new ArrayList<>()); - Map cookieMap = AuthPolicy.parseCookie(cookieList); + Map cookieMap = parseCookie(cookieList); for (String cookieKey : cookieMap.keySet()) { DataOperandFilterRequest dataOperandFilterRequest = new DataOperandFilterRequest(cookieMap.get(cookieKey), filterActionRequest.getQuerySet(), filterActionRequest.getOperand()); res = invokeFilter(dataOperandFilterRequest); diff --git a/apps/mini-testing/src/main/java/com/akto/test_editor/filter/data_operands_impl/CookieExpireFilter.java b/apps/mini-testing/src/main/java/com/akto/test_editor/filter/data_operands_impl/CookieExpireFilter.java index 0edce0afeb..fd031d6bd1 100644 --- a/apps/mini-testing/src/main/java/com/akto/test_editor/filter/data_operands_impl/CookieExpireFilter.java +++ b/apps/mini-testing/src/main/java/com/akto/test_editor/filter/data_operands_impl/CookieExpireFilter.java @@ -10,7 +10,7 @@ import java.util.Map; import com.akto.dto.test_editor.DataOperandFilterRequest; -import com.akto.runtime.policies.AuthPolicy; +import static com.akto.runtime.utils.Utils.parseCookie; public class CookieExpireFilter extends DataOperandsImpl { @@ -33,7 +33,7 @@ public Boolean isValid(DataOperandFilterRequest dataOperandFilterRequest) { return false; } - Map cookieMap = AuthPolicy.parseCookie(Arrays.asList(data)); + Map cookieMap = parseCookie(Arrays.asList(data)); boolean result = queryVal; boolean res = false; diff --git a/apps/mini-testing/src/main/java/com/akto/testing/workflow_node_executor/ApiNodeExecutor.java b/apps/mini-testing/src/main/java/com/akto/testing/workflow_node_executor/ApiNodeExecutor.java index 92645aa6c6..1ebf62bdb5 100644 --- a/apps/mini-testing/src/main/java/com/akto/testing/workflow_node_executor/ApiNodeExecutor.java +++ b/apps/mini-testing/src/main/java/com/akto/testing/workflow_node_executor/ApiNodeExecutor.java @@ -20,7 +20,8 @@ import com.akto.testing.ApiExecutor; import com.akto.testing.Main; import com.akto.testing.Utils; -import com.akto.utils.RedactSampleData; + +import static com.akto.runtime.utils.Utils.convertOriginalReqRespToString; public class ApiNodeExecutor extends NodeExecutor { @@ -94,7 +95,7 @@ public NodeResult processNode(Node node, Map valuesMap, Boolean String message = null; try { - message = RedactSampleData.convertOriginalReqRespToString(request, response); + message = convertOriginalReqRespToString(request, response); } catch (Exception e) { ; } diff --git a/apps/mini-testing/src/main/java/com/akto/testing/workflow_node_executor/YamlNodeExecutor.java b/apps/mini-testing/src/main/java/com/akto/testing/workflow_node_executor/YamlNodeExecutor.java index 5ac37218a1..d38e4499ba 100644 --- a/apps/mini-testing/src/main/java/com/akto/testing/workflow_node_executor/YamlNodeExecutor.java +++ b/apps/mini-testing/src/main/java/com/akto/testing/workflow_node_executor/YamlNodeExecutor.java @@ -45,9 +45,10 @@ import com.akto.testing.ApiExecutor; import com.akto.testing.TestExecutor; import com.akto.util.Constants; -import com.akto.utils.RedactSampleData; import com.google.gson.Gson; +import static com.akto.runtime.utils.Utils.convertOriginalReqRespToString; + public class YamlNodeExecutor extends NodeExecutor { private static final Gson gson = new Gson(); @@ -163,7 +164,7 @@ public NodeResult processNode(Node node, Map varMap, Boolean all } vulnerable = res.getVulnerable(); try { - message.add(RedactSampleData.convertOriginalReqRespToString(testReq.getRequest(), testResponse)); + message.add(convertOriginalReqRespToString(testReq.getRequest(), testResponse)); } catch (Exception e) { ; } diff --git a/apps/testing/src/main/java/com/akto/rules/TestPlugin.java b/apps/testing/src/main/java/com/akto/rules/TestPlugin.java index 030aa94005..0390f51867 100644 --- a/apps/testing/src/main/java/com/akto/rules/TestPlugin.java +++ b/apps/testing/src/main/java/com/akto/rules/TestPlugin.java @@ -11,17 +11,14 @@ import com.akto.log.LoggerMaker; import com.akto.log.LoggerMaker.LogDb; import com.akto.runtime.APICatalogSync; -import com.akto.runtime.RelationshipSync; import com.akto.store.SampleMessageStore; import com.akto.store.TestingUtil; import com.akto.test_editor.filter.Filter; +import com.akto.test_editor.filter.data_operands_impl.ValidationResult; import com.akto.testing.StatusCodeAnalyser; import com.akto.types.CappedSet; import com.akto.util.JSONUtils; -import com.akto.utils.RedactSampleData; import com.fasterxml.jackson.core.JsonFactory; -import com.fasterxml.jackson.core.JsonParser; -import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.gson.Gson; import com.mongodb.BasicDBObject; @@ -36,6 +33,7 @@ import java.util.regex.Pattern; import static com.akto.runtime.APICatalogSync.trimAndSplit; +import static com.akto.runtime.utils.Utils.convertOriginalReqRespToString; public abstract class TestPlugin { @@ -56,9 +54,7 @@ public static boolean isStatusGood(int statusCode) { } public static void extractAllValuesFromPayload(String payload, Map> payloadMap) throws Exception{ - JsonParser jp = factory.createParser(payload); - JsonNode node = mapper.readTree(jp); - RelationshipSync.extractAllValuesFromPayload(node,new ArrayList<>(),payloadMap); + com.akto.runtime.RuntimeUtil.extractAllValuesFromPayload(payload,payloadMap); } public String decrementUrlVersion(String url, int decrementValue, int limit) { @@ -92,55 +88,7 @@ public String decrementUrlVersion(String url, int decrementValue, int limit) { } public static double compareWithOriginalResponse(String originalPayload, String currentPayload, Map comparisonExcludedKeys) { - if (originalPayload == null && currentPayload == null) return 100; - if (originalPayload == null || currentPayload == null) return 0; - - String trimmedOriginalPayload = originalPayload.trim(); - String trimmedCurrentPayload = currentPayload.trim(); - if (trimmedCurrentPayload.equals(trimmedOriginalPayload)) return 100; - - Map> originalResponseParamMap = new HashMap<>(); - Map> currentResponseParamMap = new HashMap<>(); - try { - extractAllValuesFromPayload(originalPayload, originalResponseParamMap); - extractAllValuesFromPayload(currentPayload, currentResponseParamMap); - } catch (Exception e) { - return 0.0; - } - - if (originalResponseParamMap.keySet().size() == 0 && currentResponseParamMap.keySet().size() == 0) { - return 100.0; - } - - Set visited = new HashSet<>(); - int matched = 0; - for (String k1: originalResponseParamMap.keySet()) { - if (visited.contains(k1) || comparisonExcludedKeys.containsKey(k1)) continue; - visited.add(k1); - Set v1 = originalResponseParamMap.get(k1); - Set v2 = currentResponseParamMap.get(k1); - if (Objects.equals(v1, v2)) matched +=1; - } - - for (String k1: currentResponseParamMap.keySet()) { - if (visited.contains(k1) || comparisonExcludedKeys.containsKey(k1)) continue; - visited.add(k1); - Set v1 = originalResponseParamMap.get(k1); - Set v2 = currentResponseParamMap.get(k1); - if (Objects.equals(v1, v2)) matched +=1; - } - - int visitedSize = visited.size(); - if (visitedSize == 0) return 0.0; - - double result = (100.0*matched)/visitedSize; - - if (Double.isFinite(result)) { - return result; - } else { - return 0.0; - } - + return com.akto.testing.Utils.compareWithOriginalResponse(originalPayload, currentPayload, comparisonExcludedKeys); } public Result addWithoutRequestError(String originalMessage, TestResult.TestError testError) { @@ -152,7 +100,7 @@ public Result addWithoutRequestError(String originalMessage, TestResult.TestErro public TestResult buildFailedTestResultWithOriginalMessage(String originalMessage, TestResult.TestError testError, OriginalHttpRequest request, TestInfo testInfo) { String message = null; try { - message = RedactSampleData.convertOriginalReqRespToString(request, null); + message = convertOriginalReqRespToString(request, null); } catch (Exception e) { loggerMaker.errorAndAddToDb("Error while converting testRequest object to string : " + e, LogDb.TESTING); } @@ -173,11 +121,11 @@ public TestResult buildTestResult(OriginalHttpRequest request, OriginalHttpRespo List errors = new ArrayList<>(); String message = null; try { - message = RedactSampleData.convertOriginalReqRespToString(request, response); + message = convertOriginalReqRespToString(request, response); } catch (Exception e) { // TODO: logger.error("Error while converting OriginalHttpRequest to string", e); - message = RedactSampleData.convertOriginalReqRespToString(new OriginalHttpRequest(), new OriginalHttpResponse()); + message = convertOriginalReqRespToString(new OriginalHttpRequest(), new OriginalHttpResponse()); errors.add(TestResult.TestError.FAILED_TO_CONVERT_TEST_REQUEST_TO_STRING.getMessage()); } @@ -363,10 +311,8 @@ public static Map getComparisonExcludedKeys(SampleRequestReplay return comparisonExcludedKeys; } - public static boolean validateFilter(FilterNode filterNode, RawApi rawApi, ApiInfoKey apiInfoKey, Map varMap, String logId) { - if (filterNode == null) return true; - if (rawApi == null) return false; - return validate(filterNode, rawApi, null, apiInfoKey,"filter", varMap, logId); + public static ValidationResult validateFilter(FilterNode filterNode, RawApi rawApi, ApiInfoKey apiInfoKey, Map varMap, String logId) { + return com.akto.testing.Utils.validateFilter(filterNode, rawApi, apiInfoKey, varMap, logId); } public static boolean validateValidator(FilterNode validatorNode, RawApi rawApi, RawApi testRawApi, ApiInfoKey apiInfoKey, Map varMap, String logId) { diff --git a/apps/testing/src/main/java/com/akto/test_editor/execution/Build.java b/apps/testing/src/main/java/com/akto/test_editor/execution/Build.java index 3ecab13e43..275e481029 100644 --- a/apps/testing/src/main/java/com/akto/test_editor/execution/Build.java +++ b/apps/testing/src/main/java/com/akto/test_editor/execution/Build.java @@ -13,7 +13,6 @@ import com.akto.dto.traffic.SampleData; import com.akto.dto.type.URLMethods; import com.akto.log.LoggerMaker; -import com.akto.runtime.policies.AuthPolicy; import com.akto.testing.ApiExecutor; import com.akto.util.Constants; import com.akto.util.HttpRequestResponseUtils; @@ -30,6 +29,7 @@ import static com.akto.util.HttpRequestResponseUtils.FORM_URL_ENCODED_CONTENT_TYPE; import static com.akto.util.HttpRequestResponseUtils.extractValuesFromPayload; +import static com.akto.runtime.utils.Utils.parseCookie; public class Build { @@ -408,7 +408,7 @@ public static Map> getValuesMap(OriginalHttpResponse respons if (values == null) continue; if (headerKey.equalsIgnoreCase("set-cookie")) { - Map cookieMap = AuthPolicy.parseCookie(values); + Map cookieMap = parseCookie(values); for (String cookieKey : cookieMap.keySet()) { String cookieVal = cookieMap.get(cookieKey); valuesMap.put(cookieKey, new HashSet<>(Collections.singletonList(cookieVal))); diff --git a/apps/testing/src/main/java/com/akto/test_editor/execution/Executor.java b/apps/testing/src/main/java/com/akto/test_editor/execution/Executor.java index 92a833da9d..2729622822 100644 --- a/apps/testing/src/main/java/com/akto/test_editor/execution/Executor.java +++ b/apps/testing/src/main/java/com/akto/test_editor/execution/Executor.java @@ -1,16 +1,11 @@ package com.akto.test_editor.execution; -import com.akto.billing.UsageMetricUtils; import com.akto.dao.billing.OrganizationsDao; -import com.akto.dao.billing.TokensDao; import java.util.*; import java.util.regex.Matcher; import java.util.regex.Pattern; -import com.akto.dao.CustomAuthTypeDao; import com.akto.dao.context.Context; -import com.akto.dao.test_editor.TestEditorEnums; -import com.akto.dao.test_editor.TestEditorEnums.ExecutorOperandTypes; import com.akto.dao.testing.TestRolesDao; import com.akto.dto.ApiInfo; import com.akto.dto.CustomAuthType; @@ -22,11 +17,7 @@ import com.akto.testing.*; import com.akto.util.enums.LoginFlowEnums; import com.akto.util.enums.LoginFlowEnums.AuthMechanismTypes; -import com.akto.util.enums.LoginFlowEnums.LoginStepTypesEnums; -import com.akto.utils.RedactSampleData; import com.akto.dto.api_workflow.Graph; -import com.akto.dto.billing.Organization; -import com.akto.dto.billing.Tokens; import com.akto.dto.test_editor.*; import com.akto.dto.testing.TestResult.Confidence; import com.akto.dto.testing.TestResult.TestError; @@ -37,9 +28,6 @@ import com.akto.test_editor.Utils; import com.akto.testing.TestExecutor; import com.akto.util.Constants; -import com.akto.util.UsageUtils; -import com.akto.util.enums.LoginFlowEnums; -import com.akto.util.enums.LoginFlowEnums.AuthMechanismTypes; import com.akto.util.modifier.JWTPayloadReplacer; import com.fasterxml.jackson.databind.ObjectMapper; @@ -48,19 +36,17 @@ import org.json.JSONObject; import com.mongodb.BasicDBObject; -import static com.akto.rules.TestPlugin.extractAllValuesFromPayload; import static com.akto.test_editor.Utils.bodyValuesUnchanged; import static com.akto.test_editor.Utils.headerValuesUnchanged; +import static com.akto.runtime.utils.Utils.convertOriginalReqRespToString; +import static com.akto.testing.Utils.compareWithOriginalResponse; import com.mongodb.client.model.Filters; -import org.json.JSONObject; -import org.kohsuke.github.GHRateLimit.Record; -import com.mongodb.client.model.Updates; import org.apache.commons.lang3.StringUtils; -import org.bson.conversions.Bson; import org.bson.types.ObjectId; + public class Executor { private static final LoggerMaker loggerMaker = new LoggerMaker(Executor.class); @@ -173,7 +159,7 @@ public YamlTestResult execute(ExecutorNode node, RawApi rawApi, Map() ); } @@ -538,37 +524,6 @@ private ExecutorSingleOperationResp modifyAuthTokenInRawApi(TestRoles testRole, return null; } - private static BasicDBObject getBillingTokenForAuth() { - BasicDBObject bDObject; - int accountId = Context.accountId.get(); - Organization organization = OrganizationsDao.instance.findOne( - Filters.in(Organization.ACCOUNTS, accountId) - ); - if (organization == null) { - return new BasicDBObject("error", "organization not found"); - } - - Tokens tokens; - Bson filters = Filters.and( - Filters.eq(Tokens.ORG_ID, organization.getId()), - Filters.eq(Tokens.ACCOUNT_ID, accountId) - ); - String errMessage = ""; - tokens = TokensDao.instance.findOne(filters); - if (tokens == null) { - errMessage = "error extracting ${akto_header}, token is missing"; - } - if (tokens.isOldToken()) { - errMessage = "error extracting ${akto_header}, token is old"; - } - if(errMessage.length() > 0){ - bDObject = new BasicDBObject("error", errMessage); - }else{ - bDObject = new BasicDBObject("token", tokens.getToken()); - } - return bDObject; - } - public ExecutorSingleOperationResp runOperation(String operationType, RawApi rawApi, Object key, Object value, Map varMap, AuthMechanism authMechanism, List customAuthTypes, ApiInfo.ApiInfoKey apiInfoKey) { switch (operationType.toLowerCase()) { case "send_ssrf_req": @@ -580,7 +535,7 @@ public ExecutorSingleOperationResp runOperation(String operationType, RawApi raw uuidList.add(generatedUUID); varMap.put("random_uuid", uuidList); - BasicDBObject response = getBillingTokenForAuth(); + BasicDBObject response = OrganizationsDao.getBillingTokenForAuth(); if(response.getString("token") != null){ String tokenVal = response.getString("token"); return Utils.sendRequestToSsrfServer(url + generatedUUID, redirectUrl, tokenVal); @@ -589,57 +544,9 @@ public ExecutorSingleOperationResp runOperation(String operationType, RawApi raw } case "attach_file": return Operations.addHeader(rawApi, Constants.AKTO_ATTACH_FILE , key.toString()); - case "add_body_param": - Object epochVal = Utils.getEpochTime(value); - if (epochVal != null) { - value = epochVal; - } - return Operations.addBody(rawApi, key.toString(), value); - case "modify_body_param": - epochVal = Utils.getEpochTime(value); - if (epochVal != null) { - value = epochVal; - } - return Operations.modifyBodyParam(rawApi, key.toString(), value); - case "delete_graphql_field": - return Operations.deleteGraphqlField(rawApi, key == null ? "": key.toString()); - case "add_graphql_field": - return Operations.addGraphqlField(rawApi, key == null ? "": key.toString(), value == null ? "" : value.toString()); - case "add_unique_graphql_field": - return Operations.addUniqueGraphqlField(rawApi, key == null ? "": key.toString(), value == null ? "" : value.toString()); - case "modify_graphql_field": - return Operations.modifyGraphqlField(rawApi, key == null ? "": key.toString(), value == null ? "" : value.toString()); - case "delete_body_param": - return Operations.deleteBodyParam(rawApi, key.toString()); - case "replace_body": - String newPayload = rawApi.getRequest().getBody(); - if (key instanceof Map) { - Map> regexReplace = (Map) key; - String payload = rawApi.getRequest().getBody(); - Map regexInfo = regexReplace.get("regex_replace"); - String regex = regexInfo.get("regex"); - String replaceWith = regexInfo.get("replace_with"); - newPayload = Utils.applyRegexModifier(payload, regex, replaceWith); - } else { - newPayload = key.toString(); - } - return Operations.replaceBody(rawApi, newPayload); - case "add_header": - if (value.equals("${akto_header}")) { - BasicDBObject tokenResponse = getBillingTokenForAuth(); - if(tokenResponse.getString("token") != null){ - value = tokenResponse.getString("token"); - }else{ - return new ExecutorSingleOperationResp(false, tokenResponse.getString("error")); - } - } - epochVal = Utils.getEpochTime(value); - if (epochVal != null) { - value = epochVal; - } - return Operations.addHeader(rawApi, key.toString(), value.toString()); case "modify_header": + Object epochVal = Utils.getEpochTime(value); String keyStr = key.toString(); String valStr = value.toString(); @@ -666,33 +573,6 @@ public ExecutorSingleOperationResp runOperation(String operationType, RawApi raw } return Operations.modifyHeader(rawApi, keyStr, valStr); } - case "delete_header": - return Operations.deleteHeader(rawApi, key.toString()); - case "add_query_param": - epochVal = Utils.getEpochTime(value); - if (epochVal != null) { - value = epochVal; - } - return Operations.addQueryParam(rawApi, key.toString(), value); - case "modify_query_param": - epochVal = Utils.getEpochTime(value); - if (epochVal != null) { - value = epochVal; - } - return Operations.modifyQueryParam(rawApi, key.toString(), value); - case "delete_query_param": - return Operations.deleteQueryParam(rawApi, key.toString()); - case "modify_url": - String newUrl = null; - UrlModifierPayload urlModifierPayload = Utils.fetchUrlModifyPayload(key.toString()); - if (urlModifierPayload != null) { - newUrl = Utils.buildNewUrl(urlModifierPayload, rawApi.getRequest().getUrl()); - } else { - newUrl = key.toString(); - } - return Operations.modifyUrl(rawApi, newUrl); - case "modify_method": - return Operations.modifyMethod(rawApi, key.toString()); case "remove_auth_header": List authHeaders = (List) varMap.get("auth_headers"); boolean removed = false; @@ -823,7 +703,7 @@ public ExecutorSingleOperationResp runOperation(String operationType, RawApi raw return new ExecutorSingleOperationResp(true, ""); } default: - return new ExecutorSingleOperationResp(false, "invalid operationType"); + return Utils.modifySampleDataUtil(operationType, rawApi, key, value, varMap, apiInfoKey); } } diff --git a/apps/testing/src/main/java/com/akto/test_editor/execution/Memory.java b/apps/testing/src/main/java/com/akto/test_editor/execution/Memory.java index a66cb2bcc2..4549b75f41 100644 --- a/apps/testing/src/main/java/com/akto/test_editor/execution/Memory.java +++ b/apps/testing/src/main/java/com/akto/test_editor/execution/Memory.java @@ -12,7 +12,6 @@ import com.akto.dto.traffic.SampleData; import com.akto.dto.type.SingleTypeInfo; import com.akto.testing.ApiExecutor; -import com.akto.types.CappedSet; import com.akto.util.Constants; import com.mongodb.client.model.Filters; import org.bson.conversions.Bson; diff --git a/apps/testing/src/main/java/com/akto/testing/workflow_node_executor/ApiNodeExecutor.java b/apps/testing/src/main/java/com/akto/testing/workflow_node_executor/ApiNodeExecutor.java index 92645aa6c6..d205891d76 100644 --- a/apps/testing/src/main/java/com/akto/testing/workflow_node_executor/ApiNodeExecutor.java +++ b/apps/testing/src/main/java/com/akto/testing/workflow_node_executor/ApiNodeExecutor.java @@ -5,7 +5,6 @@ import java.util.Map; import java.util.Collections; -import com.akto.dto.ApiInfo; import com.akto.dto.OriginalHttpRequest; import com.akto.dto.OriginalHttpResponse; import com.akto.dto.api_workflow.Node; @@ -18,9 +17,8 @@ import com.akto.log.LoggerMaker.LogDb; import com.akto.test_editor.execution.Memory; import com.akto.testing.ApiExecutor; -import com.akto.testing.Main; import com.akto.testing.Utils; -import com.akto.utils.RedactSampleData; +import static com.akto.runtime.utils.Utils.convertOriginalReqRespToString; public class ApiNodeExecutor extends NodeExecutor { @@ -73,7 +71,7 @@ public NodeResult processNode(Node node, Map valuesMap, Boolean Thread.sleep(sleep); } - response = ApiExecutor.sendRequest(request, followRedirects, null, debug, testLogs, Main.SKIP_SSRF_CHECK); + response = ApiExecutor.sendRequest(request, followRedirects, null, debug, testLogs, com.akto.test_editor.Utils.SKIP_SSRF_CHECK); int statusCode = response.getStatusCode(); @@ -94,7 +92,7 @@ public NodeResult processNode(Node node, Map valuesMap, Boolean String message = null; try { - message = RedactSampleData.convertOriginalReqRespToString(request, response); + message = convertOriginalReqRespToString(request, response); } catch (Exception e) { ; } diff --git a/apps/testing/src/main/java/com/akto/testing/workflow_node_executor/YamlNodeExecutor.java b/apps/testing/src/main/java/com/akto/testing/workflow_node_executor/YamlNodeExecutor.java index 6f8eb0d787..42faa5edc7 100644 --- a/apps/testing/src/main/java/com/akto/testing/workflow_node_executor/YamlNodeExecutor.java +++ b/apps/testing/src/main/java/com/akto/testing/workflow_node_executor/YamlNodeExecutor.java @@ -8,7 +8,6 @@ import java.util.Map; import com.akto.dto.type.SingleTypeInfo; -import com.akto.testing.Main; import com.akto.dto.*; import com.akto.dto.type.URLMethods; import com.akto.test_editor.execution.Memory; @@ -42,7 +41,7 @@ import com.akto.testing.ApiExecutor; import com.akto.testing.TestExecutor; import com.akto.util.Constants; -import com.akto.utils.RedactSampleData; +import static com.akto.runtime.utils.Utils.convertOriginalReqRespToString; import com.google.gson.Gson; public class YamlNodeExecutor extends NodeExecutor { @@ -146,7 +145,7 @@ public NodeResult processNode(Node node, Map varMap, Boolean all int tsAfterReq = 0; try { tsBeforeReq = Context.nowInMillis(); - testResponse = ApiExecutor.sendRequest(testReq.getRequest(), followRedirect, testingRunConfig, debug, testLogs, Main.SKIP_SSRF_CHECK); + testResponse = ApiExecutor.sendRequest(testReq.getRequest(), followRedirect, testingRunConfig, debug, testLogs, com.akto.test_editor.Utils.SKIP_SSRF_CHECK); if (apiInfoKey != null && memory != null) { memory.fillResponse(testReq.getRequest(), testResponse, apiInfoKey.getApiCollectionId(), apiInfoKey.getUrl(), apiInfoKey.getMethod().name()); memory.reset(apiInfoKey.getApiCollectionId(), apiInfoKey.getUrl(), apiInfoKey.getMethod().name()); @@ -160,7 +159,7 @@ public NodeResult processNode(Node node, Map varMap, Boolean all } vulnerable = res.getVulnerable(); try { - message.add(RedactSampleData.convertOriginalReqRespToString(testReq.getRequest(), testResponse)); + message.add(convertOriginalReqRespToString(testReq.getRequest(), testResponse)); } catch (Exception e) { ; } diff --git a/apps/testing/src/main/java/com/akto/testing/yaml_tests/YamlTestTemplate.java b/apps/testing/src/main/java/com/akto/testing/yaml_tests/YamlTestTemplate.java index 57294919c5..8712f4924a 100644 --- a/apps/testing/src/main/java/com/akto/testing/yaml_tests/YamlTestTemplate.java +++ b/apps/testing/src/main/java/com/akto/testing/yaml_tests/YamlTestTemplate.java @@ -13,6 +13,7 @@ import com.akto.test_editor.Utils; import com.akto.test_editor.auth.AuthValidator; import com.akto.test_editor.execution.Executor; +import com.akto.test_editor.filter.data_operands_impl.ValidationResult; import com.akto.testing.StatusCodeAnalyser; import java.util.ArrayList; @@ -104,9 +105,9 @@ public boolean filter() { return false; } } - boolean isValid = TestPlugin.validateFilter(this.getFilterNode(),this.getRawApi(), this.getApiInfoKey(), this.varMap, this.logId); + ValidationResult isValid = TestPlugin.validateFilter(this.getFilterNode(),this.getRawApi(), this.getApiInfoKey(), this.varMap, this.logId); // loggerMaker.infoAndAddToDb("filter status " + isValid + " " + logId, LogDb.TESTING); - return isValid; + return isValid.getIsValid(); } diff --git a/libs/dao/src/main/java/com/akto/dao/billing/OrganizationsDao.java b/libs/dao/src/main/java/com/akto/dao/billing/OrganizationsDao.java index 9da9e43177..81f90a2ec8 100644 --- a/libs/dao/src/main/java/com/akto/dao/billing/OrganizationsDao.java +++ b/libs/dao/src/main/java/com/akto/dao/billing/OrganizationsDao.java @@ -1,8 +1,13 @@ package com.akto.dao.billing; +import org.bson.conversions.Bson; + import com.akto.dao.BillingContextDao; import com.akto.dao.MCollection; +import com.akto.dao.context.Context; import com.akto.dto.billing.Organization; +import com.akto.dto.billing.Tokens; +import com.mongodb.BasicDBObject; import com.mongodb.client.model.Filters; public class OrganizationsDao extends BillingContextDao{ @@ -39,4 +44,35 @@ public Organization findOneByAccountId(int accountId) { Filters.in(Organization.ACCOUNTS, accountId)); } + public static BasicDBObject getBillingTokenForAuth() { + BasicDBObject bDObject; + int accountId = Context.accountId.get(); + Organization organization = OrganizationsDao.instance.findOne( + Filters.in(Organization.ACCOUNTS, accountId) + ); + if (organization == null) { + return new BasicDBObject("error", "organization not found"); + } + + Tokens tokens; + Bson filters = Filters.and( + Filters.eq(Tokens.ORG_ID, organization.getId()), + Filters.eq(Tokens.ACCOUNT_ID, accountId) + ); + String errMessage = ""; + tokens = TokensDao.instance.findOne(filters); + if (tokens == null) { + errMessage = "error extracting ${akto_header}, token is missing"; + } + if (tokens.isOldToken()) { + errMessage = "error extracting ${akto_header}, token is old"; + } + if(errMessage.length() > 0){ + bDObject = new BasicDBObject("error", errMessage); + }else{ + bDObject = new BasicDBObject("token", tokens.getToken()); + } + return bDObject; + } + } diff --git a/libs/dao/src/main/java/com/akto/dao/monitoring/FilterConfigYamlParser.java b/libs/dao/src/main/java/com/akto/dao/monitoring/FilterConfigYamlParser.java new file mode 100644 index 0000000000..e1593bf32c --- /dev/null +++ b/libs/dao/src/main/java/com/akto/dao/monitoring/FilterConfigYamlParser.java @@ -0,0 +1,76 @@ +package com.akto.dao.monitoring; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import com.akto.dao.test_editor.filter.ConfigParser; +import com.akto.dto.monitoring.FilterConfig; +import com.akto.dto.test_editor.ConfigParserResult; +import com.akto.dto.test_editor.ExecutorConfigParserResult; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; + +public class FilterConfigYamlParser { + + public static FilterConfig parseTemplate(String content, boolean shouldParseExecutor) throws Exception { + + ObjectMapper mapper = new ObjectMapper(new YAMLFactory()); + + Map config = mapper.readValue(content, new TypeReference>() { + }); + return parseConfig(config, shouldParseExecutor); + } + + public static FilterConfig parseConfig(Map config,boolean shouldParseExecutor) throws Exception { + + FilterConfig filterConfig = null; + boolean isFilterError = false; + + String id = (String) config.get(FilterConfig.ID); + if (id == null) { + return filterConfig; + } + + Object filterMap = config.get(FilterConfig.FILTER); + if (filterMap == null) { + isFilterError = true; + filterConfig = new FilterConfig(id, null, null); + } + + ConfigParser configParser = new ConfigParser(); + ConfigParserResult filters = configParser.parse(filterMap); + if (filters == null) { + // todo: throw error + isFilterError = true; + filterConfig = new FilterConfig(id, null, null); + } + + Map> wordListMap = new HashMap<>(); + try { + if (config.containsKey(FilterConfig.WORD_LISTS)) { + wordListMap = (Map) config.get(FilterConfig.WORD_LISTS); + } + } catch (Exception e) { + isFilterError = true; + filterConfig = new FilterConfig(id, filters, null); + } + if(!isFilterError){ + filterConfig = new FilterConfig(id, filters, wordListMap); + } + + if(shouldParseExecutor){ + com.akto.dao.test_editor.executor.ConfigParser executorConfigParser = new com.akto.dao.test_editor.executor.ConfigParser(); + Object executionMap = config.get("execute"); + if(executionMap == null){ + return filterConfig; + } + ExecutorConfigParserResult executorConfigParserResult = executorConfigParser.parseConfigMap(executionMap); + filterConfig.setExecutor(executorConfigParserResult); + } + + return filterConfig; + } + +} diff --git a/libs/dao/src/main/java/com/akto/dao/monitoring/FilterYamlTemplateDao.java b/libs/dao/src/main/java/com/akto/dao/monitoring/FilterYamlTemplateDao.java new file mode 100644 index 0000000000..8a898c387c --- /dev/null +++ b/libs/dao/src/main/java/com/akto/dao/monitoring/FilterYamlTemplateDao.java @@ -0,0 +1,51 @@ +package com.akto.dao.monitoring; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import com.akto.dao.AccountsContextDao; +import com.akto.dto.monitoring.FilterConfig; +import com.akto.dto.test_editor.YamlTemplate; +import com.mongodb.client.model.Filters; + +public class FilterYamlTemplateDao extends AccountsContextDao { + + public static final FilterYamlTemplateDao instance = new FilterYamlTemplateDao(); + + public Map fetchFilterConfig(boolean includeYamlContent, boolean shouldParseExecutor) { + List yamlTemplates = FilterYamlTemplateDao.instance.findAll(Filters.empty()); + return fetchFilterConfig(includeYamlContent, yamlTemplates, shouldParseExecutor); + } + + public Map fetchFilterConfig(boolean includeYamlContent, List yamlTemplates, boolean shouldParseExecutor) { + Map filterConfigMap = new HashMap<>(); + for (YamlTemplate yamlTemplate : yamlTemplates) { + try { + if (yamlTemplate != null) { + FilterConfig filterConfig = FilterConfigYamlParser.parseTemplate(yamlTemplate.getContent(), shouldParseExecutor); + filterConfig.setAuthor(yamlTemplate.getAuthor()); + filterConfig.setCreatedAt(yamlTemplate.getCreatedAt()); + filterConfig.setUpdatedAt(yamlTemplate.getUpdatedAt()); + if (includeYamlContent) { + filterConfig.setContent(yamlTemplate.getContent()); + } + filterConfigMap.put(filterConfig.getId(), filterConfig); + } + } catch (Exception e) { + e.printStackTrace(); + } + } + return filterConfigMap; + } + + @Override + public String getCollName() { + return "filter_yaml_templates"; + } + + @Override + public Class getClassT() { + return YamlTemplate.class; + } +} diff --git a/libs/dao/src/main/java/com/akto/dao/runtime_filters/AdvancedTrafficFiltersDao.java b/libs/dao/src/main/java/com/akto/dao/runtime_filters/AdvancedTrafficFiltersDao.java new file mode 100644 index 0000000000..cc74f15939 --- /dev/null +++ b/libs/dao/src/main/java/com/akto/dao/runtime_filters/AdvancedTrafficFiltersDao.java @@ -0,0 +1,19 @@ +package com.akto.dao.runtime_filters; + +import com.akto.dao.AccountsContextDao; +import com.akto.dto.test_editor.YamlTemplate; + +public class AdvancedTrafficFiltersDao extends AccountsContextDao { + + public static final AdvancedTrafficFiltersDao instance = new AdvancedTrafficFiltersDao(); + + @Override + public String getCollName() { + return "advanced_traffic_filters"; + } + + @Override + public Class getClassT() { + return YamlTemplate.class; + } +} diff --git a/libs/dao/src/main/java/com/akto/dto/HttpResponseParams.java b/libs/dao/src/main/java/com/akto/dto/HttpResponseParams.java index ec17fd0410..b5b96db20b 100644 --- a/libs/dao/src/main/java/com/akto/dto/HttpResponseParams.java +++ b/libs/dao/src/main/java/com/akto/dto/HttpResponseParams.java @@ -157,4 +157,8 @@ public String getDirection() { public void setDirection(String direction) { this.direction = direction; } + + public void setRequestParams(HttpRequestParams requestParams) { + this.requestParams = requestParams; + } } diff --git a/libs/dao/src/main/java/com/akto/dto/monitoring/FilterConfig.java b/libs/dao/src/main/java/com/akto/dto/monitoring/FilterConfig.java new file mode 100644 index 0000000000..31ce9e2353 --- /dev/null +++ b/libs/dao/src/main/java/com/akto/dto/monitoring/FilterConfig.java @@ -0,0 +1,117 @@ +package com.akto.dto.monitoring; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import com.akto.dto.test_editor.ConfigParserResult; +import com.akto.dto.test_editor.ExecutorConfigParserResult; + +public class FilterConfig { + private String id; + public static final String ID = "id"; + private ConfigParserResult filter; + public static final String FILTER = "filter"; + private Map> wordLists; + public static final String WORD_LISTS = "wordLists"; + public static final String CREATED_AT = "createdAt"; + private int createdAt; + public static final String UPDATED_AT = "updatedAt"; + private int updatedAt; + public static final String _AUTHOR = "author"; + private String author; + public static final String _CONTENT = "content"; + private String content; + + private ExecutorConfigParserResult executor; + + public static final String DEFAULT_ALLOW_FILTER = "DEFAULT_ALLOW_FILTER"; + public static final String DEFAULT_BLOCK_FILTER = "DEFAULT_BLOCK_FILTER"; + + public enum FILTER_TYPE{ + BLOCKED , ALLOWED, MODIFIED, UNCHANGED + } + + public FilterConfig(String id, ConfigParserResult filter, Map> wordLists) { + this.id = id; + this.filter = filter; + this.wordLists = wordLists; + } + + public FilterConfig() { + } + + public String getId() { + return id; + } + + public void setId(String id) { + this.id = id; + } + + public ConfigParserResult getFilter() { + return filter; + } + + public void setFilter(ConfigParserResult filter) { + this.filter = filter; + } + + public int getCreatedAt() { + return createdAt; + } + + public void setCreatedAt(int createdAt) { + this.createdAt = createdAt; + } + + public int getUpdatedAt() { + return updatedAt; + } + + public void setUpdatedAt(int updatedAt) { + this.updatedAt = updatedAt; + } + + public String getAuthor() { + return author; + } + + public void setAuthor(String author) { + this.author = author; + } + + public Map> getWordLists() { + return wordLists; + } + + public Map resolveVarMap() { + Map> wordListsMap = this.wordLists == null ? new HashMap<>() : this.wordLists; + Map varMap = new HashMap<>(); + + for (String key : wordListsMap.keySet()) { + varMap.put("wordList_" + key, wordListsMap.get(key)); + } + return varMap; + } + + public void setWordLists(Map> wordLists) { + this.wordLists = wordLists; + } + + public String getContent() { + return content; + } + + public void setContent(String content) { + this.content = content; + } + + public ExecutorConfigParserResult getExecutor() { + return executor; + } + + public void setExecutor(ExecutorConfigParserResult executor) { + this.executor = executor; + } +} \ No newline at end of file diff --git a/libs/dao/src/main/java/com/akto/dto/test_editor/DataOperandsFilterResponse.java b/libs/dao/src/main/java/com/akto/dto/test_editor/DataOperandsFilterResponse.java index 8b49a58203..3a0b3008d2 100644 --- a/libs/dao/src/main/java/com/akto/dto/test_editor/DataOperandsFilterResponse.java +++ b/libs/dao/src/main/java/com/akto/dto/test_editor/DataOperandsFilterResponse.java @@ -10,12 +10,22 @@ public class DataOperandsFilterResponse { private List matchedEntities; private List contextEntities; private FilterNode extractNode; + private String validationReason; public DataOperandsFilterResponse(Boolean result, List matchedEntities, List contextEntities, FilterNode extractNode) { this.result = result; this.matchedEntities = matchedEntities; this.contextEntities = contextEntities; this.extractNode = extractNode; + this.validationReason = null; + } + + public DataOperandsFilterResponse(Boolean result, List matchedEntities, List contextEntities, FilterNode extractNode, String validationReason) { + this.result = result; + this.matchedEntities = matchedEntities; + this.contextEntities = contextEntities; + this.extractNode = extractNode; + this.validationReason = validationReason; } public DataOperandsFilterResponse() { } @@ -52,4 +62,11 @@ public void setExtractNode(FilterNode extractNode) { this.extractNode = extractNode; } + public String getValidationReason() { + return validationReason; + } + + public void setValidationReason(String validationReason) { + this.validationReason = validationReason; + } } diff --git a/libs/utils/src/main/java/com/akto/data_actor/ClientActor.java b/libs/utils/src/main/java/com/akto/data_actor/ClientActor.java index 17cec6865c..21f51cd139 100644 --- a/libs/utils/src/main/java/com/akto/data_actor/ClientActor.java +++ b/libs/utils/src/main/java/com/akto/data_actor/ClientActor.java @@ -76,6 +76,9 @@ import java.util.Set; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; + +import org.bson.types.ObjectId; + import com.google.gson.Gson; public class ClientActor extends DataActor { @@ -3020,4 +3023,36 @@ public List fetchLatestEndpointsForTesting(int startTimestam return respList; } + public List fetchActiveAdvancedFilters(){ + Map> headers = buildHeaders(); + + List respList = new ArrayList<>(); + OriginalHttpRequest request = new OriginalHttpRequest(url + "/fetchActiveAdvancedFilters", "", "POST", "", headers, ""); + try { + OriginalHttpResponse response = ApiExecutor.sendRequestBackOff(request, true, null, false, null); + String responsePayload = response.getBody(); + if (response.getStatusCode() != 200 || responsePayload == null) { + loggerMaker.errorAndAddToDb("non 2xx response in fetchActiveAdvancedFilters", LoggerMaker.LogDb.RUNTIME); + return null; + } + BasicDBObject payloadObj; + + try { + payloadObj = BasicDBObject.parse(responsePayload); + BasicDBList newTemplates = (BasicDBList) payloadObj.get("activeAdvancedFilters"); + for (Object template: newTemplates) { + BasicDBObject templateObject = (BasicDBObject) template; + YamlTemplate yamlTemplate = objectMapper.readValue(templateObject.toJson(), YamlTemplate.class); + respList.add(yamlTemplate); + } + } catch (Exception e) { + loggerMaker.errorAndAddToDb("error extracting response in fetchActiveAdvancedFilters" + e, LoggerMaker.LogDb.RUNTIME); + } + } catch (Exception e) { + loggerMaker.errorAndAddToDb("error in fetching filter yaml templates" + e, LoggerMaker.LogDb.RUNTIME); + return null; + } + return respList; + } + } diff --git a/libs/utils/src/main/java/com/akto/data_actor/DataActor.java b/libs/utils/src/main/java/com/akto/data_actor/DataActor.java index da62ca5f89..9bf3661efd 100644 --- a/libs/utils/src/main/java/com/akto/data_actor/DataActor.java +++ b/libs/utils/src/main/java/com/akto/data_actor/DataActor.java @@ -225,4 +225,6 @@ public abstract class DataActor { public abstract void bulkWriteDependencyNodes(List dependencyNodeList); public abstract List fetchLatestEndpointsForTesting(int startTimestamp, int endTimestamp, int apiCollectionId); + + public abstract List fetchActiveAdvancedFilters(); } diff --git a/libs/utils/src/main/java/com/akto/data_actor/DbActor.java b/libs/utils/src/main/java/com/akto/data_actor/DbActor.java index 0ff7d26b3a..f0fd24e315 100644 --- a/libs/utils/src/main/java/com/akto/data_actor/DbActor.java +++ b/libs/utils/src/main/java/com/akto/data_actor/DbActor.java @@ -468,4 +468,8 @@ public List fetchLatestEndpointsForTesting(int startTimestam return DbLayer.fetchLatestEndpointsForTesting(startTimestamp, endTimestamp, apiCollectionId); } + public List fetchActiveAdvancedFilters(){ + return DbLayer.fetchActiveFilterTemplates(); + } + } diff --git a/libs/utils/src/main/java/com/akto/data_actor/DbLayer.java b/libs/utils/src/main/java/com/akto/data_actor/DbLayer.java index ce968a5aaa..c76151dce0 100644 --- a/libs/utils/src/main/java/com/akto/data_actor/DbLayer.java +++ b/libs/utils/src/main/java/com/akto/data_actor/DbLayer.java @@ -23,6 +23,7 @@ import com.akto.dao.billing.OrganizationsDao; import com.akto.dao.billing.TokensDao; import com.akto.dao.context.Context; +import com.akto.dao.runtime_filters.AdvancedTrafficFiltersDao; import com.akto.dao.test_editor.YamlTemplateDao; import com.akto.dao.testing.AccessMatrixTaskInfosDao; import com.akto.dao.testing.AccessMatrixUrlToRolesDao; @@ -858,4 +859,10 @@ public static void bulkWriteDependencyNodes(List dependencyNodeL public static List fetchLatestEndpointsForTesting(int startTimestamp, int endTimestamp, int apiCollectionId) { return SingleTypeInfoDao.fetchLatestEndpointsForTesting(startTimestamp, endTimestamp, apiCollectionId); } + + public static List fetchActiveFilterTemplates(){ + return AdvancedTrafficFiltersDao.instance.findAll( + Filters.ne(YamlTemplate.INACTIVE, false) + ); + } } diff --git a/libs/utils/src/main/java/com/akto/runtime/RuntimeUtil.java b/libs/utils/src/main/java/com/akto/runtime/RuntimeUtil.java index 8a3e5d8a02..dc312e1723 100644 --- a/libs/utils/src/main/java/com/akto/runtime/RuntimeUtil.java +++ b/libs/utils/src/main/java/com/akto/runtime/RuntimeUtil.java @@ -2,13 +2,18 @@ import com.akto.dto.HttpResponseParams; import com.akto.dto.settings.DefaultPayload; +import com.akto.dto.type.SingleTypeInfo.SuperType; +import com.akto.dto.type.URLMethods.Method; +import com.akto.dto.type.URLTemplate; import com.akto.log.LoggerMaker; +import com.fasterxml.jackson.core.JsonFactory; +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ArrayNode; import java.net.URL; -import java.util.ArrayList; -import java.util.Base64; -import java.util.List; -import java.util.Map; +import java.util.*; public class RuntimeUtil { private static final LoggerMaker loggerMaker = new LoggerMaker(RuntimeUtil.class); @@ -39,4 +44,100 @@ public static boolean matchesDefaultPayload(HttpResponseParams httpResponseParam return false; } + + public static boolean hasSpecialCharacters(String input) { + // Define the special characters + String specialCharacters = "<>%/?#[]@!$&'()*+,;="; + for (char c : input.toCharArray()) { + if (specialCharacters.contains(Character.toString(c))) { + return true; + } + } + + return false; + } + + private static String trim(String url) { + if (url.startsWith("/")) url = url.substring(1, url.length()); + if (url.endsWith("/")) url = url.substring(0, url.length()-1); + return url; + } + + private static String[] trimAndSplit(String url) { + return trim(url).split("/"); + } + + public static URLTemplate createUrlTemplate(String url, Method method) { + String[] tokens = trimAndSplit(url); + SuperType[] types = new SuperType[tokens.length]; + for(int i = 0; i < tokens.length; i ++ ) { + String token = tokens[i]; + + if (token.equals(SuperType.STRING.name())) { + tokens[i] = null; + types[i] = SuperType.STRING; + } else if (token.equals(SuperType.INTEGER.name())) { + tokens[i] = null; + types[i] = SuperType.INTEGER; + } else if (token.equals(SuperType.OBJECT_ID.name())) { + tokens[i] = null; + types[i] = SuperType.OBJECT_ID; + } else if (token.equals(SuperType.FLOAT.name())) { + tokens[i] = null; + types[i] = SuperType.FLOAT; + } else { + types[i] = null; + } + + } + + URLTemplate urlTemplate = new URLTemplate(tokens, types, method); + + return urlTemplate; + } + + static ObjectMapper mapper = new ObjectMapper(); + static JsonFactory factory = mapper.getFactory(); + + public static void extractAllValuesFromPayload(String payload, Map> payloadMap) throws Exception{ + JsonParser jp = factory.createParser(payload); + JsonNode node = mapper.readTree(jp); + extractAllValuesFromPayload(node,new ArrayList<>(),payloadMap); + } + + public static void extractAllValuesFromPayload(JsonNode node, List params, Map> values) { + // TODO: null values remove + if (node == null) return; + if (node.isValueNode()) { + String textValue = node.asText(); + if (textValue != null) { + String param = String.join("",params); + if (param.startsWith("#")) { + param = param.substring(1); + } + if (!values.containsKey(param)) { + values.put(param, new HashSet<>()); + } + values.get(param).add(textValue); + } + } else if (node.isArray()) { + ArrayNode arrayNode = (ArrayNode) node; + for(int i = 0; i < arrayNode.size(); i++) { + JsonNode arrayElement = arrayNode.get(i); + params.add("#$"); + extractAllValuesFromPayload(arrayElement, params, values); + params.remove(params.size()-1); + } + } else { + Iterator fieldNames = node.fieldNames(); + while(fieldNames.hasNext()) { + String fieldName = fieldNames.next(); + params.add("#"+fieldName); + JsonNode fieldValue = node.get(fieldName); + extractAllValuesFromPayload(fieldValue, params,values); + params.remove(params.size()-1); + } + } + + } } diff --git a/libs/utils/src/main/java/com/akto/runtime/utils/Utils.java b/libs/utils/src/main/java/com/akto/runtime/utils/Utils.java new file mode 100644 index 0000000000..aa17a43823 --- /dev/null +++ b/libs/utils/src/main/java/com/akto/runtime/utils/Utils.java @@ -0,0 +1,120 @@ +package com.akto.runtime.utils; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Properties; +import java.util.StringJoiner; +import java.util.regex.Pattern; + +import org.apache.kafka.clients.consumer.ConsumerConfig; +import org.apache.kafka.common.serialization.StringDeserializer; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.akto.dto.HttpRequestParams; +import com.akto.dto.HttpResponseParams; +import com.akto.dto.OriginalHttpRequest; +import com.akto.dto.OriginalHttpResponse; +import com.akto.dto.RawApi; +import com.mongodb.BasicDBObject; +import static com.akto.dto.RawApi.convertHeaders; + +public class Utils { + private static final Logger logger = LoggerFactory.getLogger(Utils.class); + + private static int debugPrintCounter = 500; + public static void printL(Object o) { + if (debugPrintCounter > 0) { + debugPrintCounter--; + logger.info(o.toString()); + } + } + + public static Properties configProperties(String kafkaBrokerUrl, String groupIdConfig, int maxPollRecordsConfig) { + Properties properties = new Properties(); + properties.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, kafkaBrokerUrl); + properties.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class); + properties.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class); + properties.put(ConsumerConfig.MAX_POLL_RECORDS_CONFIG, maxPollRecordsConfig); + properties.put(ConsumerConfig.GROUP_ID_CONFIG, groupIdConfig); + properties.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest"); + properties.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, false); + + return properties; + } + + public static String convertOriginalReqRespToString(OriginalHttpRequest request, OriginalHttpResponse response) { + BasicDBObject req = new BasicDBObject(); + if (request != null) { + req.put("url", request.getUrl()); + req.put("method", request.getMethod()); + req.put("type", request.getType()); + req.put("queryParams", request.getQueryParams()); + req.put("body", request.getBody()); + req.put("headers", convertHeaders(request.getHeaders())); + } + + BasicDBObject resp = new BasicDBObject(); + if (response != null) { + resp.put("statusCode", response.getStatusCode()); + resp.put("body", response.getBody()); + resp.put("headers", convertHeaders(response.getHeaders())); + } + + BasicDBObject ret = new BasicDBObject(); + ret.put("request", req); + ret.put("response", resp); + + return ret.toString(); + } + + public static Map parseCookie(List cookieList){ + Map cookieMap = new HashMap<>(); + if(cookieList==null)return cookieMap; + for (String cookieValues : cookieList) { + String[] cookies = cookieValues.split(";"); + for (String cookie : cookies) { + cookie=cookie.trim(); + String[] cookieFields = cookie.split("="); + boolean twoCookieFields = cookieFields.length == 2; + if (twoCookieFields) { + if(!cookieMap.containsKey(cookieFields[0])){ + cookieMap.put(cookieFields[0], cookieFields[1]); + } + } + } + } + return cookieMap; + } + + public static Pattern createRegexPatternFromList(List discardedUrlList){ + StringJoiner joiner = new StringJoiner("|", ".*\\.(", ")(\\?.*)?"); + for (String extension : discardedUrlList) { + if(extension.startsWith("CONTENT-TYPE")){ + continue; + } + joiner.add(extension); + } + String regex = joiner.toString(); + + Pattern pattern = Pattern.compile(regex); + return pattern; + } + + public static HttpResponseParams convertRawApiToHttpResponseParams(RawApi rawApi, HttpResponseParams originalHttpResponseParams){ + + HttpRequestParams ogRequestParams = originalHttpResponseParams.getRequestParams(); + OriginalHttpRequest modifiedRequest = rawApi.getRequest(); + + ogRequestParams.setHeaders(modifiedRequest.getHeaders()); + ogRequestParams.setUrl(modifiedRequest.getFullUrlWithParams()); + ogRequestParams.setPayload(modifiedRequest.getBody()); + + originalHttpResponseParams.setRequestParams(ogRequestParams); + + return originalHttpResponseParams; + } + + +} diff --git a/apps/testing/src/main/java/com/akto/test_editor/Utils.java b/libs/utils/src/main/java/com/akto/test_editor/Utils.java similarity index 85% rename from apps/testing/src/main/java/com/akto/test_editor/Utils.java rename to libs/utils/src/main/java/com/akto/test_editor/Utils.java index 983e32fe0b..fff2186a72 100644 --- a/apps/testing/src/main/java/com/akto/test_editor/Utils.java +++ b/libs/utils/src/main/java/com/akto/test_editor/Utils.java @@ -15,15 +15,17 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; -import org.bouncycastle.jce.provider.JDKDSASigner.stdDSA; - import com.akto.dto.OriginalHttpRequest; import com.akto.dto.RawApi; +import com.akto.dao.billing.OrganizationsDao; import com.akto.dao.context.Context; import com.akto.dto.ApiInfo.ApiAccessType; +import com.akto.dto.ApiInfo; import com.akto.dto.test_editor.ExecutorSingleOperationResp; import com.akto.dto.testing.UrlModifierPayload; +import com.akto.test_editor.execution.Operations; import com.akto.util.Constants; +import com.akto.util.DashboardMode; import com.akto.util.JSONUtils; import com.akto.util.http_util.CoreHTTPClient; import com.fasterxml.jackson.core.JsonFactory; @@ -36,7 +38,7 @@ import com.mongodb.BasicDBList; import com.mongodb.BasicDBObject; -import static com.akto.rules.TestPlugin.extractAllValuesFromPayload; +import static com.akto.runtime.RuntimeUtil.extractAllValuesFromPayload; import okhttp3.*; public class Utils { @@ -45,6 +47,8 @@ public class Utils { private static final JsonFactory factory = mapper.getFactory(); private static final Gson gson = new Gson(); + public static boolean SKIP_SSRF_CHECK = ("true".equalsIgnoreCase(System.getenv("SKIP_SSRF_CHECK")) || !DashboardMode.isSaasDeployment()); + private static final OkHttpClient client = CoreHTTPClient.client.newBuilder() .writeTimeout(5, TimeUnit.SECONDS) .readTimeout(5, TimeUnit.SECONDS) @@ -860,4 +864,98 @@ public static Object getEpochTime(Object value) { return val; } + public static ExecutorSingleOperationResp modifySampleDataUtil(String operationType, RawApi rawApi, Object key, Object value, Map varMap, ApiInfo.ApiInfoKey apiInfoKey){ + switch (operationType.toLowerCase()) { + case "add_body_param": + Object epochVal = Utils.getEpochTime(value); + if (epochVal != null) { + value = epochVal; + } + return Operations.addBody(rawApi, key.toString(), value); + case "modify_body_param": + epochVal = Utils.getEpochTime(value); + if (epochVal != null) { + value = epochVal; + } + return Operations.modifyBodyParam(rawApi, key.toString(), value); + case "delete_graphql_field": + return Operations.deleteGraphqlField(rawApi, key == null ? "": key.toString()); + case "add_graphql_field": + return Operations.addGraphqlField(rawApi, key == null ? "": key.toString(), value == null ? "" : value.toString()); + case "add_unique_graphql_field": + return Operations.addUniqueGraphqlField(rawApi, key == null ? "": key.toString(), value == null ? "" : value.toString()); + case "modify_graphql_field": + return Operations.modifyGraphqlField(rawApi, key == null ? "": key.toString(), value == null ? "" : value.toString()); + case "delete_body_param": + return Operations.deleteBodyParam(rawApi, key.toString()); + case "replace_body": + String newPayload = rawApi.getRequest().getBody(); + if (key instanceof Map) { + Map> regexReplace = (Map) key; + String payload = rawApi.getRequest().getBody(); + Map regexInfo = regexReplace.get("regex_replace"); + String regex = regexInfo.get("regex"); + String replaceWith = regexInfo.get("replace_with"); + newPayload = Utils.applyRegexModifier(payload, regex, replaceWith); + } else { + newPayload = key.toString(); + } + return Operations.replaceBody(rawApi, newPayload); + case "add_header": + if (value.equals("${akto_header}")) { + BasicDBObject tokenResponse = OrganizationsDao.getBillingTokenForAuth(); + if(tokenResponse.getString("token") != null){ + value = tokenResponse.getString("token"); + }else{ + return new ExecutorSingleOperationResp(false, tokenResponse.getString("error")); + } + } + epochVal = Utils.getEpochTime(value); + if (epochVal != null) { + value = epochVal; + } + + return Operations.addHeader(rawApi, key.toString(), value.toString()); + case "modify_header": + String keyStr = key.toString(); + String valStr = value.toString(); + epochVal = Utils.getEpochTime(valStr); + if (epochVal != null) { + valStr = epochVal.toString(); + } + return Operations.modifyHeader(rawApi, keyStr, valStr); + case "delete_header": + return Operations.deleteHeader(rawApi, key.toString()); + case "add_query_param": + epochVal = Utils.getEpochTime(value); + if (epochVal != null) { + value = epochVal; + } + return Operations.addQueryParam(rawApi, key.toString(), value); + case "modify_query_param": + epochVal = Utils.getEpochTime(value); + if (epochVal != null) { + value = epochVal; + } + return Operations.modifyQueryParam(rawApi, key.toString(), value); + case "delete_query_param": + return Operations.deleteQueryParam(rawApi, key.toString()); + case "modify_url": + String newUrl = null; + UrlModifierPayload urlModifierPayload = Utils.fetchUrlModifyPayload(key.toString()); + if (urlModifierPayload != null) { + newUrl = Utils.buildNewUrl(urlModifierPayload, rawApi.getRequest().getUrl()); + } else { + newUrl = key.toString(); + } + return Operations.modifyUrl(rawApi, newUrl); + case "modify_method": + return Operations.modifyMethod(rawApi, key.toString()); + default: + return new ExecutorSingleOperationResp(false, "invalid operationType"); + } + + } + + } diff --git a/apps/testing/src/main/java/com/akto/test_editor/execution/ExecutionListBuilder.java b/libs/utils/src/main/java/com/akto/test_editor/execution/ExecutionListBuilder.java similarity index 99% rename from apps/testing/src/main/java/com/akto/test_editor/execution/ExecutionListBuilder.java rename to libs/utils/src/main/java/com/akto/test_editor/execution/ExecutionListBuilder.java index 009164d838..287e134441 100644 --- a/apps/testing/src/main/java/com/akto/test_editor/execution/ExecutionListBuilder.java +++ b/libs/utils/src/main/java/com/akto/test_editor/execution/ExecutionListBuilder.java @@ -14,7 +14,7 @@ public class ExecutionListBuilder { - private static final LoggerMaker loggerMaker = new LoggerMaker(Executor.class); + private static final LoggerMaker loggerMaker = new LoggerMaker(ExecutionListBuilder.class); public ExecutionOrderResp parseExecuteOperations(ExecutorNode node, List executeOrder) { diff --git a/apps/testing/src/main/java/com/akto/test_editor/execution/Operations.java b/libs/utils/src/main/java/com/akto/test_editor/execution/Operations.java similarity index 100% rename from apps/testing/src/main/java/com/akto/test_editor/execution/Operations.java rename to libs/utils/src/main/java/com/akto/test_editor/execution/Operations.java diff --git a/libs/utils/src/main/java/com/akto/test_editor/execution/ParseAndExecute.java b/libs/utils/src/main/java/com/akto/test_editor/execution/ParseAndExecute.java new file mode 100644 index 0000000000..847c4b0a78 --- /dev/null +++ b/libs/utils/src/main/java/com/akto/test_editor/execution/ParseAndExecute.java @@ -0,0 +1,114 @@ +package com.akto.test_editor.execution; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import com.akto.dao.test_editor.TestEditorEnums; +import com.akto.dao.test_editor.TestEditorEnums.ExecutorOperandTypes; +import com.akto.dto.ApiInfo; +import com.akto.dto.ApiInfo.ApiInfoKey; +import com.akto.dto.monitoring.FilterConfig; +import com.akto.dto.RawApi; +import com.akto.dto.test_editor.ExecuteAlgoObj; +import com.akto.dto.test_editor.ExecutionOrderResp; +import com.akto.dto.test_editor.ExecutorConfigParserResult; +import com.akto.dto.test_editor.ExecutorNode; +import com.akto.dto.test_editor.ExecutorSingleOperationResp; +import com.akto.test_editor.Utils; + +public class ParseAndExecute { + + public static List getExecutorNodes(ExecutorNode executorNode){ + if(executorNode.getChildNodes() == null || executorNode.getChildNodes().isEmpty()){ + return new ArrayList<>(); + } + + ExecutionListBuilder executionListBuilder = new ExecutionListBuilder(); + List executorNodes = new ArrayList<>(); + ExecutionOrderResp executionOrderResp = executionListBuilder.parseExecuteOperations(executorNode, executorNodes); + + ExecutorNode reqNodes = executorNode.getChildNodes().get(1); + if (reqNodes.getChildNodes() == null || reqNodes.getChildNodes().size() == 0) { + return new ArrayList<>(); + } + if(executionOrderResp.getError() != null && executionOrderResp.getError().length() > 0) { + return new ArrayList<>(); + } + return executorNodes; + } + + public RawApi execute (List executorNodes, RawApi rawApi, ApiInfoKey apiInfoKey, Map varMap, String logId){ + RawApi origRawApi = rawApi.copy(); + RawApi modifiedRawApi = executeAlgorithm(origRawApi, varMap, executorNodes, null, apiInfoKey); + return modifiedRawApi; + } + + private RawApi executeAlgorithm(RawApi sampleRawApi, Map varMap, List executorNodes, Map algoMap, ApiInfo.ApiInfoKey apiInfoKey){ + RawApi copyRawApi = sampleRawApi.copy(); + for(ExecutorNode executorNode: executorNodes){ + ExecutorNode node; + if (executorNode.getNodeType().equalsIgnoreCase(TestEditorEnums.ExecutorOperandTypes.NonTerminal.toString())) { + node = executorNode.getChildNodes().get(0); + } else { + node = executorNode; + } + + Object keyOp = node.getOperationType(); + Object valueOp = node.getValues(); + if (node.getNodeType().equalsIgnoreCase(ExecutorOperandTypes.Terminal.toString())) { + if (node.getValues() instanceof Boolean) { + keyOp = Boolean.toString((Boolean) node.getValues()); + } else if (node.getValues() instanceof String) { + keyOp = (String) node.getValues(); + } else { + keyOp = (Map) node.getValues(); + } + valueOp = null; + } + + List keyList = VariableResolver.resolveExpression(varMap, keyOp); + List valList = new ArrayList<>(); + if (valueOp != null) { + valList = VariableResolver.resolveExpression(varMap, valueOp); + } + + Object key = keyList.get(0); + Object val = null; + if (valList != null && valList.size() > 0) { + val = valList.get(0); + } + + ExecutorSingleOperationResp resp = Utils.modifySampleDataUtil(executorNode.getOperationType(), copyRawApi, key, val , varMap, apiInfoKey); + if(resp.getErrMsg() != null && resp.getErrMsg().length() > 0){ + break; + } + } + + return copyRawApi; + } + + public static Map> createExecutorNodeMap (Map filterMap) { + Map> finalMap = new HashMap<>(); + + if (filterMap != null && !filterMap.isEmpty()) { + for(Map.Entry iterator: filterMap.entrySet()){ + String templateId = iterator.getKey(); + if(templateId.equals(FilterConfig.DEFAULT_ALLOW_FILTER) || templateId.equals(FilterConfig.DEFAULT_BLOCK_FILTER)){ + continue; + } + ExecutorConfigParserResult nodeObj = iterator.getValue().getExecutor(); + if(nodeObj != null && nodeObj.getIsValid()){ + ExecutorNode node = nodeObj.getNode(); + List nodes = getExecutorNodes(node); + + finalMap.put(templateId, nodes); + } + } + } + + return finalMap; + } + +} diff --git a/apps/testing/src/main/java/com/akto/test_editor/execution/VariableResolver.java b/libs/utils/src/main/java/com/akto/test_editor/execution/VariableResolver.java similarity index 99% rename from apps/testing/src/main/java/com/akto/test_editor/execution/VariableResolver.java rename to libs/utils/src/main/java/com/akto/test_editor/execution/VariableResolver.java index 70e917201f..aaf19e945c 100644 --- a/apps/testing/src/main/java/com/akto/test_editor/execution/VariableResolver.java +++ b/libs/utils/src/main/java/com/akto/test_editor/execution/VariableResolver.java @@ -24,7 +24,6 @@ import com.akto.dto.type.RequestTemplate; import com.akto.dto.type.SingleTypeInfo; import com.akto.dto.type.URLMethods; -import com.akto.parsers.HttpCallParser; import com.akto.test_editor.Utils; import com.akto.util.modifier.AddJWKModifier; import com.akto.util.modifier.AddJkuJWTModifier; @@ -36,6 +35,8 @@ import com.mongodb.client.model.Filters; import com.mongodb.client.model.Projections; +import static com.akto.runtime.parser.SampleParser.parseSampleMessage; + public class VariableResolver { public static Object getValue(Map varMap, String key) { @@ -679,7 +680,7 @@ public static Set extractValuesFromSampleData(List samples, Stri HttpResponseParams httpResponseParams; HttpRequestParams httpRequestParams; try { - httpResponseParams = HttpCallParser.parseKafkaMessage(sample); + httpResponseParams = parseSampleMessage(sample); httpRequestParams = httpResponseParams.getRequestParams(); if ("terminal_keys".equals(location)) { diff --git a/apps/testing/src/main/java/com/akto/test_editor/filter/Filter.java b/libs/utils/src/main/java/com/akto/test_editor/filter/Filter.java similarity index 100% rename from apps/testing/src/main/java/com/akto/test_editor/filter/Filter.java rename to libs/utils/src/main/java/com/akto/test_editor/filter/Filter.java diff --git a/apps/testing/src/main/java/com/akto/test_editor/filter/FilterAction.java b/libs/utils/src/main/java/com/akto/test_editor/filter/FilterAction.java similarity index 98% rename from apps/testing/src/main/java/com/akto/test_editor/filter/FilterAction.java rename to libs/utils/src/main/java/com/akto/test_editor/filter/FilterAction.java index 740eecc8b2..f255e5558a 100644 --- a/apps/testing/src/main/java/com/akto/test_editor/filter/FilterAction.java +++ b/libs/utils/src/main/java/com/akto/test_editor/filter/FilterAction.java @@ -33,20 +33,20 @@ import com.akto.dto.type.SingleTypeInfo; import com.akto.dto.type.URLMethods; import com.akto.dto.type.URLTemplate; -import com.akto.parsers.HttpCallParser; -import com.akto.rules.TestPlugin; -import com.akto.runtime.APICatalogSync; -import com.akto.runtime.policies.AuthPolicy; import com.akto.test_editor.Utils; import com.akto.test_editor.execution.VariableResolver; import com.akto.test_editor.filter.data_operands_impl.*; import com.akto.util.JSONUtils; -import com.akto.utils.RedactSampleData; import com.mongodb.BasicDBList; import com.mongodb.BasicDBObject; import com.mongodb.client.model.Filters; +import static com.akto.runtime.utils.Utils.parseCookie; import static com.akto.dto.RawApi.convertHeaders; +import static com.akto.runtime.RuntimeUtil.createUrlTemplate; +import static com.akto.testing.Utils.compareWithOriginalResponse; + +import static com.akto.runtime.parser.SampleParser.parseSampleMessage; public final class FilterAction { @@ -343,7 +343,7 @@ public DataOperandsFilterResponse applyFilterOnPayload(FilterActionRequest filte if (sampleRawApi == null) { return new DataOperandsFilterResponse(false, null, null, null); } - double percentageMatch = TestPlugin.compareWithOriginalResponse(payload, sampleRawApi.getResponse().getBody(), new HashMap<>()); + double percentageMatch = compareWithOriginalResponse(payload, sampleRawApi.getResponse().getBody(), new HashMap<>()); val = (int) percentageMatch; } else if (filterActionRequest.getBodyOperand() != null && filterActionRequest.getBodyOperand().equalsIgnoreCase(BodyOperator.PERCENTAGE_MATCH_SCHEMA.toString())) { RawApi sampleRawApi = filterActionRequest.getRawApi(); @@ -425,7 +425,7 @@ public void extractPayload(FilterActionRequest filterActionRequest, Map()); + double percentageMatch = compareWithOriginalResponse(payload, sampleRawApi.getResponse().getBody(), new HashMap<>()); val = (int) percentageMatch; } } else { @@ -562,7 +562,7 @@ public DataOperandsFilterResponse applyFiltersOnHeaders(FilterActionRequest filt if (!res && (key.equals("cookie") || key.equals("set-cookie"))) { List cookieList = headers.getOrDefault(key, new ArrayList<>()); - Map cookieMap = AuthPolicy.parseCookie(cookieList); + Map cookieMap = parseCookie(cookieList); for (String cookieKey : cookieMap.keySet()) { dataOperandFilterRequest = new DataOperandFilterRequest(cookieKey, filterActionRequest.getQuerySet(), filterActionRequest.getOperand()); res = invokeFilter(dataOperandFilterRequest); @@ -599,7 +599,7 @@ public DataOperandsFilterResponse applyFiltersOnHeaders(FilterActionRequest filt if (!res && (key.equals("cookie") || key.equals("set-cookie"))) { List cookieList = headers.getOrDefault("cookie", new ArrayList<>()); - Map cookieMap = AuthPolicy.parseCookie(cookieList); + Map cookieMap = parseCookie(cookieList); for (String cookieKey : cookieMap.keySet()) { DataOperandFilterRequest dataOperandFilterRequest = new DataOperandFilterRequest(cookieMap.get(cookieKey), filterActionRequest.getQuerySet(), filterActionRequest.getOperand()); res = invokeFilter(dataOperandFilterRequest); @@ -1229,10 +1229,10 @@ public BasicDBObject getPrivateResourceCount(OriginalHttpRequest originalHttpReq int privateCnt = 0; List privateValues = new ArrayList<>(); if (APICatalog.isTemplateUrl(url)) { - URLTemplate urlTemplate = APICatalogSync.createUrlTemplate(url, method); + URLTemplate urlTemplate = createUrlTemplate(url, method); String[] tokens = urlTemplate.getTokens(); - String[] urlWithParamsTokens = APICatalogSync.createUrlTemplate(urlWithParams, method).getTokens(); + String[] urlWithParamsTokens = createUrlTemplate(urlWithParams, method).getTokens(); for (int i = 0;i < tokens.length; i++) { if (tokens[i] == null) { SingleTypeInfo singleTypeInfo = querySti(i+"", true,apiInfoKey, false, -1); @@ -1264,7 +1264,7 @@ public BasicDBObject getPrivateResourceCount(OriginalHttpRequest originalHttpReq } for (String sample: sd.getSamples()) { try { - HttpResponseParams httpResponseParams = HttpCallParser.parseKafkaMessage(sample); + HttpResponseParams httpResponseParams = parseSampleMessage(sample); String sUrl = httpResponseParams.getRequestParams().getURL(); String[] sUrlTokens = sUrl.split("/"); String[] origUrlTokens = urlWithParams.split("/"); @@ -1320,7 +1320,7 @@ public BasicDBObject getPrivateResourceCount(OriginalHttpRequest originalHttpReq String key = SingleTypeInfo.findLastKeyFromParam(param); BasicDBObject payloadObj = new BasicDBObject(); try { - HttpResponseParams httpResponseParams = HttpCallParser.parseKafkaMessage(sample); + HttpResponseParams httpResponseParams = parseSampleMessage(sample); payloadObj = RequestTemplate.parseRequestPayload(httpResponseParams.getRequestParams().getPayload(), null); } catch (Exception e) { // TODO: handle exception diff --git a/apps/testing/src/main/java/com/akto/test_editor/filter/data_operands_impl/ApiCollectionFilter.java b/libs/utils/src/main/java/com/akto/test_editor/filter/data_operands_impl/ApiCollectionFilter.java similarity index 100% rename from apps/testing/src/main/java/com/akto/test_editor/filter/data_operands_impl/ApiCollectionFilter.java rename to libs/utils/src/main/java/com/akto/test_editor/filter/data_operands_impl/ApiCollectionFilter.java diff --git a/apps/testing/src/main/java/com/akto/test_editor/filter/data_operands_impl/ContainsAllFilter.java b/libs/utils/src/main/java/com/akto/test_editor/filter/data_operands_impl/ContainsAllFilter.java similarity index 100% rename from apps/testing/src/main/java/com/akto/test_editor/filter/data_operands_impl/ContainsAllFilter.java rename to libs/utils/src/main/java/com/akto/test_editor/filter/data_operands_impl/ContainsAllFilter.java diff --git a/apps/testing/src/main/java/com/akto/test_editor/filter/data_operands_impl/ContainsEitherFilter.java b/libs/utils/src/main/java/com/akto/test_editor/filter/data_operands_impl/ContainsEitherFilter.java similarity index 100% rename from apps/testing/src/main/java/com/akto/test_editor/filter/data_operands_impl/ContainsEitherFilter.java rename to libs/utils/src/main/java/com/akto/test_editor/filter/data_operands_impl/ContainsEitherFilter.java diff --git a/apps/testing/src/main/java/com/akto/test_editor/filter/data_operands_impl/ContainsJwt.java b/libs/utils/src/main/java/com/akto/test_editor/filter/data_operands_impl/ContainsJwt.java similarity index 100% rename from apps/testing/src/main/java/com/akto/test_editor/filter/data_operands_impl/ContainsJwt.java rename to libs/utils/src/main/java/com/akto/test_editor/filter/data_operands_impl/ContainsJwt.java diff --git a/apps/testing/src/main/java/com/akto/test_editor/filter/data_operands_impl/CookieExpireFilter.java b/libs/utils/src/main/java/com/akto/test_editor/filter/data_operands_impl/CookieExpireFilter.java similarity index 95% rename from apps/testing/src/main/java/com/akto/test_editor/filter/data_operands_impl/CookieExpireFilter.java rename to libs/utils/src/main/java/com/akto/test_editor/filter/data_operands_impl/CookieExpireFilter.java index 0edce0afeb..80155cfefa 100644 --- a/apps/testing/src/main/java/com/akto/test_editor/filter/data_operands_impl/CookieExpireFilter.java +++ b/libs/utils/src/main/java/com/akto/test_editor/filter/data_operands_impl/CookieExpireFilter.java @@ -10,7 +10,8 @@ import java.util.Map; import com.akto.dto.test_editor.DataOperandFilterRequest; -import com.akto.runtime.policies.AuthPolicy; + +import static com.akto.runtime.utils.Utils.parseCookie; public class CookieExpireFilter extends DataOperandsImpl { @@ -33,7 +34,7 @@ public Boolean isValid(DataOperandFilterRequest dataOperandFilterRequest) { return false; } - Map cookieMap = AuthPolicy.parseCookie(Arrays.asList(data)); + Map cookieMap = parseCookie(Arrays.asList(data)); boolean result = queryVal; boolean res = false; diff --git a/apps/testing/src/main/java/com/akto/test_editor/filter/data_operands_impl/DataOperandsImpl.java b/libs/utils/src/main/java/com/akto/test_editor/filter/data_operands_impl/DataOperandsImpl.java similarity index 100% rename from apps/testing/src/main/java/com/akto/test_editor/filter/data_operands_impl/DataOperandsImpl.java rename to libs/utils/src/main/java/com/akto/test_editor/filter/data_operands_impl/DataOperandsImpl.java diff --git a/apps/testing/src/main/java/com/akto/test_editor/filter/data_operands_impl/DatatypeFilter.java b/libs/utils/src/main/java/com/akto/test_editor/filter/data_operands_impl/DatatypeFilter.java similarity index 100% rename from apps/testing/src/main/java/com/akto/test_editor/filter/data_operands_impl/DatatypeFilter.java rename to libs/utils/src/main/java/com/akto/test_editor/filter/data_operands_impl/DatatypeFilter.java diff --git a/apps/testing/src/main/java/com/akto/test_editor/filter/data_operands_impl/EqFilter.java b/libs/utils/src/main/java/com/akto/test_editor/filter/data_operands_impl/EqFilter.java similarity index 100% rename from apps/testing/src/main/java/com/akto/test_editor/filter/data_operands_impl/EqFilter.java rename to libs/utils/src/main/java/com/akto/test_editor/filter/data_operands_impl/EqFilter.java diff --git a/apps/testing/src/main/java/com/akto/test_editor/filter/data_operands_impl/GreaterThanEqFilter.java b/libs/utils/src/main/java/com/akto/test_editor/filter/data_operands_impl/GreaterThanEqFilter.java similarity index 100% rename from apps/testing/src/main/java/com/akto/test_editor/filter/data_operands_impl/GreaterThanEqFilter.java rename to libs/utils/src/main/java/com/akto/test_editor/filter/data_operands_impl/GreaterThanEqFilter.java diff --git a/apps/testing/src/main/java/com/akto/test_editor/filter/data_operands_impl/GreaterThanFilter.java b/libs/utils/src/main/java/com/akto/test_editor/filter/data_operands_impl/GreaterThanFilter.java similarity index 100% rename from apps/testing/src/main/java/com/akto/test_editor/filter/data_operands_impl/GreaterThanFilter.java rename to libs/utils/src/main/java/com/akto/test_editor/filter/data_operands_impl/GreaterThanFilter.java diff --git a/apps/testing/src/main/java/com/akto/test_editor/filter/data_operands_impl/LesserThanEqFilter.java b/libs/utils/src/main/java/com/akto/test_editor/filter/data_operands_impl/LesserThanEqFilter.java similarity index 100% rename from apps/testing/src/main/java/com/akto/test_editor/filter/data_operands_impl/LesserThanEqFilter.java rename to libs/utils/src/main/java/com/akto/test_editor/filter/data_operands_impl/LesserThanEqFilter.java diff --git a/apps/testing/src/main/java/com/akto/test_editor/filter/data_operands_impl/LesserThanFilter.java b/libs/utils/src/main/java/com/akto/test_editor/filter/data_operands_impl/LesserThanFilter.java similarity index 100% rename from apps/testing/src/main/java/com/akto/test_editor/filter/data_operands_impl/LesserThanFilter.java rename to libs/utils/src/main/java/com/akto/test_editor/filter/data_operands_impl/LesserThanFilter.java diff --git a/apps/testing/src/main/java/com/akto/test_editor/filter/data_operands_impl/NeqFilter.java b/libs/utils/src/main/java/com/akto/test_editor/filter/data_operands_impl/NeqFilter.java similarity index 100% rename from apps/testing/src/main/java/com/akto/test_editor/filter/data_operands_impl/NeqFilter.java rename to libs/utils/src/main/java/com/akto/test_editor/filter/data_operands_impl/NeqFilter.java diff --git a/apps/testing/src/main/java/com/akto/test_editor/filter/data_operands_impl/NotContainsEitherFilter.java b/libs/utils/src/main/java/com/akto/test_editor/filter/data_operands_impl/NotContainsEitherFilter.java similarity index 100% rename from apps/testing/src/main/java/com/akto/test_editor/filter/data_operands_impl/NotContainsEitherFilter.java rename to libs/utils/src/main/java/com/akto/test_editor/filter/data_operands_impl/NotContainsEitherFilter.java diff --git a/apps/testing/src/main/java/com/akto/test_editor/filter/data_operands_impl/NotContainsFilter.java b/libs/utils/src/main/java/com/akto/test_editor/filter/data_operands_impl/NotContainsFilter.java similarity index 100% rename from apps/testing/src/main/java/com/akto/test_editor/filter/data_operands_impl/NotContainsFilter.java rename to libs/utils/src/main/java/com/akto/test_editor/filter/data_operands_impl/NotContainsFilter.java diff --git a/apps/testing/src/main/java/com/akto/test_editor/filter/data_operands_impl/RegexFilter.java b/libs/utils/src/main/java/com/akto/test_editor/filter/data_operands_impl/RegexFilter.java similarity index 100% rename from apps/testing/src/main/java/com/akto/test_editor/filter/data_operands_impl/RegexFilter.java rename to libs/utils/src/main/java/com/akto/test_editor/filter/data_operands_impl/RegexFilter.java diff --git a/apps/testing/src/main/java/com/akto/test_editor/filter/data_operands_impl/SsrfUrlHitFilter.java b/libs/utils/src/main/java/com/akto/test_editor/filter/data_operands_impl/SsrfUrlHitFilter.java similarity index 100% rename from apps/testing/src/main/java/com/akto/test_editor/filter/data_operands_impl/SsrfUrlHitFilter.java rename to libs/utils/src/main/java/com/akto/test_editor/filter/data_operands_impl/SsrfUrlHitFilter.java diff --git a/libs/utils/src/main/java/com/akto/test_editor/filter/data_operands_impl/ValidationResult.java b/libs/utils/src/main/java/com/akto/test_editor/filter/data_operands_impl/ValidationResult.java new file mode 100644 index 0000000000..c2204827ef --- /dev/null +++ b/libs/utils/src/main/java/com/akto/test_editor/filter/data_operands_impl/ValidationResult.java @@ -0,0 +1,19 @@ +package com.akto.test_editor.filter.data_operands_impl; + +public class ValidationResult { + public static final String GET_QUERYSET_CATCH_ERROR = "Error while parsing data"; + Boolean isValid; + String validationReason; + public ValidationResult(Boolean isValid, String validationReason) { + this.isValid = isValid; + this.validationReason = validationReason; + } + + public Boolean getIsValid() { + return isValid; + } + + public String getValidationReason() { + return validationReason; + } +} diff --git a/libs/utils/src/main/java/com/akto/testing/Utils.java b/libs/utils/src/main/java/com/akto/testing/Utils.java index c20908c3a1..31c291e2fa 100644 --- a/libs/utils/src/main/java/com/akto/testing/Utils.java +++ b/libs/utils/src/main/java/com/akto/testing/Utils.java @@ -1,7 +1,11 @@ package com.akto.testing; +import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Objects; +import java.util.Set; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -9,16 +13,25 @@ import javax.script.ScriptEngineManager; import javax.script.ScriptException; +import com.akto.dto.ApiInfo.ApiInfoKey; import com.akto.dto.OriginalHttpRequest; +import com.akto.dto.OriginalHttpResponse; +import com.akto.dto.RawApi; +import com.akto.dto.test_editor.DataOperandsFilterResponse; +import com.akto.dto.test_editor.FilterNode; import com.akto.dto.testing.WorkflowUpdatedSampleData; import com.akto.dto.type.RequestTemplate; import com.akto.log.LoggerMaker; import com.akto.log.LoggerMaker.LogDb; +import com.akto.test_editor.filter.Filter; +import com.akto.test_editor.filter.data_operands_impl.ValidationResult; import com.akto.util.JSONUtils; import com.mongodb.BasicDBObject; import okhttp3.MediaType; +import static com.akto.runtime.RuntimeUtil.extractAllValuesFromPayload;; + public class Utils { private static final LoggerMaker loggerMaker = new LoggerMaker(Utils.class); @@ -291,5 +304,69 @@ public static MediaType getMediaType(String fileUrl) { } } + + public static double compareWithOriginalResponse(String originalPayload, String currentPayload, Map comparisonExcludedKeys) { + if (originalPayload == null && currentPayload == null) return 100; + if (originalPayload == null || currentPayload == null) return 0; + + String trimmedOriginalPayload = originalPayload.trim(); + String trimmedCurrentPayload = currentPayload.trim(); + if (trimmedCurrentPayload.equals(trimmedOriginalPayload)) return 100; + + Map> originalResponseParamMap = new HashMap<>(); + Map> currentResponseParamMap = new HashMap<>(); + try { + extractAllValuesFromPayload(originalPayload, originalResponseParamMap); + extractAllValuesFromPayload(currentPayload, currentResponseParamMap); + } catch (Exception e) { + return 0.0; + } + + if (originalResponseParamMap.keySet().size() == 0 && currentResponseParamMap.keySet().size() == 0) { + return 100.0; + } + + Set visited = new HashSet<>(); + int matched = 0; + for (String k1: originalResponseParamMap.keySet()) { + if (visited.contains(k1) || comparisonExcludedKeys.containsKey(k1)) continue; + visited.add(k1); + Set v1 = originalResponseParamMap.get(k1); + Set v2 = currentResponseParamMap.get(k1); + if (Objects.equals(v1, v2)) matched +=1; + } + + for (String k1: currentResponseParamMap.keySet()) { + if (visited.contains(k1) || comparisonExcludedKeys.containsKey(k1)) continue; + visited.add(k1); + Set v1 = originalResponseParamMap.get(k1); + Set v2 = currentResponseParamMap.get(k1); + if (Objects.equals(v1, v2)) matched +=1; + } + + int visitedSize = visited.size(); + if (visitedSize == 0) return 0.0; + + double result = (100.0*matched)/visitedSize; + + if (Double.isFinite(result)) { + return result; + } else { + return 0.0; + } + + } + + public static ValidationResult validateFilter(FilterNode filterNode, RawApi rawApi, ApiInfoKey apiInfoKey, Map varMap, String logId) { + if (filterNode == null) return new ValidationResult(true, ""); + if (rawApi == null) return new ValidationResult(true, "raw api is null"); + return validate(filterNode, rawApi, null, apiInfoKey,"filter", varMap, logId); + } + + private static ValidationResult validate(FilterNode node, RawApi rawApi, RawApi testRawApi, ApiInfoKey apiInfoKey, String context, Map varMap, String logId) { + Filter filter = new Filter(); + DataOperandsFilterResponse dataOperandsFilterResponse = filter.isEndpointValid(node, rawApi, testRawApi, apiInfoKey, null, null , false,context, varMap, logId, false); + return new ValidationResult(dataOperandsFilterResponse.getResult(), dataOperandsFilterResponse.getValidationReason()); + } }