From 56acdb3e87c0677a11cded363e8eec84aa4d6fee Mon Sep 17 00:00:00 2001 From: Ark2307 Date: Wed, 11 Sep 2024 11:04:53 +0530 Subject: [PATCH 1/9] Moved filters of test editor to utils --- .../akto/dependency/DependencyAnalyser.java | 5 +- .../java/com/akto/parsers/HttpCallParser.java | 157 ++++++++++++------ .../java/com/akto/runtime/APICatalogSync.java | 8 +- .../com/akto/runtime/RelationshipSync.java | 34 +--- .../com/akto/runtime/policies/AuthPolicy.java | 20 +-- .../java/com/akto/utils/RedactSampleData.java | 30 +--- .../java/com/akto/filters/HttpCallFilter.java | 121 ++++++++++++++ .../AdvancedTrafficFiltersAction.java | 94 +++++++++++ .../FilterYamlTemplateAction.java | 70 ++++++++ .../akto/listener/InitializerListener.java | 3 +- .../dashboard/components/shared/SampleData.js | 8 + .../main/java/com/akto/rules/TestPlugin.java | 72 +------- .../com/akto/test_editor/execution/Build.java | 4 +- .../execution/ExecutionListBuilder.java | 2 +- .../akto/test_editor/execution/Executor.java | 20 +-- .../akto/test_editor/execution/Memory.java | 1 - .../ApiNodeExecutor.java | 8 +- .../YamlNodeExecutor.java | 7 +- .../monitoring/FilterConfigYamlParser.java | 76 +++++++++ .../dao/monitoring/FilterYamlTemplateDao.java | 51 ++++++ .../AdvancedTrafficFiltersDao.java | 19 +++ .../com/akto/dto/monitoring/FilterConfig.java | 110 ++++++++++++ .../java/com/akto/data_actor/ClientActor.java | 3 + .../java/com/akto/runtime/RuntimeUtil.java | 109 +++++++++++- .../java/com/akto/runtime/utils/Utils.java | 157 ++++++++++++++++++ .../main/java/com/akto/test_editor/Utils.java | 7 +- .../execution/VariableResolver.java | 5 +- .../com/akto/test_editor/filter/Filter.java | 0 .../akto/test_editor/filter/FilterAction.java | 25 ++- .../ApiCollectionFilter.java | 0 .../data_operands_impl/ContainsAllFilter.java | 0 .../ContainsEitherFilter.java | 0 .../data_operands_impl/ContainsJwt.java | 0 .../CookieExpireFilter.java | 5 +- .../data_operands_impl/DataOperandsImpl.java | 0 .../data_operands_impl/DatatypeFilter.java | 0 .../filter/data_operands_impl/EqFilter.java | 0 .../GreaterThanEqFilter.java | 0 .../data_operands_impl/GreaterThanFilter.java | 0 .../LesserThanEqFilter.java | 0 .../data_operands_impl/LesserThanFilter.java | 0 .../filter/data_operands_impl/NeqFilter.java | 0 .../NotContainsEitherFilter.java | 0 .../data_operands_impl/NotContainsFilter.java | 0 .../data_operands_impl/RegexFilter.java | 0 .../data_operands_impl/SsrfUrlHitFilter.java | 0 .../data_operands_impl/ValidationResult.java | 19 +++ .../src/main/java/com/akto/testing/Utils.java | 77 +++++++++ 48 files changed, 1076 insertions(+), 251 deletions(-) create mode 100644 apps/api-threat-detection/src/main/java/com/akto/filters/HttpCallFilter.java create mode 100644 apps/dashboard/src/main/java/com/akto/action/settings/AdvancedTrafficFiltersAction.java create mode 100644 apps/dashboard/src/main/java/com/akto/action/threat_detection/FilterYamlTemplateAction.java create mode 100644 libs/dao/src/main/java/com/akto/dao/monitoring/FilterConfigYamlParser.java create mode 100644 libs/dao/src/main/java/com/akto/dao/monitoring/FilterYamlTemplateDao.java create mode 100644 libs/dao/src/main/java/com/akto/dao/runtime_filters/AdvancedTrafficFiltersDao.java create mode 100644 libs/dao/src/main/java/com/akto/dto/monitoring/FilterConfig.java create mode 100644 libs/utils/src/main/java/com/akto/runtime/utils/Utils.java rename {apps/testing => libs/utils}/src/main/java/com/akto/test_editor/Utils.java (99%) rename {apps/testing => libs/utils}/src/main/java/com/akto/test_editor/execution/VariableResolver.java (99%) rename {apps/testing => libs/utils}/src/main/java/com/akto/test_editor/filter/Filter.java (100%) rename {apps/testing => libs/utils}/src/main/java/com/akto/test_editor/filter/FilterAction.java (98%) rename {apps/testing => libs/utils}/src/main/java/com/akto/test_editor/filter/data_operands_impl/ApiCollectionFilter.java (100%) rename {apps/testing => libs/utils}/src/main/java/com/akto/test_editor/filter/data_operands_impl/ContainsAllFilter.java (100%) rename {apps/testing => libs/utils}/src/main/java/com/akto/test_editor/filter/data_operands_impl/ContainsEitherFilter.java (100%) rename {apps/testing => libs/utils}/src/main/java/com/akto/test_editor/filter/data_operands_impl/ContainsJwt.java (100%) rename {apps/testing => libs/utils}/src/main/java/com/akto/test_editor/filter/data_operands_impl/CookieExpireFilter.java (95%) rename {apps/testing => libs/utils}/src/main/java/com/akto/test_editor/filter/data_operands_impl/DataOperandsImpl.java (100%) rename {apps/testing => libs/utils}/src/main/java/com/akto/test_editor/filter/data_operands_impl/DatatypeFilter.java (100%) rename {apps/testing => libs/utils}/src/main/java/com/akto/test_editor/filter/data_operands_impl/EqFilter.java (100%) rename {apps/testing => libs/utils}/src/main/java/com/akto/test_editor/filter/data_operands_impl/GreaterThanEqFilter.java (100%) rename {apps/testing => libs/utils}/src/main/java/com/akto/test_editor/filter/data_operands_impl/GreaterThanFilter.java (100%) rename {apps/testing => libs/utils}/src/main/java/com/akto/test_editor/filter/data_operands_impl/LesserThanEqFilter.java (100%) rename {apps/testing => libs/utils}/src/main/java/com/akto/test_editor/filter/data_operands_impl/LesserThanFilter.java (100%) rename {apps/testing => libs/utils}/src/main/java/com/akto/test_editor/filter/data_operands_impl/NeqFilter.java (100%) rename {apps/testing => libs/utils}/src/main/java/com/akto/test_editor/filter/data_operands_impl/NotContainsEitherFilter.java (100%) rename {apps/testing => libs/utils}/src/main/java/com/akto/test_editor/filter/data_operands_impl/NotContainsFilter.java (100%) rename {apps/testing => libs/utils}/src/main/java/com/akto/test_editor/filter/data_operands_impl/RegexFilter.java (100%) rename {apps/testing => libs/utils}/src/main/java/com/akto/test_editor/filter/data_operands_impl/SsrfUrlHitFilter.java (100%) create mode 100644 libs/utils/src/main/java/com/akto/test_editor/filter/data_operands_impl/ValidationResult.java diff --git a/apps/api-runtime/src/main/java/com/akto/dependency/DependencyAnalyser.java b/apps/api-runtime/src/main/java/com/akto/dependency/DependencyAnalyser.java index 05ac9bdc5a..f8d19b12fb 100644 --- a/apps/api-runtime/src/main/java/com/akto/dependency/DependencyAnalyser.java +++ b/apps/api-runtime/src/main/java/com/akto/dependency/DependencyAnalyser.java @@ -29,6 +29,7 @@ import java.util.*; import static com.akto.util.HttpRequestResponseUtils.extractValuesFromPayload; +import static com.akto.runtime.utils.Utils.parseCookie; public class DependencyAnalyser { Store valueStore; // this is to store all the values seen in response payload @@ -131,7 +132,7 @@ public void analyse(String message, int finalApiCollectionId) { for (String param: responseHeaders.keySet()) { List values = responseHeaders.get(param); if (param.equalsIgnoreCase("set-cookie")) { - Map cookieMap = AuthPolicy.parseCookie(values); + Map cookieMap = parseCookie(values); for (String cookieKey: cookieMap.keySet()) { String cookieVal = cookieMap.get(cookieKey); if (!filterValues(cookieVal)) continue; @@ -192,7 +193,7 @@ public void analyse(String message, int finalApiCollectionId) { List values = requestHeaders.get(param); if (param.equals("cookie")) { - Map cookieMap = AuthPolicy.parseCookie(values); + Map cookieMap = parseCookie(values); for (String cookieKey: cookieMap.keySet()) { String cookieValue = cookieMap.get(cookieKey); processRequestParam(cookieKey, new HashSet<>(Collections.singletonList(cookieValue)), combinedUrl, false, true, doInterCollectionMatch); diff --git a/apps/api-runtime/src/main/java/com/akto/parsers/HttpCallParser.java b/apps/api-runtime/src/main/java/com/akto/parsers/HttpCallParser.java index 2295b08f33..a9cf86bf34 100644 --- a/apps/api-runtime/src/main/java/com/akto/parsers/HttpCallParser.java +++ b/apps/api-runtime/src/main/java/com/akto/parsers/HttpCallParser.java @@ -8,17 +8,21 @@ import com.akto.dao.traffic_metrics.TrafficMetricsDao; import com.akto.dependency.DependencyAnalyser; import com.akto.dto.*; +import com.akto.dto.ApiInfo.ApiInfoKey; import com.akto.dto.billing.FeatureAccess; import com.akto.dto.billing.SyncLimit; +import com.akto.dto.monitoring.FilterConfig; import com.akto.dto.billing.Organization; import com.akto.dto.settings.DefaultPayload; import com.akto.dto.traffic_metrics.TrafficMetrics; +import com.akto.dto.type.URLMethods.Method; import com.akto.dto.usage.MetricTypes; import com.akto.graphql.GraphQLUtils; import com.akto.log.LoggerMaker; import com.akto.log.LoggerMaker.LogDb; import com.akto.runtime.APICatalogSync; import com.akto.runtime.Main; +import com.akto.runtime.RuntimeUtil; import com.akto.runtime.URLAggregator; import com.akto.runtime.parser.SampleParser; import com.akto.usage.UsageMetricCalculator; @@ -28,15 +32,17 @@ import com.akto.util.DbMode; import com.akto.util.HttpRequestResponseUtils; import com.google.gson.Gson; +import com.akto.test_editor.execution.VariableResolver; +import com.akto.test_editor.filter.data_operands_impl.ValidationResult; import com.mongodb.BasicDBObject; import com.mongodb.client.model.*; import okhttp3.*; import org.apache.commons.lang3.StringUtils; -import org.apache.commons.lang3.math.NumberUtils; import org.bson.conversions.Bson; import java.io.IOException; import java.util.*; +import java.util.Map.Entry; import java.util.concurrent.ConcurrentLinkedQueue; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; @@ -45,6 +51,7 @@ import java.util.regex.Pattern; import static com.akto.runtime.RuntimeUtil.matchesDefaultPayload; +import static com.akto.testing.Utils.validateFilter; public class HttpCallParser { private final int sync_threshold_count; @@ -162,6 +169,48 @@ public int createCollectionBasedOnHostName(int id, String host) throws Exceptio int numberOfSyncs = 0; + private List applyAdvancedFilters(List responseParams){ + Map filterMap = apiCatalogSync.advancedFilterMap; + + if (filterMap != null && !filterMap.isEmpty()) { + List filteredParams = new ArrayList<>(); + for (HttpResponseParams responseParam : responseParams) { + for (Entry apiFilterEntry : filterMap.entrySet()) { + try { + FilterConfig apiFilter = apiFilterEntry.getValue(); + String message = responseParam.getOrig(); + RawApi rawApi = RawApi.buildFromMessage(message); + int apiCollectionId = createApiCollectionId(responseParam); + responseParam.requestParams.setApiCollectionId(apiCollectionId); + String url = responseParam.getRequestParams().getURL(); + Method method = Method.valueOf(responseParam.getRequestParams().getMethod()); + ApiInfoKey apiInfoKey = new ApiInfoKey(apiCollectionId, url, method); + Map varMap = apiFilter.resolveVarMap(); + VariableResolver.resolveWordList(varMap, new HashMap>() { + { + put(apiInfoKey, Arrays.asList(message)); + } + }, apiInfoKey); + String filterExecutionLogId = UUID.randomUUID().toString(); + ValidationResult res = validateFilter(apiFilter.getFilter().getNode(), rawApi, + apiInfoKey, varMap, filterExecutionLogId); + if (res.getIsValid()) { + // TODO apply execution + + filteredParams.add(responseParam); + break; + } + } catch (Exception e) { + loggerMaker.errorAndAddToDb(e, String.format("Error in httpCallFilter %s", e.toString())); + return responseParams; + } + } + } + return filteredParams; + } + return responseParams; + } + public void syncFunction(List responseParams, boolean syncImmediately, boolean fetchAllSTI, AccountSettings accountSettings) { // USE ONLY filteredResponseParams and not responseParams List filteredResponseParams = responseParams; @@ -169,6 +218,10 @@ public void syncFunction(List responseParams, boolean syncIm filteredResponseParams = filterDefaultPayloads(filteredResponseParams, accountSettings.getDefaultPayloads()); } filteredResponseParams = filterHttpResponseParams(filteredResponseParams, accountSettings); + + // add advanced filters + filteredResponseParams = applyAdvancedFilters(filteredResponseParams); + boolean isHarOrPcap = aggregate(filteredResponseParams, aggregatorMap); for (int apiCollectionId: aggregatorMap.keySet()) { @@ -359,6 +412,54 @@ private boolean isInvalidContentType(String contentType){ return res; } + public int createApiCollectionId(HttpResponseParams httpResponseParam){ + int apiCollectionId; + String hostName = getHeaderValue(httpResponseParam.getRequestParams().getHeaders(), "host"); + + if (hostName != null && !hostNameToIdMap.containsKey(hostName) && RuntimeUtil.hasSpecialCharacters(hostName)) { + hostName = "Special_Char_Host"; + } + + int vxlanId = httpResponseParam.requestParams.getApiCollectionId(); + + if (useHostCondition(hostName, httpResponseParam.getSource())) { + hostName = hostName.toLowerCase(); + hostName = hostName.trim(); + + String key = hostName; + + if (hostNameToIdMap.containsKey(key)) { + apiCollectionId = hostNameToIdMap.get(key); + + } else { + int id = hostName.hashCode(); + try { + + apiCollectionId = createCollectionBasedOnHostName(id, hostName); + + hostNameToIdMap.put(key, apiCollectionId); + } catch (Exception e) { + loggerMaker.errorAndAddToDb("Failed to create collection for host : " + hostName, LogDb.RUNTIME); + createCollectionSimple(vxlanId); + hostNameToIdMap.put("null " + vxlanId, vxlanId); + apiCollectionId = httpResponseParam.requestParams.getApiCollectionId(); + } + } + + } else { + String key = "null" + " " + vxlanId; + if (!hostNameToIdMap.containsKey(key)) { + createCollectionSimple(vxlanId); + hostNameToIdMap.put(key, vxlanId); + } + + apiCollectionId = vxlanId; + } + return apiCollectionId; + } + + + private boolean isBlankResponseBodyForGET(String method, String contentType, String matchContentType, String responseBody) { boolean res = true; @@ -439,58 +540,8 @@ public List filterHttpResponseParams(List> reqHeaders = httpResponseParam.getRequestParams().getHeaders(); - String hostName = getHeaderValue(reqHeaders, "host"); - - if (StringUtils.isEmpty(hostName)) { - hostName = getHeaderValue(reqHeaders, "authority"); - if (StringUtils.isEmpty(hostName)) { - hostName = getHeaderValue(reqHeaders, ":authority"); - } - - if (!StringUtils.isEmpty(hostName)) { - reqHeaders.put("host", Collections.singletonList(hostName)); - } - } - - int vxlanId = httpResponseParam.requestParams.getApiCollectionId(); - int apiCollectionId ; - - if (useHostCondition(hostName, httpResponseParam.getSource())) { - hostName = hostName.toLowerCase(); - hostName = hostName.trim(); - - String key = hostName; - - if (hostNameToIdMap.containsKey(key)) { - apiCollectionId = hostNameToIdMap.get(key); - - } else { - int id = hostName.hashCode(); - try { - - apiCollectionId = createCollectionBasedOnHostName(id, hostName); - - hostNameToIdMap.put(key, apiCollectionId); - } catch (Exception e) { - loggerMaker.errorAndAddToDb("Failed to create collection for host : " + hostName, LogDb.RUNTIME); - createCollectionSimple(vxlanId); - hostNameToIdMap.put("null " + vxlanId, vxlanId); - apiCollectionId = httpResponseParam.requestParams.getApiCollectionId(); - } - } - - } else { - String key = "null" + " " + vxlanId; - if (!hostNameToIdMap.containsKey(key)) { - createCollectionSimple(vxlanId); - hostNameToIdMap.put(key, vxlanId); - } - - apiCollectionId = vxlanId; - } - + int apiCollectionId = createApiCollectionId(httpResponseParam); + httpResponseParam.requestParams.setApiCollectionId(apiCollectionId); List responseParamsList = GraphQLUtils.getUtils().parseGraphqlResponseParam(httpResponseParam); diff --git a/apps/api-runtime/src/main/java/com/akto/runtime/APICatalogSync.java b/apps/api-runtime/src/main/java/com/akto/runtime/APICatalogSync.java index 544ba667e3..ef7c091617 100644 --- a/apps/api-runtime/src/main/java/com/akto/runtime/APICatalogSync.java +++ b/apps/api-runtime/src/main/java/com/akto/runtime/APICatalogSync.java @@ -7,9 +7,13 @@ import com.akto.DaoInit; import com.akto.dao.*; import com.akto.dao.context.Context; +import com.akto.dao.monitoring.FilterYamlTemplateDao; +import com.akto.dao.runtime_filters.AdvancedTrafficFiltersDao; import com.akto.dto.*; import com.akto.dto.billing.SyncLimit; import com.akto.dto.dependency_flow.DependencyFlow; +import com.akto.dto.monitoring.FilterConfig; +import com.akto.dto.test_editor.YamlTemplate; import com.akto.dto.traffic.Key; import com.akto.dto.traffic.SampleData; import com.akto.dto.traffic.TrafficInfo; @@ -65,6 +69,7 @@ public class APICatalogSync { public AktoPolicyNew aktoPolicyNew; public Map sensitiveParamInfoBooleanMap; public static boolean mergeAsyncOutside = true; + public Map advancedFilterMap = new HashMap<>(); public BloomFilter existingAPIsInDb = BloomFilter.create(Funnels.stringFunnel(Charsets.UTF_8), 1_000_000, 0.001 ); public APICatalogSync(String userIdentifier,int thresh, boolean fetchAllSTI) { @@ -1534,7 +1539,8 @@ public void buildFromDB(boolean calcDiff, boolean fetchAllSTI) { this.delta = new HashMap<>(); } - + List advancedFilterTemplates = AdvancedTrafficFiltersDao.instance.findAll(Filters.ne(YamlTemplate.INACTIVE, true)); + advancedFilterMap = FilterYamlTemplateDao.instance.fetchFilterConfig(false, advancedFilterTemplates, true); try { // fetchAllSTI check added to make sure only runs in dashboard if (!fetchAllSTI) { diff --git a/apps/api-runtime/src/main/java/com/akto/runtime/RelationshipSync.java b/apps/api-runtime/src/main/java/com/akto/runtime/RelationshipSync.java index f13c58d553..d098746587 100644 --- a/apps/api-runtime/src/main/java/com/akto/runtime/RelationshipSync.java +++ b/apps/api-runtime/src/main/java/com/akto/runtime/RelationshipSync.java @@ -12,8 +12,6 @@ import com.fasterxml.jackson.databind.node.ArrayNode; import com.mongodb.client.model.*; import org.bson.conversions.Bson; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import com.akto.dto.HttpResponseParams; import com.akto.dto.HttpRequestParams; @@ -231,37 +229,7 @@ public static Map> extractAllValuesFromPayload(String payloa public static void extractAllValuesFromPayload(JsonNode node, List params, Map> values) { // TODO: null values remove - if (node == null) return; - if (node.isValueNode()) { - String textValue = node.asText(); - if (textValue != null) { - String param = String.join("",params); - if (param.startsWith("#")) { - param = param.substring(1); - } - if (!values.containsKey(param)) { - values.put(param, new HashSet<>()); - } - values.get(param).add(textValue); - } - } else if (node.isArray()) { - ArrayNode arrayNode = (ArrayNode) node; - for(int i = 0; i < arrayNode.size(); i++) { - JsonNode arrayElement = arrayNode.get(i); - params.add("#$"); - extractAllValuesFromPayload(arrayElement, params, values); - params.remove(params.size()-1); - } - } else { - Iterator fieldNames = node.fieldNames(); - while(fieldNames.hasNext()) { - String fieldName = fieldNames.next(); - params.add("#"+fieldName); - JsonNode fieldValue = node.get(fieldName); - extractAllValuesFromPayload(fieldValue, params,values); - params.remove(params.size()-1); - } - } + com.akto.runtime.RuntimeUtil.extractAllValuesFromPayload(node,params,values); } diff --git a/apps/api-runtime/src/main/java/com/akto/runtime/policies/AuthPolicy.java b/apps/api-runtime/src/main/java/com/akto/runtime/policies/AuthPolicy.java index 1b1a130247..d78a991232 100644 --- a/apps/api-runtime/src/main/java/com/akto/runtime/policies/AuthPolicy.java +++ b/apps/api-runtime/src/main/java/com/akto/runtime/policies/AuthPolicy.java @@ -12,6 +12,7 @@ import org.slf4j.LoggerFactory; import java.util.*; +import static com.akto.runtime.utils.Utils.parseCookie; public class AuthPolicy { @@ -33,25 +34,6 @@ private static List findBearerBasicAuth(String header, String return new ArrayList<>(); } - public static Map parseCookie(List cookieList){ - Map cookieMap = new HashMap<>(); - if(cookieList==null)return cookieMap; - for (String cookieValues : cookieList) { - String[] cookies = cookieValues.split(";"); - for (String cookie : cookies) { - cookie=cookie.trim(); - String[] cookieFields = cookie.split("="); - boolean twoCookieFields = cookieFields.length == 2; - if (twoCookieFields) { - if(!cookieMap.containsKey(cookieFields[0])){ - cookieMap.put(cookieFields[0], cookieFields[1]); - } - } - } - } - return cookieMap; - } - public static boolean findAuthType(HttpResponseParams httpResponseParams, ApiInfo apiInfo, RuntimeFilter filter, List customAuthTypes) { Set> allAuthTypesFound = apiInfo.getAllAuthTypesFound(); if (allAuthTypesFound == null) allAuthTypesFound = new HashSet<>(); diff --git a/apps/api-runtime/src/main/java/com/akto/utils/RedactSampleData.java b/apps/api-runtime/src/main/java/com/akto/utils/RedactSampleData.java index e276c87263..eed330db4e 100644 --- a/apps/api-runtime/src/main/java/com/akto/utils/RedactSampleData.java +++ b/apps/api-runtime/src/main/java/com/akto/utils/RedactSampleData.java @@ -16,11 +16,11 @@ import com.fasterxml.jackson.databind.node.ArrayNode; import com.fasterxml.jackson.databind.node.ObjectNode; import com.fasterxml.jackson.databind.node.TextNode; -import com.mongodb.BasicDBObject; import java.util.*; import static com.akto.dto.RawApi.convertHeaders; +import static com.akto.runtime.utils.Utils.parseCookie; public class RedactSampleData { static ObjectMapper mapper = new ObjectMapper(); @@ -43,7 +43,7 @@ public static String redactDataTypes(String sample) throws Exception{ public static String redactCookie(Map> headers, String header) { String cookie = ""; List cookieList = headers.getOrDefault(header, new ArrayList<>()); - Map cookieMap = AuthPolicy.parseCookie(cookieList); + Map cookieMap = parseCookie(cookieList); for (String cookieKey : cookieMap.keySet()) { cookie += cookieKey + "=" + redactValue + ";"; } @@ -186,32 +186,6 @@ public static void change(String parentName, JsonNode parent, String newValue, b } } - - public static String convertOriginalReqRespToString(OriginalHttpRequest request, OriginalHttpResponse response) { - BasicDBObject req = new BasicDBObject(); - if (request != null) { - req.put("url", request.getUrl()); - req.put("method", request.getMethod()); - req.put("type", request.getType()); - req.put("queryParams", request.getQueryParams()); - req.put("body", request.getBody()); - req.put("headers", convertHeaders(request.getHeaders())); - } - - BasicDBObject resp = new BasicDBObject(); - if (response != null) { - resp.put("statusCode", response.getStatusCode()); - resp.put("body", response.getBody()); - resp.put("headers", convertHeaders(response.getHeaders())); - } - - BasicDBObject ret = new BasicDBObject(); - ret.put("request", req); - ret.put("response", resp); - - return ret.toString(); - } - public static String convertHttpRespToOriginalString(HttpResponseParams httpResponseParams) throws JsonProcessingException { Map m = new HashMap<>(); HttpRequestParams httpRequestParams = httpResponseParams.getRequestParams(); diff --git a/apps/api-threat-detection/src/main/java/com/akto/filters/HttpCallFilter.java b/apps/api-threat-detection/src/main/java/com/akto/filters/HttpCallFilter.java new file mode 100644 index 0000000000..051e0c6519 --- /dev/null +++ b/apps/api-threat-detection/src/main/java/com/akto/filters/HttpCallFilter.java @@ -0,0 +1,121 @@ +package com.akto.filters; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.UUID; + +import com.akto.dao.context.Context; +import com.akto.dao.monitoring.FilterYamlTemplateDao; +import com.akto.data_actor.DataActor; +import com.akto.data_actor.DataActorFactory; +import com.akto.dto.ApiInfo.ApiInfoKey; +import com.akto.dto.HttpResponseParams; +import com.akto.dto.RawApi; +import com.akto.dto.bulk_updates.BulkUpdates; +import com.akto.dto.bulk_updates.UpdatePayload; +import com.akto.dto.monitoring.FilterConfig; +import com.akto.dto.test_editor.YamlTemplate; +import com.akto.dto.traffic.SuspectSampleData; +import com.akto.dto.type.URLMethods.Method; +import com.akto.hybrid_parsers.HttpCallParser; +import com.akto.log.LoggerMaker; +import com.akto.log.LoggerMaker.LogDb; +import com.akto.rules.TestPlugin; +import com.akto.runtime.policies.ApiAccessTypePolicy; +import com.akto.test_editor.execution.VariableResolver; +import com.akto.test_editor.filter.data_operands_impl.ValidationResult; + +public class HttpCallFilter { + private static final LoggerMaker loggerMaker = new LoggerMaker(HttpCallFilter.class, LogDb.THREAT_DETECTION); + + private Map apiFilters; + private List bulkUpdates = new ArrayList<>(); + private final int sync_threshold_count; + private final int sync_threshold_time; + private int last_synced; + private int sync_count; + private HttpCallParser httpCallParser; + + private static final int FILTER_REFRESH_INTERVAL = 10 * 60; + private int lastFilterFetch; + + private static final DataActor dataActor = DataActorFactory.fetchInstance(); + + public HttpCallFilter(int sync_threshold_count, int sync_threshold_time) { + apiFilters = new HashMap<>(); + bulkUpdates = new ArrayList<>(); + this.sync_threshold_count = sync_threshold_count; + this.sync_threshold_time = sync_threshold_time; + last_synced = 0; + sync_count = 0; + lastFilterFetch = 0; + httpCallParser = new HttpCallParser(sync_threshold_count, sync_threshold_time); + } + + public void filterFunction(List responseParams) { + + int now = Context.now(); + if ((lastFilterFetch + FILTER_REFRESH_INTERVAL) < now) { + // TODO: add support for only active templates. + List templates = dataActor.fetchFilterYamlTemplates(); + apiFilters = FilterYamlTemplateDao.instance.fetchFilterConfig(false, templates, false); + lastFilterFetch = now; + } + + if (apiFilters != null && !apiFilters.isEmpty()) { + for (HttpResponseParams responseParam : responseParams) { + for (Entry apiFilterEntry : apiFilters.entrySet()) { + try { + FilterConfig apiFilter = apiFilterEntry.getValue(); + String filterId = apiFilterEntry.getKey(); + String message = responseParam.getOrig(); + List sourceIps = ApiAccessTypePolicy.getSourceIps(responseParam); + RawApi rawApi = RawApi.buildFromMessage(message); + int apiCollectionId = httpCallParser.createApiCollectionId(responseParam); + responseParam.requestParams.setApiCollectionId(apiCollectionId); + String url = responseParam.getRequestParams().getURL(); + Method method = Method.valueOf(responseParam.getRequestParams().getMethod()); + ApiInfoKey apiInfoKey = new ApiInfoKey(apiCollectionId, url, method); + Map varMap = apiFilter.resolveVarMap(); + VariableResolver.resolveWordList(varMap, new HashMap>() { + { + put(apiInfoKey, Arrays.asList(message)); + } + }, apiInfoKey); + String filterExecutionLogId = UUID.randomUUID().toString(); + ValidationResult res = TestPlugin.validateFilter(apiFilter.getFilter().getNode(), rawApi, + apiInfoKey, varMap, filterExecutionLogId); + if (res.getIsValid()) { + now = Context.now(); + SuspectSampleData sampleData = new SuspectSampleData( + sourceIps, apiCollectionId, url, method, + message, now, filterId); + Map filterMap = new HashMap<>(); + UpdatePayload updatePayload = new UpdatePayload("obj", sampleData, "set"); + ArrayList updates = new ArrayList<>(); + updates.add(updatePayload.toString()); + bulkUpdates.add(new BulkUpdates(filterMap, updates)); + } + } catch (Exception e) { + loggerMaker.errorAndAddToDb(e, String.format("Error in httpCallFilter %s", e.toString())); + } + } + } + } + sync_count = bulkUpdates.size(); + if (sync_count > 0 && (sync_count >= sync_threshold_count || + (Context.now() - last_synced) > sync_threshold_time)) { + List updates = new ArrayList<>(); + updates.addAll(bulkUpdates); + dataActor.bulkWriteSuspectSampleData(updates); + loggerMaker.infoAndAddToDb(String.format("Inserting %d records in SuspectSampleData", sync_count)); + last_synced = Context.now(); + sync_count = 0; + bulkUpdates.clear(); + } + } +} \ No newline at end of file diff --git a/apps/dashboard/src/main/java/com/akto/action/settings/AdvancedTrafficFiltersAction.java b/apps/dashboard/src/main/java/com/akto/action/settings/AdvancedTrafficFiltersAction.java new file mode 100644 index 0000000000..8438e6708b --- /dev/null +++ b/apps/dashboard/src/main/java/com/akto/action/settings/AdvancedTrafficFiltersAction.java @@ -0,0 +1,94 @@ +package com.akto.action.settings; + +import java.util.List; +import java.util.Map; + +import org.bson.conversions.Bson; + +import com.akto.action.UserAction; +import com.akto.dao.monitoring.FilterConfigYamlParser; +import com.akto.dao.monitoring.FilterYamlTemplateDao; +import com.akto.dao.runtime_filters.AdvancedTrafficFiltersDao; +import com.akto.dto.monitoring.FilterConfig; +import com.akto.dto.test_editor.YamlTemplate; +import com.akto.util.Constants; +import com.akto.utils.TrafficFilterUtil; +import com.mongodb.BasicDBList; +import com.mongodb.client.model.Filters; +import com.mongodb.client.model.FindOneAndUpdateOptions; +import com.mongodb.client.model.Updates; + +public class AdvancedTrafficFiltersAction extends UserAction { + + private BasicDBList templatesList; + private String yamlContent; + private String templateId; + private boolean inactive; + + public String fetchAllFilterTemplates(){ + List yamlTemplates = AdvancedTrafficFiltersDao.instance.findAll(Filters.empty()); + Map configs = FilterYamlTemplateDao.instance.fetchFilterConfig(true, yamlTemplates, false); + this.templatesList = TrafficFilterUtil.getFilterTemplates(configs); + return SUCCESS.toUpperCase(); + } + + public String saveYamlTemplateForTrafficFilters(){ + FilterConfig filterConfig = new FilterConfig(); + try { + filterConfig = FilterConfigYamlParser.parseTemplate(yamlContent, true); + if (filterConfig.getId() == null) { + throw new Exception("id field cannot be empty"); + } + if (filterConfig.getFilter() == null) { + throw new Exception("filter field cannot be empty"); + } + List updates = TrafficFilterUtil.getDbUpdateForTemplate(this.yamlContent, getSUser().getLogin()); + AdvancedTrafficFiltersDao.instance.updateOne( + Filters.eq(Constants.ID, filterConfig.getId()), + Updates.combine(updates)); + + } catch (Exception e) { + e.printStackTrace(); + addActionError(e.getMessage()); + return ERROR.toUpperCase(); + } + return SUCCESS.toUpperCase(); + } + + public String changeActivityOfFilter() { + FindOneAndUpdateOptions options = new FindOneAndUpdateOptions(); + options.upsert(false); + AdvancedTrafficFiltersDao.instance.getMCollection().findOneAndUpdate( + Filters.eq(Constants.ID, this.templateId), + Updates.set(YamlTemplate.INACTIVE, this.inactive), + options + ); + + return SUCCESS.toUpperCase(); + } + + public String deleteAdvancedFilter(){ + AdvancedTrafficFiltersDao.instance.getMCollection().deleteOne( + Filters.eq(Constants.ID, this.templateId) + ); + return SUCCESS.toUpperCase(); + } + + public BasicDBList getTemplatesList() { + return templatesList; + } + + public void setYamlContent(String yamlContent) { + this.yamlContent = yamlContent; + } + + public void setTemplateId(String templateId) { + this.templateId = templateId; + } + + public void setInactive(boolean inactive) { + this.inactive = inactive; + } + + +} diff --git a/apps/dashboard/src/main/java/com/akto/action/threat_detection/FilterYamlTemplateAction.java b/apps/dashboard/src/main/java/com/akto/action/threat_detection/FilterYamlTemplateAction.java new file mode 100644 index 0000000000..cb144e1919 --- /dev/null +++ b/apps/dashboard/src/main/java/com/akto/action/threat_detection/FilterYamlTemplateAction.java @@ -0,0 +1,70 @@ +package com.akto.action.threat_detection; + +import java.util.List; +import java.util.Map; + +import org.bson.conversions.Bson; + +import com.akto.action.UserAction; +import com.akto.dao.monitoring.FilterConfigYamlParser; +import com.akto.dao.monitoring.FilterYamlTemplateDao; +import com.akto.dto.monitoring.FilterConfig; +import com.akto.util.Constants; +import com.akto.utils.TrafficFilterUtil; +import com.mongodb.BasicDBList; +import com.mongodb.client.model.Filters; +import com.mongodb.client.model.Updates; + +public class FilterYamlTemplateAction extends UserAction { + + BasicDBList templates; + String content; + + public String fetchFilterYamlTemplate() { + Map configs = FilterYamlTemplateDao.instance.fetchFilterConfig(true, false); + this.templates = TrafficFilterUtil.getFilterTemplates(configs); + return SUCCESS.toUpperCase(); + } + + public String saveFilterYamlTemplate() { + + FilterConfig filterConfig = new FilterConfig(); + try { + filterConfig = FilterConfigYamlParser.parseTemplate(content, false); + if (filterConfig.getId() == null) { + throw new Exception("id field cannot be empty"); + } + if (filterConfig.getFilter() == null) { + throw new Exception("filter field cannot be empty"); + } + List updates = TrafficFilterUtil.getDbUpdateForTemplate(this.content, getSUser().getLogin()); + FilterYamlTemplateDao.instance.updateOne( + Filters.eq(Constants.ID, filterConfig.getId()), + Updates.combine(updates)); + + } catch (Exception e) { + e.printStackTrace(); + addActionError(e.getMessage()); + return ERROR.toUpperCase(); + } + + return SUCCESS.toUpperCase(); + } + + public BasicDBList getTemplates() { + return templates; + } + + public void setTemplates(BasicDBList templates) { + this.templates = templates; + } + + public String getContent() { + return content; + } + + public void setContent(String content) { + this.content = content; + } + +} diff --git a/apps/dashboard/src/main/java/com/akto/listener/InitializerListener.java b/apps/dashboard/src/main/java/com/akto/listener/InitializerListener.java index cb1a361a2c..e9ad0cc832 100644 --- a/apps/dashboard/src/main/java/com/akto/listener/InitializerListener.java +++ b/apps/dashboard/src/main/java/com/akto/listener/InitializerListener.java @@ -141,6 +141,7 @@ import static com.akto.task.Cluster.callDibs; import static com.akto.utils.billing.OrganizationUtils.syncOrganizationWithAkto; import static com.mongodb.client.model.Filters.eq; +import static com.akto.runtime.utils.Utils.convertOriginalReqRespToString; public class InitializerListener implements ServletContextListener { private static final Logger logger = LoggerFactory.getLogger(InitializerListener.class); @@ -842,7 +843,7 @@ public static void webhookSenderUtil(CustomWebhook webhook) { String message = null; try { - message = RedactSampleData.convertOriginalReqRespToString(request, response); + message = convertOriginalReqRespToString(request, response); } catch (Exception e) { errors.add("Failed converting sample data"); } diff --git a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/components/shared/SampleData.js b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/components/shared/SampleData.js index 7465688c2b..546337db63 100644 --- a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/components/shared/SampleData.js +++ b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/components/shared/SampleData.js @@ -4,6 +4,7 @@ import "./style.css"; import func from "@/util/func" import editorSetup from './customEditor'; import yamlEditorSetup from "../../pages/test_editor/components/editor_config/editorSetup" +import keywords from "../../pages/test_editor/components/editor_config/keywords" function highlightPaths(highlightPathMap, ref){ highlightPathMap && Object.keys(highlightPathMap).forEach((key) => { @@ -224,6 +225,13 @@ function SampleData(props) { editorSetup.setTokenizer() yamlEditorSetup.setEditorTheme() } + if(editorLanguage.includes("custom_yaml")){ + options['theme']= "customTheme" + yamlEditorSetup.registerLanguage() + yamlEditorSetup.setTokenizer() + yamlEditorSetup.setEditorTheme() + yamlEditorSetup.setAutoComplete(keywords) + } if(showDiff){ instance = monaco.editor.createDiffEditor(ref.current, options) } else { diff --git a/apps/testing/src/main/java/com/akto/rules/TestPlugin.java b/apps/testing/src/main/java/com/akto/rules/TestPlugin.java index 030aa94005..0390f51867 100644 --- a/apps/testing/src/main/java/com/akto/rules/TestPlugin.java +++ b/apps/testing/src/main/java/com/akto/rules/TestPlugin.java @@ -11,17 +11,14 @@ import com.akto.log.LoggerMaker; import com.akto.log.LoggerMaker.LogDb; import com.akto.runtime.APICatalogSync; -import com.akto.runtime.RelationshipSync; import com.akto.store.SampleMessageStore; import com.akto.store.TestingUtil; import com.akto.test_editor.filter.Filter; +import com.akto.test_editor.filter.data_operands_impl.ValidationResult; import com.akto.testing.StatusCodeAnalyser; import com.akto.types.CappedSet; import com.akto.util.JSONUtils; -import com.akto.utils.RedactSampleData; import com.fasterxml.jackson.core.JsonFactory; -import com.fasterxml.jackson.core.JsonParser; -import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.gson.Gson; import com.mongodb.BasicDBObject; @@ -36,6 +33,7 @@ import java.util.regex.Pattern; import static com.akto.runtime.APICatalogSync.trimAndSplit; +import static com.akto.runtime.utils.Utils.convertOriginalReqRespToString; public abstract class TestPlugin { @@ -56,9 +54,7 @@ public static boolean isStatusGood(int statusCode) { } public static void extractAllValuesFromPayload(String payload, Map> payloadMap) throws Exception{ - JsonParser jp = factory.createParser(payload); - JsonNode node = mapper.readTree(jp); - RelationshipSync.extractAllValuesFromPayload(node,new ArrayList<>(),payloadMap); + com.akto.runtime.RuntimeUtil.extractAllValuesFromPayload(payload,payloadMap); } public String decrementUrlVersion(String url, int decrementValue, int limit) { @@ -92,55 +88,7 @@ public String decrementUrlVersion(String url, int decrementValue, int limit) { } public static double compareWithOriginalResponse(String originalPayload, String currentPayload, Map comparisonExcludedKeys) { - if (originalPayload == null && currentPayload == null) return 100; - if (originalPayload == null || currentPayload == null) return 0; - - String trimmedOriginalPayload = originalPayload.trim(); - String trimmedCurrentPayload = currentPayload.trim(); - if (trimmedCurrentPayload.equals(trimmedOriginalPayload)) return 100; - - Map> originalResponseParamMap = new HashMap<>(); - Map> currentResponseParamMap = new HashMap<>(); - try { - extractAllValuesFromPayload(originalPayload, originalResponseParamMap); - extractAllValuesFromPayload(currentPayload, currentResponseParamMap); - } catch (Exception e) { - return 0.0; - } - - if (originalResponseParamMap.keySet().size() == 0 && currentResponseParamMap.keySet().size() == 0) { - return 100.0; - } - - Set visited = new HashSet<>(); - int matched = 0; - for (String k1: originalResponseParamMap.keySet()) { - if (visited.contains(k1) || comparisonExcludedKeys.containsKey(k1)) continue; - visited.add(k1); - Set v1 = originalResponseParamMap.get(k1); - Set v2 = currentResponseParamMap.get(k1); - if (Objects.equals(v1, v2)) matched +=1; - } - - for (String k1: currentResponseParamMap.keySet()) { - if (visited.contains(k1) || comparisonExcludedKeys.containsKey(k1)) continue; - visited.add(k1); - Set v1 = originalResponseParamMap.get(k1); - Set v2 = currentResponseParamMap.get(k1); - if (Objects.equals(v1, v2)) matched +=1; - } - - int visitedSize = visited.size(); - if (visitedSize == 0) return 0.0; - - double result = (100.0*matched)/visitedSize; - - if (Double.isFinite(result)) { - return result; - } else { - return 0.0; - } - + return com.akto.testing.Utils.compareWithOriginalResponse(originalPayload, currentPayload, comparisonExcludedKeys); } public Result addWithoutRequestError(String originalMessage, TestResult.TestError testError) { @@ -152,7 +100,7 @@ public Result addWithoutRequestError(String originalMessage, TestResult.TestErro public TestResult buildFailedTestResultWithOriginalMessage(String originalMessage, TestResult.TestError testError, OriginalHttpRequest request, TestInfo testInfo) { String message = null; try { - message = RedactSampleData.convertOriginalReqRespToString(request, null); + message = convertOriginalReqRespToString(request, null); } catch (Exception e) { loggerMaker.errorAndAddToDb("Error while converting testRequest object to string : " + e, LogDb.TESTING); } @@ -173,11 +121,11 @@ public TestResult buildTestResult(OriginalHttpRequest request, OriginalHttpRespo List errors = new ArrayList<>(); String message = null; try { - message = RedactSampleData.convertOriginalReqRespToString(request, response); + message = convertOriginalReqRespToString(request, response); } catch (Exception e) { // TODO: logger.error("Error while converting OriginalHttpRequest to string", e); - message = RedactSampleData.convertOriginalReqRespToString(new OriginalHttpRequest(), new OriginalHttpResponse()); + message = convertOriginalReqRespToString(new OriginalHttpRequest(), new OriginalHttpResponse()); errors.add(TestResult.TestError.FAILED_TO_CONVERT_TEST_REQUEST_TO_STRING.getMessage()); } @@ -363,10 +311,8 @@ public static Map getComparisonExcludedKeys(SampleRequestReplay return comparisonExcludedKeys; } - public static boolean validateFilter(FilterNode filterNode, RawApi rawApi, ApiInfoKey apiInfoKey, Map varMap, String logId) { - if (filterNode == null) return true; - if (rawApi == null) return false; - return validate(filterNode, rawApi, null, apiInfoKey,"filter", varMap, logId); + public static ValidationResult validateFilter(FilterNode filterNode, RawApi rawApi, ApiInfoKey apiInfoKey, Map varMap, String logId) { + return com.akto.testing.Utils.validateFilter(filterNode, rawApi, apiInfoKey, varMap, logId); } public static boolean validateValidator(FilterNode validatorNode, RawApi rawApi, RawApi testRawApi, ApiInfoKey apiInfoKey, Map varMap, String logId) { diff --git a/apps/testing/src/main/java/com/akto/test_editor/execution/Build.java b/apps/testing/src/main/java/com/akto/test_editor/execution/Build.java index 3ecab13e43..275e481029 100644 --- a/apps/testing/src/main/java/com/akto/test_editor/execution/Build.java +++ b/apps/testing/src/main/java/com/akto/test_editor/execution/Build.java @@ -13,7 +13,6 @@ import com.akto.dto.traffic.SampleData; import com.akto.dto.type.URLMethods; import com.akto.log.LoggerMaker; -import com.akto.runtime.policies.AuthPolicy; import com.akto.testing.ApiExecutor; import com.akto.util.Constants; import com.akto.util.HttpRequestResponseUtils; @@ -30,6 +29,7 @@ import static com.akto.util.HttpRequestResponseUtils.FORM_URL_ENCODED_CONTENT_TYPE; import static com.akto.util.HttpRequestResponseUtils.extractValuesFromPayload; +import static com.akto.runtime.utils.Utils.parseCookie; public class Build { @@ -408,7 +408,7 @@ public static Map> getValuesMap(OriginalHttpResponse respons if (values == null) continue; if (headerKey.equalsIgnoreCase("set-cookie")) { - Map cookieMap = AuthPolicy.parseCookie(values); + Map cookieMap = parseCookie(values); for (String cookieKey : cookieMap.keySet()) { String cookieVal = cookieMap.get(cookieKey); valuesMap.put(cookieKey, new HashSet<>(Collections.singletonList(cookieVal))); diff --git a/apps/testing/src/main/java/com/akto/test_editor/execution/ExecutionListBuilder.java b/apps/testing/src/main/java/com/akto/test_editor/execution/ExecutionListBuilder.java index 009164d838..287e134441 100644 --- a/apps/testing/src/main/java/com/akto/test_editor/execution/ExecutionListBuilder.java +++ b/apps/testing/src/main/java/com/akto/test_editor/execution/ExecutionListBuilder.java @@ -14,7 +14,7 @@ public class ExecutionListBuilder { - private static final LoggerMaker loggerMaker = new LoggerMaker(Executor.class); + private static final LoggerMaker loggerMaker = new LoggerMaker(ExecutionListBuilder.class); public ExecutionOrderResp parseExecuteOperations(ExecutorNode node, List executeOrder) { diff --git a/apps/testing/src/main/java/com/akto/test_editor/execution/Executor.java b/apps/testing/src/main/java/com/akto/test_editor/execution/Executor.java index 92a833da9d..6b7bd4de19 100644 --- a/apps/testing/src/main/java/com/akto/test_editor/execution/Executor.java +++ b/apps/testing/src/main/java/com/akto/test_editor/execution/Executor.java @@ -7,10 +7,7 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; -import com.akto.dao.CustomAuthTypeDao; import com.akto.dao.context.Context; -import com.akto.dao.test_editor.TestEditorEnums; -import com.akto.dao.test_editor.TestEditorEnums.ExecutorOperandTypes; import com.akto.dao.testing.TestRolesDao; import com.akto.dto.ApiInfo; import com.akto.dto.CustomAuthType; @@ -23,7 +20,6 @@ import com.akto.util.enums.LoginFlowEnums; import com.akto.util.enums.LoginFlowEnums.AuthMechanismTypes; import com.akto.util.enums.LoginFlowEnums.LoginStepTypesEnums; -import com.akto.utils.RedactSampleData; import com.akto.dto.api_workflow.Graph; import com.akto.dto.billing.Organization; import com.akto.dto.billing.Tokens; @@ -37,9 +33,6 @@ import com.akto.test_editor.Utils; import com.akto.testing.TestExecutor; import com.akto.util.Constants; -import com.akto.util.UsageUtils; -import com.akto.util.enums.LoginFlowEnums; -import com.akto.util.enums.LoginFlowEnums.AuthMechanismTypes; import com.akto.util.modifier.JWTPayloadReplacer; import com.fasterxml.jackson.databind.ObjectMapper; @@ -48,19 +41,18 @@ import org.json.JSONObject; import com.mongodb.BasicDBObject; -import static com.akto.rules.TestPlugin.extractAllValuesFromPayload; import static com.akto.test_editor.Utils.bodyValuesUnchanged; import static com.akto.test_editor.Utils.headerValuesUnchanged; +import static com.akto.runtime.utils.Utils.convertOriginalReqRespToString; +import static com.akto.testing.Utils.compareWithOriginalResponse; import com.mongodb.client.model.Filters; -import org.json.JSONObject; -import org.kohsuke.github.GHRateLimit.Record; -import com.mongodb.client.model.Updates; import org.apache.commons.lang3.StringUtils; import org.bson.conversions.Bson; import org.bson.types.ObjectId; + public class Executor { private static final LoggerMaker loggerMaker = new LoggerMaker(Executor.class); @@ -173,7 +165,7 @@ public YamlTestResult execute(ExecutorNode node, RawApi rawApi, Map() ); } diff --git a/apps/testing/src/main/java/com/akto/test_editor/execution/Memory.java b/apps/testing/src/main/java/com/akto/test_editor/execution/Memory.java index a66cb2bcc2..4549b75f41 100644 --- a/apps/testing/src/main/java/com/akto/test_editor/execution/Memory.java +++ b/apps/testing/src/main/java/com/akto/test_editor/execution/Memory.java @@ -12,7 +12,6 @@ import com.akto.dto.traffic.SampleData; import com.akto.dto.type.SingleTypeInfo; import com.akto.testing.ApiExecutor; -import com.akto.types.CappedSet; import com.akto.util.Constants; import com.mongodb.client.model.Filters; import org.bson.conversions.Bson; diff --git a/apps/testing/src/main/java/com/akto/testing/workflow_node_executor/ApiNodeExecutor.java b/apps/testing/src/main/java/com/akto/testing/workflow_node_executor/ApiNodeExecutor.java index 92645aa6c6..d205891d76 100644 --- a/apps/testing/src/main/java/com/akto/testing/workflow_node_executor/ApiNodeExecutor.java +++ b/apps/testing/src/main/java/com/akto/testing/workflow_node_executor/ApiNodeExecutor.java @@ -5,7 +5,6 @@ import java.util.Map; import java.util.Collections; -import com.akto.dto.ApiInfo; import com.akto.dto.OriginalHttpRequest; import com.akto.dto.OriginalHttpResponse; import com.akto.dto.api_workflow.Node; @@ -18,9 +17,8 @@ import com.akto.log.LoggerMaker.LogDb; import com.akto.test_editor.execution.Memory; import com.akto.testing.ApiExecutor; -import com.akto.testing.Main; import com.akto.testing.Utils; -import com.akto.utils.RedactSampleData; +import static com.akto.runtime.utils.Utils.convertOriginalReqRespToString; public class ApiNodeExecutor extends NodeExecutor { @@ -73,7 +71,7 @@ public NodeResult processNode(Node node, Map valuesMap, Boolean Thread.sleep(sleep); } - response = ApiExecutor.sendRequest(request, followRedirects, null, debug, testLogs, Main.SKIP_SSRF_CHECK); + response = ApiExecutor.sendRequest(request, followRedirects, null, debug, testLogs, com.akto.test_editor.Utils.SKIP_SSRF_CHECK); int statusCode = response.getStatusCode(); @@ -94,7 +92,7 @@ public NodeResult processNode(Node node, Map valuesMap, Boolean String message = null; try { - message = RedactSampleData.convertOriginalReqRespToString(request, response); + message = convertOriginalReqRespToString(request, response); } catch (Exception e) { ; } diff --git a/apps/testing/src/main/java/com/akto/testing/workflow_node_executor/YamlNodeExecutor.java b/apps/testing/src/main/java/com/akto/testing/workflow_node_executor/YamlNodeExecutor.java index 6f8eb0d787..42faa5edc7 100644 --- a/apps/testing/src/main/java/com/akto/testing/workflow_node_executor/YamlNodeExecutor.java +++ b/apps/testing/src/main/java/com/akto/testing/workflow_node_executor/YamlNodeExecutor.java @@ -8,7 +8,6 @@ import java.util.Map; import com.akto.dto.type.SingleTypeInfo; -import com.akto.testing.Main; import com.akto.dto.*; import com.akto.dto.type.URLMethods; import com.akto.test_editor.execution.Memory; @@ -42,7 +41,7 @@ import com.akto.testing.ApiExecutor; import com.akto.testing.TestExecutor; import com.akto.util.Constants; -import com.akto.utils.RedactSampleData; +import static com.akto.runtime.utils.Utils.convertOriginalReqRespToString; import com.google.gson.Gson; public class YamlNodeExecutor extends NodeExecutor { @@ -146,7 +145,7 @@ public NodeResult processNode(Node node, Map varMap, Boolean all int tsAfterReq = 0; try { tsBeforeReq = Context.nowInMillis(); - testResponse = ApiExecutor.sendRequest(testReq.getRequest(), followRedirect, testingRunConfig, debug, testLogs, Main.SKIP_SSRF_CHECK); + testResponse = ApiExecutor.sendRequest(testReq.getRequest(), followRedirect, testingRunConfig, debug, testLogs, com.akto.test_editor.Utils.SKIP_SSRF_CHECK); if (apiInfoKey != null && memory != null) { memory.fillResponse(testReq.getRequest(), testResponse, apiInfoKey.getApiCollectionId(), apiInfoKey.getUrl(), apiInfoKey.getMethod().name()); memory.reset(apiInfoKey.getApiCollectionId(), apiInfoKey.getUrl(), apiInfoKey.getMethod().name()); @@ -160,7 +159,7 @@ public NodeResult processNode(Node node, Map varMap, Boolean all } vulnerable = res.getVulnerable(); try { - message.add(RedactSampleData.convertOriginalReqRespToString(testReq.getRequest(), testResponse)); + message.add(convertOriginalReqRespToString(testReq.getRequest(), testResponse)); } catch (Exception e) { ; } diff --git a/libs/dao/src/main/java/com/akto/dao/monitoring/FilterConfigYamlParser.java b/libs/dao/src/main/java/com/akto/dao/monitoring/FilterConfigYamlParser.java new file mode 100644 index 0000000000..e1593bf32c --- /dev/null +++ b/libs/dao/src/main/java/com/akto/dao/monitoring/FilterConfigYamlParser.java @@ -0,0 +1,76 @@ +package com.akto.dao.monitoring; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import com.akto.dao.test_editor.filter.ConfigParser; +import com.akto.dto.monitoring.FilterConfig; +import com.akto.dto.test_editor.ConfigParserResult; +import com.akto.dto.test_editor.ExecutorConfigParserResult; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; + +public class FilterConfigYamlParser { + + public static FilterConfig parseTemplate(String content, boolean shouldParseExecutor) throws Exception { + + ObjectMapper mapper = new ObjectMapper(new YAMLFactory()); + + Map config = mapper.readValue(content, new TypeReference>() { + }); + return parseConfig(config, shouldParseExecutor); + } + + public static FilterConfig parseConfig(Map config,boolean shouldParseExecutor) throws Exception { + + FilterConfig filterConfig = null; + boolean isFilterError = false; + + String id = (String) config.get(FilterConfig.ID); + if (id == null) { + return filterConfig; + } + + Object filterMap = config.get(FilterConfig.FILTER); + if (filterMap == null) { + isFilterError = true; + filterConfig = new FilterConfig(id, null, null); + } + + ConfigParser configParser = new ConfigParser(); + ConfigParserResult filters = configParser.parse(filterMap); + if (filters == null) { + // todo: throw error + isFilterError = true; + filterConfig = new FilterConfig(id, null, null); + } + + Map> wordListMap = new HashMap<>(); + try { + if (config.containsKey(FilterConfig.WORD_LISTS)) { + wordListMap = (Map) config.get(FilterConfig.WORD_LISTS); + } + } catch (Exception e) { + isFilterError = true; + filterConfig = new FilterConfig(id, filters, null); + } + if(!isFilterError){ + filterConfig = new FilterConfig(id, filters, wordListMap); + } + + if(shouldParseExecutor){ + com.akto.dao.test_editor.executor.ConfigParser executorConfigParser = new com.akto.dao.test_editor.executor.ConfigParser(); + Object executionMap = config.get("execute"); + if(executionMap == null){ + return filterConfig; + } + ExecutorConfigParserResult executorConfigParserResult = executorConfigParser.parseConfigMap(executionMap); + filterConfig.setExecutor(executorConfigParserResult); + } + + return filterConfig; + } + +} diff --git a/libs/dao/src/main/java/com/akto/dao/monitoring/FilterYamlTemplateDao.java b/libs/dao/src/main/java/com/akto/dao/monitoring/FilterYamlTemplateDao.java new file mode 100644 index 0000000000..8a898c387c --- /dev/null +++ b/libs/dao/src/main/java/com/akto/dao/monitoring/FilterYamlTemplateDao.java @@ -0,0 +1,51 @@ +package com.akto.dao.monitoring; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import com.akto.dao.AccountsContextDao; +import com.akto.dto.monitoring.FilterConfig; +import com.akto.dto.test_editor.YamlTemplate; +import com.mongodb.client.model.Filters; + +public class FilterYamlTemplateDao extends AccountsContextDao { + + public static final FilterYamlTemplateDao instance = new FilterYamlTemplateDao(); + + public Map fetchFilterConfig(boolean includeYamlContent, boolean shouldParseExecutor) { + List yamlTemplates = FilterYamlTemplateDao.instance.findAll(Filters.empty()); + return fetchFilterConfig(includeYamlContent, yamlTemplates, shouldParseExecutor); + } + + public Map fetchFilterConfig(boolean includeYamlContent, List yamlTemplates, boolean shouldParseExecutor) { + Map filterConfigMap = new HashMap<>(); + for (YamlTemplate yamlTemplate : yamlTemplates) { + try { + if (yamlTemplate != null) { + FilterConfig filterConfig = FilterConfigYamlParser.parseTemplate(yamlTemplate.getContent(), shouldParseExecutor); + filterConfig.setAuthor(yamlTemplate.getAuthor()); + filterConfig.setCreatedAt(yamlTemplate.getCreatedAt()); + filterConfig.setUpdatedAt(yamlTemplate.getUpdatedAt()); + if (includeYamlContent) { + filterConfig.setContent(yamlTemplate.getContent()); + } + filterConfigMap.put(filterConfig.getId(), filterConfig); + } + } catch (Exception e) { + e.printStackTrace(); + } + } + return filterConfigMap; + } + + @Override + public String getCollName() { + return "filter_yaml_templates"; + } + + @Override + public Class getClassT() { + return YamlTemplate.class; + } +} diff --git a/libs/dao/src/main/java/com/akto/dao/runtime_filters/AdvancedTrafficFiltersDao.java b/libs/dao/src/main/java/com/akto/dao/runtime_filters/AdvancedTrafficFiltersDao.java new file mode 100644 index 0000000000..cc74f15939 --- /dev/null +++ b/libs/dao/src/main/java/com/akto/dao/runtime_filters/AdvancedTrafficFiltersDao.java @@ -0,0 +1,19 @@ +package com.akto.dao.runtime_filters; + +import com.akto.dao.AccountsContextDao; +import com.akto.dto.test_editor.YamlTemplate; + +public class AdvancedTrafficFiltersDao extends AccountsContextDao { + + public static final AdvancedTrafficFiltersDao instance = new AdvancedTrafficFiltersDao(); + + @Override + public String getCollName() { + return "advanced_traffic_filters"; + } + + @Override + public Class getClassT() { + return YamlTemplate.class; + } +} diff --git a/libs/dao/src/main/java/com/akto/dto/monitoring/FilterConfig.java b/libs/dao/src/main/java/com/akto/dto/monitoring/FilterConfig.java new file mode 100644 index 0000000000..21d069205d --- /dev/null +++ b/libs/dao/src/main/java/com/akto/dto/monitoring/FilterConfig.java @@ -0,0 +1,110 @@ +package com.akto.dto.monitoring; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import com.akto.dto.test_editor.ConfigParserResult; +import com.akto.dto.test_editor.ExecutorConfigParserResult; + +public class FilterConfig { + private String id; + public static final String ID = "id"; + private ConfigParserResult filter; + public static final String FILTER = "filter"; + private Map> wordLists; + public static final String WORD_LISTS = "wordLists"; + public static final String CREATED_AT = "createdAt"; + private int createdAt; + public static final String UPDATED_AT = "updatedAt"; + private int updatedAt; + public static final String _AUTHOR = "author"; + private String author; + public static final String _CONTENT = "content"; + private String content; + + private ExecutorConfigParserResult executor; + + public FilterConfig(String id, ConfigParserResult filter, Map> wordLists) { + this.id = id; + this.filter = filter; + this.wordLists = wordLists; + } + + public FilterConfig() { + } + + public String getId() { + return id; + } + + public void setId(String id) { + this.id = id; + } + + public ConfigParserResult getFilter() { + return filter; + } + + public void setFilter(ConfigParserResult filter) { + this.filter = filter; + } + + public int getCreatedAt() { + return createdAt; + } + + public void setCreatedAt(int createdAt) { + this.createdAt = createdAt; + } + + public int getUpdatedAt() { + return updatedAt; + } + + public void setUpdatedAt(int updatedAt) { + this.updatedAt = updatedAt; + } + + public String getAuthor() { + return author; + } + + public void setAuthor(String author) { + this.author = author; + } + + public Map> getWordLists() { + return wordLists; + } + + public Map resolveVarMap() { + Map> wordListsMap = this.wordLists == null ? new HashMap<>() : this.wordLists; + Map varMap = new HashMap<>(); + + for (String key : wordListsMap.keySet()) { + varMap.put("wordList_" + key, wordListsMap.get(key)); + } + return varMap; + } + + public void setWordLists(Map> wordLists) { + this.wordLists = wordLists; + } + + public String getContent() { + return content; + } + + public void setContent(String content) { + this.content = content; + } + + public ExecutorConfigParserResult getExecutor() { + return executor; + } + + public void setExecutor(ExecutorConfigParserResult executor) { + this.executor = executor; + } +} \ No newline at end of file diff --git a/libs/utils/src/main/java/com/akto/data_actor/ClientActor.java b/libs/utils/src/main/java/com/akto/data_actor/ClientActor.java index 17cec6865c..b55bcebfff 100644 --- a/libs/utils/src/main/java/com/akto/data_actor/ClientActor.java +++ b/libs/utils/src/main/java/com/akto/data_actor/ClientActor.java @@ -76,6 +76,9 @@ import java.util.Set; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; + +import org.bson.types.ObjectId; + import com.google.gson.Gson; public class ClientActor extends DataActor { diff --git a/libs/utils/src/main/java/com/akto/runtime/RuntimeUtil.java b/libs/utils/src/main/java/com/akto/runtime/RuntimeUtil.java index 8a3e5d8a02..dc312e1723 100644 --- a/libs/utils/src/main/java/com/akto/runtime/RuntimeUtil.java +++ b/libs/utils/src/main/java/com/akto/runtime/RuntimeUtil.java @@ -2,13 +2,18 @@ import com.akto.dto.HttpResponseParams; import com.akto.dto.settings.DefaultPayload; +import com.akto.dto.type.SingleTypeInfo.SuperType; +import com.akto.dto.type.URLMethods.Method; +import com.akto.dto.type.URLTemplate; import com.akto.log.LoggerMaker; +import com.fasterxml.jackson.core.JsonFactory; +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ArrayNode; import java.net.URL; -import java.util.ArrayList; -import java.util.Base64; -import java.util.List; -import java.util.Map; +import java.util.*; public class RuntimeUtil { private static final LoggerMaker loggerMaker = new LoggerMaker(RuntimeUtil.class); @@ -39,4 +44,100 @@ public static boolean matchesDefaultPayload(HttpResponseParams httpResponseParam return false; } + + public static boolean hasSpecialCharacters(String input) { + // Define the special characters + String specialCharacters = "<>%/?#[]@!$&'()*+,;="; + for (char c : input.toCharArray()) { + if (specialCharacters.contains(Character.toString(c))) { + return true; + } + } + + return false; + } + + private static String trim(String url) { + if (url.startsWith("/")) url = url.substring(1, url.length()); + if (url.endsWith("/")) url = url.substring(0, url.length()-1); + return url; + } + + private static String[] trimAndSplit(String url) { + return trim(url).split("/"); + } + + public static URLTemplate createUrlTemplate(String url, Method method) { + String[] tokens = trimAndSplit(url); + SuperType[] types = new SuperType[tokens.length]; + for(int i = 0; i < tokens.length; i ++ ) { + String token = tokens[i]; + + if (token.equals(SuperType.STRING.name())) { + tokens[i] = null; + types[i] = SuperType.STRING; + } else if (token.equals(SuperType.INTEGER.name())) { + tokens[i] = null; + types[i] = SuperType.INTEGER; + } else if (token.equals(SuperType.OBJECT_ID.name())) { + tokens[i] = null; + types[i] = SuperType.OBJECT_ID; + } else if (token.equals(SuperType.FLOAT.name())) { + tokens[i] = null; + types[i] = SuperType.FLOAT; + } else { + types[i] = null; + } + + } + + URLTemplate urlTemplate = new URLTemplate(tokens, types, method); + + return urlTemplate; + } + + static ObjectMapper mapper = new ObjectMapper(); + static JsonFactory factory = mapper.getFactory(); + + public static void extractAllValuesFromPayload(String payload, Map> payloadMap) throws Exception{ + JsonParser jp = factory.createParser(payload); + JsonNode node = mapper.readTree(jp); + extractAllValuesFromPayload(node,new ArrayList<>(),payloadMap); + } + + public static void extractAllValuesFromPayload(JsonNode node, List params, Map> values) { + // TODO: null values remove + if (node == null) return; + if (node.isValueNode()) { + String textValue = node.asText(); + if (textValue != null) { + String param = String.join("",params); + if (param.startsWith("#")) { + param = param.substring(1); + } + if (!values.containsKey(param)) { + values.put(param, new HashSet<>()); + } + values.get(param).add(textValue); + } + } else if (node.isArray()) { + ArrayNode arrayNode = (ArrayNode) node; + for(int i = 0; i < arrayNode.size(); i++) { + JsonNode arrayElement = arrayNode.get(i); + params.add("#$"); + extractAllValuesFromPayload(arrayElement, params, values); + params.remove(params.size()-1); + } + } else { + Iterator fieldNames = node.fieldNames(); + while(fieldNames.hasNext()) { + String fieldName = fieldNames.next(); + params.add("#"+fieldName); + JsonNode fieldValue = node.get(fieldName); + extractAllValuesFromPayload(fieldValue, params,values); + params.remove(params.size()-1); + } + } + + } } diff --git a/libs/utils/src/main/java/com/akto/runtime/utils/Utils.java b/libs/utils/src/main/java/com/akto/runtime/utils/Utils.java new file mode 100644 index 0000000000..380ef76ef2 --- /dev/null +++ b/libs/utils/src/main/java/com/akto/runtime/utils/Utils.java @@ -0,0 +1,157 @@ +package com.akto.runtime.utils; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Properties; + +import org.apache.commons.lang3.math.NumberUtils; +import org.apache.kafka.clients.consumer.ConsumerConfig; +import org.apache.kafka.common.serialization.StringDeserializer; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.akto.dto.HttpRequestParams; +import com.akto.dto.HttpResponseParams; +import com.akto.dto.OriginalHttpRequest; +import com.akto.dto.OriginalHttpResponse; +import com.akto.log.LoggerMaker.LogDb; +import com.akto.util.HttpRequestResponseUtils; +import com.akto.util.JSONUtils; +import com.alibaba.fastjson2.JSON; +import com.alibaba.fastjson2.JSONObject; +import com.mongodb.BasicDBObject; +import static com.akto.dto.RawApi.convertHeaders; + +import static com.akto.util.HttpRequestResponseUtils.GRPC_CONTENT_TYPE; + +public class Utils { + private static final Logger logger = LoggerFactory.getLogger(Utils.class); + + private static int debugPrintCounter = 500; + public static void printL(Object o) { + if (debugPrintCounter > 0) { + debugPrintCounter--; + logger.info(o.toString()); + } + } + + public static Properties configProperties(String kafkaBrokerUrl, String groupIdConfig, int maxPollRecordsConfig) { + Properties properties = new Properties(); + properties.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, kafkaBrokerUrl); + properties.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class); + properties.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class); + properties.put(ConsumerConfig.MAX_POLL_RECORDS_CONFIG, maxPollRecordsConfig); + properties.put(ConsumerConfig.GROUP_ID_CONFIG, groupIdConfig); + properties.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest"); + properties.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, false); + + return properties; + } + + public static String convertOriginalReqRespToString(OriginalHttpRequest request, OriginalHttpResponse response) { + BasicDBObject req = new BasicDBObject(); + if (request != null) { + req.put("url", request.getUrl()); + req.put("method", request.getMethod()); + req.put("type", request.getType()); + req.put("queryParams", request.getQueryParams()); + req.put("body", request.getBody()); + req.put("headers", convertHeaders(request.getHeaders())); + } + + BasicDBObject resp = new BasicDBObject(); + if (response != null) { + resp.put("statusCode", response.getStatusCode()); + resp.put("body", response.getBody()); + resp.put("headers", convertHeaders(response.getHeaders())); + } + + BasicDBObject ret = new BasicDBObject(); + ret.put("request", req); + ret.put("response", resp); + + return ret.toString(); + } + + public static Map parseCookie(List cookieList){ + Map cookieMap = new HashMap<>(); + if(cookieList==null)return cookieMap; + for (String cookieValues : cookieList) { + String[] cookies = cookieValues.split(";"); + for (String cookie : cookies) { + cookie=cookie.trim(); + String[] cookieFields = cookie.split("="); + boolean twoCookieFields = cookieFields.length == 2; + if (twoCookieFields) { + if(!cookieMap.containsKey(cookieFields[0])){ + cookieMap.put(cookieFields[0], cookieFields[1]); + } + } + } + } + return cookieMap; + } + + private static int GRPC_DEBUG_COUNTER = 50; + + public static HttpResponseParams parseKafkaMessage(String message) throws Exception { + + //convert java object to JSON format + + JSONObject jsonObject = JSON.parseObject(message); + + String method = jsonObject.getString("method"); + String url = jsonObject.getString("path"); + String type = jsonObject.getString("type"); + Map> requestHeaders = OriginalHttpRequest.buildHeadersMap(jsonObject, "requestHeaders"); + + String rawRequestPayload = jsonObject.getString("requestPayload"); + String requestPayload = HttpRequestResponseUtils.rawToJsonString(rawRequestPayload,requestHeaders); + + if (GRPC_DEBUG_COUNTER > 0) { + String acceptableContentType = HttpRequestResponseUtils.getAcceptableContentType(requestHeaders); + if (acceptableContentType != null && rawRequestPayload.length() > 0) { + // only if request payload is of FORM_URL_ENCODED_CONTENT_TYPE we convert it to json + if (acceptableContentType.equals(GRPC_CONTENT_TYPE)) { + logger.info("grpc kafka payload:" + message,LogDb.RUNTIME); + GRPC_DEBUG_COUNTER--; + } + } + } + + String apiCollectionIdStr = jsonObject.getOrDefault("akto_vxlan_id", "0").toString(); + int apiCollectionId = 0; + if (NumberUtils.isDigits(apiCollectionIdStr)) { + apiCollectionId = NumberUtils.toInt(apiCollectionIdStr, 0); + } + + HttpRequestParams requestParams = new HttpRequestParams( + method,url,type, requestHeaders, requestPayload, apiCollectionId + ); + + int statusCode = jsonObject.getInteger("statusCode"); + String status = jsonObject.getString("status"); + Map> responseHeaders = OriginalHttpRequest.buildHeadersMap(jsonObject, "responseHeaders"); + String payload = jsonObject.getString("responsePayload"); + payload = HttpRequestResponseUtils.rawToJsonString(payload, responseHeaders); + payload = JSONUtils.parseIfJsonP(payload); + int time = jsonObject.getInteger("time"); + String accountId = jsonObject.getString("akto_account_id"); + String sourceIP = jsonObject.getString("ip"); + String destIP = jsonObject.getString("destIp"); + String direction = jsonObject.getString("direction"); + + String isPendingStr = (String) jsonObject.getOrDefault("is_pending", "false"); + boolean isPending = !isPendingStr.toLowerCase().equals("false"); + String sourceStr = (String) jsonObject.getOrDefault("source", HttpResponseParams.Source.OTHER.name()); + HttpResponseParams.Source source = HttpResponseParams.Source.valueOf(sourceStr); + + return new HttpResponseParams( + type,statusCode, status, responseHeaders, payload, requestParams, time, accountId, isPending, source, message, sourceIP, destIP, direction + ); + } + + + +} diff --git a/apps/testing/src/main/java/com/akto/test_editor/Utils.java b/libs/utils/src/main/java/com/akto/test_editor/Utils.java similarity index 99% rename from apps/testing/src/main/java/com/akto/test_editor/Utils.java rename to libs/utils/src/main/java/com/akto/test_editor/Utils.java index 983e32fe0b..2daf84d23b 100644 --- a/apps/testing/src/main/java/com/akto/test_editor/Utils.java +++ b/libs/utils/src/main/java/com/akto/test_editor/Utils.java @@ -15,8 +15,6 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; -import org.bouncycastle.jce.provider.JDKDSASigner.stdDSA; - import com.akto.dto.OriginalHttpRequest; import com.akto.dto.RawApi; import com.akto.dao.context.Context; @@ -24,6 +22,7 @@ import com.akto.dto.test_editor.ExecutorSingleOperationResp; import com.akto.dto.testing.UrlModifierPayload; import com.akto.util.Constants; +import com.akto.util.DashboardMode; import com.akto.util.JSONUtils; import com.akto.util.http_util.CoreHTTPClient; import com.fasterxml.jackson.core.JsonFactory; @@ -36,7 +35,7 @@ import com.mongodb.BasicDBList; import com.mongodb.BasicDBObject; -import static com.akto.rules.TestPlugin.extractAllValuesFromPayload; +import static com.akto.runtime.RuntimeUtil.extractAllValuesFromPayload; import okhttp3.*; public class Utils { @@ -45,6 +44,8 @@ public class Utils { private static final JsonFactory factory = mapper.getFactory(); private static final Gson gson = new Gson(); + public static boolean SKIP_SSRF_CHECK = ("true".equalsIgnoreCase(System.getenv("SKIP_SSRF_CHECK")) || !DashboardMode.isSaasDeployment()); + private static final OkHttpClient client = CoreHTTPClient.client.newBuilder() .writeTimeout(5, TimeUnit.SECONDS) .readTimeout(5, TimeUnit.SECONDS) diff --git a/apps/testing/src/main/java/com/akto/test_editor/execution/VariableResolver.java b/libs/utils/src/main/java/com/akto/test_editor/execution/VariableResolver.java similarity index 99% rename from apps/testing/src/main/java/com/akto/test_editor/execution/VariableResolver.java rename to libs/utils/src/main/java/com/akto/test_editor/execution/VariableResolver.java index 70e917201f..52ac862f0b 100644 --- a/apps/testing/src/main/java/com/akto/test_editor/execution/VariableResolver.java +++ b/libs/utils/src/main/java/com/akto/test_editor/execution/VariableResolver.java @@ -24,7 +24,6 @@ import com.akto.dto.type.RequestTemplate; import com.akto.dto.type.SingleTypeInfo; import com.akto.dto.type.URLMethods; -import com.akto.parsers.HttpCallParser; import com.akto.test_editor.Utils; import com.akto.util.modifier.AddJWKModifier; import com.akto.util.modifier.AddJkuJWTModifier; @@ -36,6 +35,8 @@ import com.mongodb.client.model.Filters; import com.mongodb.client.model.Projections; +import static com.akto.runtime.utils.Utils.parseKafkaMessage; + public class VariableResolver { public static Object getValue(Map varMap, String key) { @@ -679,7 +680,7 @@ public static Set extractValuesFromSampleData(List samples, Stri HttpResponseParams httpResponseParams; HttpRequestParams httpRequestParams; try { - httpResponseParams = HttpCallParser.parseKafkaMessage(sample); + httpResponseParams = parseKafkaMessage(sample); httpRequestParams = httpResponseParams.getRequestParams(); if ("terminal_keys".equals(location)) { diff --git a/apps/testing/src/main/java/com/akto/test_editor/filter/Filter.java b/libs/utils/src/main/java/com/akto/test_editor/filter/Filter.java similarity index 100% rename from apps/testing/src/main/java/com/akto/test_editor/filter/Filter.java rename to libs/utils/src/main/java/com/akto/test_editor/filter/Filter.java diff --git a/apps/testing/src/main/java/com/akto/test_editor/filter/FilterAction.java b/libs/utils/src/main/java/com/akto/test_editor/filter/FilterAction.java similarity index 98% rename from apps/testing/src/main/java/com/akto/test_editor/filter/FilterAction.java rename to libs/utils/src/main/java/com/akto/test_editor/filter/FilterAction.java index 740eecc8b2..9de13ecc53 100644 --- a/apps/testing/src/main/java/com/akto/test_editor/filter/FilterAction.java +++ b/libs/utils/src/main/java/com/akto/test_editor/filter/FilterAction.java @@ -33,20 +33,19 @@ import com.akto.dto.type.SingleTypeInfo; import com.akto.dto.type.URLMethods; import com.akto.dto.type.URLTemplate; -import com.akto.parsers.HttpCallParser; -import com.akto.rules.TestPlugin; -import com.akto.runtime.APICatalogSync; -import com.akto.runtime.policies.AuthPolicy; import com.akto.test_editor.Utils; import com.akto.test_editor.execution.VariableResolver; import com.akto.test_editor.filter.data_operands_impl.*; import com.akto.util.JSONUtils; -import com.akto.utils.RedactSampleData; import com.mongodb.BasicDBList; import com.mongodb.BasicDBObject; import com.mongodb.client.model.Filters; +import static com.akto.runtime.utils.Utils.parseCookie; import static com.akto.dto.RawApi.convertHeaders; +import static com.akto.runtime.RuntimeUtil.createUrlTemplate; +import static com.akto.testing.Utils.compareWithOriginalResponse; +import static com.akto.runtime.utils.Utils.parseKafkaMessage; public final class FilterAction { @@ -343,7 +342,7 @@ public DataOperandsFilterResponse applyFilterOnPayload(FilterActionRequest filte if (sampleRawApi == null) { return new DataOperandsFilterResponse(false, null, null, null); } - double percentageMatch = TestPlugin.compareWithOriginalResponse(payload, sampleRawApi.getResponse().getBody(), new HashMap<>()); + double percentageMatch = compareWithOriginalResponse(payload, sampleRawApi.getResponse().getBody(), new HashMap<>()); val = (int) percentageMatch; } else if (filterActionRequest.getBodyOperand() != null && filterActionRequest.getBodyOperand().equalsIgnoreCase(BodyOperator.PERCENTAGE_MATCH_SCHEMA.toString())) { RawApi sampleRawApi = filterActionRequest.getRawApi(); @@ -425,7 +424,7 @@ public void extractPayload(FilterActionRequest filterActionRequest, Map()); + double percentageMatch = compareWithOriginalResponse(payload, sampleRawApi.getResponse().getBody(), new HashMap<>()); val = (int) percentageMatch; } } else { @@ -562,7 +561,7 @@ public DataOperandsFilterResponse applyFiltersOnHeaders(FilterActionRequest filt if (!res && (key.equals("cookie") || key.equals("set-cookie"))) { List cookieList = headers.getOrDefault(key, new ArrayList<>()); - Map cookieMap = AuthPolicy.parseCookie(cookieList); + Map cookieMap = parseCookie(cookieList); for (String cookieKey : cookieMap.keySet()) { dataOperandFilterRequest = new DataOperandFilterRequest(cookieKey, filterActionRequest.getQuerySet(), filterActionRequest.getOperand()); res = invokeFilter(dataOperandFilterRequest); @@ -599,7 +598,7 @@ public DataOperandsFilterResponse applyFiltersOnHeaders(FilterActionRequest filt if (!res && (key.equals("cookie") || key.equals("set-cookie"))) { List cookieList = headers.getOrDefault("cookie", new ArrayList<>()); - Map cookieMap = AuthPolicy.parseCookie(cookieList); + Map cookieMap = parseCookie(cookieList); for (String cookieKey : cookieMap.keySet()) { DataOperandFilterRequest dataOperandFilterRequest = new DataOperandFilterRequest(cookieMap.get(cookieKey), filterActionRequest.getQuerySet(), filterActionRequest.getOperand()); res = invokeFilter(dataOperandFilterRequest); @@ -1229,10 +1228,10 @@ public BasicDBObject getPrivateResourceCount(OriginalHttpRequest originalHttpReq int privateCnt = 0; List privateValues = new ArrayList<>(); if (APICatalog.isTemplateUrl(url)) { - URLTemplate urlTemplate = APICatalogSync.createUrlTemplate(url, method); + URLTemplate urlTemplate = createUrlTemplate(url, method); String[] tokens = urlTemplate.getTokens(); - String[] urlWithParamsTokens = APICatalogSync.createUrlTemplate(urlWithParams, method).getTokens(); + String[] urlWithParamsTokens = createUrlTemplate(urlWithParams, method).getTokens(); for (int i = 0;i < tokens.length; i++) { if (tokens[i] == null) { SingleTypeInfo singleTypeInfo = querySti(i+"", true,apiInfoKey, false, -1); @@ -1264,7 +1263,7 @@ public BasicDBObject getPrivateResourceCount(OriginalHttpRequest originalHttpReq } for (String sample: sd.getSamples()) { try { - HttpResponseParams httpResponseParams = HttpCallParser.parseKafkaMessage(sample); + HttpResponseParams httpResponseParams = parseKafkaMessage(sample); String sUrl = httpResponseParams.getRequestParams().getURL(); String[] sUrlTokens = sUrl.split("/"); String[] origUrlTokens = urlWithParams.split("/"); @@ -1320,7 +1319,7 @@ public BasicDBObject getPrivateResourceCount(OriginalHttpRequest originalHttpReq String key = SingleTypeInfo.findLastKeyFromParam(param); BasicDBObject payloadObj = new BasicDBObject(); try { - HttpResponseParams httpResponseParams = HttpCallParser.parseKafkaMessage(sample); + HttpResponseParams httpResponseParams = parseKafkaMessage(sample); payloadObj = RequestTemplate.parseRequestPayload(httpResponseParams.getRequestParams().getPayload(), null); } catch (Exception e) { // TODO: handle exception diff --git a/apps/testing/src/main/java/com/akto/test_editor/filter/data_operands_impl/ApiCollectionFilter.java b/libs/utils/src/main/java/com/akto/test_editor/filter/data_operands_impl/ApiCollectionFilter.java similarity index 100% rename from apps/testing/src/main/java/com/akto/test_editor/filter/data_operands_impl/ApiCollectionFilter.java rename to libs/utils/src/main/java/com/akto/test_editor/filter/data_operands_impl/ApiCollectionFilter.java diff --git a/apps/testing/src/main/java/com/akto/test_editor/filter/data_operands_impl/ContainsAllFilter.java b/libs/utils/src/main/java/com/akto/test_editor/filter/data_operands_impl/ContainsAllFilter.java similarity index 100% rename from apps/testing/src/main/java/com/akto/test_editor/filter/data_operands_impl/ContainsAllFilter.java rename to libs/utils/src/main/java/com/akto/test_editor/filter/data_operands_impl/ContainsAllFilter.java diff --git a/apps/testing/src/main/java/com/akto/test_editor/filter/data_operands_impl/ContainsEitherFilter.java b/libs/utils/src/main/java/com/akto/test_editor/filter/data_operands_impl/ContainsEitherFilter.java similarity index 100% rename from apps/testing/src/main/java/com/akto/test_editor/filter/data_operands_impl/ContainsEitherFilter.java rename to libs/utils/src/main/java/com/akto/test_editor/filter/data_operands_impl/ContainsEitherFilter.java diff --git a/apps/testing/src/main/java/com/akto/test_editor/filter/data_operands_impl/ContainsJwt.java b/libs/utils/src/main/java/com/akto/test_editor/filter/data_operands_impl/ContainsJwt.java similarity index 100% rename from apps/testing/src/main/java/com/akto/test_editor/filter/data_operands_impl/ContainsJwt.java rename to libs/utils/src/main/java/com/akto/test_editor/filter/data_operands_impl/ContainsJwt.java diff --git a/apps/testing/src/main/java/com/akto/test_editor/filter/data_operands_impl/CookieExpireFilter.java b/libs/utils/src/main/java/com/akto/test_editor/filter/data_operands_impl/CookieExpireFilter.java similarity index 95% rename from apps/testing/src/main/java/com/akto/test_editor/filter/data_operands_impl/CookieExpireFilter.java rename to libs/utils/src/main/java/com/akto/test_editor/filter/data_operands_impl/CookieExpireFilter.java index 0edce0afeb..80155cfefa 100644 --- a/apps/testing/src/main/java/com/akto/test_editor/filter/data_operands_impl/CookieExpireFilter.java +++ b/libs/utils/src/main/java/com/akto/test_editor/filter/data_operands_impl/CookieExpireFilter.java @@ -10,7 +10,8 @@ import java.util.Map; import com.akto.dto.test_editor.DataOperandFilterRequest; -import com.akto.runtime.policies.AuthPolicy; + +import static com.akto.runtime.utils.Utils.parseCookie; public class CookieExpireFilter extends DataOperandsImpl { @@ -33,7 +34,7 @@ public Boolean isValid(DataOperandFilterRequest dataOperandFilterRequest) { return false; } - Map cookieMap = AuthPolicy.parseCookie(Arrays.asList(data)); + Map cookieMap = parseCookie(Arrays.asList(data)); boolean result = queryVal; boolean res = false; diff --git a/apps/testing/src/main/java/com/akto/test_editor/filter/data_operands_impl/DataOperandsImpl.java b/libs/utils/src/main/java/com/akto/test_editor/filter/data_operands_impl/DataOperandsImpl.java similarity index 100% rename from apps/testing/src/main/java/com/akto/test_editor/filter/data_operands_impl/DataOperandsImpl.java rename to libs/utils/src/main/java/com/akto/test_editor/filter/data_operands_impl/DataOperandsImpl.java diff --git a/apps/testing/src/main/java/com/akto/test_editor/filter/data_operands_impl/DatatypeFilter.java b/libs/utils/src/main/java/com/akto/test_editor/filter/data_operands_impl/DatatypeFilter.java similarity index 100% rename from apps/testing/src/main/java/com/akto/test_editor/filter/data_operands_impl/DatatypeFilter.java rename to libs/utils/src/main/java/com/akto/test_editor/filter/data_operands_impl/DatatypeFilter.java diff --git a/apps/testing/src/main/java/com/akto/test_editor/filter/data_operands_impl/EqFilter.java b/libs/utils/src/main/java/com/akto/test_editor/filter/data_operands_impl/EqFilter.java similarity index 100% rename from apps/testing/src/main/java/com/akto/test_editor/filter/data_operands_impl/EqFilter.java rename to libs/utils/src/main/java/com/akto/test_editor/filter/data_operands_impl/EqFilter.java diff --git a/apps/testing/src/main/java/com/akto/test_editor/filter/data_operands_impl/GreaterThanEqFilter.java b/libs/utils/src/main/java/com/akto/test_editor/filter/data_operands_impl/GreaterThanEqFilter.java similarity index 100% rename from apps/testing/src/main/java/com/akto/test_editor/filter/data_operands_impl/GreaterThanEqFilter.java rename to libs/utils/src/main/java/com/akto/test_editor/filter/data_operands_impl/GreaterThanEqFilter.java diff --git a/apps/testing/src/main/java/com/akto/test_editor/filter/data_operands_impl/GreaterThanFilter.java b/libs/utils/src/main/java/com/akto/test_editor/filter/data_operands_impl/GreaterThanFilter.java similarity index 100% rename from apps/testing/src/main/java/com/akto/test_editor/filter/data_operands_impl/GreaterThanFilter.java rename to libs/utils/src/main/java/com/akto/test_editor/filter/data_operands_impl/GreaterThanFilter.java diff --git a/apps/testing/src/main/java/com/akto/test_editor/filter/data_operands_impl/LesserThanEqFilter.java b/libs/utils/src/main/java/com/akto/test_editor/filter/data_operands_impl/LesserThanEqFilter.java similarity index 100% rename from apps/testing/src/main/java/com/akto/test_editor/filter/data_operands_impl/LesserThanEqFilter.java rename to libs/utils/src/main/java/com/akto/test_editor/filter/data_operands_impl/LesserThanEqFilter.java diff --git a/apps/testing/src/main/java/com/akto/test_editor/filter/data_operands_impl/LesserThanFilter.java b/libs/utils/src/main/java/com/akto/test_editor/filter/data_operands_impl/LesserThanFilter.java similarity index 100% rename from apps/testing/src/main/java/com/akto/test_editor/filter/data_operands_impl/LesserThanFilter.java rename to libs/utils/src/main/java/com/akto/test_editor/filter/data_operands_impl/LesserThanFilter.java diff --git a/apps/testing/src/main/java/com/akto/test_editor/filter/data_operands_impl/NeqFilter.java b/libs/utils/src/main/java/com/akto/test_editor/filter/data_operands_impl/NeqFilter.java similarity index 100% rename from apps/testing/src/main/java/com/akto/test_editor/filter/data_operands_impl/NeqFilter.java rename to libs/utils/src/main/java/com/akto/test_editor/filter/data_operands_impl/NeqFilter.java diff --git a/apps/testing/src/main/java/com/akto/test_editor/filter/data_operands_impl/NotContainsEitherFilter.java b/libs/utils/src/main/java/com/akto/test_editor/filter/data_operands_impl/NotContainsEitherFilter.java similarity index 100% rename from apps/testing/src/main/java/com/akto/test_editor/filter/data_operands_impl/NotContainsEitherFilter.java rename to libs/utils/src/main/java/com/akto/test_editor/filter/data_operands_impl/NotContainsEitherFilter.java diff --git a/apps/testing/src/main/java/com/akto/test_editor/filter/data_operands_impl/NotContainsFilter.java b/libs/utils/src/main/java/com/akto/test_editor/filter/data_operands_impl/NotContainsFilter.java similarity index 100% rename from apps/testing/src/main/java/com/akto/test_editor/filter/data_operands_impl/NotContainsFilter.java rename to libs/utils/src/main/java/com/akto/test_editor/filter/data_operands_impl/NotContainsFilter.java diff --git a/apps/testing/src/main/java/com/akto/test_editor/filter/data_operands_impl/RegexFilter.java b/libs/utils/src/main/java/com/akto/test_editor/filter/data_operands_impl/RegexFilter.java similarity index 100% rename from apps/testing/src/main/java/com/akto/test_editor/filter/data_operands_impl/RegexFilter.java rename to libs/utils/src/main/java/com/akto/test_editor/filter/data_operands_impl/RegexFilter.java diff --git a/apps/testing/src/main/java/com/akto/test_editor/filter/data_operands_impl/SsrfUrlHitFilter.java b/libs/utils/src/main/java/com/akto/test_editor/filter/data_operands_impl/SsrfUrlHitFilter.java similarity index 100% rename from apps/testing/src/main/java/com/akto/test_editor/filter/data_operands_impl/SsrfUrlHitFilter.java rename to libs/utils/src/main/java/com/akto/test_editor/filter/data_operands_impl/SsrfUrlHitFilter.java diff --git a/libs/utils/src/main/java/com/akto/test_editor/filter/data_operands_impl/ValidationResult.java b/libs/utils/src/main/java/com/akto/test_editor/filter/data_operands_impl/ValidationResult.java new file mode 100644 index 0000000000..c2204827ef --- /dev/null +++ b/libs/utils/src/main/java/com/akto/test_editor/filter/data_operands_impl/ValidationResult.java @@ -0,0 +1,19 @@ +package com.akto.test_editor.filter.data_operands_impl; + +public class ValidationResult { + public static final String GET_QUERYSET_CATCH_ERROR = "Error while parsing data"; + Boolean isValid; + String validationReason; + public ValidationResult(Boolean isValid, String validationReason) { + this.isValid = isValid; + this.validationReason = validationReason; + } + + public Boolean getIsValid() { + return isValid; + } + + public String getValidationReason() { + return validationReason; + } +} diff --git a/libs/utils/src/main/java/com/akto/testing/Utils.java b/libs/utils/src/main/java/com/akto/testing/Utils.java index c20908c3a1..31c291e2fa 100644 --- a/libs/utils/src/main/java/com/akto/testing/Utils.java +++ b/libs/utils/src/main/java/com/akto/testing/Utils.java @@ -1,7 +1,11 @@ package com.akto.testing; +import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Objects; +import java.util.Set; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -9,16 +13,25 @@ import javax.script.ScriptEngineManager; import javax.script.ScriptException; +import com.akto.dto.ApiInfo.ApiInfoKey; import com.akto.dto.OriginalHttpRequest; +import com.akto.dto.OriginalHttpResponse; +import com.akto.dto.RawApi; +import com.akto.dto.test_editor.DataOperandsFilterResponse; +import com.akto.dto.test_editor.FilterNode; import com.akto.dto.testing.WorkflowUpdatedSampleData; import com.akto.dto.type.RequestTemplate; import com.akto.log.LoggerMaker; import com.akto.log.LoggerMaker.LogDb; +import com.akto.test_editor.filter.Filter; +import com.akto.test_editor.filter.data_operands_impl.ValidationResult; import com.akto.util.JSONUtils; import com.mongodb.BasicDBObject; import okhttp3.MediaType; +import static com.akto.runtime.RuntimeUtil.extractAllValuesFromPayload;; + public class Utils { private static final LoggerMaker loggerMaker = new LoggerMaker(Utils.class); @@ -291,5 +304,69 @@ public static MediaType getMediaType(String fileUrl) { } } + + public static double compareWithOriginalResponse(String originalPayload, String currentPayload, Map comparisonExcludedKeys) { + if (originalPayload == null && currentPayload == null) return 100; + if (originalPayload == null || currentPayload == null) return 0; + + String trimmedOriginalPayload = originalPayload.trim(); + String trimmedCurrentPayload = currentPayload.trim(); + if (trimmedCurrentPayload.equals(trimmedOriginalPayload)) return 100; + + Map> originalResponseParamMap = new HashMap<>(); + Map> currentResponseParamMap = new HashMap<>(); + try { + extractAllValuesFromPayload(originalPayload, originalResponseParamMap); + extractAllValuesFromPayload(currentPayload, currentResponseParamMap); + } catch (Exception e) { + return 0.0; + } + + if (originalResponseParamMap.keySet().size() == 0 && currentResponseParamMap.keySet().size() == 0) { + return 100.0; + } + + Set visited = new HashSet<>(); + int matched = 0; + for (String k1: originalResponseParamMap.keySet()) { + if (visited.contains(k1) || comparisonExcludedKeys.containsKey(k1)) continue; + visited.add(k1); + Set v1 = originalResponseParamMap.get(k1); + Set v2 = currentResponseParamMap.get(k1); + if (Objects.equals(v1, v2)) matched +=1; + } + + for (String k1: currentResponseParamMap.keySet()) { + if (visited.contains(k1) || comparisonExcludedKeys.containsKey(k1)) continue; + visited.add(k1); + Set v1 = originalResponseParamMap.get(k1); + Set v2 = currentResponseParamMap.get(k1); + if (Objects.equals(v1, v2)) matched +=1; + } + + int visitedSize = visited.size(); + if (visitedSize == 0) return 0.0; + + double result = (100.0*matched)/visitedSize; + + if (Double.isFinite(result)) { + return result; + } else { + return 0.0; + } + + } + + public static ValidationResult validateFilter(FilterNode filterNode, RawApi rawApi, ApiInfoKey apiInfoKey, Map varMap, String logId) { + if (filterNode == null) return new ValidationResult(true, ""); + if (rawApi == null) return new ValidationResult(true, "raw api is null"); + return validate(filterNode, rawApi, null, apiInfoKey,"filter", varMap, logId); + } + + private static ValidationResult validate(FilterNode node, RawApi rawApi, RawApi testRawApi, ApiInfoKey apiInfoKey, String context, Map varMap, String logId) { + Filter filter = new Filter(); + DataOperandsFilterResponse dataOperandsFilterResponse = filter.isEndpointValid(node, rawApi, testRawApi, apiInfoKey, null, null , false,context, varMap, logId, false); + return new ValidationResult(dataOperandsFilterResponse.getResult(), dataOperandsFilterResponse.getValidationReason()); + } } From 876defbd62859b2521cff266068faac17b1b93a6 Mon Sep 17 00:00:00 2001 From: Ark2307 Date: Wed, 11 Sep 2024 14:04:03 +0530 Subject: [PATCH 2/9] Fixing efficiency for redundant urls check --- .../java/com/akto/parsers/HttpCallParser.java | 28 ++++++---------- .../java/com/akto/runtime/APICatalogSync.java | 6 +++- .../java/com/akto/parsers/TestDBSync.java | 33 ++++++++++--------- .../java/com/akto/runtime/utils/Utils.java | 16 +++++++++ 4 files changed, 49 insertions(+), 34 deletions(-) diff --git a/apps/api-runtime/src/main/java/com/akto/parsers/HttpCallParser.java b/apps/api-runtime/src/main/java/com/akto/parsers/HttpCallParser.java index a9cf86bf34..4e69dc8b80 100644 --- a/apps/api-runtime/src/main/java/com/akto/parsers/HttpCallParser.java +++ b/apps/api-runtime/src/main/java/com/akto/parsers/HttpCallParser.java @@ -25,6 +25,9 @@ import com.akto.runtime.RuntimeUtil; import com.akto.runtime.URLAggregator; import com.akto.runtime.parser.SampleParser; +import com.akto.runtime.utils.Utils; +import com.akto.test_editor.execution.VariableResolver; +import com.akto.test_editor.filter.data_operands_impl.ValidationResult; import com.akto.usage.UsageMetricCalculator; import com.akto.util.JSONUtils; import com.akto.util.http_util.CoreHTTPClient; @@ -32,8 +35,6 @@ import com.akto.util.DbMode; import com.akto.util.HttpRequestResponseUtils; import com.google.gson.Gson; -import com.akto.test_editor.execution.VariableResolver; -import com.akto.test_editor.filter.data_operands_impl.ValidationResult; import com.mongodb.BasicDBObject; import com.mongodb.client.model.*; import okhttp3.*; @@ -217,7 +218,8 @@ public void syncFunction(List responseParams, boolean syncIm if (accountSettings != null && accountSettings.getDefaultPayloads() != null) { filteredResponseParams = filterDefaultPayloads(filteredResponseParams, accountSettings.getDefaultPayloads()); } - filteredResponseParams = filterHttpResponseParams(filteredResponseParams, accountSettings); + Pattern regexPattern = Utils.createRegexPatternFromList(apiCatalogSync.ignoredEndpointsList); + filteredResponseParams = filterHttpResponseParams(filteredResponseParams, apiCatalogSync.ignoredEndpointsList, regexPattern); // add advanced filters filteredResponseParams = applyAdvancedFilters(filteredResponseParams); @@ -389,17 +391,7 @@ public void incTrafficMetrics(TrafficMetrics.Key key, int value) { public static final String CONTENT_TYPE = "CONTENT-TYPE"; - public boolean isRedundantEndpoint(String url, List discardedUrlList){ - StringJoiner joiner = new StringJoiner("|", ".*\\.(", ")(\\?.*)?"); - for (String extension : discardedUrlList) { - if(extension.startsWith(CONTENT_TYPE)){ - continue; - } - joiner.add(extension); - } - String regex = joiner.toString(); - - Pattern pattern = Pattern.compile(regex); + public boolean isRedundantEndpoint(String url, Pattern pattern){ Matcher matcher = pattern.matcher(url); return matcher.matches(); } @@ -482,7 +474,7 @@ private boolean isBlankResponseBodyForGET(String method, String contentType, Str return res; } - public List filterHttpResponseParams(List httpResponseParamsList, AccountSettings accountSettings) { + public List filterHttpResponseParams(List httpResponseParamsList, List redundantUrlsList, Pattern pattern) { List filteredResponseParams = new ArrayList<>(); int originalSize = httpResponseParamsList.size(); for (HttpResponseParams httpResponseParam: httpResponseParamsList) { @@ -503,8 +495,8 @@ public List filterHttpResponseParams(List contentTypeList = (List) httpResponseParam.getRequestParams().getHeaders().getOrDefault("content-type", new ArrayList<>()); @@ -522,7 +514,7 @@ public List filterHttpResponseParams(List sensitiveParamInfoBooleanMap; public static boolean mergeAsyncOutside = true; - public Map advancedFilterMap = new HashMap<>(); public BloomFilter existingAPIsInDb = BloomFilter.create(Funnels.stringFunnel(Charsets.UTF_8), 1_000_000, 0.001 ); + public Map advancedFilterMap = new HashMap<>(); + public List ignoredEndpointsList = new ArrayList<>(); public APICatalogSync(String userIdentifier,int thresh, boolean fetchAllSTI) { this(userIdentifier, thresh, fetchAllSTI, true); @@ -1460,6 +1461,9 @@ public void buildFromDB(boolean calcDiff, boolean fetchAllSTI) { try { List allCollections = ApiCollectionsDao.instance.getMetaAll(); AccountSettings accountSettings = AccountSettingsDao.instance.findOne(AccountSettingsDao.generateFilter()); + if(accountSettings != null && accountSettings.getAllowRedundantEndpointsList().size() > 0){ + ignoredEndpointsList = accountSettings.getAllowRedundantEndpointsList(); + } Boolean urlRegexMatchingEnabled = accountSettings == null || accountSettings.getUrlRegexMatchingEnabled(); loggerMaker.infoAndAddToDb("url regex matching enabled status is " + urlRegexMatchingEnabled, LogDb.RUNTIME); for(ApiCollection apiCollection: allCollections) { diff --git a/apps/api-runtime/src/test/java/com/akto/parsers/TestDBSync.java b/apps/api-runtime/src/test/java/com/akto/parsers/TestDBSync.java index 416e766186..c1b99d5f9a 100644 --- a/apps/api-runtime/src/test/java/com/akto/parsers/TestDBSync.java +++ b/apps/api-runtime/src/test/java/com/akto/parsers/TestDBSync.java @@ -4,6 +4,7 @@ import static org.junit.jupiter.api.Assertions.assertFalse; import java.util.*; +import java.util.regex.Pattern; import com.akto.MongoBasedTest; import com.akto.dao.*; @@ -25,6 +26,7 @@ import com.akto.runtime.APICatalogSync; import com.akto.runtime.Main; import com.akto.runtime.URLAggregator; +import com.akto.runtime.utils.Utils; import com.mongodb.BasicDBObject; import com.mongodb.client.model.Filters; @@ -254,7 +256,7 @@ public void testInitialiseFilters() throws InterruptedException { @Test public void testFilterHttpResponseParamsEmpty() { HttpCallParser httpCallParser = new HttpCallParser("",0,0,0, true); - List ss = httpCallParser.filterHttpResponseParams(new ArrayList<>(), null); + List ss = httpCallParser.filterHttpResponseParams(new ArrayList<>(), null, null); assertEquals(ss.size(),0); } @@ -278,7 +280,7 @@ public void testFilterHttpResponseParamsIpHost() { h2.requestParams.setApiCollectionId(1000); h2.setSource(Source.MIRRORING); - List ss = httpCallParser.filterHttpResponseParams(Arrays.asList(h1, h2), null); + List ss = httpCallParser.filterHttpResponseParams(Arrays.asList(h1, h2), null, null); assertEquals(ss.size(),2); assertEquals(h1.requestParams.getApiCollectionId(), 1000); assertTrue(h2.requestParams.getApiCollectionId() != 1000); @@ -302,7 +304,7 @@ public void testFilterHttpResponseParamsWithoutHost() { h1.statusCode = 200; h1.requestParams.setApiCollectionId(vxlanId1); - List filterHttpResponseParamsList = httpCallParser.filterHttpResponseParams(Collections.singletonList(h1),null); + List filterHttpResponseParamsList = httpCallParser.filterHttpResponseParams(Collections.singletonList(h1),null, null); Assertions.assertEquals(filterHttpResponseParamsList.size(),1); Assertions.assertEquals(filterHttpResponseParamsList.get(0).requestParams.getApiCollectionId(),vxlanId1); @@ -317,7 +319,7 @@ public void testFilterHttpResponseParamsWithoutHost() { h2.statusCode = 200; h2.requestParams.setApiCollectionId(vxlanId2); - filterHttpResponseParamsList = httpCallParser.filterHttpResponseParams(Collections.singletonList(h2),null); + filterHttpResponseParamsList = httpCallParser.filterHttpResponseParams(Collections.singletonList(h2),null, null); Assertions.assertEquals(filterHttpResponseParamsList.size(),1); Assertions.assertEquals(filterHttpResponseParamsList.get(0).requestParams.getApiCollectionId(),vxlanId2); @@ -332,7 +334,7 @@ public void testFilterHttpResponseParamsWithoutHost() { h3.statusCode = 400; h3.requestParams.setApiCollectionId(vxlanId2); - filterHttpResponseParamsList = httpCallParser.filterHttpResponseParams(Collections.singletonList(h3),null); + filterHttpResponseParamsList = httpCallParser.filterHttpResponseParams(Collections.singletonList(h3),null, null); Assertions.assertEquals(filterHttpResponseParamsList.size(),0); ApiCollection apiCollection3 = ApiCollectionsDao.instance.findOne("_id", vxlanId3); @@ -364,7 +366,7 @@ public void testFilterResponseParamsWithHost() { h1.statusCode = 200; h1.setSource(Source.MIRRORING); - httpCallParser.filterHttpResponseParams(Collections.singletonList(h1),null); + httpCallParser.filterHttpResponseParams(Collections.singletonList(h1),null, null); List apiCollections = ApiCollectionsDao.instance.findAll(new BasicDBObject()); Assertions.assertEquals(apiCollections.size(),2); @@ -390,7 +392,7 @@ public void testFilterResponseParamsWithHost() { h2.statusCode = 200; h2.setSource(Source.MIRRORING); - httpCallParser.filterHttpResponseParams(Collections.singletonList(h2),null); + httpCallParser.filterHttpResponseParams(Collections.singletonList(h2),null, null); apiCollections = ApiCollectionsDao.instance.findAll(new BasicDBObject()); Assertions.assertEquals(apiCollections.size(),3); @@ -414,7 +416,7 @@ public void testFilterResponseParamsWithHost() { h3.statusCode = 200; h3.setSource(Source.MIRRORING); - httpCallParser.filterHttpResponseParams(Collections.singletonList(h3),null); + httpCallParser.filterHttpResponseParams(Collections.singletonList(h3),null, null); apiCollections = ApiCollectionsDao.instance.findAll(new BasicDBObject()); Assertions.assertEquals(apiCollections.size(),4); @@ -447,7 +449,7 @@ public void testFilterResponseParamsWithHost() { ); httpCallParser.getHostNameToIdMap().put("hostRandom 1234", dupId); - httpCallParser.filterHttpResponseParams(Collections.singletonList(h4),null); + httpCallParser.filterHttpResponseParams(Collections.singletonList(h4),null, null); apiCollections = ApiCollectionsDao.instance.findAll(new BasicDBObject()); Assertions.assertEquals(apiCollections.size(),6); @@ -471,7 +473,7 @@ public void testCollisionHostNameCollection() { h1.statusCode = 200; HttpCallParser httpCallParser = new HttpCallParser("",0,0,0, true); - httpCallParser.filterHttpResponseParams(Collections.singletonList(h1),null); + httpCallParser.filterHttpResponseParams(Collections.singletonList(h1),null, null); List apiCollections = ApiCollectionsDao.instance.findAll(new BasicDBObject()); Assertions.assertEquals(apiCollections.size(), 1); @@ -589,10 +591,11 @@ public void testRedundantUrlCheck() { String url5 = "test5.js.js"; List allowedUrlType = Arrays.asList("js"); - assertEquals(httpCallParser.isRedundantEndpoint(url1, allowedUrlType), true); - assertEquals(httpCallParser.isRedundantEndpoint(url2, allowedUrlType), false); - assertEquals(httpCallParser.isRedundantEndpoint(url3, allowedUrlType), false); - assertEquals(httpCallParser.isRedundantEndpoint(url4, allowedUrlType), true); - assertEquals(httpCallParser.isRedundantEndpoint(url5, allowedUrlType), true); + Pattern pattern = Utils.createRegexPatternFromList(allowedUrlType); + assertEquals(httpCallParser.isRedundantEndpoint(url1, pattern), true); + assertEquals(httpCallParser.isRedundantEndpoint(url2, pattern), false); + assertEquals(httpCallParser.isRedundantEndpoint(url3, pattern), false); + assertEquals(httpCallParser.isRedundantEndpoint(url4, pattern), true); + assertEquals(httpCallParser.isRedundantEndpoint(url5, pattern), true); } } diff --git a/libs/utils/src/main/java/com/akto/runtime/utils/Utils.java b/libs/utils/src/main/java/com/akto/runtime/utils/Utils.java index 380ef76ef2..3df7d31e73 100644 --- a/libs/utils/src/main/java/com/akto/runtime/utils/Utils.java +++ b/libs/utils/src/main/java/com/akto/runtime/utils/Utils.java @@ -4,6 +4,8 @@ import java.util.List; import java.util.Map; import java.util.Properties; +import java.util.StringJoiner; +import java.util.regex.Pattern; import org.apache.commons.lang3.math.NumberUtils; import org.apache.kafka.clients.consumer.ConsumerConfig; @@ -152,6 +154,20 @@ public static HttpResponseParams parseKafkaMessage(String message) throws Except ); } + public static Pattern createRegexPatternFromList(List discardedUrlList){ + StringJoiner joiner = new StringJoiner("|", ".*\\.(", ")(\\?.*)?"); + for (String extension : discardedUrlList) { + if(extension.startsWith("CONTENT-TYPE")){ + continue; + } + joiner.add(extension); + } + String regex = joiner.toString(); + + Pattern pattern = Pattern.compile(regex); + return pattern; + } + } From 23d118aab7fb0ea4632d6e5186c6d908571017cf Mon Sep 17 00:00:00 2001 From: Ark2307 Date: Thu, 12 Sep 2024 09:10:14 +0530 Subject: [PATCH 3/9] Fixing redundant urls check --- .../src/main/java/com/akto/parsers/HttpCallParser.java | 8 ++++++-- .../src/main/java/com/akto/runtime/APICatalogSync.java | 4 ---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/apps/api-runtime/src/main/java/com/akto/parsers/HttpCallParser.java b/apps/api-runtime/src/main/java/com/akto/parsers/HttpCallParser.java index 4e69dc8b80..46fd0d3f6b 100644 --- a/apps/api-runtime/src/main/java/com/akto/parsers/HttpCallParser.java +++ b/apps/api-runtime/src/main/java/com/akto/parsers/HttpCallParser.java @@ -218,8 +218,12 @@ public void syncFunction(List responseParams, boolean syncIm if (accountSettings != null && accountSettings.getDefaultPayloads() != null) { filteredResponseParams = filterDefaultPayloads(filteredResponseParams, accountSettings.getDefaultPayloads()); } - Pattern regexPattern = Utils.createRegexPatternFromList(apiCatalogSync.ignoredEndpointsList); - filteredResponseParams = filterHttpResponseParams(filteredResponseParams, apiCatalogSync.ignoredEndpointsList, regexPattern); + List redundantList = new ArrayList<>(); + if(accountSettings !=null && !accountSettings.getAllowRedundantEndpointsList().isEmpty()){ + redundantList = accountSettings.getAllowRedundantEndpointsList(); + } + Pattern regexPattern = Utils.createRegexPatternFromList(redundantList); + filteredResponseParams = filterHttpResponseParams(filteredResponseParams, redundantList, regexPattern); // add advanced filters filteredResponseParams = applyAdvancedFilters(filteredResponseParams); diff --git a/apps/api-runtime/src/main/java/com/akto/runtime/APICatalogSync.java b/apps/api-runtime/src/main/java/com/akto/runtime/APICatalogSync.java index 7ff1d4c82e..012546bc8e 100644 --- a/apps/api-runtime/src/main/java/com/akto/runtime/APICatalogSync.java +++ b/apps/api-runtime/src/main/java/com/akto/runtime/APICatalogSync.java @@ -71,7 +71,6 @@ public class APICatalogSync { public static boolean mergeAsyncOutside = true; public BloomFilter existingAPIsInDb = BloomFilter.create(Funnels.stringFunnel(Charsets.UTF_8), 1_000_000, 0.001 ); public Map advancedFilterMap = new HashMap<>(); - public List ignoredEndpointsList = new ArrayList<>(); public APICatalogSync(String userIdentifier,int thresh, boolean fetchAllSTI) { this(userIdentifier, thresh, fetchAllSTI, true); @@ -1461,9 +1460,6 @@ public void buildFromDB(boolean calcDiff, boolean fetchAllSTI) { try { List allCollections = ApiCollectionsDao.instance.getMetaAll(); AccountSettings accountSettings = AccountSettingsDao.instance.findOne(AccountSettingsDao.generateFilter()); - if(accountSettings != null && accountSettings.getAllowRedundantEndpointsList().size() > 0){ - ignoredEndpointsList = accountSettings.getAllowRedundantEndpointsList(); - } Boolean urlRegexMatchingEnabled = accountSettings == null || accountSettings.getUrlRegexMatchingEnabled(); loggerMaker.infoAndAddToDb("url regex matching enabled status is " + urlRegexMatchingEnabled, LogDb.RUNTIME); for(ApiCollection apiCollection: allCollections) { From 6d96f0b2fe341108574d15a1ba3bcf3fb00a82ef Mon Sep 17 00:00:00 2001 From: Ark2307 Date: Sun, 15 Sep 2024 17:11:49 +0530 Subject: [PATCH 4/9] Fixed build errors in mini-testing and added filters in call parser --- .../java/com/akto/utils/CustomAuthUtil.java | 4 +- .../AdvancedTrafficFiltersAction.java | 18 +- .../com/akto/utils/TrafficFilterUtil.java | 57 ++++++ .../akto/hybrid_parsers/HttpCallParser.java | 165 ++++++++++++------ .../akto/hybrid_runtime/APICatalogSync.java | 8 + .../main/java/com/akto/rules/TestPlugin.java | 9 +- .../com/akto/test_editor/execution/Build.java | 4 +- .../akto/test_editor/execution/Executor.java | 3 +- .../akto/test_editor/filter/FilterAction.java | 8 +- .../CookieExpireFilter.java | 4 +- .../ApiNodeExecutor.java | 5 +- .../YamlNodeExecutor.java | 5 +- .../testing/yaml_tests/YamlTestTemplate.java | 5 +- .../DataOperandsFilterResponse.java | 17 ++ .../java/com/akto/data_actor/ClientActor.java | 32 ++++ .../java/com/akto/runtime/utils/Utils.java | 69 -------- .../execution/VariableResolver.java | 4 +- .../akto/test_editor/filter/FilterAction.java | 7 +- 18 files changed, 266 insertions(+), 158 deletions(-) create mode 100644 apps/dashboard/src/main/java/com/akto/utils/TrafficFilterUtil.java diff --git a/apps/api-runtime/src/main/java/com/akto/utils/CustomAuthUtil.java b/apps/api-runtime/src/main/java/com/akto/utils/CustomAuthUtil.java index 3a70e4b03b..771a123515 100644 --- a/apps/api-runtime/src/main/java/com/akto/utils/CustomAuthUtil.java +++ b/apps/api-runtime/src/main/java/com/akto/utils/CustomAuthUtil.java @@ -18,9 +18,9 @@ import com.akto.dto.type.SingleTypeInfo; import com.akto.log.LoggerMaker; import com.akto.log.LoggerMaker.LogDb; -import com.akto.runtime.policies.AuthPolicy; import static com.akto.dto.ApiInfo.ALL_AUTH_TYPES_FOUND; +import static com.akto.runtime.utils.Utils.parseCookie; public class CustomAuthUtil { @@ -97,7 +97,7 @@ public static void customAuthTypeUtil(List customAuthTypes){ } SingleTypeInfo cookieSTI = SingleTypeInfoDao.instance.findOne(getFilters(apiInfo, true, COOKIE_LIST)); if(cookieSTI!=null){ - Map cookieMap = AuthPolicy.parseCookie(new ArrayList<>(cookieSTI.getValues().getElements())); + Map cookieMap = parseCookie(new ArrayList<>(cookieSTI.getValues().getElements())); headerAndCookieKeys.addAll(cookieMap.keySet()); } diff --git a/apps/dashboard/src/main/java/com/akto/action/settings/AdvancedTrafficFiltersAction.java b/apps/dashboard/src/main/java/com/akto/action/settings/AdvancedTrafficFiltersAction.java index 8438e6708b..a42b753477 100644 --- a/apps/dashboard/src/main/java/com/akto/action/settings/AdvancedTrafficFiltersAction.java +++ b/apps/dashboard/src/main/java/com/akto/action/settings/AdvancedTrafficFiltersAction.java @@ -1,34 +1,36 @@ package com.akto.action.settings; import java.util.List; -import java.util.Map; import org.bson.conversions.Bson; import com.akto.action.UserAction; import com.akto.dao.monitoring.FilterConfigYamlParser; -import com.akto.dao.monitoring.FilterYamlTemplateDao; import com.akto.dao.runtime_filters.AdvancedTrafficFiltersDao; import com.akto.dto.monitoring.FilterConfig; import com.akto.dto.test_editor.YamlTemplate; import com.akto.util.Constants; import com.akto.utils.TrafficFilterUtil; -import com.mongodb.BasicDBList; import com.mongodb.client.model.Filters; import com.mongodb.client.model.FindOneAndUpdateOptions; +import com.mongodb.client.model.Projections; import com.mongodb.client.model.Updates; public class AdvancedTrafficFiltersAction extends UserAction { - private BasicDBList templatesList; + private List templatesList; private String yamlContent; private String templateId; private boolean inactive; public String fetchAllFilterTemplates(){ - List yamlTemplates = AdvancedTrafficFiltersDao.instance.findAll(Filters.empty()); - Map configs = FilterYamlTemplateDao.instance.fetchFilterConfig(true, yamlTemplates, false); - this.templatesList = TrafficFilterUtil.getFilterTemplates(configs); + List yamlTemplates = AdvancedTrafficFiltersDao.instance.findAll( + Filters.empty(), + Projections.include( + Constants.ID, YamlTemplate.AUTHOR, YamlTemplate.CONTENT, YamlTemplate.INACTIVE + ) + ); + this.templatesList = yamlTemplates; return SUCCESS.toUpperCase(); } @@ -74,7 +76,7 @@ public String deleteAdvancedFilter(){ return SUCCESS.toUpperCase(); } - public BasicDBList getTemplatesList() { + public List getTemplatesList() { return templatesList; } diff --git a/apps/dashboard/src/main/java/com/akto/utils/TrafficFilterUtil.java b/apps/dashboard/src/main/java/com/akto/utils/TrafficFilterUtil.java new file mode 100644 index 0000000000..980eef9084 --- /dev/null +++ b/apps/dashboard/src/main/java/com/akto/utils/TrafficFilterUtil.java @@ -0,0 +1,57 @@ +package com.akto.utils; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; + +import org.bson.conversions.Bson; + +import com.akto.dao.context.Context; +import com.akto.dto.monitoring.FilterConfig; +import com.akto.dto.test_editor.YamlTemplate; +import com.akto.util.enums.GlobalEnums.YamlTemplateSource; +import com.mongodb.BasicDBList; +import com.mongodb.BasicDBObject; +import com.mongodb.client.model.Updates; + +public class TrafficFilterUtil { + public static BasicDBList getFilterTemplates(Map configs){ + BasicDBList templates = new BasicDBList(); + if (configs != null && !configs.isEmpty()) { + for (Entry apiFilterEntry : configs.entrySet()) { + FilterConfig config = apiFilterEntry.getValue(); + BasicDBObject template = new BasicDBObject(); + template.append(FilterConfig.ID, config.getId()); + template.append(FilterConfig._CONTENT, config.getContent()); + template.append(FilterConfig._AUTHOR, config.getAuthor()); + template.append(FilterConfig.CREATED_AT, config.getCreatedAt()); + template.append(FilterConfig.UPDATED_AT, config.getUpdatedAt()); + templates.add(template); + } + } + + return templates; + } + + public static List getDbUpdateForTemplate(String content, String userEmail) throws Exception{ + try { + String author = userEmail; + int createdAt = Context.now(); + int updatedAt = Context.now(); + + List updates = new ArrayList<>( + Arrays.asList( + Updates.setOnInsert(YamlTemplate.CREATED_AT, createdAt), + Updates.setOnInsert(YamlTemplate.AUTHOR, author), + Updates.set(YamlTemplate.UPDATED_AT, updatedAt), + Updates.set(YamlTemplate.CONTENT, content), + Updates.setOnInsert(YamlTemplate.SOURCE, YamlTemplateSource.CUSTOM))); + return updates; + + } catch (Exception e) { + throw e; + } + } +} diff --git a/apps/mini-runtime/src/main/java/com/akto/hybrid_parsers/HttpCallParser.java b/apps/mini-runtime/src/main/java/com/akto/hybrid_parsers/HttpCallParser.java index de0181b428..33a019217d 100644 --- a/apps/mini-runtime/src/main/java/com/akto/hybrid_parsers/HttpCallParser.java +++ b/apps/mini-runtime/src/main/java/com/akto/hybrid_parsers/HttpCallParser.java @@ -11,17 +11,24 @@ import com.akto.data_actor.DataActorFactory; import com.akto.dto.billing.FeatureAccess; import com.akto.dto.*; +import com.akto.dto.ApiInfo.ApiInfoKey; import com.akto.dto.billing.Organization; import com.akto.dto.billing.SyncLimit; import com.akto.dto.bulk_updates.BulkUpdates; import com.akto.dto.bulk_updates.UpdatePayload; +import com.akto.dto.monitoring.FilterConfig; import com.akto.dto.settings.DefaultPayload; import com.akto.dto.traffic_metrics.TrafficMetrics; +import com.akto.dto.type.URLMethods.Method; import com.akto.dto.usage.MetricTypes; import com.akto.graphql.GraphQLUtils; import com.akto.log.LoggerMaker; import com.akto.log.LoggerMaker.LogDb; +import com.akto.runtime.RuntimeUtil; import com.akto.runtime.parser.SampleParser; +import com.akto.runtime.utils.Utils; +import com.akto.test_editor.execution.VariableResolver; +import com.akto.test_editor.filter.data_operands_impl.ValidationResult; import com.akto.hybrid_runtime.APICatalogSync; import com.akto.hybrid_runtime.Main; import com.akto.hybrid_runtime.MergeLogicLocal; @@ -33,11 +40,10 @@ import com.mongodb.BasicDBObject; import com.mongodb.client.model.*; import okhttp3.*; -import org.apache.commons.lang3.math.NumberUtils; -import org.bson.conversions.Bson; import java.io.IOException; import java.util.*; +import java.util.Map.Entry; import java.util.concurrent.ConcurrentLinkedQueue; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; @@ -47,6 +53,7 @@ import java.util.regex.Pattern; import static com.akto.runtime.RuntimeUtil.matchesDefaultPayload; +import static com.akto.testing.Utils.validateFilter; public class HttpCallParser { private final int sync_threshold_count; @@ -192,6 +199,48 @@ private SyncLimit fetchSyncLimit() { return syncLimit; } + private List applyAdvancedFilters(List responseParams){ + Map filterMap = apiCatalogSync.advancedFilterMap; + + if (filterMap != null && !filterMap.isEmpty()) { + List filteredParams = new ArrayList<>(); + for (HttpResponseParams responseParam : responseParams) { + for (Entry apiFilterEntry : filterMap.entrySet()) { + try { + FilterConfig apiFilter = apiFilterEntry.getValue(); + String message = responseParam.getOrig(); + RawApi rawApi = RawApi.buildFromMessage(message); + int apiCollectionId = createApiCollectionId(responseParam); + responseParam.requestParams.setApiCollectionId(apiCollectionId); + String url = responseParam.getRequestParams().getURL(); + Method method = Method.valueOf(responseParam.getRequestParams().getMethod()); + ApiInfoKey apiInfoKey = new ApiInfoKey(apiCollectionId, url, method); + Map varMap = apiFilter.resolveVarMap(); + VariableResolver.resolveWordList(varMap, new HashMap>() { + { + put(apiInfoKey, Arrays.asList(message)); + } + }, apiInfoKey); + String filterExecutionLogId = UUID.randomUUID().toString(); + ValidationResult res = validateFilter(apiFilter.getFilter().getNode(), rawApi, + apiInfoKey, varMap, filterExecutionLogId); + if (res.getIsValid()) { + // TODO apply execution + + filteredParams.add(responseParam); + break; + } + } catch (Exception e) { + loggerMaker.errorAndAddToDb(e, String.format("Error in httpCallFilter %s", e.toString())); + return responseParams; + } + } + } + return filteredParams; + } + return responseParams; + } + public void syncFunction(List responseParams, boolean syncImmediately, boolean fetchAllSTI, AccountSettings accountSettings) { // USE ONLY filteredResponseParams and not responseParams List filteredResponseParams = responseParams; @@ -199,6 +248,7 @@ public void syncFunction(List responseParams, boolean syncIm filteredResponseParams = filterDefaultPayloads(filteredResponseParams, accountSettings.getDefaultPayloads()); } filteredResponseParams = filterHttpResponseParams(filteredResponseParams, accountSettings); + filteredResponseParams = applyAdvancedFilters(filteredResponseParams); boolean isHarOrPcap = aggregate(filteredResponseParams, aggregatorMap); for (int apiCollectionId: aggregatorMap.keySet()) { @@ -389,17 +439,7 @@ public void incTrafficMetrics(TrafficMetrics.Key key, int value) { public static final String CONTENT_TYPE = "CONTENT-TYPE"; - public boolean isRedundantEndpoint(String url, List discardedUrlList){ - StringJoiner joiner = new StringJoiner("|", ".*\\.(", ")(\\?.*)?"); - for (String extension : discardedUrlList) { - if(extension.startsWith(CONTENT_TYPE)){ - continue; - } - joiner.add(extension); - } - String regex = joiner.toString(); - - Pattern pattern = Pattern.compile(regex); + public boolean isRedundantEndpoint(String url, Pattern pattern){ Matcher matcher = pattern.matcher(url); return matcher.matches(); } @@ -412,6 +452,54 @@ private boolean isInvalidContentType(String contentType){ return res; } + public int createApiCollectionId(HttpResponseParams httpResponseParam){ + int apiCollectionId; + String hostName = getHeaderValue(httpResponseParam.getRequestParams().getHeaders(), "host"); + + if (hostName != null && !hostNameToIdMap.containsKey(hostName) && RuntimeUtil.hasSpecialCharacters(hostName)) { + hostName = "Special_Char_Host"; + } + + int vxlanId = httpResponseParam.requestParams.getApiCollectionId(); + + if (useHostCondition(hostName, httpResponseParam.getSource())) { + hostName = hostName.toLowerCase(); + hostName = hostName.trim(); + + String key = hostName; + + if (hostNameToIdMap.containsKey(key)) { + apiCollectionId = hostNameToIdMap.get(key); + + } else { + int id = hostName.hashCode(); + try { + + apiCollectionId = createCollectionBasedOnHostName(id, hostName); + + hostNameToIdMap.put(key, apiCollectionId); + } catch (Exception e) { + loggerMaker.errorAndAddToDb("Failed to create collection for host : " + hostName, LogDb.RUNTIME); + createCollectionSimple(vxlanId); + hostNameToIdMap.put("null " + vxlanId, vxlanId); + apiCollectionId = httpResponseParam.requestParams.getApiCollectionId(); + } + } + + } else { + String key = "null" + " " + vxlanId; + if (!hostNameToIdMap.containsKey(key)) { + createCollectionSimple(vxlanId); + hostNameToIdMap.put(key, vxlanId); + } + + apiCollectionId = vxlanId; + } + return apiCollectionId; + } + + + private boolean isBlankResponseBodyForGET(String method, String contentType, String matchContentType, String responseBody) { boolean res = true; @@ -437,6 +525,13 @@ private boolean isBlankResponseBodyForGET(String method, String contentType, Str public List filterHttpResponseParams(List httpResponseParamsList, AccountSettings accountSettings) { List filteredResponseParams = new ArrayList<>(); int originalSize = httpResponseParamsList.size(); + + List redundantList = new ArrayList<>(); + if(accountSettings !=null && !accountSettings.getAllowRedundantEndpointsList().isEmpty()){ + redundantList = accountSettings.getAllowRedundantEndpointsList(); + } + Pattern regexPattern = Utils.createRegexPatternFromList(redundantList); + for (HttpResponseParams httpResponseParam: httpResponseParamsList) { if (httpResponseParam.getSource().equals(HttpResponseParams.Source.MIRRORING)) { @@ -459,8 +554,8 @@ public List filterHttpResponseParams(List contentTypeList = (List) httpResponseParam.getRequestParams().getHeaders().getOrDefault("content-type", new ArrayList<>()); @@ -497,45 +592,7 @@ public List filterHttpResponseParams(List existingAPIsInDb = BloomFilter.create(Funnels.stringFunnel(Charsets.UTF_8), 1_000_000, 0.001 ); + public Map advancedFilterMap = new HashMap<>(); private static DataActor dataActor = DataActorFactory.fetchInstance(); @@ -1105,6 +1110,9 @@ public void buildFromDB(boolean calcDiff, boolean fetchAllSTI) { this.sensitiveParamInfoBooleanMap.put(sensitiveParamInfo, false); } + List advancedFilterTemplates = dataActor.fetchActiveAdvancedFilters(); + advancedFilterMap = FilterYamlTemplateDao.instance.fetchFilterConfig(false, advancedFilterTemplates, true); + try { loggerMaker.infoAndAddToDb("Started clearing values in db ", LogDb.RUNTIME); clearValuesInDB(); diff --git a/apps/mini-testing/src/main/java/com/akto/rules/TestPlugin.java b/apps/mini-testing/src/main/java/com/akto/rules/TestPlugin.java index 8627fc8963..bb5756bb6b 100644 --- a/apps/mini-testing/src/main/java/com/akto/rules/TestPlugin.java +++ b/apps/mini-testing/src/main/java/com/akto/rules/TestPlugin.java @@ -15,7 +15,6 @@ import com.akto.store.TestingUtil; import com.akto.test_editor.filter.Filter; import com.akto.testing.StatusCodeAnalyser; -import com.akto.utils.RedactSampleData; import com.fasterxml.jackson.core.JsonFactory; import com.fasterxml.jackson.core.JsonParser; import com.fasterxml.jackson.databind.JsonNode; @@ -28,6 +27,8 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; +import static com.akto.runtime.utils.Utils.convertOriginalReqRespToString; + public abstract class TestPlugin { static ObjectMapper mapper = new ObjectMapper(); @@ -144,7 +145,7 @@ public Result addWithoutRequestError(String originalMessage, TestResult.TestErro public TestResult buildFailedTestResultWithOriginalMessage(String originalMessage, TestResult.TestError testError, OriginalHttpRequest request, TestInfo testInfo) { String message = null; try { - message = RedactSampleData.convertOriginalReqRespToString(request, null); + message = convertOriginalReqRespToString(request, null); } catch (Exception e) { loggerMaker.errorAndAddToDb("Error while converting testRequest object to string : " + e, LogDb.TESTING); } @@ -165,11 +166,11 @@ public TestResult buildTestResult(OriginalHttpRequest request, OriginalHttpRespo List errors = new ArrayList<>(); String message = null; try { - message = RedactSampleData.convertOriginalReqRespToString(request, response); + message = convertOriginalReqRespToString(request, response); } catch (Exception e) { // TODO: logger.error("Error while converting OriginalHttpRequest to string", e); - message = RedactSampleData.convertOriginalReqRespToString(new OriginalHttpRequest(), new OriginalHttpResponse()); + message = convertOriginalReqRespToString(new OriginalHttpRequest(), new OriginalHttpResponse()); errors.add(TestResult.TestError.FAILED_TO_CONVERT_TEST_REQUEST_TO_STRING.getMessage()); } diff --git a/apps/mini-testing/src/main/java/com/akto/test_editor/execution/Build.java b/apps/mini-testing/src/main/java/com/akto/test_editor/execution/Build.java index 3ecab13e43..8d4b50bb4d 100644 --- a/apps/mini-testing/src/main/java/com/akto/test_editor/execution/Build.java +++ b/apps/mini-testing/src/main/java/com/akto/test_editor/execution/Build.java @@ -13,7 +13,6 @@ import com.akto.dto.traffic.SampleData; import com.akto.dto.type.URLMethods; import com.akto.log.LoggerMaker; -import com.akto.runtime.policies.AuthPolicy; import com.akto.testing.ApiExecutor; import com.akto.util.Constants; import com.akto.util.HttpRequestResponseUtils; @@ -30,6 +29,7 @@ import static com.akto.util.HttpRequestResponseUtils.FORM_URL_ENCODED_CONTENT_TYPE; import static com.akto.util.HttpRequestResponseUtils.extractValuesFromPayload; +import static com.akto.runtime.utils.Utils.parseCookie; public class Build { @@ -408,7 +408,7 @@ public static Map> getValuesMap(OriginalHttpResponse respons if (values == null) continue; if (headerKey.equalsIgnoreCase("set-cookie")) { - Map cookieMap = AuthPolicy.parseCookie(values); + Map cookieMap = parseCookie(values); for (String cookieKey : cookieMap.keySet()) { String cookieVal = cookieMap.get(cookieKey); valuesMap.put(cookieKey, new HashSet<>(Collections.singletonList(cookieVal))); diff --git a/apps/mini-testing/src/main/java/com/akto/test_editor/execution/Executor.java b/apps/mini-testing/src/main/java/com/akto/test_editor/execution/Executor.java index 7b9ca5f9ad..724e3228ac 100644 --- a/apps/mini-testing/src/main/java/com/akto/test_editor/execution/Executor.java +++ b/apps/mini-testing/src/main/java/com/akto/test_editor/execution/Executor.java @@ -41,6 +41,7 @@ import com.mongodb.BasicDBObject; import static com.akto.test_editor.Utils.bodyValuesUnchanged; import static com.akto.test_editor.Utils.headerValuesUnchanged; +import static com.akto.runtime.utils.Utils.convertOriginalReqRespToString; import org.apache.commons.lang3.StringUtils; @@ -365,7 +366,7 @@ public TestResult validate(ExecutionResult attempt, RawApi rawApi, Map cookieList = headers.getOrDefault(key, new ArrayList<>()); - Map cookieMap = AuthPolicy.parseCookie(cookieList); + Map cookieMap = parseCookie(cookieList); for (String cookieKey : cookieMap.keySet()) { dataOperandFilterRequest = new DataOperandFilterRequest(cookieKey, filterActionRequest.getQuerySet(), filterActionRequest.getOperand()); res = invokeFilter(dataOperandFilterRequest); @@ -596,7 +594,7 @@ public DataOperandsFilterResponse applyFiltersOnHeaders(FilterActionRequest filt if (!res && (key.equals("cookie") || key.equals("set-cookie"))) { List cookieList = headers.getOrDefault("cookie", new ArrayList<>()); - Map cookieMap = AuthPolicy.parseCookie(cookieList); + Map cookieMap = parseCookie(cookieList); for (String cookieKey : cookieMap.keySet()) { DataOperandFilterRequest dataOperandFilterRequest = new DataOperandFilterRequest(cookieMap.get(cookieKey), filterActionRequest.getQuerySet(), filterActionRequest.getOperand()); res = invokeFilter(dataOperandFilterRequest); diff --git a/apps/mini-testing/src/main/java/com/akto/test_editor/filter/data_operands_impl/CookieExpireFilter.java b/apps/mini-testing/src/main/java/com/akto/test_editor/filter/data_operands_impl/CookieExpireFilter.java index 0edce0afeb..fd031d6bd1 100644 --- a/apps/mini-testing/src/main/java/com/akto/test_editor/filter/data_operands_impl/CookieExpireFilter.java +++ b/apps/mini-testing/src/main/java/com/akto/test_editor/filter/data_operands_impl/CookieExpireFilter.java @@ -10,7 +10,7 @@ import java.util.Map; import com.akto.dto.test_editor.DataOperandFilterRequest; -import com.akto.runtime.policies.AuthPolicy; +import static com.akto.runtime.utils.Utils.parseCookie; public class CookieExpireFilter extends DataOperandsImpl { @@ -33,7 +33,7 @@ public Boolean isValid(DataOperandFilterRequest dataOperandFilterRequest) { return false; } - Map cookieMap = AuthPolicy.parseCookie(Arrays.asList(data)); + Map cookieMap = parseCookie(Arrays.asList(data)); boolean result = queryVal; boolean res = false; diff --git a/apps/mini-testing/src/main/java/com/akto/testing/workflow_node_executor/ApiNodeExecutor.java b/apps/mini-testing/src/main/java/com/akto/testing/workflow_node_executor/ApiNodeExecutor.java index 92645aa6c6..1ebf62bdb5 100644 --- a/apps/mini-testing/src/main/java/com/akto/testing/workflow_node_executor/ApiNodeExecutor.java +++ b/apps/mini-testing/src/main/java/com/akto/testing/workflow_node_executor/ApiNodeExecutor.java @@ -20,7 +20,8 @@ import com.akto.testing.ApiExecutor; import com.akto.testing.Main; import com.akto.testing.Utils; -import com.akto.utils.RedactSampleData; + +import static com.akto.runtime.utils.Utils.convertOriginalReqRespToString; public class ApiNodeExecutor extends NodeExecutor { @@ -94,7 +95,7 @@ public NodeResult processNode(Node node, Map valuesMap, Boolean String message = null; try { - message = RedactSampleData.convertOriginalReqRespToString(request, response); + message = convertOriginalReqRespToString(request, response); } catch (Exception e) { ; } diff --git a/apps/mini-testing/src/main/java/com/akto/testing/workflow_node_executor/YamlNodeExecutor.java b/apps/mini-testing/src/main/java/com/akto/testing/workflow_node_executor/YamlNodeExecutor.java index 5ac37218a1..d38e4499ba 100644 --- a/apps/mini-testing/src/main/java/com/akto/testing/workflow_node_executor/YamlNodeExecutor.java +++ b/apps/mini-testing/src/main/java/com/akto/testing/workflow_node_executor/YamlNodeExecutor.java @@ -45,9 +45,10 @@ import com.akto.testing.ApiExecutor; import com.akto.testing.TestExecutor; import com.akto.util.Constants; -import com.akto.utils.RedactSampleData; import com.google.gson.Gson; +import static com.akto.runtime.utils.Utils.convertOriginalReqRespToString; + public class YamlNodeExecutor extends NodeExecutor { private static final Gson gson = new Gson(); @@ -163,7 +164,7 @@ public NodeResult processNode(Node node, Map varMap, Boolean all } vulnerable = res.getVulnerable(); try { - message.add(RedactSampleData.convertOriginalReqRespToString(testReq.getRequest(), testResponse)); + message.add(convertOriginalReqRespToString(testReq.getRequest(), testResponse)); } catch (Exception e) { ; } diff --git a/apps/testing/src/main/java/com/akto/testing/yaml_tests/YamlTestTemplate.java b/apps/testing/src/main/java/com/akto/testing/yaml_tests/YamlTestTemplate.java index 57294919c5..8712f4924a 100644 --- a/apps/testing/src/main/java/com/akto/testing/yaml_tests/YamlTestTemplate.java +++ b/apps/testing/src/main/java/com/akto/testing/yaml_tests/YamlTestTemplate.java @@ -13,6 +13,7 @@ import com.akto.test_editor.Utils; import com.akto.test_editor.auth.AuthValidator; import com.akto.test_editor.execution.Executor; +import com.akto.test_editor.filter.data_operands_impl.ValidationResult; import com.akto.testing.StatusCodeAnalyser; import java.util.ArrayList; @@ -104,9 +105,9 @@ public boolean filter() { return false; } } - boolean isValid = TestPlugin.validateFilter(this.getFilterNode(),this.getRawApi(), this.getApiInfoKey(), this.varMap, this.logId); + ValidationResult isValid = TestPlugin.validateFilter(this.getFilterNode(),this.getRawApi(), this.getApiInfoKey(), this.varMap, this.logId); // loggerMaker.infoAndAddToDb("filter status " + isValid + " " + logId, LogDb.TESTING); - return isValid; + return isValid.getIsValid(); } diff --git a/libs/dao/src/main/java/com/akto/dto/test_editor/DataOperandsFilterResponse.java b/libs/dao/src/main/java/com/akto/dto/test_editor/DataOperandsFilterResponse.java index 8b49a58203..3a0b3008d2 100644 --- a/libs/dao/src/main/java/com/akto/dto/test_editor/DataOperandsFilterResponse.java +++ b/libs/dao/src/main/java/com/akto/dto/test_editor/DataOperandsFilterResponse.java @@ -10,12 +10,22 @@ public class DataOperandsFilterResponse { private List matchedEntities; private List contextEntities; private FilterNode extractNode; + private String validationReason; public DataOperandsFilterResponse(Boolean result, List matchedEntities, List contextEntities, FilterNode extractNode) { this.result = result; this.matchedEntities = matchedEntities; this.contextEntities = contextEntities; this.extractNode = extractNode; + this.validationReason = null; + } + + public DataOperandsFilterResponse(Boolean result, List matchedEntities, List contextEntities, FilterNode extractNode, String validationReason) { + this.result = result; + this.matchedEntities = matchedEntities; + this.contextEntities = contextEntities; + this.extractNode = extractNode; + this.validationReason = validationReason; } public DataOperandsFilterResponse() { } @@ -52,4 +62,11 @@ public void setExtractNode(FilterNode extractNode) { this.extractNode = extractNode; } + public String getValidationReason() { + return validationReason; + } + + public void setValidationReason(String validationReason) { + this.validationReason = validationReason; + } } diff --git a/libs/utils/src/main/java/com/akto/data_actor/ClientActor.java b/libs/utils/src/main/java/com/akto/data_actor/ClientActor.java index b55bcebfff..21f51cd139 100644 --- a/libs/utils/src/main/java/com/akto/data_actor/ClientActor.java +++ b/libs/utils/src/main/java/com/akto/data_actor/ClientActor.java @@ -3023,4 +3023,36 @@ public List fetchLatestEndpointsForTesting(int startTimestam return respList; } + public List fetchActiveAdvancedFilters(){ + Map> headers = buildHeaders(); + + List respList = new ArrayList<>(); + OriginalHttpRequest request = new OriginalHttpRequest(url + "/fetchActiveAdvancedFilters", "", "POST", "", headers, ""); + try { + OriginalHttpResponse response = ApiExecutor.sendRequestBackOff(request, true, null, false, null); + String responsePayload = response.getBody(); + if (response.getStatusCode() != 200 || responsePayload == null) { + loggerMaker.errorAndAddToDb("non 2xx response in fetchActiveAdvancedFilters", LoggerMaker.LogDb.RUNTIME); + return null; + } + BasicDBObject payloadObj; + + try { + payloadObj = BasicDBObject.parse(responsePayload); + BasicDBList newTemplates = (BasicDBList) payloadObj.get("activeAdvancedFilters"); + for (Object template: newTemplates) { + BasicDBObject templateObject = (BasicDBObject) template; + YamlTemplate yamlTemplate = objectMapper.readValue(templateObject.toJson(), YamlTemplate.class); + respList.add(yamlTemplate); + } + } catch (Exception e) { + loggerMaker.errorAndAddToDb("error extracting response in fetchActiveAdvancedFilters" + e, LoggerMaker.LogDb.RUNTIME); + } + } catch (Exception e) { + loggerMaker.errorAndAddToDb("error in fetching filter yaml templates" + e, LoggerMaker.LogDb.RUNTIME); + return null; + } + return respList; + } + } diff --git a/libs/utils/src/main/java/com/akto/runtime/utils/Utils.java b/libs/utils/src/main/java/com/akto/runtime/utils/Utils.java index 3df7d31e73..6c33097b23 100644 --- a/libs/utils/src/main/java/com/akto/runtime/utils/Utils.java +++ b/libs/utils/src/main/java/com/akto/runtime/utils/Utils.java @@ -7,26 +7,16 @@ import java.util.StringJoiner; import java.util.regex.Pattern; -import org.apache.commons.lang3.math.NumberUtils; import org.apache.kafka.clients.consumer.ConsumerConfig; import org.apache.kafka.common.serialization.StringDeserializer; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.akto.dto.HttpRequestParams; -import com.akto.dto.HttpResponseParams; import com.akto.dto.OriginalHttpRequest; import com.akto.dto.OriginalHttpResponse; -import com.akto.log.LoggerMaker.LogDb; -import com.akto.util.HttpRequestResponseUtils; -import com.akto.util.JSONUtils; -import com.alibaba.fastjson2.JSON; -import com.alibaba.fastjson2.JSONObject; import com.mongodb.BasicDBObject; import static com.akto.dto.RawApi.convertHeaders; -import static com.akto.util.HttpRequestResponseUtils.GRPC_CONTENT_TYPE; - public class Utils { private static final Logger logger = LoggerFactory.getLogger(Utils.class); @@ -95,65 +85,6 @@ public static Map parseCookie(List cookieList){ return cookieMap; } - private static int GRPC_DEBUG_COUNTER = 50; - - public static HttpResponseParams parseKafkaMessage(String message) throws Exception { - - //convert java object to JSON format - - JSONObject jsonObject = JSON.parseObject(message); - - String method = jsonObject.getString("method"); - String url = jsonObject.getString("path"); - String type = jsonObject.getString("type"); - Map> requestHeaders = OriginalHttpRequest.buildHeadersMap(jsonObject, "requestHeaders"); - - String rawRequestPayload = jsonObject.getString("requestPayload"); - String requestPayload = HttpRequestResponseUtils.rawToJsonString(rawRequestPayload,requestHeaders); - - if (GRPC_DEBUG_COUNTER > 0) { - String acceptableContentType = HttpRequestResponseUtils.getAcceptableContentType(requestHeaders); - if (acceptableContentType != null && rawRequestPayload.length() > 0) { - // only if request payload is of FORM_URL_ENCODED_CONTENT_TYPE we convert it to json - if (acceptableContentType.equals(GRPC_CONTENT_TYPE)) { - logger.info("grpc kafka payload:" + message,LogDb.RUNTIME); - GRPC_DEBUG_COUNTER--; - } - } - } - - String apiCollectionIdStr = jsonObject.getOrDefault("akto_vxlan_id", "0").toString(); - int apiCollectionId = 0; - if (NumberUtils.isDigits(apiCollectionIdStr)) { - apiCollectionId = NumberUtils.toInt(apiCollectionIdStr, 0); - } - - HttpRequestParams requestParams = new HttpRequestParams( - method,url,type, requestHeaders, requestPayload, apiCollectionId - ); - - int statusCode = jsonObject.getInteger("statusCode"); - String status = jsonObject.getString("status"); - Map> responseHeaders = OriginalHttpRequest.buildHeadersMap(jsonObject, "responseHeaders"); - String payload = jsonObject.getString("responsePayload"); - payload = HttpRequestResponseUtils.rawToJsonString(payload, responseHeaders); - payload = JSONUtils.parseIfJsonP(payload); - int time = jsonObject.getInteger("time"); - String accountId = jsonObject.getString("akto_account_id"); - String sourceIP = jsonObject.getString("ip"); - String destIP = jsonObject.getString("destIp"); - String direction = jsonObject.getString("direction"); - - String isPendingStr = (String) jsonObject.getOrDefault("is_pending", "false"); - boolean isPending = !isPendingStr.toLowerCase().equals("false"); - String sourceStr = (String) jsonObject.getOrDefault("source", HttpResponseParams.Source.OTHER.name()); - HttpResponseParams.Source source = HttpResponseParams.Source.valueOf(sourceStr); - - return new HttpResponseParams( - type,statusCode, status, responseHeaders, payload, requestParams, time, accountId, isPending, source, message, sourceIP, destIP, direction - ); - } - public static Pattern createRegexPatternFromList(List discardedUrlList){ StringJoiner joiner = new StringJoiner("|", ".*\\.(", ")(\\?.*)?"); for (String extension : discardedUrlList) { diff --git a/libs/utils/src/main/java/com/akto/test_editor/execution/VariableResolver.java b/libs/utils/src/main/java/com/akto/test_editor/execution/VariableResolver.java index 52ac862f0b..aaf19e945c 100644 --- a/libs/utils/src/main/java/com/akto/test_editor/execution/VariableResolver.java +++ b/libs/utils/src/main/java/com/akto/test_editor/execution/VariableResolver.java @@ -35,7 +35,7 @@ import com.mongodb.client.model.Filters; import com.mongodb.client.model.Projections; -import static com.akto.runtime.utils.Utils.parseKafkaMessage; +import static com.akto.runtime.parser.SampleParser.parseSampleMessage; public class VariableResolver { @@ -680,7 +680,7 @@ public static Set extractValuesFromSampleData(List samples, Stri HttpResponseParams httpResponseParams; HttpRequestParams httpRequestParams; try { - httpResponseParams = parseKafkaMessage(sample); + httpResponseParams = parseSampleMessage(sample); httpRequestParams = httpResponseParams.getRequestParams(); if ("terminal_keys".equals(location)) { diff --git a/libs/utils/src/main/java/com/akto/test_editor/filter/FilterAction.java b/libs/utils/src/main/java/com/akto/test_editor/filter/FilterAction.java index 9de13ecc53..f255e5558a 100644 --- a/libs/utils/src/main/java/com/akto/test_editor/filter/FilterAction.java +++ b/libs/utils/src/main/java/com/akto/test_editor/filter/FilterAction.java @@ -45,7 +45,8 @@ import static com.akto.dto.RawApi.convertHeaders; import static com.akto.runtime.RuntimeUtil.createUrlTemplate; import static com.akto.testing.Utils.compareWithOriginalResponse; -import static com.akto.runtime.utils.Utils.parseKafkaMessage; + +import static com.akto.runtime.parser.SampleParser.parseSampleMessage; public final class FilterAction { @@ -1263,7 +1264,7 @@ public BasicDBObject getPrivateResourceCount(OriginalHttpRequest originalHttpReq } for (String sample: sd.getSamples()) { try { - HttpResponseParams httpResponseParams = parseKafkaMessage(sample); + HttpResponseParams httpResponseParams = parseSampleMessage(sample); String sUrl = httpResponseParams.getRequestParams().getURL(); String[] sUrlTokens = sUrl.split("/"); String[] origUrlTokens = urlWithParams.split("/"); @@ -1319,7 +1320,7 @@ public BasicDBObject getPrivateResourceCount(OriginalHttpRequest originalHttpReq String key = SingleTypeInfo.findLastKeyFromParam(param); BasicDBObject payloadObj = new BasicDBObject(); try { - HttpResponseParams httpResponseParams = parseKafkaMessage(sample); + HttpResponseParams httpResponseParams = parseSampleMessage(sample); payloadObj = RequestTemplate.parseRequestPayload(httpResponseParams.getRequestParams().getPayload(), null); } catch (Exception e) { // TODO: handle exception From f0801bd7f3d06ddadb71688f783ff06b54fc10e3 Mon Sep 17 00:00:00 2001 From: Ark2307 Date: Sun, 15 Sep 2024 17:12:09 +0530 Subject: [PATCH 5/9] Added db calls for mini-runtime --- .../main/java/com/akto/action/DbAction.java | 18 ++++++++++++++++++ .../src/main/resources/struts.xml | 11 +++++++++++ .../java/com/akto/data_actor/DataActor.java | 2 ++ .../main/java/com/akto/data_actor/DbActor.java | 4 ++++ .../main/java/com/akto/data_actor/DbLayer.java | 7 +++++++ 5 files changed, 42 insertions(+) diff --git a/apps/database-abstractor/src/main/java/com/akto/action/DbAction.java b/apps/database-abstractor/src/main/java/com/akto/action/DbAction.java index 50dbb12ca4..a3ee7bf8c5 100644 --- a/apps/database-abstractor/src/main/java/com/akto/action/DbAction.java +++ b/apps/database-abstractor/src/main/java/com/akto/action/DbAction.java @@ -144,6 +144,7 @@ public void setTestSourceConfig(TestSourceConfig testSourceConfig) { List newEps; String logicalGroupName; BasicDBList issuesIds; + List activeAdvancedFilters; public BasicDBList getIssuesIds() { return issuesIds; @@ -1644,6 +1645,15 @@ public String fetchLatestEndpointsForTesting() { return Action.SUCCESS.toUpperCase(); } + public String fetchActiveAdvancedFilters(){ + try { + this.activeAdvancedFilters = DbLayer.fetchActiveFilterTemplates(); + } catch (Exception e) { + return Action.ERROR.toUpperCase(); + } + return Action.SUCCESS.toUpperCase(); + } + public List getCustomDataTypes() { return customDataTypes; } @@ -2522,4 +2532,12 @@ public void setNewEps(List newEps) { this.newEps = newEps; } + public List getActiveAdvancedFilters() { + return activeAdvancedFilters; + } + + public void setActiveAdvancedFilters(List activeAdvancedFilters) { + this.activeAdvancedFilters = activeAdvancedFilters; + } + } diff --git a/apps/database-abstractor/src/main/resources/struts.xml b/apps/database-abstractor/src/main/resources/struts.xml index 0b2adcc875..e4f5043569 100644 --- a/apps/database-abstractor/src/main/resources/struts.xml +++ b/apps/database-abstractor/src/main/resources/struts.xml @@ -1135,6 +1135,17 @@ + + + + + + 422 + false + ^actionErrors.* + + + diff --git a/libs/utils/src/main/java/com/akto/data_actor/DataActor.java b/libs/utils/src/main/java/com/akto/data_actor/DataActor.java index da62ca5f89..9bf3661efd 100644 --- a/libs/utils/src/main/java/com/akto/data_actor/DataActor.java +++ b/libs/utils/src/main/java/com/akto/data_actor/DataActor.java @@ -225,4 +225,6 @@ public abstract class DataActor { public abstract void bulkWriteDependencyNodes(List dependencyNodeList); public abstract List fetchLatestEndpointsForTesting(int startTimestamp, int endTimestamp, int apiCollectionId); + + public abstract List fetchActiveAdvancedFilters(); } diff --git a/libs/utils/src/main/java/com/akto/data_actor/DbActor.java b/libs/utils/src/main/java/com/akto/data_actor/DbActor.java index 0ff7d26b3a..f0fd24e315 100644 --- a/libs/utils/src/main/java/com/akto/data_actor/DbActor.java +++ b/libs/utils/src/main/java/com/akto/data_actor/DbActor.java @@ -468,4 +468,8 @@ public List fetchLatestEndpointsForTesting(int startTimestam return DbLayer.fetchLatestEndpointsForTesting(startTimestamp, endTimestamp, apiCollectionId); } + public List fetchActiveAdvancedFilters(){ + return DbLayer.fetchActiveFilterTemplates(); + } + } diff --git a/libs/utils/src/main/java/com/akto/data_actor/DbLayer.java b/libs/utils/src/main/java/com/akto/data_actor/DbLayer.java index ce968a5aaa..c76151dce0 100644 --- a/libs/utils/src/main/java/com/akto/data_actor/DbLayer.java +++ b/libs/utils/src/main/java/com/akto/data_actor/DbLayer.java @@ -23,6 +23,7 @@ import com.akto.dao.billing.OrganizationsDao; import com.akto.dao.billing.TokensDao; import com.akto.dao.context.Context; +import com.akto.dao.runtime_filters.AdvancedTrafficFiltersDao; import com.akto.dao.test_editor.YamlTemplateDao; import com.akto.dao.testing.AccessMatrixTaskInfosDao; import com.akto.dao.testing.AccessMatrixUrlToRolesDao; @@ -858,4 +859,10 @@ public static void bulkWriteDependencyNodes(List dependencyNodeL public static List fetchLatestEndpointsForTesting(int startTimestamp, int endTimestamp, int apiCollectionId) { return SingleTypeInfoDao.fetchLatestEndpointsForTesting(startTimestamp, endTimestamp, apiCollectionId); } + + public static List fetchActiveFilterTemplates(){ + return AdvancedTrafficFiltersDao.instance.findAll( + Filters.ne(YamlTemplate.INACTIVE, false) + ); + } } From 3ea3da83900cbe432f3269a71d563108e6d02326 Mon Sep 17 00:00:00 2001 From: Ark2307 Date: Sun, 15 Sep 2024 10:39:59 +0530 Subject: [PATCH 6/9] Execution part shifted to utils --- .../akto/test_editor/execution/Executor.java | 118 +----------------- .../akto/dao/billing/OrganizationsDao.java | 36 ++++++ .../java/com/akto/dto/HttpResponseParams.java | 4 + .../java/com/akto/runtime/utils/Utils.java | 16 +++ .../main/java/com/akto/test_editor/Utils.java | 97 ++++++++++++++ .../execution/ExecutionListBuilder.java | 0 .../test_editor/execution/Operations.java | 0 .../execution/ParseAndExecute.java | 89 +++++++++++++ 8 files changed, 245 insertions(+), 115 deletions(-) rename {apps/testing => libs/utils}/src/main/java/com/akto/test_editor/execution/ExecutionListBuilder.java (100%) rename {apps/testing => libs/utils}/src/main/java/com/akto/test_editor/execution/Operations.java (100%) create mode 100644 libs/utils/src/main/java/com/akto/test_editor/execution/ParseAndExecute.java diff --git a/apps/testing/src/main/java/com/akto/test_editor/execution/Executor.java b/apps/testing/src/main/java/com/akto/test_editor/execution/Executor.java index 6b7bd4de19..2729622822 100644 --- a/apps/testing/src/main/java/com/akto/test_editor/execution/Executor.java +++ b/apps/testing/src/main/java/com/akto/test_editor/execution/Executor.java @@ -1,8 +1,6 @@ package com.akto.test_editor.execution; -import com.akto.billing.UsageMetricUtils; import com.akto.dao.billing.OrganizationsDao; -import com.akto.dao.billing.TokensDao; import java.util.*; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -19,10 +17,7 @@ import com.akto.testing.*; import com.akto.util.enums.LoginFlowEnums; import com.akto.util.enums.LoginFlowEnums.AuthMechanismTypes; -import com.akto.util.enums.LoginFlowEnums.LoginStepTypesEnums; import com.akto.dto.api_workflow.Graph; -import com.akto.dto.billing.Organization; -import com.akto.dto.billing.Tokens; import com.akto.dto.test_editor.*; import com.akto.dto.testing.TestResult.Confidence; import com.akto.dto.testing.TestResult.TestError; @@ -49,7 +44,6 @@ import com.mongodb.client.model.Filters; import org.apache.commons.lang3.StringUtils; -import org.bson.conversions.Bson; import org.bson.types.ObjectId; @@ -530,37 +524,6 @@ private ExecutorSingleOperationResp modifyAuthTokenInRawApi(TestRoles testRole, return null; } - private static BasicDBObject getBillingTokenForAuth() { - BasicDBObject bDObject; - int accountId = Context.accountId.get(); - Organization organization = OrganizationsDao.instance.findOne( - Filters.in(Organization.ACCOUNTS, accountId) - ); - if (organization == null) { - return new BasicDBObject("error", "organization not found"); - } - - Tokens tokens; - Bson filters = Filters.and( - Filters.eq(Tokens.ORG_ID, organization.getId()), - Filters.eq(Tokens.ACCOUNT_ID, accountId) - ); - String errMessage = ""; - tokens = TokensDao.instance.findOne(filters); - if (tokens == null) { - errMessage = "error extracting ${akto_header}, token is missing"; - } - if (tokens.isOldToken()) { - errMessage = "error extracting ${akto_header}, token is old"; - } - if(errMessage.length() > 0){ - bDObject = new BasicDBObject("error", errMessage); - }else{ - bDObject = new BasicDBObject("token", tokens.getToken()); - } - return bDObject; - } - public ExecutorSingleOperationResp runOperation(String operationType, RawApi rawApi, Object key, Object value, Map varMap, AuthMechanism authMechanism, List customAuthTypes, ApiInfo.ApiInfoKey apiInfoKey) { switch (operationType.toLowerCase()) { case "send_ssrf_req": @@ -572,7 +535,7 @@ public ExecutorSingleOperationResp runOperation(String operationType, RawApi raw uuidList.add(generatedUUID); varMap.put("random_uuid", uuidList); - BasicDBObject response = getBillingTokenForAuth(); + BasicDBObject response = OrganizationsDao.getBillingTokenForAuth(); if(response.getString("token") != null){ String tokenVal = response.getString("token"); return Utils.sendRequestToSsrfServer(url + generatedUUID, redirectUrl, tokenVal); @@ -581,57 +544,9 @@ public ExecutorSingleOperationResp runOperation(String operationType, RawApi raw } case "attach_file": return Operations.addHeader(rawApi, Constants.AKTO_ATTACH_FILE , key.toString()); - case "add_body_param": - Object epochVal = Utils.getEpochTime(value); - if (epochVal != null) { - value = epochVal; - } - return Operations.addBody(rawApi, key.toString(), value); - case "modify_body_param": - epochVal = Utils.getEpochTime(value); - if (epochVal != null) { - value = epochVal; - } - return Operations.modifyBodyParam(rawApi, key.toString(), value); - case "delete_graphql_field": - return Operations.deleteGraphqlField(rawApi, key == null ? "": key.toString()); - case "add_graphql_field": - return Operations.addGraphqlField(rawApi, key == null ? "": key.toString(), value == null ? "" : value.toString()); - case "add_unique_graphql_field": - return Operations.addUniqueGraphqlField(rawApi, key == null ? "": key.toString(), value == null ? "" : value.toString()); - case "modify_graphql_field": - return Operations.modifyGraphqlField(rawApi, key == null ? "": key.toString(), value == null ? "" : value.toString()); - case "delete_body_param": - return Operations.deleteBodyParam(rawApi, key.toString()); - case "replace_body": - String newPayload = rawApi.getRequest().getBody(); - if (key instanceof Map) { - Map> regexReplace = (Map) key; - String payload = rawApi.getRequest().getBody(); - Map regexInfo = regexReplace.get("regex_replace"); - String regex = regexInfo.get("regex"); - String replaceWith = regexInfo.get("replace_with"); - newPayload = Utils.applyRegexModifier(payload, regex, replaceWith); - } else { - newPayload = key.toString(); - } - return Operations.replaceBody(rawApi, newPayload); - case "add_header": - if (value.equals("${akto_header}")) { - BasicDBObject tokenResponse = getBillingTokenForAuth(); - if(tokenResponse.getString("token") != null){ - value = tokenResponse.getString("token"); - }else{ - return new ExecutorSingleOperationResp(false, tokenResponse.getString("error")); - } - } - epochVal = Utils.getEpochTime(value); - if (epochVal != null) { - value = epochVal; - } - return Operations.addHeader(rawApi, key.toString(), value.toString()); case "modify_header": + Object epochVal = Utils.getEpochTime(value); String keyStr = key.toString(); String valStr = value.toString(); @@ -658,33 +573,6 @@ public ExecutorSingleOperationResp runOperation(String operationType, RawApi raw } return Operations.modifyHeader(rawApi, keyStr, valStr); } - case "delete_header": - return Operations.deleteHeader(rawApi, key.toString()); - case "add_query_param": - epochVal = Utils.getEpochTime(value); - if (epochVal != null) { - value = epochVal; - } - return Operations.addQueryParam(rawApi, key.toString(), value); - case "modify_query_param": - epochVal = Utils.getEpochTime(value); - if (epochVal != null) { - value = epochVal; - } - return Operations.modifyQueryParam(rawApi, key.toString(), value); - case "delete_query_param": - return Operations.deleteQueryParam(rawApi, key.toString()); - case "modify_url": - String newUrl = null; - UrlModifierPayload urlModifierPayload = Utils.fetchUrlModifyPayload(key.toString()); - if (urlModifierPayload != null) { - newUrl = Utils.buildNewUrl(urlModifierPayload, rawApi.getRequest().getUrl()); - } else { - newUrl = key.toString(); - } - return Operations.modifyUrl(rawApi, newUrl); - case "modify_method": - return Operations.modifyMethod(rawApi, key.toString()); case "remove_auth_header": List authHeaders = (List) varMap.get("auth_headers"); boolean removed = false; @@ -815,7 +703,7 @@ public ExecutorSingleOperationResp runOperation(String operationType, RawApi raw return new ExecutorSingleOperationResp(true, ""); } default: - return new ExecutorSingleOperationResp(false, "invalid operationType"); + return Utils.modifySampleDataUtil(operationType, rawApi, key, value, varMap, apiInfoKey); } } diff --git a/libs/dao/src/main/java/com/akto/dao/billing/OrganizationsDao.java b/libs/dao/src/main/java/com/akto/dao/billing/OrganizationsDao.java index 9da9e43177..81f90a2ec8 100644 --- a/libs/dao/src/main/java/com/akto/dao/billing/OrganizationsDao.java +++ b/libs/dao/src/main/java/com/akto/dao/billing/OrganizationsDao.java @@ -1,8 +1,13 @@ package com.akto.dao.billing; +import org.bson.conversions.Bson; + import com.akto.dao.BillingContextDao; import com.akto.dao.MCollection; +import com.akto.dao.context.Context; import com.akto.dto.billing.Organization; +import com.akto.dto.billing.Tokens; +import com.mongodb.BasicDBObject; import com.mongodb.client.model.Filters; public class OrganizationsDao extends BillingContextDao{ @@ -39,4 +44,35 @@ public Organization findOneByAccountId(int accountId) { Filters.in(Organization.ACCOUNTS, accountId)); } + public static BasicDBObject getBillingTokenForAuth() { + BasicDBObject bDObject; + int accountId = Context.accountId.get(); + Organization organization = OrganizationsDao.instance.findOne( + Filters.in(Organization.ACCOUNTS, accountId) + ); + if (organization == null) { + return new BasicDBObject("error", "organization not found"); + } + + Tokens tokens; + Bson filters = Filters.and( + Filters.eq(Tokens.ORG_ID, organization.getId()), + Filters.eq(Tokens.ACCOUNT_ID, accountId) + ); + String errMessage = ""; + tokens = TokensDao.instance.findOne(filters); + if (tokens == null) { + errMessage = "error extracting ${akto_header}, token is missing"; + } + if (tokens.isOldToken()) { + errMessage = "error extracting ${akto_header}, token is old"; + } + if(errMessage.length() > 0){ + bDObject = new BasicDBObject("error", errMessage); + }else{ + bDObject = new BasicDBObject("token", tokens.getToken()); + } + return bDObject; + } + } diff --git a/libs/dao/src/main/java/com/akto/dto/HttpResponseParams.java b/libs/dao/src/main/java/com/akto/dto/HttpResponseParams.java index ec17fd0410..b5b96db20b 100644 --- a/libs/dao/src/main/java/com/akto/dto/HttpResponseParams.java +++ b/libs/dao/src/main/java/com/akto/dto/HttpResponseParams.java @@ -157,4 +157,8 @@ public String getDirection() { public void setDirection(String direction) { this.direction = direction; } + + public void setRequestParams(HttpRequestParams requestParams) { + this.requestParams = requestParams; + } } diff --git a/libs/utils/src/main/java/com/akto/runtime/utils/Utils.java b/libs/utils/src/main/java/com/akto/runtime/utils/Utils.java index 6c33097b23..aa17a43823 100644 --- a/libs/utils/src/main/java/com/akto/runtime/utils/Utils.java +++ b/libs/utils/src/main/java/com/akto/runtime/utils/Utils.java @@ -12,8 +12,11 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import com.akto.dto.HttpRequestParams; +import com.akto.dto.HttpResponseParams; import com.akto.dto.OriginalHttpRequest; import com.akto.dto.OriginalHttpResponse; +import com.akto.dto.RawApi; import com.mongodb.BasicDBObject; import static com.akto.dto.RawApi.convertHeaders; @@ -99,6 +102,19 @@ public static Pattern createRegexPatternFromList(List discardedUrlList){ return pattern; } + public static HttpResponseParams convertRawApiToHttpResponseParams(RawApi rawApi, HttpResponseParams originalHttpResponseParams){ + + HttpRequestParams ogRequestParams = originalHttpResponseParams.getRequestParams(); + OriginalHttpRequest modifiedRequest = rawApi.getRequest(); + + ogRequestParams.setHeaders(modifiedRequest.getHeaders()); + ogRequestParams.setUrl(modifiedRequest.getFullUrlWithParams()); + ogRequestParams.setPayload(modifiedRequest.getBody()); + + originalHttpResponseParams.setRequestParams(ogRequestParams); + + return originalHttpResponseParams; + } } diff --git a/libs/utils/src/main/java/com/akto/test_editor/Utils.java b/libs/utils/src/main/java/com/akto/test_editor/Utils.java index 2daf84d23b..fff2186a72 100644 --- a/libs/utils/src/main/java/com/akto/test_editor/Utils.java +++ b/libs/utils/src/main/java/com/akto/test_editor/Utils.java @@ -17,10 +17,13 @@ import com.akto.dto.OriginalHttpRequest; import com.akto.dto.RawApi; +import com.akto.dao.billing.OrganizationsDao; import com.akto.dao.context.Context; import com.akto.dto.ApiInfo.ApiAccessType; +import com.akto.dto.ApiInfo; import com.akto.dto.test_editor.ExecutorSingleOperationResp; import com.akto.dto.testing.UrlModifierPayload; +import com.akto.test_editor.execution.Operations; import com.akto.util.Constants; import com.akto.util.DashboardMode; import com.akto.util.JSONUtils; @@ -861,4 +864,98 @@ public static Object getEpochTime(Object value) { return val; } + public static ExecutorSingleOperationResp modifySampleDataUtil(String operationType, RawApi rawApi, Object key, Object value, Map varMap, ApiInfo.ApiInfoKey apiInfoKey){ + switch (operationType.toLowerCase()) { + case "add_body_param": + Object epochVal = Utils.getEpochTime(value); + if (epochVal != null) { + value = epochVal; + } + return Operations.addBody(rawApi, key.toString(), value); + case "modify_body_param": + epochVal = Utils.getEpochTime(value); + if (epochVal != null) { + value = epochVal; + } + return Operations.modifyBodyParam(rawApi, key.toString(), value); + case "delete_graphql_field": + return Operations.deleteGraphqlField(rawApi, key == null ? "": key.toString()); + case "add_graphql_field": + return Operations.addGraphqlField(rawApi, key == null ? "": key.toString(), value == null ? "" : value.toString()); + case "add_unique_graphql_field": + return Operations.addUniqueGraphqlField(rawApi, key == null ? "": key.toString(), value == null ? "" : value.toString()); + case "modify_graphql_field": + return Operations.modifyGraphqlField(rawApi, key == null ? "": key.toString(), value == null ? "" : value.toString()); + case "delete_body_param": + return Operations.deleteBodyParam(rawApi, key.toString()); + case "replace_body": + String newPayload = rawApi.getRequest().getBody(); + if (key instanceof Map) { + Map> regexReplace = (Map) key; + String payload = rawApi.getRequest().getBody(); + Map regexInfo = regexReplace.get("regex_replace"); + String regex = regexInfo.get("regex"); + String replaceWith = regexInfo.get("replace_with"); + newPayload = Utils.applyRegexModifier(payload, regex, replaceWith); + } else { + newPayload = key.toString(); + } + return Operations.replaceBody(rawApi, newPayload); + case "add_header": + if (value.equals("${akto_header}")) { + BasicDBObject tokenResponse = OrganizationsDao.getBillingTokenForAuth(); + if(tokenResponse.getString("token") != null){ + value = tokenResponse.getString("token"); + }else{ + return new ExecutorSingleOperationResp(false, tokenResponse.getString("error")); + } + } + epochVal = Utils.getEpochTime(value); + if (epochVal != null) { + value = epochVal; + } + + return Operations.addHeader(rawApi, key.toString(), value.toString()); + case "modify_header": + String keyStr = key.toString(); + String valStr = value.toString(); + epochVal = Utils.getEpochTime(valStr); + if (epochVal != null) { + valStr = epochVal.toString(); + } + return Operations.modifyHeader(rawApi, keyStr, valStr); + case "delete_header": + return Operations.deleteHeader(rawApi, key.toString()); + case "add_query_param": + epochVal = Utils.getEpochTime(value); + if (epochVal != null) { + value = epochVal; + } + return Operations.addQueryParam(rawApi, key.toString(), value); + case "modify_query_param": + epochVal = Utils.getEpochTime(value); + if (epochVal != null) { + value = epochVal; + } + return Operations.modifyQueryParam(rawApi, key.toString(), value); + case "delete_query_param": + return Operations.deleteQueryParam(rawApi, key.toString()); + case "modify_url": + String newUrl = null; + UrlModifierPayload urlModifierPayload = Utils.fetchUrlModifyPayload(key.toString()); + if (urlModifierPayload != null) { + newUrl = Utils.buildNewUrl(urlModifierPayload, rawApi.getRequest().getUrl()); + } else { + newUrl = key.toString(); + } + return Operations.modifyUrl(rawApi, newUrl); + case "modify_method": + return Operations.modifyMethod(rawApi, key.toString()); + default: + return new ExecutorSingleOperationResp(false, "invalid operationType"); + } + + } + + } diff --git a/apps/testing/src/main/java/com/akto/test_editor/execution/ExecutionListBuilder.java b/libs/utils/src/main/java/com/akto/test_editor/execution/ExecutionListBuilder.java similarity index 100% rename from apps/testing/src/main/java/com/akto/test_editor/execution/ExecutionListBuilder.java rename to libs/utils/src/main/java/com/akto/test_editor/execution/ExecutionListBuilder.java diff --git a/apps/testing/src/main/java/com/akto/test_editor/execution/Operations.java b/libs/utils/src/main/java/com/akto/test_editor/execution/Operations.java similarity index 100% rename from apps/testing/src/main/java/com/akto/test_editor/execution/Operations.java rename to libs/utils/src/main/java/com/akto/test_editor/execution/Operations.java diff --git a/libs/utils/src/main/java/com/akto/test_editor/execution/ParseAndExecute.java b/libs/utils/src/main/java/com/akto/test_editor/execution/ParseAndExecute.java new file mode 100644 index 0000000000..495cc90aed --- /dev/null +++ b/libs/utils/src/main/java/com/akto/test_editor/execution/ParseAndExecute.java @@ -0,0 +1,89 @@ +package com.akto.test_editor.execution; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +import com.akto.dao.test_editor.TestEditorEnums; +import com.akto.dao.test_editor.TestEditorEnums.ExecutorOperandTypes; +import com.akto.dto.ApiInfo; +import com.akto.dto.ApiInfo.ApiInfoKey; +import com.akto.dto.RawApi; +import com.akto.dto.test_editor.ExecuteAlgoObj; +import com.akto.dto.test_editor.ExecutionOrderResp; +import com.akto.dto.test_editor.ExecutorNode; +import com.akto.dto.test_editor.ExecutorSingleOperationResp; +import com.akto.test_editor.Utils; + +public class ParseAndExecute { + + public static List getExecutorNodes(ExecutorNode executorNode){ + if(executorNode.getChildNodes() == null || executorNode.getChildNodes().isEmpty()){ + return new ArrayList<>(); + } + + ExecutionListBuilder executionListBuilder = new ExecutionListBuilder(); + List executorNodes = new ArrayList<>(); + ExecutionOrderResp executionOrderResp = executionListBuilder.parseExecuteOperations(executorNode, executorNodes); + + ExecutorNode reqNodes = executorNode.getChildNodes().get(1); + if (reqNodes.getChildNodes() == null || reqNodes.getChildNodes().size() == 0) { + return new ArrayList<>(); + } + if(executionOrderResp.getError() != null && executionOrderResp.getError().length() > 0) { + return new ArrayList<>(); + } + return executorNodes; + } + + public RawApi execute (List executorNodes, RawApi rawApi, ApiInfoKey apiInfoKey, Map varMap, String logId){ + RawApi origRawApi = rawApi.copy(); + RawApi modifiedRawApi = executeAlgorithm(origRawApi, varMap, executorNodes, null, apiInfoKey); + return modifiedRawApi; + } + + private RawApi executeAlgorithm(RawApi sampleRawApi, Map varMap, List executorNodes, Map algoMap, ApiInfo.ApiInfoKey apiInfoKey){ + RawApi copyRawApi = sampleRawApi.copy(); + for(ExecutorNode executorNode: executorNodes){ + ExecutorNode node; + if (executorNode.getNodeType().equalsIgnoreCase(TestEditorEnums.ExecutorOperandTypes.NonTerminal.toString())) { + node = executorNode.getChildNodes().get(0); + } else { + node = executorNode; + } + + Object keyOp = node.getOperationType(); + Object valueOp = node.getValues(); + if (node.getNodeType().equalsIgnoreCase(ExecutorOperandTypes.Terminal.toString())) { + if (node.getValues() instanceof Boolean) { + keyOp = Boolean.toString((Boolean) node.getValues()); + } else if (node.getValues() instanceof String) { + keyOp = (String) node.getValues(); + } else { + keyOp = (Map) node.getValues(); + } + valueOp = null; + } + + List keyList = VariableResolver.resolveExpression(varMap, keyOp); + List valList = new ArrayList<>(); + if (valueOp != null) { + valList = VariableResolver.resolveExpression(varMap, valueOp); + } + + Object key = keyList.get(0); + Object val = null; + if (valList != null && valList.size() > 0) { + val = valList.get(0); + } + + ExecutorSingleOperationResp resp = Utils.modifySampleDataUtil(executorNode.getOperationType(), copyRawApi, key, val , varMap, apiInfoKey); + if(resp.getErrMsg() != null && resp.getErrMsg().length() > 0){ + break; + } + } + + return copyRawApi; + } + +} From 237a6aca958b118c15676de59f16d6c610ced4fc Mon Sep 17 00:00:00 2001 From: Ark2307 Date: Mon, 16 Sep 2024 17:05:24 +0530 Subject: [PATCH 7/9] Handling execution in mini-runtime --- .../akto/hybrid_parsers/HttpCallParser.java | 86 +++++++++++-------- .../com/akto/dto/monitoring/FilterConfig.java | 3 + .../execution/ParseAndExecute.java | 25 ++++++ 3 files changed, 80 insertions(+), 34 deletions(-) diff --git a/apps/mini-runtime/src/main/java/com/akto/hybrid_parsers/HttpCallParser.java b/apps/mini-runtime/src/main/java/com/akto/hybrid_parsers/HttpCallParser.java index 33a019217d..b7de08463b 100644 --- a/apps/mini-runtime/src/main/java/com/akto/hybrid_parsers/HttpCallParser.java +++ b/apps/mini-runtime/src/main/java/com/akto/hybrid_parsers/HttpCallParser.java @@ -18,6 +18,7 @@ import com.akto.dto.bulk_updates.UpdatePayload; import com.akto.dto.monitoring.FilterConfig; import com.akto.dto.settings.DefaultPayload; +import com.akto.dto.test_editor.ExecutorNode; import com.akto.dto.traffic_metrics.TrafficMetrics; import com.akto.dto.type.URLMethods.Method; import com.akto.dto.usage.MetricTypes; @@ -27,6 +28,7 @@ import com.akto.runtime.RuntimeUtil; import com.akto.runtime.parser.SampleParser; import com.akto.runtime.utils.Utils; +import com.akto.test_editor.execution.ParseAndExecute; import com.akto.test_editor.execution.VariableResolver; import com.akto.test_editor.filter.data_operands_impl.ValidationResult; import com.akto.hybrid_runtime.APICatalogSync; @@ -199,41 +201,51 @@ private SyncLimit fetchSyncLimit() { return syncLimit; } - private List applyAdvancedFilters(List responseParams){ - Map filterMap = apiCatalogSync.advancedFilterMap; - + private static boolean isValidResponseParam(HttpResponseParams responseParam, Map filterMap, Map> executorNodesMap){ + boolean isValidResponseParam = false; + String message = responseParam.getOrig(); + RawApi rawApi = RawApi.buildFromMessage(message); + int apiCollectionId = responseParam.requestParams.getApiCollectionId(); + String url = responseParam.getRequestParams().getURL(); + Method method = Method.fromString(responseParam.getRequestParams().getMethod()); + ApiInfoKey apiInfoKey = new ApiInfoKey(apiCollectionId, url, method); + for (Entry apiFilterEntry : filterMap.entrySet()) { + try { + FilterConfig apiFilter = apiFilterEntry.getValue(); + Map varMap = apiFilter.resolveVarMap(); + VariableResolver.resolveWordList(varMap, new HashMap>() { + { + put(apiInfoKey, Arrays.asList(message)); + } + }, apiInfoKey); + String filterExecutionLogId = UUID.randomUUID().toString(); + ValidationResult res = validateFilter(apiFilter.getFilter().getNode(), rawApi, + apiInfoKey, varMap, filterExecutionLogId); + if (res.getIsValid()) { + // handle custom filters here + if(apiFilter.getId().equals(FilterConfig.DEFAULT_BLOCK_FILTER)){ + return false; + } + + // handle execute here + RawApi modifiedApi = new ParseAndExecute().execute(executorNodesMap.getOrDefault(apiFilter.getId(), new ArrayList<>()), rawApi, apiInfoKey, varMap, filterExecutionLogId); + responseParam = Utils.convertRawApiToHttpResponseParams(modifiedApi, responseParam); + isValidResponseParam = true; + } + } catch (Exception e) { + loggerMaker.errorAndAddToDb(e, String.format("Error in httpCallFilter %s", e.toString())); + isValidResponseParam = true; + } + } + return isValidResponseParam; + } + + private List applyAdvancedFilters(List responseParams, Map> executorNodesMap, Map filterMap){ if (filterMap != null && !filterMap.isEmpty()) { List filteredParams = new ArrayList<>(); for (HttpResponseParams responseParam : responseParams) { - for (Entry apiFilterEntry : filterMap.entrySet()) { - try { - FilterConfig apiFilter = apiFilterEntry.getValue(); - String message = responseParam.getOrig(); - RawApi rawApi = RawApi.buildFromMessage(message); - int apiCollectionId = createApiCollectionId(responseParam); - responseParam.requestParams.setApiCollectionId(apiCollectionId); - String url = responseParam.getRequestParams().getURL(); - Method method = Method.valueOf(responseParam.getRequestParams().getMethod()); - ApiInfoKey apiInfoKey = new ApiInfoKey(apiCollectionId, url, method); - Map varMap = apiFilter.resolveVarMap(); - VariableResolver.resolveWordList(varMap, new HashMap>() { - { - put(apiInfoKey, Arrays.asList(message)); - } - }, apiInfoKey); - String filterExecutionLogId = UUID.randomUUID().toString(); - ValidationResult res = validateFilter(apiFilter.getFilter().getNode(), rawApi, - apiInfoKey, varMap, filterExecutionLogId); - if (res.getIsValid()) { - // TODO apply execution - - filteredParams.add(responseParam); - break; - } - } catch (Exception e) { - loggerMaker.errorAndAddToDb(e, String.format("Error in httpCallFilter %s", e.toString())); - return responseParams; - } + if(isValidResponseParam(responseParam, filterMap, executorNodesMap)){ + filteredParams.add(responseParam); } } return filteredParams; @@ -248,7 +260,6 @@ public void syncFunction(List responseParams, boolean syncIm filteredResponseParams = filterDefaultPayloads(filteredResponseParams, accountSettings.getDefaultPayloads()); } filteredResponseParams = filterHttpResponseParams(filteredResponseParams, accountSettings); - filteredResponseParams = applyAdvancedFilters(filteredResponseParams); boolean isHarOrPcap = aggregate(filteredResponseParams, aggregatorMap); for (int apiCollectionId: aggregatorMap.keySet()) { @@ -531,7 +542,7 @@ public List filterHttpResponseParams(List> executorNodesMap = ParseAndExecute.createExecutorNodeMap(apiCatalogSync.advancedFilterMap); for (HttpResponseParams httpResponseParam: httpResponseParamsList) { if (httpResponseParam.getSource().equals(HttpResponseParams.Source.MIRRORING)) { @@ -592,6 +603,13 @@ public List filterHttpResponseParams(List temp = applyAdvancedFilters(Arrays.asList(httpResponseParam), executorNodesMap, apiCatalogSync.advancedFilterMap); + if(temp.isEmpty()){ + continue; + }else{ + httpResponseParam = temp.get(0); + } + int apiCollectionId = createApiCollectionId(httpResponseParam); httpResponseParam.requestParams.setApiCollectionId(apiCollectionId); diff --git a/libs/dao/src/main/java/com/akto/dto/monitoring/FilterConfig.java b/libs/dao/src/main/java/com/akto/dto/monitoring/FilterConfig.java index 21d069205d..8fa1c5f0a5 100644 --- a/libs/dao/src/main/java/com/akto/dto/monitoring/FilterConfig.java +++ b/libs/dao/src/main/java/com/akto/dto/monitoring/FilterConfig.java @@ -25,6 +25,9 @@ public class FilterConfig { private ExecutorConfigParserResult executor; + public static final String DEFAULT_ALLOW_FILTER = "DEFAULT_ALLOW_FILTER"; + public static final String DEFAULT_BLOCK_FILTER = "DEFAULT_BLOCK_FILTER"; + public FilterConfig(String id, ConfigParserResult filter, Map> wordLists) { this.id = id; this.filter = filter; diff --git a/libs/utils/src/main/java/com/akto/test_editor/execution/ParseAndExecute.java b/libs/utils/src/main/java/com/akto/test_editor/execution/ParseAndExecute.java index 495cc90aed..847c4b0a78 100644 --- a/libs/utils/src/main/java/com/akto/test_editor/execution/ParseAndExecute.java +++ b/libs/utils/src/main/java/com/akto/test_editor/execution/ParseAndExecute.java @@ -1,6 +1,7 @@ package com.akto.test_editor.execution; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; import java.util.Map; @@ -8,9 +9,11 @@ import com.akto.dao.test_editor.TestEditorEnums.ExecutorOperandTypes; import com.akto.dto.ApiInfo; import com.akto.dto.ApiInfo.ApiInfoKey; +import com.akto.dto.monitoring.FilterConfig; import com.akto.dto.RawApi; import com.akto.dto.test_editor.ExecuteAlgoObj; import com.akto.dto.test_editor.ExecutionOrderResp; +import com.akto.dto.test_editor.ExecutorConfigParserResult; import com.akto.dto.test_editor.ExecutorNode; import com.akto.dto.test_editor.ExecutorSingleOperationResp; import com.akto.test_editor.Utils; @@ -86,4 +89,26 @@ private RawApi executeAlgorithm(RawApi sampleRawApi, Map varMap, return copyRawApi; } + public static Map> createExecutorNodeMap (Map filterMap) { + Map> finalMap = new HashMap<>(); + + if (filterMap != null && !filterMap.isEmpty()) { + for(Map.Entry iterator: filterMap.entrySet()){ + String templateId = iterator.getKey(); + if(templateId.equals(FilterConfig.DEFAULT_ALLOW_FILTER) || templateId.equals(FilterConfig.DEFAULT_BLOCK_FILTER)){ + continue; + } + ExecutorConfigParserResult nodeObj = iterator.getValue().getExecutor(); + if(nodeObj != null && nodeObj.getIsValid()){ + ExecutorNode node = nodeObj.getNode(); + List nodes = getExecutorNodes(node); + + finalMap.put(templateId, nodes); + } + } + } + + return finalMap; + } + } From be8b419c8fb9b92c6693adf3b18c0729ddfaf093 Mon Sep 17 00:00:00 2001 From: Ark2307 Date: Mon, 16 Sep 2024 23:36:37 +0530 Subject: [PATCH 8/9] Making code consistent --- .../akto/hybrid_parsers/HttpCallParser.java | 47 +++++++++++-------- .../com/akto/dto/monitoring/FilterConfig.java | 4 ++ 2 files changed, 32 insertions(+), 19 deletions(-) diff --git a/apps/mini-runtime/src/main/java/com/akto/hybrid_parsers/HttpCallParser.java b/apps/mini-runtime/src/main/java/com/akto/hybrid_parsers/HttpCallParser.java index b7de08463b..8ed682dda2 100644 --- a/apps/mini-runtime/src/main/java/com/akto/hybrid_parsers/HttpCallParser.java +++ b/apps/mini-runtime/src/main/java/com/akto/hybrid_parsers/HttpCallParser.java @@ -17,6 +17,7 @@ import com.akto.dto.bulk_updates.BulkUpdates; import com.akto.dto.bulk_updates.UpdatePayload; import com.akto.dto.monitoring.FilterConfig; +import com.akto.dto.monitoring.FilterConfig.FILTER_TYPE; import com.akto.dto.settings.DefaultPayload; import com.akto.dto.test_editor.ExecutorNode; import com.akto.dto.traffic_metrics.TrafficMetrics; @@ -36,6 +37,7 @@ import com.akto.hybrid_runtime.MergeLogicLocal; import com.akto.hybrid_runtime.URLAggregator; import com.akto.util.JSONUtils; +import com.akto.util.Pair; import com.akto.util.Constants; import com.akto.util.HttpRequestResponseUtils; import com.akto.util.http_util.CoreHTTPClient; @@ -201,8 +203,8 @@ private SyncLimit fetchSyncLimit() { return syncLimit; } - private static boolean isValidResponseParam(HttpResponseParams responseParam, Map filterMap, Map> executorNodesMap){ - boolean isValidResponseParam = false; + public static FILTER_TYPE isValidResponseParam(HttpResponseParams responseParam, Map filterMap, Map> executorNodesMap){ + FILTER_TYPE filterType = FILTER_TYPE.UNCHANGED; String message = responseParam.getOrig(); RawApi rawApi = RawApi.buildFromMessage(message); int apiCollectionId = responseParam.requestParams.getApiCollectionId(); @@ -224,33 +226,39 @@ private static boolean isValidResponseParam(HttpResponseParams responseParam, Ma if (res.getIsValid()) { // handle custom filters here if(apiFilter.getId().equals(FilterConfig.DEFAULT_BLOCK_FILTER)){ - return false; + return FILTER_TYPE.BLOCKED; } // handle execute here - RawApi modifiedApi = new ParseAndExecute().execute(executorNodesMap.getOrDefault(apiFilter.getId(), new ArrayList<>()), rawApi, apiInfoKey, varMap, filterExecutionLogId); - responseParam = Utils.convertRawApiToHttpResponseParams(modifiedApi, responseParam); - isValidResponseParam = true; + List nodes = executorNodesMap.getOrDefault(apiFilter.getId(), new ArrayList<>()); + if(!nodes.isEmpty()){ + RawApi modifiedApi = new ParseAndExecute().execute(nodes, rawApi, apiInfoKey, varMap, filterExecutionLogId); + responseParam = Utils.convertRawApiToHttpResponseParams(modifiedApi, responseParam); + filterType = FILTER_TYPE.MODIFIED; + }else{ + filterType = FILTER_TYPE.ALLOWED; + } + } } catch (Exception e) { loggerMaker.errorAndAddToDb(e, String.format("Error in httpCallFilter %s", e.toString())); - isValidResponseParam = true; + filterType = FILTER_TYPE.UNCHANGED; } } - return isValidResponseParam; + return filterType; } - private List applyAdvancedFilters(List responseParams, Map> executorNodesMap, Map filterMap){ + + public static Pair applyAdvancedFilters(HttpResponseParams responseParams, Map> executorNodesMap, Map filterMap){ if (filterMap != null && !filterMap.isEmpty()) { - List filteredParams = new ArrayList<>(); - for (HttpResponseParams responseParam : responseParams) { - if(isValidResponseParam(responseParam, filterMap, executorNodesMap)){ - filteredParams.add(responseParam); - } + FILTER_TYPE filterType = isValidResponseParam(responseParams, filterMap, executorNodesMap); + if(filterType.equals(FILTER_TYPE.BLOCKED)){ + return null; + }else{ + return new Pair(responseParams, filterType); } - return filteredParams; } - return responseParams; + return new Pair(responseParams, FILTER_TYPE.UNCHANGED); } public void syncFunction(List responseParams, boolean syncImmediately, boolean fetchAllSTI, AccountSettings accountSettings) { @@ -603,11 +611,12 @@ public List filterHttpResponseParams(List temp = applyAdvancedFilters(Arrays.asList(httpResponseParam), executorNodesMap, apiCatalogSync.advancedFilterMap); - if(temp.isEmpty()){ + Pair temp = applyAdvancedFilters(httpResponseParam, executorNodesMap, apiCatalogSync.advancedFilterMap); + HttpResponseParams param = temp.getFirst(); + if(param == null){ continue; }else{ - httpResponseParam = temp.get(0); + httpResponseParam = param; } int apiCollectionId = createApiCollectionId(httpResponseParam); diff --git a/libs/dao/src/main/java/com/akto/dto/monitoring/FilterConfig.java b/libs/dao/src/main/java/com/akto/dto/monitoring/FilterConfig.java index 8fa1c5f0a5..31ce9e2353 100644 --- a/libs/dao/src/main/java/com/akto/dto/monitoring/FilterConfig.java +++ b/libs/dao/src/main/java/com/akto/dto/monitoring/FilterConfig.java @@ -28,6 +28,10 @@ public class FilterConfig { public static final String DEFAULT_ALLOW_FILTER = "DEFAULT_ALLOW_FILTER"; public static final String DEFAULT_BLOCK_FILTER = "DEFAULT_BLOCK_FILTER"; + public enum FILTER_TYPE{ + BLOCKED , ALLOWED, MODIFIED, UNCHANGED + } + public FilterConfig(String id, ConfigParserResult filter, Map> wordLists) { this.id = id; this.filter = filter; From fb2a04e3825dd1b3c65b4a421fb7b2bbbdf397a4 Mon Sep 17 00:00:00 2001 From: Ark2307 Date: Tue, 17 Sep 2024 09:47:05 +0530 Subject: [PATCH 9/9] Fixing null pointer --- .../src/main/java/com/akto/hybrid_parsers/HttpCallParser.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/mini-runtime/src/main/java/com/akto/hybrid_parsers/HttpCallParser.java b/apps/mini-runtime/src/main/java/com/akto/hybrid_parsers/HttpCallParser.java index 8ed682dda2..f24493a98e 100644 --- a/apps/mini-runtime/src/main/java/com/akto/hybrid_parsers/HttpCallParser.java +++ b/apps/mini-runtime/src/main/java/com/akto/hybrid_parsers/HttpCallParser.java @@ -253,7 +253,7 @@ public static Pair applyAdvancedFilters(HttpResp if (filterMap != null && !filterMap.isEmpty()) { FILTER_TYPE filterType = isValidResponseParam(responseParams, filterMap, executorNodesMap); if(filterType.equals(FILTER_TYPE.BLOCKED)){ - return null; + return new Pair(null, FILTER_TYPE.BLOCKED); }else{ return new Pair(responseParams, filterType); }