diff --git a/apps/api-runtime/src/main/java/com/akto/parsers/HttpCallParser.java b/apps/api-runtime/src/main/java/com/akto/parsers/HttpCallParser.java index 62b14cfb04..35af4fe27c 100644 --- a/apps/api-runtime/src/main/java/com/akto/parsers/HttpCallParser.java +++ b/apps/api-runtime/src/main/java/com/akto/parsers/HttpCallParser.java @@ -11,6 +11,7 @@ import com.akto.dto.billing.FeatureAccess; import com.akto.dto.billing.SyncLimit; import com.akto.dto.monitoring.FilterConfig; +import com.akto.dto.monitoring.FilterConfig.FILTER_TYPE; import com.akto.dto.billing.Organization; import com.akto.dto.settings.DefaultPayload; import com.akto.dto.test_editor.ExecutorNode; @@ -31,6 +32,7 @@ import com.akto.test_editor.filter.data_operands_impl.ValidationResult; import com.akto.usage.UsageMetricCalculator; import com.akto.util.DbMode; +import com.akto.util.Pair; import com.akto.util.http_util.CoreHTTPClient; import com.akto.util.Constants; import com.mongodb.BasicDBObject; @@ -173,8 +175,8 @@ public int createCollectionBasedOnHostName(int id, String host) throws Exceptio } } - public static boolean isValidResponseParam(HttpResponseParams responseParam, Map filterMap, Map> executorNodesMap){ - boolean isValidResponseParam = false; + public static FILTER_TYPE isValidResponseParam(HttpResponseParams responseParam, Map filterMap, Map> executorNodesMap){ + FILTER_TYPE filterType = FILTER_TYPE.UNCHANGED; String message = responseParam.getOrig(); RawApi rawApi = RawApi.buildFromMessage(message); int apiCollectionId = responseParam.requestParams.getApiCollectionId(); @@ -196,35 +198,40 @@ public static boolean isValidResponseParam(HttpResponseParams responseParam, Map if (res.getIsValid()) { // handle custom filters here if(apiFilter.getId().equals(FilterConfig.DEFAULT_BLOCK_FILTER)){ - return false; + return FILTER_TYPE.BLOCKED; } // handle execute here - RawApi modifiedApi = new ParseAndExecute().execute(executorNodesMap.getOrDefault(apiFilter.getId(), new ArrayList<>()), rawApi, apiInfoKey, varMap, filterExecutionLogId); - responseParam = Utils.convertRawApiToHttpResponseParams(modifiedApi, responseParam); - isValidResponseParam = true; + List nodes = executorNodesMap.getOrDefault(apiFilter.getId(), new ArrayList<>()); + if(!nodes.isEmpty()){ + RawApi modifiedApi = new ParseAndExecute().execute(nodes, rawApi, apiInfoKey, varMap, filterExecutionLogId); + responseParam = Utils.convertRawApiToHttpResponseParams(modifiedApi, responseParam); + filterType = FILTER_TYPE.MODIFIED; + }else{ + filterType = FILTER_TYPE.ALLOWED; + } + } } catch (Exception e) { loggerMaker.errorAndAddToDb(e, String.format("Error in httpCallFilter %s", e.toString())); - isValidResponseParam = true; + filterType = FILTER_TYPE.UNCHANGED; } } - return isValidResponseParam; + return filterType; } int numberOfSyncs = 0; - public static List applyAdvancedFilters(List responseParams, Map> executorNodesMap, Map filterMap){ + public static Pair applyAdvancedFilters(HttpResponseParams responseParams, Map> executorNodesMap, Map filterMap){ if (filterMap != null && !filterMap.isEmpty()) { - List filteredParams = new ArrayList<>(); - for (HttpResponseParams responseParam : responseParams) { - if(isValidResponseParam(responseParam, filterMap, executorNodesMap)){ - filteredParams.add(responseParam); - } + FILTER_TYPE filterType = isValidResponseParam(responseParams, filterMap, executorNodesMap); + if(filterType.equals(FILTER_TYPE.BLOCKED)){ + return null; + }else{ + return new Pair(responseParams, filterType); } - return filteredParams; } - return responseParams; + return new Pair(responseParams, FILTER_TYPE.UNCHANGED); } public void syncFunction(List responseParams, boolean syncImmediately, boolean fetchAllSTI, AccountSettings accountSettings) { @@ -568,11 +575,12 @@ public List filterHttpResponseParams(List temp = applyAdvancedFilters(Arrays.asList(httpResponseParam), executorNodesMap, apiCatalogSync.advancedFilterMap); - if(temp.isEmpty()){ + Pair temp = applyAdvancedFilters(httpResponseParam, executorNodesMap, apiCatalogSync.advancedFilterMap); + HttpResponseParams param = temp.getFirst(); + if(param == null){ continue; }else{ - httpResponseParam = temp.get(0); + httpResponseParam = param; } int apiCollectionId = createApiCollectionId(httpResponseParam); diff --git a/apps/dashboard/src/main/java/com/akto/utils/jobs/CleanInventory.java b/apps/dashboard/src/main/java/com/akto/utils/jobs/CleanInventory.java index 7c4148ae73..7a6a8bc45f 100644 --- a/apps/dashboard/src/main/java/com/akto/utils/jobs/CleanInventory.java +++ b/apps/dashboard/src/main/java/com/akto/utils/jobs/CleanInventory.java @@ -33,6 +33,7 @@ import com.akto.dto.ApiCollection; import com.akto.dto.HttpResponseParams; import com.akto.dto.monitoring.FilterConfig; +import com.akto.dto.monitoring.FilterConfig.FILTER_TYPE; import com.akto.dto.test_editor.ExecutorNode; import com.akto.dto.test_editor.YamlTemplate; import com.akto.dto.traffic.Key; @@ -45,6 +46,7 @@ import com.akto.parsers.HttpCallParser; import com.akto.test_editor.execution.ParseAndExecute; import com.akto.util.AccountTask; +import com.akto.util.Pair; import com.mongodb.BasicDBObject; import com.mongodb.client.model.Filters; import com.mongodb.client.model.Sorts; @@ -140,6 +142,7 @@ public static void cleanFilteredSampleDataFromAdvancedFilters(List toBeDeleted = new ArrayList<>(); + List toMove = new ArrayList<>(); for(SampleData sampleData: sampleDataList) { try { List samples = sampleData.getSamples(); @@ -155,8 +158,9 @@ public static void cleanFilteredSampleDataFromAdvancedFilters(List> executorNodesMap = ParseAndExecute.createExecutorNodeMap(filterMap); - List temp = HttpCallParser.applyAdvancedFilters(Arrays.asList(httpResponseParams), executorNodesMap, filterMap); - - if(!temp.isEmpty()){ + Pair temp = HttpCallParser.applyAdvancedFilters(httpResponseParams, executorNodesMap, filterMap); + HttpResponseParams param = temp.getFirst(); + + if(param != null){ + allMatchDefault = false; + if(temp.getSecond().equals(FILTER_TYPE.MODIFIED)){ + movingApi = true; + } + }else{ allMatchDefault = true; - httpResponseParams = temp.get(0); - - // to do moving of sample data to new collections } } } } - if (allMatchDefault) { + if(movingApi){ + toMove.add(sampleData.getId()); + logger.info("[BadApisUpdater] Updating bad from template API: " + sampleData.getId(), LogDb.DASHBOARD); + } + + else if (allMatchDefault) { // writer.write(sampleData.toString()); toBeDeleted.add(sampleData.getId()); - logger.info("[BadApisRemover] " + isNetsparkerPresent + " Deleting bad API: " + sampleData.getId(), LogDb.DASHBOARD); + logger.info("[BadApisRemover] " + isNetsparkerPresent + " Deleting bad API from template: " + sampleData.getId(), LogDb.DASHBOARD); } else { - logger.info("[BadApisRemover] " + isNetsparkerPresent + " Keeping bad API: " + sampleData.getId(), LogDb.DASHBOARD); + logger.info("[BadApisRemover] " + isNetsparkerPresent + " Keeping bad API from template: " + sampleData.getId(), LogDb.DASHBOARD); } } catch (Exception e) { loggerMaker.errorAndAddToDb("[BadApisRemover] Couldn't delete an api for default payload: " + sampleData.getId() + e.getMessage(), LogDb.DASHBOARD); @@ -193,6 +205,8 @@ public static void cleanFilteredSampleDataFromAdvancedFilters(List> wordLists) {