Skip to content

Commit

Permalink
Merge pull request #1500 from akto-api-security/hotfix/fix_clean_up
Browse files Browse the repository at this point in the history 
Fix logs
  • Loading branch information
@notshivansh
notshivansh authored Sep 16, 2024
2 parents 60b3805 + c7556ed commit f13d95e
Show file tree
Hide file tree
Showing 3 changed files with 55 additions and 29 deletions.
46 changes: 27 additions & 19 deletions apps/api-runtime/src/main/java/com/akto/parsers/HttpCallParser.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@
import com.akto.dto.billing.FeatureAccess;
import com.akto.dto.billing.SyncLimit;
import com.akto.dto.monitoring.FilterConfig;
import com.akto.dto.monitoring.FilterConfig.FILTER_TYPE;
import com.akto.dto.billing.Organization;
import com.akto.dto.settings.DefaultPayload;
import com.akto.dto.test_editor.ExecutorNode;
Expand All @@ -31,6 +32,7 @@
import com.akto.test_editor.filter.data_operands_impl.ValidationResult;
import com.akto.usage.UsageMetricCalculator;
import com.akto.util.DbMode;
import com.akto.util.Pair;
import com.akto.util.http_util.CoreHTTPClient;
import com.akto.util.Constants;
import com.mongodb.BasicDBObject;
Expand Down Expand Up @@ -173,8 +175,8 @@ public int createCollectionBasedOnHostName(int id, String host) throws Exceptio
}
}

public static boolean isValidResponseParam(HttpResponseParams responseParam, Map<String, FilterConfig> filterMap, Map<String, List<ExecutorNode>> executorNodesMap){
boolean isValidResponseParam = false;
public static FILTER_TYPE isValidResponseParam(HttpResponseParams responseParam, Map<String, FilterConfig> filterMap, Map<String, List<ExecutorNode>> executorNodesMap){
FILTER_TYPE filterType = FILTER_TYPE.UNCHANGED;
String message = responseParam.getOrig();
RawApi rawApi = RawApi.buildFromMessage(message);
int apiCollectionId = responseParam.requestParams.getApiCollectionId();
Expand All @@ -196,35 +198,40 @@ public static boolean isValidResponseParam(HttpResponseParams responseParam, Map
if (res.getIsValid()) {
// handle custom filters here
if(apiFilter.getId().equals(FilterConfig.DEFAULT_BLOCK_FILTER)){
return false;
return FILTER_TYPE.BLOCKED;
}

// handle execute here
RawApi modifiedApi = new ParseAndExecute().execute(executorNodesMap.getOrDefault(apiFilter.getId(), new ArrayList<>()), rawApi, apiInfoKey, varMap, filterExecutionLogId);
responseParam = Utils.convertRawApiToHttpResponseParams(modifiedApi, responseParam);
isValidResponseParam = true;
List<ExecutorNode> nodes = executorNodesMap.getOrDefault(apiFilter.getId(), new ArrayList<>());
if(!nodes.isEmpty()){
RawApi modifiedApi = new ParseAndExecute().execute(nodes, rawApi, apiInfoKey, varMap, filterExecutionLogId);
responseParam = Utils.convertRawApiToHttpResponseParams(modifiedApi, responseParam);
filterType = FILTER_TYPE.MODIFIED;
}else{
filterType = FILTER_TYPE.ALLOWED;
}

}
} catch (Exception e) {
loggerMaker.errorAndAddToDb(e, String.format("Error in httpCallFilter %s", e.toString()));
isValidResponseParam = true;
filterType = FILTER_TYPE.UNCHANGED;
}
}
return isValidResponseParam;
return filterType;
}

int numberOfSyncs = 0;

public static List<HttpResponseParams> applyAdvancedFilters(List<HttpResponseParams> responseParams, Map<String, List<ExecutorNode>> executorNodesMap, Map<String,FilterConfig> filterMap){
public static Pair<HttpResponseParams,FILTER_TYPE> applyAdvancedFilters(HttpResponseParams responseParams, Map<String, List<ExecutorNode>> executorNodesMap, Map<String,FilterConfig> filterMap){
if (filterMap != null && !filterMap.isEmpty()) {
List<HttpResponseParams> filteredParams = new ArrayList<>();
for (HttpResponseParams responseParam : responseParams) {
if(isValidResponseParam(responseParam, filterMap, executorNodesMap)){
filteredParams.add(responseParam);
}
FILTER_TYPE filterType = isValidResponseParam(responseParams, filterMap, executorNodesMap);
if(filterType.equals(FILTER_TYPE.BLOCKED)){
return null;
}else{
return new Pair<HttpResponseParams,FilterConfig.FILTER_TYPE>(responseParams, filterType);
}
return filteredParams;
}
return responseParams;
return new Pair<HttpResponseParams,FilterConfig.FILTER_TYPE>(responseParams, FILTER_TYPE.UNCHANGED);
}

public void syncFunction(List<HttpResponseParams> responseParams, boolean syncImmediately, boolean fetchAllSTI, AccountSettings accountSettings) {
Expand Down Expand Up @@ -568,11 +575,12 @@ public List<HttpResponseParams> filterHttpResponseParams(List<HttpResponseParams

}

List<HttpResponseParams> temp = applyAdvancedFilters(Arrays.asList(httpResponseParam), executorNodesMap, apiCatalogSync.advancedFilterMap);
if(temp.isEmpty()){
Pair<HttpResponseParams,FILTER_TYPE> temp = applyAdvancedFilters(httpResponseParam, executorNodesMap, apiCatalogSync.advancedFilterMap);
HttpResponseParams param = temp.getFirst();
if(param == null){
continue;
}else{
httpResponseParam = temp.get(0);
httpResponseParam = param;
}
int apiCollectionId = createApiCollectionId(httpResponseParam);

Expand Down
34 changes: 24 additions & 10 deletions apps/dashboard/src/main/java/com/akto/utils/jobs/CleanInventory.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@
import com.akto.dto.ApiCollection;
import com.akto.dto.HttpResponseParams;
import com.akto.dto.monitoring.FilterConfig;
import com.akto.dto.monitoring.FilterConfig.FILTER_TYPE;
import com.akto.dto.test_editor.ExecutorNode;
import com.akto.dto.test_editor.YamlTemplate;
import com.akto.dto.traffic.Key;
Expand All @@ -45,6 +46,7 @@
import com.akto.parsers.HttpCallParser;
import com.akto.test_editor.execution.ParseAndExecute;
import com.akto.util.AccountTask;
import com.akto.util.Pair;
import com.mongodb.BasicDBObject;
import com.mongodb.client.model.Filters;
import com.mongodb.client.model.Sorts;
Expand Down Expand Up @@ -140,6 +142,7 @@ public static void cleanFilteredSampleDataFromAdvancedFilters(List<ApiCollection
sampleDataList = SampleDataDao.instance.findAll(filters, skip, limit, sort);
skip += limit;
List<Key> toBeDeleted = new ArrayList<>();
List<Key> toMove = new ArrayList<>();
for(SampleData sampleData: sampleDataList) {
try {
List<String> samples = sampleData.getSamples();
Expand All @@ -155,33 +158,42 @@ public static void cleanFilteredSampleDataFromAdvancedFilters(List<ApiCollection
}


boolean allMatchDefault = true;
boolean allMatchDefault = false;
boolean isNetsparkerPresent = false;
boolean movingApi = false;
for (String sample : samples) {
HttpResponseParams httpResponseParams = HttpCallParser.parseKafkaMessage(sample);
isNetsparkerPresent |= sample.toLowerCase().contains("netsparker");
if(httpResponseParams != null){
allMatchDefault = HttpCallParser.isRedundantEndpoint(httpResponseParams.getRequestParams().getURL(), pattern);
if(!allMatchDefault){
Map<String, List<ExecutorNode>> executorNodesMap = ParseAndExecute.createExecutorNodeMap(filterMap);
List<HttpResponseParams> temp = HttpCallParser.applyAdvancedFilters(Arrays.asList(httpResponseParams), executorNodesMap, filterMap);

if(!temp.isEmpty()){
Pair<HttpResponseParams,FILTER_TYPE> temp = HttpCallParser.applyAdvancedFilters(httpResponseParams, executorNodesMap, filterMap);
HttpResponseParams param = temp.getFirst();

if(param != null){
allMatchDefault = false;
if(temp.getSecond().equals(FILTER_TYPE.MODIFIED)){
movingApi = true;
}
}else{
allMatchDefault = true;
httpResponseParams = temp.get(0);

// to do moving of sample data to new collections
}
}
}
}

if (allMatchDefault) {
if(movingApi){
toMove.add(sampleData.getId());
logger.info("[BadApisUpdater] Updating bad from template API: " + sampleData.getId(), LogDb.DASHBOARD);
}

else if (allMatchDefault) {
// writer.write(sampleData.toString());
toBeDeleted.add(sampleData.getId());
logger.info("[BadApisRemover] " + isNetsparkerPresent + " Deleting bad API: " + sampleData.getId(), LogDb.DASHBOARD);
logger.info("[BadApisRemover] " + isNetsparkerPresent + " Deleting bad API from template: " + sampleData.getId(), LogDb.DASHBOARD);
} else {
logger.info("[BadApisRemover] " + isNetsparkerPresent + " Keeping bad API: " + sampleData.getId(), LogDb.DASHBOARD);
logger.info("[BadApisRemover] " + isNetsparkerPresent + " Keeping bad API from template: " + sampleData.getId(), LogDb.DASHBOARD);
}
} catch (Exception e) {
loggerMaker.errorAndAddToDb("[BadApisRemover] Couldn't delete an api for default payload: " + sampleData.getId() + e.getMessage(), LogDb.DASHBOARD);
Expand All @@ -193,6 +205,8 @@ public static void cleanFilteredSampleDataFromAdvancedFilters(List<ApiCollection
deleteApis(toBeDeleted);
}

String shouldMove = System.getenv("MOVE_REDUNDANT_APIS");

} while (!sampleDataList.isEmpty());

// writer.flush();
Expand Down
4 changes: 4 additions & 0 deletions libs/dao/src/main/java/com/akto/dto/monitoring/FilterConfig.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,10 @@ public class FilterConfig {
public static final String DEFAULT_ALLOW_FILTER = "DEFAULT_ALLOW_FILTER";
public static final String DEFAULT_BLOCK_FILTER = "DEFAULT_BLOCK_FILTER";

public enum FILTER_TYPE{
BLOCKED , ALLOWED, MODIFIED, UNCHANGED
}

private ExecutorConfigParserResult executor;

public FilterConfig(String id, ConfigParserResult filter, Map<String, List<String>> wordLists) {
Expand Down

0 comments on commit f13d95e

Please sign in to comment.