Skip to content

Commit

Permalink
Merge pull request #1499 from akto-api-security/feature/add_execution_mr
Browse files Browse the repository at this point in the history 
Feature/add execution mr
  • Loading branch information
@Ark2307
Ark2307 authored Sep 17, 2024
2 parents 6f1485e + fb2a04e commit 5ceba2e
Show file tree
Hide file tree
Showing 71 changed files with 1,651 additions and 472 deletions.
5 changes: 3 additions & 2 deletions apps/api-runtime/src/main/java/com/akto/dependency/DependencyAnalyser.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@
import java.util.*;

import static com.akto.util.HttpRequestResponseUtils.extractValuesFromPayload;
import static com.akto.runtime.utils.Utils.parseCookie;

public class DependencyAnalyser {
Store valueStore; // this is to store all the values seen in response payload
Expand Down Expand Up @@ -131,7 +132,7 @@ public void analyse(String message, int finalApiCollectionId) {
for (String param: responseHeaders.keySet()) {
List<String> values = responseHeaders.get(param);
if (param.equalsIgnoreCase("set-cookie")) {
Map<String,String> cookieMap = AuthPolicy.parseCookie(values);
Map<String,String> cookieMap = parseCookie(values);
for (String cookieKey: cookieMap.keySet()) {
String cookieVal = cookieMap.get(cookieKey);
if (!filterValues(cookieVal)) continue;
Expand Down Expand Up @@ -192,7 +193,7 @@ public void analyse(String message, int finalApiCollectionId) {
List<String> values = requestHeaders.get(param);

if (param.equals("cookie")) {
Map<String,String> cookieMap = AuthPolicy.parseCookie(values);
Map<String,String> cookieMap = parseCookie(values);
for (String cookieKey: cookieMap.keySet()) {
String cookieValue = cookieMap.get(cookieKey);
processRequestParam(cookieKey, new HashSet<>(Collections.singletonList(cookieValue)), combinedUrl, false, true, doInterCollectionMatch);
Expand Down
185 changes: 116 additions & 69 deletions apps/api-runtime/src/main/java/com/akto/parsers/HttpCallParser.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,19 +8,26 @@
import com.akto.dao.traffic_metrics.TrafficMetricsDao;
import com.akto.dependency.DependencyAnalyser;
import com.akto.dto.*;
import com.akto.dto.ApiInfo.ApiInfoKey;
import com.akto.dto.billing.FeatureAccess;
import com.akto.dto.billing.SyncLimit;
import com.akto.dto.monitoring.FilterConfig;
import com.akto.dto.billing.Organization;
import com.akto.dto.settings.DefaultPayload;
import com.akto.dto.traffic_metrics.TrafficMetrics;
import com.akto.dto.type.URLMethods.Method;
import com.akto.dto.usage.MetricTypes;
import com.akto.graphql.GraphQLUtils;
import com.akto.log.LoggerMaker;
import com.akto.log.LoggerMaker.LogDb;
import com.akto.runtime.APICatalogSync;
import com.akto.runtime.Main;
import com.akto.runtime.RuntimeUtil;
import com.akto.runtime.URLAggregator;
import com.akto.runtime.parser.SampleParser;
import com.akto.runtime.utils.Utils;
import com.akto.test_editor.execution.VariableResolver;
import com.akto.test_editor.filter.data_operands_impl.ValidationResult;
import com.akto.usage.UsageMetricCalculator;
import com.akto.util.JSONUtils;
import com.akto.util.http_util.CoreHTTPClient;
Expand All @@ -32,11 +39,11 @@
import com.mongodb.client.model.*;
import okhttp3.*;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.math.NumberUtils;
import org.bson.conversions.Bson;

import java.io.IOException;
import java.util.*;
import java.util.Map.Entry;
import java.util.concurrent.ConcurrentLinkedQueue;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
Expand All @@ -45,6 +52,7 @@
import java.util.regex.Pattern;

import static com.akto.runtime.RuntimeUtil.matchesDefaultPayload;
import static com.akto.testing.Utils.validateFilter;

public class HttpCallParser {
private final int sync_threshold_count;
Expand Down Expand Up @@ -162,13 +170,64 @@ public int createCollectionBasedOnHostName(int id, String host) throws Exceptio

int numberOfSyncs = 0;

private List<HttpResponseParams> applyAdvancedFilters(List<HttpResponseParams> responseParams){
Map<String,FilterConfig> filterMap = apiCatalogSync.advancedFilterMap;

if (filterMap != null && !filterMap.isEmpty()) {
List<HttpResponseParams> filteredParams = new ArrayList<>();
for (HttpResponseParams responseParam : responseParams) {
for (Entry<String, FilterConfig> apiFilterEntry : filterMap.entrySet()) {
try {
FilterConfig apiFilter = apiFilterEntry.getValue();
String message = responseParam.getOrig();
RawApi rawApi = RawApi.buildFromMessage(message);
int apiCollectionId = createApiCollectionId(responseParam);
responseParam.requestParams.setApiCollectionId(apiCollectionId);
String url = responseParam.getRequestParams().getURL();
Method method = Method.valueOf(responseParam.getRequestParams().getMethod());
ApiInfoKey apiInfoKey = new ApiInfoKey(apiCollectionId, url, method);
Map<String, Object> varMap = apiFilter.resolveVarMap();
VariableResolver.resolveWordList(varMap, new HashMap<ApiInfoKey, List<String>>() {
{
put(apiInfoKey, Arrays.asList(message));
}
}, apiInfoKey);
String filterExecutionLogId = UUID.randomUUID().toString();
ValidationResult res = validateFilter(apiFilter.getFilter().getNode(), rawApi,
apiInfoKey, varMap, filterExecutionLogId);
if (res.getIsValid()) {
// TODO apply execution

filteredParams.add(responseParam);
break;
}
} catch (Exception e) {
loggerMaker.errorAndAddToDb(e, String.format("Error in httpCallFilter %s", e.toString()));
return responseParams;
}
}
}
return filteredParams;
}
return responseParams;
}

public void syncFunction(List<HttpResponseParams> responseParams, boolean syncImmediately, boolean fetchAllSTI, AccountSettings accountSettings) {
// USE ONLY filteredResponseParams and not responseParams
List<HttpResponseParams> filteredResponseParams = responseParams;
if (accountSettings != null && accountSettings.getDefaultPayloads() != null) {
filteredResponseParams = filterDefaultPayloads(filteredResponseParams, accountSettings.getDefaultPayloads());
}
filteredResponseParams = filterHttpResponseParams(filteredResponseParams, accountSettings);
List<String> redundantList = new ArrayList<>();
if(accountSettings !=null && !accountSettings.getAllowRedundantEndpointsList().isEmpty()){
redundantList = accountSettings.getAllowRedundantEndpointsList();
}
Pattern regexPattern = Utils.createRegexPatternFromList(redundantList);
filteredResponseParams = filterHttpResponseParams(filteredResponseParams, redundantList, regexPattern);

// add advanced filters
filteredResponseParams = applyAdvancedFilters(filteredResponseParams);

boolean isHarOrPcap = aggregate(filteredResponseParams, aggregatorMap);

for (int apiCollectionId: aggregatorMap.keySet()) {
Expand Down Expand Up @@ -336,17 +395,7 @@ public void incTrafficMetrics(TrafficMetrics.Key key, int value) {

public static final String CONTENT_TYPE = "CONTENT-TYPE";

public boolean isRedundantEndpoint(String url, List<String> discardedUrlList){
StringJoiner joiner = new StringJoiner("|", ".*\\.(", ")(\\?.*)?");
for (String extension : discardedUrlList) {
if(extension.startsWith(CONTENT_TYPE)){
continue;
}
joiner.add(extension);
}
String regex = joiner.toString();

Pattern pattern = Pattern.compile(regex);
public boolean isRedundantEndpoint(String url, Pattern pattern){
Matcher matcher = pattern.matcher(url);
return matcher.matches();
}
Expand All @@ -359,6 +408,54 @@ private boolean isInvalidContentType(String contentType){
return res;
}

public int createApiCollectionId(HttpResponseParams httpResponseParam){
int apiCollectionId;
String hostName = getHeaderValue(httpResponseParam.getRequestParams().getHeaders(), "host");

if (hostName != null && !hostNameToIdMap.containsKey(hostName) && RuntimeUtil.hasSpecialCharacters(hostName)) {
hostName = "Special_Char_Host";
}

int vxlanId = httpResponseParam.requestParams.getApiCollectionId();

if (useHostCondition(hostName, httpResponseParam.getSource())) {
hostName = hostName.toLowerCase();
hostName = hostName.trim();

String key = hostName;

if (hostNameToIdMap.containsKey(key)) {
apiCollectionId = hostNameToIdMap.get(key);

} else {
int id = hostName.hashCode();
try {

apiCollectionId = createCollectionBasedOnHostName(id, hostName);

hostNameToIdMap.put(key, apiCollectionId);
} catch (Exception e) {
loggerMaker.errorAndAddToDb("Failed to create collection for host : " + hostName, LogDb.RUNTIME);
createCollectionSimple(vxlanId);
hostNameToIdMap.put("null " + vxlanId, vxlanId);
apiCollectionId = httpResponseParam.requestParams.getApiCollectionId();
}
}

} else {
String key = "null" + " " + vxlanId;
if (!hostNameToIdMap.containsKey(key)) {
createCollectionSimple(vxlanId);
hostNameToIdMap.put(key, vxlanId);
}

apiCollectionId = vxlanId;
}
return apiCollectionId;
}



private boolean isBlankResponseBodyForGET(String method, String contentType, String matchContentType,
String responseBody) {
boolean res = true;
Expand All @@ -381,7 +478,7 @@ private boolean isBlankResponseBodyForGET(String method, String contentType, Str
return res;
}

public List<HttpResponseParams> filterHttpResponseParams(List<HttpResponseParams> httpResponseParamsList, AccountSettings accountSettings) {
public List<HttpResponseParams> filterHttpResponseParams(List<HttpResponseParams> httpResponseParamsList, List<String> redundantUrlsList, Pattern pattern) {
List<HttpResponseParams> filteredResponseParams = new ArrayList<>();
int originalSize = httpResponseParamsList.size();
for (HttpResponseParams httpResponseParam: httpResponseParamsList) {
Expand All @@ -402,8 +499,8 @@ public List<HttpResponseParams> filterHttpResponseParams(List<HttpResponseParams
if (ignoreAktoFlag != null) continue;

// check for garbage points here
if(accountSettings != null && accountSettings.getAllowRedundantEndpointsList() != null){
if(isRedundantEndpoint(httpResponseParam.getRequestParams().getURL(), accountSettings.getAllowRedundantEndpointsList())){
if(redundantUrlsList != null && !redundantUrlsList.isEmpty()){
if(isRedundantEndpoint(httpResponseParam.getRequestParams().getURL(), pattern)){
continue;
}
List<String> contentTypeList = (List<String>) httpResponseParam.getRequestParams().getHeaders().getOrDefault("content-type", new ArrayList<>());
Expand All @@ -421,7 +518,7 @@ public List<HttpResponseParams> filterHttpResponseParams(List<HttpResponseParams
String method = httpResponseParam.getRequestParams().getMethod();
String responseBody = httpResponseParam.getPayload();
boolean ignore = false;
for (String extension : accountSettings.getAllowRedundantEndpointsList()) {
for (String extension : redundantUrlsList) {
if(extension.startsWith(CONTENT_TYPE)){
String matchContentType = extension.split(" ")[1];
if(isBlankResponseBodyForGET(method, allContentTypes, matchContentType, responseBody)){
Expand All @@ -439,58 +536,8 @@ public List<HttpResponseParams> filterHttpResponseParams(List<HttpResponseParams
}

}

Map<String, List<String>> reqHeaders = httpResponseParam.getRequestParams().getHeaders();
String hostName = getHeaderValue(reqHeaders, "host");

if (StringUtils.isEmpty(hostName)) {
hostName = getHeaderValue(reqHeaders, "authority");
if (StringUtils.isEmpty(hostName)) {
hostName = getHeaderValue(reqHeaders, ":authority");
}

if (!StringUtils.isEmpty(hostName)) {
reqHeaders.put("host", Collections.singletonList(hostName));
}
}

int vxlanId = httpResponseParam.requestParams.getApiCollectionId();
int apiCollectionId ;

if (useHostCondition(hostName, httpResponseParam.getSource())) {
hostName = hostName.toLowerCase();
hostName = hostName.trim();

String key = hostName;

if (hostNameToIdMap.containsKey(key)) {
apiCollectionId = hostNameToIdMap.get(key);

} else {
int id = hostName.hashCode();
try {

apiCollectionId = createCollectionBasedOnHostName(id, hostName);

hostNameToIdMap.put(key, apiCollectionId);
} catch (Exception e) {
loggerMaker.errorAndAddToDb("Failed to create collection for host : " + hostName, LogDb.RUNTIME);
createCollectionSimple(vxlanId);
hostNameToIdMap.put("null " + vxlanId, vxlanId);
apiCollectionId = httpResponseParam.requestParams.getApiCollectionId();
}
}

} else {
String key = "null" + " " + vxlanId;
if (!hostNameToIdMap.containsKey(key)) {
createCollectionSimple(vxlanId);
hostNameToIdMap.put(key, vxlanId);
}

apiCollectionId = vxlanId;
}

int apiCollectionId = createApiCollectionId(httpResponseParam);

httpResponseParam.requestParams.setApiCollectionId(apiCollectionId);

List<HttpResponseParams> responseParamsList = GraphQLUtils.getUtils().parseGraphqlResponseParam(httpResponseParam);
Expand Down
8 changes: 7 additions & 1 deletion apps/api-runtime/src/main/java/com/akto/runtime/APICatalogSync.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,9 +7,13 @@
import com.akto.DaoInit;
import com.akto.dao.*;
import com.akto.dao.context.Context;
import com.akto.dao.monitoring.FilterYamlTemplateDao;
import com.akto.dao.runtime_filters.AdvancedTrafficFiltersDao;
import com.akto.dto.*;
import com.akto.dto.billing.SyncLimit;
import com.akto.dto.dependency_flow.DependencyFlow;
import com.akto.dto.monitoring.FilterConfig;
import com.akto.dto.test_editor.YamlTemplate;
import com.akto.dto.traffic.Key;
import com.akto.dto.traffic.SampleData;
import com.akto.dto.traffic.TrafficInfo;
Expand Down Expand Up @@ -66,6 +70,7 @@ public class APICatalogSync {
public Map<SensitiveParamInfo, Boolean> sensitiveParamInfoBooleanMap;
public static boolean mergeAsyncOutside = true;
public BloomFilter<CharSequence> existingAPIsInDb = BloomFilter.create(Funnels.stringFunnel(Charsets.UTF_8), 1_000_000, 0.001 );
public Map<String, FilterConfig> advancedFilterMap = new HashMap<>();

public APICatalogSync(String userIdentifier,int thresh, boolean fetchAllSTI) {
this(userIdentifier, thresh, fetchAllSTI, true);
Expand Down Expand Up @@ -1534,7 +1539,8 @@ public void buildFromDB(boolean calcDiff, boolean fetchAllSTI) {
this.delta = new HashMap<>();
}


List<YamlTemplate> advancedFilterTemplates = AdvancedTrafficFiltersDao.instance.findAll(Filters.ne(YamlTemplate.INACTIVE, true));
advancedFilterMap = FilterYamlTemplateDao.instance.fetchFilterConfig(false, advancedFilterTemplates, true);
try {
// fetchAllSTI check added to make sure only runs in dashboard
if (!fetchAllSTI) {
Expand Down
34 changes: 1 addition & 33 deletions apps/api-runtime/src/main/java/com/akto/runtime/RelationshipSync.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,8 +12,6 @@
import com.fasterxml.jackson.databind.node.ArrayNode;
import com.mongodb.client.model.*;
import org.bson.conversions.Bson;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.akto.dto.HttpResponseParams;
import com.akto.dto.HttpRequestParams;

Expand Down Expand Up @@ -231,37 +229,7 @@ public static Map<String, Set<String>> extractAllValuesFromPayload(String payloa

public static void extractAllValuesFromPayload(JsonNode node, List<String> params, Map<String, Set<String>> values) {
// TODO: null values remove
if (node == null) return;
if (node.isValueNode()) {
String textValue = node.asText();
if (textValue != null) {
String param = String.join("",params);
if (param.startsWith("#")) {
param = param.substring(1);
}
if (!values.containsKey(param)) {
values.put(param, new HashSet<>());
}
values.get(param).add(textValue);
}
} else if (node.isArray()) {
ArrayNode arrayNode = (ArrayNode) node;
for(int i = 0; i < arrayNode.size(); i++) {
JsonNode arrayElement = arrayNode.get(i);
params.add("#$");
extractAllValuesFromPayload(arrayElement, params, values);
params.remove(params.size()-1);
}
} else {
Iterator<String> fieldNames = node.fieldNames();
while(fieldNames.hasNext()) {
String fieldName = fieldNames.next();
params.add("#"+fieldName);
JsonNode fieldValue = node.get(fieldName);
extractAllValuesFromPayload(fieldValue, params,values);
params.remove(params.size()-1);
}
}
com.akto.runtime.RuntimeUtil.extractAllValuesFromPayload(node,params,values);

}

Expand Down
Loading

0 comments on commit 5ceba2e

Please sign in to comment.