Skip to content

Commit

Permalink
Merge pull request #748 from akto-api-security/develop
Browse files Browse the repository at this point in the history 
Develop
  • Loading branch information
@notshivansh
notshivansh authored Nov 30, 2023
2 parents 72db65f + d1e53f4 commit 2fadb9b
Show file tree
Hide file tree
Showing 7 changed files with 51 additions and 47 deletions.
3 changes: 2 additions & 1 deletion apps/dashboard/web/polaris_web/web/src/util/request.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -99,8 +99,9 @@ service.interceptors.response.use((response) => {
return response.data
}, err)

const black_list_apis = ['dashboard/accesstoken', 'api/fetchBurpPluginInfo', 'api/fetchActiveLoaders', 'api/fetchAllSubCategories']
async function raiseMixpanelEvent(api) {
if (api && api.indexOf("/api/fetchActiveLoaders") == -1) {
if (api && !black_list_apis.some(black_list_api => api.includes(black_list_api))) {
window.mixpanel.track(api)
}
}
Expand Down
5 changes: 3 additions & 2 deletions apps/dashboard/web/src/util/request.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -135,9 +135,10 @@ service.interceptors.response.use((response) => {
return response.data
}, err)

const black_list_apis = ['dashboard/accesstoken', 'api/fetchBurpPluginInfo', 'api/fetchActiveLoaders', 'api/fetchAllSubCategories']
async function raiseMixpanelEvent(api){
if (api && api.indexOf("/api/fetchActiveLoaders")==-1) {
window.mixpanel.track(api)
if (api && !black_list_apis.some(black_list_api => api.includes(black_list_api))) {
window.mixpanel.track(api);
}
}

Expand Down
11 changes: 8 additions & 3 deletions apps/testing/src/main/java/com/akto/test_editor/execution/Executor.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,8 @@ public List<TestResult> execute(ExecutorNode node, RawApi rawApi, Map<String, Ob

boolean requestSent = false;

List<String> error_messages = new ArrayList<>();

for (ExecutorNode reqNode: reqNodes.getChildNodes()) {
// make copy of varMap as well
List<RawApi> sampleRawApis = new ArrayList<>();
Expand All @@ -59,6 +61,7 @@ public List<TestResult> execute(ExecutorNode node, RawApi rawApi, Map<String, Ob
List<RawApi> testRawApis = new ArrayList<>();
testRawApis = singleReq.getRawApis();
if (testRawApis == null) {
error_messages.add(singleReq.getErrMsg());
continue;
}
boolean vulnerable = false;
Expand All @@ -77,17 +80,19 @@ public List<TestResult> execute(ExecutorNode node, RawApi rawApi, Map<String, Ob
}
vulnerable = res.getVulnerable();
} catch(Exception e) {
loggerMaker.errorAndAddToDb("error executing test request " + logId + " " + e.getMessage(), LogDb.TESTING);
error_messages.add("Error executing test request: " + e.getMessage());
loggerMaker.errorAndAddToDb("Error executing test request " + logId + " " + e.getMessage(), LogDb.TESTING);
}
}
}

if(result.isEmpty()){
if(requestSent){
result.add(new TestResult(null, rawApi.getOriginalMessage(), Collections.singletonList(TestError.API_REQUEST_FAILED.getMessage()), 0, false, TestResult.Confidence.HIGH, null));
error_messages.add(TestError.API_REQUEST_FAILED.getMessage());
} else {
result.add(new TestResult(null, rawApi.getOriginalMessage(), Collections.singletonList(TestError.NO_API_REQUEST.getMessage()), 0, false, TestResult.Confidence.HIGH, null));
error_messages.add(TestError.NO_API_REQUEST.getMessage());
}
result.add(new TestResult(null, rawApi.getOriginalMessage(), error_messages, 0, false, TestResult.Confidence.HIGH, null));
}

return result;
Expand Down
5 changes: 4 additions & 1 deletion apps/testing/src/main/java/com/akto/testing/ApiExecutor.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,9 @@
public class ApiExecutor {
private static final LoggerMaker loggerMaker = new LoggerMaker(ApiExecutor.class);

// Load only first 1 MiB of response body into memory.
private static final int MAX_RESPONSE_SIZE = 1024*1024;

private static OriginalHttpResponse common(Request request, boolean followRedirects) throws Exception {

Integer accountId = Context.accountId.get();
Expand Down Expand Up @@ -51,7 +54,7 @@ private static OriginalHttpResponse common(Request request, boolean followRedire
String body;
try {
response = call.execute();
ResponseBody responseBody = response.body();
ResponseBody responseBody = response.peekBody(MAX_RESPONSE_SIZE);
if (responseBody == null) {
throw new Exception("Couldn't read response body");
}
Expand Down
70 changes: 30 additions & 40 deletions apps/testing/src/main/java/com/akto/testing/TestExecutor.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,9 +36,13 @@
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.gson.Gson;
import com.mongodb.BasicDBObject;
import com.mongodb.client.model.Aggregates;
import com.mongodb.client.model.Filters;
import com.mongodb.client.model.Projections;
import com.mongodb.client.model.Sorts;
import com.mongodb.client.model.Updates;

import org.bson.conversions.Bson;
import org.bson.types.ObjectId;
import org.json.JSONObject;
import org.mortbay.util.ajax.JSON;
Expand Down Expand Up @@ -110,7 +114,7 @@ public void workflowInit (TestingRun testingRun, ObjectId summaryId) {
public void apiWiseInit(TestingRun testingRun, ObjectId summaryId) {
int accountId = Context.accountId.get();
int now = Context.now();
int maxConcurrentRequests = testingRun.getMaxConcurrentRequests() > 0 ? testingRun.getMaxConcurrentRequests() : 100;
int maxConcurrentRequests = testingRun.getMaxConcurrentRequests() > 0 ? Math.min( testingRun.getMaxConcurrentRequests(), 100) : 10;
TestingEndpoints testingEndpoints = testingRun.getTestingEndpoints();

SampleMessageStore sampleMessageStore = SampleMessageStore.create();
Expand Down Expand Up @@ -165,7 +169,7 @@ public void apiWiseInit(TestingRun testingRun, ObjectId summaryId) {

CountDownLatch latch = new CountDownLatch(apiInfoKeyList.size());
ExecutorService threadPool = Executors.newFixedThreadPool(maxConcurrentRequests);
List<Future<List<TestingRunResult>>> futureTestingRunResults = new ArrayList<>();
List<Future<Void>> futureTestingRunResults = new ArrayList<>();
Map<String, Integer> hostsToApiCollectionMap = new HashMap<>();

ConcurrentHashMap<String, String> subCategoryEndpointMap = new ConcurrentHashMap<>();
Expand Down Expand Up @@ -200,7 +204,7 @@ public void apiWiseInit(TestingRun testingRun, ObjectId summaryId) {
loggerMaker.errorAndAddToDb("Error while finding host: " + e, LogDb.TESTING);
}
try {
Future<List<TestingRunResult>> future = threadPool.submit(
Future<Void> future = threadPool.submit(
() -> startWithLatch(apiInfoKey,
testingRun.getTestIdConfig(),
testingRun.getId(),testingRun.getTestingRunConfig(), testingUtil, summaryId,
Expand All @@ -225,20 +229,6 @@ public void apiWiseInit(TestingRun testingRun, ObjectId summaryId) {

loggerMaker.infoAndAddToDb("Finished testing", LogDb.TESTING);

int totalResults = 0;
for (Future<List<TestingRunResult>> future: futureTestingRunResults) {
if (!future.isDone()) continue;
try {
if (!future.get().isEmpty()) {
int resultSize = future.get().size();
totalResults += resultSize;
}
} catch (InterruptedException | ExecutionException e) {
loggerMaker.errorAndAddToDb("Error while after running test : " + e, LogDb.TESTING);
}
}

loggerMaker.infoAndAddToDb("Finished adding " + totalResults + " testingRunResults", LogDb.TESTING);
}

public static void updateTestSummary(ObjectId summaryId){
Expand Down Expand Up @@ -269,8 +259,7 @@ public static void updateTestSummary(ObjectId summaryId){
Filters.eq(TestingRunResult.VULNERABLE, true)),
limit,
skip,
Projections.include(
TestingRunResult.TEST_RESULTS));
Projections.include("testResults.confidence"));

loggerMaker.infoAndAddToDb("Reading " + testingRunResults.size() + " vulnerable testingRunResults",
LogDb.TESTING);
Expand Down Expand Up @@ -463,7 +452,7 @@ public Map<String, Object> generateResponseMap(String payloadStr, Map<String, Li
return respMap;
}

public List<TestingRunResult> startWithLatch(
public Void startWithLatch(
ApiInfo.ApiInfoKey apiInfoKey, int testIdConfig, ObjectId testRunId, TestingRunConfig testingRunConfig,
TestingUtil testingUtil, ObjectId testRunResultSummaryId, int accountId, CountDownLatch latch, int startTime,
int timeToKill, Map<String, TestConfig> testConfigMap, TestingRun testingRun,
Expand All @@ -472,35 +461,20 @@ public List<TestingRunResult> startWithLatch(
loggerMaker.infoAndAddToDb("Starting test for " + apiInfoKey, LogDb.TESTING);

Context.accountId.set(accountId);
List<TestingRunResult> testingRunResults = new ArrayList<>();
int now = Context.now();
if ( timeToKill <= 0 || now - startTime <= timeToKill) {
try {
// todo: commented out older one
// testingRunResults = start(apiInfoKey, testIdConfig, testRunId, testingRunConfig, testingUtil, testRunResultSummaryId, testConfigMap);
testingRunResults = startTestNew(apiInfoKey, testRunId, testingRunConfig, testingUtil, testRunResultSummaryId, testConfigMap, subCategoryEndpointMap, apiInfoKeyToHostMap);
String size = testingRunResults.size()+"";
loggerMaker.infoAndAddToDb("testingRunResults size: " + size, LogDb.TESTING);
if (!testingRunResults.isEmpty()) {
trim(testingRunResults);
TestingRunResultDao.instance.insertMany(testingRunResults);
loggerMaker.infoAndAddToDb("Inserted testing results", LogDb.TESTING);
//Creating issues from testingRunResults
TestingIssuesHandler handler = new TestingIssuesHandler();
boolean triggeredByTestEditor = false;
if (testingRun.getTriggeredBy() != null) {
triggeredByTestEditor = testingRun.getTriggeredBy().equals("test_editor");
}
handler.handleIssuesCreationFromTestingRunResults(testingRunResults, triggeredByTestEditor); // pass new field here
}
startTestNew(apiInfoKey, testRunId, testingRunConfig, testingUtil, testRunResultSummaryId, testConfigMap, subCategoryEndpointMap, apiInfoKeyToHostMap);
} catch (Exception e) {
e.printStackTrace();
loggerMaker.errorAndAddToDb("error while running tests: " + e, LogDb.TESTING);
}
}

latch.countDown();
return testingRunResults;
return null;
}

public static void trim(TestingRunResult testingRunResult) {
Expand Down Expand Up @@ -535,15 +509,30 @@ public void trim(List<TestingRunResult> testingRunResults) {
}
}

public List<TestingRunResult> startTestNew(ApiInfo.ApiInfoKey apiInfoKey, ObjectId testRunId,
public void insertResultsAndMakeIssues(List<TestingRunResult> testingRunResults) {
int resultSize = testingRunResults.size();
if (resultSize > 0) {
loggerMaker.infoAndAddToDb("testingRunResults size: " + resultSize, LogDb.TESTING);
trim(testingRunResults);
TestingRunResultDao.instance.insertMany(testingRunResults);
loggerMaker.infoAndAddToDb("Inserted testing results", LogDb.TESTING);
TestingIssuesHandler handler = new TestingIssuesHandler();
boolean triggeredByTestEditor = false;
handler.handleIssuesCreationFromTestingRunResults(testingRunResults, triggeredByTestEditor);
testingRunResults.clear();
}
}

public void startTestNew(ApiInfo.ApiInfoKey apiInfoKey, ObjectId testRunId,
TestingRunConfig testingRunConfig, TestingUtil testingUtil,
ObjectId testRunResultSummaryId, Map<String, TestConfig> testConfigMap,
ConcurrentHashMap<String, String> subCategoryEndpointMap, Map<ApiInfoKey, String> apiInfoKeyToHostMap) {
List<TestingRunResult> testingRunResults = new ArrayList<>();

List<String> testSubCategories = testingRunConfig == null ? new ArrayList<>() : testingRunConfig.getTestSubCategoryList();

for (String testSubCategory: testSubCategories) {
List<TestingRunResult> testingRunResults = new ArrayList<>();

TestConfig testConfig = testConfigMap.get(testSubCategory);
if (testConfig == null) continue;
TestingRunResult testingRunResult = null;
Expand All @@ -557,9 +546,10 @@ public List<TestingRunResult> startTestNew(ApiInfo.ApiInfoKey apiInfoKey, Object
e.printStackTrace();
}
if (testingRunResult != null) testingRunResults.add(testingRunResult);

insertResultsAndMakeIssues(testingRunResults);
}

return testingRunResults;
}

public boolean applyRunOnceCheck(ApiInfoKey apiInfoKey, TestConfig testConfig, ConcurrentHashMap<String, String> subCategoryEndpointMap, Map<ApiInfoKey, String> apiInfoKeyToHostMap, String testSubCategory) {
Expand Down
3 changes: 3 additions & 0 deletions libs/dao/src/main/java/com/akto/dao/testing/TestingRunResultDao.java
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package com.akto.dao.testing;

import com.akto.dao.AccountsContextDao;
import com.akto.dao.MCollection;
import com.akto.dao.context.Context;
import com.akto.dto.ApiInfo;
import com.akto.dto.testing.TestingRunResult;
Expand Down Expand Up @@ -87,6 +88,8 @@ public void createIndicesIfAbsent() {
Bson summaryIndex = Indexes.descending(Arrays.asList(TestingRunResult.TEST_RUN_RESULT_SUMMARY_ID, Constants.ID));
createIndexIfAbsent(dbName, getCollName(), summaryIndex, new IndexOptions().name("testRunResultSummaryId_-1__id_-1"));

MCollection.createIndexIfAbsent(getDBName(), getCollName(),
new String[] { TestingRunResult.TEST_RUN_RESULT_SUMMARY_ID, TestingRunResult.VULNERABLE, Constants.ID }, false);
}

}
1 change: 1 addition & 0 deletions libs/dao/src/main/java/com/akto/dto/testing/GenericTestResult.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
public class GenericTestResult {

private boolean vulnerable;
public static final String _CONFIDENCE = "confidence";
private Confidence confidence = Confidence.HIGH;

public GenericTestResult() {
Expand Down

0 comments on commit 2fadb9b

Please sign in to comment.