diff --git a/apps/dashboard/web/polaris_web/web/src/util/request.js b/apps/dashboard/web/polaris_web/web/src/util/request.js index c2cc48dfb7..e935508b48 100644 --- a/apps/dashboard/web/polaris_web/web/src/util/request.js +++ b/apps/dashboard/web/polaris_web/web/src/util/request.js @@ -99,8 +99,9 @@ service.interceptors.response.use((response) => { return response.data }, err) +const black_list_apis = ['dashboard/accesstoken', 'api/fetchBurpPluginInfo', 'api/fetchActiveLoaders', 'api/fetchAllSubCategories'] async function raiseMixpanelEvent(api) { - if (api && api.indexOf("/api/fetchActiveLoaders") == -1) { + if (api && !black_list_apis.some(black_list_api => api.includes(black_list_api))) { window.mixpanel.track(api) } } diff --git a/apps/dashboard/web/src/util/request.js b/apps/dashboard/web/src/util/request.js index 0ab1cf3329..696e93d95e 100644 --- a/apps/dashboard/web/src/util/request.js +++ b/apps/dashboard/web/src/util/request.js @@ -135,9 +135,10 @@ service.interceptors.response.use((response) => { return response.data }, err) +const black_list_apis = ['dashboard/accesstoken', 'api/fetchBurpPluginInfo', 'api/fetchActiveLoaders', 'api/fetchAllSubCategories'] async function raiseMixpanelEvent(api){ - if (api && api.indexOf("/api/fetchActiveLoaders")==-1) { - window.mixpanel.track(api) + if (api && !black_list_apis.some(black_list_api => api.includes(black_list_api))) { + window.mixpanel.track(api); } } diff --git a/apps/testing/src/main/java/com/akto/test_editor/execution/Executor.java b/apps/testing/src/main/java/com/akto/test_editor/execution/Executor.java index 361ddb2aa6..585d628d6a 100644 --- a/apps/testing/src/main/java/com/akto/test_editor/execution/Executor.java +++ b/apps/testing/src/main/java/com/akto/test_editor/execution/Executor.java @@ -50,6 +50,8 @@ public List execute(ExecutorNode node, RawApi rawApi, Map error_messages = new ArrayList<>(); + for (ExecutorNode reqNode: reqNodes.getChildNodes()) { // make copy of varMap as well List sampleRawApis = new ArrayList<>(); @@ -59,6 +61,7 @@ public List execute(ExecutorNode node, RawApi rawApi, Map testRawApis = new ArrayList<>(); testRawApis = singleReq.getRawApis(); if (testRawApis == null) { + error_messages.add(singleReq.getErrMsg()); continue; } boolean vulnerable = false; @@ -77,17 +80,19 @@ public List execute(ExecutorNode node, RawApi rawApi, Map 0 ? testingRun.getMaxConcurrentRequests() : 100; + int maxConcurrentRequests = testingRun.getMaxConcurrentRequests() > 0 ? Math.min( testingRun.getMaxConcurrentRequests(), 100) : 10; TestingEndpoints testingEndpoints = testingRun.getTestingEndpoints(); SampleMessageStore sampleMessageStore = SampleMessageStore.create(); @@ -165,7 +169,7 @@ public void apiWiseInit(TestingRun testingRun, ObjectId summaryId) { CountDownLatch latch = new CountDownLatch(apiInfoKeyList.size()); ExecutorService threadPool = Executors.newFixedThreadPool(maxConcurrentRequests); - List>> futureTestingRunResults = new ArrayList<>(); + List> futureTestingRunResults = new ArrayList<>(); Map hostsToApiCollectionMap = new HashMap<>(); ConcurrentHashMap subCategoryEndpointMap = new ConcurrentHashMap<>(); @@ -200,7 +204,7 @@ public void apiWiseInit(TestingRun testingRun, ObjectId summaryId) { loggerMaker.errorAndAddToDb("Error while finding host: " + e, LogDb.TESTING); } try { - Future> future = threadPool.submit( + Future future = threadPool.submit( () -> startWithLatch(apiInfoKey, testingRun.getTestIdConfig(), testingRun.getId(),testingRun.getTestingRunConfig(), testingUtil, summaryId, @@ -225,20 +229,6 @@ public void apiWiseInit(TestingRun testingRun, ObjectId summaryId) { loggerMaker.infoAndAddToDb("Finished testing", LogDb.TESTING); - int totalResults = 0; - for (Future> future: futureTestingRunResults) { - if (!future.isDone()) continue; - try { - if (!future.get().isEmpty()) { - int resultSize = future.get().size(); - totalResults += resultSize; - } - } catch (InterruptedException | ExecutionException e) { - loggerMaker.errorAndAddToDb("Error while after running test : " + e, LogDb.TESTING); - } - } - - loggerMaker.infoAndAddToDb("Finished adding " + totalResults + " testingRunResults", LogDb.TESTING); } public static void updateTestSummary(ObjectId summaryId){ @@ -269,8 +259,7 @@ public static void updateTestSummary(ObjectId summaryId){ Filters.eq(TestingRunResult.VULNERABLE, true)), limit, skip, - Projections.include( - TestingRunResult.TEST_RESULTS)); + Projections.include("testResults.confidence")); loggerMaker.infoAndAddToDb("Reading " + testingRunResults.size() + " vulnerable testingRunResults", LogDb.TESTING); @@ -463,7 +452,7 @@ public Map generateResponseMap(String payloadStr, Map startWithLatch( + public Void startWithLatch( ApiInfo.ApiInfoKey apiInfoKey, int testIdConfig, ObjectId testRunId, TestingRunConfig testingRunConfig, TestingUtil testingUtil, ObjectId testRunResultSummaryId, int accountId, CountDownLatch latch, int startTime, int timeToKill, Map testConfigMap, TestingRun testingRun, @@ -472,27 +461,12 @@ public List startWithLatch( loggerMaker.infoAndAddToDb("Starting test for " + apiInfoKey, LogDb.TESTING); Context.accountId.set(accountId); - List testingRunResults = new ArrayList<>(); int now = Context.now(); if ( timeToKill <= 0 || now - startTime <= timeToKill) { try { // todo: commented out older one // testingRunResults = start(apiInfoKey, testIdConfig, testRunId, testingRunConfig, testingUtil, testRunResultSummaryId, testConfigMap); - testingRunResults = startTestNew(apiInfoKey, testRunId, testingRunConfig, testingUtil, testRunResultSummaryId, testConfigMap, subCategoryEndpointMap, apiInfoKeyToHostMap); - String size = testingRunResults.size()+""; - loggerMaker.infoAndAddToDb("testingRunResults size: " + size, LogDb.TESTING); - if (!testingRunResults.isEmpty()) { - trim(testingRunResults); - TestingRunResultDao.instance.insertMany(testingRunResults); - loggerMaker.infoAndAddToDb("Inserted testing results", LogDb.TESTING); - //Creating issues from testingRunResults - TestingIssuesHandler handler = new TestingIssuesHandler(); - boolean triggeredByTestEditor = false; - if (testingRun.getTriggeredBy() != null) { - triggeredByTestEditor = testingRun.getTriggeredBy().equals("test_editor"); - } - handler.handleIssuesCreationFromTestingRunResults(testingRunResults, triggeredByTestEditor); // pass new field here - } + startTestNew(apiInfoKey, testRunId, testingRunConfig, testingUtil, testRunResultSummaryId, testConfigMap, subCategoryEndpointMap, apiInfoKeyToHostMap); } catch (Exception e) { e.printStackTrace(); loggerMaker.errorAndAddToDb("error while running tests: " + e, LogDb.TESTING); @@ -500,7 +474,7 @@ public List startWithLatch( } latch.countDown(); - return testingRunResults; + return null; } public static void trim(TestingRunResult testingRunResult) { @@ -535,15 +509,30 @@ public void trim(List testingRunResults) { } } - public List startTestNew(ApiInfo.ApiInfoKey apiInfoKey, ObjectId testRunId, + public void insertResultsAndMakeIssues(List testingRunResults) { + int resultSize = testingRunResults.size(); + if (resultSize > 0) { + loggerMaker.infoAndAddToDb("testingRunResults size: " + resultSize, LogDb.TESTING); + trim(testingRunResults); + TestingRunResultDao.instance.insertMany(testingRunResults); + loggerMaker.infoAndAddToDb("Inserted testing results", LogDb.TESTING); + TestingIssuesHandler handler = new TestingIssuesHandler(); + boolean triggeredByTestEditor = false; + handler.handleIssuesCreationFromTestingRunResults(testingRunResults, triggeredByTestEditor); + testingRunResults.clear(); + } + } + + public void startTestNew(ApiInfo.ApiInfoKey apiInfoKey, ObjectId testRunId, TestingRunConfig testingRunConfig, TestingUtil testingUtil, ObjectId testRunResultSummaryId, Map testConfigMap, ConcurrentHashMap subCategoryEndpointMap, Map apiInfoKeyToHostMap) { - List testingRunResults = new ArrayList<>(); List testSubCategories = testingRunConfig == null ? new ArrayList<>() : testingRunConfig.getTestSubCategoryList(); for (String testSubCategory: testSubCategories) { + List testingRunResults = new ArrayList<>(); + TestConfig testConfig = testConfigMap.get(testSubCategory); if (testConfig == null) continue; TestingRunResult testingRunResult = null; @@ -557,9 +546,10 @@ public List startTestNew(ApiInfo.ApiInfoKey apiInfoKey, Object e.printStackTrace(); } if (testingRunResult != null) testingRunResults.add(testingRunResult); + + insertResultsAndMakeIssues(testingRunResults); } - return testingRunResults; } public boolean applyRunOnceCheck(ApiInfoKey apiInfoKey, TestConfig testConfig, ConcurrentHashMap subCategoryEndpointMap, Map apiInfoKeyToHostMap, String testSubCategory) { diff --git a/libs/dao/src/main/java/com/akto/dao/testing/TestingRunResultDao.java b/libs/dao/src/main/java/com/akto/dao/testing/TestingRunResultDao.java index 65990ecf12..51efd161f1 100644 --- a/libs/dao/src/main/java/com/akto/dao/testing/TestingRunResultDao.java +++ b/libs/dao/src/main/java/com/akto/dao/testing/TestingRunResultDao.java @@ -1,6 +1,7 @@ package com.akto.dao.testing; import com.akto.dao.AccountsContextDao; +import com.akto.dao.MCollection; import com.akto.dao.context.Context; import com.akto.dto.ApiInfo; import com.akto.dto.testing.TestingRunResult; @@ -87,6 +88,8 @@ public void createIndicesIfAbsent() { Bson summaryIndex = Indexes.descending(Arrays.asList(TestingRunResult.TEST_RUN_RESULT_SUMMARY_ID, Constants.ID)); createIndexIfAbsent(dbName, getCollName(), summaryIndex, new IndexOptions().name("testRunResultSummaryId_-1__id_-1")); + MCollection.createIndexIfAbsent(getDBName(), getCollName(), + new String[] { TestingRunResult.TEST_RUN_RESULT_SUMMARY_ID, TestingRunResult.VULNERABLE, Constants.ID }, false); } } diff --git a/libs/dao/src/main/java/com/akto/dto/testing/GenericTestResult.java b/libs/dao/src/main/java/com/akto/dto/testing/GenericTestResult.java index 7791ef5bfa..42c041427a 100644 --- a/libs/dao/src/main/java/com/akto/dto/testing/GenericTestResult.java +++ b/libs/dao/src/main/java/com/akto/dto/testing/GenericTestResult.java @@ -5,6 +5,7 @@ public class GenericTestResult { private boolean vulnerable; + public static final String _CONFIDENCE = "confidence"; private Confidence confidence = Confidence.HIGH; public GenericTestResult() {