Skip to content

Commit

Permalink
merge testing & mini-testing
Browse files Browse the repository at this point in the history
  • Loading branch information
@ankush-jain-akto
ankush-jain-akto committed Aug 4, 2024
1 parent ed3662e commit 57bbfb7
Showing 16 changed files with 400 additions and 572 deletions.
9 changes: 5 additions & 4 deletions apps/testing/src/main/java/com/akto/rules/BFLATest.java
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@
package com.akto.rules;

import com.akto.dao.testing.AccessMatrixUrlToRolesDao;
import com.akto.data_actor.DataActor;
import com.akto.data_actor.DataActorFactory;
import com.akto.dto.ApiInfo;
import com.akto.dto.OriginalHttpRequest;
import com.akto.dto.OriginalHttpResponse;
@@ -30,6 +32,8 @@

public class BFLATest {

private static final DataActor dataActor = DataActorFactory.fetchInstance();

public TestPlugin.ApiExecutionDetails executeApiAndReturnDetails(OriginalHttpRequest testRequest, boolean followRedirects, RawApi rawApi) throws Exception {
OriginalHttpResponse testResponse = ApiExecutor.sendRequest(testRequest, followRedirects, null, false, new ArrayList<>());

@@ -98,10 +102,7 @@ public List<String> updateAllowedRoles(RawApi rawApi, ApiInfo.ApiInfoKey apiInfo

}

Bson q = Filters.eq(Constants.ID, apiInfoKey);
Bson update = Updates.addEachToSet(AccessMatrixUrlToRole.ROLES, ret);
UpdateOptions opts = new UpdateOptions().upsert(true);
AccessMatrixUrlToRolesDao.instance.getMCollection().updateOne(q, update, opts);
dataActor.updateAccessMatrixUrlToRoles(apiInfoKey, ret);
loggerMaker.infoAndAddToDb("updated for " + apiInfoKey.getUrl() + " role: " + StringUtils.join(ret, ","), LogDb.TESTING);
return ret;
}
8 changes: 2 additions & 6 deletions apps/testing/src/main/java/com/akto/rules/RequiredConfigs.java
View file
Original file line number Diff line number Diff line change
@@ -6,6 +6,7 @@
import java.util.Map;

import com.akto.dao.testing.TestRolesDao;
import com.akto.data_actor.DataActorFactory;
import com.akto.dto.testing.TestRoles;
import com.mongodb.client.model.Filters;
import com.mongodb.client.model.Projections;
@@ -24,12 +25,7 @@ public static RequiredConfigs getRequiredConfigs() {

public static void initiate() {
validRolesExist.clear();
testRolesList = TestRolesDao.instance.findAll(
Filters.empty(),
Projections.fields(
Projections.include(TestRoles.NAME)
)
);
testRolesList = DataActorFactory.fetchInstance().fetchTestRoles();
for(TestRoles role: testRolesList){
validRolesExist.put(role.getName(), true);
}
36 changes: 20 additions & 16 deletions apps/testing/src/main/java/com/akto/rules/TestPlugin.java
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package com.akto.rules;

import com.akto.dao.SingleTypeInfoDao;
import com.akto.data_actor.DataActorFactory;
import com.akto.dto.*;
import com.akto.dto.ApiInfo.ApiInfoKey;
import com.akto.dto.test_editor.DataOperandsFilterResponse;
@@ -218,19 +219,19 @@ public Set<String> findPrivateParams() {
}

public static SingleTypeInfo findSti(String param, boolean isUrlParam,
ApiInfo.ApiInfoKey apiInfoKey, boolean isHeader, int responseCode,
Map<String, SingleTypeInfo> singleTypeInfoMap) {

Bson filter = Filters.and(
Filters.eq("apiCollectionId", apiInfoKey.getApiCollectionId()),
Filters.eq("url", apiInfoKey.url),
Filters.eq("method", apiInfoKey.method.name()),
Filters.eq("responseCode", responseCode),
Filters.eq("isHeader", isHeader),
Filters.eq("param", param),
Filters.eq("isUrlParam", isUrlParam)
);
SingleTypeInfo singleTypeInfo = SingleTypeInfoDao.instance.findOne(filter);
ApiInfo.ApiInfoKey apiInfoKey, boolean isHeader, int responseCode) {

SingleTypeInfo singleTypeInfo =
DataActorFactory.fetchInstance().findStiWithUrlParamFilters(
apiInfoKey.getApiCollectionId(),
apiInfoKey.getUrl(),
apiInfoKey.getMethod().name(),
responseCode,
isHeader,
param,
isUrlParam
);

if (singleTypeInfo == null) return null;

@@ -259,7 +260,7 @@ public ContainsPrivateResourceResult containsPrivateResource(OriginalHttpRequest
for (int i = 0;i < tokens.length; i++) {
if (tokens[i] == null) {
atLeastOneValueInRequest = true;
SingleTypeInfo singleTypeInfo = findSti(i+"", true,apiInfoKey, false, -1, sampleMessageStore.getSingleTypeInfos());
SingleTypeInfo singleTypeInfo = findSti(i+"", true,apiInfoKey, false, -1);
if (singleTypeInfo != null) {
String v = ogTokens[i];
Set<String> values = new HashSet<>();
@@ -277,7 +278,7 @@ public ContainsPrivateResourceResult containsPrivateResource(OriginalHttpRequest
Map<String, Set<Object>> flattened = JSONUtils.flatten(payload);
for (String param: flattened.keySet()) {
atLeastOneValueInRequest = true;
SingleTypeInfo singleTypeInfo = findSti(param,false,apiInfoKey, false, -1, sampleMessageStore.getSingleTypeInfos());
SingleTypeInfo singleTypeInfo = findSti(param,false,apiInfoKey, false, -1);
if (singleTypeInfo != null) {
Set<Object> valSet = flattened.get(param);
Set<String> valStringSet = new HashSet<>();
@@ -454,7 +455,10 @@ public TestRoleMatcher(List<TestRoles> testRolesList, ApiInfo.ApiInfoKey apiInfo
this.enemies = new ArrayList<>();

for (TestRoles testRoles: testRolesList) {
EndpointLogicalGroup endpointLogicalGroup = testRoles.fetchEndpointLogicalGroup();
EndpointLogicalGroup endpointLogicalGroup = testRoles.getEndpointLogicalGroup();
if (endpointLogicalGroup == null) {
endpointLogicalGroup = DataActorFactory.fetchInstance().fetchEndpointLogicalGroupById(testRoles.getEndpointLogicalGroupId().toHexString());
}
if (endpointLogicalGroup == null) continue;
TestingEndpoints testingEndpoints = endpointLogicalGroup.getTestingEndpoints();
if (testingEndpoints == null) continue;
5 changes: 4 additions & 1 deletion apps/testing/src/main/java/com/akto/store/AuthMechanismStore.java
View file
Original file line number Diff line number Diff line change
@@ -2,7 +2,9 @@

import com.akto.dao.AuthMechanismsDao;
import com.akto.dao.testing.TestRolesDao;
import com.akto.data_actor.DataActorFactory;
import com.akto.dto.testing.AuthMechanism;
import com.akto.dto.testing.TestRoles;
import com.mongodb.BasicDBObject;

public class AuthMechanismStore {
@@ -12,7 +14,8 @@ private AuthMechanismStore() {}

public static AuthMechanismStore create() {
AuthMechanismStore ret = new AuthMechanismStore();
ret.authMechanism = TestRolesDao.instance.fetchAttackerToken(0);
TestRoles testRoles = DataActorFactory.fetchInstance().fetchTestRole("ATTACKER_TOKEN_ALL");
ret.authMechanism = TestRolesDao.instance.fetchAttackerToken(0, testRoles);
return ret;
}

88 changes: 34 additions & 54 deletions apps/testing/src/main/java/com/akto/store/SampleMessageStore.java
View file
Original file line number Diff line number Diff line change
@@ -3,18 +3,19 @@
import com.akto.dao.SampleDataDao;
import com.akto.dao.testing.EndpointLogicalGroupDao;
import com.akto.dao.testing.TestRolesDao;
import com.akto.data_actor.DataActorFactory;
import com.akto.dto.*;
import com.akto.dao.SingleTypeInfoDao;
import com.akto.dto.ApiInfo;
import com.akto.dto.HttpRequestParams;
import com.akto.dto.HttpResponseParams;
import com.akto.dto.ApiInfo.ApiInfoKey;
import com.akto.dto.testing.*;
import com.akto.dto.traffic.Key;
import com.akto.dto.traffic.SampleData;
import com.akto.dto.type.SingleTypeInfo;
import com.akto.log.LoggerMaker;
import com.akto.log.LoggerMaker.LogDb;
import com.akto.metrics.AllMetrics;
import com.akto.sql.Main;
import com.akto.sql.SampleDataAltDb;
import com.mongodb.BasicDBObject;
import com.mongodb.client.model.Filters;
import org.bson.conversions.Bson;
@@ -29,46 +30,6 @@ public class SampleMessageStore {
private static final LoggerMaker loggerMaker = new LoggerMaker(SampleMessageStore.class);
private Map<ApiInfo.ApiInfoKey, List<String>> sampleDataMap = new HashMap<>();
private Map<String, SingleTypeInfo> singleTypeInfos = new HashMap<>();
public void buildSingleTypeInfoMap(TestingEndpoints testingEndpoints) {
if (testingEndpoints == null) return;
TestingEndpoints.Type type = testingEndpoints.getType();
List<SingleTypeInfo> singleTypeInfoList = new ArrayList<>();
try {
if (type.equals(TestingEndpoints.Type.COLLECTION_WISE)) {
CollectionWiseTestingEndpoints collectionWiseTestingEndpoints = (CollectionWiseTestingEndpoints) testingEndpoints;
int apiCollectionId = collectionWiseTestingEndpoints.getApiCollectionId();
singleTypeInfoList = SingleTypeInfoDao.instance.findAll(
Filters.and(
Filters.eq(SingleTypeInfo._API_COLLECTION_ID, apiCollectionId),
Filters.eq(SingleTypeInfo._RESPONSE_CODE, -1),
Filters.eq(SingleTypeInfo._IS_HEADER, false)
)
);
} else {
CustomTestingEndpoints customTestingEndpoints = (CustomTestingEndpoints) testingEndpoints;
List<ApiInfoKey> apiInfoKeys = customTestingEndpoints.getApisList();

if (apiInfoKeys.isEmpty()) {
return;
} else {
int apiCollectionId = apiInfoKeys.get(0).getApiCollectionId();
singleTypeInfoList = SingleTypeInfoDao.instance.findAll(
Filters.and(
Filters.eq(SingleTypeInfo._API_COLLECTION_ID, apiCollectionId),
Filters.eq(SingleTypeInfo._RESPONSE_CODE, -1),
Filters.eq(SingleTypeInfo._IS_HEADER, false)
)
);
}
}

for (SingleTypeInfo singleTypeInfo: singleTypeInfoList) {
singleTypeInfos.put(singleTypeInfo.composeKeyWithCustomSubType(SingleTypeInfo.GENERIC), singleTypeInfo);
}
} catch (Exception e) {
loggerMaker.errorAndAddToDb("Error while building STI map: " + e, LogDb.TESTING);
}
}

private SampleMessageStore() {}

@@ -83,13 +44,17 @@ public static SampleMessageStore create(Map<ApiInfo.ApiInfoKey, List<String>> sa
}

public List<TestRoles> fetchTestRoles() {
return TestRolesDao.instance.findAll(new BasicDBObject());
return DataActorFactory.fetchInstance().fetchTestRoles();
}


public void fetchSampleMessages(Set<Integer> apiCollectionIds) {
Bson filterQ = Filters.in("_id.apiCollectionId", apiCollectionIds);
List<SampleData> sampleDataList = SampleDataDao.instance.findAll(filterQ, 0, 10_000, null);
List<SampleData> sampleDataList = new ArrayList<>();
for (int i = 0; i < 20; i++) {
List<SampleData> sampleDataBatch = DataActorFactory.fetchInstance().fetchSampleData(apiCollectionIds, i*500);
sampleDataList.addAll(sampleDataBatch);
}

Map<ApiInfo.ApiInfoKey, List<String>> tempSampleDataMap = new HashMap<>();
for (SampleData sampleData: sampleDataList) {
if (sampleData.getSamples() == null) continue;
@@ -109,17 +74,32 @@ public void fetchSampleMessages(Set<Integer> apiCollectionIds) {

public List<RawApi> fetchAllOriginalMessages(ApiInfoKey apiInfoKey) {
List<RawApi> messages = new ArrayList<>();

List<String> samples = sampleDataMap.get(apiInfoKey);
if (samples == null || samples.isEmpty()) return messages;

for (String message: samples) {
if (Main.IS_PG_DB_USED) {
try {
messages.add(RawApi.buildFromMessage(message));
} catch(Exception e) {
loggerMaker.errorAndAddToDb("Error while building RawAPI for "+ apiInfoKey +" : " + e, LogDb.TESTING);
long start = System.currentTimeMillis();
List<String> samples = SampleDataAltDb.findSamplesByApiInfoKey(apiInfoKey);
AllMetrics.instance.setMultipleSampleDataFetchLatency(System.currentTimeMillis() - start);
for(String message: samples){
messages.add(RawApi.buildFromMessage(message));
}
return messages;
} catch (Exception e) {
loggerMaker.errorAndAddToDb(e, "Error while fetching all original messages for "+ apiInfoKey +" : " + e, LogDb.TESTING);
}

} else {

List<String> samples = sampleDataMap.get(apiInfoKey);
if (samples == null || samples.isEmpty()) return messages;

for (String message: samples) {
try {
messages.add(RawApi.buildFromMessage(message));
} catch(Exception e) {
loggerMaker.errorAndAddToDb("Error while building RawAPI for "+ apiInfoKey +" : " + e, LogDb.TESTING);
}

}
}

return messages;
28 changes: 15 additions & 13 deletions apps/testing/src/main/java/com/akto/test_editor/execution/Executor.java
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
package com.akto.test_editor.execution;

import com.akto.RuntimeMode;
import com.akto.billing.UsageMetricUtils;
import com.akto.dao.billing.OrganizationsDao;
import com.akto.dao.billing.TokensDao;
@@ -12,6 +13,8 @@
import com.akto.dao.test_editor.TestEditorEnums;
import com.akto.dao.test_editor.TestEditorEnums.ExecutorOperandTypes;
import com.akto.dao.testing.TestRolesDao;
import com.akto.data_actor.DataActor;
import com.akto.data_actor.DataActorFactory;
import com.akto.dto.ApiInfo;
import com.akto.dto.CustomAuthType;
import com.akto.dto.OriginalHttpResponse;
@@ -35,7 +38,6 @@
import com.akto.log.LoggerMaker.LogDb;
import com.akto.rules.TestPlugin;
import com.akto.test_editor.Utils;
import com.akto.testing.TestExecutor;
import com.akto.util.Constants;
import com.akto.util.UsageUtils;
import com.akto.util.enums.LoginFlowEnums;
@@ -117,9 +119,12 @@ public YamlTestResult execute(ExecutorNode node, RawApi rawApi, Map<String, Obje
return yamlTestResult;
}
if (testingRunConfig != null && StringUtils.isNotBlank(testingRunConfig.getTestRoleId())) {
TestRoles role = TestRolesDao.instance.findOne(Filters.eq("_id", new ObjectId(testingRunConfig.getTestRoleId())));
TestRoles role = DataActorFactory.fetchInstance().fetchTestRolesforId(testingRunConfig.getTestRoleId());
if (role != null) {
EndpointLogicalGroup endpointLogicalGroup = role.fetchEndpointLogicalGroup();
EndpointLogicalGroup endpointLogicalGroup = role.getEndpointLogicalGroup();
if (endpointLogicalGroup == null) {
endpointLogicalGroup = DataActorFactory.fetchInstance().fetchEndpointLogicalGroupById(role.getEndpointLogicalGroupId().toHexString());
}
if (endpointLogicalGroup != null && endpointLogicalGroup.getTestingEndpoints() != null && endpointLogicalGroup.getTestingEndpoints().containsApi(apiInfoKey)) {
if (role.getDefaultAuthMechanism() != null) {
loggerMaker.infoAndAddToDb("attempting to override auth " + logId, LogDb.TESTING);
@@ -143,7 +148,9 @@ public YamlTestResult execute(ExecutorNode node, RawApi rawApi, Map<String, Obje
if (executionType.equals("graph")) {
List<ApiInfo.ApiInfoKey> apiInfoKeys = new ArrayList<>();
apiInfoKeys.add(apiInfoKey);
memory = new Memory(apiInfoKeys, new HashMap<>());
if (!RuntimeMode.isHybridDeployment()) {
memory = new Memory(apiInfoKeys, new HashMap<>());
}
}
workflowTest = buildWorkflowGraph(reqNodes, rawApi, authMechanism, customAuthTypes, apiInfoKey, varMap, validatorNode);
result.add(triggerMultiExecution(workflowTest, reqNodes, rawApi, authMechanism, customAuthTypes, apiInfoKey, varMap, validatorNode, debug, testLogs, memory));
@@ -541,20 +548,15 @@ private ExecutorSingleOperationResp modifyAuthTokenInRawApi(TestRoles testRole,
private static BasicDBObject getBillingTokenForAuth() {
BasicDBObject bDObject;
int accountId = Context.accountId.get();
Organization organization = OrganizationsDao.instance.findOne(
Filters.in(Organization.ACCOUNTS, accountId)
);
Organization organization = DataActorFactory.fetchInstance().fetchOrganization(accountId);
if (organization == null) {
return new BasicDBObject("error", "organization not found");
}

Tokens tokens;
Bson filters = Filters.and(
Filters.eq(Tokens.ORG_ID, organization.getId()),
Filters.eq(Tokens.ACCOUNT_ID, accountId)
);

String errMessage = "";
tokens = TokensDao.instance.findOne(filters);
tokens = DataActorFactory.fetchInstance().fetchToken(organization.getId(), accountId);
if (tokens == null) {
errMessage = "error extracting ${akto_header}, token is missing";
}
@@ -648,7 +650,7 @@ public ExecutorSingleOperationResp runOperation(String operationType, RawApi raw

keyStr = keyStr.replace(ACCESS_ROLES_CONTEXT, "");
keyStr = keyStr.substring(0,keyStr.length()-1).trim();
TestRoles testRole = TestRolesDao.instance.findOne(TestRoles.NAME, keyStr);
TestRoles testRole = DataActorFactory.fetchInstance().fetchTestRole(keyStr);
if (testRole == null) {
return new ExecutorSingleOperationResp(false, "Test Role " + keyStr + " Doesn't Exist ");
}
Loading

0 comments on commit 57bbfb7

Please sign in to comment.