Skip to content

Commit 372121e

Browse files
ankush-jain-aktoankush-jain-akto
authored
Merge pull request #1945 from akto-api-security/feature/test_templates_settings
Feature/test templates settings
2 parents f9cf22a + 3f4bac6 commit 372121e

File tree

14 files changed

+292
-31
lines changed

14 files changed

+292
-31
lines changed

‎apps/dashboard/src/main/java/com/akto/action/AccountAction.java

+4
Original file line numberDiff line numberDiff line change
@@ -329,6 +329,10 @@ public void run() {
329329
AccountSettingsDao.instance.updateOnboardingFlag(true);
330330
InitializerListener.insertPiiSources();
331331

332+
if (DashboardMode.isMetered()) {
333+
AccountSettings accountSettings = AccountSettingsDao.instance.findOne(AccountSettingsDao.generateFilter());
334+
InitializerListener.insertAktoTestLibraries(accountSettings);
335+
}
332336
try {
333337
InitializerListener.executePIISourceFetch();
334338
} catch (Exception e) {

‎apps/dashboard/src/main/java/com/akto/action/ProfileAction.java

+4
Original file line numberDiff line numberDiff line change
@@ -135,6 +135,10 @@ public static void executeMeta1(Utility utility, User user, HttpServletRequest r
135135
} catch (Exception e) {
136136
}
137137

138+
if (DashboardMode.isMetered()) {
139+
InitializerListener.insertAktoTestLibraries(accountSettings);
140+
}
141+
138142
Organization organization = OrganizationsDao.instance.findOne(
139143
Filters.in(Organization.ACCOUNTS, sessionAccId)
140144
);

‎apps/dashboard/src/main/java/com/akto/action/settings/AdvancedTrafficFiltersAction.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -115,7 +115,7 @@ public String syncTrafficFromFilters(){
115115

116116
List<ApiCollection> apiCollections = ApiCollectionsDao.instance.findAll(
117117
Filters.nin(Constants.ID, deactivatedCollections), Projections.include(ApiCollection.HOST_NAME, ApiCollection.NAME));
118-
YamlTemplate yamlTemplate = new YamlTemplate(filterConfig.getId(), Context.now(), getSUser().getLogin(), Context.now(), this.yamlContent, null);
118+
YamlTemplate yamlTemplate = new YamlTemplate(filterConfig.getId(), Context.now(), getSUser().getLogin(), Context.now(), this.yamlContent, null, null);
119119
int accountId = Context.accountId.get();
120120
executorService.schedule( new Runnable() {
121121
public void run() {

‎apps/dashboard/src/main/java/com/akto/listener/InitializerListener.java

+63-3
Original file line numberDiff line numberDiff line change
@@ -54,6 +54,7 @@
5454
import com.akto.dto.settings.DefaultPayload;
5555
import com.akto.dto.sso.SAMLConfig;
5656
import com.akto.dto.test_editor.TestConfig;
57+
import com.akto.dto.test_editor.TestLibrary;
5758
import com.akto.dto.test_editor.YamlTemplate;
5859
import com.akto.dto.testing.*;
5960
import com.akto.dto.testing.custom_groups.AllAPIsGroup;
@@ -78,12 +79,14 @@
7879
import com.akto.stigg.StiggReporterClient;
7980
import com.akto.task.Cluster;
8081
import com.akto.telemetry.TelemetryJob;
82+
import com.akto.test_editor.TemplateSettingsUtil;
8183
import com.akto.testing.ApiExecutor;
8284
import com.akto.testing.HostDNSLookup;
8385
import com.akto.testing.TemplateMapper;
8486
import com.akto.usage.UsageMetricCalculator;
8587
import com.akto.usage.UsageMetricHandler;
8688
import com.akto.testing.workflow_node_executor.Utils;
89+
import com.akto.util.enums.GlobalEnums;
8790
import com.akto.util.filter.DictionaryFilter;
8891
import com.akto.utils.jobs.JobUtils;
8992
import com.akto.utils.jobs.MatchingJob;
@@ -97,6 +100,7 @@
97100
import com.akto.util.Pair;
98101
import com.akto.util.UsageUtils;
99102
import com.akto.util.enums.GlobalEnums.Severity;
103+
import com.akto.util.enums.GlobalEnums.TemplatePlan;
100104
import com.akto.util.enums.GlobalEnums.TestCategory;
101105
import com.akto.util.enums.GlobalEnums.YamlTemplateSource;
102106
import com.akto.util.http_util.CoreHTTPClient;
@@ -2433,6 +2437,30 @@ public static void clearRbacCollection(AccountsContextDaoWithRbac mCollection, i
24332437
}
24342438
}
24352439

2440+
public static Set<String> getAktoDefaultTestLibs() {
2441+
return new HashSet<>(Arrays.asList("akto-api-security/tests-library:standard", "akto-api-security/tests-library:pro"));
2442+
}
2443+
2444+
public static void insertAktoTestLibraries(AccountSettings accountSettings) {
2445+
List<TestLibrary> testLibraries = accountSettings == null ? new ArrayList<>() : accountSettings.getTestLibraries();
2446+
Set<String> aktoTestLibraries = getAktoDefaultTestLibs();
2447+
2448+
if (testLibraries != null) {
2449+
for (TestLibrary testLibrary: testLibraries) {
2450+
String author = testLibrary.getAuthor();
2451+
if (author.equals(Constants._AKTO)) {
2452+
aktoTestLibraries.remove(testLibrary.getRepositoryUrl());
2453+
}
2454+
}
2455+
}
2456+
2457+
for (String pendingLib: aktoTestLibraries) {
2458+
AccountSettingsDao.instance.updateOne(
2459+
AccountSettingsDao.generateFilter(),
2460+
Updates.addToSet(AccountSettings.TEST_LIBRARIES, new TestLibrary(pendingLib, Constants._AKTO, Context.now())));
2461+
}
2462+
}
2463+
24362464
public static void insertPiiSources(){
24372465
Map<String, String> map = new HashMap<>();
24382466
String fileUrl = "https://raw.githubusercontent.com/akto-api-security/pii-types/master/general.json";
@@ -3113,10 +3141,29 @@ public void run() {
31133141
loggerMaker.errorAndAddToDb("Error while fetching Test Editor Templates from Github and local", LogDb.DASHBOARD);
31143142
return;
31153143
}
3144+
3145+
Map<String, byte[]> allYamlTemplates = TestTemplateUtils.getZipFromMultipleRepoAndBranch(getAktoDefaultTestLibs());
31163146
AccountTask.instance.executeTask((account) -> {
31173147
try {
3118-
loggerMaker.infoAndAddToDb("Updating Test Editor Templates for accountId: " + account.getId(), LogDb.DASHBOARD);
3148+
loggerMaker.infoAndAddToDb("Updating Test Editor Templates for accountId: " + account.getId(), LogDb.DASHBOARD);
31193149
processTemplateFilesZip(testingTemplates, Constants._AKTO, YamlTemplateSource.AKTO_TEMPLATES.toString(), "");
3150+
3151+
if (!DashboardMode.isMetered()) return;
3152+
3153+
loggerMaker.infoAndAddToDb("Updating Pro and Standard Templates for accountId: " + account.getId(), LogDb.DASHBOARD);
3154+
3155+
AccountSettings accountSettings = AccountSettingsDao.instance.findOne(AccountSettingsDao.generateFilter());
3156+
3157+
if (accountSettings == null ||accountSettings.getTestLibraries() == null) return;
3158+
3159+
for(TestLibrary testLibrary: accountSettings.getTestLibraries()) {
3160+
String repoUrl = testLibrary.getRepositoryUrl();
3161+
if (repoUrl.contains("akto-api-security/tests-library")) {
3162+
byte[] zipFile = allYamlTemplates.get(testLibrary.getRepositoryUrl());
3163+
processTemplateFilesZip(zipFile, Constants._AKTO, YamlTemplateSource.AKTO_TEMPLATES.toString(), "");
3164+
}
3165+
}
3166+
31203167
} catch (Exception e) {
31213168
cacheLoggerMaker.errorAndAddToDb(e,
31223169
String.format("Error while updating Test Editor Files %s", e.toString()),
@@ -3142,7 +3189,7 @@ public static void processTemplateFilesZip(byte[] zipFile, String author, String
31423189
int countTotalTemplates = 0;
31433190
int countUnchangedTemplates = 0;
31443191
Set<String> multiNodesIds = new HashSet<>();
3145-
3192+
int skipped = 0;
31463193
while ((entry = zipInputStream.getNextEntry()) != null) {
31473194
if (!entry.isDirectory()) {
31483195
String entryName = entry.getName();
@@ -3193,6 +3240,13 @@ public static void processTemplateFilesZip(byte[] zipFile, String author, String
31933240

31943241
// new or updated template
31953242
if (testConfig != null) {
3243+
boolean hasSettings = testConfig.getAttributes() != null;
3244+
3245+
if (hasSettings && !testConfig.getAttributes().getPlan().equals(TemplatePlan.FREE) && DashboardMode.isLocalDeployment()) {
3246+
skipped++;
3247+
continue;
3248+
}
3249+
31963250
String id = testConfig.getId();
31973251
int createdAt = Context.now();
31983252
int updatedAt = Context.now();
@@ -3212,7 +3266,8 @@ public static void processTemplateFilesZip(byte[] zipFile, String author, String
32123266
Updates.set(YamlTemplate.UPDATED_AT, updatedAt),
32133267
Updates.set(YamlTemplate.HASH, templateContent.hashCode()),
32143268
Updates.set(YamlTemplate.CONTENT, templateContent),
3215-
Updates.set(YamlTemplate.INFO, testConfig.getInfo())));
3269+
Updates.set(YamlTemplate.INFO, testConfig.getInfo()),
3270+
Updates.set(YamlTemplate.SETTINGS, testConfig.getAttributes())));
32163271

32173272
try {
32183273
Object inactiveObject = TestConfigYamlParser.getFieldIfExists(templateContent,
@@ -3266,11 +3321,16 @@ public static void processTemplateFilesZip(byte[] zipFile, String author, String
32663321
if (countTotalTemplates != countUnchangedTemplates) {
32673322
loggerMaker.infoAndAddToDb(countUnchangedTemplates + "/" + countTotalTemplates + " unchanged", LogDb.DASHBOARD);
32683323
}
3324+
3325+
loggerMaker.infoAndAddToDb("Skipped " + skipped + " test templates for account: " + Context.accountId.get());
3326+
32693327
} catch (Exception ex) {
32703328
cacheLoggerMaker.errorAndAddToDb(ex,
32713329
String.format("Error while processing Test template files zip. Error %s", ex.getMessage()),
32723330
LogDb.DASHBOARD);
32733331
}
3332+
} else {
3333+
loggerMaker.infoAndAddToDb("Received null zip file");
32743334
}
32753335
}
32763336

‎apps/dashboard/src/main/java/com/akto/utils/TestTemplateUtils.java

+59-8
Original file line numberDiff line numberDiff line change
@@ -1,28 +1,79 @@
11
package com.akto.utils;
22

33
import com.akto.log.LoggerMaker;
4+
import com.akto.util.DashboardMode;
45

56
import static com.akto.listener.InitializerListener.loadTemplateFilesFromDirectory;
67

8+
import java.util.HashMap;
9+
import java.util.Map;
10+
import java.util.Set;
11+
712
public class TestTemplateUtils {
813

914
private static final LoggerMaker loggerMaker = new LoggerMaker(TestTemplateUtils.class, LoggerMaker.LogDb.DASHBOARD);
1015

1116
public static byte[] getTestingTemplates() {
12-
GithubSync githubSync = new GithubSync();
13-
byte[] repoZip = githubSync.syncRepo("akto-api-security/tests-library", "master");
14-
if(repoZip == null) {
15-
loggerMaker.infoAndAddToDb("Failed to load test templates from github, trying to load from local directory");
17+
byte[] repoZip = null;
18+
19+
if (DashboardMode.isMetered()) {
20+
GithubSync githubSync = new GithubSync();
21+
repoZip = githubSync.syncRepo("akto-api-security/tests-library", "master");
22+
23+
if(repoZip == null) {
24+
loggerMaker.infoAndAddToDb("Failed to load test templates from github, trying to load from local directory");
25+
} else {
26+
loggerMaker.infoAndAddToDb("Loaded test templates from github");
27+
}
28+
}
29+
30+
if (repoZip == null) {
1631
repoZip = loadTemplateFilesFromDirectory();
32+
1733
if(repoZip == null) {
18-
loggerMaker.errorAndAddToDb("Failed to load test templates from github or local directory");
19-
return null;
34+
loggerMaker.errorAndAddToDb("Failed to load test templates from local directory");
2035
} else {
2136
loggerMaker.infoAndAddToDb("Loaded test templates from local directory");
2237
}
23-
} else {
24-
loggerMaker.infoAndAddToDb("Loaded test templates from github");
2538
}
39+
40+
return repoZip;
41+
}
42+
43+
public static Map<String, byte[]> getZipFromMultipleRepoAndBranch(Set<String> repoAndBranchList) {
44+
45+
Map<String, byte[]> ret = new HashMap<>();
46+
47+
for (String repoAndBranch : repoAndBranchList) {
48+
ret.put(repoAndBranch, getZipFromRepoAndBranch(repoAndBranch));
49+
}
50+
51+
return ret;
52+
}
53+
54+
public static byte[] getZipFromRepoAndBranch(String repositoryUrlAndBranch) {
55+
56+
String[] tokens = repositoryUrlAndBranch.split(":");
57+
58+
boolean isBranchMaster = (tokens.length == 1 || tokens[1].length() == 0);
59+
60+
return getProTemplates(tokens[0], isBranchMaster ? "master" : tokens[1]);
61+
}
62+
63+
public static byte[] getProTemplates(String repositoryUrl, String branch) {
64+
byte[] repoZip = null;
65+
66+
if (DashboardMode.isMetered()) {
67+
GithubSync githubSync = new GithubSync();
68+
repoZip = githubSync.syncRepo(repositoryUrl, branch);
69+
70+
if(repoZip == null) {
71+
loggerMaker.infoAndAddToDb("Failed to load test templates from github, trying to load from local directory");
72+
} else {
73+
loggerMaker.infoAndAddToDb("Loaded test templates from github");
74+
}
75+
}
76+
2677
return repoZip;
2778
}
2879

‎apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/test_editor/components/SampleApi.jsx

+1-1
Original file line numberDiff line numberDiff line change
@@ -338,7 +338,7 @@ const SampleApi = () => {
338338
testingRunResult={testResult?.testingRunResult}
339339
runIssues={testResult?.testingRunIssues}
340340
testSubCategoryMap={testResult?.subCategoryMap}
341-
testId={selectedTest.value}
341+
testId={selectedTest?.value}
342342
source="editor"
343343
/>
344344
</Box>

‎libs/dao/src/main/java/com/akto/dao/test_editor/TestConfigYamlParser.java

+21-16
Original file line numberDiff line numberDiff line change
@@ -7,14 +7,9 @@
77
import com.akto.dao.test_editor.auth.Parser;
88
import com.akto.dao.test_editor.filter.ConfigParser;
99
import com.akto.dao.test_editor.info.InfoParser;
10+
import com.akto.dao.test_editor.settings.SettingsParser;
1011
import com.akto.dao.test_editor.strategy.StrategyParser;
11-
import com.akto.dto.test_editor.Auth;
12-
import com.akto.dto.test_editor.ConfigParserResult;
13-
import com.akto.dto.test_editor.ExecutorConfigParserResult;
14-
import com.akto.dto.test_editor.Info;
15-
import com.akto.dto.test_editor.SeverityParserResult;
16-
import com.akto.dto.test_editor.Strategy;
17-
import com.akto.dto.test_editor.TestConfig;
12+
import com.akto.dto.test_editor.*;
1813
import com.fasterxml.jackson.core.type.TypeReference;
1914
import com.fasterxml.jackson.databind.ObjectMapper;
2015
import com.fasterxml.jackson.dataformat.yaml.YAMLFactory;
@@ -51,27 +46,37 @@ public static TestConfig parseConfig(Map<String, Object> config) throws Exceptio
5146
return testConfig;
5247
}
5348

49+
Object settingsMap = config.get("attributes");
50+
TemplateSettings attributes = null;
51+
if (settingsMap != null) {
52+
SettingsParser settingsParser = new SettingsParser();
53+
attributes = settingsParser.parse(settingsMap);
54+
if (attributes == null) {
55+
return new TestConfig(id, info, null, null, null, null, null, null, null);
56+
}
57+
}
58+
5459
Object authMap = config.get("auth");
5560
Auth auth = null;
5661
if (authMap != null) {
5762
Parser authParser = new Parser();
5863
auth = authParser.parse(authMap);
5964
if (auth == null) {
60-
return new TestConfig(id, info, null, null, null, null, null, null);
65+
return new TestConfig(id, info, null, null, null, null, null, null, attributes);
6166
}
6267
}
6368

6469
Object filterMap = config.get("api_selection_filters");
6570
if (filterMap == null) {
6671
// todo: should not be null, throw error
67-
return new TestConfig(id, info, auth, null, null, null, null, null);
72+
return new TestConfig(id, info, auth, null, null, null, null, null, attributes);
6873
}
6974

7075
ConfigParser configParser = new ConfigParser();
7176
ConfigParserResult filters = configParser.parse(filterMap);
7277
if (filters == null) {
7378
// todo: throw error
74-
new TestConfig(id, info, auth, null, null, null, null, null);
79+
new TestConfig(id, info, auth, null, null, null, null, null, attributes);
7580
}
7681

7782
Map<String, List<String>> wordListMap = new HashMap<>();
@@ -80,32 +85,32 @@ public static TestConfig parseConfig(Map<String, Object> config) throws Exceptio
8085
wordListMap = (Map) config.get("wordLists");
8186
}
8287
} catch (Exception e) {
83-
return new TestConfig(id, info, null, null, null, null, null, null);
88+
return new TestConfig(id, info, null, null, null, null, null, null, attributes);
8489
}
8590

8691
Object executionMap = config.get("execute");
8792
if (executionMap == null) {
8893
// todo: should not be null, throw error
89-
return new TestConfig(id, info, auth, filters, wordListMap, null, null, null);
94+
return new TestConfig(id, info, auth, filters, wordListMap, null, null, null, attributes);
9095
}
9196

9297
com.akto.dao.test_editor.executor.ConfigParser executorConfigParser = new com.akto.dao.test_editor.executor.ConfigParser();
9398
ExecutorConfigParserResult executeOperations = executorConfigParser.parseConfigMap(executionMap);
9499
if (executeOperations == null) {
95100
// todo: throw error
96-
new TestConfig(id, info, auth, filters, wordListMap, null, null, null);
101+
new TestConfig(id, info, auth, filters, wordListMap, null, null, null, attributes);
97102
}
98103

99104
Object validationMap = config.get("validate");
100105
if (validationMap == null) {
101106
// todo: should not be null, throw error
102-
return new TestConfig(id, info, auth, filters, wordListMap, executeOperations, null, null);
107+
return new TestConfig(id, info, auth, filters, wordListMap, executeOperations, null, null, attributes);
103108
}
104109

105110
ConfigParserResult validations = configParser.parse(validationMap);
106111
if (validations == null) {
107112
// todo: throw error
108-
new TestConfig(id, info, auth, filters, wordListMap, executeOperations, null, null);
113+
new TestConfig(id, info, auth, filters, wordListMap, executeOperations, null, null, attributes);
109114
}
110115

111116
List<Object> apiSeverityTemp = new ArrayList<>();
@@ -135,7 +140,7 @@ public static TestConfig parseConfig(Map<String, Object> config) throws Exceptio
135140
strategy = strategyParser.parse(strategyObject);
136141
}
137142

138-
testConfig = new TestConfig(id, info, auth, filters, wordListMap, executeOperations, validations, strategy);
143+
testConfig = new TestConfig(id, info, auth, filters, wordListMap, executeOperations, validations, strategy, attributes);
139144
testConfig.setDynamicSeverityList(dynamicSeverityList);
140145
return testConfig;
141146
}

‎libs/dao/src/main/java/com/akto/dao/test_editor/settings/SettingsParser.java

+17
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
package com.akto.dao.test_editor.settings;
2+
3+
import com.akto.dto.test_editor.TemplateSettings;
4+
import com.fasterxml.jackson.databind.ObjectMapper;
5+
6+
import java.util.Map;
7+
8+
public class SettingsParser {
9+
10+
public TemplateSettings parse(Object settingsObj) {
11+
Map<String, Object> settingsMap = (Map) settingsObj;
12+
ObjectMapper objectMapper = new ObjectMapper();
13+
TemplateSettings settings = objectMapper.convertValue(settingsMap, TemplateSettings.class);
14+
return settings;
15+
}
16+
17+
}

0 commit comments

Comments
 (0)