Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fetch only template urs while building dbState in runtime #1683

Open
wants to merge 2 commits into
base: master
Choose a base branch
from
Open

fetch only template urs while building dbState in runtime #1683

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
61bb4f6
fetch only template urs while building dbState in runtime
avneesh-akto Nov 4, 2024
3e2dd68
back merge from master
avneesh-akto Nov 7, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
41 changes: 21 additions & 20 deletions apps/api-runtime/src/main/java/com/akto/runtime/APICatalogSync.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
import com.akto.dao.*;
import com.akto.dao.context.Context;
import com.akto.dao.filter.MergedUrlsDao;
import com.akto.data_actor.DbLayer;
import com.akto.dto.*;
import com.akto.dto.billing.SyncLimit;
import com.akto.dto.dependency_flow.DependencyFlow;
Expand Down Expand Up @@ -1570,30 +1571,30 @@ public void buildFromDB(boolean calcDiff, boolean fetchAllSTI) {
}

loggerMaker.infoAndAddToDb("Fetching STIs: " + fetchAllSTI, LogDb.RUNTIME);
List<SingleTypeInfo> allParams;
List<SingleTypeInfo> allParams = new ArrayList<>();

// when fetchAllSTI = true, then we need to minimise memory, we do that by keeping 1 STI per API
if (fetchAllSTI) {
Bson filterForHostHeader = SingleTypeInfoDao.filterForHostHeader(-1,false);
Bson filterQ = Filters.and(filterForHostHeader, Filters.regex(SingleTypeInfo._URL, "STRING|INTEGER"));
allParams = SingleTypeInfoDao.instance.findAll(filterQ, Projections.exclude(SingleTypeInfo._VALUES));
allParams.addAll(SingleTypeInfoDao.instance.findAll(new BasicDBObject(), Projections.exclude(SingleTypeInfo._VALUES)));

int dependencyFlowLimit = 1_000;
if (mergingCalled && allParams.size() < dependencyFlowLimit) {
loggerMaker.infoAndAddToDb("ALl params less than " + dependencyFlowLimit +", running dependency flow", LogDb.RUNTIME);
try {
DependencyFlow dependencyFlow = new DependencyFlow();
dependencyFlow.run(null);
dependencyFlow.syncWithDb();
loggerMaker.infoAndAddToDb("Finished running dependency flow", LogDb.RUNTIME);
} catch (Exception e) {
loggerMaker.errorAndAddToDb(e, "Error while running dependency flow in runtime: " + e.getMessage(), LogDb.RUNTIME);
}
}
allParams = DbLayer.fetchStiBasedOnHostHeaders();
}

List<Integer> nonHostsApiCollectionIds = ApiCollectionsDao.instance.fetchNonTrafficApiCollectionsIds();
List<SingleTypeInfo> nonHostsSTI = SingleTypeInfoDao.instance.fetchStiOfCollections(nonHostsApiCollectionIds);
if (fetchAllSTI) {
// for collections without hosts, we need to get all STIs of such collections and make sure only unique ones are kept
Set<String> seen = new HashSet<>();
for (SingleTypeInfo singleTypeInfo: nonHostsSTI) {
String nonHostUrl = singleTypeInfo.getUrl();
String nonHostMethod = singleTypeInfo.getMethod();
String key = nonHostMethod + " " + nonHostUrl;
if (seen.contains(key)) continue;
seen.add(key);
allParams.add(singleTypeInfo);
}
} else {
List<Integer> apiCollectionIds = ApiCollectionsDao.instance.fetchNonTrafficApiCollectionsIds();
allParams = SingleTypeInfoDao.instance.fetchStiOfCollections(apiCollectionIds);
allParams.addAll(nonHostsSTI);
}

loggerMaker.infoAndAddToDb("Fetched STIs count: " + allParams.size(), LogDb.RUNTIME);
this.dbState.clear();
loggerMaker.infoAndAddToDb("Starting building dbState", LogDb.RUNTIME);
Expand Down
61 changes: 39 additions & 22 deletions apps/api-runtime/src/test/java/com/akto/parsers/TestDBSync.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,11 +17,7 @@
import com.akto.dto.*;
import com.akto.dto.runtime_filters.RuntimeFilter;
import com.akto.dto.traffic.SampleData;
import com.akto.dto.type.AccountDataTypesInfo;
import com.akto.dto.type.CollectionReplaceDetails;
import com.akto.dto.type.RequestTemplate;
import com.akto.dto.type.SingleTypeInfo;
import com.akto.dto.type.URLTemplate;
import com.akto.dto.type.*;
import com.akto.dto.HttpResponseParams.Source;
import com.akto.runtime.APICatalogSync;
import com.akto.runtime.Main;
Expand Down Expand Up @@ -101,19 +97,12 @@ public void testParameterizedURL() {
APICatalogSync.mergeUrlsAndSave(123, true, false, sync.existingAPIsInDb, false);
sync.buildFromDB(false, true);

assertEquals(0, sync.getDbState(123).getStrictURLToMethods().size());
assertEquals(1, sync.getDbState(123).getTemplateURLToMethods().size());

Map.Entry<URLTemplate, RequestTemplate> entry = sync.getDbState(123).getTemplateURLToMethods().entrySet().iterator().next();
Map<String, Integer> staticAndTemplateUrlsCount = fetchStaticAndTemplateUrlsCount(123);
assertEquals(0, (int) staticAndTemplateUrlsCount.get(STATIC_URLS));
assertEquals(1, (int) staticAndTemplateUrlsCount.get(TEMPLATE_URLS));

assertEquals(url+"INTEGER", entry.getKey().getTemplateString());
RequestTemplate reqTemplate = entry.getValue();

assertEquals(2, reqTemplate.getParameters().size());

RequestTemplate respTemplate = reqTemplate.getResponseTemplates().get(resp.getStatusCode());
assertEquals(3, respTemplate.getParameters().size());
}
}

// @Test
public void testImmediateSync() {
Expand Down Expand Up @@ -204,7 +193,33 @@ public void testAllPaths() {
responseParams.get(0).setSource(Source.HAR);
parser.syncFunction(responseParams, false, true, null);

}
}

static String STATIC_URLS = "staticUrls";
static String TEMPLATE_URLS = "templateUrls";

public static Map<String, Integer> fetchStaticAndTemplateUrlsCount(int apiCollectionId) {
List<SingleTypeInfo> singleTypeInfos = SingleTypeInfoDao.instance.findAll(Filters.eq(SingleTypeInfo._API_COLLECTION_ID, apiCollectionId));
Set<String> staticUrls = new HashSet<>();
Set<String> templateUrls = new HashSet<>();
for (SingleTypeInfo singleTypeInfo: singleTypeInfos) {
String url = singleTypeInfo.getUrl();
String method = singleTypeInfo.getMethod();

if (APICatalog.isTemplateUrl(url)) {
URLTemplate urlTemplate = APICatalogSync.createUrlTemplate(url, URLMethods.Method.fromString(method));
templateUrls.add(urlTemplate.getTemplateString() + " " + method);
} else {
staticUrls.add(url + " " + method);
}
}

Map<String, Integer> count = new HashMap<>();
count.put(STATIC_URLS, staticUrls.size());
count.put(TEMPLATE_URLS, templateUrls.size());

return count;
}

@Test
public void testInvalidMergeParameterizedURL() {
Expand All @@ -219,8 +234,9 @@ public void testInvalidMergeParameterizedURL() {
sync.syncWithDB(false, true, SyncLimit.noLimit);


assertEquals(30, sync.getDbState(123).getStrictURLToMethods().size());
assertEquals(0, sync.getDbState(123).getTemplateURLToMethods().size());
Map<String, Integer> staticAndTemplateUrlsCount = fetchStaticAndTemplateUrlsCount(123);
assertEquals(30, (int) staticAndTemplateUrlsCount.get(STATIC_URLS));
assertEquals(0, (int) staticAndTemplateUrlsCount.get(TEMPLATE_URLS));

HttpResponseParams resp2 = TestDump2.createSampleParams("user1", "/payment/history");
ArrayList<String> newHeader = new ArrayList<>();
Expand All @@ -234,9 +250,10 @@ public void testInvalidMergeParameterizedURL() {
APICatalogSync.mergeUrlsAndSave(123, true, false, sync.existingAPIsInDb, false);
sync.buildFromDB(false, true);

assertEquals(1, sync.getDbState(123).getStrictURLToMethods().size());
assertEquals(1, sync.getDbState(123).getTemplateURLToMethods().size());

staticAndTemplateUrlsCount = fetchStaticAndTemplateUrlsCount(123);
assertEquals(1, (int)staticAndTemplateUrlsCount.get(STATIC_URLS));
assertEquals(1, (int) staticAndTemplateUrlsCount.get(TEMPLATE_URLS));

}

Expand Down Expand Up @@ -601,4 +618,4 @@ public void testRedundantUrlCheck() {
assertEquals(httpCallParser.isRedundantEndpoint(url4, pattern), true);
assertEquals(httpCallParser.isRedundantEndpoint(url5, pattern), true);
}
}
}
Loading
Loading