Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Feature/find shadow apis #1784

Open
wants to merge 11 commits into
base: master
Choose a base branch
from
Open

Feature/find shadow apis #1784

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
11 commits
Select commit Hold shift + click to select a range
364dc18
OpenAPI file upload changes
rishabh-akto Nov 28, 2024
789894e
backward compatibilty of source in sti and api info added
rishabh-akto Nov 28, 2024
5496a25
Merge branch 'master' into feature/find-shadow-apis
avneesh-akto Dec 12, 2024
fea5ff9
back merge from master
avneesh-akto Dec 12, 2024
309128a
Merge branch 'master' into feature/find-shadow-apis
avneesh-akto Dec 16, 2024
562a042
ask user for source if not there
avneesh-akto Dec 18, 2024
26b8ea1
added burp source
avneesh-akto Dec 19, 2024
c09e51c
null pointer check in aktoPolicyNew
avneesh-akto Dec 19, 2024
38fa862
fixed timestamp in sources in API info
avneesh-akto Dec 19, 2024
530355e
disable openAPI upload for logical groups
avneesh-akto Dec 19, 2024
e66151d
handle apiCollection type null
avneesh-akto Dec 19, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion apps/api-runtime/src/main/java/com/akto/parsers/HttpCallParser.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -281,7 +281,7 @@ public void syncFunction(List<HttpResponseParams> responseParams, boolean syncIm
SyncLimit syncLimit = featureAccess.fetchSyncLimit();

numberOfSyncs++;
apiCatalogSync.syncWithDB(syncImmediately, fetchAllSTI, syncLimit);
apiCatalogSync.syncWithDB(syncImmediately, fetchAllSTI, syncLimit, responseParams.get(0).getSource());
if (DbMode.dbType.equals(DbMode.DbType.MONGO_DB)) {
dependencyAnalyser.dbState = apiCatalogSync.dbState;
dependencyAnalyser.syncWithDb();
Expand Down
16 changes: 9 additions & 7 deletions apps/api-runtime/src/main/java/com/akto/runtime/APICatalogSync.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,9 +13,6 @@
import com.akto.dto.filter.MergedUrls;
import com.akto.dao.monitoring.FilterYamlTemplateDao;
import com.akto.dao.runtime_filters.AdvancedTrafficFiltersDao;
import com.akto.dto.*;
import com.akto.dto.billing.SyncLimit;
import com.akto.dto.dependency_flow.DependencyFlow;
import com.akto.dto.monitoring.FilterConfig;
import com.akto.dto.test_editor.YamlTemplate;
import com.akto.dto.traffic.Key;
Expand Down Expand Up @@ -48,6 +45,7 @@
import com.mongodb.client.model.*;
import com.mongodb.client.result.UpdateResult;
import org.apache.commons.lang3.math.NumberUtils;
import org.bson.Document;
import org.bson.conversions.Bson;
import org.bson.json.JsonParseException;
import org.bson.types.ObjectId;
Expand Down Expand Up @@ -1279,7 +1277,7 @@ public ArrayList<WriteModel<TrafficInfo>> getDBUpdatesForTraffic(int apiCollecti
return bulkUpdates;
}

public DbUpdateReturn getDBUpdatesForParams(APICatalog currentDelta, APICatalog currentState, boolean redactSampleData, boolean collectionLevelRedact) {
public DbUpdateReturn getDBUpdatesForParams(APICatalog currentDelta, APICatalog currentState, boolean redactSampleData, boolean collectionLevelRedact, HttpResponseParams.Source source) {
Map<String, SingleTypeInfo> dbInfoMap = convertToMap(currentState.getAllTypeInfo());
Map<String, SingleTypeInfo> deltaInfoMap = convertToMap(currentDelta.getAllTypeInfo());

Expand Down Expand Up @@ -1312,6 +1310,10 @@ public DbUpdateReturn getDBUpdatesForParams(APICatalog currentDelta, APICatalog
update = Updates.combine(update, Updates.max(SingleTypeInfo.LAST_SEEN, deltaInfo.getLastSeen()));
update = Updates.combine(update, Updates.max(SingleTypeInfo.MAX_VALUE, deltaInfo.getMaxValue()));
update = Updates.combine(update, Updates.min(SingleTypeInfo.MIN_VALUE, deltaInfo.getMinValue()));
if (source != null) {
Bson updateSourceMap = Updates.set(SingleTypeInfo.SOURCES + "." + source.name(), new Document("timestamp", timestamp) );
update = Updates.combine(update, updateSourceMap);
}

if (!Main.isOnprem) {
if (dbInfo != null) {
Expand Down Expand Up @@ -1795,7 +1797,7 @@ public static Map<Integer, APICatalog> build(List<SingleTypeInfo> allParams, Blo
int counter = 0;
List<String> partnerIpsList = new ArrayList<>();

public void syncWithDB(boolean syncImmediately, boolean fetchAllSTI, SyncLimit syncLimit) {
public void syncWithDB(boolean syncImmediately, boolean fetchAllSTI, SyncLimit syncLimit, HttpResponseParams.Source source) {
loggerMaker.infoAndAddToDb("Started sync with db! syncImmediately="+syncImmediately + " fetchAllSTI="+fetchAllSTI, LogDb.RUNTIME);
List<WriteModel<SingleTypeInfo>> writesForParams = new ArrayList<>();
List<WriteModel<SensitiveSampleData>> writesForSensitiveSampleData = new ArrayList<>();
Expand Down Expand Up @@ -1873,7 +1875,7 @@ public void syncWithDB(boolean syncImmediately, boolean fetchAllSTI, SyncLimit s

APICatalog dbCatalog = this.dbState.getOrDefault(apiCollectionId, new APICatalog(apiCollectionId, new HashMap<>(), new HashMap<>()));
boolean redactCollectionLevel = apiCollectionToRedactPayload.getOrDefault(apiCollectionId, false);
DbUpdateReturn dbUpdateReturn = getDBUpdatesForParams(deltaCatalog, dbCatalog, redact, redactCollectionLevel);
DbUpdateReturn dbUpdateReturn = getDBUpdatesForParams(deltaCatalog, dbCatalog, redact, redactCollectionLevel, source);
writesForParams.addAll(dbUpdateReturn.bulkUpdatesForSingleTypeInfo);
writesForSensitiveSampleData.addAll(dbUpdateReturn.bulkUpdatesForSampleData);
writesForSensitiveParamInfo.addAll(dbUpdateReturn.bulkUpdatesForSensitiveParamInfo);
Expand Down Expand Up @@ -1904,7 +1906,7 @@ public void syncWithDB(boolean syncImmediately, boolean fetchAllSTI, SyncLimit s
} while (from < writesForParams.size());
}

aktoPolicyNew.syncWithDb();
aktoPolicyNew.syncWithDb(source);

loggerMaker.infoAndAddToDb("adding " + writesForTraffic.size() + " updates for traffic", LogDb.RUNTIME);
if(writesForTraffic.size() > 0) {
Expand Down
18 changes: 11 additions & 7 deletions apps/api-runtime/src/main/java/com/akto/runtime/policies/AktoPolicyNew.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,9 +19,8 @@
import com.mongodb.BasicDBObject;
import com.mongodb.client.model.*;
import org.apache.commons.lang3.StringUtils;
import org.bson.Document;
import org.bson.conversions.Bson;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;


import java.util.*;
Expand Down Expand Up @@ -92,9 +91,9 @@ public void buildFromDb(boolean fetchAllSTI) {
loggerMaker.infoAndAddToDb("Built AktoPolicyNew", LogDb.RUNTIME);
}

public void syncWithDb() {
public void syncWithDb(HttpResponseParams.Source source) {
loggerMaker.infoAndAddToDb("Syncing with db", LogDb.RUNTIME);
UpdateReturn updateReturn = getUpdates(apiInfoCatalogMap);
UpdateReturn updateReturn = getUpdates(apiInfoCatalogMap, source);
List<WriteModel<ApiInfo>> writesForApiInfo = updateReturn.updatesForApiInfo;
List<WriteModel<FilterSampleData>> writesForSampleData = updateReturn.updatesForSampleData;
loggerMaker.infoAndAddToDb("Writing to db: " + "writesForApiInfoSize="+writesForApiInfo.size() + " writesForSampleData="+ writesForSampleData.size(), LogDb.RUNTIME);
Expand Down Expand Up @@ -264,7 +263,7 @@ public PolicyCatalog getApiInfoFromMap(ApiInfo.ApiInfoKey apiInfoKey) {
return newPolicyCatalog;
}

public static UpdateReturn getUpdates(Map<Integer, ApiInfoCatalog> apiInfoCatalogMap) {
public static UpdateReturn getUpdates(Map<Integer, ApiInfoCatalog> apiInfoCatalogMap, HttpResponseParams.Source source) {
List<ApiInfo> apiInfoList = new ArrayList<>();
List<FilterSampleData> filterSampleDataList = new ArrayList<>();
for (ApiInfoCatalog apiInfoCatalog: apiInfoCatalogMap.values()) {
Expand Down Expand Up @@ -297,7 +296,7 @@ public static UpdateReturn getUpdates(Map<Integer, ApiInfoCatalog> apiInfoCatalo
}
}

List<WriteModel<ApiInfo>> updatesForApiInfo = getUpdatesForApiInfo(apiInfoList);
List<WriteModel<ApiInfo>> updatesForApiInfo = getUpdatesForApiInfo(apiInfoList, source);
List<WriteModel<FilterSampleData>> updatesForSampleData = getUpdatesForSampleData(filterSampleDataList);
Map<ApiInfoKey, List<Integer>> updatesForApiGroups = getUpdatesForApiGroups(apiInfoList);

Expand Down Expand Up @@ -381,7 +380,7 @@ public UpdateReturn(List<WriteModel<ApiInfo>> updatesForApiInfo, List<WriteModel
}
}

public static List<WriteModel<ApiInfo>> getUpdatesForApiInfo(List<ApiInfo> apiInfoList) {
public static List<WriteModel<ApiInfo>> getUpdatesForApiInfo(List<ApiInfo> apiInfoList, HttpResponseParams.Source source) {

List<WriteModel<ApiInfo>> updates = new ArrayList<>();
for (ApiInfo apiInfo: apiInfoList) {
Expand Down Expand Up @@ -420,6 +419,11 @@ public static List<WriteModel<ApiInfo>> getUpdatesForApiInfo(List<ApiInfo> apiIn
// discovered timestamp
subUpdates.add(Updates.setOnInsert(ApiInfo.DISCOVERED_TIMESTAMP, apiInfo.getDiscoveredTimestamp()));

// sources
if (source != null) {
subUpdates.add(Updates.set(SingleTypeInfo.SOURCES + "." + source.name(), new Document("timestamp", Context.now())));
}

// last seen
subUpdates.add(Updates.set(ApiInfo.LAST_SEEN, apiInfo.getLastSeen()));

Expand Down
8 changes: 4 additions & 4 deletions apps/api-runtime/src/test/java/com/akto/parsers/TestDBSync.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -97,7 +97,7 @@ public void testParameterizedURL() {
aggr.addURL(TestDump2.createSampleParams("user"+i, url+i));
}
sync.computeDelta(aggr, true, 0, false);
sync.syncWithDB(false, true, SyncLimit.noLimit);
sync.syncWithDB(false, true, SyncLimit.noLimit, Source.HAR);
APICatalogSync.mergeUrlsAndSave(123, true, false, sync.existingAPIsInDb, false);
sync.buildFromDB(false, true);

Expand Down Expand Up @@ -216,7 +216,7 @@ public void testInvalidMergeParameterizedURL() {
aggr.addURL(TestDump2.createSampleParams("user"+i, "/payment/id"+i));
}
sync.computeDelta(aggr, true, 123, false);
sync.syncWithDB(false, true, SyncLimit.noLimit);
sync.syncWithDB(false, true, SyncLimit.noLimit, Source.HAR);


assertEquals(30, sync.getDbState(123).getStrictURLToMethods().size());
Expand All @@ -230,7 +230,7 @@ public void testInvalidMergeParameterizedURL() {
aggr2.addURL(resp2);

sync.computeDelta(aggr2, true, 123, false);
sync.syncWithDB(false, true, SyncLimit.noLimit);
sync.syncWithDB(false, true, SyncLimit.noLimit, Source.HAR);
APICatalogSync.mergeUrlsAndSave(123, true, false, sync.existingAPIsInDb, false);
sync.buildFromDB(false, true);

Expand Down Expand Up @@ -575,7 +575,7 @@ public void testHostNameForSourceOther() throws Exception {

HttpCallParser httpCallParser = new HttpCallParser("", 100000, 10000, 10000, true);
httpCallParser.syncFunction(responseParamsList,true, true, null);
httpCallParser.apiCatalogSync.syncWithDB(true, true, SyncLimit.noLimit);
httpCallParser.apiCatalogSync.syncWithDB(true, true, SyncLimit.noLimit, Source.HAR);

ApiCollection one = ApiCollectionsDao.instance.findOne(new BasicDBObject());
String host = "dev-1.akto.io";
Expand Down
13 changes: 7 additions & 6 deletions apps/api-runtime/src/test/java/com/akto/parsers/TestDump2.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@
import com.akto.dto.type.URLMethods.Method;
import com.akto.runtime.APICatalogSync;
import com.akto.runtime.URLAggregator;
import com.akto.dto.HttpResponseParams.Source;
import com.akto.types.CappedSet;
import com.mongodb.BasicDBList;
import com.mongodb.BasicDBObject;
Expand Down Expand Up @@ -125,7 +126,7 @@ public void testHappyPath() {

aggr.addURL(httpResponseParams);
sync.computeDelta(aggr, false, 0, false);
APICatalogSync.DbUpdateReturn dbUpdateReturn = sync.getDBUpdatesForParams(sync.getDelta(0), sync.getDbState(0), false, false);
APICatalogSync.DbUpdateReturn dbUpdateReturn = sync.getDBUpdatesForParams(sync.getDelta(0), sync.getDbState(0), false, false, Source.HAR);
assertEquals(15, dbUpdateReturn.bulkUpdatesForSingleTypeInfo.size());
assertEquals(2, sync.getDBUpdatesForTraffic(0, sync.getDelta(0)).size());
assertEquals(1, sync.getDBUpdatesForSampleData(0, sync.getDelta(0), sync.getDbState(0),true, false, false).size());
Expand Down Expand Up @@ -154,7 +155,7 @@ public void simpleTestForSingleCollection(int collectionId, APICatalogSync sync)
RequestTemplate respTemplate = reqTemplate.getResponseTemplates().get(resp.statusCode);
assertEquals(1, respTemplate.getUserIds().size());
assertEquals(3, respTemplate.getParameters().size());
APICatalogSync.DbUpdateReturn dbUpdateReturn = sync.getDBUpdatesForParams(sync.getDelta(collectionId), sync.getDbState(collectionId), false, false);
APICatalogSync.DbUpdateReturn dbUpdateReturn = sync.getDBUpdatesForParams(sync.getDelta(collectionId), sync.getDbState(collectionId), false, false, Source.HAR);
assertEquals(24, dbUpdateReturn.bulkUpdatesForSingleTypeInfo.size());
assertEquals(2, sync.getDBUpdatesForTraffic(collectionId, sync.getDelta(collectionId)).size());
}
Expand All @@ -167,9 +168,9 @@ public void simpleTest() {
simpleTestForSingleCollection(0, sync);
simpleTestForSingleCollection(1, sync);
simpleTestForSingleCollection(2, sync);
assertEquals(24, sync.getDBUpdatesForParams(sync.getDelta(0), sync.getDbState(0),false, false).bulkUpdatesForSingleTypeInfo.size());
assertEquals(24, sync.getDBUpdatesForParams(sync.getDelta(1), sync.getDbState(1),false, false).bulkUpdatesForSingleTypeInfo.size());
assertEquals(24, sync.getDBUpdatesForParams(sync.getDelta(2), sync.getDbState(2),false, false).bulkUpdatesForSingleTypeInfo.size());
assertEquals(24, sync.getDBUpdatesForParams(sync.getDelta(0), sync.getDbState(0),false, false, Source.HAR).bulkUpdatesForSingleTypeInfo.size());
assertEquals(24, sync.getDBUpdatesForParams(sync.getDelta(1), sync.getDbState(1),false, false, Source.HAR).bulkUpdatesForSingleTypeInfo.size());
assertEquals(24, sync.getDBUpdatesForParams(sync.getDelta(2), sync.getDbState(2),false, false, Source.HAR).bulkUpdatesForSingleTypeInfo.size());
}

@Test
Expand Down Expand Up @@ -383,7 +384,7 @@ public void repetitiveKeyTest() {
// TODO: investigate and fix this
// assertEquals(1, respTemplate.getParameters().size());

List updates = sync.getDBUpdatesForParams(sync.getDelta(0), sync.getDbState(0), false, false).bulkUpdatesForSingleTypeInfo;
List updates = sync.getDBUpdatesForParams(sync.getDelta(0), sync.getDbState(0), false, false, Source.HAR).bulkUpdatesForSingleTypeInfo;
}

@Test
Expand Down
Loading