From 364dc186d0f232c09177ab0b68d5c07e0f0106e8 Mon Sep 17 00:00:00 2001 From: rishabh-akto Date: Thu, 28 Nov 2024 15:00:30 +0530 Subject: [PATCH 1/8] OpenAPI file upload changes --- .../java/com/akto/parsers/HttpCallParser.java | 2 +- .../java/com/akto/runtime/APICatalogSync.java | 14 +-- .../akto/runtime/policies/AktoPolicyNew.java | 17 +-- .../java/com/akto/parsers/TestDBSync.java | 8 +- .../test/java/com/akto/parsers/TestDump2.java | 13 +- .../java/com/akto/parsers/TestMergingNew.java | 118 +++++++++--------- .../java/com/akto/action/OpenApiAction.java | 35 +++--- .../akto/action/observe/InventoryAction.java | 2 +- .../akto/utils/TestUpdatesInCollections.java | 3 +- .../src/apps/dashboard/pages/observe/api.js | 7 ++ .../observe/api_collections/ApiEndpoints.jsx | 52 +++++++- .../web/polaris_web/web/src/util/func.js | 4 +- .../src/main/java/com/akto/dto/ApiInfo.java | 13 ++ .../java/com/akto/dto/HttpResponseParams.java | 2 +- .../com/akto/dto/type/SingleTypeInfo.java | 11 +- .../utils/src/main/java/com/akto/har/HAR.java | 12 +- .../java/com/akto/open_api/parser/Parser.java | 12 +- 17 files changed, 204 insertions(+), 121 deletions(-) diff --git a/apps/api-runtime/src/main/java/com/akto/parsers/HttpCallParser.java b/apps/api-runtime/src/main/java/com/akto/parsers/HttpCallParser.java index c1eaaaf9f6..f357f44471 100644 --- a/apps/api-runtime/src/main/java/com/akto/parsers/HttpCallParser.java +++ b/apps/api-runtime/src/main/java/com/akto/parsers/HttpCallParser.java @@ -281,7 +281,7 @@ public void syncFunction(List responseParams, boolean syncIm SyncLimit syncLimit = featureAccess.fetchSyncLimit(); numberOfSyncs++; - apiCatalogSync.syncWithDB(syncImmediately, fetchAllSTI, syncLimit); + apiCatalogSync.syncWithDB(syncImmediately, fetchAllSTI, syncLimit, responseParams.get(0).getSource()); if (DbMode.dbType.equals(DbMode.DbType.MONGO_DB)) { dependencyAnalyser.dbState = apiCatalogSync.dbState; dependencyAnalyser.syncWithDb(); diff --git a/apps/api-runtime/src/main/java/com/akto/runtime/APICatalogSync.java b/apps/api-runtime/src/main/java/com/akto/runtime/APICatalogSync.java index 3f0d5ee645..dc8b430dd9 100644 --- a/apps/api-runtime/src/main/java/com/akto/runtime/APICatalogSync.java +++ b/apps/api-runtime/src/main/java/com/akto/runtime/APICatalogSync.java @@ -13,9 +13,6 @@ import com.akto.dto.filter.MergedUrls; import com.akto.dao.monitoring.FilterYamlTemplateDao; import com.akto.dao.runtime_filters.AdvancedTrafficFiltersDao; -import com.akto.dto.*; -import com.akto.dto.billing.SyncLimit; -import com.akto.dto.dependency_flow.DependencyFlow; import com.akto.dto.monitoring.FilterConfig; import com.akto.dto.test_editor.YamlTemplate; import com.akto.dto.traffic.Key; @@ -48,6 +45,7 @@ import com.mongodb.client.model.*; import com.mongodb.client.result.UpdateResult; import org.apache.commons.lang3.math.NumberUtils; +import org.bson.Document; import org.bson.conversions.Bson; import org.bson.json.JsonParseException; import org.bson.types.ObjectId; @@ -1279,7 +1277,7 @@ public ArrayList> getDBUpdatesForTraffic(int apiCollecti return bulkUpdates; } - public DbUpdateReturn getDBUpdatesForParams(APICatalog currentDelta, APICatalog currentState, boolean redactSampleData, boolean collectionLevelRedact) { + public DbUpdateReturn getDBUpdatesForParams(APICatalog currentDelta, APICatalog currentState, boolean redactSampleData, boolean collectionLevelRedact, HttpResponseParams.Source source) { Map dbInfoMap = convertToMap(currentState.getAllTypeInfo()); Map deltaInfoMap = convertToMap(currentDelta.getAllTypeInfo()); @@ -1312,6 +1310,8 @@ public DbUpdateReturn getDBUpdatesForParams(APICatalog currentDelta, APICatalog update = Updates.combine(update, Updates.max(SingleTypeInfo.LAST_SEEN, deltaInfo.getLastSeen())); update = Updates.combine(update, Updates.max(SingleTypeInfo.MAX_VALUE, deltaInfo.getMaxValue())); update = Updates.combine(update, Updates.min(SingleTypeInfo.MIN_VALUE, deltaInfo.getMinValue())); + Bson updateSourceMap = Updates.set(SingleTypeInfo.SOURCES + "." + source.name(), new Document("timestamp", timestamp) ); + update = Updates.combine(update, updateSourceMap); if (!Main.isOnprem) { if (dbInfo != null) { @@ -1795,7 +1795,7 @@ public static Map build(List allParams, Blo int counter = 0; List partnerIpsList = new ArrayList<>(); - public void syncWithDB(boolean syncImmediately, boolean fetchAllSTI, SyncLimit syncLimit) { + public void syncWithDB(boolean syncImmediately, boolean fetchAllSTI, SyncLimit syncLimit, HttpResponseParams.Source source) { loggerMaker.infoAndAddToDb("Started sync with db! syncImmediately="+syncImmediately + " fetchAllSTI="+fetchAllSTI, LogDb.RUNTIME); List> writesForParams = new ArrayList<>(); List> writesForSensitiveSampleData = new ArrayList<>(); @@ -1873,7 +1873,7 @@ public void syncWithDB(boolean syncImmediately, boolean fetchAllSTI, SyncLimit s APICatalog dbCatalog = this.dbState.getOrDefault(apiCollectionId, new APICatalog(apiCollectionId, new HashMap<>(), new HashMap<>())); boolean redactCollectionLevel = apiCollectionToRedactPayload.getOrDefault(apiCollectionId, false); - DbUpdateReturn dbUpdateReturn = getDBUpdatesForParams(deltaCatalog, dbCatalog, redact, redactCollectionLevel); + DbUpdateReturn dbUpdateReturn = getDBUpdatesForParams(deltaCatalog, dbCatalog, redact, redactCollectionLevel, source); writesForParams.addAll(dbUpdateReturn.bulkUpdatesForSingleTypeInfo); writesForSensitiveSampleData.addAll(dbUpdateReturn.bulkUpdatesForSampleData); writesForSensitiveParamInfo.addAll(dbUpdateReturn.bulkUpdatesForSensitiveParamInfo); @@ -1904,7 +1904,7 @@ public void syncWithDB(boolean syncImmediately, boolean fetchAllSTI, SyncLimit s } while (from < writesForParams.size()); } - aktoPolicyNew.syncWithDb(); + aktoPolicyNew.syncWithDb(source); loggerMaker.infoAndAddToDb("adding " + writesForTraffic.size() + " updates for traffic", LogDb.RUNTIME); if(writesForTraffic.size() > 0) { diff --git a/apps/api-runtime/src/main/java/com/akto/runtime/policies/AktoPolicyNew.java b/apps/api-runtime/src/main/java/com/akto/runtime/policies/AktoPolicyNew.java index 942da594ca..97b3f0dec5 100644 --- a/apps/api-runtime/src/main/java/com/akto/runtime/policies/AktoPolicyNew.java +++ b/apps/api-runtime/src/main/java/com/akto/runtime/policies/AktoPolicyNew.java @@ -1,7 +1,6 @@ package com.akto.runtime.policies; import com.akto.dao.*; -import com.akto.dao.context.Context; import com.akto.dao.filter.MergedUrlsDao; import com.akto.dto.*; import com.akto.dto.ApiInfo.ApiInfoKey; @@ -19,9 +18,8 @@ import com.mongodb.BasicDBObject; import com.mongodb.client.model.*; import org.apache.commons.lang3.StringUtils; +import org.bson.Document; import org.bson.conversions.Bson; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import java.util.*; @@ -92,9 +90,9 @@ public void buildFromDb(boolean fetchAllSTI) { loggerMaker.infoAndAddToDb("Built AktoPolicyNew", LogDb.RUNTIME); } - public void syncWithDb() { + public void syncWithDb(HttpResponseParams.Source source) { loggerMaker.infoAndAddToDb("Syncing with db", LogDb.RUNTIME); - UpdateReturn updateReturn = getUpdates(apiInfoCatalogMap); + UpdateReturn updateReturn = getUpdates(apiInfoCatalogMap, source); List> writesForApiInfo = updateReturn.updatesForApiInfo; List> writesForSampleData = updateReturn.updatesForSampleData; loggerMaker.infoAndAddToDb("Writing to db: " + "writesForApiInfoSize="+writesForApiInfo.size() + " writesForSampleData="+ writesForSampleData.size(), LogDb.RUNTIME); @@ -264,7 +262,7 @@ public PolicyCatalog getApiInfoFromMap(ApiInfo.ApiInfoKey apiInfoKey) { return newPolicyCatalog; } - public static UpdateReturn getUpdates(Map apiInfoCatalogMap) { + public static UpdateReturn getUpdates(Map apiInfoCatalogMap, HttpResponseParams.Source source) { List apiInfoList = new ArrayList<>(); List filterSampleDataList = new ArrayList<>(); for (ApiInfoCatalog apiInfoCatalog: apiInfoCatalogMap.values()) { @@ -297,7 +295,7 @@ public static UpdateReturn getUpdates(Map apiInfoCatalo } } - List> updatesForApiInfo = getUpdatesForApiInfo(apiInfoList); + List> updatesForApiInfo = getUpdatesForApiInfo(apiInfoList, source); List> updatesForSampleData = getUpdatesForSampleData(filterSampleDataList); Map> updatesForApiGroups = getUpdatesForApiGroups(apiInfoList); @@ -381,7 +379,7 @@ public UpdateReturn(List> updatesForApiInfo, List> getUpdatesForApiInfo(List apiInfoList) { + public static List> getUpdatesForApiInfo(List apiInfoList, HttpResponseParams.Source source) { List> updates = new ArrayList<>(); for (ApiInfo apiInfo: apiInfoList) { @@ -420,6 +418,9 @@ public static List> getUpdatesForApiInfo(List apiIn // discovered timestamp subUpdates.add(Updates.setOnInsert(ApiInfo.DISCOVERED_TIMESTAMP, apiInfo.getDiscoveredTimestamp())); + // sources + subUpdates.add(Updates.set(SingleTypeInfo.SOURCES + "." + source.name(), new Document("timestamp", apiInfo.getDiscoveredTimestamp()))); + // last seen subUpdates.add(Updates.set(ApiInfo.LAST_SEEN, apiInfo.getLastSeen())); diff --git a/apps/api-runtime/src/test/java/com/akto/parsers/TestDBSync.java b/apps/api-runtime/src/test/java/com/akto/parsers/TestDBSync.java index cf8904f39d..f9f0b64b41 100644 --- a/apps/api-runtime/src/test/java/com/akto/parsers/TestDBSync.java +++ b/apps/api-runtime/src/test/java/com/akto/parsers/TestDBSync.java @@ -97,7 +97,7 @@ public void testParameterizedURL() { aggr.addURL(TestDump2.createSampleParams("user"+i, url+i)); } sync.computeDelta(aggr, true, 0, false); - sync.syncWithDB(false, true, SyncLimit.noLimit); + sync.syncWithDB(false, true, SyncLimit.noLimit, Source.HAR); APICatalogSync.mergeUrlsAndSave(123, true, false, sync.existingAPIsInDb, false); sync.buildFromDB(false, true); @@ -216,7 +216,7 @@ public void testInvalidMergeParameterizedURL() { aggr.addURL(TestDump2.createSampleParams("user"+i, "/payment/id"+i)); } sync.computeDelta(aggr, true, 123, false); - sync.syncWithDB(false, true, SyncLimit.noLimit); + sync.syncWithDB(false, true, SyncLimit.noLimit, Source.HAR); assertEquals(30, sync.getDbState(123).getStrictURLToMethods().size()); @@ -230,7 +230,7 @@ public void testInvalidMergeParameterizedURL() { aggr2.addURL(resp2); sync.computeDelta(aggr2, true, 123, false); - sync.syncWithDB(false, true, SyncLimit.noLimit); + sync.syncWithDB(false, true, SyncLimit.noLimit, Source.HAR); APICatalogSync.mergeUrlsAndSave(123, true, false, sync.existingAPIsInDb, false); sync.buildFromDB(false, true); @@ -575,7 +575,7 @@ public void testHostNameForSourceOther() throws Exception { HttpCallParser httpCallParser = new HttpCallParser("", 100000, 10000, 10000, true); httpCallParser.syncFunction(responseParamsList,true, true, null); - httpCallParser.apiCatalogSync.syncWithDB(true, true, SyncLimit.noLimit); + httpCallParser.apiCatalogSync.syncWithDB(true, true, SyncLimit.noLimit, Source.HAR); ApiCollection one = ApiCollectionsDao.instance.findOne(new BasicDBObject()); String host = "dev-1.akto.io"; diff --git a/apps/api-runtime/src/test/java/com/akto/parsers/TestDump2.java b/apps/api-runtime/src/test/java/com/akto/parsers/TestDump2.java index c95e822238..415228e69c 100644 --- a/apps/api-runtime/src/test/java/com/akto/parsers/TestDump2.java +++ b/apps/api-runtime/src/test/java/com/akto/parsers/TestDump2.java @@ -16,6 +16,7 @@ import com.akto.dto.type.URLMethods.Method; import com.akto.runtime.APICatalogSync; import com.akto.runtime.URLAggregator; +import com.akto.dto.HttpResponseParams.Source; import com.akto.types.CappedSet; import com.mongodb.BasicDBList; import com.mongodb.BasicDBObject; @@ -125,7 +126,7 @@ public void testHappyPath() { aggr.addURL(httpResponseParams); sync.computeDelta(aggr, false, 0, false); - APICatalogSync.DbUpdateReturn dbUpdateReturn = sync.getDBUpdatesForParams(sync.getDelta(0), sync.getDbState(0), false, false); + APICatalogSync.DbUpdateReturn dbUpdateReturn = sync.getDBUpdatesForParams(sync.getDelta(0), sync.getDbState(0), false, false, Source.HAR); assertEquals(15, dbUpdateReturn.bulkUpdatesForSingleTypeInfo.size()); assertEquals(2, sync.getDBUpdatesForTraffic(0, sync.getDelta(0)).size()); assertEquals(1, sync.getDBUpdatesForSampleData(0, sync.getDelta(0), sync.getDbState(0),true, false, false).size()); @@ -154,7 +155,7 @@ public void simpleTestForSingleCollection(int collectionId, APICatalogSync sync) RequestTemplate respTemplate = reqTemplate.getResponseTemplates().get(resp.statusCode); assertEquals(1, respTemplate.getUserIds().size()); assertEquals(3, respTemplate.getParameters().size()); - APICatalogSync.DbUpdateReturn dbUpdateReturn = sync.getDBUpdatesForParams(sync.getDelta(collectionId), sync.getDbState(collectionId), false, false); + APICatalogSync.DbUpdateReturn dbUpdateReturn = sync.getDBUpdatesForParams(sync.getDelta(collectionId), sync.getDbState(collectionId), false, false, Source.HAR); assertEquals(24, dbUpdateReturn.bulkUpdatesForSingleTypeInfo.size()); assertEquals(2, sync.getDBUpdatesForTraffic(collectionId, sync.getDelta(collectionId)).size()); } @@ -167,9 +168,9 @@ public void simpleTest() { simpleTestForSingleCollection(0, sync); simpleTestForSingleCollection(1, sync); simpleTestForSingleCollection(2, sync); - assertEquals(24, sync.getDBUpdatesForParams(sync.getDelta(0), sync.getDbState(0),false, false).bulkUpdatesForSingleTypeInfo.size()); - assertEquals(24, sync.getDBUpdatesForParams(sync.getDelta(1), sync.getDbState(1),false, false).bulkUpdatesForSingleTypeInfo.size()); - assertEquals(24, sync.getDBUpdatesForParams(sync.getDelta(2), sync.getDbState(2),false, false).bulkUpdatesForSingleTypeInfo.size()); + assertEquals(24, sync.getDBUpdatesForParams(sync.getDelta(0), sync.getDbState(0),false, false, Source.HAR).bulkUpdatesForSingleTypeInfo.size()); + assertEquals(24, sync.getDBUpdatesForParams(sync.getDelta(1), sync.getDbState(1),false, false, Source.HAR).bulkUpdatesForSingleTypeInfo.size()); + assertEquals(24, sync.getDBUpdatesForParams(sync.getDelta(2), sync.getDbState(2),false, false, Source.HAR).bulkUpdatesForSingleTypeInfo.size()); } @Test @@ -383,7 +384,7 @@ public void repetitiveKeyTest() { // TODO: investigate and fix this // assertEquals(1, respTemplate.getParameters().size()); - List updates = sync.getDBUpdatesForParams(sync.getDelta(0), sync.getDbState(0), false, false).bulkUpdatesForSingleTypeInfo; + List updates = sync.getDBUpdatesForParams(sync.getDelta(0), sync.getDbState(0), false, false, Source.HAR).bulkUpdatesForSingleTypeInfo; } @Test diff --git a/apps/api-runtime/src/test/java/com/akto/parsers/TestMergingNew.java b/apps/api-runtime/src/test/java/com/akto/parsers/TestMergingNew.java index ec8ba49895..2e09265a7d 100644 --- a/apps/api-runtime/src/test/java/com/akto/parsers/TestMergingNew.java +++ b/apps/api-runtime/src/test/java/com/akto/parsers/TestMergingNew.java @@ -9,6 +9,7 @@ import com.akto.dto.HttpRequestParams; import com.akto.dto.HttpResponseParams; import com.akto.dto.IgnoreData; +import com.akto.dto.HttpResponseParams.Source; import com.akto.dto.billing.SyncLimit; import com.akto.dto.*; import com.akto.dto.traffic.SampleData; @@ -17,25 +18,20 @@ import com.akto.runtime.APICatalogSync; import com.akto.types.CappedSet; import com.akto.util.filter.DictionaryFilter; -import com.akto.utils.RedactSampleData; import com.google.api.client.util.Charsets; import com.google.common.hash.BloomFilter; import com.google.common.hash.Funnels; -import com.google.protobuf.Api; import com.mongodb.BasicDBObject; import com.mongodb.client.model.Filters; -import com.mongodb.client.model.Updates; import org.bson.conversions.Bson; import org.bson.types.ObjectId; import org.junit.Before; -import org.junit.BeforeClass; import org.junit.Test; import java.util.*; import static com.akto.parsers.TestDump2.createList; import static com.akto.runtime.APICatalogSync.mergeUrlsAndSave; -import static com.akto.runtime.APICatalogSync.tryMergeURLsInCollection; import static org.junit.Assert.*; public class TestMergingNew extends MongoBasedTest { @@ -83,15 +79,15 @@ public void testMultipleIntegerMerging() { } parser.syncFunction(responseParams.subList(0,10), false, true, null); - parser.apiCatalogSync.syncWithDB(false, true, SyncLimit.noLimit); + parser.apiCatalogSync.syncWithDB(false, true, SyncLimit.noLimit, Source.HAR); parser.syncFunction(responseParams.subList(10,15), false, true, null); - parser.apiCatalogSync.syncWithDB(false, true, SyncLimit.noLimit); + parser.apiCatalogSync.syncWithDB(false, true, SyncLimit.noLimit, Source.HAR); APICatalogSync.mergeUrlsAndSave(123,true, false, parser.apiCatalogSync.existingAPIsInDb, false); parser.apiCatalogSync.buildFromDB(false, true); assertEquals(0, getStaticURLsSize(parser)); parser.syncFunction(responseParams.subList(15,20), false, true, null); - parser.apiCatalogSync.syncWithDB(false, true, SyncLimit.noLimit); + parser.apiCatalogSync.syncWithDB(false, true, SyncLimit.noLimit, Source.HAR); APICatalogSync.mergeUrlsAndSave(123,true, false, parser.apiCatalogSync.existingAPIsInDb, false); parser.apiCatalogSync.buildFromDB(false, true); APICatalogSync.mergeUrlsAndSave(123,true, false, parser.apiCatalogSync.existingAPIsInDb, false); @@ -131,7 +127,7 @@ public void testStringMerging() { } parser.syncFunction(responseParams, false, true, null); - parser.apiCatalogSync.syncWithDB(false, true, SyncLimit.noLimit); + parser.apiCatalogSync.syncWithDB(false, true, SyncLimit.noLimit, Source.HAR); APICatalogSync.mergeUrlsAndSave(123, true, false, parser.apiCatalogSync.existingAPIsInDb, false); parser.apiCatalogSync.buildFromDB(true, true); Map urlTemplateMap = parser.apiCatalogSync.getDbState(123).getTemplateURLToMethods(); @@ -171,7 +167,7 @@ public void testCaseInsensitiveApisMerging(){ } parser.syncFunction(responseParams, false, true, null); - parser.apiCatalogSync.syncWithDB(false, true, SyncLimit.noLimit); + parser.apiCatalogSync.syncWithDB(false, true, SyncLimit.noLimit, Source.HAR); APICatalogSync.mergeUrlsAndSave(123, true, false, parser.apiCatalogSync.existingAPIsInDb, true); parser.apiCatalogSync.buildFromDB(true, true); Map strictUrlMap = parser.apiCatalogSync.getDbState(123).getStrictURLToMethods(); @@ -213,17 +209,17 @@ public void testEnglishWordsUrlTestString() { } parser.syncFunction(responseParams.subList(0,23), false, true, null); - parser.apiCatalogSync.syncWithDB(false, true, SyncLimit.noLimit); + parser.apiCatalogSync.syncWithDB(false, true, SyncLimit.noLimit, Source.HAR); assertEquals(23, getStaticURLsSize(parser)); parser.syncFunction(responseParams.subList(23,28), false, true, null); - parser.apiCatalogSync.syncWithDB(false, true, SyncLimit.noLimit); + parser.apiCatalogSync.syncWithDB(false, true, SyncLimit.noLimit, Source.HAR); APICatalogSync.mergeUrlsAndSave(123,true, false, parser.apiCatalogSync.existingAPIsInDb, false); parser.apiCatalogSync.buildFromDB(false, true); assertEquals(28, getStaticURLsSize(parser)); parser.syncFunction(responseParams.subList(28,33), false, true, null); - parser.apiCatalogSync.syncWithDB(false, true, SyncLimit.noLimit); + parser.apiCatalogSync.syncWithDB(false, true, SyncLimit.noLimit, Source.HAR); assertEquals(33, getStaticURLsSize(parser)); } @@ -260,13 +256,13 @@ public void testmultipleUUIDForceMerge(){ } parser.syncFunction(responseParams.subList(0,1), false,true, null); - parser.apiCatalogSync.syncWithDB(false,true, SyncLimit.noLimit); + parser.apiCatalogSync.syncWithDB(false,true, SyncLimit.noLimit, Source.HAR); APICatalogSync.mergeUrlsAndSave(123,true, false, parser.apiCatalogSync.existingAPIsInDb, false); parser.apiCatalogSync.buildFromDB(false, true); assertEquals(0, getStaticURLsSize(parser)); parser.syncFunction(responseParams.subList(1,2), false,true, null); - parser.apiCatalogSync.syncWithDB(false,true, SyncLimit.noLimit); + parser.apiCatalogSync.syncWithDB(false,true, SyncLimit.noLimit, Source.HAR); APICatalogSync.mergeUrlsAndSave(123,true, false, parser.apiCatalogSync.existingAPIsInDb, false); parser.apiCatalogSync.buildFromDB(false, true); assertEquals(0, getStaticURLsSize(parser)); @@ -278,7 +274,7 @@ public void testmultipleUUIDForceMerge(){ parser.syncFunction(Collections.singletonList(createDifferentHttpResponseParams(10000, url + "avneesh@akto.io" + "/subproduct/" + "avneesh@akto.io" + "/subitem/" + "avneesh@akto.io" + "/id/" + "112" )), false,true, null); // adding this just to see if multiple subTypes of urlParams are recorded or not (not for UUID merging) - parser.apiCatalogSync.syncWithDB(false,true, SyncLimit.noLimit); + parser.apiCatalogSync.syncWithDB(false,true, SyncLimit.noLimit, Source.HAR); APICatalogSync.mergeUrlsAndSave(123,true, false, parser.apiCatalogSync.existingAPIsInDb, false); parser.apiCatalogSync.buildFromDB(false, true); assertEquals(0, getStaticURLsSize(parser)); @@ -318,7 +314,7 @@ public void testFirstUrlParameterMerging(){ } parser.syncFunction(responseParams, false, true, null); - parser.apiCatalogSync.syncWithDB(false, true, SyncLimit.noLimit); + parser.apiCatalogSync.syncWithDB(false, true, SyncLimit.noLimit, Source.HAR); parser.apiCatalogSync.buildFromDB(false, true); assertEquals(1, parser.apiCatalogSync.getDbState(123).getStrictURLToMethods().size()); assertEquals(3, parser.apiCatalogSync.getDbState(123).getTemplateURLToMethods().size()); @@ -347,13 +343,13 @@ public void testUUIDForceMerge() { } parser.syncFunction(responseParams.subList(0,1), false, true, null); - parser.apiCatalogSync.syncWithDB(false, true, SyncLimit.noLimit); + parser.apiCatalogSync.syncWithDB(false, true, SyncLimit.noLimit, Source.HAR); APICatalogSync.mergeUrlsAndSave(123,true, false, parser.apiCatalogSync.existingAPIsInDb, false); parser.apiCatalogSync.buildFromDB(false, true); assertEquals(0, getStaticURLsSize(parser)); parser.syncFunction(responseParams.subList(1,2), false, true, null); - parser.apiCatalogSync.syncWithDB(false, true, SyncLimit.noLimit); + parser.apiCatalogSync.syncWithDB(false, true, SyncLimit.noLimit, Source.HAR); APICatalogSync.mergeUrlsAndSave(123,true, false, parser.apiCatalogSync.existingAPIsInDb, false); parser.apiCatalogSync.buildFromDB(false, true); assertEquals(0, getStaticURLsSize(parser)); @@ -363,7 +359,7 @@ public void testUUIDForceMerge() { parser.syncFunction(responseParams.subList(3,10), false, true, null); parser.syncFunction(Collections.singletonList(createDifferentHttpResponseParams(10000, url+"avneesh@akto.io"+"/received")), false, true, null); // adding this just to see if multiple subTypes of urlParams are recorded or not (not for UUID merging) - parser.apiCatalogSync.syncWithDB(false, true, SyncLimit.noLimit); + parser.apiCatalogSync.syncWithDB(false, true, SyncLimit.noLimit, Source.HAR); APICatalogSync.mergeUrlsAndSave(123,true, false, parser.apiCatalogSync.existingAPIsInDb, false); parser.apiCatalogSync.buildFromDB(false, true); assertEquals(0, getStaticURLsSize(parser)); @@ -401,17 +397,17 @@ public void testParameterizedURLsTestString() { } parser.syncFunction(responseParams.subList(0,23), false, true, null); - parser.apiCatalogSync.syncWithDB(false, true, SyncLimit.noLimit); + parser.apiCatalogSync.syncWithDB(false, true, SyncLimit.noLimit, Source.HAR); assertEquals(23, getStaticURLsSize(parser)); parser.syncFunction(responseParams.subList(23,28), false, true, null); - parser.apiCatalogSync.syncWithDB(false, true, SyncLimit.noLimit); + parser.apiCatalogSync.syncWithDB(false, true, SyncLimit.noLimit, Source.HAR); APICatalogSync.mergeUrlsAndSave(123,true, false, parser.apiCatalogSync.existingAPIsInDb, false); parser.apiCatalogSync.buildFromDB(false, true); assertEquals(0, getStaticURLsSize(parser)); parser.syncFunction(responseParams.subList(28,33), false, true, null); - parser.apiCatalogSync.syncWithDB(false, true, SyncLimit.noLimit); + parser.apiCatalogSync.syncWithDB(false, true, SyncLimit.noLimit, Source.HAR); assertEquals(0, getStaticURLsSize(parser)); } @@ -431,17 +427,17 @@ public void testNonJsonResponsePayloadPayload() throws Exception { HttpCallParser parser = new HttpCallParser("userIdentifier", 1, 1, 1, true); parser.syncFunction(responseParams.subList(0,10), false, true, null); - parser.apiCatalogSync.syncWithDB(false, true, SyncLimit.noLimit); + parser.apiCatalogSync.syncWithDB(false, true, SyncLimit.noLimit, Source.HAR); parser = new HttpCallParser("userIdentifier", 1, 1, 1, true); parser.syncFunction(responseParams.subList(10,25), false, true, null); - parser.apiCatalogSync.syncWithDB(false, true, SyncLimit.noLimit); + parser.apiCatalogSync.syncWithDB(false, true, SyncLimit.noLimit, Source.HAR); APICatalogSync.mergeUrlsAndSave(123,true, false, parser.apiCatalogSync.existingAPIsInDb, false); parser.apiCatalogSync.buildFromDB(false, true); parser.syncFunction(responseParams.subList(25,30), false, true, null); - parser.apiCatalogSync.syncWithDB(false, true, SyncLimit.noLimit); + parser.apiCatalogSync.syncWithDB(false, true, SyncLimit.noLimit, Source.HAR); APICatalogSync.mergeUrlsAndSave(123,true, false, parser.apiCatalogSync.existingAPIsInDb, false); parser.apiCatalogSync.buildFromDB(false, true); @@ -475,11 +471,11 @@ public void testEmptyResponsePayload() throws Exception { HttpCallParser parser = new HttpCallParser("userIdentifier", 1, 1, 1, true); parser.syncFunction(responseParams.subList(0,10), false, true, null); - parser.apiCatalogSync.syncWithDB(false, true, SyncLimit.noLimit); + parser.apiCatalogSync.syncWithDB(false, true, SyncLimit.noLimit, Source.HAR); parser.syncFunction(responseParams.subList(10,25), false, true, null); - parser.apiCatalogSync.syncWithDB(false, true, SyncLimit.noLimit); + parser.apiCatalogSync.syncWithDB(false, true, SyncLimit.noLimit, Source.HAR); parser.syncFunction(responseParams.subList(25,30), false, true, null); - parser.apiCatalogSync.syncWithDB(false, true, SyncLimit.noLimit); + parser.apiCatalogSync.syncWithDB(false, true, SyncLimit.noLimit, Source.HAR); APICatalogSync.mergeUrlsAndSave(123,true, false, parser.apiCatalogSync.existingAPIsInDb, false); parser.apiCatalogSync.buildFromDB(false, true); @@ -516,11 +512,11 @@ public void testStrictIntoTemplate() { parser.apiCatalogSync.getDbState(123).getTemplateURLToMethods().put(urlTemplate, new RequestTemplate(new HashMap<>(), new HashMap<>(), new HashMap<>(), new TrafficRecorder(new HashMap<>()))); parser.syncFunction(responseParams.subList(0,15), false, true, null); - parser.apiCatalogSync.syncWithDB(false, true, SyncLimit.noLimit); + parser.apiCatalogSync.syncWithDB(false, true, SyncLimit.noLimit, Source.HAR); parser.syncFunction(responseParams.subList(15,25), false, true, null); - parser.apiCatalogSync.syncWithDB(false, true, SyncLimit.noLimit); + parser.apiCatalogSync.syncWithDB(false, true, SyncLimit.noLimit, Source.HAR); parser.syncFunction(responseParams.subList(25,30), false, true, null); - parser.apiCatalogSync.syncWithDB(false, true, SyncLimit.noLimit); + parser.apiCatalogSync.syncWithDB(false, true, SyncLimit.noLimit, Source.HAR); APICatalogSync.mergeUrlsAndSave(123,true, false, parser.apiCatalogSync.existingAPIsInDb, false); parser.apiCatalogSync.buildFromDB(false, true); @@ -551,11 +547,11 @@ public void test20percentCondition() { } parser.syncFunction(responseParams.subList(0,23), false, true, null); - parser.apiCatalogSync.syncWithDB(false, true, SyncLimit.noLimit); + parser.apiCatalogSync.syncWithDB(false, true, SyncLimit.noLimit, Source.HAR); parser.syncFunction(responseParams.subList(23,28), false, true, null); - parser.apiCatalogSync.syncWithDB(false, true, SyncLimit.noLimit); + parser.apiCatalogSync.syncWithDB(false, true, SyncLimit.noLimit, Source.HAR); parser.syncFunction(responseParams.subList(28,33), false, true, null); - parser.apiCatalogSync.syncWithDB(false, true, SyncLimit.noLimit); + parser.apiCatalogSync.syncWithDB(false, true, SyncLimit.noLimit, Source.HAR); APICatalogSync.mergeUrlsAndSave(123,true, false, parser.apiCatalogSync.existingAPIsInDb, false); parser.apiCatalogSync.buildFromDB(false, true); @@ -757,13 +753,13 @@ public void testUrlParamSingleTypeInfoAndValues() { } parser.syncFunction(responseParams.subList(0,10), false, true, null); - parser.apiCatalogSync.syncWithDB(false, true, SyncLimit.noLimit); + parser.apiCatalogSync.syncWithDB(false, true, SyncLimit.noLimit, Source.HAR); // dbState doesn't have any template URLs initially so no urlParams are considered testSampleSizeAndDomainOfSti(parser,10, 10, SingleTypeInfo.Domain.ENUM, SingleTypeInfo.Domain.ENUM); parser.syncFunction(responseParams.subList(10,55), false, true, null); - parser.apiCatalogSync.syncWithDB(false, true, SyncLimit.noLimit); + parser.apiCatalogSync.syncWithDB(false, true, SyncLimit.noLimit, Source.HAR); APICatalogSync.mergeUrlsAndSave(123,true, false, parser.apiCatalogSync.existingAPIsInDb, false); parser.apiCatalogSync.buildFromDB(false, true); @@ -771,8 +767,8 @@ public void testUrlParamSingleTypeInfoAndValues() { assertEquals(0,getStaticURLsSize(parser)); testSampleSizeAndDomainOfSti(parser,55, 55, SingleTypeInfo.Domain.ENUM, SingleTypeInfo.Domain.ENUM); - parser.apiCatalogSync.syncWithDB(false, true, SyncLimit.noLimit); - parser.apiCatalogSync.syncWithDB(false, true, SyncLimit.noLimit); + parser.apiCatalogSync.syncWithDB(false, true, SyncLimit.noLimit, Source.HAR); + parser.apiCatalogSync.syncWithDB(false, true, SyncLimit.noLimit, Source.HAR); APICatalogSync.mergeUrlsAndSave(123,true, false, parser.apiCatalogSync.existingAPIsInDb, false); parser.apiCatalogSync.buildFromDB(false, true); @@ -782,14 +778,14 @@ public void testUrlParamSingleTypeInfoAndValues() { // using the new or old parser shouldn't change the result HttpCallParser parserNew = new HttpCallParser("userIdentifier", 1, 1, 1, true); parserNew.syncFunction(responseParams.subList(55,70), false, true, null); - parserNew.apiCatalogSync.syncWithDB(false, true, SyncLimit.noLimit); + parserNew.apiCatalogSync.syncWithDB(false, true, SyncLimit.noLimit, Source.HAR); APICatalogSync.mergeUrlsAndSave(123,true, false, parser.apiCatalogSync.existingAPIsInDb, false); parserNew.apiCatalogSync.buildFromDB(false, true); parserNew.syncFunction(responseParams.subList(70,150), false, true, null); - parserNew.apiCatalogSync.syncWithDB(false, true, SyncLimit.noLimit); + parserNew.apiCatalogSync.syncWithDB(false, true, SyncLimit.noLimit, Source.HAR); parserNew.syncFunction(responseParams.subList(150,200), false, true, null); - parserNew.apiCatalogSync.syncWithDB(false, true, SyncLimit.noLimit); + parserNew.apiCatalogSync.syncWithDB(false, true, SyncLimit.noLimit, Source.HAR); APICatalogSync.mergeUrlsAndSave(123,true, false, parser.apiCatalogSync.existingAPIsInDb, false); parserNew.apiCatalogSync.buildFromDB(false, true); @@ -831,7 +827,7 @@ public void testMinMaxAndLastSeenNew() { HttpResponseParams httpResponseParams3 = createHttpResponseForMinMax(url+"books1", 2500.9F,-200F ); parser.syncFunction(Arrays.asList(httpResponseParams1, httpResponseParams2, httpResponseParams3), false, true, null); - parser.apiCatalogSync.syncWithDB(false, true, SyncLimit.noLimit); + parser.apiCatalogSync.syncWithDB(false, true, SyncLimit.noLimit, Source.HAR); Collection requestTemplates = parser.apiCatalogSync.getDbState(123).getStrictURLToMethods().values(); validateMinMax(requestTemplates, 2500, 2, -98, -200); @@ -851,17 +847,17 @@ public void testMinMaxAndLastSeenNew() { httpResponseParams = createHttpResponseForMinMax(url+"books"+i, reqMin, respMin); parser.syncFunction(Collections.singletonList(httpResponseParams), false, true, null); } - parser.apiCatalogSync.syncWithDB(false, true, SyncLimit.noLimit); + parser.apiCatalogSync.syncWithDB(false, true, SyncLimit.noLimit, Source.HAR); APICatalogSync.mergeUrlsAndSave(123, true, false, parser.apiCatalogSync.existingAPIsInDb, false); parser.apiCatalogSync.buildFromDB(false, true); HttpResponseParams httpResponseParams = createHttpResponseForMinMax(url+"books99", 190f, -190f); parser.syncFunction(Collections.singletonList(httpResponseParams), false, true, null); - parser.apiCatalogSync.syncWithDB(false, true, SyncLimit.noLimit); + parser.apiCatalogSync.syncWithDB(false, true, SyncLimit.noLimit, Source.HAR); httpResponseParams = createHttpResponseForMinMax(url+"books100", 190f, -190f); parser.syncFunction(Collections.singletonList(httpResponseParams), false, true, null); - parser.apiCatalogSync.syncWithDB(false, true, SyncLimit.noLimit); + parser.apiCatalogSync.syncWithDB(false, true, SyncLimit.noLimit, Source.HAR); APICatalogSync.mergeUrlsAndSave(123, true, false, parser.apiCatalogSync.existingAPIsInDb, false); @@ -879,7 +875,7 @@ public void testMinMaxAndLastSeenNew() { parserNew.syncFunction(Collections.singletonList(httpResponseParams), false, true, null); httpResponseParams = createHttpResponseForMinMax(url+"books15", 19f, -19000f); parserNew.syncFunction(Collections.singletonList(httpResponseParams), false, true, null); - parserNew.apiCatalogSync.syncWithDB(false, true, SyncLimit.noLimit); + parserNew.apiCatalogSync.syncWithDB(false, true, SyncLimit.noLimit, Source.HAR); requestTemplates = parserNew.apiCatalogSync.getDbState(123).getTemplateURLToMethods().values(); validateMinMax(requestTemplates, 19000, Double.valueOf(reqMin+"").longValue(), Double.valueOf(respMax+"").longValue(), -19000); @@ -947,8 +943,8 @@ private APICatalogSync.DbUpdateReturn cleanSync(HttpResponseParams httpResponseP parser.syncFunction(Collections.singletonList(httpResponseParams),false, true, null); APICatalogSync apiCatalogSync = parser.apiCatalogSync; return apiCatalogSync.getDBUpdatesForParams( - apiCatalogSync.getDelta(collectionId), apiCatalogSync.getDbState(collectionId), false, false - ); + apiCatalogSync.getDelta(collectionId), apiCatalogSync.getDbState(collectionId), false, false, + Source.HAR); } @@ -965,11 +961,11 @@ public void testSpecialCharHostMerging() { try { HttpResponseParams httpResponseParams1 = HttpCallParser.parseKafkaMessage(payload1); httpCallParser.syncFunction(Collections.singletonList(httpResponseParams1),false, true, null); - httpCallParser.apiCatalogSync.syncWithDB(false, true, SyncLimit.noLimit); + httpCallParser.apiCatalogSync.syncWithDB(false, true, SyncLimit.noLimit, Source.HAR); HttpResponseParams httpResponseParams2 = HttpCallParser.parseKafkaMessage(payload2); httpCallParser.syncFunction(Collections.singletonList(httpResponseParams2),false, true, null); - httpCallParser.apiCatalogSync.syncWithDB(false, true, SyncLimit.noLimit); + httpCallParser.apiCatalogSync.syncWithDB(false, true, SyncLimit.noLimit, Source.HAR); } catch(Exception e) { System.out.println("dfg"); @@ -1001,7 +997,7 @@ public void testSampleDataUpdate() throws Exception { ); httpCallParser.syncFunction(Collections.singletonList(httpResponseParams1),false, true, null); - httpCallParser.apiCatalogSync.syncWithDB(false, true, SyncLimit.noLimit); + httpCallParser.apiCatalogSync.syncWithDB(false, true, SyncLimit.noLimit, Source.HAR); SampleData sampleData1 = SampleDataDao.instance.findOne(filter1); assertEquals(1, sampleData1.getSamples().size()); @@ -1010,7 +1006,7 @@ public void testSampleDataUpdate() throws Exception { String payload1UpdatedRequestPayload = "{\"method\":\"PUT\",\"requestPayload\":\"{\\\"photoUrls\\\":[\\\"string\\\"],\\\"name\\\":\\\"tommie\\\",\\\"id\\\":0,\\\"category\\\":{\\\"id\\\":0,\\\"name\\\":\\\"string\\\"},\\\"tags\\\":[{\\\"id\\\":0,\\\"name\\\":\\\"string\\\"}],\\\"status\\\":\\\"available\\\"}\",\"responsePayload\":\"{\\\"id\\\":9223372036854775807,\\\"category\\\":{\\\"id\\\":0,\\\"name\\\":\\\"string\\\"},\\\"name\\\":\\\"doggie\\\",\\\"photoUrls\\\":[\\\"string\\\"],\\\"tags\\\":[{\\\"id\\\":0,\\\"name\\\":\\\"string\\\"}],\\\"status\\\":\\\"available\\\"}\",\"ip\":\"null\",\"source\":\"MIRRORING\",\"type\":\"HTTP/2\",\"akto_vxlan_id\":\"1661807253\",\"path\":\"https://petstore.swagger.io/v2/pet\",\"requestHeaders\":\"{\\\"Origin\\\":\\\"https://petstore.swagger.io\\\",\\\"Accept\\\":\\\"application/json\\\",\\\"User-Agent\\\":\\\"Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:95.0) Gecko/20100101 Firefox/95.0\\\",\\\"Referer\\\":\\\"https://petstore.swagger.io/\\\",\\\"Connection\\\":\\\"keep-alive\\\",\\\"Sec-Fetch-Dest\\\":\\\"empty\\\",\\\"Sec-Fetch-Site\\\":\\\"same-origin\\\",\\\"Host\\\":\\\"petstore.swagger.io\\\",\\\"Accept-Encoding\\\":\\\"gzip, deflate, br\\\",\\\"Sec-Fetch-Mode\\\":\\\"cors\\\",\\\"TE\\\":\\\"trailers\\\",\\\"Accept-Language\\\":\\\"en-US,en;q=0.5\\\",\\\"Content-Length\\\":\\\"215\\\",\\\"Content-Type\\\":\\\"application/json\\\"}\",\"responseHeaders\":\"{\\\"date\\\":\\\"Tue, 04 Jan 2022 20:11:58 GMT\\\",\\\"access-control-allow-origin\\\":\\\"*\\\",\\\"server\\\":\\\"Jetty(9.2.9.v20150224)\\\",\\\"access-control-allow-headers\\\":\\\"Content-Type, api_key, Authorization\\\",\\\"X-Firefox-Spdy\\\":\\\"h2\\\",\\\"content-type\\\":\\\"application/json\\\",\\\"access-control-allow-methods\\\":\\\"GET, POST, DELETE, PUT\\\"}\",\"time\":\"1641327118\",\"contentType\":\"application/json\",\"akto_account_id\":\"1000000\",\"statusCode\":\"200\",\"status\":\"OK\"}"; HttpResponseParams httpResponseParams2 = HttpCallParser.parseKafkaMessage(payload1UpdatedRequestPayload); httpCallParser.syncFunction(Collections.singletonList(httpResponseParams2), false, true, null); - httpCallParser.apiCatalogSync.syncWithDB(false, true, SyncLimit.noLimit); + httpCallParser.apiCatalogSync.syncWithDB(false, true, SyncLimit.noLimit, Source.HAR); sampleData1 = SampleDataDao.instance.findOne(filter1); assertEquals(1, sampleData1.getSamples().size()); @@ -1020,7 +1016,7 @@ public void testSampleDataUpdate() throws Exception { String payload2 = "{\"method\":\"POST\",\"requestPayload\":\"{\\\"photoUrls\\\":[\\\"string\\\"],\\\"name\\\":\\\"tommie\\\",\\\"id\\\":0,\\\"category\\\":{\\\"id\\\":0,\\\"name\\\":\\\"string\\\"},\\\"tags\\\":[{\\\"id\\\":0,\\\"name\\\":\\\"string\\\"}],\\\"status\\\":\\\"available\\\"}\",\"responsePayload\":\"{\\\"id\\\":9223372036854775807,\\\"category\\\":{\\\"id\\\":0,\\\"name\\\":\\\"string\\\"},\\\"name\\\":\\\"doggie\\\",\\\"photoUrls\\\":[\\\"string\\\"],\\\"tags\\\":[{\\\"id\\\":0,\\\"name\\\":\\\"string\\\"}],\\\"status\\\":\\\"available\\\"}\",\"ip\":\"null\",\"source\":\"MIRRORING\",\"type\":\"HTTP/2\",\"akto_vxlan_id\":\"1661807253\",\"path\":\"https://petstore.swagger.io/v2/books/1\",\"requestHeaders\":\"{\\\"Origin\\\":\\\"https://petstore.swagger.io\\\",\\\"Accept\\\":\\\"application/json\\\",\\\"User-Agent\\\":\\\"Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:95.0) Gecko/20100101 Firefox/95.0\\\",\\\"Referer\\\":\\\"https://petstore.swagger.io/\\\",\\\"Connection\\\":\\\"keep-alive\\\",\\\"Sec-Fetch-Dest\\\":\\\"empty\\\",\\\"Sec-Fetch-Site\\\":\\\"same-origin\\\",\\\"Host\\\":\\\"petstore.swagger.io\\\",\\\"Accept-Encoding\\\":\\\"gzip, deflate, br\\\",\\\"Sec-Fetch-Mode\\\":\\\"cors\\\",\\\"TE\\\":\\\"trailers\\\",\\\"Accept-Language\\\":\\\"en-US,en;q=0.5\\\",\\\"Content-Length\\\":\\\"215\\\",\\\"Content-Type\\\":\\\"application/json\\\"}\",\"responseHeaders\":\"{\\\"date\\\":\\\"Tue, 04 Jan 2022 20:11:58 GMT\\\",\\\"access-control-allow-origin\\\":\\\"*\\\",\\\"server\\\":\\\"Jetty(9.2.9.v20150224)\\\",\\\"access-control-allow-headers\\\":\\\"Content-Type, api_key, Authorization\\\",\\\"X-Firefox-Spdy\\\":\\\"h2\\\",\\\"content-type\\\":\\\"application/json\\\",\\\"access-control-allow-methods\\\":\\\"GET, POST, DELETE, PUT\\\"}\",\"time\":\"1641327118\",\"contentType\":\"application/json\",\"akto_account_id\":\"1000000\",\"statusCode\":\"200\",\"status\":\"OK\"}"; HttpResponseParams httpResponseParams3 = HttpCallParser.parseKafkaMessage(payload2); httpCallParser.syncFunction(Collections.singletonList(httpResponseParams3), false, true, null); - httpCallParser.apiCatalogSync.syncWithDB(false, true, SyncLimit.noLimit); + httpCallParser.apiCatalogSync.syncWithDB(false, true, SyncLimit.noLimit, Source.HAR); Bson filter2 = Filters.and( Filters.eq("_id.url", "https://petstore.swagger.io/v2/books/INTEGER"), @@ -1034,7 +1030,7 @@ public void testSampleDataUpdate() throws Exception { String payload3 = "{\"method\":\"POST\",\"requestPayload\":\"{\\\"photoUrls\\\":[\\\"string\\\"],\\\"name\\\":\\\"charlie\\\",\\\"id\\\":0,\\\"category\\\":{\\\"id\\\":0,\\\"name\\\":\\\"string\\\"},\\\"tags\\\":[{\\\"id\\\":0,\\\"name\\\":\\\"string\\\"}],\\\"status\\\":\\\"available\\\"}\",\"responsePayload\":\"{\\\"id\\\":9223372036854775807,\\\"category\\\":{\\\"id\\\":0,\\\"name\\\":\\\"string\\\"},\\\"name\\\":\\\"doggie\\\",\\\"photoUrls\\\":[\\\"string\\\"],\\\"tags\\\":[{\\\"id\\\":0,\\\"name\\\":\\\"string\\\"}],\\\"status\\\":\\\"available\\\"}\",\"ip\":\"null\",\"source\":\"MIRRORING\",\"type\":\"HTTP/2\",\"akto_vxlan_id\":\"1661807253\",\"path\":\"https://petstore.swagger.io/v2/books/2\",\"requestHeaders\":\"{\\\"Origin\\\":\\\"https://petstore.swagger.io\\\",\\\"Accept\\\":\\\"application/json\\\",\\\"User-Agent\\\":\\\"Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:95.0) Gecko/20100101 Firefox/95.0\\\",\\\"Referer\\\":\\\"https://petstore.swagger.io/\\\",\\\"Connection\\\":\\\"keep-alive\\\",\\\"Sec-Fetch-Dest\\\":\\\"empty\\\",\\\"Sec-Fetch-Site\\\":\\\"same-origin\\\",\\\"Host\\\":\\\"petstore.swagger.io\\\",\\\"Accept-Encoding\\\":\\\"gzip, deflate, br\\\",\\\"Sec-Fetch-Mode\\\":\\\"cors\\\",\\\"TE\\\":\\\"trailers\\\",\\\"Accept-Language\\\":\\\"en-US,en;q=0.5\\\",\\\"Content-Length\\\":\\\"215\\\",\\\"Content-Type\\\":\\\"application/json\\\"}\",\"responseHeaders\":\"{\\\"date\\\":\\\"Tue, 04 Jan 2022 20:11:58 GMT\\\",\\\"access-control-allow-origin\\\":\\\"*\\\",\\\"server\\\":\\\"Jetty(9.2.9.v20150224)\\\",\\\"access-control-allow-headers\\\":\\\"Content-Type, api_key, Authorization\\\",\\\"X-Firefox-Spdy\\\":\\\"h2\\\",\\\"content-type\\\":\\\"application/json\\\",\\\"access-control-allow-methods\\\":\\\"GET, POST, DELETE, PUT\\\"}\",\"time\":\"1641327118\",\"contentType\":\"application/json\",\"akto_account_id\":\"1000000\",\"statusCode\":\"200\",\"status\":\"OK\"}"; HttpResponseParams httpResponseParams4 = HttpCallParser.parseKafkaMessage(payload3); httpCallParser.syncFunction(Collections.singletonList(httpResponseParams4), false, true, null); - httpCallParser.apiCatalogSync.syncWithDB(false, true, SyncLimit.noLimit); + httpCallParser.apiCatalogSync.syncWithDB(false, true, SyncLimit.noLimit, Source.HAR); APICatalogSync.mergeUrlsAndSave(httpResponseParams4.requestParams.getApiCollectionId(), true, false, httpCallParser.apiCatalogSync.existingAPIsInDb, false); @@ -1051,7 +1047,7 @@ public void testSampleDataUpdate() throws Exception { String payload4 = "{\"method\":\"POST\",\"requestPayload\":\"{\\\"photoUrls\\\":[\\\"string\\\"],\\\"name\\\":\\\"brandon\\\",\\\"id\\\":0,\\\"category\\\":{\\\"id\\\":0,\\\"name\\\":\\\"string\\\"},\\\"tags\\\":[{\\\"id\\\":0,\\\"name\\\":\\\"string\\\"}],\\\"status\\\":\\\"available\\\"}\",\"responsePayload\":\"{\\\"id\\\":9223372036854775807,\\\"category\\\":{\\\"id\\\":0,\\\"name\\\":\\\"string\\\"},\\\"name\\\":\\\"doggie\\\",\\\"photoUrls\\\":[\\\"string\\\"],\\\"tags\\\":[{\\\"id\\\":0,\\\"name\\\":\\\"string\\\"}],\\\"status\\\":\\\"available\\\"}\",\"ip\":\"null\",\"source\":\"MIRRORING\",\"type\":\"HTTP/2\",\"akto_vxlan_id\":\"1661807253\",\"path\":\"https://petstore.swagger.io/v2/books/3\",\"requestHeaders\":\"{\\\"Origin\\\":\\\"https://petstore.swagger.io\\\",\\\"Accept\\\":\\\"application/json\\\",\\\"User-Agent\\\":\\\"Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:95.0) Gecko/20100101 Firefox/95.0\\\",\\\"Referer\\\":\\\"https://petstore.swagger.io/\\\",\\\"Connection\\\":\\\"keep-alive\\\",\\\"Sec-Fetch-Dest\\\":\\\"empty\\\",\\\"Sec-Fetch-Site\\\":\\\"same-origin\\\",\\\"Host\\\":\\\"petstore.swagger.io\\\",\\\"Accept-Encoding\\\":\\\"gzip, deflate, br\\\",\\\"Sec-Fetch-Mode\\\":\\\"cors\\\",\\\"TE\\\":\\\"trailers\\\",\\\"Accept-Language\\\":\\\"en-US,en;q=0.5\\\",\\\"Content-Length\\\":\\\"215\\\",\\\"Content-Type\\\":\\\"application/json\\\"}\",\"responseHeaders\":\"{\\\"date\\\":\\\"Tue, 04 Jan 2022 20:11:58 GMT\\\",\\\"access-control-allow-origin\\\":\\\"*\\\",\\\"server\\\":\\\"Jetty(9.2.9.v20150224)\\\",\\\"access-control-allow-headers\\\":\\\"Content-Type, api_key, Authorization\\\",\\\"X-Firefox-Spdy\\\":\\\"h2\\\",\\\"content-type\\\":\\\"application/json\\\",\\\"access-control-allow-methods\\\":\\\"GET, POST, DELETE, PUT\\\"}\",\"time\":\"1641327118\",\"contentType\":\"application/json\",\"akto_account_id\":\"1000000\",\"statusCode\":\"200\",\"status\":\"OK\"}"; HttpResponseParams httpResponseParams5 = HttpCallParser.parseKafkaMessage(payload4); httpCallParser.syncFunction(Collections.singletonList(httpResponseParams5), false, true, null); - httpCallParser.apiCatalogSync.syncWithDB(false, true, SyncLimit.noLimit); + httpCallParser.apiCatalogSync.syncWithDB(false, true, SyncLimit.noLimit, Source.HAR); SampleData sampleData4 = SampleDataDao.instance.findOne(filter3); assertEquals(1, sampleData4.getSamples().size()); @@ -1186,7 +1182,7 @@ public void testHarIPMerging() { } parser.syncFunction(responseParams, false, true, null); - parser.apiCatalogSync.syncWithDB(false, true, SyncLimit.noLimit); + parser.apiCatalogSync.syncWithDB(false, true, SyncLimit.noLimit, Source.HAR); APICatalogSync.mergeUrlsAndSave(123,true, false, parser.apiCatalogSync.existingAPIsInDb, false); parser.apiCatalogSync.buildFromDB(false, true); @@ -1209,7 +1205,7 @@ public void testMultipleLongMerging() { responseParams.add(resp); parser.syncFunction(responseParams, true, true, null); - parser.apiCatalogSync.syncWithDB(true, true, SyncLimit.noLimit); + parser.apiCatalogSync.syncWithDB(true, true, SyncLimit.noLimit, Source.HAR); Map templateURLToMethods = parser.apiCatalogSync.getDbState(123).getTemplateURLToMethods(); assertEquals(1, templateURLToMethods.size()); @@ -1218,7 +1214,7 @@ public void testMultipleLongMerging() { assertEquals("api/books/INTEGER/cars/INTEGER", urlTemplate.getTemplateString()); parser.syncFunction(responseParams, true, true, null); - parser.apiCatalogSync.syncWithDB(true, true, SyncLimit.noLimit); + parser.apiCatalogSync.syncWithDB(true, true, SyncLimit.noLimit, Source.HAR); } @Test @@ -1235,7 +1231,7 @@ public void testMultipleFloatMerging() { responseParams.add(resp); parser.syncFunction(responseParams, true, true, null); - parser.apiCatalogSync.syncWithDB(true, true, SyncLimit.noLimit); + parser.apiCatalogSync.syncWithDB(true, true, SyncLimit.noLimit, Source.HAR); Map templateURLToMethods = parser.apiCatalogSync.getDbState(123).getTemplateURLToMethods(); assertEquals(1, templateURLToMethods.size()); @@ -1244,7 +1240,7 @@ public void testMultipleFloatMerging() { assertEquals("api/books/FLOAT/cars/FLOAT", urlTemplate.getTemplateString()); parser.syncFunction(responseParams, true, true, null); - parser.apiCatalogSync.syncWithDB(true, true, SyncLimit.noLimit); + parser.apiCatalogSync.syncWithDB(true, true, SyncLimit.noLimit, Source.HAR); } diff --git a/apps/dashboard/src/main/java/com/akto/action/OpenApiAction.java b/apps/dashboard/src/main/java/com/akto/action/OpenApiAction.java index 19abfae463..c5aaa52ef3 100644 --- a/apps/dashboard/src/main/java/com/akto/action/OpenApiAction.java +++ b/apps/dashboard/src/main/java/com/akto/action/OpenApiAction.java @@ -27,7 +27,6 @@ import com.mongodb.BasicDBObject; import com.mongodb.client.model.Filters; -import com.mongodb.client.model.InsertManyOptions; import com.mongodb.client.model.UpdateOptions; import com.mongodb.client.model.Updates; import com.mongodb.client.result.InsertOneResult; @@ -42,10 +41,8 @@ import org.bson.types.ObjectId; import javax.servlet.http.HttpServletResponse; -import java.io.IOException; -import java.io.PrintWriter; +import java.io.*; import java.util.ArrayList; -import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.concurrent.Executors; @@ -71,6 +68,7 @@ public class OpenApiAction extends UserAction implements ServletResponseAware { private static final ScheduledExecutorService executorService = Executors.newSingleThreadScheduledExecutor(); + @Override public String execute() { try { @@ -189,17 +187,26 @@ public void run() { for(List chunk : chunkedLists){ loggerMaker.infoAndAddToDb("Inserting chunk of size " + chunk.size(), LogDb.DASHBOARD); - FileUploadLogsDao.instance.getSwaggerMCollection().insertMany(chunk, new InsertManyOptions().ordered(true)); - } - loggerMaker.infoAndAddToDb("Inserted " + chunkedLists.size() + " chunks of logs", LogDb.DASHBOARD); - FileUploadsDao.instance.updateOne(Filters.eq(Constants.ID, new ObjectId(fileUploadId)), Updates.combine( - Updates.set("uploadStatus", FileUpload.UploadStatus.SUCCEEDED), - Updates.set("collectionName", title), - Updates.set("errors", fileErrors), - Updates.set("count",parsedSwagger.getTotalCount()) - )); - loggerMaker.infoAndAddToDb("Finished processing openAPI file", LogDb.DASHBOARD); + List stringMessages = messages.stream() + .map(SwaggerUploadLog::getAktoFormat) + .collect(Collectors.toList()); + + List stringErrors = fileErrors.stream() + .map(FileUploadError::getError) + .collect(Collectors.toList()); + + String topic = System.getenv("AKTO_KAFKA_TOPIC_NAME"); + if (topic == null) topic = "akto.api.logs"; + + try { + loggerMaker.infoAndAddToDb("Calling Utils.pushDataToKafka for openapi file, for apiCollection id " + apiCollectionId, LogDb.DASHBOARD); + Utils.pushDataToKafka(apiCollectionId, topic, stringMessages, stringErrors, true); + } catch (Exception e) { + throw new RuntimeException(e); + } + + } } catch (Exception e) { loggerMaker.errorAndAddToDb(e, "ERROR while parsing openAPI file", LogDb.DASHBOARD); diff --git a/apps/dashboard/src/main/java/com/akto/action/observe/InventoryAction.java b/apps/dashboard/src/main/java/com/akto/action/observe/InventoryAction.java index 335257740d..5644374ecf 100644 --- a/apps/dashboard/src/main/java/com/akto/action/observe/InventoryAction.java +++ b/apps/dashboard/src/main/java/com/akto/action/observe/InventoryAction.java @@ -381,7 +381,7 @@ public String fetchApiInfosForCollection(){ response.put("apiInfoList", apiInfos); if(apiCollectionId != -1){ ApiCollection apiCollection = ApiCollectionsDao.instance.findOne(Filters.eq(Constants.ID, apiCollectionId)); - response.put("redacted", apiCollection.getRedact()); + response.put("redacted", apiCollection.getRedact()); } return Action.SUCCESS.toUpperCase(); } diff --git a/apps/dashboard/src/test/java/com/akto/utils/TestUpdatesInCollections.java b/apps/dashboard/src/test/java/com/akto/utils/TestUpdatesInCollections.java index 24f3984c23..710ee87ada 100644 --- a/apps/dashboard/src/test/java/com/akto/utils/TestUpdatesInCollections.java +++ b/apps/dashboard/src/test/java/com/akto/utils/TestUpdatesInCollections.java @@ -25,6 +25,7 @@ import com.akto.dto.BackwardCompatibility; import com.akto.dto.HttpResponseParams; import com.akto.dto.ApiInfo.ApiInfoKey; +import com.akto.dto.HttpResponseParams.Source; import com.akto.dto.billing.SyncLimit; import com.akto.dto.test_run_findings.TestingIssuesId; import com.akto.dto.test_run_findings.TestingRunIssues; @@ -70,7 +71,7 @@ public void fillDBValues() throws Exception{ } httpCallParser.syncFunction(httpResponseParamsList, false, true, null); - httpCallParser.apiCatalogSync.syncWithDB(false, true, SyncLimit.noLimit); + httpCallParser.apiCatalogSync.syncWithDB(false, true, SyncLimit.noLimit, Source.HAR); } public TestingRunIssues generateTestResultIssue(String url, String method, Severity severity, String testSubCategory) throws Exception{ diff --git a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/observe/api.js b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/observe/api.js index 170ec02da4..6a91982e29 100644 --- a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/observe/api.js +++ b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/observe/api.js @@ -204,6 +204,13 @@ export default { data: formData, }) }, + uploadOpenApiFile(formData) { + return request({ + url: '/api/importDataFromOpenApiSpec', + method: 'post', + data: formData, + }) + }, uploadTcpFile(content, apiCollectionId, skipKafka) { return request({ url: '/api/uploadTcp', diff --git a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/observe/api_collections/ApiEndpoints.jsx b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/observe/api_collections/ApiEndpoints.jsx index 0668b426cc..574aeaaf35 100644 --- a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/observe/api_collections/ApiEndpoints.jsx +++ b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/observe/api_collections/ApiEndpoints.jsx @@ -163,7 +163,7 @@ function ApiEndpoints(props) { const [showEmptyScreen, setShowEmptyScreen] = useState(false) const [runTests, setRunTests ] = useState(false) - const [endpointData, setEndpointData] = useState({"all":[], 'sensitive': [], 'new': [], 'high_risk': [], 'no_auth': [], 'shadow': []}) + const [endpointData, setEndpointData] = useState({"all":[], 'sensitive': [], 'new': [], 'high_risk': [], 'no_auth': [], 'shadow': [], 'undocumented' : [], 'zombie': []}) const [selectedTab, setSelectedTab] = useState("all") const [selected, setSelected] = useState(0) const [selectedResourcesForPrimaryAction, setSelectedResourcesForPrimaryAction] = useState([]) @@ -187,7 +187,7 @@ function ApiEndpoints(props) { const selectedMethod = queryParams.get('selected_method') // the values used here are defined at the server. - const definedTableTabs = apiCollectionId === 111111999 ? ['All', 'New', 'High risk', 'No auth', 'Shadow'] : ( apiCollectionId === 111111120 ? ['All', 'New', 'Sensitive', 'High risk', 'Shadow'] : ['All', 'New', 'Sensitive', 'High risk', 'No auth', 'Shadow'] ) + const definedTableTabs = apiCollectionId === 111111999 ? ['All', 'New', 'High risk', 'No auth', 'Shadow'] : ( apiCollectionId === 111111120 ? ['All', 'New', 'Sensitive', 'High risk', 'Shadow'] : ['All', 'New', 'Sensitive', 'High risk', 'No auth', 'Shadow', 'Undocumented', 'Zombie'] ) const { tabsInfo } = useTable() @@ -334,6 +334,14 @@ function ApiEndpoints(props) { const prettifyData = transform.prettifyEndpointsData(allEndpoints) + const zombie = prettifyData.filter( + obj => Object.keys(obj.sources).length === 1 && obj.sources.hasOwnProperty("OPEN_API") + ); + + const undocumented = prettifyData.filter( + obj => Object.keys(obj.sources).length === 1 && obj.sources.hasOwnProperty("HAR") + ); + // append shadow endpoints to all endpoints data['all'] = [ ...prettifyData, ...shadowApis ] data['sensitive'] = prettifyData.filter(x => x.sensitive && x.sensitive.size > 0) @@ -341,6 +349,8 @@ function ApiEndpoints(props) { data['new'] = prettifyData.filter(x=> x.isNew) data['no_auth'] = prettifyData.filter(x => x.open) data['shadow'] = [ ...shadowApis ] + data['undocumented'] = undocumented + data['zombie'] = zombie setEndpointData(data) setSelectedTab("all") setSelected(0) @@ -582,8 +592,9 @@ function ApiEndpoints(props) { return } let isJson = file.name.endsWith(".json") + let isYaml = file.name.endsWith(".yaml") || file.name.endsWith(".yml") let isPcap = file.name.endsWith(".pcap") - if (isHar || isJson) { + if (isHar || isJson || isYaml) { reader.readAsText(file) } else if (isPcap) { reader.readAsArrayBuffer(new Blob([file])) @@ -620,6 +631,30 @@ function ApiEndpoints(props) { var bytes = new Uint8Array(arrayBuffer); await api.uploadTcpFile([...bytes], apiCollectionId, skipKafka) + } else if (isJson || isYaml) { + const formData = new FormData(); + formData.append("openAPIString", reader.result) + // formData.append("hsFile", reader.result) + formData.append("skipKafka", skipKafka) + formData.append("apiCollectionId", apiCollectionId); + func.setToast(true, false, "We are uploading your openapi file, please dont refresh the page!") + + api.uploadOpenApiFile(formData).then(resp => { + if (file.size > 2097152) { + func.setToast(true, false, "We have successfully read your file") + } + else { + func.setToast(true, false, "Your Openapi file has been successfully processed") + } + fetchData() + }).catch(err => { + if (err.message.includes(404)) { + func.setToast(true, true, "Please limit the file size to less than 50 MB") + } else { + let message = err?.response?.data?.actionErrors?.[0] || "Something went wrong while processing the file" + func.setToast(true, true, message) + } + }) } } } @@ -672,6 +707,17 @@ function ApiEndpoints(props) { } + + + handleFileChange(file)} + tooltipText="Upload openapi file" + label={Upload OpenAPI file} + primary={false} + /> + + Export as diff --git a/apps/dashboard/web/polaris_web/web/src/util/func.js b/apps/dashboard/web/polaris_web/web/src/util/func.js index 14b253cb3e..41dd80547f 100644 --- a/apps/dashboard/web/polaris_web/web/src/util/func.js +++ b/apps/dashboard/web/polaris_web/web/src/util/func.js @@ -879,7 +879,6 @@ mergeApiInfoAndApiCollection(listEndpoints, apiInfoList, idToName) { let authTypeTag = authType.replace(",", ""); let riskScore = apiInfoMap[key] ? apiInfoMap[key]?.riskScore : 0 let responseCodesArr = apiInfoMap[key] ? apiInfoMap[key]?.responseCodes : [] - ret[key] = { id: x.method + "###" + x.url + "###" + x.apiCollectionId + "###" + Math.random(), shadow: x.shadow ? x.shadow : false, @@ -911,7 +910,8 @@ mergeApiInfoAndApiCollection(listEndpoints, apiInfoList, idToName) { riskScore: riskScore, sensitiveInReq: [...this.convertSensitiveTags(x.sensitiveInReq)], sensitiveInResp: [...this.convertSensitiveTags(x.sensitiveInResp)], - responseCodes: responseCodesArr + responseCodes: responseCodesArr, + sources: apiInfoMap[key]?apiInfoMap[key]['sources']:{} } } diff --git a/libs/dao/src/main/java/com/akto/dto/ApiInfo.java b/libs/dao/src/main/java/com/akto/dto/ApiInfo.java index f335ec9de3..8ec176d0b9 100644 --- a/libs/dao/src/main/java/com/akto/dto/ApiInfo.java +++ b/libs/dao/src/main/java/com/akto/dto/ApiInfo.java @@ -21,6 +21,8 @@ public class ApiInfo { public static final String ALL_AUTH_TYPES_FOUND = "allAuthTypesFound"; private Set> allAuthTypesFound; + + // this annotation makes sure that data is not stored in mongo @BsonIgnore private List actualAuthType; @@ -55,6 +57,9 @@ public class ApiInfo { public static final String DISCOVERED_TIMESTAMP = "discoveredTimestamp"; private int discoveredTimestamp; + public static final String SOURCES = "sources"; + Map sources; + public enum ApiType { REST, GRAPHQL, GRPC, SOAP } @@ -445,4 +450,12 @@ public int getDiscoveredTimestamp() { public void setDiscoveredTimestamp(int discoveredTimestamp) { this.discoveredTimestamp = discoveredTimestamp; } + + public Map getSources() { + return this.sources; + } + + public void setSources(Map sources) { + this.sources = sources; + } } diff --git a/libs/dao/src/main/java/com/akto/dto/HttpResponseParams.java b/libs/dao/src/main/java/com/akto/dto/HttpResponseParams.java index 7fb7ed7e5b..df6838a753 100644 --- a/libs/dao/src/main/java/com/akto/dto/HttpResponseParams.java +++ b/libs/dao/src/main/java/com/akto/dto/HttpResponseParams.java @@ -8,7 +8,7 @@ public class HttpResponseParams { public enum Source { - HAR, PCAP, MIRRORING, SDK, OTHER, POSTMAN + HAR, PCAP, MIRRORING, SDK, OTHER, POSTMAN, OPEN_API } public String accountId; diff --git a/libs/dao/src/main/java/com/akto/dto/type/SingleTypeInfo.java b/libs/dao/src/main/java/com/akto/dto/type/SingleTypeInfo.java index f179ff0a9b..b4b3076c49 100644 --- a/libs/dao/src/main/java/com/akto/dto/type/SingleTypeInfo.java +++ b/libs/dao/src/main/java/com/akto/dto/type/SingleTypeInfo.java @@ -12,7 +12,6 @@ import com.mongodb.BasicDBObject; import io.swagger.v3.oas.models.media.*; import org.apache.commons.lang3.StringUtils; -import org.apache.commons.lang3.math.NumberUtils; import org.bson.codecs.pojo.annotations.BsonIgnore; import org.bson.codecs.pojo.annotations.BsonProperty; import org.bson.types.ObjectId; @@ -531,6 +530,8 @@ public String toString() { long maxValue = ACCEPTED_MIN_VALUE; // this value will be used when field doesn't exist in db public static final String LAST_SEEN = "lastSeen"; long lastSeen; + public static final String SOURCES = "sources"; + Map sources; @BsonIgnore private boolean isPrivate; // do not use this field anywhere else. This was added to convey if STI is private or not to frontend @@ -778,6 +779,14 @@ public void setStrId(String strId) { this.strId = strId; } + public Map getSources() { + return this.sources; + } + + public void setSources(Map sources) { + this.sources = sources; + } + @Override public boolean equals(Object o) { if (o == this) diff --git a/libs/utils/src/main/java/com/akto/har/HAR.java b/libs/utils/src/main/java/com/akto/har/HAR.java index dac07a935d..fc8961be77 100644 --- a/libs/utils/src/main/java/com/akto/har/HAR.java +++ b/libs/utils/src/main/java/com/akto/har/HAR.java @@ -3,17 +3,11 @@ import com.akto.dao.context.Context; import com.akto.log.LoggerMaker; import com.akto.log.LoggerMaker.LogDb; -import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; import de.sstoehr.harreader.HarReader; import de.sstoehr.harreader.HarReaderException; import de.sstoehr.harreader.HarReaderMode; import de.sstoehr.harreader.model.*; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.File; -import java.net.URLDecoder; import java.util.*; public class HAR { @@ -65,7 +59,11 @@ public static Map getResultMap(HarEntry entry, int accountId) thr if (requestPayload == null) requestPayload = ""; String akto_account_id = accountId + ""; - String path = getPath(request); + String urlString = getPath(request); + + String path = urlString; + + String requestHeaders = mapper.writeValueAsString(requestHeaderMap); String responseHeaders = mapper.writeValueAsString(responseHeaderMap); String method = request.getMethod().toString(); diff --git a/libs/utils/src/main/java/com/akto/open_api/parser/Parser.java b/libs/utils/src/main/java/com/akto/open_api/parser/Parser.java index e9592b9555..cbfd31a25f 100644 --- a/libs/utils/src/main/java/com/akto/open_api/parser/Parser.java +++ b/libs/utils/src/main/java/com/akto/open_api/parser/Parser.java @@ -1,5 +1,6 @@ package com.akto.open_api.parser; +import java.net.URL; import java.util.*; import javax.ws.rs.core.Response.Status; @@ -206,6 +207,10 @@ public static ParserResult convertOpenApiToAkto(OpenAPI openAPI, String uploadId } String requestHeadersString = ""; + URL url = new URL(path); + String urlPath = url.getPath(); + // Get the domain (including scheme) + requestHeaders.putIfAbsent("Host", url.getHost()); if (requestHeaders != null && !requestHeaders.isEmpty()) { try { requestHeadersString = mapper.writeValueAsString(requestHeaders); @@ -317,7 +322,7 @@ public static ParserResult convertOpenApiToAkto(OpenAPI openAPI, String uploadId } messageObject.put(mKeys.akto_account_id, Context.accountId.get().toString()); - messageObject.put(mKeys.path, path); + messageObject.put(mKeys.path, urlPath); messageObject.put(mKeys.method, method.toString().toUpperCase()); messageObject.put(mKeys.requestHeaders, requestHeadersString); messageObject.put(mKeys.requestPayload, requestString); @@ -325,7 +330,7 @@ public static ParserResult convertOpenApiToAkto(OpenAPI openAPI, String uploadId messageObject.put(mKeys.time, Context.now() + ""); messageObject.put(mKeys.type, "HTTP"); // swagger uploads are treated as HAR files. - messageObject.put(mKeys.source, Source.HAR.name()); + messageObject.put(mKeys.source, Source.OPEN_API.name()); if (responseObjectList.isEmpty()) { responseObjectList.add(emptyResponseObject); @@ -343,9 +348,8 @@ public static ParserResult convertOpenApiToAkto(OpenAPI openAPI, String uploadId fillDummyIfEmptyMessage(messageObject); SwaggerUploadLog log = new SwaggerUploadLog(); log.setMethod(method.toString()); - log.setUrl(path); + log.setUrl(urlPath); log.setUploadId(uploadId); - try { String s = mapper.writeValueAsString(messageObject); log.setAktoFormat(s); From 789894ed2f44191391e362fe9fbf4562791e4332 Mon Sep 17 00:00:00 2001 From: rishabh-akto Date: Thu, 28 Nov 2024 23:56:23 +0530 Subject: [PATCH 2/8] backward compatibilty of source in sti and api info added --- .../akto/listener/InitializerListener.java | 52 +++++++++++++++++++ .../observe/api_collections/ApiEndpoints.jsx | 15 +++--- .../com/akto/dto/BackwardCompatibility.java | 14 ++++- 3 files changed, 73 insertions(+), 8 deletions(-) diff --git a/apps/dashboard/src/main/java/com/akto/listener/InitializerListener.java b/apps/dashboard/src/main/java/com/akto/listener/InitializerListener.java index f0b64e996e..101ad4ecf8 100644 --- a/apps/dashboard/src/main/java/com/akto/listener/InitializerListener.java +++ b/apps/dashboard/src/main/java/com/akto/listener/InitializerListener.java @@ -132,6 +132,7 @@ import org.apache.commons.csv.CSVRecord; import org.apache.commons.io.FileUtils; import org.apache.commons.lang3.StringUtils; +import org.bson.Document; import org.bson.conversions.Bson; import org.bson.types.ObjectId; import org.json.JSONObject; @@ -1643,6 +1644,55 @@ public static void createAutomatedAPIGroups(BackwardCompatibility backwardCompat } } + public static void addSourceToStiApiInfo(BackwardCompatibility backwardCompatibility) { + if (backwardCompatibility.getAddSourceToStiApiInfo() == 0) { + List apiCollections = ApiCollectionsDao.instance.getMetaAll(); + List trafficList = new ArrayList<>(); + List customList = new ArrayList<>(); + + for (ApiCollection apiCollection : apiCollections) { + if (apiCollection.getType() == ApiCollection.Type.API_GROUP) continue; + if(apiCollection.getHostName() != null) { + trafficList.add(apiCollection); + } else { + customList.add(apiCollection); + } + } + + List trafficIdList = trafficList.stream().map(ApiCollection::getId).collect(Collectors.toList()); + Bson apiCollectionTrafficIdFilter = Filters.in("apiCollectionId", trafficIdList); + List> updates = new ArrayList<>(); + + SingleTypeInfoDao.instance.getMCollection().find(apiCollectionTrafficIdFilter).forEach(action -> { + if (action.getSources() == null) { + Bson idFilter = Filters.eq("_id", action.getId()); + Bson sourceUpdate = Updates.set(SingleTypeInfo.SOURCES + "." + HttpResponseParams.Source.MIRRORING, new Document("timestamp", action.getTimestamp()) ); + updates.add(new UpdateOneModel<>(idFilter, sourceUpdate)); + } + + }); + + + List customIdList = customList.stream().map(ApiCollection::getId).collect(Collectors.toList()); + Bson apiCollectionCustomIdFilter = Filters.in("apiCollectionId", customIdList); + + SingleTypeInfoDao.instance.getMCollection().find(apiCollectionCustomIdFilter).forEach(action -> { + if (action.getSources() == null) { + Bson idFilter = Filters.eq("_id", action.getId()); + Bson sourceUpdate = Updates.set(SingleTypeInfo.SOURCES + "." + HttpResponseParams.Source.HAR, new Document("timestamp", action.getTimestamp()) ); + updates.add(new UpdateOneModel<>(idFilter, sourceUpdate)); + } + + + }); + + // Perform bulk update + if (!updates.isEmpty()) { + SingleTypeInfoDao.instance.getMCollection().bulkWrite(updates); + } + } + } + public static void dropWorkflowTestResultCollection(BackwardCompatibility backwardCompatibility) { if (backwardCompatibility.getDropWorkflowTestResult() == 0) { WorkflowTestResultsDao.instance.getMCollection().drop(); @@ -2896,6 +2946,8 @@ public static void setBackwardCompatibilities(BackwardCompatibility backwardComp dropSpecialCharacterApiCollections(backwardCompatibility); addDefaultAdvancedFilters(backwardCompatibility); moveAzureSamlConfig(backwardCompatibility); + addSourceToStiApiInfo(backwardCompatibility); + } public static void printMultipleHosts(int apiCollectionId) { diff --git a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/observe/api_collections/ApiEndpoints.jsx b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/observe/api_collections/ApiEndpoints.jsx index 574aeaaf35..61271c7f20 100644 --- a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/observe/api_collections/ApiEndpoints.jsx +++ b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/observe/api_collections/ApiEndpoints.jsx @@ -163,7 +163,7 @@ function ApiEndpoints(props) { const [showEmptyScreen, setShowEmptyScreen] = useState(false) const [runTests, setRunTests ] = useState(false) - const [endpointData, setEndpointData] = useState({"all":[], 'sensitive': [], 'new': [], 'high_risk': [], 'no_auth': [], 'shadow': [], 'undocumented' : [], 'zombie': []}) + const [endpointData, setEndpointData] = useState({"all":[], 'sensitive': [], 'new': [], 'high_risk': [], 'no_auth': [], 'shadow': [], 'zombie': []}) const [selectedTab, setSelectedTab] = useState("all") const [selected, setSelected] = useState(0) const [selectedResourcesForPrimaryAction, setSelectedResourcesForPrimaryAction] = useState([]) @@ -187,7 +187,7 @@ function ApiEndpoints(props) { const selectedMethod = queryParams.get('selected_method') // the values used here are defined at the server. - const definedTableTabs = apiCollectionId === 111111999 ? ['All', 'New', 'High risk', 'No auth', 'Shadow'] : ( apiCollectionId === 111111120 ? ['All', 'New', 'Sensitive', 'High risk', 'Shadow'] : ['All', 'New', 'Sensitive', 'High risk', 'No auth', 'Shadow', 'Undocumented', 'Zombie'] ) + const definedTableTabs = apiCollectionId === 111111999 ? ['All', 'New', 'High risk', 'No auth', 'Shadow'] : ( apiCollectionId === 111111120 ? ['All', 'New', 'Sensitive', 'High risk', 'Shadow'] : ['All', 'New', 'Sensitive', 'High risk', 'No auth', 'Shadow', 'Zombie'] ) const { tabsInfo } = useTable() @@ -331,15 +331,17 @@ function ApiEndpoints(props) { } }) } - + const prettifyData = transform.prettifyEndpointsData(allEndpoints) const zombie = prettifyData.filter( - obj => Object.keys(obj.sources).length === 1 && obj.sources.hasOwnProperty("OPEN_API") + obj => obj.sources && // Check that obj.sources is not null or undefined + Object.keys(obj.sources).length === 1 && + obj.sources.hasOwnProperty("OPEN_API") ); const undocumented = prettifyData.filter( - obj => Object.keys(obj.sources).length === 1 && obj.sources.hasOwnProperty("HAR") + obj => obj.sources && Object.keys(obj.sources).length === 1 && obj.sources.hasOwnProperty("HAR") ); // append shadow endpoints to all endpoints @@ -348,8 +350,7 @@ function ApiEndpoints(props) { data['high_risk'] = prettifyData.filter(x=> x.riskScore >= 4) data['new'] = prettifyData.filter(x=> x.isNew) data['no_auth'] = prettifyData.filter(x => x.open) - data['shadow'] = [ ...shadowApis ] - data['undocumented'] = undocumented + data['shadow'] = [ ...shadowApis, ...undocumented ] data['zombie'] = zombie setEndpointData(data) setSelectedTab("all") diff --git a/libs/dao/src/main/java/com/akto/dto/BackwardCompatibility.java b/libs/dao/src/main/java/com/akto/dto/BackwardCompatibility.java index 7378433d99..a83ca9ea4e 100644 --- a/libs/dao/src/main/java/com/akto/dto/BackwardCompatibility.java +++ b/libs/dao/src/main/java/com/akto/dto/BackwardCompatibility.java @@ -100,6 +100,9 @@ public class BackwardCompatibility { public static final String DELETE_OPTIONS_API = "deleteOptionsAPIs"; private int deleteOptionsAPIs; + public static final String ADD_SOURCE_TO_STI_API_INFO = "addSourceToStiApiInfo"; + private int addSourceToStiApiInfo; + public BackwardCompatibility(int id, int dropFilterSampleData, int resetSingleTypeInfoCount, int dropWorkflowTestResult, int readyForNewTestingFramework,int addAktoDataTypes, boolean deploymentStatusUpdated, int authMechanismData, boolean mirroringLambdaTriggered, int deleteAccessListFromApiToken, @@ -109,7 +112,7 @@ public BackwardCompatibility(int id, int dropFilterSampleData, int resetSingleTy int loginSignupGroups, int vulnerableApiUpdationVersionV1, int riskScoreGroups, int deactivateCollections, int disableAwsSecretPii, int apiCollectionAutomatedField, int automatedApiGroups, int addAdminRoleIfAbsent, int dropSpecialCharacterApiCollections, int fixApiAccessType, - int addDefaultFilters, int moveAzureSamlToNormalSaml, int deleteOptionsAPIs) { + int addDefaultFilters, int moveAzureSamlToNormalSaml, int deleteOptionsAPIs, int addSourceToStiApiInfo) { this.id = id; this.dropFilterSampleData = dropFilterSampleData; this.resetSingleTypeInfoCount = resetSingleTypeInfoCount; @@ -141,6 +144,7 @@ public BackwardCompatibility(int id, int dropFilterSampleData, int resetSingleTy this.fixApiAccessType = fixApiAccessType; this.moveAzureSamlToNormalSaml = moveAzureSamlToNormalSaml; this.deleteOptionsAPIs = deleteOptionsAPIs; + this.addSourceToStiApiInfo = addSourceToStiApiInfo; } public BackwardCompatibility() { @@ -425,4 +429,12 @@ public int getDeleteOptionsAPIs() { public void setDeleteOptionsAPIs(int deleteOptionsAPIs) { this.deleteOptionsAPIs = deleteOptionsAPIs; } + + public int getAddSourceToStiApiInfo() { + return addSourceToStiApiInfo; + } + + public void setAddSourceToStiApiInfo(int addSourceToStiApiInfo) { + this.addSourceToStiApiInfo = addSourceToStiApiInfo; + } } From 562a0425a42526e7d72f5de82436de74d63825cd Mon Sep 17 00:00:00 2001 From: Avneesh Hota Date: Wed, 18 Dec 2024 18:16:26 +0530 Subject: [PATCH 3/8] ask user for source if not there --- .../java/com/akto/runtime/APICatalogSync.java | 6 +- .../java/com/akto/action/OpenApiAction.java | 46 +++++++- .../akto/listener/InitializerListener.java | 49 --------- .../observe/api_collections/ApiEndpoints.jsx | 102 ++++++++++++------ .../observe/api_collections/SelectSource.jsx | 37 +++++++ 5 files changed, 158 insertions(+), 82 deletions(-) create mode 100644 apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/observe/api_collections/SelectSource.jsx diff --git a/apps/api-runtime/src/main/java/com/akto/runtime/APICatalogSync.java b/apps/api-runtime/src/main/java/com/akto/runtime/APICatalogSync.java index dc8b430dd9..c424d63623 100644 --- a/apps/api-runtime/src/main/java/com/akto/runtime/APICatalogSync.java +++ b/apps/api-runtime/src/main/java/com/akto/runtime/APICatalogSync.java @@ -1310,8 +1310,10 @@ public DbUpdateReturn getDBUpdatesForParams(APICatalog currentDelta, APICatalog update = Updates.combine(update, Updates.max(SingleTypeInfo.LAST_SEEN, deltaInfo.getLastSeen())); update = Updates.combine(update, Updates.max(SingleTypeInfo.MAX_VALUE, deltaInfo.getMaxValue())); update = Updates.combine(update, Updates.min(SingleTypeInfo.MIN_VALUE, deltaInfo.getMinValue())); - Bson updateSourceMap = Updates.set(SingleTypeInfo.SOURCES + "." + source.name(), new Document("timestamp", timestamp) ); - update = Updates.combine(update, updateSourceMap); + if (source != null) { + Bson updateSourceMap = Updates.set(SingleTypeInfo.SOURCES + "." + source.name(), new Document("timestamp", timestamp) ); + update = Updates.combine(update, updateSourceMap); + } if (!Main.isOnprem) { if (dbInfo != null) { diff --git a/apps/dashboard/src/main/java/com/akto/action/OpenApiAction.java b/apps/dashboard/src/main/java/com/akto/action/OpenApiAction.java index c5aaa52ef3..248bbd400b 100644 --- a/apps/dashboard/src/main/java/com/akto/action/OpenApiAction.java +++ b/apps/dashboard/src/main/java/com/akto/action/OpenApiAction.java @@ -1,13 +1,17 @@ package com.akto.action; import com.akto.dao.ApiCollectionsDao; +import com.akto.dao.ApiInfoDao; import com.akto.dao.SampleDataDao; +import com.akto.dao.SingleTypeInfoDao; import com.akto.dao.context.Context; import com.akto.dao.file.FilesDao; import com.akto.dao.upload.FileUploadLogsDao; import com.akto.dao.upload.FileUploadsDao; import com.akto.dto.ApiCollection; import com.akto.dto.ApiInfo; +import com.akto.dto.HttpResponseParams; +import com.akto.dto.HttpResponseParams.Source; import com.akto.dto.files.File; import com.akto.dto.traffic.SampleData; import com.akto.dto.type.SingleTypeInfo; @@ -138,6 +142,7 @@ public String burpSwagger() throws IOException { } private static final String OPEN_API = "OpenAPI"; + private Source source = null; public String importDataFromOpenApiSpec(){ @@ -153,10 +158,44 @@ public String importDataFromOpenApiSpec(){ InsertOneResult insertOneResult = FileUploadsDao.instance.insertOne(fileUpload); String fileUploadId = insertOneResult.getInsertedId().asObjectId().getValue().toString(); this.uploadId = fileUploadId; + ApiCollection apiCollection = ApiCollectionsDao.instance.getMeta(apiCollectionId); + if (apiCollection.getHostName() != null) { + source = HttpResponseParams.Source.MIRRORING; + } + + ApiInfo apiInfoWithSource = ApiInfoDao.instance.findOne( + Filters.and( + Filters.eq(ApiInfo.ID_API_COLLECTION_ID, apiCollectionId), + Filters.exists(ApiInfo.SOURCES, true) + ) + ); + + if (source == null && apiInfoWithSource == null) { + addActionError("Source is null"); + return ERROR.toUpperCase(); + } + executorService.schedule(new Runnable() { public void run() { Context.accountId.set(accountId); - loggerMaker.infoAndAddToDb("Starting thread to process openAPI file", LogDb.DASHBOARD); + + + if (apiInfoWithSource == null) { + loggerMaker.infoAndAddToDb("Starting to add sources to existing STIs and ApiInfos", LogDb.DASHBOARD); + + Bson sourceUpdate = Updates.set(SingleTypeInfo.SOURCES + "." + source, new BasicDBObject("timestamp", Context.now()) ); + + Bson apiCollectionIdFilterForStis = Filters.eq(SingleTypeInfo._API_COLLECTION_ID, apiCollectionId); + SingleTypeInfoDao.instance.updateManyNoUpsert(apiCollectionIdFilterForStis, sourceUpdate); + + Bson apiCollectionIdFilterForApiInfos = Filters.eq(ApiInfo.ID_API_COLLECTION_ID, apiCollectionId); + ApiInfoDao.instance.updateManyNoUpsert(apiCollectionIdFilterForApiInfos, sourceUpdate); + + loggerMaker.infoAndAddToDb("Starting to process openAPI file", LogDb.DASHBOARD); + } else { + loggerMaker.infoAndAddToDb("No need to add sources to STIs and ApiInfos", LogDb.DASHBOARD); + } + SwaggerFileUpload fileUpload = FileUploadsDao.instance.getSwaggerMCollection().find(Filters.eq(Constants.ID, new ObjectId(fileUploadId))).first(); String title = OPEN_API + " "; @@ -406,4 +445,9 @@ public List getApiInfoKeyList() { public void setApiInfoKeyList(List apiInfoKeyList) { this.apiInfoKeyList = apiInfoKeyList; } + + public void setSource(Source source) { + this.source = source; + } + } diff --git a/apps/dashboard/src/main/java/com/akto/listener/InitializerListener.java b/apps/dashboard/src/main/java/com/akto/listener/InitializerListener.java index 114dde3e39..30d9f8cd3d 100644 --- a/apps/dashboard/src/main/java/com/akto/listener/InitializerListener.java +++ b/apps/dashboard/src/main/java/com/akto/listener/InitializerListener.java @@ -1649,54 +1649,6 @@ public static void createAutomatedAPIGroups(BackwardCompatibility backwardCompat } } - public static void addSourceToStiApiInfo(BackwardCompatibility backwardCompatibility) { - if (backwardCompatibility.getAddSourceToStiApiInfo() == 0) { - List apiCollections = ApiCollectionsDao.instance.getMetaAll(); - List trafficList = new ArrayList<>(); - List customList = new ArrayList<>(); - - for (ApiCollection apiCollection : apiCollections) { - if (apiCollection.getType() == ApiCollection.Type.API_GROUP) continue; - if(apiCollection.getHostName() != null) { - trafficList.add(apiCollection); - } else { - customList.add(apiCollection); - } - } - - List trafficIdList = trafficList.stream().map(ApiCollection::getId).collect(Collectors.toList()); - Bson apiCollectionTrafficIdFilter = Filters.in("apiCollectionId", trafficIdList); - List> updates = new ArrayList<>(); - - SingleTypeInfoDao.instance.getMCollection().find(apiCollectionTrafficIdFilter).forEach(action -> { - if (action.getSources() == null) { - Bson idFilter = Filters.eq("_id", action.getId()); - Bson sourceUpdate = Updates.set(SingleTypeInfo.SOURCES + "." + HttpResponseParams.Source.MIRRORING, new Document("timestamp", action.getTimestamp()) ); - updates.add(new UpdateOneModel<>(idFilter, sourceUpdate)); - } - - }); - - - List customIdList = customList.stream().map(ApiCollection::getId).collect(Collectors.toList()); - Bson apiCollectionCustomIdFilter = Filters.in("apiCollectionId", customIdList); - - SingleTypeInfoDao.instance.getMCollection().find(apiCollectionCustomIdFilter).forEach(action -> { - if (action.getSources() == null) { - Bson idFilter = Filters.eq("_id", action.getId()); - Bson sourceUpdate = Updates.set(SingleTypeInfo.SOURCES + "." + HttpResponseParams.Source.HAR, new Document("timestamp", action.getTimestamp()) ); - updates.add(new UpdateOneModel<>(idFilter, sourceUpdate)); - } - - - }); - - // Perform bulk update - if (!updates.isEmpty()) { - SingleTypeInfoDao.instance.getMCollection().bulkWrite(updates); - } - } - } public static void dropWorkflowTestResultCollection(BackwardCompatibility backwardCompatibility) { if (backwardCompatibility.getDropWorkflowTestResult() == 0) { @@ -2961,7 +2913,6 @@ public static void setBackwardCompatibilities(BackwardCompatibility backwardComp dropSpecialCharacterApiCollections(backwardCompatibility); addDefaultAdvancedFilters(backwardCompatibility); moveAzureSamlConfig(backwardCompatibility); - addSourceToStiApiInfo(backwardCompatibility); } diff --git a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/observe/api_collections/ApiEndpoints.jsx b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/observe/api_collections/ApiEndpoints.jsx index 99fd473fd5..a2986da47d 100644 --- a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/observe/api_collections/ApiEndpoints.jsx +++ b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/observe/api_collections/ApiEndpoints.jsx @@ -29,6 +29,7 @@ import GetPrettifyEndpoint from "../GetPrettifyEndpoint" import SourceLocation from "./component/SourceLocation" import useTable from "../../../components/tables/TableContext" import HeadingWithTooltip from "../../../components/shared/HeadingWithTooltip" +import { SelectSource } from "./SelectSource" const headings = [ { @@ -163,6 +164,9 @@ function ApiEndpoints(props) { const [unusedEndpoints, setUnusedEndpoints] = useState([]) const [showEmptyScreen, setShowEmptyScreen] = useState(false) const [runTests, setRunTests ] = useState(false) + const [showSourceDialog, setShowSourceDialog] = useState(false) + const [openAPIfile, setOpenAPIfile] = useState(null) + const [sourcesBackfilled, setSourcesBackfilled] = useState(false) const [endpointData, setEndpointData] = useState({"all":[], 'sensitive': [], 'new': [], 'high_risk': [], 'no_auth': [], 'shadow': [], 'zombie': []}) const [selectedTab, setSelectedTab] = useState("all") @@ -344,9 +348,21 @@ function ApiEndpoints(props) { obj.sources.hasOwnProperty("OPEN_API") ); - const undocumented = prettifyData.filter( - obj => obj.sources && Object.keys(obj.sources).length === 1 && obj.sources.hasOwnProperty("HAR") - ); + var hasOpenAPI = false + prettifyData.forEach((obj) => { + if (obj.sources && obj.sources.hasOwnProperty("OPEN_API")) { + hasOpenAPI = true + } + + if (obj.sources && !sourcesBackfilled) { + setSourcesBackfilled(true) + } + }) + + // check if openAPI file has been uploaded or not.. else show there no shadow APIs + const undocumented = hasOpenAPI ? prettifyData.filter( + obj => obj.sources && !obj.sources.hasOwnProperty("OPEN_API") + ) : []; // append shadow endpoints to all endpoints data['all'] = [ ...prettifyData, ...shadowApis ] @@ -587,6 +603,51 @@ function ApiEndpoints(props) { } } + function uploadOpenApiFile(file) { + setOpenAPIfile(file) + if (!isApiGroup && !(collectionsObj?.hostName && collectionsObj?.hostName?.length > 0) && !sourcesBackfilled) { + setShowSourceDialog(true) + } else { + uploadOpenFileWithSource(null, file) + } + } + + function uploadOpenFileWithSource(source, file) { + const reader = new FileReader(); + if (!file) { + file = openAPIfile + } + reader.readAsText(file) + + reader.onload = async () => { + const formData = new FormData(); + formData.append("openAPIString", reader.result) + formData.append("apiCollectionId", apiCollectionId); + if (source) { + formData.append("source", source) + } + func.setToast(true, false, "We are uploading your openapi file, please dont refresh the page!") + + api.uploadOpenApiFile(formData).then(resp => { + if (file.size > 2097152) { + func.setToast(true, false, "We have successfully read your file") + } + else { + func.setToast(true, false, "Your Openapi file has been successfully processed") + } + fetchData() + }).catch(err => { + console.log(err); + if (err.message.includes(404)) { + func.setToast(true, true, "Please limit the file size to less than 50 MB") + } else { + let message = err?.response?.data?.actionErrors?.[0] || "Something went wrong while processing the file" + func.setToast(true, true, message) + } + }) + } + } + function handleFileChange(file) { if (file) { const reader = new FileReader(); @@ -636,30 +697,6 @@ function ApiEndpoints(props) { var bytes = new Uint8Array(arrayBuffer); await api.uploadTcpFile([...bytes], apiCollectionId, skipKafka) - } else if (isJson || isYaml) { - const formData = new FormData(); - formData.append("openAPIString", reader.result) - // formData.append("hsFile", reader.result) - formData.append("skipKafka", skipKafka) - formData.append("apiCollectionId", apiCollectionId); - func.setToast(true, false, "We are uploading your openapi file, please dont refresh the page!") - - api.uploadOpenApiFile(formData).then(resp => { - if (file.size > 2097152) { - func.setToast(true, false, "We have successfully read your file") - } - else { - func.setToast(true, false, "Your Openapi file has been successfully processed") - } - fetchData() - }).catch(err => { - if (err.message.includes(404)) { - func.setToast(true, true, "Please limit the file size to less than 50 MB") - } else { - let message = err?.response?.data?.actionErrors?.[0] || "Something went wrong while processing the file" - func.setToast(true, true, message) - } - }) } } } @@ -712,17 +749,17 @@ function ApiEndpoints(props) { } - + {!isApiGroup ? handleFileChange(file)} + fileChanged={file => uploadOpenApiFile(file)} tooltipText="Upload openapi file" label={Upload OpenAPI file} primary={false} /> - + : null} Export as @@ -776,6 +813,11 @@ function ApiEndpoints(props) { disabled={showEmptyScreen} selectedResourcesForPrimaryAction={selectedResourcesForPrimaryAction} /> + setShowSourceDialog(val)} + primaryAction={(val) => uploadOpenFileWithSource(val)} + /> ) diff --git a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/observe/api_collections/SelectSource.jsx b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/observe/api_collections/SelectSource.jsx new file mode 100644 index 0000000000..7d3a06778f --- /dev/null +++ b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/observe/api_collections/SelectSource.jsx @@ -0,0 +1,37 @@ +import { Modal} from "@shopify/polaris" +import DropdownSearch from "../../../components/shared/DropdownSearch" +import { useState } from "react" + + +const sourcesAvailable = [ + { label: "Postman", value: "POSTMAN" }, + { label: "HAR file", value: "HAR" }, + { label: "Burp", value: "BURP" }, +] + +export function SelectSource({show, setShow, primaryAction}) { + const [currentSource, setCurrentSource] = useState(null) + return ( + setShow(false)} + title="Select the source for your APIs" + primaryAction={{ + content: "Upload", + onAction: () => { + primaryAction(currentSource) + setShow(false) + } + }} + > + + { setCurrentSource(val)}} + value={currentSource} + /> + + + ) +} \ No newline at end of file From 26b8ea145e6313bcfd5597dcc3f77aa09d10246d Mon Sep 17 00:00:00 2001 From: Avneesh Hota Date: Thu, 19 Dec 2024 11:10:40 +0530 Subject: [PATCH 4/8] added burp source --- .../src/main/java/com/akto/action/HarAction.java | 4 +++- .../src/main/java/com/akto/dto/HttpResponseParams.java | 2 +- libs/utils/src/main/java/com/akto/har/HAR.java | 4 +++- .../utils/src/test/java/com/akto/TestGraphQLUtils.java | 10 +++++----- 4 files changed, 12 insertions(+), 8 deletions(-) diff --git a/apps/dashboard/src/main/java/com/akto/action/HarAction.java b/apps/dashboard/src/main/java/com/akto/action/HarAction.java index cf861147b4..38b6b345b8 100644 --- a/apps/dashboard/src/main/java/com/akto/action/HarAction.java +++ b/apps/dashboard/src/main/java/com/akto/action/HarAction.java @@ -146,8 +146,10 @@ public String execute() throws IOException { return ERROR.toUpperCase(); } + HttpResponseParams.Source source = HttpResponseParams.Source.HAR; if (getSession().getOrDefault("utility","").equals(Utility.BURP.toString())) { BurpPluginInfoDao.instance.updateLastDataSentTimestamp(getSUser().getLogin()); + source = HttpResponseParams.Source.BURP; } try { @@ -156,7 +158,7 @@ public String execute() throws IOException { String zippedString = GzipUtils.zipString(harString); com.akto.dto.files.File file = new com.akto.dto.files.File(HttpResponseParams.Source.HAR.toString(),zippedString); FilesDao.instance.insertOne(file); - List messages = har.getMessages(harString, apiCollectionId, Context.accountId.get()); + List messages = har.getMessages(harString, apiCollectionId, Context.accountId.get(), source); harErrors = har.getErrors(); Utils.pushDataToKafka(apiCollectionId, topic, messages, harErrors, skipKafka); loggerMaker.infoAndAddToDb("Har file upload processing for collectionId:" + apiCollectionId + " finished", LoggerMaker.LogDb.DASHBOARD); diff --git a/libs/dao/src/main/java/com/akto/dto/HttpResponseParams.java b/libs/dao/src/main/java/com/akto/dto/HttpResponseParams.java index df6838a753..4397b8a26f 100644 --- a/libs/dao/src/main/java/com/akto/dto/HttpResponseParams.java +++ b/libs/dao/src/main/java/com/akto/dto/HttpResponseParams.java @@ -8,7 +8,7 @@ public class HttpResponseParams { public enum Source { - HAR, PCAP, MIRRORING, SDK, OTHER, POSTMAN, OPEN_API + HAR, PCAP, MIRRORING, SDK, OTHER, POSTMAN, OPEN_API, BURP } public String accountId; diff --git a/libs/utils/src/main/java/com/akto/har/HAR.java b/libs/utils/src/main/java/com/akto/har/HAR.java index fc8961be77..d9c033f784 100644 --- a/libs/utils/src/main/java/com/akto/har/HAR.java +++ b/libs/utils/src/main/java/com/akto/har/HAR.java @@ -1,6 +1,7 @@ package com.akto.har; import com.akto.dao.context.Context; +import com.akto.dto.HttpResponseParams; import com.akto.log.LoggerMaker; import com.akto.log.LoggerMaker.LogDb; import com.fasterxml.jackson.databind.ObjectMapper; @@ -16,7 +17,7 @@ public class HAR { private static final LoggerMaker loggerMaker = new LoggerMaker(Har.class); public static final String JSON_CONTENT_TYPE = "application/json"; public static final String FORM_URL_ENCODED_CONTENT_TYPE = "application/x-www-form-urlencoded"; - public List getMessages(String harString, int collection_id, int accountId) throws HarReaderException { + public List getMessages(String harString, int collection_id, int accountId, HttpResponseParams.Source source) throws HarReaderException { HarReader harReader = new HarReader(); Har har = harReader.readFromString(harString, HarReaderMode.LAX); @@ -31,6 +32,7 @@ public List getMessages(String harString, int collection_id, int account Map result = getResultMap(entry, accountId); if (result != null) { result.put("akto_vxlan_id", collection_id+""); + result.put("source", source.name()); entriesList.add(mapper.writeValueAsString(result)); } diff --git a/libs/utils/src/test/java/com/akto/TestGraphQLUtils.java b/libs/utils/src/test/java/com/akto/TestGraphQLUtils.java index 2fa6ffc0d3..c8a81a6e8c 100644 --- a/libs/utils/src/test/java/com/akto/TestGraphQLUtils.java +++ b/libs/utils/src/test/java/com/akto/TestGraphQLUtils.java @@ -28,7 +28,7 @@ public class TestGraphQLUtils{ @Test public void testGraphQLParser() throws Exception { HAR har = new HAR(); - List requests = har.getMessages(harString, 0, 1_000_000); + List requests = har.getMessages(harString, 0, 1_000_000, HttpResponseParams.Source.HAR); //Even with 2 har entries, we get 10 endpoints for (String request : requests) { @@ -41,7 +41,7 @@ public void testGraphQLParser() throws Exception { @Test public void testAddGraphqlField() throws Exception { HAR har = new HAR(); - List requests = har.getMessages(harString, 0, 1_000_000); + List requests = har.getMessages(harString, 0, 1_000_000, HttpResponseParams.Source.HAR); for (String request : requests) { HttpResponseParams responseParams = parseKafkaMessage(request); @@ -56,7 +56,7 @@ public void testAddGraphqlField() throws Exception { @Test public void testDeleteGraphqlField() throws Exception { HAR har = new HAR(); - List requests = har.getMessages(harString, 0, 1_000_000); + List requests = har.getMessages(harString, 0, 1_000_000, HttpResponseParams.Source.HAR); for (String request : requests) { HttpResponseParams responseParams = parseKafkaMessage(request); @@ -71,7 +71,7 @@ public void testDeleteGraphqlField() throws Exception { @Test public void testModifyGraphqlField() throws Exception { HAR har = new HAR(); - List requests = har.getMessages(harString, 0, 1_000_000); + List requests = har.getMessages(harString, 0, 1_000_000, HttpResponseParams.Source.HAR); for (String request : requests) { HttpResponseParams responseParams = parseKafkaMessage(request); @@ -86,7 +86,7 @@ public void testModifyGraphqlField() throws Exception { @Test public void testAddUniqueGraphqlField() throws Exception { HAR har = new HAR(); - List requests = har.getMessages(harString, 0, 1_000_000); + List requests = har.getMessages(harString, 0, 1_000_000, HttpResponseParams.Source.HAR); for (String request : requests) { HttpResponseParams responseParams = parseKafkaMessage(request); From c09e51cf40c346194565a3f579e9d1bd4aa37c74 Mon Sep 17 00:00:00 2001 From: Avneesh Hota Date: Thu, 19 Dec 2024 11:13:02 +0530 Subject: [PATCH 5/8] null pointer check in aktoPolicyNew --- .../main/java/com/akto/runtime/policies/AktoPolicyNew.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/apps/api-runtime/src/main/java/com/akto/runtime/policies/AktoPolicyNew.java b/apps/api-runtime/src/main/java/com/akto/runtime/policies/AktoPolicyNew.java index 97b3f0dec5..a2e4875aa8 100644 --- a/apps/api-runtime/src/main/java/com/akto/runtime/policies/AktoPolicyNew.java +++ b/apps/api-runtime/src/main/java/com/akto/runtime/policies/AktoPolicyNew.java @@ -419,7 +419,9 @@ public static List> getUpdatesForApiInfo(List apiIn subUpdates.add(Updates.setOnInsert(ApiInfo.DISCOVERED_TIMESTAMP, apiInfo.getDiscoveredTimestamp())); // sources - subUpdates.add(Updates.set(SingleTypeInfo.SOURCES + "." + source.name(), new Document("timestamp", apiInfo.getDiscoveredTimestamp()))); + if (source != null) { + subUpdates.add(Updates.set(SingleTypeInfo.SOURCES + "." + source.name(), new Document("timestamp", apiInfo.getDiscoveredTimestamp()))); + } // last seen subUpdates.add(Updates.set(ApiInfo.LAST_SEEN, apiInfo.getLastSeen())); From 38fa862f64b81cc1937a7e105df7af4a56d8b0aa Mon Sep 17 00:00:00 2001 From: Avneesh Hota Date: Thu, 19 Dec 2024 11:15:16 +0530 Subject: [PATCH 6/8] fixed timestamp in sources in API info --- .../src/main/java/com/akto/runtime/policies/AktoPolicyNew.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/apps/api-runtime/src/main/java/com/akto/runtime/policies/AktoPolicyNew.java b/apps/api-runtime/src/main/java/com/akto/runtime/policies/AktoPolicyNew.java index a2e4875aa8..f1e80e0fde 100644 --- a/apps/api-runtime/src/main/java/com/akto/runtime/policies/AktoPolicyNew.java +++ b/apps/api-runtime/src/main/java/com/akto/runtime/policies/AktoPolicyNew.java @@ -1,6 +1,7 @@ package com.akto.runtime.policies; import com.akto.dao.*; +import com.akto.dao.context.Context; import com.akto.dao.filter.MergedUrlsDao; import com.akto.dto.*; import com.akto.dto.ApiInfo.ApiInfoKey; @@ -420,7 +421,7 @@ public static List> getUpdatesForApiInfo(List apiIn // sources if (source != null) { - subUpdates.add(Updates.set(SingleTypeInfo.SOURCES + "." + source.name(), new Document("timestamp", apiInfo.getDiscoveredTimestamp()))); + subUpdates.add(Updates.set(SingleTypeInfo.SOURCES + "." + source.name(), new Document("timestamp", Context.now()))); } // last seen From 530355e23eb2b3d1c870332f8b3a9944045da6d7 Mon Sep 17 00:00:00 2001 From: Avneesh Hota Date: Thu, 19 Dec 2024 11:18:25 +0530 Subject: [PATCH 7/8] disable openAPI upload for logical groups --- .../src/main/java/com/akto/action/OpenApiAction.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/apps/dashboard/src/main/java/com/akto/action/OpenApiAction.java b/apps/dashboard/src/main/java/com/akto/action/OpenApiAction.java index 248bbd400b..8d77e2784b 100644 --- a/apps/dashboard/src/main/java/com/akto/action/OpenApiAction.java +++ b/apps/dashboard/src/main/java/com/akto/action/OpenApiAction.java @@ -163,6 +163,11 @@ public String importDataFromOpenApiSpec(){ source = HttpResponseParams.Source.MIRRORING; } + if (apiCollection.getType().equals(ApiCollection.Type.API_GROUP)) { + addActionError("Can't upload OpenAPI file for collection groups"); + return ERROR.toUpperCase(); + } + ApiInfo apiInfoWithSource = ApiInfoDao.instance.findOne( Filters.and( Filters.eq(ApiInfo.ID_API_COLLECTION_ID, apiCollectionId), From e66151d7cb83612dac99163f6ab632f3b3a2b78c Mon Sep 17 00:00:00 2001 From: Avneesh Hota Date: Thu, 19 Dec 2024 14:53:24 +0530 Subject: [PATCH 8/8] handle apiCollection type null --- apps/dashboard/src/main/java/com/akto/action/OpenApiAction.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/dashboard/src/main/java/com/akto/action/OpenApiAction.java b/apps/dashboard/src/main/java/com/akto/action/OpenApiAction.java index 8d77e2784b..6bca2b5a27 100644 --- a/apps/dashboard/src/main/java/com/akto/action/OpenApiAction.java +++ b/apps/dashboard/src/main/java/com/akto/action/OpenApiAction.java @@ -163,7 +163,7 @@ public String importDataFromOpenApiSpec(){ source = HttpResponseParams.Source.MIRRORING; } - if (apiCollection.getType().equals(ApiCollection.Type.API_GROUP)) { + if (apiCollection.getType() != null && apiCollection.getType().equals(ApiCollection.Type.API_GROUP)) { addActionError("Can't upload OpenAPI file for collection groups"); return ERROR.toUpperCase(); }