diff --git a/apps/dashboard/src/main/java/com/akto/action/ApiCollectionsAction.java b/apps/dashboard/src/main/java/com/akto/action/ApiCollectionsAction.java index 8192330b8c..f16308f4a2 100644 --- a/apps/dashboard/src/main/java/com/akto/action/ApiCollectionsAction.java +++ b/apps/dashboard/src/main/java/com/akto/action/ApiCollectionsAction.java @@ -26,9 +26,6 @@ import com.akto.dto.CollectionConditions.ConditionUtils; import com.akto.dto.billing.Organization; import com.akto.dto.type.SingleTypeInfo; -import com.akto.dto.usage.MetricTypes; -import com.akto.dto.usage.UsageMetric; -import com.akto.listener.InitializerListener; import com.akto.listener.RuntimeListener; import com.akto.log.LoggerMaker; import com.akto.log.LoggerMaker.LogDb; @@ -78,10 +75,10 @@ public void setApiList(List apiList) { boolean redacted; - public List fillApiCollectionsUrlCount(List apiCollections) { + public List fillApiCollectionsUrlCount(List apiCollections, Bson filter) { int tsRandom = Context.now(); loggerMaker.infoAndAddToDb("fillApiCollectionsUrlCount started: " + tsRandom, LoggerMaker.LogDb.DASHBOARD); - Map countMap = ApiCollectionsDao.instance.buildEndpointsCountToApiCollectionMap(); + Map countMap = ApiCollectionsDao.instance.buildEndpointsCountToApiCollectionMap(filter); loggerMaker.infoAndAddToDb("fillApiCollectionsUrlCount buildEndpointsCountToApiCollectionMap done: " + tsRandom, LoggerMaker.LogDb.DASHBOARD); for (ApiCollection apiCollection: apiCollections) { @@ -93,11 +90,7 @@ public List fillApiCollectionsUrlCount(List apiCol apiCollection.setUrlsCount(count); } else if(ApiCollection.Type.API_GROUP.equals(apiCollection.getType())){ if (count == null) { - List filters = SingleTypeInfoDao.filterForHostHostHeaderRaw(); - filters.add(Filters.in(SingleTypeInfo._COLLECTION_IDS, apiCollectionId)); - count = (int) SingleTypeInfoDao.instance.count( - Filters.and(filters) - ); + count = fallbackCount; } apiCollection.setUrlsCount(count); } else { @@ -116,9 +109,35 @@ public List fillApiCollectionsUrlCount(List apiCol return apiCollections; } + private Map deactivatedHostnameCountMap; + + public String getCountForHostnameDeactivatedCollections(){ + this.deactivatedHostnameCountMap = new HashMap<>(); + if(deactivatedCollections == null || deactivatedCollections.isEmpty()){ + return SUCCESS.toUpperCase(); + } + Bson filter = Filters.and(Filters.exists(ApiCollection.HOST_NAME), Filters.in(Constants.ID, deactivatedCollections)); + List hCollections = ApiCollectionsDao.instance.findAll(filter, Projections.include(Constants.ID)); + List deactivatedIds = new ArrayList<>(); + for(ApiCollection collection : hCollections){ + if(deactivatedCollections.contains(collection.getId())){ + deactivatedIds.add(collection.getId()); + } + } + + if(deactivatedIds.isEmpty()){ + return SUCCESS.toUpperCase(); + } + + this.deactivatedHostnameCountMap = ApiCollectionsDao.instance.buildEndpointsCountToApiCollectionMap( + Filters.in(SingleTypeInfo._COLLECTION_IDS, deactivatedIds) + ); + return SUCCESS.toUpperCase(); + } + public String fetchAllCollections() { this.apiCollections = ApiCollectionsDao.instance.findAll(new BasicDBObject()); - this.apiCollections = fillApiCollectionsUrlCount(this.apiCollections); + this.apiCollections = fillApiCollectionsUrlCount(this.apiCollections, Filters.empty()); return Action.SUCCESS.toUpperCase(); } @@ -178,7 +197,7 @@ public String fetchAllCollectionsBasic() { } } - this.apiCollections = fillApiCollectionsUrlCount(this.apiCollections); + this.apiCollections = fillApiCollectionsUrlCount(this.apiCollections, Filters.nin(SingleTypeInfo._API_COLLECTION_ID, deactivatedCollections)); return Action.SUCCESS.toUpperCase(); } @@ -599,7 +618,7 @@ private List filterCollections(List apiCollections public String deactivateCollections() { this.apiCollections = filterCollections(this.apiCollections, false); - this.apiCollections = fillApiCollectionsUrlCount(this.apiCollections); + this.apiCollections = fillApiCollectionsUrlCount(this.apiCollections,Filters.empty()); int deltaUsage = (-1) * this.apiCollections.stream().mapToInt(apiCollection -> apiCollection.getUrlsCount()).sum(); List apiCollectionIds = reduceApiCollectionToId(this.apiCollections); ApiCollectionsDao.instance.updateMany(Filters.in(Constants.ID, apiCollectionIds), @@ -613,7 +632,7 @@ public String activateCollections() { if (this.apiCollections.isEmpty()) { return Action.SUCCESS.toUpperCase(); } - this.apiCollections = fillApiCollectionsUrlCount(this.apiCollections); + this.apiCollections = fillApiCollectionsUrlCount(this.apiCollections,Filters.empty()); int accountId = Context.accountId.get(); FeatureAccess featureAccess = UsageMetricUtils.getFeatureAccess(accountId, MetricTypes.ACTIVE_ENDPOINTS); @@ -788,4 +807,8 @@ public void setStartTimestamp(int startTimestamp) { public void setEndTimestamp(int endTimestamp) { this.endTimestamp = endTimestamp; } + + public Map getDeactivatedHostnameCountMap() { + return deactivatedHostnameCountMap; + } } diff --git a/apps/dashboard/src/main/java/com/akto/action/observe/InventoryAction.java b/apps/dashboard/src/main/java/com/akto/action/observe/InventoryAction.java index 6a0a67c49e..5055fb11dc 100644 --- a/apps/dashboard/src/main/java/com/akto/action/observe/InventoryAction.java +++ b/apps/dashboard/src/main/java/com/akto/action/observe/InventoryAction.java @@ -421,10 +421,25 @@ public String fetchSensitiveParamsForEndpoints() { return Action.SUCCESS.toUpperCase(); } + public String loadRecentApiInfos(){ + Bson filter = Filters.and( + Filters.nin(ApiInfo.ID_API_COLLECTION_ID,deactivatedCollections), + Filters.gte(ApiInfo.DISCOVERED_TIMESTAMP, startTimestamp), + Filters.lte(ApiInfo.DISCOVERED_TIMESTAMP, endTimestamp) + ); + List apiInfos = ApiInfoDao.instance.findAll(filter); + for(ApiInfo info: apiInfos){ + info.calculateActualAuth(); + } + response = new BasicDBObject(); + response.put("apiInfoList", apiInfos); + return Action.SUCCESS.toUpperCase(); + } + public String loadRecentEndpoints() { List list = fetchRecentEndpoints(startTimestamp, endTimestamp); - attachTagsInAPIList(list); - attachAPIInfoListInResponse(list, -1); + response = new BasicDBObject(); + response.put("endpoints", list); return Action.SUCCESS.toUpperCase(); } diff --git a/apps/dashboard/src/main/java/com/akto/utils/notifications/TrafficUpdates.java b/apps/dashboard/src/main/java/com/akto/utils/notifications/TrafficUpdates.java index d37795d41a..9754a5d327 100644 --- a/apps/dashboard/src/main/java/com/akto/utils/notifications/TrafficUpdates.java +++ b/apps/dashboard/src/main/java/com/akto/utils/notifications/TrafficUpdates.java @@ -64,7 +64,7 @@ public void sendAlerts(String webhookUrl, String metricsUrl, int thresholdSecond public List filterTrafficMetricsAlertsList(List trafficMetricsAlertList) { List apiCollections = ApiCollectionsDao.instance.getMetaAll(); - Map countMap = ApiCollectionsDao.instance.buildEndpointsCountToApiCollectionMap(); + Map countMap = ApiCollectionsDao.instance.buildEndpointsCountToApiCollectionMap(Filters.empty()); Set allowedHosts = new HashSet<>(); for (ApiCollection apiCollection: apiCollections) { int apiCollectionId = apiCollection.getId(); diff --git a/apps/dashboard/src/main/resources/struts.xml b/apps/dashboard/src/main/resources/struts.xml index 4bc9b0107c..b4066b54dd 100644 --- a/apps/dashboard/src/main/resources/struts.xml +++ b/apps/dashboard/src/main/resources/struts.xml @@ -865,6 +865,27 @@ + + + + + API_COLLECTIONS + READ + + + + 403 + false + ^actionErrors.* + + + response + + + 401 + + + @@ -1472,6 +1493,26 @@ 401 + + + + + + + API_COLLECTIONS + READ + + + 403 + false + ^actionErrors.* + + + deactivatedHostnameCountMap + + + 401 + diff --git a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/observe/api.js b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/observe/api.js index df4e73204d..8ec8b84c5b 100644 --- a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/observe/api.js +++ b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/observe/api.js @@ -329,6 +329,15 @@ export default { }) return resp }, + + async loadRecentApiInfos (startTimestamp, endTimestamp) { + const resp = await request({ + url: '/api/loadRecentApiInfos', + method: 'post', + data: { startTimestamp, endTimestamp } + }) + return resp + }, async fetchSensitiveParamsForEndpoints (urls) { const resp = await request({ url: '/api/fetchSensitiveParamsForEndpoints', diff --git a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/observe/api_collections/ApiChanges.jsx b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/observe/api_collections/ApiChanges.jsx index d99ffc5c8d..ff59d24502 100644 --- a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/observe/api_collections/ApiChanges.jsx +++ b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/observe/api_collections/ApiChanges.jsx @@ -61,29 +61,40 @@ function ApiChanges() { }) } + async function fetchData() { + let apiCollection, apiCollectionUrls, apiInfoList; + let apiPromises = [ + api.loadRecentEndpoints(startTimestamp, endTimestamp), + api.loadRecentApiInfos(startTimestamp, endTimestamp), + api.fetchNewParametersTrend(startTimestamp, endTimestamp) + ]; + let results = await Promise.allSettled(apiPromises); + let endpointsFromStiResp = results[0].status === 'fulfilled' ? results[0].value : {"endpoints": []} + let endpointsFromApiInfos = results[1].status === 'fulfilled' ? results[1].value : {"apiInfoList" : []} + let parametersResp = results[2].status === 'fulfilled' ? results[2].value : {} + + apiCollection = endpointsFromStiResp.endpoints.map(x => { return { ...x._id, startTs: x.startTs } }) + apiCollectionUrls = endpointsFromStiResp.endpoints.map(x => x._id.url) + apiInfoList = endpointsFromApiInfos.apiInfoList + + await api.fetchSensitiveParamsForEndpoints(apiCollectionUrls).then(allSensitiveFields => { + let sensitiveParams = allSensitiveFields.data.endpoints + setSensitiveParams([...sensitiveParams]); + apiCollection = transform.fillSensitiveParams(sensitiveParams, apiCollection); + }) + + let data = func.mergeApiInfoAndApiCollection(apiCollection, apiInfoList, collectionsMap); + const prettifiedData = transform.prettifyEndpointsData(data) + setNewEndpoints({prettify: prettifiedData, normal: data}); + + const trendObj = transform.findNewParametersCountTrend(parametersResp, startTimestamp, endTimestamp) + setNewParametersCount(trendObj.count) + setParametersTrend(trendObj.trend) + + setLoading(false); + } + useEffect(() => { - async function fetchData() { - let apiCollection, apiCollectionUrls, apiInfoList; - await api.loadRecentEndpoints(startTimestamp, endTimestamp).then((res) => { - apiCollection = res.data.endpoints.map(x => { return { ...x._id, startTs: x.startTs } }) - apiCollectionUrls = res.data.endpoints.map(x => x._id.url) - apiInfoList = res.data.apiInfoList - }) - await api.fetchSensitiveParamsForEndpoints(apiCollectionUrls).then(allSensitiveFields => { - let sensitiveParams = allSensitiveFields.data.endpoints - setSensitiveParams([...sensitiveParams]); - apiCollection = transform.fillSensitiveParams(sensitiveParams, apiCollection); - }) - let data = func.mergeApiInfoAndApiCollection(apiCollection, apiInfoList, collectionsMap); - const prettifiedData = transform.prettifyEndpointsData(data) - setNewEndpoints({prettify: prettifiedData, normal: data}); - await api.fetchNewParametersTrend(startTimestamp, endTimestamp).then((resp) => { - const trendObj = transform.findNewParametersCountTrend(resp, startTimestamp, endTimestamp) - setNewParametersCount(trendObj.count) - setParametersTrend(trendObj.trend) - }) - setLoading(false); - } if (allCollections.length > 0) { fetchData(); } diff --git a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/observe/api_collections/ApiCollections.jsx b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/observe/api_collections/ApiCollections.jsx index 73e8f13b28..afd5c95166 100644 --- a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/observe/api_collections/ApiCollections.jsx +++ b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/observe/api_collections/ApiCollections.jsx @@ -175,12 +175,12 @@ const convertToNewData = (collectionsArr, sensitiveInfoMap, severityInfoMap, cov return{ ...c, displayNameComp: (), - testedEndpoints: coverageMap[c.id] ? coverageMap[c.id] : 0, + testedEndpoints: c.urlsCount === 0 ? 0 : (coverageMap[c.id] ? coverageMap[c.id] : 0), sensitiveInRespTypes: sensitiveInfoMap[c.id] ? sensitiveInfoMap[c.id] : [], severityInfo: severityInfoMap[c.id] ? severityInfoMap[c.id] : {}, detected: func.prettifyEpoch(trafficInfoMap[c.id] || 0), - detectedTimestamp: trafficInfoMap[c.id] || 0, - riskScore: riskScoreMap[c.id] ? riskScoreMap[c.id] : 0, + detectedTimestamp: c.urlsCount === 0 ? 0 : (trafficInfoMap[c.id] || 0), + riskScore: c.urlsCount === 0 ? 0 : (riskScoreMap[c.id] ? riskScoreMap[c.id] : 0), discovered: func.prettifyEpoch(c.startTs || 0), } }) @@ -295,6 +295,7 @@ function ApiCollections() { let apiPromises = [ api.getCoverageInfoForCollections(), api.getLastTrafficSeen(), + collectionApi.fetchCountForHostnameDeactivatedCollections() ]; if(shouldCallHeavyApis){ apiPromises = [ @@ -307,30 +308,31 @@ function ApiCollections() { let coverageInfo = results[0].status === 'fulfilled' ? results[0].value : {}; // let coverageInfo = dummyData.coverageMap let trafficInfo = results[1].status === 'fulfilled' ? results[1].value : {}; + let deactivatedCountInfo = results[2].status === 'fulfilled' ? results[2].value : {}; let riskScoreObj = lastFetchedResp let sensitiveInfo = lastFetchedSensitiveResp let severityObj = lastFetchedSeverityResp if(shouldCallHeavyApis){ - if(results[2]?.status === "fulfilled"){ - const res = results[2].value + if(results[3]?.status === "fulfilled"){ + const res = results[3].value riskScoreObj = { criticalUrls: res.criticalEndpointsCount, riskScoreMap: res.riskScoreOfCollectionsMap } } - if(results[3]?.status === "fulfilled"){ - const res = results[3].value + if(results[4]?.status === "fulfilled"){ + const res = results[4].value sensitiveInfo ={ sensitiveUrls: res.sensitiveUrlsInResponse, sensitiveInfoMap: res.sensitiveSubtypesInCollection } } - if(results[4]?.status === "fulfilled"){ - const res = results[4].value + if(results[5]?.status === "fulfilled"){ + const res = results[5].value severityObj = res } @@ -349,7 +351,12 @@ function ApiCollections() { setNormalData(dataObj.normal) // Separate active and deactivated collections - const deactivatedCollections = dataObj.prettify.filter(c => c.deactivated); + const deactivatedCollections = dataObj.prettify.filter(c => c.deactivated).map((c)=>{ + if(deactivatedCountInfo.hasOwnProperty(c.id)){ + c.urlsCount = deactivatedCountInfo[c.id] + } + return c + }); // Calculate summary data only for active collections const summary = transform.getSummaryData(dataObj.normal) diff --git a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/observe/api_collections/api.js b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/observe/api_collections/api.js index 9eea7c81d1..f2748e29a1 100644 --- a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/observe/api_collections/api.js +++ b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/observe/api_collections/api.js @@ -14,5 +14,12 @@ export default { method: 'post', data: { apiCollections: items } }) + }, + fetchCountForHostnameDeactivatedCollections(){ + return request({ + url: '/api/getCountForHostnameDeactivatedCollections', + method: 'post', + data: {} + }) } } \ No newline at end of file diff --git a/libs/dao/src/main/java/com/akto/dao/ApiCollectionsDao.java b/libs/dao/src/main/java/com/akto/dao/ApiCollectionsDao.java index 5f3c918926..cb8154a1bf 100644 --- a/libs/dao/src/main/java/com/akto/dao/ApiCollectionsDao.java +++ b/libs/dao/src/main/java/com/akto/dao/ApiCollectionsDao.java @@ -137,11 +137,15 @@ public List fetchNonTrafficApiCollectionsIds() { return apiCollectionIds; } - public Map buildEndpointsCountToApiCollectionMap() { + public Map buildEndpointsCountToApiCollectionMap(Bson filter) { Map countMap = new HashMap<>(); List pipeline = new ArrayList<>(); - pipeline.add(Aggregates.match(SingleTypeInfoDao.filterForHostHeader(0, false))); + pipeline.add(Aggregates.match(Filters.and( + SingleTypeInfoDao.filterForHostHeader(0, false), + filter + ) + )); BasicDBObject groupedId = new BasicDBObject(SingleTypeInfo._COLLECTION_IDS, "$" + SingleTypeInfo._COLLECTION_IDS); pipeline.add(Aggregates.unwind("$" + SingleTypeInfo._COLLECTION_IDS)); pipeline.add(Aggregates.group(groupedId, Accumulators.sum("count",1))); diff --git a/libs/dao/src/main/java/com/akto/dao/ApiInfoDao.java b/libs/dao/src/main/java/com/akto/dao/ApiInfoDao.java index b402cfdb0c..17fbf5269b 100644 --- a/libs/dao/src/main/java/com/akto/dao/ApiInfoDao.java +++ b/libs/dao/src/main/java/com/akto/dao/ApiInfoDao.java @@ -80,6 +80,9 @@ public void createIndicesIfAbsent() { MCollection.createIndexIfAbsent(getDBName(), getCollName(), new String[] { ApiInfo.RISK_SCORE, ApiInfo.ID_API_COLLECTION_ID }, false); + + MCollection.createIndexIfAbsent(getDBName(), getCollName(), + new String[] {ApiInfo.DISCOVERED_TIMESTAMP }, false); } diff --git a/libs/dao/src/main/java/com/akto/dao/SingleTypeInfoDao.java b/libs/dao/src/main/java/com/akto/dao/SingleTypeInfoDao.java index b8a5ac7af5..9b92fdceb0 100644 --- a/libs/dao/src/main/java/com/akto/dao/SingleTypeInfoDao.java +++ b/libs/dao/src/main/java/com/akto/dao/SingleTypeInfoDao.java @@ -719,7 +719,7 @@ public List fetchRecentEndpoints(int startTimestamp, int endTimes Filters.nin(SingleTypeInfo._API_COLLECTION_ID, nonHostApiCollectionIds), hostFilterQ ); - List latestHosts = SingleTypeInfoDao.instance.findAll(filterQWithTs, 0, 1_000, Sorts.descending("timestamp"), Projections.exclude("values")); + List latestHosts = SingleTypeInfoDao.instance.findAll(filterQWithTs, 0, 20_000, Sorts.descending("timestamp"), Projections.exclude("values")); for(SingleTypeInfo sti: latestHosts) { BasicDBObject id = new BasicDBObject("apiCollectionId", sti.getApiCollectionId())