From 66fbbfc7e8e2b854598834055ed3f54ff935118f Mon Sep 17 00:00:00 2001
From: Ark2307 <kr.aryan2307@gmail.com>
Date: Fri, 20 Sep 2024 12:28:12 +0530
Subject: [PATCH 1/4] Fixing in groups
---
.../com/akto/action/ApiCollectionsAction.java | 51 ++++++++++++++-----
.../utils/notifications/TrafficUpdates.java | 2 +-
apps/dashboard/src/main/resources/struts.xml | 20 ++++++++
.../api_collections/ApiCollections.jsx | 21 +++++---
.../pages/observe/api_collections/api.js | 7 +++
.../java/com/akto/dao/ApiCollectionsDao.java | 8 ++-
6 files changed, 85 insertions(+), 24 deletions(-)
diff --git a/apps/dashboard/src/main/java/com/akto/action/ApiCollectionsAction.java b/apps/dashboard/src/main/java/com/akto/action/ApiCollectionsAction.java
index 8192330b8c..f16308f4a2 100644
--- a/apps/dashboard/src/main/java/com/akto/action/ApiCollectionsAction.java
+++ b/apps/dashboard/src/main/java/com/akto/action/ApiCollectionsAction.java
@@ -26,9 +26,6 @@
import com.akto.dto.CollectionConditions.ConditionUtils;
import com.akto.dto.billing.Organization;
import com.akto.dto.type.SingleTypeInfo;
-import com.akto.dto.usage.MetricTypes;
-import com.akto.dto.usage.UsageMetric;
-import com.akto.listener.InitializerListener;
import com.akto.listener.RuntimeListener;
import com.akto.log.LoggerMaker;
import com.akto.log.LoggerMaker.LogDb;
@@ -78,10 +75,10 @@ public void setApiList(List<ApiInfoKey> apiList) {
boolean redacted;
- public List<ApiCollection> fillApiCollectionsUrlCount(List<ApiCollection> apiCollections) {
+ public List<ApiCollection> fillApiCollectionsUrlCount(List<ApiCollection> apiCollections, Bson filter) {
int tsRandom = Context.now();
loggerMaker.infoAndAddToDb("fillApiCollectionsUrlCount started: " + tsRandom, LoggerMaker.LogDb.DASHBOARD);
- Map<Integer, Integer> countMap = ApiCollectionsDao.instance.buildEndpointsCountToApiCollectionMap();
+ Map<Integer, Integer> countMap = ApiCollectionsDao.instance.buildEndpointsCountToApiCollectionMap(filter);
loggerMaker.infoAndAddToDb("fillApiCollectionsUrlCount buildEndpointsCountToApiCollectionMap done: " + tsRandom, LoggerMaker.LogDb.DASHBOARD);
for (ApiCollection apiCollection: apiCollections) {
@@ -93,11 +90,7 @@ public List<ApiCollection> fillApiCollectionsUrlCount(List<ApiCollection> apiCol
apiCollection.setUrlsCount(count);
} else if(ApiCollection.Type.API_GROUP.equals(apiCollection.getType())){
if (count == null) {
- List<Bson> filters = SingleTypeInfoDao.filterForHostHostHeaderRaw();
- filters.add(Filters.in(SingleTypeInfo._COLLECTION_IDS, apiCollectionId));
- count = (int) SingleTypeInfoDao.instance.count(
- Filters.and(filters)
- );
+ count = fallbackCount;
}
apiCollection.setUrlsCount(count);
} else {
@@ -116,9 +109,35 @@ public List<ApiCollection> fillApiCollectionsUrlCount(List<ApiCollection> apiCol
return apiCollections;
}
+ private Map<Integer, Integer> deactivatedHostnameCountMap;
+
+ public String getCountForHostnameDeactivatedCollections(){
+ this.deactivatedHostnameCountMap = new HashMap<>();
+ if(deactivatedCollections == null || deactivatedCollections.isEmpty()){
+ return SUCCESS.toUpperCase();
+ }
+ Bson filter = Filters.and(Filters.exists(ApiCollection.HOST_NAME), Filters.in(Constants.ID, deactivatedCollections));
+ List<ApiCollection> hCollections = ApiCollectionsDao.instance.findAll(filter, Projections.include(Constants.ID));
+ List<Integer> deactivatedIds = new ArrayList<>();
+ for(ApiCollection collection : hCollections){
+ if(deactivatedCollections.contains(collection.getId())){
+ deactivatedIds.add(collection.getId());
+ }
+ }
+
+ if(deactivatedIds.isEmpty()){
+ return SUCCESS.toUpperCase();
+ }
+
+ this.deactivatedHostnameCountMap = ApiCollectionsDao.instance.buildEndpointsCountToApiCollectionMap(
+ Filters.in(SingleTypeInfo._COLLECTION_IDS, deactivatedIds)
+ );
+ return SUCCESS.toUpperCase();
+ }
+
public String fetchAllCollections() {
this.apiCollections = ApiCollectionsDao.instance.findAll(new BasicDBObject());
- this.apiCollections = fillApiCollectionsUrlCount(this.apiCollections);
+ this.apiCollections = fillApiCollectionsUrlCount(this.apiCollections, Filters.empty());
return Action.SUCCESS.toUpperCase();
}
@@ -178,7 +197,7 @@ public String fetchAllCollectionsBasic() {
}
}
- this.apiCollections = fillApiCollectionsUrlCount(this.apiCollections);
+ this.apiCollections = fillApiCollectionsUrlCount(this.apiCollections, Filters.nin(SingleTypeInfo._API_COLLECTION_ID, deactivatedCollections));
return Action.SUCCESS.toUpperCase();
}
@@ -599,7 +618,7 @@ private List<ApiCollection> filterCollections(List<ApiCollection> apiCollections
public String deactivateCollections() {
this.apiCollections = filterCollections(this.apiCollections, false);
- this.apiCollections = fillApiCollectionsUrlCount(this.apiCollections);
+ this.apiCollections = fillApiCollectionsUrlCount(this.apiCollections,Filters.empty());
int deltaUsage = (-1) * this.apiCollections.stream().mapToInt(apiCollection -> apiCollection.getUrlsCount()).sum();
List<Integer> apiCollectionIds = reduceApiCollectionToId(this.apiCollections);
ApiCollectionsDao.instance.updateMany(Filters.in(Constants.ID, apiCollectionIds),
@@ -613,7 +632,7 @@ public String activateCollections() {
if (this.apiCollections.isEmpty()) {
return Action.SUCCESS.toUpperCase();
}
- this.apiCollections = fillApiCollectionsUrlCount(this.apiCollections);
+ this.apiCollections = fillApiCollectionsUrlCount(this.apiCollections,Filters.empty());
int accountId = Context.accountId.get();
FeatureAccess featureAccess = UsageMetricUtils.getFeatureAccess(accountId, MetricTypes.ACTIVE_ENDPOINTS);
@@ -788,4 +807,8 @@ public void setStartTimestamp(int startTimestamp) {
public void setEndTimestamp(int endTimestamp) {
this.endTimestamp = endTimestamp;
}
+
+ public Map<Integer, Integer> getDeactivatedHostnameCountMap() {
+ return deactivatedHostnameCountMap;
+ }
}
diff --git a/apps/dashboard/src/main/java/com/akto/utils/notifications/TrafficUpdates.java b/apps/dashboard/src/main/java/com/akto/utils/notifications/TrafficUpdates.java
index d37795d41a..9754a5d327 100644
--- a/apps/dashboard/src/main/java/com/akto/utils/notifications/TrafficUpdates.java
+++ b/apps/dashboard/src/main/java/com/akto/utils/notifications/TrafficUpdates.java
@@ -64,7 +64,7 @@ public void sendAlerts(String webhookUrl, String metricsUrl, int thresholdSecond
public List<TrafficMetricsAlert> filterTrafficMetricsAlertsList(List<TrafficMetricsAlert> trafficMetricsAlertList) {
List<ApiCollection> apiCollections = ApiCollectionsDao.instance.getMetaAll();
- Map<Integer, Integer> countMap = ApiCollectionsDao.instance.buildEndpointsCountToApiCollectionMap();
+ Map<Integer, Integer> countMap = ApiCollectionsDao.instance.buildEndpointsCountToApiCollectionMap(Filters.empty());
Set<String> allowedHosts = new HashSet<>();
for (ApiCollection apiCollection: apiCollections) {
int apiCollectionId = apiCollection.getId();
diff --git a/apps/dashboard/src/main/resources/struts.xml b/apps/dashboard/src/main/resources/struts.xml
index 4bc9b0107c..ebbe01d95e 100644
--- a/apps/dashboard/src/main/resources/struts.xml
+++ b/apps/dashboard/src/main/resources/struts.xml
@@ -1472,6 +1472,26 @@
<result name="ERROR" type="httpheader">
<param name="status">401</param>
</result>
+ </action>
+
+ <action name="api/getCountForHostnameDeactivatedCollections" class="com.akto.action.ApiCollectionsAction" method="getCountForHostnameDeactivatedCollections">
+ <interceptor-ref name="json"/>
+ <interceptor-ref name="defaultStack" />
+ <interceptor-ref name="roleAccessInterceptor">
+ <param name="featureLabel">API_COLLECTIONS</param>
+ <param name="accessType">READ</param>
+ </interceptor-ref>
+ <result name="FORBIDDEN" type="json">
+ <param name="statusCode">403</param>
+ <param name="ignoreHierarchy">false</param>
+ <param name="includeProperties">^actionErrors.*</param>
+ </result>
+ <result name="SUCCESS" type="json">
+ <param name="root">deactivatedHostnameCountMap</param>
+ </result>
+ <result name="ERROR" type="httpheader">
+ <param name="status">401</param>
+ </result>
</action>
<action name="api/getSensitiveInfoForCollections" class="com.akto.action.ApiCollectionsAction" method="fetchSensitiveInfoInCollections">
diff --git a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/observe/api_collections/ApiCollections.jsx b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/observe/api_collections/ApiCollections.jsx
index 73e8f13b28..6cacc5ca8e 100644
--- a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/observe/api_collections/ApiCollections.jsx
+++ b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/observe/api_collections/ApiCollections.jsx
@@ -295,6 +295,7 @@ function ApiCollections() {
let apiPromises = [
api.getCoverageInfoForCollections(),
api.getLastTrafficSeen(),
+ collectionApi.fetchCountForHostnameDeactivatedCollections()
];
if(shouldCallHeavyApis){
apiPromises = [
@@ -307,30 +308,31 @@ function ApiCollections() {
let coverageInfo = results[0].status === 'fulfilled' ? results[0].value : {};
// let coverageInfo = dummyData.coverageMap
let trafficInfo = results[1].status === 'fulfilled' ? results[1].value : {};
+ let deactivatedCountInfo = results[2].status === 'fulfilled' ? results[2].value : {};
let riskScoreObj = lastFetchedResp
let sensitiveInfo = lastFetchedSensitiveResp
let severityObj = lastFetchedSeverityResp
if(shouldCallHeavyApis){
- if(results[2]?.status === "fulfilled"){
- const res = results[2].value
+ if(results[3]?.status === "fulfilled"){
+ const res = results[3].value
riskScoreObj = {
criticalUrls: res.criticalEndpointsCount,
riskScoreMap: res.riskScoreOfCollectionsMap
}
}
- if(results[3]?.status === "fulfilled"){
- const res = results[3].value
+ if(results[4]?.status === "fulfilled"){
+ const res = results[4].value
sensitiveInfo ={
sensitiveUrls: res.sensitiveUrlsInResponse,
sensitiveInfoMap: res.sensitiveSubtypesInCollection
}
}
- if(results[4]?.status === "fulfilled"){
- const res = results[4].value
+ if(results[5]?.status === "fulfilled"){
+ const res = results[5].value
severityObj = res
}
@@ -349,7 +351,12 @@ function ApiCollections() {
setNormalData(dataObj.normal)
// Separate active and deactivated collections
- const deactivatedCollections = dataObj.prettify.filter(c => c.deactivated);
+ const deactivatedCollections = dataObj.prettify.filter(c => c.deactivated).map((c)=>{
+ if(c.type !== "API_GROUP" && deactivatedCountInfo.hasOwnProperty(c.id)){
+ c.urlsCount = deactivatedCountInfo[c.id]
+ }
+ return c
+ });
// Calculate summary data only for active collections
const summary = transform.getSummaryData(dataObj.normal)
diff --git a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/observe/api_collections/api.js b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/observe/api_collections/api.js
index 9eea7c81d1..f2748e29a1 100644
--- a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/observe/api_collections/api.js
+++ b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/observe/api_collections/api.js
@@ -14,5 +14,12 @@ export default {
method: 'post',
data: { apiCollections: items }
})
+ },
+ fetchCountForHostnameDeactivatedCollections(){
+ return request({
+ url: '/api/getCountForHostnameDeactivatedCollections',
+ method: 'post',
+ data: {}
+ })
}
}
\ No newline at end of file
diff --git a/libs/dao/src/main/java/com/akto/dao/ApiCollectionsDao.java b/libs/dao/src/main/java/com/akto/dao/ApiCollectionsDao.java
index 5f3c918926..cb8154a1bf 100644
--- a/libs/dao/src/main/java/com/akto/dao/ApiCollectionsDao.java
+++ b/libs/dao/src/main/java/com/akto/dao/ApiCollectionsDao.java
@@ -137,11 +137,15 @@ public List<Integer> fetchNonTrafficApiCollectionsIds() {
return apiCollectionIds;
}
- public Map<Integer, Integer> buildEndpointsCountToApiCollectionMap() {
+ public Map<Integer, Integer> buildEndpointsCountToApiCollectionMap(Bson filter) {
Map<Integer, Integer> countMap = new HashMap<>();
List<Bson> pipeline = new ArrayList<>();
- pipeline.add(Aggregates.match(SingleTypeInfoDao.filterForHostHeader(0, false)));
+ pipeline.add(Aggregates.match(Filters.and(
+ SingleTypeInfoDao.filterForHostHeader(0, false),
+ filter
+ )
+ ));
BasicDBObject groupedId = new BasicDBObject(SingleTypeInfo._COLLECTION_IDS, "$" + SingleTypeInfo._COLLECTION_IDS);
pipeline.add(Aggregates.unwind("$" + SingleTypeInfo._COLLECTION_IDS));
pipeline.add(Aggregates.group(groupedId, Accumulators.sum("count",1)));
From 80395f9f80d2c8fe2b6b0599f972800d8a11dd59 Mon Sep 17 00:00:00 2001
From: Ark2307 <kr.aryan2307@gmail.com>
Date: Fri, 20 Sep 2024 16:03:41 +0530
Subject: [PATCH 2/4] Fixing fields for groups
---
.../pages/observe/api_collections/ApiCollections.jsx | 8 ++++----
1 file changed, 4 insertions(+), 4 deletions(-)
diff --git a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/observe/api_collections/ApiCollections.jsx b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/observe/api_collections/ApiCollections.jsx
index 6cacc5ca8e..afd5c95166 100644
--- a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/observe/api_collections/ApiCollections.jsx
+++ b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/observe/api_collections/ApiCollections.jsx
@@ -175,12 +175,12 @@ const convertToNewData = (collectionsArr, sensitiveInfoMap, severityInfoMap, cov
return{
...c,
displayNameComp: (<Box maxWidth="20vw"><TooltipText tooltip={c.displayName} text={c.displayName} textProps={{fontWeight: 'medium'}}/></Box>),
- testedEndpoints: coverageMap[c.id] ? coverageMap[c.id] : 0,
+ testedEndpoints: c.urlsCount === 0 ? 0 : (coverageMap[c.id] ? coverageMap[c.id] : 0),
sensitiveInRespTypes: sensitiveInfoMap[c.id] ? sensitiveInfoMap[c.id] : [],
severityInfo: severityInfoMap[c.id] ? severityInfoMap[c.id] : {},
detected: func.prettifyEpoch(trafficInfoMap[c.id] || 0),
- detectedTimestamp: trafficInfoMap[c.id] || 0,
- riskScore: riskScoreMap[c.id] ? riskScoreMap[c.id] : 0,
+ detectedTimestamp: c.urlsCount === 0 ? 0 : (trafficInfoMap[c.id] || 0),
+ riskScore: c.urlsCount === 0 ? 0 : (riskScoreMap[c.id] ? riskScoreMap[c.id] : 0),
discovered: func.prettifyEpoch(c.startTs || 0),
}
})
@@ -352,7 +352,7 @@ function ApiCollections() {
// Separate active and deactivated collections
const deactivatedCollections = dataObj.prettify.filter(c => c.deactivated).map((c)=>{
- if(c.type !== "API_GROUP" && deactivatedCountInfo.hasOwnProperty(c.id)){
+ if(deactivatedCountInfo.hasOwnProperty(c.id)){
c.urlsCount = deactivatedCountInfo[c.id]
}
return c
From 3442d6ff28b634544dd3415861eac7ca1bbf4fda Mon Sep 17 00:00:00 2001
From: Ark2307 <kr.aryan2307@gmail.com>
Date: Fri, 20 Sep 2024 16:24:56 +0530
Subject: [PATCH 3/4] increasing limit for recent endpoints
---
libs/dao/src/main/java/com/akto/dao/SingleTypeInfoDao.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/libs/dao/src/main/java/com/akto/dao/SingleTypeInfoDao.java b/libs/dao/src/main/java/com/akto/dao/SingleTypeInfoDao.java
index b8a5ac7af5..9b92fdceb0 100644
--- a/libs/dao/src/main/java/com/akto/dao/SingleTypeInfoDao.java
+++ b/libs/dao/src/main/java/com/akto/dao/SingleTypeInfoDao.java
@@ -719,7 +719,7 @@ public List<BasicDBObject> fetchRecentEndpoints(int startTimestamp, int endTimes
Filters.nin(SingleTypeInfo._API_COLLECTION_ID, nonHostApiCollectionIds),
hostFilterQ
);
- List<SingleTypeInfo> latestHosts = SingleTypeInfoDao.instance.findAll(filterQWithTs, 0, 1_000, Sorts.descending("timestamp"), Projections.exclude("values"));
+ List<SingleTypeInfo> latestHosts = SingleTypeInfoDao.instance.findAll(filterQWithTs, 0, 20_000, Sorts.descending("timestamp"), Projections.exclude("values"));
for(SingleTypeInfo sti: latestHosts) {
BasicDBObject id =
new BasicDBObject("apiCollectionId", sti.getApiCollectionId())
From 1ec7c980803fbbe6c4db072874f7c8900715eb53 Mon Sep 17 00:00:00 2001
From: Ark2307 <kr.aryan2307@gmail.com>
Date: Fri, 20 Sep 2024 18:09:16 +0530
Subject: [PATCH 4/4] Optimising api changes page
---
.../akto/action/observe/InventoryAction.java | 19 ++++++-
apps/dashboard/src/main/resources/struts.xml | 21 +++++++
.../src/apps/dashboard/pages/observe/api.js | 9 +++
.../observe/api_collections/ApiChanges.jsx | 55 +++++++++++--------
.../main/java/com/akto/dao/ApiInfoDao.java | 3 +
5 files changed, 83 insertions(+), 24 deletions(-)
diff --git a/apps/dashboard/src/main/java/com/akto/action/observe/InventoryAction.java b/apps/dashboard/src/main/java/com/akto/action/observe/InventoryAction.java
index 83dc5d355c..98b52c2805 100644
--- a/apps/dashboard/src/main/java/com/akto/action/observe/InventoryAction.java
+++ b/apps/dashboard/src/main/java/com/akto/action/observe/InventoryAction.java
@@ -419,10 +419,25 @@ public String fetchSensitiveParamsForEndpoints() {
return Action.SUCCESS.toUpperCase();
}
+ public String loadRecentApiInfos(){
+ Bson filter = Filters.and(
+ Filters.nin(ApiInfo.ID_API_COLLECTION_ID,deactivatedCollections),
+ Filters.gte(ApiInfo.DISCOVERED_TIMESTAMP, startTimestamp),
+ Filters.lte(ApiInfo.DISCOVERED_TIMESTAMP, endTimestamp)
+ );
+ List<ApiInfo> apiInfos = ApiInfoDao.instance.findAll(filter);
+ for(ApiInfo info: apiInfos){
+ info.calculateActualAuth();
+ }
+ response = new BasicDBObject();
+ response.put("apiInfoList", apiInfos);
+ return Action.SUCCESS.toUpperCase();
+ }
+
public String loadRecentEndpoints() {
List<BasicDBObject> list = fetchRecentEndpoints(startTimestamp, endTimestamp);
- attachTagsInAPIList(list);
- attachAPIInfoListInResponse(list, -1);
+ response = new BasicDBObject();
+ response.put("endpoints", list);
return Action.SUCCESS.toUpperCase();
}
diff --git a/apps/dashboard/src/main/resources/struts.xml b/apps/dashboard/src/main/resources/struts.xml
index ebbe01d95e..b4066b54dd 100644
--- a/apps/dashboard/src/main/resources/struts.xml
+++ b/apps/dashboard/src/main/resources/struts.xml
@@ -865,6 +865,27 @@
</result>
</action>
+ <action name="api/loadRecentApiInfos" class="com.akto.action.observe.InventoryAction" method="loadRecentApiInfos">
+ <interceptor-ref name="json"/>
+ <interceptor-ref name="defaultStack" />
+ <interceptor-ref name="roleAccessInterceptor">
+ <param name="featureLabel">API_COLLECTIONS</param>
+ <param name="accessType">READ</param>
+ </interceptor-ref>
+
+ <result name="FORBIDDEN" type="json">
+ <param name="statusCode">403</param>
+ <param name="ignoreHierarchy">false</param>
+ <param name="includeProperties">^actionErrors.*</param>
+ </result>
+ <result name="SUCCESS" type="json">
+ <param name="root">response</param>
+ </result>
+ <result name="ERROR" type="httpheader">
+ <param name="status">401</param>
+ </result>
+ </action>
+
<action name="api/fetchNewParametersTrend" class="com.akto.action.observe.InventoryAction" method="fetchNewParametersTrend">
<interceptor-ref name="json"/>
<interceptor-ref name="defaultStack" />
diff --git a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/observe/api.js b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/observe/api.js
index df4e73204d..8ec8b84c5b 100644
--- a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/observe/api.js
+++ b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/observe/api.js
@@ -329,6 +329,15 @@ export default {
})
return resp
},
+
+ async loadRecentApiInfos (startTimestamp, endTimestamp) {
+ const resp = await request({
+ url: '/api/loadRecentApiInfos',
+ method: 'post',
+ data: { startTimestamp, endTimestamp }
+ })
+ return resp
+ },
async fetchSensitiveParamsForEndpoints (urls) {
const resp = await request({
url: '/api/fetchSensitiveParamsForEndpoints',
diff --git a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/observe/api_collections/ApiChanges.jsx b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/observe/api_collections/ApiChanges.jsx
index d99ffc5c8d..ff59d24502 100644
--- a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/observe/api_collections/ApiChanges.jsx
+++ b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/observe/api_collections/ApiChanges.jsx
@@ -61,29 +61,40 @@ function ApiChanges() {
})
}
+ async function fetchData() {
+ let apiCollection, apiCollectionUrls, apiInfoList;
+ let apiPromises = [
+ api.loadRecentEndpoints(startTimestamp, endTimestamp),
+ api.loadRecentApiInfos(startTimestamp, endTimestamp),
+ api.fetchNewParametersTrend(startTimestamp, endTimestamp)
+ ];
+ let results = await Promise.allSettled(apiPromises);
+ let endpointsFromStiResp = results[0].status === 'fulfilled' ? results[0].value : {"endpoints": []}
+ let endpointsFromApiInfos = results[1].status === 'fulfilled' ? results[1].value : {"apiInfoList" : []}
+ let parametersResp = results[2].status === 'fulfilled' ? results[2].value : {}
+
+ apiCollection = endpointsFromStiResp.endpoints.map(x => { return { ...x._id, startTs: x.startTs } })
+ apiCollectionUrls = endpointsFromStiResp.endpoints.map(x => x._id.url)
+ apiInfoList = endpointsFromApiInfos.apiInfoList
+
+ await api.fetchSensitiveParamsForEndpoints(apiCollectionUrls).then(allSensitiveFields => {
+ let sensitiveParams = allSensitiveFields.data.endpoints
+ setSensitiveParams([...sensitiveParams]);
+ apiCollection = transform.fillSensitiveParams(sensitiveParams, apiCollection);
+ })
+
+ let data = func.mergeApiInfoAndApiCollection(apiCollection, apiInfoList, collectionsMap);
+ const prettifiedData = transform.prettifyEndpointsData(data)
+ setNewEndpoints({prettify: prettifiedData, normal: data});
+
+ const trendObj = transform.findNewParametersCountTrend(parametersResp, startTimestamp, endTimestamp)
+ setNewParametersCount(trendObj.count)
+ setParametersTrend(trendObj.trend)
+
+ setLoading(false);
+ }
+
useEffect(() => {
- async function fetchData() {
- let apiCollection, apiCollectionUrls, apiInfoList;
- await api.loadRecentEndpoints(startTimestamp, endTimestamp).then((res) => {
- apiCollection = res.data.endpoints.map(x => { return { ...x._id, startTs: x.startTs } })
- apiCollectionUrls = res.data.endpoints.map(x => x._id.url)
- apiInfoList = res.data.apiInfoList
- })
- await api.fetchSensitiveParamsForEndpoints(apiCollectionUrls).then(allSensitiveFields => {
- let sensitiveParams = allSensitiveFields.data.endpoints
- setSensitiveParams([...sensitiveParams]);
- apiCollection = transform.fillSensitiveParams(sensitiveParams, apiCollection);
- })
- let data = func.mergeApiInfoAndApiCollection(apiCollection, apiInfoList, collectionsMap);
- const prettifiedData = transform.prettifyEndpointsData(data)
- setNewEndpoints({prettify: prettifiedData, normal: data});
- await api.fetchNewParametersTrend(startTimestamp, endTimestamp).then((resp) => {
- const trendObj = transform.findNewParametersCountTrend(resp, startTimestamp, endTimestamp)
- setNewParametersCount(trendObj.count)
- setParametersTrend(trendObj.trend)
- })
- setLoading(false);
- }
if (allCollections.length > 0) {
fetchData();
}
diff --git a/libs/dao/src/main/java/com/akto/dao/ApiInfoDao.java b/libs/dao/src/main/java/com/akto/dao/ApiInfoDao.java
index b402cfdb0c..17fbf5269b 100644
--- a/libs/dao/src/main/java/com/akto/dao/ApiInfoDao.java
+++ b/libs/dao/src/main/java/com/akto/dao/ApiInfoDao.java
@@ -80,6 +80,9 @@ public void createIndicesIfAbsent() {
MCollection.createIndexIfAbsent(getDBName(), getCollName(),
new String[] { ApiInfo.RISK_SCORE, ApiInfo.ID_API_COLLECTION_ID }, false);
+
+ MCollection.createIndexIfAbsent(getDBName(), getCollName(),
+ new String[] {ApiInfo.DISCOVERED_TIMESTAMP }, false);
}