Merge pull request #1736 from akto-api-security/hotfix/fix_new_params…

…_api Adding new limits and filters
akto-api-security · Nov 18, 2024 · e6ad7b5 · e6ad7b5
2 parents 9e33253 + 9e07935
commit e6ad7b5
Show file tree

Hide file tree

Showing 3 changed files with 26 additions and 12 deletions.
diff --git a/apps/dashboard/src/main/java/com/akto/action/observe/InventoryAction.java b/apps/dashboard/src/main/java/com/akto/action/observe/InventoryAction.java
@@ -844,7 +844,11 @@ public String fetchRecentParams (){
         long countApiInfosInvalid = ApiInfoDao.instance.count(apiInfoFilter);
 
         Bson useFilter = totalCount >= (2 * countApiInfosInvalid) ? apiInfoFilter : Filters.lt(ApiInfo.DISCOVERED_TIMESTAMP, this.startTimestamp - (Utils.DELTA_PERIOD_VALUE/4));
-        List<ApiInfo> apiInfos = ApiInfoDao.instance.findAll(useFilter, 0, 10000, Sorts.descending(ApiInfo.DISCOVERED_TIMESTAMP), Projections.include(Constants.ID));
+        List<ApiInfo> apiInfos = ApiInfoDao.instance.findAll(useFilter, 0, 2000, Sorts.descending(ApiInfo.DISCOVERED_TIMESTAMP), Projections.include(Constants.ID));
+        Set<Integer> uniqueApiCollections = new HashSet<>();
+        for(ApiInfo info: apiInfos){
+            uniqueApiCollections.add(info.getId().getApiCollectionId());
+        }
         Set<String> apiInfosHash = new HashSet<>();
         for(ApiInfo apiInfo: apiInfos){
             apiInfosHash.add(apiInfo.getId().toString());
@@ -854,11 +858,14 @@ public String fetchRecentParams (){
         pipeline.add(Aggregates.sort(Sorts.descending(SingleTypeInfo._TIMESTAMP)));
         pipeline.add(Aggregates.match(
             Filters.and(
+                Filters.in(SingleTypeInfo._API_COLLECTION_ID, uniqueApiCollections),
+                Filters.nin(SingleTypeInfo._API_COLLECTION_ID, deactivatedCollections),
                 Filters.gte(SingleTypeInfo._TIMESTAMP, this.startTimestamp),
                 Filters.lte(SingleTypeInfo._TIMESTAMP, this.endTimestamp)
             )
         ));
         pipeline.add(Aggregates.project(Projections.exclude(SingleTypeInfo._VALUES)));
+        pipeline.add(Aggregates.limit(20_000));
         List<SingleTypeInfo> singleTypeInfos = new ArrayList<>();
 
         MongoCursor<SingleTypeInfo> cursor = SingleTypeInfoDao.instance.getMCollection().aggregate(pipeline, SingleTypeInfo.class).cursor();

diff --git a/...board/web/polaris_web/web/src/apps/dashboard/pages/observe/api_collections/ApiChanges.jsx b/...board/web/polaris_web/web/src/apps/dashboard/pages/observe/api_collections/ApiChanges.jsx
@@ -24,6 +24,7 @@ function ApiChanges() {
     const [endpointsTrend, setEndpointsTrend] = useState([])
     const [loading, setLoading] = useState(false);
     const [apiDetail, setApiDetail] = useState({})
+    const [newParams, setNewParams] = useState([])
     const [tableHeaders,setTableHeaders] = useState([])
     const [summaryCountObj,setSummaryCountObj] = useState({
         "newEndpointsCount": 0,
@@ -96,8 +97,18 @@ function ApiChanges() {
         setParametersTrend(trendObj.trend)
 
         const endpointsTrendObj = transform.findNewParametersCountTrend(mergedArrObj, startTimestamp, endTimestamp)
-        setEndpointsTrend(endpointsTrendObj.trend)
         setLoading(false)
+        setEndpointsTrend(endpointsTrendObj.trend)
+        await api.fetchRecentParams(startTimestamp, endTimestamp).then((res) => {
+            const ret = res.data.endpoints.map((x,index) => transform.prepareEndpointForTable(x,index));
+            setNewParams(ret)
+            setSummaryCountObj((prev) => {
+                return{
+                    ...prev,
+                    newParamsCount: ret.length
+                }
+            })
+        })
     }
 
     useEffect(() => {
@@ -135,6 +146,7 @@ function ApiChanges() {
             endTimeStamp={endTimestamp}
             key="table"
             tab={(location.state)?(location.state.tab):0 }
+            newParams={newParams}
         />
     )
 

diff --git a/...is_web/web/src/apps/dashboard/pages/observe/api_collections/component/ApiChangesTable.jsx b/...is_web/web/src/apps/dashboard/pages/observe/api_collections/component/ApiChangesTable.jsx
@@ -11,7 +11,7 @@ import api from '../../api';
 
 function ApiChangesTable(props) {
 
-  const { handleRowClick, tableLoading, startTimeStamp, endTimeStamp} = props ;
+  const { handleRowClick, tableLoading, startTimeStamp, endTimeStamp, newParams} = props ;
   const [selectedTab, setSelectedTab] = useState("new_endpoints") ;
   const [selected, setSelected] = useState(0) ;
   const apiCollectionMap = PersistStore(state => state.collectionsMap)
@@ -20,10 +20,10 @@ function ApiChangesTable(props) {
 
   const definedTableTabs = ['New endpoints', 'New params']
 
-  const initialCount = [newEndpointsCount , 0]
+  const initialCount = [newEndpointsCount , newParams.length]
 
   const { tabsInfo } = useTable()
-  const tableCountObj = func.getTabsCount(definedTableTabs, [], initialCount)
+  const tableCountObj = func.getTabsCount(definedTableTabs, newParams, initialCount)
   const tableTabs = func.getTableTabsContent(definedTableTabs, tableCountObj, setSelectedTab, selectedTab, tabsInfo)
 
   const tableDataObj = apiChangesData.getData(selectedTab);
@@ -83,21 +83,16 @@ function ApiChangesTable(props) {
       })
       return { value: ret, total: total };
     }else{
-      let data = []
-      await api.fetchRecentParams(startTimeStamp, endTimeStamp).then((res) => {
-        const ret = res.data.endpoints.map((x,index) => transform.prepareEndpointForTable(x,index));
-        data = ret;
-      })
       const dataObj = {
         "headers": tableDataObj.headers,
-        "data": data,
+        "data": newParams,
         "sortOptions": tableDataObj.sortOptions,
       }
       return tableFunc.fetchDataSync(sortKey, sortOrder, skip, limit, filters, filterOperators, queryValue, () => {}, dataObj)
     }
   }
 
-  const key = selectedTab + startTimeStamp + endTimeStamp 
+  const key = selectedTab + startTimeStamp + endTimeStamp + newParams.length
   const filterOptions = func.getCollectionFilters(tableDataObj?.filters || [])
 
   return (