feat: created a new endpoint to fetch merged urls

akto-api-security · Sep 21, 2024 · 390325f · 390325f
1 parent 925880f
commit 390325f
Show file tree

Hide file tree

Showing 8 changed files with 81 additions and 2 deletions.
diff --git a/apps/database-abstractor/src/main/java/com/akto/action/DbAction.java b/apps/database-abstractor/src/main/java/com/akto/action/DbAction.java
@@ -10,6 +10,7 @@
 import com.akto.dto.billing.Tokens;
 import com.akto.dto.bulk_updates.BulkUpdates;
 import com.akto.dto.bulk_updates.UpdatePayload;
+import com.akto.dto.filter.MergedUrls;
 import com.akto.dto.runtime_filters.RuntimeFilter;
 import com.akto.dto.settings.DataControlSettings;
 import com.akto.dto.test_editor.YamlTemplate;
@@ -145,6 +146,7 @@ public void setTestSourceConfig(TestSourceConfig testSourceConfig) {
     String logicalGroupName;
     BasicDBList issuesIds;
     List<YamlTemplate> activeAdvancedFilters;
+    Set<MergedUrls> mergedUrls;
 
     public BasicDBList getIssuesIds() {
         return issuesIds;
@@ -1654,6 +1656,15 @@ public String fetchActiveAdvancedFilters(){
         return Action.SUCCESS.toUpperCase();
     }
 
+    public String fetchMergedUrls() {
+        try {
+            this.mergedUrls = DbLayer.fetchMergedUrls();
+        } catch (Exception e) {
+            return Action.ERROR.toUpperCase();
+        }
+        return Action.SUCCESS.toUpperCase();
+    }
+
     public List<CustomDataTypeMapper> getCustomDataTypes() {
         return customDataTypes;
     }
@@ -2540,4 +2551,11 @@ public void setActiveAdvancedFilters(List<YamlTemplate> activeAdvancedFilters) {
         this.activeAdvancedFilters = activeAdvancedFilters;
     }
 
+    public Set<MergedUrls> getMergedUrls() {
+        return mergedUrls;
+    }
+
+    public void setMergedUrls(Set<MergedUrls> mergedUrls) {
+        this.mergedUrls = mergedUrls;
+    }
 }
diff --git a/apps/database-abstractor/src/main/resources/struts.xml b/apps/database-abstractor/src/main/resources/struts.xml
@@ -1146,6 +1146,17 @@
             </result>
         </action>
 
+        <action name="api/fetchMergedUrls" class="com.akto.action.DbAction" method="fetchMergedUrls">
+            <interceptor-ref name="json"/>
+            <interceptor-ref name="defaultStack" />
+            <result name="SUCCESS" type="json"/>
+            <result name="ERROR" type="json">
+                <param name="statusCode">422</param>
+                <param name="ignoreHierarchy">false</param>
+                <param name="includeProperties">^actionErrors.*</param>
+            </result>
+        </action>
+
     </package>
 
 </struts>
diff --git a/apps/mini-runtime/src/main/java/com/akto/hybrid_runtime/APICatalogSync.java b/apps/mini-runtime/src/main/java/com/akto/hybrid_runtime/APICatalogSync.java
@@ -7,7 +7,6 @@
 
 import com.akto.dao.*;
 import com.akto.dao.context.Context;
-import com.akto.dao.filter.MergedUrlsDao;
 import com.akto.dao.monitoring.FilterYamlTemplateDao;
 import com.akto.dao.runtime_filters.AdvancedTrafficFiltersDao;
 import com.akto.dto.*;
@@ -1155,7 +1154,7 @@ public void buildFromDB(boolean calcDiff, boolean fetchAllSTI) {
             loggerMaker.infoAndAddToDb("Error while clearing values in db: " + e.getMessage(), LogDb.RUNTIME);
         }
 
-        mergedUrls = MergedUrlsDao.instance.getMergedUrls();
+        mergedUrls = dataActor.fetchMergedUrls();
 
         aktoPolicyNew.buildFromDb(fetchAllSTI);
     }

diff --git a/libs/dao/src/main/java/com/akto/dao/filter/MergedUrlsDao.java b/libs/dao/src/main/java/com/akto/dao/filter/MergedUrlsDao.java
@@ -26,6 +26,8 @@ public Set<MergedUrls> getMergedUrls() {
             mergedUrls.add(mergedUrlsObj);
         }
 
+        cursor.close();
+
         return mergedUrls;
     }
 

diff --git a/libs/utils/src/main/java/com/akto/data_actor/ClientActor.java b/libs/utils/src/main/java/com/akto/data_actor/ClientActor.java
@@ -1,6 +1,7 @@
 package com.akto.data_actor;
 
 import com.akto.DaoInit;
+import com.akto.dto.filter.MergedUrls;
 import com.akto.dto.settings.DataControlSettings;
 import com.akto.testing.ApiExecutor;
 import com.auth0.jwt.JWT;
@@ -3055,4 +3056,38 @@ public List<YamlTemplate> fetchActiveAdvancedFilters(){
         return respList;
     }
 
+    public Set<MergedUrls> fetchMergedUrls() {
+        Map<String, List<String>> headers = buildHeaders();
+
+        List<MergedUrls> respList = new ArrayList<>();
+        OriginalHttpRequest request = new OriginalHttpRequest(url + "/fetchMergedUrls", "", "POST", "", headers, "");
+
+        try {
+            OriginalHttpResponse response = ApiExecutor.sendRequestBackOff(request, true, null, false, null);
+            String responsePayload = response.getBody();
+            if (response.getStatusCode() != 200 || responsePayload == null) {
+                loggerMaker.errorAndAddToDb("non 2xx response in fetchMergedUrls", LoggerMaker.LogDb.RUNTIME);
+                return null;
+            }
+            BasicDBObject payloadObj;
+
+            try {
+                payloadObj = BasicDBObject.parse(responsePayload);
+                BasicDBList newUrls = (BasicDBList) payloadObj.get("mergedUrls");
+                for (Object url: newUrls) {
+                    BasicDBObject urlObj = (BasicDBObject) url;
+                    MergedUrls mergedUrl = objectMapper.readValue(urlObj.toJson(), MergedUrls.class);
+                    respList.add(mergedUrl);
+                }
+            } catch (Exception e) {
+                loggerMaker.errorAndAddToDb("error extracting response in fetchMergedUrls" + e, LoggerMaker.LogDb.RUNTIME);
+            }
+        } catch (Exception e) {
+            loggerMaker.errorAndAddToDb("error in fetching merged urls: " + e, LoggerMaker.LogDb.RUNTIME);
+            return null;
+        }
+
+        return new HashSet<>(respList);
+    }
+
 }
diff --git a/libs/utils/src/main/java/com/akto/data_actor/DataActor.java b/libs/utils/src/main/java/com/akto/data_actor/DataActor.java
@@ -3,6 +3,7 @@
 import com.akto.dto.*;
 import com.akto.dto.billing.Organization;
 import com.akto.dto.billing.Tokens;
+import com.akto.dto.filter.MergedUrls;
 import com.akto.dto.runtime_filters.RuntimeFilter;
 import com.akto.dto.settings.DataControlSettings;
 import com.akto.dto.test_editor.YamlTemplate;
@@ -227,4 +228,6 @@ public abstract class DataActor {
     public abstract List<ApiInfo.ApiInfoKey> fetchLatestEndpointsForTesting(int startTimestamp, int endTimestamp, int apiCollectionId);
 
     public abstract List<YamlTemplate> fetchActiveAdvancedFilters();
+
+    public abstract Set<MergedUrls> fetchMergedUrls();
 }
diff --git a/libs/utils/src/main/java/com/akto/data_actor/DbActor.java b/libs/utils/src/main/java/com/akto/data_actor/DbActor.java
@@ -4,6 +4,7 @@
 import com.akto.dto.ApiInfo.ApiInfoKey;
 import com.akto.dto.billing.Organization;
 import com.akto.dto.billing.Tokens;
+import com.akto.dto.filter.MergedUrls;
 import com.akto.dto.runtime_filters.RuntimeFilter;
 import com.akto.dto.settings.DataControlSettings;
 import com.akto.dto.test_editor.YamlTemplate;
@@ -472,4 +473,8 @@ public List<YamlTemplate> fetchActiveAdvancedFilters(){
         return DbLayer.fetchActiveFilterTemplates();
     }
 
+    public Set<MergedUrls> fetchMergedUrls() {
+        return DbLayer.fetchMergedUrls();
+    }
+
 }
diff --git a/libs/utils/src/main/java/com/akto/data_actor/DbLayer.java b/libs/utils/src/main/java/com/akto/data_actor/DbLayer.java
@@ -12,9 +12,11 @@
 
 import com.akto.bulk_update_util.ApiInfoBulkUpdate;
 import com.akto.dao.*;
+import com.akto.dao.filter.MergedUrlsDao;
 import com.akto.dao.settings.DataControlSettingsDao;
 import com.akto.dependency_analyser.DependencyAnalyserUtils;
 import com.akto.dto.*;
+import com.akto.dto.filter.MergedUrls;
 import com.akto.dto.settings.DataControlSettings;
 import com.mongodb.client.model.*;
 import org.bson.conversions.Bson;
@@ -865,4 +867,8 @@ public static List<YamlTemplate> fetchActiveFilterTemplates(){
             Filters.ne(YamlTemplate.INACTIVE, false)
         );
     }
+
+    public static Set<MergedUrls> fetchMergedUrls() {
+        return MergedUrlsDao.instance.getMergedUrls();
+    }
 }
-Original file line number
+Diff line change
@@ Expand Up / @@ -26,6 +26,8 @@ public Set<MergedUrls> getMergedUrls() { @@
                 mergedUrls.add(mergedUrlsObj);
             }
+            cursor.close();
             return mergedUrls;
         }
@@ Expand Down @@