Merge pull request #1529 from akto-api-security/feature/bloom_filter_…

…for_dict feat: added bloomfilter implementation for mini-runtime
akto-api-security · Sep 21, 2024 · 9534801 · 9534801
2 parents 391034b + 4d9fda4
commit 9534801
Show file tree

Hide file tree

Showing 6 changed files with 55 additions and 5 deletions.
diff --git a/apps/mini-runtime/src/main/java/com/akto/hybrid_runtime/APICatalogSync.java b/apps/mini-runtime/src/main/java/com/akto/hybrid_runtime/APICatalogSync.java
@@ -1117,7 +1117,7 @@ public void buildFromDB(boolean calcDiff, boolean fetchAllSTI) {
             loggerMaker.infoAndAddToDb("Error while clearing values in db: " + e.getMessage(), LogDb.RUNTIME);
         }
 
-        mergedUrls = MergedUrlsDao.instance.getMergedUrls();
+        mergedUrls = dataActor.fetchMergedUrls();
 
         aktoPolicyNew.buildFromDb(fetchAllSTI);
     }

diff --git a/libs/dao/src/main/java/com/akto/dao/filter/MergedUrlsDao.java b/libs/dao/src/main/java/com/akto/dao/filter/MergedUrlsDao.java
@@ -26,6 +26,8 @@ public Set<MergedUrls> getMergedUrls() {
             mergedUrls.add(mergedUrlsObj);
         }
 
+        cursor.close();
+
         return mergedUrls;
     }
 

diff --git a/libs/utils/src/main/java/com/akto/data_actor/ClientActor.java b/libs/utils/src/main/java/com/akto/data_actor/ClientActor.java
@@ -1,5 +1,6 @@
 package com.akto.data_actor;
 
+import com.akto.dto.filter.MergedUrls;
 import com.akto.testing.ApiExecutor;
 import com.auth0.jwt.JWT;
 import com.auth0.jwt.interfaces.DecodedJWT;
@@ -1307,4 +1308,38 @@ public List<YamlTemplate> fetchFilterYamlTemplates() {
         }
         return templates;
     }
+
+    public Set<MergedUrls> fetchMergedUrls() {
+        Map<String, List<String>> headers = buildHeaders();
+
+        List<MergedUrls> respList = new ArrayList<>();
+        OriginalHttpRequest request = new OriginalHttpRequest(url + "/fetchMergedUrls", "", "POST", "", headers, "");
+
+        try {
+            OriginalHttpResponse response = ApiExecutor.sendRequest(request, true, null, false, null);
+            String responsePayload = response.getBody();
+            if (response.getStatusCode() != 200 || responsePayload == null) {
+                loggerMaker.errorAndAddToDb("non 2xx response in fetchMergedUrls", LoggerMaker.LogDb.RUNTIME);
+                return null;
+            }
+            BasicDBObject payloadObj;
+
+            try {
+                payloadObj = BasicDBObject.parse(responsePayload);
+                BasicDBList newUrls = (BasicDBList) payloadObj.get("mergedUrls");
+                for (Object url: newUrls) {
+                    BasicDBObject urlObj = (BasicDBObject) url;
+                    MergedUrls mergedUrl = objectMapper.readValue(urlObj.toJson(), MergedUrls.class);
+                    respList.add(mergedUrl);
+                }
+            } catch (Exception e) {
+                loggerMaker.errorAndAddToDb("error extracting response in fetchMergedUrls" + e, LoggerMaker.LogDb.RUNTIME);
+            }
+        } catch (Exception e) {
+            loggerMaker.errorAndAddToDb("error in fetching merged urls: " + e, LoggerMaker.LogDb.RUNTIME);
+            return null;
+        }
+
+        return new HashSet<>(respList);
+    }
 }
diff --git a/libs/utils/src/main/java/com/akto/data_actor/DataActor.java b/libs/utils/src/main/java/com/akto/data_actor/DataActor.java
@@ -2,11 +2,13 @@
 
 import com.akto.dto.*;
 import com.akto.dto.billing.Organization;
+import com.akto.dto.filter.MergedUrls;
 import com.akto.dto.runtime_filters.RuntimeFilter;
 import com.akto.dto.test_editor.YamlTemplate;
 import com.akto.dto.type.SingleTypeInfo;
 
 import java.util.List;
+import java.util.Set;
 
 public abstract class DataActor {
 
@@ -89,4 +91,6 @@ public abstract class DataActor {
 
     public abstract void syncExtractedAPIs( CodeAnalysisRepo codeAnalysisRepo   , List<CodeAnalysisApi> codeAnalysisApisList, boolean isLastBatch);
     public abstract void updateRepoLastRun( CodeAnalysisRepo codeAnalysisRepo);
+
+    public abstract Set<MergedUrls> fetchMergedUrls();
 }
diff --git a/libs/utils/src/main/java/com/akto/data_actor/DbActor.java b/libs/utils/src/main/java/com/akto/data_actor/DbActor.java
@@ -2,6 +2,7 @@
 
 import com.akto.dto.*;
 import com.akto.dto.billing.Organization;
+import com.akto.dto.filter.MergedUrls;
 import com.akto.dto.runtime_filters.RuntimeFilter;
 import com.akto.dto.test_editor.YamlTemplate;
 import com.akto.dto.traffic.SampleData;
@@ -14,6 +15,7 @@
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
+import java.util.Set;
 
 public class DbActor extends DataActor {
 
@@ -218,4 +220,8 @@ public void syncExtractedAPIs( CodeAnalysisRepo codeAnalysisRepo, List<CodeAnaly
     public void updateRepoLastRun(CodeAnalysisRepo codeAnalysisRepo) {
         return;
     }
+
+    public Set<MergedUrls> fetchMergedUrls() {
+        return DbLayer.fetchMergedUrls();
+    }
 }
diff --git a/libs/utils/src/main/java/com/akto/data_actor/DbLayer.java b/libs/utils/src/main/java/com/akto/data_actor/DbLayer.java
@@ -1,11 +1,10 @@
 package com.akto.data_actor;
 
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.HashSet;
-import java.util.List;
+import java.util.*;
 
 import com.akto.bulk_update_util.ApiInfoBulkUpdate;
+import com.akto.dao.filter.MergedUrlsDao;
+import com.akto.dto.filter.MergedUrls;
 import org.bson.conversions.Bson;
 import org.bson.types.ObjectId;
 
@@ -390,4 +389,8 @@ public static void insertTestingLog(Log log) {
     public static void insertProtectionLog(Log log) {
         ProtectionLogsDao.instance.insertOne(log);
     }
+
+    public static Set<MergedUrls> fetchMergedUrls() {
+        return MergedUrlsDao.instance.getMergedUrls();
+    }
 }
-Original file line number
+Diff line change
@@ Expand Up / @@ -26,6 +26,8 @@ public Set<MergedUrls> getMergedUrls() { @@
                 mergedUrls.add(mergedUrlsObj);
             }
+            cursor.close();
             return mergedUrls;
         }
@@ Expand Down @@