diff --git a/apps/database-abstractor/src/main/java/com/akto/action/DbAction.java b/apps/database-abstractor/src/main/java/com/akto/action/DbAction.java index 824f39e157..b7ad37d8b9 100644 --- a/apps/database-abstractor/src/main/java/com/akto/action/DbAction.java +++ b/apps/database-abstractor/src/main/java/com/akto/action/DbAction.java @@ -12,6 +12,7 @@ import com.akto.dto.billing.Tokens; import com.akto.dto.bulk_updates.BulkUpdates; import com.akto.dto.bulk_updates.UpdatePayload; +import com.akto.dto.filter.MergedUrls; import com.akto.dto.runtime_filters.RuntimeFilter; import com.akto.dto.settings.DataControlSettings; import com.akto.dto.test_editor.YamlTemplate; @@ -163,6 +164,7 @@ public void setTestSourceConfig(TestSourceConfig testSourceConfig) { String logicalGroupName; BasicDBList issuesIds; List activeAdvancedFilters; + Set mergedUrls; public BasicDBList getIssuesIds() { return issuesIds; @@ -1977,6 +1979,15 @@ public String fetchActiveAdvancedFilters(){ return Action.SUCCESS.toUpperCase(); } + public String fetchMergedUrls() { + try { + this.mergedUrls = DbLayer.fetchMergedUrls(); + } catch (Exception e) { + return Action.ERROR.toUpperCase(); + } + return Action.SUCCESS.toUpperCase(); + } + public List getCustomDataTypes() { return customDataTypes; } @@ -2883,4 +2894,11 @@ public void setActiveAdvancedFilters(List activeAdvancedFilters) { this.activeAdvancedFilters = activeAdvancedFilters; } + public Set getMergedUrls() { + return mergedUrls; + } + + public void setMergedUrls(Set mergedUrls) { + this.mergedUrls = mergedUrls; + } } diff --git a/apps/database-abstractor/src/main/resources/struts.xml b/apps/database-abstractor/src/main/resources/struts.xml index 38f82e1c29..921a13822f 100644 --- a/apps/database-abstractor/src/main/resources/struts.xml +++ b/apps/database-abstractor/src/main/resources/struts.xml @@ -1211,6 +1211,17 @@ + + + + + + 422 + false + ^actionErrors.* + + + diff --git a/apps/mini-runtime/src/main/java/com/akto/hybrid_runtime/APICatalogSync.java b/apps/mini-runtime/src/main/java/com/akto/hybrid_runtime/APICatalogSync.java index d6a49a0a39..48a75aaa55 100644 --- a/apps/mini-runtime/src/main/java/com/akto/hybrid_runtime/APICatalogSync.java +++ b/apps/mini-runtime/src/main/java/com/akto/hybrid_runtime/APICatalogSync.java @@ -7,6 +7,8 @@ import com.akto.dao.*; import com.akto.dao.context.Context; import com.akto.dao.filter.MergedUrlsDao; +import com.akto.dao.monitoring.FilterYamlTemplateDao; +import com.akto.dao.runtime_filters.AdvancedTrafficFiltersDao; import com.akto.dto.*; import com.akto.dto.bulk_updates.BulkUpdates; import com.akto.dto.bulk_updates.UpdatePayload; @@ -1132,7 +1134,7 @@ public void buildFromDB(boolean calcDiff, boolean fetchAllSTI) { loggerMaker.infoAndAddToDb("Error while clearing values in db: " + e.getMessage(), LogDb.RUNTIME); } - mergedUrls = MergedUrlsDao.instance.getMergedUrls(); + mergedUrls = dataActor.fetchMergedUrls(); aktoPolicyNew.buildFromDb(fetchAllSTI); } @@ -1259,7 +1261,7 @@ private static void buildHelper(SingleTypeInfo param, Map r private static Map build(List allParams) { Map ret = new HashMap<>(); - + for (SingleTypeInfo param: allParams) { try { buildHelper(param, ret); diff --git a/libs/dao/src/main/java/com/akto/dao/filter/MergedUrlsDao.java b/libs/dao/src/main/java/com/akto/dao/filter/MergedUrlsDao.java index cd78fde362..3d3a7ea4ee 100644 --- a/libs/dao/src/main/java/com/akto/dao/filter/MergedUrlsDao.java +++ b/libs/dao/src/main/java/com/akto/dao/filter/MergedUrlsDao.java @@ -26,6 +26,8 @@ public Set getMergedUrls() { mergedUrls.add(mergedUrlsObj); } + cursor.close(); + return mergedUrls; } diff --git a/libs/utils/src/main/java/com/akto/data_actor/ClientActor.java b/libs/utils/src/main/java/com/akto/data_actor/ClientActor.java index d1b6cd1df3..bc60a70fe2 100644 --- a/libs/utils/src/main/java/com/akto/data_actor/ClientActor.java +++ b/libs/utils/src/main/java/com/akto/data_actor/ClientActor.java @@ -1,6 +1,7 @@ package com.akto.data_actor; import com.akto.DaoInit; +import com.akto.dto.filter.MergedUrls; import com.akto.dto.settings.DataControlSettings; import com.akto.testing.ApiExecutor; import com.auth0.jwt.JWT; @@ -3128,4 +3129,38 @@ public List fetchActiveAdvancedFilters(){ return respList; } + public Set fetchMergedUrls() { + Map> headers = buildHeaders(); + + List respList = new ArrayList<>(); + OriginalHttpRequest request = new OriginalHttpRequest(url + "/fetchMergedUrls", "", "POST", "", headers, ""); + + try { + OriginalHttpResponse response = ApiExecutor.sendRequestBackOff(request, true, null, false, null); + String responsePayload = response.getBody(); + if (response.getStatusCode() != 200 || responsePayload == null) { + loggerMaker.errorAndAddToDb("non 2xx response in fetchMergedUrls", LoggerMaker.LogDb.RUNTIME); + return null; + } + BasicDBObject payloadObj; + + try { + payloadObj = BasicDBObject.parse(responsePayload); + BasicDBList newUrls = (BasicDBList) payloadObj.get("mergedUrls"); + for (Object url: newUrls) { + BasicDBObject urlObj = (BasicDBObject) url; + MergedUrls mergedUrl = objectMapper.readValue(urlObj.toJson(), MergedUrls.class); + respList.add(mergedUrl); + } + } catch (Exception e) { + loggerMaker.errorAndAddToDb("error extracting response in fetchMergedUrls" + e, LoggerMaker.LogDb.RUNTIME); + } + } catch (Exception e) { + loggerMaker.errorAndAddToDb("error in fetching merged urls: " + e, LoggerMaker.LogDb.RUNTIME); + return null; + } + + return new HashSet<>(respList); + } + } diff --git a/libs/utils/src/main/java/com/akto/data_actor/DataActor.java b/libs/utils/src/main/java/com/akto/data_actor/DataActor.java index 5b9c7499d7..d8814648ab 100644 --- a/libs/utils/src/main/java/com/akto/data_actor/DataActor.java +++ b/libs/utils/src/main/java/com/akto/data_actor/DataActor.java @@ -3,6 +3,7 @@ import com.akto.dto.*; import com.akto.dto.billing.Organization; import com.akto.dto.billing.Tokens; +import com.akto.dto.filter.MergedUrls; import com.akto.dto.runtime_filters.RuntimeFilter; import com.akto.dto.settings.DataControlSettings; import com.akto.dto.test_editor.YamlTemplate; @@ -237,4 +238,6 @@ public abstract class DataActor { public abstract void insertProtectionLog(Log log); public abstract List fetchActiveAdvancedFilters(); + + public abstract Set fetchMergedUrls(); } diff --git a/libs/utils/src/main/java/com/akto/data_actor/DbActor.java b/libs/utils/src/main/java/com/akto/data_actor/DbActor.java index 43e119f338..050b05c72b 100644 --- a/libs/utils/src/main/java/com/akto/data_actor/DbActor.java +++ b/libs/utils/src/main/java/com/akto/data_actor/DbActor.java @@ -4,6 +4,7 @@ import com.akto.dto.ApiInfo.ApiInfoKey; import com.akto.dto.billing.Organization; import com.akto.dto.billing.Tokens; +import com.akto.dto.filter.MergedUrls; import com.akto.dto.runtime_filters.RuntimeFilter; import com.akto.dto.settings.DataControlSettings; import com.akto.dto.test_editor.YamlTemplate; @@ -495,4 +496,8 @@ public List fetchActiveAdvancedFilters(){ return DbLayer.fetchActiveFilterTemplates(); } + public Set fetchMergedUrls() { + return DbLayer.fetchMergedUrls(); + } + } diff --git a/libs/utils/src/main/java/com/akto/data_actor/DbLayer.java b/libs/utils/src/main/java/com/akto/data_actor/DbLayer.java index ffadce61bc..3e3d5209c4 100644 --- a/libs/utils/src/main/java/com/akto/data_actor/DbLayer.java +++ b/libs/utils/src/main/java/com/akto/data_actor/DbLayer.java @@ -12,9 +12,11 @@ import com.akto.bulk_update_util.ApiInfoBulkUpdate; import com.akto.dao.*; +import com.akto.dao.filter.MergedUrlsDao; import com.akto.dao.settings.DataControlSettingsDao; import com.akto.dependency_analyser.DependencyAnalyserUtils; import com.akto.dto.*; +import com.akto.dto.filter.MergedUrls; import com.akto.dto.settings.DataControlSettings; import com.mongodb.BasicDBList; import com.mongodb.client.model.*; @@ -910,4 +912,8 @@ public static List fetchActiveFilterTemplates(){ Filters.ne(YamlTemplate.INACTIVE, false) ); } + + public static Set fetchMergedUrls() { + return MergedUrlsDao.instance.getMergedUrls(); + } }