diff --git a/apps/mini-runtime/src/main/java/com/akto/hybrid_runtime/APICatalogSync.java b/apps/mini-runtime/src/main/java/com/akto/hybrid_runtime/APICatalogSync.java index f5a8cdba1d..c1ba13af1b 100644 --- a/apps/mini-runtime/src/main/java/com/akto/hybrid_runtime/APICatalogSync.java +++ b/apps/mini-runtime/src/main/java/com/akto/hybrid_runtime/APICatalogSync.java @@ -1117,7 +1117,7 @@ public void buildFromDB(boolean calcDiff, boolean fetchAllSTI) { loggerMaker.infoAndAddToDb("Error while clearing values in db: " + e.getMessage(), LogDb.RUNTIME); } - mergedUrls = MergedUrlsDao.instance.getMergedUrls(); + mergedUrls = dataActor.fetchMergedUrls(); aktoPolicyNew.buildFromDb(fetchAllSTI); } diff --git a/libs/dao/src/main/java/com/akto/dao/filter/MergedUrlsDao.java b/libs/dao/src/main/java/com/akto/dao/filter/MergedUrlsDao.java index 028cdaf66f..a463b476c1 100644 --- a/libs/dao/src/main/java/com/akto/dao/filter/MergedUrlsDao.java +++ b/libs/dao/src/main/java/com/akto/dao/filter/MergedUrlsDao.java @@ -26,6 +26,8 @@ public Set getMergedUrls() { mergedUrls.add(mergedUrlsObj); } + cursor.close(); + return mergedUrls; } diff --git a/libs/utils/src/main/java/com/akto/data_actor/ClientActor.java b/libs/utils/src/main/java/com/akto/data_actor/ClientActor.java index 5323417def..bbd3b5c454 100644 --- a/libs/utils/src/main/java/com/akto/data_actor/ClientActor.java +++ b/libs/utils/src/main/java/com/akto/data_actor/ClientActor.java @@ -1,5 +1,6 @@ package com.akto.data_actor; +import com.akto.dto.filter.MergedUrls; import com.akto.testing.ApiExecutor; import com.auth0.jwt.JWT; import com.auth0.jwt.interfaces.DecodedJWT; @@ -1307,4 +1308,38 @@ public List fetchFilterYamlTemplates() { } return templates; } + + public Set fetchMergedUrls() { + Map> headers = buildHeaders(); + + List respList = new ArrayList<>(); + OriginalHttpRequest request = new OriginalHttpRequest(url + "/fetchMergedUrls", "", "POST", "", headers, ""); + + try { + OriginalHttpResponse response = ApiExecutor.sendRequest(request, true, null, false, null); + String responsePayload = response.getBody(); + if (response.getStatusCode() != 200 || responsePayload == null) { + loggerMaker.errorAndAddToDb("non 2xx response in fetchMergedUrls", LoggerMaker.LogDb.RUNTIME); + return null; + } + BasicDBObject payloadObj; + + try { + payloadObj = BasicDBObject.parse(responsePayload); + BasicDBList newUrls = (BasicDBList) payloadObj.get("mergedUrls"); + for (Object url: newUrls) { + BasicDBObject urlObj = (BasicDBObject) url; + MergedUrls mergedUrl = objectMapper.readValue(urlObj.toJson(), MergedUrls.class); + respList.add(mergedUrl); + } + } catch (Exception e) { + loggerMaker.errorAndAddToDb("error extracting response in fetchMergedUrls" + e, LoggerMaker.LogDb.RUNTIME); + } + } catch (Exception e) { + loggerMaker.errorAndAddToDb("error in fetching merged urls: " + e, LoggerMaker.LogDb.RUNTIME); + return null; + } + + return new HashSet<>(respList); + } } diff --git a/libs/utils/src/main/java/com/akto/data_actor/DataActor.java b/libs/utils/src/main/java/com/akto/data_actor/DataActor.java index e61349b027..293692ba30 100644 --- a/libs/utils/src/main/java/com/akto/data_actor/DataActor.java +++ b/libs/utils/src/main/java/com/akto/data_actor/DataActor.java @@ -2,11 +2,13 @@ import com.akto.dto.*; import com.akto.dto.billing.Organization; +import com.akto.dto.filter.MergedUrls; import com.akto.dto.runtime_filters.RuntimeFilter; import com.akto.dto.test_editor.YamlTemplate; import com.akto.dto.type.SingleTypeInfo; import java.util.List; +import java.util.Set; public abstract class DataActor { @@ -89,4 +91,6 @@ public abstract class DataActor { public abstract void syncExtractedAPIs( CodeAnalysisRepo codeAnalysisRepo , List codeAnalysisApisList, boolean isLastBatch); public abstract void updateRepoLastRun( CodeAnalysisRepo codeAnalysisRepo); + + public abstract Set fetchMergedUrls(); } diff --git a/libs/utils/src/main/java/com/akto/data_actor/DbActor.java b/libs/utils/src/main/java/com/akto/data_actor/DbActor.java index 22f795a0df..93d8652c80 100644 --- a/libs/utils/src/main/java/com/akto/data_actor/DbActor.java +++ b/libs/utils/src/main/java/com/akto/data_actor/DbActor.java @@ -2,6 +2,7 @@ import com.akto.dto.*; import com.akto.dto.billing.Organization; +import com.akto.dto.filter.MergedUrls; import com.akto.dto.runtime_filters.RuntimeFilter; import com.akto.dto.test_editor.YamlTemplate; import com.akto.dto.traffic.SampleData; @@ -14,6 +15,7 @@ import java.util.ArrayList; import java.util.Collections; import java.util.List; +import java.util.Set; public class DbActor extends DataActor { @@ -218,4 +220,8 @@ public void syncExtractedAPIs( CodeAnalysisRepo codeAnalysisRepo, List fetchMergedUrls() { + return DbLayer.fetchMergedUrls(); + } } diff --git a/libs/utils/src/main/java/com/akto/data_actor/DbLayer.java b/libs/utils/src/main/java/com/akto/data_actor/DbLayer.java index 6bf423f874..4f9a530c64 100644 --- a/libs/utils/src/main/java/com/akto/data_actor/DbLayer.java +++ b/libs/utils/src/main/java/com/akto/data_actor/DbLayer.java @@ -1,11 +1,10 @@ package com.akto.data_actor; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashSet; -import java.util.List; +import java.util.*; import com.akto.bulk_update_util.ApiInfoBulkUpdate; +import com.akto.dao.filter.MergedUrlsDao; +import com.akto.dto.filter.MergedUrls; import org.bson.conversions.Bson; import org.bson.types.ObjectId; @@ -390,4 +389,8 @@ public static void insertTestingLog(Log log) { public static void insertProtectionLog(Log log) { ProtectionLogsDao.instance.insertOne(log); } + + public static Set fetchMergedUrls() { + return MergedUrlsDao.instance.getMergedUrls(); + } }