diff --git a/libs/dao/src/main/java/com/akto/dao/ApiInfoDao.java b/libs/dao/src/main/java/com/akto/dao/ApiInfoDao.java index 558951b307..c3d096bc5c 100644 --- a/libs/dao/src/main/java/com/akto/dao/ApiInfoDao.java +++ b/libs/dao/src/main/java/com/akto/dao/ApiInfoDao.java @@ -3,10 +3,11 @@ import com.akto.dao.context.Context; import com.akto.dto.ApiInfo; import com.akto.dto.type.SingleTypeInfo; -import com.mongodb.client.model.CreateCollectionOptions; +import com.mongodb.client.MongoCursor; import com.mongodb.client.model.Filters; import com.mongodb.client.model.Indexes; +import org.bson.Document; import org.bson.conversions.Bson; import java.util.ArrayList; @@ -19,17 +20,46 @@ public class ApiInfoDao extends AccountsContextDao{ public void createIndicesIfAbsent() { - String dbName = Context.accountId.get()+""; - createCollectionIfAbsent(dbName, getCollName(), new CreateCollectionOptions()); + boolean exists = false; + for (String col: clients[0].getDatabase(Context.accountId.get()+"").listCollectionNames()){ + if (getCollName().equalsIgnoreCase(col)){ + exists = true; + break; + } + }; - List indices = new ArrayList<>(Arrays.asList( - Indexes.ascending(new String[] { ApiInfo.ID_API_COLLECTION_ID }), - Indexes.ascending(new String[] { ApiInfo.ID_API_COLLECTION_ID, ApiInfo.ID_URL }), - Indexes.ascending(new String[] { ApiInfo.ID_URL }), - Indexes.ascending(new String[] { SingleTypeInfo._COLLECTION_IDS, ApiInfo.ID_URL }) - )); + if (!exists) { + clients[0].getDatabase(Context.accountId.get()+"").createCollection(getCollName()); + } + + MongoCursor cursor = instance.getMCollection().listIndexes().cursor(); + int counter = 0; + while (cursor.hasNext()) { + counter++; + cursor.next(); + } + + if (counter == 1) { + String[] fieldNames = {"_id.apiCollectionId"}; + ApiInfoDao.instance.getMCollection().createIndex(Indexes.ascending(fieldNames)); + counter++; + } + + if (counter == 2) { + String[] fieldNames = {"_id.url"}; + ApiInfoDao.instance.getMCollection().createIndex(Indexes.ascending(fieldNames)); + counter++; + } + + if (counter == 3) { + String[] fieldNames = {"_id.apiCollectionId", "_id.url"}; + ApiInfoDao.instance.getMCollection().createIndex(Indexes.ascending(fieldNames)); + counter++; + } + + MCollection.createIndexIfAbsent(getDBName(), getCollName(), + new String[] { SingleTypeInfo._COLLECTION_IDS, ApiInfo.ID_URL }, true); - createIndices(indices); } @Override diff --git a/libs/dao/src/main/java/com/akto/dao/MCollection.java b/libs/dao/src/main/java/com/akto/dao/MCollection.java index 5739288f52..ea60a531ec 100644 --- a/libs/dao/src/main/java/com/akto/dao/MCollection.java +++ b/libs/dao/src/main/java/com/akto/dao/MCollection.java @@ -191,39 +191,6 @@ public Logger getLogger() { return logger; } - public void dropIndexes(List indexes){ - try { - for(Bson index: indexes) { - this.getMCollection().dropIndex(index); - } - } catch (Exception e) { - // error outs on non-existent indices - e.printStackTrace(); - } - } - - public String getIndexName(Bson idx){ - String ret = ""; - BsonDocument id = idx.toBsonDocument(); - int c = 0; - for(String key : id.keySet()){ - if(c > 0){ - ret += "_"; - } - int value = id.getInt32(key).getValue(); - ret += key + "_" + value; - c++; - } - return ret; - } - - public void createIndices(List indices){ - for (Bson index : indices) { - String name = this.getIndexName(index); - createIndexIfAbsent(this.getDBName(), this.getCollName(), index, new IndexOptions().name(name)); - } - } - public static boolean createCollectionIfAbsent(String dbName, String collName, CreateCollectionOptions options){ try{ boolean exists = false; @@ -272,4 +239,29 @@ public static boolean createIndexIfAbsent(String dbName, String collName, Bson i } + public static boolean createIndexIfAbsent(String dbName, String collName, String[] fieldNames, boolean isAscending) { + + Bson indexInfo = isAscending ? Indexes.ascending(fieldNames) : Indexes.descending(fieldNames); + + String name = ""; + + int lenPerField = 30/fieldNames.length - 1; + + for (String field: fieldNames) { + + String[] tokens = field.split("\\."); + String lastToken = tokens[tokens.length-1]; + lastToken = lastToken.substring(0, Math.min(lenPerField, lastToken.length())); + if (!name.isEmpty()) { + name += "-"; + } + name += lastToken; + } + + name += ("_"); + name += (isAscending ? "1" : "-1"); + + return createIndexIfAbsent(dbName, collName, indexInfo, new IndexOptions().name(name)); + } + } diff --git a/libs/dao/src/main/java/com/akto/dao/SampleDataDao.java b/libs/dao/src/main/java/com/akto/dao/SampleDataDao.java index 960ebdbd96..4fe17b3448 100644 --- a/libs/dao/src/main/java/com/akto/dao/SampleDataDao.java +++ b/libs/dao/src/main/java/com/akto/dao/SampleDataDao.java @@ -5,15 +5,14 @@ import com.akto.dto.traffic.SampleData; import com.akto.dto.type.SingleTypeInfo; import com.mongodb.client.MongoCursor; -import com.mongodb.client.model.CreateCollectionOptions; import com.mongodb.client.model.Filters; import com.mongodb.client.model.Indexes; import com.mongodb.client.model.Projections; import com.mongodb.client.model.Sorts; +import org.bson.Document; import org.bson.conversions.Bson; import java.util.ArrayList; -import java.util.Arrays; import java.util.List; public class SampleDataDao extends AccountsContextDao { @@ -32,16 +31,38 @@ public Class getClassT() { public void createIndicesIfAbsent() { - String dbName = Context.accountId.get() + ""; - createCollectionIfAbsent(dbName, getCollName(), new CreateCollectionOptions()); + boolean exists = false; + for (String col: clients[0].getDatabase(Context.accountId.get()+"").listCollectionNames()){ + if (getCollName().equalsIgnoreCase(col)){ + exists = true; + break; + } + }; - List indices = new ArrayList<>( - Arrays.asList( - Indexes.ascending(new String[] { ApiInfo.ID_API_COLLECTION_ID, ApiInfo.ID_URL, ApiInfo.ID_METHOD }), - Indexes.ascending(new String[] { ApiInfo.ID_API_COLLECTION_ID }), - Indexes.ascending(new String[] { SingleTypeInfo._COLLECTION_IDS, ApiInfo.ID_URL, ApiInfo.ID_METHOD }))); + if (!exists) { + clients[0].getDatabase(Context.accountId.get()+"").createCollection(getCollName()); + } + + MongoCursor cursor = instance.getMCollection().listIndexes().cursor(); + int counter = 0; + while (cursor.hasNext()) { + counter++; + cursor.next(); + } + + if (counter == 1) { + String[] fieldNames = {"_id.apiCollectionId", "_id.url", "_id.method"}; + MCollection.createIndexIfAbsent(getDBName(), getCollName(), fieldNames, true); + counter++; + } + + if (counter == 2) { + instance.getMCollection().createIndex(Indexes.ascending("_id.apiCollectionId")); + } + + MCollection.createIndexIfAbsent(getDBName(), getCollName(), + new String[] { SingleTypeInfo._COLLECTION_IDS, ApiInfo.ID_URL, ApiInfo.ID_METHOD }, true); - createIndices(indices); } public List fetchSampleDataPaginated(int apiCollectionId, String lastFetchedUrl, diff --git a/libs/dao/src/main/java/com/akto/dao/SensitiveSampleDataDao.java b/libs/dao/src/main/java/com/akto/dao/SensitiveSampleDataDao.java index 2660ebc2b7..c5fa798da4 100644 --- a/libs/dao/src/main/java/com/akto/dao/SensitiveSampleDataDao.java +++ b/libs/dao/src/main/java/com/akto/dao/SensitiveSampleDataDao.java @@ -4,13 +4,10 @@ import com.akto.dto.ApiInfo; import com.akto.dto.SensitiveSampleData; import com.akto.dto.type.SingleTypeInfo; -import com.mongodb.client.model.CreateCollectionOptions; +import com.mongodb.client.MongoCursor; import com.mongodb.client.model.Filters; import com.mongodb.client.model.Indexes; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; +import org.bson.Document; import org.bson.conversions.Bson; @@ -40,16 +37,35 @@ public static Bson getFilters(SingleTypeInfo singleTypeInfo) { } public void createIndicesIfAbsent() { + boolean exists = false; + for (String col: clients[0].getDatabase(Context.accountId.get()+"").listCollectionNames()){ + if (getCollName().equalsIgnoreCase(col)){ + exists = true; + break; + } + }; + + if (!exists) { + clients[0].getDatabase(Context.accountId.get()+"").createCollection(getCollName()); + } + + MongoCursor cursor = instance.getMCollection().listIndexes().cursor(); + int counter = 0; + while (cursor.hasNext()) { + counter++; + cursor.next(); + } + + if (counter == 1) { + String[] fieldNames = {"_id.url", "_id.apiCollectionId", "_id.method"}; + MCollection.createIndexIfAbsent(getDBName(), getCollName(), fieldNames, true); + } - String dbName = Context.accountId.get() + ""; - createCollectionIfAbsent(dbName, getCollName(), new CreateCollectionOptions()); + MCollection.createIndexIfAbsent(getDBName(), getCollName(), + new String[] { ApiInfo.ID_URL, SingleTypeInfo._COLLECTION_IDS, ApiInfo.ID_METHOD }, true); - List indices = new ArrayList<>(Arrays.asList( - Indexes.ascending(new String[] { ApiInfo.ID_URL, ApiInfo.ID_API_COLLECTION_ID, ApiInfo.ID_METHOD }), - Indexes.ascending(new String[] { ApiInfo.ID_URL, SingleTypeInfo._COLLECTION_IDS, ApiInfo.ID_METHOD }), - Indexes.ascending(new String[] { SingleTypeInfo._COLLECTION_IDS }) - )); + MCollection.createIndexIfAbsent(getDBName(), getCollName(), + new String[] { SingleTypeInfo._COLLECTION_IDS }, true); - createIndices(indices); } } diff --git a/libs/dao/src/main/java/com/akto/dao/SingleTypeInfoDao.java b/libs/dao/src/main/java/com/akto/dao/SingleTypeInfoDao.java index 38efd221dc..89523d68e4 100644 --- a/libs/dao/src/main/java/com/akto/dao/SingleTypeInfoDao.java +++ b/libs/dao/src/main/java/com/akto/dao/SingleTypeInfoDao.java @@ -37,20 +37,59 @@ public Class getClassT() { public void createIndicesIfAbsent() { - String dbName = Context.accountId.get() + ""; - createCollectionIfAbsent(dbName, getCollName(), new CreateCollectionOptions()); - - List indices = new ArrayList<>(Arrays.asList( - Indexes.ascending(new String[] { SingleTypeInfo._URL, SingleTypeInfo._METHOD, SingleTypeInfo._RESPONSE_CODE,SingleTypeInfo._IS_HEADER, SingleTypeInfo._PARAM, SingleTypeInfo.SUB_TYPE, SingleTypeInfo._API_COLLECTION_ID }), - Indexes.ascending(new String[] { SingleTypeInfo._API_COLLECTION_ID }), - Indexes.ascending(new String[] { SingleTypeInfo._PARAM, SingleTypeInfo._API_COLLECTION_ID }), - Indexes.ascending(new String[] { SingleTypeInfo._RESPONSE_CODE, SingleTypeInfo._IS_HEADER, SingleTypeInfo._PARAM, SingleTypeInfo.SUB_TYPE, SingleTypeInfo._API_COLLECTION_ID }), - Indexes.ascending(new String[] { SingleTypeInfo.SUB_TYPE, SingleTypeInfo._RESPONSE_CODE }), - Indexes.ascending(new String[] { SingleTypeInfo._RESPONSE_CODE, SingleTypeInfo.SUB_TYPE, SingleTypeInfo._TIMESTAMP }), - Indexes.ascending(new String[] { SingleTypeInfo._COLLECTION_IDS }) - )); - - createIndices(indices); + boolean exists = false; + for (String col: clients[0].getDatabase(Context.accountId.get()+"").listCollectionNames()){ + if (getCollName().equalsIgnoreCase(col)){ + exists = true; + break; + } + }; + + if (!exists) { + clients[0].getDatabase(Context.accountId.get()+"").createCollection(getCollName()); + } + + MongoCursor cursor = instance.getMCollection().listIndexes().cursor(); + int counter = 0; + while (cursor.hasNext()) { + counter++; + cursor.next(); + } + + if (counter == 1) { + String[] fieldNames = {"url", "method", "responseCode", "isHeader", "param", "subType", "apiCollectionId"}; + MCollection.createIndexIfAbsent(getDBName(), getCollName(), fieldNames, true); + } + + if (counter == 2) { + SingleTypeInfoDao.instance.getMCollection().createIndex(Indexes.ascending(new String[]{"apiCollectionId"})); + counter++; + } + + if (counter == 3) { + SingleTypeInfoDao.instance.getMCollection().createIndex(Indexes.ascending(new String[]{"param", "apiCollectionId"})); + counter++; + } + + if (counter == 4) { + String[] fieldNames = new String[]{SingleTypeInfo._RESPONSE_CODE, SingleTypeInfo._IS_HEADER, SingleTypeInfo._PARAM, SingleTypeInfo.SUB_TYPE, SingleTypeInfo._API_COLLECTION_ID}; + MCollection.createIndexIfAbsent(getDBName(), getCollName(), fieldNames, true); + counter++; + } + + if (counter == 5) { + SingleTypeInfoDao.instance.getMCollection().createIndex(Indexes.ascending(new String[]{SingleTypeInfo.SUB_TYPE, SingleTypeInfo._RESPONSE_CODE})); + counter++; + } + + if (counter == 6) { + String[] fieldNames = {"responseCode", "subType", "timestamp",}; + SingleTypeInfoDao.instance.getMCollection().createIndex(Indexes.ascending(fieldNames)); + } + + MCollection.createIndexIfAbsent(getDBName(), getCollName(), + new String[] { SingleTypeInfo._COLLECTION_IDS }, true); + } public static Bson filterForHostHeader(int apiCollectionId, boolean useApiCollectionId) { diff --git a/libs/dao/src/main/java/com/akto/dao/traffic_metrics/TrafficMetricsDao.java b/libs/dao/src/main/java/com/akto/dao/traffic_metrics/TrafficMetricsDao.java index 22377a12d7..cb8ce08a2a 100644 --- a/libs/dao/src/main/java/com/akto/dao/traffic_metrics/TrafficMetricsDao.java +++ b/libs/dao/src/main/java/com/akto/dao/traffic_metrics/TrafficMetricsDao.java @@ -1,6 +1,7 @@ package com.akto.dao.traffic_metrics; import com.akto.dao.AccountsContextDao; +import com.akto.dao.MCollection; import com.akto.dao.SingleTypeInfoDao; import com.akto.dao.context.Context; import com.akto.dto.traffic_metrics.TrafficMetrics; @@ -70,7 +71,7 @@ public void createIndicesIfAbsent() { ID+ TrafficMetrics.Key.HOST, ID+ TrafficMetrics.Key.VXLAN_ID, }; - instance.getMCollection().createIndex(Indexes.ascending(fieldNames)); + MCollection.createIndexIfAbsent(getDBName(), getCollName(), fieldNames, true); }