Skip to content

Commit

Permalink
Merge pull request #726 from akto-api-security/feature/fix_index_crea…
Browse files Browse the repository at this point in the history 
…tion

Feature/fix index creation
  • Loading branch information
@notshivansh
notshivansh authored Nov 18, 2023
2 parents 8dd2dbd + 69cacfa commit ff4935c
Show file tree
Hide file tree
Showing 6 changed files with 180 additions and 81 deletions.
50 changes: 40 additions & 10 deletions libs/dao/src/main/java/com/akto/dao/ApiInfoDao.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,10 +3,11 @@
import com.akto.dao.context.Context;
import com.akto.dto.ApiInfo;
import com.akto.dto.type.SingleTypeInfo;
import com.mongodb.client.model.CreateCollectionOptions;
import com.mongodb.client.MongoCursor;
import com.mongodb.client.model.Filters;
import com.mongodb.client.model.Indexes;

import org.bson.Document;
import org.bson.conversions.Bson;

import java.util.ArrayList;
Expand All @@ -19,17 +20,46 @@ public class ApiInfoDao extends AccountsContextDao<ApiInfo>{

public void createIndicesIfAbsent() {

String dbName = Context.accountId.get()+"";
createCollectionIfAbsent(dbName, getCollName(), new CreateCollectionOptions());
boolean exists = false;
for (String col: clients[0].getDatabase(Context.accountId.get()+"").listCollectionNames()){
if (getCollName().equalsIgnoreCase(col)){
exists = true;
break;
}
};

List<Bson> indices = new ArrayList<>(Arrays.asList(
Indexes.ascending(new String[] { ApiInfo.ID_API_COLLECTION_ID }),
Indexes.ascending(new String[] { ApiInfo.ID_API_COLLECTION_ID, ApiInfo.ID_URL }),
Indexes.ascending(new String[] { ApiInfo.ID_URL }),
Indexes.ascending(new String[] { SingleTypeInfo._COLLECTION_IDS, ApiInfo.ID_URL })
));
if (!exists) {
clients[0].getDatabase(Context.accountId.get()+"").createCollection(getCollName());
}

MongoCursor<Document> cursor = instance.getMCollection().listIndexes().cursor();
int counter = 0;
while (cursor.hasNext()) {
counter++;
cursor.next();
}

if (counter == 1) {
String[] fieldNames = {"_id.apiCollectionId"};
ApiInfoDao.instance.getMCollection().createIndex(Indexes.ascending(fieldNames));
counter++;
}

if (counter == 2) {
String[] fieldNames = {"_id.url"};
ApiInfoDao.instance.getMCollection().createIndex(Indexes.ascending(fieldNames));
counter++;
}

if (counter == 3) {
String[] fieldNames = {"_id.apiCollectionId", "_id.url"};
ApiInfoDao.instance.getMCollection().createIndex(Indexes.ascending(fieldNames));
counter++;
}

MCollection.createIndexIfAbsent(getDBName(), getCollName(),
new String[] { SingleTypeInfo._COLLECTION_IDS, ApiInfo.ID_URL }, true);

createIndices(indices);
}

@Override
Expand Down
58 changes: 25 additions & 33 deletions libs/dao/src/main/java/com/akto/dao/MCollection.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -191,39 +191,6 @@ public Logger getLogger() {
return logger;
}

public void dropIndexes(List<Bson> indexes){
try {
for(Bson index: indexes) {
this.getMCollection().dropIndex(index);
}
} catch (Exception e) {
// error outs on non-existent indices
e.printStackTrace();
}
}

public String getIndexName(Bson idx){
String ret = "";
BsonDocument id = idx.toBsonDocument();
int c = 0;
for(String key : id.keySet()){
if(c > 0){
ret += "_";
}
int value = id.getInt32(key).getValue();
ret += key + "_" + value;
c++;
}
return ret;
}

public void createIndices(List<Bson> indices){
for (Bson index : indices) {
String name = this.getIndexName(index);
createIndexIfAbsent(this.getDBName(), this.getCollName(), index, new IndexOptions().name(name));
}
}

public static boolean createCollectionIfAbsent(String dbName, String collName, CreateCollectionOptions options){
try{
boolean exists = false;
Expand Down Expand Up @@ -272,4 +239,29 @@ public static boolean createIndexIfAbsent(String dbName, String collName, Bson i

}

public static boolean createIndexIfAbsent(String dbName, String collName, String[] fieldNames, boolean isAscending) {

Bson indexInfo = isAscending ? Indexes.ascending(fieldNames) : Indexes.descending(fieldNames);

String name = "";

int lenPerField = 30/fieldNames.length - 1;

for (String field: fieldNames) {

String[] tokens = field.split("\\.");
String lastToken = tokens[tokens.length-1];
lastToken = lastToken.substring(0, Math.min(lenPerField, lastToken.length()));
if (!name.isEmpty()) {
name += "-";
}
name += lastToken;
}

name += ("_");
name += (isAscending ? "1" : "-1");

return createIndexIfAbsent(dbName, collName, indexInfo, new IndexOptions().name(name));
}

}
41 changes: 31 additions & 10 deletions libs/dao/src/main/java/com/akto/dao/SampleDataDao.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,15 +5,14 @@
import com.akto.dto.traffic.SampleData;
import com.akto.dto.type.SingleTypeInfo;
import com.mongodb.client.MongoCursor;
import com.mongodb.client.model.CreateCollectionOptions;
import com.mongodb.client.model.Filters;
import com.mongodb.client.model.Indexes;
import com.mongodb.client.model.Projections;
import com.mongodb.client.model.Sorts;
import org.bson.Document;
import org.bson.conversions.Bson;

import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;

public class SampleDataDao extends AccountsContextDao<SampleData> {
Expand All @@ -32,16 +31,38 @@ public Class<SampleData> getClassT() {

public void createIndicesIfAbsent() {

String dbName = Context.accountId.get() + "";
createCollectionIfAbsent(dbName, getCollName(), new CreateCollectionOptions());
boolean exists = false;
for (String col: clients[0].getDatabase(Context.accountId.get()+"").listCollectionNames()){
if (getCollName().equalsIgnoreCase(col)){
exists = true;
break;
}
};

List<Bson> indices = new ArrayList<>(
Arrays.asList(
Indexes.ascending(new String[] { ApiInfo.ID_API_COLLECTION_ID, ApiInfo.ID_URL, ApiInfo.ID_METHOD }),
Indexes.ascending(new String[] { ApiInfo.ID_API_COLLECTION_ID }),
Indexes.ascending(new String[] { SingleTypeInfo._COLLECTION_IDS, ApiInfo.ID_URL, ApiInfo.ID_METHOD })));
if (!exists) {
clients[0].getDatabase(Context.accountId.get()+"").createCollection(getCollName());
}

MongoCursor<Document> cursor = instance.getMCollection().listIndexes().cursor();
int counter = 0;
while (cursor.hasNext()) {
counter++;
cursor.next();
}

if (counter == 1) {
String[] fieldNames = {"_id.apiCollectionId", "_id.url", "_id.method"};
MCollection.createIndexIfAbsent(getDBName(), getCollName(), fieldNames, true);
counter++;
}

if (counter == 2) {
instance.getMCollection().createIndex(Indexes.ascending("_id.apiCollectionId"));
}

MCollection.createIndexIfAbsent(getDBName(), getCollName(),
new String[] { SingleTypeInfo._COLLECTION_IDS, ApiInfo.ID_URL, ApiInfo.ID_METHOD }, true);

createIndices(indices);
}

public List<SampleData> fetchSampleDataPaginated(int apiCollectionId, String lastFetchedUrl,
Expand Down
42 changes: 29 additions & 13 deletions libs/dao/src/main/java/com/akto/dao/SensitiveSampleDataDao.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,13 +4,10 @@
import com.akto.dto.ApiInfo;
import com.akto.dto.SensitiveSampleData;
import com.akto.dto.type.SingleTypeInfo;
import com.mongodb.client.model.CreateCollectionOptions;
import com.mongodb.client.MongoCursor;
import com.mongodb.client.model.Filters;
import com.mongodb.client.model.Indexes;

import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import org.bson.Document;

import org.bson.conversions.Bson;

Expand Down Expand Up @@ -40,16 +37,35 @@ public static Bson getFilters(SingleTypeInfo singleTypeInfo) {
}

public void createIndicesIfAbsent() {
boolean exists = false;
for (String col: clients[0].getDatabase(Context.accountId.get()+"").listCollectionNames()){
if (getCollName().equalsIgnoreCase(col)){
exists = true;
break;
}
};

if (!exists) {
clients[0].getDatabase(Context.accountId.get()+"").createCollection(getCollName());
}

MongoCursor<Document> cursor = instance.getMCollection().listIndexes().cursor();
int counter = 0;
while (cursor.hasNext()) {
counter++;
cursor.next();
}

if (counter == 1) {
String[] fieldNames = {"_id.url", "_id.apiCollectionId", "_id.method"};
MCollection.createIndexIfAbsent(getDBName(), getCollName(), fieldNames, true);
}

String dbName = Context.accountId.get() + "";
createCollectionIfAbsent(dbName, getCollName(), new CreateCollectionOptions());
MCollection.createIndexIfAbsent(getDBName(), getCollName(),
new String[] { ApiInfo.ID_URL, SingleTypeInfo._COLLECTION_IDS, ApiInfo.ID_METHOD }, true);

List<Bson> indices = new ArrayList<>(Arrays.asList(
Indexes.ascending(new String[] { ApiInfo.ID_URL, ApiInfo.ID_API_COLLECTION_ID, ApiInfo.ID_METHOD }),
Indexes.ascending(new String[] { ApiInfo.ID_URL, SingleTypeInfo._COLLECTION_IDS, ApiInfo.ID_METHOD }),
Indexes.ascending(new String[] { SingleTypeInfo._COLLECTION_IDS })
));
MCollection.createIndexIfAbsent(getDBName(), getCollName(),
new String[] { SingleTypeInfo._COLLECTION_IDS }, true);

createIndices(indices);
}
}
67 changes: 53 additions & 14 deletions libs/dao/src/main/java/com/akto/dao/SingleTypeInfoDao.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,20 +37,59 @@ public Class<SingleTypeInfo> getClassT() {

public void createIndicesIfAbsent() {

String dbName = Context.accountId.get() + "";
createCollectionIfAbsent(dbName, getCollName(), new CreateCollectionOptions());

List<Bson> indices = new ArrayList<>(Arrays.asList(
Indexes.ascending(new String[] { SingleTypeInfo._URL, SingleTypeInfo._METHOD, SingleTypeInfo._RESPONSE_CODE,SingleTypeInfo._IS_HEADER, SingleTypeInfo._PARAM, SingleTypeInfo.SUB_TYPE, SingleTypeInfo._API_COLLECTION_ID }),
Indexes.ascending(new String[] { SingleTypeInfo._API_COLLECTION_ID }),
Indexes.ascending(new String[] { SingleTypeInfo._PARAM, SingleTypeInfo._API_COLLECTION_ID }),
Indexes.ascending(new String[] { SingleTypeInfo._RESPONSE_CODE, SingleTypeInfo._IS_HEADER, SingleTypeInfo._PARAM, SingleTypeInfo.SUB_TYPE, SingleTypeInfo._API_COLLECTION_ID }),
Indexes.ascending(new String[] { SingleTypeInfo.SUB_TYPE, SingleTypeInfo._RESPONSE_CODE }),
Indexes.ascending(new String[] { SingleTypeInfo._RESPONSE_CODE, SingleTypeInfo.SUB_TYPE, SingleTypeInfo._TIMESTAMP }),
Indexes.ascending(new String[] { SingleTypeInfo._COLLECTION_IDS })
));

createIndices(indices);
boolean exists = false;
for (String col: clients[0].getDatabase(Context.accountId.get()+"").listCollectionNames()){
if (getCollName().equalsIgnoreCase(col)){
exists = true;
break;
}
};

if (!exists) {
clients[0].getDatabase(Context.accountId.get()+"").createCollection(getCollName());
}

MongoCursor<Document> cursor = instance.getMCollection().listIndexes().cursor();
int counter = 0;
while (cursor.hasNext()) {
counter++;
cursor.next();
}

if (counter == 1) {
String[] fieldNames = {"url", "method", "responseCode", "isHeader", "param", "subType", "apiCollectionId"};
MCollection.createIndexIfAbsent(getDBName(), getCollName(), fieldNames, true);
}

if (counter == 2) {
SingleTypeInfoDao.instance.getMCollection().createIndex(Indexes.ascending(new String[]{"apiCollectionId"}));
counter++;
}

if (counter == 3) {
SingleTypeInfoDao.instance.getMCollection().createIndex(Indexes.ascending(new String[]{"param", "apiCollectionId"}));
counter++;
}

if (counter == 4) {
String[] fieldNames = new String[]{SingleTypeInfo._RESPONSE_CODE, SingleTypeInfo._IS_HEADER, SingleTypeInfo._PARAM, SingleTypeInfo.SUB_TYPE, SingleTypeInfo._API_COLLECTION_ID};
MCollection.createIndexIfAbsent(getDBName(), getCollName(), fieldNames, true);
counter++;
}

if (counter == 5) {
SingleTypeInfoDao.instance.getMCollection().createIndex(Indexes.ascending(new String[]{SingleTypeInfo.SUB_TYPE, SingleTypeInfo._RESPONSE_CODE}));
counter++;
}

if (counter == 6) {
String[] fieldNames = {"responseCode", "subType", "timestamp",};
SingleTypeInfoDao.instance.getMCollection().createIndex(Indexes.ascending(fieldNames));
}

MCollection.createIndexIfAbsent(getDBName(), getCollName(),
new String[] { SingleTypeInfo._COLLECTION_IDS }, true);

}

public static Bson filterForHostHeader(int apiCollectionId, boolean useApiCollectionId) {
Expand Down
3 changes: 2 additions & 1 deletion libs/dao/src/main/java/com/akto/dao/traffic_metrics/TrafficMetricsDao.java
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package com.akto.dao.traffic_metrics;

import com.akto.dao.AccountsContextDao;
import com.akto.dao.MCollection;
import com.akto.dao.SingleTypeInfoDao;
import com.akto.dao.context.Context;
import com.akto.dto.traffic_metrics.TrafficMetrics;
Expand Down Expand Up @@ -70,7 +71,7 @@ public void createIndicesIfAbsent() {
ID+ TrafficMetrics.Key.HOST,
ID+ TrafficMetrics.Key.VXLAN_ID,
};
instance.getMCollection().createIndex(Indexes.ascending(fieldNames));
MCollection.createIndexIfAbsent(getDBName(), getCollName(), fieldNames, true);
}


Expand Down

0 comments on commit ff4935c

Please sign in to comment.