Merge pull request #1526 from akto-api-security/feature_redaction_cron

reset sample data
akto-api-security · Sep 21, 2024 · 391034b · 391034b
2 parents 97af1ce + fbcd5f7
commit 391034b
Show file tree

Hide file tree

Showing 5 changed files with 173 additions and 2 deletions.
diff --git a/apps/dashboard/src/main/java/com/akto/action/CustomDataTypeAction.java b/apps/dashboard/src/main/java/com/akto/action/CustomDataTypeAction.java
@@ -14,6 +14,7 @@
 import com.akto.parsers.HttpCallParser;
 import com.akto.util.JSONUtils;
 import com.akto.utils.AktoCustomException;
+import com.akto.utils.RedactSampleData;
 import com.fasterxml.jackson.core.JsonFactory;
 import com.fasterxml.jackson.core.JsonParser;
 import com.fasterxml.jackson.databind.JsonNode;
@@ -269,6 +270,103 @@ public String saveAktoDataType(){
         return Action.SUCCESS.toUpperCase();
     }
 
+    public String resetSampleData(){
+        try {
+            int limit = 30;
+            List<SampleData> sampleDataList = new ArrayList<>();
+            loggerMaker.infoAndAddToDb("triggered sample data redaction cron", LogDb.DASHBOARD);
+            String lastFetchedUrl = null;
+            String lastFetchedMethod = null;
+            while (true) {
+                ArrayList<WriteModel<SampleData>> bulkUpdatesForSampleData = new ArrayList<>();
+                sampleDataList = SampleDataDao.instance.fetchSampleDataPaginated(lastFetchedUrl, lastFetchedMethod, limit);
+                if (sampleDataList == null || sampleDataList.size() == 0) {
+                    break;
+                }
+
+                loggerMaker.infoAndAddToDb("Read " + sampleDataList.size() + " samples", LogDb.DASHBOARD);
+
+                for (SampleData sd: sampleDataList) {
+                    lastFetchedUrl = sd.getId().getUrl();
+                    lastFetchedMethod = sd.getId().getMethod().name();
+                    List<String> samples = sd.getSamples();
+                    if (samples == null || samples.size() == 0) {
+                        continue;
+                    }
+                    List<String> newSamples = new ArrayList<>();
+                    for (String sample: samples) {
+                        newSamples.add(RedactSampleData.redactIfRequired(sample, false, false));
+                    }
+                    Bson bson = Updates.combine(
+                        Updates.set("samples", newSamples)
+                    );
+                    Bson filters = Filters.and(
+                        Filters.eq("_id.url", sd.getId().getUrl()),
+                        Filters.eq("_id.method", sd.getId().getMethod()),
+                        Filters.eq("_id.apiCollectionId", sd.getId().getApiCollectionId())
+                    );
+                    bulkUpdatesForSampleData.add(
+                        new UpdateOneModel<>(
+                                filters,
+                                bson
+                        )
+                    );
+                }
+                if (bulkUpdatesForSampleData.size() > 0) {
+                    SampleDataDao.instance.getMCollection().bulkWrite(bulkUpdatesForSampleData);
+                }
+            }
+
+        } catch (Exception e) {
+            loggerMaker.errorAndAddToDb(e, "Error in redact data sd " + e.toString(), LogDb.DASHBOARD);
+        }
+
+        try {
+            int limit = 30;
+            List<SensitiveSampleData> sampleDataList = new ArrayList<>();
+            int skip = 0;
+            while (true) {
+                ArrayList<WriteModel<SensitiveSampleData>> bulkUpdatesForSensitiveSampleData = new ArrayList<>();
+                sampleDataList = SensitiveSampleDataDao.instance.findAll(Filters.empty(), skip, limit, null);
+                if (sampleDataList == null || sampleDataList.size() == 0) {
+                    break;
+                }
+                loggerMaker.infoAndAddToDb("Read " + sampleDataList.size() + " sensitive samples", LogDb.DASHBOARD);
+                skip+=limit;
+                for (SensitiveSampleData sd: sampleDataList) {
+                    List<String> samples = sd.getSampleData();
+                    if (samples == null || samples.size() == 0) {
+                        continue;
+                    }
+                    List<String> newSamples = new ArrayList<>();
+                    for (String sample: samples) {
+                        newSamples.add(RedactSampleData.redactIfRequired(sample, false, false));
+                    }
+                    Bson sensitiveSampleBson = Updates.combine(
+                        Updates.set("sampleData", newSamples)
+                    );
+                    Bson filters = Filters.and(
+                        Filters.eq("_id.url", sd.getId().getUrl()),
+                        Filters.eq("_id.method", sd.getId().getMethod()),
+                        Filters.eq("_id.apiCollectionId", sd.getId().getApiCollectionId())
+                    );
+                    bulkUpdatesForSensitiveSampleData.add(
+                        new UpdateOneModel<>(
+                                filters,
+                                sensitiveSampleBson
+                        )
+                    );
+                }
+                if (bulkUpdatesForSensitiveSampleData.size() > 0) {
+                    SensitiveSampleDataDao.instance.getMCollection().bulkWrite(bulkUpdatesForSensitiveSampleData);
+                }
+            }
+        } catch (Exception e) {
+            loggerMaker.errorAndAddToDb(e, "Error in redact data ssd " + e.toString(), LogDb.DASHBOARD);
+        }
+        return Action.SUCCESS.toUpperCase();
+    }
+
     public static void handleDataTypeRedaction(){
         try{
             fetchCustomDataTypes(Context.accountId.get());

diff --git a/apps/dashboard/src/main/resources/struts.xml b/apps/dashboard/src/main/resources/struts.xml
@@ -2526,6 +2526,26 @@
             </result>
         </action>
 
+        <action name="api/resetSampleData" class="com.akto.action.CustomDataTypeAction" method="resetSampleData">
+            <interceptor-ref name="json"/>
+            <interceptor-ref name="defaultStack" />
+            <interceptor-ref name="roleAccessInterceptor">
+                <param name="featureLabel">SENSITIVE_DATA</param>
+                <param name="accessType">READ_WRITE</param>
+            </interceptor-ref>
+            <result name="FORBIDDEN" type="json">
+                <param name="statusCode">403</param>
+                <param name="ignoreHierarchy">false</param>
+                <param name="includeProperties">^actionErrors.*</param>
+            </result>
+            <result name="SUCCESS"  type="json"/>
+            <result name="ERROR" type="json">
+                <param name="statusCode">422</param>
+                <param name="ignoreHierarchy">false</param>
+                <param name="includeProperties">^actionErrors.*</param>
+            </result>
+        </action>
+
         <action name="api/reviewCustomDataType" class="com.akto.action.CustomDataTypeAction" method="reviewCustomDataType">
             <interceptor-ref name="json"/>
             <interceptor-ref name="defaultStack" />

diff --git a/...eb/polaris_web/web/src/apps/dashboard/pages/observe/AllSensitiveData/AllSensitiveData.jsx b/...eb/polaris_web/web/src/apps/dashboard/pages/observe/AllSensitiveData/AllSensitiveData.jsx
@@ -1,5 +1,5 @@
 import PageWithMultipleCards from "../../../components/layouts/PageWithMultipleCards"
-import { Text, Button, Modal } from "@shopify/polaris"
+import { Text, Button, Modal, HorizontalStack } from "@shopify/polaris"
 import api from "../api"
 import { useEffect,useState } from "react"
 import func from "@/util/func"
@@ -160,6 +160,17 @@ function AllSensitiveData() {
         const activePrompts = dashboardFunc.getPrompts(requestObj)
         setPrompts(activePrompts)
     }
+
+    function resetSampleData(){
+        api.resetSampleData();
+    }
+
+    const secondaryActionsComp = (
+        <HorizontalStack gap={"2"}>
+            { (func.checkOnPrem() && window.USER_NAME.contains("razorpay")) ? <Button onClick={resetSampleData}>Reset Sample Data</Button> : <></>}
+            <Button onClick={displayGPT}>Ask AktoGPT</Button>
+        </HorizontalStack>
+    )
 
     return (
         <PageWithMultipleCards
@@ -171,7 +182,7 @@ function AllSensitiveData() {
                 />
             }
             primaryAction={<Button id={"all-data-types"} primary onClick={handleRedirect}>Create custom data types</Button>}
-            secondaryActions={<Button onClick={displayGPT}>Ask AktoGPT</Button>}
+            secondaryActions={secondaryActionsComp}
             isFirstPage={true}
             components={[
                 <GithubSimpleTable

diff --git a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/observe/api.js b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/observe/api.js
@@ -43,6 +43,13 @@ export default {
             }
         })
     },
+    resetSampleData() {
+        return request({
+            url: '/api/resetSampleData',
+            method: 'post',
+            data: {}
+        })
+    },
     async fetchSampleData(url, apiCollectionId, method) {
         const resp = await request({
             url: '/api/fetchSampleData',

diff --git a/libs/dao/src/main/java/com/akto/dao/SampleDataDao.java b/libs/dao/src/main/java/com/akto/dao/SampleDataDao.java
@@ -126,5 +126,40 @@ public List<SampleData> fetchSampleDataPaginated(int apiCollectionId, String las
         return sampleDataList;
     }
 
+    public List<SampleData> fetchSampleDataPaginated(String lastFetchedUrl,
+                                                     String lastFetchedMethod, int limit) {
+       Bson filters = Filters.empty();
+
+        if (lastFetchedUrl != null && lastFetchedMethod != null) {
+            Bson f1 = Filters.gt("_id.url", lastFetchedUrl);
+            Bson f2 = Filters.and(
+                    Filters.eq("_id.url", lastFetchedUrl),
+                    Filters.gt("_id.method", lastFetchedMethod)
+            );
+
+            filters = Filters.or(f1, f2);
+        }
+
+        Bson sort = Sorts.ascending("_id.url", "_id.method");
+
+        MongoCursor<SampleData> cursor = SampleDataDao.instance.getMCollection()
+                .find(Filters.and(filters))
+                .skip(0)
+                .limit(limit)
+                .sort(sort)
+                .cursor();
+
+        List<SampleData> sampleDataList = new ArrayList<>();
+
+        while (cursor.hasNext()) {
+            SampleData sampleData = cursor.next();
+            sampleDataList.add(sampleData);
+        }
+
+        cursor.close();
+
+        return sampleDataList;
+    }
+
 
 }