From b205037c1024dadcd650e93872a54c7e0a2fc7ec Mon Sep 17 00:00:00 2001 From: Ankush Jain Date: Wed, 17 Jul 2024 01:32:03 -0700 Subject: [PATCH 1/5] improve changes graph --- .../dashboard/components/charts/LineChart.jsx | 138 ++++++++++++++++++ .../observe/api_collections/ApiChanges.jsx | 24 +-- .../apps/dashboard/pages/observe/transform.js | 3 +- 3 files changed, 146 insertions(+), 19 deletions(-) create mode 100644 apps/dashboard/web/polaris_web/web/src/apps/dashboard/components/charts/LineChart.jsx diff --git a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/components/charts/LineChart.jsx b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/components/charts/LineChart.jsx new file mode 100644 index 0000000000..0c4e9253d9 --- /dev/null +++ b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/components/charts/LineChart.jsx @@ -0,0 +1,138 @@ +import HighchartsReact from "highcharts-react-official" +import Highcharts from "highcharts" +import { useEffect, useRef, useState } from "react"; +import func from "../../../../util/func"; +import SpinnerCentered from "../progress/SpinnerCentered"; + +function LineChart(props) { + + const { type, height, backgroundColor, data, graphPointClick, tooltipFormatter, yAxisTitle, title, text, defaultChartOptions, areaFillHex, color, width, noGap, showGridLines } = props; + const chartComponentRef = useRef(null) + + const fillColor = { + linearGradient: { x1: 0, y1: 0, x2: 0, y2: 1 }, + stops: [ + [0, color], + [1, '#000000'], + ], + }; + + const coreChartOptions = { + chart: { + type: type, + height: (height) + 'px', + spacing: [5, 0, 0, 0], + backgroundColor: backgroundColor + }, + credits: { + enabled: false, + }, + title: { + text: title, + align: 'left', + margin: 20 + }, + tooltip:{ + shared: false, + }, + plotOptions: { + column: { + dataLabels: { + enabled: true + } + }, + series: { + minPointLength: noGap ? 0 : 5, + pointWidth: width, + cursor: 'pointer', + point: { + events: { + click: () => { + } + } + } + } + }, + xAxis: { + type: 'datetime', + dateTimeLabelFormats: { + day: '%b %e', + month: '%b', + }, + title: { + text: 'Date', + }, + visible: text, + gridLineWidth: 0, + }, + yAxis: [ + { + title: { + text: yAxisTitle, + }, + visible: text, + gridLineWidth: showGridLines ? 1 : 0, + min: 1, + } + ], + time: { + useUTC: false + }, + ...defaultChartOptions + } + + function processChartData(data) { + return data.map((x, i) => { + return { + data: x.data, + color: x.color, + name: x.name, + fillColor: areaFillHex ? fillColor : {}, + marker: { + enabled: false + }, + yAxis: i==1 ? 1 : 0, + lineWidth: i==1 ? 1: 3 + + } + }) + } + + const [chartOptions, setChartOptions] = useState({...coreChartOptions, series: []}); + + useEffect(() => { + setChartOptions((prev) => { + let tmp = processChartData(data); + if (func.deepComparison(prev.series, tmp)) { + return prev; + } + prev.series = tmp; + prev.yAxis = tmp.map((x, i) => { + return { + title: { + text: x.name, + color: x.color + }, + visible: x.name, + gridLineWidth: 1, + min: 1, + opposite: i==1, + } + }) + prev.plotOptions.series.point.events.click = graphPointClick + prev.tooltip.formatter = tooltipFormatter + return {...prev}; + }) + }, [data]) + + return ( + data.length > 0 ? : + ) +} + +export default LineChart; \ No newline at end of file diff --git a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/observe/api_collections/ApiChanges.jsx b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/observe/api_collections/ApiChanges.jsx index b247c9c69f..d99ffc5c8d 100644 --- a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/observe/api_collections/ApiChanges.jsx +++ b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/observe/api_collections/ApiChanges.jsx @@ -12,7 +12,7 @@ import ApiDetails from "./ApiDetails"; import PersistStore from "../../../../main/PersistStore"; import ApiChangesTable from "./component/ApiChangesTable"; import SummaryCardInfo from "../../../components/shared/SummaryCardInfo"; -import StackedChart from "../../../components/charts/StackedChart"; +import LineChart from "../../../components/charts/LineChart"; import TitleWithInfo from "@/apps/dashboard/components/shared/TitleWithInfo"; import { useLocation } from "react-router-dom"; @@ -145,19 +145,19 @@ function ApiChanges() { return [ { data: endpointsTrend, - color: "#AEE9D1", + color: "#61affe", name: "New endpoints" }, { data: parametersTrend, - color: "#A4E8F2", + color: "#fca130", name: "New parameters" } ] } const defaultChartOptions = { "legend": { - enabled: false + enabled: true }, } @@ -169,21 +169,9 @@ function ApiChanges() { - - - -
- Sensitive params - - -
- New endpoints - - - - { - let detectDate = func.toYMD(new Date(e._id * 86400 * 1000)) + + let detectDate = func.toYMD(new Date((e._id+1) * 86400 * 1000)) m[detectDate] = (m[detectDate] || 0) + e.count newParametersCount += e.count return m From 8e8a431df1983b60600c4d496e562594c5c8aaa6 Mon Sep 17 00:00:00 2001 From: Ankush Jain Date: Wed, 17 Jul 2024 01:43:28 -0700 Subject: [PATCH 2/5] fix width --- .../dashboard/pages/observe/api_collections/ApiSchema.jsx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/observe/api_collections/ApiSchema.jsx b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/observe/api_collections/ApiSchema.jsx index 640439ed1f..baba610c23 100644 --- a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/observe/api_collections/ApiSchema.jsx +++ b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/observe/api_collections/ApiSchema.jsx @@ -106,7 +106,7 @@ function ApiSingleSchema(props) { Header - + {headerCount} @@ -116,7 +116,7 @@ function ApiSingleSchema(props) { Payload - + {payloadCount} From 4ce59e5099730ea7b1106aa6baffa5c6340bd7e1 Mon Sep 17 00:00:00 2001 From: Ankush Jain Date: Fri, 19 Jul 2024 02:17:58 -0700 Subject: [PATCH 3/5] create groups by host --- .../akto/listener/InitializerListener.java | 2 +- .../components/CollectionComponent.jsx | 21 +++- libs/dao/src/main/java/com/akto/DaoInit.java | 3 +- .../testing/HostRegexTestingEndpoints.java | 103 ++++++++++++++++++ .../akto/dto/testing/TestingEndpoints.java | 6 +- 5 files changed, 130 insertions(+), 5 deletions(-) create mode 100644 libs/dao/src/main/java/com/akto/dto/testing/HostRegexTestingEndpoints.java diff --git a/apps/dashboard/src/main/java/com/akto/listener/InitializerListener.java b/apps/dashboard/src/main/java/com/akto/listener/InitializerListener.java index 2c7240a0b9..682e2a79a3 100644 --- a/apps/dashboard/src/main/java/com/akto/listener/InitializerListener.java +++ b/apps/dashboard/src/main/java/com/akto/listener/InitializerListener.java @@ -1184,7 +1184,7 @@ public static void createUnauthenticatedApiGroup() { Filters.eq("_id", UnauthenticatedEndpoint.UNAUTHENTICATED_GROUP_ID)) == null) { loggerMaker.infoAndAddToDb("AccountId: " + Context.accountId.get() + " Creating unauthenticated api group.", LogDb.DASHBOARD); ApiCollection unauthenticatedApisGroup = new ApiCollection(UnauthenticatedEndpoint.UNAUTHENTICATED_GROUP_ID, - "Unauthenticated Apis", Context.now(), new HashSet<>(), null, 0, false, false); + "Unauthenticated APIs", Context.now(), new HashSet<>(), null, 0, false, false); unauthenticatedApisGroup.setAutomated(true); unauthenticatedApisGroup.setType(ApiCollection.Type.API_GROUP); diff --git a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/components/CollectionComponent.jsx b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/components/CollectionComponent.jsx index 803ebd4e98..512da4c767 100644 --- a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/components/CollectionComponent.jsx +++ b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/components/CollectionComponent.jsx @@ -25,6 +25,7 @@ function CollectionComponent(props) { const { condition, index, dispatch, operatorComponent } = props const [apiEndpoints, setApiEndpoints] = useState({}) const [regexText, setRegexText] = useState('') + const [hostRegexText, setHostRegexText] = useState('') useEffect(() => { fetchApiEndpoints(condition.data) @@ -144,6 +145,8 @@ function CollectionComponent(props) { return {method:"GET"} case "REGEX": return {} + case "HOST_REGEX": + return {} default: return {} } @@ -161,9 +164,13 @@ function CollectionComponent(props) { value: 'METHOD' }, { - label: 'Matches regex', + label: 'Path matches regex', value: 'REGEX' }, + { + label: 'Host name matches regex', + value: 'HOST_REGEX' + } ]} initial={condition.type} selected={(value) => { @@ -177,6 +184,11 @@ function CollectionComponent(props) { dispatch({ type: "overwrite", index: index, key: "data", obj: {"regex":val } }) } + const handleHostRegexText = (val) => { + setHostRegexText(val) + dispatch({ type: "overwrite", index: index, key: "data", obj: {"host_regex":val } }) + } + const component = (condition, index) => { switch (condition.type) { case "CUSTOM": @@ -197,7 +209,12 @@ function CollectionComponent(props) { return( handleRegexText(val)} value={regexText} /> ) - default: break; + case "HOST_REGEX": + return( + handleHostRegexText(val)} value={hostRegexText} /> + ) + default: + break; } } diff --git a/libs/dao/src/main/java/com/akto/DaoInit.java b/libs/dao/src/main/java/com/akto/DaoInit.java index 5a8a88d575..e51c845990 100644 --- a/libs/dao/src/main/java/com/akto/DaoInit.java +++ b/libs/dao/src/main/java/com/akto/DaoInit.java @@ -225,6 +225,7 @@ public static CodecRegistry createCodecRegistry(){ ClassModel jiraintegrationClassModel = ClassModel.builder(JiraIntegration.class).enableDiscriminator(true).build(); ClassModel methodConditionClassModel = ClassModel.builder(MethodCondition.class).enableDiscriminator(true).build(); ClassModel regexTestingEndpointsClassModel = ClassModel.builder(RegexTestingEndpoints.class).enableDiscriminator(true).build(); + ClassModel hostRegexTestingEndpointsClassModel = ClassModel.builder(HostRegexTestingEndpoints.class).enableDiscriminator(true).build(); ClassModel dependencyNodeClassModel = ClassModel.builder(DependencyNode.class).enableDiscriminator(true).build(); ClassModel paramInfoClassModel = ClassModel.builder(ParamInfo.class).enableDiscriminator(true).build(); ClassModel nodeClassModel = ClassModel.builder(Node.class).enableDiscriminator(true).build(); @@ -277,7 +278,7 @@ public static CodecRegistry createCodecRegistry(){ loaderClassModel, normalLoaderClassModel, postmanUploadLoaderClassModel, aktoGptConfigClassModel, vulnerableRequestForTemplateClassModel, trafficMetricsAlertClassModel,jiraintegrationClassModel, setupClassModel, cronTimersClassModel, connectionInfoClassModel, testLibraryClassModel, - methodConditionClassModel, regexTestingEndpointsClassModel, allTestingEndpointsClassModel, + methodConditionClassModel, regexTestingEndpointsClassModel, hostRegexTestingEndpointsClassModel, allTestingEndpointsClassModel, UsageMetricClassModel, UsageMetricInfoClassModel, UsageSyncClassModel, OrganizationClassModel, yamlNodeDetails, multiExecTestResultClassModel, workflowTestClassModel, dependencyNodeClassModel, paramInfoClassModel, nodeClassModel, connectionClassModel, edgeClassModel, replaceDetailClassModel, modifyHostDetailClassModel, fileUploadClassModel diff --git a/libs/dao/src/main/java/com/akto/dto/testing/HostRegexTestingEndpoints.java b/libs/dao/src/main/java/com/akto/dto/testing/HostRegexTestingEndpoints.java new file mode 100644 index 0000000000..1fea02c5fe --- /dev/null +++ b/libs/dao/src/main/java/com/akto/dto/testing/HostRegexTestingEndpoints.java @@ -0,0 +1,103 @@ +package com.akto.dto.testing; + +import com.akto.dao.ApiCollectionsDao; +import com.akto.dto.ApiCollection; +import com.akto.dto.ApiCollectionUsers; +import com.akto.dto.ApiInfo; +import com.akto.dto.type.SingleTypeInfo; +import com.mongodb.client.model.Filters; +import org.apache.commons.lang3.NotImplementedException; +import org.apache.commons.lang3.StringUtils; +import org.bson.codecs.pojo.annotations.BsonIgnore; +import org.bson.conversions.Bson; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.regex.Pattern; + +public class HostRegexTestingEndpoints extends TestingEndpoints { + String regex; + + @BsonIgnore + Pattern pattern; + + @BsonIgnore + Map idToHost; + + public HostRegexTestingEndpoints(Operator operator, String regex) { + super(Type.REGEX, operator); + this.regex = regex; + } + + public HostRegexTestingEndpoints() { + super(Type.REGEX, Operator.OR); + } + + @Override + public List returnApis() { + throw new NotImplementedException(); + } + + @Override + public boolean containsApi(ApiInfo.ApiInfoKey key) { + if (pattern == null) { + pattern = Pattern.compile(regex); + } + + if (idToHost == null) { + createIdToHostMap(); + } + + String apiCollectionName = idToHost.getOrDefault(key.getApiCollectionId(), ""); + if (apiCollectionName == null) return false; + + return pattern.matcher(apiCollectionName).matches(); + } + + @Override + public Bson createFilters(ApiCollectionUsers.CollectionType type) { + if (pattern == null) { + pattern = Pattern.compile(regex); + } + + String prefix = getFilterPrefix(type); + + if (idToHost == null) { + createIdToHostMap(); + } + + List apiCollectionIds = new ArrayList<>(); + if (regex != null) { + for(Map.Entry entry: idToHost.entrySet()) { + int apiCollectionId = entry.getKey(); + String hostName = entry.getValue(); + if (StringUtils.isEmpty(hostName)) continue; + if (pattern.matcher(hostName).matches()) { + apiCollectionIds.add(apiCollectionId); + } + } + } + + return Filters.in(prefix + SingleTypeInfo._API_COLLECTION_ID, apiCollectionIds); + } + + private void createIdToHostMap() { + idToHost = new HashMap<>(); + for(ApiCollection apiCollection: ApiCollectionsDao.instance.getMetaAll()) { + idToHost.put(apiCollection.getId(), apiCollection.getHostName()); + } + } + + public String getRegex() { + return regex; + } + + public void setRegex(String regex) { + this.regex = regex; + if (pattern == null) { + pattern = Pattern.compile(regex); + } + } +} diff --git a/libs/dao/src/main/java/com/akto/dto/testing/TestingEndpoints.java b/libs/dao/src/main/java/com/akto/dto/testing/TestingEndpoints.java index 353dc40ce6..ca2887053f 100644 --- a/libs/dao/src/main/java/com/akto/dto/testing/TestingEndpoints.java +++ b/libs/dao/src/main/java/com/akto/dto/testing/TestingEndpoints.java @@ -38,7 +38,7 @@ public enum Operator { public enum Type { - CUSTOM, COLLECTION_WISE, WORKFLOW, LOGICAL_GROUP, METHOD, ALL, REGEX, RISK_SCORE, SENSITIVE_DATA, UNAUTHENTICATED + CUSTOM, COLLECTION_WISE, WORKFLOW, LOGICAL_GROUP, METHOD, ALL, REGEX, RISK_SCORE, SENSITIVE_DATA, UNAUTHENTICATED, HOST_REGEX } public Type getType() { @@ -78,6 +78,10 @@ public static TestingEndpoints generateCondition(Type type, Operator operator, B break; case REGEX: condition = new RegexTestingEndpoints(operator, data.getString("regex")); + break; + case HOST_REGEX: + condition = new HostRegexTestingEndpoints(operator, data.getString("host_regex")); + break; default: break; } From 7ca89b406a4facd6d5314d65bf264aca586884c9 Mon Sep 17 00:00:00 2001 From: Ankush Jain Date: Fri, 19 Jul 2024 09:24:41 -0700 Subject: [PATCH 4/5] skip count query --- .../src/main/java/com/akto/action/ApiCollectionsAction.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/apps/dashboard/src/main/java/com/akto/action/ApiCollectionsAction.java b/apps/dashboard/src/main/java/com/akto/action/ApiCollectionsAction.java index 81850b015a..8e3f68515a 100644 --- a/apps/dashboard/src/main/java/com/akto/action/ApiCollectionsAction.java +++ b/apps/dashboard/src/main/java/com/akto/action/ApiCollectionsAction.java @@ -116,7 +116,10 @@ public List fillApiCollectionsUrlCount(List apiCol if (count != null && (apiCollection.getHostName() != null)) { apiCollection.setUrlsCount(count); } else if(ApiCollection.Type.API_GROUP.equals(apiCollection.getType())){ - count = SingleTypeInfoDao.instance.countEndpoints(Filters.in(SingleTypeInfo._COLLECTION_IDS, apiCollectionId)); + if (count == null) { + count = 0; +// count = SingleTypeInfoDao.instance.countEndpoints(Filters.in(SingleTypeInfo._COLLECTION_IDS, apiCollectionId)); + } apiCollection.setUrlsCount(count); } else { apiCollection.setUrlsCount(fallbackCount); From 78ab770c83deb6a2f35698a6cbbce9e441e0799e Mon Sep 17 00:00:00 2001 From: Ankush Jain Date: Fri, 19 Jul 2024 10:24:54 -0700 Subject: [PATCH 5/5] do query only for null counts --- .../src/main/java/com/akto/action/ApiCollectionsAction.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/dashboard/src/main/java/com/akto/action/ApiCollectionsAction.java b/apps/dashboard/src/main/java/com/akto/action/ApiCollectionsAction.java index 8e3f68515a..ee09dd56bf 100644 --- a/apps/dashboard/src/main/java/com/akto/action/ApiCollectionsAction.java +++ b/apps/dashboard/src/main/java/com/akto/action/ApiCollectionsAction.java @@ -117,8 +117,8 @@ public List fillApiCollectionsUrlCount(List apiCol apiCollection.setUrlsCount(count); } else if(ApiCollection.Type.API_GROUP.equals(apiCollection.getType())){ if (count == null) { - count = 0; -// count = SingleTypeInfoDao.instance.countEndpoints(Filters.in(SingleTypeInfo._COLLECTION_IDS, apiCollectionId)); +// count = 0; + count = SingleTypeInfoDao.instance.countEndpoints(Filters.in(SingleTypeInfo._COLLECTION_IDS, apiCollectionId)); } apiCollection.setUrlsCount(count); } else {