Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add logs #1828

Open
wants to merge 2 commits into
base: master
Choose a base branch
from
Open

Add logs #1828

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
2042811
Add logs
ayushaga14 Dec 14, 2024
61f7e64
Add logs
ayushaga14 Dec 14, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
76 changes: 5 additions & 71 deletions .github/workflows/prod.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,28 +27,15 @@ jobs:
- uses: actions/setup-node@v2
with:
node-version: '17'
- name: Download Akto templates zip and PII files
working-directory: ./apps/dashboard/src/main/resources
run: |
wget -O tests-library-master.zip https://github.com/akto-api-security/tests-library/archive/refs/heads/master.zip
wget -O general.json https://raw.githubusercontent.com/akto-api-security/pii-types/master/general.json
wget -O fintech.json https://raw.githubusercontent.com/akto-api-security/akto/master/pii-types/fintech.json
wget -O filetypes.json https://raw.githubusercontent.com/akto-api-security/akto/master/pii-types/filetypes.json
wget -O automated_api_groups.csv https://raw.githubusercontent.com/akto-api-security/akto/master/automated-api-groups/automated-api-groups.csv

- name: Prepare Dashboard polaris UI
working-directory: ./apps/dashboard/web/polaris_web
run: npm install && export RELEASE_VERSION=${{github.event.inputs.release_version}} && npm run build
- name: Configure AWS Credentials
uses: aws-actions/configure-aws-credentials@v1
with:
aws-access-key-id: ${{secrets.AWS_ACCESS_KEY_ID}}
aws-secret-access-key: ${{secrets.AWS_SECRET_ACCESS_KEY}}
aws-region: ap-south-1

- name: Deploy polaris site to S3 bucket
run: aws s3 sync ./apps/dashboard/web/polaris_web/web/dist s3://dashboard-on-cdn/polaris_web/${{github.event.inputs.release_version}}/dist --delete
- run: mvn package -Dakto-image-tag=${{ github.event.inputs.release_version }} -Dakto-build-time=$(eval "date +%s") -Dakto-release-version=${{github.event.inputs.release_version}} -DskipTests
- run: mvn package -DskipTests
- name: DockerHub login
env:
DOCKER_USERNAME: ${{secrets.DOCKER_USERNAME}}
Expand All @@ -61,30 +48,14 @@ jobs:
env:
ECR_REGISTRY: aktosecurity
ECR_REPOSITORY: akto-api-security
IMAGE_TAG_1: local
IMAGE_TAG_2: ${{ github.event.inputs.release_version }}_local
IMAGE_TAG_3: latest
IMAGE_TAG_1: testingdebuglogs
run: |
docker buildx create --use
# Build a docker container and push it to DockerHub
cd apps/dashboard
docker buildx build --platform linux/arm64/v8,linux/amd64 -t $ECR_REGISTRY/$ECR_REPOSITORY-dashboard:$IMAGE_TAG_1 -t $ECR_REGISTRY/$ECR_REPOSITORY-dashboard:$IMAGE_TAG_2 . --push
echo "::set-output name=image::$ECR_REGISTRY/$ECR_REPOSITORY-dashboard:$IMAGE_TAG"
cd ../testing
docker buildx build --platform linux/arm64/v8,linux/amd64 -t $ECR_REGISTRY/akto-api-testing:$IMAGE_TAG_1 -t $ECR_REGISTRY/akto-api-testing:$IMAGE_TAG_2 . --push
cd apps/testing
docker buildx build --platform linux/arm64/v8,linux/amd64 -t $ECR_REGISTRY/akto-api-testing:$IMAGE_TAG_1 . --push
echo "::set-output name=image::$ECR_REGISTRY/akto-api-testing:$IMAGE_TAG"
cd ../testing-cli
docker buildx build --platform linux/arm64/v8,linux/amd64 -t $ECR_REGISTRY/akto-api-testing-cli:$IMAGE_TAG_1 -t $ECR_REGISTRY/akto-api-testing-cli:$IMAGE_TAG_2 . --push
echo "::set-output name=image::$ECR_REGISTRY/akto-api-testing-cli:$IMAGE_TAG"
cd ../billing
docker buildx build --platform linux/arm64/v8,linux/amd64 -t $ECR_REGISTRY/akto-billing:$IMAGE_TAG_1 -t $ECR_REGISTRY/akto-billing:$IMAGE_TAG_2 . --push
echo "::set-output name=image::$ECR_REGISTRY/akto-billing:$IMAGE_TAG"
cd ../internal
docker buildx build --platform linux/arm64/v8,linux/amd64 -t $ECR_REGISTRY/akto-internal:$IMAGE_TAG_1 -t $ECR_REGISTRY/akto-internal:$IMAGE_TAG_2 . --push
echo "::set-output name=image::$ECR_REGISTRY/akto-internal:$IMAGE_TAG"
cd ../api-threat-detection
docker buildx build --platform linux/arm64/v8,linux/amd64 -t $ECR_REGISTRY/akto-api-protection:$IMAGE_TAG_1 -t $ECR_REGISTRY/akto-api-protection:$IMAGE_TAG_2 -t $ECR_REGISTRY/akto-api-protection:$IMAGE_TAG_3 . --push


- name: Configure AWS Credentials for ECR
uses: aws-actions/configure-aws-credentials@v1
with:
Expand All @@ -98,43 +69,6 @@ jobs:
mask-password: 'true'
registry-type: public

- name: Build, tag, and push docker image to Amazon ECR
env:
ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }}
ECR_REPOSITORY: akto-api-security
REGISTRY_ALIAS: p7q3h0z2
IMAGE_TAG: local
IMAGE_TAG2: latest
run: |
docker buildx create --use
# Build a docker container and push it to DockerHub
cd apps/api-threat-detection
docker buildx build --platform linux/arm64/v8,linux/amd64 -t $ECR_REGISTRY/$REGISTRY_ALIAS/akto-api-protection:$IMAGE_TAG -t $ECR_REGISTRY/$REGISTRY_ALIAS/akto-api-protection:$IMAGE_TAG2 . --push
echo "::set-output name=image::$ECR_REGISTRY/akto-api-protection:$IMAGE_TAG"

- name: Push git tag
id: tag_version
uses: mathieudutour/[email protected]
with:
github_token: ${{ secrets.GITHUB_TOKEN }}
custom_tag: ${{ github.event.inputs.release_version }}
- name: Create a GitHub release
uses: ncipollo/[email protected]
with:
tag: ${{ steps.tag_version.outputs.new_tag }}
name: Release ${{ steps.tag_version.outputs.new_tag }}
omitBody: true
- name: Send Github release notification to Slack
id: slack
uses: slackapi/[email protected]
with:
payload: |
{
"text": "Akto Open Source v${{ github.event.inputs.release_version }} released!"
}
env:
SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK
- name: Set up JDK 11
uses: actions/setup-java@v1
with:
Expand Down
39 changes: 37 additions & 2 deletions apps/testing/src/main/java/com/akto/testing/TestExecutor.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -493,7 +493,13 @@ public Void startWithLatch(
loggerMaker.infoAndAddToDb("Starting test for " + apiInfoKey, LogDb.TESTING);

try {
if (accountId == 1665011467) {
logger.info("initiated startTestNew for api " + apiInfoKey.getUrl()+ " at " + Context.now());
}
startTestNew(apiInfoKey, testRunId, testingRunConfig, testingUtil, testRunResultSummaryId, testConfigMap, subCategoryEndpointMap, apiInfoKeyToHostMap, debug, testLogs, startTime, maxRunTime, syncLimit, authMechanism);
if (accountId == 1665011467) {
logger.info("finished startTestNew for api " + apiInfoKey.getUrl()+ " at " + Context.now());
}
} catch (Exception e) {
loggerMaker.errorAndAddToDb(e, "error while running tests: " + e);
}
Expand Down Expand Up @@ -581,9 +587,9 @@ public void startTestNew(ApiInfo.ApiInfoKey apiInfoKey, ObjectId testRunId,

int countSuccessfulTests = 0;
for (String testSubCategory: testSubCategories) {
loggerMaker.infoAndAddToDb("Trying to run test for category: " + testSubCategory + " with summary state: " + GetRunningTestsStatus.getRunningTests().getCurrentState(testRunResultSummaryId) );
loggerMaker.infoAndAddToDb("Trying to run test for category: " + testSubCategory + " with summary state: " + GetRunningTestsStatus.getRunningTests().getCurrentState(testRunResultSummaryId) + " " + Context.now());
if(GetRunningTestsStatus.getRunningTests().isTestRunning(testRunResultSummaryId, true)){
loggerMaker.infoAndAddToDb("Entered tests for api: " + apiInfoKey.toString() + " : " + testSubCategory);
loggerMaker.infoAndAddToDb("Entered tests for api: " + apiInfoKey.toString() + " : " + testSubCategory + " " + Context.now());
if (Context.now() - startTime > timeToKill) {
loggerMaker.infoAndAddToDb("Timed out in " + (Context.now()-startTime) + "seconds");
return;
Expand Down Expand Up @@ -620,6 +626,8 @@ public void startTestNew(ApiInfo.ApiInfoKey apiInfoKey, ObjectId testRunId,
Context.now(), testRunResultSummaryId, null, Collections.singletonList(new TestingRunResult.TestLog(TestingRunResult.TestLogType.INFO, "No samples messages found")));
}

int accountId = Context.accountId.get();

try {
if(testingRunResult==null){

Expand All @@ -630,7 +638,13 @@ public void startTestNew(ApiInfo.ApiInfoKey apiInfoKey, ObjectId testRunId,
checkAndUpdateAuthMechanism(Context.now(), authMechanism);
}

if (accountId == 1665011467) {
logger.info("initiated runTestNew for api " + apiInfoKey.getUrl()+ " at " + Context.now());
}
testingRunResult = runTestNew(apiInfoKey,testRunId,testingUtil,testRunResultSummaryId, testConfig, testingRunConfig, debug, testLogs);
if (accountId == 1665011467) {
logger.info("finished runTestNew for api " + apiInfoKey.getUrl()+ " at " + Context.now());
}
}
} catch (Exception e) {
loggerMaker.errorAndAddToDb("Error while running tests for " + testSubCategory + ": " + e.getMessage(), LogDb.TESTING);
Expand All @@ -644,7 +658,13 @@ public void startTestNew(ApiInfo.ApiInfoKey apiInfoKey, ObjectId testRunId,
}
}

if (accountId == 1665011467) {
logger.info("initiated insertResultsAndMakeIssues run for api " + apiInfoKey.getUrl()+ " at " + Context.now());
}
insertResultsAndMakeIssues(testingRunResults, testRunResultSummaryId);
if (accountId == 1665011467) {
logger.info("finished insertResultsAndMakeIssues run for api " + apiInfoKey.getUrl()+ " at " + Context.now());
}
}else{
if(GetRunningTestsStatus.getRunningTests().getCurrentState(testRunId) != null && GetRunningTestsStatus.getRunningTests().getCurrentState(testRunId).equals(TestingRun.State.STOPPED)){
logger.info("Test stopped for id: " + testRunId.toString());
Expand Down Expand Up @@ -729,7 +749,15 @@ public TestingRunResult runTestNew(ApiInfo.ApiInfoKey apiInfoKey, ObjectId testR
varMap.put("wordList_" + key, wordListsMap.get(key));
}

int accountId = Context.accountId.get();

if (accountId == 1665011467) {
logger.info("initiated resolveWordList for api " + apiInfoKey.getUrl()+ " at " + Context.now());
}
VariableResolver.resolveWordList(varMap, testingUtil.getSampleMessages(), apiInfoKey);
if (accountId == 1665011467) {
logger.info("finished resolveWordList for api " + apiInfoKey.getUrl()+ " at " + Context.now());
}

String testExecutionLogId = UUID.randomUUID().toString();

Expand All @@ -743,7 +771,14 @@ public TestingRunResult runTestNew(ApiInfo.ApiInfoKey apiInfoKey, ObjectId testR
executor.overrideTestUrl(rawApi, testingRunConfig);
YamlTestTemplate yamlTestTemplate = new YamlTestTemplate(apiInfoKey,filterNode, validatorNode, executorNode,
rawApi, varMap, auth, testingUtil.getAuthMechanism(), testExecutionLogId, testingRunConfig, customAuthTypes, testConfig.getStrategy());

if (accountId == 1665011467) {
logger.info("initiated yamlTestTemplate run for api " + apiInfoKey.getUrl()+ " at " + Context.now());
}
YamlTestResult testResults = yamlTestTemplate.run(debug, testLogs);
if (accountId == 1665011467) {
logger.info("finished yamlTestTemplate run for api " + apiInfoKey.getUrl()+ " at " + Context.now());
}
if (testResults == null || testResults.getTestResults().isEmpty()) {
List<GenericTestResult> res = new ArrayList<>();
res.add(new TestResult(null, rawApi.getOriginalMessage(), Collections.singletonList(TestError.SOMETHING_WENT_WRONG.getMessage()), 0, false, TestResult.Confidence.HIGH, null));
Expand Down
28 changes: 28 additions & 0 deletions apps/testing/src/main/java/com/akto/testing/yaml_tests/SecurityTestTemplate.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
package com.akto.testing.yaml_tests;

import com.akto.dao.context.Context;
import com.akto.dto.ApiInfo;
import com.akto.dto.RawApi;
import com.akto.dto.test_editor.Auth;
Expand All @@ -10,13 +11,17 @@
import com.akto.dto.testing.TestResult.TestError;
import com.akto.test_editor.execution.Memory;
import com.akto.test_editor.filter.data_operands_impl.ValidationResult;
import com.akto.testing.TestExecutor;

import java.util.Collections;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Set;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import static com.akto.dto.testing.TestResult.TestError.*;

public abstract class SecurityTestTemplate {
Expand All @@ -35,6 +40,7 @@ public abstract class SecurityTestTemplate {
Strategy strategy;

Memory memory;
private static final Logger logger = LoggerFactory.getLogger(TestExecutor.class);

public SecurityTestTemplate(ApiInfo.ApiInfoKey apiInfoKey, FilterNode filterNode, FilterNode validatorNode, ExecutorNode executorNode ,RawApi rawApi, Map<String, Object> varMap, Auth auth, AuthMechanism authMechanism, String logId, TestingRunConfig testingRunConfig, Strategy strategy) {
this.apiInfoKey = apiInfoKey;
Expand Down Expand Up @@ -86,20 +92,42 @@ public YamlTestResult run(boolean debug, List<TestingRunResult.TestLog> testLogs
}
return getResultWithError(missingConfigs + " " + ROLE_NOT_FOUND.getMessage(), true);
}

int accountId = Context.accountId.get();

if (accountId == 1665011467) {
logger.info("initiated filter check for api " + apiInfoKey.getUrl()+ " at " + Context.now());
}

ValidationResult validationResult = filter();

if (accountId == 1665011467) {
logger.info("finished filter check for api " + apiInfoKey.getUrl()+ " at " + Context.now());
}
boolean valid = validationResult.getIsValid();
if (!valid) {
List<String> errorList = new ArrayList<>();
errorList.add(SKIPPING_EXECUTION_BECAUSE_FILTERS.getMessage());
errorList.add(validationResult.getValidationReason().replaceFirst("and:", "detailed reason for skipping execution:\n").replaceAll("\n\t","\n"));
return getResultWithError(errorList, false);
}
if (accountId == 1665011467) {
logger.info("initiated checkAuthBeforeExecution check for api " + apiInfoKey.getUrl()+ " at " + Context.now());
}
valid = checkAuthBeforeExecution(debug, testLogs);
if (accountId == 1665011467) {
logger.info("finished checkAuthBeforeExecution check for api " + apiInfoKey.getUrl()+ " at " + Context.now());
}
if (!valid) {
return getResultWithError(SKIPPING_EXECUTION_BECAUSE_AUTH.getMessage(), false);
}
if (accountId == 1665011467) {
logger.info("initiated executor yaml logic for api " + apiInfoKey.getUrl()+ " at " + Context.now());
}
YamlTestResult attempts = executor(debug, testLogs);
if (accountId == 1665011467) {
logger.info("finished executor yaml logic for api " + apiInfoKey.getUrl()+ " at " + Context.now());
}
if(attempts == null || attempts.getTestResults().isEmpty()){
List<GenericTestResult> res = new ArrayList<>();
res.add(new TestResult(null, rawApi.getOriginalMessage(), Collections.singletonList(TestError.EXECUTION_FAILED.getMessage()), 0, false, TestResult.Confidence.HIGH, null));
Expand Down
Loading