multiple account handling for source-code-analyser

.github/workflows/staging.yml

@@ -29,27 +29,8 @@ jobs:
       - name: Convert github branch name to be compatible with docker tag name convention and generate tag name
         id: docker_tag
         run: echo "IMAGE_TAG=a-$(echo ${{ github.ref_name }} | sed 's/[^a-zA-Z0-9]/-/g')" >> $GITHUB_OUTPUT
-      - name: Download Akto templates zip and PII files
-        working-directory: ./apps/dashboard/src/main/resources
-        run: |
-          wget -O tests-library-master.zip https://github.com/akto-api-security/tests-library/archive/refs/heads/master.zip
-          wget -O general.json https://raw.githubusercontent.com/akto-api-security/pii-types/master/general.json
-          wget -O fintech.json https://raw.githubusercontent.com/akto-api-security/akto/master/pii-types/fintech.json
-          wget -O filetypes.json https://raw.githubusercontent.com/akto-api-security/akto/master/pii-types/filetypes.json
-      - name: Prepare Dashboard polaris UI
-        working-directory: ./apps/dashboard/web/polaris_web
-        run: npm install && export RELEASE_VERSION=${{steps.docker_tag.outputs.IMAGE_TAG}} && npm run build
-      - name: Configure AWS Credentials
-        uses: aws-actions/configure-aws-credentials@v1
-        with:
-          aws-access-key-id: ${{secrets.AWS_ACCESS_KEY_ID}}
-          aws-secret-access-key: ${{secrets.AWS_SECRET_ACCESS_KEY}}
-          aws-region: ap-south-1
-
-      - name: Deploy polaris site to S3 bucket
-        run: aws s3 sync ./apps/dashboard/web/polaris_web/web/dist s3://dashboard-on-cdn/polaris_web/${{steps.docker_tag.outputs.IMAGE_TAG}}/dist --delete
 
-      - run: mvn package -Dakto-image-tag=${{ github.event.inputs.Tag }} -Dakto-build-time=$(eval "date +%s") -Dakto-release-version=${{steps.docker_tag.outputs.IMAGE_TAG}}
+      - run: mvn package -Dakto-image-tag=${{ github.event.inputs.Tag }} -Dakto-build-time=$(eval "date +%s") -Dakto-release-version=${{steps.docker_tag.outputs.IMAGE_TAG}} -DskipTests=true
       - name: DockerHub login
         env:
           DOCKER_USERNAME: ${{secrets.DOCKER_USERNAME}}
@@ -69,19 +50,7 @@ jobs:
           echo $IMAGE_TAG >> $GITHUB_STEP_SUMMARY
           docker buildx create --use
           # Build a docker container and push it to DockerHub 
-          cd apps/dashboard
-          docker buildx build --platform linux/arm64/v8,linux/amd64 -t $ECR_REGISTRY/$ECR_REPOSITORY-dashboard:$IMAGE_TAG $IMAGE_TAG_DASHBOARD . --push
-          cd ../testing
-          docker buildx build --platform linux/arm64/v8,linux/amd64 -t $ECR_REGISTRY/akto-api-testing:$IMAGE_TAG $IMAGE_TAG_TESTING . --push
-          cd ../testing-cli
-          docker buildx build --platform linux/arm64/v8,linux/amd64 -t $ECR_REGISTRY/akto-api-testing-cli:$IMAGE_TAG $IMAGE_TAG_TESTING_CLI . --push
-          cd ../billing
-          docker buildx build --platform linux/arm64/v8,linux/amd64 -t $ECR_REGISTRY/akto-billing:$IMAGE_TAG . --push
-          cd ../internal
-          docker buildx build --platform linux/arm64/v8,linux/amd64 -t $ECR_REGISTRY/akto-internal:$IMAGE_TAG . --push
-          cd ../api-threat-detection
-          docker buildx build --platform linux/arm64/v8,linux/amd64 -t $ECR_REGISTRY/akto-api-protection:$IMAGE_TAG . --push
-          cd ../source-code-analyser
+          cd apps/source-code-analyser
           docker buildx build --platform linux/arm64/v8,linux/amd64 -t $ECR_REGISTRY/source-code-analyser:$IMAGE_TAG . --push
 
       - name: Set up JDK 11

apps/database-abstractor/src/main/java/com/akto/action/CodeAnalysisAction.java

@@ -368,12 +368,12 @@ public void setProjectName(String projectName) {
         this.projectName = projectName;
     }
 
-    public boolean isLastBatch() {
+    public boolean getIsLastBatch() {
         return isLastBatch;
     }
 
-    public void setLastBatch(boolean lastBatch) {
-        isLastBatch = lastBatch;
+    public void setIsLastBatch(boolean isLastBatch) {
+        this.isLastBatch = isLastBatch;
     }
 
     public CodeAnalysisRepo getCodeAnalysisRepo() {

apps/source-code-analyser/src/main/java/com/akto/BitbucketRepo.java

@@ -96,6 +96,7 @@ public BasicDBObject getCodeAnalysisBody(String path) {
         requestBody.put("repoName",this.getRepoToBeAnalysed().getRepoName());
         requestBody.put("bitbucketHost",BITBUCKET_URL);
         requestBody.put("is_bitbucket",true);
+        requestBody.put("is_aktogpt",false);
         return requestBody;
     }
 

apps/source-code-analyser/src/main/java/com/akto/GithubRepo.java

@@ -34,7 +34,7 @@ public String getRepoUrl() {
 
     @Override
     public BasicDBObject getCodeAnalysisBody(String path) {
-        if (path == null || StringUtils.isEmpty(GITHUB_ACCESS_TOKEN)) {
+        if (path == null) {
             return null;
         }
 
@@ -43,6 +43,7 @@ public BasicDBObject getCodeAnalysisBody(String path) {
         requestBody.put("orgName",this.getRepoToBeAnalysed().getProjectName());
         requestBody.put("repoName",this.getRepoToBeAnalysed().getRepoName());
         requestBody.put("is_github",true);
+        requestBody.put("is_aktogpt",false);
         return requestBody;
     }
 

apps/source-code-analyser/src/main/java/com/akto/Main.java

@@ -1,17 +1,55 @@
 package com.akto;
 
+import com.akto.dao.AccountsDao;
 import com.akto.data_actor.DataActor;
 import com.akto.data_actor.DataActorFactory;
 import com.akto.dto.CodeAnalysisRepo;
 import com.akto.log.LoggerMaker;
+import com.akto.util.AccountTask;
+import com.mongodb.ConnectionString;
+import org.apache.commons.lang3.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import java.util.List;
 
-
 public class Main {
     private static final LoggerMaker loggerMaker = new LoggerMaker(Main.class, LoggerMaker.LogDb.RUNTIME);
     private static final DataActor dataActor = DataActorFactory.fetchInstance();
     private static final int SLEEP_TIME = 10 * 1000;
+    private static final Logger logger = LoggerFactory.getLogger(Main.class);
+
+    private static boolean connectToMongo() {
+
+        String mongoURI = System.getenv("AKTO_MONGO_CONN");
+        if (StringUtils.isEmpty(mongoURI)) {
+            return false;
+        }
+        boolean connectedToMongo = false;
+        boolean calledOnce = false;
+        do {
+            try {
+
+                if (!calledOnce) {
+                    DaoInit.init(new ConnectionString(mongoURI));
+                    calledOnce = true;
+                }
+                AccountsDao.instance.getStats();
+                connectedToMongo = true;
+
+                logger.info("connected to mongo");
+            } catch (Exception e) {
+                logger.error("error connecting to mongo", e);
+            } finally {
+                try {
+                    Thread.sleep(1000);
+                } catch (InterruptedException e) {
+                }
+            }
+        } while (!connectedToMongo);
+        return connectedToMongo;
+    }
+
 
     private static List<CodeAnalysisRepo> fetchReposToSync() {
         List<CodeAnalysisRepo> repos = dataActor.findReposToRun();
@@ -21,39 +59,48 @@ private static List<CodeAnalysisRepo> fetchReposToSync() {
         return null;
     }
 
-    public static void main(String[] args) throws InterruptedException {
-        while (true) {
-            if (!BitbucketRepo.doesEnvVariablesExists() && !GithubRepo.doesEnvVariablesExists()) {
-                loggerMaker.infoAndAddToDb("No tokens found");
-                Thread.sleep(SLEEP_TIME);
-                continue;
-            }
-            List<CodeAnalysisRepo> repos = fetchReposToSync();
-            if (repos == null) {
-                loggerMaker.infoAndAddToDb("No repos to run, skipping");
-                Thread.sleep(SLEEP_TIME);
-                continue;
+    private static void runForRepo(List<CodeAnalysisRepo> repos) {
+        if (repos == null) {
+            loggerMaker.infoAndAddToDb("No repos to run, skipping");
+            return;
+        }
+        if (BitbucketRepo.doesEnvVariablesExists()) {
+            BitbucketRepo.fetchAllProjectKeys();
+        }
+        for (CodeAnalysisRepo repo : repos) {
+            SourceCodeAnalyserRepo sourceCodeAnalyserRepo;
+            if (repo.getSourceCodeType() == CodeAnalysisRepo.SourceCodeType.BITBUCKET) {
+                sourceCodeAnalyserRepo = new BitbucketRepo(repo);
+            } else {
+                sourceCodeAnalyserRepo = new GithubRepo(repo);
             }
-            if (BitbucketRepo.doesEnvVariablesExists()) {
-                BitbucketRepo.fetchAllProjectKeys();
-            }
-            for (CodeAnalysisRepo repo : repos) {
-                SourceCodeAnalyserRepo sourceCodeAnalyserRepo;
-                if (repo.getSourceCodeType() == CodeAnalysisRepo.SourceCodeType.BITBUCKET) {
-                    sourceCodeAnalyserRepo = new BitbucketRepo(repo);
-                } else {
-                    sourceCodeAnalyserRepo = new GithubRepo(repo);
-                }
-                try {
-                    sourceCodeAnalyserRepo.fetchEndpointsUsingAnalyser();
-                } catch (Exception e) {
-                    loggerMaker.errorAndAddToDb("Error while fetching endpoints:" + e.getMessage());
-                }
+            try {
+                sourceCodeAnalyserRepo.fetchEndpointsUsingAnalyser();
+            } catch (Exception e) {
+                loggerMaker.errorAndAddToDb("Error while fetching endpoints:" + e.getMessage());
             }
+        }
+    }
+
+    public static void main(String[] args) throws InterruptedException {
+        boolean isConnectedToMongo = connectToMongo();//When mongo connection, fetch for all accounts
 
+        while (true) {
+//            if (!BitbucketRepo.doesEnvVariablesExists() && !GithubRepo.doesEnvVariablesExists()) {
+//                loggerMaker.infoAndAddToDb("No tokens found");
+//                Thread.sleep(SLEEP_TIME);
+//                continue;
+//            }
+            if (isConnectedToMongo) {
+                AccountTask.instance.executeTask(t -> {
+                    List<CodeAnalysisRepo> repos = fetchReposToSync();
+                    runForRepo(repos);
+                }, "initialize-runtime-task");
+            } else {
+                List<CodeAnalysisRepo> repos = fetchReposToSync();
+                runForRepo(repos);
+            }
             Thread.sleep(SLEEP_TIME);
         }
-
     }
-
 }