refactor language module (#1251)

* refactor language module * more refactor * refactor in test cases * remove deprecated module
Privado-Inc · Aug 19, 2024 · d2b9591 · d2b9591
1 parent dcd4b3d
commit d2b9591
Show file tree

Hide file tree

Showing 86 changed files with 193 additions and 247 deletions.
diff --git a/src/main/scala/ai/privado/audit/DataElementDiscovery.scala b/src/main/scala/ai/privado/audit/DataElementDiscovery.scala
@@ -4,7 +4,7 @@ import ai.privado.cache.TaggerCache
 import ai.privado.dataflow.Dataflow
 import ai.privado.model.Language.Language
 import ai.privado.model.{CatLevelOne, Constants, InternalTag, Language}
-import ai.privado.semantic.Language.*
+import ai.privado.semantic.language.*
 import io.shiftleft.codepropertygraph.generated.Cpg
 import io.shiftleft.codepropertygraph.generated.nodes.*
 import io.shiftleft.semanticcpg.language.*

diff --git a/src/main/scala/ai/privado/audit/DependencyReport.scala b/src/main/scala/ai/privado/audit/DependencyReport.scala
@@ -3,7 +3,7 @@ package ai.privado.audit
 import ai.privado.audit.DependencyReport.getClass
 import ai.privado.languageEngine.java.cache.DependencyModuleCache
 import ai.privado.languageEngine.java.cache.DependencyModuleCache.RuleCategoryInfo
-import ai.privado.languageEngine.java.language.module.{NodeStarters, StepsForModule}
+import ai.privado.semantic.language.*
 import io.shiftleft.codepropertygraph.generated.Cpg
 import io.shiftleft.codepropertygraph.generated.nodes.{Dependency, ModuleDependency}
 import io.shiftleft.semanticcpg.language._

diff --git a/src/main/scala/ai/privado/audit/LiteralReport.scala b/src/main/scala/ai/privado/audit/LiteralReport.scala
@@ -2,7 +2,7 @@ package ai.privado.audit
 
 import io.shiftleft.codepropertygraph.generated.Cpg
 
-import ai.privado.languageEngine.java.language.NodeStarters
+import ai.privado.semantic.language.*
 import scala.collection.mutable.ListBuffer
 import io.shiftleft.semanticcpg.language._
 import org.slf4j.LoggerFactory

diff --git a/src/main/scala/ai/privado/cache/DataFlowCache.scala b/src/main/scala/ai/privado/cache/DataFlowCache.scala
@@ -31,7 +31,7 @@ import ai.privado.model.exporter.{
   DataFlowSubCategoryPathIntermediateModel
 }
 import ai.privado.model.DataFlowPathModel
-import ai.privado.semantic.Language.finder
+import ai.privado.semantic.language.finder
 import io.joern.dataflowengineoss.language.Path
 import io.shiftleft.semanticcpg.language._
 

diff --git a/src/main/scala/ai/privado/exporter/AndroidPermissionsExporter.scala b/src/main/scala/ai/privado/exporter/AndroidPermissionsExporter.scala
@@ -31,7 +31,7 @@ import ai.privado.model.exporter.{
   AndroidPermissionModel,
   DataFlowSubCategoryPathExcerptModel
 }
-import ai.privado.semantic.Language.NodeStarterForAndroidXmlPermissionNode
+import ai.privado.semantic.language.*
 import io.shiftleft.codepropertygraph.generated.nodes.*
 import io.shiftleft.semanticcpg.language.*
 import org.slf4j.LoggerFactory

diff --git a/src/main/scala/ai/privado/exporter/CollectionExporter.scala b/src/main/scala/ai/privado/exporter/CollectionExporter.scala
@@ -37,7 +37,7 @@ import io.shiftleft.semanticcpg.language.*
 import org.slf4j.LoggerFactory
 import overflowdb.traversal.Traversal
 import io.shiftleft.semanticcpg.language.*
-import ai.privado.semantic.Language.*
+import ai.privado.semantic.language.*
 
 import scala.collection.mutable
 import scala.collection.mutable.ListBuffer

diff --git a/src/main/scala/ai/privado/exporter/ExporterUtility.scala b/src/main/scala/ai/privado/exporter/ExporterUtility.scala
@@ -62,13 +62,13 @@ import ai.privado.model.exporter.CollectionEncoderDecoder.*
 import ai.privado.model.exporter.AndroidPermissionsEncoderDecoder.*
 import ai.privado.model.exporter.SinkEncoderDecoder.*
 import ai.privado.model.exporter.PropertyNodesEncoderDecoder.*
-import ai.privado.semantic.Language.{NodeStarterForSqlQueryNode, finder}
+import ai.privado.semantic.language.*
 import io.shiftleft.codepropertygraph.generated.{Cpg, Languages, NodeTypes}
 import ai.privado.utility.Utilities
 import ai.privado.utility.Utilities.{dump, getTruncatedText}
 import ai.privado.tagger.sink.SinkArgumentUtility
 import io.shiftleft.codepropertygraph.generated.nodes.*
-import ai.privado.languageEngine.java.language.*
+import ai.privado.semantic.*
 import better.files.File
 import io.circe.Json
 import io.circe.syntax.EncoderOps

diff --git a/src/main/scala/ai/privado/exporter/HttpConnectionMetadataExporter.scala b/src/main/scala/ai/privado/exporter/HttpConnectionMetadataExporter.scala
@@ -24,8 +24,8 @@
 package ai.privado.exporter
 
 import ai.privado.cache.{AppCache, RuleCache}
-import ai.privado.semantic.Language.*
-import ai.privado.languageEngine.java.language.*
+import ai.privado.semantic.language.*
+import ai.privado.semantic.*
 import io.shiftleft.codepropertygraph.generated.Cpg
 import org.slf4j.LoggerFactory
 import ai.privado.model.{Constants, Language}

diff --git a/src/main/scala/ai/privado/exporter/JSONExporter.scala b/src/main/scala/ai/privado/exporter/JSONExporter.scala
@@ -74,7 +74,7 @@ import scala.concurrent.*
 import scala.language.postfixOps
 import scala.util.{Failure, Success, Try}
 import io.shiftleft.semanticcpg.language.*
-import ai.privado.languageEngine.java.language.{NodeStarters, StepsForProperty}
+import ai.privado.semantic.language.*
 object JSONExporter {
 
   private val logger = LoggerFactory.getLogger(getClass)

diff --git a/src/main/scala/ai/privado/exporter/RepoConfigMetaDataExporter.scala b/src/main/scala/ai/privado/exporter/RepoConfigMetaDataExporter.scala
@@ -5,7 +5,7 @@ import ai.privado.cache.RuleCache
 import scala.collection.mutable
 import io.shiftleft.codepropertygraph.generated.Cpg
 import io.shiftleft.semanticcpg.language.*
-import ai.privado.languageEngine.java.language.NodeStarters
+import ai.privado.semantic.language.*
 import ai.privado.model.Constants
 
 import scala.collection.immutable.HashMap

diff --git a/src/main/scala/ai/privado/exporter/SinkExporter.scala b/src/main/scala/ai/privado/exporter/SinkExporter.scala
@@ -28,7 +28,7 @@ import ai.privado.entrypoint.{PrivadoInput, ScanProcessor}
 import ai.privado.languageEngine.default.NodeStarters
 import ai.privado.model.exporter.{DataFlowSubCategoryModel, SinkModel, SinkProcessingModel}
 import ai.privado.model.exporter.DataFlowEncoderDecoder.*
-import ai.privado.semantic.Language.*
+import ai.privado.semantic.language.*
 import ai.privado.model.{CatLevelOne, Constants, DatabaseDetails, InternalTag, NodeType}
 import ai.privado.utility.Utilities
 import io.shiftleft.codepropertygraph.generated.{Cpg, nodes}

diff --git a/src/main/scala/ai/privado/exporter/SourceExporter.scala b/src/main/scala/ai/privado/exporter/SourceExporter.scala
@@ -31,7 +31,7 @@ import ai.privado.tagger.utility.SourceTaggerUtility.getFilteredSourcesByTagging
 import ai.privado.utility.Utilities
 import io.shiftleft.codepropertygraph.generated.Cpg
 import io.shiftleft.codepropertygraph.generated.nodes.{AstNode, Tag}
-import ai.privado.semantic.Language.*
+import ai.privado.semantic.language.*
 import io.shiftleft.semanticcpg.language.*
 import overflowdb.traversal.Traversal
 import io.joern.dataflowengineoss.language.Path

diff --git a/src/main/scala/ai/privado/languageEngine/base/processor/BaseProcessor.scala b/src/main/scala/ai/privado/languageEngine/base/processor/BaseProcessor.scala
@@ -13,7 +13,7 @@ import ai.privado.model.Constants.*
 import ai.privado.model.Language.Language
 import ai.privado.model.{CpgWithOutputMap, Language}
 import ai.privado.passes.ExperimentalLambdaDataFlowSupportPass
-import ai.privado.semantic.Language.*
+import ai.privado.semantic.language.*
 import ai.privado.tagger.PrivadoParallelCpgPass
 import ai.privado.utility.{PropertyParserPass, StatsRecorder, UnresolvedReportUtility}
 import io.circe.Json

diff --git a/src/main/scala/ai/privado/languageEngine/csharp/processor/CSharpProcessor.scala b/src/main/scala/ai/privado/languageEngine/csharp/processor/CSharpProcessor.scala
@@ -38,7 +38,7 @@ import ai.privado.model.Constants.*
 import ai.privado.model.Language.Language
 import ai.privado.model.{CatLevelOne, Constants, CpgWithOutputMap, Language}
 import ai.privado.passes.*
-import ai.privado.semantic.Language.*
+import ai.privado.semantic.language.*
 import ai.privado.utility.Utilities.createCpgFolder
 import ai.privado.utility.{PropertyParserPass, StatsRecorder}
 import better.files.File

diff --git a/src/main/scala/ai/privado/languageEngine/default/passes/HighTouchDataflow.scala b/src/main/scala/ai/privado/languageEngine/default/passes/HighTouchDataflow.scala
@@ -4,7 +4,7 @@ import ai.privado.cache.DataFlowCache
 import ai.privado.model.{CatLevelOne, Constants, DataFlowPathModel, InternalTag, NodeType}
 import io.joern.dataflowengineoss.language.Path
 import io.shiftleft.codepropertygraph.generated.Cpg
-import ai.privado.semantic.Language.*
+import ai.privado.semantic.language.*
 import ai.privado.languageEngine.default.*
 import io.shiftleft.semanticcpg.language.*
 

diff --git a/src/main/scala/ai/privado/languageEngine/default/passes/PropertyVerificationPass.scala b/src/main/scala/ai/privado/languageEngine/default/passes/PropertyVerificationPass.scala
@@ -1,12 +1,11 @@
 package ai.privado.languageEngine.default.passes
 
 import ai.privado.cache.RuleCache
-import ai.privado.languageEngine.java.language.NodeStarters
+import ai.privado.semantic.language.*
 import ai.privado.tagger.PrivadoParallelCpgPass
 import io.shiftleft.codepropertygraph.generated.Cpg
 import io.shiftleft.codepropertygraph.generated.nodes.JavaProperty
 import overflowdb.traversal.ImplicitsTmp.toTraversalSugarExt
-import ai.privado.languageEngine.java.language.StepsForProperty
 import io.shiftleft.codepropertygraph.generated.traversal.{FileTraversalExtGen, toFileTraversalExtGen}
 
 class PropertyVerificationPass(cpg: Cpg) extends PrivadoParallelCpgPass[JavaProperty](cpg) {

diff --git a/src/main/scala/ai/privado/languageEngine/go/processor/GoProcessor.scala b/src/main/scala/ai/privado/languageEngine/go/processor/GoProcessor.scala
@@ -4,17 +4,17 @@ import ai.privado.audit.AuditReportEntryPoint
 import ai.privado.cache.*
 import ai.privado.dataflow.Dataflow
 import ai.privado.entrypoint.ScanProcessor.config
-import ai.privado.entrypoint.{PrivadoInput}
+import ai.privado.entrypoint.PrivadoInput
 import ai.privado.languageEngine.base.processor.BaseProcessor
 import ai.privado.languageEngine.go.passes.SQLQueryParser
 import ai.privado.languageEngine.go.passes.config.GoYamlLinkerPass
 import ai.privado.languageEngine.go.passes.orm.ORMParserPass
 import ai.privado.languageEngine.go.semantic.Language.tagger
-import ai.privado.languageEngine.java.language.*
+import ai.privado.semantic.*
 import ai.privado.model.Constants.*
 import ai.privado.model.{CatLevelOne, Constants, CpgWithOutputMap, Language}
 import ai.privado.passes.*
-import ai.privado.semantic.Language.*
+import ai.privado.semantic.language.*
 import ai.privado.utility.Utilities.createCpgFolder
 import ai.privado.utility.{PropertyParserPass, StatsRecorder}
 import io.joern.gosrc2cpg.{Config, GoSrc2Cpg}

diff --git a/src/main/scala/ai/privado/languageEngine/go/tagger/config/GoDBConfigTagger.scala b/src/main/scala/ai/privado/languageEngine/go/tagger/config/GoDBConfigTagger.scala
@@ -24,7 +24,7 @@
 package ai.privado.languageEngine.go.tagger.config
 
 import ai.privado.cache.DatabaseDetailsCache
-import ai.privado.languageEngine.java.language.NodeStarters
+import ai.privado.semantic.language.*
 import ai.privado.tagger.PrivadoDBConfigBaseTagger
 import io.shiftleft.codepropertygraph.generated.Cpg
 import org.slf4j.LoggerFactory

diff --git a/src/main/scala/ai/privado/languageEngine/java/language/module.scala b/src/main/scala/ai/privado/languageEngine/java/language/module.scala
diff --git a/src/main/scala/ai/privado/languageEngine/java/language/package.scala b/src/main/scala/ai/privado/languageEngine/java/language/package.scala
diff --git a/src/main/scala/ai/privado/languageEngine/java/passes/config/JavaPropertyLinkerPass.scala b/src/main/scala/ai/privado/languageEngine/java/passes/config/JavaPropertyLinkerPass.scala
@@ -23,7 +23,7 @@
 
 package ai.privado.languageEngine.java.passes.config
 
-import ai.privado.languageEngine.java.language.NodeStarters
+import ai.privado.semantic.language.*
 import ai.privado.tagger.PrivadoParallelCpgPass
 import io.shiftleft.codepropertygraph.generated.nodes.*
 import io.shiftleft.codepropertygraph.generated.{Cpg, EdgeTypes}

diff --git a/src/main/scala/ai/privado/languageEngine/java/processor/JavaProcessor.scala b/src/main/scala/ai/privado/languageEngine/java/processor/JavaProcessor.scala
@@ -38,7 +38,7 @@ import ai.privado.model.Constants.*
 import ai.privado.model.Language.Language
 import ai.privado.model.{CatLevelOne, Constants, CpgWithOutputMap, Language}
 import ai.privado.passes.*
-import ai.privado.semantic.Language.*
+import ai.privado.semantic.language.*
 import ai.privado.tagger.PrivadoParallelCpgPass
 import ai.privado.utility.Utilities.createCpgFolder
 import ai.privado.utility.{PropertyParserPass, StatsRecorder, UnresolvedReportUtility}

diff --git a/src/main/scala/ai/privado/languageEngine/java/tagger/JavaS3Tagger.scala b/src/main/scala/ai/privado/languageEngine/java/tagger/JavaS3Tagger.scala
@@ -2,7 +2,7 @@ package ai.privado.languageEngine.java.tagger
 
 import ai.privado.tagger.S3BaseTagger
 import ai.privado.cache.{DatabaseDetailsCache, S3DatabaseDetailsCache}
-import ai.privado.languageEngine.java.language.NodeStarters
+import ai.privado.semantic.language.*
 import ai.privado.model.DatabaseDetails
 import io.shiftleft.codepropertygraph.generated.{Cpg, Languages, Operators}
 import org.slf4j.LoggerFactory

diff --git a/src/main/scala/ai/privado/languageEngine/java/tagger/config/JavaDBConfigTagger.scala b/src/main/scala/ai/privado/languageEngine/java/tagger/config/JavaDBConfigTagger.scala
@@ -1,7 +1,7 @@
 package ai.privado.languageEngine.java.tagger.config
 
 import ai.privado.cache.DatabaseDetailsCache
-import ai.privado.languageEngine.java.language.NodeStarters
+import ai.privado.semantic.language.*
 import ai.privado.tagger.PrivadoDBConfigBaseTagger
 import ai.privado.utility.Utilities.addDatabaseDetailsMultiple
 import io.shiftleft.codepropertygraph.generated.Cpg

diff --git a/src/main/scala/ai/privado/languageEngine/java/tagger/sink/FeignAPI.scala b/src/main/scala/ai/privado/languageEngine/java/tagger/sink/FeignAPI.scala
@@ -27,7 +27,7 @@ import ai.privado.cache.{AppCache, RuleCache}
 import ai.privado.dataflow.DuplicateFlowProcessor
 import ai.privado.entrypoint.{PrivadoInput, ScanProcessor}
 import ai.privado.model.{Constants, RuleInfo}
-import ai.privado.languageEngine.java.language.*
+import ai.privado.semantic.language.*
 import ai.privado.languageEngine.java.semantic.JavaSemanticGenerator
 import ai.privado.utility.Utilities
 import ai.privado.utility.Utilities.{addRuleTags, getDomainFromString, storeForTag}

diff --git a/...rivado/languageEngine/java/tagger/sink/api/JavaAPISinkEndpointMapperByNonInitMethod.scala b/...rivado/languageEngine/java/tagger/sink/api/JavaAPISinkEndpointMapperByNonInitMethod.scala
@@ -5,7 +5,7 @@ import ai.privado.model.{Constants, InternalTag, NodeType, RuleInfo}
 import ai.privado.tagger.PrivadoParallelCpgPass
 import io.shiftleft.codepropertygraph.generated.Cpg
 import io.shiftleft.semanticcpg.language.*
-import ai.privado.languageEngine.java.language.*
+import ai.privado.semantic.*
 import ai.privado.languageEngine.java.tagger.sink.api.Utility.tagAPICallByItsUrlMethod
 import ai.privado.tagger.utility.APITaggerUtility.{
   getLiteralCode,

diff --git a/src/main/scala/ai/privado/languageEngine/java/tagger/sink/api/JavaAPITagger.scala b/src/main/scala/ai/privado/languageEngine/java/tagger/sink/api/JavaAPITagger.scala
@@ -24,7 +24,7 @@ package ai.privado.languageEngine.java.tagger.sink.api
 
 import ai.privado.cache.{AppCache, FileLinkingMetadata, RuleCache}
 import ai.privado.entrypoint.{PrivadoInput, ScanProcessor}
-import ai.privado.languageEngine.java.language.*
+import ai.privado.semantic.language.*
 import ai.privado.languageEngine.java.semantic.JavaSemanticGenerator
 import ai.privado.languageEngine.java.tagger.Utility.{GRPCTaggerUtility, SOAPTaggerUtility}
 import ai.privado.languageEngine.java.tagger.sink.FeignAPI