Add high-level Project API for configuring analysis targets

codyze-console/src/main/kotlin/de/fraunhofer/aisec/codyze/console/ConsoleService.kt

@@ -31,7 +31,7 @@ import com.fasterxml.jackson.dataformat.yaml.YAMLFactory
 import de.fraunhofer.aisec.codyze.AnalysisProject
 import de.fraunhofer.aisec.codyze.AnalysisResult
 import de.fraunhofer.aisec.codyze.console.ai.McpServerHelper
-import de.fraunhofer.aisec.cpg.TranslationConfiguration
+import de.fraunhofer.aisec.cpg.TranslationResult.Companion.DEFAULT_APPLICATION_NAME
 import de.fraunhofer.aisec.cpg.graph.concepts.Concept
 import de.fraunhofer.aisec.cpg.graph.concepts.conceptBuildHelper
 import de.fraunhofer.aisec.cpg.graph.declarations.TranslationUnit
@@ -40,6 +40,7 @@ import de.fraunhofer.aisec.cpg.passes.concepts.LoadPersistedConcepts
 import de.fraunhofer.aisec.cpg.passes.concepts.LoadPersistedConcepts.PersistedConceptEntry
 import de.fraunhofer.aisec.cpg.passes.concepts.LoadPersistedConcepts.PersistedConcepts
 import de.fraunhofer.aisec.cpg.passes.concepts.config.python.PythonStdLibConfigurationPass
+import de.fraunhofer.aisec.cpg.project.Project
 import de.fraunhofer.aisec.cpg.query.QueryTree
 import de.fraunhofer.aisec.cpg.serialization.NodeJSON
 import de.fraunhofer.aisec.cpg.serialization.toJSON
@@ -85,49 +86,42 @@ class ConsoleService {
     suspend fun analyze(request: AnalyzeRequestJSON): AnalysisResultJSON =
         withContext(Dispatchers.IO) {
             val path = Path.of(request.sourceDir)
-            val builder =
-                TranslationConfiguration.builder()
-                    .sourceLocations(path.toFile())
-                    .defaultPasses()
-                    .loadIncludes(true)
-                    .registerPass<PythonStdLibConfigurationPass>()
-                    .registerPass<LoadPersistedConcepts>()
-                    .optionalLanguage("de.fraunhofer.aisec.cpg.frontends.cxx.CLanguage")
-                    .optionalLanguage("de.fraunhofer.aisec.cpg.frontends.cxx.CPPLanguage")
-                    .optionalLanguage("de.fraunhofer.aisec.cpg.frontends.java.JavaLanguage")
-                    .optionalLanguage("de.fraunhofer.aisec.cpg.frontends.golang.GoLanguage")
-                    .optionalLanguage("de.fraunhofer.aisec.cpg.frontends.llvm.LLVMIRLanguage")
-                    .optionalLanguage("de.fraunhofer.aisec.cpg.frontends.python.PythonLanguage")
-                    .optionalLanguage(
-                        "de.fraunhofer.aisec.cpg.frontends.typescript.TypeScriptLanguage"
-                    )
-                    .optionalLanguage("de.fraunhofer.aisec.cpg.frontends.ruby.RubyLanguage")
-                    .optionalLanguage("de.fraunhofer.aisec.cpg.frontends.jvm.JVMLanguage")
-                    .optionalLanguage("de.fraunhofer.aisec.cpg.frontends.ini.IniFileLanguage")
-                    .codeInNodes(true)
-
-            if (request.includeDir != null) {
-                builder.includePath(request.includeDir)
-            }
-
-            if (request.topLevel != null) {
-                builder.topLevel(File(request.topLevel))
-            }
-
-            if (request.conceptsFile != null) {
-                builder.configurePass<LoadPersistedConcepts>(
-                    LoadPersistedConcepts.Configuration(
-                        conceptFiles = listOf(File(request.conceptsFile))
-                    )
-                )
-            }
+            val project =
+                Project.from(path) {
+                    // If an explicit top-level is requested, we place the sources in a single
+                    // component rooted there instead of relying on auto-detection
+                    request.topLevel?.let {
+                        component(
+                            DEFAULT_APPLICATION_NAME,
+                            root = Path.of(it),
+                            sources = listOf(path),
+                        )
+                    }
 
-            val config = builder.build()
+                    translation {
+                        it.loadIncludes(true)
+                        it.registerPass<PythonStdLibConfigurationPass>()
+                        it.registerPass<LoadPersistedConcepts>()
+
+                        request.includeDir?.let { dir -> it.includePath(dir) }
+                        request.conceptsFile?.let { file ->
+                            it.configurePass<LoadPersistedConcepts>(
+                                LoadPersistedConcepts.Configuration(
+                                    conceptFiles = listOf(File(file))
+                                )
+                            )
+                        }
+                    }
+                }
 
             // Build an ad-hoc project
-            val project =
-                AnalysisProject(name = AD_HOC_PROJECT_NAME, projectDir = null, config = config)
-            analyzeProject(project)
+            val analysisProject =
+                AnalysisProject(
+                    name = AD_HOC_PROJECT_NAME,
+                    projectDir = null,
+                    config = project.config,
+                )
+            analyzeProject(analysisProject)
         }
 
     /** Analyzes the given project and returns the analysis result as [AnalysisResultJSON]. */

codyze-core/src/main/kotlin/de/fraunhofer/aisec/codyze/Project.kt

@@ -36,15 +36,18 @@ import de.fraunhofer.aisec.codyze.dsl.RequirementCategoryBuilder
 import de.fraunhofer.aisec.cpg.TranslationConfiguration
 import de.fraunhofer.aisec.cpg.TranslationManager
 import de.fraunhofer.aisec.cpg.TranslationResult
+import de.fraunhofer.aisec.cpg.TranslationResult.Companion.DEFAULT_APPLICATION_NAME
 import de.fraunhofer.aisec.cpg.assumptions.Assumption
 import de.fraunhofer.aisec.cpg.assumptions.AssumptionStatus
 import de.fraunhofer.aisec.cpg.graph.ContextProvider
+import de.fraunhofer.aisec.cpg.project.Project
 import de.fraunhofer.aisec.cpg.query.QueryTree
 import io.github.detekt.sarif4k.*
 import java.io.File
 import java.nio.file.Path
 import kotlin.io.path.Path
 import kotlin.io.path.isDirectory
+import kotlin.io.path.listDirectoryEntries
 
 /** Options common to all subcommands dealing projects. */
 class ProjectOptions : OptionGroup("Project Options") {
@@ -238,7 +241,9 @@ class AnalysisProject(
 
         /**
          * Builds a temporary [AnalysisProject] from the given values without having a `.codyze.kts`
-         * file in place.
+         * file in place. This is based on the [Project] API of the CPG: if neither [sources] nor
+         * [components] are specified, the project structure is auto-detected from the [projectDir],
+         * e.g., based on Go modules or a C/C++ compilation database.
          */
         fun temporary(
             projectDir: Path,
@@ -256,72 +261,37 @@ class AnalysisProject(
                 ((TranslationConfiguration.Builder) -> TranslationConfiguration.Builder)? =
                 null,
         ): AnalysisProject {
-            var builder =
-                TranslationConfiguration.builder()
-                    .defaultPasses()
-                    .optionalLanguage("de.fraunhofer.aisec.cpg.frontends.cxx.CLanguage")
-                    .optionalLanguage("de.fraunhofer.aisec.cpg.frontends.cxx.CPPLanguage")
-                    .optionalLanguage("de.fraunhofer.aisec.cpg.frontends.java.JavaLanguage")
-                    .optionalLanguage("de.fraunhofer.aisec.cpg.frontends.golang.GoLanguage")
-                    .optionalLanguage("de.fraunhofer.aisec.cpg.frontends.llvm.LLVMIRLanguage")
-                    .optionalLanguage("de.fraunhofer.aisec.cpg.frontends.python.PythonLanguage")
-                    .optionalLanguage(
-                        "de.fraunhofer.aisec.cpg.frontends.typescript.TypeScriptLanguage"
-                    )
-                    .optionalLanguage("de.fraunhofer.aisec.cpg.frontends.ruby.RubyLanguage")
-                    .optionalLanguage("de.fraunhofer.aisec.cpg.frontends.jvm.JVMLanguage")
-                    .optionalLanguage("de.fraunhofer.aisec.cpg.frontends.ini.IniFileLanguage")
-
-            // We can either have a single source (using --sources) or multiple components (using
-            // --components)
-            sources?.let {
-                builder =
-                    builder
-                        .sourceLocations(it.map { source -> source.toFile() })
-                        .topLevel(projectDir.toFile())
-            }
+            val project =
+                Project.from(projectDir) {
+                    // A single list of source files (using --sources) becomes one component
+                    sources?.let {
+                        component(DEFAULT_APPLICATION_NAME, root = projectDir, sources = it)
+                    }
 
-            components?.let {
-                val componentDir = projectDir.resolve("components")
-                val pairs =
-                    it.map { component ->
-                        Pair(
-                            component,
-                            mutableListOf<File>(componentDir.resolve(component).toFile()),
-                        )
+                    // Explicitly named components (using --components) are located inside the
+                    // "components" folder
+                    components?.forEach {
+                        component(it, root = projectDir.resolve("components").resolve(it))
                     }
-                builder =
-                    builder
-                        .softwareComponents(
-                            pairs
-                                .groupingBy { it.first }
-                                .aggregate { _, accumulator: MutableList<File>?, element, _ ->
-                                    if (accumulator != null) {
-                                        accumulator.addAll(element.second)
-                                        accumulator
-                                    } else {
-                                        element.second
-                                    }
-                                }
-                                .toMutableMap()
-                        )
-                        .topLevels(it.associateWith { componentDir.resolve(it).toFile() })
-            }
 
-            val addSourcesFolder = librariesPath?.toFile()
+                    exclusionPatterns?.forEach { exclude(it) }
 
-            if (librariesPath?.isDirectory() == true) {
-                builder.loadIncludes(true)
-                addSourcesFolder?.listFiles()?.forEach {
-                    builder = builder.includePath(it.toPath())
-                }
-            }
+                    translation {
+                        // The "libraries" folder can contain additional libraries (or stubs) that
+                        // are added as includes
+                        if (librariesPath?.isDirectory() == true) {
+                            it.loadIncludes(true)
+                            librariesPath.listDirectoryEntries().forEach { library ->
+                                it.includePath(library)
+                            }
+                        }
 
-            exclusionPatterns?.forEach { builder = builder.exclusionPatterns(it) }
-            configModifier?.invoke(builder)
+                        configModifier?.invoke(it)
+                    }
+                }
 
             return AnalysisProject(
-                config = builder.build(),
+                config = project.config,
                 name = projectDir.fileName.toString(),
                 librariesPath = librariesPath,
                 projectDir = projectDir,

codyze-core/src/main/kotlin/de/fraunhofer/aisec/codyze/dsl/Dsl.kt

@@ -37,12 +37,12 @@ import de.fraunhofer.aisec.cpg.assumptions.AssumptionStatus
 import de.fraunhofer.aisec.cpg.graph.Component
 import de.fraunhofer.aisec.cpg.passes.concepts.TagOverlaysPass
 import de.fraunhofer.aisec.cpg.passes.concepts.TaggingContext
+import de.fraunhofer.aisec.cpg.project.Project
 import de.fraunhofer.aisec.cpg.query.NotYetEvaluated
 import de.fraunhofer.aisec.cpg.query.QueryTree
 import de.fraunhofer.aisec.cpg.query.toQueryTree
 import io.github.detekt.sarif4k.ReportingDescriptor
 import io.github.detekt.sarif4k.Result
-import java.io.File
 import java.nio.file.Path
 import kotlin.io.path.Path
 import kotlin.uuid.Uuid
@@ -234,59 +234,45 @@ class ProjectBuilder(val projectDir: Path = Path(".")) {
     ): AnalysisProject {
         val name = name
 
-        val configBuilder =
-            TranslationConfiguration.builder()
-                .defaultPasses()
-                .registerPass<TagOverlaysPass>()
-                .optionalLanguage("de.fraunhofer.aisec.cpg.frontends.cxx.CLanguage")
-                .optionalLanguage("de.fraunhofer.aisec.cpg.frontends.cxx.CPPLanguage")
-                .optionalLanguage("de.fraunhofer.aisec.cpg.frontends.java.JavaLanguage")
-                .optionalLanguage("de.fraunhofer.aisec.cpg.frontends.golang.GoLanguage")
-                .optionalLanguage("de.fraunhofer.aisec.cpg.frontends.llvm.LLVMIRLanguage")
-                .optionalLanguage("de.fraunhofer.aisec.cpg.frontends.python.PythonLanguage")
-                .optionalLanguage("de.fraunhofer.aisec.cpg.frontends.typescript.TypeScriptLanguage")
-                .optionalLanguage("de.fraunhofer.aisec.cpg.frontends.ruby.RubyLanguage")
-                .optionalLanguage("de.fraunhofer.aisec.cpg.frontends.jvm.JVMLanguage")
-                .optionalLanguage("de.fraunhofer.aisec.cpg.frontends.ini.IniFileLanguage")
-
         if (name == null) {
             throw IllegalArgumentException("Project name must be set")
         }
 
-        val components = mutableMapOf<String, List<File>>()
-        val topLevels = mutableMapOf<String, File>()
-
-        // Build software components and "top levels" from the specified architecture
-        toeBuilder.architectureBuilder.modulesBuilder.modules.forEach { it ->
-            // Exclude all files in the exclude list
-            it.exclude.forEach { exclude -> configBuilder.exclusionPatterns(exclude) }
-
-            // Build the file list from the include list
-            val componentTopLevel = projectDir.resolve(it.directory).toFile()
-            var files = it.include.map { include -> componentTopLevel.resolve(include) }
-
-            // If the include list is empty, we include the directory itself
-            if (files.isEmpty()) {
-                files = listOf(componentTopLevel)
+        val project =
+            Project.from(projectDir) {
+                // Build components from the specified architecture. If no modules are specified,
+                // the project structure is auto-detected, e.g., based on Go modules or a C/C++
+                // compilation database.
+                toeBuilder.architectureBuilder.modulesBuilder.modules.forEach { module ->
+                    // Exclude all files in the exclude list
+                    module.exclude.forEach { exclude(it) }
+
+                    // Build the file list from the include list. If the include list is empty, we
+                    // include the directory itself
+                    val componentTopLevel = projectDir.resolve(module.directory)
+                    val files =
+                        module.include
+                            .map { componentTopLevel.resolve(it) }
+                            .ifEmpty { listOf(componentTopLevel) }
+
+                    component(module.name, root = componentTopLevel, sources = files)
+                }
+
+                translation {
+                    it.registerPass<TagOverlaysPass>()
+
+                    // Adjust config from the "external" config modifier as well as from any
+                    // configuration builder inside the script
+                    configModifier?.invoke(it)
+                    toolBuilder.translationConfigurationBuilder?.invoke(it)
+
+                    // Configure tagging from tagging builder
+                    it.configurePass<TagOverlaysPass>(
+                        TagOverlaysPass.Configuration(tag = taggingCtx)
+                    )
+                }
             }
 
-            components += it.name to files
-            topLevels += it.name to componentTopLevel
-        }
-
-        configBuilder.softwareComponents(components)
-        configBuilder.topLevels(topLevels)
-
-        // Adjust config from the "external" config modifier as well as from any configuration
-        // builder inside the script
-        configModifier?.invoke(configBuilder)
-        toolBuilder.translationConfigurationBuilder?.invoke(configBuilder)
-
-        // Configure tagging from tagging builder
-        configBuilder.configurePass<TagOverlaysPass>(
-            TagOverlaysPass.Configuration(tag = taggingCtx)
-        )
-
         // Collect all requirements functions from all categories
         val requirementFunctions =
             requirementsBuilder.categoryBuilders
@@ -303,7 +289,7 @@ class ProjectBuilder(val projectDir: Path = Path(".")) {
             assumptionStatusFunctions =
                 assumptionsBuilder.decisionBuilder.assumptionStatusFunctions,
             suppressedQueryTreeIDs = suppressionsBuilder.suppressions,
-            config = configBuilder.build(),
+            config = project.config,
             postProcess = postProcess,
         )
     }

cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/TranslationConfiguration.kt

@@ -44,6 +44,7 @@ import de.fraunhofer.aisec.cpg.passes.configuration.RegisterExtraPass
 import de.fraunhofer.aisec.cpg.passes.configuration.ReplacePass
 import de.fraunhofer.aisec.cpg.passes.inference.DFGFunctionSummaries
 import de.fraunhofer.aisec.cpg.persistence.DoNotPersist
+import de.fraunhofer.aisec.cpg.project.TargetEnvironment
 import java.io.File
 import java.nio.file.Path
 import kotlin.reflect.KClass
@@ -134,6 +135,13 @@ private constructor(
     val exclusionPatternsByRegex: List<Regex>,
     /** Whether the type propagation system using [TypeObserver] should be disabled. */
     val disableTypeObserver: Boolean,
+    /**
+     * The external environment (operating system, architecture, environment variables) the analyzed
+     * project is assumed to run on. Language frontends can use this to configure
+     * environment-specific behaviour, such as built-in preprocessor macros (C/C++) or build
+     * constraints (Go). Defaults to the environment of the current host.
+     */
+    val targetEnvironment: TargetEnvironment,
 ) {
     /** This list contains all languages which we want to translate. */
     @JsonIgnore val languages: Set<KClass<out Language<*>>>
@@ -283,6 +291,7 @@ private constructor(
         private val exclusionPatternsByRegex = mutableListOf<Regex>()
         private val exclusionPatternsByString = mutableListOf<String>()
         private var disableTypeObserver = false
+        private var targetEnvironment = TargetEnvironment.host()
 
         fun symbols(symbols: Map<String, String>): Builder {
             this.symbols = symbols
@@ -739,6 +748,15 @@ private constructor(
             return this
         }
 
+        /**
+         * Sets the external environment (operating system, architecture, environment variables) the
+         * analyzed project is assumed to run on. Defaults to the environment of the current host.
+         */
+        fun targetEnvironment(environment: TargetEnvironment): Builder {
+            targetEnvironment = environment
+            return this
+        }
+
         @Throws(ConfigurationException::class)
         fun build(): TranslationConfiguration {
             registerExtraFrontendPasses()
@@ -772,6 +790,7 @@ private constructor(
                 exclusionPatternsByString,
                 exclusionPatternsByRegex,
                 disableTypeObserver,
+                targetEnvironment,
             )
         }