diff --git a/codyze-console/src/main/kotlin/de/fraunhofer/aisec/codyze/console/ConsoleService.kt b/codyze-console/src/main/kotlin/de/fraunhofer/aisec/codyze/console/ConsoleService.kt index 4c5dbf71cc3..7645b090bf3 100644 --- a/codyze-console/src/main/kotlin/de/fraunhofer/aisec/codyze/console/ConsoleService.kt +++ b/codyze-console/src/main/kotlin/de/fraunhofer/aisec/codyze/console/ConsoleService.kt @@ -31,7 +31,7 @@ import com.fasterxml.jackson.dataformat.yaml.YAMLFactory import de.fraunhofer.aisec.codyze.AnalysisProject import de.fraunhofer.aisec.codyze.AnalysisResult import de.fraunhofer.aisec.codyze.console.ai.McpServerHelper -import de.fraunhofer.aisec.cpg.TranslationConfiguration +import de.fraunhofer.aisec.cpg.TranslationResult.Companion.DEFAULT_APPLICATION_NAME import de.fraunhofer.aisec.cpg.graph.concepts.Concept import de.fraunhofer.aisec.cpg.graph.concepts.conceptBuildHelper import de.fraunhofer.aisec.cpg.graph.declarations.TranslationUnit @@ -40,6 +40,7 @@ import de.fraunhofer.aisec.cpg.passes.concepts.LoadPersistedConcepts import de.fraunhofer.aisec.cpg.passes.concepts.LoadPersistedConcepts.PersistedConceptEntry import de.fraunhofer.aisec.cpg.passes.concepts.LoadPersistedConcepts.PersistedConcepts import de.fraunhofer.aisec.cpg.passes.concepts.config.python.PythonStdLibConfigurationPass +import de.fraunhofer.aisec.cpg.project.Project import de.fraunhofer.aisec.cpg.query.QueryTree import de.fraunhofer.aisec.cpg.serialization.NodeJSON import de.fraunhofer.aisec.cpg.serialization.toJSON @@ -85,49 +86,42 @@ class ConsoleService { suspend fun analyze(request: AnalyzeRequestJSON): AnalysisResultJSON = withContext(Dispatchers.IO) { val path = Path.of(request.sourceDir) - val builder = - TranslationConfiguration.builder() - .sourceLocations(path.toFile()) - .defaultPasses() - .loadIncludes(true) - .registerPass() - .registerPass() - .optionalLanguage("de.fraunhofer.aisec.cpg.frontends.cxx.CLanguage") - .optionalLanguage("de.fraunhofer.aisec.cpg.frontends.cxx.CPPLanguage") - .optionalLanguage("de.fraunhofer.aisec.cpg.frontends.java.JavaLanguage") - .optionalLanguage("de.fraunhofer.aisec.cpg.frontends.golang.GoLanguage") - .optionalLanguage("de.fraunhofer.aisec.cpg.frontends.llvm.LLVMIRLanguage") - .optionalLanguage("de.fraunhofer.aisec.cpg.frontends.python.PythonLanguage") - .optionalLanguage( - "de.fraunhofer.aisec.cpg.frontends.typescript.TypeScriptLanguage" - ) - .optionalLanguage("de.fraunhofer.aisec.cpg.frontends.ruby.RubyLanguage") - .optionalLanguage("de.fraunhofer.aisec.cpg.frontends.jvm.JVMLanguage") - .optionalLanguage("de.fraunhofer.aisec.cpg.frontends.ini.IniFileLanguage") - .codeInNodes(true) - - if (request.includeDir != null) { - builder.includePath(request.includeDir) - } - - if (request.topLevel != null) { - builder.topLevel(File(request.topLevel)) - } - - if (request.conceptsFile != null) { - builder.configurePass( - LoadPersistedConcepts.Configuration( - conceptFiles = listOf(File(request.conceptsFile)) - ) - ) - } + val project = + Project.from(path) { + // If an explicit top-level is requested, we place the sources in a single + // component rooted there instead of relying on auto-detection + request.topLevel?.let { + component( + DEFAULT_APPLICATION_NAME, + root = Path.of(it), + sources = listOf(path), + ) + } - val config = builder.build() + translation { + it.loadIncludes(true) + it.registerPass() + it.registerPass() + + request.includeDir?.let { dir -> it.includePath(dir) } + request.conceptsFile?.let { file -> + it.configurePass( + LoadPersistedConcepts.Configuration( + conceptFiles = listOf(File(file)) + ) + ) + } + } + } // Build an ad-hoc project - val project = - AnalysisProject(name = AD_HOC_PROJECT_NAME, projectDir = null, config = config) - analyzeProject(project) + val analysisProject = + AnalysisProject( + name = AD_HOC_PROJECT_NAME, + projectDir = null, + config = project.config, + ) + analyzeProject(analysisProject) } /** Analyzes the given project and returns the analysis result as [AnalysisResultJSON]. */ diff --git a/codyze-core/src/main/kotlin/de/fraunhofer/aisec/codyze/Project.kt b/codyze-core/src/main/kotlin/de/fraunhofer/aisec/codyze/Project.kt index eef4d2d96c7..0c69ca9b6c0 100644 --- a/codyze-core/src/main/kotlin/de/fraunhofer/aisec/codyze/Project.kt +++ b/codyze-core/src/main/kotlin/de/fraunhofer/aisec/codyze/Project.kt @@ -36,15 +36,18 @@ import de.fraunhofer.aisec.codyze.dsl.RequirementCategoryBuilder import de.fraunhofer.aisec.cpg.TranslationConfiguration import de.fraunhofer.aisec.cpg.TranslationManager import de.fraunhofer.aisec.cpg.TranslationResult +import de.fraunhofer.aisec.cpg.TranslationResult.Companion.DEFAULT_APPLICATION_NAME import de.fraunhofer.aisec.cpg.assumptions.Assumption import de.fraunhofer.aisec.cpg.assumptions.AssumptionStatus import de.fraunhofer.aisec.cpg.graph.ContextProvider +import de.fraunhofer.aisec.cpg.project.Project import de.fraunhofer.aisec.cpg.query.QueryTree import io.github.detekt.sarif4k.* import java.io.File import java.nio.file.Path import kotlin.io.path.Path import kotlin.io.path.isDirectory +import kotlin.io.path.listDirectoryEntries /** Options common to all subcommands dealing projects. */ class ProjectOptions : OptionGroup("Project Options") { @@ -238,7 +241,9 @@ class AnalysisProject( /** * Builds a temporary [AnalysisProject] from the given values without having a `.codyze.kts` - * file in place. + * file in place. This is based on the [Project] API of the CPG: if neither [sources] nor + * [components] are specified, the project structure is auto-detected from the [projectDir], + * e.g., based on Go modules or a C/C++ compilation database. */ fun temporary( projectDir: Path, @@ -256,72 +261,37 @@ class AnalysisProject( ((TranslationConfiguration.Builder) -> TranslationConfiguration.Builder)? = null, ): AnalysisProject { - var builder = - TranslationConfiguration.builder() - .defaultPasses() - .optionalLanguage("de.fraunhofer.aisec.cpg.frontends.cxx.CLanguage") - .optionalLanguage("de.fraunhofer.aisec.cpg.frontends.cxx.CPPLanguage") - .optionalLanguage("de.fraunhofer.aisec.cpg.frontends.java.JavaLanguage") - .optionalLanguage("de.fraunhofer.aisec.cpg.frontends.golang.GoLanguage") - .optionalLanguage("de.fraunhofer.aisec.cpg.frontends.llvm.LLVMIRLanguage") - .optionalLanguage("de.fraunhofer.aisec.cpg.frontends.python.PythonLanguage") - .optionalLanguage( - "de.fraunhofer.aisec.cpg.frontends.typescript.TypeScriptLanguage" - ) - .optionalLanguage("de.fraunhofer.aisec.cpg.frontends.ruby.RubyLanguage") - .optionalLanguage("de.fraunhofer.aisec.cpg.frontends.jvm.JVMLanguage") - .optionalLanguage("de.fraunhofer.aisec.cpg.frontends.ini.IniFileLanguage") - - // We can either have a single source (using --sources) or multiple components (using - // --components) - sources?.let { - builder = - builder - .sourceLocations(it.map { source -> source.toFile() }) - .topLevel(projectDir.toFile()) - } + val project = + Project.from(projectDir) { + // A single list of source files (using --sources) becomes one component + sources?.let { + component(DEFAULT_APPLICATION_NAME, root = projectDir, sources = it) + } - components?.let { - val componentDir = projectDir.resolve("components") - val pairs = - it.map { component -> - Pair( - component, - mutableListOf(componentDir.resolve(component).toFile()), - ) + // Explicitly named components (using --components) are located inside the + // "components" folder + components?.forEach { + component(it, root = projectDir.resolve("components").resolve(it)) } - builder = - builder - .softwareComponents( - pairs - .groupingBy { it.first } - .aggregate { _, accumulator: MutableList?, element, _ -> - if (accumulator != null) { - accumulator.addAll(element.second) - accumulator - } else { - element.second - } - } - .toMutableMap() - ) - .topLevels(it.associateWith { componentDir.resolve(it).toFile() }) - } - val addSourcesFolder = librariesPath?.toFile() + exclusionPatterns?.forEach { exclude(it) } - if (librariesPath?.isDirectory() == true) { - builder.loadIncludes(true) - addSourcesFolder?.listFiles()?.forEach { - builder = builder.includePath(it.toPath()) - } - } + translation { + // The "libraries" folder can contain additional libraries (or stubs) that + // are added as includes + if (librariesPath?.isDirectory() == true) { + it.loadIncludes(true) + librariesPath.listDirectoryEntries().forEach { library -> + it.includePath(library) + } + } - exclusionPatterns?.forEach { builder = builder.exclusionPatterns(it) } - configModifier?.invoke(builder) + configModifier?.invoke(it) + } + } return AnalysisProject( - config = builder.build(), + config = project.config, name = projectDir.fileName.toString(), librariesPath = librariesPath, projectDir = projectDir, diff --git a/codyze-core/src/main/kotlin/de/fraunhofer/aisec/codyze/dsl/Dsl.kt b/codyze-core/src/main/kotlin/de/fraunhofer/aisec/codyze/dsl/Dsl.kt index e177f72b633..bfe413fe8a6 100644 --- a/codyze-core/src/main/kotlin/de/fraunhofer/aisec/codyze/dsl/Dsl.kt +++ b/codyze-core/src/main/kotlin/de/fraunhofer/aisec/codyze/dsl/Dsl.kt @@ -37,12 +37,12 @@ import de.fraunhofer.aisec.cpg.assumptions.AssumptionStatus import de.fraunhofer.aisec.cpg.graph.Component import de.fraunhofer.aisec.cpg.passes.concepts.TagOverlaysPass import de.fraunhofer.aisec.cpg.passes.concepts.TaggingContext +import de.fraunhofer.aisec.cpg.project.Project import de.fraunhofer.aisec.cpg.query.NotYetEvaluated import de.fraunhofer.aisec.cpg.query.QueryTree import de.fraunhofer.aisec.cpg.query.toQueryTree import io.github.detekt.sarif4k.ReportingDescriptor import io.github.detekt.sarif4k.Result -import java.io.File import java.nio.file.Path import kotlin.io.path.Path import kotlin.uuid.Uuid @@ -234,59 +234,45 @@ class ProjectBuilder(val projectDir: Path = Path(".")) { ): AnalysisProject { val name = name - val configBuilder = - TranslationConfiguration.builder() - .defaultPasses() - .registerPass() - .optionalLanguage("de.fraunhofer.aisec.cpg.frontends.cxx.CLanguage") - .optionalLanguage("de.fraunhofer.aisec.cpg.frontends.cxx.CPPLanguage") - .optionalLanguage("de.fraunhofer.aisec.cpg.frontends.java.JavaLanguage") - .optionalLanguage("de.fraunhofer.aisec.cpg.frontends.golang.GoLanguage") - .optionalLanguage("de.fraunhofer.aisec.cpg.frontends.llvm.LLVMIRLanguage") - .optionalLanguage("de.fraunhofer.aisec.cpg.frontends.python.PythonLanguage") - .optionalLanguage("de.fraunhofer.aisec.cpg.frontends.typescript.TypeScriptLanguage") - .optionalLanguage("de.fraunhofer.aisec.cpg.frontends.ruby.RubyLanguage") - .optionalLanguage("de.fraunhofer.aisec.cpg.frontends.jvm.JVMLanguage") - .optionalLanguage("de.fraunhofer.aisec.cpg.frontends.ini.IniFileLanguage") - if (name == null) { throw IllegalArgumentException("Project name must be set") } - val components = mutableMapOf>() - val topLevels = mutableMapOf() - - // Build software components and "top levels" from the specified architecture - toeBuilder.architectureBuilder.modulesBuilder.modules.forEach { it -> - // Exclude all files in the exclude list - it.exclude.forEach { exclude -> configBuilder.exclusionPatterns(exclude) } - - // Build the file list from the include list - val componentTopLevel = projectDir.resolve(it.directory).toFile() - var files = it.include.map { include -> componentTopLevel.resolve(include) } - - // If the include list is empty, we include the directory itself - if (files.isEmpty()) { - files = listOf(componentTopLevel) + val project = + Project.from(projectDir) { + // Build components from the specified architecture. If no modules are specified, + // the project structure is auto-detected, e.g., based on Go modules or a C/C++ + // compilation database. + toeBuilder.architectureBuilder.modulesBuilder.modules.forEach { module -> + // Exclude all files in the exclude list + module.exclude.forEach { exclude(it) } + + // Build the file list from the include list. If the include list is empty, we + // include the directory itself + val componentTopLevel = projectDir.resolve(module.directory) + val files = + module.include + .map { componentTopLevel.resolve(it) } + .ifEmpty { listOf(componentTopLevel) } + + component(module.name, root = componentTopLevel, sources = files) + } + + translation { + it.registerPass() + + // Adjust config from the "external" config modifier as well as from any + // configuration builder inside the script + configModifier?.invoke(it) + toolBuilder.translationConfigurationBuilder?.invoke(it) + + // Configure tagging from tagging builder + it.configurePass( + TagOverlaysPass.Configuration(tag = taggingCtx) + ) + } } - components += it.name to files - topLevels += it.name to componentTopLevel - } - - configBuilder.softwareComponents(components) - configBuilder.topLevels(topLevels) - - // Adjust config from the "external" config modifier as well as from any configuration - // builder inside the script - configModifier?.invoke(configBuilder) - toolBuilder.translationConfigurationBuilder?.invoke(configBuilder) - - // Configure tagging from tagging builder - configBuilder.configurePass( - TagOverlaysPass.Configuration(tag = taggingCtx) - ) - // Collect all requirements functions from all categories val requirementFunctions = requirementsBuilder.categoryBuilders @@ -303,7 +289,7 @@ class ProjectBuilder(val projectDir: Path = Path(".")) { assumptionStatusFunctions = assumptionsBuilder.decisionBuilder.assumptionStatusFunctions, suppressedQueryTreeIDs = suppressionsBuilder.suppressions, - config = configBuilder.build(), + config = project.config, postProcess = postProcess, ) } diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/TranslationConfiguration.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/TranslationConfiguration.kt index 9c3f0de0580..8c5c98c6308 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/TranslationConfiguration.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/TranslationConfiguration.kt @@ -44,6 +44,7 @@ import de.fraunhofer.aisec.cpg.passes.configuration.RegisterExtraPass import de.fraunhofer.aisec.cpg.passes.configuration.ReplacePass import de.fraunhofer.aisec.cpg.passes.inference.DFGFunctionSummaries import de.fraunhofer.aisec.cpg.persistence.DoNotPersist +import de.fraunhofer.aisec.cpg.project.TargetEnvironment import java.io.File import java.nio.file.Path import kotlin.reflect.KClass @@ -134,6 +135,13 @@ private constructor( val exclusionPatternsByRegex: List, /** Whether the type propagation system using [TypeObserver] should be disabled. */ val disableTypeObserver: Boolean, + /** + * The external environment (operating system, architecture, environment variables) the analyzed + * project is assumed to run on. Language frontends can use this to configure + * environment-specific behaviour, such as built-in preprocessor macros (C/C++) or build + * constraints (Go). Defaults to the environment of the current host. + */ + val targetEnvironment: TargetEnvironment, ) { /** This list contains all languages which we want to translate. */ @JsonIgnore val languages: Set>> @@ -283,6 +291,7 @@ private constructor( private val exclusionPatternsByRegex = mutableListOf() private val exclusionPatternsByString = mutableListOf() private var disableTypeObserver = false + private var targetEnvironment = TargetEnvironment.host() fun symbols(symbols: Map): Builder { this.symbols = symbols @@ -739,6 +748,15 @@ private constructor( return this } + /** + * Sets the external environment (operating system, architecture, environment variables) the + * analyzed project is assumed to run on. Defaults to the environment of the current host. + */ + fun targetEnvironment(environment: TargetEnvironment): Builder { + targetEnvironment = environment + return this + } + @Throws(ConfigurationException::class) fun build(): TranslationConfiguration { registerExtraFrontendPasses() @@ -772,6 +790,7 @@ private constructor( exclusionPatternsByString, exclusionPatternsByRegex, disableTypeObserver, + targetEnvironment, ) } diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/NodeBuilder.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/NodeBuilder.kt index 8c98c0adcef..318ee0ba82b 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/NodeBuilder.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/NodeBuilder.kt @@ -405,16 +405,18 @@ private fun Node.setCodeAndLocation( /** * This function tries to find the top-level file for a given [Path]. It first checks if the current - * component has a top-level file, then checks if the path is part of any configured include paths, - * and finally returns the parent directory of the path as a fallback. + * component has a top-level file that contains the path, then checks if the path is part of any + * configured include paths (e.g., for dependencies that live outside the component, such as a + * standard library), then falls back to the component top-level and finally to the parent directory + * of the path. */ context(provider: ContextProvider) val Path.topLevel: File get() { - // First, try to see if the current component has a top-level + // First, try to see if the current component has a top-level that contains the path val topLevel = provider.ctx.currentComponent?.topLevel() - if (topLevel != null) { - return topLevel + if (topLevel != null && toAbsolutePath().startsWith(topLevel.absoluteFile.toPath())) { + return topLevel.absoluteFile } // Otherwise, we can try to see if the path is from a specified include @@ -426,6 +428,11 @@ val Path.topLevel: File } } + // Fall back to the component top-level, even though it does not contain the path + if (topLevel != null) { + return topLevel + } + // If no top-level was found, we return the path's parent as a file return parent.toFile() } diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/project/Project.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/project/Project.kt new file mode 100644 index 00000000000..a8beb59a9fb --- /dev/null +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/project/Project.kt @@ -0,0 +1,582 @@ +/* + * Copyright (c) 2026, Fraunhofer AISEC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * $$$$$$\ $$$$$$$\ $$$$$$\ + * $$ __$$\ $$ __$$\ $$ __$$\ + * $$ / \__|$$ | $$ |$$ / \__| + * $$ | $$$$$$$ |$$ |$$$$\ + * $$ | $$ ____/ $$ |\_$$ | + * $$ | $$\ $$ | $$ | $$ | + * \$$$$$ |$$ | \$$$$$ | + * \______/ \__| \______/ + * + */ +package de.fraunhofer.aisec.cpg.project + +import de.fraunhofer.aisec.cpg.TranslationConfiguration +import de.fraunhofer.aisec.cpg.TranslationManager +import de.fraunhofer.aisec.cpg.TranslationResult +import de.fraunhofer.aisec.cpg.TranslationResult.Companion.DEFAULT_APPLICATION_NAME +import de.fraunhofer.aisec.cpg.frontends.Language +import de.fraunhofer.aisec.cpg.passes.Pass +import java.nio.file.Path +import kotlin.io.path.isDirectory +import kotlin.reflect.KClass +import org.slf4j.LoggerFactory + +/** + * Defines a single component of a [Project], e.g., an application, a service or a library that is + * analyzed together with the rest of the project but represented as an individual + * [de.fraunhofer.aisec.cpg.graph.Component] in the resulting graph. + */ +class ComponentDefinition( + /** The name of the component. */ + val name: String, + /** The top-level directory of the component. */ + val root: Path, + /** The source files or directories of this component. Defaults to the whole [root]. */ + val sources: List = listOf(root), +) + +/** + * A [Project] is the primary, high-level entry point for analyzing source code with the CPG. It + * describes *what* is analyzed (a single file or a whole repository, optionally split into + * components and filtered), and the [TargetEnvironment] it is assumed to run on. From this, the + * lower-level [TranslationConfiguration] is derived automatically. + * + * The simplest usage is a fully automatic analysis — an empty block is enough: + * ```kotlin + * val result = project(Path("/path/to/repo")) { }.analyze() + * ``` + * + * This **auto-mode** uses all default languages available on the classpath, all default passes, and + * runs every registered [Detector] to auto-detect the project structure. Specifying a block + * *overrides* the respective defaults: + * ```kotlin + * val project = + * project(Path("/path/to/repo")) { + * // Only Go is registered; no other language is considered. + * languages { use() } + * + * // Default languages plus Go (when Go is not already in the defaults). + * languages { default(); use() } + * + * // No default passes; only the explicitly listed ones run. + * passes { use() } + * + * // Auto-detect components AND add an extra explicit one. + * components { default(); component("extra", root = Path("extra")) } + * } + * val result = project.analyze() + * ``` + * + * When a project is created from a directory, every registered language that implements [Detector] + * is called once on the project root to auto-detect components (e.g., Go modules, compilation + * database targets) and project-wide settings (symbols, include paths, a compilation database). The + * outcome is recorded in [components] and [detectionResults] so that it can be inspected before the + * analysis is started. + */ +class Project +internal constructor( + /** The name of the project. */ + val name: String, + /** The project directory. Null, if this is an ad-hoc project for a single file. */ + val directory: Path?, + /** The components of this project. */ + val components: List, + /** The external environment this project is assumed to run on. */ + val environment: TargetEnvironment, + /** The results of the project auto-detection, mainly for diagnostic purposes. */ + val detectionResults: List, + /** The derived low-level translation configuration. */ + val config: TranslationConfiguration, +) { + /** Translates the project into a CPG and returns the result. */ + fun analyze(): TranslationResult { + return TranslationManager.builder().config(config).build().analyze().get() + } + + companion object { + /** + * Fully qualified class names of all known languages. They are registered if they are + * available on the classpath and if no language was explicitly registered with + * [LanguagesBuilder.use]. + */ + val defaultLanguages = + listOf( + "de.fraunhofer.aisec.cpg.frontends.cxx.CLanguage", + "de.fraunhofer.aisec.cpg.frontends.cxx.CPPLanguage", + "de.fraunhofer.aisec.cpg.frontends.java.JavaLanguage", + "de.fraunhofer.aisec.cpg.frontends.golang.GoLanguage", + "de.fraunhofer.aisec.cpg.frontends.llvm.LLVMIRLanguage", + "de.fraunhofer.aisec.cpg.frontends.python.PythonLanguage", + "de.fraunhofer.aisec.cpg.frontends.typescript.TypeScriptLanguage", + "de.fraunhofer.aisec.cpg.frontends.ruby.RubyLanguage", + "de.fraunhofer.aisec.cpg.frontends.jvm.JVMLanguage", + "de.fraunhofer.aisec.cpg.frontends.ini.IniFileLanguage", + ) + + /** + * Creates a [Project] from a [path], which can either point to a single source file or to a + * directory (e.g., a repository). Additional configuration can be supplied with the + * [configure] block, see [ProjectBuilder]. + */ + fun from(path: Path, configure: ProjectBuilder.() -> Unit = {}): Project { + val builder = ProjectBuilder(path) + builder.configure() + return builder.resolve() + } + } +} + +/** Creates a [Project] from [path]. This is a DSL-style alias for [Project.from]. */ +fun project(path: Path, configure: ProjectBuilder.() -> Unit = {}): Project { + return Project.from(path, configure) +} + +/** + * Controls which languages are registered for a [Project]. + * + * **Auto-mode** (no `languages {}` block): all languages from [Project.defaultLanguages] that are + * present on the classpath are used. + * + * **Explicit block**: only the languages added with [use] are registered, unless [default] is also + * called to include the defaults. + * + * ```kotlin + * // Only Go — no other language. + * languages { use() } + * + * // Default languages plus Go. + * languages { default(); use() } + * ``` + */ +class LanguagesBuilder { + @PublishedApi internal val explicit = mutableSetOf>>() + internal var includeDefaults = false + + /** Includes all languages from [Project.defaultLanguages] that are on the classpath. */ + fun default() { + includeDefaults = true + } + + /** Registers language [T]. */ + inline fun > use() { + explicit += T::class + } + + /** Registers language [clazz]. */ + fun use(clazz: KClass>) { + explicit += clazz + } +} + +/** + * Controls which passes are registered for a [Project]. + * + * **Auto-mode** (no `passes {}` block): the default passes (see + * [TranslationConfiguration.Builder.defaultPasses]) are registered. + * + * **Explicit block**: only the passes added with [use] are registered, unless [default] is also + * called to include the defaults. + * + * ```kotlin + * // Only SymbolResolver — no other pass. + * passes { use() } + * + * // Default passes plus a custom one. + * passes { default(); use() } + * ``` + */ +class PassesBuilder { + internal var includeDefaults = false + @PublishedApi internal val explicit = mutableListOf>>() + + /** Includes all default passes (see [TranslationConfiguration.Builder.defaultPasses]). */ + fun default() { + includeDefaults = true + } + + /** Registers pass [T]. */ + inline fun > use() { + explicit += T::class + } + + /** Registers pass [clazz]. */ + fun use(clazz: KClass>) { + explicit += clazz + } +} + +/** + * Controls how the components of a [Project] are determined. + * + * **Auto-mode** (no `components {}` block): [ComponentDetector]s and [ProjectDetector]s from all + * registered languages are run automatically. If no detector finds anything, a single default + * component spanning the whole project directory is used. + * + * **Explicit block**: only the components added with [component] are used; detectors do **not** run + * unless [default] is called. [default] and [detector] can be combined with explicit [component] + * definitions. + * + * ```kotlin + * // Only an explicit "backend" component; no auto-detection. + * components { component("backend", root = Path("services/backend")) } + * + * // Auto-detect everything and additionally add an explicit component. + * components { default(); component("extra", root = Path("extra")) } + * + * // Auto-detect using a custom standalone detector. + * components { detector(DirectoryComponentDetector("services")) } + * ``` + */ +class ComponentsBuilder { + internal val explicit = mutableListOf() + internal val detectors = mutableListOf() + internal var includeAutoDetect = false + + /** + * Enables auto-detection: runs [ComponentDetector]s and [ProjectDetector]s from all registered + * languages and any [detector]s added to this block. + */ + fun default() { + includeAutoDetect = true + } + + /** + * Adds a standalone [Detector] to this block. Also enables auto-detection ([default] is implied + * when a detector is added). + */ + fun detector(detector: Detector) { + detectors += detector + includeAutoDetect = true + } + + /** Adds an explicit component definition. */ + fun component(name: String, root: Path, sources: List = listOf(root)) { + explicit += ComponentDefinition(name, root, sources) + } +} + +/** + * Collects all user-supplied configuration for a [Project] and resolves it into an immutable + * [Project], including the derived [TranslationConfiguration]. + * + * Leaving a block uncalled enables **auto-mode** for that aspect of the project: + * - No `languages {}` → all [Project.defaultLanguages] present on the classpath. + * - No `passes {}` → the default pass pipeline. + * - No `components {}` → detectors from all registered languages run automatically. + */ +class ProjectBuilder( + /** The path to the project: either a single source file or a directory. */ + val path: Path +) { + /** The name of the project. Defaults to the file or directory name of [path]. */ + var name: String = path.fileName?.toString() ?: DEFAULT_APPLICATION_NAME + + private var languagesBuilder: LanguagesBuilder? = null + private var passesBuilder: PassesBuilder? = null + private var componentsBuilder: ComponentsBuilder? = null + private val standaloneDetectors = mutableListOf() + + /** + * Overrides the set of candidate languages used by [detectLanguages]. Setting this is only + * intended for tests that need to inject specific language classes without depending on the + * classpath contents of [Project.defaultLanguages]. + */ + internal var defaultLanguagesOverride: Set>>? = null + private val excludesByString = mutableListOf() + private val excludesByRegex = mutableListOf() + private val configModifiers = mutableListOf<(TranslationConfiguration.Builder) -> Unit>() + private var environment = TargetEnvironment.host() + + /** Configures the [TargetEnvironment] this project is assumed to run on. */ + fun environment(init: TargetEnvironmentBuilder.() -> Unit) { + val builder = TargetEnvironmentBuilder() + builder.init() + environment = builder.build() + } + + /** + * Configures which languages are registered. Calling this block switches from auto-mode (all + * default languages) to explicit mode; use [LanguagesBuilder.default] to re-include the + * defaults. + */ + fun languages(init: LanguagesBuilder.() -> Unit) { + val builder = languagesBuilder ?: LanguagesBuilder().also { languagesBuilder = it } + builder.init() + } + + /** + * Configures which passes are registered. Calling this block switches from auto-mode (default + * pass pipeline) to explicit mode; use [PassesBuilder.default] to re-include the defaults. + */ + fun passes(init: PassesBuilder.() -> Unit) { + val builder = passesBuilder ?: PassesBuilder().also { passesBuilder = it } + builder.init() + } + + /** + * Configures how components are determined. Calling this block switches from auto-mode (run all + * language-based detectors) to explicit mode; use [ComponentsBuilder.default] to re-enable + * auto-detection. + */ + fun components(init: ComponentsBuilder.() -> Unit) { + val builder = componentsBuilder ?: ComponentsBuilder().also { componentsBuilder = it } + builder.init() + } + + /** Excludes files and directories matching the given [patterns] from the analysis. */ + fun exclude(vararg patterns: String) { + excludesByString += patterns + } + + /** Excludes files and directories matching the given regex [patterns] from the analysis. */ + fun exclude(vararg patterns: Regex) { + excludesByRegex += patterns + } + + /** + * Adds a standalone [Detector] (a [ComponentDetector] and/or [ProjectDetector]) that is not + * tied to a [Language], such as the [DirectoryComponentDetector]. Standalone detectors run + * before the language-based ones, so they take precedence in case of conflicts. + * + * This is equivalent to `components { detector(detector) }`. + */ + fun detector(detector: Detector) { + standaloneDetectors += detector + } + + /** + * Explicitly registers a [Language]. Equivalent to `languages { use(clazz) }`. + * + * Calling this method implicitly switches languages from auto-mode to explicit mode, so only + * the registered languages (plus any loaded via [LanguagesBuilder.default]) are used. + */ + fun registerLanguage(clazz: KClass>) { + languages { use(clazz) } + } + + /** Explicitly registers a [Language]. Equivalent to `languages { use() }`. */ + inline fun > registerLanguage() { + registerLanguage(T::class) + } + + /** + * Adds an explicit component definition. Equivalent to `components { component(...) }`. + * + * Calling this method implicitly switches components from auto-mode to explicit mode. Combine + * with `components { default() }` to keep auto-detection alongside the explicit component. + */ + fun component(name: String, root: Path = path, sources: List = listOf(root)) { + components { component(name, root, sources) } + } + + /** + * An escape hatch for options that are not (yet) exposed through the project API. The + * [modifier] is applied to the underlying [TranslationConfiguration.Builder] after all + * project-level configuration, so it can override any derived option. + */ + fun translation(modifier: (TranslationConfiguration.Builder) -> Unit) { + configModifiers += modifier + } + + /** Resolves this builder into a [Project], running project auto-detection if enabled. */ + internal fun resolve(): Project { + val languages = resolveLanguages() + + // Determine whether language-based detectors should run. + val cb = componentsBuilder + val languageAutoDetect = path.isDirectory() && (cb == null || cb.includeAutoDetect) + + // Collect detectors: standalone (always present when path is a directory) + language-based + // (only in auto-detect mode). Each detector is called once on the project root. + val allDetectors = buildList { + if (path.isDirectory()) { + addAll(standaloneDetectors) + addAll(cb?.detectors ?: emptyList()) + } + if (languageAutoDetect) { + addAll(languages.mapNotNull { instantiate(it) }.filterIsInstance()) + } + } + + // Run every detector once on the project root and deduplicate by detector name. + val detectionResults = runDetectors(allDetectors) + + // Explicit components win over detected ones; fall back to a single default component. + val explicitComponents = cb?.explicit ?: emptyList() + val detectedComponents = detectionResults.flatMap { it.components }.distinctBy { it.name } + val components = + explicitComponents.ifEmpty { detectedComponents.ifEmpty { defaultComponents() } } + + components.forEach { log.info("Component '{}' rooted at {}", it.name, it.root) } + + val builder = TranslationConfiguration.builder().targetEnvironment(environment) + + resolvePasses(builder) + + languages.forEach { builder.registerLanguage(it) } + + builder.softwareComponents( + components.associate { it.name to it.sources.map(Path::toFile) }.toMutableMap() + ) + builder.topLevels(components.associate { it.name to it.root.toFile() }) + + builder.exclusionPatterns(*excludesByString.toTypedArray()) + builder.exclusionPatterns(*excludesByRegex.toTypedArray()) + + val symbols = detectionResults.flatMap { it.symbols.entries }.associate { it.toPair() } + if (symbols.isNotEmpty()) { + builder.symbols(symbols) + } + detectionResults.flatMap { it.includePaths }.forEach { builder.includePath(it) } + detectionResults + .firstNotNullOfOrNull { it.compilationDatabase } + ?.let { builder.useCompilationDatabase(it) } + + configModifiers.forEach { it(builder) } + + return Project( + name = name, + directory = if (path.isDirectory()) path else null, + components = components, + environment = environment, + detectionResults = detectionResults, + config = builder.build(), + ) + } + + /** + * Runs all [detectors] on the project root. Each detector is called once; results are + * deduplicated by [DetectionResult.detector] name so that two related detectors (e.g., + * `CLanguage` and `CPPLanguage` sharing the same C/C++ detection logic) only contribute one + * result. + */ + private fun runDetectors(detectors: List): List { + return detectors + .mapNotNull { detector -> + val result = detector.detect(path, environment) + if (result != null) { + log.info("Project detection ({}): {}", result.detector, result) + } + result + } + .distinctBy { it.detector } + } + + private fun resolveLanguages(): Set>> { + return when (val lb = languagesBuilder) { + null -> detectLanguages() + else -> + buildSet { + if (lb.includeDefaults) addAll(detectLanguages()) + addAll(lb.explicit) + } + } + } + + private fun resolvePasses(builder: TranslationConfiguration.Builder) { + when (val pb = passesBuilder) { + null -> builder.defaultPasses() + else -> { + if (pb.includeDefaults) builder.defaultPasses() + pb.explicit.forEach { builder.registerPass(it) } + } + } + } + + private fun defaultComponents(): List { + return if (path.isDirectory()) { + listOf(ComponentDefinition(DEFAULT_APPLICATION_NAME, root = path)) + } else { + // An ad-hoc project for a single file; its parent directory serves as top-level + listOf( + ComponentDefinition( + DEFAULT_APPLICATION_NAME, + root = path.toAbsolutePath().parent, + sources = listOf(path), + ) + ) + } + } + + /** + * Loads all languages from [Project.defaultLanguages] that are on the classpath, then filters + * them down to those that can actually handle files in [path]: + * - A language with no declared [Language.fileExtensions] is always included (it uses its own + * detection logic, e.g. [ComponentDetector]). + * - A language with declared extensions is only included when at least one file with a matching + * extension exists anywhere under [path]. + * + * Falls back to loading all available languages when [path] is not a directory (single-file + * projects) or when the directory walk produces no extensions at all. + */ + private fun detectLanguages(): Set>> { + val allAvailable = loadDefaultLanguages() + if (!path.isDirectory()) return allAvailable + + val presentExtensions = scanExtensions() + if (presentExtensions.isEmpty()) return allAvailable + + return allAvailable + .filter { clazz -> + val lang = instantiate(clazz) ?: return@filter false + // No declared extensions → always activate (language uses its own detection) + lang.fileExtensions.isEmpty() || + lang.fileExtensions.any { it.lowercase() in presentExtensions } + } + .toSet() + } + + /** Collects every unique (lowercased) file extension found under [path]. */ + private fun scanExtensions(): Set { + return path + .toFile() + .walkTopDown() + .onEnter { !it.name.startsWith(".") && it.name !in SKIPPED_DIRECTORIES } + .filter { it.isFile } + .mapNotNullTo(mutableSetOf()) { file -> + file.extension.lowercase().takeIf { it.isNotEmpty() } + } + } + + private fun loadDefaultLanguages(): Set>> { + return defaultLanguagesOverride + ?: Project.defaultLanguages + .mapNotNull { + try { + @Suppress("UNCHECKED_CAST") + Class.forName(it).kotlin as? KClass> + } catch (_: ClassNotFoundException) { + null + } + } + .toSet() + } + + private fun instantiate(clazz: KClass>): Language<*>? { + return try { + clazz.constructors.firstOrNull()?.call() + } catch (e: Exception) { + log.warn("Could not instantiate language {} for project detection", clazz.simpleName) + null + } + } + + companion object { + private val log = LoggerFactory.getLogger(ProjectBuilder::class.java) + } +} diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/project/ProjectDetector.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/project/ProjectDetector.kt new file mode 100644 index 00000000000..47aa7403d15 --- /dev/null +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/project/ProjectDetector.kt @@ -0,0 +1,135 @@ +/* + * Copyright (c) 2026, Fraunhofer AISEC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * $$$$$$\ $$$$$$$\ $$$$$$\ + * $$ __$$\ $$ __$$\ $$ __$$\ + * $$ / \__|$$ | $$ |$$ / \__| + * $$ | $$$$$$$ |$$ |$$$$\ + * $$ | $$ ____/ $$ |\_$$ | + * $$ | $$\ $$ | $$ | $$ | + * \$$$$$ |$$ | \$$$$$ | + * \______/ \__| \______/ + * + */ +package de.fraunhofer.aisec.cpg.project + +import de.fraunhofer.aisec.cpg.frontends.CompilationDatabase +import java.nio.file.Path +import kotlin.io.path.isDirectory +import kotlin.io.path.listDirectoryEntries +import kotlin.io.path.name + +/** + * Directory names that detection walks should never enter. Exported so that language-specific + * [Detector] implementations can reuse the same skip list. + */ +val SKIPPED_DIRECTORIES = setOf("vendor", "node_modules", "testdata") + +/** + * A project auto-detector. Detectors answer two related questions about the project rooted at a + * given directory: + * - *Which components does it consist of?* (e.g., Go modules, CMake targets) + * - *Which project-wide settings apply?* (pre-defined symbols, include paths, a compilation + * database) + * + * Both answers are returned together from a single [detect] call on the project root. The detector + * is responsible for any sub-directory traversal it needs (e.g., walking for `go.mod` files). + * + * There are two ways to contribute a detector: + * - A [de.fraunhofer.aisec.cpg.frontends.Language] can implement [Detector]. All registered + * languages are asked automatically when a [Project] is created from a directory. + * - A standalone detector (not tied to a language) can be added with [ProjectBuilder.detector], for + * example the [DirectoryComponentDetector]. + * + * All detection results are only suggestions: they are recorded in [Project.detectionResults] for + * inspection, and any configuration explicitly made by the user in the [ProjectBuilder] takes + * precedence. + */ +interface Detector { + /** + * Inspects the project [root] directory and returns a [DetectionResult] describing the detected + * components and project-wide settings, or `null` if this detector does not recognise the + * project. Called at most once per [Project] resolution, on the project root. + * + * The [environment] describes the target environment and can be used to make + * environment-specific decisions (e.g., deriving Go build constraints from the target + * architecture). + */ + fun detect(root: Path, environment: TargetEnvironment): DetectionResult? +} + +/** + * The combined result of a [Detector.detect] invocation. All properties are suggestions that are + * merged into the [Project] during resolution, unless the user has explicitly configured + * conflicting values. If multiple detectors produce a result with the same [detector] name, only + * the first result is used. + */ +class DetectionResult( + /** A human-readable name of the detector that produced this result. */ + val detector: String, + /** + * The components detected in the project, e.g., Go modules or CMake targets. If empty, + * detection falls through to subsequent detectors or the default single-component behaviour. + */ + val components: List = listOf(), + /** Additional pre-defined symbols, e.g., `GOOS`/`GOARCH` derived from the environment. */ + val symbols: Map = mapOf(), + /** Additional include paths, e.g., derived from a build configuration. */ + val includePaths: List = listOf(), + /** + * A detected [CompilationDatabase] (e.g., from a `compile_commands.json`), which supplies + * per-file include paths and defines to the C/C++ frontend. + */ + val compilationDatabase: CompilationDatabase? = null, + /** Human-readable notes about what was detected, mainly for diagnostic purposes. */ + val notes: List = listOf(), +) { + override fun toString(): String { + return "DetectionResult(detector=$detector, components=${components.map { it.name }}, " + + "symbols=$symbols, notes=$notes)" + } +} + +/** + * A standalone [Detector] that derives one component per direct subdirectory of [folder] under the + * project root. Useful for repositories that follow a convention-based layout, such as a monorepo + * with one service per directory. + * + * This detector is not active by default; add it with [ProjectBuilder.detector]: + * ```kotlin + * project(dir) { components { detector(DirectoryComponentDetector("services")) } } + * ``` + */ +class DirectoryComponentDetector(private val folder: String = "components") : Detector { + override fun detect(root: Path, environment: TargetEnvironment): DetectionResult? { + val base = root.resolve(folder) + if (!base.isDirectory()) { + return null + } + + val components = + base + .listDirectoryEntries() + .filter { it.isDirectory() && !it.name.startsWith(".") } + .sortedBy { it.name } + .map { ComponentDefinition(it.name, root = it) } + + if (components.isEmpty()) { + return null + } + + return DetectionResult(detector = "directory/$folder", components = components) + } +} diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/project/TargetEnvironment.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/project/TargetEnvironment.kt new file mode 100644 index 00000000000..f52aeb632d0 --- /dev/null +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/project/TargetEnvironment.kt @@ -0,0 +1,132 @@ +/* + * Copyright (c) 2026, Fraunhofer AISEC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * $$$$$$\ $$$$$$$\ $$$$$$\ + * $$ __$$\ $$ __$$\ $$ __$$\ + * $$ / \__|$$ | $$ |$$ / \__| + * $$ | $$$$$$$ |$$ |$$$$\ + * $$ | $$ ____/ $$ |\_$$ | + * $$ | $$\ $$ | $$ | $$ | + * \$$$$$ |$$ | \$$$$$ | + * \______/ \__| \______/ + * + */ +package de.fraunhofer.aisec.cpg.project + +import java.nio.file.Path + +/** The operating system a [Project] is targeting. */ +enum class OperatingSystem { + LINUX, + MACOS, + WINDOWS, + FREEBSD, + UNKNOWN; + + companion object { + /** Determines the [OperatingSystem] of the current host. */ + fun host(): OperatingSystem { + val name = System.getProperty("os.name").lowercase() + return when { + name.contains("linux") -> LINUX + name.contains("mac") -> MACOS + name.contains("windows") -> WINDOWS + name.contains("freebsd") -> FREEBSD + else -> UNKNOWN + } + } + } +} + +/** The processor architecture a [Project] is targeting. */ +enum class Architecture( + /** The pointer width (in bits) of this architecture. */ + val bits: Int +) { + X86_64(64), + ARM64(64), + X86(32), + ARM(32), + RISCV64(64), + UNKNOWN(64); + + companion object { + /** Determines the [Architecture] of the current host. */ + fun host(): Architecture { + return when (System.getProperty("os.arch").lowercase()) { + "x86_64", + "amd64" -> X86_64 + "aarch64", + "arm64" -> ARM64 + "x86", + "i386", + "i686" -> X86 + "arm" -> ARM + "riscv64" -> RISCV64 + else -> UNKNOWN + } + } + } +} + +/** + * Describes the external environment a [Project] is running on (or being built for). This + * information can be used by language frontends to configure language-specific behaviour, for + * example: + * - The C/C++ frontend can derive built-in preprocessor macros (such as `__linux__` or + * `__aarch64__`) as well as the width of pointer and integer types from [os] and [architecture]. + * - The Go frontend can derive the `GOOS` and `GOARCH` build constraints from it. + * + * By default, the environment of the current host (see [host]) is assumed. It can be overridden to + * analyze a project for a different target than the machine the analysis runs on (cross-compilation + * style analysis). + */ +class TargetEnvironment( + /** The target operating system. */ + val os: OperatingSystem = OperatingSystem.host(), + /** The target processor architecture. */ + val architecture: Architecture = Architecture.host(), + /** Environment variables that are assumed to be present in the target environment. */ + val env: Map = mapOf(), + /** + * An optional system root containing headers and libraries of the target environment, mainly + * useful for C/C++. + */ + val sysroot: Path? = null, +) { + companion object { + /** Creates a [TargetEnvironment] that mirrors the current host. */ + fun host(): TargetEnvironment { + return TargetEnvironment() + } + } +} + +/** A builder DSL for [TargetEnvironment], used by [ProjectBuilder.environment]. */ +class TargetEnvironmentBuilder { + var os: OperatingSystem = OperatingSystem.host() + var architecture: Architecture = Architecture.host() + var sysroot: Path? = null + private val env = mutableMapOf() + + /** Adds an environment variable to the target environment. */ + fun env(key: String, value: String) { + env[key] = value + } + + fun build(): TargetEnvironment { + return TargetEnvironment(os = os, architecture = architecture, env = env, sysroot = sysroot) + } +} diff --git a/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/project/ProjectTest.kt b/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/project/ProjectTest.kt new file mode 100644 index 00000000000..cf42def924c --- /dev/null +++ b/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/project/ProjectTest.kt @@ -0,0 +1,234 @@ +/* + * Copyright (c) 2026, Fraunhofer AISEC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * $$$$$$\ $$$$$$$\ $$$$$$\ + * $$ __$$\ $$ __$$\ $$ __$$\ + * $$ / \__|$$ | $$ |$$ / \__| + * $$ | $$$$$$$ |$$ |$$$$\ + * $$ | $$ ____/ $$ |\_$$ | + * $$ | $$\ $$ | $$ | $$ | + * \$$$$$ |$$ | \$$$$$ | + * \______/ \__| \______/ + * + */ +package de.fraunhofer.aisec.cpg.project + +import de.fraunhofer.aisec.cpg.TranslationResult.Companion.DEFAULT_APPLICATION_NAME +import de.fraunhofer.aisec.cpg.frontends.TestLanguage +import java.nio.file.Path +import kotlin.io.path.createDirectories +import kotlin.io.path.exists +import kotlin.io.path.writeText +import kotlin.test.Test +import kotlin.test.assertContains +import kotlin.test.assertEquals +import kotlin.test.assertNotNull +import kotlin.test.assertNull +import kotlin.test.assertTrue +import org.junit.jupiter.api.io.TempDir + +/** A [TestLanguage] that detects projects marked by a `test.mod` file. */ +class DetectingTestLanguage : TestLanguage(), Detector { + override fun detect(root: Path, environment: TargetEnvironment): DetectionResult? { + if (!root.resolve("test.mod").exists()) { + return null + } + + return DetectionResult( + detector = "test.mod", + components = listOf(ComponentDefinition("module", root = root)), + symbols = mapOf("TEST_OS" to environment.os.name.lowercase()), + notes = listOf("found test.mod"), + ) + } +} + +class ProjectTest { + @Test + fun testSingleFile(@TempDir tmp: Path) { + val file = tmp.resolve("main.test") + file.writeText("") + + val project = Project.from(file) { registerLanguage() } + + assertNull(project.directory, "a single-file project should be an ad-hoc project") + assertEquals(listOf(file.toFile()), project.config.sourceLocations) + assertEquals(tmp.toFile(), project.config.topLevels[DEFAULT_APPLICATION_NAME]) + assertTrue(project.config.registeredPasses.isNotEmpty()) + } + + @Test + fun testDirectory(@TempDir tmp: Path) { + tmp.resolve("main.test").writeText("") + + val project = + project(tmp) { + registerLanguage() + exclude("tests") + environment { + os = OperatingSystem.LINUX + architecture = Architecture.ARM64 + } + } + + assertEquals(tmp, project.directory) + + val component = project.components.singleOrNull() + assertNotNull(component) + assertEquals(DEFAULT_APPLICATION_NAME, component.name) + assertEquals(tmp, component.root) + + assertContains(project.config.exclusionPatternsByString, "tests") + assertEquals(OperatingSystem.LINUX, project.config.targetEnvironment.os) + assertEquals(64, project.config.targetEnvironment.architecture.bits) + } + + @Test + fun testDetection(@TempDir tmp: Path) { + tmp.resolve("test.mod").writeText("module test") + tmp.resolve("main.test").writeText("") + + val project = + Project.from(tmp) { + registerLanguage() + environment { os = OperatingSystem.LINUX } + } + + val result = project.detectionResults.singleOrNull() + assertNotNull(result) + assertEquals("test.mod", result.detector) + + val component = project.components.singleOrNull() + assertNotNull(component) + assertEquals("module", component.name) + assertEquals("linux", project.config.symbols["TEST_OS"]) + } + + @Test + fun testUserConfigurationWinsOverDetection(@TempDir tmp: Path) { + tmp.resolve("test.mod").writeText("module test") + + val project = + Project.from(tmp) { + // Explicit components disable auto-detection entirely. + languages { use() } + components { component("backend", root = tmp) } + } + + assertEquals(listOf("backend"), project.components.map { it.name }) + assertTrue(project.detectionResults.isEmpty()) + } + + @Test + fun testDetectionRunsAlongsideExplicitComponents(@TempDir tmp: Path) { + tmp.resolve("test.mod").writeText("module test") + + val project = + Project.from(tmp) { + // default() keeps auto-detection running even with an explicit component. + languages { use() } + components { + default() + component("backend", root = tmp) + } + } + + // Detection results are still recorded when default() is used. + assertEquals(1, project.detectionResults.size) + // Explicit component wins over the detected one. + assertEquals(listOf("backend"), project.components.map { it.name }) + } + + @Test + fun testDirectoryComponentDetector(@TempDir tmp: Path) { + tmp.resolve("components/backend").createDirectories() + tmp.resolve("components/frontend").createDirectories() + tmp.resolve("components/backend/main.test").writeText("") + + val project = + Project.from(tmp) { + registerLanguage() + detector(DirectoryComponentDetector()) + } + + assertEquals(listOf("backend", "frontend"), project.components.map { it.name }.sorted()) + assertEquals( + tmp.resolve("components/backend").toFile(), + project.config.topLevels["backend"], + ) + } + + @Test + fun testLanguageAutoDetection(@TempDir tmp: Path) { + // FooLanguage handles .foo, BarLanguage handles .bar. Only .foo files exist. + class FooLanguage : TestLanguage() { + override val fileExtensions = listOf("foo") + } + class BarLanguage : TestLanguage() { + override val fileExtensions = listOf("bar") + } + + tmp.resolve("main.foo").writeText("") + + // Auto-mode: no languages {} block. The builder is given a custom available set so the + // test does not depend on what language modules are on the classpath. + val project = + ProjectBuilder(tmp) + .apply { defaultLanguagesOverride = setOf(FooLanguage::class, BarLanguage::class) } + .resolve() + + // Only FooLanguage should be registered because no .bar file exists. + assertEquals(setOf(FooLanguage::class), project.config.languages) + } + + @Test + fun testLanguageAutoDetectionAlwaysIncludesExtensionlessLanguages(@TempDir tmp: Path) { + // A language with no declared extensions is always included regardless of what's in the + // dir. + class NoExtLanguage : TestLanguage() { + override val fileExtensions = listOf() + } + class FooLanguage : TestLanguage() { + override val fileExtensions = listOf("foo") + } + + tmp.resolve("unrelated.xyz").writeText("") + + val project = + ProjectBuilder(tmp) + .apply { + defaultLanguagesOverride = setOf(NoExtLanguage::class, FooLanguage::class) + } + .resolve() + + // NoExtLanguage is always active; FooLanguage is not because no .foo file exists. + assertEquals(setOf(NoExtLanguage::class), project.config.languages) + } + + @Test + fun testAutoDetectCanBeDisabled(@TempDir tmp: Path) { + tmp.resolve("test.mod").writeText("module test") + + val project = + Project.from(tmp) { + // An explicit empty components block disables auto-detection. + languages { use() } + components {} + } + + assertTrue(project.detectionResults.isEmpty()) + assertEquals(listOf(DEFAULT_APPLICATION_NAME), project.components.map { it.name }) + } +} diff --git a/cpg-language-cxx/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/cxx/CLanguage.kt b/cpg-language-cxx/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/cxx/CLanguage.kt index 79b9bfc81a7..2140a1a7e4b 100644 --- a/cpg-language-cxx/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/cxx/CLanguage.kt +++ b/cpg-language-cxx/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/cxx/CLanguage.kt @@ -29,6 +29,10 @@ import com.fasterxml.jackson.annotation.JsonIgnore import de.fraunhofer.aisec.cpg.frontends.* import de.fraunhofer.aisec.cpg.graph.types.* import de.fraunhofer.aisec.cpg.persistence.DoNotPersist +import de.fraunhofer.aisec.cpg.project.DetectionResult +import de.fraunhofer.aisec.cpg.project.Detector +import de.fraunhofer.aisec.cpg.project.TargetEnvironment +import java.nio.file.Path import kotlin.reflect.KClass const val CONST = "const" @@ -42,7 +46,13 @@ open class CLanguage : HasElaboratedTypeSpecifier, HasShortCircuitOperators, HasGlobalVariables, - HasGlobalFunctions { + HasGlobalFunctions, + Detector { + + override fun detect(root: Path, environment: TargetEnvironment): DetectionResult? { + return detectCxx(root) + } + override val fileExtensions = listOf("c", "h") override val namespaceDelimiter = "::" @DoNotPersist diff --git a/cpg-language-cxx/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/cxx/CXXProjectDetection.kt b/cpg-language-cxx/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/cxx/CXXProjectDetection.kt new file mode 100644 index 00000000000..7a06f49dfc4 --- /dev/null +++ b/cpg-language-cxx/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/cxx/CXXProjectDetection.kt @@ -0,0 +1,78 @@ +/* + * Copyright (c) 2026, Fraunhofer AISEC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * $$$$$$\ $$$$$$$\ $$$$$$\ + * $$ __$$\ $$ __$$\ $$ __$$\ + * $$ / \__|$$ | $$ |$$ / \__| + * $$ | $$$$$$$ |$$ |$$$$\ + * $$ | $$ ____/ $$ |\_$$ | + * $$ | $$\ $$ | $$ | $$ | + * \$$$$$ |$$ | \$$$$$ | + * \______/ \__| \______/ + * + */ +package de.fraunhofer.aisec.cpg.frontends.cxx + +import de.fraunhofer.aisec.cpg.frontends.CompilationDatabase +import de.fraunhofer.aisec.cpg.project.ComponentDefinition +import de.fraunhofer.aisec.cpg.project.DetectionResult +import java.io.File +import java.nio.file.Path +import kotlin.io.path.exists +import org.slf4j.LoggerFactory + +private val log = LoggerFactory.getLogger(CLanguage::class.java) + +/** The file name of a JSON compilation database, as emitted by CMake, bear and others. */ +private const val COMPILE_COMMANDS = "compile_commands.json" + +/** Returns the path to a [COMPILE_COMMANDS] file in [directory] or its `build` folder, if any. */ +private fun findCompilationDatabase(directory: Path): Path? { + return sequenceOf( + directory.resolve(COMPILE_COMMANDS), + directory.resolve("build").resolve(COMPILE_COMMANDS), + ) + .firstOrNull { it.exists() } +} + +/** + * Detects C/C++ project structure and settings in one pass. Looks for a [COMPILE_COMMANDS] file in + * [root] or its `build` folder. If found, the database supplies per-file include paths and + * preprocessor defines, and its targets are translated into [ComponentDefinition]s rooted in + * [root]. Returns `null` if no compilation database is found. + */ +internal fun detectCxx(root: Path): DetectionResult? { + val file = findCompilationDatabase(root) ?: return null + val db = tryLoadCompilationDatabase(file) ?: return null + val components = + db.components.map { (name, files) -> + ComponentDefinition(name, root = root, sources = files.map(File::toPath)) + } + return DetectionResult( + detector = COMPILE_COMMANDS, + components = components, + compilationDatabase = db, + notes = listOf("using compilation database at $file with ${db.size} entries"), + ) +} + +private fun tryLoadCompilationDatabase(file: Path): CompilationDatabase? { + return try { + CompilationDatabase.fromFile(file.toFile()) + } catch (e: Exception) { + log.warn("Could not parse compilation database at {}: {}", file, e.message) + null + } +} diff --git a/cpg-language-cxx/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/cxx/CXXProjectDetectionTest.kt b/cpg-language-cxx/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/cxx/CXXProjectDetectionTest.kt new file mode 100644 index 00000000000..2f5c41e7a41 --- /dev/null +++ b/cpg-language-cxx/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/cxx/CXXProjectDetectionTest.kt @@ -0,0 +1,121 @@ +/* + * Copyright (c) 2026, Fraunhofer AISEC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * $$$$$$\ $$$$$$$\ $$$$$$\ + * $$ __$$\ $$ __$$\ $$ __$$\ + * $$ / \__|$$ | $$ |$$ / \__| + * $$ | $$$$$$$ |$$ |$$$$\ + * $$ | $$ ____/ $$ |\_$$ | + * $$ | $$\ $$ | $$ | $$ | + * \$$$$$ |$$ | \$$$$$ | + * \______/ \__| \______/ + * + */ +package de.fraunhofer.aisec.cpg.frontends.cxx + +import de.fraunhofer.aisec.cpg.project.Project +import java.nio.file.Path +import kotlin.io.path.createDirectories +import kotlin.io.path.writeText +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertNotNull +import org.junit.jupiter.api.io.TempDir + +class CXXProjectDetectionTest { + @Test + fun testDetectCompilationDatabaseInBuildFolder(@TempDir tmp: Path) { + val src = tmp.resolve("src/libfoo") + src.createDirectories() + src.resolve("foo.c").writeText("int foo() { return 1; }") + + tmp.resolve("build").createDirectories() + tmp.resolve("build/compile_commands.json") + .writeText( + """ + [ + { + "directory": "$src", + "command": "gcc -c foo.c", + "file": "$src/foo.c", + "output": "foo.o" + } + ] + """ + .trimIndent() + ) + + val project = Project.from(tmp) { registerLanguage() } + + // The component must be rooted in the project directory, not in "build", so that + // translation unit names stay relative to the project + val component = project.components.singleOrNull() + assertNotNull(component) + assertEquals("libfoo", component.name) + assertEquals(tmp, component.root) + assertEquals(tmp.toFile(), project.config.topLevels["libfoo"]) + } + + @Test + fun testDetectCompilationDatabase(@TempDir tmp: Path) { + val libFoo = tmp.resolve("src/libfoo") + val tool = tmp.resolve("src/tool") + libFoo.createDirectories() + tool.createDirectories() + libFoo.resolve("foo.c").writeText("int foo() { return 1; }") + tool.resolve("main.c").writeText("int main() { return 0; }") + + tmp.resolve("compile_commands.json") + .writeText( + """ + [ + { + "directory": "$libFoo", + "command": "gcc -DFOO=1 -c foo.c", + "file": "$libFoo/foo.c", + "output": "foo.o" + }, + { + "directory": "$tool", + "command": "gcc -c main.c", + "file": "$tool/main.c", + "output": "main.o" + } + ] + """ + .trimIndent() + ) + + val project = + Project.from(tmp) { + registerLanguage() + registerLanguage() + } + + // One component per compilation database component (derived from the src/ layout here) + assertEquals(listOf("libfoo", "tool"), project.components.map { it.name }.sorted()) + + // Even though both CLanguage and CPPLanguage share the detection logic, the result must + // only appear once + val result = project.detectionResults.singleOrNull() + assertNotNull(result) + assertEquals("compile_commands.json", result.detector) + + val db = project.config.compilationDatabase + assertNotNull(db) + assertEquals(2, db.size) + assertEquals(mapOf("FOO" to "1"), db.getAllSymbols("libfoo").filterKeys { it == "FOO" }) + } +} diff --git a/cpg-language-go/src/integrationTest/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/IntegrationTest.kt b/cpg-language-go/src/integrationTest/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/IntegrationTest.kt index cf5f9a731a0..90d907ecaf6 100644 --- a/cpg-language-go/src/integrationTest/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/IntegrationTest.kt +++ b/cpg-language-go/src/integrationTest/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/IntegrationTest.kt @@ -25,13 +25,15 @@ */ package de.fraunhofer.aisec.cpg.frontends.golang -import de.fraunhofer.aisec.cpg.TranslationConfiguration -import de.fraunhofer.aisec.cpg.TranslationResult import de.fraunhofer.aisec.cpg.graph.calls import de.fraunhofer.aisec.cpg.graph.functions import de.fraunhofer.aisec.cpg.graph.get -import de.fraunhofer.aisec.cpg.test.analyzeWithBuilder +import de.fraunhofer.aisec.cpg.project.Architecture +import de.fraunhofer.aisec.cpg.project.OperatingSystem +import de.fraunhofer.aisec.cpg.project.Project import de.fraunhofer.aisec.cpg.test.assertInvokes +import kotlin.io.path.Path +import kotlin.test.assertEquals import kotlin.test.assertNotNull import kotlin.test.assertNull import org.junit.jupiter.api.Test @@ -41,27 +43,35 @@ class IntegrationTest { @Test fun testProject() { val project = - Project.buildProject("src/test/resources/golang/integration", "darwin", "arm64") + Project.from(Path("src/test/resources/golang/integration")) { + registerLanguage() + environment { + os = OperatingSystem.MACOS + architecture = Architecture.ARM64 + } + detector(GoBuildDetector()) + } - val app = project.components[TranslationResult.DEFAULT_APPLICATION_NAME] + // The component is named after the Go module and its sources are resolved (and + // pre-filtered by build constraints) using the Go toolchain + val app = project.components.singleOrNull() assertNotNull(app) - assertNotNull(app.firstOrNull { it.endsWith("main.go") }) - assertNotNull(app.firstOrNull { it.endsWith("func_darwin.go") }) - assertNotNull(app.firstOrNull { it.endsWith("func_darwin_arm64.go") }) - assertNull(app.firstOrNull { it.endsWith("func_darwin_ios.go") }) - assertNull(app.firstOrNull { it.endsWith("func_linux_arm64.go") }) - assertNotNull(app.firstOrNull { it.endsWith("fmt/print.go") }) + assertEquals("integration", app.name) - val tus = - analyzeWithBuilder( - TranslationConfiguration.builder() - .softwareComponents(project.components) - .symbols(project.symbols) - .includePath(project.includePaths.first().path) - .registerLanguage() - .defaultPasses() - ) - assertNotNull(tus) + val sources = app.sources.map { it.toString() } + assertNotNull(sources.firstOrNull { it.endsWith("main.go") }) + assertNotNull(sources.firstOrNull { it.endsWith("func_darwin.go") }) + assertNotNull(sources.firstOrNull { it.endsWith("func_darwin_arm64.go") }) + assertNull(sources.firstOrNull { it.endsWith("func_darwin_ios.go") }) + assertNull(sources.firstOrNull { it.endsWith("func_linux_arm64.go") }) + assertNotNull(sources.firstOrNull { it.endsWith("fmt/print.go") }) + + // GOOS/GOARCH are derived from the target environment + assertEquals("darwin", project.config.symbols["GOOS"]) + assertEquals("arm64", project.config.symbols["GOARCH"]) + + val result = project.analyze() + val tus = result.components.flatMap { it.translationUnits } val printTU = tus.firstOrNull { it.name.endsWith("fmt/print.go") } assertNotNull(printTU) diff --git a/cpg-language-go/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/GoBuildDetector.kt b/cpg-language-go/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/GoBuildDetector.kt new file mode 100644 index 00000000000..19c62360209 --- /dev/null +++ b/cpg-language-go/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/GoBuildDetector.kt @@ -0,0 +1,175 @@ +/* + * Copyright (c) 2026, Fraunhofer AISEC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * $$$$$$\ $$$$$$$\ $$$$$$\ + * $$ __$$\ $$ __$$\ $$ __$$\ + * $$ / \__|$$ | $$ |$$ / \__| + * $$ | $$$$$$$ |$$ |$$$$\ + * $$ | $$ ____/ $$ |\_$$ | + * $$ | $$\ $$ | $$ | $$ | + * \$$$$$ |$$ | \$$$$$ | + * \______/ \__| \______/ + * + */ +package de.fraunhofer.aisec.cpg.frontends.golang + +import de.fraunhofer.aisec.cpg.project.ComponentDefinition +import de.fraunhofer.aisec.cpg.project.DetectionResult +import de.fraunhofer.aisec.cpg.project.Detector +import de.fraunhofer.aisec.cpg.project.TargetEnvironment +import java.io.File +import java.nio.file.Path +import java.util.concurrent.TimeUnit +import kotlin.io.path.exists +import kotlin.io.path.name +import org.slf4j.LoggerFactory + +/** + * A tool-backed detector that emulates building a Go project using an installed Go toolchain. It + * invokes `go list` to resolve the package dependencies of a module (currently limited to packages + * of the standard library and vendored dependencies) and contributes their source files to the + * component, so that dependency code is analyzed together with the module. Additionally, the + * standard library sources (`GOROOT/src`) and a possible `vendor` folder are contributed as include + * paths. + * + * In contrast to the detection built into [GoLanguage], this detector requires a `go` binary on the + * path and is therefore not active by default; add it with + * [de.fraunhofer.aisec.cpg.project.ProjectBuilder.detector]: + * ```kotlin + * project(dir) { detector(GoBuildDetector()) } + * ``` + */ +class GoBuildDetector( + /** Additional build tags to pass to `go list` and the build constraint evaluation. */ + private val extraTags: List = listOf() +) : Detector { + + /** The Go installation root, determined by `go env GOROOT`. */ + private val goRoot: File? by lazy { + runGo(listOf("go", "env", "GOROOT"), directory = null)?.firstOrNull()?.let(::File) + } + + override fun detect(root: Path, environment: TargetEnvironment): DetectionResult? { + val goMod = root.resolve("go.mod") + if (!goMod.exists()) { + return null + } + + val module = parseGoMod(goMod) + val syms = symbols(module, environment) + val topLevel = root.toFile() + val stdLib = goRoot?.resolve("src") ?: return null + val deps = goList(topLevel, environment) ?: return null + + log.debug("Identified {} package dependencies (stdlib only)", deps.size) + + val dirs = + deps.mapNotNull { + if (!it.contains(".")) { + stdLib.resolve(it) + } else if (it.startsWith("vendor")) { + null + } else if (module != null && it.startsWith(module.path)) { + topLevel.resolve(it.substringAfter(module.path)) + } else { + topLevel.resolve("vendor").resolve(it) + } + } + + var files = dirs.flatMap { gatherGoFiles(it, false) }.toMutableList() + files += gatherGoFiles(topLevel.resolve("cmd")) + val sources = files.filter { shouldBeBuild(it, syms) }.map(File::toPath) + + return DetectionResult( + detector = "go-build", + components = + listOf( + ComponentDefinition( + name = module?.name ?: root.name, + root = root, + sources = sources, + ) + ), + symbols = syms, + includePaths = + listOfNotNull(stdLib.toPath(), root.resolve("vendor").takeIf { it.exists() }), + notes = listOf("using Go standard library at $stdLib"), + ) + } + + /** + * Derives the symbols used for build constraint evaluation: `GOOS`/`GOARCH` from the target + * [environment] and the build tags from [extraTags] plus the Go version tags (`go1.1` .. + * `go1.N`) based on the `go` directive of the [module]. + */ + private fun symbols(module: GoModule?, environment: TargetEnvironment): Map { + val tags = extraTags.toMutableList() + module?.minorVersion?.let { minor -> + for (i in 1..minor) { + tags += "go1.$i" + } + } + + val symbols = mutableMapOf() + environment.os.goos?.let { symbols["GOOS"] = it } + environment.architecture.goarch?.let { symbols["GOARCH"] = it } + symbols["-tags"] = tags.joinToString(" ") + + return symbols + } + + /** Invokes `go list` to gather the package dependencies of the module in [topLevel]. */ + private fun goList(topLevel: File, environment: TargetEnvironment): List? { + return runGo( + listOf("go", "list", "-deps", extraTags.joinToString(",", "-tags="), "all"), + directory = topLevel, + environment = environment, + ) + } + + /** Runs a go [command] and returns its standard output lines, or null if it failed. */ + private fun runGo( + command: List, + directory: File?, + environment: TargetEnvironment? = null, + ): List? { + return try { + val pb = ProcessBuilder(command).redirectOutput(ProcessBuilder.Redirect.PIPE) + directory?.let { pb.directory(it) } + environment?.os?.goos?.let { pb.environment()["GOOS"] = it } + environment?.architecture?.goarch?.let { pb.environment()["GOARCH"] = it } + + val proc = pb.start() + proc.waitFor(5, TimeUnit.MINUTES) + if (proc.exitValue() != 0) { + log.warn( + "'{}' failed: {}", + command.joinToString(" "), + proc.errorStream.bufferedReader().readLine(), + ) + return null + } + + proc.inputStream.bufferedReader().readLines() + } catch (e: Exception) { + log.warn("Could not run '{}': {}", command.joinToString(" "), e.message) + null + } + } + + companion object { + private val log = LoggerFactory.getLogger(GoBuildDetector::class.java) + } +} diff --git a/cpg-language-go/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/GoLanguage.kt b/cpg-language-go/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/GoLanguage.kt index 20773a4c78e..e527bf7b057 100644 --- a/cpg-language-go/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/GoLanguage.kt +++ b/cpg-language-go/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/GoLanguage.kt @@ -33,6 +33,10 @@ import de.fraunhofer.aisec.cpg.graph.primitiveType import de.fraunhofer.aisec.cpg.graph.types.* import de.fraunhofer.aisec.cpg.graph.unknownType import de.fraunhofer.aisec.cpg.persistence.DoNotPersist +import de.fraunhofer.aisec.cpg.project.DetectionResult +import de.fraunhofer.aisec.cpg.project.Detector +import de.fraunhofer.aisec.cpg.project.TargetEnvironment +import java.nio.file.Path import kotlin.math.max /** The Go language. */ @@ -43,7 +47,13 @@ open class GoLanguage : HasStructs, HasFirstClassFunctions, HasAnonymousIdentifier, - HasFunctionStyleCasts { + HasFunctionStyleCasts, + Detector { + + override fun detect(root: Path, environment: TargetEnvironment): DetectionResult? { + return detectGo(root, environment) + } + override val fileExtensions = listOf("go") override val namespaceDelimiter = "." @DoNotPersist override val frontend = GoLanguageFrontend::class diff --git a/cpg-language-go/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/GoProjectDetection.kt b/cpg-language-go/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/GoProjectDetection.kt new file mode 100644 index 00000000000..06b5157ff0d --- /dev/null +++ b/cpg-language-go/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/GoProjectDetection.kt @@ -0,0 +1,155 @@ +/* + * Copyright (c) 2026, Fraunhofer AISEC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * $$$$$$\ $$$$$$$\ $$$$$$\ + * $$ __$$\ $$ __$$\ $$ __$$\ + * $$ / \__|$$ | $$ |$$ / \__| + * $$ | $$$$$$$ |$$ |$$$$\ + * $$ | $$ ____/ $$ |\_$$ | + * $$ | $$\ $$ | $$ | $$ | + * \$$$$$ |$$ | \$$$$$ | + * \______/ \__| \______/ + * + */ +package de.fraunhofer.aisec.cpg.frontends.golang + +import de.fraunhofer.aisec.cpg.project.Architecture +import de.fraunhofer.aisec.cpg.project.ComponentDefinition +import de.fraunhofer.aisec.cpg.project.DetectionResult +import de.fraunhofer.aisec.cpg.project.OperatingSystem +import de.fraunhofer.aisec.cpg.project.SKIPPED_DIRECTORIES +import de.fraunhofer.aisec.cpg.project.TargetEnvironment +import java.nio.file.Path +import kotlin.io.path.exists +import kotlin.io.path.isDirectory +import kotlin.io.path.listDirectoryEntries +import kotlin.io.path.name +import kotlin.io.path.readLines + +/** The `GOOS` value corresponding to this [OperatingSystem], or null if there is none. */ +val OperatingSystem.goos: String? + get() = + when (this) { + OperatingSystem.LINUX -> "linux" + OperatingSystem.MACOS -> "darwin" + OperatingSystem.WINDOWS -> "windows" + OperatingSystem.FREEBSD -> "freebsd" + OperatingSystem.UNKNOWN -> null + } + +/** The `GOARCH` value corresponding to this [Architecture], or null if there is none. */ +val Architecture.goarch: String? + get() = + when (this) { + Architecture.X86_64 -> "amd64" + Architecture.ARM64 -> "arm64" + Architecture.X86 -> "386" + Architecture.ARM -> "arm" + Architecture.RISCV64 -> "riscv64" + Architecture.UNKNOWN -> null + } + +/** Information parsed from a `go.mod` file. */ +internal data class GoModule( + /** The module path, e.g., `example.com/app`. */ + val path: String, + /** The minor part of the version in the `go` directive, e.g., 22 for `go 1.22.1`. */ + val minorVersion: Int?, +) { + /** The short name of the module, i.e., the last segment of the module path. */ + val name: String + get() = path.substringAfterLast('/') +} + +/** + * Parses the `module` and `go` directives out of the given [goMod] file, or returns null if it + * cannot be parsed. This is intentionally a lightweight line-based parser, so that project + * detection does not require the native Go helper library. + */ +internal fun parseGoMod(goMod: Path): GoModule? { + val lines = + try { + goMod.readLines() + } catch (_: Exception) { + return null + } + + val path = + lines + .firstOrNull { it.startsWith("module ") } + ?.removePrefix("module ") + ?.substringBefore("//") + ?.trim() ?: return null + val version = + lines + .map { it.trim() } + .firstOrNull { it.startsWith("go ") } + ?.removePrefix("go ") + ?.substringBefore("//") + ?.trim() + + return GoModule(path, version?.split('.')?.getOrNull(1)?.toIntOrNull()) +} + +/** + * Detects Go project structure and settings in one pass. Walks [root] looking for `go.mod` files + * (each one becomes a component) and derives `GOOS`/`GOARCH` symbols from the target [environment]. + * Returns `null` if [root] does not look like a Go project at all. + */ +internal fun detectGo(root: Path, environment: TargetEnvironment): DetectionResult? { + // Walk for go.mod files to find all modules (supports monorepos with multiple modules). + val moduleRoots = + root + .toFile() + .walkTopDown() + .onEnter { !it.name.startsWith(".") && it.name !in SKIPPED_DIRECTORIES } + .filter { it.name == "go.mod" && it.isFile } + .map { it.parentFile.toPath() } + .toList() + + val looksLikeGo = + moduleRoots.isNotEmpty() || + root.resolve("go.work").exists() || + (root.isDirectory() && root.listDirectoryEntries("*.go").isNotEmpty()) + + if (!looksLikeGo) return null + + val components = + moduleRoots.map { moduleRoot -> + val goMod = moduleRoot.resolve("go.mod") + ComponentDefinition( + name = parseGoMod(goMod)?.name ?: moduleRoot.name, + root = moduleRoot, + ) + } + + val symbols = mutableMapOf() + val notes = mutableListOf() + environment.os.goos?.let { + symbols["GOOS"] = it + notes += "derived GOOS=$it from target environment" + } + environment.architecture.goarch?.let { + symbols["GOARCH"] = it + notes += "derived GOARCH=$it from target environment" + } + + return DetectionResult( + detector = "go", + components = components, + symbols = symbols, + notes = notes, + ) +} diff --git a/cpg-language-go/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/GoUtils.kt b/cpg-language-go/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/GoUtils.kt index 166666e989b..ac9fa447516 100644 --- a/cpg-language-go/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/GoUtils.kt +++ b/cpg-language-go/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/GoUtils.kt @@ -26,12 +26,7 @@ package de.fraunhofer.aisec.cpg.frontends.golang import de.fraunhofer.aisec.cpg.TranslationManager -import de.fraunhofer.aisec.cpg.TranslationResult -import de.fraunhofer.aisec.cpg.frontends.CompilationDatabase import java.io.File -import java.util.concurrent.TimeUnit -import org.slf4j.Logger -import org.slf4j.LoggerFactory /** * This functions checks whether the file specified in [file] should be processed in the @@ -143,130 +138,3 @@ internal fun gatherGoFiles(root: File, includeSubDir: Boolean = true): List = mutableMapOf() - - var components: MutableMap> = mutableMapOf() - - var includePaths: List = mutableListOf() - - var topLevel: File? = null - - companion object { - val log: Logger = LoggerFactory.getLogger(Project::class.java) - - /** - * This function emulates building a Go project. It requires an installed Go environment and - * uses the Go binary to compile a list of package dependencies, which are then included - * into the [Project] as includes. - * - * Note: This currently is limited to packages of the standard library - */ - fun buildProject( - modulePath: String, - goos: String? = null, - goarch: String? = null, - goVersion: Int? = null, - tags: MutableList = mutableListOf(), - ): Project { - val project = Project() - val symbols = mutableMapOf() - var files = mutableListOf() - - val topLevel = File(modulePath) - - val goModFile = topLevel.resolve("go.mod") - val module = - GoStandardLibrary.Modfile.parse(goModFile.absolutePath, goModFile.readText()) - - val pb = - ProcessBuilder("go", "list", "-deps", tags.joinToString(",", "-tags="), "all") - .directory(topLevel) - .redirectOutput(ProcessBuilder.Redirect.PIPE) - - val env = pb.environment() - env["GOOS"] = goos - env["GOARCH"] = goarch - - var proc = pb.start() - proc.waitFor(5, TimeUnit.MINUTES) - if (proc.exitValue() != 0) { - log.debug(proc.errorStream.bufferedReader().readLine()) - } - - // Read deps from standard input - val deps = proc.inputStream.bufferedReader().readLines() - - log.debug("Identified {} package dependencies (stdlib only)", deps.size) - - proc = - ProcessBuilder("go", "env", "GOROOT") - .redirectOutput(ProcessBuilder.Redirect.PIPE) - .start() - proc.waitFor(5, TimeUnit.MINUTES) - if (proc.exitValue() != 0) { - log.debug(proc.errorStream.bufferedReader().readLine()) - } - - val stdLib = File(proc.inputStream.bufferedReader().readLine()).resolve("src") - - log.debug("GOROOT/src is located @ {}", stdLib) - - // Build directories out of deps - val dirs: List = - deps.mapNotNull { - if (!it.contains(".")) { - // without a dot, it is a stdlib package - stdLib.resolve(it) - } else if (it.startsWith("vendor")) { - // if the dependency path starts with "vendor", then it is a dependency that - // is vendored within the standard library (and not in the project). we - // don't really include these - // for now since they blow up the stdlib - null - } else if (it.startsWith(module.module.mod.path)) { - topLevel.resolve(it.substringAfter(module.module.mod.path)) - } else { - // for all other dependencies, we try whether they are vendored within the - // current project. Note, this differs from the above case, where a - // dependency is vendored in the stdlib. - topLevel.resolve("vendor").resolve(it) - } - } - - files += dirs.flatMap { gatherGoFiles(it, false) } - // add cmd folder - files += gatherGoFiles(topLevel.resolve("cmd")) - - goos?.let { symbols["GOOS"] = it } - goarch?.let { symbols["GOARCH"] = it } - - if (goVersion != null) { - // Populate tags with go-version - for (i in 1..goVersion) { - tags += "go1.$i" - } - } - - tags.let { symbols["-tags"] = tags.joinToString(" ") } - - // Pre-filter any files we are not building anyway based on our symbols - files = files.filter { shouldBeBuild(it, symbols) }.toMutableList() - - // TODO(oxisto): look for binaries in cmd folder - project.components[TranslationResult.DEFAULT_APPLICATION_NAME] = files - project.symbols = symbols - // TODO(oxisto): support vendor includes - project.includePaths = listOf(stdLib, topLevel.resolve("vendor")) - project.topLevel = File(modulePath) - - return project - } - } -} diff --git a/cpg-language-go/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/GoProjectDetectionTest.kt b/cpg-language-go/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/GoProjectDetectionTest.kt new file mode 100644 index 00000000000..b1dfce66ae9 --- /dev/null +++ b/cpg-language-go/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/GoProjectDetectionTest.kt @@ -0,0 +1,83 @@ +/* + * Copyright (c) 2026, Fraunhofer AISEC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * $$$$$$\ $$$$$$$\ $$$$$$\ + * $$ __$$\ $$ __$$\ $$ __$$\ + * $$ / \__|$$ | $$ |$$ / \__| + * $$ | $$$$$$$ |$$ |$$$$\ + * $$ | $$ ____/ $$ |\_$$ | + * $$ | $$\ $$ | $$ | $$ | + * \$$$$$ |$$ | \$$$$$ | + * \______/ \__| \______/ + * + */ +package de.fraunhofer.aisec.cpg.frontends.golang + +import de.fraunhofer.aisec.cpg.project.Architecture +import de.fraunhofer.aisec.cpg.project.OperatingSystem +import de.fraunhofer.aisec.cpg.project.Project +import java.nio.file.Path +import kotlin.io.path.createDirectories +import kotlin.io.path.writeText +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertNotNull +import kotlin.test.assertTrue +import org.junit.jupiter.api.io.TempDir + +class GoProjectDetectionTest { + @Test + fun testDetectModules(@TempDir tmp: Path) { + tmp.resolve("go.mod").writeText("module example.com/app // main module\n\ngo 1.22\n") + tmp.resolve("main.go").writeText("package main") + + val worker = tmp.resolve("services/worker") + worker.createDirectories() + worker.resolve("go.mod").writeText("module example.com/worker\n") + worker.resolve("main.go").writeText("package main") + + val project = + Project.from(tmp) { + registerLanguage() + environment { + os = OperatingSystem.LINUX + architecture = Architecture.ARM64 + } + } + + // One component per detected Go module, named after the module path + assertEquals(listOf("app", "worker"), project.components.map { it.name }.sorted()) + assertEquals(tmp.toFile(), project.config.topLevels["app"]) + assertEquals(worker.toFile(), project.config.topLevels["worker"]) + + // GOOS/GOARCH are derived from the target environment, not the host + assertEquals("linux", project.config.symbols["GOOS"]) + assertEquals("arm64", project.config.symbols["GOARCH"]) + + val result = project.detectionResults.singleOrNull() + assertNotNull(result) + assertEquals("go", result.detector) + } + + @Test + fun testNoGoProject(@TempDir tmp: Path) { + tmp.resolve("README.md").writeText("nothing to see here") + + val project = Project.from(tmp) { registerLanguage() } + + assertTrue(project.detectionResults.isEmpty()) + assertEquals(listOf("application"), project.components.map { it.name }) + } +} diff --git a/cpg-mcp/build.gradle.kts b/cpg-mcp/build.gradle.kts index 389ec0219b0..73b39ba7e05 100644 --- a/cpg-mcp/build.gradle.kts +++ b/cpg-mcp/build.gradle.kts @@ -72,6 +72,10 @@ dependencies { // Test dependencies testImplementation("org.jetbrains.kotlinx:kotlinx-coroutines-test:1.10.2") + // We depend on the C/C++ frontend for testing project analysis with a compilation database, + // but the frontend is only available if enabled. The corresponding tests are skipped if it is + // not available. + findProject(":cpg-language-cxx")?.also { testImplementation(it) } // Command line interface support implementation(libs.clikt) diff --git a/cpg-mcp/src/main/kotlin/de/fraunhofer/aisec/cpg/mcp/SetupConfig.kt b/cpg-mcp/src/main/kotlin/de/fraunhofer/aisec/cpg/mcp/SetupConfig.kt deleted file mode 100644 index 921dcf2ac2f..00000000000 --- a/cpg-mcp/src/main/kotlin/de/fraunhofer/aisec/cpg/mcp/SetupConfig.kt +++ /dev/null @@ -1,115 +0,0 @@ -/* - * Copyright (c) 2025, Fraunhofer AISEC. All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * $$$$$$\ $$$$$$$\ $$$$$$\ - * $$ __$$\ $$ __$$\ $$ __$$\ - * $$ / \__|$$ | $$ |$$ / \__| - * $$ | $$$$$$$ |$$ |$$$$\ - * $$ | $$ ____/ $$ |\_$$ | - * $$ | $$\ $$ | $$ | $$ | - * \$$$$$ |$$ | \$$$$$ | - * \______/ \__| \______/ - * - */ -package de.fraunhofer.aisec.cpg.mcp - -import de.fraunhofer.aisec.cpg.InferenceConfiguration -import de.fraunhofer.aisec.cpg.TranslationConfiguration -import de.fraunhofer.aisec.cpg.passes.ControlDependenceGraphPass -import de.fraunhofer.aisec.cpg.passes.ControlFlowSensitiveDFGPass -import de.fraunhofer.aisec.cpg.passes.PrepareSerialization -import de.fraunhofer.aisec.cpg.passes.ProgramDependenceGraphPass -import de.fraunhofer.aisec.cpg.passes.concepts.file.python.PythonFileConceptPass -import java.io.File -import java.nio.file.Path -import java.nio.file.Paths -import kotlin.collections.forEach - -private const val DEBUG_PARSER = true - -/** Checks if all elements in the parameter are a valid file and returns a list of files. */ -private fun getFilesOfList(filenames: Collection): List { - val filePaths = filenames.map { Paths.get(it).toAbsolutePath().normalize().toFile() } - filePaths.forEach { require(it.exists()) { "Please use a correct path. It was: ${it.path}" } } - return filePaths -} - -fun setupTranslationConfiguration( - topLevel: File?, - files: Collection, - includePaths: List, - includesFile: File? = null, - maxComplexity: Int = -1, - loadIncludes: Boolean = true, - exclusionPatterns: Collection = listOf(), - useUnityBuild: Boolean = false, - runPasses: Boolean, -): TranslationConfiguration { - val translationConfiguration = - TranslationConfiguration.builder() - .optionalLanguage("de.fraunhofer.aisec.cpg.frontends.cxx.CLanguage") - .optionalLanguage("de.fraunhofer.aisec.cpg.frontends.cxx.CPPLanguage") - .optionalLanguage("de.fraunhofer.aisec.cpg.frontends.java.JavaLanguage") - .optionalLanguage("de.fraunhofer.aisec.cpg.frontends.golang.GoLanguage") - .optionalLanguage("de.fraunhofer.aisec.cpg.frontends.llvm.LLVMIRLanguage") - .optionalLanguage("de.fraunhofer.aisec.cpg.frontends.python.PythonLanguage") - .optionalLanguage("de.fraunhofer.aisec.cpg.frontends.typescript.TypeScriptLanguage") - .optionalLanguage("de.fraunhofer.aisec.cpg.frontends.ruby.RubyLanguage") - .optionalLanguage("de.fraunhofer.aisec.cpg.frontends.jvm.JVMLanguage") - .optionalLanguage("de.fraunhofer.aisec.cpg.frontends.ini.IniFileLanguage") - .loadIncludes(loadIncludes) - .exclusionPatterns(*exclusionPatterns.toTypedArray()) - .addIncludesToGraph(loadIncludes) - .debugParser(DEBUG_PARSER) - .useUnityBuild(useUnityBuild) - - topLevel?.let { translationConfiguration.topLevel(it) } - - if (maxComplexity != -1) { - translationConfiguration.configurePass( - ControlFlowSensitiveDFGPass.Configuration(maxComplexity = maxComplexity) - ) - } - - includePaths.forEach { translationConfiguration.includePath(it) } - - val filePaths = getFilesOfList(files) - translationConfiguration.sourceLocations(filePaths) - - if (runPasses) { - translationConfiguration.defaultPasses() - translationConfiguration.registerPass() - translationConfiguration.registerPass() - translationConfiguration.registerPass() - } - - translationConfiguration.registerPass(PrepareSerialization::class) - - includesFile?.let { theFile -> - val baseDir = File(theFile.toString()).parentFile?.toString() ?: "" - theFile - .inputStream() - .bufferedReader() - .lines() - .map(String::trim) - .map { if (Paths.get(it).isAbsolute) it else Paths.get(baseDir, it).toString() } - .forEach { translationConfiguration.includePath(it) } - } - - translationConfiguration.inferenceConfiguration( - InferenceConfiguration.builder().inferRecords(true).build() - ) - return translationConfiguration.build() -} diff --git a/cpg-mcp/src/main/kotlin/de/fraunhofer/aisec/cpg/mcp/mcpserver/tools/CpgAnalyzeTool.kt b/cpg-mcp/src/main/kotlin/de/fraunhofer/aisec/cpg/mcp/mcpserver/tools/CpgAnalyzeTool.kt index 893d9f8f662..a39f7073bf2 100644 --- a/cpg-mcp/src/main/kotlin/de/fraunhofer/aisec/cpg/mcp/mcpserver/tools/CpgAnalyzeTool.kt +++ b/cpg-mcp/src/main/kotlin/de/fraunhofer/aisec/cpg/mcp/mcpserver/tools/CpgAnalyzeTool.kt @@ -25,31 +25,6 @@ */ @file:Suppress("UNCHECKED_CAST") -/* - * Copyright (c) 2025, Fraunhofer AISEC. All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * $$$$$$\ $$$$$$$\ $$$$$$\ - * $$ __$$\ $$ __$$\ $$ __$$\ - * $$ / \__|$$ | $$ |$$ / \__| - * $$ | $$$$$$$ |$$ |$$$$\ - * $$ | $$ ____/ $$ |\_$$ | - * $$ | $$\ $$ | $$ | $$ | - * \$$$$$ |$$ | \$$$$$ | - * \______/ \__| \______/ - * - */ package de.fraunhofer.aisec.cpg.mcp.mcpserver.tools import de.fraunhofer.aisec.cpg.* @@ -71,7 +46,6 @@ import de.fraunhofer.aisec.cpg.mcp.mcpserver.tools.utils.PassInfo import de.fraunhofer.aisec.cpg.mcp.mcpserver.tools.utils.addTool import de.fraunhofer.aisec.cpg.mcp.mcpserver.tools.utils.toObject import de.fraunhofer.aisec.cpg.mcp.mcpserver.tools.utils.toSchema -import de.fraunhofer.aisec.cpg.mcp.setupTranslationConfiguration import de.fraunhofer.aisec.cpg.passes.BasicBlockCollectorPass import de.fraunhofer.aisec.cpg.passes.ComponentPass import de.fraunhofer.aisec.cpg.passes.ControlDependenceGraphPass @@ -93,18 +67,22 @@ import de.fraunhofer.aisec.cpg.passes.TranslationUnitPass import de.fraunhofer.aisec.cpg.passes.TypeHierarchyResolver import de.fraunhofer.aisec.cpg.passes.TypeResolver import de.fraunhofer.aisec.cpg.passes.briefDescription +import de.fraunhofer.aisec.cpg.passes.concepts.file.python.PythonFileConceptPass import de.fraunhofer.aisec.cpg.passes.configuration.PassOrderingHelper import de.fraunhofer.aisec.cpg.passes.configuration.ReplacePass import de.fraunhofer.aisec.cpg.passes.consumeTargets import de.fraunhofer.aisec.cpg.passes.hardDependencies import de.fraunhofer.aisec.cpg.passes.softDependencies +import de.fraunhofer.aisec.cpg.project.Project import io.modelcontextprotocol.kotlin.sdk.server.Server import io.modelcontextprotocol.kotlin.sdk.types.CallToolResult import io.modelcontextprotocol.kotlin.sdk.types.TextContent import io.modelcontextprotocol.kotlin.sdk.types.ToolSchema import java.io.File +import java.nio.file.Paths import java.util.IdentityHashMap import kotlin.String +import kotlin.io.path.exists import kotlin.reflect.KClass import kotlin.reflect.full.findAnnotations import kotlin.reflect.full.primaryConstructor @@ -119,15 +97,21 @@ var ctx: TranslationContext? = null val toolDescription = """ Analyze source code using CPG (Code Property Graph). - + $cpgDescription - - This tool parses source code and creates a comprehensive graph representation + + This tool parses source code and creates a comprehensive graph representation containing all nodes, functions, variables, and call expressions. - + + It can either analyze a small code snippet (using 'content') or a file or whole + project directory on the local filesystem (using 'path'). For project directories, + the project structure is detected automatically, e.g., components based on Go + modules or a C/C++ compilation database (compile_commands.json). + Example usage: - "Analyze this code: print('hello')" - "Analyze this uploaded file" + - "Analyze the project in /path/to/repo" """ .trimIndent() @@ -162,8 +146,14 @@ fun runCpgAnalyze( runPasses: Boolean, cleanup: Boolean, ): CpgAnalysisResult { - val file = + val path = when { + payload?.path != null -> { + val path = Paths.get(payload.path).toAbsolutePath().normalize() + require(path.exists()) { "Please use a correct path. It was: $path" } + path + } + payload?.content != null -> { val extension = if (payload.extension != null) { @@ -178,20 +168,32 @@ fun runCpgAnalyze( val tempFile = File.createTempFile("cpg_analysis", extension) tempFile.writeText(payload.content) tempFile.deleteOnExit() - tempFile + tempFile.toPath() } - else -> throw IllegalArgumentException("Must provide content") + else -> throw IllegalArgumentException("Must provide either a path or content") } - val config = - setupTranslationConfiguration( - topLevel = file, - files = listOf(file.absolutePath), - includePaths = emptyList(), - runPasses = runPasses, - ) - config.disableCleanup = !cleanup + val project = + Project.from(path) { + if (!runPasses) passes { } + translation { + it.debugParser(true) + it.loadIncludes(true) + it.addIncludesToGraph(true) + it.inferenceConfiguration( + InferenceConfiguration.builder().inferRecords(true).build() + ) + + if (runPasses) { + it.registerPass() + it.registerPass() + it.registerPass() + } + it.registerPass() + } + } + project.config.disableCleanup = !cleanup if (ctx != null) { ctx?.executedFrontends?.forEach { frontend -> @@ -203,11 +205,8 @@ fun runCpgAnalyze( ctx = null } - val analyzer = TranslationManager.builder().config(config).build() - ctx = TranslationContext(config) - val result = - ctx?.let { ctx -> analyzer.analyze(ctx).get() } - ?: throw IllegalStateException("Translation context is not initialized") + val result = project.analyze() + ctx = result.ctx // Store the result globally globalAnalysisResult = result @@ -222,6 +221,11 @@ fun runCpgAnalyze( functions = functions.size, variables = variables.size, callExpressions = callExpressions.size, + components = project.components.map { it.name }, + detectionNotes = + project.detectionResults.flatMap { result -> + result.notes.map { "${result.detector}: $it" } + }, ) } @@ -238,15 +242,21 @@ fun Server.addCpgTranslate() { description = """ Translates the source code into the AST of the CPG (Code Property Graph). This serves as a basis for subsequent passes and analyses. - + $cpgDescription - - This tool parses source code and creates a comprehensive graph representation + + This tool parses source code and creates a comprehensive graph representation containing all nodes, functions, variables, and call expressions. - + + It can either translate a small code snippet (using 'content') or a file or whole + project directory on the local filesystem (using 'path'). For project directories, + the project structure is detected automatically, e.g., components based on Go + modules or a C/C++ compilation database (compile_commands.json). + Example usage: - "Analyze this code: print('hello')" - "Analyze this uploaded file" + - "Analyze the project in /path/to/repo" """ .trimIndent(), inputSchema = CpgAnalyzePayload::class.toSchema(), diff --git a/cpg-mcp/src/main/kotlin/de/fraunhofer/aisec/cpg/mcp/mcpserver/tools/utils/Payloads.kt b/cpg-mcp/src/main/kotlin/de/fraunhofer/aisec/cpg/mcp/mcpserver/tools/utils/Payloads.kt index 3c8361910ea..4de807c9c6a 100644 --- a/cpg-mcp/src/main/kotlin/de/fraunhofer/aisec/cpg/mcp/mcpserver/tools/utils/Payloads.kt +++ b/cpg-mcp/src/main/kotlin/de/fraunhofer/aisec/cpg/mcp/mcpserver/tools/utils/Payloads.kt @@ -30,11 +30,18 @@ import kotlinx.serialization.Serializable @Serializable data class CpgAnalyzePayload( - @Description("The contents of the file which should be analyzed.") val content: String? = null, @Description( - "The file extension. This is required to identify the programming language and should resemble the typical file ending (e.g. '.py' for python, '.c' for C code)." + "The contents of the file which should be analyzed. Alternatively, 'path' can be used to analyze files or whole projects on the local filesystem." + ) + val content: String? = null, + @Description( + "The file extension. This is required to identify the programming language when providing 'content' and should resemble the typical file ending (e.g. '.py' for python, '.c' for C code)." ) val extension: String? = null, + @Description( + "The path to a source file or a project directory (e.g., a repository checkout) on the local filesystem. For directories, the project structure is detected automatically, e.g., components based on Go modules or a C/C++ compilation database (compile_commands.json). Either 'path' or 'content' must be provided." + ) + val path: String? = null, ) @Serializable diff --git a/cpg-mcp/src/main/kotlin/de/fraunhofer/aisec/cpg/mcp/mcpserver/tools/utils/Serializable.kt b/cpg-mcp/src/main/kotlin/de/fraunhofer/aisec/cpg/mcp/mcpserver/tools/utils/Serializable.kt index 4a79424eb33..8bc7adebb43 100644 --- a/cpg-mcp/src/main/kotlin/de/fraunhofer/aisec/cpg/mcp/mcpserver/tools/utils/Serializable.kt +++ b/cpg-mcp/src/main/kotlin/de/fraunhofer/aisec/cpg/mcp/mcpserver/tools/utils/Serializable.kt @@ -216,6 +216,10 @@ data class CpgAnalysisResult( val functions: Int, val variables: Int, val callExpressions: Int, + /** The names of the components of the analyzed project. */ + val components: List = listOf(), + /** Notes about what the project auto-detection recognized, e.g., a compilation database. */ + val detectionNotes: List = listOf(), ) @Serializable diff --git a/cpg-mcp/src/test/kotlin/de/fraunhofer/aisec/cpg/mcp/CpgAnalyzeProjectTest.kt b/cpg-mcp/src/test/kotlin/de/fraunhofer/aisec/cpg/mcp/CpgAnalyzeProjectTest.kt new file mode 100644 index 00000000000..2c9f23f49d0 --- /dev/null +++ b/cpg-mcp/src/test/kotlin/de/fraunhofer/aisec/cpg/mcp/CpgAnalyzeProjectTest.kt @@ -0,0 +1,106 @@ +/* + * Copyright (c) 2026, Fraunhofer AISEC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * $$$$$$\ $$$$$$$\ $$$$$$\ + * $$ __$$\ $$ __$$\ $$ __$$\ + * $$ / \__|$$ | $$ |$$ / \__| + * $$ | $$$$$$$ |$$ |$$$$\ + * $$ | $$ ____/ $$ |\_$$ | + * $$ | $$\ $$ | $$ | $$ | + * \$$$$$ |$$ | \$$$$$ | + * \______/ \__| \______/ + * + */ +package de.fraunhofer.aisec.cpg.mcp + +import de.fraunhofer.aisec.cpg.mcp.mcpserver.tools.runCpgAnalyze +import de.fraunhofer.aisec.cpg.mcp.mcpserver.tools.utils.CpgAnalyzePayload +import java.nio.file.Path +import kotlin.io.path.createDirectories +import kotlin.io.path.writeText +import kotlin.test.Test +import kotlin.test.assertContains +import kotlin.test.assertEquals +import kotlin.test.assertFailsWith +import kotlin.test.assertTrue +import org.junit.jupiter.api.Assumptions.assumeTrue +import org.junit.jupiter.api.io.TempDir + +class CpgAnalyzeProjectTest { + + @Test + fun testAnalyzeProjectWithCompilationDatabase(@TempDir tmp: Path) { + assumeTrue( + runCatching { Class.forName("de.fraunhofer.aisec.cpg.frontends.cxx.CLanguage") } + .isSuccess, + "The C/C++ frontend is not on the classpath", + ) + + val libFoo = tmp.resolve("src/libfoo") + val tool = tmp.resolve("src/tool") + libFoo.createDirectories() + tool.createDirectories() + libFoo.resolve("foo.c").writeText("int foo() { return 1; }") + tool.resolve("main.c").writeText("int main() { return foo(); }") + + tmp.resolve("compile_commands.json") + .writeText( + """ + [ + { + "directory": "$libFoo", + "command": "gcc -c foo.c", + "file": "$libFoo/foo.c", + "output": "foo.o" + }, + { + "directory": "$tool", + "command": "gcc -c main.c", + "file": "$tool/main.c", + "output": "main.o" + } + ] + """ + .trimIndent() + ) + + val result = + runCpgAnalyze( + CpgAnalyzePayload(path = tmp.toString()), + runPasses = true, + cleanup = true, + ) + + // The components are derived from the compilation database + assertEquals(listOf("libfoo", "tool"), result.components.sorted()) + assertTrue( + result.detectionNotes.any { it.startsWith("compile_commands.json:") }, + "expected a detection note about the compilation database, but got ${result.detectionNotes}", + ) + assertEquals(2, result.functions) + assertContains(1..Int.MAX_VALUE, result.callExpressions) + } + + @Test + fun testAnalyzeInvalidPath() { + assertFailsWith { + runCpgAnalyze( + CpgAnalyzePayload(path = "/does/not/exist"), + runPasses = false, + cleanup = true, + ) + } + } +} diff --git a/cpg-neo4j/src/main/kotlin/de/fraunhofer/aisec/cpg_vis_neo4j/Application.kt b/cpg-neo4j/src/main/kotlin/de/fraunhofer/aisec/cpg_vis_neo4j/Application.kt index d27873eb60e..1bfc7ec06e5 100644 --- a/cpg-neo4j/src/main/kotlin/de/fraunhofer/aisec/cpg_vis_neo4j/Application.kt +++ b/cpg-neo4j/src/main/kotlin/de/fraunhofer/aisec/cpg_vis_neo4j/Application.kt @@ -32,6 +32,7 @@ import de.fraunhofer.aisec.cpg.passes.concepts.file.python.PythonFileConceptPass import de.fraunhofer.aisec.cpg.persistence.Neo4jConnectionDefaults import de.fraunhofer.aisec.cpg.persistence.persistJson import de.fraunhofer.aisec.cpg.persistence.pushToNeo4j +import de.fraunhofer.aisec.cpg.project.Project import java.io.File import java.net.ConnectException import java.nio.file.Paths @@ -301,16 +302,9 @@ class Application : Callable { fun setupTranslationConfiguration(): TranslationConfiguration { val translationConfiguration = TranslationConfiguration.builder() - .optionalLanguage("de.fraunhofer.aisec.cpg.frontends.cxx.CLanguage") - .optionalLanguage("de.fraunhofer.aisec.cpg.frontends.cxx.CPPLanguage") - .optionalLanguage("de.fraunhofer.aisec.cpg.frontends.java.JavaLanguage") - .optionalLanguage("de.fraunhofer.aisec.cpg.frontends.golang.GoLanguage") - .optionalLanguage("de.fraunhofer.aisec.cpg.frontends.llvm.LLVMIRLanguage") - .optionalLanguage("de.fraunhofer.aisec.cpg.frontends.python.PythonLanguage") - .optionalLanguage("de.fraunhofer.aisec.cpg.frontends.typescript.TypeScriptLanguage") - .optionalLanguage("de.fraunhofer.aisec.cpg.frontends.ruby.RubyLanguage") - .optionalLanguage("de.fraunhofer.aisec.cpg.frontends.jvm.JVMLanguage") - .optionalLanguage("de.fraunhofer.aisec.cpg.frontends.ini.IniFileLanguage") + .also { builder -> + Project.defaultLanguages.forEach { builder.optionalLanguage(it) } + } .loadIncludes(loadIncludes) .exclusionPatterns(*exclusionPatterns.toTypedArray()) .addIncludesToGraph(loadIncludes) diff --git a/docs/docs/GettingStarted/project-api.md b/docs/docs/GettingStarted/project-api.md new file mode 100644 index 00000000000..d5932855e16 --- /dev/null +++ b/docs/docs/GettingStarted/project-api.md @@ -0,0 +1,185 @@ +--- +title: "Project API" +linkTitle: "Project API" +weight: 2 +date: 2026-01-01 +description: > + High-level API for configuring and running CPG analyses +--- + +The **Project API** is the recommended, high-level entry point for building a CPG from source code. +It handles language registration, pass configuration, and component detection automatically, so you +only need to override the parts that differ from the defaults. + +## Quick start + +The simplest possible analysis — pass a directory and get a result: + +```kotlin +val result = project(Path("/path/to/repo")) { }.analyze() +``` + +This **auto-mode** uses every language frontend available on the classpath, the full default pass +pipeline, and runs all registered auto-detectors to discover the project structure (Go modules, +C/C++ compilation databases, etc.). + +For a single file you don't need a project at all: + +```kotlin +val result = Project.from(Path("main.cpp")).analyze() +``` + +## Auto-mode vs. explicit override + +All three main configuration blocks — `languages {}`, `passes {}`, and `components {}` — follow the +same pattern: + +| Block | Not called | Called without `default()` | Called with `default()` | +|---|---|---|---| +| `languages {}` | All default languages on the classpath | Only the listed languages | Default languages + listed languages | +| `passes {}` | Default pass pipeline | Only the listed passes | Default passes + listed passes | +| `components {}` | Language detectors run, auto-detect | Explicit components only, no detection | Auto-detect + explicit components | + +## Languages + +```kotlin +// Auto-mode: every language frontend available on the classpath is used. +project(path) { } + +// Explicit: only Go is registered. +project(path) { + languages { use() } +} + +// Combination: all defaults plus an extra language (e.g., an in-house frontend). +project(path) { + languages { default(); use() } +} +``` + +## Passes + +```kotlin +// Auto-mode: the full default pass pipeline runs. +project(path) { } + +// No default passes; only the explicitly listed ones run. +// Useful for raw-AST inspection or very fast single-pass analyses. +project(path) { + passes { use() } +} + +// Default passes plus a custom one. +project(path) { + passes { default(); use() } +} +``` + +## Components + +A *component* maps to a [`Component`](../CPG/specs/graph.md) node in the graph. By default the +project auto-detects components through language-specific detectors (e.g., `go.mod` files for Go, +`compile_commands.json` for C/C++). + +```kotlin +// Auto-mode: detectors discover the components. +project(path) { } + +// Explicit: a single "backend" component — no detection runs. +project(path) { + components { + component("backend", root = path.resolve("services/backend")) + } +} + +// Auto-detect AND add an extra component (e.g., a generated stub directory). +project(path) { + components { + default() + component("stubs", root = path.resolve("generated/stubs")) + } +} + +// Disable auto-detection entirely (empty block). +project(path) { + components { } +} +``` + +### Standalone detectors + +[`DirectoryComponentDetector`][1] creates one component per direct subdirectory of a given folder. +It is useful for convention-based monorepos: + +```kotlin +project(path) { + components { + detector(DirectoryComponentDetector("services")) + } +} +``` + +Adding a `detector()` inside the block automatically enables auto-detection (equivalent to calling +`default()` first). + +## Environment + +Specify the target OS and architecture when analysing cross-compiled code: + +```kotlin +project(path) { + environment { + os = OperatingSystem.LINUX + architecture = Architecture.ARM64 + } +} +``` + +The environment is forwarded to [detectors][2] so they can derive the correct build +constraints (e.g., `GOOS`/`GOARCH` symbols for Go). + +## Exclusions + +```kotlin +project(path) { + exclude("vendor", "testdata", "node_modules") + exclude(Regex(".*_test\\.go")) +} +``` + +## Low-level escape hatch + +For options not yet exposed by the Project API, use `translation {}` to modify the underlying +[`TranslationConfiguration`](library.md) directly. This modifier runs after all project-level +settings, so it takes precedence: + +```kotlin +project(path) { + translation { + it.loadIncludes(true) + it.registerPass() + it.inferenceConfiguration( + InferenceConfiguration.builder().inferRecords(true).build() + ) + } +} +``` + +## Inspecting the resolved project + +`Project.from()` / `project()` return a `Project` object *before* the analysis runs. You can +inspect the resolved configuration — for example, to verify which components were detected — before +calling `analyze()`: + +```kotlin +val p = project(Path("/path/to/repo")) { } + +println("Components: ${p.components.map { it.name }}") +println("Detection notes:") +p.detectionResults.flatMap { it.notes }.forEach { println(" $it") } + +val result = p.analyze() +``` + +[1]: ../../API/de.fraunhofer.aisec.cpg.project/-directory-component-detector/index.html +[2]: ../../API/de.fraunhofer.aisec.cpg.project/-detector/index.html diff --git a/docs/mkdocs.yaml b/docs/mkdocs.yaml index cbe4b09bae0..db907f2bd0e 100755 --- a/docs/mkdocs.yaml +++ b/docs/mkdocs.yaml @@ -160,6 +160,7 @@ nav: - "Using Codyze": GettingStarted/codyze.md - "Usage as library": - "Adding the Dependency": GettingStarted/library.md + - "Project API": GettingStarted/project-api.md - "Using the Query API": GettingStarted/query.md - "Shortcuts to Explore the Graph": GettingStarted/shortcuts.md - "Graph Traversal (followXXX)":