From 33c35919c636ed7245fb0b92d34aa731de5cefae Mon Sep 17 00:00:00 2001 From: Christian Banse Date: Fri, 17 Apr 2026 16:28:27 +0200 Subject: [PATCH 01/10] Go (speed) improvements --- .../aisec/cpg/graph/edges/EdgeWalker.kt | 2 +- .../aisec/cpg/graph/types/HasType.kt | 48 ++++++++++++------- .../aisec/cpg/graph/types/PointerType.kt | 6 ++- .../aisec/cpg/helpers/SubgraphWalker.kt | 6 ++- .../frontends/golang/GoLanguageFrontend.kt | 16 ++++++- 5 files changed, 57 insertions(+), 21 deletions(-) diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/EdgeWalker.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/EdgeWalker.kt index d6df07a3c4d..b2889bef8fc 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/EdgeWalker.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/EdgeWalker.kt @@ -56,7 +56,7 @@ inline fun > Node.edges( // Gather all edges if (obj is EdgeCollection<*, *>) { - for (edge in obj.toList()) { + for (edge in obj) { if (edge is EdgeType && predicate.invoke(edge)) { edges += edge } diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/types/HasType.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/types/HasType.kt index 9c0b1a33cb9..1369b04e783 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/types/HasType.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/types/HasType.kt @@ -39,6 +39,9 @@ import de.fraunhofer.aisec.cpg.graph.expressions.Reference import de.fraunhofer.aisec.cpg.graph.expressions.UnaryOperator import de.fraunhofer.aisec.cpg.graph.unknownType +/** Thread-local set to track nodes currently propagating types to prevent infinite recursion */ +private val propagatingNodes = ThreadLocal.withInitial { mutableSetOf() } + /** * This interfaces denotes that the given [Node] has a "type". Currently, we only have two known * implementations of this class, an [Expression] and a [ValueDeclaration]. All other nodes with @@ -195,24 +198,35 @@ interface HasType : LanguageProvider { return } - if (changeType == TypeObserver.ChangeType.ASSIGNED_TYPE) { - val assignedTypes = this.assignedTypes - if (assignedTypes.isEmpty()) { - return - } - // Inform all type observers about the changes - for (observer in typeObservers) { - observer.assignedTypeChanged(assignedTypes, this) - } - } else { - val newType = this.type - if (newType is UnknownType) { - return - } - // Inform all type observers about the changes - for (observer in typeObservers) { - observer.typeChanged(newType, this) + // Guard against infinite recursion in type propagation cycles + val currentlyPropagating = propagatingNodes.get() + if (this in currentlyPropagating) { + return + } + + currentlyPropagating.add(this) + try { + if (changeType == TypeObserver.ChangeType.ASSIGNED_TYPE) { + val assignedTypes = this.assignedTypes + if (assignedTypes.isEmpty()) { + return + } + // Inform all type observers about the changes + for (observer in typeObservers) { + observer.assignedTypeChanged(assignedTypes, this) + } + } else { + val newType = this.type + if (newType is UnknownType) { + return + } + // Inform all type observers about the changes + for (observer in typeObservers) { + observer.typeChanged(newType, this) + } } + } finally { + currentlyPropagating.remove(this) } } diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/types/PointerType.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/types/PointerType.kt index c1a730c84e6..4314885adea 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/types/PointerType.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/types/PointerType.kt @@ -104,5 +104,9 @@ class PointerType : Type, SecondOrderType { pointerOrigin == other.pointerOrigin } - override fun hashCode() = Objects.hash(super.hashCode(), elementType, pointerOrigin) + override fun hashCode(): Int { + // Use System.identityHashCode for elementType to avoid infinite recursion + // when dealing with circular type references (e.g., struct with pointer to itself) + return Objects.hash(super.hashCode(), System.identityHashCode(elementType), pointerOrigin) + } } diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/helpers/SubgraphWalker.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/helpers/SubgraphWalker.kt index 4098a7ee2fe..aba2d2d5540 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/helpers/SubgraphWalker.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/helpers/SubgraphWalker.kt @@ -125,7 +125,11 @@ object SubgraphWalker { when (obj) { is EdgeCollection<*, *> -> { - obj.toNodeCollection({ it is AstEdge<*> }).filterIsInstanceTo(children) + for (edge in obj) { + if (edge is AstEdge<*>) { + children.add(edge.end) + } + } } else -> { throw AnnotationFormatError( diff --git a/cpg-language-go/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/GoLanguageFrontend.kt b/cpg-language-go/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/GoLanguageFrontend.kt index 840025feb84..2c68cb25f61 100644 --- a/cpg-language-go/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/GoLanguageFrontend.kt +++ b/cpg-language-go/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/GoLanguageFrontend.kt @@ -28,6 +28,7 @@ package de.fraunhofer.aisec.cpg.frontends.golang import de.fraunhofer.aisec.cpg.TranslationContext import de.fraunhofer.aisec.cpg.frontends.Language import de.fraunhofer.aisec.cpg.frontends.LanguageFrontend +import de.fraunhofer.aisec.cpg.frontends.SupportsNewParse import de.fraunhofer.aisec.cpg.frontends.SupportsParallelParsing import de.fraunhofer.aisec.cpg.frontends.TranslationException import de.fraunhofer.aisec.cpg.frontends.golang.GoStandardLibrary.Modfile @@ -52,6 +53,7 @@ import de.fraunhofer.aisec.cpg.sarif.PhysicalLocation import de.fraunhofer.aisec.cpg.sarif.Region import java.io.File import java.net.URI +import java.nio.file.Path /** * A language frontend for the [GoLanguage]. It makes use the internal @@ -67,7 +69,8 @@ import java.net.URI ) @SupportsParallelParsing(false) class GoLanguageFrontend(ctx: TranslationContext, language: Language) : - LanguageFrontend(ctx, language) { + LanguageFrontend(ctx, language), + SupportsNewParse { private var currentFileSet: GoStandardLibrary.Ast.FileSet? = null private var currentModule: GoStandardLibrary.Modfile.File? = null @@ -253,6 +256,17 @@ class GoLanguageFrontend(ctx: TranslationContext, language: Language Date: Fri, 17 Apr 2026 18:38:31 +0200 Subject: [PATCH 02/10] make Go frontend parallel --- .../de/fraunhofer/aisec/cpg/ScopeManager.kt | 11 +++++++ .../frontends/golang/GoLanguageFrontend.kt | 2 +- .../golang/GoLanguageFrontendTest.kt | 30 ++++++++----------- 3 files changed, 24 insertions(+), 19 deletions(-) diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/ScopeManager.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/ScopeManager.kt index c3b82d6ea64..927e5c27b9b 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/ScopeManager.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/ScopeManager.kt @@ -151,6 +151,17 @@ class ScopeManager(override var ctx: TranslationContext) : ScopeProvider, Contex // also update the AST node of the existing scope to the "latest" we have seen existing.astNode = entry.value.astNode + // Re-parent the children of the duplicate scope (entry.value) to the surviving + // scope (existing). Without this, inner scopes such as FunctionScopes whose + // parent still points to the stale duplicate scope would miss symbols that are + // only declared in the surviving scope when walking the parent chain during + // symbol resolution. + for (child in entry.value.children) { + child.parent = existing + existing.children.add(child) + } + entry.value.children.clear() + // now it gets more tricky. we also need to "redirect" the AST nodes in the sub // scope manager to our existing NameScope (currently, they point to their own, // invalid copy of the NameScope). diff --git a/cpg-language-go/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/GoLanguageFrontend.kt b/cpg-language-go/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/GoLanguageFrontend.kt index 2c68cb25f61..62e5634411c 100644 --- a/cpg-language-go/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/GoLanguageFrontend.kt +++ b/cpg-language-go/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/GoLanguageFrontend.kt @@ -67,7 +67,7 @@ import java.nio.file.Path old = EvaluationOrderGraphPass::class, with = GoEvaluationOrderGraphPass::class, ) -@SupportsParallelParsing(false) +@SupportsParallelParsing(true) class GoLanguageFrontend(ctx: TranslationContext, language: Language) : LanguageFrontend(ctx, language), SupportsNewParse { diff --git a/cpg-language-go/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/GoLanguageFrontendTest.kt b/cpg-language-go/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/GoLanguageFrontendTest.kt index c5ca376f61c..f174a1594f8 100644 --- a/cpg-language-go/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/GoLanguageFrontendTest.kt +++ b/cpg-language-go/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/GoLanguageFrontendTest.kt @@ -190,10 +190,7 @@ class GoLanguageFrontendTest : BaseTest() { } assertNotNull(result) - val tu = result.components["application"]?.translationUnits?.firstOrNull() - assertNotNull(tu) - - val p = tu.namespaces["p"] + val p = result.namespaces["p"] assertNotNull(p) val a = p.variables["a"] @@ -201,26 +198,26 @@ class GoLanguageFrontendTest : BaseTest() { assertNotNull(a.location) assertLocalName("a", a) - assertEquals(tu.primitiveType("int"), a.type) + assertEquals(result.primitiveType("int"), a.type) val s = p.variables["s"] assertNotNull(s) assertLocalName("s", s) - assertEquals(tu.primitiveType("string"), s.type) + assertEquals(result.primitiveType("string"), s.type) val f = p.variables["f"] assertNotNull(f) assertLocalName("f", f) - assertEquals(tu.primitiveType("float64"), f.type) + assertEquals(result.primitiveType("float64"), f.type) val f32 = p.variables["f32"] assertNotNull(f32) assertLocalName("f32", f32) - assertEquals(tu.primitiveType("float32"), f32.type) + assertEquals(result.primitiveType("float32"), f32.type) val n = p.variables["n"] assertNotNull(n) - assertEquals(tu.primitiveType("int").pointer(), n.type) + assertEquals(result.primitiveType("int").pointer(), n.type) val nil = n.initializer as? Literal<*> assertNotNull(nil) @@ -243,7 +240,7 @@ class GoLanguageFrontendTest : BaseTest() { assertNotNull(o) assertLocalName("MyStruct[]", o.type) - val myStruct = tu.records["MyStruct"] + val myStruct = result.records["MyStruct"] assertNotNull(myStruct) val field = myStruct.fields["Field"] @@ -277,7 +274,7 @@ class GoLanguageFrontendTest : BaseTest() { assertRefersTo(keyValue.key, field) assertLiteralValue(10, keyValue.value) - val rr = tu.variables["rr"] + val rr = result.variables["rr"] assertNotNull(rr) var init = rr.initializer @@ -293,7 +290,7 @@ class GoLanguageFrontendTest : BaseTest() { assertNotNull(zero) assertRefersTo(key, zero) - val mapr = tu.variables["mapr"] + val mapr = result.variables["mapr"] assertNotNull(mapr) init = mapr.initializer @@ -968,8 +965,8 @@ class GoLanguageFrontendTest : BaseTest() { fun testResolveStdLibImport() { val stdLib = Path.of("src", "test", "resources", "golang-std") val topLevel = Path.of("src", "test", "resources", "golang") - val tu = - analyzeAndGetFirstTU( + val result = + analyze( listOf(topLevel.resolve("function.go").toFile(), stdLib.resolve("fmt").toFile()), topLevel, true, @@ -978,9 +975,7 @@ class GoLanguageFrontendTest : BaseTest() { it.includePath(stdLib) } - assertNotNull(tu) - - val p = tu.namespaces["p"] + val p = result.namespaces["p"] assertNotNull(p) val main = p.functions["main"] @@ -1018,7 +1013,6 @@ class GoLanguageFrontendTest : BaseTest() { analyze(files, topLevel, true) { it.registerLanguage() it.includePath(stdLib) - it.useParallelFrontends(false) it.symbols(mapOf("GOOS" to "darwin", "GOARCH" to "arm64")) } From 67be90efc3fa064b5f79082b6e4b4ad9e8e60da3 Mon Sep 17 00:00:00 2001 From: Alexander Kuechler Date: Wed, 27 May 2026 14:35:40 +0200 Subject: [PATCH 03/10] Identity map in symbol resolver to speed up things --- .../kotlin/de/fraunhofer/aisec/cpg/passes/SymbolResolver.kt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/SymbolResolver.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/SymbolResolver.kt index f8efcb5e68b..1180ee6f9f9 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/SymbolResolver.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/SymbolResolver.kt @@ -49,6 +49,7 @@ import de.fraunhofer.aisec.cpg.passes.inference.tryFunctionInferenceFromFunction import de.fraunhofer.aisec.cpg.passes.inference.tryVariableInference import de.fraunhofer.aisec.cpg.processing.IVisitor import de.fraunhofer.aisec.cpg.processing.strategy.Strategy +import java.util.IdentityHashMap import kotlin.collections.firstOrNull import org.slf4j.Logger import org.slf4j.LoggerFactory @@ -721,7 +722,7 @@ open class SymbolResolver(ctx: TranslationContext) : EOGStarterPass(ctx) { companion object { val LOGGER: Logger = LoggerFactory.getLogger(SymbolResolver::class.java) - val componentsToTemplates = mutableMapOf>() + val componentsToTemplates = IdentityHashMap>() /** * Adds implicit duplicates of the TemplateParams to the implicit Construction From 5fc10cd1550d7b563a2f15eabf3648385bb39146 Mon Sep 17 00:00:00 2001 From: Alexander Kuechler Date: Wed, 27 May 2026 14:36:06 +0200 Subject: [PATCH 04/10] Loop-detection in ast parent traversal --- .../src/main/kotlin/de/fraunhofer/aisec/cpg/graph/Extensions.kt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/Extensions.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/Extensions.kt index 535982c20b4..ceb1de8d6d9 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/Extensions.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/Extensions.kt @@ -1525,6 +1525,7 @@ val AstNode?.assigns: List inline fun Node.firstParentOrNull( noinline predicate: ((T) -> Boolean)? = null ): T? { + val alreadySeen = identitySetOf() // start at searchNodes parent var node = this.astParent @@ -1535,6 +1536,7 @@ inline fun Node.firstParentOrNull( // go upwards in the ast tree node = node.astParent + if (node == null || !alreadySeen.add(node)) return null } return null From f42e3dbe35fec2d43f1a18c5017e3f71af996209 Mon Sep 17 00:00:00 2001 From: Alexander Kuechler Date: Wed, 27 May 2026 16:15:22 +0200 Subject: [PATCH 05/10] Speed improvements in Points to Pass (no AI!) --- .../aisec/cpg/passes/PointsToPass.kt | 256 ++++++++---------- 1 file changed, 114 insertions(+), 142 deletions(-) diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/PointsToPass.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/PointsToPass.kt index 3e86c011584..1a145054986 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/PointsToPass.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/PointsToPass.kt @@ -2053,32 +2053,27 @@ open class PointsToPass(ctx: TranslationContext) : EOGStarterPass(ctx, orderDepe ) .mapTo(identitySetOf()) { it.first } else identitySetOf(currentNode.arguments[srcNode.argumentIndex]) - values.forEach { value -> - destinationAddresses - .filter { it.first != null } - .forEach { (d, partialWrite) -> - // The extracted value might come from a state we - // created for a short function summary. If so, we - // have to store that info in the map - val updatedPropertySet = - equalLinkedHashSetOf().apply { - addAll(propertySet) - add(shortFS) - } - partialWrite?.let { - updatedPropertySet.add(PartialDataflowGranularity(it)) + + destinationAddresses + .filter { it.first != null } + .forEach { (d, partialWrite) -> + // The extracted value might come from a state we + // created for a short function summary. If so, we + // have to store that info in the map + val updatedPropertySet = + createPropertySet(propertySet, shortFS, partialWrite) + val currentSet = + mapDstToSrc.computeIfAbsent(d!!) { concurrentIdentitySetOf() } + + val filteredCurrentSet = + currentSet.filter { + it.lastWrites.parallelEquals( + PowersetLattice.Element(lastWrites.singleOrNull()) + ) && it.propertySet == updatedPropertySet } - val currentSet = - mapDstToSrc.computeIfAbsent(d!!) { concurrentIdentitySetOf() } - if ( - currentSet.none { - it.srcNode === value && - it.lastWrites.parallelEquals( - PowersetLattice.Element(lastWrites.singleOrNull()) - ) && - it.propertySet == updatedPropertySet - } - ) { + + values.forEach { value -> + if (filteredCurrentSet.none { it.srcNode === value }) { currentSet += MapDstToSrcEntry( param, @@ -2089,7 +2084,7 @@ open class PointsToPass(ctx: TranslationContext) : EOGStarterPass(ctx, orderDepe ) } } - } + } } } @@ -2097,40 +2092,29 @@ open class PointsToPass(ctx: TranslationContext) : EOGStarterPass(ctx, orderDepe // In case the FunctionSummary says that we have to use the // dereferenced value here, we look up the argument, // dereference it, and then add it to the sources - currentNode.invokes - .flatMap { it.parameters } - .filterTo(identitySetOf()) { - it.name.localName == srcNode.name.parent?.localName - } - .forEach { - if (it.argumentIndex < currentNode.arguments.size) { - val arg = currentNode.arguments[it.argumentIndex] - destinationAddresses - .filter { it.first != null } - .forEach { (d, partialWrite) -> - val updatedPropertySet = - equalLinkedHashSetOf().apply { - addAll(propertySet) - add(shortFS) - } - partialWrite?.let { - updatedPropertySet.add(PartialDataflowGranularity(it)) - } - val currentSet = - mapDstToSrc.computeIfAbsent(d!!) { - concurrentIdentitySetOf() - } + destinationAddresses + .filter { it.first != null } + .forEach { (d, partialWrite) -> + val updatedPropertySet = + createPropertySet(propertySet, shortFS, partialWrite) + val currentSet = + mapDstToSrc.computeIfAbsent(d!!) { concurrentIdentitySetOf() } + val filteredCurrentSet = + currentSet.filter { + it.lastWrites.parallelEquals(PowersetLattice.Element(lastWrites)) && + it.propertySet == updatedPropertySet + } + currentNode.invokes + .flatMap { it.parameters } + .filterTo(identitySetOf()) { + it.name.localName == srcNode.name.parent?.localName + } + .forEach { parameter -> + if (parameter.argumentIndex < currentNode.arguments.size) { + val arg = currentNode.arguments[parameter.argumentIndex] doubleState.getNestedValues(arg, srcValueDepth).forEach { (value, _) -> - if ( - currentSet.none { - it.srcNode === value && - it.lastWrites.parallelEquals( - PowersetLattice.Element(lastWrites) - ) && - it.propertySet == updatedPropertySet - } - ) { + if (filteredCurrentSet.none { it.srcNode === value }) { currentSet += MapDstToSrcEntry( param, @@ -2142,7 +2126,7 @@ open class PointsToPass(ctx: TranslationContext) : EOGStarterPass(ctx, orderDepe } } } - } + } } } @@ -2153,11 +2137,7 @@ open class PointsToPass(ctx: TranslationContext) : EOGStarterPass(ctx, orderDepe val currentSet = mapDstToSrc.computeIfAbsent(d!!) { concurrentIdentitySetOf() } val updatedPropertySet = - equalLinkedHashSetOf().apply { - addAll(propertySet) - add(shortFS) - } - partialWrite?.let { updatedPropertySet.add(PartialDataflowGranularity(it)) } + createPropertySet(propertySet, shortFS, partialWrite) if ( currentSet.none { it.srcNode === srcNode && @@ -2194,22 +2174,15 @@ open class PointsToPass(ctx: TranslationContext) : EOGStarterPass(ctx, orderDepe } } ?: PowersetLattice.Element(Pair(null, shortFS)) + val filteredCurrentSet = currentSet.filter { it.lastWrites === lastWrites } newSet.forEach { pair -> if ( - currentSet.none { - it.srcNode === pair.first && - it.lastWrites === lastWrites && - pair.second in it.propertySet + filteredCurrentSet.none { + it.srcNode === pair.first && pair.second in it.propertySet } ) { val updatedPropertySet = - equalLinkedHashSetOf().apply { - addAll(propertySet) - add(shortFS) - } - partialWrite?.let { - updatedPropertySet.add(PartialDataflowGranularity(it)) - } + createPropertySet(propertySet, shortFS, partialWrite) currentSet += MapDstToSrcEntry( param, @@ -2226,6 +2199,20 @@ open class PointsToPass(ctx: TranslationContext) : EOGStarterPass(ctx, orderDepe return mapDstToSrc } + private fun createPropertySet( + propertySet: EqualLinkedHashSet, + shortFS: Boolean, + partialWrite: String?, + ): EqualLinkedHashSet { + val updatedPropertySet = + equalLinkedHashSetOf().apply { + addAll(propertySet) + add(shortFS) + } + partialWrite?.let { updatedPropertySet.add(PartialDataflowGranularity(it)) } + return updatedPropertySet + } + /** Returns a Pair of destination (for the general State) and destinationAddresses */ private fun calculateCallDestinations( doubleState: PointsToState.Element, @@ -2991,7 +2978,7 @@ suspend fun PointsToState.push( // If we already have exactly that entry, no need to re-write it, otherwise we might confuse the // iterateEOG function val newLatticeCopy = newLatticeElement.duplicate() - newLatticeElement.third.forEach { existingEntry -> + newLatticeElement.third.forEachMaybeParallel { existingEntry -> if ( currentState.generalState[newNode]?.third?.any { it.node === existingEntry.node && it.properties == existingEntry.properties @@ -3021,20 +3008,18 @@ suspend fun PointsToState.pushToDeclarationsState( // iterateEOG function val newLatticeCopy = newLatticeElement.duplicate() - coroutineScope { - newLatticeElement.second.forEachMaybeParallel { pair -> - if ( - currentState.declarationsState[newNode]?.second?.any { - it.first === pair.first && it.second == pair.second - } == true - ) - newLatticeCopy.second.remove(pair) - } + newLatticeElement.second.forEachMaybeParallel { pair -> + if ( + currentState.declarationsState[newNode]?.second?.any { + it.first === pair.first && it.second == pair.second + } == true + ) + newLatticeCopy.second.remove(pair) + } - newLatticeElement.third.forEachMaybeParallel { nwpk -> - if (currentState.declarationsState[newNode]?.third?.contains(nwpk) == true) { - newLatticeCopy.third.remove(nwpk) - } + newLatticeElement.third.forEachMaybeParallel { nwpk -> + if (currentState.declarationsState[newNode]?.third?.contains(nwpk) == true) { + newLatticeCopy.third.remove(nwpk) } } @@ -3201,22 +3186,17 @@ fun PointsToState.Element.fetchValueFromDeclarationState( it != node && !this.getAddresses(node, node).contains(it) } fields?.forEach { field -> - this.declarationsState[field] - ?.second - ?.filter { if (excludeShortFSValues) !it.second else true } - ?.let { - it.forEach { - ret.add( - FetchElementFromDeclarationStateEntry( - it.first, - it.second, - field.name.localName, - this.declarationsState[field]?.third - ?: PowersetLattice.Element(), - ) + this.declarationsState[field]?.second?.forEach { + if (excludeShortFSValues && !it.second || !excludeShortFSValues) + ret.add( + FetchElementFromDeclarationStateEntry( + it.first, + it.second, + field.name.localName, + this.declarationsState[field]?.third ?: PowersetLattice.Element(), ) - } - } + ) + } } } } @@ -3533,10 +3513,6 @@ fun PointsToState.Element.getValues( this.getValues(it, it) } } - /* is New -> { - // New returns a new MemoryAddress each time - PowersetLattice.Element(Pair(MemoryAddress(Name("new", node.name)), false)) - }*/ // For BinaryOperators, UnaryOperators, Literals etc. we'll end up here // For those, we define that they are the values of themselves // However, if we have a DeclarationState entry, we will return this one. @@ -3642,7 +3618,7 @@ fun PointsToState.Element.getAddresses(node: Node, startNode: Node): IdentitySet } is Cast -> { /* - * For Casts we take the expression as the cast itself does not have any impact on the address + * For casts, we take the expression as the cast itself does not have any impact on the address */ this.getAddresses(node.expression, startNode) } @@ -3673,12 +3649,10 @@ fun PointsToState.Element.getAddresses(node: Node, startNode: Node): IdentitySet // TODO: This should work for all HasMemoryAddresses // is HasMemoryAddress -> { is BinaryOperator -> { - // synchronized(node.memoryAddresses) { if (node.memoryAddresses.isEmpty()) { node.memoryAddresses += MemoryAddress(node.name, isGlobal(node)) } node.memoryAddresses.toIdentitySet() - // } } else -> identitySetOf(node) } @@ -3811,6 +3785,16 @@ suspend fun PointsToState.Element.updateValues( ): PointsToState.Element = coroutineScope { var doubleState = doubleState + val noDestinationWithBody = + destinations.none { it is Call && it.invokes.singleOrNull()?.body == null } + + val mappedSources = + sources.mapNotNullTo(PowersetLattice.Element()) { + it.first?.let { first -> + NodeWithPropertiesKey(first, equalLinkedHashSetOf().apply { add(it.second) }) + } + } + /* Update the declarationState for the addresses */ destinationAddresses.forEach { destAddr -> if (!isGlobal(destAddr)) { @@ -3821,16 +3805,13 @@ suspend fun PointsToState.Element.updateValues( // If we want to update the State with exactly the same elements as are already in the // state, we do nothing in order not to confuse the iterateEOG function val newSources: PowersetLattice.Element> = - sources - .mapTo(PowersetLattice.Element()) { triple -> - val existingPair = - this@updateValues.declarationsState[destAddr]?.second?.firstOrNull { - it.first === triple.first && it.second == triple.second - } - existingPair ?: Pair(triple.first, triple.second) - } - .filterTo(PowersetLattice.Element()) { it.first != null } - .mapTo(PowersetLattice.Element()) { Pair(it.first!!, it.second) } + sources.mapNotNullTo(PowersetLattice.Element()) { triple -> + val existingPair = + this@updateValues.declarationsState[destAddr]?.second?.firstOrNull { + it.first === triple.first && it.second == triple.second + } ?: Pair(triple.first, triple.second) + existingPair.first?.let { first -> Pair(first, existingPair.second) } + } // Check if we have any full writes val fullSourcesExist = sources.any { !it.third } @@ -3879,7 +3860,7 @@ suspend fun PointsToState.Element.updateValues( newLastWrites.forEachMaybeParallel { lw -> if ( - destinations.none { it is Call && it.invokes.singleOrNull()?.body == null } && + noDestinationWithBody && (sources.any { src -> src.first === lw.node && src.second in lw.properties } || lw.node in destinations) @@ -3893,21 +3874,21 @@ suspend fun PointsToState.Element.updateValues( d, GeneralStateEntryElement( PowersetLattice.Element(destinationAddresses), - PowersetLattice.Element( - sources - .filter { it.first != null } - .mapTo(PowersetLattice.Element()) { - NodeWithPropertiesKey( - it.first!!, - equalLinkedHashSetOf().apply { add(it.second) }, - ) - } - ), + PowersetLattice.Element(mappedSources), PowersetLattice.Element(newLastWrites), ), ) } } else { + val mappedSources = + sources.mapNotNullTo(IdentitySet()) { + it.first?.let { first -> + NodeWithPropertiesKey( + first, + equalLinkedHashSetOf().apply { add(it.second) }, + ) + } + } // For globals, we draw a DFG Edge from the source to the destination destinations.forEachMaybeParallel { d -> val entry = @@ -3918,16 +3899,7 @@ suspend fun PointsToState.Element.updateValues( PowersetLattice.Element(), ) } - sources - .filter { it.first != null } - .forEach { - entry.third.add( - NodeWithPropertiesKey( - it.first!!, - equalLinkedHashSetOf().apply { add(it.second) }, - ) - ) - } + entry.third.addAll(mappedSources) } } } From 9967efaa63106abe0aa75b3dacb3b458d03b63af Mon Sep 17 00:00:00 2001 From: Alexander Kuechler Date: Wed, 27 May 2026 17:14:05 +0200 Subject: [PATCH 06/10] Some more cleanup --- .../aisec/cpg/passes/PointsToPass.kt | 135 +++++++++--------- 1 file changed, 68 insertions(+), 67 deletions(-) diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/PointsToPass.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/PointsToPass.kt index 1a145054986..4622f4f1337 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/PointsToPass.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/PointsToPass.kt @@ -802,13 +802,10 @@ open class PointsToPass(ctx: TranslationContext) : EOGStarterPass(ctx, orderDepe node.functionSummary.computeIfAbsent(param) { ConcurrentHashMap.newKeySet() } val filteredLastWrites = lastWrites - // for shortFS,only use these, and for !shortFS, - // only those + // for shortFS,only use these, and for !shortFS, only those .filterTo(PowersetLattice.Element()) { shortFS in it.properties } - // If the value is a newly created MemoryAddress, we only - // set the - // name so that we know later that we have to create a new - // MemoryAddress for each Call + // If the value is a newly created MemoryAddress, we only set the name so that we know later + // that we have to create a new MemoryAddress for each Call val addressName = (value as? MemoryAddress)?.name?.localName val v = if (addressName?.startsWith("NewMemoryAddress") == true) Name(addressName, node.name) @@ -824,7 +821,7 @@ open class PointsToPass(ctx: TranslationContext) : EOGStarterPass(ctx, orderDepe ) ) // Additionally, we store this as a shortFunctionSummary - // were the function writes to the parameter + // where the function writes to the parameter val shortFSEntry = FSEntry( dstValueDepth, @@ -836,14 +833,15 @@ open class PointsToPass(ctx: TranslationContext) : EOGStarterPass(ctx, orderDepe ) // Add the new entry if it doesn't exist yet synchronized(existingEntry) { + // TODO: Do we need the synchronized? Can we be more efficient in finding matching + // entries? if (existingEntry.none { it == shortFSEntry }) existingEntry.add(shortFSEntry) } val propertySet = identitySetOf(true) if (subAccessName != "") propertySet.add(Field().apply { name = Name(subAccessName) }) // Create the detailed shortFS. Like, which parameter - // influences - // what. + // influences what. // This may take a lot of time, so this is optional if ((passConfig()?.detailedShortFS ?: true)) { if (!shortFS) { @@ -851,6 +849,7 @@ open class PointsToPass(ctx: TranslationContext) : EOGStarterPass(ctx, orderDepe // Parameter and // if so, add this information to the // functionSummary + // TODO: Use memory value edges instead of DFG because these are shortcuts. val paths = value.followDFGEdgesUntilHit( collectFailedPaths = false, @@ -867,16 +866,16 @@ open class PointsToPass(ctx: TranslationContext) : EOGStarterPass(ctx, orderDepe predicate = { it is ParameterMemoryValue && /* If it's a ParameterMemoryValue from the node's - parameters, it has to have a DFG Node to one + parameters, it has to have a DFG edge to one of the node's parameters. Either partial to a derefvalue or full to the Parameter */ it.memoryValueUsageEdges - .filter { - ((it.granularity is PartialDataflowGranularity<*> && - ((it.granularity as PartialDataflowGranularity<*>) - .partialTarget as? String) - ?.endsWith("derefvalue") == true) || - (it.granularity is FullDataflowGranularity && - it.end is Parameter)) && it.end in node.parameters + .filter { edge -> + ((((edge.granularity as? PartialDataflowGranularity<*>) + ?.partialTarget as? String) + ?.endsWith("derefvalue") == true) || + (edge.granularity is FullDataflowGranularity && + edge.end is Parameter)) && + edge.end in node.parameters } .size == 1 && node.parameters.any { param -> @@ -885,7 +884,7 @@ open class PointsToPass(ctx: TranslationContext) : EOGStarterPass(ctx, orderDepe }, ) paths.fulfilled - .map { it.nodes.last() } + .mapTo(IdentitySet()) { it.nodes.last() } .forEach { sourceParamValue -> val matchingDeclarations = if (sourceParamValue is ParameterMemoryValue) @@ -949,63 +948,65 @@ open class PointsToPass(ctx: TranslationContext) : EOGStarterPass(ctx, orderDepe // We look at the deref and the derefderef, hence for depth 2 and 3 // We have to look up the index of the ParameterMemoryValue to check out // changes on the dereferences - values - .filterTo(concurrentIdentitySetOf()) { - doubleState.hasDeclarationStateValueEntry(it) - } - .forEach { indexes.add(IndexKey(it, 2)) } - // Additionally, we can check out the "dereference" itself to look for - // "derefdereferences" - values - .filterTo(identitySetOf()) { doubleState.hasDeclarationStateValueEntry(it) } - .flatMap { value -> - doubleState.getValues(value, value).mapTo(PowersetLattice.Element()) { - it.first + values.forEach { value -> + if (doubleState.hasDeclarationStateValueEntry(value)) { + indexes.add(IndexKey(value, 2)) + + // Additionally, we can check out the "dereference" itself to look for + // "derefdereferences" + val derefValues = + doubleState.getValues(value, value).mapTo( + PowersetLattice.Element() + ) { + it.first + } + // We are already inside $paramCount coroutines, so we have to divide + // the CPU_CORES by these routines + derefValues.forEachMaybeParallel(parallelism = innerCoroutineCounter) { + value -> + if (doubleState.hasDeclarationStateValueEntry(value)) + indexes.add(IndexKey(value, 3)) } } - // We are already inside $paramCount coroutines, so we have to divide - // the CPU_CORES by these routines - .forEachMaybeParallel(parallelism = innerCoroutineCounter) { value -> - if (doubleState.hasDeclarationStateValueEntry(value)) - indexes.add(IndexKey(value, 3)) - } + } indexes.forEach { (idx, dstValueDepth) -> - val stateEntries = - doubleState - .fetchValueFromDeclarationState( - idx, - fetchFields = true, - excludeShortFSValues = true, - ) - .filterTo(PowersetLattice.Element()) { it.value.name != param.name } - stateEntries - /* See if we can find something that is different from the initial value*/ - .filterTo(PowersetLattice.Element()) { - /* Filter the PMVs from this parameter*/ - !(it.value is ParameterMemoryValue && - it.value.name.localName.contains("derefvalue") && - it.value.name.parent?.localName == param.name.localName) - /* Filter the unknownMemoryValues that weren't written to*/ - && !(it.value is UnknownMemoryValue && it.lastWrites.isEmpty()) - } - // If so, store the information for the parameter in the - // FunctionSummary - // We are already inside $paramCount coroutines, so we have to - // divide the CPU_CORES by these routines + doubleState + .fetchValueFromDeclarationState( + idx, + fetchFields = true, + excludeShortFSValues = true, + ) .forEachMaybeParallel( parallelism = innerCoroutineCounter, minChunkSize = 1, ) { (value, shortFS, subAccessName, lastWrites) -> - addParameterInfoToFS( - node, - param, - dstValueDepth, - value, - shortFS, - subAccessName, - lastWrites, - ) + /* See if we can find something that is different from the initial value.*/ + if ( + value.name != param.name && + /*Filter the PMVs from this parameter*/ + !(value is ParameterMemoryValue && + value.name.localName.contains("derefvalue") && + value.name.parent?.localName == param.name.localName) + /* Filter the unknownMemoryValues that weren't written to*/ + && + !(value is UnknownMemoryValue && lastWrites.isEmpty()) + ) { + + // If so, store the information for the parameter in the + // FunctionSummary + // We are already inside $paramCount coroutines, so we have to + // divide the CPU_CORES by these routines + addParameterInfoToFS( + node, + param, + dstValueDepth, + value, + shortFS, + subAccessName, + lastWrites, + ) + } } } } From 9221188aa8c367825eabd743692bcc39da6eda88 Mon Sep 17 00:00:00 2001 From: Alexander Kuechler Date: Wed, 27 May 2026 17:47:02 +0200 Subject: [PATCH 07/10] Reduce some more loops --- .../aisec/cpg/passes/PointsToPass.kt | 110 +++++++++--------- 1 file changed, 53 insertions(+), 57 deletions(-) diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/PointsToPass.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/PointsToPass.kt index 4622f4f1337..4f8f76105ea 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/PointsToPass.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/PointsToPass.kt @@ -1111,11 +1111,11 @@ open class PointsToPass(ctx: TranslationContext) : EOGStarterPass(ctx, orderDepe ): PointsToState.Element { // For now, all we do here is to initialize the parameters var doubleState = doubleState - function.parameters - .filter { it.memoryValues.filterIsInstance().isEmpty() } - .forEach { param -> + function.parameters.forEach { param -> + if (param.memoryValues.filterIsInstance().isEmpty()) { doubleState = initializeParameter(lattice, function, param, doubleState) } + } return doubleState } @@ -1281,10 +1281,9 @@ open class PointsToPass(ctx: TranslationContext) : EOGStarterPass(ctx, orderDepe } // Filter shortFS Values var values = - doubleState - .getValues(rV, rV) - .filter { !it.second } - .mapTo(mutableSetOf()) { it.first } + doubleState.getValues(rV, rV).mapNotNullTo(mutableSetOf()) { + if (!it.second) it.first else null + } var addresses = mutableSetOf() for (depth in 1..3) { fsEntry.addAll( @@ -1312,14 +1311,13 @@ open class PointsToPass(ctx: TranslationContext) : EOGStarterPass(ctx, orderDepe // Try to deref the values. If we have nothing there, stop, otherwise, // continue val derefValues = - values - .filter { value -> - doubleState.hasDeclarationStateValueEntry(value) - } - .flatMap { value -> - doubleState.getValues(value, value).filter { !it.second } - } - .mapTo(mutableSetOf()) { it.first } + values.flatMapTo(mutableSetOf()) { value -> + if (doubleState.hasDeclarationStateValueEntry(value)) { + doubleState.getValues(value, value).mapNotNull { + if (!it.second) it.first else null + } + } else emptyList() + } if (derefValues.isEmpty()) break else { // When we deref once, the values will become the addresses, and the @@ -2055,16 +2053,15 @@ open class PointsToPass(ctx: TranslationContext) : EOGStarterPass(ctx, orderDepe .mapTo(identitySetOf()) { it.first } else identitySetOf(currentNode.arguments[srcNode.argumentIndex]) - destinationAddresses - .filter { it.first != null } - .forEach { (d, partialWrite) -> + destinationAddresses.forEach { (d, partialWrite) -> + if (d != null) { // The extracted value might come from a state we // created for a short function summary. If so, we // have to store that info in the map val updatedPropertySet = createPropertySet(propertySet, shortFS, partialWrite) val currentSet = - mapDstToSrc.computeIfAbsent(d!!) { concurrentIdentitySetOf() } + mapDstToSrc.computeIfAbsent(d) { concurrentIdentitySetOf() } val filteredCurrentSet = currentSet.filter { @@ -2086,6 +2083,7 @@ open class PointsToPass(ctx: TranslationContext) : EOGStarterPass(ctx, orderDepe } } } + } } } @@ -2093,13 +2091,12 @@ open class PointsToPass(ctx: TranslationContext) : EOGStarterPass(ctx, orderDepe // In case the FunctionSummary says that we have to use the // dereferenced value here, we look up the argument, // dereference it, and then add it to the sources - destinationAddresses - .filter { it.first != null } - .forEach { (d, partialWrite) -> + destinationAddresses.forEach { (d, partialWrite) -> + if (d != null) { val updatedPropertySet = createPropertySet(propertySet, shortFS, partialWrite) val currentSet = - mapDstToSrc.computeIfAbsent(d!!) { concurrentIdentitySetOf() } + mapDstToSrc.computeIfAbsent(d) { concurrentIdentitySetOf() } val filteredCurrentSet = currentSet.filter { it.lastWrites.parallelEquals(PowersetLattice.Element(lastWrites)) && @@ -2129,14 +2126,14 @@ open class PointsToPass(ctx: TranslationContext) : EOGStarterPass(ctx, orderDepe } } } + } } is MemoryAddress -> { - destinationAddresses - .filter { it.first != null } - .forEach { (d, partialWrite) -> + destinationAddresses.forEach { (d, partialWrite) -> + if (d != null) { val currentSet = - mapDstToSrc.computeIfAbsent(d!!) { concurrentIdentitySetOf() } + mapDstToSrc.computeIfAbsent(d) { concurrentIdentitySetOf() } val updatedPropertySet = createPropertySet(propertySet, shortFS, partialWrite) if ( @@ -2156,14 +2153,14 @@ open class PointsToPass(ctx: TranslationContext) : EOGStarterPass(ctx, orderDepe ) } } + } } else -> { - destinationAddresses - .filter { it.first != null } - .forEach { (d, partialWrite) -> + destinationAddresses.forEach { (d, partialWrite) -> + if (d != null) { val currentSet = - mapDstToSrc.computeIfAbsent(d!!) { concurrentIdentitySetOf() } + mapDstToSrc.computeIfAbsent(d) { concurrentIdentitySetOf() } val newSet = if (srcValueDepth == 0) PowersetLattice.Element(Pair(srcNode, shortFS)) else @@ -2195,6 +2192,7 @@ open class PointsToPass(ctx: TranslationContext) : EOGStarterPass(ctx, orderDepe } } } + } } } return mapDstToSrc @@ -2256,25 +2254,25 @@ open class PointsToPass(ctx: TranslationContext) : EOGStarterPass(ctx, orderDepe // there won't be any effect on the first depth (AKA the value) if (argument is Call && dstValueDepth > 1) { val updatedAddresses: IdentitySet> = - mapDstToSrc.entries - .filter { it.key == argument } - .flatMap { it.value } - // Make sure we only fetch the entries from the respective parameter - .filter { it.param == param && it.srcNode != null } - .mapTo(IdentitySet()) { it.srcNode to null } + mapDstToSrc.entries.flatMapTo(IdentitySet()) { + if (it.key == argument) { + it.value.mapNotNullTo(IdentitySet()) { + if (it.param == param && it.srcNode != null) { + it.srcNode to null + } else null + } + } else emptySet() + } if (updatedAddresses.isNotEmpty()) return Pair(updatedAddresses, destination) } else if (dstValueDepth > 2) { + val argumentValues = + doubleState.getValues(argument, argument).mapTo(IdentitySet()) { it.first } val updatedAddresses: IdentitySet> = - mapDstToSrc.entries - .filter { - it.key in - doubleState.getValues(argument, argument).mapTo(IdentitySet()) { - it.first - } - } - .flatMap { it.value } - .filter { it.srcNode != null } - .mapTo(IdentitySet()) { it.srcNode to null } + mapDstToSrc.entries.flatMapTo(IdentitySet()) { + if (it.key in argumentValues) { + it.value.mapNotNullTo(IdentitySet()) { it.srcNode?.let { it to null } } + } else emptySet() + } if (updatedAddresses.isNotEmpty()) return Pair(updatedAddresses, destination) } @@ -2520,8 +2518,7 @@ open class PointsToPass(ctx: TranslationContext) : EOGStarterPass(ctx, orderDepe .getValues(currentNode, currentNode) // Filter only the values that are not stored for short FunctionSummaries (aka // it.second set to true) - .filter { !it.second } - .mapTo(IdentitySet()) { it.first } + .mapNotNullTo(IdentitySet()) { if (!it.second) it.first else null } val prevDFGs = doubleState.getLastWrites(currentNode) // If we have any information from the dereferenced value, we also fetch that (if it's @@ -2530,14 +2527,12 @@ open class PointsToPass(ctx: TranslationContext) : EOGStarterPass(ctx, orderDepe (passConfig()?.drawCurrentDerefDFG != false) && (currentNode.astParent as? PointerDereference)?.access != AccessValues.WRITE ) { - values - .filterTo(identitySetOf()) { - /* If all we have here as a PMV value, we can skip it - Note: This only applies to the value, the deref and derefderefvalues - might be from different functions, so we leave those */ - doubleState.hasDeclarationStateValueEntry(it, true) - } - .forEach { value -> + values.forEach { value -> + // TODO: This probably can be optimized + /* If all we have here as a PMV value, we can skip it + Note: This only applies to the value, the deref and derefderefvalues + might be from different functions, so we leave those */ + if (doubleState.hasDeclarationStateValueEntry(value, true)) { // draw the DFG Edges doubleState .getLastWrites(value) @@ -2595,6 +2590,7 @@ open class PointsToPass(ctx: TranslationContext) : EOGStarterPass(ctx, orderDepe } } } + } } doubleState = From dcd317611c5ce06166a2eb941d7377bdab6aba9a Mon Sep 17 00:00:00 2001 From: Alexander Kuechler Date: Mon, 1 Jun 2026 18:44:00 +0200 Subject: [PATCH 08/10] Revert change in hashCode --- .../de/fraunhofer/aisec/cpg/graph/types/PointerType.kt | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/types/PointerType.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/types/PointerType.kt index 4314885adea..3cd3cef6e32 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/types/PointerType.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/types/PointerType.kt @@ -105,8 +105,9 @@ class PointerType : Type, SecondOrderType { } override fun hashCode(): Int { - // Use System.identityHashCode for elementType to avoid infinite recursion - // when dealing with circular type references (e.g., struct with pointer to itself) - return Objects.hash(super.hashCode(), System.identityHashCode(elementType), pointerOrigin) + return Objects.hash(super.hashCode(), elementType, pointerOrigin) + // TODO: Use System.identityHashCode for elementType to avoid infinite recursion + // when dealing with circular type references (e.g., struct with pointer to itself). + // However, this needs a re-design of the current type handling } } From 1288a11507b50c1e309f273b1c45a719c539c24f Mon Sep 17 00:00:00 2001 From: Alexander Kuechler Date: Mon, 1 Jun 2026 18:57:08 +0200 Subject: [PATCH 09/10] Cycle-aware type hashcode --- .../cpg/graph/types/FunctionPointerType.kt | 3 +- .../aisec/cpg/graph/types/NumericType.kt | 3 +- .../aisec/cpg/graph/types/ObjectType.kt | 3 +- .../aisec/cpg/graph/types/PointerType.kt | 6 +- .../aisec/cpg/graph/types/ReferenceType.kt | 3 +- .../fraunhofer/aisec/cpg/graph/types/Type.kt | 76 +++++++++++++++++++ .../aisec/cpg/graph/types/TypeTest.kt | 41 ++++++++++ 7 files changed, 122 insertions(+), 13 deletions(-) diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/types/FunctionPointerType.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/types/FunctionPointerType.kt index f9da7f70536..0a83fe8f013 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/types/FunctionPointerType.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/types/FunctionPointerType.kt @@ -29,7 +29,6 @@ import de.fraunhofer.aisec.cpg.frontends.Language import de.fraunhofer.aisec.cpg.graph.Name import de.fraunhofer.aisec.cpg.graph.types.PointerType.PointerOrigin import de.fraunhofer.aisec.cpg.persistence.Relationship -import java.util.* import org.apache.commons.lang3.builder.ToStringBuilder /** @@ -72,7 +71,7 @@ class FunctionPointerType : Type { returnType == other.returnType } - override fun hashCode() = Objects.hash(super.hashCode(), parameters, returnType) + override fun hashCode() = structuralHashCode() override fun toString(): String { return ToStringBuilder(this, TO_STRING_STYLE) diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/types/NumericType.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/types/NumericType.kt index 5320e749d8f..593dafebcc4 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/types/NumericType.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/types/NumericType.kt @@ -26,7 +26,6 @@ package de.fraunhofer.aisec.cpg.graph.types import de.fraunhofer.aisec.cpg.frontends.Language -import java.util.* /** This type collects all kind of numeric types. */ open class NumericType( @@ -61,5 +60,5 @@ open class NumericType( override fun equals(other: Any?) = super.equals(other) && this.modifier == (other as? NumericType)?.modifier - override fun hashCode() = Objects.hash(super.hashCode(), generics, modifier, isPrimitive) + override fun hashCode() = structuralHashCode() } diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/types/ObjectType.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/types/ObjectType.kt index c50d516fb71..50f753bd120 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/types/ObjectType.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/types/ObjectType.kt @@ -34,7 +34,6 @@ import de.fraunhofer.aisec.cpg.graph.unknownType import de.fraunhofer.aisec.cpg.helpers.identitySetOf import de.fraunhofer.aisec.cpg.passes.TypeResolver import de.fraunhofer.aisec.cpg.persistence.Relationship -import java.util.* /** * This is the main type in the Type system. ObjectTypes describe objects, as instances of a class. @@ -114,7 +113,7 @@ open class ObjectType : Type, HasSecondaryTypeEdge { return generics == other.generics && isPrimitive == other.isPrimitive } - override fun hashCode() = Objects.hash(super.hashCode(), generics, isPrimitive) + override fun hashCode() = structuralHashCode() override val secondaryTypes: List get() = generics diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/types/PointerType.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/types/PointerType.kt index 3cd3cef6e32..5a263108ad4 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/types/PointerType.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/types/PointerType.kt @@ -27,7 +27,6 @@ package de.fraunhofer.aisec.cpg.graph.types import de.fraunhofer.aisec.cpg.graph.Name import de.fraunhofer.aisec.cpg.persistence.Relationship -import java.util.* /** * PointerTypes represent all references to other Types. For C/CPP this includes pointers, as well @@ -105,9 +104,6 @@ class PointerType : Type, SecondOrderType { } override fun hashCode(): Int { - return Objects.hash(super.hashCode(), elementType, pointerOrigin) - // TODO: Use System.identityHashCode for elementType to avoid infinite recursion - // when dealing with circular type references (e.g., struct with pointer to itself). - // However, this needs a re-design of the current type handling + return structuralHashCode() } } diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/types/ReferenceType.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/types/ReferenceType.kt index dd3e3a1b008..285508ae8e5 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/types/ReferenceType.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/types/ReferenceType.kt @@ -27,7 +27,6 @@ package de.fraunhofer.aisec.cpg.graph.types import de.fraunhofer.aisec.cpg.graph.types.PointerType.PointerOrigin import de.fraunhofer.aisec.cpg.graph.unknownType -import java.util.* import org.apache.commons.lang3.builder.ToStringBuilder /** @@ -69,7 +68,7 @@ class ReferenceType : Type, SecondOrderType { return super.equals(other) && elementType == other.elementType } - override fun hashCode() = Objects.hash(super.hashCode(), elementType) + override fun hashCode() = structuralHashCode() override fun toString(): String { return ToStringBuilder(this, TO_STRING_STYLE) diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/types/Type.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/types/Type.kt index f3fef878c6b..8602faa3899 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/types/Type.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/types/Type.kt @@ -266,6 +266,82 @@ abstract class Type : Node { } } +private object TypeStructuralHash { + private const val VISITING = Int.MIN_VALUE + + private class HashContext { + val cache = IdentityHashMap() + var depth = 0 + } + + private val context = ThreadLocal.withInitial(::HashContext) + + fun hash(type: Type): Int { + val ctx = context.get() + val isTopLevel = ctx.depth == 0 + ctx.depth++ + + return try { + hash(type, ctx.cache) + } finally { + ctx.depth-- + if (isTopLevel) { + ctx.cache.clear() + } + } + } + + private fun hash(type: Type, cache: IdentityHashMap): Int { + val cached = cache[type] + if (cached != null) { + return if (cached == VISITING) shallowHash(type) else cached + } + + cache[type] = VISITING + + var result = shallowHash(type) + when (type) { + is PointerType -> { + result = 31 * result + (type.pointerOrigin?.hashCode() ?: 0) + result = 31 * result + hash(type.elementType, cache) + } + + is ReferenceType -> { + result = 31 * result + hash(type.elementType, cache) + } + + is FunctionPointerType -> { + type.parameters.forEach { parameter -> + result = 31 * result + hash(parameter, cache) + } + result = 31 * result + hash(type.returnType, cache) + } + + is NumericType -> { + type.generics.forEach { generic -> result = 31 * result + hash(generic, cache) } + result = 31 * result + type.modifier.hashCode() + } + + is ObjectType -> { + type.generics.forEach { generic -> result = 31 * result + hash(generic, cache) } + } + } + + cache[type] = result + return result + } + + private fun shallowHash(type: Type): Int { + var result = type::class.hashCode() + result = 31 * result + type.name.hashCode() + result = 31 * result + (type.scope?.let(System::identityHashCode) ?: 0) + result = 31 * result + type.language.hashCode() + return result + } +} + +internal fun Type.structuralHashCode(): Int = TypeStructuralHash.hash(this) + /** * Wraps the given [Type] into a chain of [PointerType]s and [ReferenceType]s, given the operations * in [TypeOperations]. diff --git a/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/graph/types/TypeTest.kt b/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/graph/types/TypeTest.kt index 2744f798b7d..82d4b9fe618 100644 --- a/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/graph/types/TypeTest.kt +++ b/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/graph/types/TypeTest.kt @@ -27,10 +27,12 @@ package de.fraunhofer.aisec.cpg.graph.types import de.fraunhofer.aisec.cpg.frontends.TestLanguageFrontend import de.fraunhofer.aisec.cpg.frontends.TranslationException +import de.fraunhofer.aisec.cpg.frontends.UnknownLanguage import de.fraunhofer.aisec.cpg.graph.* import de.fraunhofer.aisec.cpg.graph.objectType import de.fraunhofer.aisec.cpg.graph.pointer import de.fraunhofer.aisec.cpg.test.assertLocalName +import kotlin.system.measureTimeMillis import kotlin.test.* import org.junit.jupiter.api.assertThrows @@ -84,4 +86,43 @@ class TypeTest { assertIs(type.dereference()) } } + + @Test + fun testPointerHashCodeIsStableForSelfReferentialType() { + val pointer = + PointerType(UnknownType.getUnknownType(null), PointerType.PointerOrigin.POINTER) + pointer.elementType = pointer + + val hash1 = pointer.hashCode() + val hash2 = pointer.hashCode() + + assertEquals(hash1, hash2) + } + + @Test + fun testPointerHashCodeForEqualButDistinctTypes() { + with(TestLanguageFrontend()) { + val p1 = objectType("Node").pointer() + val p2 = objectType("Node").pointer() + + assertEquals(p1, p2) + assertEquals(p1.hashCode(), p2.hashCode()) + } + } + + @Test + fun testStructuralHashPerformanceSmoke() { + val base = ObjectType("Node", listOf(), false, true, UnknownLanguage) + var type: Type = base + repeat(64) { type = type.pointer() } + + var accumulator = 0 + val elapsedMs = measureTimeMillis { + repeat(200_000) { accumulator = accumulator xor type.hashCode() } + } + + assertEquals(type.hashCode(), type.hashCode()) + assertTrue(elapsedMs >= 0) + assertTrue(accumulator != Int.MIN_VALUE) + } } From fa6d122f8f2cc9914377f19dbd4180825a16807b Mon Sep 17 00:00:00 2001 From: Alexander Kuechler Date: Tue, 2 Jun 2026 16:41:42 +0200 Subject: [PATCH 10/10] Revert "Cycle-aware type hashcode" This reverts commit 1288a11507b50c1e309f273b1c45a719c539c24f. --- .../cpg/graph/types/FunctionPointerType.kt | 3 +- .../aisec/cpg/graph/types/NumericType.kt | 3 +- .../aisec/cpg/graph/types/ObjectType.kt | 3 +- .../aisec/cpg/graph/types/PointerType.kt | 6 +- .../aisec/cpg/graph/types/ReferenceType.kt | 3 +- .../fraunhofer/aisec/cpg/graph/types/Type.kt | 76 ------------------- .../aisec/cpg/graph/types/TypeTest.kt | 41 ---------- 7 files changed, 13 insertions(+), 122 deletions(-) diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/types/FunctionPointerType.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/types/FunctionPointerType.kt index 0a83fe8f013..f9da7f70536 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/types/FunctionPointerType.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/types/FunctionPointerType.kt @@ -29,6 +29,7 @@ import de.fraunhofer.aisec.cpg.frontends.Language import de.fraunhofer.aisec.cpg.graph.Name import de.fraunhofer.aisec.cpg.graph.types.PointerType.PointerOrigin import de.fraunhofer.aisec.cpg.persistence.Relationship +import java.util.* import org.apache.commons.lang3.builder.ToStringBuilder /** @@ -71,7 +72,7 @@ class FunctionPointerType : Type { returnType == other.returnType } - override fun hashCode() = structuralHashCode() + override fun hashCode() = Objects.hash(super.hashCode(), parameters, returnType) override fun toString(): String { return ToStringBuilder(this, TO_STRING_STYLE) diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/types/NumericType.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/types/NumericType.kt index 593dafebcc4..5320e749d8f 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/types/NumericType.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/types/NumericType.kt @@ -26,6 +26,7 @@ package de.fraunhofer.aisec.cpg.graph.types import de.fraunhofer.aisec.cpg.frontends.Language +import java.util.* /** This type collects all kind of numeric types. */ open class NumericType( @@ -60,5 +61,5 @@ open class NumericType( override fun equals(other: Any?) = super.equals(other) && this.modifier == (other as? NumericType)?.modifier - override fun hashCode() = structuralHashCode() + override fun hashCode() = Objects.hash(super.hashCode(), generics, modifier, isPrimitive) } diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/types/ObjectType.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/types/ObjectType.kt index 50f753bd120..c50d516fb71 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/types/ObjectType.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/types/ObjectType.kt @@ -34,6 +34,7 @@ import de.fraunhofer.aisec.cpg.graph.unknownType import de.fraunhofer.aisec.cpg.helpers.identitySetOf import de.fraunhofer.aisec.cpg.passes.TypeResolver import de.fraunhofer.aisec.cpg.persistence.Relationship +import java.util.* /** * This is the main type in the Type system. ObjectTypes describe objects, as instances of a class. @@ -113,7 +114,7 @@ open class ObjectType : Type, HasSecondaryTypeEdge { return generics == other.generics && isPrimitive == other.isPrimitive } - override fun hashCode() = structuralHashCode() + override fun hashCode() = Objects.hash(super.hashCode(), generics, isPrimitive) override val secondaryTypes: List get() = generics diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/types/PointerType.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/types/PointerType.kt index 5a263108ad4..3cd3cef6e32 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/types/PointerType.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/types/PointerType.kt @@ -27,6 +27,7 @@ package de.fraunhofer.aisec.cpg.graph.types import de.fraunhofer.aisec.cpg.graph.Name import de.fraunhofer.aisec.cpg.persistence.Relationship +import java.util.* /** * PointerTypes represent all references to other Types. For C/CPP this includes pointers, as well @@ -104,6 +105,9 @@ class PointerType : Type, SecondOrderType { } override fun hashCode(): Int { - return structuralHashCode() + return Objects.hash(super.hashCode(), elementType, pointerOrigin) + // TODO: Use System.identityHashCode for elementType to avoid infinite recursion + // when dealing with circular type references (e.g., struct with pointer to itself). + // However, this needs a re-design of the current type handling } } diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/types/ReferenceType.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/types/ReferenceType.kt index 285508ae8e5..dd3e3a1b008 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/types/ReferenceType.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/types/ReferenceType.kt @@ -27,6 +27,7 @@ package de.fraunhofer.aisec.cpg.graph.types import de.fraunhofer.aisec.cpg.graph.types.PointerType.PointerOrigin import de.fraunhofer.aisec.cpg.graph.unknownType +import java.util.* import org.apache.commons.lang3.builder.ToStringBuilder /** @@ -68,7 +69,7 @@ class ReferenceType : Type, SecondOrderType { return super.equals(other) && elementType == other.elementType } - override fun hashCode() = structuralHashCode() + override fun hashCode() = Objects.hash(super.hashCode(), elementType) override fun toString(): String { return ToStringBuilder(this, TO_STRING_STYLE) diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/types/Type.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/types/Type.kt index 8602faa3899..f3fef878c6b 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/types/Type.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/types/Type.kt @@ -266,82 +266,6 @@ abstract class Type : Node { } } -private object TypeStructuralHash { - private const val VISITING = Int.MIN_VALUE - - private class HashContext { - val cache = IdentityHashMap() - var depth = 0 - } - - private val context = ThreadLocal.withInitial(::HashContext) - - fun hash(type: Type): Int { - val ctx = context.get() - val isTopLevel = ctx.depth == 0 - ctx.depth++ - - return try { - hash(type, ctx.cache) - } finally { - ctx.depth-- - if (isTopLevel) { - ctx.cache.clear() - } - } - } - - private fun hash(type: Type, cache: IdentityHashMap): Int { - val cached = cache[type] - if (cached != null) { - return if (cached == VISITING) shallowHash(type) else cached - } - - cache[type] = VISITING - - var result = shallowHash(type) - when (type) { - is PointerType -> { - result = 31 * result + (type.pointerOrigin?.hashCode() ?: 0) - result = 31 * result + hash(type.elementType, cache) - } - - is ReferenceType -> { - result = 31 * result + hash(type.elementType, cache) - } - - is FunctionPointerType -> { - type.parameters.forEach { parameter -> - result = 31 * result + hash(parameter, cache) - } - result = 31 * result + hash(type.returnType, cache) - } - - is NumericType -> { - type.generics.forEach { generic -> result = 31 * result + hash(generic, cache) } - result = 31 * result + type.modifier.hashCode() - } - - is ObjectType -> { - type.generics.forEach { generic -> result = 31 * result + hash(generic, cache) } - } - } - - cache[type] = result - return result - } - - private fun shallowHash(type: Type): Int { - var result = type::class.hashCode() - result = 31 * result + type.name.hashCode() - result = 31 * result + (type.scope?.let(System::identityHashCode) ?: 0) - result = 31 * result + type.language.hashCode() - return result - } -} - -internal fun Type.structuralHashCode(): Int = TypeStructuralHash.hash(this) - /** * Wraps the given [Type] into a chain of [PointerType]s and [ReferenceType]s, given the operations * in [TypeOperations]. diff --git a/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/graph/types/TypeTest.kt b/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/graph/types/TypeTest.kt index 82d4b9fe618..2744f798b7d 100644 --- a/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/graph/types/TypeTest.kt +++ b/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/graph/types/TypeTest.kt @@ -27,12 +27,10 @@ package de.fraunhofer.aisec.cpg.graph.types import de.fraunhofer.aisec.cpg.frontends.TestLanguageFrontend import de.fraunhofer.aisec.cpg.frontends.TranslationException -import de.fraunhofer.aisec.cpg.frontends.UnknownLanguage import de.fraunhofer.aisec.cpg.graph.* import de.fraunhofer.aisec.cpg.graph.objectType import de.fraunhofer.aisec.cpg.graph.pointer import de.fraunhofer.aisec.cpg.test.assertLocalName -import kotlin.system.measureTimeMillis import kotlin.test.* import org.junit.jupiter.api.assertThrows @@ -86,43 +84,4 @@ class TypeTest { assertIs(type.dereference()) } } - - @Test - fun testPointerHashCodeIsStableForSelfReferentialType() { - val pointer = - PointerType(UnknownType.getUnknownType(null), PointerType.PointerOrigin.POINTER) - pointer.elementType = pointer - - val hash1 = pointer.hashCode() - val hash2 = pointer.hashCode() - - assertEquals(hash1, hash2) - } - - @Test - fun testPointerHashCodeForEqualButDistinctTypes() { - with(TestLanguageFrontend()) { - val p1 = objectType("Node").pointer() - val p2 = objectType("Node").pointer() - - assertEquals(p1, p2) - assertEquals(p1.hashCode(), p2.hashCode()) - } - } - - @Test - fun testStructuralHashPerformanceSmoke() { - val base = ObjectType("Node", listOf(), false, true, UnknownLanguage) - var type: Type = base - repeat(64) { type = type.pointer() } - - var accumulator = 0 - val elapsedMs = measureTimeMillis { - repeat(200_000) { accumulator = accumulator xor type.hashCode() } - } - - assertEquals(type.hashCode(), type.hashCode()) - assertTrue(elapsedMs >= 0) - assertTrue(accumulator != Int.MIN_VALUE) - } }