[c2cpg] Implemented cached header file loading (#4997)

joernio · Oct 11, 2024 · e0e493d · e0e493d
1 parent cf3834d
commit e0e493d
Show file tree

Hide file tree

Showing 2 changed files with 34 additions and 7 deletions.
diff --git a/joern-cli/frontends/c2cpg/src/main/scala/io/joern/c2cpg/parser/CdtParser.scala b/joern-cli/frontends/c2cpg/src/main/scala/io/joern/c2cpg/parser/CdtParser.scala
@@ -30,9 +30,13 @@ object CdtParser {
     failure: Option[Throwable] = None
   )
 
-  def readFileAsFileContent(path: Path): FileContent = {
+  def loadLinesAsFileContent(path: Path, lines: Array[Char]): InternalFileContent = {
+    FileContent.create(path.toString, true, lines).asInstanceOf[InternalFileContent]
+  }
+
+  def readFileAsFileContent(path: Path): InternalFileContent = {
     val lines = IOUtils.readLinesInFile(path).mkString("\n").toArray
-    FileContent.create(path.toString, true, lines)
+    loadLinesAsFileContent(path, lines)
   }
 
 }
@@ -97,8 +101,8 @@ class CdtParser(config: Config) extends ParseProblemsLogger with PreprocessorSta
       try {
         val fileContent         = readFileAsFileContent(realPath.path)
         val fileContentProvider = new CustomFileContentProvider(headerFileFinder)
-        val lang            = createParseLanguage(realPath.path, fileContent.asInstanceOf[InternalFileContent].toString)
-        val scannerInfo     = createScannerInfo(realPath.path)
+        val lang                = createParseLanguage(realPath.path, fileContent.toString)
+        val scannerInfo         = createScannerInfo(realPath.path)
         val translationUnit = lang.getASTTranslationUnit(fileContent, scannerInfo, fileContentProvider, null, opts, log)
         val problems        = CPPVisitor.getProblems(translationUnit)
         if (parserConfig.logProblems) logProblems(problems.toList)

diff --git a/...-cli/frontends/c2cpg/src/main/scala/io/joern/c2cpg/parser/CustomFileContentProvider.scala b/...-cli/frontends/c2cpg/src/main/scala/io/joern/c2cpg/parser/CustomFileContentProvider.scala
@@ -1,14 +1,24 @@
 package io.joern.c2cpg.parser
 
+import io.joern.c2cpg.parser.CustomFileContentProvider.missingHeaderFiles
+import io.shiftleft.utils.IOUtils
 import org.eclipse.cdt.core.index.IIndexFileLocation
 import org.eclipse.cdt.internal.core.parser.IMacroDictionary
 import org.eclipse.cdt.internal.core.parser.scanner.{InternalFileContent, InternalFileContentProvider}
 import org.slf4j.LoggerFactory
 
 import java.nio.file.Paths
+import java.util.concurrent.ConcurrentHashMap
+
+object CustomFileContentProvider {
+  private val headerFileToLines: ConcurrentHashMap[String, Array[Char]] = new ConcurrentHashMap()
+  private val missingHeaderFiles: ConcurrentHashMap[String, Boolean]    = new ConcurrentHashMap()
+}
 
 class CustomFileContentProvider(headerFileFinder: HeaderFileFinder) extends InternalFileContentProvider {
 
+  import io.joern.c2cpg.parser.CustomFileContentProvider.headerFileToLines
+
   private val logger = LoggerFactory.getLogger(classOf[CustomFileContentProvider])
 
   private def loadContent(path: String): InternalFileContent = {
@@ -19,11 +29,24 @@ class CustomFileContentProvider(headerFileFinder: HeaderFileFinder) extends Inte
     }
     maybeFullPath
       .map { foundPath =>
-        logger.debug(s"Loading header file '$foundPath'")
-        CdtParser.readFileAsFileContent(Paths.get(foundPath)).asInstanceOf[InternalFileContent]
+        val p = Paths.get(foundPath)
+        val content = headerFileToLines.computeIfAbsent(
+          foundPath,
+          _ => {
+            logger.debug(s"Loading header file '$foundPath'")
+            IOUtils.readLinesInFile(p).mkString("\n").toArray
+          }
+        )
+        CdtParser.loadLinesAsFileContent(p, content)
       }
       .getOrElse {
-        logger.debug(s"Cannot find header file for '$path'")
+        missingHeaderFiles.computeIfAbsent(
+          path,
+          _ => {
+            logger.debug(s"Cannot find header file for '$path'")
+            true
+          }
+        )
         null
       }