Fix language detection logic (#7245)

DataDog · Jun 28, 2024 · 97b5fe7 · 97b5fe7
1 parent dddda34
commit 97b5fe7
Show file tree

Hide file tree

Showing 27 changed files with 463 additions and 330 deletions.
diff --git a/dd-java-agent/agent-ci-visibility/build.gradle b/dd-java-agent/agent-ci-visibility/build.gradle
@@ -1,10 +1,22 @@
+buildscript {
+  repositories {
+    mavenCentral()
+  }
+
+  dependencies {
+    classpath group: 'org.jetbrains.kotlin', name: 'kotlin-gradle-plugin', version: libs.versions.kotlin.get()
+  }
+}
+
 plugins {
   id 'com.github.johnrengelman.shadow'
   id 'java-test-fixtures'
 }
 
 apply from: "$rootDir/gradle/java.gradle"
 apply from: "$rootDir/gradle/version.gradle"
+apply from: "$rootDir/gradle/test-with-kotlin.gradle"
+apply from: "$rootDir/gradle/test-with-scala.gradle"
 
 minimumBranchCoverage = 0.0
 minimumInstructionCoverage = 0.0
@@ -23,6 +35,9 @@ dependencies {
   testImplementation project(":utils:test-utils")
   testImplementation("com.google.jimfs:jimfs:1.1") // an in-memory file system for testing code that works with files
 
+  testImplementation libs.scala
+  testImplementation libs.kotlin
+
   testFixturesApi project(':dd-java-agent:testing')
   testFixturesApi project(':utils:test-utils')
 

diff --git a/...gent-ci-visibility/src/main/java/datadog/trace/civisibility/CiVisibilityRepoServices.java b/...gent-ci-visibility/src/main/java/datadog/trace/civisibility/CiVisibilityRepoServices.java
@@ -72,7 +72,7 @@ public class CiVisibilityRepoServices {
             services.gitClientFactory,
             services.backendApi,
             repoRoot);
-    repoIndexProvider = services.repoIndexProviderFactory.create(repoRoot, repoRoot);
+    repoIndexProvider = services.repoIndexProviderFactory.create(repoRoot);
     codeowners = buildCodeowners(repoRoot);
     sourcePathResolver = buildSourcePathResolver(repoRoot, repoIndexProvider);
 
@@ -175,14 +175,12 @@ private static GitDataUploader buildGitDataUploader(
 
   private static SourcePathResolver buildSourcePathResolver(
       String repoRoot, RepoIndexProvider indexProvider) {
-    if (repoRoot != null) {
-      RepoIndexSourcePathResolver indexSourcePathResolver =
-          new RepoIndexSourcePathResolver(repoRoot, indexProvider);
-      return new BestEffortSourcePathResolver(
-          new CompilerAidedSourcePathResolver(repoRoot), indexSourcePathResolver);
-    } else {
-      return NoOpSourcePathResolver.INSTANCE;
-    }
+    SourcePathResolver compilerAidedResolver =
+        repoRoot != null
+            ? new CompilerAidedSourcePathResolver(repoRoot)
+            : NoOpSourcePathResolver.INSTANCE;
+    RepoIndexSourcePathResolver indexResolver = new RepoIndexSourcePathResolver(indexProvider);
+    return new BestEffortSourcePathResolver(compilerAidedResolver, indexResolver);
   }
 
   private static Codeowners buildCodeowners(String repoRoot) {

diff --git a/...nt/agent-ci-visibility/src/main/java/datadog/trace/civisibility/CiVisibilityServices.java b/...nt/agent-ci-visibility/src/main/java/datadog/trace/civisibility/CiVisibilityServices.java
@@ -82,7 +82,7 @@ public class CiVisibilityServices {
       this.signalClientFactory = new SignalClient.Factory(signalServerAddress, config);
 
       RepoIndexProvider indexFetcher = new RepoIndexFetcher(signalClientFactory);
-      this.repoIndexProviderFactory = (repoRoot, scanRoot) -> indexFetcher;
+      this.repoIndexProviderFactory = (repoRoot) -> indexFetcher;
 
     } else {
       this.signalClientFactory = null;

diff --git a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/source/Utils.java b/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/source/Utils.java
@@ -1,11 +1,22 @@
 package datadog.trace.civisibility.source;
 
+import java.io.File;
 import java.io.IOException;
 import java.io.InputStream;
+import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
+import org.objectweb.asm.ClassReader;
+import org.objectweb.asm.ClassVisitor;
+import org.objectweb.asm.Opcodes;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 public abstract class Utils {
 
-  public static InputStream getClassStream(Class<?> clazz) throws IOException {
+  private static final Logger log = LoggerFactory.getLogger(Utils.class);
+
+  @Nullable
+  public static InputStream getClassStream(@Nonnull Class<?> clazz) {
     String className = clazz.getName();
     InputStream classStream = clazz.getResourceAsStream(toResourceName(className));
     if (classStream != null) {
@@ -17,16 +28,59 @@ public static InputStream getClassStream(Class<?> clazz) throws IOException {
     }
   }
 
+  @Nullable
+  public static String getFileName(@Nonnull Class<?> clazz) throws IOException {
+    try (InputStream classStream = Utils.getClassStream(clazz)) {
+      if (classStream == null) {
+        log.debug("Could not get input stream for class {}", clazz.getName());
+        return null;
+      }
+
+      ClassReader classReader = new ClassReader(classStream);
+      SourceFileAttributeVisitor visitor = new SourceFileAttributeVisitor();
+      classReader.accept(visitor, ClassReader.SKIP_CODE | ClassReader.SKIP_FRAMES);
+      return visitor.getSource();
+    }
+  }
+
+  private static final class SourceFileAttributeVisitor extends ClassVisitor {
+    private String source;
+
+    SourceFileAttributeVisitor() {
+      super(Opcodes.ASM9);
+    }
+
+    @Override
+    public void visitSource(String source, String debug) {
+      this.source = source;
+    }
+
+    public String getSource() {
+      return source;
+    }
+  }
+
   private static String toResourceName(String className) {
     return "/" + className.replace('.', '/') + ".class";
   }
 
-  public static String stripNestedClassNames(String className) {
+  @Nonnull
+  public static String stripNestedClassNames(@Nonnull String className) {
     int innerClassNameIdx = className.indexOf('$');
     if (innerClassNameIdx >= 0) {
       return className.substring(0, innerClassNameIdx);
     } else {
       return className;
     }
   }
+
+  @Nonnull
+  public static String toTrieKey(@Nonnull String relativePath) {
+    return stripExtension(relativePath).replace(File.separatorChar, '.');
+  }
+
+  @Nonnull
+  public static String stripExtension(@Nonnull String fileName) {
+    return fileName.substring(0, fileName.lastIndexOf('.'));
+  }
 }
diff --git a/...src/main/java/datadog/trace/civisibility/source/index/CachingRepoIndexBuilderFactory.java b/...src/main/java/datadog/trace/civisibility/source/index/CachingRepoIndexBuilderFactory.java
@@ -1,15 +1,13 @@
 package datadog.trace.civisibility.source.index;
 
 import datadog.trace.api.Config;
-import datadog.trace.api.Pair;
 import datadog.trace.api.cache.DDCache;
 import datadog.trace.api.cache.DDCaches;
 import java.nio.file.FileSystem;
 
 public class CachingRepoIndexBuilderFactory implements RepoIndexProvider.Factory {
 
-  private final DDCache<Pair<String, String>, RepoIndexProvider> cache =
-      DDCaches.newFixedSizeCache(8);
+  private final DDCache<String, RepoIndexProvider> cache = DDCaches.newFixedSizeCache(8);
   private final Config config;
   private final PackageResolver packageResolver;
   private final ResourceResolver resourceResolver;
@@ -27,15 +25,11 @@ public CachingRepoIndexBuilderFactory(
   }
 
   @Override
-  public RepoIndexProvider create(String repoRoot, String scanRoot) {
-    Pair<String, String> key = Pair.of(repoRoot, scanRoot);
-    return cache.computeIfAbsent(key, this::doCreate);
+  public RepoIndexProvider create(String repoRoot) {
+    return cache.computeIfAbsent(repoRoot, this::doCreate);
   }
 
-  private RepoIndexProvider doCreate(Pair<String, String> key) {
-    String repoRoot = key.getLeft();
-    String scanRoot = key.getRight();
-    return new RepoIndexBuilder(
-        config, repoRoot, scanRoot, packageResolver, resourceResolver, fileSystem);
+  private RepoIndexProvider doCreate(String repoRoot) {
+    return new RepoIndexBuilder(config, repoRoot, packageResolver, resourceResolver, fileSystem);
   }
 }
diff --git a/...rc/main/java/datadog/trace/civisibility/source/index/ConventionBasedResourceResolver.java b/...rc/main/java/datadog/trace/civisibility/source/index/ConventionBasedResourceResolver.java
@@ -1,6 +1,5 @@
 package datadog.trace.civisibility.source.index;
 
-import java.io.IOException;
 import java.nio.file.FileSystem;
 import java.nio.file.Path;
 import java.util.List;
@@ -28,7 +27,7 @@ public ConventionBasedResourceResolver(FileSystem fileSystem, List<String> resou
    * @return Resource root
    */
   @Override
-  public Path getResourceRoot(Path resourceFile) throws IOException {
+  public Path getResourceRoot(Path resourceFile) {
     String pathAsString = resourceFile.toString();
     for (String resourceFolderName : resourceFolderNames) {
       int idx = pathAsString.indexOf(resourceFolderName);

diff --git a/...t/agent-ci-visibility/src/main/java/datadog/trace/civisibility/source/index/Language.java b/...t/agent-ci-visibility/src/main/java/datadog/trace/civisibility/source/index/Language.java
@@ -0,0 +1,43 @@
+package datadog.trace.civisibility.source.index;
+
+import javax.annotation.Nullable;
+
+enum Language {
+  JAVA(".java", false),
+  GROOVY(".groovy", false),
+  KOTLIN(".kt", false),
+  SCALA(".scala", false),
+  GHERKIN(".feature", true);
+
+  private static final Language[] UNIVERSE = Language.values();
+
+  private final String extension;
+  private final boolean nonCode;
+
+  Language(String extension, boolean nonCode) {
+    this.extension = extension;
+    this.nonCode = nonCode;
+  }
+
+  public String getExtension() {
+    return extension;
+  }
+
+  public boolean isNonCode() {
+    return nonCode;
+  }
+
+  @Nullable
+  static Language getByFileName(String fileName) {
+    for (Language language : UNIVERSE) {
+      if (fileName.endsWith(language.extension)) {
+        return language;
+      }
+    }
+    return null;
+  }
+
+  static Language getByOrdinal(int ordinal) {
+    return UNIVERSE[ordinal];
+  }
+}
diff --git a/...ci-visibility/src/main/java/datadog/trace/civisibility/source/index/LanguageDetector.java b/...ci-visibility/src/main/java/datadog/trace/civisibility/source/index/LanguageDetector.java
@@ -0,0 +1,54 @@
+package datadog.trace.civisibility.source.index;
+
+import datadog.trace.civisibility.source.Utils;
+import java.io.IOException;
+import java.lang.annotation.Annotation;
+import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+class LanguageDetector {
+
+  private static final Logger log = LoggerFactory.getLogger(LanguageDetector.class);
+
+  @Nullable
+  public Language detect(@Nonnull Class<?> clazz) {
+    Class<?> c = clazz;
+    while (c != null) {
+      Class<?>[] interfaces = c.getInterfaces();
+      for (Class<?> anInterface : interfaces) {
+        String interfaceName = anInterface.getName();
+        if ("groovy.lang.GroovyObject".equals(interfaceName)) {
+          return Language.GROOVY;
+        }
+      }
+
+      Annotation[] annotations = c.getAnnotations();
+      for (Annotation annotation : annotations) {
+        Class<? extends Annotation> annotationType = annotation.annotationType();
+        if ("kotlin.Metadata".equals(annotationType.getName())) {
+          return Language.KOTLIN;
+        }
+        if ("scala.reflect.ScalaSignature".equals(annotationType.getName())) {
+          return Language.SCALA;
+        }
+      }
+
+      c = c.getSuperclass();
+    }
+
+    try {
+      // try to parse the class file to see if it contains filename attribute
+      String fileName = Utils.getFileName(clazz);
+      if (fileName != null) {
+        return Language.getByFileName(fileName);
+      }
+    } catch (IOException e) {
+      log.debug("Error while trying to read filename from class {}", clazz.getName(), e);
+    }
+
+    // assuming Java
+    return Language.JAVA;
+  }
+}