Merge branch 'master' into dougqh/resource-sampling

.circleci/config.continue.yml.j2

@@ -36,7 +36,7 @@ instrumentation_modules: &instrumentation_modules "dd-java-agent/instrumentation
 debugger_modules: &debugger_modules "dd-java-agent/agent-debugger|dd-java-agent/agent-bootstrap|dd-java-agent/agent-builder|internal-api|communication|dd-trace-core"
 profiling_modules: &profiling_modules "dd-java-agent/agent-profiling"
 
-default_system_tests_commit: &default_system_tests_commit 2487cea5160a398549743d2cfd927a863792e3bd
+default_system_tests_commit: &default_system_tests_commit 6998b307ce0e8d206ca6eb99045651dcc8bc853f
 
 parameters:
   nightly:
@@ -735,6 +735,7 @@ jobs:
           name: Generate muzzle dep report
           command: >-
             SKIP_BUILDSCAN="true"
+            GRADLE_OPTS="-Dorg.gradle.jvmargs='-Xmx2G -Xms2G -XX:ErrorFile=/tmp/hs_err_pid%p.log -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/tmp'"
             ./gradlew generateMuzzleReport muzzleInstrumentationReport
       - run:
           name: Collect Reports

.github/CODEOWNERS

@@ -19,6 +19,7 @@ dd-java-agent/instrumentation/cucumber/    @DataDog/ci-app-libraries-java
 dd-java-agent/instrumentation/jacoco/      @DataDog/ci-app-libraries-java
 dd-java-agent/instrumentation/junit-4.10/  @DataDog/ci-app-libraries-java
 dd-java-agent/instrumentation/junit-5.3/   @DataDog/ci-app-libraries-java
+dd-java-agent/instrumentation/karate/      @DataDog/ci-app-libraries-java
 dd-java-agent/instrumentation/testng/      @DataDog/ci-app-libraries-java
 dd-java-agent/instrumentation/gradle/      @DataDog/ci-app-libraries-java
 dd-java-agent/instrumentation/maven-3.2.1/ @DataDog/ci-app-libraries-java
@@ -36,6 +37,9 @@ dd-smoke-tests/debugger-integration-tests/ @DataDog/debugger-java
 dd-java-agent/agent-iast/              @DataDog/asm-java
 dd-java-agent/instrumentation/*iast*   @DataDog/asm-java
 dd-java-agent/instrumentation/*appsec* @DataDog/asm-java
+dd-java-agent/instrumentation/json/    @DataDog/asm-java
+dd-smoke-tests/iast-util/              @DataDog/asm-java
+dd-java-agent/instrumentation/commons-fileupload/    @DataDog/asm-java
 **/appsec/                             @DataDog/asm-java
 **/iast/                               @DataDog/asm-java
 **/Iast*.java                          @DataDog/asm-java

.gitlab-ci.yml

@@ -82,6 +82,12 @@ package:
   script:
     - ../.gitlab/build_java_package.sh
 
+package-arm:
+  extends: .package-arm
+  when: on_success # this can't use 'needs: [build]', since build is not available in the scheduled pipeline
+  script:
+    - ../.gitlab/build_java_package.sh
+
 .release-package:
   stage: deploy
   variables:

communication/src/main/java/datadog/communication/ddagent/SharedCommunicationObjects.java

@@ -39,10 +39,15 @@ public void createRemaining(Config config) {
       String namedPipe = config.getAgentNamedPipe();
       okHttpClient =
           OkHttpUtils.buildHttpClient(
-              agentUrl,
-              unixDomainSocket,
-              namedPipe,
-              TimeUnit.SECONDS.toMillis(config.getAgentTimeout()));
+              agentUrl, unixDomainSocket, namedPipe, getHttpClientTimeout(config));
+    }
+  }
+
+  private static long getHttpClientTimeout(Config config) {
+    if (!config.isCiVisibilityEnabled() || !config.isCiVisibilityAgentlessEnabled()) {
+      return TimeUnit.SECONDS.toMillis(config.getAgentTimeout());
+    } else {
+      return config.getCiVisibilityBackendApiTimeoutMillis();
     }
   }
 

communication/src/main/java/datadog/communication/monitor/DDAgentStatsDConnection.java

@@ -115,7 +115,6 @@ private void doConnect() {
         // when using UDS, set "entity-id" to "none" to avoid having the DogStatsD
         // server add origin tags (see https://github.com/DataDog/jmxfetch/pull/264)
         if (this.port == 0) {
-          clientBuilder.constantTags("dd.internal.card:none");
           clientBuilder.entityID("none");
         } else {
           clientBuilder.entityID(null);

dd-java-agent/agent-ci-visibility/build.gradle

@@ -27,11 +27,12 @@ excludedClassesCoverage += [
   "datadog.trace.civisibility.ci.CIInfo",
   "datadog.trace.civisibility.ci.CIInfo.Builder",
   "datadog.trace.civisibility.communication.*",
-  "datadog.trace.civisibility.config.JvmInfoFactory",
+  "datadog.trace.civisibility.config.CachingJvmInfoFactory",
+  "datadog.trace.civisibility.config.JvmInfoFactoryImpl",
+  "datadog.trace.civisibility.config.JvmInfoFactoryImpl.JvmVersionOutputParser",
   "datadog.trace.civisibility.config.ConfigurationApi",
   "datadog.trace.civisibility.config.ModuleExecutionSettingsFactory",
   "datadog.trace.civisibility.config.JvmInfo",
-  "datadog.trace.civisibility.config.JvmInfoFactory.JvmVersionOutputParser",
   "datadog.trace.civisibility.config.CachingModuleExecutionSettingsFactory",
   "datadog.trace.civisibility.config.CachingModuleExecutionSettingsFactory.Key",
   "datadog.trace.civisibility.config.CiVisibilitySettings",
@@ -73,6 +74,7 @@ excludedClassesCoverage += [
   "datadog.trace.civisibility.source.index.RepoIndexBuilder.RepoIndexingFileVisitor",
   "datadog.trace.civisibility.source.index.RepoIndexFetcher",
   "datadog.trace.civisibility.source.index.RepoIndexSourcePathResolver",
+  "datadog.trace.civisibility.source.Utils",
   "datadog.trace.civisibility.utils.ShellCommandExecutor",
   "datadog.trace.civisibility.utils.ShellCommandExecutor.OutputParser",
   "datadog.trace.civisibility.utils.SpanUtils"

dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/CiVisibilitySystem.java

@@ -1,6 +1,7 @@
 package datadog.trace.civisibility;
 
 import datadog.communication.ddagent.SharedCommunicationObjects;
+import datadog.communication.ddagent.TracerVersion;
 import datadog.trace.api.Config;
 import datadog.trace.api.civisibility.CIVisibility;
 import datadog.trace.api.civisibility.InstrumentationBridge;
@@ -17,10 +18,12 @@
 import datadog.trace.civisibility.codeowners.CodeownersProvider;
 import datadog.trace.civisibility.communication.BackendApi;
 import datadog.trace.civisibility.communication.BackendApiFactory;
+import datadog.trace.civisibility.config.CachingJvmInfoFactory;
 import datadog.trace.civisibility.config.CachingModuleExecutionSettingsFactory;
 import datadog.trace.civisibility.config.ConfigurationApi;
 import datadog.trace.civisibility.config.ConfigurationApiImpl;
 import datadog.trace.civisibility.config.JvmInfoFactory;
+import datadog.trace.civisibility.config.JvmInfoFactoryImpl;
 import datadog.trace.civisibility.config.ModuleExecutionSettingsFactory;
 import datadog.trace.civisibility.config.ModuleExecutionSettingsFactoryImpl;
 import datadog.trace.civisibility.coverage.CoverageProbeStoreFactory;
@@ -52,6 +55,7 @@
 import datadog.trace.civisibility.source.index.RepoIndexProvider;
 import datadog.trace.civisibility.source.index.RepoIndexSourcePathResolver;
 import datadog.trace.util.Strings;
+import datadog.trace.util.throwable.FatalAgentMisconfigurationError;
 import java.net.InetSocketAddress;
 import java.nio.file.FileSystems;
 import java.nio.file.Path;
@@ -74,6 +78,19 @@ public static void start(SharedCommunicationObjects sco) {
       return;
     }
 
+    String injectedTracerVersion = config.getCiVisibilityInjectedTracerVersion();
+    if (injectedTracerVersion != null
+        && !injectedTracerVersion.equals(TracerVersion.TRACER_VERSION)) {
+      throw new FatalAgentMisconfigurationError(
+          "Running JVM with tracer version "
+              + TracerVersion.TRACER_VERSION
+              + " however parent process attempted to inject "
+              + injectedTracerVersion
+              + ". Do not inject the tracer into the forked JVMs manually, or ensure the manually injected version is the same as the one injected automatically");
+    }
+
+    sco.createRemaining(config);
+
     GitClient.Factory gitClientFactory = buildGitClientFactory(config);
     CoverageProbeStoreFactory coverageProbeStoreFactory = buildTestProbesFactory(config);
 
@@ -117,10 +134,11 @@ private static BuildEventsHandler.Factory buildEventsHandlerFactory(
     DDBuildSystemSession.Factory sessionFactory =
         buildSystemSessionFactory(
             config, sco, gitInfoProvider, coverageProbeStoreFactory, gitClientFactory);
+    JvmInfoFactory jvmInfoFactory = new CachingJvmInfoFactory(config, new JvmInfoFactoryImpl());
     return new BuildEventsHandler.Factory() {
       @Override
       public <U> BuildEventsHandler<U> create() {
-        return new BuildEventsHandlerImpl<>(sessionFactory, new JvmInfoFactory());
+        return new BuildEventsHandlerImpl<>(sessionFactory, jvmInfoFactory);
       }
     };
   }
@@ -132,8 +150,6 @@ private static DDBuildSystemSession.Factory buildSystemSessionFactory(
       CoverageProbeStoreFactory coverageProbeStoreFactory,
       GitClient.Factory gitClientFactory) {
     BackendApiFactory backendApiFactory = new BackendApiFactory(config, sco);
-    BackendApi backendApi = backendApiFactory.createBackendApi();
-
     return (String projectName,
         Path projectRoot,
         String startCommand,
@@ -163,6 +179,7 @@ private static DDBuildSystemSession.Factory buildSystemSessionFactory(
       TestDecorator testDecorator = new TestDecoratorImpl(buildSystemName, ciTags);
       TestModuleRegistry testModuleRegistry = new TestModuleRegistry();
 
+      BackendApi backendApi = backendApiFactory.createBackendApi();
       GitDataUploader gitDataUploader =
           buildGitDataUploader(config, gitInfoProvider, gitClientFactory, backendApi, repoRoot);
       ModuleExecutionSettingsFactory moduleExecutionSettingsFactory =
@@ -210,7 +227,9 @@ private static TestEventsHandler.Factory testEventsHandlerFactory(
       CIInfo ciInfo = ciProviderInfo.buildCIInfo();
       String repoRoot = ciInfo.getCiWorkspace();
       String moduleName =
-          (repoRoot != null) ? Paths.get(repoRoot).relativize(path).toString() : path.toString();
+          repoRoot != null && path.startsWith(repoRoot)
+              ? Paths.get(repoRoot).relativize(path).toString()
+              : config.getServiceName();
 
       DDTestFrameworkSession testSession =
           sessionFactory.startSession(moduleName, path, component, null);
@@ -226,8 +245,6 @@ private static DDTestFrameworkSession.Factory testFrameworkSessionFactory(
       CoverageProbeStoreFactory coverageProbeStoreFactory,
       GitClient.Factory gitClientFactory) {
     BackendApiFactory backendApiFactory = new BackendApiFactory(config, sco);
-    BackendApi backendApi = backendApiFactory.createBackendApi();
-
     return (String projectName, Path projectRoot, String component, Long startTime) -> {
       CIProviderInfoFactory ciProviderInfoFactory = new CIProviderInfoFactory(config);
       CIProviderInfo ciProviderInfo = ciProviderInfoFactory.createCIProviderInfo(projectRoot);
@@ -273,6 +290,7 @@ private static DDTestFrameworkSession.Factory testFrameworkSessionFactory(
       // either we are in the build system
       // or we are in the tests JVM and the build system is not instrumented
       if (parentProcessSessionId == null || parentProcessModuleId == null) {
+        BackendApi backendApi = backendApiFactory.createBackendApi();
         GitDataUploader gitDataUploader =
             buildGitDataUploader(config, gitInfoProvider, gitClientFactory, backendApi, repoRoot);
         RepoIndexProvider indexProvider = new RepoIndexBuilder(repoRoot, FileSystems.getDefault());

dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/communication/BackendApiFactory.java

@@ -23,7 +23,6 @@ public BackendApiFactory(Config config, SharedCommunicationObjects sharedCommuni
   }
 
   public @Nullable BackendApi createBackendApi() {
-    long timeoutMillis = config.getCiVisibilityBackendApiTimeoutMillis();
     HttpRetryPolicy.Factory retryPolicyFactory = new HttpRetryPolicy.Factory(5, 100, 2.0);
 
     if (config.isCiVisibilityAgentlessEnabled()) {
@@ -33,15 +32,20 @@ public BackendApiFactory(Config config, SharedCommunicationObjects sharedCommuni
         throw new FatalAgentMisconfigurationError(
             "Agentless mode is enabled and api key is not set. Please set application key");
       }
-      return new IntakeApi(site, apiKey, timeoutMillis, retryPolicyFactory);
+      long timeoutMillis = config.getCiVisibilityBackendApiTimeoutMillis();
+      String traceId = config.getIdGenerationStrategy().generateTraceId().toString();
+      return new IntakeApi(site, apiKey, traceId, timeoutMillis, retryPolicyFactory);
     }
 
     DDAgentFeaturesDiscovery featuresDiscovery =
         sharedCommunicationObjects.featuresDiscovery(config);
+    featuresDiscovery.discoverIfOutdated();
     if (featuresDiscovery.supportsEvpProxy()) {
+      String traceId = config.getIdGenerationStrategy().generateTraceId().toString();
       String evpProxyEndpoint = featuresDiscovery.getEvpProxyEndpoint();
       HttpUrl evpProxyUrl = sharedCommunicationObjects.agentUrl.resolve(evpProxyEndpoint);
-      return new EvpProxyApi(evpProxyUrl, timeoutMillis, retryPolicyFactory);
+      return new EvpProxyApi(
+          traceId, evpProxyUrl, retryPolicyFactory, sharedCommunicationObjects.okHttpClient);
     }
 
     log.warn(

dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/communication/EvpProxyApi.java

@@ -19,17 +19,24 @@ public class EvpProxyApi implements BackendApi {
 
   private static final String API_VERSION = "v2";
   private static final String X_DATADOG_EVP_SUBDOMAIN_HEADER = "X-Datadog-EVP-Subdomain";
+  private static final String X_DATADOG_TRACE_ID_HEADER = "x-datadog-trace-id";
+  private static final String X_DATADOG_PARENT_ID_HEADER = "x-datadog-parent-id";
   private static final String API_SUBDOMAIN = "api";
 
+  private final String traceId;
   private final HttpRetryPolicy.Factory retryPolicyFactory;
   private final HttpUrl evpProxyUrl;
   private final OkHttpClient httpClient;
 
   public EvpProxyApi(
-      HttpUrl evpProxyUrl, long timeoutMillis, HttpRetryPolicy.Factory retryPolicyFactory) {
+      String traceId,
+      HttpUrl evpProxyUrl,
+      HttpRetryPolicy.Factory retryPolicyFactory,
+      OkHttpClient httpClient) {
+    this.traceId = traceId;
     this.evpProxyUrl = evpProxyUrl.resolve(String.format("api/%s/", API_VERSION));
     this.retryPolicyFactory = retryPolicyFactory;
-    httpClient = OkHttpUtils.buildHttpClient(evpProxyUrl, timeoutMillis);
+    this.httpClient = httpClient;
   }
 
   @Override
@@ -41,6 +48,8 @@ public <T> T post(
         new Request.Builder()
             .url(url)
             .addHeader(X_DATADOG_EVP_SUBDOMAIN_HEADER, API_SUBDOMAIN)
+            .addHeader(X_DATADOG_TRACE_ID_HEADER, traceId)
+            .addHeader(X_DATADOG_PARENT_ID_HEADER, traceId)
             .post(requestBody)
             .build();
 

dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/communication/IntakeApi.java

@@ -20,15 +20,23 @@ public class IntakeApi implements BackendApi {
 
   private static final String API_VERSION = "v2";
   private static final String DD_API_KEY_HEADER = "dd-api-key";
+  private static final String X_DATADOG_TRACE_ID_HEADER = "x-datadog-trace-id";
+  private static final String X_DATADOG_PARENT_ID_HEADER = "x-datadog-parent-id";
 
   private final String apiKey;
+  private final String traceId;
   private final HttpRetryPolicy.Factory retryPolicyFactory;
   private final HttpUrl hostUrl;
   private final OkHttpClient httpClient;
 
   public IntakeApi(
-      String site, String apiKey, long timeoutMillis, HttpRetryPolicy.Factory retryPolicyFactory) {
+      String site,
+      String apiKey,
+      String traceId,
+      long timeoutMillis,
+      HttpRetryPolicy.Factory retryPolicyFactory) {
     this.apiKey = apiKey;
+    this.traceId = traceId;
     this.retryPolicyFactory = retryPolicyFactory;
 
     final String ciVisibilityAgentlessUrlStr = Config.get().getCiVisibilityAgentlessUrl();
@@ -47,7 +55,12 @@ public <T> T post(
       throws IOException {
     HttpUrl url = hostUrl.resolve(uri);
     Request.Builder requestBuilder =
-        new Request.Builder().url(url).post(requestBody).addHeader(DD_API_KEY_HEADER, apiKey);
+        new Request.Builder()
+            .url(url)
+            .post(requestBody)
+            .addHeader(DD_API_KEY_HEADER, apiKey)
+            .addHeader(X_DATADOG_TRACE_ID_HEADER, traceId)
+            .addHeader(X_DATADOG_PARENT_ID_HEADER, traceId);
 
     Request request = requestBuilder.build();
     HttpRetryPolicy retryPolicy = retryPolicyFactory.create();

dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/config/CachingJvmInfoFactory.java

@@ -0,0 +1,23 @@
+package datadog.trace.civisibility.config;
+
+import datadog.trace.api.Config;
+import datadog.trace.api.cache.DDCache;
+import datadog.trace.api.cache.DDCaches;
+import java.nio.file.Path;
+
+public class CachingJvmInfoFactory implements JvmInfoFactory {
+
+  private final DDCache<Path, JvmInfo> cache;
+  private final JvmInfoFactoryImpl delegate;
+
+  public CachingJvmInfoFactory(Config config, JvmInfoFactoryImpl delegate) {
+    this.delegate = delegate;
+    this.cache =
+        DDCaches.newFixedSizeCache(config.getCiVisibilityModuleExecutionSettingsCacheSize());
+  }
+
+  @Override
+  public JvmInfo getJvmInfo(Path jvmExecutablePath) {
+    return cache.computeIfAbsent(jvmExecutablePath, delegate::getJvmInfo);
+  }
+}

dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/config/ConfigurationsJson.java

@@ -4,6 +4,7 @@
 import com.squareup.moshi.Json;
 import com.squareup.moshi.ToJson;
 import datadog.trace.api.civisibility.config.Configurations;
+import java.util.Map;
 
 public final class ConfigurationsJson {
   @Json(name = "os.platform")
@@ -33,6 +34,9 @@ public final class ConfigurationsJson {
   @Json(name = "test.bundle")
   private final String testBundle;
 
+  @Json(name = "custom")
+  private final Map<String, String> custom;
+
   public ConfigurationsJson(
       String osPlatform,
       String osArchitecture,
@@ -41,7 +45,8 @@ public ConfigurationsJson(
       String runtimeVersion,
       String runtimeVendor,
       String runtimeArchitecture,
-      String testBundle) {
+      String testBundle,
+      Map<String, String> custom) {
     this.osPlatform = osPlatform;
     osArch = osArchitecture;
     this.osArchitecture = osArchitecture;
@@ -51,6 +56,7 @@ public ConfigurationsJson(
     this.runtimeVendor = runtimeVendor;
     this.runtimeArchitecture = runtimeArchitecture;
     this.testBundle = testBundle;
+    this.custom = custom;
   }
 
   public static final class ConfigurationsJsonAdapter {
@@ -64,7 +70,8 @@ public Configurations fromJson(ConfigurationsJson configurationsJson) {
           configurationsJson.runtimeVersion,
           configurationsJson.runtimeVendor,
           configurationsJson.runtimeArchitecture,
-          configurationsJson.testBundle);
+          configurationsJson.testBundle,
+          configurationsJson.custom);
     }
 
     @ToJson
@@ -77,7 +84,8 @@ public ConfigurationsJson toJson(Configurations configurations) {
           configurations.getRuntimeVersion(),
           configurations.getRuntimeVendor(),
           configurations.getRuntimeArchitecture(),
-          configurations.getTestBundle());
+          configurations.getTestBundle(),
+          configurations.getCustom());
     }
   }
 }