Improved finding docker container id on cgroup v2 enabled systems

CHANGELOG.asciidoc

@@ -23,6 +23,10 @@ endif::[]
 [[release-notes-1.30.1]]
 ==== 1.30.1 - YYYY/MM/DD
 
+[float]
+===== Features
+* Improved finding docker container id on cgroup v2 enabled systems - {pull}2352[#2352]
+
 [float]
 ===== Bug fixes
 

apm-agent-core/src/main/java/co/elastic/apm/agent/impl/metadata/SystemInfo.java

@@ -264,7 +264,7 @@ SystemInfo findContainerDetails() {
             if (path.toFile().exists()) {
                 List<String> lines = Files.readAllLines(path, StandardCharsets.UTF_8);
                 for (final String line : lines) {
-                    parseContainerId(line);
+                    parseCGroup(line);
                     if (container != null) {
                         containerId = container.getId();
                         break;
@@ -275,6 +275,24 @@ SystemInfo findContainerDetails() {
             logger.warn("Failed to read/parse container ID from '/proc/self/cgroup'", e);
         }
 
+        if (containerId == null) {
+            try {
+                Path path = FileSystems.getDefault().getPath("/proc/self/mountinfo");
+                if (path.toFile().exists()) {
+                    List<String> lines = Files.readAllLines(path, StandardCharsets.UTF_8);
+                    for (final String line : lines) {
+                        parseMountinfo(line);
+                        if (container != null) {
+                            containerId = container.getId();
+                            break;
+                        }
+                    }
+                }
+            } catch (Throwable e) {
+                logger.warn("Failed to read/parse container ID from '/proc/self/mountinfo'", e);
+            }
+        }
+
         try {
             // Kubernetes Downward API enables setting environment variables. We are looking for the relevant ones to this discovery
             String podUid = System.getenv("KUBERNETES_POD_UID");
@@ -318,7 +336,7 @@ SystemInfo findContainerDetails() {
      * @param line a line from the /proc/self/cgroup file
      * @return this SystemInfo object after parsing
      */
-    SystemInfo parseContainerId(String line) {
+    SystemInfo parseCGroup(String line) {
         final String[] fields = line.split(":", 3);
         if (fields.length == 3) {
             String cGroupPath = fields[2];
@@ -371,6 +389,36 @@ SystemInfo parseContainerId(String line) {
         return this;
     }
 
+    /**
+     * The virtual file /proc/self/mountinfo lists the mounts that the current process sees.
+     * Each line contains the space separated fields:
+     *  mountID parentID major:minor root mountPoint ...
+     *
+     * @param line a line from the /proc/self/mountinfo file
+     * @return this SystemInfo object after parsing
+     */
+    SystemInfo parseMountinfo(String line) {
+        final String[] fields = line.split(" ", 6);
+        if (fields.length >= 5) {
+            String rootPath = fields[3];
+            int containerIdFromIndex = rootPath.indexOf("/docker/containers/");
+            if (containerIdFromIndex >= 0) {
+                containerIdFromIndex += "/docker/containers/".length();
+                int containerIdEndIndex = rootPath.indexOf('/', containerIdFromIndex);
+                if (containerIdEndIndex >= 0) {
+                    String containerId = rootPath.substring(containerIdFromIndex, containerIdEndIndex);
+                    if (containerId.matches(CONTAINER_UID_REGEX) || containerId.matches(SHORTENED_UUID_PATTERN)) {
+                        container = new Container(containerId);
+                    }
+                }
+            }
+        }
+        if (container == null) {
+            logger.debug("Could not parse container ID from '/proc/self/mountinfo' line: {}", line);
+        }
+        return this;
+    }
+
     /**
      * Architecture of the system the agent is running on.
      */

apm-agent-core/src/test/java/co/elastic/apm/agent/impl/metadata/ContainerInfoTest.java

@@ -34,10 +34,10 @@ public class ContainerInfoTest extends CustomEnvVariables {
     void testContainerIdParsing() {
         String validId = "3741401135a8d27237e2fb9c0fb2ecd93922c0d1dd708345451e479613f8d4ae";
         String validLinePrefix = "7:freezer:/path/";
-        assertContainerId(validLinePrefix + validId, validId);
+        assertCGroupContainerId(validLinePrefix + validId, validId);
         assertContainerInfoIsNull(validLinePrefix.substring(2) + validId);
-        assertContainerId(validLinePrefix + "docker-" + validId + ".scope", validId);
-        assertContainerId(validLinePrefix + validId + ".scope", validId);
+        assertCGroupContainerId(validLinePrefix + "docker-" + validId + ".scope", validId);
+        assertCGroupContainerId(validLinePrefix + validId + ".scope", validId);
         assertContainerInfoIsNull(validLinePrefix + validId.replace('f', 'g'));
         assertContainerInfoIsNull(validLinePrefix + validId.substring(1));
         assertContainerInfoIsNull(validLinePrefix + validId.concat("7"));
@@ -50,24 +50,24 @@ void testContainerIdParsing() {
     void testCloudFoundryContainerIdParsing() {
         String validId = "70eb4ce5-a065-4401-6990-88ed";
         String validLinePrefix = "9:net_cls,net_prio:/garden/";
-        assertContainerId(validLinePrefix + validId, validId);
+        assertCGroupContainerId(validLinePrefix + validId, validId);
         assertContainerInfoIsNull(validLinePrefix.substring(2) + validId);
         assertContainerInfoIsNull(validLinePrefix + validId.replace('a', 'g'));
         assertContainerInfoIsNull(validLinePrefix.substring(0, validLinePrefix.length() - 1) + validId);
         assertContainerInfoIsNull(validLinePrefix + validId.substring(0, validId.length() - 1));
         String uuid = validId.concat("abcd1234");
-        assertContainerId(validLinePrefix + uuid, uuid);
+        assertCGroupContainerId(validLinePrefix + uuid, uuid);
         assertContainerInfoIsNull(validLinePrefix + validId.concat("/"));
-        assertContainerId("5:blkio:/system.slice/garden.service/garden/" + validId, validId);
+        assertCGroupContainerId("5:blkio:/system.slice/garden.service/garden/" + validId, validId);
     }
 
     @Test
     void testEcsFormat() {
         String id = "7e9139716d9e5d762d22f9f877b87d1be8b1449ac912c025a984750c5dbff157";
-        assertContainerId("3:cpuacct:/ecs/eb9d3d0c-8936-42d7-80d8-f82b2f1a629e/" + id, id);
+        assertCGroupContainerId("3:cpuacct:/ecs/eb9d3d0c-8936-42d7-80d8-f82b2f1a629e/" + id, id);
 
         // based on https://github.com/aws/amazon-ecs-agent/issues/1119 - supporting Docker on ECS
-        assertContainerId("9:perf_event:/ecs/task-arn/" + id, id);
+        assertCGroupContainerId("9:perf_event:/ecs/task-arn/" + id, id);
     }
 
     @Test
@@ -76,36 +76,36 @@ void testKubernetesInfo() {
         String containerId = "15aa6e53-b09a-40c7-8558-c6c31e36c88a";
         String podId = "e9b90526-f47d-11e8-b2a5-080027b9f4fb";
         String line = "1:name=systemd:/kubepods/besteffort/pod" + podId + "/" + containerId;
-        SystemInfo systemInfo = assertContainerId(line, containerId);
+        SystemInfo systemInfo = assertCGroupContainerId(line, containerId);
         assertKubernetesInfo(systemInfo, podId, "my-host", null, null);
 
         // 12:pids:/kubepods/kubepods/besteffort/pod0e886e9a-3879-45f9-b44d-86ef9df03224/244a65edefdffe31685c42317c9054e71dc1193048cf9459e2a4dd35cbc1dba4
         containerId = "244a65edefdffe31685c42317c9054e71dc1193048cf9459e2a4dd35cbc1dba4";
         podId = "0e886e9a-3879-45f9-b44d-86ef9df03224";
         line = "12:pids:/kubepods/kubepods/besteffort/pod" + podId + "/" + containerId;
-        systemInfo = assertContainerId(line, containerId);
+        systemInfo = assertCGroupContainerId(line, containerId);
         assertKubernetesInfo(systemInfo, podId, "my-host", null, null);
 
         // 10:cpuset:/kubepods/pod5eadac96-ab58-11ea-b82b-0242ac110009/7fe41c8a2d1da09420117894f11dd91f6c3a44dfeb7d125dc594bd53468861df
         containerId = "7fe41c8a2d1da09420117894f11dd91f6c3a44dfeb7d125dc594bd53468861df";
         podId = "5eadac96-ab58-11ea-b82b-0242ac110009";
         line = "10:cpuset:/kubepods/pod" + podId + "/" + containerId;
-        systemInfo = assertContainerId(line, containerId);
+        systemInfo = assertCGroupContainerId(line, containerId);
         assertKubernetesInfo(systemInfo, podId, "my-host", null, null);
     }
 
     @Test
     void testUbuntuCgroup() {
         String line = "1:name=systemd:/user.slice/user-1000.slice/[email protected]/apps.slice/apps-org.gnome.Terminal" +
             ".slice/vte-spawn-75bc72bd-6642-4cf5-b62c-0674e11bfc84.scope";
-        assertThat(createSystemInfo().parseContainerId(line).getContainerInfo()).isNull();
+        assertThat(createSystemInfo().parseCGroup(line).getContainerInfo()).isNull();
     }
 
     @Test
     void testOpenshiftFormDisney() {
         String line = "9:freezer:/kubepods.slice/kubepods-pod22949dce_fd8b_11ea_8ede_98f2b32c645c.slice" +
             "/docker-b15a5bdedd2e7645c3be271364324321b908314e4c77857bbfd32a041148c07f.scope";
-        SystemInfo systemInfo = assertContainerId(line, "b15a5bdedd2e7645c3be271364324321b908314e4c77857bbfd32a041148c07f");
+        SystemInfo systemInfo = assertCGroupContainerId(line, "b15a5bdedd2e7645c3be271364324321b908314e4c77857bbfd32a041148c07f");
         assertKubernetesInfo(systemInfo, "22949dce-fd8b-11ea-8ede-98f2b32c645c", "my-host", null, null);
     }
 
@@ -115,7 +115,7 @@ void testKubernetesInfo_podUid_with_underscores() {
         String line = "1:name=systemd:/kubepods.slice/kubepods-burstable.slice/" +
             "kubepods-burstable-pod90d81341_92de_11e7_8cf2_507b9d4141fa.slice/" +
             "crio-2227daf62df6694645fee5df53c1f91271546a9560e8600a525690ae252b7f63.scope";
-        SystemInfo systemInfo = assertContainerId(line, "2227daf62df6694645fee5df53c1f91271546a9560e8600a525690ae252b7f63");
+        SystemInfo systemInfo = assertCGroupContainerId(line, "2227daf62df6694645fee5df53c1f91271546a9560e8600a525690ae252b7f63");
         assertKubernetesInfo(systemInfo, "90d81341-92de-11e7-8cf2-507b9d4141fa", "my-host", null, null);
     }
 
@@ -124,15 +124,15 @@ void testKubernetesInfo_containerd_cri() {
         // In such cases- underscores should be replaced with hyphens in the pod UID
         String line = "1:name=systemd:/system.slice/containerd.service/kubepods-burstable-podff49d0be_16b7_4a49_bb9e_8ec1f1f4e27f.slice" +
             ":cri-containerd:0f99ad5f45163ed14ab8eaf92ed34bb4a631d007f8755a7d79be614bcb0df0ef";
-        SystemInfo systemInfo = assertContainerId(line, "0f99ad5f45163ed14ab8eaf92ed34bb4a631d007f8755a7d79be614bcb0df0ef");
+        SystemInfo systemInfo = assertCGroupContainerId(line, "0f99ad5f45163ed14ab8eaf92ed34bb4a631d007f8755a7d79be614bcb0df0ef");
         assertKubernetesInfo(systemInfo, "ff49d0be-16b7-4a49-bb9e-8ec1f1f4e27f", "my-host", null, null);
     }
 
     @Test
     void testKubernetesDownwardApi() throws Exception {
         String line = "1:name=systemd:/kubepods/besteffort/pode9b90526-f47d-11e8-b2a5-080027b9f4fb/15aa6e53-b09a-40c7-8558-c6c31e36c88a";
         String containerId = "15aa6e53-b09a-40c7-8558-c6c31e36c88a";
-        SystemInfo systemInfo = assertContainerId(line, containerId);
+        SystemInfo systemInfo = assertCGroupContainerId(line, containerId);
 
         String originalPodUid = "e9b90526-f47d-11e8-b2a5-080027b9f4fb";
         String hostName = "my-host";
@@ -152,30 +152,43 @@ void testKubernetesDownwardApi() throws Exception {
         assertKubernetesInfo(systemInfo, podUid, podName, nodeName, namespace);
 
         // test partial settings
-        systemInfo = assertContainerId(line, containerId);
+        systemInfo = assertCGroupContainerId(line, containerId);
         assertKubernetesInfo(systemInfo, originalPodUid, hostName, null, null);
         mockedEnv.put("KUBERNETES_POD_NAME", null);
         mockedEnv.put("KUBERNETES_POD_UID", null);
         runWithCustomEnvVariables(mockedEnv, systemInfo::findContainerDetails);
         assertKubernetesInfo(systemInfo, originalPodUid, hostName, nodeName, namespace);
     }
 
+    @Test
+    void testMountinfo() {
+        assertMountinfoContainerId("533 525 254:1 /var/lib/docker/containers/c677b374c4747e7297ec4e1792421158dfbaac39820348b6a506739f6947d19d/resolv.conf /etc/resolv.conf rw,relatime - ext4 /dev/vda1 rw", "c677b374c4747e7297ec4e1792421158dfbaac39820348b6a506739f6947d19d");
+        assertMountinfoContainerId("534 525 254:1 /docker/containers/c677b374c4747e7297ec4e1792421158dfbaac39820348b6a506739f6947d19d/hostname /etc/hostname rw,relatime - ext4 /dev/vda1 rw", "c677b374c4747e7297ec4e1792421158dfbaac39820348b6a506739f6947d19d");
+    }
 
     private SystemInfo createSystemInfo() {
         return new SystemInfo("arch", "my-host", null, "platform");
     }
 
-    private SystemInfo assertContainerId(String line, String containerId) {
+    private SystemInfo assertCGroupContainerId(String line, String containerId) {
+        SystemInfo systemInfo = createSystemInfo();
+        assertThat(systemInfo.parseCGroup(line).getContainerInfo()).isNotNull();
+        //noinspection ConstantConditions
+        assertThat(systemInfo.getContainerInfo().getId()).isEqualTo(containerId);
+        return systemInfo;
+    }
+
+    private SystemInfo assertMountinfoContainerId(String line, String containerId) {
         SystemInfo systemInfo = createSystemInfo();
-        assertThat(systemInfo.parseContainerId(line).getContainerInfo()).isNotNull();
+        assertThat(systemInfo.parseMountinfo(line).getContainerInfo()).isNotNull();
         //noinspection ConstantConditions
         assertThat(systemInfo.getContainerInfo().getId()).isEqualTo(containerId);
         return systemInfo;
     }
 
     private void assertContainerInfoIsNull(String line) {
         SystemInfo systemInfo = createSystemInfo();
-        assertThat(systemInfo.parseContainerId(line).getContainerInfo()).isNull();
+        assertThat(systemInfo.parseCGroup(line).getContainerInfo()).isNull();
     }
 
     private void assertKubernetesInfo(SystemInfo systemInfo, @Nullable String podUid, @Nullable String podName, @Nullable String nodeName,