diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index 6e6e0d8a0c..416513b4f2 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -23,6 +23,10 @@ endif::[] [[release-notes-1.30.1]] ==== 1.30.1 - YYYY/MM/DD +[float] +===== Features +* Improved finding docker container id on cgroup v2 enabled systems - {pull}2352[#2352] + [float] ===== Bug fixes diff --git a/apm-agent-core/src/main/java/co/elastic/apm/agent/impl/metadata/SystemInfo.java b/apm-agent-core/src/main/java/co/elastic/apm/agent/impl/metadata/SystemInfo.java index caa6543ba5..4fff3aaab1 100644 --- a/apm-agent-core/src/main/java/co/elastic/apm/agent/impl/metadata/SystemInfo.java +++ b/apm-agent-core/src/main/java/co/elastic/apm/agent/impl/metadata/SystemInfo.java @@ -264,7 +264,7 @@ SystemInfo findContainerDetails() { if (path.toFile().exists()) { List lines = Files.readAllLines(path, StandardCharsets.UTF_8); for (final String line : lines) { - parseContainerId(line); + parseCGroup(line); if (container != null) { containerId = container.getId(); break; @@ -275,6 +275,24 @@ SystemInfo findContainerDetails() { logger.warn("Failed to read/parse container ID from '/proc/self/cgroup'", e); } + if (containerId == null) { + try { + Path path = FileSystems.getDefault().getPath("/proc/self/mountinfo"); + if (path.toFile().exists()) { + List lines = Files.readAllLines(path, StandardCharsets.UTF_8); + for (final String line : lines) { + parseMountinfo(line); + if (container != null) { + containerId = container.getId(); + break; + } + } + } + } catch (Throwable e) { + logger.warn("Failed to read/parse container ID from '/proc/self/mountinfo'", e); + } + } + try { // Kubernetes Downward API enables setting environment variables. We are looking for the relevant ones to this discovery String podUid = System.getenv("KUBERNETES_POD_UID"); @@ -318,7 +336,7 @@ SystemInfo findContainerDetails() { * @param line a line from the /proc/self/cgroup file * @return this SystemInfo object after parsing */ - SystemInfo parseContainerId(String line) { + SystemInfo parseCGroup(String line) { final String[] fields = line.split(":", 3); if (fields.length == 3) { String cGroupPath = fields[2]; @@ -371,6 +389,36 @@ SystemInfo parseContainerId(String line) { return this; } + /** + * The virtual file /proc/self/mountinfo lists the mounts that the current process sees. + * Each line contains the space separated fields: + * mountID parentID major:minor root mountPoint ... + * + * @param line a line from the /proc/self/mountinfo file + * @return this SystemInfo object after parsing + */ + SystemInfo parseMountinfo(String line) { + final String[] fields = line.split(" ", 6); + if (fields.length >= 5) { + String rootPath = fields[3]; + int containerIdFromIndex = rootPath.indexOf("/docker/containers/"); + if (containerIdFromIndex >= 0) { + containerIdFromIndex += "/docker/containers/".length(); + int containerIdEndIndex = rootPath.indexOf('/', containerIdFromIndex); + if (containerIdEndIndex >= 0) { + String containerId = rootPath.substring(containerIdFromIndex, containerIdEndIndex); + if (containerId.matches(CONTAINER_UID_REGEX) || containerId.matches(SHORTENED_UUID_PATTERN)) { + container = new Container(containerId); + } + } + } + } + if (container == null) { + logger.debug("Could not parse container ID from '/proc/self/mountinfo' line: {}", line); + } + return this; + } + /** * Architecture of the system the agent is running on. */ diff --git a/apm-agent-core/src/test/java/co/elastic/apm/agent/impl/metadata/ContainerInfoTest.java b/apm-agent-core/src/test/java/co/elastic/apm/agent/impl/metadata/ContainerInfoTest.java index d779e2cbd6..d70cd9a038 100644 --- a/apm-agent-core/src/test/java/co/elastic/apm/agent/impl/metadata/ContainerInfoTest.java +++ b/apm-agent-core/src/test/java/co/elastic/apm/agent/impl/metadata/ContainerInfoTest.java @@ -34,10 +34,10 @@ public class ContainerInfoTest extends CustomEnvVariables { void testContainerIdParsing() { String validId = "3741401135a8d27237e2fb9c0fb2ecd93922c0d1dd708345451e479613f8d4ae"; String validLinePrefix = "7:freezer:/path/"; - assertContainerId(validLinePrefix + validId, validId); + assertCGroupContainerId(validLinePrefix + validId, validId); assertContainerInfoIsNull(validLinePrefix.substring(2) + validId); - assertContainerId(validLinePrefix + "docker-" + validId + ".scope", validId); - assertContainerId(validLinePrefix + validId + ".scope", validId); + assertCGroupContainerId(validLinePrefix + "docker-" + validId + ".scope", validId); + assertCGroupContainerId(validLinePrefix + validId + ".scope", validId); assertContainerInfoIsNull(validLinePrefix + validId.replace('f', 'g')); assertContainerInfoIsNull(validLinePrefix + validId.substring(1)); assertContainerInfoIsNull(validLinePrefix + validId.concat("7")); @@ -50,24 +50,24 @@ void testContainerIdParsing() { void testCloudFoundryContainerIdParsing() { String validId = "70eb4ce5-a065-4401-6990-88ed"; String validLinePrefix = "9:net_cls,net_prio:/garden/"; - assertContainerId(validLinePrefix + validId, validId); + assertCGroupContainerId(validLinePrefix + validId, validId); assertContainerInfoIsNull(validLinePrefix.substring(2) + validId); assertContainerInfoIsNull(validLinePrefix + validId.replace('a', 'g')); assertContainerInfoIsNull(validLinePrefix.substring(0, validLinePrefix.length() - 1) + validId); assertContainerInfoIsNull(validLinePrefix + validId.substring(0, validId.length() - 1)); String uuid = validId.concat("abcd1234"); - assertContainerId(validLinePrefix + uuid, uuid); + assertCGroupContainerId(validLinePrefix + uuid, uuid); assertContainerInfoIsNull(validLinePrefix + validId.concat("/")); - assertContainerId("5:blkio:/system.slice/garden.service/garden/" + validId, validId); + assertCGroupContainerId("5:blkio:/system.slice/garden.service/garden/" + validId, validId); } @Test void testEcsFormat() { String id = "7e9139716d9e5d762d22f9f877b87d1be8b1449ac912c025a984750c5dbff157"; - assertContainerId("3:cpuacct:/ecs/eb9d3d0c-8936-42d7-80d8-f82b2f1a629e/" + id, id); + assertCGroupContainerId("3:cpuacct:/ecs/eb9d3d0c-8936-42d7-80d8-f82b2f1a629e/" + id, id); // based on https://github.com/aws/amazon-ecs-agent/issues/1119 - supporting Docker on ECS - assertContainerId("9:perf_event:/ecs/task-arn/" + id, id); + assertCGroupContainerId("9:perf_event:/ecs/task-arn/" + id, id); } @Test @@ -76,21 +76,21 @@ void testKubernetesInfo() { String containerId = "15aa6e53-b09a-40c7-8558-c6c31e36c88a"; String podId = "e9b90526-f47d-11e8-b2a5-080027b9f4fb"; String line = "1:name=systemd:/kubepods/besteffort/pod" + podId + "/" + containerId; - SystemInfo systemInfo = assertContainerId(line, containerId); + SystemInfo systemInfo = assertCGroupContainerId(line, containerId); assertKubernetesInfo(systemInfo, podId, "my-host", null, null); // 12:pids:/kubepods/kubepods/besteffort/pod0e886e9a-3879-45f9-b44d-86ef9df03224/244a65edefdffe31685c42317c9054e71dc1193048cf9459e2a4dd35cbc1dba4 containerId = "244a65edefdffe31685c42317c9054e71dc1193048cf9459e2a4dd35cbc1dba4"; podId = "0e886e9a-3879-45f9-b44d-86ef9df03224"; line = "12:pids:/kubepods/kubepods/besteffort/pod" + podId + "/" + containerId; - systemInfo = assertContainerId(line, containerId); + systemInfo = assertCGroupContainerId(line, containerId); assertKubernetesInfo(systemInfo, podId, "my-host", null, null); // 10:cpuset:/kubepods/pod5eadac96-ab58-11ea-b82b-0242ac110009/7fe41c8a2d1da09420117894f11dd91f6c3a44dfeb7d125dc594bd53468861df containerId = "7fe41c8a2d1da09420117894f11dd91f6c3a44dfeb7d125dc594bd53468861df"; podId = "5eadac96-ab58-11ea-b82b-0242ac110009"; line = "10:cpuset:/kubepods/pod" + podId + "/" + containerId; - systemInfo = assertContainerId(line, containerId); + systemInfo = assertCGroupContainerId(line, containerId); assertKubernetesInfo(systemInfo, podId, "my-host", null, null); } @@ -98,14 +98,14 @@ void testKubernetesInfo() { void testUbuntuCgroup() { String line = "1:name=systemd:/user.slice/user-1000.slice/user@1000.service/apps.slice/apps-org.gnome.Terminal" + ".slice/vte-spawn-75bc72bd-6642-4cf5-b62c-0674e11bfc84.scope"; - assertThat(createSystemInfo().parseContainerId(line).getContainerInfo()).isNull(); + assertThat(createSystemInfo().parseCGroup(line).getContainerInfo()).isNull(); } @Test void testOpenshiftFormDisney() { String line = "9:freezer:/kubepods.slice/kubepods-pod22949dce_fd8b_11ea_8ede_98f2b32c645c.slice" + "/docker-b15a5bdedd2e7645c3be271364324321b908314e4c77857bbfd32a041148c07f.scope"; - SystemInfo systemInfo = assertContainerId(line, "b15a5bdedd2e7645c3be271364324321b908314e4c77857bbfd32a041148c07f"); + SystemInfo systemInfo = assertCGroupContainerId(line, "b15a5bdedd2e7645c3be271364324321b908314e4c77857bbfd32a041148c07f"); assertKubernetesInfo(systemInfo, "22949dce-fd8b-11ea-8ede-98f2b32c645c", "my-host", null, null); } @@ -115,7 +115,7 @@ void testKubernetesInfo_podUid_with_underscores() { String line = "1:name=systemd:/kubepods.slice/kubepods-burstable.slice/" + "kubepods-burstable-pod90d81341_92de_11e7_8cf2_507b9d4141fa.slice/" + "crio-2227daf62df6694645fee5df53c1f91271546a9560e8600a525690ae252b7f63.scope"; - SystemInfo systemInfo = assertContainerId(line, "2227daf62df6694645fee5df53c1f91271546a9560e8600a525690ae252b7f63"); + SystemInfo systemInfo = assertCGroupContainerId(line, "2227daf62df6694645fee5df53c1f91271546a9560e8600a525690ae252b7f63"); assertKubernetesInfo(systemInfo, "90d81341-92de-11e7-8cf2-507b9d4141fa", "my-host", null, null); } @@ -124,7 +124,7 @@ void testKubernetesInfo_containerd_cri() { // In such cases- underscores should be replaced with hyphens in the pod UID String line = "1:name=systemd:/system.slice/containerd.service/kubepods-burstable-podff49d0be_16b7_4a49_bb9e_8ec1f1f4e27f.slice" + ":cri-containerd:0f99ad5f45163ed14ab8eaf92ed34bb4a631d007f8755a7d79be614bcb0df0ef"; - SystemInfo systemInfo = assertContainerId(line, "0f99ad5f45163ed14ab8eaf92ed34bb4a631d007f8755a7d79be614bcb0df0ef"); + SystemInfo systemInfo = assertCGroupContainerId(line, "0f99ad5f45163ed14ab8eaf92ed34bb4a631d007f8755a7d79be614bcb0df0ef"); assertKubernetesInfo(systemInfo, "ff49d0be-16b7-4a49-bb9e-8ec1f1f4e27f", "my-host", null, null); } @@ -132,7 +132,7 @@ void testKubernetesInfo_containerd_cri() { void testKubernetesDownwardApi() throws Exception { String line = "1:name=systemd:/kubepods/besteffort/pode9b90526-f47d-11e8-b2a5-080027b9f4fb/15aa6e53-b09a-40c7-8558-c6c31e36c88a"; String containerId = "15aa6e53-b09a-40c7-8558-c6c31e36c88a"; - SystemInfo systemInfo = assertContainerId(line, containerId); + SystemInfo systemInfo = assertCGroupContainerId(line, containerId); String originalPodUid = "e9b90526-f47d-11e8-b2a5-080027b9f4fb"; String hostName = "my-host"; @@ -152,7 +152,7 @@ void testKubernetesDownwardApi() throws Exception { assertKubernetesInfo(systemInfo, podUid, podName, nodeName, namespace); // test partial settings - systemInfo = assertContainerId(line, containerId); + systemInfo = assertCGroupContainerId(line, containerId); assertKubernetesInfo(systemInfo, originalPodUid, hostName, null, null); mockedEnv.put("KUBERNETES_POD_NAME", null); mockedEnv.put("KUBERNETES_POD_UID", null); @@ -160,14 +160,27 @@ void testKubernetesDownwardApi() throws Exception { assertKubernetesInfo(systemInfo, originalPodUid, hostName, nodeName, namespace); } + @Test + void testMountinfo() { + assertMountinfoContainerId("533 525 254:1 /var/lib/docker/containers/c677b374c4747e7297ec4e1792421158dfbaac39820348b6a506739f6947d19d/resolv.conf /etc/resolv.conf rw,relatime - ext4 /dev/vda1 rw", "c677b374c4747e7297ec4e1792421158dfbaac39820348b6a506739f6947d19d"); + assertMountinfoContainerId("534 525 254:1 /docker/containers/c677b374c4747e7297ec4e1792421158dfbaac39820348b6a506739f6947d19d/hostname /etc/hostname rw,relatime - ext4 /dev/vda1 rw", "c677b374c4747e7297ec4e1792421158dfbaac39820348b6a506739f6947d19d"); + } private SystemInfo createSystemInfo() { return new SystemInfo("arch", "my-host", null, "platform"); } - private SystemInfo assertContainerId(String line, String containerId) { + private SystemInfo assertCGroupContainerId(String line, String containerId) { + SystemInfo systemInfo = createSystemInfo(); + assertThat(systemInfo.parseCGroup(line).getContainerInfo()).isNotNull(); + //noinspection ConstantConditions + assertThat(systemInfo.getContainerInfo().getId()).isEqualTo(containerId); + return systemInfo; + } + + private SystemInfo assertMountinfoContainerId(String line, String containerId) { SystemInfo systemInfo = createSystemInfo(); - assertThat(systemInfo.parseContainerId(line).getContainerInfo()).isNotNull(); + assertThat(systemInfo.parseMountinfo(line).getContainerInfo()).isNotNull(); //noinspection ConstantConditions assertThat(systemInfo.getContainerInfo().getId()).isEqualTo(containerId); return systemInfo; @@ -175,7 +188,7 @@ private SystemInfo assertContainerId(String line, String containerId) { private void assertContainerInfoIsNull(String line) { SystemInfo systemInfo = createSystemInfo(); - assertThat(systemInfo.parseContainerId(line).getContainerInfo()).isNull(); + assertThat(systemInfo.parseCGroup(line).getContainerInfo()).isNull(); } private void assertKubernetesInfo(SystemInfo systemInfo, @Nullable String podUid, @Nullable String podName, @Nullable String nodeName,