From 4b9d441cfde2e4bae05298a48b33f92a317156b2 Mon Sep 17 00:00:00 2001 From: Sylvain Juge <763082+SylvainJuge@users.noreply.github.com> Date: Fri, 23 Jun 2023 14:17:36 +0200 Subject: [PATCH 1/6] add spec for cgroups v2 parsing --- specs/agents/metadata.md | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/specs/agents/metadata.md b/specs/agents/metadata.md index d5c05307..da202682 100644 --- a/specs/agents/metadata.md +++ b/specs/agents/metadata.md @@ -112,6 +112,17 @@ If the Kubernetes pod name is not the hostname, it can be overridden by the `KUB *Note:* [cgroup_parsing.json](../../tests/agents/json-specs/cgroup_parsing.json) provides test cases for parsing cgroup lines. +With cgroups v2, the `/proc/self/cgroup does not contain the container ID and we have to parse the `/proc/self/mounts` with the following algorithm: + + 1. filter the line containing `/etc/hostname` to retrieve the file mount that provides the host name to the container. + + 2. split the line on spaces and take the 3rd element containing the host path. + + 3. extract the container ID from file path by using a regular expression matching a 64 character hexadecimal ID. + +*Note:* [mounts_parsing.json](../../tests/agents/json-specs/múnts_parsing.json) provides a test case for parsing mounts lines. + + ### Process metadata Process level metadata relates to the process running the service being monitored: From 68966885c3b2fdbe8d597f15611afe6751d4c933 Mon Sep 17 00:00:00 2001 From: Sylvain Juge <763082+SylvainJuge@users.noreply.github.com> Date: Fri, 23 Jun 2023 14:18:21 +0200 Subject: [PATCH 2/6] add test json --- tests/agents/json-specs/mounts_parsing.json | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 tests/agents/json-specs/mounts_parsing.json diff --git a/tests/agents/json-specs/mounts_parsing.json b/tests/agents/json-specs/mounts_parsing.json new file mode 100644 index 00000000..651b8c97 --- /dev/null +++ b/tests/agents/json-specs/mounts_parsing.json @@ -0,0 +1,21 @@ +{ + "lines": [ + "3984 3905 0:73 / / rw,relatime shared:1863 master:1733 - overlay overlay rw,lowerdir=/var/lib/docker/overlay2/l/KEX7CWLHQCXQY2RHPGTXJ3C26N:/var/lib/docker/overlay2/l/2PVS7JRTRSTVZS4KSUAFML3BIV:/var/lib/docker/overlay2/l/52M7ARM4JDVHCJAYUI6JIKBO4B,upperdir=/var/lib/docker/overlay2/267f825fb89e584605bf161177451879c0ba8b15f7df9b51fb7843c7beb9ed25/diff,workdir=/var/lib/docker/overlay2/267f825fb89e584605bf161177451879c0ba8b15f7df9b51fb7843c7beb9ed25/work", + "3985 3984 0:77 / /proc rw,nosuid,nodev,noexec,relatime shared:1864 - proc proc rw", + "3986 3984 0:78 / /dev rw,nosuid shared:1865 - tmpfs tmpfs rw,size=65536k,mode=755,inode64", + "3987 3986 0:79 / /dev/pts rw,nosuid,noexec,relatime shared:1866 - devpts devpts rw,gid=5,mode=620,ptmxmode=666", + "3988 3984 0:80 / /sys ro,nosuid,nodev,noexec,relatime shared:1870 - sysfs sysfs ro", + "3989 3988 0:30 / /sys/fs/cgroup rw,nosuid,nodev,noexec,relatime shared:1871 - cgroup2 cgroup rw", + "3990 3986 0:76 / /dev/mqueue rw,nosuid,nodev,noexec,relatime shared:1867 - mqueue mqueue rw", + "3991 3986 0:81 / /dev/shm rw,nosuid,nodev,noexec,relatime shared:1868 - tmpfs shm rw,size=65536k,inode64", + "3992 3984 253:1 /var/lib/docker/volumes/9d18ce5b36572d85358fa936afe5a4bf95cca5c822b04941aa08c6118f6e0d33/_data /var rw,relatime shared:1872 master:1 - ext4 /dev/mapper/vgubuntu-root rw,errors=remount-ro", + "3993 3984 0:82 / /run rw,nosuid,nodev,noexec,relatime shared:1873 - tmpfs tmpfs rw,inode64", + "3994 3984 0:83 / /tmp rw,nosuid,nodev,noexec,relatime shared:1874 - tmpfs tmpfs rw,inode64", + "3995 3984 253:1 /usr/lib/modules /usr/lib/modules ro,relatime shared:1875 - ext4 /dev/mapper/vgubuntu-root rw,errors=remount-ro", + "3996 3984 253:1 /var/lib/docker/containers/6548c6863fb748e72d1e2a4f824fde92f720952d062dede1318c2d6219a672d6/resolv.conf /etc/resolv.conf rw,relatime shared:1876 - ext4 /dev/mapper/vgubuntu-root rw,errors=remount-ro", + "3997 3984 253:1 /var/lib/docker/containers/6548c6863fb748e72d1e2a4f824fde92f720952d062dede1318c2d6219a672d6/hostname /etc/hostname rw,relatime shared:1877 - ext4 /dev/mapper/vgubuntu-root rw,errors=remount-ro", + "3998 3984 253:1 /var/lib/docker/containers/6548c6863fb748e72d1e2a4f824fde92f720952d062dede1318c2d6219a672d6/hosts /etc/hosts rw,relatime shared:1878 - ext4 /dev/mapper/vgubuntu-root rw,errors=remount-ro" + ], + "containerId": "6548c6863fb748e72d1e2a4f824fde92f720952d062dede1318c2d6219a672d6" +} + From 4e3dc140be8a35ba1c76c813772dc1dbf273859a Mon Sep 17 00:00:00 2001 From: SylvainJuge <763082+SylvainJuge@users.noreply.github.com> Date: Tue, 27 Jun 2023 16:38:44 +0200 Subject: [PATCH 3/6] Apply suggestions from code review Co-authored-by: Colton Myers --- specs/agents/metadata.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/specs/agents/metadata.md b/specs/agents/metadata.md index da202682..64e96f1d 100644 --- a/specs/agents/metadata.md +++ b/specs/agents/metadata.md @@ -112,7 +112,7 @@ If the Kubernetes pod name is not the hostname, it can be overridden by the `KUB *Note:* [cgroup_parsing.json](../../tests/agents/json-specs/cgroup_parsing.json) provides test cases for parsing cgroup lines. -With cgroups v2, the `/proc/self/cgroup does not contain the container ID and we have to parse the `/proc/self/mounts` with the following algorithm: +With cgroups v2, the `/proc/self/cgroup` does not contain the container ID and we have to parse the `/proc/self/mounts` with the following algorithm: 1. filter the line containing `/etc/hostname` to retrieve the file mount that provides the host name to the container. @@ -120,7 +120,7 @@ With cgroups v2, the `/proc/self/cgroup does not contain the container ID and we 3. extract the container ID from file path by using a regular expression matching a 64 character hexadecimal ID. -*Note:* [mounts_parsing.json](../../tests/agents/json-specs/múnts_parsing.json) provides a test case for parsing mounts lines. +*Note:* [mounts_parsing.json](../../tests/agents/json-specs/mounts_parsing.json) provides a test case for parsing mounts lines. ### Process metadata From ca828bcdb6ed62b58a4476dedc9370d9de96d73d Mon Sep 17 00:00:00 2001 From: Sylvain Juge <763082+SylvainJuge@users.noreply.github.com> Date: Tue, 27 Jun 2023 16:43:52 +0200 Subject: [PATCH 4/6] clarify the cgroup file contents with cgroups v2 --- specs/agents/metadata.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/agents/metadata.md b/specs/agents/metadata.md index 64e96f1d..8cd826bd 100644 --- a/specs/agents/metadata.md +++ b/specs/agents/metadata.md @@ -112,7 +112,7 @@ If the Kubernetes pod name is not the hostname, it can be overridden by the `KUB *Note:* [cgroup_parsing.json](../../tests/agents/json-specs/cgroup_parsing.json) provides test cases for parsing cgroup lines. -With cgroups v2, the `/proc/self/cgroup` does not contain the container ID and we have to parse the `/proc/self/mounts` with the following algorithm: +With cgroups v2, the `/proc/self/cgroup` contains only `0::/` and does not contain the container ID and we have to parse the `/proc/self/mounts` with the following algorithm as a fallback. 1. filter the line containing `/etc/hostname` to retrieve the file mount that provides the host name to the container. From 5a43d10e1c2a50e6855a6468b7047465937e551f Mon Sep 17 00:00:00 2001 From: Sylvain Juge <763082+SylvainJuge@users.noreply.github.com> Date: Wed, 28 Jun 2023 13:19:45 +0200 Subject: [PATCH 5/6] post review change: common test json --- specs/agents/metadata.md | 7 +- .../container_metadata_discovery.json | 65 +++++++++++++++++++ tests/agents/json-specs/mounts_parsing.json | 21 ------ 3 files changed, 67 insertions(+), 26 deletions(-) create mode 100644 tests/agents/json-specs/container_metadata_discovery.json delete mode 100644 tests/agents/json-specs/mounts_parsing.json diff --git a/specs/agents/metadata.md b/specs/agents/metadata.md index 8cd826bd..588c1291 100644 --- a/specs/agents/metadata.md +++ b/specs/agents/metadata.md @@ -110,9 +110,7 @@ On Linux, the container ID and some of the Kubernetes metadata can be extracted If the Kubernetes pod name is not the hostname, it can be overridden by the `KUBERNETES_POD_NAME` environment variable, using the [Downward API](https://kubernetes.io/docs/tasks/inject-data-application/environment-variable-expose-pod-information/). In a similar manner, you can inform the agent of the node name and namespace, using the environment variables `KUBERNETES_NODE_NAME` and `KUBERNETES_NAMESPACE`. -*Note:* [cgroup_parsing.json](../../tests/agents/json-specs/cgroup_parsing.json) provides test cases for parsing cgroup lines. - -With cgroups v2, the `/proc/self/cgroup` contains only `0::/` and does not contain the container ID and we have to parse the `/proc/self/mounts` with the following algorithm as a fallback. +With cgroups v2, the `/proc/self/cgroup` contains only `0::/` and does not contain the container ID and we have to parse the `/proc/self/mountinfo` with the following algorithm as a fallback. 1. filter the line containing `/etc/hostname` to retrieve the file mount that provides the host name to the container. @@ -120,8 +118,7 @@ With cgroups v2, the `/proc/self/cgroup` contains only `0::/` and does not conta 3. extract the container ID from file path by using a regular expression matching a 64 character hexadecimal ID. -*Note:* [mounts_parsing.json](../../tests/agents/json-specs/mounts_parsing.json) provides a test case for parsing mounts lines. - +*Note:* [container_metadata_discovery.json](../../tests/agents/json-specs/container_metadata_discovery.json) provides test cases for parsing `/self/proc/*` files. ### Process metadata diff --git a/tests/agents/json-specs/container_metadata_discovery.json b/tests/agents/json-specs/container_metadata_discovery.json new file mode 100644 index 00000000..510fd19d --- /dev/null +++ b/tests/agents/json-specs/container_metadata_discovery.json @@ -0,0 +1,65 @@ +{ + "cgroup_v1_underscores": { + "files": { + "/proc/self/cgroup": [ + "1:name=systemd:/kubepods.slice/kubepods-burstable.slice/kubepods-burstable-pod90d81341_92de_11e7_8cf2_507b9d4141fa.slice/crio-2227daf62df6694645fee5df53c1f91271546a9560e8600a525690ae252b7f63.scope" + ] + }, + "containerId": "2227daf62df6694645fee5df53c1f91271546a9560e8600a525690ae252b7f63", + "podId": "90d81341-92de-11e7-8cf2-507b9d4141fa" + }, + "cgroup_v1_openshift": { + "files": { + "/proc/self/cgroup": [ + "9:freezer:/kubepods.slice/kubepods-pod22949dce_fd8b_11ea_8ede_98f2b32c645c.slice/docker-b15a5bdedd2e7645c3be271364324321b908314e4c77857bbfd32a041148c07f.scope" + ] + }, + "containerId": "b15a5bdedd2e7645c3be271364324321b908314e4c77857bbfd32a041148c07f", + "podId": "22949dce-fd8b-11ea-8ede-98f2b32c645c" + }, + "cgroup_v1_ubuntu": { + "files": { + "/proc/self/cgroup": [ + "1:name=systemd:/user.slice/user-1000.slice/user@1000.service/apps.slice/apps-org.gnome.Terminal.slice/vte-spawn-75bc72bd-6642-4cf5-b62c-0674e11bfc84.scope" + ] + }, + "containerId": null, + "podId": null + }, + "cgroup_v1_awsEcs": { + "files": { + "/proc/self/cgroup": [ + "1:name=systemd:/ecs/03752a671e744971a862edcee6195646/03752a671e744971a862edcee6195646-4015103728" + ] + }, + "containerId": "03752a671e744971a862edcee6195646-4015103728", + "podId": null + }, + "cgroup_v2": { + "files": { + "/proc/self/cgroup": [ + "0::/" + ], + "/proc/self/mountinfo": [ + "3984 3905 0:73 / / rw,relatime shared:1863 master:1733 - overlay overlay rw,lowerdir=/var/lib/docker/overlay2/l/KEX7CWLHQCXQY2RHPGTXJ3C26N:/var/lib/docker/overlay2/l/2PVS7JRTRSTVZS4KSUAFML3BIV:/var/lib/docker/overlay2/l/52M7ARM4JDVHCJAYUI6JIKBO4B,upperdir=/var/lib/docker/overlay2/267f825fb89e584605bf161177451879c0ba8b15f7df9b51fb7843c7beb9ed25/diff,workdir=/var/lib/docker/overlay2/267f825fb89e584605bf161177451879c0ba8b15f7df9b51fb7843c7beb9ed25/work", + "3985 3984 0:77 / /proc rw,nosuid,nodev,noexec,relatime shared:1864 - proc proc rw", + "3986 3984 0:78 / /dev rw,nosuid shared:1865 - tmpfs tmpfs rw,size=65536k,mode=755,inode64", + "3987 3986 0:79 / /dev/pts rw,nosuid,noexec,relatime shared:1866 - devpts devpts rw,gid=5,mode=620,ptmxmode=666", + "3988 3984 0:80 / /sys ro,nosuid,nodev,noexec,relatime shared:1870 - sysfs sysfs ro", + "3989 3988 0:30 / /sys/fs/cgroup rw,nosuid,nodev,noexec,relatime shared:1871 - cgroup2 cgroup rw", + "3990 3986 0:76 / /dev/mqueue rw,nosuid,nodev,noexec,relatime shared:1867 - mqueue mqueue rw", + "3991 3986 0:81 / /dev/shm rw,nosuid,nodev,noexec,relatime shared:1868 - tmpfs shm rw,size=65536k,inode64", + "3992 3984 253:1 /var/lib/docker/volumes/9d18ce5b36572d85358fa936afe5a4bf95cca5c822b04941aa08c6118f6e0d33/_data /var rw,relatime shared:1872 master:1 - ext4 /dev/mapper/vgubuntu-root rw,errors=remount-ro", + "3993 3984 0:82 / /run rw,nosuid,nodev,noexec,relatime shared:1873 - tmpfs tmpfs rw,inode64", + "3994 3984 0:83 / /tmp rw,nosuid,nodev,noexec,relatime shared:1874 - tmpfs tmpfs rw,inode64", + "3995 3984 253:1 /usr/lib/modules /usr/lib/modules ro,relatime shared:1875 - ext4 /dev/mapper/vgubuntu-root rw,errors=remount-ro", + "3996 3984 253:1 /var/lib/docker/containers/6548c6863fb748e72d1e2a4f824fde92f720952d062dede1318c2d6219a672d6/resolv.conf /etc/resolv.conf rw,relatime shared:1876 - ext4 /dev/mapper/vgubuntu-root rw,errors=remount-ro", + "3997 3984 253:1 /var/lib/docker/containers/6548c6863fb748e72d1e2a4f824fde92f720952d062dede1318c2d6219a672d6/hostname /etc/hostname rw,relatime shared:1877 - ext4 /dev/mapper/vgubuntu-root rw,errors=remount-ro", + "3998 3984 253:1 /var/lib/docker/containers/6548c6863fb748e72d1e2a4f824fde92f720952d062dede1318c2d6219a672d6/hosts /etc/hosts rw,relatime shared:1878 - ext4 /dev/mapper/vgubuntu-root rw,errors=remount-ro" + ] + }, + "containerId": "6548c6863fb748e72d1e2a4f824fde92f720952d062dede1318c2d6219a672d6", + "podId": null + } +} + diff --git a/tests/agents/json-specs/mounts_parsing.json b/tests/agents/json-specs/mounts_parsing.json deleted file mode 100644 index 651b8c97..00000000 --- a/tests/agents/json-specs/mounts_parsing.json +++ /dev/null @@ -1,21 +0,0 @@ -{ - "lines": [ - "3984 3905 0:73 / / rw,relatime shared:1863 master:1733 - overlay overlay rw,lowerdir=/var/lib/docker/overlay2/l/KEX7CWLHQCXQY2RHPGTXJ3C26N:/var/lib/docker/overlay2/l/2PVS7JRTRSTVZS4KSUAFML3BIV:/var/lib/docker/overlay2/l/52M7ARM4JDVHCJAYUI6JIKBO4B,upperdir=/var/lib/docker/overlay2/267f825fb89e584605bf161177451879c0ba8b15f7df9b51fb7843c7beb9ed25/diff,workdir=/var/lib/docker/overlay2/267f825fb89e584605bf161177451879c0ba8b15f7df9b51fb7843c7beb9ed25/work", - "3985 3984 0:77 / /proc rw,nosuid,nodev,noexec,relatime shared:1864 - proc proc rw", - "3986 3984 0:78 / /dev rw,nosuid shared:1865 - tmpfs tmpfs rw,size=65536k,mode=755,inode64", - "3987 3986 0:79 / /dev/pts rw,nosuid,noexec,relatime shared:1866 - devpts devpts rw,gid=5,mode=620,ptmxmode=666", - "3988 3984 0:80 / /sys ro,nosuid,nodev,noexec,relatime shared:1870 - sysfs sysfs ro", - "3989 3988 0:30 / /sys/fs/cgroup rw,nosuid,nodev,noexec,relatime shared:1871 - cgroup2 cgroup rw", - "3990 3986 0:76 / /dev/mqueue rw,nosuid,nodev,noexec,relatime shared:1867 - mqueue mqueue rw", - "3991 3986 0:81 / /dev/shm rw,nosuid,nodev,noexec,relatime shared:1868 - tmpfs shm rw,size=65536k,inode64", - "3992 3984 253:1 /var/lib/docker/volumes/9d18ce5b36572d85358fa936afe5a4bf95cca5c822b04941aa08c6118f6e0d33/_data /var rw,relatime shared:1872 master:1 - ext4 /dev/mapper/vgubuntu-root rw,errors=remount-ro", - "3993 3984 0:82 / /run rw,nosuid,nodev,noexec,relatime shared:1873 - tmpfs tmpfs rw,inode64", - "3994 3984 0:83 / /tmp rw,nosuid,nodev,noexec,relatime shared:1874 - tmpfs tmpfs rw,inode64", - "3995 3984 253:1 /usr/lib/modules /usr/lib/modules ro,relatime shared:1875 - ext4 /dev/mapper/vgubuntu-root rw,errors=remount-ro", - "3996 3984 253:1 /var/lib/docker/containers/6548c6863fb748e72d1e2a4f824fde92f720952d062dede1318c2d6219a672d6/resolv.conf /etc/resolv.conf rw,relatime shared:1876 - ext4 /dev/mapper/vgubuntu-root rw,errors=remount-ro", - "3997 3984 253:1 /var/lib/docker/containers/6548c6863fb748e72d1e2a4f824fde92f720952d062dede1318c2d6219a672d6/hostname /etc/hostname rw,relatime shared:1877 - ext4 /dev/mapper/vgubuntu-root rw,errors=remount-ro", - "3998 3984 253:1 /var/lib/docker/containers/6548c6863fb748e72d1e2a4f824fde92f720952d062dede1318c2d6219a672d6/hosts /etc/hosts rw,relatime shared:1878 - ext4 /dev/mapper/vgubuntu-root rw,errors=remount-ro" - ], - "containerId": "6548c6863fb748e72d1e2a4f824fde92f720952d062dede1318c2d6219a672d6" -} - From 98eef2947f686e5d3dd75ee1b5da6f4eef798f66 Mon Sep 17 00:00:00 2001 From: Sylvain Juge <763082+SylvainJuge@users.noreply.github.com> Date: Wed, 28 Jun 2023 13:32:11 +0200 Subject: [PATCH 6/6] remove cgroup_parsing.json --- tests/agents/json-specs/cgroup_parsing.json | 22 --------------------- 1 file changed, 22 deletions(-) delete mode 100644 tests/agents/json-specs/cgroup_parsing.json diff --git a/tests/agents/json-specs/cgroup_parsing.json b/tests/agents/json-specs/cgroup_parsing.json deleted file mode 100644 index f28d87d4..00000000 --- a/tests/agents/json-specs/cgroup_parsing.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "testUnderscores": { - "groupLine": "1:name=systemd:/kubepods.slice/kubepods-burstable.slice/kubepods-burstable-pod90d81341_92de_11e7_8cf2_507b9d4141fa.slice/crio-2227daf62df6694645fee5df53c1f91271546a9560e8600a525690ae252b7f63.scope", - "containerId": "2227daf62df6694645fee5df53c1f91271546a9560e8600a525690ae252b7f63", - "podId": "90d81341-92de-11e7-8cf2-507b9d4141fa" - }, - "testOpenshiftForm": { - "groupLine": "9:freezer:/kubepods.slice/kubepods-pod22949dce_fd8b_11ea_8ede_98f2b32c645c.slice/docker-b15a5bdedd2e7645c3be271364324321b908314e4c77857bbfd32a041148c07f.scope", - "containerId": "b15a5bdedd2e7645c3be271364324321b908314e4c77857bbfd32a041148c07f", - "podId": "22949dce-fd8b-11ea-8ede-98f2b32c645c" - }, - "testUbuntuCGroup": { - "groupLine": "1:name=systemd:/user.slice/user-1000.slice/user@1000.service/apps.slice/apps-org.gnome.Terminal.slice/vte-spawn-75bc72bd-6642-4cf5-b62c-0674e11bfc84.scope", - "containerId": null, - "podId": null - }, - "testAwsEcsCGroup": { - "groupLine": "1:name=systemd:/ecs/03752a671e744971a862edcee6195646/03752a671e744971a862edcee6195646-4015103728", - "containerId": "03752a671e744971a862edcee6195646-4015103728", - "podId": null - } -}