feat: Collect the containerdebug logs (#1106)

* feat: Collect the containerdebug logs

* chore: Apply findings from other operators

* chore: Upgrade serde-saphyr

* chore: Revert downgrade of base64

Author: siegfriedweber

CHANGELOG.md

@@ -10,6 +10,8 @@ All notable changes to this project will be documented in this file.
   ([#1086]).
 - BREAKING: Enable auto-purge feature for data directory cleanup ([#1068]).
 - Add `/ready` endpoint to the operator Deployment, which reports the CRD installation status ([#1090]).
+- The logs of the `containerdebug` process are now collected by the Vector agent and forwarded to
+  the Vector aggregator ([#1106]).
 
 ### Changed
 
@@ -49,6 +51,10 @@ All notable changes to this project will be documented in this file.
   See [our internal issue](https://github.com/stackabletech/hdfs-operator/issues/626) and [the fix](https://github.com/kube-rs/kube/pull/2042) for details ([#1070]).
 - The operator now watches all resources that it creates and early-exits the reconcile action when the
   cluster is marked for deletion ([#1079]).
+- The `containerdebug` process writes its log file again. The operator set
+  `CONTAINERDEBUG_LOG_DIRECTORY`, which containerdebug stopped reading in its version 0.2.0 when it
+  moved to `stackable-telemetry`; the directory is now set via `FILE_LOG_DIRECTORY`. The file is
+  rotated hourly and at most two files are kept, because rotation is opt-in ([#1106]).
 
 [#1053]: https://github.com/stackabletech/zookeeper-operator/pull/1053
 [#1060]: https://github.com/stackabletech/zookeeper-operator/pull/1060
@@ -68,6 +74,7 @@ All notable changes to this project will be documented in this file.
 [#1100]: https://github.com/stackabletech/zookeeper-operator/pull/1100
 [#1101]: https://github.com/stackabletech/zookeeper-operator/pull/1101
 [#1102]: https://github.com/stackabletech/zookeeper-operator/pull/1102
+[#1106]: https://github.com/stackabletech/zookeeper-operator/pull/1106
 
 ## [26.7.0] - 2026-07-21
 

Cargo.lock

@@ -26,4 +26,12 @@ spec:
         replicas: 1
 ----
 
+NOTE: The logs of the xref:concepts:observability/containerdebug.adoc[`containerdebug` process] are collected as well.
+They are labelled `container: containerdebug`, although no such Kubernetes container exists.
+The process runs in the background of the `zookeeper` container.
+The label comes from `/stackable/log/containerdebug`, the directory which the operator sets with the `FILE_LOG_DIRECTORY` environment variable and from which the Vector agent derives the container name.
+Overriding `FILE_LOG_DIRECTORY` with `envOverrides` therefore changes how these logs are aggregated.
+The agent only collects files one level below `/stackable/log`, so a sibling directory such as `/stackable/log/zookeeper` relabels the events instead of dropping them, while any deeper or shallower path stops the aggregation.
+A path outside `/stackable/log` also leaves the size-capped log volume.
+
 Further information on how to configure logging, can be found in xref:concepts:logging.adoc[].

Cargo.nix

@@ -80,5 +80,7 @@ mod tests {
         // ZooKeeper's logback writes log4j-format XML to `*.log4j.xml`, so the `files_log4j`
         // source matches it.
         assert!(content.contains("files_log4j"));
+        assert!(content.contains("files_tracing_rs"));
+        assert!(content.contains("*.tracing-rs.json"));
     }
 }

docs/modules/zookeeper/pages/usage_guide/log_aggregation.adoc

@@ -137,6 +137,285 @@ tests:
                 "timestamp": t'2025-10-02T09:27:28.582Z'
               }
 
+              assert_eq!(expected_log_event, .)
+  - name: Test tracing-rs log
+    inputs:
+      - type: log
+        insert_at: processed_files_tracing_rs
+        log_fields:
+          file: /stackable/log/containerdebug/containerdebug.2026-11-02-14.tracing-rs.json
+          message: '{"timestamp":"2026-11-02T14:49:28.028922Z","level":"INFO","fields":{"message":"found disk","disk.name":"overlay"},"target":"containerdebug::system_information::disk"}'
+          pod: test-zk-server-default-0
+          source_type: file
+          timestamp: 2026-11-02T14:49:29.473487331Z
+    outputs:
+      - extract_from: extended_logs
+        expected_event_count: 1
+        conditions:
+          - type: vrl
+            source: |
+              expected_log_event = {
+                "cluster": "test-zk",
+                "container": "containerdebug",
+                "file": "containerdebug.2026-11-02-14.tracing-rs.json",
+                "level": "INFO",
+                "logger": "containerdebug::system_information::disk",
+                "message": "found disk\n\ndisk.name=overlay",
+                "timestamp": t'2026-11-02T14:49:28.028922Z',
+                "namespace": "default",
+                "pod": "test-zk-server-default-0",
+                "role": "server",
+                "roleGroup": "default"
+              }
+
+              assert_eq!(expected_log_event, .)
+  - name: Test unparsable tracing-rs log
+    inputs:
+      - type: log
+        insert_at: processed_files_tracing_rs
+        log_fields:
+          file: /stackable/log/containerdebug/containerdebug.2026-11-02-14.tracing-rs.json
+          message: '{ unparsable'
+          pod: test-zk-server-default-0
+          source_type: file
+          timestamp: 2026-11-02T14:49:29.473487331Z
+    outputs:
+      - extract_from: extended_logs
+        expected_event_count: 1
+        conditions:
+          - type: vrl
+            source: |
+              expected_log_event = {
+                "cluster": "test-zk",
+                "container": "containerdebug",
+                "errors": [
+                  .errors[0]
+                ],
+                "file": "containerdebug.2026-11-02-14.tracing-rs.json",
+                "level": "INFO",
+                "logger": "",
+                "message": "{ unparsable",
+                "timestamp": .timestamp,
+                "namespace": "default",
+                "pod": "test-zk-server-default-0",
+                "role": "server",
+                "roleGroup": "default"
+              }
+
+              assert_eq!(expected_log_event, .)
+
+              # The remainder of the message contains character offsets into
+              # the VRL source, so only the stable prefix is asserted.
+              assert!(starts_with!(.errors[0], "JSON not parsable: "))
+  - name: Test tracing-rs log which is not a JSON object
+    inputs:
+      - type: log
+        insert_at: processed_files_tracing_rs
+        log_fields:
+          file: /stackable/log/containerdebug/containerdebug.2026-11-02-14.tracing-rs.json
+          message: 'false'
+          pod: test-zk-server-default-0
+          source_type: file
+          timestamp: 2026-11-02T14:49:29.473487331Z
+    outputs:
+      - extract_from: extended_logs
+        expected_event_count: 1
+        conditions:
+          - type: vrl
+            source: |
+              expected_log_event = {
+                "cluster": "test-zk",
+                "container": "containerdebug",
+                "errors": [
+                  "Parsed event is not a JSON object."
+                ],
+                "file": "containerdebug.2026-11-02-14.tracing-rs.json",
+                "level": "INFO",
+                "logger": "",
+                "message": "false",
+                "timestamp": .timestamp,
+                "namespace": "default",
+                "pod": "test-zk-server-default-0",
+                "role": "server",
+                "roleGroup": "default"
+              }
+
+              assert_eq!(expected_log_event, .)
+  - name: Test tracing-rs log with invalid fields
+    inputs:
+      - type: log
+        insert_at: processed_files_tracing_rs
+        log_fields:
+          file: /stackable/log/containerdebug/containerdebug.2026-11-02-14.tracing-rs.json
+          message: '{"timestamp":"unparsable","level":"INVALID","fields":"none","target":null}'
+          pod: test-zk-server-default-0
+          source_type: file
+          timestamp: 2026-11-02T14:49:29.473487331Z
+    outputs:
+      - extract_from: extended_logs
+        expected_event_count: 1
+        conditions:
+          - type: vrl
+            source: |
+              expected_log_event = {
+                "cluster": "test-zk",
+                "container": "containerdebug",
+                "errors": [
+                  .errors[0],
+                  "Logger/target not found.",
+                  "Level \"INVALID\" unknown, using \"INFO\" instead.",
+                  "Fields are not an object."
+                ],
+                "file": "containerdebug.2026-11-02-14.tracing-rs.json",
+                "level": "INFO",
+                "logger": "",
+                "message": "{\"timestamp\":\"unparsable\",\"level\":\"INVALID\",\"fields\":\"none\",\"target\":null}",
+                "timestamp": .timestamp,
+                "namespace": "default",
+                "pod": "test-zk-server-default-0",
+                "role": "server",
+                "roleGroup": "default"
+              }
+
+              assert_eq!(expected_log_event, .)
+
+              # The remainder of the message contains character offsets into
+              # the VRL source, so only the stable prefix is asserted.
+              assert!(starts_with!(.errors[0], "Timestamp not parsable, using current time instead: "))
+  - name: Test tracing-rs log without a timestamp
+    inputs:
+      - type: log
+        insert_at: processed_files_tracing_rs
+        log_fields:
+          file: /stackable/log/containerdebug/containerdebug.2026-11-02-14.tracing-rs.json
+          message: '{"level":"INFO","fields":{"message":"found disk"},"target":"containerdebug::system_information::disk"}'
+          pod: test-zk-server-default-0
+          source_type: file
+          timestamp: 2026-11-02T14:49:29.473487331Z
+    outputs:
+      - extract_from: extended_logs
+        expected_event_count: 1
+        conditions:
+          - type: vrl
+            source: |
+              expected_log_event = {
+                "cluster": "test-zk",
+                "container": "containerdebug",
+                "errors": [
+                  "Timestamp not found, using current time instead."
+                ],
+                "file": "containerdebug.2026-11-02-14.tracing-rs.json",
+                "level": "INFO",
+                "logger": "containerdebug::system_information::disk",
+                "message": "found disk",
+                "timestamp": .timestamp,
+                "namespace": "default",
+                "pod": "test-zk-server-default-0",
+                "role": "server",
+                "roleGroup": "default"
+              }
+
+              assert_eq!(expected_log_event, .)
+
+              # The current time is used and not the timestamp which the file
+              # source attached to the event. The latter reaches the transform
+              # as a string, whereas now() yields a timestamp.
+              assert!(is_timestamp(.timestamp))
+  - name: Test tracing-rs log without a message but with other fields
+    inputs:
+      - type: log
+        insert_at: processed_files_tracing_rs
+        log_fields:
+          file: /stackable/log/containerdebug/containerdebug.2026-11-02-14.tracing-rs.json
+          message: '{"timestamp":"2026-11-02T14:49:28.028922Z","level":"INFO","fields":{"disk.name":"overlay"},"target":"containerdebug::system_information::disk"}'
+          pod: test-zk-server-default-0
+          source_type: file
+          timestamp: 2026-11-02T14:49:29.473487331Z
+    outputs:
+      - extract_from: extended_logs
+        expected_event_count: 1
+        conditions:
+          - type: vrl
+            source: |
+              expected_log_event = {
+                "cluster": "test-zk",
+                "container": "containerdebug",
+                "file": "containerdebug.2026-11-02-14.tracing-rs.json",
+                "level": "INFO",
+                "logger": "containerdebug::system_information::disk",
+                "message": "disk.name=overlay",
+                "timestamp": t'2026-11-02T14:49:28.028922Z',
+                "namespace": "default",
+                "pod": "test-zk-server-default-0",
+                "role": "server",
+                "roleGroup": "default"
+              }
+
+              assert_eq!(expected_log_event, .)
+  - name: Test tracing-rs log with a message which is not a string
+    inputs:
+      - type: log
+        insert_at: processed_files_tracing_rs
+        log_fields:
+          file: /stackable/log/containerdebug/containerdebug.2026-11-02-14.tracing-rs.json
+          message: '{"timestamp":"2026-11-02T14:49:28.028922Z","level":"INFO","fields":{"message":42,"disk.name":"overlay"},"target":"containerdebug::system_information::disk"}'
+          pod: test-zk-server-default-0
+          source_type: file
+          timestamp: 2026-11-02T14:49:29.473487331Z
+    outputs:
+      - extract_from: extended_logs
+        expected_event_count: 1
+        conditions:
+          - type: vrl
+            source: |
+              expected_log_event = {
+                "cluster": "test-zk",
+                "container": "containerdebug",
+                "file": "containerdebug.2026-11-02-14.tracing-rs.json",
+                "level": "INFO",
+                "logger": "containerdebug::system_information::disk",
+                "message": "disk.name=overlay\nmessage=42",
+                "timestamp": t'2026-11-02T14:49:28.028922Z',
+                "namespace": "default",
+                "pod": "test-zk-server-default-0",
+                "role": "server",
+                "roleGroup": "default"
+              }
+
+              assert_eq!(expected_log_event, .)
+  - name: Test tracing-rs log without any fields
+    inputs:
+      - type: log
+        insert_at: processed_files_tracing_rs
+        log_fields:
+          file: /stackable/log/containerdebug/containerdebug.2026-11-02-14.tracing-rs.json
+          message: '{"timestamp":"2026-11-02T14:49:28.028922Z","level":"INFO","fields":{},"target":"containerdebug::system_information::disk"}'
+          pod: test-zk-server-default-0
+          source_type: file
+          timestamp: 2026-11-02T14:49:29.473487331Z
+    outputs:
+      - extract_from: extended_logs
+        expected_event_count: 1
+        conditions:
+          - type: vrl
+            source: |
+              expected_log_event = {
+                "cluster": "test-zk",
+                "container": "containerdebug",
+                "errors": [
+                  "Message not found."
+                ],
+                "file": "containerdebug.2026-11-02-14.tracing-rs.json",
+                "level": "INFO",
+                "logger": "containerdebug::system_information::disk",
+                "message": "",
+                "timestamp": t'2026-11-02T14:49:28.028922Z',
+                "namespace": "default",
+                "pod": "test-zk-server-default-0",
+                "role": "server",
+                "roleGroup": "default"
+              }
+
               assert_eq!(expected_log_event, .)
   - name: Test Vector internal logs
     inputs: