feat: Collect the containerdebug logs (#832)

* feat: Collect the containerdebug logs

* chore: Update changelog

* chore: Improve code style

* chore: Upgrade serde-saphyr

* chore: Inline CONTAINERDEBUG_LOG_DIR

* test(smoke): Fix test assertion

* chore: Revert downgrade of base64

Author: siegfriedweber

CHANGELOG.md

@@ -9,6 +9,8 @@
 - Add `/ready` endpoint to the operator Deployment, which reports the CRD installation status ([#812]).
 - Add support for 2.6.7 and 3.0.0 (experimental)  ([#815]).
 - Masters and region servers now have a default affinity to the OPA Pods when OPA authorization is configured ([#816]).
+- The logs of the `containerdebug` process are now collected by the Vector agent and forwarded to
+  the Vector aggregator ([#832]).
 
 ### Changed
 
@@ -56,6 +58,10 @@
 - BREAKING: The rest-server listener PVC template now carries the recommended labels without the
   version label, so that the labels stay stable across upgrades.
   Existing rest-server StatefulSets must be deleted once before the new operator can reconcile them ([#799]).
+- The `containerdebug` process writes its log file again. The operator set
+  `CONTAINERDEBUG_LOG_DIRECTORY`, which containerdebug stopped reading in its version 0.2.0 when it
+  moved to `stackable-telemetry`; the directory is now set via `FILE_LOG_DIRECTORY`. The file is
+  rotated hourly and at most two files are kept, because rotation is opt-in ([#832]).
 
 [#776]: https://github.com/stackabletech/hbase-operator/pull/776
 [#782]: https://github.com/stackabletech/hbase-operator/pull/782
@@ -76,6 +82,7 @@
 [#825]: https://github.com/stackabletech/hbase-operator/pull/825
 [#827]: https://github.com/stackabletech/hbase-operator/pull/827
 [#829]: https://github.com/stackabletech/hbase-operator/pull/829
+[#832]: https://github.com/stackabletech/hbase-operator/pull/832
 
 ## [26.7.0] - 2026-07-21
 

Cargo.lock

@@ -22,4 +22,12 @@ spec:
         enableVectorAgent: true
 ----
 
+NOTE: The logs of the xref:concepts:observability/containerdebug.adoc[`containerdebug` process] are collected as well.
+They are labelled `container: containerdebug`, although no such Kubernetes container exists.
+The process runs in the background of the `hbase` container.
+The label comes from `/stackable/log/containerdebug`, the directory which the operator sets with the `FILE_LOG_DIRECTORY` environment variable and from which the Vector agent derives the container name.
+Overriding `FILE_LOG_DIRECTORY` with `envOverrides` therefore changes how these logs are aggregated.
+The agent only collects files one level below `/stackable/log`, so a sibling directory such as `/stackable/log/hbase` relabels the events instead of dropping them, while any deeper or shallower path stops the aggregation.
+A path outside `/stackable/log` also leaves the size-capped log volume.
+
 Further information on how to configure logging, can be found in xref:concepts:logging.adoc[].

Cargo.nix

@@ -31,4 +31,5 @@ built.workspace = true
 
 [dev-dependencies]
 rstest.workspace = true
+stackable-operator = { workspace = true, features = ["test-support"] }
 yaml_serde.workspace = true

docs/modules/hbase/pages/usage-guide/logging.adoc

@@ -7,6 +7,7 @@
 
 use std::str::FromStr;
 
+use const_format::concatcp;
 use indoc::formatdoc;
 use snafu::{ResultExt, Snafu};
 use stackable_operator::{
@@ -56,7 +57,9 @@ constant!(HADOOP_CONF_DIR_ENV: EnvVarName = "HADOOP_CONF_DIR");
 constant!(pub(crate) REGION_MOVER_OPTS_ENV: EnvVarName = "REGION_MOVER_OPTS");
 constant!(pub(crate) RUN_REGION_MOVER_ENV: EnvVarName = "RUN_REGION_MOVER");
 constant!(STACKABLE_LOG_DIR_ENV: EnvVarName = "STACKABLE_LOG_DIR");
-constant!(CONTAINERDEBUG_LOG_DIRECTORY_ENV: EnvVarName = "CONTAINERDEBUG_LOG_DIRECTORY");
+constant!(FILE_LOG_DIRECTORY_ENV: EnvVarName = "FILE_LOG_DIRECTORY");
+constant!(FILE_LOG_MAX_FILES_ENV: EnvVarName = "FILE_LOG_MAX_FILES");
+constant!(FILE_LOG_ROTATION_PERIOD_ENV: EnvVarName = "FILE_LOG_ROTATION_PERIOD");
 
 // These constants are hard coded in hbase-entrypoint.sh
 // You need to change them there too.
@@ -145,11 +148,18 @@ pub(crate) fn containers_and_volumes(inputs: &RoleGroupInputs) -> Result<Contain
         .with_value(&REGION_MOVER_OPTS_ENV, inputs.region_mover_opts.clone())
         .with_value(&RUN_REGION_MOVER_ENV, inputs.run_region_mover.to_string())
         .with_value(&STACKABLE_LOG_DIR_ENV, STACKABLE_LOG_DIR)
-        // Needed for the `containerdebug` process to log its tracing information to.
+        // Let the `containerdebug` process write its tracing information to a file. These are the
+        // generic `stackable-telemetry` variables and apply to the whole HBase container, so any
+        // other telemetry-based process in it would log into the same directory. It must be
+        // exactly one level below the log directory, because the Vector agent derives the
+        // container name from that path segment.
         .with_value(
-            &CONTAINERDEBUG_LOG_DIRECTORY_ENV,
-            format!("{STACKABLE_LOG_DIR}/containerdebug"),
+            &FILE_LOG_DIRECTORY_ENV,
+            concatcp!(STACKABLE_LOG_DIR, "/containerdebug"),
         )
+        // Rotation is opt-in, so without these the file would grow unbounded.
+        .with_value(&FILE_LOG_ROTATION_PERIOD_ENV, "hourly")
+        .with_value(&FILE_LOG_MAX_FILES_ENV, "2")
         // Empty when Kerberos is disabled.
         .merge(kerberos::kerberos_env_vars(cluster))
         // apply overrides last of all; `EnvVarSet` is keyed by name, so iteration is already
@@ -217,6 +227,9 @@ pub(crate) fn containers_and_volumes(inputs: &RoleGroupInputs) -> Result<Contain
         VolumeBuilder::new(&*LOG_VOLUME_NAME)
             .with_empty_dir(
                 None::<String>,
+                // Only the HBase log files are budgeted. The `containerdebug` files in this
+                // volume are rotated by time, not by size, so they have no budget; raising
+                // `FILE_LOG_LEVEL` via `envOverrides` can therefore evict the Pod.
                 Some(product_logging::framework::calculate_log_volume_size_limit(
                     &[MAX_HBASE_LOG_FILES_SIZE],
                 )),
@@ -303,6 +316,8 @@ mod tests {
         let _ = *REGION_MOVER_OPTS_ENV;
         let _ = *RUN_REGION_MOVER_ENV;
         let _ = *STACKABLE_LOG_DIR_ENV;
-        let _ = *CONTAINERDEBUG_LOG_DIRECTORY_ENV;
+        let _ = *FILE_LOG_DIRECTORY_ENV;
+        let _ = *FILE_LOG_MAX_FILES_ENV;
+        let _ = *FILE_LOG_ROTATION_PERIOD_ENV;
     }
 }

rust/operator-binary/Cargo.toml

@@ -112,7 +112,7 @@ mod tests {
             false,
             "-Xtest".to_string(),
             BTreeMap::from([(
-                EnvVarName::from_str("CUSTOM_VAR").expect("valid variable name"),
+                EnvVarName::from_str_unsafe("CUSTOM_VAR"),
                 "custom_value".to_string(),
             )]),
         )

rust/operator-binary/src/controller/build/container.rs

@@ -61,7 +61,9 @@ mod tests {
         assert!(!content.is_empty());
         // A kept source must be present ...
         assert!(content.contains("files_log4j2"));
+        assert!(content.contains("files_tracing_rs"));
+        assert!(content.contains("*.tracing-rs.json"));
         // ... while a product-specific source we don't emit must not.
-        assert!(!content.contains("files_tracing_rs"));
+        assert!(!content.contains("files_py"));
     }
 }