chore: Follow-ups to the containerdebug log collection (#892)

* chore: Follow-ups to the containerdebug log collection

* chore: Create the containerdebug log directory via the environment variable

* fix: Do not hard-code the environment variable name "FILE_LOG_DIRECTORY"

Author: siegfriedweber

CHANGELOG.md

@@ -13,7 +13,7 @@
 - Add `/ready` endpoint to the operator Deployment, which reports the CRD installation status ([#868]).
 - Webservers now have a default affinity to the OPA Pods when OPA authorization is configured ([#872]).
 - The logs of the `containerdebug` process are now collected by the Vector agent and forwarded to
-  the Vector aggregator ([#888]).
+  the Vector aggregator ([#888], [#892]).
 
 ### Changed
 
@@ -112,6 +112,7 @@
 [#886]: https://github.com/stackabletech/airflow-operator/pull/886
 [#887]: https://github.com/stackabletech/airflow-operator/pull/887
 [#888]: https://github.com/stackabletech/airflow-operator/pull/888
+[#892]: https://github.com/stackabletech/airflow-operator/pull/892
 
 ## [26.7.0] - 2026-07-21
 

rust/operator-binary/src/controller/build/properties/env_vars.rs

@@ -1,5 +1,6 @@
 use std::{collections::BTreeSet, str::FromStr};
 
+use const_format::concatcp;
 use stackable_operator::{
     constant,
     crd::{authentication::oidc, git_sync},
@@ -15,7 +16,8 @@ use stackable_operator::{
 use crate::{
     controller::{ValidatedCluster, ValidatedLogging},
     crd::{
-        AirflowExecutor, AirflowRole, HTTP_PORT, LOG_CONFIG_DIR, TEMPLATE_LOCATION, TEMPLATE_NAME,
+        AirflowExecutor, AirflowRole, FILE_LOG_DIRECTORY, FILE_LOG_MAX_FILES,
+        FILE_LOG_ROTATION_PERIOD, HTTP_PORT, LOG_CONFIG_DIR, TEMPLATE_LOCATION, TEMPLATE_NAME,
         authentication::{
             AirflowAuthenticationClassResolved, AirflowClientAuthenticationDetailsResolved,
         },
@@ -77,9 +79,6 @@ constant!(ADMIN_PASSWORD_SECRET_KEY: SecretKey = "adminUser.password");
 
 constant!(PYTHONPATH: EnvVarName = "PYTHONPATH");
 
-constant!(FILE_LOG_DIRECTORY: EnvVarName = "FILE_LOG_DIRECTORY");
-constant!(FILE_LOG_ROTATION_PERIOD: EnvVarName = "FILE_LOG_ROTATION_PERIOD");
-constant!(FILE_LOG_MAX_FILES: EnvVarName = "FILE_LOG_MAX_FILES");
 constant!(STACKABLE_POST_HOOK: EnvVarName = "_STACKABLE_POST_HOOK");
 
 /// Return environment variables to be applied to the statefulsets for the scheduler, webserver (and worker,
@@ -188,9 +187,11 @@ pub fn build_airflow_statefulset_envs(
     // other telemetry-based process in it would log into the same directory and the Vector
     // agent would label its events with `container: containerdebug`.
     env_vars = env_vars
+        // The directory must be exactly one level below the log directory, because the Vector
+        // agent derives the container name from that path segment.
         .with_value(
             &FILE_LOG_DIRECTORY,
-            format!("{STACKABLE_LOG_DIR}/containerdebug"),
+            concatcp!(STACKABLE_LOG_DIR, "/containerdebug"),
         )
         .with_value(&FILE_LOG_ROTATION_PERIOD, "hourly")
         .with_value(&FILE_LOG_MAX_FILES, "2")
@@ -536,9 +537,6 @@ mod tests {
         let _ = *AIRFLOW_WEBSERVER_ENABLE_PROXY_FIX;
         let _ = *AIRFLOW_WEBSERVER_EXPOSE_CONFIG;
         let _ = *AIRFLOW_WEBSERVER_PROXY_FIX_X_FOR;
-        let _ = *FILE_LOG_DIRECTORY;
-        let _ = *FILE_LOG_MAX_FILES;
-        let _ = *FILE_LOG_ROTATION_PERIOD;
         let _ = *FORWARDED_ALLOW_IPS;
         let _ = *PYTHONPATH;
         let _ = *STACKABLE_POST_HOOK;

rust/operator-binary/src/controller/build/volumes.rs

@@ -32,11 +32,9 @@ pub fn create_volumes(
         name: LOG_VOLUME_NAME.to_string(),
         empty_dir: Some(EmptyDirVolumeSource {
             medium: None,
-            // Only the Airflow log files are budgeted here. The `containerdebug` files in this
-            // volume are rotated by time, which bounds their number but not their size, so they
-            // are an unbudgeted writer. They stay negligible at the default INFO level and a 30
-            // minute loop, but raising `FILE_LOG_LEVEL` via `envOverrides` can exceed the limit,
-            // which makes the kubelet evict the Pod.
+            // Only the Airflow log files are budgeted. The `containerdebug` files in this
+            // volume are rotated by time, not by size, so they have no budget; raising
+            // `FILE_LOG_LEVEL` via `envOverrides` can therefore evict the Pod.
             size_limit: Some(product_logging::framework::calculate_log_volume_size_limit(
                 &[MAX_LOG_FILES_SIZE],
             )),

rust/operator-binary/src/crd/mod.rs

@@ -1,5 +1,6 @@
 use std::{collections::BTreeSet, ops::Deref, str::FromStr};
 
+use indoc::formatdoc;
 use serde::{Deserialize, Serialize};
 use snafu::{ResultExt, Snafu};
 use stackable_operator::{
@@ -38,6 +39,7 @@ use stackable_operator::{
     status::condition::{ClusterCondition, HasStatusCondition},
     utils::{COMMON_BASH_TRAP_FUNCTIONS, crds::raw_object_list_schema},
     v2::{
+        builder::pod::container::EnvVarName,
         config_overrides::KeyValueConfigOverrides,
         flask_config_writer::{FlaskAppConfigOptions, PythonType},
         product_logging::framework::STACKABLE_LOG_DIR,
@@ -84,6 +86,14 @@ pub const CONFIG_PATH: &str = "/stackable/app/config";
 pub const LOG_CONFIG_DIR: &str = "/stackable/app/log_config";
 pub const AIRFLOW_HOME: &str = "/stackable/airflow";
 
+// The `stackable-telemetry` variables which configure the file logs of the `containerdebug`
+// process. They are set on the Airflow container in `env_vars`, and the start command below
+// references the directory as a shell variable, so that an `envOverrides` entry for it takes
+// effect in both places.
+constant!(pub FILE_LOG_DIRECTORY: EnvVarName = "FILE_LOG_DIRECTORY");
+constant!(pub FILE_LOG_MAX_FILES: EnvVarName = "FILE_LOG_MAX_FILES");
+constant!(pub FILE_LOG_ROTATION_PERIOD: EnvVarName = "FILE_LOG_ROTATION_PERIOD");
+
 constant!(pub TEMPLATE_VOLUME_NAME: VolumeName = "airflow-executor-pod-template");
 pub const TEMPLATE_LOCATION: &str = "/templates";
 pub const TEMPLATE_NAME: &str = "airflow_executor_pod_template.yaml";
@@ -776,7 +786,15 @@ impl Deref for AirflowRole {
 }
 
 fn container_debug_command() -> String {
-    format!("containerdebug --output={STACKABLE_LOG_DIR}/containerdebug-state.json --loop &")
+    // The log directory is created here, because `FILE_LOG_MAX_FILES` makes the telemetry library
+    // prune old log files before it creates the first one, which complains about the directory not
+    // existing yet.
+    formatdoc!(
+        r#"
+        mkdir --parents "${file_log_directory}" || true
+        containerdebug --output={STACKABLE_LOG_DIR}/containerdebug-state.json --loop &"#,
+        file_log_directory = FILE_LOG_DIRECTORY.as_ref(),
+    )
 }
 
 #[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)]
@@ -1029,6 +1047,9 @@ mod tests {
         let _ = *WORKER_ROLE_NAME;
         let _ = *DAG_PROCESSOR_ROLE_NAME;
         let _ = *TRIGGERER_ROLE_NAME;
+        let _ = *FILE_LOG_DIRECTORY;
+        let _ = *FILE_LOG_MAX_FILES;
+        let _ = *FILE_LOG_ROTATION_PERIOD;
         let _ = *TEMPLATE_VOLUME_NAME;
         let _ = *LISTENER_PVC_NAME;
         let _ = *METRICS_CONTAINER_NAME;