CNS-136: expose statement logging sample rate in the operator chart

misc/helm-charts/operator/README.md

@@ -145,6 +145,8 @@ The following table lists the configurable parameters of the Materialize operato
 | `operator.args.enableLicenseKeyChecks` |  | ``false`` |
 | `operator.args.installV1CRD` | Whether to install the v1 version of the Materialize CRD and the conversion webhook that converts between v1 and v1alpha1. When false, only the v1alpha1 CRD version is installed and no webhook serving certificate or service is created. | ``false`` |
 | `operator.args.startupLogFilter` | Log filtering settings for startup logs | ``"INFO,mz_orchestratord=TRACE"`` |
+| `operator.args.statementLoggingMaxSampleRate` | Caps the fraction of statements recorded in query history, via environmentd's `statement_logging_max_sample_rate`. Sampling costs CPU on environmentd, and the recorded history is retained for the lifetime of the environment, so this samples a fraction of statements rather than environmentd's own default of 0.99. This bounds the sampled fraction, not the total size of the history. Sustained write volume is capped separately by `statement_logging_target_data_rate`, which is the binding limit on busy environments. Raise this for more complete query history, set it to `0` to disable statement logging entirely, or set it to `null` to inherit environmentd's default. | ``0.1`` |
+| `operator.args.statementLoggingTargetDataRate` | Caps the sustained volume statement logging writes, in bytes per second, via environmentd's `statement_logging_target_data_rate`. This is the binding limit on busy environments, where it throttles sampled statements that would exceed it. Lower it to hold query history growth down on environments with limited storage, keeping in mind that the recorded history is retained for the lifetime of the environment. Set it to `null` to inherit environmentd's default of 2071 bytes per second. | ``nil`` |
 | `operator.args.webhookCertReloadInterval` | How often orchestratord reloads its webhook TLS certificate from disk and, when the CA changes, refreshes the conversion webhook's CA bundle. Must be shorter than the certificate's lifetime. Accepts a humantime duration (e.g. "1h", "30m"). Leave null to use the binary default. Only used if `installV1CRD` is true. | ``nil`` |
 | `operator.certificate.caDuration` | Lifetime of the root CA that signs the webhook serving certificate, when `source` is "cert-manager". The serving certificate is signed by this CA, so the CA outlives individual serving-certificate rotations. | ``"87600h"`` |
 | `operator.certificate.caRenewBefore` | How long before the root CA expires to renew it. Must be less than `caDuration`. | ``"8760h"`` |

misc/helm-charts/operator/templates/deployment.yaml

@@ -76,9 +76,12 @@ spec:
         - "--cloud-provider={{ .Values.operator.cloudProvider.type }}"
         - "--region={{ .Values.operator.cloudProvider.region }}"
         - "--secrets-controller={{ .Values.operator.secretsController }}"
-        # (SangJunBak) For self-managed, we disable statement logging due to performance issues
-        # https://github.com/MaterializeInc/cloud/issues/10755
-        - "--disable-statement-logging"
+        {{- if not (kindIs "invalid" .Values.operator.args.statementLoggingMaxSampleRate) }}
+        - "--statement-logging-max-sample-rate={{ .Values.operator.args.statementLoggingMaxSampleRate }}"
+        {{- end }}
+        {{- if not (kindIs "invalid" .Values.operator.args.statementLoggingTargetDataRate) }}
+        - "--statement-logging-target-data-rate={{ int64 .Values.operator.args.statementLoggingTargetDataRate }}"
+        {{- end }}
         {{- range $key, $value := include "materialize-operator.selectorLabels" . | fromYaml }}
         - "--orchestratord-pod-selector-labels={{ $key }}={{ $value }}"
         {{- end }}

misc/helm-charts/operator/tests/deployment_test.yaml

@@ -47,6 +47,61 @@ tests:
       path: spec.template.spec.containers[0].args
       content: "--startup-log-filter=INFO,mz_orchestratord=TRACE"
 
+- it: should set a non-zero statement logging sample rate by default
+  asserts:
+  - contains:
+      path: spec.template.spec.containers[0].args
+      content: "--statement-logging-max-sample-rate=0.1"
+
+- it: should set a custom statement logging sample rate
+  set:
+    operator.args.statementLoggingMaxSampleRate: 0.5
+  asserts:
+  - contains:
+      path: spec.template.spec.containers[0].args
+      content: "--statement-logging-max-sample-rate=0.5"
+
+- it: should pass a zero statement logging sample rate to disable statement logging
+  set:
+    operator.args.statementLoggingMaxSampleRate: 0
+  asserts:
+  - contains:
+      path: spec.template.spec.containers[0].args
+      content: "--statement-logging-max-sample-rate=0"
+
+- it: should omit the statement logging sample rate when unset
+  set:
+    operator.args.statementLoggingMaxSampleRate: null
+  asserts:
+  - notContains:
+      path: spec.template.spec.containers[0].args
+      content: "--statement-logging-max-sample-rate=0.1"
+  - notContains:
+      path: spec.template.spec.containers[0].args
+      content: "--statement-logging-max-sample-rate="
+
+- it: should omit the statement logging target data rate by default
+  asserts:
+  - notContains:
+      path: spec.template.spec.containers[0].args
+      content: "--statement-logging-target-data-rate="
+
+- it: should set a custom statement logging target data rate
+  set:
+    operator.args.statementLoggingTargetDataRate: 512
+  asserts:
+  - contains:
+      path: spec.template.spec.containers[0].args
+      content: "--statement-logging-target-data-rate=512"
+
+- it: should render a large statement logging target data rate as an integer
+  values:
+  - values/large-target-data-rate.yaml
+  asserts:
+  - contains:
+      path: spec.template.spec.containers[0].args
+      content: "--statement-logging-target-data-rate=1048576"
+
 - it: should set resources correctly
   asserts:
   - equal:

misc/helm-charts/operator/tests/values/large-target-data-rate.yaml

@@ -0,0 +1,15 @@
+# Copyright Materialize, Inc. and contributors. All rights reserved.
+#
+# Use of this software is governed by the Business Source License
+# included in the LICENSE file at the root of this repository.
+#
+# As of the Change Date specified in that file, in accordance with
+# the Business Source License, use of this software will be governed
+# by the Apache License, Version 2.0.
+
+# Helm parses values files through YAML to JSON, so numbers arrive as float64
+# and render in exponential form at or above 1e6. A rate this size has to come
+# from a values file to reproduce that, `set:` values stay integers.
+operator:
+  args:
+    statementLoggingTargetDataRate: 1048576

misc/helm-charts/operator/values.yaml

@@ -37,6 +37,25 @@ operator:
     # -- Log filtering settings for startup logs
     startupLogFilter: "INFO,mz_orchestratord=TRACE"
     enableInternalStatementLogging: true
+    # -- (float) Caps the fraction of statements recorded in query history, via
+    # environmentd's `statement_logging_max_sample_rate`. Sampling costs CPU on
+    # environmentd, and the recorded history is retained for the lifetime of the
+    # environment, so this samples a fraction of statements rather than
+    # environmentd's own default of 0.99. This bounds the sampled fraction, not
+    # the total size of the history. Sustained write volume is capped separately
+    # by `statement_logging_target_data_rate`, which is the binding limit on
+    # busy environments. Raise this for more complete query history, set it to
+    # `0` to disable statement logging entirely, or set it to `null` to inherit
+    # environmentd's default.
+    statementLoggingMaxSampleRate: 0.1
+    # -- (int) Caps the sustained volume statement logging writes, in bytes per
+    # second, via environmentd's `statement_logging_target_data_rate`. This is
+    # the binding limit on busy environments, where it throttles sampled
+    # statements that would exceed it. Lower it to hold query history growth
+    # down on environments with limited storage, keeping in mind that the
+    # recorded history is retained for the lifetime of the environment. Set it
+    # to `null` to inherit environmentd's default of 2071 bytes per second.
+    statementLoggingTargetDataRate:
     # Newer versions ignore this setting and always enforce license key checks.
     enableLicenseKeyChecks: false
     # -- Whether to install the v1 version of the Materialize CRD and the

src/orchestratord/src/bin/orchestratord.rs

@@ -213,8 +213,17 @@ pub struct Args {
     enable_security_context: bool,
     #[clap(long)]
     enable_internal_statement_logging: bool,
-    #[clap(long, default_value = "false")]
-    disable_statement_logging: bool,
+    /// Overrides environmentd's default `statement_logging_max_sample_rate`. A
+    /// rate of 0 disables statement logging entirely. Leave unset to keep
+    /// environmentd's own default.
+    #[clap(long, value_parser = parse_sample_rate)]
+    statement_logging_max_sample_rate: Option<f64>,
+    /// Overrides environmentd's default `statement_logging_target_data_rate`,
+    /// in bytes per second. Unlike the sample rate, this caps the sustained
+    /// volume statement logging writes. Leave unset to keep environmentd's own
+    /// default.
+    #[clap(long)]
+    statement_logging_target_data_rate: Option<usize>,
 
     #[clap(long)]
     orchestratord_pod_selector_labels: Vec<KeyValueArg<String, String>>,
@@ -342,6 +351,17 @@ fn parse_crd_columns(val: &str) -> Result<Vec<CustomResourceColumnDefinition>, s
     serde_json::from_str(val)
 }
 
+/// Rejects rates environmentd's `NUMERIC_BOUNDED_0_1_INCLUSIVE` constraint
+/// would reject, which it does by refusing to open its catalog. Validating here
+/// surfaces the mistake where the rate was configured.
+fn parse_sample_rate(s: &str) -> anyhow::Result<f64> {
+    let rate: f64 = s.parse()?;
+    if !(0.0..=1.0).contains(&rate) {
+        anyhow::bail!("sample rate must be between 0 and 1, got {rate}");
+    }
+    Ok(rate)
+}
+
 #[tokio::main]
 async fn main() {
     mz_ore::panic::install_enhanced_handler();
@@ -628,7 +648,8 @@ async fn run(args: Args) -> Result<(), anyhow::Error> {
             scheduler_name: args.scheduler_name.clone(),
             enable_security_context: args.enable_security_context,
             enable_internal_statement_logging: args.enable_internal_statement_logging,
-            disable_statement_logging: args.disable_statement_logging,
+            statement_logging_max_sample_rate: args.statement_logging_max_sample_rate,
+            statement_logging_target_data_rate: args.statement_logging_target_data_rate,
             orchestratord_pod_selector_labels: args.orchestratord_pod_selector_labels,
             environmentd_node_selector: args.environmentd_node_selector,
             environmentd_affinity: args.environmentd_affinity,

src/orchestratord/src/controller/materialize.rs

@@ -81,7 +81,8 @@ pub struct Config {
     pub scheduler_name: Option<String>,
     pub enable_security_context: bool,
     pub enable_internal_statement_logging: bool,
-    pub disable_statement_logging: bool,
+    pub statement_logging_max_sample_rate: Option<f64>,
+    pub statement_logging_target_data_rate: Option<usize>,
 
     pub orchestratord_pod_selector_labels: Vec<KeyValueArg<String, String>>,
     pub environmentd_node_selector: Vec<KeyValueArg<String, String>>,

src/orchestratord/src/controller/materialize/generation.rs

@@ -715,8 +715,16 @@ fn create_environmentd_statefulset_object(
         args.push("--system-parameter-default=enable_internal_statement_logging=true".into());
     }
 
-    if config.disable_statement_logging {
-        args.push("--system-parameter-default=statement_logging_max_sample_rate=0".into());
+    if let Some(rate) = config.statement_logging_max_sample_rate {
+        args.push(format!(
+            "--system-parameter-default=statement_logging_max_sample_rate={rate}"
+        ));
+    }
+
+    if let Some(rate) = config.statement_logging_target_data_rate {
+        args.push(format!(
+            "--system-parameter-default=statement_logging_target_data_rate={rate}"
+        ));
     }
 
     if !mz.spec.enable_rbac {