doc: design for removing the legacy cluster scheduling and staged reconfig paths

doc/user/content/self-managed-deployments/troubleshooting.md

@@ -113,13 +113,19 @@ To increase the cluster's size, you can follow the following steps:
    show clusters;
    ```
 
+   Resizing a cluster is a graceful reconfiguration: Materialize brings up a
+   replica at the new size, waits for it to hydrate, and only then retires the
+   old one. Until that finishes, `SHOW CLUSTERS` reports the old size, and
+   briefly both. Re-run the statement until it settles. The replacement replica
+   also gets a fresh name, so the resized cluster reports `r2` rather than `r1`.
+
    The output should include the `mz_catalog_server` cluster with a size of `50cc`:
 
    ```none
           name        | replicas  | comment
     -------------------+-----------+---------
     mz_analytics      |           |
-    mz_catalog_server | r1 (50cc) |
+    mz_catalog_server | r2 (50cc) |
     mz_probe          |           |
     mz_support        |           |
     mz_system         |           |

doc/user/content/sql/alter-cluster.md

@@ -177,10 +177,6 @@ by default), Materialize rolls back the resize and the cluster keeps its current
 size. To customize the timeout behavior, use the `WAIT UNTIL READY` or `WAIT FOR` options.
 The resize still proceeds in the background.
 
-{{< private-preview >}}
-Customizing the resize timeout with `WAIT UNTIL READY` or `WAIT FOR`
-{{< /private-preview >}}
-
 - `WAIT UNTIL READY (TIMEOUT = ..., ON TIMEOUT = ...)` sets the timeout for the
   resize. On timeout, `ON TIMEOUT` selects whether to `COMMIT` (retire the old
   replicas and proceed with the not-yet-hydrated new ones, which can cause
@@ -225,7 +221,6 @@ current size. Materialize drops the pending replicas and keeps the current
 configuration.
 
 #### Downtime considerations for v26.34 or before
-{{< private-preview />}}
 
 You can use the `WAIT UNTIL READY` option to perform a zero-downtime resizing,
 which incurs **no downtime**. Instead of restarting the cluster, this approach

doc/user/data/metrics.yml

@@ -453,28 +453,6 @@ metrics:
   help: Total number of started transactions.
   source: src/catalog/src/durable/metrics.rs
   visibility: internal
-- name: mz_check_scheduling_policies_seconds_bucket
-  help: The time each policy in `check_scheduling_policies` takes.
-  labels:
-  - le
-  - policy
-  - thread
-  source: src/adapter/src/metrics.rs
-  visibility: internal
-- name: mz_check_scheduling_policies_seconds_count
-  help: The time each policy in `check_scheduling_policies` takes.
-  labels:
-  - policy
-  - thread
-  source: src/adapter/src/metrics.rs
-  visibility: internal
-- name: mz_check_scheduling_policies_seconds_sum
-  help: The time each policy in `check_scheduling_policies` takes.
-  labels:
-  - policy
-  - thread
-  source: src/adapter/src/metrics.rs
-  visibility: internal
 - name: mz_cluster_handle_command_duration_seconds_bucket
   help: Time spent in handling commands.
   labels:
@@ -971,25 +949,6 @@ metrics:
   help: The time it takes to advance the catalog shard upper for a txns-shard write (group commits and table register/forget).
   source: src/adapter/src/metrics.rs
   visibility: internal
-- name: mz_handle_scheduling_decisions_seconds_bucket
-  help: The time `handle_scheduling_decisions` takes.
-  labels:
-  - altered_a_cluster
-  - le
-  source: src/adapter/src/metrics.rs
-  visibility: internal
-- name: mz_handle_scheduling_decisions_seconds_count
-  help: The time `handle_scheduling_decisions` takes.
-  labels:
-  - altered_a_cluster
-  source: src/adapter/src/metrics.rs
-  visibility: internal
-- name: mz_handle_scheduling_decisions_seconds_sum
-  help: The time `handle_scheduling_decisions` takes.
-  labels:
-  - altered_a_cluster
-  source: src/adapter/src/metrics.rs
-  visibility: internal
 - name: mz_index_peek_cursor_setup_seconds_bucket
   help: Time setting up cursor and literal constraints.
   labels:

misc/python/materialize/checks/all_checks/builtin_cluster_replication_factor.py

@@ -21,6 +21,10 @@ class BuiltinClusterReplicationFactor(Check):
     the break-glass path a support engineer scales up by hand. A catalog open that
     does not read the cluster's replication factor tears that replica down, leaving
     the cluster reporting a factor it is not honoring.
+
+    The replica arrives asynchronously: the cluster controller materializes it a
+    tick after the ALTER commits. Every assertion below is a retrying testdrive
+    query for that reason.
     """
 
     def _can_run(self, e: Executor) -> bool:

misc/python/materialize/checks/all_checks/cluster.py

@@ -152,9 +152,6 @@ def _can_run(self, e: Executor) -> bool:
 
     def initialize(self) -> Testdrive:
         return Testdrive(dedent("""
-            $ postgres-execute connection=postgres://mz_system@${testdrive.materialize-internal-sql-addr}
-            ALTER SYSTEM SET enable_zero_downtime_cluster_reconfiguration = true
-
             $ postgres-execute connection=postgres://postgres:postgres@postgres
             CREATE USER graceful_reconfig WITH SUPERUSER PASSWORD 'postgres';
             ALTER USER graceful_reconfig WITH replication;

misc/python/materialize/mzcompose/__init__.py

@@ -97,14 +97,9 @@ def get_minimal_system_parameters(
         "enable_refresh_every_mvs": "true",
         "enable_replacement_materialized_views": "true",
         "enable_cluster_schedule_refresh": "true",
-        # The cluster controller and background ALTER CLUSTER dyncfgs default on
-        # in current versions. Pin them explicitly so runs against older versions
-        # (which predate the flags or defaulted them off) exercise the legacy
-        # paths while current versions exercise the controller owning the
-        # managed-cluster replica set.
-        "enable_cluster_controller": (
-            "true" if version >= MzVersion.parse_mz("v26.29.0-dev") else "false"
-        ),
+        # Pinned explicitly so runs against older versions (which predate the
+        # flag or defaulted it off) behave like current ones, where it defaults
+        # on.
         "enable_background_alter_cluster": (
             "true" if version >= MzVersion.parse_mz("v26.29.0-dev") else "false"
         ),
@@ -127,6 +122,23 @@ def get_minimal_system_parameters(
     if version < MzVersion.parse_mz("v0.163.0-dev"):
         config["enable_compute_active_dataflow_cancelation"] = "true"
 
+    # The cluster controller's break-glass gate. Removed in v26.38, where the
+    # controller runs unconditionally. Older binaries still read it, and
+    # defaulted it off before v26.29, so pin it on for them to keep mixed-version
+    # runs exercising the same path as current versions.
+    if version < MzVersion.parse_mz("v26.38.0-dev"):
+        config["enable_cluster_controller"] = (
+            "true" if version >= MzVersion.parse_mz("v26.29.0-dev") else "false"
+        )
+
+    # The `WITH (WAIT ...)` graceful-reconfiguration surface. Always accepted
+    # from v26.38 on. Older binaries still gate it behind this feature flag, so
+    # pin it on for them: the tests that use the surface no longer enable it
+    # themselves, and in a mixed-version run some of their phases execute
+    # against the old binary.
+    if version < MzVersion.parse_mz("v26.38.0-dev"):
+        config["enable_zero_downtime_cluster_reconfiguration"] = "true"
+
     return config
 
 

misc/python/materialize/parallel_workload/action.py

@@ -3270,7 +3270,6 @@ def __init__(
             "oidc_group_role_sync_strict",
             "console_oidc_client_id",
             "console_oidc_scopes",
-            "enable_cluster_controller",
             "cluster_controller_tick_interval",
             "enable_background_alter_cluster",
             "default_cluster_reconfiguration_timeout",

misc/python/materialize/parallel_workload/parallel_workload.py

@@ -168,10 +168,6 @@ def run(
         system_exe.execute("ALTER SYSTEM SET max_sql_server_connections = 1000000")
         system_exe.execute("ALTER SYSTEM SET max_kafka_connections = 1000000")
         system_exe.execute("ALTER SYSTEM SET idle_in_transaction_session_timeout = 0")
-        # Gates the WITH (WAIT ...) clause used by ReconfigureClusterAction.
-        system_exe.execute(
-            "ALTER SYSTEM SET enable_zero_downtime_cluster_reconfiguration = true"
-        )
         # Most queries should not fail because of privileges
         for object_type in [
             "TABLES",

src/adapter-types/src/dyncfgs.rs

@@ -353,23 +353,7 @@ pub const ENABLE_SCOPED_SYSTEM_PARAMETERS: Config<bool> = Config::new(
     "Whether per-cluster and per-replica scoped system parameters are evaluated and applied.",
 );
 
-/// Top-level gate for the cluster controller. When on, the controller owns the
-/// managed-cluster replica set and the legacy paths (the graceful 3-stage
-/// machine and `cluster_scheduling.rs`) are bypassed. The replica set cannot
-/// have two writers, so this is a clean switch, not a per-strategy toggle.
-///
-/// Defaults on. This is the break-glass switch to fall back to the legacy
-/// paths if the controller misbehaves.
-pub const ENABLE_CLUSTER_CONTROLLER: Config<bool> = Config::new(
-    "enable_cluster_controller",
-    true,
-    "Whether the cluster controller owns the managed-cluster replica set. When false, the legacy scheduling and graceful-reconfiguration paths run instead.",
-);
-
 /// Cadence of the cluster controller's reconcile tick.
-///
-/// Replaces `cluster_check_scheduling_policies_interval` once the controller is
-/// the sole owner; while the controller is dark both intervals exist.
 pub const CLUSTER_CONTROLLER_TICK_INTERVAL: Config<Duration> = Config::new(
     "cluster_controller_tick_interval",
     Duration::from_secs(5),
@@ -380,9 +364,6 @@ pub const CLUSTER_CONTROLLER_TICK_INTERVAL: Config<Duration> = Config::new(
 /// controller converging in the background, or blocks the session on a
 /// wait-shim until the reconfiguration completes or its deadline passes.
 ///
-/// Only consulted while [`ENABLE_CLUSTER_CONTROLLER`] is on, when the
-/// controller owns the reconfiguration.
-///
 /// Defaults on. This is the break-glass switch back to the blocking wait-shim
 /// if returning immediately causes trouble.
 pub const ENABLE_BACKGROUND_ALTER_CLUSTER: Config<bool> = Config::new(
@@ -404,9 +385,9 @@ pub const DEFAULT_CLUSTER_RECONFIGURATION_TIMEOUT: Config<Duration> = Config::ne
 /// runs a burst replica; graceful reconfiguration and `ON REFRESH` scheduling
 /// are unaffected.
 ///
-/// Only consulted while [`ENABLE_CLUSTER_CONTROLLER`] is on. A cluster can only
-/// carry an `AUTO SCALING STRATEGY` while its SQL acceptance feature flag is
-/// on, so this is the second of the two gates burst sits behind.
+/// A cluster can only carry an `AUTO SCALING STRATEGY` while its SQL acceptance
+/// feature flag is on, so this is the second of the two gates burst sits
+/// behind.
 pub const ENABLE_HYDRATION_BURST: Config<bool> = Config::new(
     "enable_hydration_burst",
     true,
@@ -426,7 +407,6 @@ pub const DEFAULT_HYDRATION_BURST_LINGER: Config<Duration> = Config::new(
 pub fn all_dyncfgs(configs: ConfigSet) -> ConfigSet {
     configs
         .add(&ALLOW_USER_SESSIONS)
-        .add(&ENABLE_CLUSTER_CONTROLLER)
         .add(&CLUSTER_CONTROLLER_TICK_INTERVAL)
         .add(&ENABLE_BACKGROUND_ALTER_CLUSTER)
         .add(&DEFAULT_CLUSTER_RECONFIGURATION_TIMEOUT)

src/adapter/src/catalog/open.rs

@@ -1155,9 +1155,14 @@ fn add_new_remove_old_builtin_roles_migration(
 /// recorded durably and runs before the cluster controller is spawned, and the
 /// controller does not run at all while a deployment is read-only.
 ///
-/// The controller derives its target from the same cluster config, so it converges
-/// on the same replica set rather than competing for it. It excludes system
-/// clusters today, but nothing here depends on that staying true.
+/// The cluster controller owns these replica sets at runtime and derives its
+/// target from the same cluster config, so the two converge on the same set
+/// rather than competing for it. The controller matches replicas by shape and
+/// count, never by name, so the `r1..rN` this creates satisfy it. This converges
+/// by name, so a boot after the controller reshaped a cluster renames or
+/// re-creates replicas it had materialized under generator names. That is
+/// harmless: every replica is a cold process at boot anyway, so the cost is
+/// replica-id and audit-log noise.
 fn reconcile_builtin_cluster_replicas(
     txn: &mut Transaction<'_>,
     builtin_cluster_config_map: &BuiltinBootstrapClusterConfigMap,
@@ -1231,9 +1236,9 @@ fn reconcile_builtin_cluster_replicas(
         }
 
         // Reading the cluster's factor is what makes this compose with the other
-        // writers of a replica set. The refresh scheduler parks a scheduled cluster
-        // by writing its factor to 0, so converging on the factor honors that
-        // instead of resurrecting a replica the scheduler just dropped.
+        // writers of a replica set. The controller's on-refresh strategy parks a
+        // scheduled cluster by writing its factor to 0, so converging on the
+        // factor honors that instead of resurrecting a replica it just dropped.
         let mut surplus = replicas_by_cluster.remove(&cluster.id).unwrap_or_default();
         for index in 0..managed.replication_factor {
             let replica_name = managed_cluster_replica_name(index);
@@ -1425,8 +1430,14 @@ fn remove_invalid_config_param_role_defaults_migration(
     Ok(())
 }
 
-/// Cluster Replicas may be created ephemerally during an alter statement, these replicas
-/// are marked as pending and should be cleaned up on catalog open.
+/// Drops replicas left durably marked `pending`.
+///
+/// No runtime path creates one anymore. An upgrade can still come from a version
+/// whose staged reconfiguration machine crashed between the pending-create commit
+/// and the finalize, and those replicas are excluded from the cluster
+/// controller's ownership test, so this catalog-open sweep is their only
+/// remaining cleaner. It goes away together with the durable `pending` field,
+/// once no supported upgrade source can still write one.
 fn remove_pending_cluster_replicas_migration(
     tx: &mut Transaction,
     boot_ts: mz_repr::Timestamp,

src/adapter/src/catalog/transact.rs

@@ -86,7 +86,6 @@ use crate::catalog::{
 use crate::config::{ScopedParameters, ScopedParametersScope};
 use crate::coord::ConnMeta;
 use crate::coord::catalog_implications::parsed_state_updates::ParsedStateUpdate;
-use crate::coord::cluster_scheduling::SchedulingDecision;
 use crate::util::ResultExt;
 
 /// A manually injected audit event.
@@ -360,9 +359,6 @@ pub enum ReplicaCreateDropReason {
     /// - ALTERing various options on a managed cluster,
     /// - CREATE/DROP CLUSTER REPLICA on an unmanaged cluster.
     Manual,
-    /// The automated cluster scheduling initiated the replica create or drop, e.g., a
-    /// materialized view is needing a refresh on a SCHEDULE ON REFRESH cluster.
-    ClusterScheduling(Vec<SchedulingDecision>),
     /// The cluster controller's graceful-reconfiguration strategy created the replica while
     /// converging a cluster onto an in-flight `reconfiguration` target (a background
     /// `ALTER CLUSTER`).
@@ -373,11 +369,7 @@ pub enum ReplicaCreateDropReason {
     /// The cluster controller's on-refresh strategy created the replica for a refresh window on
     /// a `SCHEDULE = ON REFRESH` cluster. Audited as the `schedule` reason, carrying the tick's
     /// window decision (which MVs needed a refresh or compaction time, and the hydration-time
-    /// estimate) as the `scheduling_policies` detail, the same detail the legacy scheduler's
-    /// [`ReplicaCreateDropReason::ClusterScheduling`] records. Deliberately not that variant
-    /// itself: its legacy shape carries a per-policy `Vec` and an on/off flag for auditing
-    /// off-decisions, neither of which the controller has (controller drops are uniformly
-    /// `Retired`), and it is removed together with the legacy scheduler.
+    /// estimate) as the `scheduling_policies` detail.
     OnRefresh(RefreshWindowDecision),
     /// The cluster controller dropped the replica because the cluster's configuration no longer
     /// calls for it. The uniform reason on every controller-emitted drop (e.g. a
@@ -394,12 +386,6 @@ impl ReplicaCreateDropReason {
     ) {
         match self {
             ReplicaCreateDropReason::Manual => (CreateOrDropClusterReplicaReasonV1::Manual, None),
-            ReplicaCreateDropReason::ClusterScheduling(scheduling_decisions) => (
-                CreateOrDropClusterReplicaReasonV1::Schedule,
-                Some(SchedulingDecision::reasons_to_audit_log_reasons(
-                    &scheduling_decisions,
-                )),
-            ),
             ReplicaCreateDropReason::GracefulReconfiguration => {
                 (CreateOrDropClusterReplicaReasonV1::Reconfiguration, None)
             }
@@ -416,8 +402,8 @@ impl ReplicaCreateDropReason {
 }
 
 /// Convert the controller's on-refresh window decision into the audit log's
-/// `scheduling_policies` detail, the same shape the legacy scheduler records:
-/// ids as strings and the hydration-time estimate as an interval string.
+/// `scheduling_policies` detail: ids as strings and the hydration-time estimate
+/// as an interval string.
 fn refresh_window_decision_to_audit_log(
     decision: RefreshWindowDecision,
 ) -> SchedulingDecisionsWithReasonsV2 {
@@ -514,8 +500,8 @@ impl Catalog {
     /// status change, a fresh record, or the drop of an in-progress record.
     ///
     /// Every such movement is an audit-log transition, so a write performing
-    /// one must declare the matching intent. Status-preserving copies (legacy
-    /// paths carrying a record forward, re-targets that stay in progress with a
+    /// one must declare the matching intent. Status-preserving copies (a write
+    /// carrying a record forward, re-targets that stay in progress with a
     /// declared `Started`) and drops of already-settled records move nothing.
     fn reconfiguration_lifecycle_moved(
         old_config: &ClusterConfig,
@@ -3827,8 +3813,8 @@ mod tests {
             &unmanaged,
         ));
 
-        // Not movements: no record at all, a status-preserving copy (legacy
-        // paths carry the record forward), and dropping a settled record.
+        // Not movements: no record at all, a status-preserving copy (a write
+        // that carries the record forward), and dropping a settled record.
         assert!(!Catalog::reconfiguration_lifecycle_moved(
             &managed(None),
             &managed(None),
@@ -3985,9 +3971,8 @@ mod tests {
 
         use crate::catalog::ReplicaCreateDropReason;
 
-        // `OnRefresh` shares the `schedule` audit word with the legacy
-        // `ClusterScheduling` variant and converts the controller's window
-        // decision into the same `scheduling_policies` detail blob: ids as
+        // `OnRefresh` audits the `schedule` word and converts the controller's
+        // window decision into the `scheduling_policies` detail blob: ids as
         // strings, the hydration-time estimate as an interval string, and the
         // decision hardcoded `on` (the controller produces a create, and so
         // this detail, only for an open window).